Dear Alex,

I only had a cursory look, but could the problem be in your set_matrix?

OxValSetMat takes a copy of the matrix, but I don't see a free in set_matrix.

Jurgen

oxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxox
oxox 20th OxMetrics user conference 10-11 Sept 2018
oxox Centre for Econometric Analysis, Cass Business School
oxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxox
Dr Jurgen A Doornik
James Martin Fellow, Institute for New Economic Thinking
at the Oxford Martin School, University of Oxford
http://www.doornik.com
oxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxoxox

On 2018-07-03 11:38, Alex de Geus wrote:
> Hello,
>
> I am experiencing some large memory consumptions in an application using Cython
> to communicate with Ox.
>
> I am using the Ox Professional version 7.10 with the Development Kit with this
> manual as reference https://www.doornik.com/ox/OxDeveloper.pdf.
>
> Below are some code snippets with a simplified unit test that hands 2 large
> matrices to ox and back to python again.
>
> I have tried to clean up all the pointer allocations to Ox very carefully,
> however this does not seem to work correctly as memory keeps accumulating for
> every Ox call.
>
> I am pretty sure I am not freeing the memory allocated correctly for both the
> input and output arguments, however I have been unsuccessful in finding a solution.
>
> I have tried the void OxFreeByValue(OxVALUE*) function for the output arguments,
> but it does not seem to help, in some cases it even increases memory consumption.
>
> I have tried the void MatFreeBlock(MATRIX) function for the input and output
> arguments of type matrix, but this always seems to crash.
>
> Any idea on how to fix this or how to debug this code more effectively?
>
> Best,
>
> Alex
>
> Memory consumption (using the memory_profiler package from python):
>
> Line #    Mem usage    Increment   Line Contents
>
> ================================================
>
>     31     25.1 MiB     25.1 MiB    @profile
>
>     32                                         def test_getterssetters(self):
>
>     33
>
>     34     27.1 MiB      2.1 MiB                 with
> oxcy.create_ox_environment_from_docstring(self.docstring) as OxEnv:
>
>     35
>
>     36    790.1 MiB    762.9 MiB                           mat1_in =
> np.ones((10000, 10000))
>
>     37   1553.0 MiB    762.9 MiB                          mat2_in =
> np.ones((10000, 10000))
>
>     38
>
>     39   7096.0 MiB   5543.0 MiB                         mat1_out, mat2_out =
> OxEnv.TestGettersSetters(mat1_in, mat2_in)
>
>     40
>
>     41   7096.1 MiB      0.1 MiB
> np.testing.assert_allclose(mat1_in, mat1_out)
>
>     42   7095.4 MiB     -0.7 MiB
> np.testing.assert_allclose(mat2_in, mat2_out)
>
>     43
>
>     44   4043.7 MiB  -3051.8 MiB              del mat1_in, mat2_in, mat1_out,
> mat2_out
>
>     45
>
>     46   4044.4 MiB      0.8 MiB                with
> oxcy.create_ox_environment_from_docstring(self.docstring) as OxEnv:
>
>     47
>
>     48   4807.3 MiB    762.9 MiB                          mat1_in =
> np.ones((10000, 10000))
>
>     49   5570.3 MiB    762.9 MiB                          mat2_in =
> np.ones((10000, 10000))
>
>     50
>
>     51   8024.9 MiB   2454.6 MiB                          mat1_out, mat2_out =
> OxEnv.TestGettersSetters(mat1_in, mat2_in)
>
>     52
>
>     53   8079.7 MiB     54.8 MiB
> np.testing.assert_allclose(mat1_in, mat1_out)
>
>     54   8079.2 MiB     -0.5 MiB
> np.testing.assert_allclose(mat2_in, mat2_out)
>
>     55
>
>     56   5027.4 MiB  -3051.8 MiB              del mat1_in, mat2_in, mat1_out,
> mat2_out
>
> Python snippet:
>
> classTest_oxcy(unittest.TestCase):
>
> defsetUp(self):
>
> self.docstring = \
>
> """
>
> #include <oxstd.h>
>
> #include <packages/ssfpack/ssfpack_ex.h>
>
> TestGettersSetters(const mat1, const mat2);
>
> TestGettersSetters(const mat1, const mat2)
>
> {
>
>     decl output = new array[2];
>
>     output[0] = mat1;
>
>     output[1] = mat2;
>
>     return output;
>
> }
>
> """
>
> deftest_getterssetters(self):
>
> withoxcy.create_ox_environment_from_docstring(self.docstring) asOxEnv:
>
>             mat1_in = np.ones((10000, 10000))
>
>             mat2_in = np.ones((10000, 10000))
>
>             mat1_out, mat2_out = OxEnv.TestGettersSetters(mat1_in, mat2_in)
>
>             np.testing.assert_allclose(mat1_in, mat1_out)
>
>             np.testing.assert_allclose(mat2_in, mat2_out)
>
> delmat1_in, mat2_in, mat1_out, mat2_out
>
> withoxcy.create_ox_environment_from_docstring(self.docstring) asOxEnv:
>
>             mat1_in = np.ones((10000, 10000))
>
>             mat2_in = np.ones((10000, 10000))
>
>             mat1_out, mat2_out = OxEnv.TestGettersSetters(mat1_in, mat2_in)
>
>             np.testing.assert_allclose(mat1_in, mat1_out)
>
>             np.testing.assert_allclose(mat2_in, mat2_out)
>
> delmat1_in, mat2_in, mat1_out, mat2_out
>
> deftearDown(self):
>
> pass
>
> if__name__== "__main__":
>
>     unittest.main()
>
> Cython snippets:
>
> cdef classOxEnv:
>
> """ ox environment wrapper """
>
>     ...
>
> def__enter__(self):
>
> """ load ox source file into ox compiler """
>
> # TODO: check return value for response?
>
>         cox.OxMainCmd('-r- {}'.format(self.src))
>
> returnself
>
> def__exit__(self, exc_type, exc_message, exc_traceback):
>
> """ exit ox runtime environment """
>
>         ...
>
>         cox.OxRunExit()
>
>         cox.OxMainExit()
>
>     ...
>
> cdefcall_ox_func(strf_name):
>
> """ call ox static function """
>
> deffunc(*args):
>
> try:
>
>             f_ptr = cox.OxStoreCreate(1)
>
>             cox.OxValSetString(f_ptr, f_name)
>
>             in_ptr = cox.OxStoreCreate(len(args))
>
> fori in range(len(args)):
>
>                 set_ox_var(cox.OxValGetVal(in_ptr, i), args[i])
>
>             out_ptr = cox.OxStoreCreate(1)
>
> ifcox.FOxCallBack(f_ptr, out_ptr, in_ptr, len(args)) != 1:
>
> raiseException("Failed to call function!", f_name)
>
>             val = get_ox_var(out_ptr)
>
> finally:
>
>             cox.OxStoreDelete(f_ptr, 1)
>
> cox.OxStoreDelete(in_ptr, len(args))
>
> cox.OxStoreDelete(out_ptr, 1)
>
> returnval
>
> returnfunc
>
> # called by set_ox_var for matrix types
>
> cdefset_matrix(cox.OxVALUE *ptr, np.ndarray[np.double_t, ndim=2] m):
>
> """ write matrix to ptr """
>
>     cR, cC = m.shape[0], m.shape[1]
>
> matrix = cox.MatAllocBlock(cR, cC)
>
> fori in range(cR):
>
> forj in range(cC):
>
>             cox.MatSetAt(matrix, m[i, j], i, j)
>
>     cox.OxValSetMat(ptr, matrix, cR, cC)
>
>
>
> Please consider the environment before printing this document
>
> --------------------------------------------------------------------------------
>
> To unsubscribe from the OX-USERS list, click the following link:
> https://www.jiscmail.ac.uk/cgi-bin/webadmin?SUBED1=OX-USERS&A=1
>

########################################################################

To unsubscribe from the OX-USERS list, click the following link:
https://www.jiscmail.ac.uk/cgi-bin/webadmin?SUBED1=OX-USERS&A=1