Hello,

 

I am experiencing some large memory consumptions in an application using Cython to communicate with Ox.

I am using the Ox Professional version 7.10 with the Development Kit with this manual as reference https://www.doornik.com/ox/OxDeveloper.pdf.

 

Below are some code snippets with a simplified unit test that hands 2 large matrices to ox and back to python again.

 

I have tried to clean up all the pointer allocations to Ox very carefully, however this does not seem to work correctly as memory keeps accumulating for every Ox call.

I am pretty sure I am not freeing the memory allocated correctly for both the input and output arguments, however I have been unsuccessful in finding a solution.

 

I have tried the void OxFreeByValue(OxVALUE*) function for the output arguments, but it does not seem to help, in some cases it even increases memory consumption.

I have tried the void MatFreeBlock(MATRIX) function for the input and output arguments of type matrix, but this always seems to crash.

 

Any idea on how to fix this or how to debug this code more effectively?

 

Best,

Alex

 

Memory consumption (using the memory_profiler package from python):

 

Line #    Mem usage    Increment   Line Contents

================================================

    31     25.1 MiB     25.1 MiB    @profile

    32                                         def test_getterssetters(self):

    33

    34     27.1 MiB      2.1 MiB                 with oxcy.create_ox_environment_from_docstring(self.docstring) as OxEnv:

    35

    36    790.1 MiB    762.9 MiB                           mat1_in = np.ones((10000, 10000))

    37   1553.0 MiB    762.9 MiB                          mat2_in = np.ones((10000, 10000))

    38

    39   7096.0 MiB   5543.0 MiB                         mat1_out, mat2_out = OxEnv.TestGettersSetters(mat1_in, mat2_in)

    40

    41   7096.1 MiB      0.1 MiB                           np.testing.assert_allclose(mat1_in, mat1_out)

    42   7095.4 MiB     -0.7 MiB                           np.testing.assert_allclose(mat2_in, mat2_out)

    43

    44   4043.7 MiB  -3051.8 MiB              del mat1_in, mat2_in, mat1_out, mat2_out

    45

    46   4044.4 MiB      0.8 MiB                with oxcy.create_ox_environment_from_docstring(self.docstring) as OxEnv:

    47

    48   4807.3 MiB    762.9 MiB                          mat1_in = np.ones((10000, 10000))

    49   5570.3 MiB    762.9 MiB                          mat2_in = np.ones((10000, 10000))

    50

    51   8024.9 MiB   2454.6 MiB                          mat1_out, mat2_out = OxEnv.TestGettersSetters(mat1_in, mat2_in)

    52

    53   8079.7 MiB     54.8 MiB                           np.testing.assert_allclose(mat1_in, mat1_out)

    54   8079.2 MiB     -0.5 MiB                            np.testing.assert_allclose(mat2_in, mat2_out)

    55

    56   5027.4 MiB  -3051.8 MiB              del mat1_in, mat2_in, mat1_out, mat2_out

 

Python snippet:

 

class Test_oxcy(unittest.TestCase):

 

    def setUp(self):

       

        self.docstring = \

"""

#include <oxstd.h>

#include <packages/ssfpack/ssfpack_ex.h>

 

TestGettersSetters(const mat1, const mat2);

TestGettersSetters(const mat1, const mat2)

{

    decl output = new array[2];

    output[0] = mat1;

    output[1] = mat2;

 

    return output;

}

 

"""

   

    def test_getterssetters(self):

 

        with oxcy.create_ox_environment_from_docstring(self.docstring) as OxEnv:

 

            mat1_in = np.ones((10000, 10000))

            mat2_in = np.ones((10000, 10000))

 

            mat1_out, mat2_out = OxEnv.TestGettersSetters(mat1_in, mat2_in)

 

            np.testing.assert_allclose(mat1_in, mat1_out)

            np.testing.assert_allclose(mat2_in, mat2_out)

 

        del mat1_in, mat2_in, mat1_out, mat2_out

 

        with oxcy.create_ox_environment_from_docstring(self.docstring) as OxEnv:

 

            mat1_in = np.ones((10000, 10000))

            mat2_in = np.ones((10000, 10000))

 

            mat1_out, mat2_out = OxEnv.TestGettersSetters(mat1_in, mat2_in)

 

            np.testing.assert_allclose(mat1_in, mat1_out)

            np.testing.assert_allclose(mat2_in, mat2_out)

 

        del mat1_in, mat2_in, mat1_out, mat2_out

 

    def tearDown(self):

        pass

 

if __name__ == "__main__":

    unittest.main()

 

Cython snippets:

 

cdef class OxEnv:

    """ ox environment wrapper """

 

    ...

 

    def __enter__(self):

        """ load ox source file into ox compiler """

        # TODO: check return value for response?

        cox.OxMainCmd('-r- {}'.format(self.src))       

        return self

 

    def __exit__(self, exc_type, exc_message, exc_traceback):

        """ exit ox runtime environment """

        ...

 

        cox.OxRunExit()

        cox.OxMainExit()

 

    ...

 

cdef call_ox_func(str f_name):

    """ call ox static function """

    def func(*args):

        try:

            f_ptr = cox.OxStoreCreate(1)

            cox.OxValSetString(f_ptr, f_name)

 

            in_ptr = cox.OxStoreCreate(len(args))

 

            for i in range(len(args)):

                set_ox_var(cox.OxValGetVal(in_ptr, i), args[i])

 

            out_ptr = cox.OxStoreCreate(1)

 

            if cox.FOxCallBack(f_ptr, out_ptr, in_ptr, len(args)) != 1:

                raise Exception("Failed to call function!", f_name)

 

            val = get_ox_var(out_ptr)

        finally:

            cox.OxStoreDelete(f_ptr, 1)

            cox.OxStoreDelete(in_ptr, len(args))

            cox.OxStoreDelete(out_ptr, 1)

        return val

    return func

 

# called by set_ox_var for matrix types

 

cdef set_matrix(cox.OxVALUE *ptr, np.ndarray[np.double_t, ndim=2] m):

    """ write matrix to ptr """

    cR, cC = m.shape[0], m.shape[1]

    matrix = cox.MatAllocBlock(cR, cC)

    for i in range(cR):

        for j in range(cC):

            cox.MatSetAt(matrix, m[i, j], i, j)

    cox.OxValSetMat(ptr, matrix, cR, cC)