Hello,
I am experiencing some large memory consumptions in an application using Cython to communicate with Ox.
I am using the Ox Professional version 7.10 with the Development Kit with this manual as reference
https://www.doornik.com/ox/OxDeveloper.pdf.
Below are some code snippets with a simplified unit test that hands 2 large matrices to ox and back to python again.
I have tried to clean up all the pointer allocations to Ox very carefully, however this does not seem to work correctly as memory keeps accumulating for every Ox
call.
I am pretty sure I am not freeing the memory allocated correctly for both the input and output arguments, however I have been unsuccessful in finding a solution.
I have tried the void OxFreeByValue(OxVALUE*) function for the output arguments, but it does not seem to help, in some cases it even increases memory consumption.
I have tried the void MatFreeBlock(MATRIX) function for the input and output arguments of type matrix, but this always seems to crash.
Any idea on how to fix this or how to debug this code more effectively?
Best,
Alex
Memory consumption (using the memory_profiler package from python):
Line # Mem usage Increment Line Contents
================================================
31 25.1 MiB 25.1 MiB @profile
32 def test_getterssetters(self):
33
34 27.1 MiB 2.1 MiB with oxcy.create_ox_environment_from_docstring(self.docstring) as OxEnv:
35
36 790.1 MiB 762.9 MiB mat1_in = np.ones((10000, 10000))
37 1553.0 MiB 762.9 MiB mat2_in = np.ones((10000, 10000))
38
39 7096.0 MiB 5543.0 MiB mat1_out, mat2_out = OxEnv.TestGettersSetters(mat1_in, mat2_in)
40
41 7096.1 MiB 0.1 MiB np.testing.assert_allclose(mat1_in, mat1_out)
42 7095.4 MiB -0.7 MiB np.testing.assert_allclose(mat2_in, mat2_out)
43
44 4043.7 MiB -3051.8 MiB del mat1_in, mat2_in, mat1_out, mat2_out
45
46 4044.4 MiB 0.8 MiB with oxcy.create_ox_environment_from_docstring(self.docstring) as OxEnv:
47
48 4807.3 MiB 762.9 MiB mat1_in = np.ones((10000, 10000))
49 5570.3 MiB 762.9 MiB mat2_in = np.ones((10000, 10000))
50
51 8024.9 MiB 2454.6 MiB mat1_out, mat2_out = OxEnv.TestGettersSetters(mat1_in, mat2_in)
52
53 8079.7 MiB 54.8 MiB np.testing.assert_allclose(mat1_in, mat1_out)
54 8079.2 MiB -0.5 MiB np.testing.assert_allclose(mat2_in, mat2_out)
55
56 5027.4 MiB -3051.8 MiB del mat1_in, mat2_in, mat1_out, mat2_out
Python snippet:
class
Test_oxcy(unittest.TestCase):
def
setUp(self):
self.docstring
= \
"""
#include <oxstd.h>
#include <packages/ssfpack/ssfpack_ex.h>
TestGettersSetters(const mat1, const mat2);
TestGettersSetters(const mat1, const mat2)
{
decl output = new array[2];
output[0] = mat1;
output[1] = mat2;
return output;
}
"""
def
test_getterssetters(self):
with oxcy.create_ox_environment_from_docstring(self.docstring)
as OxEnv:
mat1_in = np.ones((10000,
10000))
mat2_in = np.ones((10000,
10000))
mat1_out, mat2_out = OxEnv.TestGettersSetters(mat1_in, mat2_in)
np.testing.assert_allclose(mat1_in, mat1_out)
np.testing.assert_allclose(mat2_in, mat2_out)
del mat1_in, mat2_in, mat1_out, mat2_out
with oxcy.create_ox_environment_from_docstring(self.docstring)
as OxEnv:
mat1_in = np.ones((10000,
10000))
mat2_in = np.ones((10000,
10000))
mat1_out, mat2_out = OxEnv.TestGettersSetters(mat1_in, mat2_in)
np.testing.assert_allclose(mat1_in, mat1_out)
np.testing.assert_allclose(mat2_in, mat2_out)
del mat1_in, mat2_in, mat1_out, mat2_out
def
tearDown(self):
pass
if
__name__ ==
"__main__":
unittest.main()
Cython snippets:
cdef class
OxEnv:
""" ox environment wrapper """
...
def
__enter__(self):
""" load ox source file into ox compiler """
# TODO: check return value for response?
cox.OxMainCmd('-r- {}'.format(self.src))
return
self
def
__exit__(self,
exc_type,
exc_message,
exc_traceback):
""" exit ox runtime environment """
...
cox.OxRunExit()
cox.OxMainExit()
...
cdef
call_ox_func(str
f_name):
""" call ox static function """
def
func(*args):
try:
f_ptr = cox.OxStoreCreate(1)
cox.OxValSetString(f_ptr, f_name)
in_ptr = cox.OxStoreCreate(len(args))
for i in
range(len(args)):
set_ox_var(cox.OxValGetVal(in_ptr, i), args[i])
out_ptr = cox.OxStoreCreate(1)
if cox.FOxCallBack(f_ptr, out_ptr, in_ptr,
len(args)) !=
1:
raise
Exception("Failed to call function!",
f_name)
val = get_ox_var(out_ptr)
finally:
cox.OxStoreDelete(f_ptr,
1)
cox.OxStoreDelete(in_ptr,
len(args))
cox.OxStoreDelete(out_ptr,
1)
return val
return func
# called by set_ox_var for matrix types
cdef
set_matrix(cox.OxVALUE
*ptr, np.ndarray[np.double_t,
ndim=2]
m):
""" write matrix to ptr """
cR, cC = m.shape[0],
m.shape[1]
matrix = cox.MatAllocBlock(cR, cC)
for i in
range(cR):
for j in
range(cC):
cox.MatSetAt(matrix, m[i, j], i, j)
cox.OxValSetMat(ptr, matrix, cR, cC)