SIP memory leak using std::vector

[Phil Thompson]

On 21/05/2021 14:22, Marian Thomsen wrote:
> Hello,
> 
> I struggle for some time now to close a memory leak using std::vector
> with SIP (first SIP v6.0.3 and now 6.1).
> 
> The valgrind output:
> 
> 
> ==26606== 16 bytes in 1 blocks are definitely lost in loss record 6 of 
> 4,537
> ==26606==    at 0x4C3217F: operator new(unsigned long) (in
> /usr/lib/valgrind/vgpreload_memcheck-amd64-linux.so)
> ==26606==    by 0x6FB1542: allocate (new_allocator.h:111)
> ==26606==    by 0x6FB1542: allocate (alloc_traits.h:436)
> ==26606==    by 0x6FB1542: _M_allocate (stl_vector.h:172)
> ==26606==    by 0x6FB1542:
> _M_allocate_and_copy<__gnu_cxx::__normal_iterator<ActionID* const*,
> std::vector<ActionID*> > > (stl_vector.h:1260)
> ==26606==    by 0x6FB1542: std::vector<ActionID*,
> std::allocator<ActionID*> >::operator=(std::vector<ActionID*,
> std::allocator<ActionID*> > const&) (vector.tcc:206)
> ==26606==    by 0x6FBE60C:
> varset_RoadWorksContainerExtended_referenceDenms
> (sipdenm_pduRoadWorksContainerExtended.cpp:347)
> ==26606==    by 0x7CEF538: sipVariableDescr_descr_set 
> (descriptors.c:415)
> ==26606==    by 0x5CDD3F: _PyObject_GenericSetAttrWithDict (in
> /usr/bin/python3.8)
> ==26606==    by 0x5D613F: PyObject_SetAttr (in /usr/bin/python3.8)
> ==26606==    by 0x578DE9: _PyEval_EvalFrameDefault (in 
> /usr/bin/python3.8)
> ==26606==    by 0x5760EC: _PyEval_EvalCodeWithName (in 
> /usr/bin/python3.8)
> ==26606==    by 0x6025C6: _PyFunction_Vectorcall (in 
> /usr/bin/python3.8)
> ==26606==    by 0x578798: _PyEval_EvalFrameDefault (in 
> /usr/bin/python3.8)
> ==26606==    by 0x5760EC: _PyEval_EvalCodeWithName (in 
> /usr/bin/python3.8)
> ==26606==    by 0x66299D: ??? (in /usr/bin/python3.8)
> 
> 
> The wrapper snipped mentioned in valgrind ->
> (sipdenm_pduRoadWorksContainerExtended.cpp:347):
> 
> 
> extern "C" {static int
> varset_RoadWorksContainerExtended_referenceDenms(void *, PyObject *,
> PyObject *);}
> static int varset_RoadWorksContainerExtended_referenceDenms(void
> *sipSelf, PyObject *sipPy, PyObject *)
> {
>      ::ReferenceDenms*sipVal;
>      ::RoadWorksContainerExtended *sipCpp = reinterpret_cast<
> ::RoadWorksContainerExtended *>(sipSelf);
> 
>     int sipIsErr = 0;
>     int sipValState;
>     sipVal = reinterpret_cast< ::ReferenceDenms
> *>(sipForceConvertToType(sipPy, sipType_std_vector_0101ActionID,
> SIP_NULLPTR, SIP_NOT_NONE, &sipValState, &sipIsErr));
> 
>     if (sipIsErr)
>         return -1;
> 
>     sipCpp->referenceDenms = *sipVal; <- 347
> 
>     sipReleaseType(sipVal, sipType_std_vector_0101ActionID, 
> sipValState);
> 
>     return 0;
> }
> 
> 
> The python code:
> 
> 
> actionId1 = denm_pdu.ActionID()
> actionId1.originatingStationID = 4294902015
> actionId1.sequenceNumber = 12
> actionId2 = denm_pdu.ActionID()
> actionId2.originatingStationID = 4294902016
> actionId2.sequenceNumber = 13
> roadWorksContainerExtended.referenceDenms = [actionId1, actionId2]
> 
> The sip-file:
> 
> ...
> 
> typedef std::vector<ActionID*> ReferenceDenms;
> 
> ...
> 
> 
> struct RoadWorksContainerExtended {
> 
> %TypeHeaderCode
> #include <denm-desc.h>
> %End
> 
>    ReferenceDenms referenceDenms;
>    RoadWorksContainerExtended ();
> } ;
> 
> 
> 
> The MappedType:
> 
> 
> template<TYPE>
> %MappedType std::vector<TYPE *>
> {
> %TypeHeaderCode
> #include <vector>
> #include <iostream>
> %End
> 
> // from python
> %ConvertToTypeCode
>     const sipTypeDef* kpTypeDef = sipFindType("TYPE");
> 
>     if (!kpTypeDef) {
>         std::cout << "Error in vector_ptr.sip - ConvertToTypeCode: "
>                      "Could not find a matching type for 'TYPE'\n";
>         return 0;
>     }
> 
>     // See if we are just being asked to check the type of the Python 
> object.
>     if (sipIsErr == NULL) {
>         // Check it is a list.
>         if (!PyList_Check(sipPy)) {
>             return 0;
>         }
> 
>         // Now check each element of the list is of the type we expect.
>         // The template is for a pointer type so we don't disallow 
> None.
>         for (Py_ssize_t i = 0; i < PyList_GET_SIZE(sipPy); ++i) {
> 
>             PyObject *item = PyList_GET_ITEM(sipPy, i);
> 
>             if (!sipCanConvertToType(item, kpTypeDef, SIP_NOT_NONE)) {
>                 std::cout << "Error in vector_ptr.sip - 
> ConvertToTypeCode: "
>                              "Object cannot be converted to TYPE'\n";
>                 return 0;
>             }
>         }
>         return 1;
>     }
> 
>     Py_ssize_t len = PyList_GET_SIZE(sipPy);
>     // Create the instance on the heap.
>     std::vector<TYPE *> *v = new std::vector<TYPE *>;
>     v->reserve(len);
> 
>     for (Py_ssize_t i = 0; i < len; ++i) {
> 
>         int state;
>         // Use the SIP API to convert the Python object to the
> corresponding C++ instance.  Note that we apply any
>         // ownership transfer to the list itself, not the individual 
> elements.
>         TYPE *t = static_cast<TYPE
> *>(sipConvertToType(PyList_GET_ITEM(sipPy, i), kpTypeDef, NULL,
> SIP_NOT_NONE, &state, sipIsErr));
> 
>         // Deal with any errors
>         if (*sipIsErr) {
>             std::cout << "Error in vector_ptr.sip - ConvertToTypeCode: 
> "
>                          "Error occurred - cleanup on the heap ...'\n";
>             // Tidy up.
>             sipReleaseType(t, kpTypeDef, state);
>             delete v;
> 
>             // There is nothing on the heap.
>             return 0;
>         }
> 
>         // Add the pointer to the C++ instance.
>         v->push_back(t);
>     }
> 
>     // Return the instance on the heap.
>     *sipCppPtr = v;
> 
>     // Apply the normal transfer.
>     return sipGetState(sipTransferObj);
> %End
> 
> // from c++
> %ConvertFromTypeCode
>     PyObject *l;
> 
>     const sipTypeDef* kpTypeDef = sipFindType("TYPE");
> 
>     if (!kpTypeDef) {
>         std::cout << "Error in vector_ptr.sip - ConvertFromTypeCode: "
>                      "Could not find a matching type for 'TYPE'\n";
>         return NULL;
>     }
> 
>     // Create the Python list of the correct length.
>     if ((l = PyList_New(sipCpp->size())) == NULL) {
>         std::cout << "Error in vector_ptr.sip - ConvertFromTypeCode: "
>                      "Could not create python list'\n";
>         return NULL;
>     }
> 
>     int i = 0;
>     // Go through each element in the C++ instance and convert it to
> the corresponding Python object.
>     for(std::vector<TYPE *>::iterator iter = sipCpp->begin(); iter !=
> sipCpp->end(); ++iter) {
>         TYPE *t = *iter;
>         PyObject *tobj;
> 
>         if ((tobj = sipConvertFromType(t, kpTypeDef, sipTransferObj)) 
> == NULL) {
> 
>             // There was an error so garbage collect the Python list.
>             Py_XDECREF(tobj);
>             Py_XDECREF(l);
>             std::cout << "Error in vector_ptr.sip - 
> ConvertFromTypeCode: "
>                          "Could not convert type in Python list'\n";
>             return NULL;
>         }
> 
>         PyList_SET_ITEM(l, i++, tobj);
>     }
> 
>     // Return the Python list.
>     return l;
> %End
> };
> 
> 
> 
> I tried changing the ownership to c++ of every item when creating
> vector in the MappedType - ConvertToTypeCode but no effect.
> 
> Running sip-install with tracing or debug option gives no further 
> information.
> 
> 
> Does someone have an idea how this leak could be closed or what else
> one could try to close it?
> 
> Any help would be appreciated.

Your best chance of getting help is to produce a short, complete example 
that demonstrates the problem and that people can actually run.

Phil