From 96fae38be2cba25ba53c8d6743e635ed123ab04f Mon Sep 17 00:00:00 2001 From: William S Fulton Date: Thu, 13 Oct 2016 08:08:27 +0100 Subject: [PATCH] Fix Python pickling and metaclass for builtin wrappers The metaclass (SwigPyObjectType) for SWIG objects was not defined in a way that let importlib successfully import the Python wrappers. The pickle module failed because it couldn't determine what module the SWIG wrapped objects are in. I've changed the definition of SwigPyObjectType using more normal builtin type definitions. There are still some open questions: - None of the builtin types, like swig_static_var_getset_descriptor and SwigPyObject are added into any module. No call to PyModule_AddObject is made, so isinstance cannot be used for any wrapped type, all of which are derived from SwigPyObject. Closes #808 --- Examples/test-suite/python/Makefile.in | 1 + .../test-suite/python/python_pickle_runme.py | 20 ++++ Examples/test-suite/python_pickle.i | 51 ++++++++++ Lib/python/builtin.swg | 94 ++++++++++++++++++- Lib/python/pyinit.swg | 10 +- 5 files changed, 167 insertions(+), 9 deletions(-) create mode 100644 Examples/test-suite/python/python_pickle_runme.py create mode 100644 Examples/test-suite/python_pickle.i diff --git a/Examples/test-suite/python/Makefile.in b/Examples/test-suite/python/Makefile.in index bfc5450b0..033b83103 100644 --- a/Examples/test-suite/python/Makefile.in +++ b/Examples/test-suite/python/Makefile.in @@ -64,6 +64,7 @@ CPP_TEST_CASES += \ python_docstring \ python_nondynamic \ python_overload_simple_cast \ + python_pickle \ python_pythoncode \ python_richcompare \ python_strict_unicode \ diff --git a/Examples/test-suite/python/python_pickle_runme.py b/Examples/test-suite/python/python_pickle_runme.py new file mode 100644 index 000000000..f7f8395c2 --- /dev/null +++ b/Examples/test-suite/python/python_pickle_runme.py @@ -0,0 +1,20 @@ +import python_pickle + +import pickle + +def check(p): + msg = p.msg + if msg != "hi there": + raise RuntimeError("Bad, got: " + msg) + +python_pickle.cvar.debug = False + +p = python_pickle.PickleMe("hi there") +check(p) + +r = p.__reduce__() +if python_pickle.cvar.debug: + print "__reduce__ returned:", r +pickle_string = pickle.dumps(p) +newp = pickle.loads(pickle_string) +check(newp) diff --git a/Examples/test-suite/python_pickle.i b/Examples/test-suite/python_pickle.i new file mode 100644 index 000000000..3c03b177c --- /dev/null +++ b/Examples/test-suite/python_pickle.i @@ -0,0 +1,51 @@ +%module python_pickle + + +%include + +%extend PickleMe { +#if 0 +// Note: %pythoncode can't be used with -builtin +%pythoncode %{ +def __reduce__(self): + print "In Python __reduce__" + return (type(self), (self.msg, )) +%} +#else + // Equivalent to Python code above + PyObject *__reduce__() { + if (debug) + std::cout << "In C++ __reduce__" << std::endl; + PyObject *args = PyTuple_New(1); + PyTuple_SetItem(args, 0, SWIG_From_std_string(self->msg)); + + swig_type_info *ty = SWIGTYPE_p_PickleMe; + SwigPyClientData *data = (SwigPyClientData *)ty->clientdata; +#if defined(SWIGPYTHON_BUILTIN) + PyObject *callable = (PyObject *)data->pytype; +#else + PyObject *callable = data->klass; +#endif + Py_INCREF(callable); + + PyObject *ret = PyTuple_New(2); + PyTuple_SetItem(ret, 0, callable); + PyTuple_SetItem(ret, 1, args); + return ret; + } +#endif +} + +%inline %{ +#include + +bool debug = false; + +struct PickleMe { + std::string msg; + PickleMe(const std::string& msg) : msg(msg) { + if (debug) + std::cout << "In C++ constructor " << " [" << msg << "]" << std::endl; + } +}; +%} diff --git a/Lib/python/builtin.swg b/Lib/python/builtin.swg index 314d2b385..bd2141cd4 100644 --- a/Lib/python/builtin.swg +++ b/Lib/python/builtin.swg @@ -164,9 +164,13 @@ SwigPyStaticVar_set(PyGetSetDescrObject *descr, PyObject *obj, PyObject *value) } SWIGINTERN int -SwigPyObjectType_setattro(PyTypeObject *type, PyObject *name, PyObject *value) { +SwigPyObjectType_setattro(PyObject *typeobject, PyObject *name, PyObject *value) { PyObject *attribute; + PyTypeObject *type; descrsetfunc local_set; + + assert(PyType_Check(typeobject)); + type = (PyTypeObject *)typeobject; attribute = _PyType_Lookup(type, name); if (attribute != NULL) { /* Implement descriptor functionality, if any */ @@ -276,6 +280,94 @@ SwigPyStaticVar_Type(void) { return &staticvar_type; } +SWIGINTERN PyTypeObject* +SwigPyObjectType(void) { + static char swigpyobjecttype_doc[] = "Metaclass for SWIG wrapped types"; + static PyTypeObject swigpyobjecttype_type; + static int type_init = 0; + if (!type_init) { + const PyTypeObject tmp = { +#if PY_VERSION_HEX >= 0x03000000 + PyVarObject_HEAD_INIT(&PyType_Type, 0) +#else + PyObject_HEAD_INIT(&PyType_Type) + 0, /* ob_size */ +#endif + "SwigPyObjectType", /* tp_name */ + PyType_Type.tp_basicsize, /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + SwigPyObjectType_setattro, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_CLASS, /* tp_flags */ + swigpyobjecttype_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyType_Type, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ +#if PY_VERSION_HEX >= 0x02030000 + 0, /* tp_del */ +#endif +#if PY_VERSION_HEX >= 0x02060000 + 0, /* tp_version_tag */ +#endif +#if PY_VERSION_HEX >= 0x03040000 + 0, /* tp_finalize */ +#endif +#ifdef COUNT_ALLOCS + 0, /* tp_allocs */ + 0, /* tp_frees */ + 0, /* tp_maxalloc */ +#if PY_VERSION_HEX >= 0x02050000 + 0, /* tp_prev */ +#endif + 0 /* tp_next */ +#endif + }; + swigpyobjecttype_type = tmp; + type_init = 1; +#if PY_VERSION_HEX < 0x02020000 + swigpyobjecttype_type.ob_type = &PyType_Type; +#else + if (PyType_Ready(&swigpyobjecttype_type) < 0) + return NULL; +#endif + } + return &swigpyobjecttype_type; +} + SWIGINTERN PyGetSetDescrObject * SwigPyStaticVar_new_getset(PyTypeObject *type, PyGetSetDef *getset) { diff --git a/Lib/python/pyinit.swg b/Lib/python/pyinit.swg index e671731ac..df70e6566 100644 --- a/Lib/python/pyinit.swg +++ b/Lib/python/pyinit.swg @@ -374,7 +374,6 @@ SWIG_init(void) { static PyGetSetDef thisown_getset_def = { (char *)"thisown", SwigPyBuiltin_GetterClosure, SwigPyBuiltin_SetterClosure, NULL, &thisown_getset_closure }; - PyObject *metatype_args; PyTypeObject *builtin_pytype; int builtin_base_count; swig_type_info *builtin_basetype; @@ -395,14 +394,9 @@ SWIG_init(void) { (void)static_getset; (void)self; - /* metatype is used to implement static member variables. */ - metatype_args = Py_BuildValue("(s(O){})", "SwigPyObjectType", &PyType_Type); - assert(metatype_args); - metatype = (PyTypeObject *) PyType_Type.tp_call((PyObject *) &PyType_Type, metatype_args, NULL); + /* Metaclass is used to implement static member variables */ + metatype = SwigPyObjectType(); assert(metatype); - Py_DECREF(metatype_args); - metatype->tp_setattro = (setattrofunc) &SwigPyObjectType_setattro; - assert(PyType_Ready(metatype) >= 0); #endif /* Fix SwigMethods to carry the callback ptrs when needed */