From 9846f3f1eac0835e9d1870f41ea0c03f3006a687 Mon Sep 17 00:00:00 2001 From: Harvey Falcic Date: Wed, 16 Apr 2014 11:13:21 -0400 Subject: [PATCH 1/3] Python 3 byte string output: use errors="surrogateescape" ... if available on the version of Python that's in use. This allows obtaining the original byte string (and potentially trying a fallback encoding) if the bytes can't be decoded as UTF-8. Previously, a UnicodeDecodeError would be raised with no way to treat the data as bytes or try another codec. --- Lib/python/pystrings.swg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/python/pystrings.swg b/Lib/python/pystrings.swg index f6a4eba8a..2b14547ad 100644 --- a/Lib/python/pystrings.swg +++ b/Lib/python/pystrings.swg @@ -89,7 +89,11 @@ SWIG_FromCharPtrAndSize(const char* carray, size_t size) SWIG_InternalNewPointerObj(%const_cast(carray,char *), pchar_descriptor, 0) : SWIG_Py_Void(); } else { %#if PY_VERSION_HEX >= 0x03000000 +%#if PY_VERSION_HEX >= 0x03010000 + return PyUnicode_DecodeUTF8(carray, %numeric_cast(size,int), "surrogateescape"); +%#else return PyUnicode_FromStringAndSize(carray, %numeric_cast(size,int)); +%#endif %#else return PyString_FromStringAndSize(carray, %numeric_cast(size,int)); %#endif From 791f070e66ee4cf96f0df0a4d00e1fe688f6bb80 Mon Sep 17 00:00:00 2001 From: Harvey Falcic Date: Thu, 22 May 2014 22:53:07 -0400 Subject: [PATCH 2/3] Add "unicode_strings" test case for new Python 3 behavior --- Examples/test-suite/common.mk | 1 + .../test-suite/python/unicode_strings_runme.py | 4 ++++ Examples/test-suite/unicode_strings.i | 15 +++++++++++++++ 3 files changed, 20 insertions(+) create mode 100644 Examples/test-suite/python/unicode_strings_runme.py create mode 100644 Examples/test-suite/unicode_strings.i diff --git a/Examples/test-suite/common.mk b/Examples/test-suite/common.mk index bfb960fe5..bc1ca5cb8 100644 --- a/Examples/test-suite/common.mk +++ b/Examples/test-suite/common.mk @@ -473,6 +473,7 @@ CPP_TEST_CASES += \ typemap_various \ typename \ types_directive \ + unicode_strings \ union_scope \ using1 \ using2 \ diff --git a/Examples/test-suite/python/unicode_strings_runme.py b/Examples/test-suite/python/unicode_strings_runme.py new file mode 100644 index 000000000..2d26599aa --- /dev/null +++ b/Examples/test-suite/python/unicode_strings_runme.py @@ -0,0 +1,4 @@ +import unicode_strings + +unicode_strings.test_c_str() +unicode_strings.test_std_string() diff --git a/Examples/test-suite/unicode_strings.i b/Examples/test-suite/unicode_strings.i new file mode 100644 index 000000000..f4a8b8b50 --- /dev/null +++ b/Examples/test-suite/unicode_strings.i @@ -0,0 +1,15 @@ +%module unicode_strings + +%include + +%inline %{ + +const char* test_c_str(void) { + return "h\xe9llo"; +} + +std::string test_std_string(void) { + return std::string("h\xe9llo"); +} + +%} From 5fc851a1e0928d5c179c2b9336e76d5e19324d25 Mon Sep 17 00:00:00 2001 From: Harvey Falcic Date: Fri, 23 May 2014 15:24:35 -0400 Subject: [PATCH 3/3] Add Python 3 'surrogateescape' documentation --- Doc/Manual/Python.html | 87 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/Doc/Manual/Python.html b/Doc/Manual/Python.html index bdb1ada30..45725065d 100644 --- a/Doc/Manual/Python.html +++ b/Doc/Manual/Python.html @@ -116,6 +116,7 @@
  • Function annotation
  • Buffer interface
  • Abstract base classes +
  • Byte string output conversion @@ -5928,6 +5929,92 @@ For details of abstract base class, please see PEP 3119.

    +

    35.12.4 Byte string output conversion

    + + +

    +By default, any byte string (char* or std::string) returned +from C or C++ code is decoded to text as UTF-8. This decoding uses the +surrogateescape error handler under Python 3.1 or higher -- this +error handler decodes invalid byte sequences to high surrogate characters +in the range U+DC80 to U+DCFF. + +As an example, consider the following SWIG interface, which exposes a byte +string that cannot be completely decoded as UTF-8: +

    + +
    +%module example
+
+%include <std_string.i>
+
+%inline %{
+
+const char* non_utf8_c_str(void) {
+        return "h\xe9llo w\xc3\xb6rld";
+}
+
+%}
+
    + +

    +When this method is called from Python 3, the return value is the following +text string: +

    + +
    +>>> s = test.non_utf8_c_str()
+>>> s
+'h\udce9llo wörld'
+
    + +

    +Since the C string contains bytes that cannot be decoded as UTF-8, those raw +bytes are represented as high surrogate characters that can be used to obtain +the original byte sequence: +

    + +
    +>>> b = s.encode('utf-8', errors='surrogateescape')
+>>> b
+b'h\xe9llo w\xc3\xb6rld'
+
    + +

    +One can then attempt a different encoding, if desired (or simply leave the +byte string as a raw sequence of bytes for use in binary protocols): +

    + +
    +>>> b.decode('latin-1')
+'héllo wörld'
+
    + +

    +Note, however, that text strings containing surrogate characters are rejected +with the default strict codec error handler. For example: +

    + +
    +>>> with open('test', 'w') as f:
+...     print(s, file=f)
+...
+Traceback (most recent call last):
+  File "<stdin>", line 2, in <module>
+UnicodeEncodeError: 'utf-8' codec can't encode character '\udce9' in position 1: surrogates not allowed
+
    + +

    +This requires the user to check most strings returned by SWIG bindings, but +the alternative is for a non-UTF8 byte string to be completely inaccessible +in Python 3 code. +

    + +

    +For more details about the surrogateescape error handler, please see +PEP 383. +

    +