Ensure that JSON is utf-8 encoded and that bytes sent/received on
the stream sockets are in utf-8 form. Add a test case to verify
that unicode column data can be sent/received successfully using
Python module.

JSON encoder magic to ensure utf-8 encoding  suggested by Terry
Wilson.

Suggested-by: Terry Wilson <twil...@redhat.com>
Signed-off-by: Lance Richardson <lrich...@redhat.com>
---
 python/ovs/json.py    |  9 +++++++--
 python/ovs/jsonrpc.py |  3 +--
 python/ovs/stream.py  |  6 ++++--
 tests/ovsdb-idl.at    | 13 +++++++++++++
 tests/test-ovsdb.py   |  5 ++++-
 5 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/python/ovs/json.py b/python/ovs/json.py
index 37ea9cf38..e84063fc2 100644
--- a/python/ovs/json.py
+++ b/python/ovs/json.py
@@ -29,8 +29,13 @@ except ImportError:
 __pychecker__ = 'no-stringiter'
 
 SPACES_PER_LEVEL = 2
-dumper = functools.partial(json.dumps, separators=(",", ":"),
-                           ensure_ascii=False)
+_dumper = functools.partial(json.dumps, separators=(",", ":"))
+
+if six.PY2:
+    def dumper(*args, **kwargs):
+        return _dumper(*args, **kwargs).decode('raw-unicode-escape')
+else:
+    dumper = _dumper
 
 
 def to_stream(obj, stream, pretty=False, sort_keys=True):
diff --git a/python/ovs/jsonrpc.py b/python/ovs/jsonrpc.py
index 09e9c8b0a..d284190e0 100644
--- a/python/ovs/jsonrpc.py
+++ b/python/ovs/jsonrpc.py
@@ -268,8 +268,7 @@ class Connection(object):
                 # Python 3 has separate types for strings and bytes.  We
                 # received bytes from a socket.  We expect it to be string
                 # data, so we convert it here as soon as possible.
-                if (data and not error
-                        and not isinstance(data, six.string_types)):
+                if data and not error:
                     try:
                         data = data.decode('utf-8')
                     except UnicodeError:
diff --git a/python/ovs/stream.py b/python/ovs/stream.py
index 660d8bbf2..57e7a6eef 100644
--- a/python/ovs/stream.py
+++ b/python/ovs/stream.py
@@ -386,8 +386,10 @@ class Stream(object):
         try:
             # Python 3 has separate types for strings and bytes.  We must have
             # bytes here.
-            if six.PY3 and not isinstance(buf, six.binary_type):
-                buf = six.binary_type(buf, 'utf-8')
+            if six.PY3 and not isinstance(buf, bytes):
+                buf = bytes(buf, 'utf-8')
+            elif six.PY2:
+                buf = buf.encode('utf-8')
             return self.socket.send(buf)
         except socket.error as e:
             return -ovs.socket_util.get_exception_errno(e)
diff --git a/tests/ovsdb-idl.at b/tests/ovsdb-idl.at
index 920feb471..fc541ad50 100644
--- a/tests/ovsdb-idl.at
+++ b/tests/ovsdb-idl.at
@@ -315,6 +315,19 @@ OVSDB_CHECK_IDL([simple idl, writing via IDL],
 005: done
 ]])
 
+OVSDB_CHECK_IDL([simple idl, writing via IDL with unicode],
+  [['["idltest",
+      {"op": "insert",
+       "table": "simple",
+       "row": {"s": "٩(͡๏̯͡๏)۶"}}]']],
+  [['set 0 b 1, insert 1, set 1 s "¯\_(ツ)_/¯"']],
+  [[000: i=0 r=0 b=false s=٩(͡๏̯͡๏)۶ u=<0> ia=[] ra=[] ba=[] sa=[] ua=[] 
uuid=<1>
+001: commit, status=success
+002: i=0 r=0 b=true s=٩(͡๏̯͡๏)۶ u=<0> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<1>
+002: i=1 r=0 b=false s="¯\_(ツ)_/¯" u=<0> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<2>
+003: done
+]])
+
 OVSDB_CHECK_IDL([simple idl, handling verification failure],
   [['["idltest",
       {"op": "insert",
diff --git a/tests/test-ovsdb.py b/tests/test-ovsdb.py
index 4dcc91e91..933e5bd5c 100644
--- a/tests/test-ovsdb.py
+++ b/tests/test-ovsdb.py
@@ -334,7 +334,10 @@ def idl_set(idl, commands, step):
             if args[1] == "b":
                 s.b = args[2] == "1"
             elif args[1] == "s":
-                s.s = args[2]
+                if six.PY2:
+                    s.s = args[2].decode('utf-8')
+                else:
+                    s.s = args[2].encode('utf-8', 
'surrogateescape').decode('utf-8', 'replace')
             elif args[1] == "u":
                 s.u = uuid.UUID(args[2])
             elif args[1] == "r":
-- 
2.13.3

_______________________________________________
dev mailing list
d...@openvswitch.org
https://mail.openvswitch.org/mailman/listinfo/ovs-dev

Reply via email to