Ensure that JSON is utf-8 encoded and that bytes sent/received on the stream sockets are in utf-8 form. Add a test case to verify that unicode data can be sent/received successfully using Python IDL module.
Co-authored-by: Terry Wilson <[email protected]> Signed-off-by: Terry Wilson <[email protected]> Signed-off-by: Lance Richardson <[email protected]> --- v1-v2: - Changed a unicode string in ovsdb-idl.at to avoid an issue in some installations of gnome-terminal (coredump). - Fixed a flake8 line length warning in test-ovsdb.py. - Added co-authored-by/acked-by for Terry Wilson (was suggested-by). python/ovs/json.py | 9 +++++++-- python/ovs/jsonrpc.py | 3 +-- python/ovs/stream.py | 6 ++++-- tests/ovsdb-idl.at | 13 +++++++++++++ tests/test-ovsdb.py | 6 +++++- 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/python/ovs/json.py b/python/ovs/json.py index 37ea9cf38..e84063fc2 100644 --- a/python/ovs/json.py +++ b/python/ovs/json.py @@ -29,8 +29,13 @@ except ImportError: __pychecker__ = 'no-stringiter' SPACES_PER_LEVEL = 2 -dumper = functools.partial(json.dumps, separators=(",", ":"), - ensure_ascii=False) +_dumper = functools.partial(json.dumps, separators=(",", ":")) + +if six.PY2: + def dumper(*args, **kwargs): + return _dumper(*args, **kwargs).decode('raw-unicode-escape') +else: + dumper = _dumper def to_stream(obj, stream, pretty=False, sort_keys=True): diff --git a/python/ovs/jsonrpc.py b/python/ovs/jsonrpc.py index 09e9c8b0a..d284190e0 100644 --- a/python/ovs/jsonrpc.py +++ b/python/ovs/jsonrpc.py @@ -268,8 +268,7 @@ class Connection(object): # Python 3 has separate types for strings and bytes. We # received bytes from a socket. We expect it to be string # data, so we convert it here as soon as possible. - if (data and not error - and not isinstance(data, six.string_types)): + if data and not error: try: data = data.decode('utf-8') except UnicodeError: diff --git a/python/ovs/stream.py b/python/ovs/stream.py index 660d8bbf2..57e7a6eef 100644 --- a/python/ovs/stream.py +++ b/python/ovs/stream.py @@ -386,8 +386,10 @@ class Stream(object): try: # Python 3 has separate types for strings and bytes. We must have # bytes here. - if six.PY3 and not isinstance(buf, six.binary_type): - buf = six.binary_type(buf, 'utf-8') + if six.PY3 and not isinstance(buf, bytes): + buf = bytes(buf, 'utf-8') + elif six.PY2: + buf = buf.encode('utf-8') return self.socket.send(buf) except socket.error as e: return -ovs.socket_util.get_exception_errno(e) diff --git a/tests/ovsdb-idl.at b/tests/ovsdb-idl.at index 920feb471..a5f75febf 100644 --- a/tests/ovsdb-idl.at +++ b/tests/ovsdb-idl.at @@ -315,6 +315,19 @@ OVSDB_CHECK_IDL([simple idl, writing via IDL], 005: done ]]) +OVSDB_CHECK_IDL([simple idl, writing via IDL with unicode], + [['["idltest", + {"op": "insert", + "table": "simple", + "row": {"s": "(╯°□°）╯︵ ┻━┻"}}]']], + [['set 0 b 1, insert 1, set 1 s "¯\_(ツ)_/¯"']], + [[000: i=0 r=0 b=false s=(╯°□°）╯︵ ┻━┻ u=<0> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<1> +001: commit, status=success +002: i=0 r=0 b=true s=(╯°□°）╯︵ ┻━┻ u=<0> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<1> +002: i=1 r=0 b=false s="¯\_(ツ)_/¯" u=<0> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<2> +003: done +]]) + OVSDB_CHECK_IDL([simple idl, handling verification failure], [['["idltest", {"op": "insert", diff --git a/tests/test-ovsdb.py b/tests/test-ovsdb.py index 4dcc91e91..df9d6d58c 100644 --- a/tests/test-ovsdb.py +++ b/tests/test-ovsdb.py @@ -334,7 +334,11 @@ def idl_set(idl, commands, step): if args[1] == "b": s.b = args[2] == "1" elif args[1] == "s": - s.s = args[2] + if six.PY2: + s.s = args[2].decode('utf-8') + else: + s.s = args[2].encode('utf-8', 'surrogateescape') \ + .decode('utf-8', 'replace') elif args[1] == "u": s.u = uuid.UUID(args[2]) elif args[1] == "r": -- 2.13.3 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
