THRIFT-1857 Python 3 Support Client: Python Patch: Thomas Bartelmess, Eevee (Alex Munroe), helgridly, Christian Verkerk, Jeroen Vlek, Nobuaki Sukegawa
This closes #213 and closes #680 Project: http://git-wip-us.apache.org/repos/asf/thrift/repo Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/760511f5 Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/760511f5 Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/760511f5 Branch: refs/heads/master Commit: 760511f59b349c59982a64e249e6cf24c2b2f8f6 Parents: 49f4dc0 Author: Nobuaki Sukegawa <[email protected]> Authored: Fri Nov 6 21:24:16 2015 +0900 Committer: Nobuaki Sukegawa <[email protected]> Committed: Fri Nov 6 21:24:16 2015 +0900 ---------------------------------------------------------------------- build/docker/ubuntu/Dockerfile | 2 +- build/travis/installDependencies.sh | 2 +- compiler/cpp/src/generate/t_py_generator.cc | 138 +++++++++++++-------- contrib/Vagrantfile | 2 +- lib/py/setup.py | 13 +- lib/py/src/TSCons.py | 2 + lib/py/src/TSerialization.py | 4 +- lib/py/src/TTornado.py | 3 +- lib/py/src/compat.py | 27 ++++ lib/py/src/protocol/TBase.py | 24 ++-- lib/py/src/protocol/TBinaryProtocol.py | 10 +- lib/py/src/protocol/TCompactProtocol.py | 24 ++-- lib/py/src/protocol/TJSONProtocol.py | 149 ++++++++++++++--------- lib/py/src/protocol/TProtocol.py | 31 +++-- lib/py/src/server/THttpServer.py | 2 +- lib/py/src/server/TNonblockingServer.py | 25 ++-- lib/py/src/server/TProcessPoolServer.py | 2 +- lib/py/src/server/TServer.py | 8 +- lib/py/src/transport/THttpClient.py | 38 +++--- lib/py/src/transport/TSocket.py | 2 +- lib/py/src/transport/TTransport.py | 46 +++---- lib/py/src/transport/TTwisted.py | 9 +- lib/py/src/transport/TZlibTransport.py | 18 +-- test/py.tornado/test_suite.py | 8 +- test/py.twisted/test_suite.py | 2 +- test/py/RunClientServer.py | 45 +++---- test/py/SerializationTest.py | 10 +- test/py/TSimpleJSONProtocolTest.py | 14 +-- test/py/TestClient.py | 21 +++- test/py/TestEof.py | 4 +- test/py/TestServer.py | 4 +- test/py/TestSocket.py | 6 +- test/py/TestSyntax.py | 2 +- tutorial/py.tornado/PythonClient.py | 18 +-- tutorial/py.tornado/PythonServer.py | 16 +-- tutorial/py.twisted/PythonClient.py | 16 +-- tutorial/py.twisted/PythonServer.py | 14 +-- tutorial/py.twisted/PythonServer.tac | 2 +- tutorial/py/PythonClient.py | 20 +-- tutorial/py/PythonServer.py | 16 +-- 40 files changed, 460 insertions(+), 339 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/build/docker/ubuntu/Dockerfile ---------------------------------------------------------------------- diff --git a/build/docker/ubuntu/Dockerfile b/build/docker/ubuntu/Dockerfile index 4ad94e1..0b02a70 100644 --- a/build/docker/ubuntu/Dockerfile +++ b/build/docker/ubuntu/Dockerfile @@ -37,7 +37,7 @@ RUN apt-get install -y ant openjdk-7-jdk maven && \ # Python dependencies RUN apt-get install -y python-all python-all-dev python-all-dbg python-setuptools python-support \ - python-twisted python-zope.interface + python-twisted python-zope.interface python-six # Ruby dependencies RUN apt-get install -y ruby ruby-dev && \ http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/build/travis/installDependencies.sh ---------------------------------------------------------------------- diff --git a/build/travis/installDependencies.sh b/build/travis/installDependencies.sh index 453494c..df12640 100755 --- a/build/travis/installDependencies.sh +++ b/build/travis/installDependencies.sh @@ -27,7 +27,7 @@ sudo apt-get install -qq ant openjdk-7-jdk sudo update-java-alternatives -s java-1.7.0-openjdk-amd64 # Python dependencies -sudo apt-get install -qq python-all python-all-dev python-all-dbg python-setuptools python-support python-twisted +sudo apt-get install -qq python-all python-all-dev python-all-dbg python-setuptools python-support python-twisted python-six # Ruby dependencies sudo apt-get install -qq ruby ruby-dev http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/compiler/cpp/src/generate/t_py_generator.cc ---------------------------------------------------------------------- diff --git a/compiler/cpp/src/generate/t_py_generator.cc b/compiler/cpp/src/generate/t_py_generator.cc index fce29cf..74904ac 100644 --- a/compiler/cpp/src/generate/t_py_generator.cc +++ b/compiler/cpp/src/generate/t_py_generator.cc @@ -349,11 +349,18 @@ void t_py_generator::init_generator() { f_init.close(); // Print header - f_types_ << py_autogen_comment() << endl << py_imports() << endl << render_includes() << endl - << render_fastbinary_includes() << endl << endl; - - f_consts_ << py_autogen_comment() << endl << py_imports() << endl << "from ttypes import *" - << endl << endl; + f_types_ << + py_autogen_comment() << endl << + py_imports() << endl << + render_includes() << endl << + render_fastbinary_includes() << + endl << endl; + + f_consts_ << + py_autogen_comment() << endl << + py_imports() << endl << + "from .ttypes import *" << endl << + endl; } /** @@ -759,10 +766,12 @@ void t_py_generator::generate_py_struct_definition(ofstream& out, if (!gen_slots_) { // Printing utilities so that on the command line thrift // structs look pretty like dictionaries - out << indent() << "def __repr__(self):" << endl << indent() << " L = ['%s=%r' % (key, value)" - << endl << indent() << " for key, value in self.__dict__.iteritems()]" << endl - << indent() << " return '%s(%s)' % (self.__class__.__name__, ', '.join(L))" << endl - << endl; + out << + indent() << "def __repr__(self):" << endl << + indent() << " L = ['%s=%r' % (key, value)" << endl << + indent() << " for key, value in self.__dict__.items()]" << endl << + indent() << " return '%s(%s)' % (self.__class__.__name__, ', '.join(L))" << endl << + endl; // Equality and inequality methods that compare by value out << indent() << "def __eq__(self, other):" << endl; @@ -961,7 +970,7 @@ void t_py_generator::generate_service(t_service* tservice) { } f_service_ << "import logging" << endl - << "from ttypes import *" << endl + << "from .ttypes import *" << endl << "from thrift.Thrift import TProcessor" << endl << render_fastbinary_includes() << endl; @@ -1141,26 +1150,32 @@ void t_py_generator::generate_service_client(t_service* tservice) { } if (gen_tornado_ && extends.empty()) { - f_service_ << indent() << "@gen.engine" << endl << indent() - << "def _start_receiving(self):" << endl << indent() << " while True:" << endl - << indent() << " try:" << endl << indent() - << " frame = yield self._transport.readFrame()" << endl << indent() - << " except TTransport.TTransportException as e:" << endl << indent() - << " for future in self._reqs.itervalues():" << endl << indent() - << " future.set_exception(e)" << endl << indent() << " self._reqs = {}" - << endl << indent() << " return" << endl << indent() - << " tr = TTransport.TMemoryBuffer(frame)" << endl << indent() - << " iprot = self._iprot_factory.getProtocol(tr)" << endl << indent() - << " (fname, mtype, rseqid) = iprot.readMessageBegin()" << endl << indent() - << " future = self._reqs.pop(rseqid, None)" << endl << indent() - << " if not future:" << endl << indent() - << " # future has already been discarded" << endl << indent() - << " continue" << endl << indent() - << " method = getattr(self, 'recv_' + fname)" << endl << indent() - << " try:" << endl << indent() << " result = method(iprot, mtype, rseqid)" - << endl << indent() << " except Exception as e:" << endl << indent() - << " future.set_exception(e)" << endl << indent() << " else:" << endl - << indent() << " future.set_result(result)" << endl << endl; + f_service_ << + indent() << "@gen.engine" << endl << + indent() << "def _start_receiving(self):" << endl << + indent() << " while True:" << endl << + indent() << " try:" << endl << + indent() << " frame = yield self._transport.readFrame()" << endl << + indent() << " except TTransport.TTransportException as e:" << endl << + indent() << " for future in self._reqs.values():" << endl << + indent() << " future.set_exception(e)" << endl << + indent() << " self._reqs = {}" << endl << + indent() << " return" << endl << + indent() << " tr = TTransport.TMemoryBuffer(frame)" << endl << + indent() << " iprot = self._iprot_factory.getProtocol(tr)" << endl << + indent() << " (fname, mtype, rseqid) = iprot.readMessageBegin()" << endl << + indent() << " method = getattr(self, 'recv_' + fname)" << endl << + indent() << " future = self._reqs.pop(rseqid, None)" << endl << + indent() << " if not future:" << endl << + indent() << " # future has already been discarded" << endl << + indent() << " continue" << endl << + indent() << " try:" << endl << + indent() << " result = method(iprot, mtype, rseqid)" << endl << + indent() << " except Exception as e:" << endl << + indent() << " future.set_exception(e)" << endl << + indent() << " else:" << endl << + indent() << " future.set_result(result)" << endl << + endl; } // Generate client method implementations @@ -1409,21 +1424,33 @@ void t_py_generator::generate_service_remote(t_service* tservice) { ofstream f_remote; f_remote.open(f_remote_name.c_str()); - f_remote << "#!/usr/bin/env python" << endl << py_autogen_comment() << endl << "import sys" - << endl << "import pprint" << endl << "from urlparse import urlparse" << endl - << "from thrift.transport import TTransport" << endl - << "from thrift.transport import TSocket" << endl - << "from thrift.transport import TSSLSocket" << endl - << "from thrift.transport import THttpClient" << endl - << "from thrift.protocol import TBinaryProtocol" << endl << endl; - - f_remote << "from " << module_ << " import " << service_name_ << endl << "from " << module_ - << ".ttypes import *" << endl << endl; - - f_remote << "if len(sys.argv) <= 1 or sys.argv[1] == '--help':" << endl << " print('')" << endl - << " print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] " - "function [arg1 [arg2...]]')" << endl << " print('')" << endl - << " print('Functions:')" << endl; + f_remote << + "#!/usr/bin/env python" << endl << + py_autogen_comment() << endl << + "import sys" << endl << + "import pprint" << endl << + "if sys.version_info[0] == 3:" << endl << + " from urllib.parse import urlparse" << endl << + "else:" << endl << + " from urlparse import urlparse" << endl << + "from thrift.transport import TTransport" << endl << + "from thrift.transport import TSocket" << endl << + "from thrift.transport import TSSLSocket" << endl << + "from thrift.transport import THttpClient" << endl << + "from thrift.protocol import TBinaryProtocol" << endl << + endl; + + f_remote << + "from " << module_ << " import " << service_name_ << endl << + "from " << module_ << ".ttypes import *" << endl << + endl; + + f_remote << + "if len(sys.argv) <= 1 or sys.argv[1] == '--help':" << endl << + " print('')" << endl << + " print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] function [arg1 [arg2...]]')" << endl << + " print('')" << endl << + " print('Functions:')" << endl; for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { f_remote << " print(' " << (*f_iter)->get_returntype()->get_name() << " " << (*f_iter)->get_name() << "("; @@ -1720,8 +1747,8 @@ void t_py_generator::generate_process_function(t_service* tservice, t_function* // Kinda absurd f_service_ << indent() << " error.raiseException()" << endl; for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { - f_service_ << indent() << "except " << type_name((*x_iter)->get_type()) << " as " - << (*x_iter)->get_name() << ":" << endl; + f_service_ << + indent() << "except " << type_name((*x_iter)->get_type()) << " as " << (*x_iter)->get_name() << ":" << endl; if (!tfunction->is_oneway()) { indent_up(); f_service_ << indent() << "result." << (*x_iter)->get_name() << " = " @@ -1854,11 +1881,15 @@ void t_py_generator::generate_process_function(t_service* tservice, t_function* for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { f_service_ << indent() << "except " << type_name((*x_iter)->get_type()) << " as " << (*x_iter)->get_name() << ":" << endl; - indent_up(); - f_service_ << indent() << "msg_type = TMessageType.REPLY" << endl - << indent() << "result." << (*x_iter)->get_name() << " = " - << (*x_iter)->get_name() << endl; - indent_down(); + if (!tfunction->is_oneway()) { + indent_up(); + f_service_ << indent() << "msg_type = TMessageType.REPLY" << endl; + f_service_ << indent() << "result." << (*x_iter)->get_name() << " = " + << (*x_iter)->get_name() << endl; + indent_down(); + } else { + f_service_ << indent() << "pass" << endl; + } } f_service_ << indent() << "except Exception as ex:" << endl @@ -1989,7 +2020,8 @@ void t_py_generator::generate_deserialize_container(ofstream& out, t_type* ttype // For loop iterates over elements string i = tmp("_i"); - indent(out) << "for " << i << " in xrange(" << size << "):" << endl; + indent(out) << + "for " << i << " in range(" << size << "):" << endl; indent_up(); http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/contrib/Vagrantfile ---------------------------------------------------------------------- diff --git a/contrib/Vagrantfile b/contrib/Vagrantfile index f967f04..2091110 100644 --- a/contrib/Vagrantfile +++ b/contrib/Vagrantfile @@ -46,7 +46,7 @@ sudo apt-get install -qq libboost-dev libboost-test-dev libboost-program-options sudo apt-get install -qq ant openjdk-7-jdk maven # Python dependencies -sudo apt-get install -qq python-all python-all-dev python-all-dbg python-setuptools python-support +sudo apt-get install -qq python-all python-all-dev python-all-dbg python-setuptools python-support python-six # Ruby dependencies sudo apt-get install -qq ruby ruby-dev http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/setup.py ---------------------------------------------------------------------- diff --git a/lib/py/setup.py b/lib/py/setup.py index 7e0a964..090544c 100644 --- a/lib/py/setup.py +++ b/lib/py/setup.py @@ -19,6 +19,7 @@ # under the License. # +import platform import sys try: from setuptools import setup, Extension @@ -69,7 +70,7 @@ def run_setup(with_binary): ) else: extensions = dict() - + setup(name = 'thrift', version = '1.0.0-dev', description = 'Python bindings for the Apache Thrift RPC system', @@ -77,6 +78,7 @@ def run_setup(with_binary): author_email = '[email protected]', url = 'http://thrift.apache.org', license = 'Apache License 2.0', + install_requires=['six>=1.7.2'], packages = [ 'thrift', 'thrift.protocol', @@ -90,15 +92,20 @@ def run_setup(with_binary): 'Intended Audience :: Developers', 'Programming Language :: Python', 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 3', 'Topic :: Software Development :: Libraries', 'Topic :: System :: Networking' ], - use_2to3 = True, **extensions ) try: - run_setup(True) + with_binary = False + # Don't even try to build the C module unless we're on CPython 2.x. + # TODO: fix it for CPython 3.x + if platform.python_implementation() == 'CPython' and sys.version_info < (3,): + with_binary = True + run_setup(with_binary) except BuildFailed: print() print('*' * 80) http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/TSCons.py ---------------------------------------------------------------------- diff --git a/lib/py/src/TSCons.py b/lib/py/src/TSCons.py index da8d283..ed2601a 100644 --- a/lib/py/src/TSCons.py +++ b/lib/py/src/TSCons.py @@ -19,6 +19,8 @@ from os import path from SCons.Builder import Builder +from six.moves import map +from six.moves import zip def scons_env(env, add=''): http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/TSerialization.py ---------------------------------------------------------------------- diff --git a/lib/py/src/TSerialization.py b/lib/py/src/TSerialization.py index 8a58d89..fbbe768 100644 --- a/lib/py/src/TSerialization.py +++ b/lib/py/src/TSerialization.py @@ -17,8 +17,8 @@ # under the License. # -from protocol import TBinaryProtocol -from transport import TTransport +from .protocol import TBinaryProtocol +from .transport import TTransport def serialize(thrift_object, http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/TTornado.py ---------------------------------------------------------------------- diff --git a/lib/py/src/TTornado.py b/lib/py/src/TTornado.py index 65795ab..e3b4df7 100644 --- a/lib/py/src/TTornado.py +++ b/lib/py/src/TTornado.py @@ -20,11 +20,10 @@ from __future__ import absolute_import import socket import struct - import logging logger = logging.getLogger(__name__) -from thrift.transport.TTransport import TTransportException, TTransportBase, TMemoryBuffer +from .transport.TTransport import TTransportException, TTransportBase, TMemoryBuffer from io import BytesIO from collections import deque http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/compat.py ---------------------------------------------------------------------- diff --git a/lib/py/src/compat.py b/lib/py/src/compat.py new file mode 100644 index 0000000..b2f47dc --- /dev/null +++ b/lib/py/src/compat.py @@ -0,0 +1,27 @@ +import sys + +if sys.version_info[0] == 2: + + from cStringIO import StringIO as BufferIO + + def binary_to_str(bin_val): + return bin_val + + def str_to_binary(str_val): + return str_val + +else: + + from io import BytesIO as BufferIO + + def binary_to_str(bin_val): + try: + return bin_val.decode('utf8') + except: + return bin_val + + def str_to_binary(str_val): + try: + return bytearray(str_val, 'utf8') + except: + return str_val http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/protocol/TBase.py ---------------------------------------------------------------------- diff --git a/lib/py/src/protocol/TBase.py b/lib/py/src/protocol/TBase.py index 6cbd5f3..118a679 100644 --- a/lib/py/src/protocol/TBase.py +++ b/lib/py/src/protocol/TBase.py @@ -17,7 +17,6 @@ # under the License. # -from thrift.Thrift import * from thrift.protocol import TBinaryProtocol from thrift.transport import TTransport @@ -31,8 +30,7 @@ class TBase(object): __slots__ = [] def __repr__(self): - L = ['%s=%r' % (key, getattr(self, key)) - for key in self.__slots__] + L = ['%s=%r' % (key, getattr(self, key)) for key in self.__slots__] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): @@ -50,9 +48,9 @@ class TBase(object): def read(self, iprot): if (iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and - isinstance(iprot.trans, TTransport.CReadableTransport) and - self.thrift_spec is not None and - fastbinary is not None): + isinstance(iprot.trans, TTransport.CReadableTransport) and + self.thrift_spec is not None and + fastbinary is not None): fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) @@ -61,21 +59,13 @@ class TBase(object): def write(self, oprot): if (oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and - self.thrift_spec is not None and - fastbinary is not None): + self.thrift_spec is not None and + fastbinary is not None): oprot.trans.write( fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStruct(self, self.thrift_spec) -class TExceptionBase(Exception): - # old style class so python2.4 can raise exceptions derived from this - # This can't inherit from TBase because of that limitation. +class TExceptionBase(TBase, Exception): __slots__ = [] - - __repr__ = TBase.__repr__.im_func - __eq__ = TBase.__eq__.im_func - __ne__ = TBase.__ne__.im_func - read = TBase.read.im_func - write = TBase.write.im_func http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/protocol/TBinaryProtocol.py ---------------------------------------------------------------------- diff --git a/lib/py/src/protocol/TBinaryProtocol.py b/lib/py/src/protocol/TBinaryProtocol.py index 6fdd08c..f92f558 100644 --- a/lib/py/src/protocol/TBinaryProtocol.py +++ b/lib/py/src/protocol/TBinaryProtocol.py @@ -17,7 +17,7 @@ # under the License. # -from TProtocol import * +from .TProtocol import TType, TProtocolBase, TProtocolException from struct import pack, unpack @@ -118,7 +118,7 @@ class TBinaryProtocol(TProtocolBase): buff = pack("!d", dub) self.trans.write(buff) - def writeString(self, str): + def writeBinary(self, str): self.writeI32(len(str)) self.trans.write(str) @@ -217,10 +217,10 @@ class TBinaryProtocol(TProtocolBase): val, = unpack('!d', buff) return val - def readString(self): + def readBinary(self): len = self.readI32() - str = self.trans.readAll(len) - return str + s = self.trans.readAll(len) + return s class TBinaryProtocolFactory: http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/protocol/TCompactProtocol.py ---------------------------------------------------------------------- diff --git a/lib/py/src/protocol/TCompactProtocol.py b/lib/py/src/protocol/TCompactProtocol.py index 7054ab0..b8d171e 100644 --- a/lib/py/src/protocol/TCompactProtocol.py +++ b/lib/py/src/protocol/TCompactProtocol.py @@ -17,9 +17,11 @@ # under the License. # -from TProtocol import * +from .TProtocol import TType, TProtocolBase, TProtocolException, checkIntegerLimits from struct import pack, unpack +from ..compat import binary_to_str, str_to_binary + __all__ = ['TCompactProtocol', 'TCompactProtocolFactory'] CLEAR = 0 @@ -62,7 +64,7 @@ def writeVarint(trans, n): else: out.append((n & 0xff) | 0x80) n = n >> 7 - trans.write(''.join(map(chr, out))) + trans.write(bytearray(out)) def readVarint(trans): @@ -141,7 +143,7 @@ class TCompactProtocol(TProtocolBase): self.__writeUByte(self.PROTOCOL_ID) self.__writeUByte(self.VERSION | (type << self.TYPE_SHIFT_AMOUNT)) self.__writeVarint(seqid) - self.__writeString(name) + self.__writeBinary(str_to_binary(name)) self.state = VALUE_WRITE def writeMessageEnd(self): @@ -254,10 +256,10 @@ class TCompactProtocol(TProtocolBase): def writeDouble(self, dub): self.trans.write(pack('<d', dub)) - def __writeString(self, s): + def __writeBinary(self, s): self.__writeSize(len(s)) self.trans.write(s) - writeString = writer(__writeString) + writeBinary = writer(__writeBinary) def readFieldBegin(self): assert self.state == FIELD_READ, self.state @@ -302,7 +304,7 @@ class TCompactProtocol(TProtocolBase): def __readSize(self): result = self.__readVarint() if result < 0: - raise TException("Length < 0") + raise TProtocolException("Length < 0") return result def readMessageBegin(self): @@ -310,15 +312,15 @@ class TCompactProtocol(TProtocolBase): proto_id = self.__readUByte() if proto_id != self.PROTOCOL_ID: raise TProtocolException(TProtocolException.BAD_VERSION, - 'Bad protocol id in the message: %d' % proto_id) + 'Bad protocol id in the message: %d' % proto_id) ver_type = self.__readUByte() type = (ver_type >> self.TYPE_SHIFT_AMOUNT) & self.TYPE_BITS version = ver_type & self.VERSION_MASK if version != self.VERSION: raise TProtocolException(TProtocolException.BAD_VERSION, - 'Bad version: %d (expect %d)' % (version, self.VERSION)) + 'Bad version: %d (expect %d)' % (version, self.VERSION)) seqid = self.__readVarint() - name = self.__readString() + name = binary_to_str(self.__readBinary()) return (name, type, seqid) def readMessageEnd(self): @@ -388,10 +390,10 @@ class TCompactProtocol(TProtocolBase): val, = unpack('<d', buff) return val - def __readString(self): + def __readBinary(self): len = self.__readSize() return self.trans.readAll(len) - readString = reader(__readString) + readBinary = reader(__readBinary) def __getTType(self, byte): return TTYPES[byte & 0x0f] http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/protocol/TJSONProtocol.py ---------------------------------------------------------------------- diff --git a/lib/py/src/protocol/TJSONProtocol.py b/lib/py/src/protocol/TJSONProtocol.py index 7807a6c..3ed8bcb 100644 --- a/lib/py/src/protocol/TJSONProtocol.py +++ b/lib/py/src/protocol/TJSONProtocol.py @@ -17,11 +17,13 @@ # under the License. # -from TProtocol import TType, TProtocolBase, TProtocolException, \ - checkIntegerLimits +from .TProtocol import TType, TProtocolBase, TProtocolException, checkIntegerLimits import base64 -import json import math +import sys + +from ..compat import str_to_binary + __all__ = ['TJSONProtocol', 'TJSONProtocolFactory', @@ -30,20 +32,39 @@ __all__ = ['TJSONProtocol', VERSION = 1 -COMMA = ',' -COLON = ':' -LBRACE = '{' -RBRACE = '}' -LBRACKET = '[' -RBRACKET = ']' -QUOTE = '"' -BACKSLASH = '\\' -ZERO = '0' - -ESCSEQ = '\\u00' -ESCAPE_CHAR = '"\\bfnrt/' -ESCAPE_CHAR_VALS = ['"', '\\', '\b', '\f', '\n', '\r', '\t', '/'] -NUMERIC_CHAR = '+-.0123456789Ee' +COMMA = b',' +COLON = b':' +LBRACE = b'{' +RBRACE = b'}' +LBRACKET = b'[' +RBRACKET = b']' +QUOTE = b'"' +BACKSLASH = b'\\' +ZERO = b'0' + +ESCSEQ0 = ord('\\') +ESCSEQ1 = ord('u') +ESCAPE_CHAR_VALS = { + '"': '\\"', + '\\': '\\\\', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', + # '/': '\\/', +} +ESCAPE_CHARS = { + b'"': '"', + b'\\': '\\', + b'b': '\b', + b'f': '\f', + b'n': '\n', + b'r': '\r', + b't': '\t', + b'/': '/', +} +NUMERIC_CHAR = b'+-.0123456789Ee' CTYPES = {TType.BOOL: 'tf', TType.BYTE: 'i8', @@ -70,7 +91,7 @@ class JSONBaseContext(object): def doIO(self, function): pass - + def write(self): pass @@ -85,7 +106,7 @@ class JSONBaseContext(object): class JSONListContext(JSONBaseContext): - + def doIO(self, function): if self.first is True: self.first = False @@ -100,7 +121,7 @@ class JSONListContext(JSONBaseContext): class JSONPairContext(JSONBaseContext): - + def __init__(self, protocol): super(JSONPairContext, self).__init__(protocol) self.colon = True @@ -146,6 +167,7 @@ class LookaheadReader(): self.hasData = True return self.data + class TJSONProtocolBase(TProtocolBase): def __init__(self, trans): @@ -174,14 +196,22 @@ class TJSONProtocolBase(TProtocolBase): def writeJSONString(self, string): self.context.write() - self.trans.write(json.dumps(string, ensure_ascii=False)) + json_str = ['"'] + for s in string: + escaped = ESCAPE_CHAR_VALS.get(s, s) + json_str.append(escaped) + json_str.append('"') + self.trans.write(str_to_binary(''.join(json_str))) def writeJSONNumber(self, number, formatter='{}'): self.context.write() - jsNumber = formatter.format(number) + jsNumber = str(formatter.format(number)).encode('ascii') if self.context.escapeNum(): - jsNumber = "%s%s%s" % (QUOTE, jsNumber, QUOTE) - self.trans.write(jsNumber) + self.trans.write(QUOTE) + self.trans.write(jsNumber) + self.trans.write(QUOTE) + else: + self.trans.write(jsNumber) def writeJSONBase64(self, binary): self.context.write() @@ -222,18 +252,23 @@ class TJSONProtocolBase(TProtocolBase): character = self.reader.read() if character == QUOTE: break - if character == ESCSEQ[0]: + if ord(character) == ESCSEQ0: character = self.reader.read() - if character == ESCSEQ[1]: - self.readJSONSyntaxChar(ZERO) - self.readJSONSyntaxChar(ZERO) - character = json.JSONDecoder().decode('"\u00%s"' % self.trans.read(2)) + if ord(character) == ESCSEQ1: + character = chr(int(self.trans.read(4))) else: - off = ESCAPE_CHAR.find(character) - if off == -1: + if character not in ESCAPE_CHARS: raise TProtocolException(TProtocolException.INVALID_DATA, "Expected control char") - character = ESCAPE_CHAR_VALS[off] + character = ESCAPE_CHARS[character] + elif character in ESCAPE_CHAR_VALS: + raise TProtocolException(TProtocolException.INVALID_DATA, + "Unescaped control char") + elif sys.version_info[0] > 2: + utf8_bytes = bytearray([ord(character)]) + while ord(self.reader.peek()) >= 0x80: + utf8_bytes.append(ord(self.reader.read())) + character = utf8_bytes.decode('utf8') string.append(character) return ''.join(string) @@ -251,7 +286,7 @@ class TJSONProtocolBase(TProtocolBase): if self.isJSONNumeric(character) is False: break numeric.append(self.reader.read()) - return ''.join(numeric) + return b''.join(numeric).decode('ascii') def readJSONInteger(self): self.context.read() @@ -267,12 +302,12 @@ class TJSONProtocolBase(TProtocolBase): def readJSONDouble(self): self.context.read() if self.reader.peek() == QUOTE: - string = self.readJSONString(True) + string = self.readJSONString(True) try: double = float(string) if (self.context.escapeNum is False and - not math.isinf(double) and - not math.isnan(double)): + not math.isinf(double) and + not math.isnan(double)): raise TProtocolException(TProtocolException.INVALID_DATA, "Numeric data unexpectedly quoted") return double @@ -430,12 +465,12 @@ class TJSONProtocol(TJSONProtocolBase): def writeMapEnd(self): self.writeJSONObjectEnd() self.writeJSONArrayEnd() - + def writeListBegin(self, etype, size): self.writeJSONArrayStart() self.writeJSONString(CTYPES[etype]) self.writeJSONNumber(size) - + def writeListEnd(self): self.writeJSONArrayEnd() @@ -443,7 +478,7 @@ class TJSONProtocol(TJSONProtocolBase): self.writeJSONArrayStart() self.writeJSONString(CTYPES[etype]) self.writeJSONNumber(size) - + def writeSetEnd(self): self.writeJSONArrayEnd() @@ -472,7 +507,7 @@ class TJSONProtocol(TJSONProtocolBase): def writeString(self, string): self.writeJSONString(string) - + def writeBinary(self, binary): self.writeJSONBase64(binary) @@ -485,49 +520,49 @@ class TJSONProtocolFactory: class TSimpleJSONProtocol(TJSONProtocolBase): """Simple, readable, write-only JSON protocol. - + Useful for interacting with scripting languages. """ def readMessageBegin(self): raise NotImplementedError() - + def readMessageEnd(self): raise NotImplementedError() - + def readStructBegin(self): raise NotImplementedError() - + def readStructEnd(self): raise NotImplementedError() - + def writeMessageBegin(self, name, request_type, seqid): self.resetWriteContext() - + def writeMessageEnd(self): pass - + def writeStructBegin(self, name): self.writeJSONObjectStart() - + def writeStructEnd(self): self.writeJSONObjectEnd() - + def writeFieldBegin(self, name, ttype, fid): self.writeJSONString(name) - + def writeFieldEnd(self): pass - + def writeMapBegin(self, ktype, vtype, size): self.writeJSONObjectStart() - + def writeMapEnd(self): self.writeJSONObjectEnd() - + def _writeCollectionBegin(self, etype, size): self.writeJSONArrayStart() - + def _writeCollectionEnd(self): self.writeJSONArrayEnd() writeListBegin = _writeCollectionBegin @@ -550,16 +585,16 @@ class TSimpleJSONProtocol(TJSONProtocolBase): def writeI64(self, i64): checkIntegerLimits(i64, 64) self.writeJSONNumber(i64) - + def writeBool(self, boolean): self.writeJSONNumber(1 if boolean is True else 0) def writeDouble(self, dbl): self.writeJSONNumber(dbl) - + def writeString(self, string): self.writeJSONString(string) - + def writeBinary(self, binary): self.writeJSONBase64(binary) http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/protocol/TProtocol.py ---------------------------------------------------------------------- diff --git a/lib/py/src/protocol/TProtocol.py b/lib/py/src/protocol/TProtocol.py index 311a635..22339c0 100644 --- a/lib/py/src/protocol/TProtocol.py +++ b/lib/py/src/protocol/TProtocol.py @@ -17,7 +17,10 @@ # under the License. # -from thrift.Thrift import * +from thrift.Thrift import TException, TType +import six + +from ..compat import binary_to_str, str_to_binary class TProtocolException(TException): @@ -100,6 +103,9 @@ class TProtocolBase: pass def writeString(self, str_val): + self.writeBinary(str_to_binary(str_val)) + + def writeBinary(self, str_val): pass def readMessageBegin(self): @@ -157,6 +163,9 @@ class TProtocolBase: pass def readString(self): + return binary_to_str(self.readBinary()) + + def readBinary(self): pass def skip(self, ttype): @@ -187,18 +196,18 @@ class TProtocolBase: self.readStructEnd() elif ttype == TType.MAP: (ktype, vtype, size) = self.readMapBegin() - for i in xrange(size): + for i in range(size): self.skip(ktype) self.skip(vtype) self.readMapEnd() elif ttype == TType.SET: (etype, size) = self.readSetBegin() - for i in xrange(size): + for i in range(size): self.skip(etype) self.readSetEnd() elif ttype == TType.LIST: (etype, size) = self.readListBegin() - for i in xrange(size): + for i in range(size): self.skip(etype) self.readListEnd() @@ -246,13 +255,13 @@ class TProtocolBase: (list_type, list_len) = self.readListBegin() if tspec is None: # list values are simple types - for idx in xrange(list_len): + for idx in range(list_len): results.append(reader()) else: # this is like an inlined readFieldByTType container_reader = self._TTYPE_HANDLERS[list_type][0] val_reader = getattr(self, container_reader) - for idx in xrange(list_len): + for idx in range(list_len): val = val_reader(tspec) results.append(val) self.readListEnd() @@ -266,12 +275,12 @@ class TProtocolBase: (set_type, set_len) = self.readSetBegin() if tspec is None: # set members are simple types - for idx in xrange(set_len): + for idx in range(set_len): results.add(reader()) else: container_reader = self._TTYPE_HANDLERS[set_type][0] val_reader = getattr(self, container_reader) - for idx in xrange(set_len): + for idx in range(set_len): results.add(val_reader(tspec)) self.readSetEnd() return results @@ -292,7 +301,7 @@ class TProtocolBase: key_reader = getattr(self, self._TTYPE_HANDLERS[key_ttype][0]) val_reader = getattr(self, self._TTYPE_HANDLERS[val_ttype][0]) # list values are simple types - for idx in xrange(map_len): + for idx in range(map_len): if key_spec is None: k_val = key_reader() else: @@ -363,7 +372,7 @@ class TProtocolBase: k_writer = getattr(self, ktype_name) v_writer = getattr(self, vtype_name) self.writeMapBegin(k_type, v_type, len(val)) - for m_key, m_val in val.iteritems(): + for m_key, m_val in six.iteritems(val): if not k_is_container: k_writer(m_key) else: @@ -402,6 +411,7 @@ class TProtocolBase: else: writer(val) + def checkIntegerLimits(i, bits): if bits == 8 and (i < -128 or i > 127): raise TProtocolException(TProtocolException.INVALID_DATA, @@ -416,6 +426,7 @@ def checkIntegerLimits(i, bits): raise TProtocolException(TProtocolException.INVALID_DATA, "i64 requires -9223372036854775808 <= number <= 9223372036854775807") + class TProtocolFactory: def getProtocol(self, trans): pass http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/server/THttpServer.py ---------------------------------------------------------------------- diff --git a/lib/py/src/server/THttpServer.py b/lib/py/src/server/THttpServer.py index 6f92173..bf3b0e3 100644 --- a/lib/py/src/server/THttpServer.py +++ b/lib/py/src/server/THttpServer.py @@ -17,7 +17,7 @@ # under the License. # -import BaseHTTPServer +from six.moves import BaseHTTPServer from thrift.server import TServer from thrift.transport import TTransport http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/server/TNonblockingServer.py ---------------------------------------------------------------------- diff --git a/lib/py/src/server/TNonblockingServer.py b/lib/py/src/server/TNonblockingServer.py index 39486cd..79748b7 100644 --- a/lib/py/src/server/TNonblockingServer.py +++ b/lib/py/src/server/TNonblockingServer.py @@ -26,13 +26,14 @@ maximum connections """ import threading import socket -import Queue import select import struct import logging logger = logging.getLogger(__name__) +from six.moves import queue + from thrift.transport import TTransport from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory @@ -57,7 +58,7 @@ class Worker(threading.Thread): callback(True, otrans.getvalue()) except Exception: logger.exception("Exception while processing request") - callback(False, '') + callback(False, b'') WAIT_LEN = 0 WAIT_MESSAGE = 1 @@ -104,7 +105,7 @@ class Connection: self.socket.setblocking(False) self.status = WAIT_LEN self.len = 0 - self.message = '' + self.message = b'' self.lock = threading.Lock() self.wake_up = wake_up @@ -132,7 +133,7 @@ class Connection: logger.error("empty frame, it's really strange") self.close() else: - self.message = '' + self.message = b'' self.status = WAIT_MESSAGE @socket_exception @@ -162,7 +163,7 @@ class Connection: sent = self.socket.send(self.message) if sent == len(self.message): self.status = WAIT_LEN - self.message = '' + self.message = b'' self.len = 0 else: self.message = self.message[sent:] @@ -185,10 +186,10 @@ class Connection: self.close() self.wake_up() return - self.len = '' + self.len = 0 if len(message) == 0: # it was a oneway request, do not write answer - self.message = '' + self.message = b'' self.status = WAIT_LEN else: self.message = struct.pack('!i', len(message)) + message @@ -236,7 +237,7 @@ class TNonblockingServer: self.out_protocol = outputProtocolFactory or self.in_protocol self.threads = int(threads) self.clients = {} - self.tasks = Queue.Queue() + self.tasks = queue.Queue() self._read, self._write = socket.socketpair() self.prepared = False self._stop = False @@ -252,7 +253,7 @@ class TNonblockingServer: if self.prepared: return self.socket.listen() - for _ in xrange(self.threads): + for _ in range(self.threads): thread = Worker(self.tasks) thread.setDaemon(True) thread.start() @@ -269,7 +270,7 @@ class TNonblockingServer: In this case, we can just write anything to the second socket from socketpair. """ - self._write.send('1') + self._write.send(b'1') def stop(self): """Stop the server. @@ -290,7 +291,7 @@ class TNonblockingServer: """Does select on open connections.""" readable = [self.socket.handle.fileno(), self._read.fileno()] writable = [] - for i, connection in self.clients.items(): + for i, connection in list(self.clients.items()): if connection.is_readable(): readable.append(connection.fileno()) if connection.is_writeable(): @@ -332,7 +333,7 @@ class TNonblockingServer: def close(self): """Closes the server.""" - for _ in xrange(self.threads): + for _ in range(self.threads): self.tasks.put([None, None, None, None, None]) self.socket.close() self.prepared = False http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/server/TProcessPoolServer.py ---------------------------------------------------------------------- diff --git a/lib/py/src/server/TProcessPoolServer.py b/lib/py/src/server/TProcessPoolServer.py index ae7fe1c..b2c2308 100644 --- a/lib/py/src/server/TProcessPoolServer.py +++ b/lib/py/src/server/TProcessPoolServer.py @@ -23,7 +23,7 @@ logger = logging.getLogger(__name__) from multiprocessing import Process, Value, Condition, reduction -from TServer import TServer +from .TServer import TServer from thrift.transport.TTransport import TTransportException http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/server/TServer.py ---------------------------------------------------------------------- diff --git a/lib/py/src/server/TServer.py b/lib/py/src/server/TServer.py index 8c58e39..130f730 100644 --- a/lib/py/src/server/TServer.py +++ b/lib/py/src/server/TServer.py @@ -17,7 +17,7 @@ # under the License. # -import Queue +from six.moves import queue import os import sys import threading @@ -139,7 +139,7 @@ class TThreadPoolServer(TServer): def __init__(self, *args, **kwargs): TServer.__init__(self, *args) - self.clients = Queue.Queue() + self.clients = queue.Queue() self.threads = 10 self.daemon = kwargs.get("daemon", False) @@ -250,7 +250,7 @@ class TForkingServer(TServer): try: while True: self.processor.process(iprot, oprot) - except TTransport.TTransportException as tx: + except TTransport.TTransportException: pass except Exception as e: logger.exception(e) @@ -261,7 +261,7 @@ class TForkingServer(TServer): os._exit(ecode) - except TTransport.TTransportException as tx: + except TTransport.TTransportException: pass except Exception as x: logger.exception(x) http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/transport/THttpClient.py ---------------------------------------------------------------------- diff --git a/lib/py/src/transport/THttpClient.py b/lib/py/src/transport/THttpClient.py index 5851fa2..5abd41c 100644 --- a/lib/py/src/transport/THttpClient.py +++ b/lib/py/src/transport/THttpClient.py @@ -17,17 +17,17 @@ # under the License. # -import httplib +from io import BytesIO import os import socket import sys -import urllib -import urlparse import warnings -from cStringIO import StringIO +from six.moves import urllib +from six.moves import http_client -from TTransport import * +from .TTransport import * +import six class THttpClient(TTransportBase): @@ -52,31 +52,33 @@ class THttpClient(TTransportBase): self.path = path self.scheme = 'http' else: - parsed = urlparse.urlparse(uri_or_host) + parsed = urllib.parse.urlparse(uri_or_host) self.scheme = parsed.scheme assert self.scheme in ('http', 'https') if self.scheme == 'http': - self.port = parsed.port or httplib.HTTP_PORT + self.port = parsed.port or http_client.HTTP_PORT elif self.scheme == 'https': - self.port = parsed.port or httplib.HTTPS_PORT + self.port = parsed.port or http_client.HTTPS_PORT self.host = parsed.hostname self.path = parsed.path if parsed.query: self.path += '?%s' % parsed.query - self.__wbuf = StringIO() + self.__wbuf = BytesIO() self.__http = None + self.__http_response = None self.__timeout = None self.__custom_headers = None def open(self): if self.scheme == 'http': - self.__http = httplib.HTTP(self.host, self.port) + self.__http = http_client.HTTPConnection(self.host, self.port) else: - self.__http = httplib.HTTPS(self.host, self.port) + self.__http = http_client.HTTPSConnection(self.host, self.port) def close(self): self.__http.close() self.__http = None + self.__http_response = None def isOpen(self): return self.__http is not None @@ -94,7 +96,7 @@ class THttpClient(TTransportBase): self.__custom_headers = headers def read(self, sz): - return self.__http.file.read(sz) + return self.__http_response.read(sz) def write(self, buf): self.__wbuf.write(buf) @@ -117,13 +119,12 @@ class THttpClient(TTransportBase): # Pull data out of buffer data = self.__wbuf.getvalue() - self.__wbuf = StringIO() + self.__wbuf = BytesIO() # HTTP request self.__http.putrequest('POST', self.path) # Write headers - self.__http.putheader('Host', self.host) self.__http.putheader('Content-Type', 'application/x-thrift') self.__http.putheader('Content-Length', str(len(data))) @@ -131,11 +132,11 @@ class THttpClient(TTransportBase): user_agent = 'Python/THttpClient' script = os.path.basename(sys.argv[0]) if script: - user_agent = '%s (%s)' % (user_agent, urllib.quote(script)) + user_agent = '%s (%s)' % (user_agent, urllib.parse.quote(script)) self.__http.putheader('User-Agent', user_agent) if self.__custom_headers: - for key, val in self.__custom_headers.iteritems(): + for key, val in six.iteritems(self.__custom_headers): self.__http.putheader(key, val) self.__http.endheaders() @@ -144,7 +145,10 @@ class THttpClient(TTransportBase): self.__http.send(data) # Get reply to flush the request - self.code, self.message, self.headers = self.__http.getreply() + self.__http_response = self.__http.getresponse() + self.code = self.__http_response.status + self.message = self.__http_response.reason + self.headers = self.__http_response.msg # Decorate if we know how to timeout if hasattr(socket, 'getdefaulttimeout'): http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/transport/TSocket.py ---------------------------------------------------------------------- diff --git a/lib/py/src/transport/TSocket.py b/lib/py/src/transport/TSocket.py index 7b564aa..cb204a4 100644 --- a/lib/py/src/transport/TSocket.py +++ b/lib/py/src/transport/TSocket.py @@ -22,7 +22,7 @@ import os import socket import sys -from TTransport import * +from .TTransport import * class TSocketBase(TTransportBase): http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/transport/TTransport.py ---------------------------------------------------------------------- diff --git a/lib/py/src/transport/TTransport.py b/lib/py/src/transport/TTransport.py index 5914aca..3fe289a 100644 --- a/lib/py/src/transport/TTransport.py +++ b/lib/py/src/transport/TTransport.py @@ -17,9 +17,9 @@ # under the License. # -from cStringIO import StringIO from struct import pack, unpack from thrift.Thrift import TException +from ..compat import BufferIO class TTransportException(TException): @@ -52,7 +52,7 @@ class TTransportBase: pass def readAll(self, sz): - buff = '' + buff = b'' have = 0 while (have < sz): chunk = self.read(sz - have) @@ -138,8 +138,8 @@ class TBufferedTransport(TTransportBase, CReadableTransport): def __init__(self, trans, rbuf_size=DEFAULT_BUFFER): self.__trans = trans - self.__wbuf = StringIO() - self.__rbuf = StringIO("") + self.__wbuf = BufferIO() + self.__rbuf = BufferIO() self.__rbuf_size = rbuf_size def isOpen(self): @@ -155,8 +155,7 @@ class TBufferedTransport(TTransportBase, CReadableTransport): ret = self.__rbuf.read(sz) if len(ret) != 0: return ret - - self.__rbuf = StringIO(self.__trans.read(max(sz, self.__rbuf_size))) + self.__rbuf = BufferIO(self.__trans.read(max(sz, self.__rbuf_size))) return self.__rbuf.read(sz) def write(self, buf): @@ -164,13 +163,14 @@ class TBufferedTransport(TTransportBase, CReadableTransport): self.__wbuf.write(buf) except Exception as e: # on exception reset wbuf so it doesn't contain a partial function call - self.__wbuf = StringIO() + self.__wbuf = BufferIO() raise e + self.__wbuf.getvalue() def flush(self): out = self.__wbuf.getvalue() # reset wbuf before write/flush to preserve state on underlying failure - self.__wbuf = StringIO() + self.__wbuf = BufferIO() self.__trans.write(out) self.__trans.flush() @@ -189,12 +189,12 @@ class TBufferedTransport(TTransportBase, CReadableTransport): if len(retstring) < reqlen: retstring += self.__trans.readAll(reqlen - len(retstring)) - self.__rbuf = StringIO(retstring) + self.__rbuf = BufferIO(retstring) return self.__rbuf class TMemoryBuffer(TTransportBase, CReadableTransport): - """Wraps a cStringIO object as a TTransport. + """Wraps a cBytesIO object as a TTransport. NOTE: Unlike the C++ version of this class, you cannot write to it then immediately read from it. If you want to read from a @@ -208,9 +208,9 @@ class TMemoryBuffer(TTransportBase, CReadableTransport): If value is set, this will be a transport for reading, otherwise, it is for writing""" if value is not None: - self._buffer = StringIO(value) + self._buffer = BufferIO(value) else: - self._buffer = StringIO() + self._buffer = BufferIO() def isOpen(self): return not self._buffer.closed @@ -256,8 +256,8 @@ class TFramedTransport(TTransportBase, CReadableTransport): def __init__(self, trans,): self.__trans = trans - self.__rbuf = StringIO() - self.__wbuf = StringIO() + self.__rbuf = BufferIO() + self.__wbuf = BufferIO() def isOpen(self): return self.__trans.isOpen() @@ -279,7 +279,7 @@ class TFramedTransport(TTransportBase, CReadableTransport): def readFrame(self): buff = self.__trans.readAll(4) sz, = unpack('!i', buff) - self.__rbuf = StringIO(self.__trans.readAll(sz)) + self.__rbuf = BufferIO(self.__trans.readAll(sz)) def write(self, buf): self.__wbuf.write(buf) @@ -288,7 +288,7 @@ class TFramedTransport(TTransportBase, CReadableTransport): wout = self.__wbuf.getvalue() wsz = len(wout) # reset wbuf before write/flush to preserve state on underlying failure - self.__wbuf = StringIO() + self.__wbuf = BufferIO() # N.B.: Doing this string concatenation is WAY cheaper than making # two separate calls to the underlying socket object. Socket writes in # Python turn out to be REALLY expensive, but it seems to do a pretty @@ -309,7 +309,7 @@ class TFramedTransport(TTransportBase, CReadableTransport): while len(prefix) < reqlen: self.readFrame() prefix += self.__rbuf.getvalue() - self.__rbuf = StringIO(prefix) + self.__rbuf = BufferIO(prefix) return self.__rbuf @@ -337,7 +337,7 @@ class TFileObjectTransport(TTransportBase): class TSaslClientTransport(TTransportBase, CReadableTransport): """ - SASL transport + SASL transport """ START = 1 @@ -363,8 +363,8 @@ class TSaslClientTransport(TTransportBase, CReadableTransport): self.transport = transport self.sasl = SASLClient(host, service, mechanism, **sasl_kwargs) - self.__wbuf = StringIO() - self.__rbuf = StringIO() + self.__wbuf = BufferIO() + self.__rbuf = BufferIO() def open(self): if not self.transport.isOpen(): @@ -409,7 +409,7 @@ class TSaslClientTransport(TTransportBase, CReadableTransport): encoded = self.sasl.wrap(data) self.transport.write(''.join((pack("!i", len(encoded)), encoded))) self.transport.flush() - self.__wbuf = StringIO() + self.__wbuf = BufferIO() def read(self, sz): ret = self.__rbuf.read(sz) @@ -423,7 +423,7 @@ class TSaslClientTransport(TTransportBase, CReadableTransport): header = self.transport.readAll(4) length, = unpack('!i', header) encoded = self.transport.readAll(length) - self.__rbuf = StringIO(self.sasl.unwrap(encoded)) + self.__rbuf = BufferIO(self.sasl.unwrap(encoded)) def close(self): self.sasl.dispose() @@ -441,6 +441,6 @@ class TSaslClientTransport(TTransportBase, CReadableTransport): while len(prefix) < reqlen: self._read_frame() prefix += self.__rbuf.getvalue() - self.__rbuf = StringIO(prefix) + self.__rbuf = BufferIO(prefix) return self.__rbuf http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/transport/TTwisted.py ---------------------------------------------------------------------- diff --git a/lib/py/src/transport/TTwisted.py b/lib/py/src/transport/TTwisted.py index 29bbd4c..cabe345 100644 --- a/lib/py/src/transport/TTwisted.py +++ b/lib/py/src/transport/TTwisted.py @@ -17,8 +17,8 @@ # under the License. # +from io import BytesIO import struct -from cStringIO import StringIO from zope.interface import implements, Interface, Attribute from twisted.internet.protocol import ServerFactory, ClientFactory, \ @@ -29,19 +29,20 @@ from twisted.protocols import basic from twisted.web import server, resource, http from thrift.transport import TTransport +import six class TMessageSenderTransport(TTransport.TTransportBase): def __init__(self): - self.__wbuf = StringIO() + self.__wbuf = BytesIO() def write(self, buf): self.__wbuf.write(buf) def flush(self): msg = self.__wbuf.getvalue() - self.__wbuf = StringIO() + self.__wbuf = BytesIO() return self.sendMessage(msg) def sendMessage(self, message): @@ -82,7 +83,7 @@ class ThriftClientProtocol(basic.Int32StringReceiver): self.started.callback(self.client) def connectionLost(self, reason=connectionDone): - for k, v in self.client._reqs.iteritems(): + for k, v in six.iteritems(self.client._reqs): tex = TTransport.TTransportException( type=TTransport.TTransportException.END_OF_FILE, message='Connection closed') http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/lib/py/src/transport/TZlibTransport.py ---------------------------------------------------------------------- diff --git a/lib/py/src/transport/TZlibTransport.py b/lib/py/src/transport/TZlibTransport.py index a2f42a5..7fe5853 100644 --- a/lib/py/src/transport/TZlibTransport.py +++ b/lib/py/src/transport/TZlibTransport.py @@ -24,8 +24,8 @@ data compression. from __future__ import division import zlib -from cStringIO import StringIO -from TTransport import TTransportBase, CReadableTransport +from .TTransport import TTransportBase, CReadableTransport +from ..compat import BufferIO class TZlibTransportFactory(object): @@ -88,8 +88,8 @@ class TZlibTransport(TTransportBase, CReadableTransport): """ self.__trans = trans self.compresslevel = compresslevel - self.__rbuf = StringIO() - self.__wbuf = StringIO() + self.__rbuf = BufferIO() + self.__wbuf = BufferIO() self._init_zlib() self._init_stats() @@ -97,8 +97,8 @@ class TZlibTransport(TTransportBase, CReadableTransport): """Internal method to initialize/reset the internal StringIO objects for read and write buffers. """ - self.__rbuf = StringIO() - self.__wbuf = StringIO() + self.__rbuf = BufferIO() + self.__wbuf = BufferIO() def _init_stats(self): """Internal method to reset the internal statistics counters @@ -203,7 +203,7 @@ class TZlibTransport(TTransportBase, CReadableTransport): self.bytes_in += len(zbuf) self.bytes_in_comp += len(buf) old = self.__rbuf.read() - self.__rbuf = StringIO(old + buf) + self.__rbuf = BufferIO(old + buf) if len(old) + len(buf) == 0: return False return True @@ -228,7 +228,7 @@ class TZlibTransport(TTransportBase, CReadableTransport): ztail = self._zcomp_write.flush(zlib.Z_SYNC_FLUSH) self.bytes_out_comp += len(ztail) if (len(zbuf) + len(ztail)) > 0: - self.__wbuf = StringIO() + self.__wbuf = BufferIO() self.__trans.write(zbuf + ztail) self.__trans.flush() @@ -244,5 +244,5 @@ class TZlibTransport(TTransportBase, CReadableTransport): retstring += self.read(self.DEFAULT_BUFFSIZE) while len(retstring) < reqlen: retstring += self.read(reqlen - len(retstring)) - self.__rbuf = StringIO(retstring) + self.__rbuf = BufferIO(retstring) return self.__rbuf http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/test/py.tornado/test_suite.py ---------------------------------------------------------------------- diff --git a/test/py.tornado/test_suite.py b/test/py.tornado/test_suite.py index b63ea2d..e0bf913 100755 --- a/test/py.tornado/test_suite.py +++ b/test/py.tornado/test_suite.py @@ -28,12 +28,12 @@ import unittest basepath = os.path.abspath(os.path.dirname(__file__)) sys.path.insert(0, basepath+'/gen-py.tornado') -sys.path.insert(0, glob.glob(os.path.join(basepath, '../../lib/py/build/lib.*'))[0]) +sys.path.insert(0, glob.glob(os.path.join(basepath, '../../lib/py/build/lib*'))[0]) try: __import__('tornado') except ImportError: - print "module `tornado` not found, skipping test" + print("module `tornado` not found, skipping test") sys.exit(0) from tornado import gen @@ -188,9 +188,9 @@ class ThriftTestCase(AsyncTestCase): self.assertEqual(y.i64_thing, -5) def test_oneway(self): - self.client.testOneway(0.5) + self.client.testOneway(0) start, end, seconds = self.wait(timeout=1) - self.assertAlmostEquals(seconds, (end - start), places=3) + self.assertAlmostEqual(seconds, (end - start), places=3) @gen_test def test_map(self): http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/test/py.twisted/test_suite.py ---------------------------------------------------------------------- diff --git a/test/py.twisted/test_suite.py b/test/py.twisted/test_suite.py index 048abc5..2c07baa 100755 --- a/test/py.twisted/test_suite.py +++ b/test/py.twisted/test_suite.py @@ -172,7 +172,7 @@ class ThriftTestCase(unittest.TestCase): try: yield self.client.testException('Xception') self.fail("should have gotten exception") - except Xception, x: + except Xception as x: self.assertEquals(x.errorCode, 1001) self.assertEquals(x.message, 'Xception') http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/test/py/RunClientServer.py ---------------------------------------------------------------------- diff --git a/test/py/RunClientServer.py b/test/py/RunClientServer.py index b7f4332..7224bac 100755 --- a/test/py/RunClientServer.py +++ b/test/py/RunClientServer.py @@ -20,6 +20,7 @@ # from __future__ import division +from __future__ import print_function import time import socket import subprocess @@ -77,13 +78,13 @@ SERVERS = [ try: import multiprocessing except: - print 'Warning: the multiprocessing module is unavailable. Skipping tests for TProcessPoolServer' + print('Warning: the multiprocessing module is unavailable. Skipping tests for TProcessPoolServer') SERVERS.remove('TProcessPoolServer') try: import ssl except: - print 'Warning, no ssl module available. Skipping all SSL tests.' + print('Warning, no ssl module available. Skipping all SSL tests.') SKIP_SSL.extend(SERVERS) # commandline permits a single class name to be specified to override SERVERS=[...] @@ -91,7 +92,7 @@ if len(args) == 1: if args[0] in SERVERS: SERVERS = args else: - print 'Unavailable server type "%s", please choose one of: %s' % (args[0], SERVERS) + print('Unavailable server type "%s", please choose one of: %s' % (args[0], SERVERS)) sys.exit(0) @@ -102,7 +103,7 @@ def runScriptTest(genpydir, script): script_args = [sys.executable, relfile(script) ] script_args.append('--genpydir=%s' % genpydir) serverproc = subprocess.Popen(script_args) - print '\nTesting script: %s\n----' % (' '.join(script_args)) + print('\nTesting script: %s\n----' % (' '.join(script_args))) ret = subprocess.call(script_args) if ret != 0: raise Exception("Script subprocess failed, retcode=%d, args: %s" % (ret, ' '.join(script_args))) @@ -133,12 +134,12 @@ def runServiceTest(genpydir, server_class, proto, port, use_zlib, use_ssl): if server_class == 'THttpServer': cli_args.append('--http=/') if options.verbose > 0: - print 'Testing server %s: %s' % (server_class, ' '.join(server_args)) + print('Testing server %s: %s' % (server_class, ' '.join(server_args))) serverproc = subprocess.Popen(server_args) def ensureServerAlive(): if serverproc.poll() is not None: - print ('FAIL: Server process (%s) failed with retcode %d' + print(('FAIL: Server process (%s) failed with retcode %d') % (' '.join(server_args), serverproc.returncode)) raise Exception('Server subprocess %s died, args: %s' % (server_class, ' '.join(server_args))) @@ -161,7 +162,7 @@ def runServiceTest(genpydir, server_class, proto, port, use_zlib, use_ssl): try: if options.verbose > 0: - print 'Testing client: %s' % (' '.join(cli_args)) + print('Testing client: %s' % (' '.join(cli_args))) ret = subprocess.call(cli_args) if ret != 0: raise Exception("Client subprocess failed, retcode=%d, args: %s" % (ret, ' '.join(cli_args))) @@ -170,7 +171,7 @@ def runServiceTest(genpydir, server_class, proto, port, use_zlib, use_ssl): ensureServerAlive() extra_sleep = EXTRA_DELAY.get(server_class, 0) if extra_sleep > 0 and options.verbose > 0: - print ('Giving %s (proto=%s,zlib=%s,ssl=%s) an extra %d seconds for child' + print('Giving %s (proto=%s,zlib=%s,ssl=%s) an extra %d seconds for child' 'processes to terminate via alarm' % (server_class, proto, use_zlib, use_ssl, extra_sleep)) time.sleep(extra_sleep) @@ -179,22 +180,22 @@ def runServiceTest(genpydir, server_class, proto, port, use_zlib, use_ssl): test_count = 0 # run tests without a client/server first -print '----------------' -print ' Executing individual test scripts with various generated code directories' -print ' Directories to be tested: ' + ', '.join(generated_dirs) -print ' Scripts to be tested: ' + ', '.join(SCRIPTS) -print '----------------' +print('----------------') +print(' Executing individual test scripts with various generated code directories') +print(' Directories to be tested: ' + ', '.join(generated_dirs)) +print(' Scripts to be tested: ' + ', '.join(SCRIPTS)) +print('----------------') for genpydir in generated_dirs: for script in SCRIPTS: runScriptTest(genpydir, script) -print '----------------' -print ' Executing Client/Server tests with various generated code directories' -print ' Servers to be tested: ' + ', '.join(SERVERS) -print ' Directories to be tested: ' + ', '.join(generated_dirs) -print ' Protocols to be tested: ' + ', '.join(PROTOS) -print ' Options to be tested: ZLIB(yes/no), SSL(yes/no)' -print '----------------' +print('----------------') +print(' Executing Client/Server tests with various generated code directories') +print(' Servers to be tested: ' + ', '.join(SERVERS)) +print(' Directories to be tested: ' + ', '.join(generated_dirs)) +print(' Protocols to be tested: ' + ', '.join(PROTOS)) +print(' Options to be tested: ZLIB(yes/no), SSL(yes/no)') +print('----------------') for try_server in SERVERS: for genpydir in generated_dirs: for try_proto in PROTOS: @@ -208,7 +209,7 @@ for try_server in SERVERS: continue test_count += 1 if options.verbose > 0: - print '\nTest run #%d: (includes %s) Server=%s, Proto=%s, zlib=%s, SSL=%s' % (test_count, genpydir, try_server, try_proto, with_zlib, with_ssl) + print('\nTest run #%d: (includes %s) Server=%s, Proto=%s, zlib=%s, SSL=%s' % (test_count, genpydir, try_server, try_proto, with_zlib, with_ssl)) runServiceTest(genpydir, try_server, try_proto, options.port, with_zlib, with_ssl) if options.verbose > 0: - print 'OK: Finished (includes %s) %s / %s proto / zlib=%s / SSL=%s. %d combinations tested.' % (genpydir, try_server, try_proto, with_zlib, with_ssl, test_count) + print('OK: Finished (includes %s) %s / %s proto / zlib=%s / SSL=%s. %d combinations tested.' % (genpydir, try_server, try_proto, with_zlib, with_ssl, test_count)) http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/test/py/SerializationTest.py ---------------------------------------------------------------------- diff --git a/test/py/SerializationTest.py b/test/py/SerializationTest.py index 40a52e6..29f3656 100755 --- a/test/py/SerializationTest.py +++ b/test/py/SerializationTest.py @@ -26,7 +26,7 @@ parser.add_option('--genpydir', type='string', dest='genpydir', default='gen-py' options, args = parser.parse_args() del sys.argv[1:] # clean up hack so unittest doesn't complain sys.path.insert(0, options.genpydir) -sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) +sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0]) from ThriftTest.ttypes import * from DebugProtoTest.ttypes import CompactProtoTestStruct, Empty @@ -305,7 +305,7 @@ class AcceleratedFramedTest(unittest.TestCase): prot.writeString(bigstring) prot.writeI16(24) data = databuf.getvalue() - cutpoint = len(data)/2 + cutpoint = len(data) // 2 parts = [ data[:cutpoint], data[cutpoint:] ] framed_buffer = TTransport.TMemoryBuffer() @@ -346,14 +346,14 @@ class SerializersTest(unittest.TestCase): objcopy = Bools() deserialize(objcopy, serialize(obj)) self.assertEquals(obj, objcopy) - + # test enums - for num, name in Numberz._VALUES_TO_NAMES.iteritems(): + for num, name in Numberz._VALUES_TO_NAMES.items(): obj = Bonk(message='enum Numberz value %d is string %s' % (num, name), type=num) objcopy = Bonk() deserialize(objcopy, serialize(obj)) self.assertEquals(obj, objcopy) - + def suite(): suite = unittest.TestSuite() http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/test/py/TSimpleJSONProtocolTest.py ---------------------------------------------------------------------- diff --git a/test/py/TSimpleJSONProtocolTest.py b/test/py/TSimpleJSONProtocolTest.py index 080293a..b8db932 100644 --- a/test/py/TSimpleJSONProtocolTest.py +++ b/test/py/TSimpleJSONProtocolTest.py @@ -27,7 +27,7 @@ parser.add_option('--genpydir', type='string', dest='genpydir', default='gen-py' options, args = parser.parse_args() del sys.argv[1:] # clean up hack so unittest doesn't complain sys.path.insert(0, options.genpydir) -sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) +sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0]) from ThriftTest.ttypes import * from thrift.protocol import TJSONProtocol @@ -45,7 +45,7 @@ class SimpleJSONProtocolTest(unittest.TestCase): # assertDictEqual only in Python 2.7. Depends on your machine. self.assertDictEqual(a, b, msg) return - + # Substitute implementation not as good as unittest library's self.assertEquals(len(a), len(b), msg) for k, v in a.iteritems(): @@ -66,7 +66,7 @@ class SimpleJSONProtocolTest(unittest.TestCase): def testWriteOnly(self): self.assertRaises(NotImplementedError, - self._deserialize, VersioningTestV1, '{}') + self._deserialize, VersioningTestV1, b'{}') def testSimpleMessage(self): v1obj = VersioningTestV1( @@ -76,10 +76,10 @@ class SimpleJSONProtocolTest(unittest.TestCase): expected = dict(begin_in_both=v1obj.begin_in_both, old_string=v1obj.old_string, end_in_both=v1obj.end_in_both) - actual = json.loads(self._serialize(v1obj)) + actual = json.loads(self._serialize(v1obj).decode('ascii')) self._assertDictEqual(expected, actual) - + def testComplicated(self): v2obj = VersioningTestV2( begin_in_both=12345, @@ -107,10 +107,10 @@ class SimpleJSONProtocolTest(unittest.TestCase): newmap=v2obj.newmap, newstring=v2obj.newstring, end_in_both=v2obj.end_in_both) - + # Need to load/dump because map keys get escaped. expected = json.loads(json.dumps(expected)) - actual = json.loads(self._serialize(v2obj)) + actual = json.loads(self._serialize(v2obj).decode('ascii')) self._assertDictEqual(expected, actual) http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/test/py/TestClient.py ---------------------------------------------------------------------- diff --git a/test/py/TestClient.py b/test/py/TestClient.py index 3a74353..4689d63 100755 --- a/test/py/TestClient.py +++ b/test/py/TestClient.py @@ -54,11 +54,11 @@ parser.set_defaults(framed=False, http_path=None, verbose=1, host='localhost', p options, args = parser.parse_args() script_dir = os.path.abspath(os.path.dirname(__file__)) -lib_dir = os.path.join(os.path.dirname(os.path.dirname(script_dir)), 'lib', 'py', 'build', 'lib.*') +lib_dir = os.path.join(os.path.dirname(os.path.dirname(script_dir)), 'lib', 'py', 'build', 'lib*') sys.path.insert(0, os.path.join(script_dir, options.genpydir)) sys.path.insert(0, glob.glob(lib_dir)[0]) -from ThriftTest import ThriftTest, SecondService +from ThriftTest import ThriftTest from ThriftTest.ttypes import * from thrift.transport import TTransport from thrift.transport import TSocket @@ -105,8 +105,8 @@ class AbstractTest(unittest.TestCase): print('testString') self.assertEqual(self.client.testString('Python' * 20), 'Python' * 20) self.assertEqual(self.client.testString(''), '') - self.assertEqual(self.client.testString(u'ãã¤ã½ã³'.encode('utf8')), u'ãã¤ã½ã³'.encode('utf8')) - s = u"""Afrikaans, Alemannisch, Aragonés, اÙعربÙØ©, ٠صرÙ, + s1 = u'\b\t\n/\\\\\r{}:ãã¤ã½ã³"' + s2 = u"""Afrikaans, Alemannisch, Aragonés, اÙعربÙØ©, ٠صرÙ, Asturianu, Aymar aru, AzÉrbaycan, ÐаÑҡоÑÑ, Boarisch, ŽemaitÄÅ¡ka, ÐелаÑÑÑкаÑ, ÐелаÑÑÑÐºÐ°Ñ (ÑаÑаÑкевÑÑа), ÐÑлгаÑÑки, Bamanankan, বাà¦à¦²à¦¾, Brezhoneg, Bosanski, Català , Mìng-dÄ̤ng-ngá¹³Ì, ÐÐ¾Ñ Ñийн, @@ -131,7 +131,11 @@ class AbstractTest(unittest.TestCase): Türkçe, ТаÑаÑÑа/Tatarça, УкÑаÑнÑÑка, اردÙ, Tiếng Viá»t, Volapük, Walon, Winaray, å´è¯, isiXhosa, ××Ö´××ש, Yorùbá, Zeêuws, ä¸æ, Bân-lâm-gú, ç²µèª""" - self.assertEqual(self.client.testString(s.encode('utf8')), s.encode('utf8')) + if sys.version_info[0] == 2: + s1 = s1.encode('utf8') + s2 = s2.encode('utf8') + self.assertEqual(self.client.testString(s1), s1) + self.assertEqual(self.client.testString(s2), s2) def testBool(self): print('testBool') @@ -290,19 +294,23 @@ class AbstractTest(unittest.TestCase): class NormalBinaryTest(AbstractTest): protocol_factory = TBinaryProtocol.TBinaryProtocolFactory() + class CompactTest(AbstractTest): protocol_factory = TCompactProtocol.TCompactProtocolFactory() + class JSONTest(AbstractTest): protocol_factory = TJSONProtocol.TJSONProtocolFactory() + class AcceleratedBinaryTest(AbstractTest): protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory() + def suite(): suite = unittest.TestSuite() loader = unittest.TestLoader() - if options.proto == 'binary': # look for --proto on cmdline + if options.proto == 'binary': # look for --proto on cmdline suite.addTest(loader.loadTestsFromTestCase(NormalBinaryTest)) elif options.proto == 'accel': suite.addTest(loader.loadTestsFromTestCase(AcceleratedBinaryTest)) @@ -314,6 +322,7 @@ def suite(): raise AssertionError('Unknown protocol given with --protocol: %s' % options.proto) return suite + class OwnArgsTestProgram(unittest.TestProgram): def parseArgs(self, argv): if args: http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/test/py/TestEof.py ---------------------------------------------------------------------- diff --git a/test/py/TestEof.py b/test/py/TestEof.py index a9d81f1..7677de8 100755 --- a/test/py/TestEof.py +++ b/test/py/TestEof.py @@ -26,7 +26,7 @@ parser.add_option('--genpydir', type='string', dest='genpydir', default='gen-py' options, args = parser.parse_args() del sys.argv[1:] # clean up hack so unittest doesn't complain sys.path.insert(0, options.genpydir) -sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) +sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0]) from ThriftTest import ThriftTest from ThriftTest.ttypes import * @@ -99,7 +99,7 @@ class TestEof(unittest.TestCase): # TODO: we should make sure this covers more of the code paths data = self.make_data(pfactory) - for i in xrange(0, len(data) + 1): + for i in range(0, len(data) + 1): trans = TTransport.TMemoryBuffer(data[0:i]) prot = pfactory.getProtocol(trans) try: http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/test/py/TestServer.py ---------------------------------------------------------------------- diff --git a/test/py/TestServer.py b/test/py/TestServer.py index e7478e4..bc221c0 100755 --- a/test/py/TestServer.py +++ b/test/py/TestServer.py @@ -18,7 +18,7 @@ # specific language governing permissions and limitations # under the License. # -from __future__ import division +from __future__ import division, print_function import glob import logging import os @@ -53,7 +53,7 @@ parser.set_defaults(port=9090, verbose=1, proto='binary') options, args = parser.parse_args() script_dir = os.path.realpath(os.path.dirname(__file__)) # <-- absolute dir the script is in -lib_dir = os.path.join(os.path.dirname(os.path.dirname(script_dir)), 'lib', 'py', 'build', 'lib.*') +lib_dir = os.path.join(os.path.dirname(os.path.dirname(script_dir)), 'lib', 'py', 'build', 'lib*') sys.path.insert(0, os.path.join(script_dir, options.genpydir)) sys.path.insert(0, glob.glob(lib_dir)[0]) http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/test/py/TestSocket.py ---------------------------------------------------------------------- diff --git a/test/py/TestSocket.py b/test/py/TestSocket.py index b9bdf27..55e4996 100755 --- a/test/py/TestSocket.py +++ b/test/py/TestSocket.py @@ -26,7 +26,7 @@ parser.add_option('--genpydir', type='string', dest='genpydir', default='gen-py' options, args = parser.parse_args() del sys.argv[1:] # clean up hack so unittest doesn't complain sys.path.insert(0, options.genpydir) -sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) +sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0]) from ThriftTest import ThriftTest from ThriftTest.ttypes import * @@ -41,7 +41,7 @@ from optparse import OptionParser class TimeoutTest(unittest.TestCase): def setUp(self): - for i in xrange(50): + for i in range(50): try: # find a port we can use self.listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -58,7 +58,7 @@ class TimeoutTest(unittest.TestCase): try: leaky = [] - for i in xrange(100): + for i in range(100): socket = TSocket.TSocket('localhost', self.port) socket.setTimeout(10) socket.open() http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/test/py/TestSyntax.py ---------------------------------------------------------------------- diff --git a/test/py/TestSyntax.py b/test/py/TestSyntax.py index 9f71cf5..cdf0e0d 100755 --- a/test/py/TestSyntax.py +++ b/test/py/TestSyntax.py @@ -26,7 +26,7 @@ parser.add_option('--genpydir', type='string', dest='genpydir', default='gen-py' options, args = parser.parse_args() del sys.argv[1:] # clean up hack so unittest doesn't complain sys.path.insert(0, options.genpydir) -sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) +sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0]) # Just import these generated files to make sure they are syntactically valid from DebugProtoTest import EmptyService http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/tutorial/py.tornado/PythonClient.py ---------------------------------------------------------------------- diff --git a/tutorial/py.tornado/PythonClient.py b/tutorial/py.tornado/PythonClient.py index 95d78b8..005ad3c 100755 --- a/tutorial/py.tornado/PythonClient.py +++ b/tutorial/py.tornado/PythonClient.py @@ -22,7 +22,7 @@ import sys import glob sys.path.append('gen-py.tornado') -sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) +sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0]) import logging @@ -56,21 +56,21 @@ def communicate(callback=None): # ping yield gen.Task(client.ping) - print "ping()" + print("ping()") # add sum_ = yield gen.Task(client.add, 1, 1) - print "1 + 1 = {}".format(sum_) + print("1 + 1 = {}".format(sum_)) # make a oneway call without a callback (schedule the write and continue # without blocking) client.zip() - print "zip() without callback" + print("zip() without callback") # make a oneway call with a callback (we'll wait for the stream write to # complete before continuing) yield gen.Task(client.zip) - print "zip() with callback" + print("zip() with callback") # calculate 1/0 work = Work() @@ -80,9 +80,9 @@ def communicate(callback=None): try: quotient = yield gen.Task(client.calculate, 1, work) - print "Whoa? You know how to divide by zero?" + print("Whoa? You know how to divide by zero?") except InvalidOperation as io: - print "InvalidOperation: {}".format(io) + print("InvalidOperation: {}".format(io)) # calculate 15-10 work.op = Operation.SUBTRACT @@ -90,11 +90,11 @@ def communicate(callback=None): work.num2 = 10 diff = yield gen.Task(client.calculate, 1, work) - print "15 - 10 = {}".format(diff) + print("15 - 10 = {}".format(diff)) # getStruct log = yield gen.Task(client.getStruct, 1) - print "Check log: {}".format(log.value) + print("Check log: {}".format(log.value)) # close the transport client._transport.close() http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/tutorial/py.tornado/PythonServer.py ---------------------------------------------------------------------- diff --git a/tutorial/py.tornado/PythonServer.py b/tutorial/py.tornado/PythonServer.py index 7a34107..4198214 100755 --- a/tutorial/py.tornado/PythonServer.py +++ b/tutorial/py.tornado/PythonServer.py @@ -22,7 +22,7 @@ import sys import glob sys.path.append('gen-py.tornado') -sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) +sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0]) from tutorial import Calculator from tutorial.ttypes import Operation, InvalidOperation @@ -43,15 +43,15 @@ class CalculatorHandler(object): self.log = {} def ping(self, callback): - print "ping()" + print("ping()") callback() def add(self, n1, n2, callback): - print "add({}, {})".format(n1, n2) + print("add({}, {})".format(n1, n2)) callback(n1 + n2) def calculate(self, logid, work, callback): - print "calculate({}, {})".format(logid, work) + print("calculate({}, {})".format(logid, work)) if work.op == Operation.ADD: val = work.num1 + work.num2 @@ -79,11 +79,11 @@ class CalculatorHandler(object): callback(val) def getStruct(self, key, callback): - print "getStruct({})".format(key) + print("getStruct({})".format(key)) callback(self.log[key]) def zip(self, callback): - print "zip()" + print("zip()") callback() @@ -93,11 +93,11 @@ def main(): pfactory = TBinaryProtocol.TBinaryProtocolFactory() server = TTornado.TTornadoServer(processor, pfactory) - print "Starting the server..." + print("Starting the server...") server.bind(9090) server.start(1) ioloop.IOLoop.instance().start() - print "done." + print("done.") if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/tutorial/py.twisted/PythonClient.py ---------------------------------------------------------------------- diff --git a/tutorial/py.twisted/PythonClient.py b/tutorial/py.twisted/PythonClient.py index 9e086f0..e80c0fc 100755 --- a/tutorial/py.twisted/PythonClient.py +++ b/tutorial/py.twisted/PythonClient.py @@ -21,7 +21,7 @@ import sys, glob sys.path.append('gen-py.twisted') -sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) +sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0]) from tutorial import Calculator from tutorial.ttypes import * @@ -37,10 +37,10 @@ from thrift.protocol import TBinaryProtocol @inlineCallbacks def main(client): yield client.ping() - print 'ping()' + print('ping()') sum = yield client.add(1,1) - print '1+1=%d' % (sum) + print(('1+1=%d' % (sum))) work = Work() @@ -50,19 +50,19 @@ def main(client): try: quotient = yield client.calculate(1, work) - print 'Whoa? You know how to divide by zero?' - except InvalidOperation, io: - print 'InvalidOperation: %r' % io + print('Whoa? You know how to divide by zero?') + except InvalidOperation as e: + print(('InvalidOperation: %r' % e)) work.op = Operation.SUBTRACT work.num1 = 15 work.num2 = 10 diff = yield client.calculate(1, work) - print '15-10=%d' % (diff) + print(('15-10=%d' % (diff))) log = yield client.getStruct(1) - print 'Check log: %s' % (log.value) + print(('Check log: %s' % (log.value))) reactor.stop() if __name__ == '__main__': http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/tutorial/py.twisted/PythonServer.py ---------------------------------------------------------------------- diff --git a/tutorial/py.twisted/PythonServer.py b/tutorial/py.twisted/PythonServer.py index 227f6d4..c578321 100755 --- a/tutorial/py.twisted/PythonServer.py +++ b/tutorial/py.twisted/PythonServer.py @@ -21,7 +21,7 @@ import sys, glob sys.path.append('gen-py.twisted') -sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) +sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0]) from tutorial import Calculator from tutorial.ttypes import * @@ -36,19 +36,19 @@ from thrift.protocol import TBinaryProtocol from thrift.server import TServer class CalculatorHandler: - implements(Calculator.Iface) + implements(Calculator.Iface) def __init__(self): self.log = {} def ping(self): - print 'ping()' + print('ping()') def add(self, n1, n2): - print 'add(%d,%d)' % (n1, n2) + print('add(%d,%d)' % (n1, n2)) return n1+n2 def calculate(self, logid, work): - print 'calculate(%d, %r)' % (logid, work) + print('calculate(%d, %r)' % (logid, work)) if work.op == Operation.ADD: val = work.num1 + work.num2 @@ -77,11 +77,11 @@ class CalculatorHandler: return val def getStruct(self, key): - print 'getStruct(%d)' % (key) + print('getStruct(%d)' % (key)) return self.log[key] def zip(self): - print 'zip()' + print('zip()') if __name__ == '__main__': handler = CalculatorHandler() http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/tutorial/py.twisted/PythonServer.tac ---------------------------------------------------------------------- diff --git a/tutorial/py.twisted/PythonServer.tac b/tutorial/py.twisted/PythonServer.tac index 1d0b6c4..08493ff 100755 --- a/tutorial/py.twisted/PythonServer.tac +++ b/tutorial/py.twisted/PythonServer.tac @@ -24,7 +24,7 @@ from thrift.transport import TTwisted import sys, glob sys.path.append('gen-py.twisted') -sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) +sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0]) from tutorial import Calculator from tutorial.ttypes import * from PythonServer import CalculatorHandler http://git-wip-us.apache.org/repos/asf/thrift/blob/760511f5/tutorial/py/PythonClient.py ---------------------------------------------------------------------- diff --git a/tutorial/py/PythonClient.py b/tutorial/py/PythonClient.py index 0554ee1..c4559ff 100755 --- a/tutorial/py/PythonClient.py +++ b/tutorial/py/PythonClient.py @@ -21,7 +21,7 @@ import sys, glob sys.path.append('gen-py') -sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) +sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0]) from tutorial import Calculator from tutorial.ttypes import * @@ -49,10 +49,10 @@ try: transport.open() client.ping() - print 'ping()' + print('ping()') sum = client.add(1,1) - print '1+1=%d' % (sum) + print(('1+1=%d' % (sum))) work = Work() @@ -62,22 +62,22 @@ try: try: quotient = client.calculate(1, work) - print 'Whoa? You know how to divide by zero?' - except InvalidOperation, io: - print 'InvalidOperation: %r' % io + print('Whoa? You know how to divide by zero?') + except InvalidOperation as e: + print(('InvalidOperation: %r' % e)) work.op = Operation.SUBTRACT work.num1 = 15 work.num2 = 10 diff = client.calculate(1, work) - print '15-10=%d' % (diff) + print(('15-10=%d' % (diff))) log = client.getStruct(1) - print 'Check log: %s' % (log.value) + print(('Check log: %s' % (log.value))) # Close! transport.close() -except Thrift.TException, tx: - print '%s' % (tx.message) +except Thrift.TException as tx: + print(('%s' % (tx.message)))
