This is an automated email from the ASF dual-hosted git repository. spmallette pushed a commit to branch TINKERPOP-2279 in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 97cff5eeb4e7599fbe56a7098e06a5652b2e0d6a Author: Stephen Mallette <[email protected]> AuthorDate: Wed Aug 7 10:31:28 2019 -0400 Major refactoring of GraphBinary Better support for null/value flag. Basic traversal connectivity to Gremlin Server working. Still a bit of a working state at this point. --- .../main/jython/gremlin_python/driver/protocol.py | 5 +- .../jython/gremlin_python/driver/serializer.py | 125 +++++++- .../gremlin_python/structure/io/graphbinaryV1.py | 331 +++++++++++++-------- gremlin-python/src/main/jython/tests/conftest.py | 32 +- .../main/jython/tests/driver/test_serializer.py | 20 +- .../tests/structure/io/test_functionalityio.py | 64 ++++ .../tests/structure/io/test_graphbinaryV1.py | 2 +- .../jython/tests/structure/io/test_graphsonV3d0.py | 42 --- 8 files changed, 426 insertions(+), 195 deletions(-) diff --git a/gremlin-python/src/main/jython/gremlin_python/driver/protocol.py b/gremlin-python/src/main/jython/gremlin_python/driver/protocol.py index c033fe1..bb3ab24 100644 --- a/gremlin-python/src/main/jython/gremlin_python/driver/protocol.py +++ b/gremlin-python/src/main/jython/gremlin_python/driver/protocol.py @@ -76,9 +76,10 @@ class GremlinServerWSProtocol(AbstractBaseProtocol): def data_received(self, message, results_dict): # if Gremlin Server cuts off then we get a None for the message if message is None: - raise GremlinServerError({'code': 500, 'message':'Server disconnected - please try to reconnect', 'attributes': {}}) + raise GremlinServerError({'code': 500, + 'message': 'Server disconnected - please try to reconnect', 'attributes': {}}) - message = self._message_serializer.deserialize_message(json.loads(message.decode('utf-8'))) + message = self._message_serializer.deserialize_message(message) request_id = message['requestId'] result_set = results_dict[request_id] status_code = message['status']['code'] diff --git a/gremlin-python/src/main/jython/gremlin_python/driver/serializer.py b/gremlin-python/src/main/jython/gremlin_python/driver/serializer.py index 007e162..be28140 100644 --- a/gremlin-python/src/main/jython/gremlin_python/driver/serializer.py +++ b/gremlin-python/src/main/jython/gremlin_python/driver/serializer.py @@ -20,7 +20,11 @@ try: import ujson as json except ImportError: import json +import struct +import uuid +import io +from gremlin_python.structure.io import graphbinaryV1 from gremlin_python.structure.io import graphsonV2d0 from gremlin_python.structure.io import graphsonV3d0 @@ -31,7 +35,7 @@ class Processor: """Base class for OpProcessor serialization system.""" def __init__(self, writer): - self._graphson_writer = writer + self._writer = writer def get_op_args(self, op, args): op_method = getattr(self, op, None) @@ -56,7 +60,7 @@ class Traversal(Processor): def bytecode(self, args): gremlin = args['gremlin'] - args['gremlin'] = self._graphson_writer.toDict(gremlin) + args['gremlin'] = self._writer.toDict(gremlin) aliases = args.get('aliases', '') if not aliases: aliases = {'g': 'g'} @@ -143,7 +147,8 @@ class GraphSONMessageSerializer(object): return message def deserialize_message(self, message): - return self._graphson_reader.toObject(message) + msg = json.loads(message.decode('utf-8')) + return self._graphson_reader.toObject(msg) class GraphSONSerializersV2d0(GraphSONMessageSerializer): @@ -162,3 +167,117 @@ class GraphSONSerializersV3d0(GraphSONMessageSerializer): writer = graphsonV3d0.GraphSONWriter() version = b"application/vnd.gremlin-v3.0+json" super(GraphSONSerializersV3d0, self).__init__(reader, writer, version) + + +class GraphBinaryMessageSerializerV1(object): + DEFAULT_READER_CLASS = graphbinaryV1.GraphBinaryReader + DEFAULT_WRITER_CLASS = graphbinaryV1.GraphBinaryWriter + DEFAULT_VERSION = b"application/vnd.graphbinary-v1.0" + + def __init__(self, reader=None, writer=None, version=None): + if not version: + version = self.DEFAULT_VERSION + self._version = version + if not reader: + reader = self.DEFAULT_READER_CLASS() + self._graphbinary_reader = reader + if not writer: + writer = self.DEFAULT_WRITER_CLASS() + self._graphbinary_writer = writer + self.standard = Standard(writer) + self.traversal = Traversal(writer) + + @property + def version(self): + """Read only property""" + return self._version + + def get_processor(self, processor): + processor = getattr(self, processor, None) + if not processor: + raise Exception("Unknown processor") + return processor + + def serialize_message(self, request_id, request_message): + processor = request_message.processor + op = request_message.op + args = request_message.args + if not processor: + processor_obj = self.get_processor('standard') + else: + processor_obj = self.get_processor(processor) + args = processor_obj.get_op_args(op, args) + message = self.build_message(request_id, processor, op, args) + return message + + def build_message(self, request_id, processor, op, args): + message = { + 'requestId': request_id, + 'processor': processor, + 'op': op, + 'args': args + } + return self.finalize_message(message, 0x20, self.version) + + def finalize_message(self, message, mime_len, mime_type): + ba = bytearray() + ba.extend(struct.pack(">b", mime_len)) + ba.extend(mime_type) + ba.extend([0x81]) + ba.extend(uuid.UUID(message['requestId']).bytes) + + ba.extend(struct.pack(">i", len(message['op']))) + ba.extend(message['op'].encode("utf-8")) + + ba.extend(struct.pack(">i", len(message['processor']))) + ba.extend(message['processor'].encode("utf-8")) + + args = message["args"] + ba.extend(struct.pack(">i", len(args))) + for k, v in args.items(): + ba.extend(self._graphbinary_writer.writeObject(k)) + ba.extend(self._graphbinary_writer.writeObject(v)) + + return bytes(ba) + + def deserialize_message(self, message): + b = io.BytesIO(message) + + #TODO: lots of hardcode null checks need better resolution + + b.read(1) # version + + b.read(1) # requestid nullable + request_id = str(uuid.UUID(bytes=b.read(16))) # result queue uses string as a key + + status_code = struct.unpack(">i", b.read(4))[0] + + b.read(1) # status message nullable + status_msg = b.read(struct.unpack(">i", b.read(4))[0]).decode("utf-8") + + attr_count = struct.unpack(">i", b.read(4))[0] + status_attrs = {} + while attr_count > 0: + k = self._graphbinary_reader.toObject(b) + v = self._graphbinary_reader.toObject(b) + status_attrs[k] = v + attr_count = attr_count - 1 + + meta_count = struct.unpack(">i", b.read(4))[0] + meta_attrs = {} + while meta_count > 0: + k = self._graphbinary_reader.toObject(b) + v = self._graphbinary_reader.toObject(b) + meta_attrs[k] = v + meta_count = meta_count - 1 + + result = self._graphbinary_reader.toObject(b) + + msg = {'requestId': request_id, + 'status': {'code': status_code, + 'message': status_msg, + 'attributes': status_attrs}, + 'result': {'meta': meta_attrs, + 'data': result}} + + return msg diff --git a/gremlin-python/src/main/jython/gremlin_python/structure/io/graphbinaryV1.py b/gremlin-python/src/main/jython/gremlin_python/structure/io/graphbinaryV1.py index e282dda..045c7ed 100644 --- a/gremlin-python/src/main/jython/gremlin_python/structure/io/graphbinaryV1.py +++ b/gremlin-python/src/main/jython/gremlin_python/structure/io/graphbinaryV1.py @@ -95,10 +95,10 @@ class DataType(Enum): textp = 0x28 traversalstrategy = 0x29 #todo bulkset = 0x2a - tree = 0x2b - metrics = 0x2c - traversalmetrics = 0x2d - custom = 0x00 + tree = 0x2b #todo + metrics = 0x2c #todo + traversalmetrics = 0x2d #todo + custom = 0x00 #todo class GraphBinaryTypeType(type): @@ -153,6 +153,9 @@ class GraphBinaryReader(object): def toObject(self, buff): bt = buff.read(1) + if bt[0] == DataType.null.value: + return None + bt_value = struct.unpack('>b', bt)[0] return self.deserializers[DataType(bt_value)].objectify(buff, self) @@ -170,6 +173,10 @@ class _GraphBinaryTypeIO(object): @classmethod def as_bytes(cls, graphbin_type=None, size=None, *args): ba = bytearray() if graphbin_type is None else bytearray([graphbin_type.value]) + + # todo: empty value flag just hardcoded in + ba.extend(struct.pack(">b", 0)) + if size is not None: ba.extend(struct.pack(">i", size)) @@ -182,7 +189,10 @@ class _GraphBinaryTypeIO(object): @classmethod def string_as_bytes(cls, s): - return cls.as_bytes(None, len(s), s.encode("utf-8")) + ba = bytearray() + ba.extend(struct.pack(">i", len(s))) + ba.extend(s.encode("utf-8")) + return ba @classmethod def read_int(cls, buff): @@ -195,13 +205,21 @@ class _GraphBinaryTypeIO(object): @classmethod def unmangleKeyword(cls, symbol): return cls.symbolMap.get(symbol, symbol) + + @classmethod + def write_as_value(cls, graph_binary_type, as_value): + return None if as_value else graph_binary_type - def dictify(self, obj, writer): - raise NotImplementedError() + @classmethod + def is_null(cls, buff, reader, else_opt): + return None if buff.read(1)[0] == 0x01 else else_opt(buff, reader) - def objectify(self, d, reader): + def dictify(self, obj, writer, as_value=False): raise NotImplementedError() + def objectify(self, d, reader, as_value=False): + raise NotImplementedError() + class LongIO(_GraphBinaryTypeIO): @@ -210,15 +228,16 @@ class LongIO(_GraphBinaryTypeIO): byte_format = ">q" @classmethod - def dictify(cls, obj, writer): + def dictify(cls, obj, writer, as_value=False): if obj < -9223372036854775808 or obj > 9223372036854775807: raise Exception("TODO: don't forget bigint") else: - return cls.as_bytes(cls.graphbinary_type, None, struct.pack(cls.byte_format, obj)) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), + None, struct.pack(cls.byte_format, obj)) @classmethod - def objectify(cls, buff, reader): - return struct.unpack(cls.byte_format, buff.read(8))[0] + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, lambda b, r: struct.unpack(cls.byte_format, b.read(8))[0]) class IntIO(LongIO): @@ -228,8 +247,8 @@ class IntIO(LongIO): byte_format = ">i" @classmethod - def objectify(cls, buff, reader): - return cls.read_int(buff) + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, lambda b, r: cls.read_int(b)) class DateIO(_GraphBinaryTypeIO): @@ -238,19 +257,21 @@ class DateIO(_GraphBinaryTypeIO): graphbinary_type = DataType.date @classmethod - def dictify(cls, obj, writer): + def dictify(cls, obj, writer, as_value=False): try: timestamp_seconds = calendar.timegm(obj.utctimetuple()) pts = timestamp_seconds * 1e3 + getattr(obj, 'microsecond', 0) / 1e3 except AttributeError: pts = calendar.timegm(obj.timetuple()) * 1e3 - ts = int(round(pts * 100)) - return cls.as_bytes(cls.graphbinary_type, None, struct.pack(">q", ts)) + ts = int(round(pts)) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), + None, struct.pack(">q", ts)) @classmethod - def objectify(cls, buff, reader): - return datetime.datetime.utcfromtimestamp(struct.unpack(">q", buff.read(8))[0] / 1000.0) + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, + lambda b, r: datetime.datetime.utcfromtimestamp(struct.unpack(">q", b.read(8))[0] / 1000.0)) # Based on current implementation, this class must always be declared before FloatIO. @@ -260,15 +281,16 @@ class TimestampIO(_GraphBinaryTypeIO): graphbinary_type = DataType.timestamp @classmethod - def dictify(cls, obj, writer): + def dictify(cls, obj, writer, as_value=False): # Java timestamp expects milliseconds integer - Have to use int because of legacy Python ts = int(round(obj * 1000)) - return cls.as_bytes(cls.graphbinary_type, None, struct.pack(">q", ts)) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), + None, struct.pack(">q", ts)) @classmethod def objectify(cls, buff, reader): # Python timestamp expects seconds - return statics.timestamp(struct.unpack(">q", buff.read(8))[0] / 1000.0) + return cls.is_null(buff, reader, lambda b, r: statics.timestamp(struct.unpack(">q", b.read(8))[0] / 1000.0)) def _long_bits_to_double(bits): @@ -288,19 +310,23 @@ class FloatIO(LongIO): byte_format = ">f" @classmethod - def dictify(cls, obj, writer): + def dictify(cls, obj, writer, as_value=False): if math.isnan(obj): - return cls.as_bytes(cls.graphbinary_base_type, None, struct.pack(cls.byte_format, NAN)) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), + None, struct.pack(cls.byte_format, NAN)) elif math.isinf(obj) and obj > 0: - return cls.as_bytes(cls.graphbinary_base_type, None, struct.pack(cls.byte_format, POSITIVE_INFINITY)) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), + None, struct.pack(cls.byte_format, POSITIVE_INFINITY)) elif math.isinf(obj) and obj < 0: - return cls.as_bytes(cls.graphbinary_base_type, None, struct.pack(cls.byte_format, NEGATIVE_INFINITY)) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), + None, struct.pack(cls.byte_format, NEGATIVE_INFINITY)) else: - return cls.as_bytes(cls.graphbinary_base_type, None, struct.pack(cls.byte_format, obj)) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), + None, struct.pack(cls.byte_format, obj)) @classmethod - def objectify(cls, buff, reader): - return struct.unpack(cls.byte_format, buff.read(4))[0] + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, lambda b, r: struct.unpack(cls.byte_format, b.read(4))[0]) class DoubleIO(FloatIO): @@ -313,15 +339,15 @@ class DoubleIO(FloatIO): byte_format = ">d" @classmethod - def objectify(cls, buff, reader): - return struct.unpack(cls.byte_format, buff.read(8))[0] + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, lambda b, r: struct.unpack(cls.byte_format, b.read(8))[0]) class TypeSerializer(_GraphBinaryTypeIO): python_type = TypeType @classmethod - def dictify(cls, typ, writer): + def dictify(cls, typ, writer, as_value=False): return writer.toDict(typ()) @@ -331,12 +357,13 @@ class StringIO(_GraphBinaryTypeIO): graphbinary_type = DataType.string @classmethod - def dictify(cls, obj, writer): - return cls.as_bytes(cls.graphbinary_type, len(obj), obj.encode("utf-8")) + def dictify(cls, obj, writer, as_value=False): + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), + len(obj), obj.encode("utf-8")) @classmethod - def objectify(cls, b, reader): - return cls.read_string(b) + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, lambda b, r: b.read(cls.read_int(b)).decode("utf-8")) class ListIO(_GraphBinaryTypeIO): @@ -345,19 +372,24 @@ class ListIO(_GraphBinaryTypeIO): graphbinary_type = DataType.list @classmethod - def dictify(cls, obj, writer): + def dictify(cls, obj, writer, as_value=False): list_data = bytearray() for item in obj: list_data.extend(writer.writeObject(item)) - return cls.as_bytes(cls.graphbinary_type, len(obj), list_data) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), + len(obj), list_data) @classmethod - def objectify(cls, buff, reader): - size = cls.read_int(buff) + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, cls._read_list) + + @classmethod + def _read_list(cls, b, r): + size = cls.read_int(b) the_list = [] while size > 0: - the_list.append(reader.readObject(buff)) + the_list.append(r.readObject(b)) size = size - 1 return the_list @@ -369,8 +401,8 @@ class SetIO(ListIO): graphbinary_type = DataType.set @classmethod - def objectify(cls, buff, reader): - return set(ListIO.objectify(buff, reader)) + def objectify(cls, buff, reader, as_value=False): + return set(ListIO.objectify(buff, reader, as_value)) class MapIO(_GraphBinaryTypeIO): @@ -379,21 +411,26 @@ class MapIO(_GraphBinaryTypeIO): graphbinary_type = DataType.map @classmethod - def dictify(cls, obj, writer): + def dictify(cls, obj, writer, as_value=False): map_data = bytearray() for k, v in obj.items(): map_data.extend(writer.writeObject(k)) map_data.extend(writer.writeObject(v)) - return cls.as_bytes(cls.graphbinary_type, len(obj), map_data) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), + len(obj), map_data) @classmethod - def objectify(cls, buff, reader): - size = cls.read_int(buff) + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, cls._read_map) + + @classmethod + def _read_map(cls, b, r): + size = cls.read_int(b) the_dict = {} while size > 0: - k = reader.readObject(buff) - v = reader.readObject(buff) + k = r.readObject(b) + v = r.readObject(b) the_dict[k] = v size = size - 1 @@ -406,14 +443,13 @@ class UuidIO(_GraphBinaryTypeIO): graphbinary_type = DataType.uuid @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) - ba.extend(obj.bytes) - return ba + def dictify(cls, obj, writer, as_value=False): + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), + None, obj.bytes) @classmethod - def objectify(cls, b, reader): - return uuid.UUID(bytes=b.read(16)) + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, lambda b, r: uuid.UUID(bytes=b.read(16))) class EdgeIO(_GraphBinaryTypeIO): @@ -422,8 +458,8 @@ class EdgeIO(_GraphBinaryTypeIO): graphbinary_type = DataType.edge @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) + def dictify(cls, obj, writer, as_value=False): + ba = bytearray() ba.extend(writer.writeObject(obj.id)) ba.extend(cls.string_as_bytes(obj.label)) ba.extend(writer.writeObject(obj.inV.id)) @@ -432,14 +468,18 @@ class EdgeIO(_GraphBinaryTypeIO): ba.extend(cls.string_as_bytes(obj.outV.label)) ba.extend([DataType.null.value]) ba.extend([DataType.null.value]) - return ba + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) @classmethod - def objectify(cls, b, reader): - edgeid = reader.readObject(b) + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, cls._read_edge) + + @classmethod + def _read_edge(cls, b, r): + edgeid = r.readObject(b) edgelbl = cls.read_string(b) - edge = Edge(edgeid, Vertex(reader.readObject(b), cls.read_string(b)), - edgelbl, Vertex(reader.readObject(b), cls.read_string(b))) + edge = Edge(edgeid, Vertex(r.readObject(b), cls.read_string(b)), + edgelbl, Vertex(r.readObject(b), cls.read_string(b))) b.read(2) return edge @@ -450,15 +490,15 @@ class PathIO(_GraphBinaryTypeIO): graphbinary_type = DataType.path @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) + def dictify(cls, obj, writer, as_value=False): + ba = bytearray() ba.extend(writer.writeObject(obj.labels)) ba.extend(writer.writeObject(obj.objects)) - return ba + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) @classmethod - def objectify(cls, b, reader): - return Path(reader.readObject(b), reader.readObject(b)) + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, lambda b, r: Path(r.readObject(b), r.readObject(b))) class PropertyIO(_GraphBinaryTypeIO): @@ -467,16 +507,20 @@ class PropertyIO(_GraphBinaryTypeIO): graphbinary_type = DataType.property @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) + def dictify(cls, obj, writer, as_value=False): + ba = bytearray() ba.extend(cls.string_as_bytes(obj.key)) ba.extend(writer.writeObject(obj.value)) ba.extend([DataType.null.value]) - return ba + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) @classmethod - def objectify(cls, b, reader): - p = Property(cls.read_string(b), reader.readObject(b), None) + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, cls._read_property) + + @classmethod + def _read_property(cls, b, r): + p = Property(cls.read_string(b), r.readObject(b), None) b.read(1) return p @@ -487,11 +531,11 @@ class TinkerGraphIO(_GraphBinaryTypeIO): graphbinary_type = DataType.graph @classmethod - def dictify(cls, obj, writer): + def dictify(cls, obj, writer, as_value=False): raise AttributeError("TinkerGraph serialization is not currently supported by gremlin-python") @classmethod - def objectify(cls, b, reader): + def objectify(cls, b, reader, as_value=False): raise AttributeError("TinkerGraph deserialization is not currently supported by gremlin-python") @@ -501,16 +545,20 @@ class VertexIO(_GraphBinaryTypeIO): graphbinary_type = DataType.vertex @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) + def dictify(cls, obj, writer, as_value=False): + ba = bytearray() ba.extend(writer.writeObject(obj.id)) ba.extend(cls.string_as_bytes(obj.label)) ba.extend([DataType.null.value]) - return ba + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) + + @classmethod + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, cls._read_vertex) @classmethod - def objectify(cls, b, reader): - vertex = Vertex(reader.readObject(b), cls.read_string(b)) + def _read_vertex(cls, b, r): + vertex = Vertex(r.readObject(b), cls.read_string(b)) b.read(1) return vertex @@ -521,18 +569,22 @@ class VertexPropertyIO(_GraphBinaryTypeIO): graphbinary_type = DataType.vertexproperty @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) + def dictify(cls, obj, writer, as_value=False): + ba = bytearray() ba.extend(writer.writeObject(obj.id)) ba.extend(cls.string_as_bytes(obj.label)) ba.extend(writer.writeObject(obj.value)) ba.extend([DataType.null.value]) ba.extend([DataType.null.value]) - return ba + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) + + @classmethod + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, cls._read_vertexproperty) @classmethod - def objectify(cls, b, reader): - vp = VertexProperty(reader.readObject(b), cls.read_string(b), reader.readObject(b), None) + def _read_vertexproperty(cls, b, r): + vp = VertexProperty(r.readObject(b), cls.read_string(b), r.readObject(b), None) b.read(1) b.read(1) return vp @@ -541,14 +593,14 @@ class VertexPropertyIO(_GraphBinaryTypeIO): class _EnumIO(_GraphBinaryTypeIO): @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) + def dictify(cls, obj, writer, as_value=False): + ba = bytearray() ba.extend(cls.string_as_bytes(cls.unmangleKeyword(str(obj.name)))) - return ba + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) @classmethod - def objectify(cls, b, reader): - return cls.python_type[cls.read_string(b)] + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, lambda b, r: cls.python_type[cls.read_string(b)]) class BarrierIO(_EnumIO): @@ -597,15 +649,15 @@ class BindingIO(_GraphBinaryTypeIO): graphbinary_type = DataType.binding @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) + def dictify(cls, obj, writer, as_value=False): + ba = bytearray() ba.extend(cls.string_as_bytes(obj.key)) ba.extend(writer.writeObject(obj.value)) - return ba + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) @classmethod - def objectify(cls, b, reader): - return Binding(cls.read_string(b), reader.readObject(b)) + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, lambda b, r: Binding(cls.read_string(b), reader.readObject(b))) class BytecodeIO(_GraphBinaryTypeIO): @@ -613,8 +665,8 @@ class BytecodeIO(_GraphBinaryTypeIO): graphbinary_type = DataType.bytecode @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) + def dictify(cls, obj, writer, as_value=False): + ba = bytearray() ba.extend(struct.pack(">i", len(obj.step_instructions))) for inst in obj.step_instructions: inst_name, inst_args = inst[0], inst[1:] if len(inst) > 1 else [] @@ -630,11 +682,15 @@ class BytecodeIO(_GraphBinaryTypeIO): ba.extend(struct.pack(">i", len(inst_args))) for arg in inst_args: ba.extend(writer.writeObject(arg)) - - return ba - + + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) + @classmethod - def objectify(cls, b, reader): + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, cls._read_bytecode) + + @classmethod + def _read_bytecode(cls, b, r): bytecode = Bytecode() step_count = cls.read_int(b) @@ -644,7 +700,7 @@ class BytecodeIO(_GraphBinaryTypeIO): inst_ct = cls.read_int(b) iy = 0 while iy < inst_ct: - inst.append(reader.readObject(b)) + inst.append(r.readObject(b)) iy += 1 bytecode.step_instructions.append(inst) ix += 1 @@ -656,7 +712,7 @@ class BytecodeIO(_GraphBinaryTypeIO): inst_ct = cls.read_int(b) iy = 0 while iy < inst_ct: - inst.append(reader.readObject(b)) + inst.append(r.readObject(b)) iy += 1 bytecode.source_instructions.append(inst) ix += 1 @@ -670,8 +726,8 @@ class LambdaIO(_GraphBinaryTypeIO): graphbinary_type = DataType.lambda_ @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) + def dictify(cls, obj, writer, as_value=False): + ba = bytearray() if as_value else bytearray([cls.graphbinary_type.value]) lambda_result = obj() script = lambda_result if isinstance(lambda_result, str) else lambda_result[0] language = statics.default_lambda_language if isinstance(lambda_result, str) else lambda_result[1] @@ -697,8 +753,8 @@ class PIO(_GraphBinaryTypeIO): python_type = P @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) + def dictify(cls, obj, writer, as_value=False): + ba = bytearray() if as_value else bytearray([cls.graphbinary_type.value]) ba.extend(cls.string_as_bytes(obj.operator)) additional = [writer.writeObject(obj.value), writer.writeObject(obj.other)] \ if obj.other is not None else [writer.writeObject(obj.value)] @@ -724,12 +780,21 @@ class TraverserIO(_GraphBinaryTypeIO): python_type = Traverser @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) - ba.extend(struct.pack(">i", obj.bulk)) + def dictify(cls, obj, writer, as_value=False): + ba = bytearray() + ba.extend(struct.pack(">q", obj.bulk)) ba.extend(writer.writeObject(obj.object)) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) - return ba + @classmethod + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, cls._read_traverser) + + @classmethod + def _read_traverser(cls, b, r): + bulk = struct.unpack(">q", b.read(8))[0] + obj = r.readObject(b) + return Traverser(obj, bulk=bulk) class ByteIO(_GraphBinaryTypeIO): @@ -737,14 +802,12 @@ class ByteIO(_GraphBinaryTypeIO): graphbinary_type = DataType.byte @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) - ba.extend(struct.pack(">b", obj)) - return ba + def dictify(cls, obj, writer, as_value=False): + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, struct.pack(">b", obj)) @classmethod - def objectify(cls, b, reader): - return int.__new__(SingleByte, struct.unpack_from(">b", b.read(1))[0]) + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, lambda b, r: int.__new__(SingleByte, struct.unpack_from(">b", b.read(1))[0])) class ByteBufferIO(_GraphBinaryTypeIO): @@ -752,11 +815,15 @@ class ByteBufferIO(_GraphBinaryTypeIO): graphbinary_type = DataType.bytebuffer @classmethod - def dictify(cls, obj, writer): - return cls.as_bytes(cls.graphbinary_type, len(obj), obj) + def dictify(cls, obj, writer, as_value=False): + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), len(obj), obj) + + @classmethod + def objectify(cls, buff, reader): + return cls.is_null(buff, reader, cls._read_bytebuffer) @classmethod - def objectify(cls, b, reader): + def _read_bytebuffer(cls, b, r): size = cls.read_int(b) return ByteBufferType(b.read(size)) @@ -766,14 +833,12 @@ class BooleanIO(_GraphBinaryTypeIO): graphbinary_type = DataType.boolean @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) - ba.extend(struct.pack(">b", 0x01 if obj else 0x00)) - return ba + def dictify(cls, obj, writer, as_value=False): + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, struct.pack(">b", 0x01 if obj else 0x00)) @classmethod - def objectify(cls, b, reader): - return True if struct.unpack_from(">b", b.read(1))[0] == 0x01 else False + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, lambda b, r: True if struct.unpack_from(">b", b.read(1))[0] == 0x01 else False) class TextPIO(_GraphBinaryTypeIO): @@ -781,8 +846,8 @@ class TextPIO(_GraphBinaryTypeIO): python_type = TextP @classmethod - def dictify(cls, obj, writer): - ba = bytearray([cls.graphbinary_type.value]) + def dictify(cls, obj, writer, as_value=False): + ba = bytearray() if as_value else bytearray([cls.graphbinary_type.value]) ba.extend(cls.string_as_bytes(obj.operator)) additional = [writer.writeObject(obj.value), writer.writeObject(obj.other)] \ if obj.other is not None else [writer.writeObject(obj.value)] @@ -798,12 +863,16 @@ class BulkSetIO(_GraphBinaryTypeIO): graphbinary_type = DataType.bulkset @classmethod - def objectify(cls, buff, reader): - size = cls.read_int(buff) + def objectify(cls, buff, reader, as_value=False): + return cls.is_null(buff, reader, cls._read_bulkset) + + @classmethod + def _read_bulkset(cls, b, r): + size = cls.read_int(b) the_list = [] while size > 0: - itm = reader.readObject(buff) - bulk = cls.read_int(buff) + itm = r.readObject(b) + bulk = cls.read_int(b) for y in range(bulk): the_list.append(itm) size = size - 1 diff --git a/gremlin-python/src/main/jython/tests/conftest.py b/gremlin-python/src/main/jython/tests/conftest.py index 408c469..fb31c31 100644 --- a/gremlin-python/src/main/jython/tests/conftest.py +++ b/gremlin-python/src/main/jython/tests/conftest.py @@ -28,10 +28,13 @@ from gremlin_python.driver.driver_remote_connection import ( DriverRemoteConnection) from gremlin_python.driver.protocol import GremlinServerWSProtocol from gremlin_python.driver.serializer import ( - GraphSONMessageSerializer, GraphSONSerializersV2d0, - GraphSONSerializersV3d0) + GraphSONMessageSerializer, GraphSONSerializersV2d0, GraphSONSerializersV3d0, + GraphBinaryMessageSerializerV1) from gremlin_python.driver.tornado.transport import TornadoTransport +gremlin_server_host = "localhost" +gremlin_server_url = 'ws://' + gremlin_server_host + ':45940/gremlin' + @pytest.fixture def connection(request): @@ -41,7 +44,7 @@ def connection(request): executor = concurrent.futures.ThreadPoolExecutor(5) pool = queue.Queue() try: - conn = Connection('ws://localhost:45941/gremlin', 'gmodern', protocol, + conn = Connection(gremlin_server_url, 'gmodern', protocol, lambda: TornadoTransport(), executor, pool) except OSError: executor.shutdown() @@ -57,7 +60,7 @@ def connection(request): @pytest.fixture def client(request): try: - client = Client('ws://localhost:45940/gremlin', 'gmodern') + client = Client(gremlin_server_url, 'gmodern') except OSError: pytest.skip('Gremlin Server is not running') else: @@ -70,7 +73,7 @@ def client(request): @pytest.fixture def secure_client(request): try: - client = Client('ws://localhost:45941/gremlin', 'gmodern', username='stephen', password='password') + client = Client('ws://' + gremlin_server_host + ':45941/gremlin', 'gmodern', username='stephen', password='password') except OSError: pytest.skip('Gremlin Server is not running') else: @@ -80,14 +83,17 @@ def secure_client(request): return client [email protected](params=['v2', 'v3']) [email protected](params=['graphsonv2', 'graphsonv3', 'graphbinaryv1']) def remote_connection(request): try: - if request.param == 'v2': - remote_conn = DriverRemoteConnection('ws://localhost:45940/gremlin', 'gmodern', + if request.param == 'graphbinaryv1': + remote_conn = DriverRemoteConnection(gremlin_server_url, 'gmodern', + message_serializer=serializer.GraphBinaryMessageSerializerV1()) + elif request.param == 'graphsonv2': + remote_conn = DriverRemoteConnection(gremlin_server_url, 'gmodern', message_serializer=serializer.GraphSONSerializersV2d0()) else: - remote_conn = DriverRemoteConnection('ws://localhost:45940/gremlin', 'gmodern') + remote_conn = DriverRemoteConnection(gremlin_server_url, 'gmodern') except OSError: pytest.skip('Gremlin Server is not running') else: @@ -100,7 +106,8 @@ def remote_connection(request): @pytest.fixture def remote_connection_v2(request): try: - remote_conn = DriverRemoteConnection('ws://localhost:45940/gremlin', 'g', message_serializer=serializer.GraphSONSerializersV2d0()) + remote_conn = DriverRemoteConnection(gremlin_server_url, 'g', + message_serializer=serializer.GraphSONSerializersV2d0()) except OSError: pytest.skip('Gremlin Server is not running') else: @@ -118,3 +125,8 @@ def graphson_serializer_v2(request): @pytest.fixture def graphson_serializer_v3(request): return GraphSONSerializersV3d0() + + [email protected] +def graphbinary_serializer_v1(request): + return GraphBinaryMessageSerializerV1() diff --git a/gremlin-python/src/main/jython/tests/driver/test_serializer.py b/gremlin-python/src/main/jython/tests/driver/test_serializer.py index f1b4ccb..80c5966 100644 --- a/gremlin-python/src/main/jython/tests/driver/test_serializer.py +++ b/gremlin-python/src/main/jython/tests/driver/test_serializer.py @@ -18,20 +18,28 @@ under the License. ''' from gremlin_python.structure.io import graphsonV2d0 from gremlin_python.structure.io import graphsonV3d0 +from gremlin_python.structure.io import graphbinaryV1 __author__ = 'David M. Brown' -def test_graphson_serialzier_v2(graphson_serializer_v2): +def test_graphson_serializer_v2(graphson_serializer_v2): assert graphson_serializer_v2.version == b"application/vnd.gremlin-v2.0+json" assert isinstance(graphson_serializer_v2._graphson_reader, graphsonV2d0.GraphSONReader) - assert isinstance(graphson_serializer_v2.standard._graphson_writer, graphsonV2d0.GraphSONWriter) - assert isinstance(graphson_serializer_v2.traversal._graphson_writer, graphsonV2d0.GraphSONWriter) + assert isinstance(graphson_serializer_v2.standard._writer, graphsonV2d0.GraphSONWriter) + assert isinstance(graphson_serializer_v2.traversal._writer, graphsonV2d0.GraphSONWriter) -def test_graphson_serialzier_v3(graphson_serializer_v3): +def test_graphson_serializer_v3(graphson_serializer_v3): assert graphson_serializer_v3.version == b"application/vnd.gremlin-v3.0+json" assert isinstance(graphson_serializer_v3._graphson_reader, graphsonV3d0.GraphSONReader) - assert isinstance(graphson_serializer_v3.standard._graphson_writer, graphsonV3d0.GraphSONWriter) - assert isinstance(graphson_serializer_v3.traversal._graphson_writer, graphsonV3d0.GraphSONWriter) + assert isinstance(graphson_serializer_v3.standard._writer, graphsonV3d0.GraphSONWriter) + assert isinstance(graphson_serializer_v3.traversal._writer, graphsonV3d0.GraphSONWriter) + + +def test_graphbinary_serializer_v1(graphbinary_serializer_v1): + assert graphbinary_serializer_v1.version == b"application/vnd.graphbinary-v1.0" + assert isinstance(graphbinary_serializer_v1._graphbinary_reader, graphbinaryV1.GraphBinaryReader) + assert isinstance(graphbinary_serializer_v1.standard._writer, graphbinaryV1.GraphBinaryWriter) + assert isinstance(graphbinary_serializer_v1.traversal._writer, graphbinaryV1.GraphBinaryWriter) diff --git a/gremlin-python/src/main/jython/tests/structure/io/test_functionalityio.py b/gremlin-python/src/main/jython/tests/structure/io/test_functionalityio.py new file mode 100644 index 0000000..54dd35e --- /dev/null +++ b/gremlin-python/src/main/jython/tests/structure/io/test_functionalityio.py @@ -0,0 +1,64 @@ +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +''' + +import datetime +import uuid + +from gremlin_python.structure.graph import Graph +from gremlin_python.statics import * + + +def test_timestamp(remote_connection): + g = Graph().traversal().withRemote(remote_connection) + ts = timestamp(1481750076295 / 1000) + resp = g.addV('test_vertex').property('ts', ts) + resp = resp.toList() + vid = resp[0].id + try: + ts_prop = g.V(vid).properties('ts').toList()[0] + assert isinstance(ts_prop.value, timestamp) + assert ts_prop.value == ts + finally: + g.V(vid).drop().iterate() + + +def test_datetime(remote_connection): + g = Graph().traversal().withRemote(remote_connection) + dt = datetime.datetime.utcfromtimestamp(1481750076295 / 1000) + resp = g.addV('test_vertex').property('dt', dt).toList() + vid = resp[0].id + try: + dt_prop = g.V(vid).properties('dt').toList()[0] + assert isinstance(dt_prop.value, datetime.datetime) + assert dt_prop.value == dt + finally: + g.V(vid).drop().iterate() + + +def test_uuid(remote_connection): + g = Graph().traversal().withRemote(remote_connection) + uid = uuid.UUID("41d2e28a-20a4-4ab0-b379-d810dede3786") + resp = g.addV('test_vertex').property('uuid', uid).toList() + vid = resp[0].id + try: + uid_prop = g.V(vid).properties('uuid').toList()[0] + assert isinstance(uid_prop.value, uuid.UUID) + assert uid_prop.value == uid + finally: + g.V(vid).drop().iterate() diff --git a/gremlin-python/src/main/jython/tests/structure/io/test_graphbinaryV1.py b/gremlin-python/src/main/jython/tests/structure/io/test_graphbinaryV1.py index 620c3fc..bcaad3d 100644 --- a/gremlin-python/src/main/jython/tests/structure/io/test_graphbinaryV1.py +++ b/gremlin-python/src/main/jython/tests/structure/io/test_graphbinaryV1.py @@ -77,7 +77,7 @@ class TestGraphSONWriter(object): assert x == output def test_date(self): - x = calendar.timegm(datetime.datetime(2016, 12, 14, 16, 14, 36, 295000).utctimetuple()) + x = datetime.datetime(2016, 12, 14, 16, 14, 36, 295000) output = self.graphbinary_reader.readObject(self.graphbinary_writer.writeObject(x)) assert x == output diff --git a/gremlin-python/src/main/jython/tests/structure/io/test_graphsonV3d0.py b/gremlin-python/src/main/jython/tests/structure/io/test_graphsonV3d0.py index 7cc8d68..17bb177 100644 --- a/gremlin-python/src/main/jython/tests/structure/io/test_graphsonV3d0.py +++ b/gremlin-python/src/main/jython/tests/structure/io/test_graphsonV3d0.py @@ -88,7 +88,6 @@ class TestGraphSONReader(object): assert x.count("marko") == 1 assert x.count("josh") == 3 - def test_number_input(self): x = self.graphson_reader.readObject(json.dumps({ "@type": "gx:Byte", @@ -535,44 +534,3 @@ class TestGraphSONWriter(object): c = str.__new__(SingleChar, chr(76)) output = self.graphson_writer.writeObject(c) assert expected == output - - -class TestFunctionalGraphSONIO(object): - """Functional IO tests""" - - def test_timestamp(self, remote_connection): - g = Graph().traversal().withRemote(remote_connection) - ts = timestamp(1481750076295 / 1000) - resp = g.addV('test_vertex').property('ts', ts) - resp = resp.toList() - vid = resp[0].id - try: - ts_prop = g.V(vid).properties('ts').toList()[0] - assert isinstance(ts_prop.value, timestamp) - assert ts_prop.value == ts - finally: - g.V(vid).drop().iterate() - - def test_datetime(self, remote_connection): - g = Graph().traversal().withRemote(remote_connection) - dt = datetime.datetime.utcfromtimestamp(1481750076295 / 1000) - resp = g.addV('test_vertex').property('dt', dt).toList() - vid = resp[0].id - try: - dt_prop = g.V(vid).properties('dt').toList()[0] - assert isinstance(dt_prop.value, datetime.datetime) - assert dt_prop.value == dt - finally: - g.V(vid).drop().iterate() - - def test_uuid(self, remote_connection): - g = Graph().traversal().withRemote(remote_connection) - uid = uuid.UUID("41d2e28a-20a4-4ab0-b379-d810dede3786") - resp = g.addV('test_vertex').property('uuid', uid).toList() - vid = resp[0].id - try: - uid_prop = g.V(vid).properties('uuid').toList()[0] - assert isinstance(uid_prop.value, uuid.UUID) - assert uid_prop.value == uid - finally: - g.V(vid).drop().iterate()
