This is an automated email from the ASF dual-hosted git repository. xiazcy pushed a commit to branch add-http-gremlin-python in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit b2ef3dbb37a9e34e91319214a751b575eee94a14 Author: Yang Xia <[email protected]> AuthorDate: Thu Oct 12 22:23:23 2023 -0700 Enable remote connection over HTTP for gremlin python --- CHANGELOG.asciidoc | 2 + gremlin-python/docker-compose.yml | 2 + .../gremlin_python/driver/aiohttp/transport.py | 101 ++++++++++ .../main/python/gremlin_python/driver/client.py | 34 +++- .../main/python/gremlin_python/driver/protocol.py | 95 ++++++++- .../python/gremlin_python/driver/serializer.py | 7 +- .../python/gremlin_python/process/translator.py | 110 +++++++---- gremlin-python/src/main/python/tests/conftest.py | 58 +++++- .../driver/test_driver_remote_connection_http.py | 214 +++++++++++++++++++++ .../main/python/tests/process/test_translator.py | 37 +++- 10 files changed, 605 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6ee4818231..152f66a8fa 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -28,6 +28,8 @@ This release also includes changes from <<release-3-5-8, 3.5.8>>. * Fixed a javadoc comment in `GraphTraversal.not()` method. * Allowed `gremlin-driver` to be used over HTTP for experimental purposes. * Deprecated the `HandshakeInterceptor` in favor of a more generic `RequestInterceptor`. +* Allowed `gremlin-python` to be used over HTTP for experimental purposes. +* Updated `gremlin-python` translator to enable additional traversals over HTTP. * Fixed a bug in `StarGraph` where `EdgeFilter` did not remove associated Edge Properties. * Added translator to the Go GLV. * Fixed bug with filtering for `group()` when the side-effect label was defined for it. diff --git a/gremlin-python/docker-compose.yml b/gremlin-python/docker-compose.yml index 6262c61da8..02e693ee5b 100644 --- a/gremlin-python/docker-compose.yml +++ b/gremlin-python/docker-compose.yml @@ -58,6 +58,8 @@ services: - KRB5CCNAME=./test-tkt.cc - GREMLIN_SERVER_URL=ws://gremlin-server-test-python:{}/gremlin - GREMLIN_SERVER_BASIC_AUTH_URL=wss://gremlin-server-test-python:{}/gremlin + - GREMLIN_SERVER_URL_HTTP=http://gremlin-server-test-python:{}/ + - GREMLIN_SERVER_BASIC_AUTH_URL_HTTP=https://gremlin-server-test-python:{}/ - KRB_HOSTNAME=${KRB_HOSTNAME:-gremlin-server-test} - VERSION=${VERSION} working_dir: /python_app diff --git a/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py b/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py index 34dab16b53..4b9ead23cc 100644 --- a/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py +++ b/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py @@ -138,3 +138,104 @@ class AiohttpTransport(AbstractBaseTransport): def closed(self): # Connection is closed if either the websocket or the client session is closed. return self._websocket.closed or self._client_session.closed + + +class AiohttpHTTPTransport(AbstractBaseTransport): + nest_asyncio_applied = False + + def __init__(self, call_from_event_loop=None, read_timeout=None, write_timeout=None, **kwargs): + if call_from_event_loop is not None and call_from_event_loop and not AiohttpTransport.nest_asyncio_applied: + """ + The AiohttpTransport implementation uses the asyncio event loop. Because of this, it cannot be called + within an event loop without nest_asyncio. If the code is ever refactored so that it can be called + within an event loop this import and call can be removed. Without this, applications which use the + event loop to call gremlin-python (such as Jupyter) will not work. + """ + import nest_asyncio + nest_asyncio.apply() + AiohttpTransport.nest_asyncio_applied = True + + # Start event loop and initialize client session and response to None + self._loop = asyncio.new_event_loop() + self._client_session = None + self._http_req_resp = None + self._enable_ssl = False + + # Set all inner variables to parameters passed in. + self._aiohttp_kwargs = kwargs + self._write_timeout = write_timeout + self._read_timeout = read_timeout + if "ssl_options" in self._aiohttp_kwargs: + self._ssl_context = self._aiohttp_kwargs.pop("ssl_options") + self._enable_ssl = True + + # if "ssl_options" in self._aiohttp_kwargs: + # self._aiohttp_kwargs["ssl_context"] = self._aiohttp_kwargs.pop("ssl_options") + + def __del__(self): + # Close will only actually close if things are left open, so this is safe to call. + # Clean up any connection resources and close the event loop. + self.close() + + def connect(self, url, headers=None): + # Inner function to perform async connect. + async def async_connect(): + # Start client session and use it to send all HTTP requests. Base url is the endpoint, headers are set here + # Base url can only parse basic url with no path, see https://github.com/aio-libs/aiohttp/issues/6647 + if self._enable_ssl: + # ssl context is established through tcp connector + tcp_conn = aiohttp.TCPConnector(ssl_context=self._ssl_context) + self._client_session = aiohttp.ClientSession(connector=tcp_conn, + base_url=url, headers=headers, loop=self._loop) + else: + self._client_session = aiohttp.ClientSession(base_url=url, headers=headers, loop=self._loop) + + + # Execute the async connect synchronously. + self._loop.run_until_complete(async_connect()) + + def write(self, message): + # Inner function to perform async write. + async def async_write(): + basic_auth = None + # basic password authentication for https connections + if message['auth']: + basic_auth = aiohttp.BasicAuth(message['auth']['username'], message['auth']['password']) + async with async_timeout.timeout(self._write_timeout): + self._http_req_resp = await self._client_session.post(url="/gremlin", + auth=basic_auth, + data=message['payload'], + headers=message['headers'], + **self._aiohttp_kwargs) + + # Execute the async write synchronously. + self._loop.run_until_complete(async_write()) + + def read(self): + # Inner function to perform async read. + async def async_read(): + async with async_timeout.timeout(self._read_timeout): + # using http request read() + return await self._http_req_resp.text() + + return self._loop.run_until_complete(async_read()) + + def close(self): + # Inner function to perform async close. + async def async_close(): + if self._client_session is not None and not self._client_session.closed: + await self._client_session.close() + self._client_session = None + + # If the loop is not closed (connection hasn't already been closed) + if not self._loop.is_closed(): + # Execute the async close synchronously. + self._loop.run_until_complete(async_close()) + + # Close the event loop. + self._loop.close() + + @property + def closed(self): + # Connection is closed when client session is closed. + return self._client_session.closed diff --git a/gremlin-python/src/main/python/gremlin_python/driver/client.py b/gremlin-python/src/main/python/gremlin_python/driver/client.py index ec7078d593..6ad9fd5517 100644 --- a/gremlin-python/src/main/python/gremlin_python/driver/client.py +++ b/gremlin-python/src/main/python/gremlin_python/driver/client.py @@ -19,6 +19,7 @@ import logging import warnings import queue +import re from concurrent.futures import ThreadPoolExecutor from gremlin_python.driver import connection, protocol, request, serializer @@ -45,15 +46,20 @@ class Client: kerberized_service="", headers=None, session=None, enable_user_agent_on_connect=True, **transport_kwargs): log.info("Creating Client with url '%s'", url) + + # check via url that we are using http protocol + self._use_http = re.search('^http', url) + self._closed = False self._url = url self._headers = headers self._enable_user_agent_on_connect = enable_user_agent_on_connect self._traversal_source = traversal_source - if "max_content_length" not in transport_kwargs: + if not self._use_http and "max_content_length" not in transport_kwargs: transport_kwargs["max_content_length"] = 10 * 1024 * 1024 if message_serializer is None: message_serializer = serializer.GraphBinarySerializersV1() + # message_serializer = serializer.GraphSONMessageSerializer() self._message_serializer = message_serializer self._username = username @@ -63,21 +69,31 @@ class Client: if transport_factory is None: try: from gremlin_python.driver.aiohttp.transport import ( - AiohttpTransport) + AiohttpTransport, AiohttpHTTPTransport) except ImportError: raise Exception("Please install AIOHTTP or pass " "custom transport factory") else: def transport_factory(): - return AiohttpTransport(**transport_kwargs) + if self._use_http: + return AiohttpHTTPTransport(**transport_kwargs) + else: + return AiohttpTransport(**transport_kwargs) self._transport_factory = transport_factory if protocol_factory is None: - def protocol_factory(): return protocol.GremlinServerWSProtocol( - self._message_serializer, - username=self._username, - password=self._password, - kerberized_service=kerberized_service, - max_content_length=transport_kwargs["max_content_length"]) + def protocol_factory(): + if self._use_http: + return protocol.GremlinServerHTTPProtocol( + self._message_serializer, + username=self._username, + password=self._password) + else: + return protocol.GremlinServerWSProtocol( + self._message_serializer, + username=self._username, + password=self._password, + kerberized_service=kerberized_service, + max_content_length=transport_kwargs["max_content_length"]) self._protocol_factory = protocol_factory if self._session_enabled: if pool_size is None: diff --git a/gremlin-python/src/main/python/gremlin_python/driver/protocol.py b/gremlin-python/src/main/python/gremlin_python/driver/protocol.py index 9323210356..6aaa9b8efe 100644 --- a/gremlin-python/src/main/python/gremlin_python/driver/protocol.py +++ b/gremlin-python/src/main/python/gremlin_python/driver/protocol.py @@ -16,6 +16,7 @@ # specific language governing permissions and limitations # under the License. # +import json import logging import abc import base64 @@ -25,6 +26,8 @@ import struct from gremlin_python.driver import request from gremlin_python.driver.resultset import ResultSet +from gremlin_python.process.translator import Translator +from gremlin_python.process.traversal import Bytecode log = logging.getLogger("gremlinpython") @@ -63,7 +66,6 @@ class AbstractBaseProtocol(metaclass=abc.ABCMeta): class GremlinServerWSProtocol(AbstractBaseProtocol): - QOP_AUTH_BIT = 1 _kerberos_context = None _max_content_length = 10 * 1024 * 1024 @@ -133,7 +135,7 @@ class GremlinServerWSProtocol(AbstractBaseProtocol): # This message is going to be huge and kind of hard to read, but in the event of an error, # it can provide invaluable info, so space it out appropriately. log.error("\r\nReceived error message '%s'\r\n\r\nWith results dictionary '%s'", - str(message), str(results_dict)) + str(message), str(results_dict)) del results_dict[request_id] raise GremlinServerError(message['status']) @@ -185,8 +187,95 @@ class GremlinServerWSProtocol(AbstractBaseProtocol): name_length = len(self._username) fmt = '!I' + str(name_length) + 's' word = self.QOP_AUTH_BIT << 24 | self._max_content_length - out = struct.pack(fmt, word, self._username.encode("utf-8"),) + out = struct.pack(fmt, word, self._username.encode("utf-8"), ) encoded = base64.b64encode(out).decode('ascii') kerberos.authGSSClientWrap(self._kerberos_context, encoded) auth = kerberos.authGSSClientResponse(self._kerberos_context) return request.RequestMessage('', 'authentication', {'sasl': auth}) + + +class GremlinServerHTTPProtocol(AbstractBaseProtocol): + + def __init__(self, + message_serializer, + username='', password=''): + self._message_serializer = message_serializer + self._username = username + self._password = password + + def connection_made(self, transport): + super(GremlinServerHTTPProtocol, self).connection_made(transport) + + # Transforms request message into string + def write(self, request_id, request_message): + payload = { + 'requestId': request_id + } + + gremlinArgs = request_message.args['gremlin'] + useBytecode = isinstance(gremlinArgs, Bytecode) + + # translate bytecode into scripts + if useBytecode: + request_message.args['gremlin'] = Translator().of('g').translate(gremlinArgs) + payload['op'] = 'bytecode' + + # add all args into the payload + for k, v in request_message.args.items(): + payload[k] = v + + json_data = json.dumps(payload) + + basic_auth = {} + if self._username and self._password: + basic_auth['username'] = self._username + basic_auth['password'] = self._password + + message = { + 'headers': {'CONTENT-TYPE': 'application/json', + 'ACCEPT': str(self._message_serializer.version, encoding='utf-8')}, + 'payload': json_data, + 'auth': basic_auth + } + + self._transport.write(message) + + def data_received(self, message, results_dict): + # if Gremlin Server cuts off then we get a None for the message + if message is None: + log.error("Received empty message from server.") + raise GremlinServerError({'code': 500, + 'message': 'Server disconnected - please try to reconnect', 'attributes': {}}) + + # if a request query cannot be compiled by Gremlin Server, the HTTP handler will send back the exception in + # json string without a requestId + if 'message' in message and 'requestId' not in message: + log.error("\r\nReceived error message from server '%s'", str(message)) + raise GremlinServerError({'code': 400, + 'message': message, 'attributes': {}}) + + message = self._message_serializer.deserialize_message(message) + request_id = message['requestId'] + result_set = results_dict[request_id] if request_id in results_dict else ResultSet(None, None) + status_code = message['status']['code'] + aggregate_to = message['result']['meta'].get('aggregateTo', 'list') + data = message['result']['data'] + result_set.aggregate_to = aggregate_to + + if status_code == 204: + result_set.stream.put_nowait([]) + del results_dict[request_id] + return status_code + elif status_code in [200, 206]: + result_set.stream.put_nowait(data) + if status_code == 200: + result_set.status_attributes = message['status']['attributes'] + del results_dict[request_id] + return status_code + else: + # This message is going to be huge and kind of hard to read, but in the event of an error, + # it can provide invaluable info, so space it out appropriately. + log.error("\r\nReceived error message '%s'\r\n\r\nWith results dictionary '%s'", + str(message), str(results_dict)) + del results_dict[request_id] + raise GremlinServerError(message['status']) diff --git a/gremlin-python/src/main/python/gremlin_python/driver/serializer.py b/gremlin-python/src/main/python/gremlin_python/driver/serializer.py index a0b8832119..74ed99523c 100644 --- a/gremlin-python/src/main/python/gremlin_python/driver/serializer.py +++ b/gremlin-python/src/main/python/gremlin_python/driver/serializer.py @@ -17,6 +17,7 @@ # under the License. # +import base64 import logging import struct import uuid @@ -156,7 +157,8 @@ class GraphSONMessageSerializer(object): return message def deserialize_message(self, message): - msg = json.loads(message.decode('utf-8')) + # for parsing string message via HTTP connections + msg = json.loads(message if isinstance(message, str) else message.decode('utf-8')) return self._graphson_reader.to_object(msg) @@ -268,7 +270,8 @@ class GraphBinarySerializersV1(object): return bytes(ba) def deserialize_message(self, message): - b = io.BytesIO(message) + # for parsing string message via HTTP connections + b = io.BytesIO(base64.b64decode(message) if isinstance(message, str) else message) b.read(1) # version diff --git a/gremlin-python/src/main/python/gremlin_python/process/translator.py b/gremlin-python/src/main/python/gremlin_python/process/translator.py index efbc4b42e3..9f29e44eb3 100755 --- a/gremlin-python/src/main/python/gremlin_python/process/translator.py +++ b/gremlin-python/src/main/python/gremlin_python/process/translator.py @@ -24,11 +24,11 @@ sent to any TinkerPop compliant HTTP endpoint. """ __author__ = 'Kelvin R. Lawrence (gfxman)' -from gremlin_python.process.graph_traversal import __ -from gremlin_python.process.anonymous_traversal import traversal +import re + from gremlin_python.process.traversal import * -from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection from gremlin_python.process.strategies import * +from gremlin_python.structure.graph import Vertex, Edge, VertexProperty from datetime import datetime @@ -39,16 +39,16 @@ class Translator: # Dictionary used to reverse-map token IDs to strings options = { - WithOptions.tokens: 'tokens', - WithOptions.none: 'none', - WithOptions.ids: 'ids', - WithOptions.labels: 'labels', - WithOptions.keys: 'keys', - WithOptions.values: 'values', - WithOptions.all: 'all', - WithOptions.indexer: 'indexer', - WithOptions.list: 'list', - WithOptions.map: 'map' + WithOptions.tokens: 'tokens', + WithOptions.none: 'none', + WithOptions.ids: 'ids', + WithOptions.labels: 'labels', + WithOptions.keys: 'keys', + WithOptions.values: 'values', + WithOptions.all: 'all', + WithOptions.indexer: 'indexer', + WithOptions.list: 'list', + WithOptions.map: 'map' } def __init__(self, traversal_source=None): @@ -60,20 +60,22 @@ class Translator: def get_target_language(self): return "gremlin-groovy" - def of(self,traversal_source): + def of(self, traversal_source): self.traversal_source = traversal_source return self # Do any needed special processing for the representation - # of strings and dates. + # of strings and dates and boolean. def fixup(self, v): if isinstance(v, str): return f'\'{v}\'' elif type(v) == datetime: return self.process_date(v) + elif type(v) == bool: + return 'true' if v else 'false' else: return str(v) - + # Turn a Python datetime into the equivalent new Date(...) def process_date(self, date): y = date.year - 1900 @@ -97,27 +99,56 @@ class Translator: else: res += self.fixup(p.value) if p.other is not None: - res+= ',' + self.fixup(p.other) + res += ',' + self.fixup(p.other) res += ')' return res # Special processing to handle strategies def process_strategy(self, s): c = 0 - res = f'new {str(s)}(' - for key in s.configuration: - res += ',' if c > 0 else '' - res += key + ':' - val = s.configuration[key] - if isinstance(val, Traversal): - res += self.translate(val.bytecode, child=True) - else: - res += self.fixup(val) - c += 1 - res += ')' + res = '' + # if parameter is empty, only pass class name (referenced GroovyTranslator.java) + if not s.configuration: + res += s.__class__.__name__ + else: + res = f'new {str(s)}(' + for key in s.configuration: + res += ',' if c > 0 else '' + res += key + ':' + val = s.configuration[key] + if isinstance(val, Traversal): + res += self.translate(val.bytecode, child=True) + else: + res += self.fixup(val) + c += 1 + res += ')' return res pass + # Special processing to handle vertices + def process_vertex(self, vertex): + return f'new ReferenceVertex({str(vertex.id)},\'{vertex.label}\')' + + # Special processing to handle edges + def process_edge(self, edge): + return f'new ReferenceEdge({str(edge.id)},\'{edge.label}\',' \ + f'new ReferenceVertex({str(edge.inV.id)},\'{edge.inV.label}\'),' \ + f'new ReferenceVertex({str(edge.outV.id)},\'{edge.outV.label}\'))' + + # Special processing to handle vertex property + def process_vertex_property(self, vp): + return f'new ReferenceVertexProperty({str(vp.id)},\'{vp.label}\',{self.fixup(vp.value)})' + + # Special processing to handle lambda + def process_lambda(self, lam): + lambda_result = lam() + script = lambda_result if isinstance(lambda_result, str) else lambda_result[0] + return f'{script}' if re.match(r"^\{.*\}$", script) else f'{{{script}}}' + + # Special processing to handle bindings inside of traversals + def process_binding(self, binding): + return f'Bindings.instance().of(\'{binding.key}\', {str(binding.value)})' + # Main driver of the translation. Different parts of # a Traversal are handled appropriately. def do_translation(self, step): @@ -130,16 +161,21 @@ class Translator: for p in params: script += ',' if c > 0 else '' if with_opts: - script += f'WithOptions.{self.options[p]}' + script += f'WithOptions.{self.options[p]}' elif type(p) == Bytecode: script += self.translate(p, True) elif isinstance(p, P): script += self.process_predicate(p) + elif type(p) == Vertex: + script += self.process_vertex(p) + elif type(p) == Edge: + script += self.process_edge(p) + elif type(p) == VertexProperty: + script += self.process_vertex_property(p) elif type(p) in [Cardinality, Pop, Operator]: tmp = str(p) - script += tmp[0:-1] if tmp.endswith('_') else tmp - elif type(p) in [ReadOnlyStrategy, SubgraphStrategy, VertexProgramStrategy, - OptionsStrategy, PartitionStrategy]: + script += tmp[0:-1] if tmp.endswith('_') else tmp + elif isinstance(p, TraversalStrategy): # this will capture all strategies script += self.process_strategy(p) elif type(p) == datetime: script += self.process_date(p) @@ -150,8 +186,14 @@ class Translator: script += f'\'{p}\'' elif type(p) == bool: script += 'true' if p else 'false' + elif isinstance(p, type(lambda: None)) and p.__name__ == (lambda: None).__name__: + script += self.process_lambda(p) + elif type(p) == Binding: + script += self.process_binding(p) elif p is None: script += 'null' + elif isinstance(p, type): + script += p.__name__ else: script += str(p) c += 1 @@ -165,11 +207,11 @@ class Translator: # anonymous traversal style syntax. def translate(self, bytecode, child=False): script = '__' if child else self.traversal_source - + for step in bytecode.source_instructions: script += self.do_translation(step) for step in bytecode.step_instructions: script += self.do_translation(step) - + return script diff --git a/gremlin-python/src/main/python/tests/conftest.py b/gremlin-python/src/main/python/tests/conftest.py index a3b76a4362..c80d107d26 100644 --- a/gremlin-python/src/main/python/tests/conftest.py +++ b/gremlin-python/src/main/python/tests/conftest.py @@ -34,7 +34,7 @@ from gremlin_python.driver.protocol import GremlinServerWSProtocol from gremlin_python.driver.serializer import ( GraphSONMessageSerializer, GraphSONSerializersV2d0, GraphSONSerializersV3d0, GraphBinarySerializersV1) -from gremlin_python.driver.aiohttp.transport import AiohttpTransport +from gremlin_python.driver.aiohttp.transport import AiohttpTransport, AiohttpHTTPTransport gremlin_server_url = os.environ.get('GREMLIN_SERVER_URL', 'ws://localhost:{}/gremlin') gremlin_basic_auth_url = os.environ.get('GREMLIN_SERVER_BASIC_AUTH_URL', 'wss://localhost:{}/gremlin') @@ -43,6 +43,10 @@ anonymous_url = gremlin_server_url.format(45940) basic_url = gremlin_basic_auth_url.format(45941) kerberos_url = gremlin_server_url.format(45942) kerberized_service = 'test-service@{}'.format(kerberos_hostname) +gremlin_server_url_http = os.environ.get('GREMLIN_SERVER_URL_HTTP', 'http://localhost:{}/') +gremlin_basic_auth_url_http = os.environ.get('GREMLIN_SERVER_BASIC_AUTH_URL_HTTP', 'https://localhost:{}/') +anonymous_url_http = gremlin_server_url_http.format(45940) +basic_url_http = gremlin_basic_auth_url_http.format(45941) verbose_logging = False logging.basicConfig(format='%(asctime)s [%(levelname)8s] [%(filename)15s:%(lineno)d - %(funcName)10s()] - %(message)s', @@ -209,3 +213,55 @@ def graphson_serializer_v3(request): @pytest.fixture def graphbinary_serializer_v1(request): return GraphBinarySerializersV1() + + [email protected](params=['graphsonv2', 'graphsonv3', 'graphbinaryv1']) +def remote_connection_http(request): + try: + if request.param == 'graphbinaryv1': + remote_conn = DriverRemoteConnection(anonymous_url_http, 'gmodern', + message_serializer=serializer.GraphBinarySerializersV1()) + elif request.param == 'graphsonv2': + remote_conn = DriverRemoteConnection(anonymous_url_http, 'gmodern', + message_serializer=serializer.GraphSONSerializersV2d0()) + elif request.param == 'graphsonv3': + remote_conn = DriverRemoteConnection(anonymous_url_http, 'gmodern', + message_serializer=serializer.GraphSONSerializersV3d0()) + else: + raise ValueError("Invalid serializer option - " + request.param) + except OSError: + pytest.skip('Gremlin Server is not running') + else: + def fin(): + remote_conn.close() + + request.addfinalizer(fin) + return remote_conn + + +""" +# The WsAndHttpChannelizer somehow does not distinguish the ssl handlers so authenticated https remote connection will +# only work with HttpChannelizer that is currently not in the testing set up, thus this is commented out for now + [email protected](params=['basic']) +def remote_connection_http_authenticated(request): + try: + if request.param == 'basic': + # turn off certificate verification for testing purposes only + ssl_opts = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + ssl_opts.verify_mode = ssl.CERT_NONE + remote_conn = DriverRemoteConnection(basic_url_http, 'gmodern', + username='stephen', password='password', + message_serializer=serializer.GraphSONSerializersV2d0(), + transport_factory=lambda: AiohttpHTTPTransport(ssl_options=ssl_opts)) + else: + raise ValueError("Invalid authentication option - " + request.param) + except OSError: + pytest.skip('Gremlin Server is not running') + else: + def fin(): + remote_conn.close() + + request.addfinalizer(fin) + return remote_conn +""" diff --git a/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_http.py b/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_http.py new file mode 100644 index 0000000000..4d1fd82e17 --- /dev/null +++ b/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_http.py @@ -0,0 +1,214 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import os + +from gremlin_python import statics +from gremlin_python.driver.protocol import GremlinServerError +from gremlin_python.statics import long +from gremlin_python.process.traversal import Traverser +from gremlin_python.process.traversal import TraversalStrategy +from gremlin_python.process.traversal import Bindings +from gremlin_python.process.traversal import P, Order, T +from gremlin_python.process.graph_traversal import __ +from gremlin_python.process.anonymous_traversal import traversal +from gremlin_python.structure.graph import Vertex +from gremlin_python.process.strategies import SubgraphStrategy, ReservedKeysVerificationStrategy, SeedStrategy +from gremlin_python.structure.io.util import HashableDict +from gremlin_python.driver.serializer import GraphSONSerializersV2d0 + +__author__ = 'Marko A. Rodriguez (http://markorodriguez.com)' + +gremlin_server_url_http = os.environ.get('GREMLIN_SERVER_URL_HTTP', 'http://localhost:{}/') +test_no_auth_http_url = gremlin_server_url_http.format(45940) + +# due to the limitation of relying on python translator for sending scripts over HTTP, certain tests are omitted +class TestDriverRemoteConnectionHttp(object): + def test_traversals(self, remote_connection_http): + statics.load_statics(globals()) + g = traversal().withRemote(remote_connection_http) + + assert long(6) == g.V().count().toList()[0] + # # + assert Vertex(1) == g.V(1).next() + assert Vertex(1) == g.V(Vertex(1)).next() + assert 1 == g.V(1).id_().next() + assert Traverser(Vertex(1)) == g.V(1).nextTraverser() + assert 1 == len(g.V(1).toList()) + assert isinstance(g.V(1).toList(), list) + results = g.V().repeat(__.out()).times(2).name + results = results.toList() + assert 2 == len(results) + assert "lop" in results + assert "ripple" in results + # # + assert 10 == g.V().repeat(__.both()).times(5)[0:10].count().next() + assert 1 == g.V().repeat(__.both()).times(5)[0:1].count().next() + assert 0 == g.V().repeat(__.both()).times(5)[0:0].count().next() + assert 4 == g.V()[2:].count().next() + assert 2 == g.V()[:2].count().next() + # # + results = g.withSideEffect('a', ['josh', 'peter']).V(1).out('created').in_('created').values('name').where( + P.within('a')).toList() + assert 2 == len(results) + assert 'josh' in results + assert 'peter' in results + # # + results = g.V().out().profile().toList() + assert 1 == len(results) + assert 'metrics' in results[0] + assert 'dur' in results[0] + # # + results = g.V().has('name', 'peter').as_('a').out('created').as_('b').select('a', 'b').by( + __.valueMap()).toList() + assert 1 == len(results) + assert 'peter' == results[0]['a']['name'][0] + assert 35 == results[0]['a']['age'][0] + assert 'lop' == results[0]['b']['name'][0] + assert 'java' == results[0]['b']['lang'][0] + assert 2 == len(results[0]['a']) + assert 2 == len(results[0]['b']) + # # + results = g.V(1).inject(g.V(2).next()).values('name').toList() + assert 2 == len(results) + assert 'marko' in results + assert 'vadas' in results + # # + results = g.V().has('person', 'name', 'marko').map( + lambda: ("it.get().value('name')", "gremlin-groovy")).toList() + assert 1 == len(results) + assert 'marko' in results + # # + # this test just validates that the underscored versions of steps conflicting with Gremlin work + # properly and can be removed when the old steps are removed - TINKERPOP-2272 + results = g.V().filter_(__.values('age').sum_().and_( + __.max_().is_(P.gt(0)), __.min_().is_(P.gt(0)))).range_(0, 1).id_().next() + assert 1 == results + # # + # test binding in P + results = g.V().has('person', 'age', Bindings.of('x', P.lt(30))).count().next() + assert 2 == results + # # + # test dict keys which can only work on GraphBinary and GraphSON3 which include specific serialization + # types for dict + if not isinstance(remote_connection_http._client._message_serializer, GraphSONSerializersV2d0): + results = g.V().has('person', 'name', 'marko').elementMap("name").groupCount().next() + assert {HashableDict.of({T.id: 1, T.label: 'person', 'name': 'marko'}): 1} == results + if not isinstance(remote_connection_http._client._message_serializer, GraphSONSerializersV2d0): + results = g.V().has('person', 'name', 'marko').both('knows').groupCount().by( + __.values('name').fold()).next() + assert {tuple(['vadas']): 1, tuple(['josh']): 1} == results + + def test_iteration(self, remote_connection_http): + statics.load_statics(globals()) + g = traversal().withRemote(remote_connection_http) + + t = g.V().count() + assert t.hasNext() + assert t.hasNext() + assert t.hasNext() + assert t.hasNext() + assert t.hasNext() + assert 6 == t.next() + assert not (t.hasNext()) + assert not (t.hasNext()) + assert not (t.hasNext()) + assert not (t.hasNext()) + assert not (t.hasNext()) + + t = g.V().has('name', P.within('marko', 'peter')).values('name').order() + assert "marko" == t.next() + assert t.hasNext() + assert t.hasNext() + assert t.hasNext() + assert t.hasNext() + assert t.hasNext() + assert "peter" == t.next() + assert not (t.hasNext()) + assert not (t.hasNext()) + assert not (t.hasNext()) + assert not (t.hasNext()) + assert not (t.hasNext()) + + try: + t.next() + assert False + except StopIteration: + assert True + + def test_strategies(self, remote_connection_http): + statics.load_statics(globals()) + g = traversal().withRemote(remote_connection_http). \ + withStrategies(TraversalStrategy("SubgraphStrategy", + {"vertices": __.hasLabel("person"), + "edges": __.hasLabel("created")}, + "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.SubgraphStrategy")) + assert 4 == g.V().count().next() + assert 0 == g.E().count().next() + assert 1 == g.V().label().dedup().count().next() + assert 4 == g.V().filter_(lambda: ("x -> true", "gremlin-groovy")).count().next() + assert "person" == g.V().label().dedup().next() + # + g = traversal().withRemote(remote_connection_http). \ + withStrategies(SubgraphStrategy(vertices=__.hasLabel("person"), edges=__.hasLabel("created"))) + assert 4 == g.V().count().next() + assert 0 == g.E().count().next() + assert 1 == g.V().label().dedup().count().next() + assert "person" == g.V().label().dedup().next() + # + g = traversal().withRemote(remote_connection_http). \ + withStrategies(SubgraphStrategy(edges=__.hasLabel("created"))) + assert 6 == g.V().count().next() + assert 4 == g.E().count().next() + assert 1 == g.E().label().dedup().count().next() + assert "created" == g.E().label().dedup().next() + # + g = g.withoutStrategies(SubgraphStrategy). \ + withComputer(vertices=__.has("name", "marko"), edges=__.limit(0)) + assert 1 == g.V().count().next() + assert 0 == g.E().count().next() + assert "person" == g.V().label().next() + assert "marko" == g.V().name.next() + # + g = traversal().withRemote(remote_connection_http).withComputer() + assert 6 == g.V().count().next() + assert 6 == g.E().count().next() + # + g = traversal().withRemote(remote_connection_http).withStrategies(SeedStrategy(12345)) + shuffledResult = g.V().values("name").order().by(Order.shuffle).toList() + assert shuffledResult == g.V().values("name").order().by(Order.shuffle).toList() + assert shuffledResult == g.V().values("name").order().by(Order.shuffle).toList() + assert shuffledResult == g.V().values("name").order().by(Order.shuffle).toList() + + def test_clone(self, remote_connection_http): + g = traversal().withRemote(remote_connection_http) + t = g.V().both() + assert 12 == len(t.toList()) + assert 5 == t.clone().limit(5).count().next() + assert 10 == t.clone().limit(10).count().next() + + """ + # The WsAndHttpChannelizer somehow does not distinguish the ssl handlers so authenticated https remote connection + # only work with HttpChannelizer that is currently not in the testing set up, thus this is commented out for now + + def test_authenticated(self, remote_connection_http_authenticated): + statics.load_statics(globals()) + g = traversal().withRemote(remote_connection_http_authenticated) + + assert long(6) == g.V().count().toList()[0] + """ diff --git a/gremlin-python/src/main/python/tests/process/test_translator.py b/gremlin-python/src/main/python/tests/process/test_translator.py index 2afa148372..8a121a5437 100644 --- a/gremlin-python/src/main/python/tests/process/test_translator.py +++ b/gremlin-python/src/main/python/tests/process/test_translator.py @@ -21,13 +21,14 @@ Unit tests for the Translator Class. """ __author__ = 'Kelvin R. Lawrence (gfxman)' -from gremlin_python.structure.graph import Graph +from gremlin_python.structure.graph import Graph, Vertex, Edge, VertexProperty +from gremlin_python.process.anonymous_traversal import traversal from gremlin_python.process.graph_traversal import __ from gremlin_python.process.translator import * from datetime import datetime -class TestTraversalStrategies(object): +class TestTranslator(object): def test_translations(self): g = traversal().withGraph(Graph()) @@ -317,7 +318,7 @@ class TestTraversalStrategies(object): "g.V('44').valueMap().with(WithOptions.tokens)"]) # 89 tests.append([g.withStrategies(ReadOnlyStrategy()).addV('test'), - "g.withStrategies(new ReadOnlyStrategy()).addV('test')"]) + "g.withStrategies(ReadOnlyStrategy).addV('test')"]) # 90 strategy = SubgraphStrategy(vertices=__.has('region', 'US-TX'), edges=__.hasLabel('route')) tests.append([g.withStrategies(strategy).V().count(), @@ -333,11 +334,11 @@ class TestTraversalStrategies(object): # 93 strategy = SubgraphStrategy(vertices=__.has('region', 'US-TX'), edges=__.hasLabel('route')) tests.append([g.withStrategies(ReadOnlyStrategy(),strategy).V().count(), - "g.withStrategies(new ReadOnlyStrategy(),new SubgraphStrategy(vertices:__.has('region','US-TX'),edges:__.hasLabel('route'))).V().count()"]) + "g.withStrategies(ReadOnlyStrategy,new SubgraphStrategy(vertices:__.has('region','US-TX'),edges:__.hasLabel('route'))).V().count()"]) # 94 strategy = SubgraphStrategy(vertices=__.has('region', 'US-TX')) tests.append([g.withStrategies(ReadOnlyStrategy(), strategy).V().count(), - "g.withStrategies(new ReadOnlyStrategy(),new SubgraphStrategy(vertices:__.has('region','US-TX'))).V().count()"]) + "g.withStrategies(ReadOnlyStrategy,new SubgraphStrategy(vertices:__.has('region','US-TX'))).V().count()"]) # 95 tests.append([g.with_('evaluationTimeout', 500).V().count(), "g.withStrategies(new OptionsStrategy(evaluationTimeout:500)).V().count()"]) @@ -349,7 +350,7 @@ class TestTraversalStrategies(object): "g.withStrategies(new PartitionStrategy(partitionKey:'partition',writePartition:'a',readPartitions:['a'])).addV('test')"]) # 98 tests.append([g.withComputer().V().shortestPath().with_(ShortestPath.target, __.has('name','peter')), - "g.withStrategies(new VertexProgramStrategy()).V().shortestPath().with('~tinkerpop.shortestPath.target',__.has('name','peter'))"]) + "g.withStrategies(VertexProgramStrategy).V().shortestPath().with('~tinkerpop.shortestPath.target',__.has('name','peter'))"]) # 99 tests.append([g.V().has("p1", starting_with("foo")), @@ -373,6 +374,30 @@ class TestTraversalStrategies(object): tests.append([g.V().has("p1", None), "g.V().has('p1',null)"]) + # 104 + vertex = Vertex(0, "person") + tests.append([g.V(vertex), + "g.V(new ReferenceVertex(0,'person'))"]) + + # 105 + outVertex = Vertex(0, "person") + inVertex = Vertex(1, "person") + edge = Edge(2, outVertex, "knows", inVertex) + tests.append([g.inject(edge), + "g.inject(new ReferenceEdge(2,'knows',new ReferenceVertex(1,'person'),new ReferenceVertex(0,'person')))"]) + + # 106 + vp = VertexProperty(3, "time", "18:00", None) + tests.append([g.inject(vp), + "g.inject(new ReferenceVertexProperty(3,'time','18:00'))"]) + + # 107 + tests.append([g.V().has('person', 'name', 'marko').map(lambda: ("it.get().value('name')", "gremlin-groovy")), + "g.V().has('person','name','marko').map({it.get().value('name')})"]) + + # 108 + tests.append([g.V().has('person', 'age', Bindings.of('x', P.lt(30))).count(), + "g.V().has('person','age',Bindings.instance().of('x', lt(30))).count()"]) tlr = Translator().of('g')
