This is an automated email from the ASF dual-hosted git repository. jin pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-ai.git
The following commit(s) were added to refs/heads/main by this push: new e356178 feat(vermeer): add vermeer python client for graph computing (#263) e356178 is described below commit e3561782397b79a825832ad41b36634b070f2b58 Author: SoJGooo <102796027+mrjs...@users.noreply.github.com> AuthorDate: Tue Jun 10 22:08:52 2025 +0800 feat(vermeer): add vermeer python client for graph computing (#263) develop vermeer python client. Support some graph computation algorithms. --------- Co-authored-by: imbajin <j...@apache.org> --- vermeer-python-client/README.md | 26 +++ vermeer-python-client/requirements.txt | 7 + vermeer-python-client/setup.py | 42 ++++ vermeer-python-client/src/pyvermeer/__init__.py | 16 ++ vermeer-python-client/src/pyvermeer/api/base.py | 40 ++++ vermeer-python-client/src/pyvermeer/api/graph.py | 39 ++++ vermeer-python-client/src/pyvermeer/api/master.py | 16 ++ vermeer-python-client/src/pyvermeer/api/task.py | 49 ++++ vermeer-python-client/src/pyvermeer/api/worker.py | 16 ++ .../src/pyvermeer/client/__init__.py | 16 ++ .../src/pyvermeer/client/client.py | 63 +++++ .../src/pyvermeer/demo/task_demo.py | 53 +++++ .../src/pyvermeer/structure/__init__.py | 16 ++ .../src/pyvermeer/structure/base_data.py | 50 ++++ .../src/pyvermeer/structure/graph_data.py | 255 +++++++++++++++++++++ .../src/pyvermeer/structure/master_data.py | 82 +++++++ .../src/pyvermeer/structure/task_data.py | 226 ++++++++++++++++++ .../src/pyvermeer/structure/worker_data.py | 118 ++++++++++ .../src/pyvermeer/utils/__init__.py | 16 ++ .../src/pyvermeer/utils/exception.py | 44 ++++ vermeer-python-client/src/pyvermeer/utils/log.py | 70 ++++++ .../src/pyvermeer/utils/vermeer_config.py | 37 +++ .../src/pyvermeer/utils/vermeer_datetime.py | 32 +++ .../src/pyvermeer/utils/vermeer_requests.py | 115 ++++++++++ 24 files changed, 1444 insertions(+) diff --git a/vermeer-python-client/README.md b/vermeer-python-client/README.md new file mode 100644 index 0000000..83c0cc2 --- /dev/null +++ b/vermeer-python-client/README.md @@ -0,0 +1,26 @@ +# vermeer-python-client + +The `vermeer-python-client` is a Python client(SDK) for [Vermeer](https://github.com/apache/incubator-hugegraph-computer/tree/master/vermeer#readme) (A high-performance distributed graph computing platform based on memory, supporting more than 15 graph algorithms, custom algorithm extensions, and custom data source access & easy to deploy and use) + + +## Installation + +To install the `vermeer-python-client`, you can use `pip/uv` or **source code building**: + +```bash +#todo +``` + +### Install from Source (Latest Code) + +To install from the source, clone the repository and install the required dependencies: + +```bash +#todo +``` + +## Usage + +```bash +#todo +``` \ No newline at end of file diff --git a/vermeer-python-client/requirements.txt b/vermeer-python-client/requirements.txt new file mode 100644 index 0000000..30b82d7 --- /dev/null +++ b/vermeer-python-client/requirements.txt @@ -0,0 +1,7 @@ +# TODO: replace pip/current file to uv +decorator~=5.1.1 +requests~=2.32.0 +setuptools~=78.1.1 +urllib3~=2.2.2 +rich~=13.9.4 +python-dateutil~=2.9.0 diff --git a/vermeer-python-client/setup.py b/vermeer-python-client/setup.py new file mode 100644 index 0000000..753d897 --- /dev/null +++ b/vermeer-python-client/setup.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import setuptools +from pkg_resources import parse_requirements + +# TODO: replace/delete current file by uv +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +with open("requirements.txt", encoding="utf-8") as fp: + install_requires = [str(requirement) for requirement in parse_requirements(fp)] + +setuptools.setup( + name="vermeer-python", + version="0.1.0", + install_requires=install_requires, + long_description=long_description, + long_description_content_type="text/markdown", + packages=setuptools.find_packages(where="src", exclude=["tests"]), + package_dir={"": "src"}, + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + ], + python_requires=">=3.9", +) diff --git a/vermeer-python-client/src/pyvermeer/__init__.py b/vermeer-python-client/src/pyvermeer/__init__.py new file mode 100644 index 0000000..13a8339 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/vermeer-python-client/src/pyvermeer/api/base.py b/vermeer-python-client/src/pyvermeer/api/base.py new file mode 100644 index 0000000..0ab5fe0 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/api/base.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyvermeer.utils.log import log + + +class BaseModule: + """Base class""" + + def __init__(self, client): + self._client = client + self.log = log.getChild(__name__) + + @property + def session(self): + """Return the client's session object""" + return self._client.session + + def _send_request(self, method: str, endpoint: str, params: dict = None): + """Unified request entry point""" + self.log.debug(f"Sending {method} to {endpoint}") + return self._client.send_request( + method=method, + endpoint=endpoint, + params=params + ) diff --git a/vermeer-python-client/src/pyvermeer/api/graph.py b/vermeer-python-client/src/pyvermeer/api/graph.py new file mode 100644 index 0000000..1fabdca --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/api/graph.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyvermeer.structure.graph_data import GraphsResponse, GraphResponse +from .base import BaseModule + + +class GraphModule(BaseModule): + """Graph""" + + def get_graph(self, graph_name: str) -> GraphResponse: + """Get task list""" + response = self._send_request( + "GET", + f"/graphs/{graph_name}" + ) + return GraphResponse(response) + + def get_graphs(self) -> GraphsResponse: + """Get task list""" + response = self._send_request( + "GET", + "/graphs", + ) + return GraphsResponse(response) diff --git a/vermeer-python-client/src/pyvermeer/api/master.py b/vermeer-python-client/src/pyvermeer/api/master.py new file mode 100644 index 0000000..13a8339 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/api/master.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/vermeer-python-client/src/pyvermeer/api/task.py b/vermeer-python-client/src/pyvermeer/api/task.py new file mode 100644 index 0000000..12e1186 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/api/task.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyvermeer.api.base import BaseModule + +from pyvermeer.structure.task_data import TasksResponse, TaskCreateRequest, TaskCreateResponse, TaskResponse + + +class TaskModule(BaseModule): + """Task""" + + def get_tasks(self) -> TasksResponse: + """Get task list""" + response = self._send_request( + "GET", + "/tasks" + ) + return TasksResponse(response) + + def get_task(self, task_id: int) -> TaskResponse: + """Get single task information""" + response = self._send_request( + "GET", + f"/task/{task_id}" + ) + return TaskResponse(response) + + def create_task(self, create_task: TaskCreateRequest) -> TaskCreateResponse: + """Create new task""" + response = self._send_request( + method="POST", + endpoint="/tasks/create", + params=create_task.to_dict() + ) + return TaskCreateResponse(response) diff --git a/vermeer-python-client/src/pyvermeer/api/worker.py b/vermeer-python-client/src/pyvermeer/api/worker.py new file mode 100644 index 0000000..13a8339 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/api/worker.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/vermeer-python-client/src/pyvermeer/client/__init__.py b/vermeer-python-client/src/pyvermeer/client/__init__.py new file mode 100644 index 0000000..13a8339 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/client/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/vermeer-python-client/src/pyvermeer/client/client.py b/vermeer-python-client/src/pyvermeer/client/client.py new file mode 100644 index 0000000..ba6a094 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/client/client.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Dict +from typing import Optional + +from pyvermeer.api.base import BaseModule +from pyvermeer.api.graph import GraphModule +from pyvermeer.api.task import TaskModule +from pyvermeer.utils.log import log +from pyvermeer.utils.vermeer_config import VermeerConfig +from pyvermeer.utils.vermeer_requests import VermeerSession + + +class PyVermeerClient: + """Vermeer API Client""" + + def __init__( + self, + ip: str, + port: int, + token: str, + timeout: Optional[tuple[float, float]] = None, + log_level: str = "INFO", + ): + """Initialize the client, including configuration and session management + :param ip: + :param port: + :param token: + :param timeout: + :param log_level: + """ + self.cfg = VermeerConfig(ip, port, token, timeout) + self.session = VermeerSession(self.cfg) + self._modules: Dict[str, BaseModule] = { + "graph": GraphModule(self), + "tasks": TaskModule(self) + } + log.setLevel(log_level) + + def __getattr__(self, name): + """Access modules through attributes""" + if name in self._modules: + return self._modules[name] + raise AttributeError(f"Module {name} not found") + + def send_request(self, method: str, endpoint: str, params: dict = None): + """Unified request method""" + return self.session.request(method, endpoint, params) diff --git a/vermeer-python-client/src/pyvermeer/demo/task_demo.py b/vermeer-python-client/src/pyvermeer/demo/task_demo.py new file mode 100644 index 0000000..bb0a00d --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/demo/task_demo.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyvermeer.client.client import PyVermeerClient +from pyvermeer.structure.task_data import TaskCreateRequest + + +def main(): + """main""" + client = PyVermeerClient( + ip="127.0.0.1", + port=8688, + token="", + log_level="DEBUG", + ) + task = client.tasks.get_tasks() + + print(task.to_dict()) + + create_response = client.tasks.create_task( + create_task=TaskCreateRequest( + task_type='load', + graph_name='DEFAULT-example', + params={ + "load.hg_pd_peers": "[\"127.0.0.1:8686\"]", + "load.hugegraph_name": "DEFAULT/example/g", + "load.hugegraph_password": "xxx", + "load.hugegraph_username": "xxx", + "load.parallel": "10", + "load.type": "hugegraph" + }, + ) + ) + + print(create_response.to_dict()) + + +if __name__ == "__main__": + main() diff --git a/vermeer-python-client/src/pyvermeer/structure/__init__.py b/vermeer-python-client/src/pyvermeer/structure/__init__.py new file mode 100644 index 0000000..13a8339 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/structure/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/vermeer-python-client/src/pyvermeer/structure/base_data.py b/vermeer-python-client/src/pyvermeer/structure/base_data.py new file mode 100644 index 0000000..4d60780 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/structure/base_data.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +RESPONSE_ERR = 1 +RESPONSE_OK = 0 +RESPONSE_NONE = -1 + + +class BaseResponse(object): + """ + Base response class + """ + + def __init__(self, dic: dict): + """ + init + :param dic: + """ + self.__errcode = dic.get('errcode', RESPONSE_NONE) + self.__message = dic.get('message', "") + + @property + def errcode(self) -> int: + """ + get error code + :return: + """ + return self.__errcode + + @property + def message(self) -> str: + """ + get message + :return: + """ + return self.__message diff --git a/vermeer-python-client/src/pyvermeer/structure/graph_data.py b/vermeer-python-client/src/pyvermeer/structure/graph_data.py new file mode 100644 index 0000000..8f97ed1 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/structure/graph_data.py @@ -0,0 +1,255 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import datetime + +from pyvermeer.structure.base_data import BaseResponse +from pyvermeer.utils.vermeer_datetime import parse_vermeer_time + + +class BackendOpt: + """BackendOpt class""" + + def __init__(self, dic: dict): + """init""" + self.__vertex_data_backend = dic.get('vertex_data_backend', None) + + @property + def vertex_data_backend(self): + """vertex data backend""" + return self.__vertex_data_backend + + def to_dict(self): + """to dict""" + return { + 'vertex_data_backend': self.vertex_data_backend + } + + +class GraphWorker: + """GraphWorker""" + + def __init__(self, dic: dict): + """init""" + self.__name = dic.get('Name', '') + self.__vertex_count = dic.get('VertexCount', -1) + self.__vert_id_start = dic.get('VertIdStart', -1) + self.__edge_count = dic.get('EdgeCount', -1) + self.__is_self = dic.get('IsSelf', False) + self.__scatter_offset = dic.get('ScatterOffset', -1) + + @property + def name(self) -> str: + """graph worker name""" + return self.__name + + @property + def vertex_count(self) -> int: + """vertex count""" + return self.__vertex_count + + @property + def vert_id_start(self) -> int: + """vertex id start""" + return self.__vert_id_start + + @property + def edge_count(self) -> int: + """edge count""" + return self.__edge_count + + @property + def is_self(self) -> bool: + """is self worker. Nonsense """ + return self.__is_self + + @property + def scatter_offset(self) -> int: + """scatter offset""" + return self.__scatter_offset + + def to_dict(self): + """to dict""" + return { + 'name': self.name, + 'vertex_count': self.vertex_count, + 'vert_id_start': self.vert_id_start, + 'edge_count': self.edge_count, + 'is_self': self.is_self, + 'scatter_offset': self.scatter_offset + } + + +class VermeerGraph: + """VermeerGraph""" + + def __init__(self, dic: dict): + """init""" + self.__name = dic.get('name', '') + self.__space_name = dic.get('space_name', '') + self.__status = dic.get('status', '') + self.__create_time = parse_vermeer_time(dic.get('create_time', '')) + self.__update_time = parse_vermeer_time(dic.get('update_time', '')) + self.__vertex_count = dic.get('vertex_count', 0) + self.__edge_count = dic.get('edge_count', 0) + self.__workers = [GraphWorker(w) for w in dic.get('workers', [])] + self.__worker_group = dic.get('worker_group', '') + self.__use_out_edges = dic.get('use_out_edges', False) + self.__use_property = dic.get('use_property', False) + self.__use_out_degree = dic.get('use_out_degree', False) + self.__use_undirected = dic.get('use_undirected', False) + self.__on_disk = dic.get('on_disk', False) + self.__backend_option = BackendOpt(dic.get('backend_option', {})) + + @property + def name(self) -> str: + """graph name""" + return self.__name + + @property + def space_name(self) -> str: + """space name""" + return self.__space_name + + @property + def status(self) -> str: + """graph status""" + return self.__status + + @property + def create_time(self) -> datetime: + """create time""" + return self.__create_time + + @property + def update_time(self) -> datetime: + """update time""" + return self.__update_time + + @property + def vertex_count(self) -> int: + """vertex count""" + return self.__vertex_count + + @property + def edge_count(self) -> int: + """edge count""" + return self.__edge_count + + @property + def workers(self) -> list[GraphWorker]: + """graph workers""" + return self.__workers + + @property + def worker_group(self) -> str: + """worker group""" + return self.__worker_group + + @property + def use_out_edges(self) -> bool: + """whether graph has out edges""" + return self.__use_out_edges + + @property + def use_property(self) -> bool: + """whether graph has property""" + return self.__use_property + + @property + def use_out_degree(self) -> bool: + """whether graph has out degree""" + return self.__use_out_degree + + @property + def use_undirected(self) -> bool: + """whether graph is undirected""" + return self.__use_undirected + + @property + def on_disk(self) -> bool: + """whether graph is on disk""" + return self.__on_disk + + @property + def backend_option(self) -> BackendOpt: + """backend option""" + return self.__backend_option + + def to_dict(self) -> dict: + """to dict""" + return { + 'name': self.__name, + 'space_name': self.__space_name, + 'status': self.__status, + 'create_time': self.__create_time.strftime("%Y-%m-%d %H:%M:%S") if self.__create_time else '', + 'update_time': self.__update_time.strftime("%Y-%m-%d %H:%M:%S") if self.__update_time else '', + 'vertex_count': self.__vertex_count, + 'edge_count': self.__edge_count, + 'workers': [w.to_dict() for w in self.__workers], + 'worker_group': self.__worker_group, + 'use_out_edges': self.__use_out_edges, + 'use_property': self.__use_property, + 'use_out_degree': self.__use_out_degree, + 'use_undirected': self.__use_undirected, + 'on_disk': self.__on_disk, + 'backend_option': self.__backend_option.to_dict(), + } + + +class GraphsResponse(BaseResponse): + """GraphsResponse""" + + def __init__(self, dic: dict): + """init""" + super().__init__(dic) + self.__graphs = [VermeerGraph(g) for g in dic.get('graphs', [])] + + @property + def graphs(self) -> list[VermeerGraph]: + """graphs""" + return self.__graphs + + def to_dict(self) -> dict: + """to dict""" + return { + 'errcode': self.errcode, + 'message': self.message, + 'graphs': [g.to_dict() for g in self.graphs] + } + + +class GraphResponse(BaseResponse): + """GraphResponse""" + + def __init__(self, dic: dict): + """init""" + super().__init__(dic) + self.__graph = VermeerGraph(dic.get('graph', {})) + + @property + def graph(self) -> VermeerGraph: + """graph""" + return self.__graph + + def to_dict(self) -> dict: + """to dict""" + return { + 'errcode': self.errcode, + 'message': self.message, + 'graph': self.graph.to_dict() + } diff --git a/vermeer-python-client/src/pyvermeer/structure/master_data.py b/vermeer-python-client/src/pyvermeer/structure/master_data.py new file mode 100644 index 0000000..de2fac7 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/structure/master_data.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import datetime + +from pyvermeer.structure.base_data import BaseResponse +from pyvermeer.utils.vermeer_datetime import parse_vermeer_time + + +class MasterInfo: + """Master information""" + + def __init__(self, dic: dict): + """Initialization function""" + self.__grpc_peer = dic.get('grpc_peer', '') + self.__ip_addr = dic.get('ip_addr', '') + self.__debug_mod = dic.get('debug_mod', False) + self.__version = dic.get('version', '') + self.__launch_time = parse_vermeer_time(dic.get('launch_time', '')) + + @property + def grpc_peer(self) -> str: + """gRPC address""" + return self.__grpc_peer + + @property + def ip_addr(self) -> str: + """IP address""" + return self.__ip_addr + + @property + def debug_mod(self) -> bool: + """Whether it is debug mode""" + return self.__debug_mod + + @property + def version(self) -> str: + """Master version number""" + return self.__version + + @property + def launch_time(self) -> datetime: + """Master startup time""" + return self.__launch_time + + def to_dict(self): + """Return data in dictionary format""" + return { + "grpc_peer": self.__grpc_peer, + "ip_addr": self.__ip_addr, + "debug_mod": self.__debug_mod, + "version": self.__version, + "launch_time": self.__launch_time.strftime("%Y-%m-%d %H:%M:%S") if self.__launch_time else '' + } + + +class MasterResponse(BaseResponse): + """Master response""" + + def __init__(self, dic: dict): + """Initialization function""" + super().__init__(dic) + self.__master_info = MasterInfo(dic['master_info']) + + @property + def master_info(self) -> MasterInfo: + """Get master node information""" + return self.__master_info diff --git a/vermeer-python-client/src/pyvermeer/structure/task_data.py b/vermeer-python-client/src/pyvermeer/structure/task_data.py new file mode 100644 index 0000000..6fa408a --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/structure/task_data.py @@ -0,0 +1,226 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import datetime + +from pyvermeer.structure.base_data import BaseResponse +from pyvermeer.utils.vermeer_datetime import parse_vermeer_time + + +class TaskWorker: + """task worker info""" + + def __init__(self, dic): + """init""" + self.__name = dic.get('name', None) + self.__status = dic.get('status', None) + + @property + def name(self) -> str: + """worker name""" + return self.__name + + @property + def status(self) -> str: + """worker status""" + return self.__status + + def to_dict(self): + """to dict""" + return {'name': self.name, 'status': self.status} + + +class TaskInfo: + """task info""" + + def __init__(self, dic): + """init""" + self.__id = dic.get('id', 0) + self.__status = dic.get('status', '') + self.__state = dic.get('state', '') + self.__create_user = dic.get('create_user', '') + self.__create_type = dic.get('create_type', '') + self.__create_time = parse_vermeer_time(dic.get('create_time', '')) + self.__start_time = parse_vermeer_time(dic.get('start_time', '')) + self.__update_time = parse_vermeer_time(dic.get('update_time', '')) + self.__graph_name = dic.get('graph_name', '') + self.__space_name = dic.get('space_name', '') + self.__type = dic.get('type', '') + self.__params = dic.get('params', {}) + self.__workers = [TaskWorker(w) for w in dic.get('workers', [])] + + @property + def id(self) -> int: + """task id""" + return self.__id + + @property + def state(self) -> str: + """task state""" + return self.__state + + @property + def create_user(self) -> str: + """task creator""" + return self.__create_user + + @property + def create_type(self) -> str: + """task create type""" + return self.__create_type + + @property + def create_time(self) -> datetime: + """task create time""" + return self.__create_time + + @property + def start_time(self) -> datetime: + """task start time""" + return self.__start_time + + @property + def update_time(self) -> datetime: + """task update time""" + return self.__update_time + + @property + def graph_name(self) -> str: + """task graph""" + return self.__graph_name + + @property + def space_name(self) -> str: + """task space""" + return self.__space_name + + @property + def type(self) -> str: + """task type""" + return self.__type + + @property + def params(self) -> dict: + """task params""" + return self.__params + + @property + def workers(self) -> list[TaskWorker]: + """task workers""" + return self.__workers + + def to_dict(self) -> dict: + """to dict""" + return { + 'id': self.__id, + 'status': self.__status, + 'state': self.__state, + 'create_user': self.__create_user, + 'create_type': self.__create_type, + 'create_time': self.__create_time.strftime("%Y-%m-%d %H:%M:%S") if self.__start_time else '', + 'start_time': self.__start_time.strftime("%Y-%m-%d %H:%M:%S") if self.__start_time else '', + 'update_time': self.__update_time.strftime("%Y-%m-%d %H:%M:%S") if self.__update_time else '', + 'graph_name': self.__graph_name, + 'space_name': self.__space_name, + 'type': self.__type, + 'params': self.__params, + 'workers': [w.to_dict() for w in self.__workers], + } + + +class TaskCreateRequest: + """task create request""" + + def __init__(self, task_type, graph_name, params): + """init""" + self.task_type = task_type + self.graph_name = graph_name + self.params = params + + def to_dict(self) -> dict: + """to dict""" + return { + 'task_type': self.task_type, + 'graph': self.graph_name, + 'params': self.params + } + + +class TaskCreateResponse(BaseResponse): + """task create response""" + + def __init__(self, dic): + """init""" + super().__init__(dic) + self.__task = TaskInfo(dic.get('task', {})) + + @property + def task(self) -> TaskInfo: + """task info""" + return self.__task + + def to_dict(self) -> dict: + """to dict""" + return { + "errcode": self.errcode, + "message": self.message, + "task": self.task.to_dict(), + } + + +class TasksResponse(BaseResponse): + """tasks response""" + + def __init__(self, dic): + """init""" + super().__init__(dic) + self.__tasks = [TaskInfo(t) for t in dic.get('tasks', [])] + + @property + def tasks(self) -> list[TaskInfo]: + """task infos""" + return self.__tasks + + def to_dict(self) -> dict: + """to dict""" + return { + "errcode": self.errcode, + "message": self.message, + "tasks": [t.to_dict() for t in self.tasks] + } + + +class TaskResponse(BaseResponse): + """task response""" + + def __init__(self, dic): + """init""" + super().__init__(dic) + self.__task = TaskInfo(dic.get('task', {})) + + @property + def task(self) -> TaskInfo: + """task info""" + return self.__task + + def to_dict(self) -> dict: + """to dict""" + return { + "errcode": self.errcode, + "message": self.message, + "task": self.task.to_dict(), + } diff --git a/vermeer-python-client/src/pyvermeer/structure/worker_data.py b/vermeer-python-client/src/pyvermeer/structure/worker_data.py new file mode 100644 index 0000000..a35cfe9 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/structure/worker_data.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import datetime + +from pyvermeer.structure.base_data import BaseResponse +from pyvermeer.utils.vermeer_datetime import parse_vermeer_time + + +class Worker: + """worker data""" + + def __init__(self, dic): + """init""" + self.__id = dic.get('id', 0) + self.__name = dic.get('name', '') + self.__grpc_addr = dic.get('grpc_addr', '') + self.__ip_addr = dic.get('ip_addr', '') + self.__state = dic.get('state', '') + self.__version = dic.get('version', '') + self.__group = dic.get('group', '') + self.__init_time = parse_vermeer_time(dic.get('init_time', '')) + self.__launch_time = parse_vermeer_time(dic.get('launch_time', '')) + + @property + def id(self) -> int: + """worker id""" + return self.__id + + @property + def name(self) -> str: + """worker name""" + return self.__name + + @property + def grpc_addr(self) -> str: + """gRPC address""" + return self.__grpc_addr + + @property + def ip_addr(self) -> str: + """IP address""" + return self.__ip_addr + + @property + def state(self) -> int: + """worker status""" + return self.__state + + @property + def version(self) -> str: + """worker version""" + return self.__version + + @property + def group(self) -> str: + """worker group""" + return self.__group + + @property + def init_time(self) -> datetime: + """worker initialization time""" + return self.__init_time + + @property + def launch_time(self) -> datetime: + """worker launch time""" + return self.__launch_time + + def to_dict(self): + """convert object to dictionary""" + return { + "id": self.id, + "name": self.name, + "grpc_addr": self.grpc_addr, + "ip_addr": self.ip_addr, + "state": self.state, + "version": self.version, + "group": self.group, + "init_time": self.init_time, + "launch_time": self.launch_time, + } + + +class WorkersResponse(BaseResponse): + """response of workers""" + + def __init__(self, dic): + """init""" + super().__init__(dic) + self.__workers = [Worker(worker) for worker in dic['workers']] + + @property + def workers(self) -> list[Worker]: + """list of workers""" + return self.__workers + + def to_dict(self): + """convert object to dictionary""" + return { + "errcode": self.errcode, + "message": self.message, + "workers": [worker.to_dict() for worker in self.workers], + } diff --git a/vermeer-python-client/src/pyvermeer/utils/__init__.py b/vermeer-python-client/src/pyvermeer/utils/__init__.py new file mode 100644 index 0000000..e0533d9 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/utils/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/vermeer-python-client/src/pyvermeer/utils/exception.py b/vermeer-python-client/src/pyvermeer/utils/exception.py new file mode 100644 index 0000000..ddb36d8 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/utils/exception.py @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class ConnectError(Exception): + """Raised when there is an issue connecting to the server.""" + + def __init__(self, message): + super().__init__(f"Connection error: {str(message)}") + + +class TimeOutError(Exception): + """Raised when a request times out.""" + + def __init__(self, message): + super().__init__(f"Request timed out: {str(message)}") + + +class JsonDecodeError(Exception): + """Raised when the response from the server cannot be decoded as JSON.""" + + def __init__(self, message): + super().__init__(f"Failed to decode JSON response: {str(message)}") + + +class UnknownError(Exception): + """Raised for any other unknown errors.""" + + def __init__(self, message): + super().__init__(f"Unknown API error: {str(message)}") diff --git a/vermeer-python-client/src/pyvermeer/utils/log.py b/vermeer-python-client/src/pyvermeer/utils/log.py new file mode 100644 index 0000000..cc5199f --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/utils/log.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import logging +import sys + + +class VermeerLogger: + """vermeer API log""" + _instance = None + + def __new__(cls, name: str = "VermeerClient"): + """new api logger""" + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._initialize(name) + return cls._instance + + def _initialize(self, name: str): + """Initialize log configuration""" + self.logger = logging.getLogger(name) + self.logger.setLevel(logging.INFO) # Default level + + if not self.logger.handlers: + # Console output format + console_format = logging.Formatter( + '[%(asctime)s] [%(levelname)s] %(name)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + # Console handler + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(logging.INFO) # Console default level + console_handler.setFormatter(console_format) + + # file_handler = logging.FileHandler('api_client.log') + # file_handler.setLevel(logging.DEBUG) + # file_handler.setFormatter( + # logging.Formatter( + # '[%(asctime)s] [%(levelname)s] [%(threadName)s] %(name)s - %(message)s' + # ) + # ) + + self.logger.addHandler(console_handler) + # self.logger.addHandler(file_handler) + + self.logger.propagate = False + + @classmethod + def get_logger(cls) -> logging.Logger: + """Get configured logger""" + return cls().logger + + +# Global log instance +log = VermeerLogger.get_logger() diff --git a/vermeer-python-client/src/pyvermeer/utils/vermeer_config.py b/vermeer-python-client/src/pyvermeer/utils/vermeer_config.py new file mode 100644 index 0000000..dfd4d50 --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/utils/vermeer_config.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class VermeerConfig: + """The configuration of a Vermeer instance.""" + ip: str + port: int + token: str + factor: str + username: str + graph_space: str + + def __init__(self, + ip: str, + port: int, + token: str, + timeout: tuple[float, float] = (0.5, 15.0)): + """Initialize the configuration for a Vermeer instance.""" + self.ip = ip + self.port = port + self.token = token + self.timeout = timeout diff --git a/vermeer-python-client/src/pyvermeer/utils/vermeer_datetime.py b/vermeer-python-client/src/pyvermeer/utils/vermeer_datetime.py new file mode 100644 index 0000000..a76dfee --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/utils/vermeer_datetime.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import datetime + +from dateutil import parser + + +def parse_vermeer_time(vm_dt: str) -> datetime: + """Parse a vermeer time string into a Python datetime object.""" + if vm_dt is None or len(vm_dt) == 0: + return None + dt = parser.parse(vm_dt) + return dt + + +if __name__ == '__main__': + print(parse_vermeer_time('2025-02-17T15:45:05.396311145+08:00').strftime("%Y%m%d")) diff --git a/vermeer-python-client/src/pyvermeer/utils/vermeer_requests.py b/vermeer-python-client/src/pyvermeer/utils/vermeer_requests.py new file mode 100644 index 0000000..118484c --- /dev/null +++ b/vermeer-python-client/src/pyvermeer/utils/vermeer_requests.py @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +from typing import Optional +from urllib.parse import urljoin + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from pyvermeer.utils.exception import JsonDecodeError, ConnectError, TimeOutError, UnknownError +from pyvermeer.utils.log import log +from pyvermeer.utils.vermeer_config import VermeerConfig + + +class VermeerSession: + """vermeer session""" + + def __init__( + self, + cfg: VermeerConfig, + retries: int = 3, + backoff_factor: int = 0.1, + status_forcelist=(500, 502, 504), + session: Optional[requests.Session] = None, + ): + """ + Initialize the Session. + """ + self._cfg = cfg + self._retries = retries + self._backoff_factor = backoff_factor + self._status_forcelist = status_forcelist + if self._cfg.token is not None: + self._auth = self._cfg.token + else: + raise ValueError("Vermeer Token must be provided.") + self._headers = {"Content-Type": "application/json", "Authorization": self._auth} + self._timeout = cfg.timeout + self._session = session if session else requests.Session() + self.__configure_session() + + def __configure_session(self): + """ + Configure the retry strategy and connection adapter for the session. + """ + retry_strategy = Retry( + total=self._retries, + read=self._retries, + connect=self._retries, + backoff_factor=self._backoff_factor, + status_forcelist=self._status_forcelist, + ) + adapter = HTTPAdapter(max_retries=retry_strategy) + self._session.mount("http://", adapter) + self._session.mount("https://", adapter) + self._session.keep_alive = False + log.debug( + "Session configured with retries=%s and backoff_factor=%s", + self._retries, + self._backoff_factor, + ) + + def resolve(self, path: str): + """ + Resolve the path to a full URL. + """ + url = f"http://{self._cfg.ip}:{self._cfg.port}/" + return urljoin(url, path).strip("/") + + def close(self): + """ + closes the session. + """ + self._session.close() + + def request( + self, + method: str, + path: str, + params: dict = None + ) -> dict: + """request""" + try: + log.debug(f"Request made to {path} with params {json.dumps(params)}") + response = self._session.request(method, + self.resolve(path), + headers=self._headers, + data=json.dumps(params), + timeout=self._timeout) + log.debug(f"Response code:{response.status_code}, received: {response.text}") + return response.json() + except requests.ConnectionError as e: + raise ConnectError(e) from e + except requests.Timeout as e: + raise TimeOutError(e) from e + except json.JSONDecodeError as e: + raise JsonDecodeError(e) from e + except Exception as e: + raise UnknownError(e) from e