This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch main
in repository
https://gitbox.apache.org/repos/asf/dolphinscheduler-sdk-python.git
The following commit(s) were added to refs/heads/main by this push:
new 229aa8b [feat] Version check for dolphinscheduler main package (#47)
229aa8b is described below
commit 229aa8b61b4a7d712c79e6d4ee1aef332062d08d
Author: Jay Chung <[email protected]>
AuthorDate: Fri Dec 16 17:38:22 2022 +0800
[feat] Version check for dolphinscheduler main package (#47)
---
setup.py | 5 ++-
src/pydolphinscheduler/constants.py | 7 ++++
src/pydolphinscheduler/java_gateway.py | 7 ++--
src/pydolphinscheduler/utils/versions.py | 46 +++++++++++++++++++++++++
src/pydolphinscheduler/version_ext | 1 +
tests/utils/test_versions.py | 58 ++++++++++++++++++++++++++++++++
6 files changed, 121 insertions(+), 3 deletions(-)
diff --git a/setup.py b/setup.py
index 7ee2053..9f3c756 100644
--- a/setup.py
+++ b/setup.py
@@ -152,7 +152,10 @@ setup(
package_dir={"": "src"},
include_package_data=True,
package_data={
- "pydolphinscheduler": ["default_config.yaml"],
+ "pydolphinscheduler": [
+ "default_config.yaml",
+ "version_ext",
+ ],
},
platforms=["any"],
classifiers=[
diff --git a/src/pydolphinscheduler/constants.py
b/src/pydolphinscheduler/constants.py
index cdfd441..0519688 100644
--- a/src/pydolphinscheduler/constants.py
+++ b/src/pydolphinscheduler/constants.py
@@ -122,3 +122,10 @@ class Symbol(str):
POINT = "."
COMMA = ","
UNDERLINE = "_"
+
+
+class Version(str):
+ """Constants for version match."""
+
+ DS = "dolphinscheduler"
+ FILE_NAME = "version_ext"
diff --git a/src/pydolphinscheduler/java_gateway.py
b/src/pydolphinscheduler/java_gateway.py
index ea15ae5..46a284e 100644
--- a/src/pydolphinscheduler/java_gateway.py
+++ b/src/pydolphinscheduler/java_gateway.py
@@ -27,8 +27,9 @@ from py4j.java_gateway import GatewayParameters, JavaGateway
from py4j.protocol import Py4JError
from pydolphinscheduler import __version__, configuration
-from pydolphinscheduler.constants import JavaGatewayDefault
+from pydolphinscheduler.constants import JavaGatewayDefault, Version
from pydolphinscheduler.exceptions import PyDSJavaGatewayException
+from pydolphinscheduler.utils.versions import version_match
logger = getLogger(__name__)
@@ -71,7 +72,9 @@ class GatewayEntryPoint:
# 1. Java gateway version is too old: doesn't have method
'getGatewayVersion()'
# 2. Error connecting to Java gateway
gateway_version = self.get_gateway_version()
- if not __version__.endswith("dev") and gateway_version != __version__:
+ if not __version__.endswith("dev") and not version_match(
+ Version.DS, gateway_version
+ ):
warnings.warn(
f"Using unmatched version of pydolphinscheduler (version
{__version__}) "
f"and Java gateway (version {gateway_version}) may cause
errors. "
diff --git a/src/pydolphinscheduler/utils/versions.py
b/src/pydolphinscheduler/utils/versions.py
new file mode 100644
index 0000000..5fc93e5
--- /dev/null
+++ b/src/pydolphinscheduler/utils/versions.py
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Util versions."""
+
+from pathlib import Path
+
+from packaging import requirements
+from packaging.version import InvalidVersion
+from pkg_resources import parse_requirements
+
+from pydolphinscheduler.constants import Version
+
+
+def version_match(name: str, version: str) -> bool:
+ """Check if the version of external system matches current python sdk
version.
+
+ :param name: External system name in file ``Version.FILE_NAME``
+ :param version: External system current version
+ """
+ path = Path(__file__).parent.parent.joinpath(Version.FILE_NAME)
+ with path.open() as match:
+ content = match.read()
+ for reqs in parse_requirements(content):
+ if reqs.name == name:
+ try:
+ return
requirements.Requirement(str(reqs)).specifier.contains(
+ version
+ )
+ except InvalidVersion:
+ return False
+ raise ValueError("%s is not in %s" % (name, Version.FILE_NAME))
diff --git a/src/pydolphinscheduler/version_ext
b/src/pydolphinscheduler/version_ext
new file mode 100644
index 0000000..1983d1a
--- /dev/null
+++ b/src/pydolphinscheduler/version_ext
@@ -0,0 +1 @@
+dolphinscheduler>=3.2.0
\ No newline at end of file
diff --git a/tests/utils/test_versions.py b/tests/utils/test_versions.py
new file mode 100644
index 0000000..b075e5e
--- /dev/null
+++ b/tests/utils/test_versions.py
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Test utils.versions module."""
+
+from unittest import mock
+
+import pytest
+
+from pydolphinscheduler.utils.versions import version_match
+
+
[email protected](
+ "content, name, version, expect",
+ [
+ ("dolphinscheduler>=3.0.0", "dolphinscheduler", "3.0.0", True),
+ ("dolphinscheduler>=3.0.0", "dolphinscheduler", "2.1.9", False),
+ ("dolphinscheduler>=3.0.0", "dolphinscheduler", "3.1.0", True),
+ ("dolphinscheduler~=3.0.0", "dolphinscheduler", "3.0.0", True),
+ ("dolphinscheduler~=3.0.0", "dolphinscheduler", "3.0.9", True),
+ ("dolphinscheduler~=3.0.0", "dolphinscheduler", "3.1.0", False),
+ ("dolphinscheduler>=3.0.0, <3.1.0", "dolphinscheduler", "3.0.9", True),
+ ("dolphinscheduler>=3.0.0, <3.1.0", "dolphinscheduler", "3.1.0",
False),
+ ("dolphinscheduler~=3.0.0, !=3.0.5", "dolphinscheduler", "3.0.9",
True),
+ ("dolphinscheduler~=3.0.0, !=3.0.5", "dolphinscheduler", "3.0.5",
False),
+ # invalid version test
+ ("dolphinscheduler~=3.0.0, !=3.0.5", "dolphinscheduler",
"dev-SNAPSHOT", False),
+ ],
+)
[email protected]("pathlib.Path.open")
+def test_version_match(mock_open, content: str, name: str, version: str,
expect: str):
+ """Test function version_match."""
+ mock_open.return_value.__enter__.return_value.read.return_value = content
+ assert version_match(name, version) == expect
+ assert mock_open.call_count == 1
+
+
+def test_version_match_error():
+ """Test function version_match error when external system name not in file
``Version.FILE_NAME``."""
+ with pytest.raises(
+ ValueError,
+ match=".*?is not in.*",
+ ):
+ version_match("dolphinschedulerError", "1.0.0")
