This is an automated email from the ASF dual-hosted git repository.
ka94 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 695163314 Fix failing python plugin tests (#4802)
695163314 is described below
commit 695163314225ee3d48542caef839359f9341d67a
Author: Camille Teruel <[email protected]>
AuthorDate: Wed Mar 29 03:28:56 2023 +0200
Fix failing python plugin tests (#4802)
* fix: Remove debug leftover
* fix: Spread extra tasks
* fix: Add missing org plugin
* fix: Make start.sh executable
* fix: Do not write SubtaskRun if error
Reraise exception for early stop.
* fix: Fix fake plugin implementation
* test: Re-enable pipeline and blueprint tests
* fix: Resolve refs of DynamicModelInfo JSON schema
If a plugin connection, scope or transformation rule has a field with a
non-primitive type, pydantic will generate a JSON schema with a reference to
that field type.
On go side, the generation of a DynamicTabler from a JSON schema doesn't
support JSON refs.
So we need to resolve those refs on python side (thanks to jsonref library)
and add support for 'object' JSON type, which is converted to a JSONMap field
in the generated struct.
* feat: Add support for PostgreSQL in SQLAlchemy
* feat: lake-builder Docker support libpq for postgres (needed by Python)
and bumped to v0.0.10
* fix: Use postgresql protocol instead of postgres in Python due to
SqlAlchemy's requirement
---------
Co-authored-by: Camille Teruel <[email protected]>
Co-authored-by: Keon Amini <[email protected]>
---
backend/Dockerfile | 2 +-
backend/python/build_tests.sh | 0
backend/python/plugins/azuredevops/poetry.lock | 41 ++++++++++++++++++-
backend/python/pydevlake/poetry.lock | 47 +++++++++++++++++++---
backend/python/pydevlake/pydevlake/context.py | 33 ++++++++-------
backend/python/pydevlake/pydevlake/ipc.py | 1 -
backend/python/pydevlake/pydevlake/message.py | 8 +++-
backend/python/pydevlake/pydevlake/plugin.py | 2 +-
backend/python/pydevlake/pydevlake/subtasks.py | 1 +
backend/python/pydevlake/pyproject.toml | 2 +
backend/python/test/fakeplugin/fakeplugin/main.py | 21 +++++-----
backend/python/test/fakeplugin/poetry.lock | 47 +++++++++++++++++++---
backend/python/test/fakeplugin/start.sh | 0
.../server/services/remote/models/conversion.go | 4 +-
backend/test/integration/remote/helper.go | 12 +++---
.../test/integration/remote/python_plugin_test.go | 5 +--
devops/docker/lake-builder/Dockerfile | 14 ++++---
17 files changed, 184 insertions(+), 56 deletions(-)
diff --git a/backend/Dockerfile b/backend/Dockerfile
index 2e90ad574..417cc4782 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -116,7 +116,7 @@ FROM python:3.11.2-slim-bullseye as base
ENV PYTHONUNBUFFERED=1
RUN apt-get update && \
- apt-get install -y python3-dev python3-pip tar curl libssh2-1 zlib1g
libffi-dev default-libmysqlclient-dev && \
+ apt-get install -y python3-dev python3-pip tar curl libssh2-1 zlib1g
libffi-dev default-libmysqlclient-dev libpq-dev && \
apt-get clean && \
rm -fr /usr/share/doc/* \
/usr/share/info/* \
diff --git a/backend/python/build_tests.sh b/backend/python/build_tests.sh
old mode 100644
new mode 100755
diff --git a/backend/python/plugins/azuredevops/poetry.lock
b/backend/python/plugins/azuredevops/poetry.lock
index c7dc34910..74b8c9635 100644
--- a/backend/python/plugins/azuredevops/poetry.lock
+++ b/backend/python/plugins/azuredevops/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry and should not be changed by
hand.
+# This file is automatically @generated by Poetry 1.4.1 and should not be
changed by hand.
[[package]]
name = "attrs"
@@ -287,6 +287,18 @@ files = [
{file = "iso8601-1.1.0.tar.gz", hash =
"sha256:32811e7b81deee2063ea6d2e94f8819a86d1f3811e49d23623a41fa832bef03f"},
]
+[[package]]
+name = "jsonref"
+version = "1.1.0"
+description = "jsonref is a library for automatic dereferencing of JSON
Reference objects for Python."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "jsonref-1.1.0-py3-none-any.whl", hash =
"sha256:590dc7773df6c21cbf948b5dac07a72a251db28b0238ceecce0a2abfa8ec30a9"},
+ {file = "jsonref-1.1.0.tar.gz", hash =
"sha256:32fe8e1d85af0fdefbebce950af85590b22b60f9e95443176adbde4e1ecea552"},
+]
+
[[package]]
name = "mysqlclient"
version = "2.1.1"
@@ -332,6 +344,29 @@ files = [
dev = ["pre-commit", "tox"]
testing = ["pytest", "pytest-benchmark"]
+[[package]]
+name = "psycopg2"
+version = "2.9.5"
+description = "psycopg2 - Python-PostgreSQL Database Adapter"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+files = [
+ {file = "psycopg2-2.9.5-cp310-cp310-win32.whl", hash =
"sha256:d3ef67e630b0de0779c42912fe2cbae3805ebaba30cda27fea2a3de650a9414f"},
+ {file = "psycopg2-2.9.5-cp310-cp310-win_amd64.whl", hash =
"sha256:4cb9936316d88bfab614666eb9e32995e794ed0f8f6b3b718666c22819c1d7ee"},
+ {file = "psycopg2-2.9.5-cp311-cp311-win32.whl", hash =
"sha256:093e3894d2d3c592ab0945d9eba9d139c139664dcf83a1c440b8a7aa9bb21955"},
+ {file = "psycopg2-2.9.5-cp311-cp311-win_amd64.whl", hash =
"sha256:920bf418000dd17669d2904472efeab2b20546efd0548139618f8fa305d1d7ad"},
+ {file = "psycopg2-2.9.5-cp36-cp36m-win32.whl", hash =
"sha256:b9ac1b0d8ecc49e05e4e182694f418d27f3aedcfca854ebd6c05bb1cffa10d6d"},
+ {file = "psycopg2-2.9.5-cp36-cp36m-win_amd64.whl", hash =
"sha256:fc04dd5189b90d825509caa510f20d1d504761e78b8dfb95a0ede180f71d50e5"},
+ {file = "psycopg2-2.9.5-cp37-cp37m-win32.whl", hash =
"sha256:922cc5f0b98a5f2b1ff481f5551b95cd04580fd6f0c72d9b22e6c0145a4840e0"},
+ {file = "psycopg2-2.9.5-cp37-cp37m-win_amd64.whl", hash =
"sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a"},
+ {file = "psycopg2-2.9.5-cp38-cp38-win32.whl", hash =
"sha256:f5b6320dbc3cf6cfb9f25308286f9f7ab464e65cfb105b64cc9c52831748ced2"},
+ {file = "psycopg2-2.9.5-cp38-cp38-win_amd64.whl", hash =
"sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e"},
+ {file = "psycopg2-2.9.5-cp39-cp39-win32.whl", hash =
"sha256:322fd5fca0b1113677089d4ebd5222c964b1760e361f151cbb2706c4912112c5"},
+ {file = "psycopg2-2.9.5-cp39-cp39-win_amd64.whl", hash =
"sha256:190d51e8c1b25a47484e52a79638a8182451d6f6dff99f26ad9bd81e5359a0fa"},
+ {file = "psycopg2-2.9.5.tar.gz", hash =
"sha256:a5246d2e683a972e2187a8714b5c2cf8156c064629f9a9b1a873c1730d9e245a"},
+]
+
[[package]]
name = "pydantic"
version = "1.10.7"
@@ -409,7 +444,9 @@ develop = true
[package.dependencies]
fire = "^0.4.0"
inflect = "^6.0.2"
+jsonref = "^1.1.0"
mysqlclient = "^2.1.1"
+psycopg2 = "^2.9.5"
pydantic = "^1.10.2"
pydevd-pycharm = "^231.6471.3"
pytest = "^7.2.2"
@@ -530,7 +567,7 @@ files = [
]
[package.dependencies]
-greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and
(platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or
platform_machine == \"x86_64\" or platform_machine == \"amd64\" or
platform_machine == \"AMD64\" or platform_machine == \"win32\" or
platform_machine == \"WIN32\")"}
+greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and
platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine
== \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or
python_version >= \"3\" and platform_machine == \"amd64\" or python_version >=
\"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and
platform_machine == \"win32\" or python_version >= \"3\" and platform_machine
== \"WIN32\""}
[package.extras]
aiomysql = ["aiomysql", "greenlet (!=0.4.17)"]
diff --git a/backend/python/pydevlake/poetry.lock
b/backend/python/pydevlake/poetry.lock
index 518b2cd7c..14c9c6ca3 100644
--- a/backend/python/pydevlake/poetry.lock
+++ b/backend/python/pydevlake/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry and should not be changed by
hand.
+# This file is automatically @generated by Poetry 1.4.1 and should not be
changed by hand.
[[package]]
name = "attrs"
@@ -275,6 +275,18 @@ files = [
{file = "iniconfig-2.0.0.tar.gz", hash =
"sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
]
+[[package]]
+name = "jsonref"
+version = "1.1.0"
+description = "jsonref is a library for automatic dereferencing of JSON
Reference objects for Python."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "jsonref-1.1.0-py3-none-any.whl", hash =
"sha256:590dc7773df6c21cbf948b5dac07a72a251db28b0238ceecce0a2abfa8ec30a9"},
+ {file = "jsonref-1.1.0.tar.gz", hash =
"sha256:32fe8e1d85af0fdefbebce950af85590b22b60f9e95443176adbde4e1ecea552"},
+]
+
[[package]]
name = "mysqlclient"
version = "2.1.1"
@@ -320,6 +332,29 @@ files = [
dev = ["pre-commit", "tox"]
testing = ["pytest", "pytest-benchmark"]
+[[package]]
+name = "psycopg2"
+version = "2.9.5"
+description = "psycopg2 - Python-PostgreSQL Database Adapter"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+files = [
+ {file = "psycopg2-2.9.5-cp310-cp310-win32.whl", hash =
"sha256:d3ef67e630b0de0779c42912fe2cbae3805ebaba30cda27fea2a3de650a9414f"},
+ {file = "psycopg2-2.9.5-cp310-cp310-win_amd64.whl", hash =
"sha256:4cb9936316d88bfab614666eb9e32995e794ed0f8f6b3b718666c22819c1d7ee"},
+ {file = "psycopg2-2.9.5-cp311-cp311-win32.whl", hash =
"sha256:093e3894d2d3c592ab0945d9eba9d139c139664dcf83a1c440b8a7aa9bb21955"},
+ {file = "psycopg2-2.9.5-cp311-cp311-win_amd64.whl", hash =
"sha256:920bf418000dd17669d2904472efeab2b20546efd0548139618f8fa305d1d7ad"},
+ {file = "psycopg2-2.9.5-cp36-cp36m-win32.whl", hash =
"sha256:b9ac1b0d8ecc49e05e4e182694f418d27f3aedcfca854ebd6c05bb1cffa10d6d"},
+ {file = "psycopg2-2.9.5-cp36-cp36m-win_amd64.whl", hash =
"sha256:fc04dd5189b90d825509caa510f20d1d504761e78b8dfb95a0ede180f71d50e5"},
+ {file = "psycopg2-2.9.5-cp37-cp37m-win32.whl", hash =
"sha256:922cc5f0b98a5f2b1ff481f5551b95cd04580fd6f0c72d9b22e6c0145a4840e0"},
+ {file = "psycopg2-2.9.5-cp37-cp37m-win_amd64.whl", hash =
"sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a"},
+ {file = "psycopg2-2.9.5-cp38-cp38-win32.whl", hash =
"sha256:f5b6320dbc3cf6cfb9f25308286f9f7ab464e65cfb105b64cc9c52831748ced2"},
+ {file = "psycopg2-2.9.5-cp38-cp38-win_amd64.whl", hash =
"sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e"},
+ {file = "psycopg2-2.9.5-cp39-cp39-win32.whl", hash =
"sha256:322fd5fca0b1113677089d4ebd5222c964b1760e361f151cbb2706c4912112c5"},
+ {file = "psycopg2-2.9.5-cp39-cp39-win_amd64.whl", hash =
"sha256:190d51e8c1b25a47484e52a79638a8182451d6f6dff99f26ad9bd81e5359a0fa"},
+ {file = "psycopg2-2.9.5.tar.gz", hash =
"sha256:a5246d2e683a972e2187a8714b5c2cf8156c064629f9a9b1a873c1730d9e245a"},
+]
+
[[package]]
name = "pydantic"
version = "1.10.7"
@@ -494,11 +529,11 @@ files = [
]
[package.dependencies]
-greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and
(platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or
platform_machine == \"x86_64\" or platform_machine == \"amd64\" or
platform_machine == \"AMD64\" or platform_machine == \"win32\" or
platform_machine == \"WIN32\")"}
+greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and
platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine
== \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or
python_version >= \"3\" and platform_machine == \"amd64\" or python_version >=
\"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and
platform_machine == \"win32\" or python_version >= \"3\" and platform_machine
== \"WIN32\""}
[package.extras]
aiomysql = ["aiomysql", "greenlet (!=0.4.17)"]
-aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions
(!=3.10.0.1)"]
+aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions
(!=3.10.0.1)"]
asyncio = ["greenlet (!=0.4.17)"]
asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"]
mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"]
@@ -508,14 +543,14 @@ mssql-pyodbc = ["pyodbc"]
mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"]
mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"]
mysql-connector = ["mysql-connector-python"]
-oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"]
+oracle = ["cx-oracle (>=7)", "cx-oracle (>=7,<8)"]
postgresql = ["psycopg2 (>=2.7)"]
postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"]
postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"]
postgresql-psycopg2binary = ["psycopg2-binary"]
postgresql-psycopg2cffi = ["psycopg2cffi"]
pymysql = ["pymysql", "pymysql (<1)"]
-sqlcipher = ["sqlcipher3_binary"]
+sqlcipher = ["sqlcipher3-binary"]
[[package]]
name = "sqlalchemy2-stubs"
@@ -608,4 +643,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[metadata]
lock-version = "2.0"
python-versions = "^3.10"
-content-hash =
"cbf8b5f2a4bafcf413a43bd03dd6849fda956818b66bed0f3dd747968006fc7c"
+content-hash =
"7f9231d791832106dc95d2052cb5a6f11ae1ddb9c4ecfa072e1e74e0e914ea9b"
diff --git a/backend/python/pydevlake/pydevlake/context.py
b/backend/python/pydevlake/pydevlake/context.py
index 91d299afe..42a1de377 100644
--- a/backend/python/pydevlake/pydevlake/context.py
+++ b/backend/python/pydevlake/pydevlake/context.py
@@ -15,6 +15,8 @@
from urllib.parse import urlparse, parse_qsl
+
+from sqlalchemy.engine import Engine
from sqlmodel import SQLModel, create_engine
from pydevlake.model import Connection, TransformationRule
@@ -35,26 +37,29 @@ class Context:
self._engine = None
@property
- def engine(self):
+ def engine(self) -> Engine:
if not self._engine:
- db_url = self.db_url
- if not db_url:
- raise Exception("Missing db_url setting")
-
- # `parseTime` parameter is not understood by MySQL driver
- # so we have to parse query args to remove it
- connect_args = dict(parse_qsl(urlparse(self.db_url).query))
- db_url = self.db_url.split('?')[0]
-
- if 'parseTime' in connect_args:
- del connect_args['parseTime']
+ db_url, args = self.get_engine_db_url()
try:
- self._engine = create_engine(db_url, connect_args=connect_args)
+ self._engine = create_engine(db_url, connect_args=args)
SQLModel.metadata.create_all(self._engine)
except Exception as e:
raise Exception(f"Unable to make a database connection") from e
return self._engine
@property
- def incremental(self):
+ def incremental(self) -> bool:
return self.options.get('incremental') is True
+
+ def get_engine_db_url(self) -> [str, dict[str, any]]:
+ db_url = self.db_url
+ if not db_url:
+ raise Exception("Missing db_url setting")
+ db_url = db_url.replace("postgres://", "postgresql://")
+ db_url = db_url.split('?')[0]
+ # `parseTime` parameter is not understood by MySQL driver,
+ # so we have to parse query args to remove it
+ connect_args = dict(parse_qsl(urlparse(self.db_url).query))
+ if 'parseTime' in connect_args:
+ del connect_args['parseTime']
+ return db_url, connect_args
diff --git a/backend/python/pydevlake/pydevlake/ipc.py
b/backend/python/pydevlake/pydevlake/ipc.py
index 7aea247fd..e9542fe97 100644
--- a/backend/python/pydevlake/pydevlake/ipc.py
+++ b/backend/python/pydevlake/pydevlake/ipc.py
@@ -124,7 +124,6 @@ class PluginCommands:
return Context(db_url, scope, connection, transformation_rule, options)
def _parse(self, data: Union[str, dict, list]) -> Union[dict, list]:
- print(data)
if isinstance(data, (dict, list)):
return data
if isinstance(data, str):
diff --git a/backend/python/pydevlake/pydevlake/message.py
b/backend/python/pydevlake/pydevlake/message.py
index 74704372e..1985ef967 100644
--- a/backend/python/pydevlake/pydevlake/message.py
+++ b/backend/python/pydevlake/pydevlake/message.py
@@ -17,6 +17,7 @@
from typing import Optional
from pydantic import BaseModel, Field
+import jsonref
from pydevlake.model import ToolScope
@@ -41,8 +42,13 @@ class DynamicModelInfo(Message):
@staticmethod
def from_model(model_class):
+ schema = model_class.schema()
+ if 'definitions' in schema:
+ # Replace $ref with actual schema
+ schema = jsonref.replace_refs(schema, proxies=False)
+ del schema['definitions']
return DynamicModelInfo(
- json_schema=model_class.schema(),
+ json_schema=schema,
table_name=model_class.__tablename__
)
diff --git a/backend/python/pydevlake/pydevlake/plugin.py
b/backend/python/pydevlake/pydevlake/plugin.py
index 8ea9d57ff..aba5234ee 100644
--- a/backend/python/pydevlake/pydevlake/plugin.py
+++ b/backend/python/pydevlake/pydevlake/plugin.py
@@ -176,7 +176,7 @@ class Plugin(ABC):
"connectionId": connection.id
}
),
- self.extra_tasks(scope, tx_rule, entity_types, connection)
+ *self.extra_tasks(scope, tx_rule, entity_types, connection)
]
def extra_tasks(self, scope: ToolScope, tx_rule:
Optional[TransformationRule],
diff --git a/backend/python/pydevlake/pydevlake/subtasks.py
b/backend/python/pydevlake/pydevlake/subtasks.py
index ab584cb29..d7f1fe59b 100644
--- a/backend/python/pydevlake/pydevlake/subtasks.py
+++ b/backend/python/pydevlake/pydevlake/subtasks.py
@@ -71,6 +71,7 @@ class Subtask:
)
except Exception as e:
logger.error(f'{type(e).__name__}: {e}')
+ raise e
subtask_run.state = json.dumps(state)
subtask_run.completed = datetime.now()
diff --git a/backend/python/pydevlake/pyproject.toml
b/backend/python/pydevlake/pyproject.toml
index 25341c1f2..1e6f11b6a 100644
--- a/backend/python/pydevlake/pyproject.toml
+++ b/backend/python/pydevlake/pyproject.toml
@@ -31,6 +31,8 @@ fire = "^0.4.0"
pydantic = "^1.10.2"
pydevd-pycharm = "^231.6471.3"
pytest = "^7.2.2"
+jsonref = "^1.1.0"
+psycopg2 = "^2.9.5"
[tool.poetry.group.dev.dependencies]
diff --git a/backend/python/test/fakeplugin/fakeplugin/main.py
b/backend/python/test/fakeplugin/fakeplugin/main.py
index 45d0c4c6b..f1903c91f 100644
--- a/backend/python/test/fakeplugin/fakeplugin/main.py
+++ b/backend/python/test/fakeplugin/fakeplugin/main.py
@@ -16,11 +16,12 @@
from enum import Enum
from datetime import datetime
from typing import Optional
+import json
from sqlmodel import Field
from pydevlake import Plugin, Connection, TransformationRule, Stream,
ToolModel, ToolScope, RemoteScopeGroup, DomainType
-from pydevlake.domain_layer.devops import CicdScope, CICDPipeline
+from pydevlake.domain_layer.devops import CicdScope, CICDPipeline, CICDStatus,
CICDResult, CICDType
VALID_TOKEN = "this_is_a_valid_token"
@@ -51,11 +52,11 @@ class FakeStream(Stream):
def collect(self, state, context):
for p in self.fake_pipelines:
- yield p.json(), {}
+ yield json.loads(p.json()), {}
def convert(self, pipeline: FakePipeline, ctx):
- if ctx.transformationRule:
- env = ctx.transformationRule.env
+ if ctx.transformation_rule:
+ env = ctx.transformation_rule.env
else:
env = "unknown"
yield CICDPipeline(
@@ -65,22 +66,22 @@ class FakeStream(Stream):
result=self.convert_result(pipeline.state),
duration_sec=self.duration(pipeline),
environment=env,
- type=CICDPipeline.Type.CI
+ type=CICDType.BUILD
)
def convert_status(self, state: FakePipeline.State):
match state:
case FakePipeline.State.FAILURE | FakePipeline.State.SUCCESS:
- return CICDPipeline.Status.DONE
+ return CICDStatus.DONE
case _:
- return CICDPipeline.Status.IN_PROGRESS
+ return CICDStatus.IN_PROGRESS
def convert_result(self, state: FakePipeline.State):
match state:
case FakePipeline.State.SUCCESS:
- return CICDPipeline.Result.SUCCESS
+ return CICDResult.SUCCESS
case FakePipeline.State.FAILURE:
- return CICDPipeline.Status.FAILURE
+ return CICDResult.FAILURE
case _:
return None
@@ -94,7 +95,7 @@ class FakeConnection(Connection):
token: str
-class FakeProject(ToolScope):
+class FakeProject(ToolScope, table=True):
pass
diff --git a/backend/python/test/fakeplugin/poetry.lock
b/backend/python/test/fakeplugin/poetry.lock
index a6622ebdd..6aa757eed 100644
--- a/backend/python/test/fakeplugin/poetry.lock
+++ b/backend/python/test/fakeplugin/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry and should not be changed by
hand.
+# This file is automatically @generated by Poetry 1.4.1 and should not be
changed by hand.
[[package]]
name = "attrs"
@@ -275,6 +275,18 @@ files = [
{file = "iniconfig-2.0.0.tar.gz", hash =
"sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
]
+[[package]]
+name = "jsonref"
+version = "1.1.0"
+description = "jsonref is a library for automatic dereferencing of JSON
Reference objects for Python."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "jsonref-1.1.0-py3-none-any.whl", hash =
"sha256:590dc7773df6c21cbf948b5dac07a72a251db28b0238ceecce0a2abfa8ec30a9"},
+ {file = "jsonref-1.1.0.tar.gz", hash =
"sha256:32fe8e1d85af0fdefbebce950af85590b22b60f9e95443176adbde4e1ecea552"},
+]
+
[[package]]
name = "mysqlclient"
version = "2.1.1"
@@ -320,6 +332,29 @@ files = [
dev = ["pre-commit", "tox"]
testing = ["pytest", "pytest-benchmark"]
+[[package]]
+name = "psycopg2"
+version = "2.9.5"
+description = "psycopg2 - Python-PostgreSQL Database Adapter"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+files = [
+ {file = "psycopg2-2.9.5-cp310-cp310-win32.whl", hash =
"sha256:d3ef67e630b0de0779c42912fe2cbae3805ebaba30cda27fea2a3de650a9414f"},
+ {file = "psycopg2-2.9.5-cp310-cp310-win_amd64.whl", hash =
"sha256:4cb9936316d88bfab614666eb9e32995e794ed0f8f6b3b718666c22819c1d7ee"},
+ {file = "psycopg2-2.9.5-cp311-cp311-win32.whl", hash =
"sha256:093e3894d2d3c592ab0945d9eba9d139c139664dcf83a1c440b8a7aa9bb21955"},
+ {file = "psycopg2-2.9.5-cp311-cp311-win_amd64.whl", hash =
"sha256:920bf418000dd17669d2904472efeab2b20546efd0548139618f8fa305d1d7ad"},
+ {file = "psycopg2-2.9.5-cp36-cp36m-win32.whl", hash =
"sha256:b9ac1b0d8ecc49e05e4e182694f418d27f3aedcfca854ebd6c05bb1cffa10d6d"},
+ {file = "psycopg2-2.9.5-cp36-cp36m-win_amd64.whl", hash =
"sha256:fc04dd5189b90d825509caa510f20d1d504761e78b8dfb95a0ede180f71d50e5"},
+ {file = "psycopg2-2.9.5-cp37-cp37m-win32.whl", hash =
"sha256:922cc5f0b98a5f2b1ff481f5551b95cd04580fd6f0c72d9b22e6c0145a4840e0"},
+ {file = "psycopg2-2.9.5-cp37-cp37m-win_amd64.whl", hash =
"sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a"},
+ {file = "psycopg2-2.9.5-cp38-cp38-win32.whl", hash =
"sha256:f5b6320dbc3cf6cfb9f25308286f9f7ab464e65cfb105b64cc9c52831748ced2"},
+ {file = "psycopg2-2.9.5-cp38-cp38-win_amd64.whl", hash =
"sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e"},
+ {file = "psycopg2-2.9.5-cp39-cp39-win32.whl", hash =
"sha256:322fd5fca0b1113677089d4ebd5222c964b1760e361f151cbb2706c4912112c5"},
+ {file = "psycopg2-2.9.5-cp39-cp39-win_amd64.whl", hash =
"sha256:190d51e8c1b25a47484e52a79638a8182451d6f6dff99f26ad9bd81e5359a0fa"},
+ {file = "psycopg2-2.9.5.tar.gz", hash =
"sha256:a5246d2e683a972e2187a8714b5c2cf8156c064629f9a9b1a873c1730d9e245a"},
+]
+
[[package]]
name = "pydantic"
version = "1.10.7"
@@ -397,7 +432,9 @@ develop = true
[package.dependencies]
fire = "^0.4.0"
inflect = "^6.0.2"
+jsonref = "^1.1.0"
mysqlclient = "^2.1.1"
+psycopg2 = "^2.9.5"
pydantic = "^1.10.2"
pydevd-pycharm = "^231.6471.3"
pytest = "^7.2.2"
@@ -518,11 +555,11 @@ files = [
]
[package.dependencies]
-greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and
(platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or
platform_machine == \"x86_64\" or platform_machine == \"amd64\" or
platform_machine == \"AMD64\" or platform_machine == \"win32\" or
platform_machine == \"WIN32\")"}
+greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and
platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine
== \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or
python_version >= \"3\" and platform_machine == \"amd64\" or python_version >=
\"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and
platform_machine == \"win32\" or python_version >= \"3\" and platform_machine
== \"WIN32\""}
[package.extras]
aiomysql = ["aiomysql", "greenlet (!=0.4.17)"]
-aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions
(!=3.10.0.1)"]
+aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions
(!=3.10.0.1)"]
asyncio = ["greenlet (!=0.4.17)"]
asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"]
mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"]
@@ -532,14 +569,14 @@ mssql-pyodbc = ["pyodbc"]
mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"]
mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"]
mysql-connector = ["mysql-connector-python"]
-oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"]
+oracle = ["cx-oracle (>=7)", "cx-oracle (>=7,<8)"]
postgresql = ["psycopg2 (>=2.7)"]
postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"]
postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"]
postgresql-psycopg2binary = ["psycopg2-binary"]
postgresql-psycopg2cffi = ["psycopg2cffi"]
pymysql = ["pymysql", "pymysql (<1)"]
-sqlcipher = ["sqlcipher3_binary"]
+sqlcipher = ["sqlcipher3-binary"]
[[package]]
name = "sqlalchemy2-stubs"
diff --git a/backend/python/test/fakeplugin/start.sh
b/backend/python/test/fakeplugin/start.sh
old mode 100644
new mode 100755
diff --git a/backend/server/services/remote/models/conversion.go
b/backend/server/services/remote/models/conversion.go
index 6d77bf37c..0deb03dbd 100644
--- a/backend/server/services/remote/models/conversion.go
+++ b/backend/server/services/remote/models/conversion.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
+ "gorm.io/datatypes"
)
func LoadTableModel(tableName string, schema map[string]any, encrypt bool,
parentModel any) (*models.DynamicTabler, errors.Error) {
@@ -118,8 +119,9 @@ func getGoType(schema map[string]any) (reflect.Type,
errors.Error) {
case "boolean":
goType = reflect.TypeOf(false)
case "string":
- //TODO: distinguish stypes based on string format
goType = reflect.TypeOf("")
+ case "object":
+ goType = reflect.TypeOf(datatypes.JSONMap{})
default:
return nil, errors.BadInput.New(fmt.Sprintf("Unsupported type
%s", jsonType))
}
diff --git a/backend/test/integration/remote/helper.go
b/backend/test/integration/remote/helper.go
index c185778ad..469d49e28 100644
--- a/backend/test/integration/remote/helper.go
+++ b/backend/test/integration/remote/helper.go
@@ -19,13 +19,15 @@ package test
import (
"fmt"
- "github.com/apache/incubator-devlake/core/config"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/test/integration/helper"
"os"
"path/filepath"
"testing"
"time"
+
+ "github.com/apache/incubator-devlake/core/config"
+ "github.com/apache/incubator-devlake/core/plugin"
+ orgPlugin "github.com/apache/incubator-devlake/plugins/org/impl"
+ "github.com/apache/incubator-devlake/test/integration/helper"
)
const (
@@ -56,7 +58,7 @@ type (
func SetupEnv() {
fmt.Println("Setup test env")
helper.Init()
- path := filepath.Join(helper.ProjectRoot, FAKE_PLUGIN_DIR, "start.sh")
// make sure the path is correct
+ path := filepath.Join(helper.ProjectRoot, FAKE_PLUGIN_DIR, "start.sh")
_, err := os.Stat(path)
if err != nil {
panic(err)
@@ -72,7 +74,7 @@ func ConnectLocalServer(t *testing.T) *helper.DevlakeClient {
DbURL: config.GetConfig().GetString("E2E_DB_URL"),
CreateServer: true,
TruncateDb: true,
- Plugins: map[string]plugin.PluginMeta{},
+ Plugins: map[string]plugin.PluginMeta{"org":
orgPlugin.Org{}},
})
client.SetTimeout(30 * time.Second)
return client
diff --git a/backend/test/integration/remote/python_plugin_test.go
b/backend/test/integration/remote/python_plugin_test.go
index 8deaf43b2..67bf76140 100644
--- a/backend/test/integration/remote/python_plugin_test.go
+++ b/backend/test/integration/remote/python_plugin_test.go
@@ -18,11 +18,12 @@ limitations under the License.
package test
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/test/integration/helper"
"github.com/stretchr/testify/require"
- "testing"
)
func TestCreateConnection(t *testing.T) {
@@ -81,7 +82,6 @@ func TestCreateScope(t *testing.T) {
}
func TestRunPipeline(t *testing.T) {
- t.SkipNow() //Fix later
client := CreateClient(t)
conn := CreateTestConnection(client)
@@ -109,7 +109,6 @@ func TestRunPipeline(t *testing.T) {
}
func TestBlueprintV200(t *testing.T) {
- t.SkipNow() //Fix later
client := CreateClient(t)
connection := CreateTestConnection(client)
projectName := "Test project"
diff --git a/devops/docker/lake-builder/Dockerfile
b/devops/docker/lake-builder/Dockerfile
index 9eea8c03f..3e6344ffe 100644
--- a/devops/docker/lake-builder/Dockerfile
+++ b/devops/docker/lake-builder/Dockerfile
@@ -15,14 +15,14 @@
#
FROM --platform=linux/amd64 debian:bullseye as debian-amd64
-RUN apt-get update
-RUN apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev
+RUN apt-get -y update && apt -y upgrade &&\
+ apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev
FROM golang:1.19.7-bullseye as builder
# Base dependencies
-RUN apt-get update
-RUN apt-get install -y gcc binutils libfindbin-libs-perl cmake libssh2-1-dev
libssl-dev zlib1g-dev \
+RUN apt-get -y update && apt -y upgrade &&\
+ apt-get install -y gcc binutils libfindbin-libs-perl cmake libssh2-1-dev
libssl-dev zlib1g-dev \
gcc-x86-64-linux-gnu binutils-x86-64-linux-gnu
COPY --from=debian-amd64 /usr/include /rootfs-amd64/usr/include
@@ -43,8 +43,10 @@ RUN \
FROM python:3.11.2-slim-bullseye
-RUN apt update && apt upgrade && apt -y install tzdata make tar curl gcc g++
pkg-config git \
- libssh2-1 zlib1g libffi-dev default-libmysqlclient-dev
+RUN apt -y update && apt -y upgrade && apt -y install tzdata make tar curl gcc
g++ pkg-config git \
+ libssh2-1 zlib1g libffi-dev \
+ default-libmysqlclient-dev \
+ libpq-dev
# Install Libs/Headers from previous stage
COPY --from=builder /tmp/deps/*.so* /usr/lib/
