This is an automated email from the ASF dual-hosted git repository.
ka94 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new f6545ca67 fix: Set scopes `_raw_data_params` (#5454)
f6545ca67 is described below
commit f6545ca67b75843ea6c264e6406aa784f72b79de
Author: Keon Amini <[email protected]>
AuthorDate: Tue Jun 13 11:40:21 2023 -0500
fix: Set scopes `_raw_data_params` (#5454)
Set the `_raw_data_params` column on tool scopes (in make_remote_scopes)
and domain scopes (in make_pipeline).
---
backend/python/README.md | 2 +-
.../python/plugins/azuredevops/azuredevops/models.py | 2 +-
backend/python/pydevlake/pydevlake/message.py | 5 +++--
backend/python/pydevlake/pydevlake/model.py | 20 +++++++++++++++-----
backend/python/pydevlake/pydevlake/plugin.py | 9 +++++----
backend/python/pydevlake/pydevlake/subtasks.py | 7 ++-----
backend/test/e2e/manual/azuredevops/models.go | 1 +
7 files changed, 28 insertions(+), 18 deletions(-)
diff --git a/backend/python/README.md b/backend/python/README.md
index 317d5dc73..6bb36f5c1 100644
--- a/backend/python/README.md
+++ b/backend/python/README.md
@@ -377,7 +377,7 @@ from myplugin.api import MyAPI
def collect(self, state, context) -> Iterable[Tuple[object, dict]]:
api = MyAPI(context.connection.url)
- for user in api.users().json():
+ for user in api.users().json:
yield user, state
...
diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py
b/backend/python/plugins/azuredevops/azuredevops/models.py
index 85f45232f..3ce1aebb5 100644
--- a/backend/python/plugins/azuredevops/azuredevops/models.py
+++ b/backend/python/plugins/azuredevops/azuredevops/models.py
@@ -39,7 +39,7 @@ class GitRepositoryConfig(ScopeConfig):
class GitRepository(ToolScope, table=True):
url: str
- remote_url: str
+ remote_url: Optional[str]
default_branch: Optional[str]
project_id: str
org_id: str
diff --git a/backend/python/pydevlake/pydevlake/message.py
b/backend/python/pydevlake/pydevlake/message.py
index 20d320439..bf5a2d0cb 100644
--- a/backend/python/pydevlake/pydevlake/message.py
+++ b/backend/python/pydevlake/pydevlake/message.py
@@ -45,7 +45,7 @@ class DynamicModelInfo(Message):
@staticmethod
def from_model(model_class):
- schema = model_class.schema()
+ schema = model_class.schema(by_alias=True)
if 'definitions' in schema:
# Replace $ref with actual schema
schema = jsonref.replace_refs(schema, proxies=False)
@@ -54,7 +54,8 @@ class DynamicModelInfo(Message):
for prop in schema['properties'].values():
if 'type' not in prop and 'enum' in prop:
prop['type'] = 'string'
- return DynamicModelInfo( json_schema=schema,
+ return DynamicModelInfo(
+ json_schema=schema,
table_name=model_class.__tablename__
)
diff --git a/backend/python/pydevlake/pydevlake/model.py
b/backend/python/pydevlake/pydevlake/model.py
index 8d196d1c2..b9d6ec051 100644
--- a/backend/python/pydevlake/pydevlake/model.py
+++ b/backend/python/pydevlake/pydevlake/model.py
@@ -15,6 +15,7 @@
import os
+import json
from typing import Iterable, Optional
from inspect import getmodule
from datetime import datetime
@@ -40,6 +41,7 @@ class Model(SQLModel):
sa_column=Column(DateTime(), default=datetime.utcnow,
onupdate=datetime.utcnow)
)
+
class ToolTable(SQLModel):
@declared_attr
def __tablename__(cls) -> str:
@@ -71,6 +73,7 @@ class Connection(ToolTable, Model):
return None
return proxy
+
class DomainType(Enum):
CODE = "CODE"
TICKET = "TICKET"
@@ -97,10 +100,10 @@ class RawModel(SQLModel):
class RawDataOrigin(SQLModel):
# SQLModel doesn't like attributes starting with _
# so we change the names of the columns.
- raw_data_params: Optional[str] =
Field(sa_column_kwargs={'name':'_raw_data_params'})
- raw_data_table: Optional[str] =
Field(sa_column_kwargs={'name':'_raw_data_table'})
- raw_data_id: Optional[str] =
Field(sa_column_kwargs={'name':'_raw_data_id'})
- raw_data_remark: Optional[str] =
Field(sa_column_kwargs={'name':'_raw_data_remark'})
+ raw_data_params: Optional[str] = Field(sa_column_kwargs={'name':
'_raw_data_params'}, alias='_raw_data_params')
+ raw_data_table: Optional[str] = Field(sa_column_kwargs={'name':
'_raw_data_table'}, alias='_raw_data_table')
+ raw_data_id: Optional[str] = Field(sa_column_kwargs={'name':
'_raw_data_id'}, alias='_raw_data_id')
+ raw_data_remark: Optional[str] = Field(sa_column_kwargs={'name':
'_raw_data_remark'}, alias='_raw_data_remark')
def set_raw_origin(self, raw: RawModel):
self.raw_data_id = raw.id
@@ -165,6 +168,13 @@ def domain_id(model_type, connection_id, *args):
return ':'.join(segments)
+def raw_data_params(connection_id: int, scope_id: str) -> str:
+ return json.dumps({
+ "connection_id": connection_id,
+ "scope_id": scope_id
+ }, separators=(',', ':'))
+
+
def _get_plugin_name(cls):
"""
Get the plugin name from a class by looking into
@@ -188,4 +198,4 @@ class SubtaskRun(SQLModel, table=True):
connection_id: int
started: datetime
completed: Optional[datetime]
- state: str = Field(sa_column=Column(Text)) # JSON encoded dict of atomic
values
+ state: str = Field(sa_column=Column(Text)) # JSON encoded dict of atomic
values
diff --git a/backend/python/pydevlake/pydevlake/plugin.py
b/backend/python/pydevlake/pydevlake/plugin.py
index 253abacd3..e130b52e0 100644
--- a/backend/python/pydevlake/pydevlake/plugin.py
+++ b/backend/python/pydevlake/pydevlake/plugin.py
@@ -22,13 +22,12 @@ import sys
import fire
import pydevlake.message as msg
-from pydevlake.api import APIException
from pydevlake.subtasks import Subtask
from pydevlake.logger import logger
from pydevlake.ipc import PluginCommands
from pydevlake.context import Context
-from pydevlake.stream import Stream, DomainType
-from pydevlake.model import ToolScope, DomainScope, Connection, ScopeConfig
+from pydevlake.stream import Stream
+from pydevlake.model import ToolScope, DomainScope, Connection, ScopeConfig,
raw_data_params
from pydevlake.migration import MIGRATION_SCRIPTS
@@ -117,6 +116,7 @@ class Plugin(ABC):
remote_scopes = []
for tool_scope in self.remote_scopes(connection, group_id):
tool_scope.connection_id = connection.id
+ tool_scope.raw_data_params = raw_data_params(connection.id,
tool_scope.id)
remote_scopes.append(
msg.RemoteScope(
id=tool_scope.id,
@@ -139,10 +139,11 @@ class Plugin(ABC):
for tool_scope, _ in scope_config_pairs:
for scope in self.domain_scopes(tool_scope):
scope.id = tool_scope.domain_id()
+ scope.raw_data_params = raw_data_params(connection.id,
scope.id)
domain_scopes.append(
msg.DynamicDomainScope(
type_name=type(scope).__name__,
- data=scope.json(exclude_unset=True)
+ data=scope.json(exclude_unset=True, by_alias=True)
)
)
return msg.PipelineData(
diff --git a/backend/python/pydevlake/pydevlake/subtasks.py
b/backend/python/pydevlake/pydevlake/subtasks.py
index 159316db1..588e478fe 100644
--- a/backend/python/pydevlake/pydevlake/subtasks.py
+++ b/backend/python/pydevlake/pydevlake/subtasks.py
@@ -23,7 +23,7 @@ from typing import Tuple, Dict, Iterable, Generator
import sqlalchemy.sql as sql
from sqlmodel import Session, select
-from pydevlake.model import RawModel, ToolModel, DomainModel, SubtaskRun
+from pydevlake.model import RawModel, ToolModel, DomainModel, SubtaskRun,
raw_data_params
from pydevlake.context import Context
from pydevlake.message import RemoteProgress
from pydevlake import logger
@@ -127,10 +127,7 @@ class Subtask:
return {}
def _params(self, ctx: Context) -> str:
- return json.dumps({
- "connection_id": ctx.connection.id,
- "scope_id": ctx.scope.id
- }, separators=(',', ':'))
+ return raw_data_params(ctx.connection.id, ctx.scope.id)
@abstractmethod
def delete(self, session, ctx):
diff --git a/backend/test/e2e/manual/azuredevops/models.go
b/backend/test/e2e/manual/azuredevops/models.go
index d40172fdd..3ea19ffb3 100644
--- a/backend/test/e2e/manual/azuredevops/models.go
+++ b/backend/test/e2e/manual/azuredevops/models.go
@@ -44,6 +44,7 @@ type (
}
AzureGitRepo struct {
+ RawDataParams string `json:"_raw_data_params"`
Id string
Name string
ConnectionId uint64
