This is an automated email from the ASF dual-hosted git repository.
ka94 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new ba897f23d fix: Make Pydevlake ScopeConfig optional (#5470)
ba897f23d is described below
commit ba897f23ddfcab142b16d92bcedeabfc0d8f1e5a
Author: Camille Teruel <[email protected]>
AuthorDate: Thu Jun 15 22:35:01 2023 +0200
fix: Make Pydevlake ScopeConfig optional (#5470)
Co-authored-by: Camille Teruel <[email protected]>
---
backend/python/README.md | 4 ++--
backend/python/plugins/azuredevops/azuredevops/main.py | 6 +++---
backend/python/pydevlake/pydevlake/context.py | 2 +-
backend/python/pydevlake/pydevlake/ipc.py | 4 ++--
backend/python/pydevlake/pydevlake/model.py | 4 ++--
backend/python/pydevlake/pydevlake/plugin.py | 2 +-
backend/python/pydevlake/pydevlake/testing/testing.py | 8 +++++++-
backend/python/pydevlake/tests/stream_test.py | 3 ++-
backend/server/services/remote/models/conversion.go | 2 +-
backend/server/services/remote/plugin/plugin_impl.go | 7 +++++--
10 files changed, 26 insertions(+), 16 deletions(-)
diff --git a/backend/python/README.md b/backend/python/README.md
index 6bb36f5c1..148f96257 100644
--- a/backend/python/README.md
+++ b/backend/python/README.md
@@ -45,7 +45,7 @@ class MyPluginToolScope(dl.ToolScope):
class MyPlugin(dl.Plugin):
connection_type = MyPluginConnection
tool_scope_type = MyPluginToolScope
- scope_config_type = MyPluginScopeConfig
+ scope_config_type = MyPluginScopeConfig
streams = []
def domain_scopes(self, tool_scope: MyScope) -> Iterable[dl.DomainScope]:
@@ -122,7 +122,7 @@ class MyPluginScopeConfig(ScopeConfig):
issue_type_regex: str
```
-If your plugin does not require any such conversion parameter, leave this
class empty.
+If your plugin does not require any such conversion parameter, you can omit
this class and the `scope_config_type` plugin attribute.
### Tool scope type
diff --git a/backend/python/plugins/azuredevops/azuredevops/main.py
b/backend/python/plugins/azuredevops/azuredevops/main.py
index 15d2593c2..ff6007d3a 100644
--- a/backend/python/plugins/azuredevops/azuredevops/main.py
+++ b/backend/python/plugins/azuredevops/azuredevops/main.py
@@ -131,9 +131,9 @@ class AzureDevOpsPlugin(Plugin):
def extra_stages(self, scope_config_pairs: list[tuple[GitRepository,
GitRepositoryConfig]], _):
for scope, config in scope_config_pairs:
- if DomainType.CODE in config.domain_types:
- if not scope.is_external():
- yield [refdiff(scope.id, config.refdiff)]
+ if DomainType.CODE in config.domain_types and not
scope.is_external():
+ yield [refdiff(scope.id, config.refdiff)]
+
@property
def streams(self):
diff --git a/backend/python/pydevlake/pydevlake/context.py
b/backend/python/pydevlake/pydevlake/context.py
index 1c543d349..ad815da3c 100644
--- a/backend/python/pydevlake/pydevlake/context.py
+++ b/backend/python/pydevlake/pydevlake/context.py
@@ -24,7 +24,7 @@ class Context:
engine: Engine,
connection: Connection,
scope: ToolScope,
- scope_config: ScopeConfig = None,
+ scope_config: ScopeConfig,
options: dict = None):
self.engine = engine
self.connection = connection
diff --git a/backend/python/pydevlake/pydevlake/ipc.py
b/backend/python/pydevlake/pydevlake/ipc.py
index 1db11d88f..a6ff19476 100644
--- a/backend/python/pydevlake/pydevlake/ipc.py
+++ b/backend/python/pydevlake/pydevlake/ipc.py
@@ -93,7 +93,7 @@ class PluginCommands:
scope_config_pairs = [
(
self._plugin.tool_scope_type(**raw_scope),
- self._plugin.scope_config_type(**raw_config)
+ self._plugin.scope_config_type(**raw_config or {})
)
for raw_scope, raw_config in scope_config_pairs
]
@@ -114,7 +114,7 @@ class PluginCommands:
scope = self._plugin.tool_scope_type(**scope_dict)
connection_dict = data['connection']
connection = self._plugin.connection_type(**connection_dict)
- scope_config_dict = data['scope_config']
+ scope_config_dict = data.get('scope_config') or {}
scope_config = self._plugin.scope_config_type(**scope_config_dict)
options = data.get('options', {})
return Context(create_db_engine(db_url), connection, scope,
scope_config, options)
diff --git a/backend/python/pydevlake/pydevlake/model.py
b/backend/python/pydevlake/pydevlake/model.py
index b9d6ec051..1b3effb6a 100644
--- a/backend/python/pydevlake/pydevlake/model.py
+++ b/backend/python/pydevlake/pydevlake/model.py
@@ -84,8 +84,8 @@ class DomainType(Enum):
class ScopeConfig(ToolTable, Model):
- name: str
- domain_types: list[DomainType] = Field(default_factory=list,
alias="entities")
+ name: str = Field(default="default")
+ domain_types: list[DomainType] = Field(default=list(DomainType),
alias="entities")
class RawModel(SQLModel):
diff --git a/backend/python/pydevlake/pydevlake/plugin.py
b/backend/python/pydevlake/pydevlake/plugin.py
index e130b52e0..e6cbe8e96 100644
--- a/backend/python/pydevlake/pydevlake/plugin.py
+++ b/backend/python/pydevlake/pydevlake/plugin.py
@@ -65,7 +65,7 @@ class Plugin(ABC):
@property
def scope_config_type(self) -> Type[ScopeConfig]:
- return None
+ return ScopeConfig
@abstractmethod
def test_connection(self, connection: Connection) ->
msg.TestConnectionResult:
diff --git a/backend/python/pydevlake/pydevlake/testing/testing.py
b/backend/python/pydevlake/pydevlake/testing/testing.py
index e31240e1c..225f47b0e 100644
--- a/backend/python/pydevlake/pydevlake/testing/testing.py
+++ b/backend/python/pydevlake/pydevlake/testing/testing.py
@@ -17,6 +17,7 @@ import pytest
from typing import Union, Type, Iterable, Generator, Optional
+from pydantic import ValidationError
from sqlmodel import create_engine
from pydevlake.context import Context
@@ -114,6 +115,10 @@ def assert_valid_tool_scope_type(plugin: Plugin):
def assert_valid_scope_config_type(plugin: Plugin):
scope_config_type = plugin.scope_config_type
assert issubclass(scope_config_type, ScopeConfig), 'scope_config_type must
be a subclass of ScopeConfig'
+ try:
+ scope_config_type()
+ except ValidationError as e:
+ pytest.fail(f'scope_config_type must not have required fields: {e}')
def assert_valid_streams(plugin: Plugin):
@@ -189,12 +194,13 @@ def assert_valid_plugin(plugin: Plugin):
assert_valid_streams(plugin)
-def assert_plugin_run(plugin: Plugin, connection: Connection, scope_config:
ScopeConfig):
+def assert_plugin_run(plugin: Plugin, connection: Connection, scope_config:
Optional[ScopeConfig] = None):
assert_valid_plugin(plugin)
assert_valid_connection(plugin, connection)
groups = assert_valid_remote_scope_groups(plugin, connection)
scope = assert_valid_remote_scopes(plugin, connection, groups[0].id)[0]
assert_valid_domain_scopes(plugin, scope)
+ scope_config = scope_config or plugin.scope_config_type()
assert_valid_pipeline_plan(plugin, connection, scope, scope_config)
for stream in plugin.streams:
if isinstance(stream, type):
diff --git a/backend/python/pydevlake/tests/stream_test.py
b/backend/python/pydevlake/tests/stream_test.py
index 362d0da6b..07a841d66 100644
--- a/backend/python/pydevlake/tests/stream_test.py
+++ b/backend/python/pydevlake/tests/stream_test.py
@@ -20,7 +20,7 @@ import pytest
from sqlmodel import SQLModel, Session, Field, create_engine
from pydevlake import Stream, Connection, Context, DomainType
-from pydevlake.model import ToolModel, DomainModel, ToolScope
+from pydevlake.model import ScopeConfig, ToolModel, DomainModel, ToolScope
class DummyToolModel(ToolModel, table=True):
@@ -88,6 +88,7 @@ def ctx(connection, scope, engine):
engine=engine,
scope=scope,
connection=connection,
+ scope_config=ScopeConfig(),
options={}
)
diff --git a/backend/server/services/remote/models/conversion.go
b/backend/server/services/remote/models/conversion.go
index 877597ab4..affbd0b17 100644
--- a/backend/server/services/remote/models/conversion.go
+++ b/backend/server/services/remote/models/conversion.go
@@ -48,7 +48,7 @@ func GenerateStructType(schema utils.JsonObject, baseType
reflect.Type) (reflect
}
required, err := utils.GetProperty[[]string](schema, "required")
if err != nil {
- return nil, err
+ required = []string{}
}
if baseType != nil {
anonymousField := reflect.StructField{
diff --git a/backend/server/services/remote/plugin/plugin_impl.go
b/backend/server/services/remote/plugin/plugin_impl.go
index 9e2e2ad3a..0ac2daa06 100644
--- a/backend/server/services/remote/plugin/plugin_impl.go
+++ b/backend/server/services/remote/plugin/plugin_impl.go
@@ -19,6 +19,7 @@ package plugin
import (
"fmt"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
coreModels "github.com/apache/incubator-devlake/core/models"
@@ -170,14 +171,16 @@ func (p *remotePluginImpl) getScopeAndConfig(db dal.Dal,
connectionId uint64, sc
if err != nil {
return nil, nil, errors.BadInput.Wrap(err, "Invalid scope")
}
- wrappedScopeConfig := p.scopeConfigTabler.New()
if scope.ScopeConfigId != 0 {
+ wrappedScopeConfig := p.scopeConfigTabler.New()
err = api.CallDB(db.First, wrappedScopeConfig,
dal.From(p.scopeConfigTabler.TableName()), dal.Where("id = ?",
scope.ScopeConfigId))
if err != nil {
return nil, nil, err
}
+ return wrappedScope.Unwrap(), wrappedScopeConfig.Unwrap(), nil
+ } else {
+ return wrappedScope.Unwrap(), nil, nil
}
- return wrappedScope.Unwrap(), wrappedScopeConfig.Unwrap(), nil
}
func (p *remotePluginImpl) Description() string {
