This is an automated email from the ASF dual-hosted git repository.
ka94 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 3c4774392 Test azuredevops plugin (#4812)
3c4774392 is described below
commit 3c47743925c4591fab956502250e875fc8eab591
Author: Camille Teruel <[email protected]>
AuthorDate: Wed Mar 29 22:07:57 2023 +0200
Test azuredevops plugin (#4812)
* fix: Fix leftover reference to connection.pat
* feat: Add support for field aliases
Message field names are serialized by alias.
This allow to have python attributes that are snake_cased that end up being
serialized by alias, typically camelCased.
* feat: Add parentId to RemoteScope
* feat: Add more testing facilities for plugins
Add many assert methods to check a specific plugin implementation is
correct.
* test: Use pydevlake testing facilities to test AzureDevOps plugin
---------
Co-authored-by: Camille Teruel <[email protected]>
---
.../python/plugins/azuredevops/azuredevops/main.py | 14 ++-
.../azuredevops/tests/plugin_test.py} | 24 +++-
.../plugins/azuredevops/tests/streams_test.py | 10 +-
.../pydevlake/pydevlake/domain_layer/code.py | 4 +-
.../pydevlake/pydevlake/domain_layer/devops.py | 4 +-
backend/python/pydevlake/pydevlake/ipc.py | 2 +-
backend/python/pydevlake/pydevlake/message.py | 8 +-
backend/python/pydevlake/pydevlake/model.py | 2 +-
backend/python/pydevlake/pydevlake/plugin.py | 24 ++--
.../python/pydevlake/pydevlake/testing/__init__.py | 8 +-
.../python/pydevlake/pydevlake/testing/testing.py | 131 ++++++++++++++++++++-
.../services/remote/plugin/remote_scope_api.go | 9 +-
backend/test/integration/helper/models.go | 5 +-
.../test/integration/remote/python_plugin_test.go | 2 +
14 files changed, 203 insertions(+), 44 deletions(-)
diff --git a/backend/python/plugins/azuredevops/azuredevops/main.py
b/backend/python/plugins/azuredevops/azuredevops/main.py
index 8648c2bbc..23e0b0751 100644
--- a/backend/python/plugins/azuredevops/azuredevops/main.py
+++ b/backend/python/plugins/azuredevops/azuredevops/main.py
@@ -46,8 +46,7 @@ class AzureDevOpsPlugin(Plugin):
yield Repo(
name=git_repo.name,
url=git_repo.url,
- forked_from=git_repo.parentRepositoryUrl,
- deleted=git_repo.isDisabled,
+ forked_from=git_repo.parentRepositoryUrl
)
yield CicdScope(
@@ -75,8 +74,11 @@ class AzureDevOpsPlugin(Plugin):
api = AzureDevOpsAPI(connection)
for raw_repo in api.git_repos(org, proj):
url = urlparse(raw_repo['remoteUrl'])
- url =
url._replace(netloc=f'{url.username}:{connection.pat}@{url.hostname}')
- repo = GitRepository(**raw_repo, project_id=proj, org_id=org,
url=url.geturl())
+ url =
url._replace(netloc=f'{url.username}:{connection.token}@{url.hostname}')
+ raw_repo['url'] = url.geturl()
+ raw_repo['project_id'] = proj
+ raw_repo['org_id'] = org
+ repo = GitRepository(**raw_repo)
if not repo.defaultBranch:
return None
if "parentRepository" in raw_repo:
@@ -88,13 +90,13 @@ class AzureDevOpsPlugin(Plugin):
if resp.status != 200:
raise Exception(f"Invalid token: {connection.token}")
- def extra_tasks(self, scope: GitRepository, entity_types: list[str],
connection: AzureDevOpsConnection):
+ def extra_tasks(self, scope: GitRepository, tx_rule:
AzureDevOpsTransformationRule, entity_types: list[str], connection:
AzureDevOpsConnection):
if DomainType.CODE in entity_types:
return [gitextractor(scope.url, scope.id, connection.proxy)]
else:
return []
- def extra_stages(self, scope_tx_rule_pairs: list[ScopeTxRulePair],
entity_types: list[str], connection_id: int):
+ def extra_stages(self, scope_tx_rule_pairs: list[ScopeTxRulePair],
entity_types: list[str], _):
if DomainType.CODE in entity_types:
for scope, tx_rule in scope_tx_rule_pairs:
options = tx_rule.refdiff_options if tx_rule else None
diff --git a/backend/python/pydevlake/pydevlake/testing/__init__.py
b/backend/python/plugins/azuredevops/tests/plugin_test.py
similarity index 51%
copy from backend/python/pydevlake/pydevlake/testing/__init__.py
copy to backend/python/plugins/azuredevops/tests/plugin_test.py
index 987641ae5..2351f277f 100644
--- a/backend/python/pydevlake/pydevlake/testing/__init__.py
+++ b/backend/python/plugins/azuredevops/tests/plugin_test.py
@@ -13,7 +13,27 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
import pytest
-pytest.register_assert_rewrite('pydevlake.testing')
-from .testing import assert_convert, ContextBuilder
+from pydevlake.testing import assert_valid_plugin, assert_plugin_run
+
+from azuredevops.models import AzureDevOpsConnection,
AzureDevOpsTransformationRule
+from azuredevops.main import AzureDevOpsPlugin
+
+
+def test_valid_plugin():
+ assert_valid_plugin(AzureDevOpsPlugin())
+
+
+def test_valid_plugin_and_connection():
+ # TODO: Set AZURE_DEVOPS_TOKEN env variable in CI
+ token = os.environ.get('AZURE_DEVOPS_TOKEN')
+ if not(token):
+ pytest.skip("No Azure DevOps token provided")
+
+ plugin = AzureDevOpsPlugin()
+ connection = AzureDevOpsConnection(id=1, name='test_connection',
token=token)
+ tx_rule = AzureDevOpsTransformationRule(id=1, name='test_rule')
+
+ assert_plugin_run(plugin, connection, tx_rule)
diff --git a/backend/python/plugins/azuredevops/tests/streams_test.py
b/backend/python/plugins/azuredevops/tests/streams_test.py
index 47fb6cc3f..f76584174 100644
--- a/backend/python/plugins/azuredevops/tests/streams_test.py
+++ b/backend/python/plugins/azuredevops/tests/streams_test.py
@@ -15,7 +15,7 @@
import pytest
-from pydevlake.testing import assert_convert, ContextBuilder
+from pydevlake.testing import assert_stream_convert, ContextBuilder
import pydevlake.domain_layer.code as code
import pydevlake.domain_layer.devops as devops
@@ -133,7 +133,7 @@ def test_builds_stream():
)
]
- assert_convert(AzureDevOpsPlugin, 'builds', raw, expected)
+ assert_stream_convert(AzureDevOpsPlugin, 'builds', raw, expected)
def test_jobs_stream():
@@ -190,7 +190,7 @@ def test_jobs_stream():
environment=devops.CICDEnvironment.PRODUCTION,
cicd_scope_id='johndoe/test-repo'
)
- assert_convert(AzureDevOpsPlugin, 'jobs', raw, expected, ctx)
+ assert_stream_convert(AzureDevOpsPlugin, 'jobs', raw, expected, ctx)
def test_pull_requests_stream():
@@ -290,7 +290,7 @@ def test_pull_requests_stream():
base_commit_sha='4bc26d92b5dbee7837a4d221035a4e2f8df120b2'
)
- assert_convert(AzureDevOpsPlugin, 'gitpullrequests', raw, expected)
+ assert_stream_convert(AzureDevOpsPlugin, 'gitpullrequests', raw, expected)
def test_pull_request_commits_stream():
@@ -316,4 +316,4 @@ def test_pull_request_commits_stream():
pull_request_id="azuredevops:gitpullrequest:1:12345",
)
- assert_convert(AzureDevOpsPlugin, 'gitpullrequestcommits', raw, expected)
+ assert_stream_convert(AzureDevOpsPlugin, 'gitpullrequestcommits', raw,
expected)
diff --git a/backend/python/pydevlake/pydevlake/domain_layer/code.py
b/backend/python/pydevlake/pydevlake/domain_layer/code.py
index 9fca7d256..aa8f1909b 100644
--- a/backend/python/pydevlake/pydevlake/domain_layer/code.py
+++ b/backend/python/pydevlake/pydevlake/domain_layer/code.py
@@ -19,7 +19,7 @@ from typing import Optional
from sqlmodel import Field
-from pydevlake.model import DomainModel, NoPKModel
+from pydevlake.model import DomainModel, DomainScope, NoPKModel
class PullRequest(DomainModel, table=True):
@@ -143,7 +143,7 @@ class RefsPrCherryPick(DomainModel, table=True):
parent_pr_id: str = Field(primary_key=True)
-class Repo(DomainModel, table=True):
+class Repo(DomainScope, table=True):
__tablename__ = "repos"
name: str
url: str
diff --git a/backend/python/pydevlake/pydevlake/domain_layer/devops.py
b/backend/python/pydevlake/pydevlake/domain_layer/devops.py
index 934ac5b2b..f23ddc3db 100644
--- a/backend/python/pydevlake/pydevlake/domain_layer/devops.py
+++ b/backend/python/pydevlake/pydevlake/domain_layer/devops.py
@@ -20,7 +20,7 @@ from enum import Enum
from sqlmodel import Field
-from pydevlake.model import DomainModel, NoPKModel
+from pydevlake.model import DomainModel, NoPKModel, DomainScope
class CICDResult(Enum):
@@ -70,7 +70,7 @@ class CiCDPipelineCommit(NoPKModel, table=True):
repo: str
-class CicdScope(DomainModel):
+class CicdScope(DomainScope):
__tablename__ = 'cicd_scopes'
name: str
description: Optional[str]
diff --git a/backend/python/pydevlake/pydevlake/ipc.py
b/backend/python/pydevlake/pydevlake/ipc.py
index e9542fe97..f28c3af67 100644
--- a/backend/python/pydevlake/pydevlake/ipc.py
+++ b/backend/python/pydevlake/pydevlake/ipc.py
@@ -35,7 +35,7 @@ def plugin_method(func):
def send_output(send_ch: TextIO, obj: object):
if not isinstance(obj, Message):
raise Exception(f"Not a message: {obj}")
- send_ch.write(obj.json(exclude_none=True))
+ send_ch.write(obj.json(exclude_none=True, by_alias=True))
send_ch.write('\n')
send_ch.flush()
diff --git a/backend/python/pydevlake/pydevlake/message.py
b/backend/python/pydevlake/pydevlake/message.py
index 1985ef967..9d2021087 100644
--- a/backend/python/pydevlake/pydevlake/message.py
+++ b/backend/python/pydevlake/pydevlake/message.py
@@ -23,7 +23,8 @@ from pydevlake.model import ToolScope
class Message(BaseModel):
- pass
+ class Config:
+ allow_population_by_field_name = True
class SubtaskMeta(BaseModel):
@@ -84,9 +85,7 @@ class RemoteProgress(Message):
class PipelineTask(Message):
plugin: str
- # Do not snake_case this attribute,
- # it must match the json tag name in PipelineTask go struct
- skipOnFail: bool = False
+ skip_on_fail: bool = Field(default=False, alias="skipOnFail")
subtasks: list[str] = Field(default_factory=list)
options: dict[str, object] = Field(default_factory=dict)
@@ -112,6 +111,7 @@ class RemoteScopeGroup(RemoteScopeTreeNode):
class RemoteScope(RemoteScopeTreeNode):
type: str = Field("scope", const=True)
+ parent_id: str = Field(..., alias="parentId")
scope: ToolScope
diff --git a/backend/python/pydevlake/pydevlake/model.py
b/backend/python/pydevlake/pydevlake/model.py
index 852b62077..7d49bcf67 100644
--- a/backend/python/pydevlake/pydevlake/model.py
+++ b/backend/python/pydevlake/pydevlake/model.py
@@ -124,7 +124,7 @@ class ToolModel(ToolTable, NoPKModel):
class DomainModel(NoPKModel):
- id: str = Field(primary_key=True)
+ id: Optional[str] = Field(primary_key=True)
class ToolScope(ToolModel):
diff --git a/backend/python/pydevlake/pydevlake/plugin.py
b/backend/python/pydevlake/pydevlake/plugin.py
index aba5234ee..c6395fc13 100644
--- a/backend/python/pydevlake/pydevlake/plugin.py
+++ b/backend/python/pydevlake/pydevlake/plugin.py
@@ -108,18 +108,20 @@ class Plugin(ABC):
def make_remote_scopes(self, connection: Connection, group_id:
Optional[str] = None) -> msg.RemoteScopes:
if group_id:
- scopes = [
- msg.RemoteScope(
- id=tool_scope.id,
- name=tool_scope.name,
- scope=tool_scope
+ remote_scopes = []
+ for tool_scope in self.remote_scopes(connection, group_id):
+ tool_scope.connection_id = connection.id
+ remote_scopes.append(
+ msg.RemoteScope(
+ id=tool_scope.id,
+ parent_id=group_id,
+ name=tool_scope.name,
+ scope=tool_scope
+ )
)
- for tool_scope
- in self.remote_scopes(connection, group_id)
- ]
else:
- scopes = self.remote_scope_groups(connection)
- return msg.RemoteScopes(__root__=scopes)
+ remote_scopes = self.remote_scope_groups(connection)
+ return msg.RemoteScopes(__root__=remote_scopes)
def make_pipeline(self, scope_tx_rule_pairs: list[ScopeTxRulePair],
entity_types: list[str], connection: Connection):
@@ -168,7 +170,7 @@ class Plugin(ABC):
return [
msg.PipelineTask(
plugin=self.name,
- skipOnFail=False,
+ skip_on_fail=False,
subtasks=self.select_subtasks(scope, entity_types),
options={
"scopeId": scope.id,
diff --git a/backend/python/pydevlake/pydevlake/testing/__init__.py
b/backend/python/pydevlake/pydevlake/testing/__init__.py
index 987641ae5..01d7fa9e8 100644
--- a/backend/python/pydevlake/pydevlake/testing/__init__.py
+++ b/backend/python/pydevlake/pydevlake/testing/__init__.py
@@ -16,4 +16,10 @@
import pytest
pytest.register_assert_rewrite('pydevlake.testing')
-from .testing import assert_convert, ContextBuilder
+from .testing import (
+ assert_stream_convert,
+ assert_stream_run,
+ assert_valid_plugin,
+ assert_plugin_run,
+ ContextBuilder
+)
diff --git a/backend/python/pydevlake/pydevlake/testing/testing.py
b/backend/python/pydevlake/pydevlake/testing/testing.py
index 02004fb7b..4c36ddb51 100644
--- a/backend/python/pydevlake/pydevlake/testing/testing.py
+++ b/backend/python/pydevlake/pydevlake/testing/testing.py
@@ -15,11 +15,13 @@
import pytest
-from typing import Union, Type, Iterable, Generator
+from typing import Union, Type, Iterable, Generator, Optional
from pydevlake.context import Context
from pydevlake.plugin import Plugin
-from pydevlake.model import DomainModel
+from pydevlake.message import RemoteScopeGroup, PipelineTask
+from pydevlake.model import DomainModel, Connection, DomainScope, ToolModel,
ToolScope, TransformationRule
+from pydevlake.stream import DomainType, Stream
class ContextBuilder:
@@ -52,7 +54,7 @@ class ContextBuilder:
)
-def assert_convert(plugin: Union[Plugin, Type[Plugin]], stream_name: str,
+def assert_stream_convert(plugin: Union[Plugin, Type[Plugin]], stream_name:
str,
raw: dict, expected: Union[DomainModel,
Iterable[DomainModel]],
ctx=None):
if isinstance(plugin, type):
@@ -66,3 +68,126 @@ def assert_convert(plugin: Union[Plugin, Type[Plugin]],
stream_name: str,
domain_models = [domain_models]
for res, exp in zip(domain_models, expected):
assert res == exp
+
+
+def assert_stream_run(stream: Stream, connection: Connection, scope:
ToolScope, transformation_rule: Optional[TransformationRule] = None):
+ """
+ Test that a stream can run all 3 steps without error.
+ """
+ ctx = Context(db_url='sqlite:///:memory:', connection=connection,
scope=scope, transformation_rule=transformation_rule)
+ stream.collector.run(ctx)
+ stream.extractor.run(ctx)
+ stream.convertor.run(ctx)
+
+
+def assert_valid_name(plugin: Plugin):
+ name = plugin.name
+ assert isinstance(name, str), 'name must be a string'
+ assert name.isalnum(), 'name must be alphanumeric'
+
+
+def assert_valid_description(plugin: Plugin):
+ name = plugin.description
+ assert isinstance(name, str), 'description must be a string'
+
+
+def assert_valid_connection_type(plugin: Plugin):
+ connection_type = plugin.connection_type
+ assert issubclass(connection_type, Connection), 'connection_type must be a
subclass of Connection'
+
+
+def assert_valid_tool_scope_type(plugin: Plugin):
+ tool_scope_type = plugin.tool_scope_type
+ assert issubclass(tool_scope_type, ToolScope), 'tool_scope_type must be a
subclass of ToolScope'
+
+
+def assert_valid_transformation_rule_type(plugin: Plugin):
+ transformation_rule_type = plugin.transformation_rule_type
+ assert issubclass(transformation_rule_type, TransformationRule),
'transformation_rule_type must be a subclass of TransformationRule'
+
+
+def assert_valid_streams(plugin: Plugin):
+ streams = plugin.streams
+ assert isinstance(streams, list), 'streams must be a list'
+ assert len(streams) > 0, 'this plugin has no stream'
+ for stream in streams:
+ if isinstance(stream, type):
+ stream = stream(plugin.name)
+ assert isinstance(stream, Stream), 'stream must be a stream class or
instance'
+ assert_valid_stream(stream)
+
+
+def assert_valid_stream(stream: Stream):
+ assert isinstance(stream.name, str), 'name must be a string'
+ assert issubclass(stream.tool_model, ToolModel), 'tool_model must be a
subclass of ToolModel'
+ domain_types = stream.domain_types
+ assert len(domain_types) > 0, 'stream must have at least one domain type'
+ for domain_type in domain_types:
+ assert isinstance(domain_type, DomainType), 'domain type must be a
DomainType'
+
+
+def assert_valid_connection(plugin: Plugin, connection: Connection):
+ try:
+ plugin.test_connection(connection)
+ except Exception as e:
+ pytest.fail(f'Connection is not valid: {e}')
+
+
+def assert_valid_domain_scopes(plugin: Plugin, tool_scope: ToolScope) ->
list[DomainScope]:
+ domain_scopes = list(plugin.domain_scopes(tool_scope))
+ assert len(domain_scopes) > 0, 'No domain scope generated for given tool
scope'
+ for domain_scope in domain_scopes:
+ assert isinstance(domain_scope, DomainScope), 'Domain scope must be a
DomainScope'
+ return domain_scopes
+
+
+def assert_valid_remote_scope_groups(plugin: Plugin, connection: Connection)
-> list[RemoteScopeGroup]:
+ scope_groups = list(plugin.remote_scope_groups(connection))
+ assert len(scope_groups) > 0, 'This connection has no scope groups'
+ for scope_group in scope_groups:
+ assert isinstance(scope_group, RemoteScopeGroup), 'Scope group must be
a RemoteScopeGroup'
+ assert scope_group.id is not None, 'Scope group id must not be None'
+ assert bool(scope_group.name), 'Scope group name must not be empty'
+ return scope_groups
+
+
+def assert_valid_remote_scopes(plugin: Plugin, connection: Connection,
group_id: str) -> list[ToolScope]:
+ tool_scopes = list(plugin.remote_scopes(connection, group_id))
+ assert len(tool_scopes) > 0, 'This connection has no scopes'
+ for tool_scope in tool_scopes:
+ assert isinstance(tool_scope, ToolScope), 'Remote scope must be a
ToolScope'
+ return tool_scopes
+
+
+def assert_valid_pipeline_plan(plugin: Plugin, connection: Connection,
tool_scope: ToolScope, transformation_rule: Optional[TransformationRule] =
None) -> list[list[PipelineTask]]:
+ plan = plugin.make_pipeline_plan(
+ [(tool_scope, transformation_rule)],
+ [domain_type.value for domain_type in DomainType],
+ connection
+ )
+ assert len(plan) > 0, 'Pipeline plan has no stage'
+ for stage in plan:
+ assert len(stage) > 0, 'Pipeline stage has no task'
+ return plan
+
+
+def assert_valid_plugin(plugin: Plugin):
+ assert_valid_name(plugin)
+ assert_valid_description(plugin)
+ assert_valid_connection_type(plugin)
+ assert_valid_tool_scope_type(plugin)
+ assert_valid_transformation_rule_type(plugin)
+ assert_valid_streams(plugin)
+
+
+def assert_plugin_run(plugin: Plugin, connection: Connection,
transformation_rule: Optional[TransformationRule] = None):
+ assert_valid_plugin(plugin)
+ assert_valid_connection(plugin, connection)
+ groups = assert_valid_remote_scope_groups(plugin, connection)
+ scope = assert_valid_remote_scopes(plugin, connection, groups[0].id)[0]
+ assert_valid_domain_scopes(plugin, scope)
+ assert_valid_pipeline_plan(plugin, connection, scope, transformation_rule)
+ for stream in plugin.streams:
+ if isinstance(stream, type):
+ stream = stream(plugin.name)
+ assert_stream_run(stream, connection, scope, transformation_rule)
diff --git a/backend/server/services/remote/plugin/remote_scope_api.go
b/backend/server/services/remote/plugin/remote_scope_api.go
index 0e5485739..3ae77a907 100644
--- a/backend/server/services/remote/plugin/remote_scope_api.go
+++ b/backend/server/services/remote/plugin/remote_scope_api.go
@@ -31,10 +31,11 @@ type RemoteScopesOutput struct {
}
type RemoteScopesTreeNode struct {
- Type string `json:"type"`
- Id string `json:"id"`
- Name string `json:"name"`
- Data interface{} `json:"data"`
+ Type string `json:"type"`
+ ParentId string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
}
func (pa *pluginAPI) GetRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
diff --git a/backend/test/integration/helper/models.go
b/backend/test/integration/helper/models.go
index 29d2f2840..37da37282 100644
--- a/backend/test/integration/helper/models.go
+++ b/backend/test/integration/helper/models.go
@@ -18,9 +18,10 @@ limitations under the License.
package helper
import (
+ "time"
+
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "time"
)
type (
@@ -50,7 +51,7 @@ type BlueprintV2Config struct {
}
type RemoteScopesChild struct {
Type string `json:"type"`
- ParentId *string `json:"parentId"`
+ ParentId string `json:"parentId"`
Id string `json:"id"`
Name string `json:"name"`
Data interface{} `json:"data"`
diff --git a/backend/test/integration/remote/python_plugin_test.go
b/backend/test/integration/remote/python_plugin_test.go
index 67bf76140..077005c32 100644
--- a/backend/test/integration/remote/python_plugin_test.go
+++ b/backend/test/integration/remote/python_plugin_test.go
@@ -50,6 +50,7 @@ func TestRemoteScopeGroups(t *testing.T) {
scope := scopeGroups[0]
require.Equal(t, "Group 1", scope.Name)
require.Equal(t, "group1", scope.Id)
+ require.Equal(t, "", scope.ParentId)
require.Equal(t, "group", scope.Type)
}
@@ -68,6 +69,7 @@ func TestRemoteScopes(t *testing.T) {
scope := scopes[0]
require.Equal(t, "Project 1", scope.Name)
require.Equal(t, "p1", scope.Id)
+ require.Equal(t, "group1", scope.ParentId)
require.Equal(t, "scope", scope.Type)
}
