This is an automated email from the ASF dual-hosted git repository.
lynwee pushed a commit to branch release-v0.21
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.21 by this push:
new ca32a17eb cherry pick #7001 fix(azuredevops): fix some fields' value
to v0.21 (#7003)
ca32a17eb is described below
commit ca32a17ebd1c28dd11433a9a151917e28e510975
Author: Lynwee <[email protected]>
AuthorDate: Fri Feb 23 21:53:08 2024 +0800
cherry pick #7001 fix(azuredevops): fix some fields' value to v0.21 (#7003)
* fix(azuredevops): fix some fields' value
* fix(azuredevops): fix updated_date in cicd_scopes
* fix(azuredevops): fix test
* fix(azuredevops): fix e2e test errors
---
backend/core/models/domainlayer/code/repo.go | 10 +++++-----
backend/core/plugin/hub.go | 2 ++
backend/python/plugins/azuredevops/azuredevops/main.py | 11 ++++++-----
backend/python/plugins/azuredevops/azuredevops/migrations.py | 7 +++++++
backend/python/plugins/azuredevops/azuredevops/models.py | 3 ++-
.../plugins/azuredevops/azuredevops/streams/pull_requests.py | 4 ++--
backend/python/plugins/azuredevops/tests/streams_test.py | 2 +-
backend/python/pydevlake/pydevlake/plugin.py | 2 +-
8 files changed, 26 insertions(+), 15 deletions(-)
diff --git a/backend/core/models/domainlayer/code/repo.go
b/backend/core/models/domainlayer/code/repo.go
index 90469c1cb..f6ad9faf9 100644
--- a/backend/core/models/domainlayer/code/repo.go
+++ b/backend/core/models/domainlayer/code/repo.go
@@ -19,7 +19,7 @@ package code
import (
"github.com/apache/incubator-devlake/core/models/domainlayer"
- plugin "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/plugin"
"time"
)
@@ -30,11 +30,11 @@ type Repo struct {
Name string `json:"name"`
Url string `json:"url"`
Description string `json:"description"`
- OwnerId string `json:"ownerId" gorm:"type:varchar(255)"`
+ OwnerId string `json:"owner_id" gorm:"type:varchar(255)"`
Language string `json:"language" gorm:"type:varchar(255)"`
- ForkedFrom string `json:"forkedFrom"`
- CreatedDate *time.Time `json:"createdDate"`
- UpdatedDate *time.Time `json:"updatedDate"`
+ ForkedFrom string `json:"forked_from"`
+ CreatedDate *time.Time `json:"created_date"`
+ UpdatedDate *time.Time `json:"updated_date"`
Deleted bool `json:"deleted"`
}
diff --git a/backend/core/plugin/hub.go b/backend/core/plugin/hub.go
index 911bfb6ba..6788c9057 100644
--- a/backend/core/plugin/hub.go
+++ b/backend/core/plugin/hub.go
@@ -79,6 +79,8 @@ func InitPlugins(basicRes context.BasicRes) {
if err != nil {
panic(fmt.Errorf("failed to initialize plugin
%v due to %w", pluginName, err))
}
+ } else {
+ basicRes.GetLogger().Info("plugin: %s doesn't implement
'PluginInit', it will be skipped.", pluginName)
}
}
}
diff --git a/backend/python/plugins/azuredevops/azuredevops/main.py
b/backend/python/plugins/azuredevops/azuredevops/main.py
index a58d53591..966aca974 100644
--- a/backend/python/plugins/azuredevops/azuredevops/main.py
+++ b/backend/python/plugins/azuredevops/azuredevops/main.py
@@ -4,7 +4,6 @@
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
-
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
@@ -27,6 +26,7 @@ from pydevlake.domain_layer.code import Repo
from pydevlake.domain_layer.devops import CicdScope
from pydevlake.pipeline_tasks import gitextractor, refdiff
from pydevlake.api import APIException
+from pydevlake.extractor import autoextract
_SUPPORTED_EXTERNAL_SOURCE_PROVIDERS = ['github', 'githubenterprise',
'bitbucket', 'git']
@@ -50,13 +50,14 @@ class AzureDevOpsPlugin(Plugin):
yield Repo(
name=git_repo.name,
url=git_repo.url,
- forked_from=git_repo.parent_repository_url
+ forked_from=git_repo.parent_repository_url,
+ updated_date=git_repo.updated_date,
)
-
yield CicdScope(
name=git_repo.name,
description=git_repo.name,
- url=git_repo.url
+ url=git_repo.url,
+ updatedDate=git_repo.updated_date,
)
def remote_scope_groups(self, connection) -> list[RemoteScopeGroup]:
@@ -88,7 +89,7 @@ class AzureDevOpsPlugin(Plugin):
url = urlparse(raw_repo['remoteUrl'])
url = url._replace(netloc=url.hostname)
raw_repo['url'] = url.geturl()
- repo = GitRepository(**raw_repo)
+ repo = autoextract(raw_repo, GitRepository)
if not repo.default_branch:
continue
if "parentRepository" in raw_repo:
diff --git a/backend/python/plugins/azuredevops/azuredevops/migrations.py
b/backend/python/plugins/azuredevops/azuredevops/migrations.py
index fd3b0d58e..b6924e037 100644
--- a/backend/python/plugins/azuredevops/azuredevops/migrations.py
+++ b/backend/python/plugins/azuredevops/azuredevops/migrations.py
@@ -192,3 +192,10 @@ def add_queue_time_field_in_tool_azuredevops_builds(b:
MigrationScriptBuilder):
table = '_tool_azuredevops_builds'
b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time timestamptz',
Dialect.POSTGRESQL)
b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time datetime',
Dialect.MYSQL)
+
+
+@migration(20240223170000, name="add updated_date field in
_tool_azuredevops_gitrepositories")
+def add_updated_date_field_in_tool_azuredevops_gitrepositories(b:
MigrationScriptBuilder):
+ table = "_tool_azuredevops_gitrepositories"
+ b.execute(f'ALTER TABLE {table} add COLUMN updated_date datetime(3)
DEFAULT NULL', Dialect.MYSQL)
+ b.execute(f'ALTER TABLE {table} add COLUMN updated_date TIMESTAMPTZ
DEFAULT NULL', Dialect.POSTGRESQL)
diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py
b/backend/python/plugins/azuredevops/azuredevops/models.py
index cbddd9d1d..3a295aa7c 100644
--- a/backend/python/plugins/azuredevops/azuredevops/models.py
+++ b/backend/python/plugins/azuredevops/azuredevops/models.py
@@ -45,8 +45,9 @@ class GitRepository(ToolScope, table=True):
default_branch: Optional[str]
project_id: str
org_id: str
- parent_repository_url: Optional[str] = Field(source='parentRepository/url')
+ parent_repository_url: Optional[str] =
Field(source='/parentRepository/url')
provider: Optional[str]
+ updated_date: datetime.datetime = Field(source='/project/lastUpdateTime')
def is_external(self):
return bool(self.provider)
diff --git
a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
index 6922d1b7e..bebb7bed8 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py
@@ -37,7 +37,7 @@ class GitPullRequests(Stream):
def convert(self, pr: GitPullRequest, ctx):
repo_id = ctx.scope.domain_id()
- # If the PR is from a fork, we forge a new repo ID for the base repo
but it doesn't correspond to a real repo
+ # If the PR is from a fork, we forge a new repo ID for the base repo,
but it doesn't correspond to a real repo
base_repo_id = domain_id(GitRepository, ctx.connection.id,
pr.fork_repo_id) if pr.fork_repo_id is not None else repo_id
# Use the same status values as GitHub plugin
@@ -56,7 +56,7 @@ class GitPullRequests(Stream):
original_status=pr.status.value,
title=pr.title,
description=pr.description,
- url=pr.url,
+ url=f"{ctx.scope.url}/pullrequest/{pr.pull_request_id}",
author_name=pr.created_by_name,
author_id=pr.created_by_id,
pull_request_key=pr.pull_request_id,
diff --git a/backend/python/plugins/azuredevops/tests/streams_test.py
b/backend/python/plugins/azuredevops/tests/streams_test.py
index 97b963a7d..110cb3280 100644
--- a/backend/python/plugins/azuredevops/tests/streams_test.py
+++ b/backend/python/plugins/azuredevops/tests/streams_test.py
@@ -287,7 +287,7 @@ def test_pull_requests_stream(context):
original_status='active',
title='ticket-2 PR',
description='Updated main.java by ticket-2',
-
url='https://dev.azure.com/johndoe/7a3fd40e-2aed-4fac-bac9-511bf1a70206/_apis/git/repositories/0d50ba13-f9ad-49b0-9b21-d29eda50ca33/pullRequests/1',
+ url='https://github.com/johndoe/test-repo/pullrequest/1',
author_name='John Doe',
author_id='bc538feb-9fdd-6cf8-80e1-7c56950d0289',
pull_request_key=1,
diff --git a/backend/python/pydevlake/pydevlake/plugin.py
b/backend/python/pydevlake/pydevlake/plugin.py
index fc368261a..e3303300e 100644
--- a/backend/python/pydevlake/pydevlake/plugin.py
+++ b/backend/python/pydevlake/pydevlake/plugin.py
@@ -71,7 +71,7 @@ class Plugin(ABC):
@abstractmethod
def test_connection(self, connection: Connection) ->
msg.TestConnectionResult:
"""
- Test if the the connection with the datasource can be established with
the given connection.
+ Test if the connection with the datasource can be established with the
given connection.
Must raise an exception if the connection can't be established.
"""
pass
