This is an automated email from the ASF dual-hosted git repository.
lynwee pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new b25ff1355 Update AzureDevops plugin (#6548)
b25ff1355 is described below
commit b25ff1355de4c003101cae02be95c63ab9870cbe
Author: Lynwee <[email protected]>
AuthorDate: Fri Dec 8 09:58:17 2023 +0800
Update AzureDevops plugin (#6548)
* feat(azuredevops): update date related fields and add queued_duration_sec
* feat(azuredevops): add original_status and original_result field
* feat(azuredevops): update transformation rules for status and result
* feat(azuredevops): add new test connection api
* fix(azuredevops): fix new test connection api
* fix(azuredevops): fix test
]
* refactor(remote): remove debug codes
* fix: e2e test
* fix: e2e for postges
* fix(azure): fix tests
---------
Co-authored-by: Klesh Wong <[email protected]>
---
.../plugins/azuredevops/azuredevops/migrations.py | 7 ++++
.../plugins/azuredevops/azuredevops/models.py | 16 ++++++++-
.../azuredevops/azuredevops/streams/builds.py | 30 +++++++++-------
.../azuredevops/azuredevops/streams/jobs.py | 17 +++++----
.../plugins/azuredevops/tests/streams_test.py | 25 ++++++++-----
.../pydevlake/pydevlake/domain_layer/devops.py | 42 +++++++++++++++-------
.../services/remote/plugin/connection_api.go | 39 ++++++++++++++++++++
.../server/services/remote/plugin/default_api.go | 3 ++
8 files changed, 138 insertions(+), 41 deletions(-)
diff --git a/backend/python/plugins/azuredevops/azuredevops/migrations.py
b/backend/python/plugins/azuredevops/azuredevops/migrations.py
index 4681f8d90..fd3b0d58e 100644
--- a/backend/python/plugins/azuredevops/azuredevops/migrations.py
+++ b/backend/python/plugins/azuredevops/azuredevops/migrations.py
@@ -185,3 +185,10 @@ def
add_missing_field_in_tool_azuredevops_gitrepositoryconfigs(b: MigrationScrip
@migration(20231013130201, name="add missing field in
_tool_azuredevops_gitrepositories")
def add_missing_field_in_tool_azuredevops_gitrepositories(b:
MigrationScriptBuilder):
b.add_column('_tool_azuredevops_gitrepositories', 'scope_config_id',
'bigint')
+
+
+@migration(20231130163000, name="add queue_time field in
_tool_azuredevops_builds")
+def add_queue_time_field_in_tool_azuredevops_builds(b: MigrationScriptBuilder):
+ table = '_tool_azuredevops_builds'
+ b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time timestamptz',
Dialect.POSTGRESQL)
+ b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time datetime',
Dialect.MYSQL)
diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py
b/backend/python/plugins/azuredevops/azuredevops/models.py
index 437b36301..cbddd9d1d 100644
--- a/backend/python/plugins/azuredevops/azuredevops/models.py
+++ b/backend/python/plugins/azuredevops/azuredevops/models.py
@@ -19,12 +19,13 @@ from enum import Enum
from typing import Optional
from pydantic import SecretStr
+
from pydevlake import ScopeConfig, Field
from pydevlake.model import ToolScope, ToolModel, Connection
from pydevlake.pipeline_tasks import RefDiffOptions
# needed to be able to run migrations
-import azuredevops.migrations
+from azuredevops.migrations import *
class AzureDevOpsConnection(Connection):
@@ -91,6 +92,9 @@ class Build(ToolModel, table=True):
NotStarted = "notStarted"
Postponed = "postponed"
+ def __str__(self) -> str:
+ return self.name
+
class BuildResult(Enum):
Canceled = "canceled"
Failed = "failed"
@@ -98,8 +102,12 @@ class Build(ToolModel, table=True):
PartiallySucceeded = "partiallySucceeded"
Succeeded = "succeeded"
+ def __str__(self) -> str:
+ return self.name
+
id: int = Field(primary_key=True)
name: str = Field(source='/definition/name')
+ queue_time: Optional[datetime.datetime] = Field(source='/queueTime')
start_time: Optional[datetime.datetime]
finish_time: Optional[datetime.datetime]
status: BuildStatus
@@ -114,6 +122,9 @@ class Job(ToolModel, table=True):
InProgress = "inProgress"
Pending = "pending"
+ def __str__(self) -> str:
+ return self.name
+
class JobResult(Enum):
Abandoned = "abandoned"
Canceled = "canceled"
@@ -122,6 +133,9 @@ class Job(ToolModel, table=True):
Succeeded = "succeeded"
SucceededWithIssues = "succeededWithIssues"
+ def __str__(self) -> str:
+ return self.name
+
id: str = Field(primary_key=True)
build_id: str = Field(primary_key=True)
name: str
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
b/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
index 9b6deda78..aba07c90d 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
@@ -15,11 +15,11 @@
from typing import Iterable
+import pydevlake.domain_layer.devops as devops
from azuredevops.api import AzureDevOpsAPI
-from azuredevops.models import GitRepository
from azuredevops.models import Build
+from azuredevops.models import GitRepository
from pydevlake import Context, DomainType, Stream
-import pydevlake.domain_layer.devops as devops
class Builds(Stream):
@@ -37,26 +37,26 @@ class Builds(Stream):
if not b.start_time:
return
- result = None
+ result = devops.CICDResult.RESULT_DEFAULT
if b.result == Build.BuildResult.Canceled:
- result = devops.CICDResult.ABORT
+ result = devops.CICDResult.FAILURE
elif b.result == Build.BuildResult.Failed:
result = devops.CICDResult.FAILURE
elif b.result == Build.BuildResult.PartiallySucceeded:
- result = devops.CICDResult.SUCCESS
- elif b.result == Build.BuildResult.Succeeded:
+ result = devops.CICDResult.FAILURE
+ elif b.result == Build.BuildResult.Succeeded:
result = devops.CICDResult.SUCCESS
- status = None
+ status = devops.CICDStatus.STATUS_OTHER
if b.status == Build.BuildStatus.Cancelling:
status = devops.CICDStatus.DONE
elif b.status == Build.BuildStatus.Completed:
status = devops.CICDStatus.DONE
- elif b.status == Build.BuildStatus.InProgress:
+ elif b.status == Build.BuildStatus.InProgress:
status = devops.CICDStatus.IN_PROGRESS
elif b.status == Build.BuildStatus.NotStarted:
status = devops.CICDStatus.IN_PROGRESS
- elif b.status == Build.BuildStatus.Postponed:
+ elif b.status == Build.BuildStatus.Postponed:
status = devops.CICDStatus.IN_PROGRESS
type = devops.CICDType.BUILD
@@ -67,16 +67,20 @@ class Builds(Stream):
environment = devops.CICDEnvironment.PRODUCTION
if b.finish_time:
- duration_sec = abs(b.finish_time.second - b.start_time.second)
+ duration_sec = abs(b.finish_time.timestamp() -
b.start_time.timestamp())
else:
- duration_sec = 0
+ duration_sec = float(0.0)
yield devops.CICDPipeline(
name=b.name,
status=status,
- created_date=b.start_time,
- finished_date=b.finish_time,
result=result,
+ original_status=str(b.status),
+ original_result=str(b.result),
+ created_date=b.queue_time,
+ queued_date=b.queue_time,
+ started_date=b.start_time,
+ finished_date=b.finish_time,
duration_sec=duration_sec,
environment=environment,
type=type,
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
b/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
index cc1c1bbe4..06a6e481a 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/jobs.py
@@ -56,21 +56,21 @@ class Jobs(Substream):
if not j.start_time:
return
- result = None
+ result = devops.CICDResult.RESULT_DEFAULT
if j.result == Job.JobResult.Abandoned:
- result = devops.CICDResult.ABORT
+ result = devops.CICDResult.RESULT_DEFAULT
elif j.result == Job.JobResult.Canceled:
- result = devops.CICDResult.ABORT
+ result = devops.CICDResult.FAILURE
elif j.result == Job.JobResult.Failed:
result = devops.CICDResult.FAILURE
elif j.result == Job.JobResult.Skipped:
- result = devops.CICDResult.ABORT
+ result = devops.CICDResult.RESULT_DEFAULT
elif j.result == Job.JobResult.Succeeded:
result = devops.CICDResult.SUCCESS
elif j.result == Job.JobResult.SucceededWithIssues:
result = devops.CICDResult.FAILURE
- status = None
+ status = devops.CICDStatus.STATUS_OTHER
if j.state == Job.JobState.Completed:
status = devops.CICDStatus.DONE
elif j.state == Job.JobState.InProgress:
@@ -86,16 +86,19 @@ class Jobs(Substream):
environment = devops.CICDEnvironment.PRODUCTION
if j.finish_time:
- duration_sec = abs(j.finish_time.second-j.start_time.second)
+ duration_sec =
abs(j.finish_time.timestamp()-j.start_time.timestamp())
else:
- duration_sec = 0
+ duration_sec = float(0.0)
yield devops.CICDTask(
id=j.id,
name=j.name,
pipeline_id=j.build_id,
status=status,
+ original_status = str(j.state),
+ original_result = str(j.result),
created_date=j.start_time,
+ started_date =j.start_time,
finished_date=j.finish_time,
result=result,
type=type,
diff --git a/backend/python/plugins/azuredevops/tests/streams_test.py
b/backend/python/plugins/azuredevops/tests/streams_test.py
index da5889ee9..97b963a7d 100644
--- a/backend/python/plugins/azuredevops/tests/streams_test.py
+++ b/backend/python/plugins/azuredevops/tests/streams_test.py
@@ -15,11 +15,10 @@
import pytest
-from pydevlake.testing import assert_stream_convert, ContextBuilder
import pydevlake.domain_layer.code as code
import pydevlake.domain_layer.devops as devops
-
from azuredevops.main import AzureDevOpsPlugin
+from pydevlake.testing import assert_stream_convert, ContextBuilder
@pytest.fixture
@@ -28,11 +27,12 @@ def context():
ContextBuilder(AzureDevOpsPlugin())
.with_connection(token='token')
.with_scope_config(deployment_pattern='deploy',
- production_pattern='prod')
+ production_pattern='prod')
.with_scope('johndoe/test-repo',
url='https://github.com/johndoe/test-repo')
.build()
)
+
def test_builds_stream(context):
raw = {
'properties': {},
@@ -127,10 +127,14 @@ def test_builds_stream(context):
devops.CICDPipeline(
name='deploy_to_prod',
status=devops.CICDStatus.DONE,
- created_date='2023-02-25T06:22:32.8097789Z',
+ created_date='2023-02-25T06:22:21.2237625Z',
+ queued_date='2023-02-25T06:22:21.2237625Z',
+ started_date='2023-02-25T06:22:32.8097789Z',
finished_date='2023-02-25T06:23:04.0061884Z',
result=devops.CICDResult.SUCCESS,
- duration_sec=28,
+ original_status='Completed',
+ original_result='Succeeded',
+ duration_sec=31.196409940719604,
environment=devops.CICDEnvironment.PRODUCTION,
type=devops.CICDType.DEPLOYMENT,
cicd_scope_id=context.scope.domain_id()
@@ -186,11 +190,14 @@ def test_jobs_stream(context):
name='deploy production',
pipeline_id='azuredevops:Build:1:12',
status=devops.CICDStatus.DONE,
+ original_status='Completed',
+ original_result='Succeeded',
created_date='2023-02-25T06:22:36.8066667Z',
+ started_date='2023-02-25T06:22:36.8066667Z',
finished_date='2023-02-25T06:22:43.2333333Z',
result=devops.CICDResult.SUCCESS,
type=devops.CICDType.DEPLOYMENT,
- duration_sec=7,
+ duration_sec=6.426667213439941,
environment=devops.CICDEnvironment.PRODUCTION,
cicd_scope_id=context.scope.domain_id()
)
@@ -255,7 +262,8 @@ def test_pull_requests_stream(context):
'isFlagged': False,
'displayName': 'John Doe',
'url':
'https://spsprodcus5.vssps.visualstudio.com/A1def512a-251e-4668-9a5d-a4bc1f0da4aa/_apis/Identities/bc538feb-9fdd-6cf8-80e1-7c56950d0289',
- '_links': {'avatar': {'href':
'https://dev.azure.com/johndoe/_apis/GraphProfile/MemberAvatars/aad.YmM1MzhmZWItOWZkZC03Y2Y4LT3wXTXtN2M1Njk1MGQwMjg5'}},
+ '_links': {'avatar': {
+ 'href':
'https://dev.azure.com/johndoe/_apis/GraphProfile/MemberAvatars/aad.YmM1MzhmZWItOWZkZC03Y2Y4LT3wXTXtN2M1Njk1MGQwMjg5'}},
'id': 'bc538feb-9fdd-6cf8-80e1-7c56950d0289',
'uniqueName': '[email protected]',
'imageUrl':
'https://dev.azure.com/johndoe/_api/_common/identityImage?id=bc538feb-9fdd-6cf8-80e1-7c56950d0289'
@@ -313,7 +321,8 @@ def test_pull_request_commits_stream():
},
'comment': 'Fixed main.java',
'url':
'https://dev.azure.com/johndoe/7a3fd40e-2aed-4fac-bac9-511bf1a70206/_apis/git/repositories/0d50ba13-f9ad-49b0-9b21-d29eda50ca33/commits/85ede91717145a1e6e2bdab4cab689ac8f2fa3a2',
- 'pull_request_id': "azuredevops:gitpullrequest:1:12345" # This is not
part of the API response, but is added in collect method
+ 'pull_request_id': "azuredevops:gitpullrequest:1:12345"
+ # This is not part of the API response, but is added in collect method
}
expected = code.PullRequestCommit(
diff --git a/backend/python/pydevlake/pydevlake/domain_layer/devops.py
b/backend/python/pydevlake/pydevlake/domain_layer/devops.py
index b47bfd922..3ac97fedd 100644
--- a/backend/python/pydevlake/pydevlake/domain_layer/devops.py
+++ b/backend/python/pydevlake/pydevlake/domain_layer/devops.py
@@ -14,25 +14,23 @@
# limitations under the License.
-from typing import Optional
from datetime import datetime
from enum import Enum
-
-from sqlmodel import Field
+from typing import Optional
from pydevlake.model import DomainModel, NoPKModel, DomainScope
+from sqlmodel import Field
class CICDResult(Enum):
SUCCESS = "SUCCESS"
FAILURE = "FAILURE"
- ABORT = "ABORT"
- MANUAL = "MANUAL"
-
+ RESULT_DEFAULT = ""
class CICDStatus(Enum):
IN_PROGRESS = "IN_PROGRESS"
DONE = "DONE"
+ STATUS_OTHER = "OTHER"
class CICDType(Enum):
@@ -50,15 +48,25 @@ class CICDEnvironment(Enum):
class CICDPipeline(DomainModel, table=True):
__tablename__ = 'cicd_pipelines'
+
name: str
+ cicd_scope_id: Optional[str]
+
status: Optional[CICDStatus]
+ result: Optional[CICDResult]
+ original_status: Optional[str]
+ original_result: Optional[str]
+
created_date: Optional[datetime]
+ started_date: Optional[datetime]
+ queued_date: Optional[datetime]
finished_date: Optional[datetime]
- result: Optional[CICDResult]
- duration_sec: Optional[int]
- environment: Optional[str]
+
+ duration_sec: Optional[float]
+ queued_duration_sec: Optional[float]
+
type: Optional[CICDType]
- cicd_scope_id: Optional[str]
+ environment: Optional[str]
class CiCDPipelineCommit(NoPKModel, table=True):
@@ -81,13 +89,23 @@ class CicdScope(DomainScope):
class CICDTask(DomainModel, table=True):
__tablename__ = 'cicd_tasks'
+
name: str
pipeline_id: str
+ cicd_scope_id: str
+
result: Optional[CICDResult]
status: Optional[CICDStatus]
+ original_status: Optional[str]
+ original_result: Optional[str]
+
type: Optional[CICDType]
environment: Optional[CICDEnvironment]
- duration_sec: int
+
+ created_date: Optional[datetime]
+ queued_date: Optional[datetime]
started_date: Optional[datetime]
finished_date: Optional[datetime]
- cicd_scope_id: str
+
+ duration_sec: float
+ queued_duration_sec: Optional[float]
diff --git a/backend/server/services/remote/plugin/connection_api.go
b/backend/server/services/remote/plugin/connection_api.go
index 552c7f64a..38c832ee7 100644
--- a/backend/server/services/remote/plugin/connection_api.go
+++ b/backend/server/services/remote/plugin/connection_api.go
@@ -20,6 +20,8 @@ package plugin
import (
"encoding/json"
"fmt"
+ "github.com/spf13/cast"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/server/api/shared"
@@ -79,6 +81,43 @@ func (pa *pluginAPI) TestConnection(input
*plugin.ApiResourceInput) (*plugin.Api
}
}
+func (pa *pluginAPI) TestExistingConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ connection := pa.connType.New()
+ err := pa.connhelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+ conn := connection.Unwrap()
+ params := make(map[string]interface{})
+ if data, err := json.Marshal(conn); err != nil {
+ return nil, errors.Convert(err)
+ } else {
+ if err := json.Unmarshal(data, ¶ms); err != nil {
+ return nil, errors.Convert(err)
+ }
+ }
+
+ necessaryParams := make(map[string]string)
+ necessaryParams["proxy"] = cast.ToString(params["proxy"])
+ necessaryParams["token"] = cast.ToString(params["token"])
+
+ var result TestConnectionResult
+ rpcCallErr := pa.invoker.Call("test-connection", bridge.DefaultContext,
necessaryParams).Get(&result)
+ if rpcCallErr != nil {
+ body := shared.ApiBody{
+ Success: false,
+ Message: fmt.Sprintf("Error while testing connection:
%s", rpcCallErr.Error()),
+ }
+ return &plugin.ApiResourceOutput{Body: body, Status: 500}, nil
+ } else {
+ body := shared.ApiBody{
+ Success: result.Success,
+ Message: result.Message,
+ }
+ return &plugin.ApiResourceOutput{Body: body, Status:
result.Status}, nil
+ }
+}
+
func (pa *pluginAPI) PostConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
connection := pa.connType.New()
err := pa.connhelper.Create(connection, input)
diff --git a/backend/server/services/remote/plugin/default_api.go
b/backend/server/services/remote/plugin/default_api.go
index b0c2c6751..2a6c570ca 100644
--- a/backend/server/services/remote/plugin/default_api.go
+++ b/backend/server/services/remote/plugin/default_api.go
@@ -61,6 +61,9 @@ func GetDefaultAPI(
"PATCH": papi.PatchConnection,
"DELETE": papi.DeleteConnection,
},
+ "connections/:connectionId/test": {
+ "POST": papi.TestExistingConnection,
+ },
"connections/:connectionId/scopes": {
"PUT": papi.PutScope,
"GET": papi.ListScopes,
