This is an automated email from the ASF dual-hosted git repository.
mappjzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new bddf3e857 fix: fix jenkins stage convertor error (#5003)
bddf3e857 is described below
commit bddf3e857971be9bafdcea71a866598cbc6a0e13
Author: Leric Zhang <[email protected]>
AuthorDate: Wed Apr 26 19:48:39 2023 +0800
fix: fix jenkins stage convertor error (#5003)
* fix: fix jenkins stage convertor error
* all stage env is set to production when ProductionPattern is not set
* stage status is set to IN_PROGRESS when task status is not SUCCESS
* fix: cicd_scopes is empty
* fix: unittest
---
.github/workflows/test-e2e.yml | 6 ++----
.github/workflows/test.yml | 8 +++++---
backend/helpers/pluginhelper/api/enrich_with_regex.go | 5 ++---
backend/plugins/dora/impl/impl_test.go | 19 +++++++++++++++++--
.../e2e/snapshot_tables/cicd_tasks_after_stages.csv | 10 +++++-----
backend/plugins/jenkins/impl/impl.go | 12 +-----------
backend/plugins/jenkins/tasks/stage_convertor.go | 4 ++--
backend/plugins/jenkins/tasks/task_data.go | 2 +-
8 files changed, 35 insertions(+), 31 deletions(-)
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml
index 9e8ed95f7..2f422c7d1 100644
--- a/.github/workflows/test-e2e.yml
+++ b/.github/workflows/test-e2e.yml
@@ -18,6 +18,8 @@
# This action run e2e test for pull requests
name: test-e2e
on:
+ push:
+ branches: [main]
pull_request:
branches: [main]
paths:
@@ -49,8 +51,6 @@ jobs:
uses: actions/checkout@v3
- run: git config --global --add safe.directory $(pwd)
- name: Build Python
- with:
- python-version: 3.9.9
run: |
cd backend
echo "Building Python"
@@ -90,8 +90,6 @@ jobs:
uses: actions/checkout@v3
- run: git config --global --add safe.directory $(pwd)
- name: Build Python
- with:
- python-version: 3.9.9
run: |
cd backend
echo "Building Python"
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 57d9aceb3..9f0df7cee 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -18,6 +18,8 @@
name: unit-test
on:
+ push:
+ branches: [main]
pull_request:
branches: [ main ]
paths:
@@ -43,12 +45,12 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v3
- - run: git config --global --add safe.directory $(pwd)
+ - name: Set git config
+ run: git config --global --add safe.directory $(pwd)
- name: Build Python
- with:
- python-version: 3.9.9
run: |
cd backend
+ echo "Building Python"
make build-python
- name: Cache unit-test
id: cache-unit-test
diff --git a/backend/helpers/pluginhelper/api/enrich_with_regex.go
b/backend/helpers/pluginhelper/api/enrich_with_regex.go
index 1c25c3913..a4d31697e 100644
--- a/backend/helpers/pluginhelper/api/enrich_with_regex.go
+++ b/backend/helpers/pluginhelper/api/enrich_with_regex.go
@@ -22,7 +22,6 @@ import (
"regexp"
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
)
// RegexEnricher process value with regex pattern
@@ -57,8 +56,8 @@ func (r *RegexEnricher) AddRegexp(patterns ...string)
errors.Error {
// lastly, will return corresponding value(result or empty)
// TODO: to be removed
func (r *RegexEnricher) GetEnrichResult(pattern string, v string, result
string) string {
- if result == devops.PRODUCTION && pattern == "" {
- return result
+ if pattern == "" {
+ return ""
}
regex := r.regexpMap[pattern]
if regex != nil {
diff --git a/backend/plugins/dora/impl/impl_test.go
b/backend/plugins/dora/impl/impl_test.go
index 05f79bcc1..7c5d8715a 100644
--- a/backend/plugins/dora/impl/impl_test.go
+++ b/backend/plugins/dora/impl/impl_test.go
@@ -19,9 +19,10 @@ package impl
import (
"encoding/json"
+ "testing"
+
"github.com/apache/incubator-devlake/core/plugin"
"github.com/stretchr/testify/assert"
- "testing"
)
func TestMakeMetricPluginPipelinePlanV200(t *testing.T) {
@@ -36,16 +37,30 @@ func TestMakeMetricPluginPipelinePlanV200(t *testing.T) {
assert.Nil(t, err)
plan, err := dora.MakeMetricPluginPipelinePlanV200(projectName,
optionJson)
doraOutputPlan := plugin.PipelinePlan{
+ plugin.PipelineStage{
+ {
+ Plugin: "dora",
+ Subtasks: []string{
+ "generateDeploymentCommits",
+ "enrichPrevSuccessDeploymentCommits",
+ },
+ Options: map[string]interface{}{"projectName":
projectName},
+ },
+ },
plugin.PipelineStage{
{
Plugin: "refdiff",
- Subtasks:
[]string{"calculateProjectDeploymentCommitsDiff"},
+ Subtasks:
[]string{"calculateDeploymentCommitsDiff"},
Options: map[string]interface{}{"projectName":
projectName},
},
},
plugin.PipelineStage{
{
Plugin: "dora",
+ Subtasks: []string{
+ "calculateChangeLeadTime",
+ "ConnectIncidentToDeployment",
+ },
Options: map[string]interface{}{"projectName":
projectName},
},
},
diff --git
a/backend/plugins/jenkins/e2e/snapshot_tables/cicd_tasks_after_stages.csv
b/backend/plugins/jenkins/e2e/snapshot_tables/cicd_tasks_after_stages.csv
index c9015fb4a..97c4e2f1f 100644
--- a/backend/plugins/jenkins/e2e/snapshot_tables/cicd_tasks_after_stages.csv
+++ b/backend/plugins/jenkins/e2e/snapshot_tables/cicd_tasks_after_stages.csv
@@ -1,8 +1,8 @@
id,name,pipeline_id,result,status,type,duration_sec,started_date,finished_date,environment,cicd_scope_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#10:8,scp-f/b,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#10,,IN_PROGRESS,,0,2019-10-29T04:01:34.000+00:00,2019-10-29T04:01:34.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13577,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#10:8,scp-f/b,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#10,FAILURE,DONE,,0,2019-10-29T04:01:34.000+00:00,2019-10-29T04:01:34.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13577,
jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#11:9,gitlabInit,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#11,,IN_PROGRESS,,1312,2021-03-09T13:35:10.000+00:00,2021-03-09T13:57:02.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13578,
jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#12:1,gitlabAutoSync,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#12,SUCCESS,DONE,DEPLOYMENT,14,2020-02-07T11:54:28.000+00:00,2020-02-07T11:54:42.000+00:00,PRODUCTION,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins
[...]
-jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#13:2,gitlabInit,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#13,,IN_PROGRESS,,5,2020-03-18T02:19:17.000+00:00,2020-03-18T02:19:22.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13580,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#13:2,gitlabInit,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#13,ABORT,DONE,,5,2020-03-18T02:19:17.000+00:00,2020-03-18T02:19:22.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13580,
jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#14:3,gitlabAutoSync,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#14,SUCCESS,DONE,DEPLOYMENT,83,2020-03-12T02:45:25.000+00:00,2020-03-12T02:46:48.000+00:00,PRODUCTION,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins
[...]
jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#1:6,Hello,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#1,SUCCESS,DONE,,0,2022-09-08T15:40:34.000+00:00,2022-09-08T15:40:34.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,1,
jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#2:7,Hello,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#2,SUCCESS,DONE,,0,2022-09-08T15:40:49.000+00:00,2022-09-08T15:40:49.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,2,
@@ -10,6 +10,6 @@
jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/de
jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#4:9,Hello,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#4,SUCCESS,DONE,,0,2022-09-08T15:40:17.000+00:00,2022-09-08T15:40:17.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,4,
jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#5:10,Hello,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#5,SUCCESS,DONE,,0,2022-09-08T15:40:17.000+00:00,2022-09-08T15:40:17.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,5,
jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#6:11,Hello,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#6,SUCCESS,DONE,,0,2022-09-08T15:40:34.000+00:00,2022-09-08T15:40:34.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,6,
-jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#7:12,gitlabInit,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#7,,IN_PROGRESS,,0,2020-03-04T13:47:24.000+00:00,2020-03-04T13:47:24.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13574,
-jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#8:6,gitlabInit,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#8,,IN_PROGRESS,,215,2020-03-17T15:27:15.000+00:00,2020-03-17T15:30:50.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13575,
-jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#9:7,gitlabInit,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#9,,IN_PROGRESS,,5,2020-03-18T02:19:17.000+00:00,2020-03-18T02:19:22.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13576,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#7:12,gitlabInit,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#7,FAILURE,DONE,,0,2020-03-04T13:47:24.000+00:00,2020-03-04T13:47:24.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13574,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#8:6,gitlabInit,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#8,FAILURE,DONE,,215,2020-03-17T15:27:15.000+00:00,2020-03-17T15:30:50.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13575,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#9:7,gitlabInit,jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#9,ABORT,DONE,,5,2020-03-18T02:19:17.000+00:00,2020-03-18T02:19:22.000+00:00,,jenkins:JenkinsJob:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13576,
diff --git a/backend/plugins/jenkins/impl/impl.go
b/backend/plugins/jenkins/impl/impl.go
index 59941c543..104ba38fe 100644
--- a/backend/plugins/jenkins/impl/impl.go
+++ b/backend/plugins/jenkins/impl/impl.go
@@ -238,17 +238,7 @@ func EnrichOptions(taskCtx plugin.TaskContext,
}
}
- pathSplit := strings.Split(op.JobFullName, "/")
- lastOne := len(pathSplit)
-
- // Multi-level directory assembly
- path := "job/" + strings.Join(pathSplit, "/job/")
- if path == "job/" {
- path = ""
- }
- name := pathSplit[lastOne-1]
-
- err = api.GetJob(apiClient, path, name, op.JobFullName, 100, func(job
*models.Job, isPath bool) errors.Error {
+ err = api.GetJob(apiClient, op.JobPath, op.JobName, op.JobFullName,
100, func(job *models.Job, isPath bool) errors.Error {
log.Debug(fmt.Sprintf("Current job: %s", job.FullName))
op.Name = job.Name
op.JobPath = job.Path
diff --git a/backend/plugins/jenkins/tasks/stage_convertor.go
b/backend/plugins/jenkins/tasks/stage_convertor.go
index ebae9ecf3..1a921b5b8 100644
--- a/backend/plugins/jenkins/tasks/stage_convertor.go
+++ b/backend/plugins/jenkins/tasks/stage_convertor.go
@@ -106,9 +106,9 @@ func ConvertStages(taskCtx plugin.SubTaskContext) (err
errors.Error) {
results := make([]interface{}, 0)
if body.Status == "SUCCESS" {
jenkinsTaskResult = devops.SUCCESS
- } else if body.Result == "FAILED" {
+ } else if body.Status == "FAILED" {
jenkinsTaskResult = devops.FAILURE
- } else if body.Result == "ABORTED" {
+ } else if body.Status == "ABORTED" {
jenkinsTaskResult = devops.ABORT
} else {
jenkinsTaskResult = ""
diff --git a/backend/plugins/jenkins/tasks/task_data.go
b/backend/plugins/jenkins/tasks/task_data.go
index f2bccf514..1635fb504 100644
--- a/backend/plugins/jenkins/tasks/task_data.go
+++ b/backend/plugins/jenkins/tasks/task_data.go
@@ -72,7 +72,7 @@ func ValidateTaskOptions(op *JenkinsOptions)
(*JenkinsOptions, errors.Error) {
op.JobPath = `job/` +
strings.Join(strings.Split(op.JobFullName[:i], `/`), `/job/`)
} else {
op.JobName = op.JobFullName
- op.JobPath = ``
+ op.JobPath = `view/all`
}
if op.JenkinsTransformationRule == nil && op.TransformationRuleId == 0 {
op.JenkinsTransformationRule =
new(models.JenkinsTransformationRule)
