This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v0.19
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.19 by this push:
new 73dda86d1 cherry-pick fix(bamboo): set domain layer's repo_url field
with an acceptable value to v0.19 (#6171)
73dda86d1 is described below
commit 73dda86d17f4f7229107087f8c2e9d4df45d0c2e
Author: Lynwee <[email protected]>
AuthorDate: Thu Sep 28 16:14:52 2023 +0800
cherry-pick fix(bamboo): set domain layer's repo_url field with an
acceptable value to v0.19 (#6171)
* fix(bamboo): set domain layer's repo_url field with an acceptable value
* fix(bamboo): fix e2e test
* fix(bamboo): fix conflicts
* fix(test): fix test case
---
backend/plugins/bamboo/e2e/deploy_build_test.go | 15 ++++++++--
backend/plugins/bamboo/e2e/deploy_test.go | 1 +
backend/plugins/bamboo/e2e/job_build_test.go | 1 +
backend/plugins/bamboo/e2e/job_test.go | 1 +
.../plugins/bamboo/e2e/plan_build_commits_test.go | 1 +
backend/plugins/bamboo/e2e/plan_build_test.go | 1 +
backend/plugins/bamboo/e2e/plan_test.go | 1 +
.../snapshot_tables/cicd_deployment_commits.csv | 16 +++++-----
.../e2e/snapshot_tables/cicd_pipeline_commits.csv | 22 +++++++-------
.../plugins/bamboo/tasks/deploy_build_convertor.go | 7 +++++
.../plugins/bamboo/tasks/plan_commit_convertor.go | 8 ++++-
backend/plugins/bamboo/tasks/shared.go | 12 ++++++++
backend/plugins/bamboo/tasks/shared_test.go | 35 ++++++++++++++++++++++
13 files changed, 98 insertions(+), 23 deletions(-)
diff --git a/backend/plugins/bamboo/e2e/deploy_build_test.go
b/backend/plugins/bamboo/e2e/deploy_build_test.go
index 5db8e1861..b87e9c765 100644
--- a/backend/plugins/bamboo/e2e/deploy_build_test.go
+++ b/backend/plugins/bamboo/e2e/deploy_build_test.go
@@ -18,8 +18,6 @@ limitations under the License.
package e2e
import (
- "testing"
-
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
@@ -27,8 +25,18 @@ import (
"github.com/apache/incubator-devlake/plugins/bamboo/impl"
"github.com/apache/incubator-devlake/plugins/bamboo/models"
"github.com/apache/incubator-devlake/plugins/bamboo/tasks"
+ "testing"
+ "time"
)
+func getFakeAPIClient() *helper.ApiAsyncClient {
+ client := &helper.ApiClient{}
+ client.Setup("http://127.0.0.1:8080/bamboo/";, nil, time.Second)
+ return &helper.ApiAsyncClient{
+ ApiClient: client,
+ }
+}
+
func TestBambooDeployBuildDataFlow(t *testing.T) {
var bamboo impl.Bamboo
dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo)
@@ -42,6 +50,7 @@ func TestBambooDeployBuildDataFlow(t *testing.T) {
},
},
RegexEnricher: helper.NewRegexEnricher(),
+ ApiClient: getFakeAPIClient(),
}
taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT,
taskData.Options.DeploymentPattern)
taskData.RegexEnricher.TryAdd(devops.PRODUCTION,
taskData.Options.ProductionPattern)
@@ -89,6 +98,6 @@ func TestBambooDeployBuildDataFlow(t *testing.T) {
dataflowTester.VerifyTableWithOptions(&devops.CicdDeploymentCommit{},
e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/cicd_deployment_commits.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
- IgnoreFields: []string{"created_date", "started_date",
"finished_date"},
+ IgnoreFields: []string{},
})
}
diff --git a/backend/plugins/bamboo/e2e/deploy_test.go
b/backend/plugins/bamboo/e2e/deploy_test.go
index 62c7efc9b..54bf4a4a7 100644
--- a/backend/plugins/bamboo/e2e/deploy_test.go
+++ b/backend/plugins/bamboo/e2e/deploy_test.go
@@ -35,6 +35,7 @@ func TestBambooDeployDataFlow(t *testing.T) {
PlanKey: "TEST1",
BambooScopeConfig: new(models.BambooScopeConfig),
},
+ ApiClient: getFakeAPIClient(),
}
// import raw data table
diff --git a/backend/plugins/bamboo/e2e/job_build_test.go
b/backend/plugins/bamboo/e2e/job_build_test.go
index 2f69c1963..5db603638 100644
--- a/backend/plugins/bamboo/e2e/job_build_test.go
+++ b/backend/plugins/bamboo/e2e/job_build_test.go
@@ -43,6 +43,7 @@ func TestBambooJobBuildDataFlow(t *testing.T) {
},
},
RegexEnricher: helper.NewRegexEnricher(),
+ ApiClient: getFakeAPIClient(),
}
taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT,
taskData.Options.DeploymentPattern)
// import raw data table
diff --git a/backend/plugins/bamboo/e2e/job_test.go
b/backend/plugins/bamboo/e2e/job_test.go
index 419b062f0..dc89148e7 100644
--- a/backend/plugins/bamboo/e2e/job_test.go
+++ b/backend/plugins/bamboo/e2e/job_test.go
@@ -40,6 +40,7 @@ func TestBambooJobDataFlow(t *testing.T) {
ProductionPattern: "(?i)compile",
},
},
+ ApiClient: getFakeAPIClient(),
}
// import raw data table
// SELECT * FROM _raw_bamboo_api_job INTO OUTFILE
"/tmp/_raw_bamboo_api_jobs.csv" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY
'"' LINES TERMINATED BY '\r\n';
diff --git a/backend/plugins/bamboo/e2e/plan_build_commits_test.go
b/backend/plugins/bamboo/e2e/plan_build_commits_test.go
index f52b894c8..ed8533481 100644
--- a/backend/plugins/bamboo/e2e/plan_build_commits_test.go
+++ b/backend/plugins/bamboo/e2e/plan_build_commits_test.go
@@ -42,6 +42,7 @@ func TestBambooPlanBuildCommitsDataFlow(t *testing.T) {
ProductionPattern: "(?i)compile",
},
},
+ ApiClient: getFakeAPIClient(),
}
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_bamboo_plan_build_commits.csv",
&models.BambooPlanBuildVcsRevision{})
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_bamboo_plan_builds.csv",
&models.BambooPlanBuild{})
diff --git a/backend/plugins/bamboo/e2e/plan_build_test.go
b/backend/plugins/bamboo/e2e/plan_build_test.go
index e83654e80..ca9755d35 100644
--- a/backend/plugins/bamboo/e2e/plan_build_test.go
+++ b/backend/plugins/bamboo/e2e/plan_build_test.go
@@ -44,6 +44,7 @@ func TestBambooPlanBuildDataFlow(t *testing.T) {
},
},
RegexEnricher: helper.NewRegexEnricher(),
+ ApiClient: getFakeAPIClient(),
}
taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT,
taskData.Options.DeploymentPattern)
taskData.RegexEnricher.TryAdd(devops.PRODUCTION,
taskData.Options.ProductionPattern)
diff --git a/backend/plugins/bamboo/e2e/plan_test.go
b/backend/plugins/bamboo/e2e/plan_test.go
index a8757081a..09987c8b1 100644
--- a/backend/plugins/bamboo/e2e/plan_test.go
+++ b/backend/plugins/bamboo/e2e/plan_test.go
@@ -39,6 +39,7 @@ func TestBambooPlanDataFlow(t *testing.T) {
PlanKey: "TEST1",
BambooScopeConfig: new(models.BambooScopeConfig),
},
+ ApiClient: getFakeAPIClient(),
}
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_bamboo_plans.csv",
models.BambooPlan{})
diff --git
a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv
b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv
index 677e0cca3..20182662e 100644
--- a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv
+++ b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv
@@ -1,8 +1,8 @@
-id,commit_sha,cicd_scope_id,cicd_deployment_id,name,result,status,environment,duration_sec,ref_name,repo_id,repo_url,prev_success_deployment_commit_id
-bamboo:BambooDeployBuild:1:1540097,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540097,release-1,FAILURE,DONE,dev,0,,622595,,
-bamboo:BambooDeployBuild:1:1540100,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540100,release-1,FAILURE,DONE,dev,0,,622595,,
-bamboo:BambooDeployBuild:1:1540101,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540101,release-2,FAILURE,DONE,dev,0,,622595,,
-bamboo:BambooDeployBuild:1:1540102,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540102,release-2,,DONE,dev,0,,622595,,
-bamboo:BambooDeployBuild:1:1540105,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540105,release-2,,DONE,dev,0,,622595,,
-bamboo:BambooDeployBuild:1:1540106,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540106,release-2,,DONE,dev,0,,622595,,
-bamboo:BambooDeployBuild:1:1540117,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540117,release-3,,DONE,dev,0,,622595,,
+id,commit_sha,cicd_scope_id,cicd_deployment_id,name,result,status,environment,created_date,started_date,finished_date,duration_sec,ref_name,repo_id,repo_url,prev_success_deployment_commit_id
+bamboo:BambooDeployBuild:1:1540097,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540097,release-1,FAILURE,DONE,dev,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,
+bamboo:BambooDeployBuild:1:1540100,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540100,release-1,FAILURE,DONE,dev,2023-07-31T11:50:10.000+00:00,2023-07-31T11:50:10.000+00:00,2023-07-31T11:50:10.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,
+bamboo:BambooDeployBuild:1:1540101,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540101,release-2,FAILURE,DONE,dev,2023-07-31T11:51:14.000+00:00,2023-07-31T11:51:14.000+00:00,2023-07-31T11:51:14.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,
+bamboo:BambooDeployBuild:1:1540102,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540102,release-2,,DONE,dev,2023-07-31T11:52:32.000+00:00,2023-07-31T11:52:32.000+00:00,2023-07-31T11:52:32.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,
+bamboo:BambooDeployBuild:1:1540105,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540105,release-2,,DONE,dev,2023-08-01T09:31:53.000+00:00,2023-08-01T09:31:53.000+00:00,2023-08-01T09:31:53.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,
+bamboo:BambooDeployBuild:1:1540106,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540106,release-2,,DONE,dev,2023-08-01T09:32:00.000+00:00,2023-08-01T09:32:00.000+00:00,2023-08-01T09:32:00.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,
+bamboo:BambooDeployBuild:1:1540117,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540117,release-3,,DONE,dev,2023-08-03T09:49:07.000+00:00,2023-08-03T09:49:07.000+00:00,2023-08-03T09:49:07.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,
diff --git
a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv
b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv
index 1a18d25dd..dafc2e66e 100644
--- a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv
+++ b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv
@@ -1,12 +1,12 @@
pipeline_id,commit_sha,branch,repo_id,repo_url
-bamboo:BambooPlanBuild:1:TEST-PLA2-10,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-11,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-12,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-2,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-3,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-4,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-5,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-6,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-7,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-8,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-9,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
+bamboo:BambooPlanBuild:1:TEST-PLA2-10,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-11,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-12,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-2,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-3,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-4,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-5,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-6,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-7,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-8,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-9,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
diff --git a/backend/plugins/bamboo/tasks/deploy_build_convertor.go
b/backend/plugins/bamboo/tasks/deploy_build_convertor.go
index f413ee2e2..613da69eb 100644
--- a/backend/plugins/bamboo/tasks/deploy_build_convertor.go
+++ b/backend/plugins/bamboo/tasks/deploy_build_convertor.go
@@ -59,6 +59,7 @@ func (deployBuildWithVcsRevision deployBuildWithVcsRevision)
GenerateCICDDeploym
func ConvertDeployBuilds(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
+ logger := taskCtx.GetLogger()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_JOB_BUILD_TABLE)
cursor, err := db.Cursor(
dal.Select("db.*, pbc.repository_id, pbc.repository_name,
pbc.vcs_revision_key, p.name as project_plan_name, p.project_name"),
@@ -120,6 +121,12 @@ func ConvertDeployBuilds(taskCtx plugin.SubTaskContext)
errors.Error {
duration :=
uint64(input.FinishedDate.Sub(*input.StartedDate).Seconds())
deploymentCommit.DurationSec = &duration
}
+ fakeRepoUrl, err :=
generateFakeRepoUrl(data.ApiClient.GetEndpoint(), input.RepositoryId)
+ if err != nil {
+ logger.Warn(err, "generate fake repo url,
endpoint: %s, repo id: %d", data.ApiClient.GetEndpoint(), input.RepositoryId)
+ } else {
+ deploymentCommit.RepoUrl = fakeRepoUrl
+ }
return []interface{}{deploymentCommit}, nil
},
diff --git a/backend/plugins/bamboo/tasks/plan_commit_convertor.go
b/backend/plugins/bamboo/tasks/plan_commit_convertor.go
index a6d891eb9..ec9384b49 100644
--- a/backend/plugins/bamboo/tasks/plan_commit_convertor.go
+++ b/backend/plugins/bamboo/tasks/plan_commit_convertor.go
@@ -40,6 +40,7 @@ var ConvertPlanVcsMeta = plugin.SubTaskMeta{
func ConvertPlanVcs(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
+ logger := taskCtx.GetLogger()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_PLAN_BUILD_TABLE)
cursor, err := db.Cursor(
dal.From(&models.BambooPlanBuildVcsRevision{}),
@@ -61,9 +62,14 @@ func ConvertPlanVcs(taskCtx plugin.SubTaskContext)
errors.Error {
domainPlanVcs := &devops.CiCDPipelineCommit{
PipelineId:
planBuildIdGen.Generate(data.Options.ConnectionId, line.PlanBuildKey),
CommitSha: line.VcsRevisionKey,
- RepoUrl: line.RepositoryName,
}
domainPlanVcs.RepoId = repoMap[line.RepositoryId]
+ fakeRepoUrl, err :=
generateFakeRepoUrl(data.ApiClient.GetEndpoint(), line.RepositoryId)
+ if err != nil {
+ logger.Warn(err, "generate fake repo url,
endpoint: %s, repo id: %d", data.ApiClient.GetEndpoint(), line.RepositoryId)
+ } else {
+ domainPlanVcs.RepoUrl = fakeRepoUrl
+ }
return []interface{}{
domainPlanVcs,
}, nil
diff --git a/backend/plugins/bamboo/tasks/shared.go
b/backend/plugins/bamboo/tasks/shared.go
index f74af27a1..0d834a0f0 100644
--- a/backend/plugins/bamboo/tasks/shared.go
+++ b/backend/plugins/bamboo/tasks/shared.go
@@ -103,3 +103,15 @@ func getBambooHomePage(endpoint string) (string, error) {
return fmt.Sprintf("%s://%s%s", protocol, host, bambooPath), nil
}
}
+
+// generateFakeRepoUrl will return a fake url for repo url field.
+func generateFakeRepoUrl(endpoint string, repoId int) (string, error) {
+ if endpoint == "" {
+ return "", errors.Default.New("empty endpoint")
+ }
+ endpointURL, err := url.Parse(endpoint)
+ if err != nil {
+ return "", err
+ }
+ return fmt.Sprintf("fake://%s/repos/%d", endpointURL.Host, repoId), nil
+}
diff --git a/backend/plugins/bamboo/tasks/shared_test.go
b/backend/plugins/bamboo/tasks/shared_test.go
index 462408cd2..941643c48 100644
--- a/backend/plugins/bamboo/tasks/shared_test.go
+++ b/backend/plugins/bamboo/tasks/shared_test.go
@@ -79,3 +79,38 @@ func Test_getBambooWebURL(t *testing.T) {
})
}
}
+
+func Test_generateFakeRepoURL(t *testing.T) {
+ type args struct {
+ endpoint string
+ repoID int
+ }
+ tests := []struct {
+ name string
+ args args
+ want string
+ wantErr bool
+ }{
+ {
+ name: "t-1",
+ args: args{
+ endpoint: "http://127.0.0.1:8080/abc";,
+ repoID: 123,
+ },
+ want: "fake://127.0.0.1:8080/repos/123",
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := generateFakeRepoUrl(tt.args.endpoint,
tt.args.repoID)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("generateFakeRepoUrl() error = %v,
wantErr %v", err, tt.wantErr)
+ return
+ }
+ if got != tt.want {
+ t.Errorf("generateFakeRepoUrl() got = %v, want
%v", got, tt.want)
+ }
+ })
+ }
+}
