This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 8dc569720 fix: add deploy generator e2e to dora (#5118)
8dc569720 is described below
commit 8dc56972019f6c82c34d092d4353dcbf3a6baa46
Author: mappjzc <[email protected]>
AuthorDate: Mon May 8 15:26:14 2023 +0800
fix: add deploy generator e2e to dora (#5118)
Add TestGenerateDeploymentCommitsDataFlow
Nddtfjiang <[email protected]>
---
backend/helpers/e2ehelper/data_flow_tester.go | 5 +-
.../dora/e2e/deployment_commits_generator_test.go | 61 ++++++++++++++++++++++
.../cicd_deployment_commits.csv | 7 +++
.../deployment_generator/cicd_pipeline_commits.csv | 14 +++++
.../e2e/deployment_generator/cicd_pipelines.csv | 14 +++++
.../dora/e2e/deployment_generator/cicd_tasks.csv | 12 +++++
.../e2e/deployment_generator/project_mapping.csv | 9 ++++
7 files changed, 121 insertions(+), 1 deletion(-)
diff --git a/backend/helpers/e2ehelper/data_flow_tester.go
b/backend/helpers/e2ehelper/data_flow_tester.go
index 4e99e590a..6ea736412 100644
--- a/backend/helpers/e2ehelper/data_flow_tester.go
+++ b/backend/helpers/e2ehelper/data_flow_tester.go
@@ -144,7 +144,10 @@ func (t *DataFlowTester) ImportCsvIntoRawTable(csvRelPath
string, rawTableName s
}
func (t *DataFlowTester) importCsv(csvRelPath string, dst schema.Tabler,
nullable bool) {
- csvIter, _ := pluginhelper.NewCsvFileIterator(csvRelPath)
+ csvIter, err := pluginhelper.NewCsvFileIterator(csvRelPath)
+ if err != nil {
+ panic(err)
+ }
defer csvIter.Close()
t.FlushTabler(dst)
// load rows and insert into target table
diff --git a/backend/plugins/dora/e2e/deployment_commits_generator_test.go
b/backend/plugins/dora/e2e/deployment_commits_generator_test.go
new file mode 100644
index 000000000..113c85500
--- /dev/null
+++ b/backend/plugins/dora/e2e/deployment_commits_generator_test.go
@@ -0,0 +1,61 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+
"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/dora/impl"
+ "github.com/apache/incubator-devlake/plugins/dora/tasks"
+)
+
+func TestGenerateDeploymentCommitsDataFlow(t *testing.T) {
+ var plugin impl.Dora
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "dora", plugin)
+
+ taskData := &tasks.DoraTaskData{
+ Options: &tasks.DoraOptions{
+ ProjectName: "project1",
+ },
+ }
+ // import raw data table
+
dataflowTester.ImportCsvIntoTabler("./deployment_generator/project_mapping.csv",
&crossdomain.ProjectMapping{})
+
dataflowTester.ImportCsvIntoTabler("./deployment_generator/cicd_pipeline_commits.csv",
&devops.CiCDPipelineCommit{})
+
dataflowTester.ImportCsvIntoTabler("./deployment_generator/cicd_pipelines.csv",
&devops.CICDPipeline{})
+
dataflowTester.ImportCsvIntoTabler("./deployment_generator/cicd_tasks.csv",
&devops.CICDTask{})
+
+ // verify converter
+ dataflowTester.FlushTabler(&devops.CicdDeploymentCommit{})
+ dataflowTester.Subtask(tasks.DeploymentCommitsGeneratorMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&devops.CicdDeploymentCommit{},
e2ehelper.TableOptions{
+ CSVRelPath:
"./deployment_generator/cicd_deployment_commits.csv",
+ TargetFields: []string{
+ "id",
+ "cicd_scope_id",
+ "cicd_deployment_id",
+ "result",
+ "repo_url",
+ "environment",
+ "started_date",
+ "finished_date",
+ },
+ })
+}
diff --git
a/backend/plugins/dora/e2e/deployment_generator/cicd_deployment_commits.csv
b/backend/plugins/dora/e2e/deployment_generator/cicd_deployment_commits.csv
new file mode 100644
index 000000000..781b4d596
--- /dev/null
+++ b/backend/plugins/dora/e2e/deployment_generator/cicd_deployment_commits.csv
@@ -0,0 +1,7 @@
+id,commit_sha,cicd_scope_id,cicd_deployment_id,result,repo_url,environment,started_date,finished_date
+gitlab:GitlabPipeline:1:457475337:https://gitlab.com/gitlab-data/snowflake_spend,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,cicd1,gitlab:GitlabPipeline:1:457475337,,https://gitlab.com/gitlab-data/snowflake_spend,PRODUCTION,,
+gitlab:GitlabPipeline:1:485811050:https://gitlab.com/gitlab-data/snowflake_spend,c791ea6949d6b4aadf79b15ba666cb690c6527ac,cicd1,gitlab:GitlabPipeline:1:485811050,FAILURE,https://gitlab.com/gitlab-data/snowflake_spend,PRODUCTION,2022-03-07T06:26:42.109+00:00,2022-03-07T06:26:42.109+00:00
+gitlab:GitlabPipeline:1:485813816:https://gitlab.com/gitlab-data/snowflake_spend,ecc7c0b2874c812ed882c9effbbda26e0abc7110,cicd1,gitlab:GitlabPipeline:1:485813816,FAILURE,https://gitlab.com/gitlab-data/snowflake_spend,PRODUCTION,2022-03-07T06:33:56.824+00:00,2022-03-07T06:33:56.824+00:00
+gitlab:GitlabPipeline:1:485814501:https://gitlab.com/gitlab-data/snowflake_spend,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,cicd1,gitlab:GitlabPipeline:1:485814501,FAILURE,https://gitlab.com/gitlab-data/snowflake_spend,PRODUCTION,2022-03-07T06:35:28.111+00:00,2022-03-07T06:35:28.111+00:00
+gitlab:GitlabPipeline:1:485877118:https://gitlab.com/gitlab-data/snowflake_spend,09f81b1b2d083411c0bfecd32d7728479b594503,cicd1,gitlab:GitlabPipeline:1:485877118,FAILURE,https://gitlab.com/gitlab-data/snowflake_spend,PRODUCTION,2022-03-07T08:22:49.364+00:00,2022-03-07T08:27:38.364+00:00
+gitlab:GitlabPipeline:1:485932863:https://gitlab.com/gitlab-data/snowflake_spend,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,cicd1,gitlab:GitlabPipeline:1:485932863,SUCCESS,https://gitlab.com/gitlab-data/snowflake_spend,,2022-03-07T09:34:58.267+00:00,2022-03-07T09:41:36.267+00:00
diff --git
a/backend/plugins/dora/e2e/deployment_generator/cicd_pipeline_commits.csv
b/backend/plugins/dora/e2e/deployment_generator/cicd_pipeline_commits.csv
new file mode 100644
index 000000000..e34ee8b6d
--- /dev/null
+++ b/backend/plugins/dora/e2e/deployment_generator/cicd_pipeline_commits.csv
@@ -0,0 +1,14 @@
+pipeline_id,commit_sha,branch,repo_id,repo_url
+gitlab:GitlabPipeline:1:485932863,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,refs/merge-requests/74/head,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485845850,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,refs/merge-requests/74/head,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485877118,09f81b1b2d083411c0bfecd32d7728479b594503,refs/merge-requests/74/head,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485814501,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,EE-7121,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485814516,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,refs/merge-requests/74/head,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485814871,f731732471961fed061a01fa99631f69e861c4a2,refs/merge-requests/74/head,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485817670,5b95c5aebce1eae6a1b95ecf6fbc870851455375,refs/merge-requests/74/head,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485813816,ecc7c0b2874c812ed882c9effbbda26e0abc7110,EE-7121,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485813830,ecc7c0b2874c812ed882c9effbbda26e0abc7110,refs/merge-requests/74/head,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485811059,c791ea6949d6b4aadf79b15ba666cb690c6527ac,refs/merge-requests/74/head,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485811050,c791ea6949d6b4aadf79b15ba666cb690c6527ac,EE-7121,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:457475160,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,renovate/lodash-monorepo,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:457475337,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,renovate/shx-0.x,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
diff --git a/backend/plugins/dora/e2e/deployment_generator/cicd_pipelines.csv
b/backend/plugins/dora/e2e/deployment_generator/cicd_pipelines.csv
new file mode 100644
index 000000000..32302c103
--- /dev/null
+++ b/backend/plugins/dora/e2e/deployment_generator/cicd_pipelines.csv
@@ -0,0 +1,14 @@
+id,name,result,status,type,duration_sec,environment,created_date,finished_date,cicd_scope_id
+gitlab:GitlabPipeline:1:457475160,cicd1,FAILURE,DONE,,0,,2022-01-27T10:07:26.435+00:00,2022-01-27T10:07:26.638+00:00,cicd1
+gitlab:GitlabPipeline:1:457475337,cicd1,,IN_PROGRESS,,0,,2022-01-27T10:07:36.502+00:00,,cicd1
+gitlab:GitlabPipeline:1:485811050,cicd1,FAILURE,DONE,DEPLOYMENT,0,PRODUCTION,2022-03-07T06:26:42.109+00:00,2022-03-07T06:26:42.109+00:00,cicd1
+gitlab:GitlabPipeline:1:485811059,cicd1,FAILURE,DONE,,0,,2022-03-07T06:26:43.784+00:00,2022-03-07T06:26:43.784+00:00,cicd1
+gitlab:GitlabPipeline:1:485813816,cicd1,FAILURE,DONE,DEPLOYMENT,0,PRODUCTION,2022-03-07T06:33:56.824+00:00,2022-03-07T06:33:56.824+00:00,cicd1
+gitlab:GitlabPipeline:1:485813830,cicd1,FAILURE,DONE,,0,,2022-03-07T06:33:58.889+00:00,2022-03-07T06:33:58.889+00:00,cicd1
+gitlab:GitlabPipeline:1:485814501,cicd1,FAILURE,DONE,DEPLOYMENT,0,PRODUCTION,2022-03-07T06:35:28.111+00:00,2022-03-07T06:35:28.111+00:00,cicd1
+gitlab:GitlabPipeline:1:485814516,cicd1,FAILURE,DONE,,0,,2022-03-07T06:35:31.255+00:00,2022-03-07T06:35:31.255+00:00,cicd1
+gitlab:GitlabPipeline:1:485814871,cicd1,FAILURE,DONE,,42,,2022-03-07T06:36:50.020+00:00,2022-03-07T06:37:32.103+00:00,cicd1
+gitlab:GitlabPipeline:1:485817670,cicd1,FAILURE,DONE,,1956,,2022-03-07T06:45:09.471+00:00,2022-03-07T07:17:46.305+00:00,cicd1
+gitlab:GitlabPipeline:1:485845850,cicd1,FAILURE,DONE,,419,,2022-03-07T07:38:58.611+00:00,2022-03-07T07:45:58.412+00:00,cicd1
+gitlab:GitlabPipeline:1:485877118,cicd1,FAILURE,DONE,,289,,2022-03-07T08:22:48.943+00:00,2022-03-07T08:27:38.364+00:00,cicd1
+gitlab:GitlabPipeline:1:485932863,cicd1,SUCCESS,DONE,DEPLOYMENT,398,,2022-03-07T09:34:57.476+00:00,2022-03-07T09:41:36.267+00:00,cicd1
diff --git a/backend/plugins/dora/e2e/deployment_generator/cicd_tasks.csv
b/backend/plugins/dora/e2e/deployment_generator/cicd_tasks.csv
new file mode 100644
index 000000000..7c35ccc10
--- /dev/null
+++ b/backend/plugins/dora/e2e/deployment_generator/cicd_tasks.csv
@@ -0,0 +1,12 @@
+id,name,pipeline_id,result,status,type,environment,duration_sec,started_date,finished_date,cicd_scope_id
+gitlab:GitlabJob:1:100,compile,gitlab:GitlabPipeline:1:485877118,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,2,2022-07-25T15:06:57.051+00:00,2022-07-25T15:06:59.885+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:101,format,gitlab:GitlabPipeline:1:485877118,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,3,2022-07-25T15:13:37.206+00:00,2022-07-25T15:13:40.246+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:102,format,gitlab:GitlabPipeline:1:485877118,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,2,2022-07-25T15:30:22.560+00:00,2022-07-25T15:30:25.315+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:103,format,gitlab:GitlabPipeline:1:485877118,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,2,2022-07-25T15:30:55.671+00:00,2022-07-25T15:30:58.650+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:104,format,gitlab:GitlabPipeline:1:485817670,SUCCESS,DONE,,,2,2022-07-25T15:32:04.954+00:00,2022-07-25T15:32:07.726+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:105,compile,gitlab:GitlabPipeline:1:485817670,FAILURE,DONE,,,3,2022-07-25T15:32:07.953+00:00,2022-07-25T15:32:11.077+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:106,format,gitlab:GitlabPipeline:1:485817670,SUCCESS,DONE,,,2,2022-07-25T15:33:26.382+00:00,2022-07-25T15:33:29.356+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:107,format,gitlab:GitlabPipeline:1:457475337,SUCCESS,DONE,,,2,2022-07-25T15:34:23.665+00:00,2022-07-25T15:34:26.392+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:108,format,gitlab:GitlabPipeline:1:457475337,SUCCESS,DONE,,,2,2022-07-25T15:35:11.707+00:00,2022-07-25T15:35:14.224+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:109,compile,gitlab:GitlabPipeline:1:457475337,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,3,2022-07-25T15:35:14.724+00:00,2022-07-25T15:35:17.828+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:110,format,gitlab:GitlabPipeline:1:457475337,SUCCESS,DONE,,,2,2022-07-25T15:36:18.097+00:00,2022-07-25T15:36:20.954+00:00,gitlab:GitlabProject:1:44
diff --git a/backend/plugins/dora/e2e/deployment_generator/project_mapping.csv
b/backend/plugins/dora/e2e/deployment_generator/project_mapping.csv
new file mode 100644
index 000000000..296ab9e2f
--- /dev/null
+++ b/backend/plugins/dora/e2e/deployment_generator/project_mapping.csv
@@ -0,0 +1,9 @@
+project_name,table,row_id
+project1,cicd_scopes,cicd1
+project1,cicd_scopes,cicd2
+project1,repos,repo1
+project1,repos,repo2
+project2,cicd_scopes,cicd3
+project1,boards,board1
+project1,boards,board2
+project2,boards,board3
\ No newline at end of file
