This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 5ae16b97b fix: add env and type to gitlab (#4975)
5ae16b97b is described below
commit 5ae16b97b7ac80212dc651396f9d9db5f97a7613
Author: mappjzc <[email protected]>
AuthorDate: Thu Apr 20 12:10:48 2023 +0800
fix: add env and type to gitlab (#4975)
Add env and type to gitlab pipelines.
Nddtfjiang <[email protected]>
---
backend/plugins/gitlab/e2e/pipelines_test.go | 11 ++++++
.../e2e/snapshot_tables/_tool_gitlab_pipelines.csv | 42 +++++++++++-----------
.../gitlab/e2e/snapshot_tables/cicd_pipelines.csv | 6 ++--
backend/plugins/gitlab/gitlab.go | 33 +++++++++--------
backend/plugins/gitlab/impl/impl.go | 14 ++++++--
...ter.go => 20230419_add_type_env_to_pipeline.go} | 38 ++++++++++++--------
.../gitlab/models/migrationscripts/register.go | 1 +
backend/plugins/gitlab/models/pipeline.go | 3 ++
backend/plugins/gitlab/tasks/pipeline_convertor.go | 2 ++
backend/plugins/gitlab/tasks/pipeline_extractor.go | 27 ++++++++------
backend/plugins/gitlab/tasks/task_data.go | 1 +
11 files changed, 113 insertions(+), 65 deletions(-)
diff --git a/backend/plugins/gitlab/e2e/pipelines_test.go
b/backend/plugins/gitlab/e2e/pipelines_test.go
index d313ed3b2..0f4bb38d6 100644
--- a/backend/plugins/gitlab/e2e/pipelines_test.go
+++ b/backend/plugins/gitlab/e2e/pipelines_test.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/impl"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
@@ -33,13 +34,19 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
var gitlab impl.Gitlab
dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab)
+ regexEnricher := api.NewRegexEnricher()
+ _ = regexEnricher.TryAdd(devops.DEPLOYMENT, "EE-7121")
+ _ = regexEnricher.TryAdd(devops.PRODUCTION, "EE-7121")
+
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
ConnectionId: 1,
ProjectId: 12345678,
GitlabTransformationRule:
new(models.GitlabTransformationRule),
},
+ RegexEnricher: regexEnricher,
}
+
// import raw data table
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_pipeline.csv",
"_raw_gitlab_api_pipeline")
dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_gitlab_projects.csv",
&models.GitlabProject{})
@@ -57,11 +64,15 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
"gitlab_created_at",
"project_id",
"status",
+ "ref",
+ "sha",
"web_url",
"duration",
"started_at",
"finished_at",
"coverage",
+ "type",
+ "environment",
),
)
diff --git
a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
index 425608973..46bb9b3a2 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
@@ -1,21 +1,21 @@
-connection_id,gitlab_id,gitlab_created_at,project_id,status,web_url,duration,started_at,finished_at,coverage,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,457474837,2022-01-27T10:07:09.429+00:00,12345678,running,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
-1,457474996,2022-01-27T10:07:18.884+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
-1,457475160,2022-01-27T10:07:26.435+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
-1,457475337,2022-01-27T10:07:36.502+00:00,12345678,running,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
-1,485811050,2022-03-07T06:26:42.109+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156,
-1,485811059,2022-03-07T06:26:43.784+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157,
-1,485813816,2022-03-07T06:33:56.824+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158,
-1,485813830,2022-03-07T06:33:58.889+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813830,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159,
-1,485814501,2022-03-07T06:35:28.111+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814501,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160,
-1,485814516,2022-03-07T06:35:31.255+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814516,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161,
-1,485814871,2022-03-07T06:36:50.020+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814871,42,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162,
-1,485817670,2022-03-07T06:45:09.471+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485817670,1956,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163,
-1,485837602,2022-03-07T07:20:45.859+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485837602,434,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164,
-1,485842553,2022-03-07T07:30:47.018+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485842553,287,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165,
-1,485845850,2022-03-07T07:38:58.611+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485845850,419,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166,
-1,485852752,2022-03-07T07:46:09.385+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485852752,319,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167,
-1,485865876,2022-03-07T08:04:56.406+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485865876,480,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168,
-1,485877118,2022-03-07T08:22:48.943+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485877118,289,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169,
-1,485905167,2022-03-07T09:02:09.994+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485905167,687,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170,
-1,485932863,2022-03-07T09:34:57.476+00:00,12345678,success,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171,
+connection_id,gitlab_id,gitlab_created_at,project_id,status,ref,sha,web_url,duration,started_at,finished_at,coverage,type,environment,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,457474837,2022-01-27T10:07:09.429+00:00,12345678,running,renovate/pin-dependencies,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
+1,457474996,2022-01-27T10:07:18.884+00:00,12345678,failed,renovate/jest-monorepo,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
+1,457475160,2022-01-27T10:07:26.435+00:00,12345678,failed,renovate/lodash-monorepo,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
+1,457475337,2022-01-27T10:07:36.502+00:00,12345678,running,renovate/shx-0.x,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
+1,485811050,2022-03-07T06:26:42.109+00:00,12345678,failed,EE-7121,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,,,,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156,
+1,485811059,2022-03-07T06:26:43.784+00:00,12345678,failed,refs/merge-requests/74/head,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157,
+1,485813816,2022-03-07T06:33:56.824+00:00,12345678,failed,EE-7121,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,,,,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158,
+1,485813830,2022-03-07T06:33:58.889+00:00,12345678,failed,refs/merge-requests/74/head,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813830,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159,
+1,485814501,2022-03-07T06:35:28.111+00:00,12345678,failed,EE-7121,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814501,0,,,,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160,
+1,485814516,2022-03-07T06:35:31.255+00:00,12345678,failed,refs/merge-requests/74/head,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814516,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161,
+1,485814871,2022-03-07T06:36:50.020+00:00,12345678,failed,refs/merge-requests/74/head,f731732471961fed061a01fa99631f69e861c4a2,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814871,42,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162,
+1,485817670,2022-03-07T06:45:09.471+00:00,12345678,failed,refs/merge-requests/74/head,5b95c5aebce1eae6a1b95ecf6fbc870851455375,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485817670,1956,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163,
+1,485837602,2022-03-07T07:20:45.859+00:00,12345678,failed,refs/merge-requests/74/head,a79e2aaeee9916338a11b665cb4201f24627f3f1,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485837602,434,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164,
+1,485842553,2022-03-07T07:30:47.018+00:00,12345678,failed,refs/merge-requests/74/head,9e0934cedcd83abab8d34c5bb0f597c7837a55f0,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485842553,287,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165,
+1,485845850,2022-03-07T07:38:58.611+00:00,12345678,failed,refs/merge-requests/74/head,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485845850,419,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166,
+1,485852752,2022-03-07T07:46:09.385+00:00,12345678,failed,refs/merge-requests/74/head,82df93eeb72c71271dad5a85359f39661de899fe,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485852752,319,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167,
+1,485865876,2022-03-07T08:04:56.406+00:00,12345678,failed,refs/merge-requests/74/head,db45efd231dbaca1337bd29bc1f65f404754e6bd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485865876,480,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168,
+1,485877118,2022-03-07T08:22:48.943+00:00,12345678,failed,refs/merge-requests/74/head,09f81b1b2d083411c0bfecd32d7728479b594503,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485877118,289,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169,
+1,485905167,2022-03-07T09:02:09.994+00:00,12345678,failed,refs/merge-requests/74/head,cce432655861efafcd05edc8de09e305c772d2b8,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485905167,687,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170,
+1,485932863,2022-03-07T09:34:57.476+00:00,12345678,success,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171,
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
index d73e46df0..b09f525ea 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
@@ -3,11 +3,11 @@
gitlab:GitlabPipeline:1:457474837,gitlab:GitlabProject:1:12345678,,IN_PROGRESS,,
gitlab:GitlabPipeline:1:457474996,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-01-27T10:07:18.884+00:00,2022-01-27T10:07:19.043+00:00,gitlab:GitlabProject:1:12345678
gitlab:GitlabPipeline:1:457475160,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-01-27T10:07:26.435+00:00,2022-01-27T10:07:26.638+00:00,gitlab:GitlabProject:1:12345678
gitlab:GitlabPipeline:1:457475337,gitlab:GitlabProject:1:12345678,,IN_PROGRESS,,0,,2022-01-27T10:07:36.502+00:00,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485811050,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-03-07T06:26:42.109+00:00,2022-03-07T06:26:42.109+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:485811050,gitlab:GitlabProject:1:12345678,FAILURE,DONE,DEPLOYMENT,0,PRODUCTION,2022-03-07T06:26:42.109+00:00,2022-03-07T06:26:42.109+00:00,gitlab:GitlabProject:1:12345678
gitlab:GitlabPipeline:1:485811059,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-03-07T06:26:43.784+00:00,2022-03-07T06:26:43.784+00:00,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485813816,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-03-07T06:33:56.824+00:00,2022-03-07T06:33:56.824+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:485813816,gitlab:GitlabProject:1:12345678,FAILURE,DONE,DEPLOYMENT,0,PRODUCTION,2022-03-07T06:33:56.824+00:00,2022-03-07T06:33:56.824+00:00,gitlab:GitlabProject:1:12345678
gitlab:GitlabPipeline:1:485813830,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-03-07T06:33:58.889+00:00,2022-03-07T06:33:58.889+00:00,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485814501,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-03-07T06:35:28.111+00:00,2022-03-07T06:35:28.111+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:485814501,gitlab:GitlabProject:1:12345678,FAILURE,DONE,DEPLOYMENT,0,PRODUCTION,2022-03-07T06:35:28.111+00:00,2022-03-07T06:35:28.111+00:00,gitlab:GitlabProject:1:12345678
gitlab:GitlabPipeline:1:485814516,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-03-07T06:35:31.255+00:00,2022-03-07T06:35:31.255+00:00,gitlab:GitlabProject:1:12345678
gitlab:GitlabPipeline:1:485814871,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,42,,2022-03-07T06:36:50.020+00:00,2022-03-07T06:37:32.103+00:00,gitlab:GitlabProject:1:12345678
gitlab:GitlabPipeline:1:485817670,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,1956,,2022-03-07T06:45:09.471+00:00,2022-03-07T07:17:46.305+00:00,gitlab:GitlabProject:1:12345678
diff --git a/backend/plugins/gitlab/gitlab.go b/backend/plugins/gitlab/gitlab.go
index a45b0aa27..c44faa4da 100644
--- a/backend/plugins/gitlab/gitlab.go
+++ b/backend/plugins/gitlab/gitlab.go
@@ -44,23 +44,26 @@ func main() {
issueTypeBug := cmd.Flags().String("issueTypeBug",
"^(bug|failure|error)$", "issue type bug")
issueTypeIncident := cmd.Flags().String("issueTypeIncident", "", "issue
type incident")
issueTypeRequirement := cmd.Flags().String("issueTypeRequirement",
"^(feat|feature|proposal|requirement)$", "issue type requirement")
- deployTagPattern := cmd.Flags().String("deployTagPattern",
"(?i)deploy", "deploy tag name")
-
+ deploymentPattern := cmd.Flags().String("deploymentPattern",
"(?i)deploy", "deploy Pattern for project name")
+ productionPattern := cmd.Flags().String("productionPattern", "product",
"product Pattern for project name")
cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "projectId": *projectId,
- "connectionId": *connectionId,
- "timeAfter": *timeAfter,
- "prType": *prType,
- "prComponent": *prComponent,
- "prBodyClosePattern": *prBodyClosePattern,
- "issueSeverity": *issueSeverity,
- "issuePriority": *issuePriority,
- "issueComponent": *issueComponent,
- "issueTypeBug": *issueTypeBug,
- "issueTypeIncident": *issueTypeIncident,
- "issueTypeRequirement": *issueTypeRequirement,
- "deployTagPattern": *deployTagPattern,
+ "projectId": *projectId,
+ "connectionId": *connectionId,
+ "timeAfter": *timeAfter,
+ "transformationRules": map[string]interface{}{
+ "prType": *prType,
+ "prComponent": *prComponent,
+ "prBodyClosePattern": *prBodyClosePattern,
+ "issueSeverity": *issueSeverity,
+ "issuePriority": *issuePriority,
+ "issueComponent": *issueComponent,
+ "issueTypeBug": *issueTypeBug,
+ "issueTypeIncident": *issueTypeIncident,
+ "issueTypeRequirement": *issueTypeRequirement,
+ "deploymentPattern": *deploymentPattern,
+ "productionPattern": *productionPattern,
+ },
})
}
diff --git a/backend/plugins/gitlab/impl/impl.go
b/backend/plugins/gitlab/impl/impl.go
index 561538ae5..fb04f6207 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/api"
@@ -209,9 +210,18 @@ func (p Gitlab) PrepareTaskData(taskCtx
plugin.TaskContext, options map[string]i
op.GitlabTransformationRule = &transformationRule
}
+ regexEnricher := helper.NewRegexEnricher()
+ if err := regexEnricher.TryAdd(devops.DEPLOYMENT,
op.DeploymentPattern); err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value for
`deploymentPattern`")
+ }
+ if err := regexEnricher.TryAdd(devops.PRODUCTION,
op.ProductionPattern); err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value for
`productionPattern`")
+ }
+
taskData := tasks.GitlabTaskData{
- Options: op,
- ApiClient: apiClient,
+ Options: op,
+ ApiClient: apiClient,
+ RegexEnricher: regexEnricher,
}
if !timeAfter.IsZero() {
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go
b/backend/plugins/gitlab/models/migrationscripts/20230419_add_type_env_to_pipeline.go
similarity index 52%
copy from backend/plugins/gitlab/models/migrationscripts/register.go
copy to
backend/plugins/gitlab/models/migrationscripts/20230419_add_type_env_to_pipeline.go
index 9612cf3f7..fed04a02b 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++
b/backend/plugins/gitlab/models/migrationscripts/20230419_add_type_env_to_pipeline.go
@@ -18,20 +18,30 @@ limitations under the License.
package migrationscripts
import (
- "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addGitlabCI),
- new(addPipelineID),
- new(addPipelineProjects),
- new(fixDurationToFloat8),
- new(addTransformationRule20221125),
- new(addStdTypeToIssue221230),
- new(addIsDetailRequired20230210),
- new(addConnectionIdToTransformationRule),
- }
+type addTypeEnvToPipeline struct{}
+
+type pipeline20230419 struct {
+ Type string `gorm:"type:varchar(255)"`
+ Environment string `gorm:"type:varchar(255)"`
+}
+
+func (pipeline20230419) TableName() string {
+ return "_tool_gitlab_pipelines"
+}
+
+func (u *addTypeEnvToPipeline) Up(baseRes context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(baseRes, &pipeline20230419{})
+}
+
+func (*addTypeEnvToPipeline) Version() uint64 {
+ return 20230322150357
+}
+
+func (*addTypeEnvToPipeline) Name() string {
+ return "add type/env to gitlab pipelines"
}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go
b/backend/plugins/gitlab/models/migrationscripts/register.go
index 9612cf3f7..53823fae4 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/register.go
@@ -33,5 +33,6 @@ func All() []plugin.MigrationScript {
new(addStdTypeToIssue221230),
new(addIsDetailRequired20230210),
new(addConnectionIdToTransformationRule),
+ new(addTypeEnvToPipeline),
}
}
diff --git a/backend/plugins/gitlab/models/pipeline.go
b/backend/plugins/gitlab/models/pipeline.go
index c66e78a24..920f2d077 100644
--- a/backend/plugins/gitlab/models/pipeline.go
+++ b/backend/plugins/gitlab/models/pipeline.go
@@ -40,6 +40,9 @@ type GitlabPipeline struct {
FinishedAt *time.Time
Coverage string
+ Type string `gorm:"type:varchar(255)"`
+ Environment string `gorm:"type:varchar(255)"`
+
IsDetailRequired bool
common.NoPKModel
diff --git a/backend/plugins/gitlab/tasks/pipeline_convertor.go
b/backend/plugins/gitlab/tasks/pipeline_convertor.go
index 810b84c9b..30e6b979b 100644
--- a/backend/plugins/gitlab/tasks/pipeline_convertor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_convertor.go
@@ -92,6 +92,8 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext)
errors.Error {
CreatedDate: createdAt,
FinishedDate: gitlabPipeline.GitlabUpdatedAt,
CicdScopeId:
projectIdGen.Generate(data.Options.ConnectionId, gitlabPipeline.ProjectId),
+ Environment: gitlabPipeline.Environment,
+ Type: gitlabPipeline.Type,
}
// rebuild the FinishedDate and DurationSec by Status
diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go
b/backend/plugins/gitlab/tasks/pipeline_extractor.go
index 6b1ad5875..39c840119 100644
--- a/backend/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go
@@ -21,6 +21,7 @@ import (
"encoding/json"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
@@ -80,16 +81,22 @@ func ExtractApiPipelines(taskCtx plugin.SubTaskContext)
errors.Error {
}
gitlabPipeline := &models.GitlabPipeline{
- GitlabId: gitlabApiPipeline.Id,
- ProjectId: data.Options.ProjectId,
- WebUrl: gitlabApiPipeline.WebUrl,
- Status: gitlabApiPipeline.Status,
- GitlabCreatedAt:
api.Iso8601TimeToTime(gitlabApiPipeline.CreatedAt),
- GitlabUpdatedAt:
api.Iso8601TimeToTime(gitlabApiPipeline.UpdatedAt),
- StartedAt:
api.Iso8601TimeToTime(gitlabApiPipeline.StartedAt),
- FinishedAt:
api.Iso8601TimeToTime(gitlabApiPipeline.FinishedAt),
- Duration: gitlabApiPipeline.Duration,
- ConnectionId: data.Options.ConnectionId,
+ GitlabId: gitlabApiPipeline.Id,
+ ProjectId: data.Options.ProjectId,
+ Ref: gitlabApiPipeline.Ref,
+ Sha: gitlabApiPipeline.Sha,
+ WebUrl: gitlabApiPipeline.WebUrl,
+ Status: gitlabApiPipeline.Status,
+ GitlabCreatedAt:
api.Iso8601TimeToTime(gitlabApiPipeline.CreatedAt),
+ GitlabUpdatedAt:
api.Iso8601TimeToTime(gitlabApiPipeline.UpdatedAt),
+ StartedAt:
api.Iso8601TimeToTime(gitlabApiPipeline.StartedAt),
+ FinishedAt:
api.Iso8601TimeToTime(gitlabApiPipeline.FinishedAt),
+ Duration: gitlabApiPipeline.Duration,
+ ConnectionId: data.Options.ConnectionId,
+
+ Type:
data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT,
gitlabApiPipeline.Ref),
+ Environment:
data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION,
gitlabApiPipeline.Ref),
+
IsDetailRequired: false,
}
diff --git a/backend/plugins/gitlab/tasks/task_data.go
b/backend/plugins/gitlab/tasks/task_data.go
index 9c0a5a67a..4170a4547 100644
--- a/backend/plugins/gitlab/tasks/task_data.go
+++ b/backend/plugins/gitlab/tasks/task_data.go
@@ -39,6 +39,7 @@ type GitlabTaskData struct {
ApiClient *helper.ApiAsyncClient
ProjectCommit *models.GitlabProjectCommit
TimeAfter *time.Time
+ RegexEnricher *helper.RegexEnricher
}
func DecodeAndValidateTaskOptions(options map[string]interface{})
(*GitlabOptions, errors.Error) {
