This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 1d210d409 fix: duration_sec has wrong value (#6688)
1d210d409 is described below
commit 1d210d40952ca0a5f5c4c624da487830832e5765
Author: abeizn <[email protected]>
AuthorDate: Mon Dec 25 12:52:40 2023 +0800
fix: duration_sec has wrong value (#6688)
* fix: duration_sec has wrong value
* fix: remove some useless code
* fix: pipeline detail e2e
* fix: pipeline e2e
* fix: pipeline e2e
* fix: gitlab pipelines e2e test
* fix: gitlab pipelines e2e test
---
.../plugins/gitlab/e2e/pipelines_detail_test.go | 23 +++------
backend/plugins/gitlab/e2e/pipelines_test.go | 55 +-------------------
.../_raw_gitlab_api_pipeline_details.csv | 17 ++++++
.../e2e/snapshot_tables/_tool_gitlab_pipelines.csv | 45 ++++++----------
.../e2e/snapshot_tables/cicd_pipeline_commits.csv | 60 +++++++++++-----------
.../gitlab/e2e/snapshot_tables/cicd_pipelines.csv | 45 ++++++----------
...20231221_add_time_filed_to_pipeline_projects.go | 52 +++++++++++++++++++
.../gitlab/models/migrationscripts/register.go | 1 +
backend/plugins/gitlab/models/pipeline.go | 12 +++--
.../gitlab/tasks/pipeline_detail_collector.go | 16 +++---
.../gitlab/tasks/pipeline_detail_extractor.go | 25 ++-------
backend/plugins/gitlab/tasks/pipeline_extractor.go | 38 ++------------
12 files changed, 166 insertions(+), 223 deletions(-)
diff --git a/backend/plugins/gitlab/e2e/pipelines_detail_test.go
b/backend/plugins/gitlab/e2e/pipelines_detail_test.go
index 9d1d68830..33f884f31 100644
--- a/backend/plugins/gitlab/e2e/pipelines_detail_test.go
+++ b/backend/plugins/gitlab/e2e/pipelines_detail_test.go
@@ -48,16 +48,16 @@ func TestGitlabPipelineDetailDataFlow(t *testing.T) {
}
// import raw data table
-
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_pipeline.csv",
"_raw_gitlab_api_pipeline_details")
-
dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_gitlab_projects.csv",
&models.GitlabProject{})
+ dataflowTester.FlushTabler(&models.GitlabPipelineProject{})
+ dataflowTester.FlushTabler(models.GitlabPipeline{})
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_pipeline_details.csv",
"_raw_gitlab_api_pipeline_details")
+
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_gitlab_pipeline_projects.csv",
&models.GitlabPipelineProject{})
// verify extraction
- dataflowTester.FlushTabler(&models.GitlabPipeline{})
- dataflowTester.FlushTabler(&models.GitlabPipelineProject{})
dataflowTester.Subtask(tasks.ExtractApiPipelineDetailsMeta, taskData)
dataflowTester.VerifyTable(
models.GitlabPipeline{},
- "./snapshot_tables/_raw_gitlab_api_pipeline_details.csv",
+ "./snapshot_tables/_tool_gitlab_pipelines.csv",
e2ehelper.ColumnWithRawData(
"connection_id",
"gitlab_id",
@@ -76,19 +76,8 @@ func TestGitlabPipelineDetailDataFlow(t *testing.T) {
),
)
- dataflowTester.VerifyTable(
- models.GitlabPipelineProject{},
- "./snapshot_tables/_tool_gitlab_pipeline_projects_details.csv",
- e2ehelper.ColumnWithRawData(
- "connection_id",
- "pipeline_id",
- "project_id",
- "ref",
- "sha",
- ),
- )
-
// verify conversion
+
dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_gitlab_projects.csv",
&models.GitlabProject{})
dataflowTester.FlushTabler(&devops.CICDPipeline{})
dataflowTester.FlushTabler(&devops.CiCDPipelineCommit{})
dataflowTester.Subtask(tasks.ConvertPipelineMeta, taskData)
diff --git a/backend/plugins/gitlab/e2e/pipelines_test.go
b/backend/plugins/gitlab/e2e/pipelines_test.go
index 620a11d5b..513e2e944 100644
--- a/backend/plugins/gitlab/e2e/pipelines_test.go
+++ b/backend/plugins/gitlab/e2e/pipelines_test.go
@@ -20,7 +20,6 @@ package e2e
import (
"testing"
- "github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -46,47 +45,10 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
RegexEnricher: regexEnricher,
}
- // import raw data table
-
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_pipeline.csv",
"_raw_gitlab_api_pipeline")
-
dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_gitlab_projects.csv",
&models.GitlabProject{})
-
// verify env when production is omitted
- dataflowTester.FlushTabler(&models.GitlabPipeline{})
- dataflowTester.FlushTabler(&models.GitlabPipelineProject{})
- dataflowTester.Subtask(tasks.ExtractApiPipelinesMeta, taskData)
- dataflowTester.VerifyTable(
- models.GitlabPipeline{},
- "./snapshot_tables/_tool_gitlab_pipelines_no_prod_regex.csv",
- e2ehelper.ColumnWithRawData(
- "environment",
- ),
- )
-
- // verify extraction
- _ = regexEnricher.TryAdd(devops.PRODUCTION, "EE-7121")
- dataflowTester.FlushTabler(&models.GitlabPipeline{})
dataflowTester.FlushTabler(&models.GitlabPipelineProject{})
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_pipeline.csv",
"_raw_gitlab_api_pipeline")
dataflowTester.Subtask(tasks.ExtractApiPipelinesMeta, taskData)
- dataflowTester.VerifyTable(
- models.GitlabPipeline{},
- "./snapshot_tables/_tool_gitlab_pipelines.csv",
- e2ehelper.ColumnWithRawData(
- "connection_id",
- "gitlab_id",
- "gitlab_created_at",
- "project_id",
- "status",
- "ref",
- "sha",
- "web_url",
- "duration",
- "started_at",
- "finished_at",
- "coverage",
- "type",
- "environment",
- ),
- )
dataflowTester.VerifyTable(
models.GitlabPipelineProject{},
@@ -99,19 +61,4 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
"sha",
),
)
-
- // verify conversion
- dataflowTester.FlushTabler(&devops.CICDPipeline{})
- dataflowTester.FlushTabler(&devops.CiCDPipelineCommit{})
- dataflowTester.Subtask(tasks.ConvertPipelineMeta, taskData)
- dataflowTester.Subtask(tasks.ConvertPipelineCommitMeta, taskData)
- dataflowTester.VerifyTableWithOptions(&devops.CICDPipeline{},
e2ehelper.TableOptions{
- CSVRelPath: "./snapshot_tables/cicd_pipelines.csv",
- IgnoreTypes: []interface{}{common.NoPKModel{}},
- })
-
- dataflowTester.VerifyTableWithOptions(&devops.CiCDPipelineCommit{},
e2ehelper.TableOptions{
- CSVRelPath: "./snapshot_tables/cicd_pipeline_commits.csv",
- IgnoreTypes: []interface{}{common.NoPKModel{}},
- })
}
diff --git
a/backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline_details.csv
b/backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline_details.csv
new file mode 100644
index 000000000..4377e8137
--- /dev/null
+++ b/backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline_details.csv
@@ -0,0 +1,17 @@
+"id","params","data","url","input","created_at"
+1,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1078343570,""iid"":36229,""project_id"":12345678,""sha"":""694ed6ede74c367f17c7fddb56de629354a27069"",""ref"":""refs/merge-requests/5394/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-20T07:17:08.753Z"",""updated_at"":""2023-11-21T01:54:29.757Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1078343570"",""before_sha"":""0000000000000000000000000000000000000000"",""tag
[...]
+2,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079491591,""iid"":36241,""project_id"":12345678,""sha"":""caed059a785dbfa72e1f04d2640b3580d8d67441"",""ref"":""refs/merge-requests/5394/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T01:34:32.198Z"",""updated_at"":""2023-11-21T02:00:16.216Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079491591"",""before_sha"":""0000000000000000000000000000000000000000"",""tag
[...]
+3,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079502706,""iid"":36242,""project_id"":12345678,""sha"":""919fe190728e79f3564cecc223ba8e6068addb87"",""ref"":""refs/merge-requests/5500/head"",""status"":""failed"",""source"":""merge_request_event"",""created_at"":""2023-11-21T01:57:27.619Z"",""updated_at"":""2023-11-21T02:00:57.015Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079502706"",""before_sha"":""0000000000000000000000000000000000000000"",""tag"
[...]
+4,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079509085,""iid"":36243,""project_id"":12345678,""sha"":""4b0471ca50fd1f4f8c6d2313f741285aff07a5b5"",""ref"":""refs/merge-requests/5500/head"",""status"":""failed"",""source"":""merge_request_event"",""created_at"":""2023-11-21T02:01:18.064Z"",""updated_at"":""2023-11-21T02:04:28.990Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079509085"",""before_sha"":""0000000000000000000000000000000000000000"",""tag"
[...]
+5,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079536077,""iid"":36244,""project_id"":12345678,""sha"":""eb651695e20ba35e8dee87e768eb04d46a466854"",""ref"":""refs/merge-requests/5500/head"",""status"":""failed"",""source"":""merge_request_event"",""created_at"":""2023-11-21T02:57:02.538Z"",""updated_at"":""2023-11-21T03:00:28.052Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079536077"",""before_sha"":""0000000000000000000000000000000000000000"",""tag"
[...]
+6,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079536741,""iid"":36245,""project_id"":12345678,""sha"":""a83eb5619c1132d2f5a02213ac3bce4c9e29e348"",""ref"":""refs/merge-requests/5522/head"",""status"":""failed"",""source"":""merge_request_event"",""created_at"":""2023-11-21T02:59:22.311Z"",""updated_at"":""2023-11-21T03:12:07.685Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079536741"",""before_sha"":""0000000000000000000000000000000000000000"",""tag"
[...]
+7,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079543310,""iid"":36246,""project_id"":12345678,""sha"":""e59c1ef621e438ab1632e203d67f8ab955662aee"",""ref"":""refs/merge-requests/5501/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T03:02:28.896Z"",""updated_at"":""2023-11-21T03:27:00.551Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079543310"",""before_sha"":""0000000000000000000000000000000000000000"",""tag
[...]
+8,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079545236,""iid"":36247,""project_id"":12345678,""sha"":""00bd6fb5e570661da63c4775234d84e88913e5b1"",""ref"":""refs/merge-requests/5500/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T03:05:55.578Z"",""updated_at"":""2023-11-21T03:30:29.620Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079545236"",""before_sha"":""0000000000000000000000000000000000000000"",""tag
[...]
+9,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079555385,""iid"":36248,""project_id"":12345678,""sha"":""b5b8ec19fbddefe1374e87b05775f238ac06003e"",""ref"":""master"",""status"":""success"",""source"":""push"",""created_at"":""2023-11-21T03:28:30.382Z"",""updated_at"":""2023-11-21T03:51:44.879Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079555385"",""before_sha"":""0275c5b0341fba564c615bbcc154671bc3de9b82"",""tag"":false,""yaml_errors"":null,""user""
[...]
+10,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079557580,""iid"":36249,""project_id"":12345678,""sha"":""f1aeedd7d744f02195d1cee9406a94609333abb6"",""ref"":""refs/merge-requests/5522/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T03:30:51.330Z"",""updated_at"":""2023-11-21T03:55:49.274Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079557580"",""before_sha"":""0000000000000000000000000000000000000000"",""ta
[...]
+11,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079558335,""iid"":36250,""project_id"":12345678,""sha"":""03b72da7177a59a499d08dcaf036e37d5910863b"",""ref"":""refs/merge-requests/5522/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T03:33:09.746Z"",""updated_at"":""2023-11-21T04:04:25.665Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079558335"",""before_sha"":""0000000000000000000000000000000000000000"",""ta
[...]
+12,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079577769,""iid"":36251,""project_id"":12345678,""sha"":""1e8ca37113a962d52a7bac5e738e817549a1217b"",""ref"":""refs/merge-requests/5541/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T04:04:00.904Z"",""updated_at"":""2023-11-21T04:27:18.236Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079577769"",""before_sha"":""0000000000000000000000000000000000000000"",""ta
[...]
+13,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079589242,""iid"":36252,""project_id"":12345678,""sha"":""393741a60c9395efab1d404816ff072af03d6f02"",""ref"":""refs/merge-requests/5500/head"",""status"":""failed"",""source"":""merge_request_event"",""created_at"":""2023-11-21T04:23:00.209Z"",""updated_at"":""2023-11-21T04:26:03.518Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079589242"",""before_sha"":""0000000000000000000000000000000000000000"",""tag
[...]
+14,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079594690,""iid"":36253,""project_id"":12345678,""sha"":""c124f78f8def33fd2541782d335ba177bd85f948"",""ref"":""refs/merge-requests/5500/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T04:31:16.546Z"",""updated_at"":""2023-11-21T04:55:30.486Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079594690"",""before_sha"":""0000000000000000000000000000000000000000"",""ta
[...]
+15,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079602571,""iid"":36254,""project_id"":12345678,""sha"":""76513320215d47fd384a0110e3b7f29538f5bbd7"",""ref"":""refs/merge-requests/5519/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T04:47:19.253Z"",""updated_at"":""2023-11-21T05:12:07.103Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079602571"",""before_sha"":""0000000000000000000000000000000000000000"",""ta
[...]
+16,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079620249,""iid"":36255,""project_id"":12345678,""sha"":""3534c526a3ea0be1eb4ff2f0a6b89ca152762b0e"",""ref"":""refs/merge-requests/5522/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T05:13:37.061Z"",""updated_at"":""2023-11-21T05:36:26.228Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079620249"",""before_sha"":""0000000000000000000000000000000000000000"",""ta
[...]
\ No newline at end of file
diff --git
a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
index 1d62c7baa..c44bdd788 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
@@ -1,30 +1,17 @@
connection_id,gitlab_id,gitlab_created_at,project_id,status,ref,sha,web_url,duration,started_at,finished_at,coverage,type,environment,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,457474837,2022-01-27T10:07:09.429+00:00,12345678,running,renovate/pin-dependencies,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
-1,457474996,2022-01-27T10:07:18.884+00:00,12345678,failed,renovate/jest-monorepo,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
-1,457475160,2022-01-27T10:07:26.435+00:00,12345678,failed,renovate/lodash-monorepo,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
-1,457475337,2022-01-27T10:07:36.502+00:00,12345678,running,renovate/shx-0.x,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
-1,485811050,2022-03-07T06:26:42.109+00:00,12345678,failed,EE-7121,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,,,,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156,
-1,485811059,2022-03-07T06:26:43.784+00:00,12345678,failed,refs/merge-requests/74/head,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157,
-1,485813816,2022-03-07T06:33:56.824+00:00,12345678,failed,EE-7121,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,,,,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158,
-1,485813830,2022-03-07T06:33:58.889+00:00,12345678,failed,refs/merge-requests/74/head,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813830,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159,
-1,485814501,2022-03-07T06:35:28.111+00:00,12345678,failed,EE-7121,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814501,0,,,,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160,
-1,485814516,2022-03-07T06:35:31.255+00:00,12345678,failed,refs/merge-requests/74/head,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814516,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161,
-1,485814871,2022-03-07T06:36:50.020+00:00,12345678,failed,refs/merge-requests/74/head,f731732471961fed061a01fa99631f69e861c4a2,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814871,42,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162,
-1,485817670,2022-03-07T06:45:09.471+00:00,12345678,failed,refs/merge-requests/74/head,5b95c5aebce1eae6a1b95ecf6fbc870851455375,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485817670,1956,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163,
-1,485837602,2022-03-07T07:20:45.859+00:00,12345678,failed,refs/merge-requests/74/head,a79e2aaeee9916338a11b665cb4201f24627f3f1,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485837602,434,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164,
-1,485842553,2022-03-07T07:30:47.018+00:00,12345678,failed,refs/merge-requests/74/head,9e0934cedcd83abab8d34c5bb0f597c7837a55f0,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485842553,287,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165,
-1,485845850,2022-03-07T07:38:58.611+00:00,12345678,failed,refs/merge-requests/74/head,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485845850,419,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166,
-1,485852752,2022-03-07T07:46:09.385+00:00,12345678,failed,refs/merge-requests/74/head,82df93eeb72c71271dad5a85359f39661de899fe,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485852752,319,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167,
-1,485865876,2022-03-07T08:04:56.406+00:00,12345678,failed,refs/merge-requests/74/head,db45efd231dbaca1337bd29bc1f65f404754e6bd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485865876,480,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168,
-1,485877118,2022-03-07T08:22:48.943+00:00,12345678,failed,refs/merge-requests/74/head,09f81b1b2d083411c0bfecd32d7728479b594503,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485877118,289,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169,
-1,485905167,2022-03-07T09:02:09.994+00:00,12345678,failed,refs/merge-requests/74/head,cce432655861efafcd05edc8de09e305c772d2b8,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485905167,687,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170,
-1,485932863,2022-03-07T09:34:57.476+00:00,12345678,success,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb271,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171,
-1,485932864,2022-03-07T09:34:57.476+00:00,12345678,canceled,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb272,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,172,
-1,485932865,2022-03-07T09:34:57.476+00:00,12345678,created,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb273,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,173,
-1,485932866,2022-03-07T09:34:57.476+00:00,12345678,waiting_for_resource,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb274,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,174,
-1,485932867,2022-03-07T09:34:57.476+00:00,12345678,preparing,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb275,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,175,
-1,485932868,2022-03-07T09:34:57.476+00:00,12345678,pending,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb276,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,176,
-1,485932869,2022-03-07T09:34:57.476+00:00,12345678,running,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb277,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,177,
-1,485932870,2022-03-07T09:34:57.476+00:00,12345678,manual,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb278,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,178,
-1,485932871,2022-03-07T09:34:57.476+00:00,12345678,scheduled,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,179,
-1,485932872,2022-03-07T09:34:57.476+00:00,12345678,skipped,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb270,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,180,
+1,1078343570,2023-11-20T07:17:08.753+00:00,12345678,success,refs/merge-requests/5394/head,694ed6ede74c367f17c7fddb56de629354a27069,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1078343570,1512,2023-11-20T07:17:10.123+00:00,2023-11-21T01:54:29.748+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,1,
+1,1079491591,2023-11-21T01:34:32.198+00:00,12345678,success,refs/merge-requests/5394/head,caed059a785dbfa72e1f04d2640b3580d8d67441,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079491591,1532,2023-11-21T01:34:36.759+00:00,2023-11-21T02:00:16.182+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,2,
+1,1079502706,2023-11-21T01:57:27.619+00:00,12345678,failed,refs/merge-requests/5500/head,919fe190728e79f3564cecc223ba8e6068addb87,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079502706,190,2023-11-21T01:57:31.260+00:00,2023-11-21T02:00:55.705+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,3,
+1,1079509085,2023-11-21T02:01:18.064+00:00,12345678,failed,refs/merge-requests/5500/head,4b0471ca50fd1f4f8c6d2313f741285aff07a5b5,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079509085,183,2023-11-21T02:01:29.601+00:00,2023-11-21T02:04:28.979+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,4,
+1,1079536077,2023-11-21T02:57:02.538+00:00,12345678,failed,refs/merge-requests/5500/head,eb651695e20ba35e8dee87e768eb04d46a466854,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079536077,202,2023-11-21T02:57:05.664+00:00,2023-11-21T03:00:28.040+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,5,
+1,1079536741,2023-11-21T02:59:22.311+00:00,12345678,failed,refs/merge-requests/5522/head,a83eb5619c1132d2f5a02213ac3bce4c9e29e348,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079536741,758,2023-11-21T02:59:26.579+00:00,2023-11-21T03:12:07.672+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,6,
+1,1079543310,2023-11-21T03:02:28.896+00:00,12345678,success,refs/merge-requests/5501/head,e59c1ef621e438ab1632e203d67f8ab955662aee,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079543310,1464,2023-11-21T03:02:32.680+00:00,2023-11-21T03:27:00.530+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,7,
+1,1079545236,2023-11-21T03:05:55.578+00:00,12345678,success,refs/merge-requests/5500/head,00bd6fb5e570661da63c4775234d84e88913e5b1,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079545236,1466,2023-11-21T03:05:56.822+00:00,2023-11-21T03:30:29.503+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,8,
+1,1079555385,2023-11-21T03:28:30.382+00:00,12345678,success,master,b5b8ec19fbddefe1374e87b05775f238ac06003e,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079555385,1384,2023-11-21T03:28:32.970+00:00,2023-11-21T03:51:44.872+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,9,
+1,1079557580,2023-11-21T03:30:51.330+00:00,12345678,success,refs/merge-requests/5522/head,f1aeedd7d744f02195d1cee9406a94609333abb6,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079557580,1486,2023-11-21T03:30:54.281+00:00,2023-11-21T03:55:49.264+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,10,
+1,1079558335,2023-11-21T03:33:09.746+00:00,12345678,success,refs/merge-requests/5522/head,03b72da7177a59a499d08dcaf036e37d5910863b,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079558335,1618,2023-11-21T03:33:12.402+00:00,2023-11-21T04:04:25.654+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,11,
+1,1079577769,2023-11-21T04:04:00.904+00:00,12345678,success,refs/merge-requests/5541/head,1e8ca37113a962d52a7bac5e738e817549a1217b,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079577769,1388,2023-11-21T04:04:04.131+00:00,2023-11-21T04:27:18.210+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,12,
+1,1079589242,2023-11-21T04:23:00.209+00:00,12345678,failed,refs/merge-requests/5500/head,393741a60c9395efab1d404816ff072af03d6f02,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079589242,181,2023-11-21T04:23:01.634+00:00,2023-11-21T04:26:03.503+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,13,
+1,1079594690,2023-11-21T04:31:16.546+00:00,12345678,success,refs/merge-requests/5500/head,c124f78f8def33fd2541782d335ba177bd85f948,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079594690,1447,2023-11-21T04:31:19.661+00:00,2023-11-21T04:55:30.474+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,14,
+1,1079602571,2023-11-21T04:47:19.253+00:00,12345678,success,refs/merge-requests/5519/head,76513320215d47fd384a0110e3b7f29538f5bbd7,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079602571,1470,2023-11-21T04:47:23.059+00:00,2023-11-21T05:12:07.091+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,15,
+1,1079620249,2023-11-21T05:13:37.061+00:00,12345678,success,refs/merge-requests/5522/head,3534c526a3ea0be1eb4ff2f0a6b89ca152762b0e,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079620249,1363,2023-11-21T05:13:38.637+00:00,2023-11-21T05:36:26.218+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,16,
diff --git
a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
index 01de48ed6..279c3a60a 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
@@ -1,30 +1,30 @@
-pipeline_id,commit_sha,branch,repo_id,repo_url
-gitlab:GitlabPipeline:1:457474837,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,renovate/pin-dependencies,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:457474996,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,renovate/jest-monorepo,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:457475160,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,renovate/lodash-monorepo,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:457475337,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,renovate/shx-0.x,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485811050,c791ea6949d6b4aadf79b15ba666cb690c6527ac,EE-7121,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485811059,c791ea6949d6b4aadf79b15ba666cb690c6527ac,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485813816,ecc7c0b2874c812ed882c9effbbda26e0abc7110,EE-7121,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485813830,ecc7c0b2874c812ed882c9effbbda26e0abc7110,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485814501,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,EE-7121,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485814516,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485814871,f731732471961fed061a01fa99631f69e861c4a2,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485817670,5b95c5aebce1eae6a1b95ecf6fbc870851455375,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485837602,a79e2aaeee9916338a11b665cb4201f24627f3f1,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485842553,9e0934cedcd83abab8d34c5bb0f597c7837a55f0,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485845850,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485852752,82df93eeb72c71271dad5a85359f39661de899fe,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485865876,db45efd231dbaca1337bd29bc1f65f404754e6bd,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485877118,09f81b1b2d083411c0bfecd32d7728479b594503,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485905167,cce432655861efafcd05edc8de09e305c772d2b8,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932863,12fc3a42080bb98ca520817bd4fe0ca33c0bb271,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932864,12fc3a42080bb98ca520817bd4fe0ca33c0bb272,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932865,12fc3a42080bb98ca520817bd4fe0ca33c0bb273,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932866,12fc3a42080bb98ca520817bd4fe0ca33c0bb274,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932867,12fc3a42080bb98ca520817bd4fe0ca33c0bb275,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932868,12fc3a42080bb98ca520817bd4fe0ca33c0bb276,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932869,12fc3a42080bb98ca520817bd4fe0ca33c0bb277,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932870,12fc3a42080bb98ca520817bd4fe0ca33c0bb278,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932871,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932872,12fc3a42080bb98ca520817bd4fe0ca33c0bb270,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+pipeline_id,commit_sha,commit_msg,branch,repo_id,repo_url
+gitlab:GitlabPipeline:1:457474837,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,,renovate/pin-dependencies,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:457474996,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,,renovate/jest-monorepo,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:457475160,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,,renovate/lodash-monorepo,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:457475337,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,,renovate/shx-0.x,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485811050,c791ea6949d6b4aadf79b15ba666cb690c6527ac,,EE-7121,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485811059,c791ea6949d6b4aadf79b15ba666cb690c6527ac,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485813816,ecc7c0b2874c812ed882c9effbbda26e0abc7110,,EE-7121,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485813830,ecc7c0b2874c812ed882c9effbbda26e0abc7110,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485814501,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,,EE-7121,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485814516,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485814871,f731732471961fed061a01fa99631f69e861c4a2,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485817670,5b95c5aebce1eae6a1b95ecf6fbc870851455375,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485837602,a79e2aaeee9916338a11b665cb4201f24627f3f1,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485842553,9e0934cedcd83abab8d34c5bb0f597c7837a55f0,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485845850,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485852752,82df93eeb72c71271dad5a85359f39661de899fe,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485865876,db45efd231dbaca1337bd29bc1f65f404754e6bd,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485877118,09f81b1b2d083411c0bfecd32d7728479b594503,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485905167,cce432655861efafcd05edc8de09e305c772d2b8,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932863,12fc3a42080bb98ca520817bd4fe0ca33c0bb271,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932864,12fc3a42080bb98ca520817bd4fe0ca33c0bb272,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932865,12fc3a42080bb98ca520817bd4fe0ca33c0bb273,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932866,12fc3a42080bb98ca520817bd4fe0ca33c0bb274,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932867,12fc3a42080bb98ca520817bd4fe0ca33c0bb275,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932868,12fc3a42080bb98ca520817bd4fe0ca33c0bb276,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932869,12fc3a42080bb98ca520817bd4fe0ca33c0bb277,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932870,12fc3a42080bb98ca520817bd4fe0ca33c0bb278,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932871,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932872,12fc3a42080bb98ca520817bd4fe0ca33c0bb270,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
index e2da5ced4..11576635a 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
@@ -1,30 +1,17 @@
id,name,result,status,original_status,original_result,type,duration_sec,queued_duration_sec,environment,created_date,queued_date,started_date,finished_date,cicd_scope_id
-gitlab:GitlabPipeline:1:457474837,gitlab:GitlabPipeline:1:457474837,,IN_PROGRESS,running,,,0,0,,2022-01-27T10:07:09.429+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:457474996,gitlab:GitlabPipeline:1:457474996,FAILURE,DONE,failed,,,0,0,,2022-01-27T10:07:18.884+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:457475160,gitlab:GitlabPipeline:1:457475160,FAILURE,DONE,failed,,,0,0,,2022-01-27T10:07:26.435+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:457475337,gitlab:GitlabPipeline:1:457475337,,IN_PROGRESS,running,,,0,0,,2022-01-27T10:07:36.502+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485811050,gitlab:GitlabPipeline:1:485811050,FAILURE,DONE,failed,,DEPLOYMENT,0,0,PRODUCTION,2022-03-07T06:26:42.109+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485811059,gitlab:GitlabPipeline:1:485811059,FAILURE,DONE,failed,,,0,0,,2022-03-07T06:26:43.784+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485813816,gitlab:GitlabPipeline:1:485813816,FAILURE,DONE,failed,,DEPLOYMENT,0,0,PRODUCTION,2022-03-07T06:33:56.824+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485813830,gitlab:GitlabPipeline:1:485813830,FAILURE,DONE,failed,,,0,0,,2022-03-07T06:33:58.889+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485814501,gitlab:GitlabPipeline:1:485814501,FAILURE,DONE,failed,,DEPLOYMENT,0,0,PRODUCTION,2022-03-07T06:35:28.111+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485814516,gitlab:GitlabPipeline:1:485814516,FAILURE,DONE,failed,,,0,0,,2022-03-07T06:35:31.255+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485814871,gitlab:GitlabPipeline:1:485814871,FAILURE,DONE,failed,,,42,0,,2022-03-07T06:36:50.020+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485817670,gitlab:GitlabPipeline:1:485817670,FAILURE,DONE,failed,,,1956,0,,2022-03-07T06:45:09.471+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485837602,gitlab:GitlabPipeline:1:485837602,FAILURE,DONE,failed,,,434,0,,2022-03-07T07:20:45.859+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485842553,gitlab:GitlabPipeline:1:485842553,FAILURE,DONE,failed,,,287,0,,2022-03-07T07:30:47.018+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485845850,gitlab:GitlabPipeline:1:485845850,FAILURE,DONE,failed,,,419,0,,2022-03-07T07:38:58.611+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485852752,gitlab:GitlabPipeline:1:485852752,FAILURE,DONE,failed,,,319,0,,2022-03-07T07:46:09.385+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485865876,gitlab:GitlabPipeline:1:485865876,FAILURE,DONE,failed,,,480,0,,2022-03-07T08:04:56.406+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485877118,gitlab:GitlabPipeline:1:485877118,FAILURE,DONE,failed,,,289,0,,2022-03-07T08:22:48.943+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485905167,gitlab:GitlabPipeline:1:485905167,FAILURE,DONE,failed,,,687,0,,2022-03-07T09:02:09.994+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932863,gitlab:GitlabPipeline:1:485932863,SUCCESS,DONE,success,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932864,gitlab:GitlabPipeline:1:485932864,FAILURE,DONE,canceled,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932865,gitlab:GitlabPipeline:1:485932865,,OTHER,created,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932866,gitlab:GitlabPipeline:1:485932866,,IN_PROGRESS,waiting_for_resource,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932867,gitlab:GitlabPipeline:1:485932867,,IN_PROGRESS,preparing,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932868,gitlab:GitlabPipeline:1:485932868,,IN_PROGRESS,pending,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932869,gitlab:GitlabPipeline:1:485932869,,IN_PROGRESS,running,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932870,gitlab:GitlabPipeline:1:485932870,,OTHER,manual,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932871,gitlab:GitlabPipeline:1:485932871,,OTHER,scheduled,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932872,gitlab:GitlabPipeline:1:485932872,,OTHER,skipped,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1078343570,gitlab:GitlabPipeline:1:1078343570,SUCCESS,DONE,success,,,1512,1,,2023-11-20T07:17:08.753+00:00,,2023-11-20T07:17:10.123+00:00,2023-11-21T01:54:29.748+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079491591,gitlab:GitlabPipeline:1:1079491591,SUCCESS,DONE,success,,,1532,4,,2023-11-21T01:34:32.198+00:00,,2023-11-21T01:34:36.759+00:00,2023-11-21T02:00:16.182+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079502706,gitlab:GitlabPipeline:1:1079502706,FAILURE,DONE,failed,,,190,3,,2023-11-21T01:57:27.619+00:00,,2023-11-21T01:57:31.260+00:00,2023-11-21T02:00:55.705+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079509085,gitlab:GitlabPipeline:1:1079509085,FAILURE,DONE,failed,,,183,11,,2023-11-21T02:01:18.064+00:00,,2023-11-21T02:01:29.601+00:00,2023-11-21T02:04:28.979+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079536077,gitlab:GitlabPipeline:1:1079536077,FAILURE,DONE,failed,,,202,3,,2023-11-21T02:57:02.538+00:00,,2023-11-21T02:57:05.664+00:00,2023-11-21T03:00:28.040+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079536741,gitlab:GitlabPipeline:1:1079536741,FAILURE,DONE,failed,,,758,4,,2023-11-21T02:59:22.311+00:00,,2023-11-21T02:59:26.579+00:00,2023-11-21T03:12:07.672+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079543310,gitlab:GitlabPipeline:1:1079543310,SUCCESS,DONE,success,,,1464,3,,2023-11-21T03:02:28.896+00:00,,2023-11-21T03:02:32.680+00:00,2023-11-21T03:27:00.530+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079545236,gitlab:GitlabPipeline:1:1079545236,SUCCESS,DONE,success,,,1466,1,,2023-11-21T03:05:55.578+00:00,,2023-11-21T03:05:56.822+00:00,2023-11-21T03:30:29.503+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079555385,gitlab:GitlabPipeline:1:1079555385,SUCCESS,DONE,success,,,1384,2,,2023-11-21T03:28:30.382+00:00,,2023-11-21T03:28:32.970+00:00,2023-11-21T03:51:44.872+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079557580,gitlab:GitlabPipeline:1:1079557580,SUCCESS,DONE,success,,,1486,2,,2023-11-21T03:30:51.330+00:00,,2023-11-21T03:30:54.281+00:00,2023-11-21T03:55:49.264+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079558335,gitlab:GitlabPipeline:1:1079558335,SUCCESS,DONE,success,,,1618,2,,2023-11-21T03:33:09.746+00:00,,2023-11-21T03:33:12.402+00:00,2023-11-21T04:04:25.654+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079577769,gitlab:GitlabPipeline:1:1079577769,SUCCESS,DONE,success,,,1388,3,,2023-11-21T04:04:00.904+00:00,,2023-11-21T04:04:04.131+00:00,2023-11-21T04:27:18.210+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079589242,gitlab:GitlabPipeline:1:1079589242,FAILURE,DONE,failed,,,181,1,,2023-11-21T04:23:00.209+00:00,,2023-11-21T04:23:01.634+00:00,2023-11-21T04:26:03.503+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079594690,gitlab:GitlabPipeline:1:1079594690,SUCCESS,DONE,success,,,1447,3,,2023-11-21T04:31:16.546+00:00,,2023-11-21T04:31:19.661+00:00,2023-11-21T04:55:30.474+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079602571,gitlab:GitlabPipeline:1:1079602571,SUCCESS,DONE,success,,,1470,3,,2023-11-21T04:47:19.253+00:00,,2023-11-21T04:47:23.059+00:00,2023-11-21T05:12:07.091+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079620249,gitlab:GitlabPipeline:1:1079620249,SUCCESS,DONE,success,,,1363,1,,2023-11-21T05:13:37.061+00:00,,2023-11-21T05:13:38.637+00:00,2023-11-21T05:36:26.218+00:00,gitlab:GitlabProject:1:12345678
diff --git
a/backend/plugins/gitlab/models/migrationscripts/20231221_add_time_filed_to_pipeline_projects.go
b/backend/plugins/gitlab/models/migrationscripts/20231221_add_time_filed_to_pipeline_projects.go
new file mode 100644
index 000000000..8b9e0a04f
--- /dev/null
+++
b/backend/plugins/gitlab/models/migrationscripts/20231221_add_time_filed_to_pipeline_projects.go
@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type gitlabPipelineProject20231221 struct {
+ GitlabCreatedAt *time.Time
+ GitlabUpdatedAt *time.Time
+}
+
+func (gitlabPipelineProject20231221) TableName() string {
+ return "_tool_gitlab_pipeline_projects"
+}
+
+type addTimeToGitlabPipelineProject struct{}
+
+func (script *addTimeToGitlabPipelineProject) Up(basicRes context.BasicRes)
errors.Error {
+ return migrationhelper.AutoMigrateTables(
+ basicRes,
+ &gitlabPipelineProject20231221{},
+ )
+}
+
+func (*addTimeToGitlabPipelineProject) Version() uint64 {
+ return 20231221161031
+}
+
+func (*addTimeToGitlabPipelineProject) Name() string {
+ return "add time filed to _tool_gitlab_pipeline_projects"
+}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go
b/backend/plugins/gitlab/models/migrationscripts/register.go
index 01152d1b6..c82c09db3 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/register.go
@@ -44,5 +44,6 @@ func All() []plugin.MigrationScript {
new(addEnvNamePattern),
new(addQueuedDuration20231129),
new(modifyDeploymentMessageType),
+ new(addTimeToGitlabPipelineProject),
}
}
diff --git a/backend/plugins/gitlab/models/pipeline.go
b/backend/plugins/gitlab/models/pipeline.go
index d14dd424b..37b357c2e 100644
--- a/backend/plugins/gitlab/models/pipeline.go
+++ b/backend/plugins/gitlab/models/pipeline.go
@@ -54,11 +54,13 @@ func (GitlabPipeline) TableName() string {
}
type GitlabPipelineProject struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- PipelineId int `gorm:"primaryKey"`
- ProjectId int `gorm:"primaryKey"`
- Ref string `gorm:"type:varchar(255)"`
- Sha string `gorm:"type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ PipelineId int `gorm:"primaryKey"`
+ ProjectId int `gorm:"primaryKey"`
+ Ref string `gorm:"type:varchar(255)"`
+ Sha string `gorm:"type:varchar(255)"`
+ GitlabCreatedAt *time.Time
+ GitlabUpdatedAt *time.Time
common.NoPKModel
}
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
b/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
index aa4c80148..88605af66 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
@@ -66,7 +66,7 @@ func CollectApiPipelineDetails(taskCtx plugin.SubTaskContext)
errors.Error {
ApiClient: data.ApiClient,
MinTickInterval: &tickInterval,
Input: iterator,
- UrlTemplate: "projects/{{ .Params.ProjectId
}}/pipelines/{{ .Input.GitlabId }}",
+ UrlTemplate: "projects/{{ .Params.ProjectId
}}/pipelines/{{ .Input.PipelineId }}",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("with_stats", "true")
@@ -86,11 +86,11 @@ func GetPipelinesIterator(taskCtx plugin.SubTaskContext,
collectorWithState *hel
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GitlabTaskData)
clauses := []dal.Clause{
- dal.Select("gp.gitlab_id,gp.gitlab_id as iid"),
- dal.From("_tool_gitlab_pipelines gp"),
+ dal.Select("gp.pipeline_id"),
+ dal.From("_tool_gitlab_pipeline_projects gp"),
dal.Where(
- `gp.project_id = ? and gp.connection_id = ? and
gp.is_detail_required = ?`,
- data.Options.ProjectId, data.Options.ConnectionId, true,
+ `gp.project_id = ? and gp.connection_id = ?`,
+ data.Options.ProjectId, data.Options.ConnectionId,
),
}
if collectorWithState.Since != nil {
@@ -102,5 +102,9 @@ func GetPipelinesIterator(taskCtx plugin.SubTaskContext,
collectorWithState *hel
return nil, err
}
- return helper.NewDalCursorIterator(db, cursor,
reflect.TypeOf(GitlabInput{}))
+ return helper.NewDalCursorIterator(db, cursor,
reflect.TypeOf(PipelineInput{}))
+}
+
+type PipelineInput struct {
+ PipelineId int
}
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
index 928f560cb..52c12c04a 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
@@ -53,9 +54,6 @@ func ExtractApiPipelineDetails(taskCtx plugin.SubTaskContext)
errors.Error {
return nil, err
}
- if gitlabApiPipeline.UpdatedAt != nil &&
gitlabApiPipeline.CreatedAt != nil {
- gitlabApiPipeline.Duration =
int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.CreatedAt.ToTime()).Seconds())
- }
gitlabPipeline := &models.GitlabPipeline{
GitlabId: gitlabApiPipeline.Id,
ProjectId: data.Options.ProjectId,
@@ -70,31 +68,18 @@ func ExtractApiPipelineDetails(taskCtx
plugin.SubTaskContext) errors.Error {
Duration: gitlabApiPipeline.Duration,
QueuedDuration:
gitlabApiPipeline.QueuedDuration,
ConnectionId: data.Options.ConnectionId,
-
- Type:
data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT,
gitlabApiPipeline.Ref),
- Environment:
data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION,
gitlabApiPipeline.Ref),
-
- IsDetailRequired: false,
+ Type:
data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT,
gitlabApiPipeline.Ref),
+ Environment:
data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION,
gitlabApiPipeline.Ref),
}
if err != nil {
return nil, err
}
- pipelineProject := &models.GitlabPipelineProject{
- ConnectionId: data.Options.ConnectionId,
- PipelineId: gitlabPipeline.GitlabId,
- ProjectId: data.Options.ProjectId,
- Ref: gitlabApiPipeline.Ref,
- Sha: gitlabApiPipeline.Sha,
- }
-
- results := make([]interface{}, 0, 2)
- results = append(results, gitlabPipeline,
pipelineProject)
-
+ results := make([]interface{}, 0, 1)
+ results = append(results, gitlabPipeline)
return results, nil
},
})
-
if err != nil {
return err
}
diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go
b/backend/plugins/gitlab/tasks/pipeline_extractor.go
index 39a722117..aaf756bd7 100644
--- a/backend/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go
@@ -22,7 +22,6 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
@@ -83,45 +82,18 @@ func ExtractApiPipelines(taskCtx plugin.SubTaskContext)
errors.Error {
return nil, err
}
- if gitlabApiPipeline.UpdatedAt != nil &&
gitlabApiPipeline.CreatedAt != nil {
- gitlabApiPipeline.Duration =
int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.CreatedAt.ToTime()).Seconds())
- }
-
- gitlabPipeline := &models.GitlabPipeline{
- GitlabId: gitlabApiPipeline.Id,
+ pipelineProject := &models.GitlabPipelineProject{
+ ConnectionId: data.Options.ConnectionId,
+ PipelineId: gitlabApiPipeline.Id,
ProjectId: data.Options.ProjectId,
Ref: gitlabApiPipeline.Ref,
Sha: gitlabApiPipeline.Sha,
- WebUrl: gitlabApiPipeline.WebUrl,
- Status: gitlabApiPipeline.Status,
GitlabCreatedAt:
common.Iso8601TimeToTime(gitlabApiPipeline.CreatedAt),
GitlabUpdatedAt:
common.Iso8601TimeToTime(gitlabApiPipeline.UpdatedAt),
- StartedAt:
common.Iso8601TimeToTime(gitlabApiPipeline.StartedAt),
- FinishedAt:
common.Iso8601TimeToTime(gitlabApiPipeline.FinishedAt),
- Duration: gitlabApiPipeline.Duration,
- QueuedDuration:
gitlabApiPipeline.QueuedDuration,
- ConnectionId: data.Options.ConnectionId,
-
- Type:
data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT,
gitlabApiPipeline.Ref),
- Environment:
data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION,
gitlabApiPipeline.Ref),
-
- IsDetailRequired: false,
- }
-
- if gitlabApiPipeline.StartedAt == nil &&
gitlabApiPipeline.FinishedAt == nil {
- gitlabPipeline.IsDetailRequired = true
- }
-
- pipelineProject := &models.GitlabPipelineProject{
- ConnectionId: data.Options.ConnectionId,
- PipelineId: gitlabPipeline.GitlabId,
- ProjectId: data.Options.ProjectId,
- Ref: gitlabApiPipeline.Ref,
- Sha: gitlabApiPipeline.Sha,
}
- results := make([]interface{}, 0, 2)
- results = append(results, gitlabPipeline,
pipelineProject)
+ results := make([]interface{}, 0, 1)
+ results = append(results, pipelineProject)
return results, nil
},