This is an automated email from the ASF dual-hosted git repository.

klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new 1d210d409 fix: duration_sec has wrong value (#6688)
1d210d409 is described below

commit 1d210d40952ca0a5f5c4c624da487830832e5765
Author: abeizn <[email protected]>
AuthorDate: Mon Dec 25 12:52:40 2023 +0800

    fix: duration_sec has wrong value (#6688)
    
    * fix: duration_sec has wrong value
    
    * fix: remove some useless code
    
    * fix: pipeline detail e2e
    
    * fix: pipeline e2e
    
    * fix: pipeline e2e
    
    * fix: gitlab pipelines e2e test
    
    * fix: gitlab pipelines e2e test
---
 .../plugins/gitlab/e2e/pipelines_detail_test.go    | 23 +++------
 backend/plugins/gitlab/e2e/pipelines_test.go       | 55 +-------------------
 .../_raw_gitlab_api_pipeline_details.csv           | 17 ++++++
 .../e2e/snapshot_tables/_tool_gitlab_pipelines.csv | 45 ++++++----------
 .../e2e/snapshot_tables/cicd_pipeline_commits.csv  | 60 +++++++++++-----------
 .../gitlab/e2e/snapshot_tables/cicd_pipelines.csv  | 45 ++++++----------
 ...20231221_add_time_filed_to_pipeline_projects.go | 52 +++++++++++++++++++
 .../gitlab/models/migrationscripts/register.go     |  1 +
 backend/plugins/gitlab/models/pipeline.go          | 12 +++--
 .../gitlab/tasks/pipeline_detail_collector.go      | 16 +++---
 .../gitlab/tasks/pipeline_detail_extractor.go      | 25 ++-------
 backend/plugins/gitlab/tasks/pipeline_extractor.go | 38 ++------------
 12 files changed, 166 insertions(+), 223 deletions(-)

diff --git a/backend/plugins/gitlab/e2e/pipelines_detail_test.go 
b/backend/plugins/gitlab/e2e/pipelines_detail_test.go
index 9d1d68830..33f884f31 100644
--- a/backend/plugins/gitlab/e2e/pipelines_detail_test.go
+++ b/backend/plugins/gitlab/e2e/pipelines_detail_test.go
@@ -48,16 +48,16 @@ func TestGitlabPipelineDetailDataFlow(t *testing.T) {
        }
 
        // import raw data table
-       
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_pipeline.csv",
 "_raw_gitlab_api_pipeline_details")
-       
dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_gitlab_projects.csv", 
&models.GitlabProject{})
+       dataflowTester.FlushTabler(&models.GitlabPipelineProject{})
+       dataflowTester.FlushTabler(models.GitlabPipeline{})
+       
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_pipeline_details.csv",
 "_raw_gitlab_api_pipeline_details")
+       
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_gitlab_pipeline_projects.csv",
 &models.GitlabPipelineProject{})
 
        // verify extraction
-       dataflowTester.FlushTabler(&models.GitlabPipeline{})
-       dataflowTester.FlushTabler(&models.GitlabPipelineProject{})
        dataflowTester.Subtask(tasks.ExtractApiPipelineDetailsMeta, taskData)
        dataflowTester.VerifyTable(
                models.GitlabPipeline{},
-               "./snapshot_tables/_raw_gitlab_api_pipeline_details.csv",
+               "./snapshot_tables/_tool_gitlab_pipelines.csv",
                e2ehelper.ColumnWithRawData(
                        "connection_id",
                        "gitlab_id",
@@ -76,19 +76,8 @@ func TestGitlabPipelineDetailDataFlow(t *testing.T) {
                ),
        )
 
-       dataflowTester.VerifyTable(
-               models.GitlabPipelineProject{},
-               "./snapshot_tables/_tool_gitlab_pipeline_projects_details.csv",
-               e2ehelper.ColumnWithRawData(
-                       "connection_id",
-                       "pipeline_id",
-                       "project_id",
-                       "ref",
-                       "sha",
-               ),
-       )
-
        // verify conversion
+       
dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_gitlab_projects.csv", 
&models.GitlabProject{})
        dataflowTester.FlushTabler(&devops.CICDPipeline{})
        dataflowTester.FlushTabler(&devops.CiCDPipelineCommit{})
        dataflowTester.Subtask(tasks.ConvertPipelineMeta, taskData)
diff --git a/backend/plugins/gitlab/e2e/pipelines_test.go 
b/backend/plugins/gitlab/e2e/pipelines_test.go
index 620a11d5b..513e2e944 100644
--- a/backend/plugins/gitlab/e2e/pipelines_test.go
+++ b/backend/plugins/gitlab/e2e/pipelines_test.go
@@ -20,7 +20,6 @@ package e2e
 import (
        "testing"
 
-       "github.com/apache/incubator-devlake/core/models/common"
        "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
        "github.com/apache/incubator-devlake/helpers/e2ehelper"
        "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -46,47 +45,10 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
                RegexEnricher: regexEnricher,
        }
 
-       // import raw data table
-       
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_pipeline.csv",
 "_raw_gitlab_api_pipeline")
-       
dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_gitlab_projects.csv", 
&models.GitlabProject{})
-
        // verify env when production is omitted
-       dataflowTester.FlushTabler(&models.GitlabPipeline{})
-       dataflowTester.FlushTabler(&models.GitlabPipelineProject{})
-       dataflowTester.Subtask(tasks.ExtractApiPipelinesMeta, taskData)
-       dataflowTester.VerifyTable(
-               models.GitlabPipeline{},
-               "./snapshot_tables/_tool_gitlab_pipelines_no_prod_regex.csv",
-               e2ehelper.ColumnWithRawData(
-                       "environment",
-               ),
-       )
-
-       // verify extraction
-       _ = regexEnricher.TryAdd(devops.PRODUCTION, "EE-7121")
-       dataflowTester.FlushTabler(&models.GitlabPipeline{})
        dataflowTester.FlushTabler(&models.GitlabPipelineProject{})
+       
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_pipeline.csv",
 "_raw_gitlab_api_pipeline")
        dataflowTester.Subtask(tasks.ExtractApiPipelinesMeta, taskData)
-       dataflowTester.VerifyTable(
-               models.GitlabPipeline{},
-               "./snapshot_tables/_tool_gitlab_pipelines.csv",
-               e2ehelper.ColumnWithRawData(
-                       "connection_id",
-                       "gitlab_id",
-                       "gitlab_created_at",
-                       "project_id",
-                       "status",
-                       "ref",
-                       "sha",
-                       "web_url",
-                       "duration",
-                       "started_at",
-                       "finished_at",
-                       "coverage",
-                       "type",
-                       "environment",
-               ),
-       )
 
        dataflowTester.VerifyTable(
                models.GitlabPipelineProject{},
@@ -99,19 +61,4 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
                        "sha",
                ),
        )
-
-       // verify conversion
-       dataflowTester.FlushTabler(&devops.CICDPipeline{})
-       dataflowTester.FlushTabler(&devops.CiCDPipelineCommit{})
-       dataflowTester.Subtask(tasks.ConvertPipelineMeta, taskData)
-       dataflowTester.Subtask(tasks.ConvertPipelineCommitMeta, taskData)
-       dataflowTester.VerifyTableWithOptions(&devops.CICDPipeline{}, 
e2ehelper.TableOptions{
-               CSVRelPath:  "./snapshot_tables/cicd_pipelines.csv",
-               IgnoreTypes: []interface{}{common.NoPKModel{}},
-       })
-
-       dataflowTester.VerifyTableWithOptions(&devops.CiCDPipelineCommit{}, 
e2ehelper.TableOptions{
-               CSVRelPath:  "./snapshot_tables/cicd_pipeline_commits.csv",
-               IgnoreTypes: []interface{}{common.NoPKModel{}},
-       })
 }
diff --git 
a/backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline_details.csv 
b/backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline_details.csv
new file mode 100644
index 000000000..4377e8137
--- /dev/null
+++ b/backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline_details.csv
@@ -0,0 +1,17 @@
+"id","params","data","url","input","created_at"
+1,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1078343570,""iid"":36229,""project_id"":12345678,""sha"":""694ed6ede74c367f17c7fddb56de629354a27069"",""ref"":""refs/merge-requests/5394/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-20T07:17:08.753Z"",""updated_at"":""2023-11-21T01:54:29.757Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1078343570"",""before_sha"":""0000000000000000000000000000000000000000"",""tag
 [...]
+2,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079491591,""iid"":36241,""project_id"":12345678,""sha"":""caed059a785dbfa72e1f04d2640b3580d8d67441"",""ref"":""refs/merge-requests/5394/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T01:34:32.198Z"",""updated_at"":""2023-11-21T02:00:16.216Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079491591"",""before_sha"":""0000000000000000000000000000000000000000"",""tag
 [...]
+3,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079502706,""iid"":36242,""project_id"":12345678,""sha"":""919fe190728e79f3564cecc223ba8e6068addb87"",""ref"":""refs/merge-requests/5500/head"",""status"":""failed"",""source"":""merge_request_event"",""created_at"":""2023-11-21T01:57:27.619Z"",""updated_at"":""2023-11-21T02:00:57.015Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079502706"",""before_sha"":""0000000000000000000000000000000000000000"",""tag";
 [...]
+4,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079509085,""iid"":36243,""project_id"":12345678,""sha"":""4b0471ca50fd1f4f8c6d2313f741285aff07a5b5"",""ref"":""refs/merge-requests/5500/head"",""status"":""failed"",""source"":""merge_request_event"",""created_at"":""2023-11-21T02:01:18.064Z"",""updated_at"":""2023-11-21T02:04:28.990Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079509085"",""before_sha"":""0000000000000000000000000000000000000000"",""tag";
 [...]
+5,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079536077,""iid"":36244,""project_id"":12345678,""sha"":""eb651695e20ba35e8dee87e768eb04d46a466854"",""ref"":""refs/merge-requests/5500/head"",""status"":""failed"",""source"":""merge_request_event"",""created_at"":""2023-11-21T02:57:02.538Z"",""updated_at"":""2023-11-21T03:00:28.052Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079536077"",""before_sha"":""0000000000000000000000000000000000000000"",""tag";
 [...]
+6,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079536741,""iid"":36245,""project_id"":12345678,""sha"":""a83eb5619c1132d2f5a02213ac3bce4c9e29e348"",""ref"":""refs/merge-requests/5522/head"",""status"":""failed"",""source"":""merge_request_event"",""created_at"":""2023-11-21T02:59:22.311Z"",""updated_at"":""2023-11-21T03:12:07.685Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079536741"",""before_sha"":""0000000000000000000000000000000000000000"",""tag";
 [...]
+7,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079543310,""iid"":36246,""project_id"":12345678,""sha"":""e59c1ef621e438ab1632e203d67f8ab955662aee"",""ref"":""refs/merge-requests/5501/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T03:02:28.896Z"",""updated_at"":""2023-11-21T03:27:00.551Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079543310"",""before_sha"":""0000000000000000000000000000000000000000"",""tag
 [...]
+8,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079545236,""iid"":36247,""project_id"":12345678,""sha"":""00bd6fb5e570661da63c4775234d84e88913e5b1"",""ref"":""refs/merge-requests/5500/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T03:05:55.578Z"",""updated_at"":""2023-11-21T03:30:29.620Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079545236"",""before_sha"":""0000000000000000000000000000000000000000"",""tag
 [...]
+9,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079555385,""iid"":36248,""project_id"":12345678,""sha"":""b5b8ec19fbddefe1374e87b05775f238ac06003e"",""ref"":""master"",""status"":""success"",""source"":""push"",""created_at"":""2023-11-21T03:28:30.382Z"",""updated_at"":""2023-11-21T03:51:44.879Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079555385"",""before_sha"":""0275c5b0341fba564c615bbcc154671bc3de9b82"",""tag"":false,""yaml_errors"":null,""user"";
 [...]
+10,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079557580,""iid"":36249,""project_id"":12345678,""sha"":""f1aeedd7d744f02195d1cee9406a94609333abb6"",""ref"":""refs/merge-requests/5522/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T03:30:51.330Z"",""updated_at"":""2023-11-21T03:55:49.274Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079557580"",""before_sha"":""0000000000000000000000000000000000000000"",""ta
 [...]
+11,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079558335,""iid"":36250,""project_id"":12345678,""sha"":""03b72da7177a59a499d08dcaf036e37d5910863b"",""ref"":""refs/merge-requests/5522/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T03:33:09.746Z"",""updated_at"":""2023-11-21T04:04:25.665Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079558335"",""before_sha"":""0000000000000000000000000000000000000000"",""ta
 [...]
+12,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079577769,""iid"":36251,""project_id"":12345678,""sha"":""1e8ca37113a962d52a7bac5e738e817549a1217b"",""ref"":""refs/merge-requests/5541/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T04:04:00.904Z"",""updated_at"":""2023-11-21T04:27:18.236Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079577769"",""before_sha"":""0000000000000000000000000000000000000000"",""ta
 [...]
+13,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079589242,""iid"":36252,""project_id"":12345678,""sha"":""393741a60c9395efab1d404816ff072af03d6f02"",""ref"":""refs/merge-requests/5500/head"",""status"":""failed"",""source"":""merge_request_event"",""created_at"":""2023-11-21T04:23:00.209Z"",""updated_at"":""2023-11-21T04:26:03.518Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079589242"",""before_sha"":""0000000000000000000000000000000000000000"",""tag
 [...]
+14,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079594690,""iid"":36253,""project_id"":12345678,""sha"":""c124f78f8def33fd2541782d335ba177bd85f948"",""ref"":""refs/merge-requests/5500/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T04:31:16.546Z"",""updated_at"":""2023-11-21T04:55:30.486Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079594690"",""before_sha"":""0000000000000000000000000000000000000000"",""ta
 [...]
+15,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079602571,""iid"":36254,""project_id"":12345678,""sha"":""76513320215d47fd384a0110e3b7f29538f5bbd7"",""ref"":""refs/merge-requests/5519/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T04:47:19.253Z"",""updated_at"":""2023-11-21T05:12:07.103Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079602571"",""before_sha"":""0000000000000000000000000000000000000000"",""ta
 [...]
+16,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":1079620249,""iid"":36255,""project_id"":12345678,""sha"":""3534c526a3ea0be1eb4ff2f0a6b89ca152762b0e"",""ref"":""refs/merge-requests/5522/head"",""status"":""success"",""source"":""merge_request_event"",""created_at"":""2023-11-21T05:13:37.061Z"",""updated_at"":""2023-11-21T05:36:26.228Z"",""web_url"":""https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079620249"",""before_sha"":""0000000000000000000000000000000000000000"",""ta
 [...]
\ No newline at end of file
diff --git 
a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv 
b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
index 1d62c7baa..c44bdd788 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
@@ -1,30 +1,17 @@
 
connection_id,gitlab_id,gitlab_created_at,project_id,status,ref,sha,web_url,duration,started_at,finished_at,coverage,type,environment,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,457474837,2022-01-27T10:07:09.429+00:00,12345678,running,renovate/pin-dependencies,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
-1,457474996,2022-01-27T10:07:18.884+00:00,12345678,failed,renovate/jest-monorepo,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
-1,457475160,2022-01-27T10:07:26.435+00:00,12345678,failed,renovate/lodash-monorepo,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
-1,457475337,2022-01-27T10:07:36.502+00:00,12345678,running,renovate/shx-0.x,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
-1,485811050,2022-03-07T06:26:42.109+00:00,12345678,failed,EE-7121,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,,,,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156,
-1,485811059,2022-03-07T06:26:43.784+00:00,12345678,failed,refs/merge-requests/74/head,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157,
-1,485813816,2022-03-07T06:33:56.824+00:00,12345678,failed,EE-7121,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,,,,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158,
-1,485813830,2022-03-07T06:33:58.889+00:00,12345678,failed,refs/merge-requests/74/head,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813830,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159,
-1,485814501,2022-03-07T06:35:28.111+00:00,12345678,failed,EE-7121,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814501,0,,,,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160,
-1,485814516,2022-03-07T06:35:31.255+00:00,12345678,failed,refs/merge-requests/74/head,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814516,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161,
-1,485814871,2022-03-07T06:36:50.020+00:00,12345678,failed,refs/merge-requests/74/head,f731732471961fed061a01fa99631f69e861c4a2,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814871,42,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162,
-1,485817670,2022-03-07T06:45:09.471+00:00,12345678,failed,refs/merge-requests/74/head,5b95c5aebce1eae6a1b95ecf6fbc870851455375,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485817670,1956,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163,
-1,485837602,2022-03-07T07:20:45.859+00:00,12345678,failed,refs/merge-requests/74/head,a79e2aaeee9916338a11b665cb4201f24627f3f1,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485837602,434,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164,
-1,485842553,2022-03-07T07:30:47.018+00:00,12345678,failed,refs/merge-requests/74/head,9e0934cedcd83abab8d34c5bb0f597c7837a55f0,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485842553,287,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165,
-1,485845850,2022-03-07T07:38:58.611+00:00,12345678,failed,refs/merge-requests/74/head,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485845850,419,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166,
-1,485852752,2022-03-07T07:46:09.385+00:00,12345678,failed,refs/merge-requests/74/head,82df93eeb72c71271dad5a85359f39661de899fe,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485852752,319,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167,
-1,485865876,2022-03-07T08:04:56.406+00:00,12345678,failed,refs/merge-requests/74/head,db45efd231dbaca1337bd29bc1f65f404754e6bd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485865876,480,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168,
-1,485877118,2022-03-07T08:22:48.943+00:00,12345678,failed,refs/merge-requests/74/head,09f81b1b2d083411c0bfecd32d7728479b594503,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485877118,289,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169,
-1,485905167,2022-03-07T09:02:09.994+00:00,12345678,failed,refs/merge-requests/74/head,cce432655861efafcd05edc8de09e305c772d2b8,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485905167,687,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170,
-1,485932863,2022-03-07T09:34:57.476+00:00,12345678,success,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb271,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171,
-1,485932864,2022-03-07T09:34:57.476+00:00,12345678,canceled,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb272,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,172,
-1,485932865,2022-03-07T09:34:57.476+00:00,12345678,created,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb273,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,173,
-1,485932866,2022-03-07T09:34:57.476+00:00,12345678,waiting_for_resource,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb274,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,174,
-1,485932867,2022-03-07T09:34:57.476+00:00,12345678,preparing,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb275,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,175,
-1,485932868,2022-03-07T09:34:57.476+00:00,12345678,pending,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb276,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,176,
-1,485932869,2022-03-07T09:34:57.476+00:00,12345678,running,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb277,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,177,
-1,485932870,2022-03-07T09:34:57.476+00:00,12345678,manual,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb278,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,178,
-1,485932871,2022-03-07T09:34:57.476+00:00,12345678,scheduled,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,179,
-1,485932872,2022-03-07T09:34:57.476+00:00,12345678,skipped,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb270,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,180,
+1,1078343570,2023-11-20T07:17:08.753+00:00,12345678,success,refs/merge-requests/5394/head,694ed6ede74c367f17c7fddb56de629354a27069,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1078343570,1512,2023-11-20T07:17:10.123+00:00,2023-11-21T01:54:29.748+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,1,
+1,1079491591,2023-11-21T01:34:32.198+00:00,12345678,success,refs/merge-requests/5394/head,caed059a785dbfa72e1f04d2640b3580d8d67441,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079491591,1532,2023-11-21T01:34:36.759+00:00,2023-11-21T02:00:16.182+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,2,
+1,1079502706,2023-11-21T01:57:27.619+00:00,12345678,failed,refs/merge-requests/5500/head,919fe190728e79f3564cecc223ba8e6068addb87,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079502706,190,2023-11-21T01:57:31.260+00:00,2023-11-21T02:00:55.705+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,3,
+1,1079509085,2023-11-21T02:01:18.064+00:00,12345678,failed,refs/merge-requests/5500/head,4b0471ca50fd1f4f8c6d2313f741285aff07a5b5,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079509085,183,2023-11-21T02:01:29.601+00:00,2023-11-21T02:04:28.979+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,4,
+1,1079536077,2023-11-21T02:57:02.538+00:00,12345678,failed,refs/merge-requests/5500/head,eb651695e20ba35e8dee87e768eb04d46a466854,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079536077,202,2023-11-21T02:57:05.664+00:00,2023-11-21T03:00:28.040+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,5,
+1,1079536741,2023-11-21T02:59:22.311+00:00,12345678,failed,refs/merge-requests/5522/head,a83eb5619c1132d2f5a02213ac3bce4c9e29e348,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079536741,758,2023-11-21T02:59:26.579+00:00,2023-11-21T03:12:07.672+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,6,
+1,1079543310,2023-11-21T03:02:28.896+00:00,12345678,success,refs/merge-requests/5501/head,e59c1ef621e438ab1632e203d67f8ab955662aee,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079543310,1464,2023-11-21T03:02:32.680+00:00,2023-11-21T03:27:00.530+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,7,
+1,1079545236,2023-11-21T03:05:55.578+00:00,12345678,success,refs/merge-requests/5500/head,00bd6fb5e570661da63c4775234d84e88913e5b1,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079545236,1466,2023-11-21T03:05:56.822+00:00,2023-11-21T03:30:29.503+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,8,
+1,1079555385,2023-11-21T03:28:30.382+00:00,12345678,success,master,b5b8ec19fbddefe1374e87b05775f238ac06003e,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079555385,1384,2023-11-21T03:28:32.970+00:00,2023-11-21T03:51:44.872+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,9,
+1,1079557580,2023-11-21T03:30:51.330+00:00,12345678,success,refs/merge-requests/5522/head,f1aeedd7d744f02195d1cee9406a94609333abb6,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079557580,1486,2023-11-21T03:30:54.281+00:00,2023-11-21T03:55:49.264+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,10,
+1,1079558335,2023-11-21T03:33:09.746+00:00,12345678,success,refs/merge-requests/5522/head,03b72da7177a59a499d08dcaf036e37d5910863b,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079558335,1618,2023-11-21T03:33:12.402+00:00,2023-11-21T04:04:25.654+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,11,
+1,1079577769,2023-11-21T04:04:00.904+00:00,12345678,success,refs/merge-requests/5541/head,1e8ca37113a962d52a7bac5e738e817549a1217b,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079577769,1388,2023-11-21T04:04:04.131+00:00,2023-11-21T04:27:18.210+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,12,
+1,1079589242,2023-11-21T04:23:00.209+00:00,12345678,failed,refs/merge-requests/5500/head,393741a60c9395efab1d404816ff072af03d6f02,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079589242,181,2023-11-21T04:23:01.634+00:00,2023-11-21T04:26:03.503+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,13,
+1,1079594690,2023-11-21T04:31:16.546+00:00,12345678,success,refs/merge-requests/5500/head,c124f78f8def33fd2541782d335ba177bd85f948,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079594690,1447,2023-11-21T04:31:19.661+00:00,2023-11-21T04:55:30.474+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,14,
+1,1079602571,2023-11-21T04:47:19.253+00:00,12345678,success,refs/merge-requests/5519/head,76513320215d47fd384a0110e3b7f29538f5bbd7,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079602571,1470,2023-11-21T04:47:23.059+00:00,2023-11-21T05:12:07.091+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,15,
+1,1079620249,2023-11-21T05:13:37.061+00:00,12345678,success,refs/merge-requests/5522/head,3534c526a3ea0be1eb4ff2f0a6b89ca152762b0e,https://gitlab.com/merico-dev/ee/vdev.co/-/pipelines/1079620249,1363,2023-11-21T05:13:38.637+00:00,2023-11-21T05:36:26.218+00:00,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline_details,16,
diff --git 
a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv 
b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
index 01de48ed6..279c3a60a 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
@@ -1,30 +1,30 @@
-pipeline_id,commit_sha,branch,repo_id,repo_url
-gitlab:GitlabPipeline:1:457474837,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,renovate/pin-dependencies,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:457474996,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,renovate/jest-monorepo,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:457475160,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,renovate/lodash-monorepo,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:457475337,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,renovate/shx-0.x,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485811050,c791ea6949d6b4aadf79b15ba666cb690c6527ac,EE-7121,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485811059,c791ea6949d6b4aadf79b15ba666cb690c6527ac,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485813816,ecc7c0b2874c812ed882c9effbbda26e0abc7110,EE-7121,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485813830,ecc7c0b2874c812ed882c9effbbda26e0abc7110,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485814501,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,EE-7121,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485814516,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485814871,f731732471961fed061a01fa99631f69e861c4a2,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485817670,5b95c5aebce1eae6a1b95ecf6fbc870851455375,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485837602,a79e2aaeee9916338a11b665cb4201f24627f3f1,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485842553,9e0934cedcd83abab8d34c5bb0f597c7837a55f0,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485845850,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485852752,82df93eeb72c71271dad5a85359f39661de899fe,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485865876,db45efd231dbaca1337bd29bc1f65f404754e6bd,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485877118,09f81b1b2d083411c0bfecd32d7728479b594503,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485905167,cce432655861efafcd05edc8de09e305c772d2b8,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932863,12fc3a42080bb98ca520817bd4fe0ca33c0bb271,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932864,12fc3a42080bb98ca520817bd4fe0ca33c0bb272,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932865,12fc3a42080bb98ca520817bd4fe0ca33c0bb273,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932866,12fc3a42080bb98ca520817bd4fe0ca33c0bb274,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932867,12fc3a42080bb98ca520817bd4fe0ca33c0bb275,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932868,12fc3a42080bb98ca520817bd4fe0ca33c0bb276,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932869,12fc3a42080bb98ca520817bd4fe0ca33c0bb277,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932870,12fc3a42080bb98ca520817bd4fe0ca33c0bb278,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932871,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
-gitlab:GitlabPipeline:1:485932872,12fc3a42080bb98ca520817bd4fe0ca33c0bb270,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+pipeline_id,commit_sha,commit_msg,branch,repo_id,repo_url
+gitlab:GitlabPipeline:1:457474837,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,,renovate/pin-dependencies,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:457474996,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,,renovate/jest-monorepo,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:457475160,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,,renovate/lodash-monorepo,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:457475337,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,,renovate/shx-0.x,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485811050,c791ea6949d6b4aadf79b15ba666cb690c6527ac,,EE-7121,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485811059,c791ea6949d6b4aadf79b15ba666cb690c6527ac,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485813816,ecc7c0b2874c812ed882c9effbbda26e0abc7110,,EE-7121,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485813830,ecc7c0b2874c812ed882c9effbbda26e0abc7110,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485814501,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,,EE-7121,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485814516,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485814871,f731732471961fed061a01fa99631f69e861c4a2,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485817670,5b95c5aebce1eae6a1b95ecf6fbc870851455375,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485837602,a79e2aaeee9916338a11b665cb4201f24627f3f1,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485842553,9e0934cedcd83abab8d34c5bb0f597c7837a55f0,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485845850,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485852752,82df93eeb72c71271dad5a85359f39661de899fe,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485865876,db45efd231dbaca1337bd29bc1f65f404754e6bd,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485877118,09f81b1b2d083411c0bfecd32d7728479b594503,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485905167,cce432655861efafcd05edc8de09e305c772d2b8,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932863,12fc3a42080bb98ca520817bd4fe0ca33c0bb271,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932864,12fc3a42080bb98ca520817bd4fe0ca33c0bb272,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932865,12fc3a42080bb98ca520817bd4fe0ca33c0bb273,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932866,12fc3a42080bb98ca520817bd4fe0ca33c0bb274,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932867,12fc3a42080bb98ca520817bd4fe0ca33c0bb275,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932868,12fc3a42080bb98ca520817bd4fe0ca33c0bb276,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932869,12fc3a42080bb98ca520817bd4fe0ca33c0bb277,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932870,12fc3a42080bb98ca520817bd4fe0ca33c0bb278,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932871,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932872,12fc3a42080bb98ca520817bd4fe0ca33c0bb270,,refs/merge-requests/74/head,gitlab:GitlabProject:1:12345678,https://gitlab.com/gitlab-data/snowflake_spend
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv 
b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
index e2da5ced4..11576635a 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
@@ -1,30 +1,17 @@
 
id,name,result,status,original_status,original_result,type,duration_sec,queued_duration_sec,environment,created_date,queued_date,started_date,finished_date,cicd_scope_id
-gitlab:GitlabPipeline:1:457474837,gitlab:GitlabPipeline:1:457474837,,IN_PROGRESS,running,,,0,0,,2022-01-27T10:07:09.429+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:457474996,gitlab:GitlabPipeline:1:457474996,FAILURE,DONE,failed,,,0,0,,2022-01-27T10:07:18.884+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:457475160,gitlab:GitlabPipeline:1:457475160,FAILURE,DONE,failed,,,0,0,,2022-01-27T10:07:26.435+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:457475337,gitlab:GitlabPipeline:1:457475337,,IN_PROGRESS,running,,,0,0,,2022-01-27T10:07:36.502+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485811050,gitlab:GitlabPipeline:1:485811050,FAILURE,DONE,failed,,DEPLOYMENT,0,0,PRODUCTION,2022-03-07T06:26:42.109+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485811059,gitlab:GitlabPipeline:1:485811059,FAILURE,DONE,failed,,,0,0,,2022-03-07T06:26:43.784+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485813816,gitlab:GitlabPipeline:1:485813816,FAILURE,DONE,failed,,DEPLOYMENT,0,0,PRODUCTION,2022-03-07T06:33:56.824+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485813830,gitlab:GitlabPipeline:1:485813830,FAILURE,DONE,failed,,,0,0,,2022-03-07T06:33:58.889+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485814501,gitlab:GitlabPipeline:1:485814501,FAILURE,DONE,failed,,DEPLOYMENT,0,0,PRODUCTION,2022-03-07T06:35:28.111+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485814516,gitlab:GitlabPipeline:1:485814516,FAILURE,DONE,failed,,,0,0,,2022-03-07T06:35:31.255+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485814871,gitlab:GitlabPipeline:1:485814871,FAILURE,DONE,failed,,,42,0,,2022-03-07T06:36:50.020+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485817670,gitlab:GitlabPipeline:1:485817670,FAILURE,DONE,failed,,,1956,0,,2022-03-07T06:45:09.471+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485837602,gitlab:GitlabPipeline:1:485837602,FAILURE,DONE,failed,,,434,0,,2022-03-07T07:20:45.859+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485842553,gitlab:GitlabPipeline:1:485842553,FAILURE,DONE,failed,,,287,0,,2022-03-07T07:30:47.018+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485845850,gitlab:GitlabPipeline:1:485845850,FAILURE,DONE,failed,,,419,0,,2022-03-07T07:38:58.611+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485852752,gitlab:GitlabPipeline:1:485852752,FAILURE,DONE,failed,,,319,0,,2022-03-07T07:46:09.385+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485865876,gitlab:GitlabPipeline:1:485865876,FAILURE,DONE,failed,,,480,0,,2022-03-07T08:04:56.406+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485877118,gitlab:GitlabPipeline:1:485877118,FAILURE,DONE,failed,,,289,0,,2022-03-07T08:22:48.943+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485905167,gitlab:GitlabPipeline:1:485905167,FAILURE,DONE,failed,,,687,0,,2022-03-07T09:02:09.994+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932863,gitlab:GitlabPipeline:1:485932863,SUCCESS,DONE,success,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932864,gitlab:GitlabPipeline:1:485932864,FAILURE,DONE,canceled,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932865,gitlab:GitlabPipeline:1:485932865,,OTHER,created,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932866,gitlab:GitlabPipeline:1:485932866,,IN_PROGRESS,waiting_for_resource,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932867,gitlab:GitlabPipeline:1:485932867,,IN_PROGRESS,preparing,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932868,gitlab:GitlabPipeline:1:485932868,,IN_PROGRESS,pending,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932869,gitlab:GitlabPipeline:1:485932869,,IN_PROGRESS,running,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932870,gitlab:GitlabPipeline:1:485932870,,OTHER,manual,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932871,gitlab:GitlabPipeline:1:485932871,,OTHER,scheduled,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
-gitlab:GitlabPipeline:1:485932872,gitlab:GitlabPipeline:1:485932872,,OTHER,skipped,,,398,0,,2022-03-07T09:34:57.476+00:00,,,,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1078343570,gitlab:GitlabPipeline:1:1078343570,SUCCESS,DONE,success,,,1512,1,,2023-11-20T07:17:08.753+00:00,,2023-11-20T07:17:10.123+00:00,2023-11-21T01:54:29.748+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079491591,gitlab:GitlabPipeline:1:1079491591,SUCCESS,DONE,success,,,1532,4,,2023-11-21T01:34:32.198+00:00,,2023-11-21T01:34:36.759+00:00,2023-11-21T02:00:16.182+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079502706,gitlab:GitlabPipeline:1:1079502706,FAILURE,DONE,failed,,,190,3,,2023-11-21T01:57:27.619+00:00,,2023-11-21T01:57:31.260+00:00,2023-11-21T02:00:55.705+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079509085,gitlab:GitlabPipeline:1:1079509085,FAILURE,DONE,failed,,,183,11,,2023-11-21T02:01:18.064+00:00,,2023-11-21T02:01:29.601+00:00,2023-11-21T02:04:28.979+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079536077,gitlab:GitlabPipeline:1:1079536077,FAILURE,DONE,failed,,,202,3,,2023-11-21T02:57:02.538+00:00,,2023-11-21T02:57:05.664+00:00,2023-11-21T03:00:28.040+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079536741,gitlab:GitlabPipeline:1:1079536741,FAILURE,DONE,failed,,,758,4,,2023-11-21T02:59:22.311+00:00,,2023-11-21T02:59:26.579+00:00,2023-11-21T03:12:07.672+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079543310,gitlab:GitlabPipeline:1:1079543310,SUCCESS,DONE,success,,,1464,3,,2023-11-21T03:02:28.896+00:00,,2023-11-21T03:02:32.680+00:00,2023-11-21T03:27:00.530+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079545236,gitlab:GitlabPipeline:1:1079545236,SUCCESS,DONE,success,,,1466,1,,2023-11-21T03:05:55.578+00:00,,2023-11-21T03:05:56.822+00:00,2023-11-21T03:30:29.503+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079555385,gitlab:GitlabPipeline:1:1079555385,SUCCESS,DONE,success,,,1384,2,,2023-11-21T03:28:30.382+00:00,,2023-11-21T03:28:32.970+00:00,2023-11-21T03:51:44.872+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079557580,gitlab:GitlabPipeline:1:1079557580,SUCCESS,DONE,success,,,1486,2,,2023-11-21T03:30:51.330+00:00,,2023-11-21T03:30:54.281+00:00,2023-11-21T03:55:49.264+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079558335,gitlab:GitlabPipeline:1:1079558335,SUCCESS,DONE,success,,,1618,2,,2023-11-21T03:33:09.746+00:00,,2023-11-21T03:33:12.402+00:00,2023-11-21T04:04:25.654+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079577769,gitlab:GitlabPipeline:1:1079577769,SUCCESS,DONE,success,,,1388,3,,2023-11-21T04:04:00.904+00:00,,2023-11-21T04:04:04.131+00:00,2023-11-21T04:27:18.210+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079589242,gitlab:GitlabPipeline:1:1079589242,FAILURE,DONE,failed,,,181,1,,2023-11-21T04:23:00.209+00:00,,2023-11-21T04:23:01.634+00:00,2023-11-21T04:26:03.503+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079594690,gitlab:GitlabPipeline:1:1079594690,SUCCESS,DONE,success,,,1447,3,,2023-11-21T04:31:16.546+00:00,,2023-11-21T04:31:19.661+00:00,2023-11-21T04:55:30.474+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079602571,gitlab:GitlabPipeline:1:1079602571,SUCCESS,DONE,success,,,1470,3,,2023-11-21T04:47:19.253+00:00,,2023-11-21T04:47:23.059+00:00,2023-11-21T05:12:07.091+00:00,gitlab:GitlabProject:1:12345678
+gitlab:GitlabPipeline:1:1079620249,gitlab:GitlabPipeline:1:1079620249,SUCCESS,DONE,success,,,1363,1,,2023-11-21T05:13:37.061+00:00,,2023-11-21T05:13:38.637+00:00,2023-11-21T05:36:26.218+00:00,gitlab:GitlabProject:1:12345678
diff --git 
a/backend/plugins/gitlab/models/migrationscripts/20231221_add_time_filed_to_pipeline_projects.go
 
b/backend/plugins/gitlab/models/migrationscripts/20231221_add_time_filed_to_pipeline_projects.go
new file mode 100644
index 000000000..8b9e0a04f
--- /dev/null
+++ 
b/backend/plugins/gitlab/models/migrationscripts/20231221_add_time_filed_to_pipeline_projects.go
@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+       "time"
+
+       "github.com/apache/incubator-devlake/core/context"
+       "github.com/apache/incubator-devlake/core/errors"
+       "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type gitlabPipelineProject20231221 struct {
+       GitlabCreatedAt *time.Time
+       GitlabUpdatedAt *time.Time
+}
+
+func (gitlabPipelineProject20231221) TableName() string {
+       return "_tool_gitlab_pipeline_projects"
+}
+
+type addTimeToGitlabPipelineProject struct{}
+
+func (script *addTimeToGitlabPipelineProject) Up(basicRes context.BasicRes) 
errors.Error {
+       return migrationhelper.AutoMigrateTables(
+               basicRes,
+               &gitlabPipelineProject20231221{},
+       )
+}
+
+func (*addTimeToGitlabPipelineProject) Version() uint64 {
+       return 20231221161031
+}
+
+func (*addTimeToGitlabPipelineProject) Name() string {
+       return "add time filed to _tool_gitlab_pipeline_projects"
+}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go 
b/backend/plugins/gitlab/models/migrationscripts/register.go
index 01152d1b6..c82c09db3 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/register.go
@@ -44,5 +44,6 @@ func All() []plugin.MigrationScript {
                new(addEnvNamePattern),
                new(addQueuedDuration20231129),
                new(modifyDeploymentMessageType),
+               new(addTimeToGitlabPipelineProject),
        }
 }
diff --git a/backend/plugins/gitlab/models/pipeline.go 
b/backend/plugins/gitlab/models/pipeline.go
index d14dd424b..37b357c2e 100644
--- a/backend/plugins/gitlab/models/pipeline.go
+++ b/backend/plugins/gitlab/models/pipeline.go
@@ -54,11 +54,13 @@ func (GitlabPipeline) TableName() string {
 }
 
 type GitlabPipelineProject struct {
-       ConnectionId uint64 `gorm:"primaryKey"`
-       PipelineId   int    `gorm:"primaryKey"`
-       ProjectId    int    `gorm:"primaryKey"`
-       Ref          string `gorm:"type:varchar(255)"`
-       Sha          string `gorm:"type:varchar(255)"`
+       ConnectionId    uint64 `gorm:"primaryKey"`
+       PipelineId      int    `gorm:"primaryKey"`
+       ProjectId       int    `gorm:"primaryKey"`
+       Ref             string `gorm:"type:varchar(255)"`
+       Sha             string `gorm:"type:varchar(255)"`
+       GitlabCreatedAt *time.Time
+       GitlabUpdatedAt *time.Time
        common.NoPKModel
 }
 
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_collector.go 
b/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
index aa4c80148..88605af66 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
@@ -66,7 +66,7 @@ func CollectApiPipelineDetails(taskCtx plugin.SubTaskContext) 
errors.Error {
                ApiClient:          data.ApiClient,
                MinTickInterval:    &tickInterval,
                Input:              iterator,
-               UrlTemplate:        "projects/{{ .Params.ProjectId 
}}/pipelines/{{ .Input.GitlabId }}",
+               UrlTemplate:        "projects/{{ .Params.ProjectId 
}}/pipelines/{{ .Input.PipelineId }}",
                Query: func(reqData *helper.RequestData) (url.Values, 
errors.Error) {
                        query := url.Values{}
                        query.Set("with_stats", "true")
@@ -86,11 +86,11 @@ func GetPipelinesIterator(taskCtx plugin.SubTaskContext, 
collectorWithState *hel
        db := taskCtx.GetDal()
        data := taskCtx.GetData().(*GitlabTaskData)
        clauses := []dal.Clause{
-               dal.Select("gp.gitlab_id,gp.gitlab_id as iid"),
-               dal.From("_tool_gitlab_pipelines gp"),
+               dal.Select("gp.pipeline_id"),
+               dal.From("_tool_gitlab_pipeline_projects gp"),
                dal.Where(
-                       `gp.project_id = ? and gp.connection_id = ? and 
gp.is_detail_required = ?`,
-                       data.Options.ProjectId, data.Options.ConnectionId, true,
+                       `gp.project_id = ? and gp.connection_id = ?`,
+                       data.Options.ProjectId, data.Options.ConnectionId,
                ),
        }
        if collectorWithState.Since != nil {
@@ -102,5 +102,9 @@ func GetPipelinesIterator(taskCtx plugin.SubTaskContext, 
collectorWithState *hel
                return nil, err
        }
 
-       return helper.NewDalCursorIterator(db, cursor, 
reflect.TypeOf(GitlabInput{}))
+       return helper.NewDalCursorIterator(db, cursor, 
reflect.TypeOf(PipelineInput{}))
+}
+
+type PipelineInput struct {
+       PipelineId int
 }
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go 
b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
index 928f560cb..52c12c04a 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
@@ -19,6 +19,7 @@ package tasks
 
 import (
        "encoding/json"
+
        "github.com/apache/incubator-devlake/core/errors"
        "github.com/apache/incubator-devlake/core/models/common"
        "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
@@ -53,9 +54,6 @@ func ExtractApiPipelineDetails(taskCtx plugin.SubTaskContext) 
errors.Error {
                                return nil, err
                        }
 
-                       if gitlabApiPipeline.UpdatedAt != nil && 
gitlabApiPipeline.CreatedAt != nil {
-                               gitlabApiPipeline.Duration = 
int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.CreatedAt.ToTime()).Seconds())
-                       }
                        gitlabPipeline := &models.GitlabPipeline{
                                GitlabId:        gitlabApiPipeline.Id,
                                ProjectId:       data.Options.ProjectId,
@@ -70,31 +68,18 @@ func ExtractApiPipelineDetails(taskCtx 
plugin.SubTaskContext) errors.Error {
                                Duration:        gitlabApiPipeline.Duration,
                                QueuedDuration:  
gitlabApiPipeline.QueuedDuration,
                                ConnectionId:    data.Options.ConnectionId,
-
-                               Type:        
data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, 
gitlabApiPipeline.Ref),
-                               Environment: 
data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, 
gitlabApiPipeline.Ref),
-
-                               IsDetailRequired: false,
+                               Type:            
data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, 
gitlabApiPipeline.Ref),
+                               Environment:     
data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, 
gitlabApiPipeline.Ref),
                        }
                        if err != nil {
                                return nil, err
                        }
 
-                       pipelineProject := &models.GitlabPipelineProject{
-                               ConnectionId: data.Options.ConnectionId,
-                               PipelineId:   gitlabPipeline.GitlabId,
-                               ProjectId:    data.Options.ProjectId,
-                               Ref:          gitlabApiPipeline.Ref,
-                               Sha:          gitlabApiPipeline.Sha,
-                       }
-
-                       results := make([]interface{}, 0, 2)
-                       results = append(results, gitlabPipeline, 
pipelineProject)
-
+                       results := make([]interface{}, 0, 1)
+                       results = append(results, gitlabPipeline)
                        return results, nil
                },
        })
-
        if err != nil {
                return err
        }
diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go 
b/backend/plugins/gitlab/tasks/pipeline_extractor.go
index 39a722117..aaf756bd7 100644
--- a/backend/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go
@@ -22,7 +22,6 @@ import (
 
        "github.com/apache/incubator-devlake/core/errors"
        "github.com/apache/incubator-devlake/core/models/common"
-       "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
        "github.com/apache/incubator-devlake/core/plugin"
        "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
        "github.com/apache/incubator-devlake/plugins/gitlab/models"
@@ -83,45 +82,18 @@ func ExtractApiPipelines(taskCtx plugin.SubTaskContext) 
errors.Error {
                                return nil, err
                        }
 
-                       if gitlabApiPipeline.UpdatedAt != nil && 
gitlabApiPipeline.CreatedAt != nil {
-                               gitlabApiPipeline.Duration = 
int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.CreatedAt.ToTime()).Seconds())
-                       }
-
-                       gitlabPipeline := &models.GitlabPipeline{
-                               GitlabId:        gitlabApiPipeline.Id,
+                       pipelineProject := &models.GitlabPipelineProject{
+                               ConnectionId:    data.Options.ConnectionId,
+                               PipelineId:      gitlabApiPipeline.Id,
                                ProjectId:       data.Options.ProjectId,
                                Ref:             gitlabApiPipeline.Ref,
                                Sha:             gitlabApiPipeline.Sha,
-                               WebUrl:          gitlabApiPipeline.WebUrl,
-                               Status:          gitlabApiPipeline.Status,
                                GitlabCreatedAt: 
common.Iso8601TimeToTime(gitlabApiPipeline.CreatedAt),
                                GitlabUpdatedAt: 
common.Iso8601TimeToTime(gitlabApiPipeline.UpdatedAt),
-                               StartedAt:       
common.Iso8601TimeToTime(gitlabApiPipeline.StartedAt),
-                               FinishedAt:      
common.Iso8601TimeToTime(gitlabApiPipeline.FinishedAt),
-                               Duration:        gitlabApiPipeline.Duration,
-                               QueuedDuration:  
gitlabApiPipeline.QueuedDuration,
-                               ConnectionId:    data.Options.ConnectionId,
-
-                               Type:        
data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, 
gitlabApiPipeline.Ref),
-                               Environment: 
data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, 
gitlabApiPipeline.Ref),
-
-                               IsDetailRequired: false,
-                       }
-
-                       if gitlabApiPipeline.StartedAt == nil && 
gitlabApiPipeline.FinishedAt == nil {
-                               gitlabPipeline.IsDetailRequired = true
-                       }
-
-                       pipelineProject := &models.GitlabPipelineProject{
-                               ConnectionId: data.Options.ConnectionId,
-                               PipelineId:   gitlabPipeline.GitlabId,
-                               ProjectId:    data.Options.ProjectId,
-                               Ref:          gitlabApiPipeline.Ref,
-                               Sha:          gitlabApiPipeline.Sha,
                        }
 
-                       results := make([]interface{}, 0, 2)
-                       results = append(results, gitlabPipeline, 
pipelineProject)
+                       results := make([]interface{}, 0, 1)
+                       results = append(results, pipelineProject)
 
                        return results, nil
                },

Reply via email to