This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new fb6f20d9f Fix/circleci column names (#8799)
fb6f20d9f is described below
commit fb6f20d9f36055349b21c6357dc0c3c79d947737
Author: Chris Pavlicek <[email protected]>
AuthorDate: Mon Mar 23 14:21:55 2026 +0000
Fix/circleci column names (#8799)
* fix(circleci): rename created_at to created_date in jobs/workflows
Add migration to copy created_at -> created_date and update
models/converters.
* fix(circleci): update pipeline parsing
* test(circleci): add incremental tests for collectors
---
backend/plugins/circleci/e2e/incremental_test.go | 154 +++++++++++++++++++++
.../_raw_circleci_api_jobs_incremental.csv | 73 ++++++++++
.../_raw_circleci_api_workflows_incremental.csv | 67 +++++++++
.../e2e/snapshot_tables/_tool_circleci_jobs.csv | 20 +--
.../_tool_circleci_jobs_incremental.csv | 7 +
.../snapshot_tables/_tool_circleci_pipelines.csv | 22 +--
.../_tool_circleci_workflows_incremental.csv | 7 +
.../e2e/snapshot_tables/cicd_pipelines.csv | 20 +--
backend/plugins/circleci/e2e/workflow_test.go | 2 +-
backend/plugins/circleci/models/job.go | 2 +-
.../20260322_rename_created_at_to_created_date.go | 71 ++++++++++
.../circleci/models/migrationscripts/register.go | 1 +
backend/plugins/circleci/models/pipeline.go | 4 +-
backend/plugins/circleci/models/workflow.go | 2 +-
backend/plugins/circleci/tasks/job_converter.go | 4 +-
.../plugins/circleci/tasks/workflow_converter.go | 6 +-
.../plugins/circleci/tasks/workflow_extractor.go | 4 +-
17 files changed, 423 insertions(+), 43 deletions(-)
diff --git a/backend/plugins/circleci/e2e/incremental_test.go
b/backend/plugins/circleci/e2e/incremental_test.go
new file mode 100644
index 000000000..85de8a7ba
--- /dev/null
+++ b/backend/plugins/circleci/e2e/incremental_test.go
@@ -0,0 +1,154 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "sort"
+ "testing"
+ "time"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/circleci/impl"
+ "github.com/apache/incubator-devlake/plugins/circleci/models"
+ "github.com/apache/incubator-devlake/plugins/circleci/tasks"
+ "github.com/stretchr/testify/assert"
+)
+
+// TestCircleciWorkflowIncremental verifies that the workflow collector's
incremental
+// logic correctly filters pipelines by created_date. This is a regression
test for
+// the created_at -> created_date column rename.
+func TestCircleciWorkflowIncremental(t *testing.T) {
+ var circleci impl.Circleci
+
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "circleci", circleci)
+ taskData := &tasks.CircleciTaskData{
+ Options: &tasks.CircleciOptions{
+ ConnectionId: 1,
+ ProjectSlug: "github/coldgust/coldgust.github.io",
+ },
+ RegexEnricher: api.NewRegexEnricher(),
+ }
+
+ // seed pipelines table via extraction
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_pipelines.csv",
"_raw_circleci_api_pipelines")
+ dataflowTester.FlushTabler(&models.CircleciPipeline{})
+ dataflowTester.Subtask(tasks.ExtractPipelinesMeta, taskData)
+
+ // Part 1: verify the SQL query used by BuildInputIterator works with
created_date.
+ // Pipelines #4-10 have created_date > 17:45 — assert the exact IDs
returned.
+ createdAfter := time.Date(2023, 3, 25, 17, 45, 0, 0, time.UTC)
+ var pipelines []models.CircleciPipeline
+ assert.Nil(t, dataflowTester.Dal.All(&pipelines,
+ dal.Where("connection_id = ? AND project_slug = ? AND
created_date > ?",
+ 1, "github/coldgust/coldgust.github.io", createdAfter),
+ ))
+ pipelineIds := make([]string, len(pipelines))
+ for i, p := range pipelines {
+ pipelineIds[i] = p.Id
+ }
+ sort.Strings(pipelineIds)
+ assert.Equal(t, []string{
+ "23622ee4-e150-4920-9d66-81533fa765a4", // pipeline #5
+ "2c45280f-7fb3-4025-b703-a547c4a94916", // pipeline #4
+ "70f3eb15-3b94-4f80-b65e-f23f4b74c33a", // pipeline #6
+ "7fcc1623-edcc-4a76-ad20-cd81aa83519f", // pipeline #9
+ "866e967d-f826-4470-aed6-fc0c92e98703", // pipeline #7
+ "afe0cabe-e7ee-4eb7-bf13-bb6170d139f0", // pipeline #8
+ "d323f088-02fa-4ed5-9696-fc2f89a27150", // pipeline #10
+ }, pipelineIds)
+
+ // Part 2: verify extraction with only the incrementally-collected
workflow raw data
+ // (workflows for pipelines #4-9 only).
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_workflows_incremental.csv",
"_raw_circleci_api_workflows")
+ dataflowTester.FlushTabler(&models.CircleciWorkflow{})
+ dataflowTester.Subtask(tasks.ExtractWorkflowsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(
+ models.CircleciWorkflow{},
+ e2ehelper.TableOptions{
+ CSVRelPath:
"./snapshot_tables/_tool_circleci_workflows_incremental.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ IgnoreFields: []string{"started_at", "stopped_at"},
+ },
+ )
+}
+
+// TestCircleciJobIncremental verifies that the job collector's incremental
logic
+// correctly filters workflows by created_date. Regression test for the column
rename.
+func TestCircleciJobIncremental(t *testing.T) {
+ var circleci impl.Circleci
+
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "circleci", circleci)
+ taskData := &tasks.CircleciTaskData{
+ Options: &tasks.CircleciOptions{
+ ConnectionId: 1,
+ ProjectSlug: "github/coldgust/coldgust.github.io",
+ },
+ RegexEnricher: api.NewRegexEnricher(),
+ Project: &models.CircleciProject{
+ Id: "abcd",
+ },
+ }
+
+ // seed workflows table via extraction (all 10 workflows including
b3b77371 with null created_date)
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_workflows.csv",
"_raw_circleci_api_workflows")
+ dataflowTester.FlushTabler(&models.CircleciWorkflow{})
+ dataflowTester.Subtask(tasks.ExtractWorkflowsMeta, taskData)
+
+ // Part 1: verify the SQL query used by BuildInputIterator works with
created_date.
+ // Workflows for pipelines #4-9 have created_date > 17:45 — assert the
exact IDs returned.
+ // Workflow b3b77371 (null created_date) is excluded by the >
comparison.
+ createdAfter := time.Date(2023, 3, 25, 17, 45, 0, 0, time.UTC)
+ var workflows []models.CircleciWorkflow
+ assert.Nil(t, dataflowTester.Dal.All(&workflows,
+ dal.Where("connection_id = ? AND project_slug = ? AND
created_date > ?",
+ 1, "github/coldgust/coldgust.github.io", createdAfter),
+ ))
+ workflowIds := make([]string, len(workflows))
+ for i, w := range workflows {
+ workflowIds[i] = w.Id
+ }
+ sort.Strings(workflowIds)
+ assert.Equal(t, []string{
+ "6731159f-5275-4bfa-ba70-39d343d63814", // pipeline #5
+ "7370985a-9de3-4a47-acbc-e6a1fe8e5812", // pipeline #7
+ "b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14", // pipeline #6
+ "c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943", // pipeline #4
+ "fc76deef-bcdd-4856-8e96-a8e2d1c5a85f", // pipeline #8
+ "fd0bd4f5-264f-4e3c-a151-06153c018f78", // pipeline #9
+ }, workflowIds)
+
+ // Part 2: verify extraction with only the incrementally-collected job
raw data
+ // (jobs for workflows from pipelines #4-9 only).
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_projects.csv",
"_raw_circleci_api_projects")
+ dataflowTester.FlushTabler(&models.CircleciProject{})
+ dataflowTester.Subtask(tasks.ExtractProjectsMeta, taskData)
+
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_jobs_incremental.csv",
"_raw_circleci_api_jobs")
+ dataflowTester.FlushTabler(&models.CircleciJob{})
+ dataflowTester.Subtask(tasks.ExtractJobsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(
+ models.CircleciJob{},
+ e2ehelper.TableOptions{
+ CSVRelPath:
"./snapshot_tables/_tool_circleci_jobs_incremental.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ },
+ )
+}
diff --git
a/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_jobs_incremental.csv
b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_jobs_incremental.csv
new file mode 100644
index 000000000..24e3fbdcb
--- /dev/null
+++
b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_jobs_incremental.csv
@@ -0,0 +1,73 @@
+id,params,data,url,input,created_at
+20,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{
+ ""dependencies"" : [ ],
+ ""job_number"" : 5,
+ ""id"" : ""ab8c3282-0e74-4a41-834e-152a71280bed"",
+ ""started_at"" : ""2023-03-25T17:52:20Z"",
+ ""created_at"" : ""2023-03-25T17:52:20Z"",
+ ""name"" : ""build"",
+ ""project_slug"" : ""gh/coldgust/coldgust.github.io"",
+ ""status"" : ""failed"",
+ ""type"" : ""build"",
+ ""stopped_at"" : ""2023-03-25T17:52:23Z""
+
}",https://circleci.com/api/v2/workflow/6731159f-5275-4bfa-ba70-39d343d63814/job,"{""id"":
""6731159f-5275-4bfa-ba70-39d343d63814"", ""tag"": """", ""name"": """",
""status"": """", ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"":
""0001-01-01T00:00:00Z"", ""created_at"": null, ""errored_by"": """",
""started_by"": """", ""stopped_at"": null, ""canceled_by"": """",
""pipeline_id"": ""23622ee4-e150-4920-9d66-81533fa765a4"", ""ConnectionId"": 0,
""_raw_data_id"": 0, ""duration_sec [...]
+21,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{
+ ""dependencies"" : [ ],
+ ""job_number"" : 7,
+ ""id"" : ""a00f80bc-f759-4900-97a5-2d121d80bde8"",
+ ""started_at"" : ""2023-03-25T17:56:27Z"",
+ ""created_at"" : ""2023-03-25T17:56:27Z"",
+ ""name"" : ""build"",
+ ""project_slug"" : ""gh/coldgust/coldgust.github.io"",
+ ""status"" : ""failed"",
+ ""type"" : ""build"",
+ ""stopped_at"" : ""2023-03-25T17:56:43Z""
+
}",https://circleci.com/api/v2/workflow/7370985a-9de3-4a47-acbc-e6a1fe8e5812/job,"{""id"":
""7370985a-9de3-4a47-acbc-e6a1fe8e5812"", ""tag"": """", ""name"": """",
""status"": """", ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"":
""0001-01-01T00:00:00Z"", ""created_at"": null, ""errored_by"": """",
""started_by"": """", ""stopped_at"": null, ""canceled_by"": """",
""pipeline_id"": ""866e967d-f826-4470-aed6-fc0c92e98703"", ""ConnectionId"": 0,
""_raw_data_id"": 0, ""duration_sec [...]
+23,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{
+ ""dependencies"" : [ ],
+ ""job_number"" : 9,
+ ""id"" : ""2ff3594e-9da1-4306-aefa-77b72a97971e"",
+ ""started_at"" : ""2023-03-25T18:13:25Z"",
+ ""created_at"" : ""2023-03-25T18:13:25Z"",
+ ""name"" : ""build"",
+ ""project_slug"" : ""gh/coldgust/coldgust.github.io"",
+ ""status"" : ""success"",
+ ""type"" : ""build"",
+ ""stopped_at"" : ""2023-03-25T18:13:38Z""
+
}",https://circleci.com/api/v2/workflow/fd0bd4f5-264f-4e3c-a151-06153c018f78/job,"{""id"":
""fd0bd4f5-264f-4e3c-a151-06153c018f78"", ""tag"": """", ""name"": """",
""status"": """", ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"":
""0001-01-01T00:00:00Z"", ""created_at"": null, ""errored_by"": """",
""started_by"": """", ""stopped_at"": null, ""canceled_by"": """",
""pipeline_id"": ""7fcc1623-edcc-4a76-ad20-cd81aa83519f"", ""ConnectionId"": 0,
""_raw_data_id"": 0, ""duration_sec [...]
+24,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{
+ ""dependencies"" : [ ],
+ ""job_number"" : 6,
+ ""id"" : ""76c1f2cc-27ea-47aa-8167-48d2633abdba"",
+ ""started_at"" : ""2023-03-25T17:54:11Z"",
+ ""created_at"" : ""2023-03-25T17:54:11Z"",
+ ""name"" : ""build"",
+ ""project_slug"" : ""gh/coldgust/coldgust.github.io"",
+ ""status"" : ""failed"",
+ ""type"" : ""build"",
+ ""stopped_at"" : ""2023-03-25T17:54:23Z""
+
}",https://circleci.com/api/v2/workflow/b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14/job,"{""id"":
""b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14"", ""tag"": """", ""name"": """",
""status"": """", ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"":
""0001-01-01T00:00:00Z"", ""created_at"": null, ""errored_by"": """",
""started_by"": """", ""stopped_at"": null, ""canceled_by"": """",
""pipeline_id"": ""70f3eb15-3b94-4f80-b65e-f23f4b74c33a"", ""ConnectionId"": 0,
""_raw_data_id"": 0, ""duration_sec [...]
+25,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{
+ ""dependencies"" : [ ],
+ ""job_number"" : 4,
+ ""id"" : ""a4af3dd5-a3ae-48e8-b634-e2d63aafbb5b"",
+ ""started_at"" : ""2023-03-25T17:50:22Z"",
+ ""created_at"" : ""2023-03-25T17:50:22Z"",
+ ""name"" : ""build"",
+ ""project_slug"" : ""gh/coldgust/coldgust.github.io"",
+ ""status"" : ""failed"",
+ ""type"" : ""build"",
+ ""stopped_at"" : ""2023-03-25T17:50:25Z""
+
}",https://circleci.com/api/v2/workflow/c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943/job,"{""id"":
""c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943"", ""tag"": """", ""name"": """",
""status"": """", ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"":
""0001-01-01T00:00:00Z"", ""created_at"": null, ""errored_by"": """",
""started_by"": """", ""stopped_at"": null, ""canceled_by"": """",
""pipeline_id"": ""2c45280f-7fb3-4025-b703-a547c4a94916"", ""ConnectionId"": 0,
""_raw_data_id"": 0, ""duration_sec [...]
+27,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{
+ ""dependencies"" : [ ],
+ ""job_number"" : 8,
+ ""id"" : ""004e3e27-17d7-4ccb-9b21-a7f55bcf2b3e"",
+ ""started_at"" : ""2023-03-25T18:06:15Z"",
+ ""created_at"" : ""2023-03-25T18:06:15Z"",
+ ""name"" : ""build"",
+ ""project_slug"" : ""gh/coldgust/coldgust.github.io"",
+ ""status"" : ""failed"",
+ ""type"" : ""build"",
+ ""stopped_at"" : ""2023-03-25T18:06:28Z""
+
}",https://circleci.com/api/v2/workflow/fc76deef-bcdd-4856-8e96-a8e2d1c5a85f/job,"{""id"":
""fc76deef-bcdd-4856-8e96-a8e2d1c5a85f"", ""tag"": """", ""name"": """",
""status"": """", ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"":
""0001-01-01T00:00:00Z"", ""created_at"": null, ""errored_by"": """",
""started_by"": """", ""stopped_at"": null, ""canceled_by"": """",
""pipeline_id"": ""afe0cabe-e7ee-4eb7-bf13-bb6170d139f0"", ""ConnectionId"": 0,
""_raw_data_id"": 0, ""duration_sec [...]
diff --git
a/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows_incremental.csv
b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows_incremental.csv
new file mode 100644
index 000000000..2a70cdd51
--- /dev/null
+++
b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows_incremental.csv
@@ -0,0 +1,67 @@
+id,params,data,url,input,created_at
+19,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{
+ ""pipeline_id"" : ""23622ee4-e150-4920-9d66-81533fa765a4"",
+ ""id"" : ""6731159f-5275-4bfa-ba70-39d343d63814"",
+ ""name"" : ""workflow"",
+ ""project_slug"" : ""gh/coldgust/coldgust.github.io"",
+ ""status"" : ""failed"",
+ ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"",
+ ""pipeline_number"" : 5,
+ ""created_at"" : ""2023-03-25T17:52:15Z"",
+ ""stopped_at"" : ""2023-03-25T17:52:23Z""
+
}",https://circleci.com/api/v2/pipeline/23622ee4-e150-4920-9d66-81533fa765a4/workflow,"{""id"":
""23622ee4-e150-4920-9d66-81533fa765a4"", ""vcs"": {""tag"": """", ""branch"":
"""", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """",
""review_id"": """", ""review_url"": """", ""provider_name"": """",
""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"":
"""", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"":
{""login"": "" [...]
+20,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{
+ ""pipeline_id"" : ""2c45280f-7fb3-4025-b703-a547c4a94916"",
+ ""id"" : ""c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943"",
+ ""name"" : ""workflow"",
+ ""project_slug"" : ""gh/coldgust/coldgust.github.io"",
+ ""status"" : ""failed"",
+ ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"",
+ ""pipeline_number"" : 4,
+ ""created_at"" : ""2023-03-25T17:50:20Z"",
+ ""stopped_at"" : ""2023-03-25T17:50:25Z""
+
}",https://circleci.com/api/v2/pipeline/2c45280f-7fb3-4025-b703-a547c4a94916/workflow,"{""id"":
""2c45280f-7fb3-4025-b703-a547c4a94916"", ""vcs"": {""tag"": """", ""branch"":
"""", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """",
""review_id"": """", ""review_url"": """", ""provider_name"": """",
""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"":
"""", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"":
{""login"": "" [...]
+22,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{
+ ""pipeline_id"" : ""7fcc1623-edcc-4a76-ad20-cd81aa83519f"",
+ ""id"" : ""fd0bd4f5-264f-4e3c-a151-06153c018f78"",
+ ""name"" : ""workflow"",
+ ""project_slug"" : ""gh/coldgust/coldgust.github.io"",
+ ""status"" : ""success"",
+ ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"",
+ ""pipeline_number"" : 9,
+ ""created_at"" : ""2023-03-25T18:13:21Z"",
+ ""stopped_at"" : ""2023-03-25T18:13:38Z""
+
}",https://circleci.com/api/v2/pipeline/7fcc1623-edcc-4a76-ad20-cd81aa83519f/workflow,"{""id"":
""7fcc1623-edcc-4a76-ad20-cd81aa83519f"", ""vcs"": {""tag"": """", ""branch"":
"""", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """",
""review_id"": """", ""review_url"": """", ""provider_name"": """",
""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"":
"""", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"":
{""login"": "" [...]
+24,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{
+ ""pipeline_id"" : ""866e967d-f826-4470-aed6-fc0c92e98703"",
+ ""id"" : ""7370985a-9de3-4a47-acbc-e6a1fe8e5812"",
+ ""name"" : ""workflow"",
+ ""project_slug"" : ""gh/coldgust/coldgust.github.io"",
+ ""status"" : ""failed"",
+ ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"",
+ ""pipeline_number"" : 7,
+ ""created_at"" : ""2023-03-25T17:56:24Z"",
+ ""stopped_at"" : ""2023-03-25T17:56:43Z""
+
}",https://circleci.com/api/v2/pipeline/866e967d-f826-4470-aed6-fc0c92e98703/workflow,"{""id"":
""866e967d-f826-4470-aed6-fc0c92e98703"", ""vcs"": {""tag"": """", ""branch"":
"""", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """",
""review_id"": """", ""review_url"": """", ""provider_name"": """",
""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"":
"""", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"":
{""login"": "" [...]
+26,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{
+ ""pipeline_id"" : ""70f3eb15-3b94-4f80-b65e-f23f4b74c33a"",
+ ""id"" : ""b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14"",
+ ""name"" : ""workflow"",
+ ""project_slug"" : ""gh/coldgust/coldgust.github.io"",
+ ""status"" : ""failed"",
+ ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"",
+ ""pipeline_number"" : 6,
+ ""created_at"" : ""2023-03-25T17:54:09Z"",
+ ""stopped_at"" : ""2023-03-25T17:54:23Z""
+
}",https://circleci.com/api/v2/pipeline/70f3eb15-3b94-4f80-b65e-f23f4b74c33a/workflow,"{""id"":
""70f3eb15-3b94-4f80-b65e-f23f4b74c33a"", ""vcs"": {""tag"": """", ""branch"":
"""", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """",
""review_id"": """", ""review_url"": """", ""provider_name"": """",
""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"":
"""", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"":
{""login"": "" [...]
+27,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{
+ ""pipeline_id"" : ""afe0cabe-e7ee-4eb7-bf13-bb6170d139f0"",
+ ""id"" : ""fc76deef-bcdd-4856-8e96-a8e2d1c5a85f"",
+ ""name"" : ""workflow"",
+ ""project_slug"" : ""gh/coldgust/coldgust.github.io"",
+ ""status"" : ""failed"",
+ ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"",
+ ""pipeline_number"" : 8,
+ ""created_at"" : ""2023-03-25T18:06:13Z"",
+ ""stopped_at"" : ""2023-03-25T18:06:28Z""
+
}",https://circleci.com/api/v2/pipeline/afe0cabe-e7ee-4eb7-bf13-bb6170d139f0/workflow,"{""id"":
""afe0cabe-e7ee-4eb7-bf13-bb6170d139f0"", ""vcs"": {""tag"": """", ""branch"":
"""", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """",
""review_id"": """", ""review_url"": """", ""provider_name"": """",
""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"":
"""", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"":
{""login"": "" [...]
diff --git
a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs.csv
b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs.csv
index 82f2b4c45..517060bd3 100644
--- a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs.csv
+++ b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs.csv
@@ -1,10 +1,10 @@
-connection_id,workflow_id,id,project_slug,canceled_by,dependencies,job_number,queued_at,started_at,name,approved_by,status,type,approval_request_id,stopped_at,duration_sec,pipeline_id,duration
-1,6731159f-5275-4bfa-ba70-39d343d63814,ab8c3282-0e74-4a41-834e-152a71280bed,github/coldgust/coldgust.github.io,,[],5,,2023-03-25T17:52:20.000+00:00,build,,failed,build,,2023-03-25T17:52:23.000+00:00,3,23622ee4-e150-4920-9d66-81533fa765a4,0
-1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,a00f80bc-f759-4900-97a5-2d121d80bde8,github/coldgust/coldgust.github.io,,[],7,,2023-03-25T17:56:27.000+00:00,build,,failed,build,,2023-03-25T17:56:43.000+00:00,16,866e967d-f826-4470-aed6-fc0c92e98703,0
-1,89054eb2-8e85-4f5c-9a93-66d753a0e970,c46092f9-6f82-4a52-8d8b-bd70d365dfc2,github/coldgust/coldgust.github.io,,[],3,,2023-03-25T17:39:25.000+00:00,say-hello,,success,build,,2023-03-25T17:39:28.000+00:00,3,625ca634-68fe-4515-91f0-7ba8af51dc99,0
-1,8971a56b-5547-4824-94dd-07bb467524c5,7b96e45b-b10e-47a0-95d0-96580b88bdda,github/coldgust/coldgust.github.io,,[],2,,2023-03-25T17:12:20.000+00:00,say-hello,,success,build,,2023-03-25T17:12:23.000+00:00,3,87aad008-1ad5-486a-8174-fdeed846561a,0
-1,8fe60291-68f7-40e2-acec-d99bff4da713,afde48dd-7319-4973-b3c8-e00308ff7667,github/coldgust/coldgust.github.io,,[],1,,2023-03-25T17:12:20.000+00:00,say-hello,,success,build,,2023-03-25T17:12:22.000+00:00,2,afef32b3-5ffe-48d2-8d9e-46dcedd82554,0
-1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,76c1f2cc-27ea-47aa-8167-48d2633abdba,github/coldgust/coldgust.github.io,,[],6,,2023-03-25T17:54:11.000+00:00,build,,failed,build,,2023-03-25T17:54:23.000+00:00,12,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,0
-1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,a4af3dd5-a3ae-48e8-b634-e2d63aafbb5b,github/coldgust/coldgust.github.io,,[],4,,2023-03-25T17:50:22.000+00:00,build,,failed,build,,2023-03-25T17:50:25.000+00:00,3,2c45280f-7fb3-4025-b703-a547c4a94916,0
-1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,004e3e27-17d7-4ccb-9b21-a7f55bcf2b3e,github/coldgust/coldgust.github.io,,[],8,,2023-03-25T18:06:15.000+00:00,build,,failed,build,,2023-03-25T18:06:28.000+00:00,13,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,0
-1,fd0bd4f5-264f-4e3c-a151-06153c018f78,2ff3594e-9da1-4306-aefa-77b72a97971e,github/coldgust/coldgust.github.io,,[],9,,2023-03-25T18:13:25.000+00:00,build,,success,build,,2023-03-25T18:13:38.000+00:00,13,7fcc1623-edcc-4a76-ad20-cd81aa83519f,0
+connection_id,workflow_id,id,project_slug,canceled_by,dependencies,job_number,created_date,queued_at,started_at,name,approved_by,status,type,approval_request_id,stopped_at,duration_sec,pipeline_id,duration
+1,6731159f-5275-4bfa-ba70-39d343d63814,ab8c3282-0e74-4a41-834e-152a71280bed,github/coldgust/coldgust.github.io,,[],5,2023-03-25T17:52:20.000+00:00,,2023-03-25T17:52:20.000+00:00,build,,failed,build,,2023-03-25T17:52:23.000+00:00,3,23622ee4-e150-4920-9d66-81533fa765a4,0
+1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,a00f80bc-f759-4900-97a5-2d121d80bde8,github/coldgust/coldgust.github.io,,[],7,2023-03-25T17:56:27.000+00:00,,2023-03-25T17:56:27.000+00:00,build,,failed,build,,2023-03-25T17:56:43.000+00:00,16,866e967d-f826-4470-aed6-fc0c92e98703,0
+1,89054eb2-8e85-4f5c-9a93-66d753a0e970,c46092f9-6f82-4a52-8d8b-bd70d365dfc2,github/coldgust/coldgust.github.io,,[],3,2023-03-25T17:39:25.000+00:00,,2023-03-25T17:39:25.000+00:00,say-hello,,success,build,,2023-03-25T17:39:28.000+00:00,3,625ca634-68fe-4515-91f0-7ba8af51dc99,0
+1,8971a56b-5547-4824-94dd-07bb467524c5,7b96e45b-b10e-47a0-95d0-96580b88bdda,github/coldgust/coldgust.github.io,,[],2,2023-03-25T17:12:20.000+00:00,,2023-03-25T17:12:20.000+00:00,say-hello,,success,build,,2023-03-25T17:12:23.000+00:00,3,87aad008-1ad5-486a-8174-fdeed846561a,0
+1,8fe60291-68f7-40e2-acec-d99bff4da713,afde48dd-7319-4973-b3c8-e00308ff7667,github/coldgust/coldgust.github.io,,[],1,2023-03-25T17:12:20.000+00:00,,2023-03-25T17:12:20.000+00:00,say-hello,,success,build,,2023-03-25T17:12:22.000+00:00,2,afef32b3-5ffe-48d2-8d9e-46dcedd82554,0
+1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,76c1f2cc-27ea-47aa-8167-48d2633abdba,github/coldgust/coldgust.github.io,,[],6,2023-03-25T17:54:11.000+00:00,,2023-03-25T17:54:11.000+00:00,build,,failed,build,,2023-03-25T17:54:23.000+00:00,12,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,0
+1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,a4af3dd5-a3ae-48e8-b634-e2d63aafbb5b,github/coldgust/coldgust.github.io,,[],4,2023-03-25T17:50:22.000+00:00,,2023-03-25T17:50:22.000+00:00,build,,failed,build,,2023-03-25T17:50:25.000+00:00,3,2c45280f-7fb3-4025-b703-a547c4a94916,0
+1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,004e3e27-17d7-4ccb-9b21-a7f55bcf2b3e,github/coldgust/coldgust.github.io,,[],8,2023-03-25T18:06:15.000+00:00,,2023-03-25T18:06:15.000+00:00,build,,failed,build,,2023-03-25T18:06:28.000+00:00,13,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,0
+1,fd0bd4f5-264f-4e3c-a151-06153c018f78,2ff3594e-9da1-4306-aefa-77b72a97971e,github/coldgust/coldgust.github.io,,[],9,2023-03-25T18:13:25.000+00:00,,2023-03-25T18:13:25.000+00:00,build,,success,build,,2023-03-25T18:13:38.000+00:00,13,7fcc1623-edcc-4a76-ad20-cd81aa83519f,0
diff --git
a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs_incremental.csv
b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs_incremental.csv
new file mode 100644
index 000000000..ea9a23a44
--- /dev/null
+++
b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs_incremental.csv
@@ -0,0 +1,7 @@
+connection_id,workflow_id,id,project_slug,canceled_by,dependencies,job_number,created_date,queued_at,started_at,name,approved_by,status,type,approval_request_id,stopped_at,duration_sec,pipeline_id,duration
+1,6731159f-5275-4bfa-ba70-39d343d63814,ab8c3282-0e74-4a41-834e-152a71280bed,github/coldgust/coldgust.github.io,,[],5,2023-03-25T17:52:20.000+00:00,,2023-03-25T17:52:20.000+00:00,build,,failed,build,,2023-03-25T17:52:23.000+00:00,3,23622ee4-e150-4920-9d66-81533fa765a4,0
+1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,a00f80bc-f759-4900-97a5-2d121d80bde8,github/coldgust/coldgust.github.io,,[],7,2023-03-25T17:56:27.000+00:00,,2023-03-25T17:56:27.000+00:00,build,,failed,build,,2023-03-25T17:56:43.000+00:00,16,866e967d-f826-4470-aed6-fc0c92e98703,0
+1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,76c1f2cc-27ea-47aa-8167-48d2633abdba,github/coldgust/coldgust.github.io,,[],6,2023-03-25T17:54:11.000+00:00,,2023-03-25T17:54:11.000+00:00,build,,failed,build,,2023-03-25T17:54:23.000+00:00,12,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,0
+1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,a4af3dd5-a3ae-48e8-b634-e2d63aafbb5b,github/coldgust/coldgust.github.io,,[],4,2023-03-25T17:50:22.000+00:00,,2023-03-25T17:50:22.000+00:00,build,,failed,build,,2023-03-25T17:50:25.000+00:00,3,2c45280f-7fb3-4025-b703-a547c4a94916,0
+1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,004e3e27-17d7-4ccb-9b21-a7f55bcf2b3e,github/coldgust/coldgust.github.io,,[],8,2023-03-25T18:06:15.000+00:00,,2023-03-25T18:06:15.000+00:00,build,,failed,build,,2023-03-25T18:06:28.000+00:00,13,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,0
+1,fd0bd4f5-264f-4e3c-a151-06153c018f78,2ff3594e-9da1-4306-aefa-77b72a97971e,github/coldgust/coldgust.github.io,,[],9,2023-03-25T18:13:25.000+00:00,,2023-03-25T18:13:25.000+00:00,build,,success,build,,2023-03-25T18:13:38.000+00:00,13,7fcc1623-edcc-4a76-ad20-cd81aa83519f,0
diff --git
a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv
b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv
index dd01015be..3d55ea884 100644
--- a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv
+++ b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv
@@ -1,11 +1,11 @@
-connection_id,id,project_slug,errors,updated_at,number,trigger_parameters,state,created_at,trigger,vcs,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,23622ee4-e150-4920-9d66-81533fa765a4,github/coldgust/coldgust.github.io,[],2023-03-25
17:52:15.651,5,,created,2023-03-25
17:52:15.651,"{""type"":""webhook"",""received_at"":""2023-03-25T17:52:15.446Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revisio
[...]
-1,2c45280f-7fb3-4025-b703-a547c4a94916,github/coldgust/coldgust.github.io,[],2023-03-25
17:50:20.316,4,,created,2023-03-25
17:50:20.316,"{""type"":""webhook"",""received_at"":""2023-03-25T17:50:20.150Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revisio
[...]
-1,625ca634-68fe-4515-91f0-7ba8af51dc99,github/coldgust/coldgust.github.io,[],2023-03-25
17:39:23.233,3,,created,2023-03-25
17:39:23.233,"{""type"":""webhook"",""received_at"":""2023-03-25T17:39:22.921Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url";
[...]
-1,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,github/coldgust/coldgust.github.io,[],2023-03-25
17:54:08.987,6,,created,2023-03-25
17:54:08.987,"{""type"":""webhook"",""received_at"":""2023-03-25T17:54:08.742Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revisio
[...]
-1,7fcc1623-edcc-4a76-ad20-cd81aa83519f,github/coldgust/coldgust.github.io,[],2023-03-25
18:13:21.590,9,,created,2023-03-25
18:13:21.590,"{""type"":""webhook"",""received_at"":""2023-03-25T18:13:21.404Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revisio
[...]
-1,866e967d-f826-4470-aed6-fc0c92e98703,github/coldgust/coldgust.github.io,[],2023-03-25
17:56:24.409,7,,created,2023-03-25
17:56:24.409,"{""type"":""webhook"",""received_at"":""2023-03-25T17:56:24.245Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revisio
[...]
-1,87aad008-1ad5-486a-8174-fdeed846561a,github/coldgust/coldgust.github.io,[],2023-03-25
17:12:18.747,2,,created,2023-03-25
17:12:18.747,"{""type"":""webhook"",""received_at"":""2023-03-25T17:12:18.624Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url";
[...]
-1,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,github/coldgust/coldgust.github.io,[],2023-03-25
18:06:13.404,8,,created,2023-03-25
18:06:13.404,"{""type"":""webhook"",""received_at"":""2023-03-25T18:06:13.246Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revisio
[...]
-1,afef32b3-5ffe-48d2-8d9e-46dcedd82554,github/coldgust/coldgust.github.io,[],2023-03-25
17:12:18.398,1,,created,2023-03-25
17:12:18.398,"{""type"":""api"",""received_at"":""2023-03-25T17:12:16.438Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"";
[...]
-1,d323f088-02fa-4ed5-9696-fc2f89a27150,github/coldgust/coldgust.github.io,"[{""type"":""config-fetch"",""message"":""Failed
to fetch config.yml file.""}]",2023-03-25
18:13:43.446,10,"{""circleci"":{""org_name"":""coldgust"",""project_name"":""coldgust.github.io"",""provider_name"":""github""},""git"":{""checkout_sha"":""6ba0cdaf7e7791c5bc5aa89a13b9cd184f8ad296""}}",errored,2023-03-25
18:13:43.446,"{""type"":""Decoupled Ingestion
System"",""received_at"":""1970-01-01T00:00:00.000Z"",""act [...]
+connection_id,id,project_slug,errors,updated_date,number,trigger_parameters,state,created_date,trigger,vcs,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,23622ee4-e150-4920-9d66-81533fa765a4,github/coldgust/coldgust.github.io,[],2023-03-25T17:52:15.651+00:00,5,,created,2023-03-25T17:52:15.651+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T17:52:15.446Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"";
[...]
+1,2c45280f-7fb3-4025-b703-a547c4a94916,github/coldgust/coldgust.github.io,[],2023-03-25T17:50:20.316+00:00,4,,created,2023-03-25T17:50:20.316+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T17:50:20.150Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"";
[...]
+1,625ca634-68fe-4515-91f0-7ba8af51dc99,github/coldgust/coldgust.github.io,[],2023-03-25T17:39:23.233+00:00,3,,created,2023-03-25T17:39:23.233+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T17:39:22.921Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",";
[...]
+1,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,github/coldgust/coldgust.github.io,[],2023-03-25T17:54:08.987+00:00,6,,created,2023-03-25T17:54:08.987+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T17:54:08.742Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"";
[...]
+1,7fcc1623-edcc-4a76-ad20-cd81aa83519f,github/coldgust/coldgust.github.io,[],2023-03-25T18:13:21.590+00:00,9,,created,2023-03-25T18:13:21.590+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T18:13:21.404Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"";
[...]
+1,866e967d-f826-4470-aed6-fc0c92e98703,github/coldgust/coldgust.github.io,[],2023-03-25T17:56:24.409+00:00,7,,created,2023-03-25T17:56:24.409+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T17:56:24.245Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"";
[...]
+1,87aad008-1ad5-486a-8174-fdeed846561a,github/coldgust/coldgust.github.io,[],2023-03-25T17:12:18.747+00:00,2,,created,2023-03-25T17:12:18.747+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T17:12:18.624Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",";
[...]
+1,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,github/coldgust/coldgust.github.io,[],2023-03-25T18:06:13.404+00:00,8,,created,2023-03-25T18:06:13.404+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T18:06:13.246Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"";
[...]
+1,afef32b3-5ffe-48d2-8d9e-46dcedd82554,github/coldgust/coldgust.github.io,[],2023-03-25T17:12:18.398+00:00,1,,created,2023-03-25T17:12:18.398+00:00,"{""type"":""api"",""received_at"":""2023-03-25T17:12:16.438Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""rev
[...]
+1,d323f088-02fa-4ed5-9696-fc2f89a27150,github/coldgust/coldgust.github.io,"[{""type"":""config-fetch"",""message"":""Failed
to fetch config.yml
file.""}]",2023-03-25T18:13:43.446+00:00,10,"{""circleci"":{""org_name"":""coldgust"",""project_name"":""coldgust.github.io"",""provider_name"":""github""},""git"":{""checkout_sha"":""6ba0cdaf7e7791c5bc5aa89a13b9cd184f8ad296""}}",errored,2023-03-25T18:13:43.446+00:00,"{""type"":""Decoupled
Ingestion System"",""received_at"":""1970-01-01T00:00:00. [...]
diff --git
a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows_incremental.csv
b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows_incremental.csv
new file mode 100644
index 000000000..529e84744
--- /dev/null
+++
b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows_incremental.csv
@@ -0,0 +1,7 @@
+connection_id,id,project_slug,pipeline_id,canceled_by,name,errored_by,tag,status,started_by,pipeline_number,created_date,stopped_date,duration_sec
+1,6731159f-5275-4bfa-ba70-39d343d63814,github/coldgust/coldgust.github.io,23622ee4-e150-4920-9d66-81533fa765a4,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,5,2023-03-25T17:52:15.000+00:00,2023-03-25T17:52:23.000+00:00,8
+1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,github/coldgust/coldgust.github.io,866e967d-f826-4470-aed6-fc0c92e98703,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,7,2023-03-25T17:56:24.000+00:00,2023-03-25T17:56:43.000+00:00,19
+1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,github/coldgust/coldgust.github.io,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,6,2023-03-25T17:54:09.000+00:00,2023-03-25T17:54:23.000+00:00,14
+1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,github/coldgust/coldgust.github.io,2c45280f-7fb3-4025-b703-a547c4a94916,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,4,2023-03-25T17:50:20.000+00:00,2023-03-25T17:50:25.000+00:00,5
+1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,github/coldgust/coldgust.github.io,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,8,2023-03-25T18:06:13.000+00:00,2023-03-25T18:06:28.000+00:00,15
+1,fd0bd4f5-264f-4e3c-a151-06153c018f78,github/coldgust/coldgust.github.io,7fcc1623-edcc-4a76-ad20-cd81aa83519f,,workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,9,2023-03-25T18:13:21.000+00:00,2023-03-25T18:13:38.000+00:00,17
diff --git a/backend/plugins/circleci/e2e/snapshot_tables/cicd_pipelines.csv
b/backend/plugins/circleci/e2e/snapshot_tables/cicd_pipelines.csv
index 45f61d9ce..54a7ede5d 100644
--- a/backend/plugins/circleci/e2e/snapshot_tables/cicd_pipelines.csv
+++ b/backend/plugins/circleci/e2e/snapshot_tables/cicd_pipelines.csv
@@ -1,10 +1,10 @@
-id,name,display_title,url,result,status,original_status,original_result,type,duration_sec,queued_duration_sec,environment,queued_date,started_date,cicd_scope_id
-circleci:CircleciWorkflow:1:6731159f-5275-4bfa-ba70-39d343d63814,workflow,workflow#5,,FAILURE,DONE,failed,,,8,,PRODUCTION,,2023-03-25T17:52:15.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:7370985a-9de3-4a47-acbc-e6a1fe8e5812,workflow,workflow#7,,FAILURE,DONE,failed,,,19,,PRODUCTION,,2023-03-25T17:56:24.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:89054eb2-8e85-4f5c-9a93-66d753a0e970,say-hello-workflow,say-hello-workflow#3,,SUCCESS,DONE,success,,,5,,PRODUCTION,,2023-03-25T17:39:23.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:8971a56b-5547-4824-94dd-07bb467524c5,say-hello-workflow,say-hello-workflow#2,,SUCCESS,DONE,success,,,5,,PRODUCTION,,2023-03-25T17:12:18.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:8fe60291-68f7-40e2-acec-d99bff4da713,say-hello-workflow,say-hello-workflow#1,,SUCCESS,DONE,success,,,4,,PRODUCTION,,2023-03-25T17:12:18.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,workflow,workflow#6,,FAILURE,DONE,failed,,,14,,PRODUCTION,,2023-03-25T17:54:09.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,workflow,workflow#4,,FAILURE,DONE,failed,,,5,,PRODUCTION,,2023-03-25T17:50:20.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,workflow,workflow#8,,FAILURE,DONE,failed,,,15,,PRODUCTION,,2023-03-25T18:06:13.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:fd0bd4f5-264f-4e3c-a151-06153c018f78,workflow,workflow#9,,SUCCESS,DONE,success,,,17,,PRODUCTION,,2023-03-25T18:13:21.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+id,name,display_title,url,result,status,original_status,original_result,type,duration_sec,queued_duration_sec,environment,queued_date,started_date,created_date,cicd_scope_id
+circleci:CircleciWorkflow:1:6731159f-5275-4bfa-ba70-39d343d63814,workflow,workflow#5,,FAILURE,DONE,failed,,,8,,PRODUCTION,,2023-03-25T17:52:15.000+00:00,2023-03-25T17:52:15.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:7370985a-9de3-4a47-acbc-e6a1fe8e5812,workflow,workflow#7,,FAILURE,DONE,failed,,,19,,PRODUCTION,,2023-03-25T17:56:24.000+00:00,2023-03-25T17:56:24.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:89054eb2-8e85-4f5c-9a93-66d753a0e970,say-hello-workflow,say-hello-workflow#3,,SUCCESS,DONE,success,,,5,,PRODUCTION,,2023-03-25T17:39:23.000+00:00,2023-03-25T17:39:23.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:8971a56b-5547-4824-94dd-07bb467524c5,say-hello-workflow,say-hello-workflow#2,,SUCCESS,DONE,success,,,5,,PRODUCTION,,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:18.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:8fe60291-68f7-40e2-acec-d99bff4da713,say-hello-workflow,say-hello-workflow#1,,SUCCESS,DONE,success,,,4,,PRODUCTION,,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:18.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,workflow,workflow#6,,FAILURE,DONE,failed,,,14,,PRODUCTION,,2023-03-25T17:54:09.000+00:00,2023-03-25T17:54:09.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,workflow,workflow#4,,FAILURE,DONE,failed,,,5,,PRODUCTION,,2023-03-25T17:50:20.000+00:00,2023-03-25T17:50:20.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,workflow,workflow#8,,FAILURE,DONE,failed,,,15,,PRODUCTION,,2023-03-25T18:06:13.000+00:00,2023-03-25T18:06:13.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:fd0bd4f5-264f-4e3c-a151-06153c018f78,workflow,workflow#9,,SUCCESS,DONE,success,,,17,,PRODUCTION,,2023-03-25T18:13:21.000+00:00,2023-03-25T18:13:21.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
diff --git a/backend/plugins/circleci/e2e/workflow_test.go
b/backend/plugins/circleci/e2e/workflow_test.go
index cc542fdb2..ba5194cea 100644
--- a/backend/plugins/circleci/e2e/workflow_test.go
+++ b/backend/plugins/circleci/e2e/workflow_test.go
@@ -72,7 +72,7 @@ func TestCircleciWorkflow(t *testing.T) {
devops.CICDPipeline{},
e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/cicd_pipelines.csv",
- IgnoreFields: []string{"finished_date", "created_date",
"is_child"},
+ IgnoreFields: []string{"finished_date", "is_child"},
IgnoreTypes: []interface{}{domainlayer.DomainEntity{}},
},
)
diff --git a/backend/plugins/circleci/models/job.go
b/backend/plugins/circleci/models/job.go
index bafaf5b9e..c8a3bea31 100644
--- a/backend/plugins/circleci/models/job.go
+++ b/backend/plugins/circleci/models/job.go
@@ -29,7 +29,7 @@ type CircleciJob struct {
CanceledBy string `gorm:"type:varchar(100)"
json:"canceled_by"`
Dependencies []string `gorm:"serializer:json;type:text"
json:"dependencies"`
JobNumber int64 `json:"job_number"`
- CreatedAt *common.Iso8601Time `json:"created_at"`
+ CreatedDate *common.Iso8601Time `json:"created_at"`
QueuedAt *common.Iso8601Time `json:"queued_at"`
StartedAt *common.Iso8601Time `json:"started_at"`
Name string `gorm:"type:varchar(255)"
json:"name"`
diff --git
a/backend/plugins/circleci/models/migrationscripts/20260322_rename_created_at_to_created_date.go
b/backend/plugins/circleci/models/migrationscripts/20260322_rename_created_at_to_created_date.go
new file mode 100644
index 000000000..497c0d3b5
--- /dev/null
+++
b/backend/plugins/circleci/models/migrationscripts/20260322_rename_created_at_to_created_date.go
@@ -0,0 +1,71 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+)
+
+type circleciWorkflow20260322 struct {
+ CreatedDate *time.Time
+}
+
+func (circleciWorkflow20260322) TableName() string {
+ return "_tool_circleci_workflows"
+}
+
+type circleciJob20260322 struct {
+ CreatedDate *time.Time
+}
+
+func (circleciJob20260322) TableName() string {
+ return "_tool_circleci_jobs"
+}
+
+type renameCreatedAtToCreatedDate20260322 struct{}
+
+func (*renameCreatedAtToCreatedDate20260322) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+
+ if err := db.AutoMigrate(&circleciWorkflow20260322{}); err != nil {
+ return err
+ }
+ if err := db.Exec("UPDATE _tool_circleci_workflows SET created_date =
created_at WHERE created_date IS NULL"); err != nil {
+ return err
+ }
+
+ if err := db.AutoMigrate(&circleciJob20260322{}); err != nil {
+ return err
+ }
+ if err := db.Exec("UPDATE _tool_circleci_jobs SET created_date =
created_at WHERE created_date IS NULL"); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (*renameCreatedAtToCreatedDate20260322) Version() uint64 {
+ return 20260322000001
+}
+
+func (*renameCreatedAtToCreatedDate20260322) Name() string {
+ return "circleci rename created_at to created_date in workflows and
jobs"
+}
diff --git a/backend/plugins/circleci/models/migrationscripts/register.go
b/backend/plugins/circleci/models/migrationscripts/register.go
index 5a23f27cc..af8472a3c 100644
--- a/backend/plugins/circleci/models/migrationscripts/register.go
+++ b/backend/plugins/circleci/models/migrationscripts/register.go
@@ -24,5 +24,6 @@ func All() []plugin.MigrationScript {
return []plugin.MigrationScript{
new(addInitTables),
new(addFieldsToCircleciJob20231129),
+ new(renameCreatedAtToCreatedDate20260322),
}
}
diff --git a/backend/plugins/circleci/models/pipeline.go
b/backend/plugins/circleci/models/pipeline.go
index a18cded10..a775ac4b2 100644
--- a/backend/plugins/circleci/models/pipeline.go
+++ b/backend/plugins/circleci/models/pipeline.go
@@ -49,8 +49,8 @@ type CircleciPipeline struct {
ConnectionId uint64 `gorm:"primaryKey;type:BIGINT"
json:"connectionId" mapstructure:"connectionId"`
Id string
`gorm:"primaryKey;type:varchar(100)" json:"id" mapstructure:"id"`
ProjectSlug string `gorm:"type:varchar(255)"
json:"projectSlug" mapstructure:"projectSlug"`
- UpdatedDate *common.Iso8601Time `json:"updatedDate"
mapstructure:"updatedDate"`
- CreatedDate *common.Iso8601Time `json:"createdDate"
mapstructure:"createdDate"`
+ UpdatedDate *common.Iso8601Time `json:"updated_at"`
+ CreatedDate *common.Iso8601Time `json:"created_at"`
Number int64 `json:"number"
mapstructure:"number"` // pipeline number within the project?
TriggerParameters any `gorm:"serializer:json"
json:"trigger_parameters" mapstructure:"triggerParameters"`
State string `gorm:"type:varchar(100)"
json:"state" mapstructure:"state"`
diff --git a/backend/plugins/circleci/models/workflow.go
b/backend/plugins/circleci/models/workflow.go
index f4e5e9f71..0b1a33028 100644
--- a/backend/plugins/circleci/models/workflow.go
+++ b/backend/plugins/circleci/models/workflow.go
@@ -33,7 +33,7 @@ type CircleciWorkflow struct {
Status string `gorm:"type:varchar(100)"
json:"status"`
StartedBy string `gorm:"type:varchar(100)"
json:"started_by"`
PipelineNumber int64 `json:"pipeline_number"`
- CreatedAt *common.Iso8601Time `json:"created_at"`
+ CreatedDate *common.Iso8601Time `json:"created_at"`
StoppedAt *common.Iso8601Time `json:"stopped_at"`
DurationSec float64 `json:"duration_sec"`
diff --git a/backend/plugins/circleci/tasks/job_converter.go
b/backend/plugins/circleci/tasks/job_converter.go
index 2cb8de120..5a085609d 100644
--- a/backend/plugins/circleci/tasks/job_converter.go
+++ b/backend/plugins/circleci/tasks/job_converter.go
@@ -58,8 +58,8 @@ func ConvertJobs(taskCtx plugin.SubTaskContext) errors.Error {
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
userTool := inputRow.(*models.CircleciJob)
createdAt := time.Now()
- if userTool.CreatedAt != nil {
- createdAt = userTool.CreatedAt.ToTime()
+ if userTool.CreatedDate != nil {
+ createdAt = userTool.CreatedDate.ToTime()
}
task := &devops.CICDTask{
DomainEntity: domainlayer.DomainEntity{
diff --git a/backend/plugins/circleci/tasks/workflow_converter.go
b/backend/plugins/circleci/tasks/workflow_converter.go
index a023accc7..bf6b029e3 100644
--- a/backend/plugins/circleci/tasks/workflow_converter.go
+++ b/backend/plugins/circleci/tasks/workflow_converter.go
@@ -61,11 +61,11 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext)
errors.Error {
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
userTool := inputRow.(*models.CircleciWorkflow)
// Skip if CreatedAt is null or empty string - still
enters into the `_tool_circleci_workflows` table with null values
- if userTool.CreatedAt.ToNullableTime() == nil {
- logger.Info("CreatedAt is null or empty string
in the CircleCI API response for %s", userTool.PipelineId)
+ if userTool.CreatedDate.ToNullableTime() == nil {
+ logger.Info("CreatedDate is null or empty
string in the CircleCI API response for %s", userTool.PipelineId)
return []interface{}{}, nil
}
- createdAt := userTool.CreatedAt.ToTime()
+ createdAt := userTool.CreatedDate.ToTime()
pipeline := &devops.CICDPipeline{
DomainEntity: domainlayer.DomainEntity{
Id:
getWorkflowIdGen().Generate(data.Options.ConnectionId, userTool.Id),
diff --git a/backend/plugins/circleci/tasks/workflow_extractor.go
b/backend/plugins/circleci/tasks/workflow_extractor.go
index ba0639bc8..165c6d635 100644
--- a/backend/plugins/circleci/tasks/workflow_extractor.go
+++ b/backend/plugins/circleci/tasks/workflow_extractor.go
@@ -48,8 +48,8 @@ func ExtractWorkflows(taskCtx plugin.SubTaskContext)
errors.Error {
toolL := userRes
toolL.ConnectionId = data.Options.ConnectionId
toolL.ProjectSlug = data.Options.ProjectSlug
- if userRes.CreatedAt != nil && userRes.StoppedAt != nil
{
- startTime := userRes.CreatedAt.ToTime()
+ if userRes.CreatedDate != nil && userRes.StoppedAt !=
nil {
+ startTime := userRes.CreatedDate.ToTime()
endTime := userRes.StoppedAt.ToTime()
toolL.DurationSec =
float64(endTime.Sub(startTime).Milliseconds() / 1e3)
}
