This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 24ca4782f Cherry-pick to `release-v1.0` -- fix(circleci): CircleCI
Workflow nil check for CreatedDate (#8310) (#8314)
24ca4782f is described below
commit 24ca4782fedf715953d3341fea618d5a0e04e7f7
Author: John Ibsen <[email protected]>
AuthorDate: Fri Mar 7 04:17:03 2025 -0500
Cherry-pick to `release-v1.0` -- fix(circleci): CircleCI Workflow nil check
for CreatedDate (#8310) (#8314)
* fix(circleci): CircleCI Workflow nil check for CreatedDate (#8310)
* fix(circleci): Add CircleCI workflow nil check for CreatedDate
Adds a nil check during `convertWorkflows` task when CircleCI workflows API
returns a weird malformed pipeline data with no created_date. Previously, the
entire task fails and panics when trying to get convert a nil in ToTime util.
Fixes Issue #8309
* Update to fix compile problems
* Tweak the _tool csv to allow for the NULL value entries
---------
Co-authored-by: John Ibsen <[email protected]>
* Update with latest updates in circleci plugin for createdAt
---------
Co-authored-by: John Ibsen <[email protected]>
Co-authored-by: Klesh Wong <[email protected]>
---
.../e2e/raw_tables/_raw_circleci_api_workflows.csv | 11 +++++++++++
.../snapshot_tables/_tool_circleci_workflows.csv | 21 +++++++++++----------
.../plugins/circleci/tasks/workflow_converter.go | 8 ++++++++
3 files changed, 30 insertions(+), 10 deletions(-)
diff --git
a/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv
b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv
index 43edf350c..5760d52da 100644
--- a/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv
+++ b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv
@@ -98,3 +98,14 @@ id,params,data,url,input,created_at
""created_at"" : ""2023-03-25T18:06:13Z"",
""stopped_at"" : ""2023-03-25T18:06:28Z""
}",https://circleci.com/api/v2/pipeline/afe0cabe-e7ee-4eb7-bf13-bb6170d139f0/workflow,"{""id"":
""afe0cabe-e7ee-4eb7-bf13-bb6170d139f0"", ""vcs"": {""tag"": """", ""branch"":
"""", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """",
""review_id"": """", ""review_url"": """", ""provider_name"": """",
""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"":
"""", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"":
{""login"": "" [...]
+28,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{
+ ""pipeline_id"" : ""f5a2a6e8-4430-4935-8bec-77ca2b26c61d"",
+ ""id"" : ""b3b77371-e27a-4e93-a554-c728f757cd24"",
+ ""name"" : ""workflow"",
+ ""project_slug"" : ""gh/coldgust/coldgust.github.io"",
+ ""status"" : ""failed"",
+ ""started_by"" : """",
+ ""pipeline_number"" : 16,
+ ""created_at"" : """",
+ ""stopped_at"" : """"
+
}",https://circleci.com/api/v2/pipeline/f5a2a6e8-4430-4935-8bec-77ca2b26c61d/workflow,"{""id"":
""f5a2a6e8-4430-4935-8bec-77ca2b26c61d"", ""vcs"": {""tag"": """", ""branch"":
"""", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """",
""review_id"": """", ""review_url"": """", ""provider_name"": """",
""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"":
"""", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"":
{""login"": "" [...]
diff --git
a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv
b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv
index 0c43df3ea..bd6af7cc9 100644
--- a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv
+++ b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv
@@ -1,10 +1,11 @@
-connection_id,id,project_slug,pipeline_id,canceled_by,name,errored_by,tag,status,started_by,pipeline_number,created_at,stopped_at,duration_sec,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,6731159f-5275-4bfa-ba70-39d343d63814,github/coldgust/coldgust.github.io,23622ee4-e150-4920-9d66-81533fa765a4,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,5,2023-03-25
17:52:15,2023-03-25 17:52:23,8,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,19,""
-1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,github/coldgust/coldgust.github.io,866e967d-f826-4470-aed6-fc0c92e98703,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,7,2023-03-25
17:56:24,2023-03-25 17:56:43,19,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,24,""
-1,89054eb2-8e85-4f5c-9a93-66d753a0e970,github/coldgust/coldgust.github.io,625ca634-68fe-4515-91f0-7ba8af51dc99,"",say-hello-workflow,"","",success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,3,2023-03-25
17:39:23,2023-03-25 17:39:28,5,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,21,""
-1,8971a56b-5547-4824-94dd-07bb467524c5,github/coldgust/coldgust.github.io,87aad008-1ad5-486a-8174-fdeed846561a,"",say-hello-workflow,"","",success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,2,2023-03-25
17:12:18,2023-03-25 17:12:23,5,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,23,""
-1,8fe60291-68f7-40e2-acec-d99bff4da713,github/coldgust/coldgust.github.io,afef32b3-5ffe-48d2-8d9e-46dcedd82554,"",say-hello-workflow,"","",success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,1,2023-03-25
17:12:18,2023-03-25 17:12:22,4,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,25,""
-1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,github/coldgust/coldgust.github.io,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,6,2023-03-25
17:54:09,2023-03-25 17:54:23,14,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,26,""
-1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,github/coldgust/coldgust.github.io,2c45280f-7fb3-4025-b703-a547c4a94916,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,4,2023-03-25
17:50:20,2023-03-25 17:50:25,5,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,20,""
-1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,github/coldgust/coldgust.github.io,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,8,2023-03-25
18:06:13,2023-03-25 18:06:28,15,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,27,""
-1,fd0bd4f5-264f-4e3c-a151-06153c018f78,github/coldgust/coldgust.github.io,7fcc1623-edcc-4a76-ad20-cd81aa83519f,"",workflow,"","",success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,9,2023-03-25
18:13:21,2023-03-25 18:13:38,17,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,22,""
+connection_id,id,project_slug,pipeline_id,canceled_by,name,errored_by,tag,status,started_by,pipeline_number,created_date,stopped_date,duration_sec
+1,6731159f-5275-4bfa-ba70-39d343d63814,github/coldgust/coldgust.github.io,23622ee4-e150-4920-9d66-81533fa765a4,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,5,2023-03-25T17:52:15.000+00:00,2023-03-25T17:52:23.000+00:00,8
+1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,github/coldgust/coldgust.github.io,866e967d-f826-4470-aed6-fc0c92e98703,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,7,2023-03-25T17:56:24.000+00:00,2023-03-25T17:56:43.000+00:00,19
+1,89054eb2-8e85-4f5c-9a93-66d753a0e970,github/coldgust/coldgust.github.io,625ca634-68fe-4515-91f0-7ba8af51dc99,,say-hello-workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,3,2023-03-25T17:39:23.000+00:00,2023-03-25T17:39:28.000+00:00,5
+1,8971a56b-5547-4824-94dd-07bb467524c5,github/coldgust/coldgust.github.io,87aad008-1ad5-486a-8174-fdeed846561a,,say-hello-workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,2,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:23.000+00:00,5
+1,8fe60291-68f7-40e2-acec-d99bff4da713,github/coldgust/coldgust.github.io,afef32b3-5ffe-48d2-8d9e-46dcedd82554,,say-hello-workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,1,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:22.000+00:00,4
+1,b3b77371-e27a-4e93-a554-c728f757cd24,github/coldgust/coldgust.github.io,f5a2a6e8-4430-4935-8bec-77ca2b26c61d,,workflow,,,failed,,16,,,0
+1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,github/coldgust/coldgust.github.io,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,6,2023-03-25T17:54:09.000+00:00,2023-03-25T17:54:23.000+00:00,14
+1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,github/coldgust/coldgust.github.io,2c45280f-7fb3-4025-b703-a547c4a94916,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,4,2023-03-25T17:50:20.000+00:00,2023-03-25T17:50:25.000+00:00,5
+1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,github/coldgust/coldgust.github.io,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,8,2023-03-25T18:06:13.000+00:00,2023-03-25T18:06:28.000+00:00,15
+1,fd0bd4f5-264f-4e3c-a151-06153c018f78,github/coldgust/coldgust.github.io,7fcc1623-edcc-4a76-ad20-cd81aa83519f,,workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,9,2023-03-25T18:13:21.000+00:00,2023-03-25T18:13:38.000+00:00,17
diff --git a/backend/plugins/circleci/tasks/workflow_converter.go
b/backend/plugins/circleci/tasks/workflow_converter.go
index e1af8f961..a023accc7 100644
--- a/backend/plugins/circleci/tasks/workflow_converter.go
+++ b/backend/plugins/circleci/tasks/workflow_converter.go
@@ -40,6 +40,9 @@ var ConvertWorkflowsMeta = plugin.SubTaskMeta{
func ConvertWorkflows(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_WORKFLOW_TABLE)
+ logger := taskCtx.GetLogger()
+ logger.Info("convert workflows")
+
db := taskCtx.GetDal()
clauses := []dal.Clause{
dal.From(&models.CircleciWorkflow{}),
@@ -57,6 +60,11 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext)
errors.Error {
Input: cursor,
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
userTool := inputRow.(*models.CircleciWorkflow)
+ // Skip if CreatedAt is null or empty string - still
enters into the `_tool_circleci_workflows` table with null values
+ if userTool.CreatedAt.ToNullableTime() == nil {
+ logger.Info("CreatedAt is null or empty string
in the CircleCI API response for %s", userTool.PipelineId)
+ return []interface{}{}, nil
+ }
createdAt := userTool.CreatedAt.ToTime()
pipeline := &devops.CICDPipeline{
DomainEntity: domainlayer.DomainEntity{
