This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 4dc01bffd fix(argocd): report a Degraded rollout as a failed
deployment (#9354)
4dc01bffd is described below
commit 4dc01bffd201cb02818ff17d1fcf981b29631b6e
Author: ZIHAN DAI <[email protected]>
AuthorDate: Tue Nov 10 22:15:38 2026 +1100
fix(argocd): report a Degraded rollout as a failed deployment (#9354)
A sync can succeed while the rollout it starts never becomes healthy, and
the plugin only looks at the sync phase, so such a deployment was stored
with result SUCCESS. Record the application's health on the newest history
entry when it is collected and turn a Succeeded sync whose rollout is
Degraded into result FAILURE.
Signed-off-by: Zihan Dai <[email protected]>
---
.../raw_tables/_raw_argocd_api_sync_operations.csv | 8 +++
.../_tool_argocd_sync_operations.csv | 4 ++
.../snapshot_tables/cicd_deployment_commits.csv | 4 ++
.../e2e/snapshot_tables/cicd_deployments.csv | 4 ++
backend/plugins/argocd/e2e/sync_operation_test.go | 76 ++++++++++++++++++++++
.../argocd/tasks/sync_operation_collector.go | 32 +++++++++
.../argocd/tasks/sync_operation_collector_test.go | 40 ++++++++++++
.../argocd/tasks/sync_operation_convertor.go | 16 ++++-
.../argocd/tasks/sync_operation_convertor_test.go | 21 ++++++
.../argocd/tasks/sync_operation_extractor.go | 10 +++
10 files changed, 213 insertions(+), 2 deletions(-)
diff --git
a/backend/plugins/argocd/e2e/raw_tables/_raw_argocd_api_sync_operations.csv
b/backend/plugins/argocd/e2e/raw_tables/_raw_argocd_api_sync_operations.csv
new file mode 100644
index 000000000..8cf9f5d54
--- /dev/null
+++ b/backend/plugins/argocd/e2e/raw_tables/_raw_argocd_api_sync_operations.csv
@@ -0,0 +1,8 @@
+id,params,data,url,input,created_at
+1,"{""ConnectionId"":1,""Name"":""myapp""}","{""phase"":""Succeeded"",""message"":""successfully
synced (all tasks
run)"",""startedAt"":""2026-11-03T09:59:30Z"",""finishedAt"":""2026-11-03T10:00:00Z"",""syncResult"":{""revision"":""bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb""}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-03
12:00:00.000
+2,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":1,""revision"":""aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"",""deployedAt"":""2026-11-01T10:00:00Z"",""deployStartedAt"":""2026-11-01T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-03
12:00:00.000
+3,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":2,""revision"":""bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"",""deployedAt"":""2026-11-03T10:00:00Z"",""deployStartedAt"":""2026-11-03T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true},""applicationHealth"":""Degraded""}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-03
12:00:00.000
+4,"{""ConnectionId"":1,""Name"":""myapp""}","{""phase"":""Succeeded"",""message"":""successfully
synced (all tasks
run)"",""startedAt"":""2026-11-04T09:59:30Z"",""finishedAt"":""2026-11-04T10:00:00Z"",""syncResult"":{""revision"":""cccccccccccccccccccccccccccccccccccccccc""}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-04
12:00:00.000
+5,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":1,""revision"":""aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"",""deployedAt"":""2026-11-01T10:00:00Z"",""deployStartedAt"":""2026-11-01T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-04
12:00:00.000
+6,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":2,""revision"":""bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"",""deployedAt"":""2026-11-03T10:00:00Z"",""deployStartedAt"":""2026-11-03T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-04
12:00:00.000
+7,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":3,""revision"":""cccccccccccccccccccccccccccccccccccccccc"",""deployedAt"":""2026-11-04T10:00:00Z"",""deployStartedAt"":""2026-11-04T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true},""applicationHealth"":""Healthy""}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-04
12:00:00.000
diff --git
a/backend/plugins/argocd/e2e/snapshot_tables/_tool_argocd_sync_operations.csv
b/backend/plugins/argocd/e2e/snapshot_tables/_tool_argocd_sync_operations.csv
new file mode 100644
index 000000000..a347677c5
--- /dev/null
+++
b/backend/plugins/argocd/e2e/snapshot_tables/_tool_argocd_sync_operations.csv
@@ -0,0 +1,4 @@
+connection_id,application_name,deployment_id,revision,phase,health_status
+1,myapp,1793527200,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,Succeeded,
+1,myapp,1793700000,bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb,Succeeded,Degraded
+1,myapp,1793786400,cccccccccccccccccccccccccccccccccccccccc,Succeeded,Healthy
diff --git
a/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployment_commits.csv
b/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployment_commits.csv
new file mode 100644
index 000000000..30632647e
--- /dev/null
+++ b/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployment_commits.csv
@@ -0,0 +1,4 @@
+id,cicd_deployment_id,result,commit_sha
+argocd:ArgocdSyncOperation:1:myapp:1793527200,argocd:ArgocdSyncOperation:1:myapp:1793527200,SUCCESS,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+argocd:ArgocdSyncOperation:1:myapp:1793700000,argocd:ArgocdSyncOperation:1:myapp:1793700000,FAILURE,bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+argocd:ArgocdSyncOperation:1:myapp:1793786400,argocd:ArgocdSyncOperation:1:myapp:1793786400,SUCCESS,cccccccccccccccccccccccccccccccccccccccc
diff --git a/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployments.csv
b/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployments.csv
new file mode 100644
index 000000000..ec4e2d730
--- /dev/null
+++ b/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployments.csv
@@ -0,0 +1,4 @@
+id,result,status,original_status,original_result
+argocd:ArgocdSyncOperation:1:myapp:1793527200,SUCCESS,DONE,Succeeded,Succeeded
+argocd:ArgocdSyncOperation:1:myapp:1793700000,FAILURE,DONE,Succeeded,Degraded
+argocd:ArgocdSyncOperation:1:myapp:1793786400,SUCCESS,DONE,Succeeded,Succeeded
diff --git a/backend/plugins/argocd/e2e/sync_operation_test.go
b/backend/plugins/argocd/e2e/sync_operation_test.go
new file mode 100644
index 000000000..27f24d3eb
--- /dev/null
+++ b/backend/plugins/argocd/e2e/sync_operation_test.go
@@ -0,0 +1,76 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/devlake/core/models/domainlayer/devops"
+ "github.com/apache/devlake/helpers/e2ehelper"
+ "github.com/apache/devlake/plugins/argocd/impl"
+ "github.com/apache/devlake/plugins/argocd/models"
+
"github.com/apache/devlake/plugins/argocd/models/migrationscripts/archived"
+ "github.com/apache/devlake/plugins/argocd/tasks"
+)
+
+func TestSyncOperationDataFlow(t *testing.T) {
+ var argocd impl.ArgoCD
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "argocd", argocd)
+
+ taskData := &tasks.ArgocdTaskData{
+ Options: &tasks.ArgocdOptions{
+ ConnectionId: 1,
+ ApplicationName: "myapp",
+ ScopeConfig: &models.ArgocdScopeConfig{},
+ },
+ }
+
+ // Two collections: the first saw revision bbb Degraded, the second a
newer Healthy
+ // sync, by which time bbb is an ordinary history entry again.
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_argocd_api_sync_operations.csv",
"_raw_argocd_api_sync_operations")
+ dataflowTester.FlushTabler(&archived.ArgocdApplication{})
+ dataflowTester.FlushTabler(&models.ArgocdRevisionImage{})
+
+ dataflowTester.FlushTabler(&models.ArgocdSyncOperation{})
+ dataflowTester.Subtask(tasks.ExtractSyncOperationsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(
+ models.ArgocdSyncOperation{},
+ e2ehelper.TableOptions{
+ CSVRelPath:
"./snapshot_tables/_tool_argocd_sync_operations.csv",
+ TargetFields: []string{"connection_id",
"application_name", "deployment_id", "revision", "phase", "health_status"},
+ },
+ )
+
+ dataflowTester.FlushTabler(&devops.CICDDeployment{})
+ dataflowTester.FlushTabler(&devops.CicdDeploymentCommit{})
+ dataflowTester.Subtask(tasks.ConvertSyncOperationsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(
+ devops.CICDDeployment{},
+ e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/cicd_deployments.csv",
+ TargetFields: []string{"id", "result", "status",
"original_status", "original_result"},
+ },
+ )
+ dataflowTester.VerifyTableWithOptions(
+ devops.CicdDeploymentCommit{},
+ e2ehelper.TableOptions{
+ CSVRelPath:
"./snapshot_tables/cicd_deployment_commits.csv",
+ TargetFields: []string{"id", "cicd_deployment_id",
"result", "commit_sha"},
+ },
+ )
+}
diff --git a/backend/plugins/argocd/tasks/sync_operation_collector.go
b/backend/plugins/argocd/tasks/sync_operation_collector.go
index bac7e5e64..553037607 100644
--- a/backend/plugins/argocd/tasks/sync_operation_collector.go
+++ b/backend/plugins/argocd/tasks/sync_operation_collector.go
@@ -69,6 +69,9 @@ func CollectSyncOperations(taskCtx plugin.SubTaskContext)
errors.Error {
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
var appResponse struct {
Status struct {
+ Health struct {
+ Status string `json:"status"`
+ } `json:"health"`
History []json.RawMessage
`json:"history"`
OperationState json.RawMessage
`json:"operationState"`
} `json:"status"`
@@ -79,6 +82,7 @@ func CollectSyncOperations(taskCtx plugin.SubTaskContext)
errors.Error {
return nil, err
}
+
recordHealthOnLatestSync(appResponse.Status.OperationState,
appResponse.Status.History, appResponse.Status.Health.Status)
return
sanitizeOperationEntries(appResponse.Status.OperationState,
appResponse.Status.History), nil
},
})
@@ -90,6 +94,34 @@ func CollectSyncOperations(taskCtx plugin.SubTaskContext)
errors.Error {
return collector.Execute()
}
+// recordHealthOnLatestSync stores the application's current health on the
newest
+// history entry. Argo CD only reports health for the application as a whole,
so
+// this is the one moment the value can be tied to a sync. When the operation
in
+// progress failed or is still running, that newer sync owns the current health
+// and the last history entry is left untouched.
+func recordHealthOnLatestSync(operationState json.RawMessage, history
[]json.RawMessage, health string) {
+ if health == "" {
+ return
+ }
+ var operation struct {
+ Phase string `json:"phase"`
+ }
+ if err := json.Unmarshal(operationState, &operation); err == nil &&
operation.Phase != "" && !isRedundantOperationState(operation.Phase) {
+ return
+ }
+ for i := len(history) - 1; i >= 0; i-- {
+ var entry map[string]json.RawMessage
+ if err := json.Unmarshal(history[i], &entry); err != nil ||
entry == nil {
+ continue
+ }
+ entry["applicationHealth"], _ = json.Marshal(health)
+ if tagged, err := json.Marshal(entry); err == nil {
+ history[i] = tagged
+ }
+ return
+ }
+}
+
func sanitizeOperationEntries(operationState json.RawMessage, history
[]json.RawMessage) []json.RawMessage {
results := make([]json.RawMessage, 0, len(history)+1)
diff --git a/backend/plugins/argocd/tasks/sync_operation_collector_test.go
b/backend/plugins/argocd/tasks/sync_operation_collector_test.go
index 9737e880e..62576e9b1 100644
--- a/backend/plugins/argocd/tasks/sync_operation_collector_test.go
+++ b/backend/plugins/argocd/tasks/sync_operation_collector_test.go
@@ -66,3 +66,43 @@ func TestSanitizeOperationEntries_TrimsWhitespace(t
*testing.T) {
assert.JSONEq(t, `{ "id":3 }`, string(result[0]))
}
}
+
+func TestRecordHealthOnLatestSync_TagsOnlyNewestHistoryEntry(t *testing.T) {
+ history := []json.RawMessage{
+ json.RawMessage(`{"id": 1, "revision": "aaa"}`),
+ json.RawMessage(`{"id": 2, "revision": "bbb"}`),
+ json.RawMessage("null"),
+ }
+
+ recordHealthOnLatestSync(json.RawMessage(`{"phase": "Succeeded"}`),
history, "Degraded")
+
+ assert.JSONEq(t, `{"id": 1, "revision": "aaa"}`, string(history[0]))
+ assert.JSONEq(t, `{"id": 2, "revision": "bbb", "applicationHealth":
"Degraded"}`, string(history[1]))
+ assert.Equal(t, "null", string(history[2]))
+}
+
+func TestRecordHealthOnLatestSync_NoOperationState(t *testing.T) {
+ history := []json.RawMessage{json.RawMessage(`{"id": 7}`)}
+
+ recordHealthOnLatestSync(nil, history, "Healthy")
+
+ assert.JSONEq(t, `{"id": 7, "applicationHealth": "Healthy"}`,
string(history[0]))
+}
+
+func TestRecordHealthOnLatestSync_NewerOperationOwnsTheHealth(t *testing.T) {
+ for _, phase := range []string{"Failed", "Error", "Running",
"Terminating"} {
+ history := []json.RawMessage{json.RawMessage(`{"id": 1}`)}
+
+ recordHealthOnLatestSync(json.RawMessage(`{"phase":
"`+phase+`"}`), history, "Degraded")
+
+ assert.JSONEq(t, `{"id": 1}`, string(history[0]), phase)
+ }
+}
+
+func TestRecordHealthOnLatestSync_NoHealthReported(t *testing.T) {
+ history := []json.RawMessage{json.RawMessage(`{"id": 1}`)}
+
+ recordHealthOnLatestSync(nil, history, "")
+
+ assert.JSONEq(t, `{"id": 1}`, string(history[0]))
+}
diff --git a/backend/plugins/argocd/tasks/sync_operation_convertor.go
b/backend/plugins/argocd/tasks/sync_operation_convertor.go
index debf6c5a7..5df8e6501 100644
--- a/backend/plugins/argocd/tasks/sync_operation_convertor.go
+++ b/backend/plugins/argocd/tasks/sync_operation_convertor.go
@@ -109,6 +109,8 @@ func ConvertSyncOperations(taskCtx plugin.SubTaskContext)
errors.Error {
}
created = created.UTC()
+ result, originalResult :=
convertSyncOperationResult(syncOp)
+
deployment := &devops.CICDDeployment{
DomainEntity: domainlayer.DomainEntity{
Id: deploymentId,
@@ -116,10 +118,10 @@ func ConvertSyncOperations(taskCtx plugin.SubTaskContext)
errors.Error {
CicdScopeId: scopeId,
Name: fmt.Sprintf("%s:%d",
syncOp.ApplicationName, syncOp.DeploymentId),
DisplayTitle: syncOp.Message,
- Result:
convertPhaseToResult(syncOp.Phase),
+ Result: result,
Status:
convertPhaseToStatus(syncOp.Phase),
OriginalStatus: syncOp.Phase,
- OriginalResult: syncOp.Phase,
+ OriginalResult: originalResult,
Environment: detectEnvironment(syncOp,
application, data.Options.ScopeConfig, data.RegexEnricher),
OriginalEnvironment: syncOp.ApplicationName,
TaskDatesInfo: devops.TaskDatesInfo{
@@ -183,6 +185,16 @@ func ConvertSyncOperations(taskCtx plugin.SubTaskContext)
errors.Error {
return converter.Execute()
}
+// convertSyncOperationResult returns the deployment result of a sync and the
Argo CD
+// value it comes from. A sync only reports whether the manifests were
applied, so a
+// Succeeded sync whose rollout ended up Degraded is a failed deployment.
+func convertSyncOperationResult(syncOp *models.ArgocdSyncOperation) (string,
string) {
+ if syncOp.Phase == "Succeeded" && syncOp.HealthStatus == "Degraded" {
+ return devops.RESULT_FAILURE, syncOp.HealthStatus
+ }
+ return convertPhaseToResult(syncOp.Phase), syncOp.Phase
+}
+
func convertPhaseToResult(phase string) string {
switch phase {
case "Succeeded":
diff --git a/backend/plugins/argocd/tasks/sync_operation_convertor_test.go
b/backend/plugins/argocd/tasks/sync_operation_convertor_test.go
index d905d1c02..58ecb46c3 100644
--- a/backend/plugins/argocd/tasks/sync_operation_convertor_test.go
+++ b/backend/plugins/argocd/tasks/sync_operation_convertor_test.go
@@ -94,6 +94,27 @@ func TestConvertPhaseToResult(t *testing.T) {
assert.Equal(t, devops.RESULT_DEFAULT, convertPhaseToResult("Unknown"))
}
+func TestConvertSyncOperationResult(t *testing.T) {
+ cases := []struct {
+ phase string
+ health string
+ result string
+ originalResult string
+ }{
+ {"Succeeded", "Degraded", devops.RESULT_FAILURE, "Degraded"},
+ {"Succeeded", "Healthy", devops.RESULT_SUCCESS, "Succeeded"},
+ {"Succeeded", "Progressing", devops.RESULT_SUCCESS,
"Succeeded"},
+ {"Succeeded", "", devops.RESULT_SUCCESS, "Succeeded"},
+ {"Failed", "Degraded", devops.RESULT_FAILURE, "Failed"},
+ {"Running", "Degraded", devops.RESULT_DEFAULT, "Running"},
+ }
+ for _, c := range cases {
+ result, originalResult :=
convertSyncOperationResult(&models.ArgocdSyncOperation{Phase: c.phase,
HealthStatus: c.health})
+ assert.Equal(t, c.result, result, "%s/%s", c.phase, c.health)
+ assert.Equal(t, c.originalResult, originalResult, "%s/%s",
c.phase, c.health)
+ }
+}
+
func TestConvertPhaseToStatus(t *testing.T) {
assert.Equal(t, devops.STATUS_DONE, convertPhaseToStatus("Succeeded"))
assert.Equal(t, devops.STATUS_DONE, convertPhaseToStatus("Failed"))
diff --git a/backend/plugins/argocd/tasks/sync_operation_extractor.go
b/backend/plugins/argocd/tasks/sync_operation_extractor.go
index 5138bbb04..9feef3035 100644
--- a/backend/plugins/argocd/tasks/sync_operation_extractor.go
+++ b/backend/plugins/argocd/tasks/sync_operation_extractor.go
@@ -66,6 +66,8 @@ type ArgocdApiSyncOperation struct {
} `json:"initiatedBy"`
Metadata ArgocdApiSyncOperationMetadata `json:"metadata"`
Operation ArgocdApiSyncOperationDetails `json:"operation"`
+ // ApplicationHealth is not an Argo CD field, see
recordHealthOnLatestSync
+ ApplicationHealth string `json:"applicationHealth"`
// For operationState (current operation)
Phase string `json:"phase"` // Succeeded, Failed, Error,
Running, Terminating
@@ -147,6 +149,7 @@ func ExtractSyncOperations(taskCtx plugin.SubTaskContext)
errors.Error {
}
revisionDirty := make(map[string][]string)
+ lastSeenHealth := make(map[int64]string)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
@@ -246,6 +249,13 @@ func ExtractSyncOperations(taskCtx plugin.SubTaskContext)
errors.Error {
syncOp.StartedAt = start
syncOp.Phase = "Succeeded"
+ // Raw rows are read oldest first and a sync
only carries health while it
+ // is the newest one, so keep the last value
seen for later rows.
+ if apiOp.ApplicationHealth != "" {
+ lastSeenHealth[syncOp.DeploymentId] =
apiOp.ApplicationHealth
+ }
+ syncOp.HealthStatus =
lastSeenHealth[syncOp.DeploymentId]
+
if apiOp.InitiatedBy.Automated {
syncOp.InitiatedBy = "automated"
} else {