This is an automated email from the ASF dual-hosted git repository.

klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new 4dc01bffd fix(argocd): report a Degraded rollout as a failed 
deployment (#9354)
4dc01bffd is described below

commit 4dc01bffd201cb02818ff17d1fcf981b29631b6e
Author: ZIHAN DAI <[email protected]>
AuthorDate: Tue Nov 10 22:15:38 2026 +1100

    fix(argocd): report a Degraded rollout as a failed deployment (#9354)
    
    A sync can succeed while the rollout it starts never becomes healthy, and
    the plugin only looks at the sync phase, so such a deployment was stored
    with result SUCCESS. Record the application's health on the newest history
    entry when it is collected and turn a Succeeded sync whose rollout is
    Degraded into result FAILURE.
    
    Signed-off-by: Zihan Dai <[email protected]>
---
 .../raw_tables/_raw_argocd_api_sync_operations.csv |  8 +++
 .../_tool_argocd_sync_operations.csv               |  4 ++
 .../snapshot_tables/cicd_deployment_commits.csv    |  4 ++
 .../e2e/snapshot_tables/cicd_deployments.csv       |  4 ++
 backend/plugins/argocd/e2e/sync_operation_test.go  | 76 ++++++++++++++++++++++
 .../argocd/tasks/sync_operation_collector.go       | 32 +++++++++
 .../argocd/tasks/sync_operation_collector_test.go  | 40 ++++++++++++
 .../argocd/tasks/sync_operation_convertor.go       | 16 ++++-
 .../argocd/tasks/sync_operation_convertor_test.go  | 21 ++++++
 .../argocd/tasks/sync_operation_extractor.go       | 10 +++
 10 files changed, 213 insertions(+), 2 deletions(-)

diff --git 
a/backend/plugins/argocd/e2e/raw_tables/_raw_argocd_api_sync_operations.csv 
b/backend/plugins/argocd/e2e/raw_tables/_raw_argocd_api_sync_operations.csv
new file mode 100644
index 000000000..8cf9f5d54
--- /dev/null
+++ b/backend/plugins/argocd/e2e/raw_tables/_raw_argocd_api_sync_operations.csv
@@ -0,0 +1,8 @@
+id,params,data,url,input,created_at
+1,"{""ConnectionId"":1,""Name"":""myapp""}","{""phase"":""Succeeded"",""message"":""successfully
 synced (all tasks 
run)"",""startedAt"":""2026-11-03T09:59:30Z"",""finishedAt"":""2026-11-03T10:00:00Z"",""syncResult"":{""revision"":""bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb""}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-03
 12:00:00.000
+2,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":1,""revision"":""aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"",""deployedAt"":""2026-11-01T10:00:00Z"",""deployStartedAt"":""2026-11-01T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-03
 12:00:00.000
+3,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":2,""revision"":""bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"",""deployedAt"":""2026-11-03T10:00:00Z"",""deployStartedAt"":""2026-11-03T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true},""applicationHealth"":""Degraded""}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-03
 12:00:00.000
+4,"{""ConnectionId"":1,""Name"":""myapp""}","{""phase"":""Succeeded"",""message"":""successfully
 synced (all tasks 
run)"",""startedAt"":""2026-11-04T09:59:30Z"",""finishedAt"":""2026-11-04T10:00:00Z"",""syncResult"":{""revision"":""cccccccccccccccccccccccccccccccccccccccc""}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-04
 12:00:00.000
+5,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":1,""revision"":""aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"",""deployedAt"":""2026-11-01T10:00:00Z"",""deployStartedAt"":""2026-11-01T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-04
 12:00:00.000
+6,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":2,""revision"":""bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"",""deployedAt"":""2026-11-03T10:00:00Z"",""deployStartedAt"":""2026-11-03T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-04
 12:00:00.000
+7,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":3,""revision"":""cccccccccccccccccccccccccccccccccccccccc"",""deployedAt"":""2026-11-04T10:00:00Z"",""deployStartedAt"":""2026-11-04T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true},""applicationHealth"":""Healthy""}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-04
 12:00:00.000
diff --git 
a/backend/plugins/argocd/e2e/snapshot_tables/_tool_argocd_sync_operations.csv 
b/backend/plugins/argocd/e2e/snapshot_tables/_tool_argocd_sync_operations.csv
new file mode 100644
index 000000000..a347677c5
--- /dev/null
+++ 
b/backend/plugins/argocd/e2e/snapshot_tables/_tool_argocd_sync_operations.csv
@@ -0,0 +1,4 @@
+connection_id,application_name,deployment_id,revision,phase,health_status
+1,myapp,1793527200,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,Succeeded,
+1,myapp,1793700000,bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb,Succeeded,Degraded
+1,myapp,1793786400,cccccccccccccccccccccccccccccccccccccccc,Succeeded,Healthy
diff --git 
a/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployment_commits.csv 
b/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployment_commits.csv
new file mode 100644
index 000000000..30632647e
--- /dev/null
+++ b/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployment_commits.csv
@@ -0,0 +1,4 @@
+id,cicd_deployment_id,result,commit_sha
+argocd:ArgocdSyncOperation:1:myapp:1793527200,argocd:ArgocdSyncOperation:1:myapp:1793527200,SUCCESS,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+argocd:ArgocdSyncOperation:1:myapp:1793700000,argocd:ArgocdSyncOperation:1:myapp:1793700000,FAILURE,bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+argocd:ArgocdSyncOperation:1:myapp:1793786400,argocd:ArgocdSyncOperation:1:myapp:1793786400,SUCCESS,cccccccccccccccccccccccccccccccccccccccc
diff --git a/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployments.csv 
b/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployments.csv
new file mode 100644
index 000000000..ec4e2d730
--- /dev/null
+++ b/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployments.csv
@@ -0,0 +1,4 @@
+id,result,status,original_status,original_result
+argocd:ArgocdSyncOperation:1:myapp:1793527200,SUCCESS,DONE,Succeeded,Succeeded
+argocd:ArgocdSyncOperation:1:myapp:1793700000,FAILURE,DONE,Succeeded,Degraded
+argocd:ArgocdSyncOperation:1:myapp:1793786400,SUCCESS,DONE,Succeeded,Succeeded
diff --git a/backend/plugins/argocd/e2e/sync_operation_test.go 
b/backend/plugins/argocd/e2e/sync_operation_test.go
new file mode 100644
index 000000000..27f24d3eb
--- /dev/null
+++ b/backend/plugins/argocd/e2e/sync_operation_test.go
@@ -0,0 +1,76 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+       "testing"
+
+       "github.com/apache/devlake/core/models/domainlayer/devops"
+       "github.com/apache/devlake/helpers/e2ehelper"
+       "github.com/apache/devlake/plugins/argocd/impl"
+       "github.com/apache/devlake/plugins/argocd/models"
+       
"github.com/apache/devlake/plugins/argocd/models/migrationscripts/archived"
+       "github.com/apache/devlake/plugins/argocd/tasks"
+)
+
+func TestSyncOperationDataFlow(t *testing.T) {
+       var argocd impl.ArgoCD
+       dataflowTester := e2ehelper.NewDataFlowTester(t, "argocd", argocd)
+
+       taskData := &tasks.ArgocdTaskData{
+               Options: &tasks.ArgocdOptions{
+                       ConnectionId:    1,
+                       ApplicationName: "myapp",
+                       ScopeConfig:     &models.ArgocdScopeConfig{},
+               },
+       }
+
+       // Two collections: the first saw revision bbb Degraded, the second a 
newer Healthy
+       // sync, by which time bbb is an ordinary history entry again.
+       
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_argocd_api_sync_operations.csv",
 "_raw_argocd_api_sync_operations")
+       dataflowTester.FlushTabler(&archived.ArgocdApplication{})
+       dataflowTester.FlushTabler(&models.ArgocdRevisionImage{})
+
+       dataflowTester.FlushTabler(&models.ArgocdSyncOperation{})
+       dataflowTester.Subtask(tasks.ExtractSyncOperationsMeta, taskData)
+       dataflowTester.VerifyTableWithOptions(
+               models.ArgocdSyncOperation{},
+               e2ehelper.TableOptions{
+                       CSVRelPath:   
"./snapshot_tables/_tool_argocd_sync_operations.csv",
+                       TargetFields: []string{"connection_id", 
"application_name", "deployment_id", "revision", "phase", "health_status"},
+               },
+       )
+
+       dataflowTester.FlushTabler(&devops.CICDDeployment{})
+       dataflowTester.FlushTabler(&devops.CicdDeploymentCommit{})
+       dataflowTester.Subtask(tasks.ConvertSyncOperationsMeta, taskData)
+       dataflowTester.VerifyTableWithOptions(
+               devops.CICDDeployment{},
+               e2ehelper.TableOptions{
+                       CSVRelPath:   "./snapshot_tables/cicd_deployments.csv",
+                       TargetFields: []string{"id", "result", "status", 
"original_status", "original_result"},
+               },
+       )
+       dataflowTester.VerifyTableWithOptions(
+               devops.CicdDeploymentCommit{},
+               e2ehelper.TableOptions{
+                       CSVRelPath:   
"./snapshot_tables/cicd_deployment_commits.csv",
+                       TargetFields: []string{"id", "cicd_deployment_id", 
"result", "commit_sha"},
+               },
+       )
+}
diff --git a/backend/plugins/argocd/tasks/sync_operation_collector.go 
b/backend/plugins/argocd/tasks/sync_operation_collector.go
index bac7e5e64..553037607 100644
--- a/backend/plugins/argocd/tasks/sync_operation_collector.go
+++ b/backend/plugins/argocd/tasks/sync_operation_collector.go
@@ -69,6 +69,9 @@ func CollectSyncOperations(taskCtx plugin.SubTaskContext) 
errors.Error {
                ResponseParser: func(res *http.Response) ([]json.RawMessage, 
errors.Error) {
                        var appResponse struct {
                                Status struct {
+                                       Health struct {
+                                               Status string `json:"status"`
+                                       } `json:"health"`
                                        History        []json.RawMessage 
`json:"history"`
                                        OperationState json.RawMessage   
`json:"operationState"`
                                } `json:"status"`
@@ -79,6 +82,7 @@ func CollectSyncOperations(taskCtx plugin.SubTaskContext) 
errors.Error {
                                return nil, err
                        }
 
+                       
recordHealthOnLatestSync(appResponse.Status.OperationState, 
appResponse.Status.History, appResponse.Status.Health.Status)
                        return 
sanitizeOperationEntries(appResponse.Status.OperationState, 
appResponse.Status.History), nil
                },
        })
@@ -90,6 +94,34 @@ func CollectSyncOperations(taskCtx plugin.SubTaskContext) 
errors.Error {
        return collector.Execute()
 }
 
+// recordHealthOnLatestSync stores the application's current health on the 
newest
+// history entry. Argo CD only reports health for the application as a whole, 
so
+// this is the one moment the value can be tied to a sync. When the operation 
in
+// progress failed or is still running, that newer sync owns the current health
+// and the last history entry is left untouched.
+func recordHealthOnLatestSync(operationState json.RawMessage, history 
[]json.RawMessage, health string) {
+       if health == "" {
+               return
+       }
+       var operation struct {
+               Phase string `json:"phase"`
+       }
+       if err := json.Unmarshal(operationState, &operation); err == nil && 
operation.Phase != "" && !isRedundantOperationState(operation.Phase) {
+               return
+       }
+       for i := len(history) - 1; i >= 0; i-- {
+               var entry map[string]json.RawMessage
+               if err := json.Unmarshal(history[i], &entry); err != nil || 
entry == nil {
+                       continue
+               }
+               entry["applicationHealth"], _ = json.Marshal(health)
+               if tagged, err := json.Marshal(entry); err == nil {
+                       history[i] = tagged
+               }
+               return
+       }
+}
+
 func sanitizeOperationEntries(operationState json.RawMessage, history 
[]json.RawMessage) []json.RawMessage {
        results := make([]json.RawMessage, 0, len(history)+1)
 
diff --git a/backend/plugins/argocd/tasks/sync_operation_collector_test.go 
b/backend/plugins/argocd/tasks/sync_operation_collector_test.go
index 9737e880e..62576e9b1 100644
--- a/backend/plugins/argocd/tasks/sync_operation_collector_test.go
+++ b/backend/plugins/argocd/tasks/sync_operation_collector_test.go
@@ -66,3 +66,43 @@ func TestSanitizeOperationEntries_TrimsWhitespace(t 
*testing.T) {
                assert.JSONEq(t, `{ "id":3 }`, string(result[0]))
        }
 }
+
+func TestRecordHealthOnLatestSync_TagsOnlyNewestHistoryEntry(t *testing.T) {
+       history := []json.RawMessage{
+               json.RawMessage(`{"id": 1, "revision": "aaa"}`),
+               json.RawMessage(`{"id": 2, "revision": "bbb"}`),
+               json.RawMessage("null"),
+       }
+
+       recordHealthOnLatestSync(json.RawMessage(`{"phase": "Succeeded"}`), 
history, "Degraded")
+
+       assert.JSONEq(t, `{"id": 1, "revision": "aaa"}`, string(history[0]))
+       assert.JSONEq(t, `{"id": 2, "revision": "bbb", "applicationHealth": 
"Degraded"}`, string(history[1]))
+       assert.Equal(t, "null", string(history[2]))
+}
+
+func TestRecordHealthOnLatestSync_NoOperationState(t *testing.T) {
+       history := []json.RawMessage{json.RawMessage(`{"id": 7}`)}
+
+       recordHealthOnLatestSync(nil, history, "Healthy")
+
+       assert.JSONEq(t, `{"id": 7, "applicationHealth": "Healthy"}`, 
string(history[0]))
+}
+
+func TestRecordHealthOnLatestSync_NewerOperationOwnsTheHealth(t *testing.T) {
+       for _, phase := range []string{"Failed", "Error", "Running", 
"Terminating"} {
+               history := []json.RawMessage{json.RawMessage(`{"id": 1}`)}
+
+               recordHealthOnLatestSync(json.RawMessage(`{"phase": 
"`+phase+`"}`), history, "Degraded")
+
+               assert.JSONEq(t, `{"id": 1}`, string(history[0]), phase)
+       }
+}
+
+func TestRecordHealthOnLatestSync_NoHealthReported(t *testing.T) {
+       history := []json.RawMessage{json.RawMessage(`{"id": 1}`)}
+
+       recordHealthOnLatestSync(nil, history, "")
+
+       assert.JSONEq(t, `{"id": 1}`, string(history[0]))
+}
diff --git a/backend/plugins/argocd/tasks/sync_operation_convertor.go 
b/backend/plugins/argocd/tasks/sync_operation_convertor.go
index debf6c5a7..5df8e6501 100644
--- a/backend/plugins/argocd/tasks/sync_operation_convertor.go
+++ b/backend/plugins/argocd/tasks/sync_operation_convertor.go
@@ -109,6 +109,8 @@ func ConvertSyncOperations(taskCtx plugin.SubTaskContext) 
errors.Error {
                        }
                        created = created.UTC()
 
+                       result, originalResult := 
convertSyncOperationResult(syncOp)
+
                        deployment := &devops.CICDDeployment{
                                DomainEntity: domainlayer.DomainEntity{
                                        Id: deploymentId,
@@ -116,10 +118,10 @@ func ConvertSyncOperations(taskCtx plugin.SubTaskContext) 
errors.Error {
                                CicdScopeId:         scopeId,
                                Name:                fmt.Sprintf("%s:%d", 
syncOp.ApplicationName, syncOp.DeploymentId),
                                DisplayTitle:        syncOp.Message,
-                               Result:              
convertPhaseToResult(syncOp.Phase),
+                               Result:              result,
                                Status:              
convertPhaseToStatus(syncOp.Phase),
                                OriginalStatus:      syncOp.Phase,
-                               OriginalResult:      syncOp.Phase,
+                               OriginalResult:      originalResult,
                                Environment:         detectEnvironment(syncOp, 
application, data.Options.ScopeConfig, data.RegexEnricher),
                                OriginalEnvironment: syncOp.ApplicationName,
                                TaskDatesInfo: devops.TaskDatesInfo{
@@ -183,6 +185,16 @@ func ConvertSyncOperations(taskCtx plugin.SubTaskContext) 
errors.Error {
        return converter.Execute()
 }
 
+// convertSyncOperationResult returns the deployment result of a sync and the 
Argo CD
+// value it comes from. A sync only reports whether the manifests were 
applied, so a
+// Succeeded sync whose rollout ended up Degraded is a failed deployment.
+func convertSyncOperationResult(syncOp *models.ArgocdSyncOperation) (string, 
string) {
+       if syncOp.Phase == "Succeeded" && syncOp.HealthStatus == "Degraded" {
+               return devops.RESULT_FAILURE, syncOp.HealthStatus
+       }
+       return convertPhaseToResult(syncOp.Phase), syncOp.Phase
+}
+
 func convertPhaseToResult(phase string) string {
        switch phase {
        case "Succeeded":
diff --git a/backend/plugins/argocd/tasks/sync_operation_convertor_test.go 
b/backend/plugins/argocd/tasks/sync_operation_convertor_test.go
index d905d1c02..58ecb46c3 100644
--- a/backend/plugins/argocd/tasks/sync_operation_convertor_test.go
+++ b/backend/plugins/argocd/tasks/sync_operation_convertor_test.go
@@ -94,6 +94,27 @@ func TestConvertPhaseToResult(t *testing.T) {
        assert.Equal(t, devops.RESULT_DEFAULT, convertPhaseToResult("Unknown"))
 }
 
+func TestConvertSyncOperationResult(t *testing.T) {
+       cases := []struct {
+               phase          string
+               health         string
+               result         string
+               originalResult string
+       }{
+               {"Succeeded", "Degraded", devops.RESULT_FAILURE, "Degraded"},
+               {"Succeeded", "Healthy", devops.RESULT_SUCCESS, "Succeeded"},
+               {"Succeeded", "Progressing", devops.RESULT_SUCCESS, 
"Succeeded"},
+               {"Succeeded", "", devops.RESULT_SUCCESS, "Succeeded"},
+               {"Failed", "Degraded", devops.RESULT_FAILURE, "Failed"},
+               {"Running", "Degraded", devops.RESULT_DEFAULT, "Running"},
+       }
+       for _, c := range cases {
+               result, originalResult := 
convertSyncOperationResult(&models.ArgocdSyncOperation{Phase: c.phase, 
HealthStatus: c.health})
+               assert.Equal(t, c.result, result, "%s/%s", c.phase, c.health)
+               assert.Equal(t, c.originalResult, originalResult, "%s/%s", 
c.phase, c.health)
+       }
+}
+
 func TestConvertPhaseToStatus(t *testing.T) {
        assert.Equal(t, devops.STATUS_DONE, convertPhaseToStatus("Succeeded"))
        assert.Equal(t, devops.STATUS_DONE, convertPhaseToStatus("Failed"))
diff --git a/backend/plugins/argocd/tasks/sync_operation_extractor.go 
b/backend/plugins/argocd/tasks/sync_operation_extractor.go
index 5138bbb04..9feef3035 100644
--- a/backend/plugins/argocd/tasks/sync_operation_extractor.go
+++ b/backend/plugins/argocd/tasks/sync_operation_extractor.go
@@ -66,6 +66,8 @@ type ArgocdApiSyncOperation struct {
        } `json:"initiatedBy"`
        Metadata  ArgocdApiSyncOperationMetadata `json:"metadata"`
        Operation ArgocdApiSyncOperationDetails  `json:"operation"`
+       // ApplicationHealth is not an Argo CD field, see 
recordHealthOnLatestSync
+       ApplicationHealth string `json:"applicationHealth"`
 
        // For operationState (current operation)
        Phase      string     `json:"phase"` // Succeeded, Failed, Error, 
Running, Terminating
@@ -147,6 +149,7 @@ func ExtractSyncOperations(taskCtx plugin.SubTaskContext) 
errors.Error {
        }
 
        revisionDirty := make(map[string][]string)
+       lastSeenHealth := make(map[int64]string)
 
        extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
                RawDataSubTaskArgs: api.RawDataSubTaskArgs{
@@ -246,6 +249,13 @@ func ExtractSyncOperations(taskCtx plugin.SubTaskContext) 
errors.Error {
                                syncOp.StartedAt = start
                                syncOp.Phase = "Succeeded"
 
+                               // Raw rows are read oldest first and a sync 
only carries health while it
+                               // is the newest one, so keep the last value 
seen for later rows.
+                               if apiOp.ApplicationHealth != "" {
+                                       lastSeenHealth[syncOp.DeploymentId] = 
apiOp.ApplicationHealth
+                               }
+                               syncOp.HealthStatus = 
lastSeenHealth[syncOp.DeploymentId]
+
                                if apiOp.InitiatedBy.Automated {
                                        syncOp.InitiatedBy = "automated"
                                } else {

Reply via email to