This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 9309e7ef2 fix(dora): stream the diff-based deployment fetch in
calculateChangeLeadTime (#9281)
9309e7ef2 is described below
commit 9309e7ef2e91fb070212269ae45f9b313d9a9efc
Author: Marco de la Cruz Sánchez <[email protected]>
AuthorDate: Fri Oct 30 13:53:04 2026 +0100
fix(dora): stream the diff-based deployment fetch in
calculateChangeLeadTime (#9281)
Signed-off-by: mdelacruz-avantio <[email protected]>
---
.../change_lead_time/cicd_deployment_commits.csv | 4 +-
.../dora/e2e/change_lead_time/commits_diffs.csv | 2 +
.../e2e/change_lead_time/project_pr_metrics.csv | 3 +-
.../e2e/change_lead_time/pull_request_comments.csv | 3 +-
.../e2e/change_lead_time/pull_request_commits.csv | 3 +-
.../dora/e2e/change_lead_time/pull_requests.csv | 1 +
.../dora/tasks/change_lead_time_calculator.go | 44 ++++++++++++++--------
7 files changed, 41 insertions(+), 19 deletions(-)
diff --git
a/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv
b/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv
index 9d1e488f0..13bc22dd5 100644
--- a/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv
+++ b/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv
@@ -15,4 +15,6 @@
id,result,started_date,duration_sec,cicd_deployment_id,cicd_scope_id,repo_url,en
14,FAILURE,2023-04-13T07:57:26.000+00:00,60,pipeline8,cicd3,REPO111,PRODUCTION,,,commit14,2023-4-13
7:57:26,2023-04-13T07:58:26.000+00:00
15,SUCCESS,2023-04-13T07:57:45.000+00:00,60,pipeline9,cicd3,REPO111,PRODUCTION,,,commit15,2023-4-13
7:57:45,2023-04-13T07:58:45.000+00:00
16,SUCCESS,2023-04-13T07:58:24.000+00:00,60,pipeline10,cicd3,REPO333,,,,commit16,2023-4-13
7:58:24,2023-04-13T07:59:24.000+00:00
-17,SUCCESS,2023-04-13T07:59:00.000+00:00,60,pipeline11,cicd1,REPO111,PRODUCTION,,project1,direct_commit1,2023-4-13
7:59:00,2023-04-13T08:00:00.000+00:00
\ No newline at end of file
+17,SUCCESS,2023-04-13T07:59:00.000+00:00,60,pipeline11,cicd1,REPO111,PRODUCTION,,project1,direct_commit1,2023-4-13
7:59:00,2023-04-13T08:00:00.000+00:00
+18,SUCCESS,2023-04-13T08:10:00.000+00:00,60,pipeline12,cicd1,REPO111,PRODUCTION,17,project1,commit18,2023-4-13
8:10:00,2023-04-13T08:11:00.000+00:00
+19,SUCCESS,2023-04-13T08:05:00.000+00:00,60,pipeline13,cicd2,REPO111,PRODUCTION,13,,commit19,2023-4-13
8:05:00,2023-04-13T08:06:00.000+00:00
\ No newline at end of file
diff --git a/backend/plugins/dora/e2e/change_lead_time/commits_diffs.csv
b/backend/plugins/dora/e2e/change_lead_time/commits_diffs.csv
index 440a68a4a..f3979dc5b 100644
--- a/backend/plugins/dora/e2e/change_lead_time/commits_diffs.csv
+++ b/backend/plugins/dora/e2e/change_lead_time/commits_diffs.csv
@@ -5,3 +5,5 @@ commit2,commit6,pr_merge_commit3
commit7,commit8,pr_merge_commit4
,commit11,pr_merge_commit5
,commit1,pr_merge_commit0
+direct_commit1,commit18,pr_merge_commit8
+commit13,commit19,pr_merge_commit8
diff --git a/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv
b/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv
index 1d9c736a5..2354cfc6d 100644
--- a/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv
+++ b/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv
@@ -5,4 +5,5 @@
pr2,project1,2537845559d8db99e9cda6190f32b50ec979c722,,comment04,1,60,5,1538,159
pr3,project1,55f445997abbd5918da59d202d28762cd56fbd44,5883,comment07,,5760,6,,10203,2023-04-07T04:51:47.000+00:00,2023-04-10T06:53:51.000+00:00,2023-04-11T06:53:51.000+00:00,2023-04-14T06:53:51.000+00:00,2023-04-13T07:30:34.000+00:00
pr4,project1,5ad0c09c447c19338f1dfbb65d89a3728962b3b7,11704,comment10,1500,,,,11764,2023-04-05T04:51:47.000+00:00,2023-04-14T08:55:01.000+00:00,2023-04-13T07:55:01.000+00:00,2023-04-13T08:55:01.000+00:00,
pr5,project1,62535543802631a0d3daf0b0b78c6a7e05e508fb,13144,comment12,,313068,,,13204,2023-04-04T04:51:47.000+00:00,2022-09-07T23:07:13.000+00:00,2023-04-13T07:55:01.000+00:00,2023-04-13T08:55:01.000+00:00,
-pr7,project1,pr7_commit0,1440,comment13,30,30,9,1433,2933,2023-04-11T07:00:00.000+00:00,2023-04-12T07:30:00.000+00:00,2023-04-12T07:00:00.000+00:00,2023-04-12T08:00:00.000+00:00,2023-04-13T07:52:26.000+00:00
\ No newline at end of file
+pr7,project1,pr7_commit0,1440,comment13,30,30,9,1433,2933,2023-04-11T07:00:00.000+00:00,2023-04-12T07:30:00.000+00:00,2023-04-12T07:00:00.000+00:00,2023-04-12T08:00:00.000+00:00,2023-04-13T07:52:26.000+00:00
+pr8,project1,pr8_commit0,1440,comment14,30,30,19,66,1566,2023-04-12T06:00:00.000+00:00,2023-04-13T06:30:00.000+00:00,2023-04-13T06:00:00.000+00:00,2023-04-13T07:00:00.000+00:00,2023-04-13T08:06:00.000+00:00
\ No newline at end of file
diff --git
a/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv
b/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv
index cc8d062c8..e0a95b834 100644
--- a/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv
+++ b/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv
@@ -13,4 +13,5 @@ comment09,pr3,2023-4-12 6:53:51,i
comment10,pr4,2023-4-14 8:55:01,j
comment11,pr4,2023-4-14 8:55:01,k
comment12,pr5,2022-09-07 23:07:13,l
-comment13,pr7,2023-4-12 7:30:00,m
\ No newline at end of file
+comment13,pr7,2023-4-12 7:30:00,m
+comment14,pr8,2023-4-13 6:30:00,n
\ No newline at end of file
diff --git a/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv
b/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv
index e183d14cc..544e2e802 100644
--- a/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv
+++ b/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv
@@ -12,4 +12,5 @@ pr0_commit0,pr0,2022-1-10 4:51:47,
56b895f0443730c6d7abfbc51a05ab35abd2971f,pr4,2023-4-06 4:51:47,
5ad0c09c447c19338f1dfbb65d89a3728962b3b7,pr4,2023-4-05 4:51:47,
62535543802631a0d3daf0b0b78c6a7e05e508fb,pr5,2023-4-04 4:51:47,
-pr7_commit0,pr7,2023-4-11 7:00:00,
\ No newline at end of file
+pr7_commit0,pr7,2023-4-11 7:00:00,
+pr8_commit0,pr8,2023-4-12 6:00:00,
\ No newline at end of file
diff --git a/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv
b/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv
index a83370ab0..52c068eb8 100644
--- a/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv
+++ b/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv
@@ -7,3 +7,4 @@ pr4,repo1,,pr_merge_commit4,2023-4-13 7:55:01,2023-4-13
8:55:01,,,,refs/heads/fe
pr5,repo1,,pr_merge_commit5,2023-4-13 7:55:01,2023-4-13 8:55:01,,,,
pr6,repo1,,pr_merge_commit6,2023-4-13 7:55:01,,,,,feat/e
pr7,repo1,a,commit9,2023-4-12 7:00:00,2023-4-12 8:00:00,,,,hotfix/g
+pr8,repo1,a,pr_merge_commit8,2023-4-13 6:00:00,2023-4-13 7:00:00,,,,chore/h
diff --git a/backend/plugins/dora/tasks/change_lead_time_calculator.go
b/backend/plugins/dora/tasks/change_lead_time_calculator.go
index fdcf628b5..e3b0a0cde 100644
--- a/backend/plugins/dora/tasks/change_lead_time_calculator.go
+++ b/backend/plugins/dora/tasks/change_lead_time_calculator.go
@@ -192,11 +192,13 @@ func computeTimeSpan(start, end *time.Time) *int64 {
return &minutes
}
-// deploymentCommitWithMergeSha is a helper struct to capture both the
deployment commit
-// and the associated merge_sha from the commits_diffs join query.
-type deploymentCommitWithMergeSha struct {
- devops.CicdDeploymentCommit
- MergeSha string `gorm:"column:merge_sha"`
+// diffDeployment holds the columns of a diff-based deployment match that the
+// lead time conversion reads: the deployment commit id, its finish date, and
+// the merge commit found in its commits_diffs range.
+type diffDeployment struct {
+ Id string
+ FinishedDate *time.Time
+ MergeSha string `gorm:"column:merge_sha"`
}
// batchFetchFirstCommits retrieves the first commit for all pull requests in
the given project.
@@ -289,7 +291,8 @@ func batchFetchFirstReviews(projectName string, db dal.Dal)
(map[string]*code.Pu
}
// batchFetchDeployments retrieves deployment commits for all merge commits in
the given project.
-// Returns a map indexed by merge commit SHA for O(1) lookup performance.
+// Returns a map indexed by merge commit SHA for O(1) lookup performance. The
map values carry
+// only the deployment commit Id and FinishedDate, the two fields the lead
time conversion reads.
//
// Uses a two-phase strategy to avoid the "first deployment over-mapping"
problem:
//
@@ -303,7 +306,7 @@ func batchFetchDeployments(projectName string, db dal.Dal)
(map[string]*devops.C
var directResults []*devops.CicdDeploymentCommit
err := db.All(
&directResults,
- dal.Select("dc.*"),
+ dal.Select("dc.id, dc.finished_date, dc.commit_sha"),
dal.From("cicd_deployment_commits dc"),
dal.Join("LEFT JOIN project_mapping pm ON pm.table =
'cicd_scopes' AND pm.row_id = dc.cicd_scope_id"),
dal.Where("dc.environment = 'PRODUCTION'"), // TODO: remove
when multi-environment is supported
@@ -319,10 +322,8 @@ func batchFetchDeployments(projectName string, db dal.Dal)
(map[string]*devops.C
deploymentMap[dc.CommitSha] = &deploymentCopy
}
}
- var diffResults []*deploymentCommitWithMergeSha
- err = db.All(
- &diffResults,
- dal.Select("dc.*, cd.commit_sha as merge_sha"),
+ cursor, err := db.Cursor(
+ dal.Select("dc.id, dc.finished_date, cd.commit_sha as
merge_sha"),
dal.From("cicd_deployment_commits dc"),
dal.Join("LEFT JOIN cicd_deployment_commits p ON
dc.prev_success_deployment_commit_id = p.id"),
dal.Join("INNER JOIN commits_diffs cd ON cd.new_commit_sha =
dc.commit_sha AND cd.old_commit_sha = COALESCE(p.commit_sha, '')"),
@@ -335,11 +336,24 @@ func batchFetchDeployments(projectName string, db
dal.Dal) (map[string]*devops.C
if err != nil {
return nil, errors.Default.Wrap(err, "failed to batch fetch
diff-based deployments")
}
- for _, result := range diffResults {
- if _, exists := deploymentMap[result.MergeSha]; !exists {
- deploymentCopy := result.CicdDeploymentCommit
- deploymentMap[result.MergeSha] = &deploymentCopy
+ defer cursor.Close()
+ // Rows are ordered by merge sha, then by deployment start, so the
first row
+ // seen for a merge sha is its earliest deployment. Streaming keeps
memory
+ // proportional to the distinct merge shas instead of the join size.
+ for cursor.Next() {
+ row := &diffDeployment{}
+ if err := db.Fetch(cursor, row); err != nil {
+ return nil, errors.Default.Wrap(err, "failed to read
diff-based deployment")
+ }
+ if _, exists := deploymentMap[row.MergeSha]; !exists {
+ deployment := &devops.CicdDeploymentCommit{}
+ deployment.Id = row.Id
+ deployment.FinishedDate = row.FinishedDate
+ deploymentMap[row.MergeSha] = deployment
}
}
+ if err := cursor.Err(); err != nil {
+ return nil, errors.Default.Wrap(err, "failed to read diff-based
deployments")
+ }
return deploymentMap, nil
}
