This is an automated email from the ASF dual-hosted git repository.
hez pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 70202f3b1 Refactor Change Lead Time - Step 6: refactor deployment
commits diff (#4931)
70202f3b1 is described below
commit 70202f3b1deeb20c16df1de3e9abae0b2445e572
Author: Klesh Wong <[email protected]>
AuthorDate: Sun Apr 16 14:07:58 2023 +0800
Refactor Change Lead Time - Step 6: refactor deployment commits diff (#4931)
* feat: add data_enricher helper
* fix: unit-test and comments
* fix: flush empty set should be ok
* refactor: refdiff deployment commits diff
* feat: e2e for deployment commits diff
* refactor: improve deployment commits diff performance
* fix: linting
---
backend/helpers/pluginhelper/api/batch_save.go | 8 +-
.../cicd_deployment_commits.csv | 7 +
.../commit_parents.csv | 2 +
.../commits_diffs.csv | 0
.../finished_commits_diffs.csv | 0
.../project_mapping.csv | 4 +-
.../repo_commits.csv | 3 +
...diff_test.go => deployment_commit_diff_test.go} | 21 ++-
.../e2e/raw_tables/cicd_pipeline_commits.csv | 6 -
.../refdiff/e2e/raw_tables/cicd_pipelines.csv | 6 -
.../plugins/refdiff/e2e/raw_tables/cicd_tasks.csv | 6 -
backend/plugins/refdiff/impl/impl.go | 2 +-
backend/plugins/refdiff/refdiff.go | 11 +-
.../tasks/deployment_commit_diff_calculator.go | 175 ++++++++++++++++++
.../project_deployment_commit_diff_calculator.go | 195 ---------------------
15 files changed, 211 insertions(+), 235 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/batch_save.go
b/backend/helpers/pluginhelper/api/batch_save.go
index 9b665df05..7cf57c254 100644
--- a/backend/helpers/pluginhelper/api/batch_save.go
+++ b/backend/helpers/pluginhelper/api/batch_save.go
@@ -19,12 +19,13 @@ package api
import (
"fmt"
+ "reflect"
+ "strings"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/log"
- "reflect"
- "strings"
)
// BatchSave performs mulitple records persistence of a specific type in one
sql query to improve the performance
@@ -106,6 +107,9 @@ func (c *BatchSave) Add(slot interface{}) errors.Error {
// Flush save cached records into database
func (c *BatchSave) Flush() errors.Error {
+ if c.current == 0 {
+ return nil
+ }
clauses := make([]dal.Clause, 0)
if c.tableName != "" {
clauses = append(clauses, dal.From(c.tableName))
diff --git
a/backend/plugins/refdiff/e2e/deployment_commit_diff/cicd_deployment_commits.csv
b/backend/plugins/refdiff/e2e/deployment_commit_diff/cicd_deployment_commits.csv
new file mode 100644
index 000000000..41bce1e6d
--- /dev/null
+++
b/backend/plugins/refdiff/e2e/deployment_commit_diff/cicd_deployment_commits.csv
@@ -0,0 +1,7 @@
+id,result,started_date,cicd_pipeline_id,cicd_scope_id,repo_url,environment,prev_success_deployment_commit_id,commit_sha,created_date
+1,SUCCESS,2023-04-10T06:51:47.000+00:00,pipeline1,github:GithubRepo:1:484251804,REPO111,PRODUCTION,,commit_sha1,2023-4-10
6:51:47
+2,SUCCESS,2023-04-10T06:53:51.000+00:00,pipeline2,github:GithubRepo:1:484251804,REPO111,PRODUCTION,1,commit_sha2,2023-4-10
6:53:51
+3,SUCCESS,2023-04-13T07:21:16.000+00:00,pipeline3,github:GithubRepo:1:484251804,REPO111,PRODUCTION,2,commit_sha4,2023-4-13
7:21:16
+4,SUCCESS,2023-04-13T07:22:14.000+00:00,pipeline4,github:GithubRepo:1:384111310,REPO222,PRODUCTION,,commit_sha10,2023-4-13
7:22:14
+5,SUCCESS,2023-04-13T07:28:14.000+00:00,pipeline5,github:GithubRepo:1:384111310,REPO222,PRODUCTION,5,commit_sha11,2023-4-13
7:28:14
+6,SUCCESS,2023-04-13T07:29:34.000+00:00,pipeline4,github:GithubRepo:1:384111310,REPO222,PRODUCTION,6,commit_sha12,2023-4-13
7:29:34
\ No newline at end of file
diff --git a/backend/plugins/refdiff/e2e/raw_tables/commit_parents.csv
b/backend/plugins/refdiff/e2e/deployment_commit_diff/commit_parents.csv
similarity index 74%
rename from backend/plugins/refdiff/e2e/raw_tables/commit_parents.csv
rename to backend/plugins/refdiff/e2e/deployment_commit_diff/commit_parents.csv
index a14ceb368..a8fcf849a 100644
--- a/backend/plugins/refdiff/e2e/raw_tables/commit_parents.csv
+++ b/backend/plugins/refdiff/e2e/deployment_commit_diff/commit_parents.csv
@@ -4,3 +4,5 @@ commit_sha2,commit_sha6
commit_sha3,commit_sha6
commit_sha4,commit_sha7
commit_sha5,commit_sha7
+commit_sha12,commit_sha11
+commit_sha11,commit_sha10
diff --git a/backend/plugins/refdiff/e2e/snapshot_tables/commits_diffs.csv
b/backend/plugins/refdiff/e2e/deployment_commit_diff/commits_diffs.csv
similarity index 100%
rename from backend/plugins/refdiff/e2e/snapshot_tables/commits_diffs.csv
rename to backend/plugins/refdiff/e2e/deployment_commit_diff/commits_diffs.csv
diff --git
a/backend/plugins/refdiff/e2e/snapshot_tables/finished_commits_diffs.csv
b/backend/plugins/refdiff/e2e/deployment_commit_diff/finished_commits_diffs.csv
similarity index 100%
rename from
backend/plugins/refdiff/e2e/snapshot_tables/finished_commits_diffs.csv
rename to
backend/plugins/refdiff/e2e/deployment_commit_diff/finished_commits_diffs.csv
diff --git a/backend/plugins/refdiff/e2e/raw_tables/project_mapping.csv
b/backend/plugins/refdiff/e2e/deployment_commit_diff/project_mapping.csv
similarity index 53%
rename from backend/plugins/refdiff/e2e/raw_tables/project_mapping.csv
rename to backend/plugins/refdiff/e2e/deployment_commit_diff/project_mapping.csv
index bd480d897..3ceda583e 100644
--- a/backend/plugins/refdiff/e2e/raw_tables/project_mapping.csv
+++ b/backend/plugins/refdiff/e2e/deployment_commit_diff/project_mapping.csv
@@ -1,5 +1,5 @@
project_name,table,row_id
project1,repos,github:GithubRepo:1:484251804
+project1,cicd_scopes,github:GithubRepo:1:484251804
project2,repos,github:GithubRepo:1:384111310
-project3,boards,jira:JiraBoard:1:68
-
+project2,cicd_scopes,github:GithubRepo:1:384111310
\ No newline at end of file
diff --git a/backend/plugins/refdiff/e2e/raw_tables/repo_commits.csv
b/backend/plugins/refdiff/e2e/deployment_commit_diff/repo_commits.csv
similarity index 77%
rename from backend/plugins/refdiff/e2e/raw_tables/repo_commits.csv
rename to backend/plugins/refdiff/e2e/deployment_commit_diff/repo_commits.csv
index d73934b64..a5779f1e4 100644
--- a/backend/plugins/refdiff/e2e/raw_tables/repo_commits.csv
+++ b/backend/plugins/refdiff/e2e/deployment_commit_diff/repo_commits.csv
@@ -9,3 +9,6 @@ github:GithubRepo:1:484251804,commit_sha7
github:GithubRepo:1:484251804,commit_sha8
github:GithubRepo:2:484251804,commit_sha9
github:GithubRepo:3:484251804,commit_sha1
+github:GithubRepo:1:384111310,commit_sha10
+github:GithubRepo:1:384111310,commit_sha11
+github:GithubRepo:1:384111310,commit_sha12
\ No newline at end of file
diff --git a/backend/plugins/refdiff/e2e/project_deployment_commit_diff_test.go
b/backend/plugins/refdiff/e2e/deployment_commit_diff_test.go
similarity index 68%
rename from backend/plugins/refdiff/e2e/project_deployment_commit_diff_test.go
rename to backend/plugins/refdiff/e2e/deployment_commit_diff_test.go
index 1b701e477..00a8c7c6d 100644
--- a/backend/plugins/refdiff/e2e/project_deployment_commit_diff_test.go
+++ b/backend/plugins/refdiff/e2e/deployment_commit_diff_test.go
@@ -18,16 +18,17 @@ limitations under the License.
package e2e
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/refdiff/impl"
"github.com/apache/incubator-devlake/plugins/refdiff/tasks"
- "testing"
)
-func TestRepoDataFlow(t *testing.T) {
+func TestDeploymentCommitDiffDataFlow(t *testing.T) {
var plugin impl.RefDiff
dataflowTester := e2ehelper.NewDataFlowTester(t, "refdiff", plugin)
@@ -39,23 +40,21 @@ func TestRepoDataFlow(t *testing.T) {
}
// import raw data table
- dataflowTester.ImportCsvIntoTabler("./raw_tables/project_mapping.csv",
&crossdomain.ProjectMapping{})
- dataflowTester.ImportCsvIntoTabler("./raw_tables/cicd_tasks.csv",
&devops.CICDTask{})
- dataflowTester.ImportCsvIntoTabler("./raw_tables/cicd_pipelines.csv",
&devops.CICDPipeline{})
-
dataflowTester.ImportCsvIntoTabler("./raw_tables/cicd_pipeline_commits.csv",
&devops.CiCDPipelineCommit{})
- dataflowTester.ImportCsvIntoTabler("./raw_tables/repo_commits.csv",
&code.RepoCommit{})
- dataflowTester.ImportCsvIntoTabler("./raw_tables/commit_parents.csv",
&code.CommitParent{})
+
dataflowTester.ImportCsvIntoTabler("./deployment_commit_diff/project_mapping.csv",
&crossdomain.ProjectMapping{})
+
dataflowTester.ImportCsvIntoTabler("./deployment_commit_diff/repo_commits.csv",
&code.RepoCommit{})
+
dataflowTester.ImportCsvIntoTabler("./deployment_commit_diff/commit_parents.csv",
&code.CommitParent{})
+
dataflowTester.ImportCsvIntoTabler("./deployment_commit_diff/cicd_deployment_commits.csv",
&devops.CicdDeploymentCommit{})
// verify extraction
dataflowTester.FlushTabler(&code.CommitsDiff{})
dataflowTester.FlushTabler(&code.FinishedCommitsDiff{})
- dataflowTester.Subtask(tasks.CalculateProjectDeploymentCommitsDiffMeta,
taskData)
+ dataflowTester.Subtask(tasks.CalculateDeploymentCommitsDiffMeta,
taskData)
dataflowTester.VerifyTableWithOptions(&code.CommitsDiff{},
e2ehelper.TableOptions{
- CSVRelPath: "./snapshot_tables/commits_diffs.csv",
+ CSVRelPath: "./deployment_commit_diff/commits_diffs.csv",
})
dataflowTester.VerifyTableWithOptions(&code.FinishedCommitsDiff{},
e2ehelper.TableOptions{
- CSVRelPath: "./snapshot_tables/finished_commits_diffs.csv",
+ CSVRelPath:
"./deployment_commit_diff/finished_commits_diffs.csv",
})
}
diff --git a/backend/plugins/refdiff/e2e/raw_tables/cicd_pipeline_commits.csv
b/backend/plugins/refdiff/e2e/raw_tables/cicd_pipeline_commits.csv
deleted file mode 100644
index 1567f87d2..000000000
--- a/backend/plugins/refdiff/e2e/raw_tables/cicd_pipeline_commits.csv
+++ /dev/null
@@ -1,6 +0,0 @@
-pipeline_id,commit_sha,branch,repo_id,repo_url
-github:GithubRun:1:1,commit_sha1,main,github:GithubRepo:1:484251804,""
-github:GithubRun:1:2,commit_sha2,main,github:GithubRepo:1:484251804,""
-github:GithubRun:1:3,commit_sha3,fix-edit-url,github:GithubRepo:1:484251804,""
-github:GithubRun:1:4,commit_sha4,main,github:GithubRepo:1:484251804,""
-github:GithubRun:1:5,commit_sha5,main,github:GithubRepo:1:484251804,""
diff --git a/backend/plugins/refdiff/e2e/raw_tables/cicd_pipelines.csv
b/backend/plugins/refdiff/e2e/raw_tables/cicd_pipelines.csv
deleted file mode 100644
index 1c94013e5..000000000
--- a/backend/plugins/refdiff/e2e/raw_tables/cicd_pipelines.csv
+++ /dev/null
@@ -1,6 +0,0 @@
-id,name,result,status,cicd_scope_id
-github:GithubRun:1:1,Production
workflow,SUCCESS,DONE,github:GithubRepo:1:484251804
-github:GithubRun:1:2,Production
workflow,SUCCESS,DONE,github:GithubRepo:1:484251804
-github:GithubRun:1:3,Build workflow,SUCCESS,DONE,github:GithubRepo:1:484251804
-github:GithubRun:1:4,Production
workflow,SUCCESS,DONE,github:GithubRepo:1:484251804
-github:GithubRun:1:5,Production
workflow,SUCCESS,DONE,github:GithubRepo:1:484251804
diff --git a/backend/plugins/refdiff/e2e/raw_tables/cicd_tasks.csv
b/backend/plugins/refdiff/e2e/raw_tables/cicd_tasks.csv
deleted file mode 100644
index bd1b8870b..000000000
--- a/backend/plugins/refdiff/e2e/raw_tables/cicd_tasks.csv
+++ /dev/null
@@ -1,6 +0,0 @@
-id,name,pipeline_id,result,status,type,environment,cicd_scope_id
-github:GithubJob:1:1,build-and-deploy,github:GithubRun:1:1,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,""
-github:GithubJob:1:2,build-and-deploy,github:GithubRun:1:2,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,""
-github:GithubJob:1:3,build,github:GithubRun:1:3,SUCCESS,DONE,"","",""
-github:GithubJob:1:4,build-and-deploy,github:GithubRun:1:4,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,""
-
diff --git a/backend/plugins/refdiff/impl/impl.go
b/backend/plugins/refdiff/impl/impl.go
index a4c4e25d2..518c0a117 100644
--- a/backend/plugins/refdiff/impl/impl.go
+++ b/backend/plugins/refdiff/impl/impl.go
@@ -63,7 +63,7 @@ func (p RefDiff) SubTaskMetas() []plugin.SubTaskMeta {
tasks.CalculateCommitsDiffMeta,
tasks.CalculateIssuesDiffMeta,
tasks.CalculatePrCherryPickMeta,
- tasks.CalculateProjectDeploymentCommitsDiffMeta,
+ tasks.CalculateDeploymentCommitsDiffMeta,
}
}
diff --git a/backend/plugins/refdiff/refdiff.go
b/backend/plugins/refdiff/refdiff.go
index 0c9d462f2..2c246f19b 100644
--- a/backend/plugins/refdiff/refdiff.go
+++ b/backend/plugins/refdiff/refdiff.go
@@ -37,17 +37,15 @@ func main() {
tagsLimit := refdiffCmd.Flags().IntP("tags-limit", "l", 2, "tags limit")
tagsOrder := refdiffCmd.Flags().StringP("tags-order", "d", "", "tags
order")
- _ = refdiffCmd.MarkFlagRequired("repo-id")
+ projectName := refdiffCmd.Flags().StringP("project-name", "P", "",
"project name")
+
+ // _ = refdiffCmd.MarkFlagRequired("repo-id")
//_ = refdiffCmd.MarkFlagRequired("new-ref")
//_ = refdiffCmd.MarkFlagRequired("old-ref")
refdiffCmd.Run = func(cmd *cobra.Command, args []string) {
pairs := make([]map[string]string, 0, 1)
- if *newRef == "" && *oldRef == "" {
- if *tagsPattern == "" {
- panic("You must set at least one part of '-p'
or '-n -o' for tagsPattern or newRef,oldRef")
- }
- } else {
+ if *newRef != "" && *oldRef != "" {
pairs = append(pairs, map[string]string{
"NewRef": *newRef,
"OldRef": *oldRef,
@@ -60,6 +58,7 @@ func main() {
"tagsPattern": *tagsPattern,
"tagsLimit": *tagsLimit,
"tagsOrder": *tagsOrder,
+ "projectName": *projectName,
})
}
runner.RunCmd(refdiffCmd)
diff --git a/backend/plugins/refdiff/tasks/deployment_commit_diff_calculator.go
b/backend/plugins/refdiff/tasks/deployment_commit_diff_calculator.go
new file mode 100644
index 000000000..4b34b6a7b
--- /dev/null
+++ b/backend/plugins/refdiff/tasks/deployment_commit_diff_calculator.go
@@ -0,0 +1,175 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "context"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/refdiff/utils"
+)
+
+var CalculateDeploymentCommitsDiffMeta = plugin.SubTaskMeta{
+ Name: "calculateDeploymentCommitsDiff",
+ EntryPoint: CalculateDeploymentCommitsDiff,
+ EnabledByDefault: true,
+ Description: "Calculate commits diff between deployments in the
specified project",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+}
+
+func CalculateDeploymentCommitsDiff(taskCtx plugin.SubTaskContext)
errors.Error {
+ data := taskCtx.GetData().(*RefdiffTaskData)
+ db := taskCtx.GetDal()
+ ctx := taskCtx.GetContext()
+ logger := taskCtx.GetLogger()
+
+ if data.Options.ProjectName == "" {
+ return nil
+ }
+
+ // step 1. select all deployment commits that need to be calculated
+ pairs := make([]*deploymentCommitPair, 0)
+ err := db.All(
+ &pairs,
+ dal.Select("dc.id, dc.commit_sha, p.commit_sha as
prev_commit_sha"),
+ dal.From("cicd_deployment_commits dc"),
+ dal.Join("LEFT JOIN project_mapping pm ON (pm.table =
'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)"),
+ dal.Join("LEFT JOIN cicd_deployment_commits p ON
(dc.prev_success_deployment_commit_id = p.id)"),
+ dal.Where(
+ `
+ pm.project_name = ?
+ AND dc.prev_success_deployment_commit_id IS NOT NULL
+ AND dc.prev_success_deployment_commit_id <> ''
+ AND NOT EXISTS (
+ SELECT 1
+ FROM finished_commits_diffs fcd
+ WHERE fcd.new_commit_sha = dc.commit_sha AND
fcd.old_commit_sha = p.commit_sha
+ )
+ `,
+ data.Options.ProjectName,
+ ),
+ dal.Orderby(`dc.cicd_scope_id, dc.repo_url, dc.environment,
dc.started_date`),
+ )
+ if err != nil {
+ return err
+ }
+ pairsCount := len(pairs)
+ if pairsCount == 0 {
+ // graph is expensive, we should avoid creating one for nothing
+ return nil
+ }
+
+ // step 2. construct a commit node graph and batch save
+ graph, err := loadCommitGraph(ctx, db, data)
+ if err != nil {
+ return err
+ }
+ batch_save, err := api.NewBatchSave(taskCtx,
reflect.TypeOf(&code.CommitsDiff{}), 1000)
+ if err != nil {
+ return err
+ }
+
+ // step 3. iterate all pairs and calculate diff
+ taskCtx.SetProgress(0, pairsCount)
+ for _, pair := range pairs {
+ select {
+ case <-ctx.Done():
+ return errors.Convert(ctx.Err())
+ default:
+ }
+ lostSha, oldCount, newCount :=
graph.CalculateLostSha(pair.PrevCommitSha, pair.CommitSha)
+ for i, sha := range lostSha {
+ commitsDiff := &code.CommitsDiff{
+ NewCommitSha: pair.CommitSha,
+ OldCommitSha: pair.PrevCommitSha,
+ CommitSha: sha,
+ SortingIndex: i + 1,
+ }
+ err = batch_save.Add(commitsDiff)
+ if err != nil {
+ return err
+ }
+ }
+ err = batch_save.Flush()
+ if err != nil {
+ return err
+ }
+ // mark commits_diff were calculated, no need to do it again in
the future
+ finishedCommitsDiff := &code.FinishedCommitsDiff{
+ NewCommitSha: pair.CommitSha,
+ OldCommitSha: pair.PrevCommitSha,
+ }
+ err = db.CreateOrUpdate(finishedCommitsDiff)
+ if err != nil {
+ return err
+ }
+
+ logger.Info(
+ "total %d commits of difference found between [new][%s]
and [old][%s(total:%d)]",
+ newCount,
+ pair.CommitSha,
+ pair.PrevCommitSha,
+ oldCount,
+ )
+ taskCtx.IncProgress(1)
+ }
+ return nil
+}
+
+type deploymentCommitPair struct {
+ Id string
+ CommitSha string
+ PrevCommitSha string
+}
+
+func loadCommitGraph(ctx context.Context, db dal.Dal, data *RefdiffTaskData)
(*utils.CommitNodeGraph, errors.Error) {
+ graph := utils.NewCommitNodeGraph()
+
+ cursor, err := db.Cursor(
+ dal.Select("cp.commit_sha, cp.parent_commit_sha"),
+ dal.From("commit_parents cp"),
+ dal.Join("LEFT JOIN repo_commits rc ON (rc.commit_sha =
cp.commit_sha)"),
+ dal.Join("LEFT JOIN project_mapping pm ON (pm.table = 'repos'
AND pm.row_id = rc.repo_id)"),
+ dal.Where("pm.project_name = ?", data.Options.ProjectName),
+ )
+ if err != nil {
+ return nil, err
+ }
+ defer cursor.Close()
+
+ commitParent := &code.CommitParent{}
+ for cursor.Next() {
+ select {
+ case <-ctx.Done():
+ return nil, errors.Convert(ctx.Err())
+ default:
+ }
+ err = db.Fetch(cursor, commitParent)
+ if err != nil {
+ return nil, err
+ }
+ graph.AddParent(commitParent.CommitSha,
commitParent.ParentCommitSha)
+ }
+
+ return graph, nil
+}
diff --git
a/backend/plugins/refdiff/tasks/project_deployment_commit_diff_calculator.go
b/backend/plugins/refdiff/tasks/project_deployment_commit_diff_calculator.go
deleted file mode 100644
index ab9ca75f1..000000000
--- a/backend/plugins/refdiff/tasks/project_deployment_commit_diff_calculator.go
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "github.com/apache/incubator-devlake/core/dal"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/models/domainlayer/code"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/plugins/refdiff/utils"
- "reflect"
-)
-
-func CommitDiffConvertor(pipelineCommitShaList []string,
existFinishedCommitDiff []code.FinishedCommitsDiff) (commitPairs
[]code.CommitsDiff, finishedCommitDiffs []code.FinishedCommitsDiff) {
- for i := 0; i < len(pipelineCommitShaList)-1; i++ {
- for _, item := range existFinishedCommitDiff {
- if pipelineCommitShaList[i+1] == item.NewCommitSha &&
pipelineCommitShaList[i] == item.OldCommitSha {
- i++
- break
- }
- }
- commitPairs = append(commitPairs,
code.CommitsDiff{NewCommitSha: pipelineCommitShaList[i+1], OldCommitSha:
pipelineCommitShaList[i]})
- finishedCommitDiffs = append(finishedCommitDiffs,
code.FinishedCommitsDiff{NewCommitSha: pipelineCommitShaList[i+1],
OldCommitSha: pipelineCommitShaList[i]})
- }
- return commitPairs, finishedCommitDiffs
-}
-
-func CalculateProjectDeploymentCommitsDiff(taskCtx plugin.SubTaskContext)
errors.Error {
- data := taskCtx.GetData().(*RefdiffTaskData)
- db := taskCtx.GetDal()
- ctx := taskCtx.GetContext()
- logger := taskCtx.GetLogger()
-
- projectName := data.Options.ProjectName
- if projectName == "" {
- return nil
- }
-
- cursorScope, err := db.Cursor(
- dal.Select("row_id"),
- dal.From("project_mapping"),
- dal.Where("project_name = ?", projectName),
- )
- if err != nil {
- return err
- }
- defer cursorScope.Close()
-
- var existFinishedCommitDiff []code.FinishedCommitsDiff
- err = db.All(&existFinishedCommitDiff,
- dal.Select("*"),
- dal.From("finished_commits_diffs"),
- )
- if err != nil {
- return err
- }
-
- for cursorScope.Next() {
- var scopeId string
- err = errors.Convert(cursorScope.Scan(&scopeId))
- if err != nil {
- return err
- }
-
- var pipelineCommitShaList []string
- err := db.All(&pipelineCommitShaList,
- dal.Select("commit_sha"),
- dal.From("cicd_tasks ct"),
- dal.Join("left join cicd_pipelines cp on cp.id =
ct.pipeline_id"),
- dal.Join("left join cicd_pipeline_commits cpc on
cpc.pipeline_id = cp.id"),
- dal.Where("ct.type = ? and commit_sha != ? and
repo_id=? ", "DEPLOYMENT", "", scopeId),
- dal.Orderby("ct.started_date"),
- )
- if err != nil {
- return err
- }
- // generate commitPairs and finishedCommitDiffs
- commitPairs, finishedCommitDiffs :=
CommitDiffConvertor(pipelineCommitShaList, existFinishedCommitDiff)
-
- insertCountLimitOfDeployCommitsDiff := int(65535 /
reflect.ValueOf(code.CommitsDiff{}).NumField())
- commitNodeGraph := utils.NewCommitNodeGraph()
-
- var CommitParentList []code.CommitParent
- err = db.All(&CommitParentList,
- dal.Select("cp.*"),
- dal.Join("LEFT JOIN repo_commits rc ON (rc.commit_sha =
cp.commit_sha)"),
- dal.From("commit_parents cp"),
- dal.Where("rc.repo_id = ?", scopeId),
- )
- if err != nil {
- return err
- }
-
- for i := 0; i < len(CommitParentList); i++ {
-
commitNodeGraph.AddParent(CommitParentList[i].CommitSha,
CommitParentList[i].ParentCommitSha)
- }
- logger.Info("Create a commit node graph with node count[%d]",
commitNodeGraph.Size())
-
- // calculate diffs for commits pairs and store them into
database
- commitsDiff := &code.CommitsDiff{}
- lenCommitPairs := len(commitPairs)
- taskCtx.SetProgress(0, lenCommitPairs)
-
- for _, pair := range commitPairs {
- select {
- case <-ctx.Done():
- return errors.Convert(ctx.Err())
- default:
- }
-
- commitsDiff.NewCommitSha = pair.NewCommitSha
- commitsDiff.OldCommitSha = pair.OldCommitSha
-
- if commitsDiff.NewCommitSha == commitsDiff.OldCommitSha
{
- // different deploy might point to a same
commit, it is ok
- logger.Info(
- "skipping ref pair due to they are the
same %s",
- commitsDiff.NewCommitSha,
- )
- continue
- }
-
- lostSha, oldCount, newCount :=
commitNodeGraph.CalculateLostSha(commitsDiff.OldCommitSha,
commitsDiff.NewCommitSha)
-
- commitsDiffs := []code.CommitsDiff{}
- commitsDiff.SortingIndex = 1
- for _, sha := range lostSha {
- commitsDiff.CommitSha = sha
- commitsDiffs = append(commitsDiffs,
*commitsDiff)
-
- // sql limit placeholders count only 65535
- if
commitsDiff.SortingIndex%insertCountLimitOfDeployCommitsDiff == 0 {
- logger.Info("commitsDiffs count in
limited[%d] index[%d]--exec and clean", len(commitsDiffs),
commitsDiff.SortingIndex)
- err = db.CreateIfNotExist(commitsDiffs)
- if err != nil {
- return err
- }
- commitsDiffs = []code.CommitsDiff{}
- }
-
- commitsDiff.SortingIndex++
- }
-
- if len(commitsDiffs) > 0 {
- logger.Info("insert data count [%d]",
len(commitsDiffs))
- err = db.CreateIfNotExist(commitsDiffs)
- if err != nil {
- return err
- }
- }
-
- if len(finishedCommitDiffs) > 0 {
- err = db.CreateIfNotExist(finishedCommitDiffs)
- if err != nil {
- return err
- }
- }
-
- logger.Info(
- "total %d commits of difference found between
[new][%s] and [old][%s(total:%d)]",
- newCount,
- commitsDiff.NewCommitSha,
- commitsDiff.OldCommitSha,
- oldCount,
- )
-
- }
-
- }
- taskCtx.IncProgress(1)
- return nil
-}
-
-var CalculateProjectDeploymentCommitsDiffMeta = plugin.SubTaskMeta{
- Name: "calculateProjectDeploymentCommitsDiff",
- EntryPoint: CalculateProjectDeploymentCommitsDiff,
- EnabledByDefault: true,
- Description: "Calculate diff commits between project deployments",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
-}
