This is an automated email from the ASF dual-hosted git repository.
mappjzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 9d69664fe Refactor Chage Lead Time - Step 4: generate deployments
based on cicd pipelines/tasks (#4911)
9d69664fe is described below
commit 9d69664feea42320d3a51d8f08bb595482f7d482
Author: Klesh Wong <[email protected]>
AuthorDate: Thu Apr 13 18:55:49 2023 +0800
Refactor Chage Lead Time - Step 4: generate deployments based on cicd
pipelines/tasks (#4911)
* fix: bitbucket pipeline convertor scoping / missed cicd_scope_id
* fix: bitbucket repo_convertor missed cicd_scope conversion
* feat: dora generate deployments from cicd pipelines/tasks
* fix: typo and e2e
---
.../20230411_add_cicd_deployment_commits.go | 2 +-
backend/plugins/bitbucket/e2e/repo_test.go | 16 ++-
.../bitbucket/e2e/snapshot_tables/cicd_scopes.csv | 2 +
.../plugins/bitbucket/tasks/pipeline_convertor.go | 16 ++-
backend/plugins/bitbucket/tasks/repo_convertor.go | 30 +++-
backend/plugins/dora/dora.go | 2 -
backend/plugins/dora/impl/impl.go | 2 +
.../dora/tasks/deployment_commits_generator.go | 156 +++++++++++++++++++++
8 files changed, 209 insertions(+), 17 deletions(-)
diff --git
a/backend/core/models/migrationscripts/20230411_add_cicd_deployment_commits.go
b/backend/core/models/migrationscripts/20230411_add_cicd_deployment_commits.go
index 92eb82196..0c4c16f3d 100644
---
a/backend/core/models/migrationscripts/20230411_add_cicd_deployment_commits.go
+++
b/backend/core/models/migrationscripts/20230411_add_cicd_deployment_commits.go
@@ -41,5 +41,5 @@ func (*addCicdDeploymentCommits) Version() uint64 {
}
func (*addCicdDeploymentCommits) Name() string {
- return "Rename cicd_piopeline_commits repo to repo_url"
+ return "Rename cicd_pipeline_commits repo to repo_url"
}
diff --git a/backend/plugins/bitbucket/e2e/repo_test.go
b/backend/plugins/bitbucket/e2e/repo_test.go
index dbd30692f..239bf6c88 100644
--- a/backend/plugins/bitbucket/e2e/repo_test.go
+++ b/backend/plugins/bitbucket/e2e/repo_test.go
@@ -19,8 +19,11 @@ package e2e
import (
"encoding/json"
+ "testing"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/helpers/pluginhelper"
@@ -28,7 +31,6 @@ import (
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
"github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
"github.com/stretchr/testify/assert"
- "testing"
)
func TestRepoDataFlow(t *testing.T) {
@@ -79,6 +81,7 @@ func TestRepoDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&code.Repo{})
dataflowTester.FlushTabler(&ticket.Board{})
+ dataflowTester.FlushTabler(&devops.CicdScope{})
dataflowTester.Subtask(tasks.ConvertRepoMeta, taskData)
dataflowTester.VerifyTable(
code.Repo{},
@@ -105,4 +108,15 @@ func TestRepoDataFlow(t *testing.T) {
"created_date",
),
)
+ dataflowTester.VerifyTable(
+ devops.CicdScope{},
+ "./snapshot_tables/cicd_scopes.csv",
+ e2ehelper.ColumnWithRawData(
+ "id",
+ "name",
+ "description",
+ "url",
+ "created_date",
+ ),
+ )
}
diff --git a/backend/plugins/bitbucket/e2e/snapshot_tables/cicd_scopes.csv
b/backend/plugins/bitbucket/e2e/snapshot_tables/cicd_scopes.csv
new file mode 100644
index 000000000..cf77c6767
--- /dev/null
+++ b/backend/plugins/bitbucket/e2e/snapshot_tables/cicd_scopes.csv
@@ -0,0 +1,2 @@
+id,name,description,url,created_date,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+bitbucket:BitbucketRepo:1:likyh/likyhphp,likyhphp,,https://bitbucket.org/likyh/likyhphp/issues,2022-06-17T03:27:18.865+00:00,,,0,
diff --git a/backend/plugins/bitbucket/tasks/pipeline_convertor.go
b/backend/plugins/bitbucket/tasks/pipeline_convertor.go
index 673fa09fa..3f38cd291 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_convertor.go
@@ -48,8 +48,12 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext)
errors.Error {
if err != nil {
return err
}
+ repoId :=
didgen.NewDomainIdGenerator(&models.BitbucketRepo{}).Generate(repo.ConnectionId,
repo.BitbucketId)
- cursor, err := db.Cursor(dal.From(models.BitbucketPipeline{}))
+ cursor, err := db.Cursor(
+ dal.From(models.BitbucketPipeline{}),
+ dal.Where("connection_id = ? AND repo_id = ?",
data.Options.ConnectionId, data.Options.FullName),
+ )
if err != nil {
return err
}
@@ -71,11 +75,10 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext)
errors.Error {
results := make([]interface{}, 0, 2)
domainPipelineCommit := &devops.CiCDPipelineCommit{
PipelineId:
pipelineIdGen.Generate(data.Options.ConnectionId,
bitbucketPipeline.BitbucketId),
- RepoId:
didgen.NewDomainIdGenerator(&models.BitbucketRepo{}).
-
Generate(bitbucketPipeline.ConnectionId, bitbucketPipeline.RepoId),
- CommitSha: bitbucketPipeline.CommitSha,
- Branch: bitbucketPipeline.RefName,
- RepoUrl: repo.HTMLUrl,
+ RepoId: repoId,
+ CommitSha: bitbucketPipeline.CommitSha,
+ Branch: bitbucketPipeline.RefName,
+ RepoUrl: repo.HTMLUrl,
}
domainPipeline := &devops.CICDPipeline{
DomainEntity: domainlayer.DomainEntity{
@@ -99,6 +102,7 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext)
errors.Error {
CreatedDate: createdAt,
DurationSec:
bitbucketPipeline.DurationInSeconds,
FinishedDate:
bitbucketPipeline.BitbucketCompleteOn,
+ CicdScopeId: repoId,
}
results = append(results, domainPipelineCommit,
domainPipeline)
return results, nil
diff --git a/backend/plugins/bitbucket/tasks/repo_convertor.go
b/backend/plugins/bitbucket/tasks/repo_convertor.go
index 90e6fc9c4..48a49cfc8 100644
--- a/backend/plugins/bitbucket/tasks/repo_convertor.go
+++ b/backend/plugins/bitbucket/tasks/repo_convertor.go
@@ -20,20 +20,22 @@ package tasks
import (
"encoding/json"
"fmt"
+ "io"
+ "net/http"
+ "path"
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
plugin "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
aha
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
- "io"
- "net/http"
- "path"
- "reflect"
)
const RAW_REPOSITORIES_TABLE = "bitbucket_api_repositories"
@@ -87,7 +89,7 @@ func ConvertRepo(taskCtx plugin.SubTaskContext) errors.Error {
cursor, err := db.Cursor(
dal.From(&models.BitbucketRepo{}),
- dal.Where("bitbucket_id = ?", repoId),
+ dal.Where("connection_id = ? AND bitbucket_id = ?",
data.Options.ConnectionId, repoId),
)
if err != nil {
return err
@@ -102,9 +104,12 @@ func ConvertRepo(taskCtx plugin.SubTaskContext)
errors.Error {
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
repository := inputRow.(*models.BitbucketRepo)
+
+ repoId := repoIdGen.Generate(data.Options.ConnectionId,
repository.BitbucketId)
+
domainRepository := &code.Repo{
DomainEntity: domainlayer.DomainEntity{
- Id:
repoIdGen.Generate(data.Options.ConnectionId, repository.BitbucketId),
+ Id: repoId,
},
Name: repository.Name,
Url: repository.HTMLUrl,
@@ -116,7 +121,17 @@ func ConvertRepo(taskCtx plugin.SubTaskContext)
errors.Error {
domainBoard := &ticket.Board{
DomainEntity: domainlayer.DomainEntity{
- Id:
repoIdGen.Generate(data.Options.ConnectionId, repository.BitbucketId),
+ Id: repoId,
+ },
+ Name: repository.Name,
+ Url: fmt.Sprintf("%s/%s",
repository.HTMLUrl, "issues"),
+ Description: repository.Description,
+ CreatedDate: repository.CreatedDate,
+ }
+
+ domainCicdScope := &devops.CicdScope{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: repoId,
},
Name: repository.Name,
Url: fmt.Sprintf("%s/%s",
repository.HTMLUrl, "issues"),
@@ -127,6 +142,7 @@ func ConvertRepo(taskCtx plugin.SubTaskContext)
errors.Error {
return []interface{}{
domainRepository,
domainBoard,
+ domainCicdScope,
}, nil
},
})
diff --git a/backend/plugins/dora/dora.go b/backend/plugins/dora/dora.go
index cbe95f772..a4ee412d3 100644
--- a/backend/plugins/dora/dora.go
+++ b/backend/plugins/dora/dora.go
@@ -30,12 +30,10 @@ var PluginEntry impl.Dora //nolint
func main() {
cmd := &cobra.Command{Use: "dora"}
- repoId := cmd.Flags().StringP("repoId", "r", "", "repo id")
projectName := cmd.Flags().StringP("projectName", "p", "", "project
name")
cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "repoId": *repoId,
"projectName": *projectName,
})
}
diff --git a/backend/plugins/dora/impl/impl.go
b/backend/plugins/dora/impl/impl.go
index 892343acd..e4bbcfaa8 100644
--- a/backend/plugins/dora/impl/impl.go
+++ b/backend/plugins/dora/impl/impl.go
@@ -19,6 +19,7 @@ package impl
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
@@ -81,6 +82,7 @@ func (p Dora) Settings() interface{} {
func (p Dora) SubTaskMetas() []plugin.SubTaskMeta {
return []plugin.SubTaskMeta{
+ tasks.DeploymentCommitsGeneratorMeta,
tasks.EnrichTaskEnvMeta,
tasks.CalculateChangeLeadTimeMeta,
tasks.ConnectIncidentToDeploymentMeta,
diff --git a/backend/plugins/dora/tasks/deployment_commits_generator.go
b/backend/plugins/dora/tasks/deployment_commits_generator.go
new file mode 100644
index 000000000..2aae4a4d7
--- /dev/null
+++ b/backend/plugins/dora/tasks/deployment_commits_generator.go
@@ -0,0 +1,156 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "reflect"
+ "time"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+var DeploymentCommitsGeneratorMeta = plugin.SubTaskMeta{
+ Name: "generateDeploymentCommits",
+ EntryPoint: GenerateDeploymentCommits,
+ EnabledByDefault: false, // it should be executed before
refdiff.calculateDeploymentCommitsDiff, check
https://github.com/apache/incubator-devlake/issues/4869 for detail
+ Description: "Generate deployment_commits from
cicd_pipeline_commits if cicd_pipeline.type == DEPLOYMENT or any of its
cicd_tasks is a deployment task",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
+
+type pipelineCommitEx struct {
+ devops.CiCDPipelineCommit
+ PipelineName string
+ Result string
+ Status string
+ DurationSec *uint64
+ CreatedDate *time.Time
+ FinishedDate *time.Time
+ Environment string
+ CicdScopeId string
+ HasTestingTasks bool
+ HasStagingTasks bool
+ HasProductionTasks bool
+}
+
+func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error {
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*DoraTaskData)
+ // select all cicd_pipeline_commits from all "Deployments" in the
project
+ // Note that failed records shall be included as well
+ cursor, err := db.Cursor(
+ dal.Select(`
+ pc.*, p.name as pipeline_name,
+ p.result,
+ p.status,
+ p.duration_sec,
+ p.created_date,
+ p.finished_date,
+ p.environment,
+ p.cicd_scope_id,
+ EXISTS(SELECT 1 FROM cicd_tasks t WHERE t.pipeline_id =
p.id AND t.environment = ?)
+ as has_testing_tasks,
+ EXISTS(SELECT 1 FROM cicd_tasks t WHERE t.pipeline_id =
p.id AND t.environment = ?)
+ as has_staging_tasks,
+ EXISTS( SELECT 1 FROM cicd_tasks t WHERE t.pipeline_id
= p.id AND t.environment = ?)
+ as has_production_tasks
+ `),
+ dal.From("cicd_pipeline_commits pc"),
+ dal.Join("LEFT JOIN cicd_pipelines p ON (p.id =
pc.pipeline_id)"),
+ dal.Join("LEFT JOIN project_mapping pm ON (pm.table = ? AND
pm.row_id = p.cicd_scope_id)"),
+ dal.Where(
+ `
+ pm.project_name = ? AND (
+ p.type = ? OR EXISTS(
+ SELECT 1 FROM cicd_tasks t WHERE
t.pipeline_id = p.id AND t.type = p.type
+ )
+ )
+ `, devops.STAGING, devops.PRODUCTION, "cicd_scopes",
data.Options.ProjectName, devops.DEPLOYMENT,
+ ),
+ )
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ enricher, err := api.NewDataConverter(api.DataConverterArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: DoraApiParams{
+ ProjectName: data.Options.ProjectName,
+ },
+ Table: "cicd_pipeline_commits",
+ },
+ InputRowType: reflect.TypeOf(pipelineCommitEx{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
+ pipelineCommit := inputRow.(*pipelineCommitEx)
+
+ domainDeployCommit := &devops.CicdDeploymentCommit{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: fmt.Sprintf("%s:%s",
pipelineCommit.PipelineId, pipelineCommit.RepoUrl),
+ },
+ CicdScopeId: pipelineCommit.CicdScopeId,
+ CicdPipelineId: pipelineCommit.PipelineId,
+ Name: pipelineCommit.PipelineName,
+ Result: pipelineCommit.Result,
+ Status: pipelineCommit.Status,
+ Environment: pipelineCommit.Environment,
+ CreatedDate: *pipelineCommit.CreatedDate,
+ DurationSec: pipelineCommit.DurationSec,
+ CommitSha: pipelineCommit.CommitSha,
+ RefName: pipelineCommit.Branch,
+ RepoId: pipelineCommit.RepoId,
+ RepoUrl: pipelineCommit.RepoUrl,
+ }
+ if pipelineCommit.FinishedDate != nil &&
pipelineCommit.DurationSec != nil {
+ s :=
pipelineCommit.FinishedDate.Add(-time.Duration(*pipelineCommit.DurationSec) *
time.Second)
+ domainDeployCommit.StartedDate = &s
+ }
+ // it is tricky when Environment was declared on the
cicd_tasks level
+ // lets talk common sense and assume that one pipeline
can only be deployed to one environment
+ // so if the pipeline has both staging and production
tasks, we will treat it as a production pipeline
+ // and if it has staging tasks without production
tasks, we will treat it as a staging pipeline
+ // and then a testing pipeline
+ // lastly, we will leave Environment empty if any of
the above measures didn't work out
+
+ // However, there is another catch, what if one
deployed multiple TESTING(STAGING or PRODUCTION)
+ // environments? e.g. testing1, testing2, etc., Does it
matter?
+ if pipelineCommit.Environment == "" {
+ if pipelineCommit.HasProductionTasks {
+ domainDeployCommit.Environment =
devops.PRODUCTION
+ } else if pipelineCommit.HasStagingTasks {
+ domainDeployCommit.Environment =
devops.STAGING
+ } else if pipelineCommit.HasTestingTasks {
+ domainDeployCommit.Environment =
devops.TESTING
+ }
+ }
+ return []interface{}{domainDeployCommit}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return enricher.Execute()
+}
