This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 0e3bfb572 fix(dora): add deployment generator (#6698)
0e3bfb572 is described below
commit 0e3bfb5725cc459282936006e45a598a9484b990
Author: Lynwee <[email protected]>
AuthorDate: Thu Dec 28 14:39:25 2023 +0800
fix(dora): add deployment generator (#6698)
* fix(dora): add deployment generator
* fix(test): fix unit test
* fix(deployments): check and update convertor that can generate
cicd_deployments
* fix(dora): make code better
---
backend/plugins/bamboo/e2e/deploy_build_test.go | 2 +-
backend/plugins/bamboo/impl/impl.go | 3 +-
...deploy_build_to_deployment_commit_convertor.go} | 12 +-
.../tasks/deploy_build_to_deployment_convertor.go | 122 ++++++++++++++++++
.../bitbucket/tasks/deployment_convertor.go | 2 +
backend/plugins/dora/impl/impl.go | 2 +
backend/plugins/dora/impl/impl_test.go | 1 +
.../dora/tasks/deployment_commits_generator.go | 5 +-
backend/plugins/dora/tasks/deployment_generator.go | 142 +++++++++++++++++++++
.../plugins/github/tasks/deployment_convertor.go | 2 +
.../plugins/gitlab/tasks/deployment_convertor.go | 3 +-
11 files changed, 285 insertions(+), 11 deletions(-)
diff --git a/backend/plugins/bamboo/e2e/deploy_build_test.go
b/backend/plugins/bamboo/e2e/deploy_build_test.go
index 730e36a1d..ef40e1e90 100644
--- a/backend/plugins/bamboo/e2e/deploy_build_test.go
+++ b/backend/plugins/bamboo/e2e/deploy_build_test.go
@@ -97,7 +97,7 @@ func TestBambooDeployBuildDataFlow(t *testing.T) {
dataflowTester.FlushTabler(&devops.CicdDeploymentCommit{})
dataflowTester.FlushTabler(&devops.CICDDeployment{})
- dataflowTester.Subtask(tasks.ConvertDeployBuildsMeta, taskData)
+
dataflowTester.Subtask(tasks.ConvertDeployBuildsToDeploymentCommitsMeta,
taskData)
dataflowTester.VerifyTableWithOptions(&devops.CicdDeploymentCommit{},
e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/cicd_deployment_commits.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
diff --git a/backend/plugins/bamboo/impl/impl.go
b/backend/plugins/bamboo/impl/impl.go
index 5dc45e160..382f6e4fa 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -116,7 +116,8 @@ func (p Bamboo) SubTaskMetas() []plugin.SubTaskMeta {
tasks.ConvertJobBuildsMeta,
tasks.ConvertPlanBuildsMeta,
tasks.ConvertPlanVcsMeta,
- tasks.ConvertDeployBuildsMeta,
+ tasks.ConvertDeployBuildsToDeploymentCommitsMeta,
+ tasks.ConvertDeployBuildsToDeploymentMeta,
}
}
diff --git a/backend/plugins/bamboo/tasks/deploy_build_convertor.go
b/backend/plugins/bamboo/tasks/deploy_build_to_deployment_commit_convertor.go
similarity index 94%
rename from backend/plugins/bamboo/tasks/deploy_build_convertor.go
rename to
backend/plugins/bamboo/tasks/deploy_build_to_deployment_commit_convertor.go
index a32fb4ba7..c973c5f15 100644
--- a/backend/plugins/bamboo/tasks/deploy_build_convertor.go
+++
b/backend/plugins/bamboo/tasks/deploy_build_to_deployment_commit_convertor.go
@@ -34,11 +34,11 @@ import (
"github.com/apache/incubator-devlake/plugins/bamboo/models"
)
-var ConvertDeployBuildsMeta = plugin.SubTaskMeta{
- Name: "convertDeployBuilds",
- EntryPoint: ConvertDeployBuilds,
+var ConvertDeployBuildsToDeploymentCommitsMeta = plugin.SubTaskMeta{
+ Name: "convertDeployBuildsToDeploymentCommits",
+ EntryPoint: ConvertDeployBuildsToDeploymentCommits,
EnabledByDefault: true,
- Description: "Convert tool layer table bamboo_deploy_builds into
domain layer table deployBuilds",
+ Description: "Convert tool layer table bamboo_deploy_builds into
domain layer table cicd_deployment_commits",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}
@@ -77,7 +77,7 @@ func (deployBuildWithVcsRevision deployBuildWithVcsRevision)
GenerateCICDDeploym
return deployBuildWithVcsRevision.DeploymentVersionName
}
-func ConvertDeployBuilds(taskCtx plugin.SubTaskContext) errors.Error {
+func ConvertDeployBuildsToDeploymentCommits(taskCtx plugin.SubTaskContext)
errors.Error {
db := taskCtx.GetDal()
logger := taskCtx.GetLogger()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_JOB_BUILD_TABLE)
@@ -153,7 +153,7 @@ func ConvertDeployBuilds(taskCtx plugin.SubTaskContext)
errors.Error {
deploymentCommit.RepoUrl = fakeRepoUrl
}
- return []interface{}{deploymentCommit,
deploymentCommit.ToDeployment()}, nil
+ return []interface{}{deploymentCommit}, nil
},
})
diff --git
a/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go
b/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go
new file mode 100644
index 000000000..e87fca9f7
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go
@@ -0,0 +1,122 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "reflect"
+ "time"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bamboo/models"
+)
+
+var ConvertDeployBuildsToDeploymentMeta = plugin.SubTaskMeta{
+ Name: "convertDeployBuildsToDeployments",
+ EntryPoint: ConvertDeployBuildsToDeployments,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table bamboo_deploy_builds into
domain layer table cicd_deployments",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
+
+type bambooDeployBuildEx struct {
+ models.BambooDeployBuild
+ ProjectPlanName string
+ ProjectName string
+}
+
+func ConvertDeployBuildsToDeployments(taskCtx plugin.SubTaskContext)
errors.Error {
+ db := taskCtx.GetDal()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_JOB_BUILD_TABLE)
+ cursor, err := db.Cursor(
+ dal.Select("db.*, p.name as project_plan_name, p.project_name"),
+ dal.From("_tool_bamboo_deploy_builds AS db"),
+ dal.Join("LEFT JOIN _tool_bamboo_plans as p ON db.plan_key =
p.plan_key"),
+ dal.Where("db.connection_id = ? and db.plan_key = ?",
data.Options.ConnectionId, data.Options.PlanKey),
+ )
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ planIdGen := didgen.NewDomainIdGenerator(&models.BambooPlan{})
+ converter, err := api.NewDataConverter(api.DataConverterArgs{
+ InputRowType: reflect.TypeOf(bambooDeployBuildEx{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
+ input := inputRow.(*bambooDeployBuildEx)
+ deploymentCommitId :=
didgen.NewDomainIdGenerator(&bambooDeployBuildEx{}).Generate(data.Options.ConnectionId,
input.DeployBuildId)
+ createdDate := time.Now()
+ if input.StartedDate != nil {
+ createdDate = *input.StartedDate
+ }
+ name := input.DeploymentVersionName
+ if input.ProjectPlanName != "" {
+ name = fmt.Sprintf("%s/%s",
input.ProjectPlanName, input.DeploymentVersionName)
+ }
+
+ deployment := &devops.CICDDeployment{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: deploymentCommitId,
+ },
+ CicdScopeId:
planIdGen.Generate(data.Options.ConnectionId, data.Options.PlanKey),
+ Name: name,
+ Result: devops.GetResult(&devops.ResultRule{
+ Success: []string{ResultSuccess,
ResultSuccessful},
+ Failure: []string{ResultFailed},
+ Default: devops.RESULT_DEFAULT,
+ }, input.DeploymentState),
+ OriginalResult: input.DeploymentState,
+ Status: devops.GetStatus(&devops.StatusRule{
+ Done: []string{StatusFinished},
+ InProgress: []string{StatusInProgress,
StatusPending, StatusQueued},
+ Default: devops.STATUS_OTHER,
+ }, input.LifeCycleState),
+ OriginalStatus: input.LifeCycleState,
+ Environment: input.Environment,
+ TaskDatesInfo: devops.TaskDatesInfo{
+ CreatedDate: createdDate,
+ QueuedDate: input.QueuedDate,
+ StartedDate: input.ExecutedDate,
+ FinishedDate: input.FinishedDate,
+ },
+ }
+ if
data.RegexEnricher.ReturnNameIfMatched(devops.ENV_NAME_PATTERN,
input.Environment) != "" {
+ deployment.Environment = devops.PRODUCTION
+ }
+ if input.FinishedDate != nil && input.ExecutedDate !=
nil {
+ duration :=
float64(input.FinishedDate.Sub(*input.ExecutedDate).Milliseconds() / 1e3)
+ deployment.DurationSec = &duration
+ }
+ return []interface{}{deployment}, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/backend/plugins/bitbucket/tasks/deployment_convertor.go
b/backend/plugins/bitbucket/tasks/deployment_convertor.go
index 3f2732fa7..1698016c0 100644
--- a/backend/plugins/bitbucket/tasks/deployment_convertor.go
+++ b/backend/plugins/bitbucket/tasks/deployment_convertor.go
@@ -44,6 +44,8 @@ type bitbucketDeploymentWithRefName struct {
RefName string
}
+// ConvertDeployments should be split into two task theoretically
+// But in BitBucket, all deployments have commits, and we use "LEFT JOIN" to
get "ref_name" only, so there is no need to change it.
func ConvertDeployments(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_PIPELINE_TABLE)
db := taskCtx.GetDal()
diff --git a/backend/plugins/dora/impl/impl.go
b/backend/plugins/dora/impl/impl.go
index c0d798143..b20ce189d 100644
--- a/backend/plugins/dora/impl/impl.go
+++ b/backend/plugins/dora/impl/impl.go
@@ -89,6 +89,7 @@ func (p Dora) Settings() interface{} {
func (p Dora) SubTaskMetas() []plugin.SubTaskMeta {
return []plugin.SubTaskMeta{
+ tasks.DeploymentGeneratorMeta,
tasks.DeploymentCommitsGeneratorMeta,
tasks.EnrichPrevSuccessDeploymentCommitMeta,
tasks.EnrichTaskEnvMeta,
@@ -130,6 +131,7 @@ func (p Dora) MakeMetricPluginPipelinePlanV200(projectName
string, options json.
"projectName": projectName,
},
Subtasks: []string{
+ "generateDeployments",
"generateDeploymentCommits",
"enrichPrevSuccessDeploymentCommits",
},
diff --git a/backend/plugins/dora/impl/impl_test.go
b/backend/plugins/dora/impl/impl_test.go
index 587cc517e..461897dc2 100644
--- a/backend/plugins/dora/impl/impl_test.go
+++ b/backend/plugins/dora/impl/impl_test.go
@@ -41,6 +41,7 @@ func TestMakeMetricPluginPipelinePlanV200(t *testing.T) {
{
Plugin: "dora",
Subtasks: []string{
+ "generateDeployments",
"generateDeploymentCommits",
"enrichPrevSuccessDeploymentCommits",
},
diff --git a/backend/plugins/dora/tasks/deployment_commits_generator.go
b/backend/plugins/dora/tasks/deployment_commits_generator.go
index 050f81999..d616396ec 100644
--- a/backend/plugins/dora/tasks/deployment_commits_generator.go
+++ b/backend/plugins/dora/tasks/deployment_commits_generator.go
@@ -64,7 +64,8 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext)
errors.Error {
cursor, err := db.Cursor(
dal.Select(
`
- pc.*, p.name as pipeline_name,
+ pc.*,
+ p.name as pipeline_name,
p.result,
p.status,
p.duration_sec,
@@ -165,7 +166,7 @@ func GenerateDeploymentCommits(taskCtx
plugin.SubTaskContext) errors.Error {
domainDeployCommit.Environment =
devops.TESTING
}
}
- return []interface{}{domainDeployCommit,
domainDeployCommit.ToDeployment()}, nil
+ return []interface{}{domainDeployCommit}, nil
},
})
if err != nil {
diff --git a/backend/plugins/dora/tasks/deployment_generator.go
b/backend/plugins/dora/tasks/deployment_generator.go
new file mode 100644
index 000000000..b52c99191
--- /dev/null
+++ b/backend/plugins/dora/tasks/deployment_generator.go
@@ -0,0 +1,142 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+ "time"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+var DeploymentGeneratorMeta = plugin.SubTaskMeta{
+ Name: "generateDeployments",
+ EntryPoint: GenerateDeployment,
+ EnabledByDefault: true,
+ Description: "Generate cicd_deployments from cicd_pipelines if
cicd_pipeline.type == DEPLOYMENT or any of its cicd_tasks is a deployment task",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
+
+type pipelineEx struct {
+ devops.CICDPipeline
+ HasTestingTasks bool
+ HasStagingTasks bool
+ HasProductionTasks bool
+}
+
+func GenerateDeployment(taskCtx plugin.SubTaskContext) errors.Error {
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*DoraTaskData)
+ // Note that failed records shall be included as well
+ noneSkippedResult := []string{devops.RESULT_FAILURE,
devops.RESULT_SUCCESS}
+ cursor, err := db.Cursor(
+ dal.Select(
+ `
+ p.*,
+ EXISTS(SELECT 1 FROM cicd_tasks t WHERE
t.pipeline_id = p.id AND t.environment = ? AND t.result IN ?)
+ as has_testing_tasks,
+ EXISTS(SELECT 1 FROM cicd_tasks t WHERE
t.pipeline_id = p.id AND t.environment = ? AND t.result IN ?)
+ as has_staging_tasks,
+ EXISTS( SELECT 1 FROM cicd_tasks t WHERE
t.pipeline_id = p.id AND t.environment = ? AND t.result IN ?)
+ as has_production_tasks
+ `,
+ devops.TESTING, noneSkippedResult,
+ devops.STAGING, noneSkippedResult,
+ devops.PRODUCTION, noneSkippedResult,
+ ),
+ dal.From("cicd_pipelines p"),
+ dal.Join("LEFT JOIN project_mapping pm ON (pm.table =
'cicd_scopes' AND pm.row_id = p.cicd_scope_id)"),
+ dal.Where(
+ `
+ pm.project_name = ? AND (
+ p.type = ? OR EXISTS(
+ SELECT 1 FROM cicd_tasks t WHERE
t.pipeline_id = p.id AND t.type = ? AND t.result IN ?
+ )
+ ) AND p.result IN ?
+ `,
+ data.Options.ProjectName,
+ devops.DEPLOYMENT,
+ devops.DEPLOYMENT,
+ noneSkippedResult,
+ noneSkippedResult,
+ ),
+ )
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ enricher, err := api.NewDataConverter(api.DataConverterArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: DoraApiParams{
+ ProjectName: data.Options.ProjectName,
+ },
+ Table: devops.CICDPipeline{}.TableName(),
+ },
+ InputRowType: reflect.TypeOf(pipelineEx{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
+ pipelineExInfo := inputRow.(*pipelineEx)
+ domainDeployment := &devops.CICDDeployment{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: pipelineExInfo.Id,
+ },
+ CicdScopeId: pipelineExInfo.CicdScopeId,
+ Name: pipelineExInfo.Name,
+ Result: pipelineExInfo.Result,
+ Status: pipelineExInfo.Status,
+ OriginalStatus: pipelineExInfo.OriginalStatus,
+ OriginalResult: pipelineExInfo.OriginalResult,
+ Environment: pipelineExInfo.Environment,
+ TaskDatesInfo: devops.TaskDatesInfo{
+ CreatedDate:
pipelineExInfo.CreatedDate,
+ QueuedDate: pipelineExInfo.QueuedDate,
+ StartedDate:
pipelineExInfo.StartedDate,
+ FinishedDate:
pipelineExInfo.FinishedDate,
+ },
+ DurationSec: &pipelineExInfo.DurationSec,
+ QueuedDurationSec:
pipelineExInfo.QueuedDurationSec,
+ }
+ if pipelineExInfo.FinishedDate != nil &&
pipelineExInfo.DurationSec != 0 {
+ s :=
pipelineExInfo.FinishedDate.Add(-time.Duration(pipelineExInfo.DurationSec) *
time.Second)
+ domainDeployment.StartedDate = &s
+ }
+ if pipelineExInfo.Environment == "" {
+ if pipelineExInfo.HasProductionTasks {
+ domainDeployment.Environment =
devops.PRODUCTION
+ } else if pipelineExInfo.HasStagingTasks {
+ domainDeployment.Environment =
devops.STAGING
+ } else if pipelineExInfo.HasTestingTasks {
+ domainDeployment.Environment =
devops.TESTING
+ }
+ }
+ return []interface{}{domainDeployment}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return enricher.Execute()
+}
diff --git a/backend/plugins/github/tasks/deployment_convertor.go
b/backend/plugins/github/tasks/deployment_convertor.go
index de6fada62..8cfb96e2a 100644
--- a/backend/plugins/github/tasks/deployment_convertor.go
+++ b/backend/plugins/github/tasks/deployment_convertor.go
@@ -48,6 +48,8 @@ var ConvertDeploymentsMeta = plugin.SubTaskMeta{
ProductTables: []string{devops.CicdDeploymentCommit{}.TableName(),
devops.CICDDeployment{}.TableName()},
}
+// ConvertDeployment should be split into two task theoretically
+// But in GitHub, all deployments have commits, so there is no need to change
it.
func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_DEPLOYMENT_TABLE)
diff --git a/backend/plugins/gitlab/tasks/deployment_convertor.go
b/backend/plugins/gitlab/tasks/deployment_convertor.go
index fb2ecaf28..7b0581040 100644
--- a/backend/plugins/gitlab/tasks/deployment_convertor.go
+++ b/backend/plugins/gitlab/tasks/deployment_convertor.go
@@ -46,6 +46,8 @@ var ConvertDeploymentMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&ExtractDeploymentMeta},
}
+// ConvertDeployment should be split into two task theoretically
+// But in GitLab, all deployments have commits, so there is no need to change
it.
func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_DEPLOYMENT)
db := taskCtx.GetDal()
@@ -68,7 +70,6 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext)
errors.Error {
defer cursor.Close()
idGen := didgen.NewDomainIdGenerator(&models.GitlabDeployment{})
- //pipelineIdGen :=
didgen.NewDomainIdGenerator(&models.BitbucketPipeline{})
converter, err := api.NewDataConverter(api.DataConverterArgs{
InputRowType: reflect.TypeOf(models.GitlabDeployment{}),
