This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v0.21
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.21 by this push:
new 03e4326ec fix: add flag for data convert to dora (#7143)
03e4326ec is described below
commit 03e4326ec8c37acf8563f68b0aa93deb8f2ce028
Author: abeizn <[email protected]>
AuthorDate: Fri Mar 8 15:35:09 2024 +0800
fix: add flag for data convert to dora (#7143)
* fix: add flag for data convert to dora
* fix: some minor optimizations
* fix: add subtask_name filter
---
.../models/domainlayer/devops/cicd_deployment.go | 1 +
.../domainlayer/devops/cicd_deployment_commit.go | 1 +
.../20240308_add_subtaskname_to_deployment.go | 63 ++++++++++++++++++++++
backend/core/models/migrationscripts/register.go | 1 +
.../dora/tasks/change_lead_time_calculator.go | 10 ++--
.../dora/tasks/deployment_commits_generator.go | 45 ++++++++--------
backend/plugins/dora/tasks/deployment_generator.go | 45 ++++++++--------
.../dora/tasks/incident_deploy_connector.go | 10 ++--
8 files changed, 124 insertions(+), 52 deletions(-)
diff --git a/backend/core/models/domainlayer/devops/cicd_deployment.go
b/backend/core/models/domainlayer/devops/cicd_deployment.go
index c395e32c5..b66e84f85 100644
--- a/backend/core/models/domainlayer/devops/cicd_deployment.go
+++ b/backend/core/models/domainlayer/devops/cicd_deployment.go
@@ -33,6 +33,7 @@ type CICDDeployment struct {
TaskDatesInfo
DurationSec *float64
QueuedDurationSec *float64
+ SubtaskName string `gorm:"type:varchar(255)"`
}
func (CICDDeployment) TableName() string {
diff --git a/backend/core/models/domainlayer/devops/cicd_deployment_commit.go
b/backend/core/models/domainlayer/devops/cicd_deployment_commit.go
index 4170268a9..2afbb8669 100644
--- a/backend/core/models/domainlayer/devops/cicd_deployment_commit.go
+++ b/backend/core/models/domainlayer/devops/cicd_deployment_commit.go
@@ -40,6 +40,7 @@ type CicdDeploymentCommit struct {
RepoId string `gorm:"type:varchar(255)"`
RepoUrl string `gorm:"index;not null"`
PrevSuccessDeploymentCommitId string `gorm:"type:varchar(255)"`
+ SubtaskName string `gorm:"type:varchar(255)"`
}
func (cicdDeploymentCommit CicdDeploymentCommit) TableName() string {
diff --git
a/backend/core/models/migrationscripts/20240308_add_subtaskname_to_deployment.go
b/backend/core/models/migrationscripts/20240308_add_subtaskname_to_deployment.go
new file mode 100644
index 000000000..7bac0c34c
--- /dev/null
+++
b/backend/core/models/migrationscripts/20240308_add_subtaskname_to_deployment.go
@@ -0,0 +1,63 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addSubtabknameToDeployment)(nil)
+
+type cicdDeployment20240308 struct {
+ SubtaskName string `gorm:"type:varchar(255)"`
+}
+
+func (cicdDeployment20240308) TableName() string {
+ return "cicd_deployments"
+}
+
+type cicdDeploymentCommits20240308 struct {
+ SubtaskName string `gorm:"type:varchar(255)"`
+}
+
+func (cicdDeploymentCommits20240308) TableName() string {
+ return "cicd_deployment_commits"
+}
+
+type addSubtabknameToDeployment struct{}
+
+func (*addSubtabknameToDeployment) Up(basicRes context.BasicRes) errors.Error {
+ db := basicRes.GetDal()
+ if err := db.AutoMigrate(&cicdDeployment20240308{}); err != nil {
+ return err
+ }
+ if err := db.AutoMigrate(&cicdDeploymentCommits20240308{}); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (*addSubtabknameToDeployment) Version() uint64 {
+ return 20240308142101
+}
+
+func (*addSubtabknameToDeployment) Name() string {
+ return "add subtaskname to deployment"
+}
diff --git a/backend/core/models/migrationscripts/register.go
b/backend/core/models/migrationscripts/register.go
index 667f864b0..7732c012c 100644
--- a/backend/core/models/migrationscripts/register.go
+++ b/backend/core/models/migrationscripts/register.go
@@ -106,5 +106,6 @@ func All() []plugin.MigrationScript {
new(modfiyFieldsSort),
new(modifyIssueLeadTimeMinutesToUint),
new(addUrgencyToIssues),
+ new(addSubtabknameToDeployment),
}
}
diff --git a/backend/plugins/dora/tasks/change_lead_time_calculator.go
b/backend/plugins/dora/tasks/change_lead_time_calculator.go
index c76b3d54b..8e29f1678 100644
--- a/backend/plugins/dora/tasks/change_lead_time_calculator.go
+++ b/backend/plugins/dora/tasks/change_lead_time_calculator.go
@@ -46,11 +46,11 @@ func CalculateChangeLeadTime(taskCtx plugin.SubTaskContext)
errors.Error {
db := taskCtx.GetDal()
logger := taskCtx.GetLogger()
data := taskCtx.GetData().(*DoraTaskData)
- // // Clear previous results from the project
- // err := db.Exec("DELETE FROM project_pr_metrics WHERE project_name =
? ", data.Options.ProjectName)
- // if err != nil {
- // return errors.Default.Wrap(err, "error deleting previous
project_pr_metrics")
- // }
+ // Clear previous results from the project
+ err := db.Exec("DELETE FROM project_pr_metrics WHERE project_name = ?
", data.Options.ProjectName)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting previous
project_pr_metrics")
+ }
// Get pull requests by repo project_name
var clauses = []dal.Clause{
diff --git a/backend/plugins/dora/tasks/deployment_commits_generator.go
b/backend/plugins/dora/tasks/deployment_commits_generator.go
index e8a64e239..e29846997 100644
--- a/backend/plugins/dora/tasks/deployment_commits_generator.go
+++ b/backend/plugins/dora/tasks/deployment_commits_generator.go
@@ -30,6 +30,8 @@ import (
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+const DORAGenerateDeploymentCommits = "dora.generateDeploymentCommits"
+
var DeploymentCommitsGeneratorMeta = plugin.SubTaskMeta{
Name: "generateDeploymentCommits",
EntryPoint: GenerateDeploymentCommits,
@@ -103,32 +105,32 @@ func GenerateDeploymentCommits(taskCtx
plugin.SubTaskContext) errors.Error {
}
if data.Options.ScopeId != nil {
clauses = append(clauses, dal.Where(`p.cicd_scope_id = ?`,
data.Options.ScopeId))
- // // Clear previous results from the project
- // deleteSql := `DELETE FROM cicd_deployment_commits WHERE
cicd_scope_id = ? ;`
- // err := db.Exec(deleteSql, data.Options.ScopeId)
- // if err != nil {
- // return errors.Default.Wrap(err, "error deleting
previous cicd_deployment_commits")
- // }
+ // Clear previous results from the project
+ deleteSql := `DELETE FROM cicd_deployment_commits WHERE
cicd_scope_id = ? and subtask_name = ?;`
+ err := db.Exec(deleteSql, data.Options.ScopeId,
DORAGenerateDeploymentCommits)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting
previous cicd_deployment_commits")
+ }
} else {
clauses = append(clauses,
dal.Join("LEFT JOIN project_mapping pm ON (pm.table =
'cicd_scopes' AND pm.row_id = p.cicd_scope_id)"),
dal.Where(`pm.project_name = ?`,
data.Options.ProjectName),
)
- // // Clear previous results from the project
- // deleteSql := `DELETE FROM cicd_deployment_commits
- // WHERE cicd_scope_id IN (
- // SELECT cicd_scope_id
- // FROM (
- // SELECT cdc.cicd_scope_id
- // FROM cicd_deployment_commits cdc
- // LEFT JOIN project_mapping pm ON (pm.table =
'cicd_scopes' AND pm.row_id = cdc.cicd_scope_id)
- // WHERE pm.project_name = ?
- // ) AS subquery
- // );`
- // err := db.Exec(deleteSql, data.Options.ProjectName)
- // if err != nil {
- // return errors.Default.Wrap(err, "error deleting
previous cicd_deployment_commits")
- // }
+ // Clear previous results from the project
+ deleteSql := `DELETE FROM cicd_deployment_commits
+ WHERE cicd_scope_id IN (
+ SELECT cicd_scope_id
+ FROM (
+ SELECT cdc.cicd_scope_id
+ FROM cicd_deployment_commits cdc
+ LEFT JOIN project_mapping pm ON (pm.table =
'cicd_scopes' AND pm.row_id = cdc.cicd_scope_id)
+ WHERE pm.project_name = ?
+ ) AS subquery
+ ) AND subtask_name = ?;`
+ err := db.Exec(deleteSql, data.Options.ProjectName,
DORAGenerateDeploymentCommits)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting
previous cicd_deployment_commits")
+ }
}
cursor, err := db.Cursor(clauses...)
if err != nil {
@@ -198,6 +200,7 @@ func GenerateDeploymentCommits(taskCtx
plugin.SubTaskContext) errors.Error {
domainDeployCommit.Environment =
devops.TESTING
}
}
+ domainDeployCommit.SubtaskName =
DORAGenerateDeploymentCommits
return []interface{}{domainDeployCommit}, nil
},
})
diff --git a/backend/plugins/dora/tasks/deployment_generator.go
b/backend/plugins/dora/tasks/deployment_generator.go
index cca452d6b..08102fb42 100644
--- a/backend/plugins/dora/tasks/deployment_generator.go
+++ b/backend/plugins/dora/tasks/deployment_generator.go
@@ -28,6 +28,8 @@ import (
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+const DORAGenerateDeployment = "dora.generateDeployments"
+
var DeploymentGeneratorMeta = plugin.SubTaskMeta{
Name: "generateDeployments",
EntryPoint: GenerateDeployment,
@@ -78,32 +80,32 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext)
errors.Error {
clauses = append(clauses,
dal.Where("p.cicd_scope_id = ?", data.Options.ScopeId),
)
- // // Clear previous results from the cicd_scope_id
- // deleteSql := `DELETE FROM cicd_deployments WHERE
cicd_scope_id = ?;`
- // err := db.Exec(deleteSql, data.Options.ScopeId)
- // if err != nil {
- // return errors.Default.Wrap(err, "error deleting
previous deployments")
- // }
+ // Clear previous results from the cicd_scope_id
+ deleteSql := `DELETE FROM cicd_deployments WHERE cicd_scope_id
= ? and subtask_name = ?;`
+ err := db.Exec(deleteSql, data.Options.ScopeId,
DORAGenerateDeployment)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting
previous deployments")
+ }
} else {
clauses = append(clauses,
dal.Join("LEFT JOIN project_mapping pm ON (pm.table =
'cicd_scopes' AND pm.row_id = p.cicd_scope_id)"),
dal.Where("pm.project_name = ?",
data.Options.ProjectName),
)
- // // Clear previous results from the project
- // deleteSql := `DELETE FROM cicd_deployments
- // WHERE cicd_scope_id IN (
- // SELECT cicd_scope_id
- // FROM (
- // SELECT cd.cicd_scope_id
- // FROM cicd_deployments cd
- // LEFT JOIN project_mapping pm ON
(pm.table = 'cicd_scopes' AND pm.row_id = cd.cicd_scope_id)
- // WHERE pm.project_name = ?
- // ) AS subquery
- // );`
- // err := db.Exec(deleteSql, data.Options.ProjectName)
- // if err != nil {
- // return errors.Default.Wrap(err, "error deleting
previous deployments")
- // }
+ // Clear previous results from the project
+ deleteSql := `DELETE FROM cicd_deployments
+ WHERE cicd_scope_id IN (
+ SELECT cicd_scope_id
+ FROM (
+ SELECT cd.cicd_scope_id
+ FROM cicd_deployments cd
+ LEFT JOIN project_mapping pm ON
(pm.table = 'cicd_scopes' AND pm.row_id = cd.cicd_scope_id)
+ WHERE pm.project_name = ?
+ ) AS subquery
+ ) AND subtask_name = ?;`
+ err := db.Exec(deleteSql, data.Options.ProjectName,
DORAGenerateDeployment)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting
previous deployments")
+ }
}
cursor, err := db.Cursor(clauses...)
@@ -153,6 +155,7 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext)
errors.Error {
domainDeployment.Environment =
devops.TESTING
}
}
+ domainDeployment.SubtaskName = DORAGenerateDeployment
return []interface{}{domainDeployment}, nil
},
})
diff --git a/backend/plugins/dora/tasks/incident_deploy_connector.go
b/backend/plugins/dora/tasks/incident_deploy_connector.go
index 2ec9f04cb..6d9a1365d 100644
--- a/backend/plugins/dora/tasks/incident_deploy_connector.go
+++ b/backend/plugins/dora/tasks/incident_deploy_connector.go
@@ -47,11 +47,11 @@ type simpleCicdDeploymentCommit struct {
func ConnectIncidentToDeployment(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*DoraTaskData)
- // // Clear previous results from the project
- // err := db.Exec("DELETE FROM project_issue_metrics WHERE project_name
= ?", data.Options.ProjectName)
- // if err != nil {
- // return errors.Default.Wrap(err, "error deleting previous
project_issue_metrics")
- // }
+ // Clear previous results from the project
+ err := db.Exec("DELETE FROM project_issue_metrics WHERE project_name =
?", data.Options.ProjectName)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting previous
project_issue_metrics")
+ }
// select all issues belongs to the board
clauses := []dal.Clause{
dal.From(`issues i`),
