This is an automated email from the ASF dual-hosted git repository.
mintsweet pushed a commit to branch feat-dora-config
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 133517f1c1989bd45a19fc901aa67ae69b3506ce
Author: abeizn <zikuan...@merico.dev>
AuthorDate: Fri Sep 13 14:43:07 2024 +0800
feat: gitlab dora config + bamboo fix (#8047)
---
backend/plugins/bamboo/impl/impl.go | 11 +-
.../20240906_add_env_name_list_to_scope_config.go | 6 +-
.../bamboo/models/migrationscripts/register.go | 1 +
backend/plugins/bamboo/models/scope_config.go | 1 +
.../tasks/deploy_build_to_deployment_convertor.go | 2 +-
.../20240906_add_env_name_list_to_scope_config.go | 2 +-
backend/plugins/gitlab/api/connection_api.go | 122 +++++++++++++++++++++
backend/plugins/gitlab/impl/impl.go | 17 ++-
.../20240906_add_env_name_list_to_scope_config.go | 6 +-
.../gitlab/models/migrationscripts/register.go | 1 +
backend/plugins/gitlab/models/scope_config.go | 1 +
.../plugins/gitlab/tasks/deployment_convertor.go | 2 +-
12 files changed, 159 insertions(+), 13 deletions(-)
diff --git a/backend/plugins/bamboo/impl/impl.go
b/backend/plugins/bamboo/impl/impl.go
index 419503646..15a140165 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -189,8 +189,15 @@ func (p Bamboo) PrepareTaskData(taskCtx
plugin.TaskContext, options map[string]i
if err := regexEnricher.TryAdd(devops.PRODUCTION,
op.ProductionPattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for
`productionPattern`")
}
- if err := regexEnricher.TryAdd(devops.ENV_NAME_PATTERN,
op.EnvNamePattern); err != nil {
- return nil, errors.BadInput.Wrap(err, "invalid value for
`envNamePattern`")
+ if len(op.BambooScopeConfig.EnvNameList) > 0 ||
(len(op.BambooScopeConfig.EnvNameList) == 0 &&
op.BambooScopeConfig.EnvNamePattern == "") {
+ if err = regexEnricher.TryAddList(devops.ENV_NAME_PATTERN,
op.BambooScopeConfig.EnvNameList...); err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value
for `envNameList`")
+ }
+ } else {
+ if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN,
op.BambooScopeConfig.EnvNamePattern); err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value
for `envNamePattern`")
+ }
+
}
return &tasks.BambooOptions{
Options: op,
diff --git
a/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
b/backend/plugins/bamboo/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
similarity index 93%
copy from
backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
copy to
backend/plugins/bamboo/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
index fe08552ad..e0a439e06 100644
---
a/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
+++
b/backend/plugins/bamboo/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
@@ -30,7 +30,7 @@ type scopeConfig20240906 struct {
}
func (scopeConfig20240906) TableName() string {
- return "_tool_github_scope_configs"
+ return "_tool_bamboo_scope_configs"
}
type addEnvNameListToScopeConfig struct{}
@@ -44,9 +44,9 @@ func (*addEnvNameListToScopeConfig) Up(basicRes
context.BasicRes) errors.Error {
}
func (*addEnvNameListToScopeConfig) Version() uint64 {
- return 20240906142100
+ return 20240906142102
}
func (*addEnvNameListToScopeConfig) Name() string {
- return "add is_draft to _tool_github_pull_requests"
+ return "add env_name_list to _tool_bamboo_scope_configs"
}
diff --git a/backend/plugins/bamboo/models/migrationscripts/register.go
b/backend/plugins/bamboo/models/migrationscripts/register.go
index d7ef35c7e..435368026 100644
--- a/backend/plugins/bamboo/models/migrationscripts/register.go
+++ b/backend/plugins/bamboo/models/migrationscripts/register.go
@@ -38,5 +38,6 @@ func All() []plugin.MigrationScript {
new(addMissingPrimaryKeyForBambooPlanBuildVcsRevision),
new(addQueuedFieldsInJobBuild20231128),
new(addLinkHrefToBambooPlanBuild),
+ new(addEnvNameListToScopeConfig),
}
}
diff --git a/backend/plugins/bamboo/models/scope_config.go
b/backend/plugins/bamboo/models/scope_config.go
index 587ba991e..1ec217baa 100644
--- a/backend/plugins/bamboo/models/scope_config.go
+++ b/backend/plugins/bamboo/models/scope_config.go
@@ -27,6 +27,7 @@ type BambooScopeConfig struct {
DeploymentPattern string
`mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern"
gorm:"type:varchar(255)"`
ProductionPattern string
`mapstructure:"productionPattern,omitempty" json:"productionPattern"
gorm:"type:varchar(255)"`
EnvNamePattern string
`mapstructure:"envNamePattern,omitempty" json:"envNamePattern"
gorm:"type:varchar(255)"`
+ EnvNameList []string `gorm:"type:json;serializer:json"
json:"envNameList" mapstructure:"envNameList"`
}
func (BambooScopeConfig) TableName() string {
diff --git
a/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go
b/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go
index badf42a53..ffa6568b2 100644
--- a/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go
+++ b/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go
@@ -105,7 +105,7 @@ func ConvertDeployBuildsToDeployments(taskCtx
plugin.SubTaskContext) errors.Erro
},
DisplayTitle: name,
}
- if
data.RegexEnricher.ReturnNameIfMatched(devops.ENV_NAME_PATTERN,
input.Environment) != "" {
+ if
data.RegexEnricher.ReturnNameIfMatchedList(devops.ENV_NAME_PATTERN,
input.Environment) != "" {
deployment.Environment = devops.PRODUCTION
}
if input.FinishedDate != nil && input.ExecutedDate !=
nil {
diff --git
a/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
b/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
index fe08552ad..20226b10e 100644
---
a/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
+++
b/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
@@ -48,5 +48,5 @@ func (*addEnvNameListToScopeConfig) Version() uint64 {
}
func (*addEnvNameListToScopeConfig) Name() string {
- return "add is_draft to _tool_github_pull_requests"
+ return "add env_name_list to _tool_github_scope_configs"
}
diff --git a/backend/plugins/gitlab/api/connection_api.go
b/backend/plugins/gitlab/api/connection_api.go
index 598d5b975..1a485d589 100644
--- a/backend/plugins/gitlab/api/connection_api.go
+++ b/backend/plugins/gitlab/api/connection_api.go
@@ -23,6 +23,7 @@ import (
"net/http"
"net/url"
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -174,3 +175,124 @@ func ListConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
return dsHelper.ConnApi.GetDetail(input)
}
+
+// GetConnectionDeployments return one connection deployments
+// @Summary return one connection deployments
+// @Description return one connection deployments
+// @Tags plugins/gitlab
+// @Param id path int true "id"
+// @Param connectionId path int true "connectionId"
+// @Success 200 {array} string "List of Environment Names"
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/connections/{connectionId}/deployments [GET]
+func GetConnectionDeployments(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ db := basicRes.GetDal()
+ connectionId := input.Params["connectionId"]
+ var environments []string
+ err := db.All(&environments,
+ dal.From(&models.GitlabDeployment{}),
+ dal.Where("connection_id = ?", connectionId),
+ dal.Select("DISTINCT environment"))
+ if err != nil {
+ return nil, err
+ }
+
+ return &plugin.ApiResourceOutput{
+ Body: environments,
+ }, nil
+}
+
+// GetConnectionTransformToDeployments return one connection deployments
+// @Summary return one connection deployments
+// @Description return one connection deployments
+// @Tags plugins/gitlab
+// @Param id path int true "id"
+// @Param connectionId path int true "connectionId"
+// @Success 200 {object} map[string]interface{}
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/connections/{connectionId}/transform-to-deployments
[POST]
+func GetConnectionTransformToDeployments(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ db := basicRes.GetDal()
+ connectionId := input.Params["connectionId"]
+ deploymentPattern := input.Body["deploymentPattern"]
+ productionPattern := input.Body["productionPattern"]
+ page, err := api.ParsePageParam(input.Body, "page", 1)
+ if err != nil {
+ return nil, errors.Default.New("invalid page value")
+ }
+ pageSize, err := api.ParsePageParam(input.Body, "pageSize", 10)
+ if err != nil {
+ return nil, errors.Default.New("invalid pageSize value")
+ }
+
+ cursor, err := db.RawCursor(`
+ SELECT DISTINCT name, gitlab_id, web_url, started_at
+ FROM (
+ SELECT r.name, p.gitlab_id, p.web_url, p.started_at
+ FROM _tool_gitlab_pipelines p
+ LEFT JOIN _tool_gitlab_projects r on r.gitlab_id =
p.project_id
+ WHERE p.connection_id = ? AND ref REGEXP ?
+ AND ref REGEXP ?
+ UNION
+ SELECT r.name, p.gitlab_id, p.web_url, p.started_at
+ FROM _tool_gitlab_pipelines p
+ LEFT JOIN _tool_gitlab_projects r on r.gitlab_id =
p.project_id
+ LEFT JOIN _tool_gitlab_jobs j on j.pipeline_id =
p.gitlab_id
+ WHERE j.connection_id = ? AND j.name REGEXP ?
+ AND j.name REGEXP ?
+ ) r
+ ORDER BY r.started_at DESC
+ `, connectionId, deploymentPattern, productionPattern, connectionId,
deploymentPattern, productionPattern)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on get")
+ }
+ defer cursor.Close()
+
+ type selectFileds struct {
+ Name string
+ GitlabId int
+ WebUrl string
+ }
+ type transformedFields struct {
+ Name string
+ URL string
+ }
+ var allRuns []transformedFields
+ for cursor.Next() {
+ sf := &selectFileds{}
+ err = db.Fetch(cursor, sf)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on fetch")
+ }
+ // Directly transform and append to allRuns
+ transformed := transformedFields{
+ Name: fmt.Sprintf("%s-#%d", sf.Name, sf.GitlabId),
+ URL: sf.WebUrl,
+ }
+ allRuns = append(allRuns, transformed)
+ }
+ // Calculate total count
+ totalCount := len(allRuns)
+
+ // Paginate in memory
+ start := (page - 1) * pageSize
+ end := start + pageSize
+ if start > totalCount {
+ start = totalCount
+ }
+ if end > totalCount {
+ end = totalCount
+ }
+ pagedRuns := allRuns[start:end]
+
+ // Return result containing paged runs and total count
+ result := map[string]interface{}{
+ "total": totalCount,
+ "data": pagedRuns,
+ }
+ return &plugin.ApiResourceOutput{
+ Body: result,
+ }, nil
+}
diff --git a/backend/plugins/gitlab/impl/impl.go
b/backend/plugins/gitlab/impl/impl.go
index 2d4dbb4fc..8bf6f9235 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -212,8 +212,15 @@ func (p Gitlab) PrepareTaskData(taskCtx
plugin.TaskContext, options map[string]i
if err := regexEnricher.TryAdd(devops.PRODUCTION,
op.ScopeConfig.ProductionPattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for
`productionPattern`")
}
- if err := regexEnricher.TryAdd(devops.ENV_NAME_PATTERN,
op.ScopeConfig.EnvNamePattern); err != nil {
- return nil, errors.BadInput.Wrap(err, "invalid value for
`envNamePattern`")
+ if len(op.ScopeConfig.EnvNameList) > 0 ||
(len(op.ScopeConfig.EnvNameList) == 0 && op.ScopeConfig.EnvNamePattern == "") {
+ if err = regexEnricher.TryAddList(devops.ENV_NAME_PATTERN,
op.ScopeConfig.EnvNameList...); err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value
for `envNameList`")
+ }
+ } else {
+ if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN,
op.ScopeConfig.EnvNamePattern); err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value
for `envNamePattern`")
+ }
+
}
taskData := tasks.GitlabTaskData{
@@ -268,6 +275,12 @@ func (p Gitlab) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"GET": api.GetScopeList,
"PUT": api.PutScopes,
},
+ "connections/:connectionId/deployments": {
+ "GET": api.GetConnectionDeployments,
+ },
+ "connections/:connectionId/transform-to-deployments": {
+ "POST": api.GetConnectionTransformToDeployments,
+ },
"connections/:connectionId/scope-configs": {
"POST": api.CreateScopeConfig,
"GET": api.GetScopeConfigList,
diff --git
a/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
b/backend/plugins/gitlab/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
similarity index 93%
copy from
backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
copy to
backend/plugins/gitlab/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
index fe08552ad..88fa89364 100644
---
a/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
+++
b/backend/plugins/gitlab/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
@@ -30,7 +30,7 @@ type scopeConfig20240906 struct {
}
func (scopeConfig20240906) TableName() string {
- return "_tool_github_scope_configs"
+ return "_tool_gitlab_scope_configs"
}
type addEnvNameListToScopeConfig struct{}
@@ -44,9 +44,9 @@ func (*addEnvNameListToScopeConfig) Up(basicRes
context.BasicRes) errors.Error {
}
func (*addEnvNameListToScopeConfig) Version() uint64 {
- return 20240906142100
+ return 20240906142101
}
func (*addEnvNameListToScopeConfig) Name() string {
- return "add is_draft to _tool_github_pull_requests"
+ return "add env_name_list to _tool_gitlab_scope_configs"
}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go
b/backend/plugins/gitlab/models/migrationscripts/register.go
index 1d89b2505..2fb32db0b 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/register.go
@@ -52,5 +52,6 @@ func All() []plugin.MigrationScript {
new(addGitlabAssigneeAndReviewerPrimaryKey),
new(changeIssueComponentType),
new(addIsChildToPipelines240906),
+ new(addEnvNameListToScopeConfig),
}
}
diff --git a/backend/plugins/gitlab/models/scope_config.go
b/backend/plugins/gitlab/models/scope_config.go
index 78cfd7f2d..63f6ab7d8 100644
--- a/backend/plugins/gitlab/models/scope_config.go
+++ b/backend/plugins/gitlab/models/scope_config.go
@@ -36,6 +36,7 @@ type GitlabScopeConfig struct {
DeploymentPattern string
`mapstructure:"deploymentPattern" json:"deploymentPattern"`
ProductionPattern string
`mapstructure:"productionPattern,omitempty" json:"productionPattern"
gorm:"type:varchar(255)"`
EnvNamePattern string
`mapstructure:"envNamePattern,omitempty" json:"envNamePattern"
gorm:"type:varchar(255)"`
+ EnvNameList []string
`gorm:"type:json;serializer:json" json:"envNameList" mapstructure:"envNameList"`
Refdiff datatypes.JSONMap
`mapstructure:"refdiff,omitempty" json:"refdiff" swaggertype:"object"
format:"json"`
}
diff --git a/backend/plugins/gitlab/tasks/deployment_convertor.go
b/backend/plugins/gitlab/tasks/deployment_convertor.go
index d7ff0d774..06ed5923d 100644
--- a/backend/plugins/gitlab/tasks/deployment_convertor.go
+++ b/backend/plugins/gitlab/tasks/deployment_convertor.go
@@ -129,7 +129,7 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext)
errors.Error {
Url: repo.WebUrl +
"/environments",
}
if data.RegexEnricher != nil {
- if
data.RegexEnricher.ReturnNameIfMatched(devops.ENV_NAME_PATTERN,
gitlabDeployment.Environment) != "" {
+ if
data.RegexEnricher.ReturnNameIfMatchedList(devops.ENV_NAME_PATTERN,
gitlabDeployment.Environment) != "" {
domainDeployCommit.Environment =
devops.PRODUCTION
}
}
