This is an automated email from the ASF dual-hosted git repository.
mintsweet pushed a commit to branch feat-dora-config
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 39c9f8b10491f32b19b08752f43b3608399cf86a
Author: abeizn <zikuan...@merico.dev>
AuthorDate: Tue Sep 10 16:36:01 2024 +0800
feat: github/graphql dora config and adjust regex function (#8026)
---
.../core/models/domainlayer/devops/cicd_task.go | 10 ++
.../helpers/pluginhelper/api/enrich_with_regex.go | 59 +++++++++-
backend/helpers/pluginhelper/api/pagenation.go | 24 ++++
backend/plugins/github/api/scope_config_api.go | 125 +++++++++++++++++++++
backend/plugins/github/impl/impl.go | 17 ++-
.../20240906_add_env_name_list_to_scope_config.go | 52 +++++++++
.../github/models/migrationscripts/register.go | 1 +
backend/plugins/github/models/scope_config.go | 1 +
.../github_graphql/tasks/deployment_convertor.go | 2 +-
9 files changed, 286 insertions(+), 5 deletions(-)
diff --git a/backend/core/models/domainlayer/devops/cicd_task.go
b/backend/core/models/domainlayer/devops/cicd_task.go
index 5fd00bbe4..098836fda 100644
--- a/backend/core/models/domainlayer/devops/cicd_task.go
+++ b/backend/core/models/domainlayer/devops/cicd_task.go
@@ -36,6 +36,16 @@ const (
const ENV_NAME_PATTERN = "ENV_NAME_PATTERN"
+type TransformDeployment struct {
+ Name string `json:"name"`
+ URL string `json:"url"`
+}
+
+type TransformDeploymentResponse struct {
+ Total int `json:"total"`
+ Data []TransformDeployment `json:"data"`
+}
+
type CICDTask struct {
domainlayer.DomainEntity
Name string `gorm:"type:varchar(255)"`
diff --git a/backend/helpers/pluginhelper/api/enrich_with_regex.go
b/backend/helpers/pluginhelper/api/enrich_with_regex.go
index 5e7bd2a3a..d586b8187 100644
--- a/backend/helpers/pluginhelper/api/enrich_with_regex.go
+++ b/backend/helpers/pluginhelper/api/enrich_with_regex.go
@@ -28,12 +28,13 @@ import (
// TODO: remove Enricher from naming since it is more like a util function
type RegexEnricher struct {
// This field will store compiled regular expression for every pattern
- regexpMap map[string]*regexp.Regexp
+ regexpMap map[string]*regexp.Regexp
+ regexMapList map[string][]*regexp.Regexp
}
// NewRegexEnricher initialize a regexEnricher
func NewRegexEnricher() *RegexEnricher {
- return &RegexEnricher{regexpMap: make(map[string]*regexp.Regexp)}
+ return &RegexEnricher{regexpMap: make(map[string]*regexp.Regexp),
regexMapList: make(map[string][]*regexp.Regexp)}
}
// AddRegexp will add compiled regular expression for pattern to regexpMap
@@ -105,3 +106,57 @@ func (r *RegexEnricher) ReturnNameIfOmittedOrMatched(name
string, targets ...str
}
return r.ReturnNameIfMatched(name, targets...)
}
+
+// TryAdd a named regexp if given pattern is not empty
+func (r *RegexEnricher) TryAddList(name string, patterns ...string)
errors.Error {
+ if _, ok := r.regexMapList[name]; ok {
+ return errors.Default.New(fmt.Sprintf("Regex pattern with name:
%s already exists", name))
+ }
+
+ var regexList []*regexp.Regexp
+ for _, pattern := range patterns {
+ if pattern == "" {
+ continue
+ }
+ regex, err := errors.Convert01(regexp.Compile(pattern))
+ if err != nil {
+ return errors.BadInput.Wrap(err, fmt.Sprintf("Fail to
compile pattern for regex pattern: %s", pattern))
+ }
+ regexList = append(regexList, regex)
+ }
+
+ // Only save non-empty regexList
+ if len(regexList) > 0 {
+ r.regexMapList[name] = regexList
+ }
+ return nil
+}
+
+// ReturnNameIfMatched will return name if any of the targets matches the
regex associated with the given name
+func (r *RegexEnricher) ReturnNameIfMatchedList(name string, targets
...string) string {
+ if regexList, ok := r.regexMapList[name]; !ok {
+ return ""
+ } else {
+ for _, regex := range regexList {
+ matched := false
+ for _, target := range targets {
+ if regex.MatchString(target) {
+ matched = true
+ break
+ }
+ }
+ if !matched {
+ return "" // If any regex fails to match,
return ""
+ }
+ }
+ }
+ return name // Return name if all regex conditions were fulfilled
+}
+
+// ReturnNameIfOmittedOrMatched returns the given name if regex of the given
name is omitted or fallback to ReturnNameIfMatched
+func (r *RegexEnricher) ReturnNameIfOmittedOrMatchedList(name string, targets
...string) string {
+ if _, ok := r.regexMapList[name]; !ok {
+ return name
+ }
+ return r.ReturnNameIfMatched(name, targets...)
+}
diff --git a/backend/helpers/pluginhelper/api/pagenation.go
b/backend/helpers/pluginhelper/api/pagenation.go
index 784ac61e4..6c3ea7e5f 100644
--- a/backend/helpers/pluginhelper/api/pagenation.go
+++ b/backend/helpers/pluginhelper/api/pagenation.go
@@ -20,6 +20,8 @@ package api
import (
"net/url"
"strconv"
+
+ "github.com/go-errors/errors"
)
const pageSize = 50
@@ -47,3 +49,25 @@ func GetLimitOffset(q url.Values, pageSizeKey, pageKey
string) (limit int, offse
offset = (page - 1) * limit
return limit, offset
}
+
+func ParsePageParam(body map[string]interface{}, paramName string,
defaultValue int) (int, error) {
+ value, exists := body[paramName]
+ if !exists {
+ return defaultValue, nil
+ }
+
+ switch v := value.(type) {
+ case int:
+ return v, nil
+ case float64:
+ return int(v), nil
+ case string:
+ parsedValue, err := strconv.Atoi(v)
+ if err != nil {
+ return 0, errors.New("invalid " + paramName + " value")
+ }
+ return parsedValue, nil
+ default:
+ return 0, errors.New(paramName + " must be int or string")
+ }
+}
diff --git a/backend/plugins/github/api/scope_config_api.go
b/backend/plugins/github/api/scope_config_api.go
index b109341e7..6d72f6933 100644
--- a/backend/plugins/github/api/scope_config_api.go
+++ b/backend/plugins/github/api/scope_config_api.go
@@ -18,8 +18,13 @@ limitations under the License.
package api
import (
+ "fmt"
+
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ githubModels "github.com/apache/incubator-devlake/plugins/github/models"
)
// PostScopeConfig create scope config for Github
@@ -109,3 +114,123 @@ func GetProjectsByScopeConfig(input
*plugin.ApiResourceInput) (*plugin.ApiResour
func DeleteScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
return dsHelper.ScopeConfigApi.Delete(input)
}
+
+// GetScopeConfig return one scope config deployments
+// @Summary return one scope config deployments
+// @Description return one scope config deployments
+// @Tags plugins/github
+// @Param id path int true "id"
+// @Param connectionId path int true "connectionId"
+// @Success 200 {object} models.GithubScopeConfigDeployment
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router
/plugins/github/connections/{connectionId}/scope-configs/{id}/deployments [GET]
+func GetScopeConfigDeployments(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ db := basicRes.GetDal()
+ connectionId := input.Params["connectionId"]
+ var environments []string
+ err := db.All(&environments,
+ dal.From(&githubModels.GithubDeployment{}),
+ dal.Where("connection_id = ?", connectionId),
+ dal.Select("DISTINCT environment"))
+ if err != nil {
+ return nil, err
+ }
+
+ return &plugin.ApiResourceOutput{
+ Body: environments,
+ }, nil
+}
+
+// GetScopeConfig return one scope config deployments
+// @Summary return one scope config deployments
+// @Description return one scope config deployments
+// @Tags plugins/github
+// @Param id path int true "id"
+// @Param connectionId path int true "connectionId"
+// @Success 200 {object} models.GithubScopeConfigDeployment
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router
/plugins/github/connections/{connectionId}/scope-configs/{id}/transform-to-deployments
[POST]
+func GetScopeConfigTransformToDeployments(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ db := basicRes.GetDal()
+ connectionId := input.Params["connectionId"]
+ deploymentPattern := input.Body["deploymentPattern"]
+ productionPattern := input.Body["productionPattern"]
+ page, err := api.ParsePageParam(input.Body, "page", 1)
+ if err != nil {
+ return nil, errors.Default.New("invalid page value")
+ }
+ pageSize, err := api.ParsePageParam(input.Body, "pageSize", 10)
+ if err != nil {
+ return nil, errors.Default.New("invalid pageSize value")
+ }
+
+ cursor, err := db.RawCursor(`
+ SELECT DISTINCT r.run_number, r.name, r.head_branch,
r.html_url, r.run_started_at
+ FROM (
+ SELECT id, run_number, name, head_branch, html_url,
run_started_at
+ FROM _tool_github_runs
+ WHERE connection_id = ? AND name REGEXP ?
+ AND (name REGEXP ? OR head_branch REGEXP ?)
+ UNION
+ SELECT r.id, r.run_number, r.name, r.head_branch,
r.html_url, r.run_started_at
+ FROM _tool_github_jobs j
+ LEFT JOIN _tool_github_runs r ON j.run_id = r.id
+ WHERE j.connection_id = ? AND j.name REGEXP ?
+ AND j.name REGEXP ?
+ ) r
+ ORDER BY r.run_started_at DESC
+ `, connectionId, deploymentPattern, productionPattern,
productionPattern, connectionId, deploymentPattern, productionPattern)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on get")
+ }
+ defer cursor.Close()
+
+ type selectFileds struct {
+ RunNumber int
+ Name string
+ HeadBranch string
+ HTMLURL string
+ }
+ type transformedFields struct {
+ Name string
+ URL string
+ }
+ var allRuns []transformedFields
+ for cursor.Next() {
+ sf := &selectFileds{}
+ err = db.Fetch(cursor, sf)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on fetch")
+ }
+ // Directly transform and append to allRuns
+ transformed := transformedFields{
+ Name: fmt.Sprintf("#%d - %s", sf.RunNumber, sf.Name),
+ URL: sf.HTMLURL,
+ }
+ allRuns = append(allRuns, transformed)
+ }
+ // Calculate total count
+ totalCount := len(allRuns)
+
+ // Paginate in memory
+ start := (page - 1) * pageSize
+ end := start + pageSize
+ if start > totalCount {
+ start = totalCount
+ }
+ if end > totalCount {
+ end = totalCount
+ }
+ pagedRuns := allRuns[start:end]
+
+ // Return result containing paged runs and total count
+ result := map[string]interface{}{
+ "total": totalCount,
+ "data": pagedRuns,
+ }
+ return &plugin.ApiResourceOutput{
+ Body: result,
+ }, nil
+}
diff --git a/backend/plugins/github/impl/impl.go
b/backend/plugins/github/impl/impl.go
index 07342b786..ac8f4e829 100644
--- a/backend/plugins/github/impl/impl.go
+++ b/backend/plugins/github/impl/impl.go
@@ -164,8 +164,15 @@ func (p Github) PrepareTaskData(taskCtx
plugin.TaskContext, options map[string]i
if err = regexEnricher.TryAdd(devops.PRODUCTION,
op.ScopeConfig.ProductionPattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for
`productionPattern`")
}
- if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN,
op.ScopeConfig.EnvNamePattern); err != nil {
- return nil, errors.BadInput.Wrap(err, "invalid value for
`envNamePattern`")
+ if len(op.ScopeConfig.EnvNameList) > 0 ||
(len(op.ScopeConfig.EnvNameList) == 0 && op.ScopeConfig.EnvNamePattern == "") {
+ if err = regexEnricher.TryAddList(devops.ENV_NAME_PATTERN,
op.ScopeConfig.EnvNameList...); err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value
for `envNameList`")
+ }
+ } else {
+ if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN,
op.ScopeConfig.EnvNamePattern); err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value
for `envNamePattern`")
+ }
+
}
taskData.RegexEnricher = regexEnricher
@@ -218,6 +225,12 @@ func (p Github) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"GET": api.GetScopeConfig,
"DELETE": api.DeleteScopeConfig,
},
+
"connections/:connectionId/scope-configs/:scopeConfigId/deployments": {
+ "GET": api.GetScopeConfigDeployments,
+ },
+
"connections/:connectionId/scope-configs/:scopeConfigId/transform-to-deployments":
{
+ "POST": api.GetScopeConfigTransformToDeployments,
+ },
"connections/:connectionId/remote-scopes": {
"GET": api.RemoteScopes,
},
diff --git
a/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
b/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
new file mode 100644
index 000000000..fe08552ad
--- /dev/null
+++
b/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go
@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addEnvNameListToScopeConfig)(nil)
+
+type scopeConfig20240906 struct {
+ EnvNameList []string `gorm:"type:json;serializer:json"
json:"env_name_list" mapstructure:"env_name_list"`
+}
+
+func (scopeConfig20240906) TableName() string {
+ return "_tool_github_scope_configs"
+}
+
+type addEnvNameListToScopeConfig struct{}
+
+func (*addEnvNameListToScopeConfig) Up(basicRes context.BasicRes) errors.Error
{
+ db := basicRes.GetDal()
+ if err := db.AutoMigrate(&scopeConfig20240906{}); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (*addEnvNameListToScopeConfig) Version() uint64 {
+ return 20240906142100
+}
+
+func (*addEnvNameListToScopeConfig) Name() string {
+ return "add is_draft to _tool_github_pull_requests"
+}
diff --git a/backend/plugins/github/models/migrationscripts/register.go
b/backend/plugins/github/models/migrationscripts/register.go
index b8a0722eb..b69d56b85 100644
--- a/backend/plugins/github/models/migrationscripts/register.go
+++ b/backend/plugins/github/models/migrationscripts/register.go
@@ -55,5 +55,6 @@ func All() []plugin.MigrationScript {
new(addIsDraftToPr),
new(changeIssueComponentType),
new(addIndexToGithubJobs),
+ new(addEnvNameListToScopeConfig),
}
}
diff --git a/backend/plugins/github/models/scope_config.go
b/backend/plugins/github/models/scope_config.go
index 60b298d36..e880d500e 100644
--- a/backend/plugins/github/models/scope_config.go
+++ b/backend/plugins/github/models/scope_config.go
@@ -39,6 +39,7 @@ type GithubScopeConfig struct {
DeploymentPattern string
`mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern"
gorm:"type:varchar(255)"`
ProductionPattern string
`mapstructure:"productionPattern,omitempty" json:"productionPattern"
gorm:"type:varchar(255)"`
EnvNamePattern string
`mapstructure:"envNamePattern,omitempty" json:"envNamePattern"
gorm:"type:varchar(255)"`
+ EnvNameList []string
`gorm:"type:json;serializer:json" json:"envNameList" mapstructure:"envNameList"`
Refdiff datatypes.JSONMap
`mapstructure:"refdiff,omitempty" json:"refdiff" swaggertype:"object"
format:"json"`
}
diff --git a/backend/plugins/github_graphql/tasks/deployment_convertor.go
b/backend/plugins/github_graphql/tasks/deployment_convertor.go
index fdc439553..c0b6a29aa 100644
--- a/backend/plugins/github_graphql/tasks/deployment_convertor.go
+++ b/backend/plugins/github_graphql/tasks/deployment_convertor.go
@@ -104,7 +104,7 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext)
errors.Error {
deploymentCommit.DurationSec = &durationSec
if data.RegexEnricher != nil {
- if
data.RegexEnricher.ReturnNameIfMatched(devops.ENV_NAME_PATTERN,
githubDeployment.Environment) != "" {
+ if
data.RegexEnricher.ReturnNameIfMatchedList(devops.ENV_NAME_PATTERN,
githubDeployment.Environment) != "" {
deploymentCommit.Environment =
devops.PRODUCTION
}
}
