This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new ecd422896 Refactor Chage Lead Time - Step 2: enrich type/env for
pipelines and tasks (#4886)
ecd422896 is described below
commit ecd422896ee110958be0582398bc81887de09951
Author: Klesh Wong <[email protected]>
AuthorDate: Tue Apr 11 17:45:53 2023 +0800
Refactor Chage Lead Time - Step 2: enrich type/env for pipelines and tasks
(#4886)
* refactor: add TryAdd/ReturnNameIfMatched to simplify the usage of
RegexEnricher
* refactor: enrich type/env for bitbucket pipelines/steps
---
.../helpers/pluginhelper/api/enrich_with_regex.go | 35 +++++++++++++-
backend/plugins/bitbucket/bitbucket.go | 6 +++
backend/plugins/bitbucket/e2e/pipelines_test.go | 7 +++
.../snapshot_tables/_tool_bitbucket_pipelines.csv | 22 ++++-----
.../e2e/snapshot_tables/cicd_pipelines.csv | 22 ++++-----
backend/plugins/bitbucket/impl/impl.go | 13 ++++-
.../20230410_add_type_env_to_pipeline_and_step.go | 56 ++++++++++++++++++++++
.../bitbucket/models/migrationscripts/register.go | 1 +
backend/plugins/bitbucket/models/pipeline.go | 5 +-
backend/plugins/bitbucket/models/pipeline_step.go | 5 +-
.../plugins/bitbucket/tasks/pipeline_convertor.go | 3 +-
.../plugins/bitbucket/tasks/pipeline_extractor.go | 4 ++
.../bitbucket/tasks/pipeline_steps_convertor.go | 16 -------
.../bitbucket/tasks/pipeline_steps_extractor.go | 6 ++-
backend/plugins/bitbucket/tasks/task_data.go | 7 +--
15 files changed, 160 insertions(+), 48 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/enrich_with_regex.go
b/backend/helpers/pluginhelper/api/enrich_with_regex.go
index 81bed857f..2df1dfcec 100644
--- a/backend/helpers/pluginhelper/api/enrich_with_regex.go
+++ b/backend/helpers/pluginhelper/api/enrich_with_regex.go
@@ -19,9 +19,10 @@ package api
import (
"fmt"
+ "regexp"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
- "regexp"
)
// RegexEnricher process value with regex pattern
@@ -36,6 +37,7 @@ func NewRegexEnricher() *RegexEnricher {
}
// AddRegexp will add compiled regular expression for pattern to regexpMap
+// TODO: to be removed
func (r *RegexEnricher) AddRegexp(patterns ...string) errors.Error {
for _, pattern := range patterns {
if len(pattern) > 0 {
@@ -52,6 +54,7 @@ func (r *RegexEnricher) AddRegexp(patterns ...string)
errors.Error {
// GetEnrichResult will get compiled regular expression from map by pattern,
// and check if v matches compiled regular expression,
// lastly, will return corresponding value(result or empty)
+// TODO: to be removed
func (r *RegexEnricher) GetEnrichResult(pattern string, v string, result
string) string {
if result == devops.PRODUCTION && pattern == "" {
return result
@@ -64,3 +67,33 @@ func (r *RegexEnricher) GetEnrichResult(pattern string, v
string, result string)
}
return ""
}
+
+// TryAdd a named regexp if given pattern is not empty
+func (r *RegexEnricher) TryAdd(name, pattern string) errors.Error {
+ if pattern == "" {
+ return nil
+ }
+ if _, ok := r.regexpMap[name]; ok {
+ return errors.Default.New(fmt.Sprintf("Regex pattern with name:
%s already exists", name))
+ }
+ regex, err := errors.Convert01(regexp.Compile(pattern))
+ if err != nil {
+ return errors.BadInput.Wrap(err, fmt.Sprintf("Fail to compile
pattern for regex pattern: %s", pattern))
+ }
+ r.regexpMap[name] = regex
+ return nil
+}
+
+// ReturnNameIfMatched will return name if any of the targets matches the
regex with the given name
+func (r *RegexEnricher) ReturnNameIfMatched(name string, targets ...string)
string {
+ if regex, ok := r.regexpMap[name]; !ok {
+ return ""
+ } else {
+ for _, target := range targets {
+ if regex.MatchString(target) {
+ return name
+ }
+ }
+ }
+ return ""
+}
diff --git a/backend/plugins/bitbucket/bitbucket.go
b/backend/plugins/bitbucket/bitbucket.go
index 8d6ba8b65..809a22f99 100644
--- a/backend/plugins/bitbucket/bitbucket.go
+++ b/backend/plugins/bitbucket/bitbucket.go
@@ -32,6 +32,8 @@ func main() {
connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "bitbucket
connection id")
fullName := cmd.Flags().StringP("fullName", "n", "", "bitbucket id:
owner/repo")
timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data
that are created after specified time, ie 2006-05-06T07:08:09Z")
+ deploymentPattern := cmd.Flags().StringP("deployment", "", "",
"deployment pattern")
+ productionPattern := cmd.Flags().StringP("production", "", "",
"production pattern")
_ = cmd.MarkFlagRequired("connectionId")
_ = cmd.MarkFlagRequired("fullName")
@@ -40,6 +42,10 @@ func main() {
"connectionId": *connectionId,
"fullName": *fullName,
"timeAfter": *timeAfter,
+ "transformationRules": map[string]string{
+ "deploymentPattern": *deploymentPattern,
+ "productionPattern": *productionPattern,
+ },
})
}
runner.RunCmd(cmd)
diff --git a/backend/plugins/bitbucket/e2e/pipelines_test.go
b/backend/plugins/bitbucket/e2e/pipelines_test.go
index 524d82b08..d253e8815 100644
--- a/backend/plugins/bitbucket/e2e/pipelines_test.go
+++ b/backend/plugins/bitbucket/e2e/pipelines_test.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/impl"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
"github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
@@ -33,11 +34,15 @@ func TestBitbucketPipelineDataFlow(t *testing.T) {
var bitbucket impl.Bitbucket
dataflowTester := e2ehelper.NewDataFlowTester(t, "bitbucket", bitbucket)
+ regexEnricher := helper.NewRegexEnricher()
+ _ = regexEnricher.TryAdd(devops.DEPLOYMENT, "main")
+ _ = regexEnricher.TryAdd(devops.PRODUCTION, "pipeline")
taskData := &tasks.BitbucketTaskData{
Options: &tasks.BitbucketOptions{
ConnectionId: 1,
FullName: "likyh/likyhphp",
},
+ RegexEnricher: regexEnricher,
}
// import raw data table
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_bitbucket_api_pipelines.csv",
"_raw_bitbucket_api_pipelines")
@@ -56,6 +61,8 @@ func TestBitbucketPipelineDataFlow(t *testing.T) {
"ref_name",
"web_url",
"duration_in_seconds",
+ "type",
+ "environment",
"_raw_data_params",
"_raw_data_table",
"_raw_data_id",
diff --git
a/backend/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_pipelines.csv
b/backend/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_pipelines.csv
index 84d8128b9..0554c7ace 100644
---
a/backend/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_pipelines.csv
+++
b/backend/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_pipelines.csv
@@ -1,11 +1,11 @@
-connection_id,bitbucket_id,status,result,ref_name,web_url,duration_in_seconds,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,{0af285e5-c07d-48eb-b0e9-b579f63f6f54},IN_PROGRESS,PAUSED,main,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{0af285e5-c07d-48eb-b0e9-b579f63f6f54},10,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,43,
-1,{0b0986ff-87ab-4c61-8244-72ee93270992},IN_PROGRESS,PAUSED,main,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{0b0986ff-87ab-4c61-8244-72ee93270992},10,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,45,
-1,{105b3616-0140-4f17-993e-65d8836cbfd4},IN_PROGRESS,PAUSED,pipeline,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{105b3616-0140-4f17-993e-65d8836cbfd4},9,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,46,
-1,{60bd9ab0-57d7-4da6-bf39-3b04e8133223},COMPLETED,FAILED,feature/pipelinetest,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{60bd9ab0-57d7-4da6-bf39-3b04e8133223},0,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,48,
-1,{76e9c380-bedf-48f8-ad11-9b4a60307dd6},COMPLETED,STOPPED,pipeline,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{76e9c380-bedf-48f8-ad11-9b4a60307dd6},0,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,47,
-1,{844365c2-2d8c-4b67-9e27-21c2fcda7bd7},IN_PROGRESS,PAUSED,main,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{844365c2-2d8c-4b67-9e27-21c2fcda7bd7},10,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,42,
-1,{a57ab3dc-2afd-4e23-acd3-7acf1bb0cf28},COMPLETED,SUCCESSFUL,main,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{a57ab3dc-2afd-4e23-acd3-7acf1bb0cf28},14,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,41,
-1,{accb6177-eea1-4d13-9806-037645ca3f67},COMPLETED,FAILED,,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{accb6177-eea1-4d13-9806-037645ca3f67},0,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,50,
-1,{d676e969-7294-4ca2-9173-4fba9b419fe9},COMPLETED,FAILED,pipeline,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{d676e969-7294-4ca2-9173-4fba9b419fe9},0,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,49,
-1,{fc8cfdbd-2e0f-4789-9abb-19bf326f704b},IN_PROGRESS,PAUSED,feature/pipelinetest,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{fc8cfdbd-2e0f-4789-9abb-19bf326f704b},12,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,44,
+connection_id,bitbucket_id,status,result,ref_name,type,environment,web_url,duration_in_seconds,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,{0af285e5-c07d-48eb-b0e9-b579f63f6f54},IN_PROGRESS,PAUSED,main,DEPLOYMENT,,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{0af285e5-c07d-48eb-b0e9-b579f63f6f54},10,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,43,
+1,{0b0986ff-87ab-4c61-8244-72ee93270992},IN_PROGRESS,PAUSED,main,DEPLOYMENT,,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{0b0986ff-87ab-4c61-8244-72ee93270992},10,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,45,
+1,{105b3616-0140-4f17-993e-65d8836cbfd4},IN_PROGRESS,PAUSED,pipeline,,PRODUCTION,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{105b3616-0140-4f17-993e-65d8836cbfd4},9,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,46,
+1,{60bd9ab0-57d7-4da6-bf39-3b04e8133223},COMPLETED,FAILED,feature/pipelinetest,,PRODUCTION,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{60bd9ab0-57d7-4da6-bf39-3b04e8133223},0,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,48,
+1,{76e9c380-bedf-48f8-ad11-9b4a60307dd6},COMPLETED,STOPPED,pipeline,,PRODUCTION,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{76e9c380-bedf-48f8-ad11-9b4a60307dd6},0,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,47,
+1,{844365c2-2d8c-4b67-9e27-21c2fcda7bd7},IN_PROGRESS,PAUSED,main,DEPLOYMENT,,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{844365c2-2d8c-4b67-9e27-21c2fcda7bd7},10,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,42,
+1,{a57ab3dc-2afd-4e23-acd3-7acf1bb0cf28},COMPLETED,SUCCESSFUL,main,DEPLOYMENT,,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{a57ab3dc-2afd-4e23-acd3-7acf1bb0cf28},14,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,41,
+1,{accb6177-eea1-4d13-9806-037645ca3f67},COMPLETED,FAILED,,,,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{accb6177-eea1-4d13-9806-037645ca3f67},0,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,50,
+1,{d676e969-7294-4ca2-9173-4fba9b419fe9},COMPLETED,FAILED,pipeline,,PRODUCTION,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{d676e969-7294-4ca2-9173-4fba9b419fe9},0,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,49,
+1,{fc8cfdbd-2e0f-4789-9abb-19bf326f704b},IN_PROGRESS,PAUSED,feature/pipelinetest,,PRODUCTION,http://bitbucket-pipelines.prod.public.atl-paas.net//rest/1.0/accounts/{64135f6a-3978-4297-99b1-21827e0faf0b}/repositories/{b304e403-4841-4ebe-9d4e-432025a507d2}/pipelines/{fc8cfdbd-2e0f-4789-9abb-19bf326f704b},12,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipelines,44,
diff --git a/backend/plugins/bitbucket/e2e/snapshot_tables/cicd_pipelines.csv
b/backend/plugins/bitbucket/e2e/snapshot_tables/cicd_pipelines.csv
index c7f646737..7d4f52e4f 100644
--- a/backend/plugins/bitbucket/e2e/snapshot_tables/cicd_pipelines.csv
+++ b/backend/plugins/bitbucket/e2e/snapshot_tables/cicd_pipelines.csv
@@ -1,11 +1,11 @@
-id,name,result,status,type,duration_sec,environment
-bitbucket:BitbucketPipeline:1:{0af285e5-c07d-48eb-b0e9-b579f63f6f54},bitbucket:BitbucketPipeline:1:main,SUCCESS,IN_PROGRESS,CI/CD,10,
-bitbucket:BitbucketPipeline:1:{0b0986ff-87ab-4c61-8244-72ee93270992},bitbucket:BitbucketPipeline:1:main,SUCCESS,IN_PROGRESS,CI/CD,10,
-bitbucket:BitbucketPipeline:1:{105b3616-0140-4f17-993e-65d8836cbfd4},bitbucket:BitbucketPipeline:1:pipeline,SUCCESS,IN_PROGRESS,CI/CD,9,
-bitbucket:BitbucketPipeline:1:{60bd9ab0-57d7-4da6-bf39-3b04e8133223},bitbucket:BitbucketPipeline:1:feature/pipelinetest,FAILURE,DONE,CI/CD,0,
-bitbucket:BitbucketPipeline:1:{76e9c380-bedf-48f8-ad11-9b4a60307dd6},bitbucket:BitbucketPipeline:1:pipeline,ABORT,DONE,CI/CD,0,
-bitbucket:BitbucketPipeline:1:{844365c2-2d8c-4b67-9e27-21c2fcda7bd7},bitbucket:BitbucketPipeline:1:main,SUCCESS,IN_PROGRESS,CI/CD,10,
-bitbucket:BitbucketPipeline:1:{a57ab3dc-2afd-4e23-acd3-7acf1bb0cf28},bitbucket:BitbucketPipeline:1:main,SUCCESS,DONE,CI/CD,14,
-bitbucket:BitbucketPipeline:1:{accb6177-eea1-4d13-9806-037645ca3f67},bitbucket:BitbucketPipeline:1:,FAILURE,DONE,CI/CD,0,
-bitbucket:BitbucketPipeline:1:{d676e969-7294-4ca2-9173-4fba9b419fe9},bitbucket:BitbucketPipeline:1:pipeline,FAILURE,DONE,CI/CD,0,
-bitbucket:BitbucketPipeline:1:{fc8cfdbd-2e0f-4789-9abb-19bf326f704b},bitbucket:BitbucketPipeline:1:feature/pipelinetest,SUCCESS,IN_PROGRESS,CI/CD,12,
+id,name,result,status,type,duration_sec,environment
+bitbucket:BitbucketPipeline:1:{0af285e5-c07d-48eb-b0e9-b579f63f6f54},bitbucket:BitbucketPipeline:1:main,SUCCESS,IN_PROGRESS,DEPLOYMENT,10,
+bitbucket:BitbucketPipeline:1:{0b0986ff-87ab-4c61-8244-72ee93270992},bitbucket:BitbucketPipeline:1:main,SUCCESS,IN_PROGRESS,DEPLOYMENT,10,
+bitbucket:BitbucketPipeline:1:{105b3616-0140-4f17-993e-65d8836cbfd4},bitbucket:BitbucketPipeline:1:pipeline,SUCCESS,IN_PROGRESS,,9,PRODUCTION
+bitbucket:BitbucketPipeline:1:{60bd9ab0-57d7-4da6-bf39-3b04e8133223},bitbucket:BitbucketPipeline:1:feature/pipelinetest,FAILURE,DONE,,0,PRODUCTION
+bitbucket:BitbucketPipeline:1:{76e9c380-bedf-48f8-ad11-9b4a60307dd6},bitbucket:BitbucketPipeline:1:pipeline,ABORT,DONE,,0,PRODUCTION
+bitbucket:BitbucketPipeline:1:{844365c2-2d8c-4b67-9e27-21c2fcda7bd7},bitbucket:BitbucketPipeline:1:main,SUCCESS,IN_PROGRESS,DEPLOYMENT,10,
+bitbucket:BitbucketPipeline:1:{a57ab3dc-2afd-4e23-acd3-7acf1bb0cf28},bitbucket:BitbucketPipeline:1:main,SUCCESS,DONE,DEPLOYMENT,14,
+bitbucket:BitbucketPipeline:1:{accb6177-eea1-4d13-9806-037645ca3f67},bitbucket:BitbucketPipeline:1:,FAILURE,DONE,,0,
+bitbucket:BitbucketPipeline:1:{d676e969-7294-4ca2-9173-4fba9b419fe9},bitbucket:BitbucketPipeline:1:pipeline,FAILURE,DONE,,0,PRODUCTION
+bitbucket:BitbucketPipeline:1:{fc8cfdbd-2e0f-4789-9abb-19bf326f704b},bitbucket:BitbucketPipeline:1:feature/pipelinetest,SUCCESS,IN_PROGRESS,,12,PRODUCTION
diff --git a/backend/plugins/bitbucket/impl/impl.go
b/backend/plugins/bitbucket/impl/impl.go
index 5d79126f3..bf121fad6 100644
--- a/backend/plugins/bitbucket/impl/impl.go
+++ b/backend/plugins/bitbucket/impl/impl.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/api"
@@ -155,9 +156,17 @@ func (p Bitbucket) PrepareTaskData(taskCtx
plugin.TaskContext, options map[strin
return nil, errors.BadInput.Wrap(err, "invalid value
for `timeAfter`")
}
}
+ regexEnricher := helper.NewRegexEnricher()
+ if err := regexEnricher.TryAdd(devops.DEPLOYMENT,
op.DeploymentPattern); err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value for
`deploymentPattern`")
+ }
+ if err := regexEnricher.TryAdd(devops.PRODUCTION,
op.ProductionPattern); err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value for
`productionPattern`")
+ }
taskData := &tasks.BitbucketTaskData{
- Options: op,
- ApiClient: apiClient,
+ Options: op,
+ ApiClient: apiClient,
+ RegexEnricher: regexEnricher,
}
if !timeAfter.IsZero() {
taskData.TimeAfter = &timeAfter
diff --git
a/backend/plugins/bitbucket/models/migrationscripts/20230410_add_type_env_to_pipeline_and_step.go
b/backend/plugins/bitbucket/models/migrationscripts/20230410_add_type_env_to_pipeline_and_step.go
new file mode 100644
index 000000000..af00519d2
--- /dev/null
+++
b/backend/plugins/bitbucket/models/migrationscripts/20230410_add_type_env_to_pipeline_and_step.go
@@ -0,0 +1,56 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type addTypeEnvToPipelineAndStep struct{}
+
+type pipeline20230410 struct {
+ Type string `gorm:"type:varchar(255)"`
+ Environment string `gorm:"type:varchar(255)"`
+}
+
+func (pipeline20230410) TableName() string {
+ return "_tool_bitbucket_pipelines"
+}
+
+type pipelineStep20230410 struct {
+ Type string `gorm:"type:varchar(255)"`
+ Environment string `gorm:"type:varchar(255)"`
+}
+
+func (pipelineStep20230410) TableName() string {
+ return "_tool_bitbucket_pipeline_steps"
+}
+
+func (u *addTypeEnvToPipelineAndStep) Up(baseRes context.BasicRes)
errors.Error {
+ return migrationhelper.AutoMigrateTables(baseRes, &pipeline20230410{},
&pipelineStep20230410{})
+}
+
+func (*addTypeEnvToPipelineAndStep) Version() uint64 {
+ return 20230322150357
+}
+
+func (*addTypeEnvToPipelineAndStep) Name() string {
+ return "add type/env to bitbucket pipelines and pipeline_steps"
+}
diff --git a/backend/plugins/bitbucket/models/migrationscripts/register.go
b/backend/plugins/bitbucket/models/migrationscripts/register.go
index 27a5f1fa7..ede1ab955 100644
--- a/backend/plugins/bitbucket/models/migrationscripts/register.go
+++ b/backend/plugins/bitbucket/models/migrationscripts/register.go
@@ -32,5 +32,6 @@ func All() []plugin.MigrationScript {
new(addScope20230206),
new(addPipelineStep20230215),
new(addConnectionIdToTransformationRule),
+ new(addTypeEnvToPipelineAndStep),
}
}
diff --git a/backend/plugins/bitbucket/models/pipeline.go
b/backend/plugins/bitbucket/models/pipeline.go
index 86b07c350..a8f63c67a 100644
--- a/backend/plugins/bitbucket/models/pipeline.go
+++ b/backend/plugins/bitbucket/models/pipeline.go
@@ -18,8 +18,9 @@ limitations under the License.
package models
import (
- "github.com/apache/incubator-devlake/core/models/common"
"time"
+
+ "github.com/apache/incubator-devlake/core/models/common"
)
type BitbucketPipeline struct {
@@ -31,6 +32,8 @@ type BitbucketPipeline struct {
RepoId string `gorm:"type:varchar(255)"`
CommitSha string `gorm:"type:varchar(255)"`
WebUrl string `gorm:"type:varchar(255)"`
+ Type string `gorm:"type:varchar(255)"`
+ Environment string `gorm:"type:varchar(255)"`
DurationInSeconds uint64
BitbucketCreatedOn *time.Time
diff --git a/backend/plugins/bitbucket/models/pipeline_step.go
b/backend/plugins/bitbucket/models/pipeline_step.go
index 7f08a4b1c..6125f09b6 100644
--- a/backend/plugins/bitbucket/models/pipeline_step.go
+++ b/backend/plugins/bitbucket/models/pipeline_step.go
@@ -18,8 +18,9 @@ limitations under the License.
package models
import (
- "github.com/apache/incubator-devlake/core/models/common"
"time"
+
+ "github.com/apache/incubator-devlake/core/models/common"
)
type BitbucketPipelineStep struct {
@@ -36,6 +37,8 @@ type BitbucketPipelineStep struct {
DurationInSeconds int
BuildSecondsUsed int
RunNumber int
+ Type string `gorm:"type:varchar(255)"`
+ Environment string `gorm:"type:varchar(255)"`
common.NoPKModel
}
diff --git a/backend/plugins/bitbucket/tasks/pipeline_convertor.go
b/backend/plugins/bitbucket/tasks/pipeline_convertor.go
index e715ee438..673fa09fa 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_convertor.go
@@ -94,7 +94,8 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext)
errors.Error {
InProgress:
[]string{models.IN_PROGRESS, models.PENDING, models.BUILDING},
Default: devops.DONE,
}, bitbucketPipeline.Status),
- Type: "CI/CD",
+ Type: bitbucketPipeline.Type,
+ Environment: bitbucketPipeline.Environment,
CreatedDate: createdAt,
DurationSec:
bitbucketPipeline.DurationInSeconds,
FinishedDate:
bitbucketPipeline.BitbucketCompleteOn,
diff --git a/backend/plugins/bitbucket/tasks/pipeline_extractor.go
b/backend/plugins/bitbucket/tasks/pipeline_extractor.go
index 43f903b41..1cd1eb8c3 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_extractor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_extractor.go
@@ -19,7 +19,9 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
@@ -98,6 +100,8 @@ func ExtractApiPipelines(taskCtx plugin.SubTaskContext)
errors.Error {
DurationInSeconds:
bitbucketApiPipeline.DurationInSeconds,
BitbucketCreatedOn:
api.Iso8601TimeToTime(bitbucketApiPipeline.CreatedOn),
BitbucketCompleteOn:
api.Iso8601TimeToTime(bitbucketApiPipeline.CompletedOn),
+ Type:
data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT,
bitbucketApiPipeline.Target.RefName),
+ Environment:
data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION,
bitbucketApiPipeline.Target.RefName),
}
if err != nil {
return nil, err
diff --git a/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go
b/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go
index d00ec0b7d..5e16cde81 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go
@@ -42,14 +42,6 @@ func ConvertPipelineSteps(taskCtx plugin.SubTaskContext)
errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_PIPELINE_STEPS_TABLE)
db := taskCtx.GetDal()
- deploymentPattern := data.Options.DeploymentPattern
- productionPattern := data.Options.ProductionPattern
- regexEnricher := api.NewRegexEnricher()
- err := regexEnricher.AddRegexp(deploymentPattern, productionPattern)
- if err != nil {
- return err
- }
-
cursor, err := db.Cursor(dal.From(models.BitbucketPipelineStep{}))
if err != nil {
return err
@@ -109,14 +101,6 @@ func ConvertPipelineSteps(taskCtx plugin.SubTaskContext)
errors.Error {
domainTask.Environment = devops.TESTING
}
}
- if domainTask.Type == `` {
- domainTask.Type =
regexEnricher.GetEnrichResult(deploymentPattern, bitbucketPipelineStep.Name,
devops.DEPLOYMENT)
- if domainTask.Type != `` {
- // only check env after type recognized
- domainTask.Environment =
regexEnricher.GetEnrichResult(productionPattern, bitbucketPipelineStep.Name,
devops.PRODUCTION)
- }
- }
-
return []interface{}{
domainTask,
}, nil
diff --git a/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go
b/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go
index e89c60587..08036aa66 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go
@@ -19,11 +19,13 @@ package tasks
import (
"encoding/json"
+ "time"
+
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
- "time"
)
var _ plugin.SubTaskEntryPoint = ExtractPipelineSteps
@@ -90,6 +92,8 @@ func ExtractPipelineSteps(taskCtx plugin.SubTaskContext)
errors.Error {
DurationInSeconds:
apiPipelineStep.DurationInSeconds,
BuildSecondsUsed:
apiPipelineStep.BuildSecondsUsed,
RunNumber: apiPipelineStep.RunNumber,
+ Type:
data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, apiPipelineStep.Name),
+ Environment:
data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, apiPipelineStep.Name),
}
return []interface{}{
bitbucketStep,
diff --git a/backend/plugins/bitbucket/tasks/task_data.go
b/backend/plugins/bitbucket/tasks/task_data.go
index 02eb39874..8d4ab301d 100644
--- a/backend/plugins/bitbucket/tasks/task_data.go
+++ b/backend/plugins/bitbucket/tasks/task_data.go
@@ -35,9 +35,10 @@ type BitbucketOptions struct {
}
type BitbucketTaskData struct {
- Options *BitbucketOptions
- ApiClient *api.ApiAsyncClient
- TimeAfter *time.Time
+ Options *BitbucketOptions
+ ApiClient *api.ApiAsyncClient
+ TimeAfter *time.Time
+ RegexEnricher *api.RegexEnricher
}
func DecodeAndValidateTaskOptions(options map[string]interface{})
(*BitbucketOptions, errors.Error) {
