This is an automated email from the ASF dual-hosted git repository.
mappjzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 1be2edb55 feat(bamboo): add plan build convertor (#4492)
1be2edb55 is described below
commit 1be2edb55483b347e75d884ae6a5f460da0701a2
Author: Warren Chen <[email protected]>
AuthorDate: Thu Feb 23 11:16:27 2023 +0800
feat(bamboo): add plan build convertor (#4492)
---
backend/plugins/bamboo/impl/impl.go | 4 +
.../plugins/bamboo/tasks/job_build_extractor.go | 15 +---
.../plugins/bamboo/tasks/plan_build_convertor.go | 100 +++++++++++++++++++++
.../plugins/bamboo/tasks/plan_commit_convertor.go | 83 +++++++++++++++++
backend/plugins/bamboo/tasks/shared.go | 13 +++
backend/plugins/bamboo/tasks/shared_test.go | 55 ++++++++++++
6 files changed, 256 insertions(+), 14 deletions(-)
diff --git a/backend/plugins/bamboo/impl/impl.go
b/backend/plugins/bamboo/impl/impl.go
index 0c7588b2c..ffdd96f50 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -94,6 +94,9 @@ func (p Bamboo) SubTaskMetas() []plugin.SubTaskMeta {
tasks.ExtractPlanBuildMeta,
tasks.CollectJobBuildMeta,
tasks.ExtractJobBuildMeta,
+ tasks.ConvertJobBuildsMeta,
+ tasks.ConvertPlanBuildsMeta,
+ tasks.ConvertPlanVcsMeta,
tasks.ConvertProjectsMeta,
}
}
@@ -140,6 +143,7 @@ func (p Bamboo) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]i
return nil, err
}
}
+ op.TransformationRuleId = scope.TransformationRuleId
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find project: %s", op.ProjectKey))
}
diff --git a/backend/plugins/bamboo/tasks/job_build_extractor.go
b/backend/plugins/bamboo/tasks/job_build_extractor.go
index 6d950ce6f..c0a8e5910 100644
--- a/backend/plugins/bamboo/tasks/job_build_extractor.go
+++ b/backend/plugins/bamboo/tasks/job_build_extractor.go
@@ -57,7 +57,7 @@ func ExtractJobBuild(taskCtx plugin.SubTaskContext)
errors.Error {
for _, v := range res.VcsRevisions.VcsRevision {
results = append(results,
&models.BambooPlanBuildVcsRevision{
ConnectionId:
data.Options.ConnectionId,
- PlanBuildKey: body.PlanKey,
+ PlanBuildKey: body.PlanBuildKey,
RepositoryId: v.RepositoryId,
RepositoryName: v.RepositoryName,
VcsRevisionKey: v.VcsRevisionKey,
@@ -80,16 +80,3 @@ var ExtractJobBuildMeta = plugin.SubTaskMeta{
Description: "Extract raw data into tool layer table
bamboo_plan_builds",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}
-
-// will be used in next pr
-//func getRepoMap(rawRepoMap datatypes.JSONMap) map[int]string {
-// repoMap := make(map[int]string)
-// for k, v := range rawRepoMap {
-// if list, ok := v.([]int); ok {
-// for _, id := range list {
-// repoMap[id] = k
-// }
-// }
-// }
-// return repoMap
-//}
diff --git a/backend/plugins/bamboo/tasks/plan_build_convertor.go
b/backend/plugins/bamboo/tasks/plan_build_convertor.go
new file mode 100644
index 000000000..34014e683
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/plan_build_convertor.go
@@ -0,0 +1,100 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bamboo/models"
+)
+
+var ConvertPlanBuildsMeta = plugin.SubTaskMeta{
+ Name: "convertPlanBuilds",
+ EntryPoint: ConvertPlanBuilds,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table bamboo_planBuilds into
domain layer table planBuilds",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
+
+func ConvertPlanBuilds(taskCtx plugin.SubTaskContext) errors.Error {
+ db := taskCtx.GetDal()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_PLAN_BUILD_TABLE)
+ deploymentPattern := data.Options.DeploymentPattern
+ productionPattern := data.Options.ProductionPattern
+ regexEnricher := api.NewRegexEnricher()
+ err := regexEnricher.AddRegexp(deploymentPattern, productionPattern)
+ if err != nil {
+ return err
+ }
+ cursor, err := db.Cursor(
+ dal.From(&models.BambooPlanBuild{}),
+ dal.Where("connection_id = ? and project_key = ?",
data.Options.ConnectionId, data.Options.ProjectKey))
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ planBuildIdGen := didgen.NewDomainIdGenerator(&models.BambooPlanBuild{})
+ projectIdGen := didgen.NewDomainIdGenerator(&models.BambooProject{})
+
+ converter, err := api.NewDataConverter(api.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.BambooPlanBuild{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
+ line := inputRow.(*models.BambooPlanBuild)
+ domainPlanBuild := &devops.CICDPipeline{
+ DomainEntity: domainlayer.DomainEntity{Id:
planBuildIdGen.Generate(data.Options.ConnectionId, line.PlanBuildKey)},
+ Name: line.PlanName,
+ DurationSec:
uint64(line.BuildDurationInSeconds),
+ CreatedDate: *line.BuildStartedTime,
+ FinishedDate: line.BuildCompletedDate,
+ CicdScopeId:
projectIdGen.Generate(data.Options.ConnectionId, line.ProjectKey),
+ }
+ if line.LifeCycleState != "Finished" {
+ domainPlanBuild.Status = devops.IN_PROGRESS
+ } else {
+ domainPlanBuild.Status = devops.DONE
+ }
+ if line.BuildState == "Failed" {
+ domainPlanBuild.Result = devops.FAILURE
+ } else if line.BuildState == "Successful" {
+ domainPlanBuild.Result = devops.SUCCESS
+ }
+ domainPlanBuild.Type =
regexEnricher.GetEnrichResult(deploymentPattern, line.PlanName,
devops.DEPLOYMENT)
+ domainPlanBuild.Environment =
regexEnricher.GetEnrichResult(productionPattern, line.PlanName,
devops.PRODUCTION)
+
+ return []interface{}{
+ domainPlanBuild,
+ }, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/backend/plugins/bamboo/tasks/plan_commit_convertor.go
b/backend/plugins/bamboo/tasks/plan_commit_convertor.go
new file mode 100644
index 000000000..c9ac60fa3
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/plan_commit_convertor.go
@@ -0,0 +1,83 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bamboo/models"
+)
+
+var ConvertPlanVcsMeta = plugin.SubTaskMeta{
+ Name: "convertPlanVcs",
+ EntryPoint: ConvertPlanVcs,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table bamboo_planBuilds into
domain layer table planBuilds",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
+
+func ConvertPlanVcs(taskCtx plugin.SubTaskContext) errors.Error {
+ db := taskCtx.GetDal()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_PLAN_BUILD_TABLE)
+ deploymentPattern := data.Options.DeploymentPattern
+ productionPattern := data.Options.ProductionPattern
+ regexEnricher := api.NewRegexEnricher()
+ err := regexEnricher.AddRegexp(deploymentPattern, productionPattern)
+ if err != nil {
+ return err
+ }
+ cursor, err := db.Cursor(
+ dal.From(&models.BambooPlanBuildVcsRevision{}),
+ dal.Join(`left join _tool_bamboo_plan_builds on
_tool_bamboo_plan_builds.plan_build_key =
_tool_bamboo_plan_build_commits.plan_build_key`),
+ dal.Where("_tool_bamboo_plan_build_commits.connection_id = ?
and _tool_bamboo_plan_builds.project_key = ?", data.Options.ConnectionId,
data.Options.ProjectKey))
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ planBuildIdGen := didgen.NewDomainIdGenerator(&models.BambooPlanBuild{})
+ repoMap := getRepoMap(data.Options.RepoMap)
+ converter, err := api.NewDataConverter(api.DataConverterArgs{
+ InputRowType:
reflect.TypeOf(models.BambooPlanBuildVcsRevision{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
+ line := inputRow.(*models.BambooPlanBuildVcsRevision)
+ domainPlanVcs := &devops.CiCDPipelineCommit{
+ PipelineId:
planBuildIdGen.Generate(data.Options.ConnectionId, line.PlanBuildKey),
+ CommitSha: line.VcsRevisionKey,
+ }
+ domainPlanVcs.RepoId = repoMap[line.RepositoryId]
+ return []interface{}{
+ domainPlanVcs,
+ }, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/backend/plugins/bamboo/tasks/shared.go
b/backend/plugins/bamboo/tasks/shared.go
index c49a7f339..20bc6cdc1 100644
--- a/backend/plugins/bamboo/tasks/shared.go
+++ b/backend/plugins/bamboo/tasks/shared.go
@@ -22,6 +22,7 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bamboo/models"
+ "gorm.io/datatypes"
)
func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, rawTable string)
(*api.RawDataSubTaskArgs, *BambooTaskData) {
@@ -48,3 +49,15 @@ func GetTotalPagesFromSizeInfo(sizeInfo
*models.ApiBambooSizeData, args *api.Api
}
return pages, nil
}
+
+func getRepoMap(rawRepoMap datatypes.JSONMap) map[int]string {
+ repoMap := make(map[int]string)
+ for k, v := range rawRepoMap {
+ if list, ok := v.([]interface{}); ok {
+ for _, id := range list {
+ repoMap[int(id.(float64))] = k
+ }
+ }
+ }
+ return repoMap
+}
diff --git a/backend/plugins/bamboo/tasks/shared_test.go
b/backend/plugins/bamboo/tasks/shared_test.go
new file mode 100644
index 000000000..420370340
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/shared_test.go
@@ -0,0 +1,55 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "gorm.io/datatypes"
+ "reflect"
+ "testing"
+)
+
+func TestGetRepoMap(t *testing.T) {
+ rawJSON := `
+ {
+ "repo1": [1, 2],
+ "repo2": [3],
+ "repo3": [4, 5, 6]
+ }
+ `
+ var rawRepoMap datatypes.JSONMap
+ err := json.Unmarshal([]byte(rawJSON), &rawRepoMap)
+ if err != nil {
+ t.Errorf("Failed to unmarshal rawJSON: %v", err)
+ }
+
+ expectedRepoMap := map[int]string{
+ 1: "repo1",
+ 2: "repo1",
+ 3: "repo2",
+ 4: "repo3",
+ 5: "repo3",
+ 6: "repo3",
+ }
+
+ repoMap := getRepoMap(rawRepoMap)
+
+ if !reflect.DeepEqual(repoMap, expectedRepoMap) {
+ t.Errorf("Unexpected result. Expected: %v, got: %v",
expectedRepoMap, repoMap)
+ }
+}
