This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new adc883316 feat: add deploy colle (#4630)
adc883316 is described below
commit adc8833160f7e2d2ecf4d80f8d614fc3081b74be
Author: mappjzc <[email protected]>
AuthorDate: Tue Mar 14 09:45:58 2023 +0800
feat: add deploy colle (#4630)
Add deploy collect for deploy project.
Add extract to get env data by plan.
Nddtfjiang <[email protected]>
---
backend/plugins/bamboo/e2e/deploy_test.go | 69 ++++++++++++++++++++++
.../e2e/raw_tables/_raw_bamboo_api_deploy.csv | 5 ++
.../raw_tables/_tool_bamboo_plans_for_deploy.csv | 5 ++
.../_tool_bamboo_deploy_environment.csv | 7 +++
backend/plugins/bamboo/impl/impl.go | 4 +-
backend/plugins/bamboo/models/deploy.go | 8 +--
..._init_tables.go => 20230309_add_init_tables.go} | 6 +-
.../models/migrationscripts/archived/deploy.go | 6 +-
backend/plugins/bamboo/tasks/deploy_collector.go | 30 +++-------
backend/plugins/bamboo/tasks/deploy_extractor.go | 46 ++++++++++++---
10 files changed, 142 insertions(+), 44 deletions(-)
diff --git a/backend/plugins/bamboo/e2e/deploy_test.go
b/backend/plugins/bamboo/e2e/deploy_test.go
new file mode 100644
index 000000000..ca6d00bb9
--- /dev/null
+++ b/backend/plugins/bamboo/e2e/deploy_test.go
@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/bamboo/impl"
+ "github.com/apache/incubator-devlake/plugins/bamboo/models"
+ "github.com/apache/incubator-devlake/plugins/bamboo/tasks"
+)
+
+func TestBambooDeployDataFlow(t *testing.T) {
+ var bamboo impl.Bamboo
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo)
+ taskData := &tasks.BambooTaskData{
+ Options: &tasks.BambooOptions{
+ ConnectionId: 3,
+ ProjectKey: "TEST1",
+ BambooTransformationRule:
new(models.BambooTransformationRule),
+ },
+ }
+
+ // import raw data table
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_bamboo_api_deploy.csv",
"_raw_bamboo_api_deploy")
+ // it need import plan data
+
dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_bamboo_plans_for_deploy.csv",
models.BambooPlan{})
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.BambooDeployEnvironment{})
+ dataflowTester.Subtask(tasks.ExtractDeployMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.BambooDeployEnvironment{},
+ "./snapshot_tables/_tool_bamboo_deploy_environment.csv",
+ e2ehelper.ColumnWithRawData(
+ "connection_id",
+ "env_id",
+ "name",
+ "plan_key",
+ "description",
+ "position",
+ "configuration_state",
+ "can_view",
+ "can_edit",
+ "can_delete",
+ "allowed_to_execute",
+ "can_execute",
+ "allowed_to_create_version",
+ "allowed_to_set_version_status",
+ ),
+ )
+
+}
diff --git a/backend/plugins/bamboo/e2e/raw_tables/_raw_bamboo_api_deploy.csv
b/backend/plugins/bamboo/e2e/raw_tables/_raw_bamboo_api_deploy.csv
new file mode 100644
index 000000000..6f6bf318f
--- /dev/null
+++ b/backend/plugins/bamboo/e2e/raw_tables/_raw_bamboo_api_deploy.csv
@@ -0,0 +1,5 @@
+"id","params","data","url","input","created_at"
+39,"{""connectionId"":3,""ProjectKey"":""TEST1""}","{""id"":884737,""oid"":""q3067ilzvtvl"",""key"":{""key"":""884737""},""name"":""deploy-test"",""planKey"":{""key"":""TEST1-TEST1""},""description"":""a
test deployment
project."",""environments"":[{""id"":950273,""key"":{""key"":""884737-950273""},""name"":""test-env"",""description"":""a
test
environment"",""deploymentProjectId"":884737,""operations"":{""canView"":true,""canEdit"":true,""canDelete"":true,""allowedToExecute"":true,""can
[...]
+40,"{""connectionId"":3,""ProjectKey"":""TEST1""}","{""id"":1146881,""oid"":""q3067ilzvuo1"",""key"":{""key"":""1146881""},""name"":""deploy-test2"",""planKey"":{""key"":""TEST1-TEST2""},""description"":""other
test deployment
project."",""environments"":[],""operations"":{""canView"":true,""canEdit"":true,""canDelete"":true,""allowedToExecute"":false,""canExecute"":false,""allowedToCreateVersion"":true,""allowedToSetVersionStatus"":false}}","http://18.212.108.64:8085/rest/api/latest/dep
[...]
+41,"{""connectionId"":3,""ProjectKey"":""TEST1""}","{""id"":1146882,""oid"":""q3067ilzvuo2"",""key"":{""key"":""1146882""},""name"":""deploy-test3"",""planKey"":{""key"":""TEST1-TEST3""},""description"":""other
test deployment project
again."",""environments"":[{""id"":1310721,""key"":{""key"":""1146882-1310721""},""name"":""environment2"",""description"":""other
test
environment"",""deploymentProjectId"":1146882,""operations"":{""canView"":true,""canEdit"":true,""canDelete"":true,""allo
[...]
+42,"{""connectionId"":3,""ProjectKey"":""TEST1""}","{""id"":1146883,""oid"":""q3067ilzvuo3"",""key"":{""key"":""1146883""},""name"":""deploy-test4"",""planKey"":{""key"":""TEST2-TEST1""},""description"":"""",""environments"":[{""id"":1310726,""key"":{""key"":""1146883-1310726""},""name"":""environment-for_p2"",""description"":"""",""deploymentProjectId"":1146883,""operations"":{""canView"":true,""canEdit"":true,""canDelete"":true,""allowedToExecute"":true,""canExecute"":true,""allowedToC
[...]
diff --git
a/backend/plugins/bamboo/e2e/raw_tables/_tool_bamboo_plans_for_deploy.csv
b/backend/plugins/bamboo/e2e/raw_tables/_tool_bamboo_plans_for_deploy.csv
new file mode 100644
index 000000000..eb2f4e06a
--- /dev/null
+++ b/backend/plugins/bamboo/e2e/raw_tables/_tool_bamboo_plans_for_deploy.csv
@@ -0,0 +1,5 @@
+connection_id,plan_key,name,expand,project_key,project_name,description,short_name,build_name,short_key,type,enabled,href,rel,is_favourite,is_active,is_building,average_build_time_in_seconds,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+3,TEST1-TEST1,test_project -
test_plan,"actions,stages,branches",TEST1,test_project,just a test
plan,test_plan,test_plan,TEST1,chain,1,http://54.172.92.89:8085/rest/api/latest/plan/TEST1-TEST1,self,0,0,0,0,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_plan,22,
+3,TEST1-TEST2,test_project -
test2,"actions,stages,branches",TEST1,test_project,description
plan2,test2,test2,TEST2,chain,1,http://54.172.92.89:8085/rest/api/latest/plan/TEST1-TEST2,self,0,0,0,0,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_plan,19,
+3,TEST1-TEST3,test_project -
test3,"actions,stages,branches",TEST1,test_project,description
plan23,test3,test3,TEST3,chain,1,http://54.172.92.89:8085/rest/api/latest/plan/TEST1-TEST3,self,0,0,0,0,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_plan,20,
+3,TEST1-TEST4,test_project -
test4,"actions,stages,branches",TEST1,test_project,description
plan4,test4,test4,TEST4,chain,1,http://54.172.92.89:8085/rest/api/latest/plan/TEST1-TEST4,self,0,0,0,0,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_plan,21,
diff --git
a/backend/plugins/bamboo/e2e/snapshot_tables/_tool_bamboo_deploy_environment.csv
b/backend/plugins/bamboo/e2e/snapshot_tables/_tool_bamboo_deploy_environment.csv
new file mode 100644
index 000000000..4040b177f
--- /dev/null
+++
b/backend/plugins/bamboo/e2e/snapshot_tables/_tool_bamboo_deploy_environment.csv
@@ -0,0 +1,7 @@
+connection_id,env_id,name,plan_key,description,position,configuration_state,can_view,can_edit,can_delete,allowed_to_execute,can_execute,allowed_to_create_version,allowed_to_set_version_status,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+3,950273,test-env,TEST1-TEST1,a test
environment,0,DETAILED,1,1,1,1,1,0,0,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy,39,
+3,1310721,environment2,TEST1-TEST3,other test
environment,0,DETAILED,1,1,1,1,1,0,0,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy,41,
+3,1310722,lake,TEST1-TEST3,a test
environment,1,DETAILED,1,1,1,1,1,0,0,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy,41,
+3,1310723,environment2,TEST1-TEST1,the environment2 for
project,1,DETAILED,1,1,1,1,1,0,0,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy,39,
+3,1310724,test-env3,TEST1-TEST1,a test environment
num3,2,DETAILED,1,1,1,1,1,0,0,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy,39,
+3,1310725,test-env4,TEST1-TEST1,the 4th test
environment,3,DETAILED,1,1,1,1,1,0,0,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy,39,
diff --git a/backend/plugins/bamboo/impl/impl.go
b/backend/plugins/bamboo/impl/impl.go
index 2350a0e8e..50de2945b 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -94,8 +94,8 @@ func (p Bamboo) SubTaskMetas() []plugin.SubTaskMeta {
tasks.ExtractPlanBuildMeta,
tasks.CollectJobBuildMeta,
tasks.ExtractJobBuildMeta,
- //tasks.CollectDeployMeta,
- //tasks.ExtractDeployMeta,
+ tasks.CollectDeployMeta,
+ tasks.ExtractDeployMeta,
tasks.ConvertJobBuildsMeta,
tasks.ConvertPlanBuildsMeta,
tasks.ConvertPlanVcsMeta,
diff --git a/backend/plugins/bamboo/models/deploy.go
b/backend/plugins/bamboo/models/deploy.go
index 89239d8c6..7f7f9e58d 100644
--- a/backend/plugins/bamboo/models/deploy.go
+++ b/backend/plugins/bamboo/models/deploy.go
@@ -17,12 +17,11 @@ limitations under the License.
package models
-/*
import "github.com/apache/incubator-devlake/core/models/common"
type BambooDeployEnvironment struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- ID uint64 `json:"id" gorm:"primaryKey"`
+ ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"`
+ EnvId uint64 `json:"env_id" gorm:"primaryKey"`
EnvKey string `json:"key" gorm:"index;type:varchar(255)"`
Name string `json:"name" gorm:"index;type:varchar(255)"`
PlanKey string `json:"plan_key"
gorm:"index;type:varchar(255)"`
@@ -36,7 +35,7 @@ type BambooDeployEnvironment struct {
}
func (b *BambooDeployEnvironment) Convert(apiEnv *ApiBambooEnvironment) {
- b.ID = apiEnv.ID
+ b.EnvId = apiEnv.ID
b.EnvKey = apiEnv.Key.Key
b.Name = apiEnv.Name
b.Description = apiEnv.Description
@@ -71,4 +70,3 @@ type ApiBambooDeployProject struct {
Environments []ApiBambooEnvironment `json:"environments"`
Operations ApiBambooOperations `json:"operations"`
}
-*/
diff --git
a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
b/backend/plugins/bamboo/models/migrationscripts/20230309_add_init_tables.go
similarity index 94%
rename from
backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
rename to
backend/plugins/bamboo/models/migrationscripts/20230309_add_init_tables.go
index 95ad8cf7c..7daf5f95d 100644
--- a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
+++ b/backend/plugins/bamboo/models/migrationscripts/20230309_add_init_tables.go
@@ -35,7 +35,7 @@ func (u *addInitTables) Up(baseRes context.BasicRes)
errors.Error {
&archived.BambooPlanBuildVcsRevision{},
&archived.BambooJobBuild{},
&archived.BambooTransformationRule{},
- //&archived.BambooDeployEnvironment{},
+ &archived.BambooDeployEnvironment{},
)
return migrationhelper.AutoMigrateTables(
baseRes,
@@ -47,12 +47,12 @@ func (u *addInitTables) Up(baseRes context.BasicRes)
errors.Error {
&archived.BambooPlanBuildVcsRevision{},
&archived.BambooJobBuild{},
&archived.BambooTransformationRule{},
- //&archived.BambooDeployEnvironment{},
+ &archived.BambooDeployEnvironment{},
)
}
func (*addInitTables) Version() uint64 {
- return 20230222205033
+ return 20230309205035
}
func (*addInitTables) Name() string {
diff --git a/backend/plugins/bamboo/models/migrationscripts/archived/deploy.go
b/backend/plugins/bamboo/models/migrationscripts/archived/deploy.go
index 7250df19a..05f5205d6 100644
--- a/backend/plugins/bamboo/models/migrationscripts/archived/deploy.go
+++ b/backend/plugins/bamboo/models/migrationscripts/archived/deploy.go
@@ -17,7 +17,6 @@ limitations under the License.
package archived
-/*
import
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
type ApiBambooOperations struct {
@@ -31,8 +30,8 @@ type ApiBambooOperations struct {
}
type BambooDeployEnvironment struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- ID uint64 `json:"id" gorm:"primaryKey"`
+ ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"`
+ EnvId uint64 `json:"env_id" gorm:"primaryKey"`
EnvKey string `json:"key" gorm:"index;type:varchar(255)"`
Name string `json:"name" gorm:"index;type:varchar(255)"`
PlanKey string `json:"plan_key"
gorm:"index;type:varchar(255)"`
@@ -48,4 +47,3 @@ type BambooDeployEnvironment struct {
func (BambooDeployEnvironment) TableName() string {
return "_tool_bamboo_deploy_environment"
}
-*/
diff --git a/backend/plugins/bamboo/tasks/deploy_collector.go
b/backend/plugins/bamboo/tasks/deploy_collector.go
index 7aa341348..994b83c7f 100644
--- a/backend/plugins/bamboo/tasks/deploy_collector.go
+++ b/backend/plugins/bamboo/tasks/deploy_collector.go
@@ -17,10 +17,8 @@ limitations under the License.
package tasks
-/*
import (
- "fmt"
- "net/url"
+ "net/http"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
@@ -34,29 +32,20 @@ var _ plugin.SubTaskEntryPoint = CollectDeploy
func CollectDeploy(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_DEPLOY_TABLE)
- collectorWithState, err :=
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, nil)
- if err != nil {
- return err
- }
-
- err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
- ApiClient: data.ApiClient,
- PageSize: 100,
- UrlTemplate: "deploy/project/all.json",
- Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
- query := url.Values{}
- query.Set("showEmpty", fmt.Sprintf("%v", true))
- query.Set("max-result", fmt.Sprintf("%v",
reqData.Pager.Size))
- query.Set("start-index", fmt.Sprintf("%v",
reqData.Pager.Skip))
- return query, nil
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ GetTotalPages: func(res *http.Response, args
*helper.ApiCollectorArgs) (int, errors.Error) {
+ return 1, nil
},
-
+ UrlTemplate: "deploy/project/all.json",
ResponseParser: helper.GetRawMessageArrayFromResponse,
})
if err != nil {
return err
}
- return collectorWithState.Execute()
+ return collector.Execute()
}
var CollectDeployMeta = plugin.SubTaskMeta{
@@ -66,4 +55,3 @@ var CollectDeployMeta = plugin.SubTaskMeta{
Description: "Collect Deploy data from Bamboo api",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}
-*/
diff --git a/backend/plugins/bamboo/tasks/deploy_extractor.go
b/backend/plugins/bamboo/tasks/deploy_extractor.go
index 4e5d538d5..84ca83c66 100644
--- a/backend/plugins/bamboo/tasks/deploy_extractor.go
+++ b/backend/plugins/bamboo/tasks/deploy_extractor.go
@@ -17,10 +17,10 @@ limitations under the License.
package tasks
-/*
import (
"encoding/json"
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -32,9 +32,33 @@ var _ plugin.SubTaskEntryPoint = ExtractDeploy
func ExtractDeploy(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_DEPLOY_TABLE)
+ db := taskCtx.GetDal()
+ clauses := []dal.Clause{
+ dal.Select("plan_key"),
+ dal.From(models.BambooPlan{}.TableName()),
+ dal.Where("project_key = ? and connection_id=?",
data.Options.ProjectKey, data.Options.ConnectionId),
+ }
+ cursor, err := db.Cursor(
+ clauses...,
+ )
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ Plans := make(map[string]bool)
+
+ for cursor.Next() {
+ Plan := &models.BambooPlan{}
+ err = db.Fetch(cursor, Plan)
+ if err != nil {
+ return err
+ }
+ Plans[Plan.PlanKey] = true
+ }
+
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
-
Extract: func(resData *helper.RawData) ([]interface{},
errors.Error) {
res := &models.ApiBambooDeployProject{}
err := errors.Convert(json.Unmarshal(resData.Data, res))
@@ -43,20 +67,25 @@ func ExtractDeploy(taskCtx plugin.SubTaskContext)
errors.Error {
}
plan := &SimplePlan{}
err = errors.Convert(json.Unmarshal(resData.Input,
plan))
+
if err != nil {
return nil, err
}
results := make([]interface{}, 0, len(res.Environments))
- for _, env := range res.Environments {
- body := &models.BambooDeployEnvironment{}
- body.Convert(&env)
- body.ConnectionId = data.Options.ConnectionId
- body.PlanKey = res.PlanKey.Key
+ if Plans[res.PlanKey.Key] {
+ for _, env := range res.Environments {
+ body :=
&models.BambooDeployEnvironment{}
- results = append(results, body)
+ body.Convert(&env)
+ body.ConnectionId =
data.Options.ConnectionId
+ body.PlanKey = res.PlanKey.Key
+
+ results = append(results, body)
+ }
}
+
return results, nil
},
})
@@ -74,4 +103,3 @@ var ExtractDeployMeta = plugin.SubTaskMeta{
Description: "Extract raw data into tool layer table
_tool_bamboo_deploy_environment",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}
-*/
