This is an automated email from the ASF dual-hosted git repository.
mappjzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 477674fed Bamboo deploy build (#4676)
477674fed is described below
commit 477674fed3651a40d9fa0a2ceb5cf7ddb7ea0fe0
Author: mappjzc <[email protected]>
AuthorDate: Fri Mar 17 19:57:43 2023 +0800
Bamboo deploy build (#4676)
* feat: add deploy colle
Add deploy collect for deploy project.
Add extract to get env data by plan.
Nddtfjiang <[email protected]>
* feat: add bamboo deploy build
Add CollectDeployBuild
Add ExtractDeployBuild
Add ConvertDeployBuilds
Add TestBambooDeployBuildDataFlow
Nddtfjiang <[email protected]>
---
backend/helpers/e2ehelper/data_flow_tester.go | 8 +-
.../e2e/{deploy_test.go => deploy_build_test.go} | 42 +++++---
backend/plugins/bamboo/e2e/deploy_test.go | 2 +-
backend/plugins/bamboo/e2e/job_build_test.go | 2 +-
backend/plugins/bamboo/e2e/job_test.go | 2 +-
backend/plugins/bamboo/e2e/plan_build_test.go | 2 +-
backend/plugins/bamboo/e2e/plan_test.go | 2 +-
.../raw_tables/_raw_bamboo_api_deploy_build.csv | 10 ++
.../snapshot_tables/_tool_bamboo_deploy_build.csv | 10 ++
.../e2e/snapshot_tables/cicd_tasks_deploy.csv | 10 ++
backend/plugins/bamboo/impl/impl.go | 4 +
backend/plugins/bamboo/models/deploy.go | 7 +-
backend/plugins/bamboo/models/deploy_build.go | 109 +++++++++++++++++++++
.../migrationscripts/20230309_add_init_tables.go | 4 +-
.../models/migrationscripts/archived/deploy.go | 1 +
.../migrationscripts/archived/deploy_build.go | 48 +++++++++
backend/plugins/bamboo/models/share.go | 7 ++
.../bamboo/{tasks/task_data.go => models/task.go} | 34 ++-----
.../plugins/bamboo/tasks/deploy_build_collector.go | 106 ++++++++++++++++++++
.../plugins/bamboo/tasks/deploy_build_convertor.go | 109 +++++++++++++++++++++
...ploy_extractor.go => deploy_build_extractor.go} | 68 ++++---------
backend/plugins/bamboo/tasks/deploy_extractor.go | 1 +
backend/plugins/bamboo/tasks/shared.go | 4 +-
backend/plugins/bamboo/tasks/task_data.go | 23 +----
24 files changed, 489 insertions(+), 126 deletions(-)
diff --git a/backend/helpers/e2ehelper/data_flow_tester.go
b/backend/helpers/e2ehelper/data_flow_tester.go
index d4b862383..f768ce7ae 100644
--- a/backend/helpers/e2ehelper/data_flow_tester.go
+++ b/backend/helpers/e2ehelper/data_flow_tester.go
@@ -107,6 +107,9 @@ func NewDataFlowTester(t *testing.T, pluginName string,
pluginMeta plugin.Plugin
cfg.Set(`DB_URL`, cfg.GetString(`E2E_DB_URL`))
db, err := runner.NewGormDb(cfg, logruslog.Global)
if err != nil {
+ // if here fail with error `acces denied for user` you need to
create database by your self as follow command
+ // create databases lake_test;
+ // grant all on lake_test.* to 'merico'@'%';
panic(err)
}
return &DataFlowTester{
@@ -122,7 +125,10 @@ func NewDataFlowTester(t *testing.T, pluginName string,
pluginMeta plugin.Plugin
// ImportCsvIntoRawTable imports records from specified csv file into target
raw table, note that existing data would be deleted first.
func (t *DataFlowTester) ImportCsvIntoRawTable(csvRelPath string, rawTableName
string) {
- csvIter, _ := pluginhelper.NewCsvFileIterator(csvRelPath)
+ csvIter, err := pluginhelper.NewCsvFileIterator(csvRelPath)
+ if err != nil {
+ panic(err)
+ }
defer csvIter.Close()
t.FlushRawTable(rawTableName)
// load rows and insert into target table
diff --git a/backend/plugins/bamboo/e2e/deploy_test.go
b/backend/plugins/bamboo/e2e/deploy_build_test.go
similarity index 61%
copy from backend/plugins/bamboo/e2e/deploy_test.go
copy to backend/plugins/bamboo/e2e/deploy_build_test.go
index ca6d00bb9..ad154ea8a 100644
--- a/backend/plugins/bamboo/e2e/deploy_test.go
+++ b/backend/plugins/bamboo/e2e/deploy_build_test.go
@@ -20,42 +20,47 @@ package e2e
import (
"testing"
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/bamboo/impl"
"github.com/apache/incubator-devlake/plugins/bamboo/models"
"github.com/apache/incubator-devlake/plugins/bamboo/tasks"
)
-func TestBambooDeployDataFlow(t *testing.T) {
+func TestBambooDeployBuildDataFlow(t *testing.T) {
var bamboo impl.Bamboo
dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo)
taskData := &tasks.BambooTaskData{
- Options: &tasks.BambooOptions{
- ConnectionId: 3,
+ Options: &models.BambooOptions{
+ ConnectionId: 1,
ProjectKey: "TEST1",
BambooTransformationRule:
new(models.BambooTransformationRule),
},
}
// import raw data table
-
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_bamboo_api_deploy.csv",
"_raw_bamboo_api_deploy")
- // it need import plan data
-
dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_bamboo_plans_for_deploy.csv",
models.BambooPlan{})
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_bamboo_api_deploy_build.csv",
"_raw_bamboo_api_deploy_build")
// verify extraction
- dataflowTester.FlushTabler(&models.BambooDeployEnvironment{})
- dataflowTester.Subtask(tasks.ExtractDeployMeta, taskData)
+ dataflowTester.FlushTabler(&models.BambooDeployBuild{})
+ dataflowTester.Subtask(tasks.ExtractDeployBuildMeta, taskData)
dataflowTester.VerifyTable(
- models.BambooDeployEnvironment{},
- "./snapshot_tables/_tool_bamboo_deploy_environment.csv",
+ models.BambooDeployBuild{},
+ "./snapshot_tables/_tool_bamboo_deploy_build.csv",
e2ehelper.ColumnWithRawData(
"connection_id",
- "env_id",
- "name",
+ "deploy_build_id",
+ "deployment_version_name",
+ "deployment_state",
+ "life_cycle_state",
+ "started_date",
+ "queued_date",
+ "executed_date",
+ "finished_date",
+ "reason_summary",
"plan_key",
- "description",
- "position",
- "configuration_state",
+ "project_key",
"can_view",
"can_edit",
"can_delete",
@@ -66,4 +71,11 @@ func TestBambooDeployDataFlow(t *testing.T) {
),
)
+ // verify conversion
+ dataflowTester.FlushTabler(&devops.CICDTask{})
+ dataflowTester.Subtask(tasks.ConvertDeployBuildsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&devops.CICDTask{},
e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/cicd_tasks_deploy.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
}
diff --git a/backend/plugins/bamboo/e2e/deploy_test.go
b/backend/plugins/bamboo/e2e/deploy_test.go
index ca6d00bb9..7d809a4e8 100644
--- a/backend/plugins/bamboo/e2e/deploy_test.go
+++ b/backend/plugins/bamboo/e2e/deploy_test.go
@@ -30,7 +30,7 @@ func TestBambooDeployDataFlow(t *testing.T) {
var bamboo impl.Bamboo
dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo)
taskData := &tasks.BambooTaskData{
- Options: &tasks.BambooOptions{
+ Options: &models.BambooOptions{
ConnectionId: 3,
ProjectKey: "TEST1",
BambooTransformationRule:
new(models.BambooTransformationRule),
diff --git a/backend/plugins/bamboo/e2e/job_build_test.go
b/backend/plugins/bamboo/e2e/job_build_test.go
index 6146a992a..7266a0cad 100644
--- a/backend/plugins/bamboo/e2e/job_build_test.go
+++ b/backend/plugins/bamboo/e2e/job_build_test.go
@@ -34,7 +34,7 @@ func TestBambooJobBuildDataFlow(t *testing.T) {
dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo)
taskData := &tasks.BambooTaskData{
- Options: &tasks.BambooOptions{
+ Options: &models.BambooOptions{
ConnectionId: 3,
ProjectKey: "TEST1",
BambooTransformationRule:
&models.BambooTransformationRule{
diff --git a/backend/plugins/bamboo/e2e/job_test.go
b/backend/plugins/bamboo/e2e/job_test.go
index e60ca0bab..7dab01e9a 100644
--- a/backend/plugins/bamboo/e2e/job_test.go
+++ b/backend/plugins/bamboo/e2e/job_test.go
@@ -32,7 +32,7 @@ func TestBambooJobDataFlow(t *testing.T) {
dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo)
taskData := &tasks.BambooTaskData{
- Options: &tasks.BambooOptions{
+ Options: &models.BambooOptions{
ConnectionId: 3,
ProjectKey: "TEST1",
BambooTransformationRule:
&models.BambooTransformationRule{
diff --git a/backend/plugins/bamboo/e2e/plan_build_test.go
b/backend/plugins/bamboo/e2e/plan_build_test.go
index a80714093..9229b7057 100644
--- a/backend/plugins/bamboo/e2e/plan_build_test.go
+++ b/backend/plugins/bamboo/e2e/plan_build_test.go
@@ -34,7 +34,7 @@ func TestBambooPlanBuildDataFlow(t *testing.T) {
dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo)
taskData := &tasks.BambooTaskData{
- Options: &tasks.BambooOptions{
+ Options: &models.BambooOptions{
ConnectionId: 3,
ProjectKey: "TEST1",
BambooTransformationRule:
&models.BambooTransformationRule{
diff --git a/backend/plugins/bamboo/e2e/plan_test.go
b/backend/plugins/bamboo/e2e/plan_test.go
index 4f1f51e27..648b46c45 100644
--- a/backend/plugins/bamboo/e2e/plan_test.go
+++ b/backend/plugins/bamboo/e2e/plan_test.go
@@ -32,7 +32,7 @@ func TestBambooPlanDataFlow(t *testing.T) {
dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo)
taskData := &tasks.BambooTaskData{
- Options: &tasks.BambooOptions{
+ Options: &models.BambooOptions{
ConnectionId: 3,
ProjectKey: "TEST1",
BambooTransformationRule:
new(models.BambooTransformationRule),
diff --git
a/backend/plugins/bamboo/e2e/raw_tables/_raw_bamboo_api_deploy_build.csv
b/backend/plugins/bamboo/e2e/raw_tables/_raw_bamboo_api_deploy_build.csv
new file mode 100644
index 000000000..bbf1f05b5
--- /dev/null
+++ b/backend/plugins/bamboo/e2e/raw_tables/_raw_bamboo_api_deploy_build.csv
@@ -0,0 +1,10 @@
+"id","params","data","url","input","created_at"
+1,"{""connectionId"":1,""ProjectKey"":""TEST1""}","{""deploymentVersion"":{""id"":1671170,""name"":""release-3"",""creationDate"":1678451437024,""creatorUserName"":""bamboo"",""items"":[],""operations"":{""canView"":true,""canEdit"":true,""canDelete"":true,""allowedToExecute"":false,""canExecute"":false,""allowedToCreateVersion"":true,""allowedToSetVersionStatus"":true},""creatorDisplayName"":""devlake"",""creatorGravatarUrl"":""https://secure.gravatar.com/avatar/324e7f96b867d04ccf353fe0
[...]
+2,"{""connectionId"":1,""ProjectKey"":""TEST1""}","{""deploymentVersion"":{""id"":1671170,""name"":""release-3"",""creationDate"":1678451437024,""creatorUserName"":""bamboo"",""items"":[],""operations"":{""canView"":true,""canEdit"":true,""canDelete"":true,""allowedToExecute"":false,""canExecute"":false,""allowedToCreateVersion"":true,""allowedToSetVersionStatus"":true},""creatorDisplayName"":""devlake"",""creatorGravatarUrl"":""https://secure.gravatar.com/avatar/324e7f96b867d04ccf353fe0
[...]
+3,"{""connectionId"":1,""ProjectKey"":""TEST1""}","{""deploymentVersion"":{""id"":983041,""name"":""release-1"",""creationDate"":1676989673661,""creatorUserName"":""bamboo"",""items"":[],""operations"":{""canView"":true,""canEdit"":true,""canDelete"":true,""allowedToExecute"":false,""canExecute"":false,""allowedToCreateVersion"":true,""allowedToSetVersionStatus"":true},""creatorDisplayName"":""devlake"",""creatorGravatarUrl"":""https://secure.gravatar.com/avatar/324e7f96b867d04ccf353fe0f
[...]
+4,"{""connectionId"":1,""ProjectKey"":""TEST1""}","{""deploymentVersion"":{""id"":983041,""name"":""release-1"",""creationDate"":1676989673661,""creatorUserName"":""bamboo"",""items"":[],""operations"":{""canView"":true,""canEdit"":true,""canDelete"":true,""allowedToExecute"":false,""canExecute"":false,""allowedToCreateVersion"":true,""allowedToSetVersionStatus"":true},""creatorDisplayName"":""devlake"",""creatorGravatarUrl"":""https://secure.gravatar.com/avatar/324e7f96b867d04ccf353fe0f
[...]
+5,"{""connectionId"":1,""ProjectKey"":""TEST1""}","{""deploymentVersion"":{""id"":1671169,""name"":""release-2"",""creationDate"":1678418543376,""creatorUserName"":""bamboo"",""items"":[],""operations"":{""canView"":true,""canEdit"":true,""canDelete"":true,""allowedToExecute"":false,""canExecute"":false,""allowedToCreateVersion"":true,""allowedToSetVersionStatus"":true},""creatorDisplayName"":""devlake"",""creatorGravatarUrl"":""https://secure.gravatar.com/avatar/324e7f96b867d04ccf353fe0
[...]
+6,"{""connectionId"":1,""ProjectKey"":""TEST1""}","{""deploymentVersion"":{""id"":1671169,""name"":""release-2"",""creationDate"":1678418543376,""creatorUserName"":""bamboo"",""items"":[],""operations"":{""canView"":true,""canEdit"":true,""canDelete"":true,""allowedToExecute"":false,""canExecute"":false,""allowedToCreateVersion"":true,""allowedToSetVersionStatus"":true},""creatorDisplayName"":""devlake"",""creatorGravatarUrl"":""https://secure.gravatar.com/avatar/324e7f96b867d04ccf353fe0
[...]
+7,"{""connectionId"":1,""ProjectKey"":""TEST1""}","{""deploymentVersion"":{""id"":1671169,""name"":""release-2"",""creationDate"":1678418543376,""creatorUserName"":""bamboo"",""items"":[],""operations"":{""canView"":true,""canEdit"":true,""canDelete"":true,""allowedToExecute"":false,""canExecute"":false,""allowedToCreateVersion"":true,""allowedToSetVersionStatus"":true},""creatorDisplayName"":""devlake"",""creatorGravatarUrl"":""https://secure.gravatar.com/avatar/324e7f96b867d04ccf353fe0
[...]
+8,"{""connectionId"":1,""ProjectKey"":""TEST1""}","{""deploymentVersion"":{""id"":1671169,""name"":""release-2"",""creationDate"":1678418543376,""creatorUserName"":""bamboo"",""items"":[],""operations"":{""canView"":true,""canEdit"":true,""canDelete"":true,""allowedToExecute"":false,""canExecute"":false,""allowedToCreateVersion"":true,""allowedToSetVersionStatus"":true},""creatorDisplayName"":""devlake"",""creatorGravatarUrl"":""https://secure.gravatar.com/avatar/324e7f96b867d04ccf353fe0
[...]
+9,"{""connectionId"":1,""ProjectKey"":""TEST1""}","{""deploymentVersion"":{""id"":1671169,""name"":""release-2"",""creationDate"":1678418543376,""creatorUserName"":""bamboo"",""items"":[],""operations"":{""canView"":true,""canEdit"":true,""canDelete"":true,""allowedToExecute"":false,""canExecute"":false,""allowedToCreateVersion"":true,""allowedToSetVersionStatus"":true},""creatorDisplayName"":""devlake"",""creatorGravatarUrl"":""https://secure.gravatar.com/avatar/324e7f96b867d04ccf353fe0
[...]
\ No newline at end of file
diff --git
a/backend/plugins/bamboo/e2e/snapshot_tables/_tool_bamboo_deploy_build.csv
b/backend/plugins/bamboo/e2e/snapshot_tables/_tool_bamboo_deploy_build.csv
new file mode 100644
index 000000000..d4ad2ca4b
--- /dev/null
+++ b/backend/plugins/bamboo/e2e/snapshot_tables/_tool_bamboo_deploy_build.csv
@@ -0,0 +1,10 @@
+connection_id,deploy_build_id,deployment_version_name,deployment_state,life_cycle_state,started_date,queued_date,executed_date,finished_date,reason_summary,plan_key,project_key,can_view,can_edit,can_delete,allowed_to_execute,can_execute,allowed_to_create_version,allowed_to_set_version_status,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,1769473,release-2,FAILED,FINISHED,2023-03-10T12:15:45.000+00:00,2023-03-10T12:15:45.000+00:00,2023-03-10T12:15:46.000+00:00,2023-03-10T12:15:46.000+00:00,"Manual
run by <a
href=""http://54.172.92.89:8085/browse/user/bamboo"";>devlake</a>",TEST1-TEST1,TEST1,1,1,1,1,1,0,0,"{""connectionId"":1,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy_build,9,
+1,1769474,release-2,FAILED,FINISHED,2023-03-10T12:27:28.000+00:00,2023-03-10T12:27:28.000+00:00,2023-03-10T12:27:28.000+00:00,2023-03-10T12:27:28.000+00:00,"Manual
run by <a
href=""http://54.172.92.89:8085/browse/user/bamboo"";>devlake</a>",TEST1-TEST1,TEST1,1,1,1,1,1,0,0,"{""connectionId"":1,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy_build,8,
+1,1769475,release-2,FAILED,FINISHED,2023-03-10T12:27:56.000+00:00,2023-03-10T12:27:56.000+00:00,2023-03-10T12:27:56.000+00:00,2023-03-10T12:27:56.000+00:00,"Manual
run by <a
href=""http://54.172.92.89:8085/browse/user/bamboo"";>devlake</a>",TEST1-TEST1,TEST1,1,1,1,1,1,0,0,"{""connectionId"":1,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy_build,7,
+1,1769476,release-2,FAILED,FINISHED,2023-03-10T12:28:12.000+00:00,2023-03-10T12:28:12.000+00:00,2023-03-10T12:28:12.000+00:00,2023-03-10T12:28:12.000+00:00,"Manual
run by <a
href=""http://54.172.92.89:8085/browse/user/bamboo"";>devlake</a>",TEST1-TEST1,TEST1,1,1,1,1,1,0,0,"{""connectionId"":1,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy_build,6,
+1,1769477,release-1,FAILED,FINISHED,2023-03-10T12:29:08.000+00:00,2023-03-10T12:29:09.000+00:00,2023-03-10T12:29:09.000+00:00,2023-03-10T12:29:09.000+00:00,"Manual
run by <a
href=""http://54.172.92.89:8085/browse/user/bamboo"";>devlake</a>",TEST1-TEST1,TEST1,1,1,1,1,1,0,0,"{""connectionId"":1,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy_build,4,
+1,1769478,release-1,FAILED,FINISHED,2023-03-10T12:29:28.000+00:00,2023-03-10T12:29:28.000+00:00,2023-03-10T12:29:28.000+00:00,2023-03-10T12:29:28.000+00:00,"Manual
run by <a
href=""http://54.172.92.89:8085/browse/user/bamboo"";>devlake</a>",TEST1-TEST1,TEST1,1,1,1,1,1,0,0,"{""connectionId"":1,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy_build,3,
+1,1769479,release-3,FAILED,FINISHED,2023-03-10T12:30:37.000+00:00,2023-03-10T12:30:37.000+00:00,2023-03-10T12:30:37.000+00:00,2023-03-10T12:30:37.000+00:00,"Manual
run by <a
href=""http://54.172.92.89:8085/browse/user/bamboo"";>devlake</a>",TEST1-TEST1,TEST1,1,1,1,1,1,0,0,"{""connectionId"":1,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy_build,2,
+1,1769480,release-3,FAILED,FINISHED,2023-03-10T12:30:49.000+00:00,2023-03-10T12:30:49.000+00:00,2023-03-10T12:30:49.000+00:00,2023-03-10T12:30:49.000+00:00,"Manual
run by <a
href=""http://54.172.92.89:8085/browse/user/bamboo"";>devlake</a>",TEST1-TEST1,TEST1,1,1,1,1,1,0,0,"{""connectionId"":1,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy_build,1,
+1,1769481,release-2,FAILED,FINISHED,2023-03-13T09:43:26.000+00:00,2023-03-13T09:43:26.000+00:00,2023-03-13T09:43:26.000+00:00,2023-03-13T09:43:26.000+00:00,"Manual
run by <a
href=""http://54.172.92.89:8085/browse/user/bamboo"";>devlake</a>",TEST1-TEST1,TEST1,1,1,1,1,1,0,0,"{""connectionId"":1,""ProjectKey"":""TEST1""}",_raw_bamboo_api_deploy_build,5,
diff --git a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_tasks_deploy.csv
b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_tasks_deploy.csv
new file mode 100644
index 000000000..5ba0f3921
--- /dev/null
+++ b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_tasks_deploy.csv
@@ -0,0 +1,10 @@
+id,name,pipeline_id,result,status,type,environment,duration_sec,started_date,finished_date,cicd_scope_id
+bamboo:BambooDeployBuild:1:1769473,release-2,bamboo:BambooPlanBuild:1:TEST1-TEST1,,IN_PROGRESS,DEPLOYMENT,PRODUCTION,0,2023-03-10T12:15:45.000+00:00,2023-03-10T12:15:46.000+00:00,bamboo:BambooProject:1:TEST1
+bamboo:BambooDeployBuild:1:1769474,release-2,bamboo:BambooPlanBuild:1:TEST1-TEST1,,IN_PROGRESS,DEPLOYMENT,PRODUCTION,0,2023-03-10T12:27:28.000+00:00,2023-03-10T12:27:28.000+00:00,bamboo:BambooProject:1:TEST1
+bamboo:BambooDeployBuild:1:1769475,release-2,bamboo:BambooPlanBuild:1:TEST1-TEST1,,IN_PROGRESS,DEPLOYMENT,PRODUCTION,0,2023-03-10T12:27:56.000+00:00,2023-03-10T12:27:56.000+00:00,bamboo:BambooProject:1:TEST1
+bamboo:BambooDeployBuild:1:1769476,release-2,bamboo:BambooPlanBuild:1:TEST1-TEST1,,IN_PROGRESS,DEPLOYMENT,PRODUCTION,0,2023-03-10T12:28:12.000+00:00,2023-03-10T12:28:12.000+00:00,bamboo:BambooProject:1:TEST1
+bamboo:BambooDeployBuild:1:1769477,release-1,bamboo:BambooPlanBuild:1:TEST1-TEST1,,IN_PROGRESS,DEPLOYMENT,PRODUCTION,0,2023-03-10T12:29:08.000+00:00,2023-03-10T12:29:09.000+00:00,bamboo:BambooProject:1:TEST1
+bamboo:BambooDeployBuild:1:1769478,release-1,bamboo:BambooPlanBuild:1:TEST1-TEST1,,IN_PROGRESS,DEPLOYMENT,PRODUCTION,0,2023-03-10T12:29:28.000+00:00,2023-03-10T12:29:28.000+00:00,bamboo:BambooProject:1:TEST1
+bamboo:BambooDeployBuild:1:1769479,release-3,bamboo:BambooPlanBuild:1:TEST1-TEST1,,IN_PROGRESS,DEPLOYMENT,PRODUCTION,0,2023-03-10T12:30:37.000+00:00,2023-03-10T12:30:37.000+00:00,bamboo:BambooProject:1:TEST1
+bamboo:BambooDeployBuild:1:1769480,release-3,bamboo:BambooPlanBuild:1:TEST1-TEST1,,IN_PROGRESS,DEPLOYMENT,PRODUCTION,0,2023-03-10T12:30:49.000+00:00,2023-03-10T12:30:49.000+00:00,bamboo:BambooProject:1:TEST1
+bamboo:BambooDeployBuild:1:1769481,release-2,bamboo:BambooPlanBuild:1:TEST1-TEST1,,IN_PROGRESS,DEPLOYMENT,PRODUCTION,0,2023-03-13T09:43:26.000+00:00,2023-03-13T09:43:26.000+00:00,bamboo:BambooProject:1:TEST1
diff --git a/backend/plugins/bamboo/impl/impl.go
b/backend/plugins/bamboo/impl/impl.go
index be0a3f10b..479cebd58 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -95,10 +95,14 @@ func (p Bamboo) SubTaskMetas() []plugin.SubTaskMeta {
tasks.ExtractJobBuildMeta,
tasks.CollectDeployMeta,
tasks.ExtractDeployMeta,
+ tasks.CollectDeployBuildMeta,
+ tasks.ExtractDeployBuildMeta,
+
tasks.ConvertJobBuildsMeta,
tasks.ConvertPlanBuildsMeta,
tasks.ConvertPlanVcsMeta,
tasks.ConvertProjectsMeta,
+ tasks.ConvertDeployBuildsMeta,
}
}
diff --git a/backend/plugins/bamboo/models/deploy.go
b/backend/plugins/bamboo/models/deploy.go
index 7f7f9e58d..093542eab 100644
--- a/backend/plugins/bamboo/models/deploy.go
+++ b/backend/plugins/bamboo/models/deploy.go
@@ -22,9 +22,10 @@ import
"github.com/apache/incubator-devlake/core/models/common"
type BambooDeployEnvironment struct {
ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"`
EnvId uint64 `json:"env_id" gorm:"primaryKey"`
- EnvKey string `json:"key" gorm:"index;type:varchar(255)"`
- Name string `json:"name" gorm:"index;type:varchar(255)"`
- PlanKey string `json:"plan_key"
gorm:"index;type:varchar(255)"`
+ EnvKey string `json:"key" gorm:"index"`
+ Name string `json:"name" gorm:"index"`
+ PlanKey string `json:"plan_key" gorm:"index"`
+ ProjectKey string `json:"project_key" gorm:"index"`
Description string `json:"description"`
DeploymentProjectId uint64 `json:"deploymentProjectId"`
Position uint64 `json:"position"`
diff --git a/backend/plugins/bamboo/models/deploy_build.go
b/backend/plugins/bamboo/models/deploy_build.go
new file mode 100644
index 000000000..dd43bdfb2
--- /dev/null
+++ b/backend/plugins/bamboo/models/deploy_build.go
@@ -0,0 +1,109 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/core/models/common"
+)
+
+type BambooDeployBuild struct {
+ ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"`
+ DeployBuildId uint64 `json:"deploy_build_id" gorm:"primaryKey"`
+
+ DeploymentVersionName string `json:"deploymentVersionName"`
+ DeploymentState string `json:"deploymentState"`
+ LifeCycleState string `json:"lifeCycleState"`
+ StartedDate *time.Time `json:"startedDate"`
+ QueuedDate *time.Time `json:"queuedDate"`
+ ExecutedDate *time.Time `json:"executedDate"`
+ FinishedDate *time.Time `json:"finishedDate"`
+ ReasonSummary string `json:"reasonSummary"`
+
+ ProjectKey string `json:"project_key" gorm:"index"`
+ PlanKey string `json:"plan_key" gorm:"index"`
+
+ ApiBambooOperations
+ common.NoPKModel
+}
+
+func (BambooDeployBuild) TableName() string {
+ return "_tool_bamboo_deploy_build"
+}
+
+type ApiBambooDeployBuild struct {
+ DeploymentVersionName string
`json:"deploymentVersionName"`
+ Id uint64 `json:"id"`
+ DeploymentState string `json:"deploymentState"`
+ LifeCycleState string `json:"lifeCycleState"`
+ StartedDate int64 `json:"startedDate"`
+ QueuedDate int64 `json:"queuedDate"`
+ ExecutedDate int64 `json:"executedDate"`
+ FinishedDate int64 `json:"finishedDate"`
+ ReasonSummary string `json:"reasonSummary"`
+ Key ApiBambooDeployBuildKey `json:"key"`
+ Agent ApiBambooDeployBuildAgent `json:"agent"`
+ Operations ApiBambooOperations `json:"operations"`
+}
+
+func (api *ApiBambooDeployBuild) Convert(op *BambooOptions) *BambooDeployBuild
{
+ return &BambooDeployBuild{
+ ConnectionId: op.ConnectionId,
+ ProjectKey: op.ProjectKey,
+ DeploymentVersionName: api.DeploymentVersionName,
+ DeployBuildId: api.Id,
+ DeploymentState: api.DeploymentState,
+ LifeCycleState: api.LifeCycleState,
+ StartedDate:
unixForBambooDeployBuild(api.StartedDate),
+ QueuedDate: unixForBambooDeployBuild(api.QueuedDate),
+ ExecutedDate:
unixForBambooDeployBuild(api.ExecutedDate),
+ FinishedDate:
unixForBambooDeployBuild(api.FinishedDate),
+ ReasonSummary: api.ReasonSummary,
+ ApiBambooOperations: api.Operations,
+ }
+}
+
+type ApiBambooDeployBuildAgent struct {
+ Id uint64 `json:"id"`
+ Name string `json:"name"`
+
+ Type string `json:"type"`
+ Active bool `json:"active"`
+ Enable bool `json:"enabled"`
+ Busy bool `json:"busy"`
+}
+
+type ApiBambooDeployBuildKey struct {
+ Key string `json:"key"`
+ EntityKey ApiBambooKey `json:"entityKey"`
+ ResultNumber uint64 `json:"resultNumber"`
+}
+
+type ApiBambooDeploymentVersion struct {
+ ID uint64 `json:"id"`
+ Name string `json:"name"`
+ CreationDate *time.Time `json:"creationDate"`
+ CreatorUserName string `json:"creatorUserName"`
+ Items []interface{} `json:"items"`
+ Operations ApiBambooOperations `json:"operations"`
+ CreatorDisplayName string `json:"creatorDisplayName"`
+ CreatorGravatarUrl string `json:"creatorGravatarUrl"`
+ PlanBranchName string `json:"planBranchName"`
+ AgeZeroPoint uint64 `json:"ageZeroPoint"`
+}
diff --git
a/backend/plugins/bamboo/models/migrationscripts/20230309_add_init_tables.go
b/backend/plugins/bamboo/models/migrationscripts/20230309_add_init_tables.go
index 7daf5f95d..cf6c1485e 100644
--- a/backend/plugins/bamboo/models/migrationscripts/20230309_add_init_tables.go
+++ b/backend/plugins/bamboo/models/migrationscripts/20230309_add_init_tables.go
@@ -36,6 +36,7 @@ func (u *addInitTables) Up(baseRes context.BasicRes)
errors.Error {
&archived.BambooJobBuild{},
&archived.BambooTransformationRule{},
&archived.BambooDeployEnvironment{},
+ &archived.BambooDeployBuild{},
)
return migrationhelper.AutoMigrateTables(
baseRes,
@@ -48,11 +49,12 @@ func (u *addInitTables) Up(baseRes context.BasicRes)
errors.Error {
&archived.BambooJobBuild{},
&archived.BambooTransformationRule{},
&archived.BambooDeployEnvironment{},
+ &archived.BambooDeployBuild{},
)
}
func (*addInitTables) Version() uint64 {
- return 20230309205035
+ return 20230315205035
}
func (*addInitTables) Name() string {
diff --git a/backend/plugins/bamboo/models/migrationscripts/archived/deploy.go
b/backend/plugins/bamboo/models/migrationscripts/archived/deploy.go
index 05f5205d6..a8e7e1617 100644
--- a/backend/plugins/bamboo/models/migrationscripts/archived/deploy.go
+++ b/backend/plugins/bamboo/models/migrationscripts/archived/deploy.go
@@ -35,6 +35,7 @@ type BambooDeployEnvironment struct {
EnvKey string `json:"key" gorm:"index;type:varchar(255)"`
Name string `json:"name" gorm:"index;type:varchar(255)"`
PlanKey string `json:"plan_key"
gorm:"index;type:varchar(255)"`
+ ProjectKey string `json:"project_key" gorm:"index"`
Description string `json:"description"`
DeploymentProjectId uint64 `json:"deploymentProjectId"`
Position uint64 `json:"position"`
diff --git
a/backend/plugins/bamboo/models/migrationscripts/archived/deploy_build.go
b/backend/plugins/bamboo/models/migrationscripts/archived/deploy_build.go
new file mode 100644
index 000000000..8b60cbb92
--- /dev/null
+++ b/backend/plugins/bamboo/models/migrationscripts/archived/deploy_build.go
@@ -0,0 +1,48 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+)
+
+type BambooDeployBuild struct {
+ ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"`
+ DeployBuildId uint64 `json:"deploy_build_id" gorm:"primaryKey"`
+
+ DeploymentVersionName string `json:"deploymentVersionName"`
+ DeploymentState string `json:"deploymentState"`
+ LifeCycleState string `json:"lifeCycleState"`
+ StartedDate *time.Time `json:"startedDate"`
+ QueuedDate *time.Time `json:"queuedDate"`
+ ExecutedDate *time.Time `json:"executedDate"`
+ FinishedDate *time.Time `json:"finishedDate"`
+ ReasonSummary string `json:"reasonSummary"`
+
+ ProjectKey string `json:"project_key" gorm:"index"`
+ PlanKey string `json:"plan_key" gorm:"index"`
+
+ ApiBambooOperations
+ archived.NoPKModel
+}
+
+func (BambooDeployBuild) TableName() string {
+ return "_tool_bamboo_deploy_build"
+}
diff --git a/backend/plugins/bamboo/models/share.go
b/backend/plugins/bamboo/models/share.go
index 4ac14e9ac..356a04747 100644
--- a/backend/plugins/bamboo/models/share.go
+++ b/backend/plugins/bamboo/models/share.go
@@ -17,6 +17,8 @@ limitations under the License.
package models
+import "time"
+
type ApiBambooLink struct {
Href string `json:"href"`
Rel string `json:"rel"`
@@ -40,3 +42,8 @@ type ApiBambooOperations struct {
AllowedToCreateVersion bool `json:"allowedToCreateVersion"`
AllowedToSetVersionStatus bool `json:"allowedToSetVersionStatus"`
}
+
+func unixForBambooDeployBuild(time_unix int64) *time.Time {
+ t := time.Unix(time_unix/1000, 0)
+ return &t
+}
diff --git a/backend/plugins/bamboo/tasks/task_data.go
b/backend/plugins/bamboo/models/task.go
similarity index 52%
copy from backend/plugins/bamboo/tasks/task_data.go
copy to backend/plugins/bamboo/models/task.go
index 1abee2795..827f37110 100644
--- a/backend/plugins/bamboo/tasks/task_data.go
+++ b/backend/plugins/bamboo/models/task.go
@@ -15,13 +15,7 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package tasks
-
-import (
- "github.com/apache/incubator-devlake/core/errors"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/bamboo/models"
-)
+package models
type BambooApiParams struct {
ConnectionId uint64 `json:"connectionId"`
@@ -33,25 +27,9 @@ type BambooOptions struct {
// options means some custom params required by plugin running.
// Such As How many rows do your want
// You can use it in sub tasks and you need pass it in main.go and
pipelines.
- ConnectionId uint64 `json:"connectionId"`
- ProjectKey string `json:"projectKey"`
- Tasks []string `json:"tasks,omitempty"`
- TransformationRuleId uint64
`mapstructure:"transformationRuleId" json:"transformationRuleId"`
- *models.BambooTransformationRule `mapstructure:"transformationRules"
json:"transformationRules"`
-}
-
-type BambooTaskData struct {
- Options *BambooOptions
- ApiClient *helper.ApiAsyncClient
-}
-
-func DecodeAndValidateTaskOptions(options map[string]interface{})
(*BambooOptions, errors.Error) {
- var op BambooOptions
- if err := helper.Decode(options, &op, nil); err != nil {
- return nil, err
- }
- if op.ConnectionId == 0 {
- return nil, errors.Default.New("connectionId is invalid")
- }
- return &op, nil
+ ConnectionId uint64 `json:"connectionId"`
+ ProjectKey string `json:"projectKey"`
+ Tasks []string `json:"tasks,omitempty"`
+ TransformationRuleId uint64 `mapstructure:"transformationRuleId"
json:"transformationRuleId"`
+ *BambooTransformationRule `mapstructure:"transformationRules"
json:"transformationRules"`
}
diff --git a/backend/plugins/bamboo/tasks/deploy_build_collector.go
b/backend/plugins/bamboo/tasks/deploy_build_collector.go
new file mode 100644
index 000000000..53a4d2bb5
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/deploy_build_collector.go
@@ -0,0 +1,106 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bamboo/models"
+)
+
+const RAW_DEPLOY_BUILD_TABLE = "bamboo_api_deploy_build"
+
+var _ plugin.SubTaskEntryPoint = CollectDeployBuild
+
+type InputForEnv struct {
+ EnvId uint64 `json:"env_id"`
+ PlanKey string `json:"plan_key"`
+}
+
+func CollectDeployBuild(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_DEPLOY_BUILD_TABLE)
+ db := taskCtx.GetDal()
+ clauses := []dal.Clause{
+ dal.Select("env_id,plan_key"),
+ dal.From(models.BambooDeployEnvironment{}.TableName()),
+ dal.Where("project_key = ? and connection_id=?",
data.Options.ProjectKey, data.Options.ConnectionId),
+ }
+ cursor, err := db.Cursor(
+ clauses...,
+ )
+ if err != nil {
+ return err
+ }
+ iterator, err := helper.NewDalCursorIterator(db, cursor,
reflect.TypeOf(InputForEnv{}))
+ if err != nil {
+ return err
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Input: iterator,
+ UrlTemplate: "/deploy/environment/{{ .Input.EnvId
}}/results.json",
+ Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
+ query := url.Values{}
+ query.Set("max-result", fmt.Sprintf("%v",
reqData.Pager.Size))
+ query.Set("start-index", fmt.Sprintf("%v",
reqData.Pager.Skip))
+ return query, nil
+ },
+ GetTotalPages: func(res *http.Response, args
*helper.ApiCollectorArgs) (int, errors.Error) {
+ body := &models.ApiBambooSizeData{}
+ err = helper.UnmarshalResponse(res, body)
+ if err != nil {
+ return 0, err
+ }
+ return GetTotalPagesFromSizeInfo(body, args)
+ },
+
+ ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
+ var resData struct {
+ Results []json.RawMessage `json:"results"`
+ }
+ err := helper.UnmarshalResponse(res, &resData)
+ if err != nil {
+ return nil, err
+ }
+ return resData.Results, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+ return collector.Execute()
+}
+
+var CollectDeployBuildMeta = plugin.SubTaskMeta{
+ Name: "CollectDeployBuild",
+ EntryPoint: CollectDeployBuild,
+ EnabledByDefault: true,
+ Description: "Collect DeployBuild data from Bamboo api",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
diff --git a/backend/plugins/bamboo/tasks/deploy_build_convertor.go
b/backend/plugins/bamboo/tasks/deploy_build_convertor.go
new file mode 100644
index 000000000..d87b70cca
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/deploy_build_convertor.go
@@ -0,0 +1,109 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bamboo/models"
+)
+
+var ConvertDeployBuildsMeta = plugin.SubTaskMeta{
+ Name: "convertDeployBuilds",
+ EntryPoint: ConvertDeployBuilds,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table bamboo_deploy_builds into
domain layer table deployBuilds",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
+
+func ConvertDeployBuilds(taskCtx plugin.SubTaskContext) errors.Error {
+ db := taskCtx.GetDal()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_JOB_BUILD_TABLE)
+ deploymentPattern := data.Options.DeploymentPattern
+ productionPattern := data.Options.ProductionPattern
+ regexEnricher := api.NewRegexEnricher()
+ err := regexEnricher.AddRegexp(deploymentPattern, productionPattern)
+ if err != nil {
+ return err
+ }
+ cursor, err := db.Cursor(
+ dal.From(&models.BambooDeployBuild{}),
+ dal.Where("connection_id = ? and project_key = ?",
data.Options.ConnectionId, data.Options.ProjectKey))
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ deployBuildIdGen :=
didgen.NewDomainIdGenerator(&models.BambooDeployBuild{})
+ planBuildIdGen := didgen.NewDomainIdGenerator(&models.BambooPlanBuild{})
+ projectIdGen := didgen.NewDomainIdGenerator(&models.BambooProject{})
+
+ converter, err := api.NewDataConverter(api.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.BambooDeployBuild{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
+ deployBuild := inputRow.(*models.BambooDeployBuild)
+ domainTask := &devops.CICDTask{
+ DomainEntity: domainlayer.DomainEntity{
+ Id:
deployBuildIdGen.Generate(data.Options.ConnectionId, deployBuild.DeployBuildId),
+ },
+ PipelineId:
planBuildIdGen.Generate(data.Options.ConnectionId, deployBuild.PlanKey),
+ CicdScopeId:
projectIdGen.Generate(data.Options.ConnectionId, deployBuild.ProjectKey),
+
+ Name: deployBuild.DeploymentVersionName,
+
+ Result: devops.GetResult(&devops.ResultRule{
+ Failed: []string{"Failed"},
+ Success: []string{"Successful"},
+ Default: "",
+ }, deployBuild.DeploymentState),
+
+ Status: devops.GetStatus(&devops.StatusRule{
+ Done: []string{"Finished"},
+ Default: devops.IN_PROGRESS,
+ }, deployBuild.LifeCycleState),
+
+ //DurationSec: uint64(deployBuild),
+ StartedDate: *deployBuild.StartedDate,
+ FinishedDate: deployBuild.FinishedDate,
+ }
+
+ domainTask.Type = devops.DEPLOYMENT
+ domainTask.Environment =
regexEnricher.GetEnrichResult(productionPattern,
deployBuild.DeploymentVersionName, devops.PRODUCTION)
+
+ return []interface{}{
+ domainTask,
+ }, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/backend/plugins/bamboo/tasks/deploy_extractor.go
b/backend/plugins/bamboo/tasks/deploy_build_extractor.go
similarity index 53%
copy from backend/plugins/bamboo/tasks/deploy_extractor.go
copy to backend/plugins/bamboo/tasks/deploy_build_extractor.go
index 84ca83c66..385d97cc8 100644
--- a/backend/plugins/bamboo/tasks/deploy_extractor.go
+++ b/backend/plugins/bamboo/tasks/deploy_build_extractor.go
@@ -20,73 +20,39 @@ package tasks
import (
"encoding/json"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bamboo/models"
)
-var _ plugin.SubTaskEntryPoint = ExtractDeploy
-
-func ExtractDeploy(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_DEPLOY_TABLE)
-
- db := taskCtx.GetDal()
- clauses := []dal.Clause{
- dal.Select("plan_key"),
- dal.From(models.BambooPlan{}.TableName()),
- dal.Where("project_key = ? and connection_id=?",
data.Options.ProjectKey, data.Options.ConnectionId),
- }
- cursor, err := db.Cursor(
- clauses...,
- )
- if err != nil {
- return err
- }
- defer cursor.Close()
-
- Plans := make(map[string]bool)
-
- for cursor.Next() {
- Plan := &models.BambooPlan{}
- err = db.Fetch(cursor, Plan)
- if err != nil {
- return err
- }
- Plans[Plan.PlanKey] = true
- }
+var _ plugin.SubTaskEntryPoint = ExtractDeployBuild
+func ExtractDeployBuild(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_DEPLOY_BUILD_TABLE)
+ //repoMap := getRepoMap(data.Options.BambooTransformationRule.RepoMap)
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
+
Extract: func(resData *helper.RawData) ([]interface{},
errors.Error) {
- res := &models.ApiBambooDeployProject{}
+ res := &models.ApiBambooDeployBuild{}
err := errors.Convert(json.Unmarshal(resData.Data, res))
if err != nil {
return nil, err
}
- plan := &SimplePlan{}
- err = errors.Convert(json.Unmarshal(resData.Input,
plan))
+ input := &InputForEnv{}
+ err = errors.Convert(json.Unmarshal(resData.Input,
input))
if err != nil {
return nil, err
}
- results := make([]interface{}, 0, len(res.Environments))
-
- if Plans[res.PlanKey.Key] {
- for _, env := range res.Environments {
- body :=
&models.BambooDeployEnvironment{}
-
- body.Convert(&env)
- body.ConnectionId =
data.Options.ConnectionId
- body.PlanKey = res.PlanKey.Key
-
- results = append(results, body)
- }
- }
+ build := res.Convert(data.Options)
+ build.PlanKey = input.PlanKey
- return results, nil
+ return []interface{}{
+ build,
+ }, nil
},
})
if err != nil {
@@ -96,10 +62,10 @@ func ExtractDeploy(taskCtx plugin.SubTaskContext)
errors.Error {
return extractor.Execute()
}
-var ExtractDeployMeta = plugin.SubTaskMeta{
- Name: "ExtractDeploy",
- EntryPoint: ExtractDeploy,
+var ExtractDeployBuildMeta = plugin.SubTaskMeta{
+ Name: "ExtractDeployBuild",
+ EntryPoint: ExtractDeployBuild,
EnabledByDefault: true,
- Description: "Extract raw data into tool layer table
_tool_bamboo_deploy_environment",
+ Description: "Extract raw data into tool layer table
bamboo_plan_builds",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}
diff --git a/backend/plugins/bamboo/tasks/deploy_extractor.go
b/backend/plugins/bamboo/tasks/deploy_extractor.go
index 84ca83c66..f02e51726 100644
--- a/backend/plugins/bamboo/tasks/deploy_extractor.go
+++ b/backend/plugins/bamboo/tasks/deploy_extractor.go
@@ -80,6 +80,7 @@ func ExtractDeploy(taskCtx plugin.SubTaskContext)
errors.Error {
body.Convert(&env)
body.ConnectionId =
data.Options.ConnectionId
+ body.ProjectKey =
data.Options.ProjectKey
body.PlanKey = res.PlanKey.Key
results = append(results, body)
diff --git a/backend/plugins/bamboo/tasks/shared.go
b/backend/plugins/bamboo/tasks/shared.go
index 20bc6cdc1..277b324fa 100644
--- a/backend/plugins/bamboo/tasks/shared.go
+++ b/backend/plugins/bamboo/tasks/shared.go
@@ -28,9 +28,9 @@ import (
func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, rawTable string)
(*api.RawDataSubTaskArgs, *BambooTaskData) {
data := taskCtx.GetData().(*BambooTaskData)
filteredData := *data
- filteredData.Options = &BambooOptions{}
+ filteredData.Options = &models.BambooOptions{}
*filteredData.Options = *data.Options
- var params = BambooApiParams{
+ var params = models.BambooApiParams{
ConnectionId: data.Options.ConnectionId,
ProjectKey: data.Options.ProjectKey,
}
diff --git a/backend/plugins/bamboo/tasks/task_data.go
b/backend/plugins/bamboo/tasks/task_data.go
index 1abee2795..6859edd3e 100644
--- a/backend/plugins/bamboo/tasks/task_data.go
+++ b/backend/plugins/bamboo/tasks/task_data.go
@@ -23,30 +23,13 @@ import (
"github.com/apache/incubator-devlake/plugins/bamboo/models"
)
-type BambooApiParams struct {
- ConnectionId uint64 `json:"connectionId"`
- ProjectKey string
-}
-
-type BambooOptions struct {
- // TODO add some custom options here if necessary
- // options means some custom params required by plugin running.
- // Such As How many rows do your want
- // You can use it in sub tasks and you need pass it in main.go and
pipelines.
- ConnectionId uint64 `json:"connectionId"`
- ProjectKey string `json:"projectKey"`
- Tasks []string `json:"tasks,omitempty"`
- TransformationRuleId uint64
`mapstructure:"transformationRuleId" json:"transformationRuleId"`
- *models.BambooTransformationRule `mapstructure:"transformationRules"
json:"transformationRules"`
-}
-
type BambooTaskData struct {
- Options *BambooOptions
+ Options *models.BambooOptions
ApiClient *helper.ApiAsyncClient
}
-func DecodeAndValidateTaskOptions(options map[string]interface{})
(*BambooOptions, errors.Error) {
- var op BambooOptions
+func DecodeAndValidateTaskOptions(options map[string]interface{})
(*models.BambooOptions, errors.Error) {
+ var op models.BambooOptions
if err := helper.Decode(options, &op, nil); err != nil {
return nil, err
}
