This is an automated email from the ASF dual-hosted git repository.
zky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new a7a14df66 [feat-5598]: v0.18 Migrations for _raw_data_table and
_raw_data_params (#5606)
a7a14df66 is described below
commit a7a14df669ff8f3951267f5781e79a1829ce6873
Author: Keon Amini <[email protected]>
AuthorDate: Mon Jul 10 04:52:51 2023 -0500
[feat-5598]: v0.18 Migrations for _raw_data_table and _raw_data_params
(#5606)
* feat: added common plugin migrations + migration for _raw_data_table of
tools models
* feat: migrations for domain-scope models
* refactor: plugins manually perform migrations
* fix: fix migration handling of postgres for azure
* refactor: explicit domain scope migrations
* feat: migrations for raw_data_params added
* fix: plugin should take care their own business
* fix: linting
---------
Co-authored-by: Klesh Wong <[email protected]>
---
.../20230630_add_raw_param_table_for_scopes.go | 81 +++++++++++++++++++++
.../bamboo/models/migrationscripts/register.go | 1 +
.../20230630_add_raw_param_table_for_scopes.go | 81 +++++++++++++++++++++
.../bitbucket/models/migrationscripts/register.go | 1 +
.../20230630_add_raw_param_table_for_scopes.go | 82 ++++++++++++++++++++++
.../github/models/migrationscripts/register.go | 1 +
.../20230630_add_raw_param_table_for_scopes.go | 81 +++++++++++++++++++++
.../gitlab/models/migrationscripts/register.go | 1 +
.../20230630_add_raw_param_table_for_scopes.go | 81 +++++++++++++++++++++
.../jenkins/models/migrationscripts/register.go | 1 +
.../20230630_add_raw_param_table_for_scopes.go | 81 +++++++++++++++++++++
.../jira/models/migrationscripts/register.go | 1 +
.../20230630_add_raw_param_table_for_scopes.go | 81 +++++++++++++++++++++
.../pagerduty/models/migrationscripts/register.go | 1 +
.../20230630_add_raw_param_table_for_scopes.go | 81 +++++++++++++++++++++
.../sonarqube/models/migrationscripts/register.go | 1 +
.../20230630_add_raw_param_table_for_scopes.go | 81 +++++++++++++++++++++
.../tapd/models/migrationscripts/register.go | 1 +
.../20230630_add_raw_param_table_for_scopes.go | 81 +++++++++++++++++++++
.../trello/models/migrationscripts/register.go | 1 +
...30630_add_raw_param_table_for_project_scopes.go | 81 +++++++++++++++++++++
.../zentao/models/migrationscripts/register.go | 1 +
.../plugins/azuredevops/azuredevops/models.py | 7 +-
backend/python/test/fakeplugin/fakeplugin/main.py | 11 ++-
backend/server/services/init.go | 4 +-
backend/test/helper/client.go | 2 +-
26 files changed, 922 insertions(+), 5 deletions(-)
diff --git
a/backend/plugins/bamboo/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/bamboo/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
new file mode 100644
index 000000000..5eb151949
--- /dev/null
+++
b/backend/plugins/bamboo/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addRawParamTableForScope)(nil)
+
+type scope20230630 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ProjectKey string `gorm:"primaryKey"`
+ RawDataTable string `gorm:"column:_raw_data_table"`
+ RawDataParams string `gorm:"column:_raw_data_params"`
+}
+
+func (scope20230630) TableName() string {
+ return "_tool_bamboo_projects"
+}
+
+type params20230630 struct {
+ ConnectionId uint64
+ ProjectKey string
+}
+
+type addRawParamTableForScope struct{}
+
+func (script *addRawParamTableForScope) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+ return migrationhelper.CopyTableColumns(basicRes,
+ scope20230630{}.TableName(),
+ scope20230630{}.TableName(),
+ func(src *scope20230630) (*scope20230630, errors.Error) {
+ src.RawDataTable = "_raw_bamboo_scopes"
+ src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
+ ConnectionId: src.ConnectionId,
+ ProjectKey: src.ProjectKey,
+ })))
+ updateSet := []dal.DalSet{
+ {ColumnName: "_raw_data_table", Value:
src.RawDataTable},
+ {ColumnName: "_raw_data_params", Value:
src.RawDataParams},
+ }
+ where := dal.Where("id = ?",
fmt.Sprintf("bamboo:BambooProject:%v:%v", src.ConnectionId, src.ProjectKey))
+ errors.Must(db.UpdateColumns("repos", updateSet, where))
+ errors.Must(db.UpdateColumns("boards", updateSet,
where))
+ errors.Must(db.UpdateColumns("cicd_scopes", updateSet,
where))
+ errors.Must(db.UpdateColumns("cq_projects", updateSet,
where))
+ return src, nil
+ })
+}
+
+func (*addRawParamTableForScope) Version() uint64 {
+ return 20230630000001
+}
+
+func (script *addRawParamTableForScope) Name() string {
+ return "populated _raw_data columns for bamboo projects"
+}
diff --git a/backend/plugins/bamboo/models/migrationscripts/register.go
b/backend/plugins/bamboo/models/migrationscripts/register.go
index a8b234e29..b526b2ce6 100644
--- a/backend/plugins/bamboo/models/migrationscripts/register.go
+++ b/backend/plugins/bamboo/models/migrationscripts/register.go
@@ -28,5 +28,6 @@ func All() []plugin.MigrationScript {
new(addConnectionIdToTransformationRule),
new(addTypeAndEnvironment),
new(renameTr2ScopeConfig),
+ new(addRawParamTableForScope),
}
}
diff --git
a/backend/plugins/bitbucket/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/bitbucket/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
new file mode 100644
index 000000000..ac562e65c
--- /dev/null
+++
b/backend/plugins/bitbucket/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addRawParamTableForScope)(nil)
+
+type scope20230630 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ BitbucketId string `gorm:"primaryKey"`
+ RawDataTable string `gorm:"column:_raw_data_table"`
+ RawDataParams string `gorm:"column:_raw_data_params"`
+}
+
+func (scope20230630) TableName() string {
+ return "_tool_bitbucket_repos"
+}
+
+type params20230630 struct {
+ ConnectionId uint64
+ FullName string
+}
+
+type addRawParamTableForScope struct{}
+
+func (script *addRawParamTableForScope) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+ return migrationhelper.CopyTableColumns(basicRes,
+ scope20230630{}.TableName(),
+ scope20230630{}.TableName(),
+ func(src *scope20230630) (*scope20230630, errors.Error) {
+ src.RawDataTable = "_raw_bitbucket_scopes"
+ src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
+ ConnectionId: src.ConnectionId,
+ FullName: src.BitbucketId,
+ })))
+ updateSet := []dal.DalSet{
+ {ColumnName: "_raw_data_table", Value:
src.RawDataTable},
+ {ColumnName: "_raw_data_params", Value:
src.RawDataParams},
+ }
+ where := dal.Where("id = ?",
fmt.Sprintf("bitbucket:BitbucketRepo:%v:%v", src.ConnectionId, src.BitbucketId))
+ errors.Must(db.UpdateColumns("repos", updateSet, where))
+ errors.Must(db.UpdateColumns("boards", updateSet,
where))
+ errors.Must(db.UpdateColumns("cicd_scopes", updateSet,
where))
+ errors.Must(db.UpdateColumns("cq_projects", updateSet,
where))
+ return src, nil
+ })
+}
+
+func (*addRawParamTableForScope) Version() uint64 {
+ return 20230630000001
+}
+
+func (script *addRawParamTableForScope) Name() string {
+ return "populated _raw_data columns for bitbucket repos"
+}
diff --git a/backend/plugins/bitbucket/models/migrationscripts/register.go
b/backend/plugins/bitbucket/models/migrationscripts/register.go
index cfaccffe0..f7d84de0a 100644
--- a/backend/plugins/bitbucket/models/migrationscripts/register.go
+++ b/backend/plugins/bitbucket/models/migrationscripts/register.go
@@ -37,5 +37,6 @@ func All() []plugin.MigrationScript {
new(addRepoIdToPr),
new(addBitbucketCommitAuthorInfo),
new(renameTr2ScopeConfig),
+ new(addRawParamTableForScope),
}
}
diff --git
a/backend/plugins/github/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/github/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
new file mode 100644
index 000000000..73d96402b
--- /dev/null
+++
b/backend/plugins/github/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -0,0 +1,82 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addRawParamTableForScope)(nil)
+
+type scope20230630 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ GithubId int `gorm:"primaryKey"`
+ Name string
+ RawDataTable string `gorm:"column:_raw_data_table"`
+ RawDataParams string `gorm:"column:_raw_data_params"`
+}
+
+func (scope20230630) TableName() string {
+ return "_tool_github_repos"
+}
+
+type params20230630 struct {
+ ConnectionId uint64
+ Name string
+}
+
+type addRawParamTableForScope struct{}
+
+func (script *addRawParamTableForScope) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+ return migrationhelper.CopyTableColumns(basicRes,
+ scope20230630{}.TableName(),
+ scope20230630{}.TableName(),
+ func(src *scope20230630) (*scope20230630, errors.Error) {
+ src.RawDataTable = "_raw_github_scopes"
+ src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
+ ConnectionId: src.ConnectionId,
+ Name: src.Name,
+ })))
+ updateSet := []dal.DalSet{
+ {ColumnName: "_raw_data_table", Value:
src.RawDataTable},
+ {ColumnName: "_raw_data_params", Value:
src.RawDataParams},
+ }
+ where := dal.Where("id = ?",
fmt.Sprintf("github:GithubRepo:%v:%v", src.ConnectionId, src.GithubId))
+ errors.Must(db.UpdateColumns("repos", updateSet, where))
+ errors.Must(db.UpdateColumns("boards", updateSet,
where))
+ errors.Must(db.UpdateColumns("cicd_scopes", updateSet,
where))
+ errors.Must(db.UpdateColumns("cq_projects", updateSet,
where))
+ return src, nil
+ })
+}
+
+func (*addRawParamTableForScope) Version() uint64 {
+ return 20230630000001
+}
+
+func (script *addRawParamTableForScope) Name() string {
+ return "populated _raw_data columns for github repos"
+}
diff --git a/backend/plugins/github/models/migrationscripts/register.go
b/backend/plugins/github/models/migrationscripts/register.go
index cb216fad1..f574e72e8 100644
--- a/backend/plugins/github/models/migrationscripts/register.go
+++ b/backend/plugins/github/models/migrationscripts/register.go
@@ -42,5 +42,6 @@ func All() []plugin.MigrationScript {
new(renameTr2ScopeConfig),
new(addGithubIssueAssignee),
new(addFullName),
+ new(addRawParamTableForScope),
}
}
diff --git
a/backend/plugins/gitlab/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/gitlab/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
new file mode 100644
index 000000000..e7c07715e
--- /dev/null
+++
b/backend/plugins/gitlab/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addRawParamTableForScope)(nil)
+
+type scope20230630 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ GitlabId string `gorm:"primaryKey"`
+ RawDataTable string `gorm:"column:_raw_data_table"`
+ RawDataParams string `gorm:"column:_raw_data_params"`
+}
+
+func (scope20230630) TableName() string {
+ return "_tool_gitlab_projects"
+}
+
+type params20230630 struct {
+ ConnectionId uint64
+ ProjectId string
+}
+
+type addRawParamTableForScope struct{}
+
+func (script *addRawParamTableForScope) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+ return migrationhelper.CopyTableColumns(basicRes,
+ scope20230630{}.TableName(),
+ scope20230630{}.TableName(),
+ func(src *scope20230630) (*scope20230630, errors.Error) {
+ src.RawDataTable = "_raw_gitlab_scopes"
+ src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
+ ConnectionId: src.ConnectionId,
+ ProjectId: src.GitlabId,
+ })))
+ updateSet := []dal.DalSet{
+ {ColumnName: "_raw_data_table", Value:
src.RawDataTable},
+ {ColumnName: "_raw_data_params", Value:
src.RawDataParams},
+ }
+ where := dal.Where("id = ?",
fmt.Sprintf("gitlab:GitlabProject:%v:%v", src.ConnectionId, src.GitlabId))
+ errors.Must(db.UpdateColumns("repos", updateSet, where))
+ errors.Must(db.UpdateColumns("boards", updateSet,
where))
+ errors.Must(db.UpdateColumns("cicd_scopes", updateSet,
where))
+ errors.Must(db.UpdateColumns("cq_projects", updateSet,
where))
+ return src, nil
+ })
+}
+
+func (*addRawParamTableForScope) Version() uint64 {
+ return 20230630000001
+}
+
+func (script *addRawParamTableForScope) Name() string {
+ return "populated _raw_data columns for gitlab projects"
+}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go
b/backend/plugins/gitlab/models/migrationscripts/register.go
index eba26b3ac..f12be1883 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/register.go
@@ -38,5 +38,6 @@ func All() []plugin.MigrationScript {
new(renameTr2ScopeConfig),
new(addGitlabIssueAssignee),
new(addMrCommitSha),
+ new(addRawParamTableForScope),
}
}
diff --git
a/backend/plugins/jenkins/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/jenkins/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
new file mode 100644
index 000000000..0670e2025
--- /dev/null
+++
b/backend/plugins/jenkins/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addRawParamTableForScope)(nil)
+
+type scope20230630 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ FullName string `gorm:"primaryKey"`
+ RawDataTable string `gorm:"column:_raw_data_table"`
+ RawDataParams string `gorm:"column:_raw_data_params"`
+}
+
+func (scope20230630) TableName() string {
+ return "_tool_jenkins_jobs"
+}
+
+type params20230630 struct {
+ ConnectionId uint64
+ FullName string
+}
+
+type addRawParamTableForScope struct{}
+
+func (script *addRawParamTableForScope) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+ return migrationhelper.CopyTableColumns(basicRes,
+ scope20230630{}.TableName(),
+ scope20230630{}.TableName(),
+ func(src *scope20230630) (*scope20230630, errors.Error) {
+ src.RawDataTable = "_raw_jenkins_scopes"
+ src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
+ ConnectionId: src.ConnectionId,
+ FullName: src.FullName,
+ })))
+ updateSet := []dal.DalSet{
+ {ColumnName: "_raw_data_table", Value:
src.RawDataTable},
+ {ColumnName: "_raw_data_params", Value:
src.RawDataParams},
+ }
+ where := dal.Where("id = ?",
fmt.Sprintf("jenkins:JenkinsJob:%v:%v", src.ConnectionId, src.FullName))
+ errors.Must(db.UpdateColumns("repos", updateSet, where))
+ errors.Must(db.UpdateColumns("boards", updateSet,
where))
+ errors.Must(db.UpdateColumns("cicd_scopes", updateSet,
where))
+ errors.Must(db.UpdateColumns("cq_projects", updateSet,
where))
+ return src, nil
+ })
+}
+
+func (*addRawParamTableForScope) Version() uint64 {
+ return 20230630000001
+}
+
+func (script *addRawParamTableForScope) Name() string {
+ return "populated _raw_data columns for jenkins jobs"
+}
diff --git a/backend/plugins/jenkins/models/migrationscripts/register.go
b/backend/plugins/jenkins/models/migrationscripts/register.go
index b75403117..ab7a63c66 100644
--- a/backend/plugins/jenkins/models/migrationscripts/register.go
+++ b/backend/plugins/jenkins/models/migrationscripts/register.go
@@ -34,5 +34,6 @@ func All() []plugin.MigrationScript {
new(addFullNameForBuilds),
new(addConnectionIdToTransformationRule),
new(renameTr2ScopeConfig),
+ new(addRawParamTableForScope),
}
}
diff --git
a/backend/plugins/jira/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/jira/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
new file mode 100644
index 000000000..bc1a9b0b4
--- /dev/null
+++
b/backend/plugins/jira/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addRawParamTableForScope)(nil)
+
+type scope20230630 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ BoardId string `gorm:"primaryKey"`
+ RawDataTable string `gorm:"column:_raw_data_table"`
+ RawDataParams string `gorm:"column:_raw_data_params"`
+}
+
+func (scope20230630) TableName() string {
+ return "_tool_jira_boards"
+}
+
+type params20230630 struct {
+ ConnectionId uint64
+ BoardId string
+}
+
+type addRawParamTableForScope struct{}
+
+func (script *addRawParamTableForScope) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+ return migrationhelper.CopyTableColumns(basicRes,
+ scope20230630{}.TableName(),
+ scope20230630{}.TableName(),
+ func(src *scope20230630) (*scope20230630, errors.Error) {
+ src.RawDataTable = "_raw_jira_scopes"
+ src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
+ ConnectionId: src.ConnectionId,
+ BoardId: src.BoardId,
+ })))
+ updateSet := []dal.DalSet{
+ {ColumnName: "_raw_data_table", Value:
src.RawDataTable},
+ {ColumnName: "_raw_data_params", Value:
src.RawDataParams},
+ }
+ where := dal.Where("id = ?",
fmt.Sprintf("jira:JiraBoard:%v:%v", src.ConnectionId, src.BoardId))
+ errors.Must(db.UpdateColumns("repos", updateSet, where))
+ errors.Must(db.UpdateColumns("boards", updateSet,
where))
+ errors.Must(db.UpdateColumns("cicd_scopes", updateSet,
where))
+ errors.Must(db.UpdateColumns("cq_projects", updateSet,
where))
+ return src, nil
+ })
+}
+
+func (*addRawParamTableForScope) Version() uint64 {
+ return 20230630000001
+}
+
+func (script *addRawParamTableForScope) Name() string {
+ return "populated _raw_data columns for jira boards"
+}
diff --git a/backend/plugins/jira/models/migrationscripts/register.go
b/backend/plugins/jira/models/migrationscripts/register.go
index 56a882e60..c0c9ed9a2 100644
--- a/backend/plugins/jira/models/migrationscripts/register.go
+++ b/backend/plugins/jira/models/migrationscripts/register.go
@@ -41,5 +41,6 @@ func All() []plugin.MigrationScript {
new(addRepoUrl),
new(addApplicationType),
new(clearRepoPattern),
+ new(addRawParamTableForScope),
}
}
diff --git
a/backend/plugins/pagerduty/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/pagerduty/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
new file mode 100644
index 000000000..4ed1da62d
--- /dev/null
+++
b/backend/plugins/pagerduty/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addRawParamTableForScope)(nil)
+
+type scope20230630 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Id string `gorm:"primaryKey"`
+ RawDataTable string `gorm:"column:_raw_data_table"`
+ RawDataParams string `gorm:"column:_raw_data_params"`
+}
+
+func (scope20230630) TableName() string {
+ return "_tool_pagerduty_services"
+}
+
+type params20230630 struct {
+ ConnectionId uint64
+ ScopeId string
+}
+
+type addRawParamTableForScope struct{}
+
+func (script *addRawParamTableForScope) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+ return migrationhelper.CopyTableColumns(basicRes,
+ scope20230630{}.TableName(),
+ scope20230630{}.TableName(),
+ func(src *scope20230630) (*scope20230630, errors.Error) {
+ src.RawDataTable = "_raw_pagerduty_scopes"
+ src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
+ ConnectionId: src.ConnectionId,
+ ScopeId: src.Id,
+ })))
+ updateSet := []dal.DalSet{
+ {ColumnName: "_raw_data_table", Value:
src.RawDataTable},
+ {ColumnName: "_raw_data_params", Value:
src.RawDataParams},
+ }
+ where := dal.Where("id = ?",
fmt.Sprintf("pagerduty:Service:%v:%v", src.ConnectionId, src.Id))
+ errors.Must(db.UpdateColumns("repos", updateSet, where))
+ errors.Must(db.UpdateColumns("boards", updateSet,
where))
+ errors.Must(db.UpdateColumns("cicd_scopes", updateSet,
where))
+ errors.Must(db.UpdateColumns("cq_projects", updateSet,
where))
+ return src, nil
+ })
+}
+
+func (*addRawParamTableForScope) Version() uint64 {
+ return 20230630000001
+}
+
+func (script *addRawParamTableForScope) Name() string {
+ return "populated _raw_data columns for pagerduty services"
+}
diff --git a/backend/plugins/pagerduty/models/migrationscripts/register.go
b/backend/plugins/pagerduty/models/migrationscripts/register.go
index 9d6d0c74b..e54a2dd52 100644
--- a/backend/plugins/pagerduty/models/migrationscripts/register.go
+++ b/backend/plugins/pagerduty/models/migrationscripts/register.go
@@ -30,5 +30,6 @@ func All() []plugin.MigrationScript {
new(addTransformationRulesToService20230303),
new(renameTr2ScopeConfig),
new(removeScopeConfig),
+ new(addRawParamTableForScope),
}
}
diff --git
a/backend/plugins/sonarqube/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/sonarqube/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
new file mode 100644
index 000000000..621393045
--- /dev/null
+++
b/backend/plugins/sonarqube/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addRawParamTableForScope)(nil)
+
+type scope20230630 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ProjectKey string `gorm:"primaryKey"`
+ RawDataTable string `gorm:"column:_raw_data_table"`
+ RawDataParams string `gorm:"column:_raw_data_params"`
+}
+
+func (scope20230630) TableName() string {
+ return "_tool_sonarqube_projects"
+}
+
+type params20230630 struct {
+ ConnectionId uint64
+ ProjectKey string
+}
+
+type addRawParamTableForScope struct{}
+
+func (script *addRawParamTableForScope) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+ return migrationhelper.CopyTableColumns(basicRes,
+ scope20230630{}.TableName(),
+ scope20230630{}.TableName(),
+ func(src *scope20230630) (*scope20230630, errors.Error) {
+ src.RawDataTable = "_raw_sonarqube_scopes"
+ src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
+ ConnectionId: src.ConnectionId,
+ ProjectKey: src.ProjectKey,
+ })))
+ updateSet := []dal.DalSet{
+ {ColumnName: "_raw_data_table", Value:
src.RawDataTable},
+ {ColumnName: "_raw_data_params", Value:
src.RawDataParams},
+ }
+ where := dal.Where("id = ?",
fmt.Sprintf("sonarqube:SonarqubeProject:%v:%v", src.ConnectionId,
src.ProjectKey))
+ errors.Must(db.UpdateColumns("repos", updateSet, where))
+ errors.Must(db.UpdateColumns("boards", updateSet,
where))
+ errors.Must(db.UpdateColumns("cicd_scopes", updateSet,
where))
+ errors.Must(db.UpdateColumns("cq_projects", updateSet,
where))
+ return src, nil
+ })
+}
+
+func (*addRawParamTableForScope) Version() uint64 {
+ return 20230630000001
+}
+
+func (script *addRawParamTableForScope) Name() string {
+ return "populated _raw_data columns for sonarqube projects"
+}
diff --git a/backend/plugins/sonarqube/models/migrationscripts/register.go
b/backend/plugins/sonarqube/models/migrationscripts/register.go
index cbae40708..1f2200763 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/register.go
+++ b/backend/plugins/sonarqube/models/migrationscripts/register.go
@@ -25,5 +25,6 @@ func All() []plugin.MigrationScript {
new(addInitTables),
new(modifyCharacterSet),
new(expandProjectKey20230206),
+ new(addRawParamTableForScope),
}
}
diff --git
a/backend/plugins/tapd/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/tapd/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
new file mode 100644
index 000000000..7b18e2c8c
--- /dev/null
+++
b/backend/plugins/tapd/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addRawParamTableForScope)(nil)
+
+type scope20230630 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Id string `gorm:"primaryKey"`
+ RawDataTable string `gorm:"column:_raw_data_table"`
+ RawDataParams string `gorm:"column:_raw_data_params"`
+}
+
+func (scope20230630) TableName() string {
+ return "_tool_tapd_workspaces"
+}
+
+type params20230630 struct {
+ ConnectionId uint64
+ WorkspaceId string
+}
+
+type addRawParamTableForScope struct{}
+
+func (script *addRawParamTableForScope) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+ return migrationhelper.CopyTableColumns(basicRes,
+ scope20230630{}.TableName(),
+ scope20230630{}.TableName(),
+ func(src *scope20230630) (*scope20230630, errors.Error) {
+ src.RawDataTable = "_raw_tapd_scopes"
+ src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
+ ConnectionId: src.ConnectionId,
+ WorkspaceId: src.Id,
+ })))
+ updateSet := []dal.DalSet{
+ {ColumnName: "_raw_data_table", Value:
src.RawDataTable},
+ {ColumnName: "_raw_data_params", Value:
src.RawDataParams},
+ }
+ where := dal.Where("id = ?",
fmt.Sprintf("tapd:TapdWorkspace:%v:%v", src.ConnectionId, src.Id))
+ errors.Must(db.UpdateColumns("repos", updateSet, where))
+ errors.Must(db.UpdateColumns("boards", updateSet,
where))
+ errors.Must(db.UpdateColumns("cicd_scopes", updateSet,
where))
+ errors.Must(db.UpdateColumns("cq_projects", updateSet,
where))
+ return src, nil
+ })
+}
+
+func (*addRawParamTableForScope) Version() uint64 {
+ return 20230630000001
+}
+
+func (script *addRawParamTableForScope) Name() string {
+ return "populated _raw_data columns for tapd workspaces"
+}
diff --git a/backend/plugins/tapd/models/migrationscripts/register.go
b/backend/plugins/tapd/models/migrationscripts/register.go
index 749dd3d47..c2a99a547 100644
--- a/backend/plugins/tapd/models/migrationscripts/register.go
+++ b/backend/plugins/tapd/models/migrationscripts/register.go
@@ -31,5 +31,6 @@ func All() []plugin.MigrationScript {
new(modifyCustomFieldName),
new(addCustomFieldValue),
new(renameTr2ScopeConfig),
+ new(addRawParamTableForScope),
}
}
diff --git
a/backend/plugins/trello/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/trello/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
new file mode 100644
index 000000000..f08453725
--- /dev/null
+++
b/backend/plugins/trello/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addRawParamTableForScope)(nil)
+
+type scope20230630 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ BoardId string `gorm:"primaryKey"`
+ RawDataTable string `gorm:"column:_raw_data_table"`
+ RawDataParams string `gorm:"column:_raw_data_params"`
+}
+
+func (scope20230630) TableName() string {
+ return "_tool_trello_boards"
+}
+
+type params20230630 struct {
+ ConnectionId uint64
+ BoardId string
+}
+
+type addRawParamTableForScope struct{}
+
+func (script *addRawParamTableForScope) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+ return migrationhelper.CopyTableColumns(basicRes,
+ scope20230630{}.TableName(),
+ scope20230630{}.TableName(),
+ func(src *scope20230630) (*scope20230630, errors.Error) {
+ src.RawDataTable = "_raw_trello_scopes"
+ src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
+ ConnectionId: src.ConnectionId,
+ BoardId: src.BoardId,
+ })))
+ updateSet := []dal.DalSet{
+ {ColumnName: "_raw_data_table", Value:
src.RawDataTable},
+ {ColumnName: "_raw_data_params", Value:
src.RawDataParams},
+ }
+ where := dal.Where("id = ?",
fmt.Sprintf("trello:TrelloBoard:%v:%v", src.ConnectionId, src.BoardId))
+ errors.Must(db.UpdateColumns("repos", updateSet, where))
+ errors.Must(db.UpdateColumns("boards", updateSet,
where))
+ errors.Must(db.UpdateColumns("cicd_scopes", updateSet,
where))
+ errors.Must(db.UpdateColumns("cq_projects", updateSet,
where))
+ return src, nil
+ })
+}
+
+func (*addRawParamTableForScope) Version() uint64 {
+ return 20230630000001
+}
+
+func (script *addRawParamTableForScope) Name() string {
+ return "populated _raw_data columns for trello boards"
+}
diff --git a/backend/plugins/trello/models/migrationscripts/register.go
b/backend/plugins/trello/models/migrationscripts/register.go
index 0558090db..e822bbe93 100644
--- a/backend/plugins/trello/models/migrationscripts/register.go
+++ b/backend/plugins/trello/models/migrationscripts/register.go
@@ -27,5 +27,6 @@ func All() []plugin.MigrationScript {
new(addInitTables),
new(addConnectionIdToTransformationRule),
new(renameTr2ScopeConfig),
+ new(addRawParamTableForScope),
}
}
diff --git
a/backend/plugins/zentao/models/migrationscripts/20230630_add_raw_param_table_for_project_scopes.go
b/backend/plugins/zentao/models/migrationscripts/20230630_add_raw_param_table_for_project_scopes.go
new file mode 100644
index 000000000..6b2ce1bbd
--- /dev/null
+++
b/backend/plugins/zentao/models/migrationscripts/20230630_add_raw_param_table_for_project_scopes.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addRawParamTableForScope)(nil)
+
+type scope20230630 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Id string `gorm:"primaryKey"`
+ RawDataTable string `gorm:"column:_raw_data_table"`
+ RawDataParams string `gorm:"column:_raw_data_params"`
+}
+
+func (scope20230630) TableName() string {
+ return "_tool_zentao_projects"
+}
+
+type params20230630 struct {
+ ConnectionId uint64
+ ProjectId string
+}
+
+type addRawParamTableForScope struct{}
+
+func (script *addRawParamTableForScope) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+ return migrationhelper.CopyTableColumns(basicRes,
+ scope20230630{}.TableName(),
+ scope20230630{}.TableName(),
+ func(src *scope20230630) (*scope20230630, errors.Error) {
+ src.RawDataTable = "_raw_zentao_scopes"
+ src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
+ ConnectionId: src.ConnectionId,
+ ProjectId: src.Id,
+ })))
+ updateSet := []dal.DalSet{
+ {ColumnName: "_raw_data_table", Value:
src.RawDataTable},
+ {ColumnName: "_raw_data_params", Value:
src.RawDataParams},
+ }
+ where := dal.Where("id = ?",
fmt.Sprintf("zentao:ZentaoProject:%v:%v", src.ConnectionId, src.Id))
+ errors.Must(db.UpdateColumns("repos", updateSet, where))
+ errors.Must(db.UpdateColumns("boards", updateSet,
where))
+ errors.Must(db.UpdateColumns("cicd_scopes", updateSet,
where))
+ errors.Must(db.UpdateColumns("cq_projects", updateSet,
where))
+ return src, nil
+ })
+}
+
+func (*addRawParamTableForScope) Version() uint64 {
+ return 20230630000001
+}
+
+func (script *addRawParamTableForScope) Name() string {
+ return "populated _raw_data columns for zentao projects"
+}
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go
b/backend/plugins/zentao/models/migrationscripts/register.go
index cdd64f274..4d3e3dc40 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/migrationscripts/register.go
@@ -29,5 +29,6 @@ func All() []plugin.MigrationScript {
new(addIssueRepoCommitsTables),
new(addInitChangelogTables),
new(addTaskLeft),
+ new(addRawParamTableForScope),
}
}
diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py
b/backend/python/plugins/azuredevops/azuredevops/models.py
index 3ce1aebb5..879392e4b 100644
--- a/backend/python/plugins/azuredevops/azuredevops/models.py
+++ b/backend/python/plugins/azuredevops/azuredevops/models.py
@@ -67,7 +67,7 @@ class GitPullRequest(ToolModel, table=True):
target_commit_sha: str = Field(source='/lastMergeTargetCommit/commitId')
merge_commit_sha: Optional[str] = Field(source='/lastMergeCommit/commitId')
url: Optional[str]
- type: Optional[str] = Field(source='/labels/0/name') # TODO: Add regex to
scope config
+ type: Optional[str] = Field(source='/labels/0/name') # TODO: Add regex to
scope config
title: Optional[str]
target_ref_name: Optional[str]
source_ref_name: Optional[str]
@@ -146,3 +146,8 @@ def add_build_id_as_job_primary_key(b:
MigrationScriptBuilder):
def rename_tx_rule_table_to_scope_config(b: MigrationScriptBuilder):
b.rename_table('_tool_azuredevops_azuredevopstransformationrules',
GitRepositoryConfig.__tablename__)
b.add_column(GitRepositoryConfig.__tablename__, 'entities', 'json')
+
+
+@migration(20230630000001, name="populated _raw_data_table column for
azuredevops git repos")
+def add_raw_data_params_table_to_scope(b: MigrationScriptBuilder):
+ b.execute(f'''UPDATE {GitRepository.__tablename__} SET _raw_data_table =
'_raw_azuredevops_scopes' WHERE 1=1''')
diff --git a/backend/python/test/fakeplugin/fakeplugin/main.py
b/backend/python/test/fakeplugin/fakeplugin/main.py
index ad3e625a2..a3e1ab214 100644
--- a/backend/python/test/fakeplugin/fakeplugin/main.py
+++ b/backend/python/test/fakeplugin/fakeplugin/main.py
@@ -20,8 +20,10 @@ import json
from pydantic import SecretStr
-from pydevlake import Plugin, Connection, Stream, ToolModel, ToolScope,
ScopeConfig, RemoteScopeGroup, DomainType, Field, TestConnectionResult
+from pydevlake import Plugin, Connection, Stream, ToolModel, ToolScope,
ScopeConfig, RemoteScopeGroup, DomainType, \
+ Field, TestConnectionResult
from pydevlake.domain_layer.devops import CicdScope, CICDPipeline, CICDStatus,
CICDResult, CICDType
+from pydevlake.migration import migration, MigrationScriptBuilder, Dialect
VALID_TOKEN = "this_is_a_valid_token"
@@ -165,5 +167,12 @@ class FakePlugin(Plugin):
]
+# test migration
+@migration(20230630000001, name="populated _raw_data_table column for
fakeproject")
+def add_raw_data_params_table_to_scope(b: MigrationScriptBuilder):
+ b.execute(f'UPDATE {FakeProject.__tablename__} SET _raw_data_table =
"_raw_fakeproject_scopes" WHERE 1=1', Dialect.MYSQL) #mysql only
+ b.execute(f'''UPDATE {FakeProject.__tablename__} SET _raw_data_table =
'_raw_fakeproject_scopes' WHERE 1=1''', Dialect.POSTGRESQL) #mysql and postgres
+
+
if __name__ == '__main__':
FakePlugin.start()
diff --git a/backend/server/services/init.go b/backend/server/services/init.go
index 2e27a027e..86558ce55 100644
--- a/backend/server/services/init.go
+++ b/backend/server/services/init.go
@@ -91,9 +91,9 @@ func Init() {
}
// pull migration scripts from plugins to migrator
- for pluginName, pluginInst := range plugin.AllPlugins() {
+ for _, pluginInst := range plugin.AllPlugins() {
if migratable, ok := pluginInst.(plugin.PluginMigration); ok {
- migrator.Register(migratable.MigrationScripts(),
pluginName)
+ migrator.Register(migratable.MigrationScripts(),
pluginInst.Name())
}
}
diff --git a/backend/test/helper/client.go b/backend/test/helper/client.go
index d29f70572..229b322f9 100644
--- a/backend/test/helper/client.go
+++ b/backend/test/helper/client.go
@@ -177,7 +177,7 @@ func ConnectLocalServer(t *testing.T, clientConfig
*LocalClientConfig) *DevlakeC
}()
req, err2 := http.NewRequest(http.MethodGet,
fmt.Sprintf("%s/proceed-db-migration", addr), nil)
require.NoError(t, err2)
- d.forceSendHttpRequest(20, req, func(err errors.Error) bool {
+ d.forceSendHttpRequest(100, req, func(err errors.Error) bool {
e := err.Unwrap()
return goerror.Is(e, syscall.ECONNREFUSED)
})
