This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 9a781dd0a fix: Migration fixes for problems encountered in
v0.18.0-alpha14 (#5660)
9a781dd0a is described below
commit 9a781dd0ad48dc5de22cc64aa9e17a0705d70753
Author: Keon Amini <[email protected]>
AuthorDate: Tue Jul 11 20:19:31 2023 -0500
fix: Migration fixes for problems encountered in v0.18.0-alpha14 (#5660)
* fix: fix incorrect params types in migrations
* fix: add missing migrations for azure
* fix: fixed issue with batch saving only writing the last entry
* refactor: remove noisy debug message
* refactor: bump migration version of old broken migrations from alpha
testing
* fix: use strconv.Atoi for int
---------
Co-authored-by: Klesh Wong <[email protected]>
---
backend/helpers/migrationhelper/migrationhelper.go | 2 +-
.../20230630_add_raw_param_table_for_scopes.go | 2 +-
.../20230630_add_raw_param_table_for_scopes.go | 2 +-
.../20230630_add_raw_param_table_for_scopes.go | 2 +-
.../20230630_add_raw_param_table_for_scopes.go | 7 ++--
.../20230630_add_raw_param_table_for_scopes.go | 2 +-
.../20230630_add_raw_param_table_for_scopes.go | 7 ++--
.../20230630_add_raw_param_table_for_scopes.go | 2 +-
.../20230630_add_raw_param_table_for_scopes.go | 2 +-
.../20230630_add_raw_param_table_for_scopes.go | 7 ++--
.../20230630_add_raw_param_table_for_scopes.go | 2 +-
...30630_add_raw_param_table_for_project_scopes.go | 7 ++--
backend/server/services/pipeline.go | 1 -
.../azuredevops_20230714_add_raw_data_params.go} | 37 ++++++++++-----------
.../remote/models/migrationscripts/register.go | 38 ++++++++++++++++++++++
.../server/services/remote/plugin/plugin_impl.go | 3 +-
16 files changed, 82 insertions(+), 41 deletions(-)
diff --git a/backend/helpers/migrationhelper/migrationhelper.go
b/backend/helpers/migrationhelper/migrationhelper.go
index cb89e8fac..4de96c326 100644
--- a/backend/helpers/migrationhelper/migrationhelper.go
+++ b/backend/helpers/migrationhelper/migrationhelper.go
@@ -270,8 +270,8 @@ func CopyTableColumns[S any, D any](
}
defer batch.Close()
- srcTable := new(S)
for cursor.Next() {
+ srcTable := new(S)
err1 := db.Fetch(cursor, srcTable)
if err1 != nil {
return errors.Default.Wrap(err1, fmt.Sprintf("fail to
load record from table [%s]", srcTableName))
diff --git
a/backend/plugins/bamboo/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/bamboo/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
index 5eb151949..edef3e857 100644
---
a/backend/plugins/bamboo/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
+++
b/backend/plugins/bamboo/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -73,7 +73,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
}
func (*addRawParamTableForScope) Version() uint64 {
- return 20230630000001
+ return 20230630000002
}
func (script *addRawParamTableForScope) Name() string {
diff --git
a/backend/plugins/bitbucket/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/bitbucket/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
index ac562e65c..3be360519 100644
---
a/backend/plugins/bitbucket/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
+++
b/backend/plugins/bitbucket/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -73,7 +73,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
}
func (*addRawParamTableForScope) Version() uint64 {
- return 20230630000001
+ return 20230630000002
}
func (script *addRawParamTableForScope) Name() string {
diff --git
a/backend/plugins/github/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/github/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
index 73d96402b..e8797176b 100644
---
a/backend/plugins/github/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
+++
b/backend/plugins/github/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -74,7 +74,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
}
func (*addRawParamTableForScope) Version() uint64 {
- return 20230630000001
+ return 20230630000002
}
func (script *addRawParamTableForScope) Name() string {
diff --git
a/backend/plugins/gitlab/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/gitlab/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
index e7c07715e..4d4ee1abe 100644
---
a/backend/plugins/gitlab/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
+++
b/backend/plugins/gitlab/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -20,6 +20,7 @@ package migrationscripts
import (
"encoding/json"
"fmt"
+ "strconv"
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
@@ -43,7 +44,7 @@ func (scope20230630) TableName() string {
type params20230630 struct {
ConnectionId uint64
- ProjectId string
+ ProjectId int
}
type addRawParamTableForScope struct{}
@@ -57,7 +58,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
src.RawDataTable = "_raw_gitlab_scopes"
src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
ConnectionId: src.ConnectionId,
- ProjectId: src.GitlabId,
+ ProjectId:
errors.Must1(strconv.Atoi(src.GitlabId)),
})))
updateSet := []dal.DalSet{
{ColumnName: "_raw_data_table", Value:
src.RawDataTable},
@@ -73,7 +74,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
}
func (*addRawParamTableForScope) Version() uint64 {
- return 20230630000001
+ return 20230630000002
}
func (script *addRawParamTableForScope) Name() string {
diff --git
a/backend/plugins/jenkins/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/jenkins/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
index 0670e2025..93fa479c2 100644
---
a/backend/plugins/jenkins/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
+++
b/backend/plugins/jenkins/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -73,7 +73,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
}
func (*addRawParamTableForScope) Version() uint64 {
- return 20230630000001
+ return 20230630000002
}
func (script *addRawParamTableForScope) Name() string {
diff --git
a/backend/plugins/jira/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/jira/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
index bc1a9b0b4..10d27881a 100644
---
a/backend/plugins/jira/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
+++
b/backend/plugins/jira/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -20,6 +20,7 @@ package migrationscripts
import (
"encoding/json"
"fmt"
+ "strconv"
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
@@ -43,7 +44,7 @@ func (scope20230630) TableName() string {
type params20230630 struct {
ConnectionId uint64
- BoardId string
+ BoardId uint64
}
type addRawParamTableForScope struct{}
@@ -57,7 +58,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
src.RawDataTable = "_raw_jira_scopes"
src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
ConnectionId: src.ConnectionId,
- BoardId: src.BoardId,
+ BoardId:
errors.Must1(strconv.ParseUint(src.BoardId, 10, 64)),
})))
updateSet := []dal.DalSet{
{ColumnName: "_raw_data_table", Value:
src.RawDataTable},
@@ -73,7 +74,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
}
func (*addRawParamTableForScope) Version() uint64 {
- return 20230630000001
+ return 20230630000002
}
func (script *addRawParamTableForScope) Name() string {
diff --git
a/backend/plugins/pagerduty/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/pagerduty/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
index 4ed1da62d..3cba2e1f0 100644
---
a/backend/plugins/pagerduty/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
+++
b/backend/plugins/pagerduty/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -73,7 +73,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
}
func (*addRawParamTableForScope) Version() uint64 {
- return 20230630000001
+ return 20230630000002
}
func (script *addRawParamTableForScope) Name() string {
diff --git
a/backend/plugins/sonarqube/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/sonarqube/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
index 621393045..511b7cbe4 100644
---
a/backend/plugins/sonarqube/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
+++
b/backend/plugins/sonarqube/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -73,7 +73,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
}
func (*addRawParamTableForScope) Version() uint64 {
- return 20230630000001
+ return 20230630000002
}
func (script *addRawParamTableForScope) Name() string {
diff --git
a/backend/plugins/tapd/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/tapd/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
index 7b18e2c8c..723301e38 100644
---
a/backend/plugins/tapd/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
+++
b/backend/plugins/tapd/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -20,6 +20,7 @@ package migrationscripts
import (
"encoding/json"
"fmt"
+ "strconv"
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
@@ -43,7 +44,7 @@ func (scope20230630) TableName() string {
type params20230630 struct {
ConnectionId uint64
- WorkspaceId string
+ WorkspaceId uint64
}
type addRawParamTableForScope struct{}
@@ -57,7 +58,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
src.RawDataTable = "_raw_tapd_scopes"
src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
ConnectionId: src.ConnectionId,
- WorkspaceId: src.Id,
+ WorkspaceId:
errors.Must1(strconv.ParseUint(src.Id, 10, 64)),
})))
updateSet := []dal.DalSet{
{ColumnName: "_raw_data_table", Value:
src.RawDataTable},
@@ -73,7 +74,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
}
func (*addRawParamTableForScope) Version() uint64 {
- return 20230630000001
+ return 20230630000002
}
func (script *addRawParamTableForScope) Name() string {
diff --git
a/backend/plugins/trello/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/plugins/trello/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
index f08453725..1d36131f7 100644
---
a/backend/plugins/trello/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
+++
b/backend/plugins/trello/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
@@ -73,7 +73,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
}
func (*addRawParamTableForScope) Version() uint64 {
- return 20230630000001
+ return 20230630000002
}
func (script *addRawParamTableForScope) Name() string {
diff --git
a/backend/plugins/zentao/models/migrationscripts/20230630_add_raw_param_table_for_project_scopes.go
b/backend/plugins/zentao/models/migrationscripts/20230630_add_raw_param_table_for_project_scopes.go
index 6b2ce1bbd..e291ed77c 100644
---
a/backend/plugins/zentao/models/migrationscripts/20230630_add_raw_param_table_for_project_scopes.go
+++
b/backend/plugins/zentao/models/migrationscripts/20230630_add_raw_param_table_for_project_scopes.go
@@ -20,6 +20,7 @@ package migrationscripts
import (
"encoding/json"
"fmt"
+ "strconv"
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
@@ -43,7 +44,7 @@ func (scope20230630) TableName() string {
type params20230630 struct {
ConnectionId uint64
- ProjectId string
+ ProjectId int64
}
type addRawParamTableForScope struct{}
@@ -57,7 +58,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
src.RawDataTable = "_raw_zentao_scopes"
src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
ConnectionId: src.ConnectionId,
- ProjectId: src.Id,
+ ProjectId:
errors.Must1(strconv.ParseInt(src.Id, 10, 64)),
})))
updateSet := []dal.DalSet{
{ColumnName: "_raw_data_table", Value:
src.RawDataTable},
@@ -73,7 +74,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
}
func (*addRawParamTableForScope) Version() uint64 {
- return 20230630000001
+ return 20230630000002
}
func (script *addRawParamTableForScope) Name() string {
diff --git a/backend/server/services/pipeline.go
b/backend/server/services/pipeline.go
index b451cf4dc..fd1b3e81b 100644
--- a/backend/server/services/pipeline.go
+++ b/backend/server/services/pipeline.go
@@ -173,7 +173,6 @@ func dequeuePipeline(runningParallelLabels []string)
(pipeline *models.Pipeline,
defer txHelper.End()
tx := txHelper.Begin()
// mysql read lock, not sure if it works for postgresql
- globalPipelineLog.Debug("acquire lock")
errors.Must(tx.LockTables(map[string]bool{
"_devlake_pipelines": false,
"_devlake_pipeline_labels": false,
diff --git
a/backend/plugins/pagerduty/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
b/backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_add_raw_data_params.go
similarity index 65%
copy from
backend/plugins/pagerduty/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
copy to
backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_add_raw_data_params.go
index 4ed1da62d..aaeb1d0ad 100644
---
a/backend/plugins/pagerduty/models/migrationscripts/20230630_add_raw_param_table_for_scopes.go
+++
b/backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_add_raw_data_params.go
@@ -15,12 +15,11 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package azuredevops
import (
"encoding/json"
"fmt"
-
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -28,34 +27,34 @@ import (
"github.com/apache/incubator-devlake/helpers/migrationhelper"
)
-var _ plugin.MigrationScript = (*addRawParamTableForScope)(nil)
+var _ plugin.MigrationScript = (*AddRawDataForScope)(nil)
-type scope20230630 struct {
+type azureDevopsGitRepositories20230714 struct {
ConnectionId uint64 `gorm:"primaryKey"`
Id string `gorm:"primaryKey"`
RawDataTable string `gorm:"column:_raw_data_table"`
RawDataParams string `gorm:"column:_raw_data_params"`
}
-func (scope20230630) TableName() string {
- return "_tool_pagerduty_services"
+func (azureDevopsGitRepositories20230714) TableName() string {
+ return "_tool_azuredevops_gitrepositories"
}
-type params20230630 struct {
+type rawDataParams20230714 struct {
ConnectionId uint64
ScopeId string
}
-type addRawParamTableForScope struct{}
+type AddRawDataForScope struct{}
-func (script *addRawParamTableForScope) Up(basicRes context.BasicRes)
errors.Error {
+func (script *AddRawDataForScope) Up(basicRes context.BasicRes) errors.Error {
db := basicRes.GetDal()
return migrationhelper.CopyTableColumns(basicRes,
- scope20230630{}.TableName(),
- scope20230630{}.TableName(),
- func(src *scope20230630) (*scope20230630, errors.Error) {
- src.RawDataTable = "_raw_pagerduty_scopes"
- src.RawDataParams =
string(errors.Must1(json.Marshal(¶ms20230630{
+ azureDevopsGitRepositories20230714{}.TableName(),
+ azureDevopsGitRepositories20230714{}.TableName(),
+ func(src *azureDevopsGitRepositories20230714)
(*azureDevopsGitRepositories20230714, errors.Error) {
+ src.RawDataTable = "_raw_azuredevops_scopes"
+ src.RawDataParams =
string(errors.Must1(json.Marshal(&rawDataParams20230714{
ConnectionId: src.ConnectionId,
ScopeId: src.Id,
})))
@@ -63,7 +62,7 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
{ColumnName: "_raw_data_table", Value:
src.RawDataTable},
{ColumnName: "_raw_data_params", Value:
src.RawDataParams},
}
- where := dal.Where("id = ?",
fmt.Sprintf("pagerduty:Service:%v:%v", src.ConnectionId, src.Id))
+ where := dal.Where("id = ?",
fmt.Sprintf("azuredevops:GitRepository:%v:%v", src.ConnectionId, src.Id))
errors.Must(db.UpdateColumns("repos", updateSet, where))
errors.Must(db.UpdateColumns("boards", updateSet,
where))
errors.Must(db.UpdateColumns("cicd_scopes", updateSet,
where))
@@ -72,10 +71,10 @@ func (script *addRawParamTableForScope) Up(basicRes
context.BasicRes) errors.Err
})
}
-func (*addRawParamTableForScope) Version() uint64 {
- return 20230630000001
+func (*AddRawDataForScope) Version() uint64 {
+ return 20230714000001
}
-func (script *addRawParamTableForScope) Name() string {
- return "populated _raw_data columns for pagerduty services"
+func (script *AddRawDataForScope) Name() string {
+ return "populated _raw_data columns for azuredevops"
}
diff --git a/backend/server/services/remote/models/migrationscripts/register.go
b/backend/server/services/remote/models/migrationscripts/register.go
new file mode 100644
index 000000000..af1e9f949
--- /dev/null
+++ b/backend/server/services/remote/models/migrationscripts/register.go
@@ -0,0 +1,38 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/plugin"
+
"github.com/apache/incubator-devlake/server/services/remote/models/migrationscripts/azuredevops"
+)
+
+var allMigrations = map[string][]plugin.MigrationScript{
+ "azuredevops": {
+ &azuredevops.AddRawDataForScope{},
+ },
+}
+
+// All return Go-defined migration scripts of remote plugins. These migrations
are intended for more advanced
+// use cases where they need to be defined in the Go language. Example: a
migration that relies on checking existing data
+func All(pluginName string) []plugin.MigrationScript {
+ if all, ok := allMigrations[pluginName]; ok {
+ return all
+ }
+ return nil
+}
diff --git a/backend/server/services/remote/plugin/plugin_impl.go
b/backend/server/services/remote/plugin/plugin_impl.go
index d6a033794..d76ae1fe9 100644
--- a/backend/server/services/remote/plugin/plugin_impl.go
+++ b/backend/server/services/remote/plugin/plugin_impl.go
@@ -28,6 +28,7 @@ import (
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/server/services/remote/bridge"
"github.com/apache/incubator-devlake/server/services/remote/models"
+
"github.com/apache/incubator-devlake/server/services/remote/models/migrationscripts"
"github.com/apache/incubator-devlake/server/services/remote/plugin/doc"
)
@@ -257,7 +258,7 @@ func (p *remotePluginImpl) OpenApiSpec() string {
}
func (p *remotePluginImpl) MigrationScripts() []plugin.MigrationScript {
- return p.migrationScripts
+ return append(p.migrationScripts, migrationscripts.All(p.name)...)
}
var _ models.RemotePlugin = (*remotePluginImpl)(nil)
