This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 7b0199a6 refactor: migration helper (#3169)
7b0199a6 is described below
commit 7b0199a6d8557d4f18b73200d619a96d10df0163
Author: mappjzc <[email protected]>
AuthorDate: Sat Oct 8 19:32:20 2022 +0800
refactor: migration helper (#3169)
* refactor: migration helper
Add ReBuildTable
Add ReBuildTableWithOutBak
Nddtfjiang <[email protected]>
* refactor: add param checking
Add paramCheckingForReBuildTable
Add paramCheckingForReBuildTableWithOutBak
Add paramCheckingForReBuildTableShare
Nddtfjiang <[email protected]>
* refactor: rename func
rename to TransformRowsInPlace
rename to TransformRowsBetweenTables
Nddtfjiang <[email protected]>
* refactor: change gorm db to dal
change gorm db to dal.
Nddtfjiang <[email protected]>
---
helpers/migrationhelper/helper.go | 187 ++++++++++++++++++++++++++++++++++++++
impl/dalgorm/dalgorm.go | 21 ++++-
plugins/core/dal/dal.go | 7 +-
3 files changed, 213 insertions(+), 2 deletions(-)
diff --git a/helpers/migrationhelper/helper.go
b/helpers/migrationhelper/helper.go
new file mode 100644
index 00000000..2191a1c9
--- /dev/null
+++ b/helpers/migrationhelper/helper.go
@@ -0,0 +1,187 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationhelper
+
+import (
+ "fmt"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/gitlab/api"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "gorm.io/gorm/schema"
+)
+
+// TransformRowsInPlace method can be used when we need to change the table
structure and reprocess all the data in the table.
+func TransformRowsInPlace(db dal.Dal, src schema.Tabler, bak schema.Tabler,
dst schema.Tabler, callback_transform func(src schema.Tabler) schema.Tabler)
(errs errors.Error) {
+ var err error
+
+ // param cheking
+ errs = paramCheckingForTransformRowsInPlace(db, src, bak, dst,
callback_transform)
+ if errs != nil {
+ return errors.Default.Wrap(errs, "TransformRowsInPlace param
cheking error")
+ }
+
+ // rename the src to bak for cache src table
+ err = db.RenameTable(src, bak)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error no rename
[%s] to [%s]", src.TableName(), bak.TableName()))
+ }
+
+ // rollback for rename back
+ defer func() {
+ if errs != nil {
+ err = db.RenameTable(bak, src)
+ if err != nil {
+ errs = errors.Default.Wrap(err,
fmt.Sprintf("fail to rollback table [%s] , you must to rollback by yourself.
%s", bak.TableName(), err.Error()))
+ }
+ }
+ }()
+
+ return TransformRowsBetweenTables(db, bak, dst, callback_transform)
+}
+
+// TransformRowsBetweenTables method can be used when we need to change the
table structure and reprocess all the data in the table.
+// It request the src table and the dst table with different table name.
+func TransformRowsBetweenTables(db dal.Dal, src schema.Tabler, dst
schema.Tabler, callback_transform func(src schema.Tabler) schema.Tabler) (errs
errors.Error) {
+ var err error
+
+ errs = paramCheckingForTransformRowsBetweenTables(db, src, dst,
callback_transform)
+ if errs != nil {
+ return errors.Default.Wrap(errs, "TransformRowsBetweenTables
param cheking error")
+ }
+
+ // create new commit_files table
+ err = db.AutoMigrate(dst)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error on auto
migrate [%s]", dst.TableName()))
+ }
+
+ // rollback for create new table
+ defer func() {
+ if errs != nil {
+ err = db.DropTable(dst)
+ if err != nil {
+ errs = errors.Default.Wrap(err,
fmt.Sprintf("fail to rollback table [%s] , you must to rollback by yourself.
%s", dst.TableName(), err.Error()))
+ }
+ }
+ }()
+
+ // update src id to dst id and write to the dst table
+ cursor, err := db.Cursor(
+ dal.From(src.TableName()),
+ )
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error on select
[%s]]", src.TableName()))
+ }
+ defer cursor.Close()
+
+ // caculate and save the data to new table
+ batch, err := helper.NewBatchSave(api.BasicRes, reflect.TypeOf(dst),
200)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error getting
batch from table [%s]", dst.TableName()))
+ }
+
+ defer batch.Close()
+ for cursor.Next() {
+ err = db.Fetch(cursor, src)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error scan
rows from table [%s]", src.TableName()))
+ }
+
+ cf := callback_transform(src)
+
+ err = batch.Add(&cf)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error on
[%s] batch add", dst.TableName()))
+ }
+ }
+
+ // drop the src table
+ err = db.DropTable(src)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error no drop
[%s]", src.TableName()))
+ }
+
+ return nil
+}
+
+// paramCheckingForTransformRowsInPlace check the params of
TransformRowsInPlace
+func paramCheckingForTransformRowsInPlace(db dal.Dal, src schema.Tabler, bak
schema.Tabler, dst schema.Tabler, callback_transform func(src schema.Tabler)
schema.Tabler) (errs errors.Error) {
+ errs = paramCheckingShare(db, dst, callback_transform)
+ if errs != nil {
+ return errs
+ }
+
+ if src == nil {
+ return errors.Default.New("can not working with param src nil")
+ }
+
+ if bak == nil {
+ return errors.Default.New("can not working with param bak nil")
+ }
+
+ if dst.TableName() == bak.TableName() {
+ return errors.Default.New(fmt.Sprintf("the bak and dst can not
use the same table name [%s][%s].",
+ bak.TableName(), dst.TableName()))
+ }
+
+ if src.TableName() == bak.TableName() {
+ return errors.Default.New(fmt.Sprintf("the src and bak can not
use the same table name [%s][%s].",
+ src.TableName(), bak.TableName()))
+ }
+
+ return nil
+}
+
+// paramCheckingForTransformRowsBetweenTables check the params of
ReBuildTableWithOutBak
+func paramCheckingForTransformRowsBetweenTables(db dal.Dal, src schema.Tabler,
dst schema.Tabler, callback_transform func(src schema.Tabler) schema.Tabler)
(errs errors.Error) {
+ errs = paramCheckingShare(db, dst, callback_transform)
+ if errs != nil {
+ return errs
+ }
+
+ if src == nil {
+ return errors.Default.New("can not working with param src nil")
+ }
+
+ if src.TableName() == dst.TableName() {
+ return errors.Default.New(fmt.Sprintf("src and dst can not use
the same table name [%s][%s].",
+ src.TableName(), dst.TableName()))
+ }
+
+ return nil
+}
+
+// paramCheckingShare check the Share part params of
TransformRowsBetweenTables and TransformRowsInPlace
+func paramCheckingShare(db dal.Dal, dst schema.Tabler, callback_transform
func(src schema.Tabler) schema.Tabler) (errs errors.Error) {
+ if db == nil {
+ return errors.Default.New("can not working with param db nil")
+ }
+
+ if dst == nil {
+ return errors.Default.New("can not working with param dst nil")
+ }
+
+ if callback_transform == nil {
+ return errors.Default.New("can not working with param
callback_transform nil")
+ }
+
+ return nil
+}
diff --git a/impl/dalgorm/dalgorm.go b/impl/dalgorm/dalgorm.go
index 9d16556b..003c565f 100644
--- a/impl/dalgorm/dalgorm.go
+++ b/impl/dalgorm/dalgorm.go
@@ -19,10 +19,11 @@ package dalgorm
import (
"database/sql"
- "github.com/apache/incubator-devlake/errors"
"reflect"
"strings"
+ "github.com/apache/incubator-devlake/errors"
+
"github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/utils"
"gorm.io/gorm"
@@ -220,3 +221,21 @@ func (d *Dalgorm) AllTables() ([]string, errors.Error) {
func NewDalgorm(db *gorm.DB) *Dalgorm {
return &Dalgorm{db}
}
+
+// RenameTable rename the oldName table to newName
+func (d *Dalgorm) RenameTable(oldName interface{}, newName interface{})
errors.Error {
+ err := d.db.Migrator().RenameTable(oldName, newName)
+ if err != nil {
+ return errors.Default.New(err.Error())
+ }
+ return nil
+}
+
+// DropTable drop the table
+func (d *Dalgorm) DropTable(dst ...interface{}) errors.Error {
+ err := d.db.Migrator().DropTable(dst)
+ if err != nil {
+ return errors.Default.New(err.Error())
+ }
+ return nil
+}
diff --git a/plugins/core/dal/dal.go b/plugins/core/dal/dal.go
index 257eaf5a..01bf9fa6 100644
--- a/plugins/core/dal/dal.go
+++ b/plugins/core/dal/dal.go
@@ -19,9 +19,10 @@ package dal
import (
"database/sql"
- "github.com/apache/incubator-devlake/errors"
"reflect"
+ "github.com/apache/incubator-devlake/errors"
+
"gorm.io/gorm/schema"
)
@@ -88,6 +89,10 @@ type Dal interface {
GetColumns(dst schema.Tabler, filter func(columnMeta ColumnMeta) bool)
(cms []ColumnMeta, err errors.Error)
// GetPrimarykeyFields get the PrimaryKey from `gorm` tag
GetPrimaryKeyFields(t reflect.Type) []reflect.StructField
+ // RenameTable rename the oldName table to newName
+ RenameTable(oldName interface{}, newName interface{}) errors.Error
+ // DropTable drop the table
+ DropTable(dst ...interface{}) errors.Error
}
// GetColumnNames returns table Column Names in database
