This is an automated email from the ASF dual-hosted git repository.

abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new d979b154 refactor: migration scripts of AE, Azure, bitbucket (#3509)
d979b154 is described below

commit d979b15483a6eb5bb832abfa313503274813d24a
Author: mindlesscloud <[email protected]>
AuthorDate: Thu Oct 20 19:01:24 2022 +0800

    refactor: migration scripts of AE, Azure, bitbucket (#3509)
---
 plugins/ae/impl/impl.go                            |  7 +--
 .../migrationscripts/20220714_add_init_tables.go   | 46 +++++++-------
 plugins/ae/models/migrationscripts/register.go     |  8 +--
 plugins/azure/impl/impl.go                         |  6 +-
 .../migrationscripts/20220825_add_init_tables.go   | 28 +++------
 plugins/azure/models/migrationscripts/register.go  |  8 +--
 plugins/bitbucket/impl/impl.go                     |  5 +-
 .../migrationscripts/20220803_add_init_tables.go   | 70 +++-------------------
 .../20220914_add_pipeline_tables.go                |  7 +--
 .../20221013_add_deployment_tables.go              | 11 +---
 ...4_add_RepoId_CommitSha_field_pipeline_tables.go | 42 ++++---------
 .../bitbucket/models/migrationscripts/register.go  | 10 ++--
 12 files changed, 77 insertions(+), 171 deletions(-)

diff --git a/plugins/ae/impl/impl.go b/plugins/ae/impl/impl.go
index ec97da43..da5c9bfd 100644
--- a/plugins/ae/impl/impl.go
+++ b/plugins/ae/impl/impl.go
@@ -19,9 +19,8 @@ package impl
 
 import (
        "fmt"
-       "github.com/apache/incubator-devlake/errors"
 
-       "github.com/apache/incubator-devlake/migration"
+       "github.com/apache/incubator-devlake/errors"
        "github.com/apache/incubator-devlake/plugins/ae/api"
        "github.com/apache/incubator-devlake/plugins/ae/models"
        "github.com/apache/incubator-devlake/plugins/ae/models/migrationscripts"
@@ -36,7 +35,7 @@ var _ core.PluginMeta = (*AE)(nil)
 var _ core.PluginInit = (*AE)(nil)
 var _ core.PluginTask = (*AE)(nil)
 var _ core.PluginApi = (*AE)(nil)
-var _ core.Migratable = (*AE)(nil)
+var _ core.PluginMigration = (*AE)(nil)
 var _ core.CloseablePluginTask = (*AE)(nil)
 
 type AE struct{}
@@ -100,7 +99,7 @@ func (plugin AE) RootPkgPath() string {
        return "github.com/apache/incubator-devlake/plugins/ae"
 }
 
-func (plugin AE) MigrationScripts() []migration.Script {
+func (plugin AE) MigrationScripts() []core.MigrationScript {
        return migrationscripts.All()
 }
 
diff --git a/plugins/ae/models/migrationscripts/20220714_add_init_tables.go 
b/plugins/ae/models/migrationscripts/20220714_add_init_tables.go
index cc11ec8d..1ae21f7e 100644
--- a/plugins/ae/models/migrationscripts/20220714_add_init_tables.go
+++ b/plugins/ae/models/migrationscripts/20220714_add_init_tables.go
@@ -18,68 +18,66 @@ limitations under the License.
 package migrationscripts
 
 import (
-       "context"
+       "github.com/apache/incubator-devlake/config"
        "github.com/apache/incubator-devlake/errors"
-
+       "github.com/apache/incubator-devlake/helpers/migrationhelper"
        
"github.com/apache/incubator-devlake/plugins/ae/models/migrationscripts/archived"
        "github.com/apache/incubator-devlake/plugins/core"
        "github.com/apache/incubator-devlake/plugins/helper"
-       "gorm.io/gorm"
 )
 
-type addInitTables struct {
-       config core.ConfigGetter
-}
-
-func (u *addInitTables) SetConfigGetter(config core.ConfigGetter) {
-       u.config = config
-}
+type addInitTables20220714 struct{}
 
-func (u *addInitTables) Up(ctx context.Context, db *gorm.DB) errors.Error {
-       err := db.Migrator().DropTable(
+func (u *addInitTables20220714) Up(basicRes core.BasicRes) errors.Error {
+       err := basicRes.GetDal().DropTables(
                &archived.AECommit{},
                &archived.AEProject{},
                "_raw_ae_project",
                "_raw_ae_commits",
        )
        if err != nil {
-               return errors.Convert(err)
+               return err
        }
 
-       err = db.Migrator().AutoMigrate(
+       err = migrationhelper.AutoMigrateTables(
+               basicRes,
                &archived.AECommit{},
                &archived.AEProject{},
                &archived.AeConnection{},
        )
        if err != nil {
-               return errors.Convert(err)
+               return err
        }
 
-       encodeKey := u.config.GetString(core.EncodeKeyEnvStr)
+       c := config.GetConfig()
+       encodeKey := c.GetString(core.EncodeKeyEnvStr)
        connection := &archived.AeConnection{}
-       connection.Endpoint = u.config.GetString("AE_ENDPOINT")
-       connection.Proxy = u.config.GetString("AE_PROXY")
-       connection.SecretKey = u.config.GetString("AE_SECRET_KEY")
-       connection.AppId = u.config.GetString("AE_APP_ID")
+       connection.Endpoint = c.GetString("AE_ENDPOINT")
+       connection.Proxy = c.GetString("AE_PROXY")
+       connection.SecretKey = c.GetString("AE_SECRET_KEY")
+       connection.AppId = c.GetString("AE_APP_ID")
        connection.Name = "AE"
        if connection.Endpoint != "" && connection.AppId != "" && 
connection.SecretKey != "" && encodeKey != "" {
                err = helper.UpdateEncryptFields(connection, func(plaintext 
string) (string, errors.Error) {
                        return core.Encrypt(encodeKey, plaintext)
                })
                if err != nil {
-                       return errors.Default.Wrap(err, "error encrypting 
fields")
+                       return err
                }
                // update from .env and save to db
-               db.Create(connection)
+               err = basicRes.GetDal().Create(connection)
+               if err != nil {
+                       return err
+               }
        }
 
        return nil
 }
 
-func (*addInitTables) Version() uint64 {
+func (*addInitTables20220714) Version() uint64 {
        return 20220714201133
 }
 
-func (*addInitTables) Name() string {
+func (*addInitTables20220714) Name() string {
        return "AE init schemas"
 }
diff --git a/plugins/ae/models/migrationscripts/register.go 
b/plugins/ae/models/migrationscripts/register.go
index c1365f7d..f5e9d97a 100644
--- a/plugins/ae/models/migrationscripts/register.go
+++ b/plugins/ae/models/migrationscripts/register.go
@@ -18,12 +18,12 @@ limitations under the License.
 package migrationscripts
 
 import (
-       "github.com/apache/incubator-devlake/migration"
+       "github.com/apache/incubator-devlake/plugins/core"
 )
 
 // All return all the migration scripts
-func All() []migration.Script {
-       return []migration.Script{
-               new(addInitTables),
+func All() []core.MigrationScript {
+       return []core.MigrationScript{
+               new(addInitTables20220714),
        }
 }
diff --git a/plugins/azure/impl/impl.go b/plugins/azure/impl/impl.go
index 0f306dc8..5b55d488 100644
--- a/plugins/azure/impl/impl.go
+++ b/plugins/azure/impl/impl.go
@@ -19,15 +19,14 @@ package impl
 
 import (
        "fmt"
+
        "github.com/apache/incubator-devlake/errors"
-       "github.com/apache/incubator-devlake/migration"
        "github.com/apache/incubator-devlake/plugins/azure/api"
        "github.com/apache/incubator-devlake/plugins/azure/models"
        
"github.com/apache/incubator-devlake/plugins/azure/models/migrationscripts"
        "github.com/apache/incubator-devlake/plugins/azure/tasks"
        "github.com/apache/incubator-devlake/plugins/core"
        "github.com/apache/incubator-devlake/plugins/helper"
-
        "github.com/spf13/viper"
        "gorm.io/gorm"
 )
@@ -38,6 +37,7 @@ var _ core.PluginInit = (*Azure)(nil)
 var _ core.PluginTask = (*Azure)(nil)
 var _ core.PluginApi = (*Azure)(nil)
 var _ core.CloseablePluginTask = (*Azure)(nil)
+var _ core.PluginMigration = (*Azure)(nil)
 
 // PluginEntry exports for Framework to search and load
 var PluginEntry Azure //nolint
@@ -117,7 +117,7 @@ func (plugin Azure) ApiResources() 
map[string]map[string]core.ApiResourceHandler
        }
 }
 
-func (plugin Azure) MigrationScripts() []migration.Script {
+func (plugin Azure) MigrationScripts() []core.MigrationScript {
        return migrationscripts.All()
 }
 
diff --git a/plugins/azure/models/migrationscripts/20220825_add_init_tables.go 
b/plugins/azure/models/migrationscripts/20220825_add_init_tables.go
index 47f80dea..af1ef0df 100644
--- a/plugins/azure/models/migrationscripts/20220825_add_init_tables.go
+++ b/plugins/azure/models/migrationscripts/20220825_add_init_tables.go
@@ -18,37 +18,27 @@ limitations under the License.
 package migrationscripts
 
 import (
-       "context"
        "github.com/apache/incubator-devlake/errors"
-
+       "github.com/apache/incubator-devlake/helpers/migrationhelper"
        
"github.com/apache/incubator-devlake/plugins/azure/models/migrationscripts/archived"
-       "gorm.io/gorm"
+       "github.com/apache/incubator-devlake/plugins/core"
 )
 
-type addInitTables struct{}
+type addInitTables20220825 struct{}
 
-func (*addInitTables) Up(ctx context.Context, db *gorm.DB) errors.Error {
-       if !db.Migrator().HasTable(&archived.AzureConnection{}) {
-               err := db.Migrator().AutoMigrate(&archived.AzureConnection{})
-               if err != nil {
-                       return errors.Convert(err)
-               }
-       }
-       err := db.Migrator().AutoMigrate(
+func (*addInitTables20220825) Up(basicRes core.BasicRes) errors.Error {
+       return migrationhelper.AutoMigrateTables(
+               basicRes,
+               &archived.AzureConnection{},
                &archived.AzureRepo{},
                &archived.AzureBuildDefinition{},
        )
-       if err != nil {
-               return errors.Convert(err)
-       }
-
-       return nil
 }
 
-func (*addInitTables) Version() uint64 {
+func (*addInitTables20220825) Version() uint64 {
        return 20220825231237
 }
 
-func (*addInitTables) Name() string {
+func (*addInitTables20220825) Name() string {
        return "Azure init schemas"
 }
diff --git a/plugins/azure/models/migrationscripts/register.go 
b/plugins/azure/models/migrationscripts/register.go
index c1365f7d..8e64d44b 100644
--- a/plugins/azure/models/migrationscripts/register.go
+++ b/plugins/azure/models/migrationscripts/register.go
@@ -18,12 +18,12 @@ limitations under the License.
 package migrationscripts
 
 import (
-       "github.com/apache/incubator-devlake/migration"
+       "github.com/apache/incubator-devlake/plugins/core"
 )
 
 // All return all the migration scripts
-func All() []migration.Script {
-       return []migration.Script{
-               new(addInitTables),
+func All() []core.MigrationScript {
+       return []core.MigrationScript{
+               new(addInitTables20220825),
        }
 }
diff --git a/plugins/bitbucket/impl/impl.go b/plugins/bitbucket/impl/impl.go
index 8619687a..9f23a181 100644
--- a/plugins/bitbucket/impl/impl.go
+++ b/plugins/bitbucket/impl/impl.go
@@ -19,7 +19,6 @@ package impl
 
 import (
        "github.com/apache/incubator-devlake/errors"
-       "github.com/apache/incubator-devlake/migration"
        "github.com/apache/incubator-devlake/plugins/bitbucket/api"
        "github.com/apache/incubator-devlake/plugins/bitbucket/models"
        
"github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts"
@@ -34,7 +33,7 @@ var _ core.PluginMeta = (*Bitbucket)(nil)
 var _ core.PluginInit = (*Bitbucket)(nil)
 var _ core.PluginTask = (*Bitbucket)(nil)
 var _ core.PluginApi = (*Bitbucket)(nil)
-var _ core.Migratable = (*Bitbucket)(nil)
+var _ core.PluginMigration = (*Bitbucket)(nil)
 var _ core.PluginBlueprintV100 = (*Bitbucket)(nil)
 
 type Bitbucket string
@@ -105,7 +104,7 @@ func (plugin Bitbucket) RootPkgPath() string {
        return "github.com/apache/incubator-devlake/plugins/bitbucket"
 }
 
-func (plugin Bitbucket) MigrationScripts() []migration.Script {
+func (plugin Bitbucket) MigrationScripts() []core.MigrationScript {
        return migrationscripts.All()
 }
 
diff --git 
a/plugins/bitbucket/models/migrationscripts/20220803_add_init_tables.go 
b/plugins/bitbucket/models/migrationscripts/20220803_add_init_tables.go
index ca8b59d6..a722ac76 100644
--- a/plugins/bitbucket/models/migrationscripts/20220803_add_init_tables.go
+++ b/plugins/bitbucket/models/migrationscripts/20220803_add_init_tables.go
@@ -18,20 +18,16 @@ limitations under the License.
 package migrationscripts
 
 import (
-       "context"
-       "encoding/base64"
-       "github.com/apache/incubator-devlake/config"
        "github.com/apache/incubator-devlake/errors"
+       "github.com/apache/incubator-devlake/helpers/migrationhelper"
        
"github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts/archived"
        "github.com/apache/incubator-devlake/plugins/core"
-       "gorm.io/gorm"
-       "strings"
 )
 
-type addInitTables struct{}
+type addInitTables20220803 struct{}
 
-func (*addInitTables) Up(ctx context.Context, db *gorm.DB) errors.Error {
-       err := db.Migrator().DropTable(
+func (script *addInitTables20220803) Up(basicRes core.BasicRes) errors.Error {
+       err := basicRes.GetDal().DropTables(
                //history table
                &archived.BitbucketRepo{},
                &archived.BitbucketRepoCommit{},
@@ -43,10 +39,11 @@ func (*addInitTables) Up(ctx context.Context, db *gorm.DB) 
errors.Error {
                &archived.BitbucketIssueComment{},
        )
        if err != nil {
-               return errors.Convert(err)
+               return err
        }
 
-       err = db.Migrator().AutoMigrate(
+       return migrationhelper.AutoMigrateTables(
+               basicRes,
                &archived.BitbucketRepo{},
                &archived.BitbucketRepoCommit{},
                &archived.BitbucketConnection{},
@@ -57,61 +54,12 @@ func (*addInitTables) Up(ctx context.Context, db *gorm.DB) 
errors.Error {
                &archived.BitbucketPrComment{},
                &archived.BitbucketIssueComment{},
        )
-       if err != nil {
-               return errors.Convert(err)
-       }
-
-       var result *gorm.DB
-       var bitbucketConns []archived.BitbucketConnection
-       result = db.Find(&bitbucketConns)
-       if result.Error != nil {
-               return errors.Convert(result.Error)
-       }
-
-       for _, v := range bitbucketConns {
-               conn := &archived.BitbucketConnection{}
-               conn.ID = v.ID
-               conn.Name = v.Name
-               conn.Endpoint = v.Endpoint
-               conn.Proxy = v.Proxy
-               conn.RateLimitPerHour = v.RateLimitPerHour
-
-               c := config.GetConfig()
-               encKey := c.GetString(core.EncodeKeyEnvStr)
-               if encKey == "" {
-                       return errors.BadInput.New("bitbucket invalid encKey")
-               }
-               var auth string
-               if auth, err = core.Decrypt(encKey, 
v.BasicAuth.GetEncodedToken()); err != nil {
-                       return errors.Convert(err)
-               }
-               var pk []byte
-               pk, err = base64.StdEncoding.DecodeString(auth)
-               if err != nil {
-                       return errors.Default.Wrap(err, "error creating 
connection entry for BitBucket")
-               }
-               originInfo := strings.Split(string(pk), ":")
-               if len(originInfo) == 2 {
-                       conn.Username = originInfo[0]
-                       conn.Password, err = core.Encrypt(encKey, originInfo[1])
-                       if err != nil {
-                               return errors.Convert(err)
-                       }
-                       // create
-                       tx := db.Create(&conn)
-                       if tx.Error != nil {
-                               return errors.Default.Wrap(tx.Error, "error 
adding connection to DB")
-                       }
-               }
-       }
-
-       return nil
 }
 
-func (*addInitTables) Version() uint64 {
+func (*addInitTables20220803) Version() uint64 {
        return 20220803220824
 }
 
-func (*addInitTables) Name() string {
+func (*addInitTables20220803) Name() string {
        return "Bitbucket init schema 20220803"
 }
diff --git 
a/plugins/bitbucket/models/migrationscripts/20220914_add_pipeline_tables.go 
b/plugins/bitbucket/models/migrationscripts/20220914_add_pipeline_tables.go
index 45db9377..d9a40069 100644
--- a/plugins/bitbucket/models/migrationscripts/20220914_add_pipeline_tables.go
+++ b/plugins/bitbucket/models/migrationscripts/20220914_add_pipeline_tables.go
@@ -18,16 +18,15 @@ limitations under the License.
 package migrationscripts
 
 import (
-       "context"
        "github.com/apache/incubator-devlake/errors"
        
"github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts/archived"
-       "gorm.io/gorm"
+       "github.com/apache/incubator-devlake/plugins/core"
 )
 
 type addPipeline20220914 struct{}
 
-func (*addPipeline20220914) Up(ctx context.Context, db *gorm.DB) errors.Error {
-       err := db.Migrator().AutoMigrate(&archived.BitbucketPipeline{})
+func (*addPipeline20220914) Up(basicRes core.BasicRes) errors.Error {
+       err := basicRes.GetDal().AutoMigrate(&archived.BitbucketPipeline{})
        if err != nil {
                return errors.Convert(err)
        }
diff --git 
a/plugins/bitbucket/models/migrationscripts/20221013_add_deployment_tables.go 
b/plugins/bitbucket/models/migrationscripts/20221013_add_deployment_tables.go
index 11ab6d7b..10a158a6 100644
--- 
a/plugins/bitbucket/models/migrationscripts/20221013_add_deployment_tables.go
+++ 
b/plugins/bitbucket/models/migrationscripts/20221013_add_deployment_tables.go
@@ -18,20 +18,15 @@ limitations under the License.
 package migrationscripts
 
 import (
-       "context"
        "github.com/apache/incubator-devlake/errors"
        
"github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts/archived"
-       "gorm.io/gorm"
+       "github.com/apache/incubator-devlake/plugins/core"
 )
 
 type addDeployment20221013 struct{}
 
-func (*addDeployment20221013) Up(ctx context.Context, db *gorm.DB) 
errors.Error {
-       err := db.Migrator().AutoMigrate(&archived.BitbucketDeployment{})
-       if err != nil {
-               return errors.Convert(err)
-       }
-       return nil
+func (*addDeployment20221013) Up(basicRes core.BasicRes) errors.Error {
+       return basicRes.GetDal().AutoMigrate(&archived.BitbucketDeployment{})
 }
 
 func (*addDeployment20221013) Version() uint64 {
diff --git 
a/plugins/bitbucket/models/migrationscripts/20221014_add_RepoId_CommitSha_field_pipeline_tables.go
 
b/plugins/bitbucket/models/migrationscripts/20221014_add_RepoId_CommitSha_field_pipeline_tables.go
index 9d520fb1..e9eea34f 100644
--- 
a/plugins/bitbucket/models/migrationscripts/20221014_add_RepoId_CommitSha_field_pipeline_tables.go
+++ 
b/plugins/bitbucket/models/migrationscripts/20221014_add_RepoId_CommitSha_field_pipeline_tables.go
@@ -18,51 +18,29 @@ limitations under the License.
 package migrationscripts
 
 import (
-       "context"
-       "time"
-
        "github.com/apache/incubator-devlake/errors"
-       "github.com/apache/incubator-devlake/models/migrationscripts/archived"
-       "gorm.io/gorm"
+       "github.com/apache/incubator-devlake/plugins/core"
 )
 
-type Task20221014 struct {
-       ConnectionId        uint64 `gorm:"primaryKey"`
-       BitbucketId         string `gorm:"primaryKey"`
-       Status              string `gorm:"type:varchar(100)"`
-       Result              string `gorm:"type:varchar(100)"`
-       RefName             string `gorm:"type:varchar(255)"`
-       RepoId              string `gorm:"type:varchar(255)"`
-       CommitSha           string `gorm:"type:varchar(255)"`
-       WebUrl              string `gorm:"type:varchar(255)"`
-       DurationInSeconds   uint64
-       BitbucketCreatedOn  *time.Time
-       BitbucketCompleteOn *time.Time
-       archived.NoPKModel
+type bitbucketPipeline20221014 struct {
+       RepoId    string `gorm:"type:varchar(255)"`
+       CommitSha string `gorm:"type:varchar(255)"`
 }
 
-func (Task20221014) TableName() string {
+func (bitbucketPipeline20221014) TableName() string {
        return "_tool_bitbucket_pipelines"
 }
 
-type addRepoIdAndCommitShaField struct{}
+type addRepoIdAndCommitShaField20221014 struct{}
 
-func (*addRepoIdAndCommitShaField) Up(ctx context.Context, db *gorm.DB) 
errors.Error {
-       err := db.Migrator().AddColumn(Task20221014{}, "repo_id")
-       if err != nil {
-               return errors.Convert(err)
-       }
-       err = db.Migrator().AddColumn(Task20221014{}, "commit_sha")
-       if err != nil {
-               return errors.Convert(err)
-       }
-       return nil
+func (*addRepoIdAndCommitShaField20221014) Up(basicRes core.BasicRes) 
errors.Error {
+       return basicRes.GetDal().AutoMigrate(&bitbucketPipeline20221014{})
 }
 
-func (*addRepoIdAndCommitShaField) Version() uint64 {
+func (*addRepoIdAndCommitShaField20221014) Version() uint64 {
        return 20221014114623
 }
 
-func (*addRepoIdAndCommitShaField) Name() string {
+func (*addRepoIdAndCommitShaField20221014) Name() string {
        return "add column `repo_id` and `commit_sha` at 
_tool_bitbucket_pipelines"
 }
diff --git a/plugins/bitbucket/models/migrationscripts/register.go 
b/plugins/bitbucket/models/migrationscripts/register.go
index 758a4e73..d6d7fc8e 100644
--- a/plugins/bitbucket/models/migrationscripts/register.go
+++ b/plugins/bitbucket/models/migrationscripts/register.go
@@ -18,15 +18,15 @@ limitations under the License.
 package migrationscripts
 
 import (
-       "github.com/apache/incubator-devlake/migration"
+       "github.com/apache/incubator-devlake/plugins/core"
 )
 
 // All return all the migration scripts
-func All() []migration.Script {
-       return []migration.Script{
-               new(addInitTables),
+func All() []core.MigrationScript {
+       return []core.MigrationScript{
+               new(addInitTables20220803),
                new(addPipeline20220914),
                new(addDeployment20221013),
-               new(addRepoIdAndCommitShaField),
+               new(addRepoIdAndCommitShaField20221014),
        }
 }

Reply via email to