This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 06347adc Db Migration confirmation (#2500)
06347adc is described below
commit 06347adc64f4ae44158da0f186191f92e2a0792c
Author: Klesh Wong <[email protected]>
AuthorDate: Fri Jul 15 15:35:16 2022 +0800
Db Migration confirmation (#2500)
* refactor: remove swag from `make dev`
* feat: migration confirmation
* feat: execute migration withut confirmation when FORCE_MIGRATION=true
---
Makefile | 2 +-
api/api.go | 33 +++++++++++++++++++++++++++++++++
main.go | 2 --
migration/migrator.go | 49 +++++++++++++++++++++++++++----------------------
services/init.go | 27 +++++++++++++++++++++++----
5 files changed, 84 insertions(+), 29 deletions(-)
diff --git a/Makefile b/Makefile
index 7fc86054..b685adff 100644
--- a/Makefile
+++ b/Makefile
@@ -48,7 +48,7 @@ swag:
swag init --parseDependency --parseInternal -o ./api/docs -g
./api/api.go -g plugins/*/api/*.go
echo "visit the swagger document on
http://localhost:8080/swagger/index.html";
-dev: build-plugin swag run
+dev: build-plugin run
debug: build-plugin-debug
dlv debug main.go
diff --git a/api/api.go b/api/api.go
index 26443e45..2d1994fb 100644
--- a/api/api.go
+++ b/api/api.go
@@ -18,12 +18,16 @@ limitations under the License.
package api
import (
+ "fmt"
+ "net/http"
"time"
_ "github.com/apache/incubator-devlake/api/docs"
+ "github.com/apache/incubator-devlake/api/shared"
"github.com/apache/incubator-devlake/logger"
"github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/services"
"github.com/gin-contrib/cors"
"github.com/gin-gonic/gin"
ginSwagger "github.com/swaggo/gin-swagger"
@@ -38,10 +42,39 @@ import (
// @host localhost:8080
// @BasePath /
func CreateApiService() {
+ services.Init()
v := config.GetConfig()
gin.SetMode(v.GetString("MODE"))
router := gin.Default()
+ // Wait for user confirmation if db migration is needed
+ router.GET("/proceed-db-migration", func(ctx *gin.Context) {
+ if !services.MigrationRequireConfirmation() {
+ shared.ApiOutputError(ctx, fmt.Errorf("no pending
migration"), http.StatusBadRequest)
+ return
+ }
+ err := services.ExecuteMigration()
+ if err != nil {
+ shared.ApiOutputError(ctx, err, http.StatusBadRequest)
+ return
+ }
+ shared.ApiOutputSuccess(ctx, nil, http.StatusOK)
+ })
+ router.Use(func(ctx *gin.Context) {
+ if !services.MigrationRequireConfirmation() {
+ return
+ }
+ shared.ApiOutputError(
+ ctx,
+ fmt.Errorf("Database migration is required for Apache
DevLake to function properly, it might cause the "+
+ "collected data gets wiped out for consistency.
Please send a request to `/proceed-migrations` "+
+ "if it is ok, or you may downgrade back to the
older version you previous used"),
+ http.StatusPreconditionRequired,
+ )
+ ctx.Abort()
+ return
+ })
+
router.GET("/swagger/*any",
ginSwagger.WrapHandler(swaggerFiles.Handler))
//endpoint debug log
diff --git a/main.go b/main.go
index 2ffe728d..7c3001f7 100644
--- a/main.go
+++ b/main.go
@@ -21,7 +21,6 @@ import (
"github.com/apache/incubator-devlake/api"
"github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/services"
_ "github.com/apache/incubator-devlake/version"
)
@@ -37,6 +36,5 @@ func main() {
panic(err)
}
}
- services.Init()
api.CreateApiService()
}
diff --git a/migration/migrator.go b/migration/migrator.go
index b143284a..ce7eae0f 100644
--- a/migration/migrator.go
+++ b/migration/migrator.go
@@ -25,7 +25,7 @@ import (
"sync"
)
-var m = migrator{scripts: make(map[string]scriptWithComment)}
+var m = migrator{}
type scriptWithComment struct {
Script
@@ -34,25 +34,37 @@ type scriptWithComment struct {
type migrator struct {
sync.Mutex
db *gorm.DB
- scripts map[string]scriptWithComment
+ executed map[string]bool
+ scripts []*scriptWithComment
+ pending []*scriptWithComment
}
func Init(db *gorm.DB) {
m.db = db
+ var err error
+ m.executed, err = m.getExecuted()
+ if err != nil {
+ panic(err)
+ }
}
func (m *migrator) register(scripts []Script, comment string) {
m.Lock()
defer m.Unlock()
for _, script := range scripts {
- m.scripts[fmt.Sprintf("%s:%d", script.Name(),
script.Version())] = scriptWithComment{
+ key := fmt.Sprintf("%s:%d", script.Name(), script.Version())
+ swc := &scriptWithComment{
Script: script,
comment: comment,
}
+ m.scripts = append(m.scripts, swc)
+ if !m.executed[key] {
+ m.pending = append(m.pending, swc)
+ }
}
}
-func (m *migrator) bookKeep(script scriptWithComment) error {
+func (m *migrator) bookKeep(script *scriptWithComment) error {
record := &MigrationHistory{
ScriptVersion: script.Version(),
ScriptName: script.Name(),
@@ -62,22 +74,11 @@ func (m *migrator) bookKeep(script scriptWithComment) error
{
}
func (m *migrator) execute(ctx context.Context) error {
- versions, err := m.getExecuted()
- if err != nil {
- return err
- }
- for key := range versions {
- delete(m.scripts, key)
- }
- var scriptSlice []scriptWithComment
- for _, script := range m.scripts {
- scriptSlice = append(scriptSlice, script)
- }
- sort.Slice(scriptSlice, func(i, j int) bool {
- return scriptSlice[i].Version() < scriptSlice[j].Version()
+ sort.Slice(m.pending, func(i, j int) bool {
+ return m.pending[i].Version() < m.pending[j].Version()
})
- for _, script := range scriptSlice {
- err = script.Up(ctx, m.db)
+ for _, script := range m.pending {
+ err := script.Up(ctx, m.db)
if err != nil {
return err
}
@@ -88,9 +89,9 @@ func (m *migrator) execute(ctx context.Context) error {
}
return nil
}
-func (m *migrator) getExecuted() (map[string]struct{}, error) {
+func (m *migrator) getExecuted() (map[string]bool, error) {
var err error
- versions := make(map[string]struct{})
+ versions := make(map[string]bool)
err = m.db.Migrator().AutoMigrate(&MigrationHistory{})
if err != nil {
return nil, err
@@ -101,7 +102,7 @@ func (m *migrator) getExecuted() (map[string]struct{},
error) {
return nil, err
}
for _, record := range records {
- versions[fmt.Sprintf("%s:%d", record.ScriptName,
record.ScriptVersion)] = struct{}{}
+ versions[fmt.Sprintf("%s:%d", record.ScriptName,
record.ScriptVersion)] = true
}
return versions, nil
}
@@ -113,3 +114,7 @@ func Register(scripts []Script, comment string) {
func Execute(ctx context.Context) error {
return m.execute(ctx)
}
+
+func NeedConfirmation() bool {
+ return len(m.executed) > 0 && len(m.pending) > 0
+}
diff --git a/services/init.go b/services/init.go
index 072eda7d..be3610db 100644
--- a/services/init.go
+++ b/services/init.go
@@ -38,8 +38,9 @@ var cfg *viper.Viper
var db *gorm.DB
var cronManager *cron.Cron
var log core.Logger
+var migrationRequireConfirmation bool
-// Init FIXME ...
+// Init the services module
func Init() {
var err error
cfg = config.GetConfig()
@@ -62,11 +63,29 @@ func Init() {
if err != nil {
panic(err)
}
- err = migration.Execute(context.Background())
- if err != nil {
- panic(err)
+ forceMigration := cfg.GetBool("FORCE_MIGRATION")
+ if !migration.NeedConfirmation() || forceMigration {
+ err = ExecuteMigration()
+ if err != nil {
+ panic(err)
+ }
+ } else {
+ migrationRequireConfirmation = true
}
+ log.Info("Db migration confirmation needed: %v",
migrationRequireConfirmation)
+}
+func ExecuteMigration() error {
+ err := migration.Execute(context.Background())
+ if err != nil {
+ return err
+ }
// call service init
pipelineServiceInit()
+ migrationRequireConfirmation = false
+ return nil
+}
+
+func MigrationRequireConfirmation() bool {
+ return migrationRequireConfirmation
}
