This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 38507a603 CI/Tests: Optimize GitHub workflows tests performance
(#4839)
38507a603 is described below
commit 38507a60305bd76265fa50e52dc9b35b4d6a6ab0
Author: Keon Amini <[email protected]>
AuthorDate: Mon Apr 3 03:15:06 2023 -0500
CI/Tests: Optimize GitHub workflows tests performance (#4839)
* ci: Disable plugin builds from test CI + test cleanup + adjust IT
framework for stability against DB leftovers
* fix: correction to the migration execution cache
---
.github/workflows/test-e2e.yml | 14 +--
backend/Makefile | 8 +-
backend/core/migration/migrator.go | 21 ++--
backend/test/Readme.md | 4 +
backend/test/e2e/api/task/task_test.go | 59 -----------
.../remote}/docker-compose.test.yml | 0
backend/test/{integration => e2e}/remote/helper.go | 16 +--
.../remote/python_plugin_test.go | 4 +-
..._runner_e2e_test.go => pipeline_runner_test.go} | 7 +-
.../services/server_startup_test.go} | 23 ++---
backend/test/{integration => }/helper/api.go | 0
backend/test/{e2e/init.go => helper/bootstrap.go} | 39 ++++----
backend/test/{integration => }/helper/client.go | 110 ++++++++++-----------
.../test/{e2e/init.go => helper/client_factory.go} | 37 +++----
.../test/{integration => }/helper/db_selector.go | 0
backend/test/{utils.go => helper/init.go} | 16 ++-
.../test/{integration => }/helper/json_helper.go | 12 +++
backend/test/{integration => }/helper/models.go | 0
backend/test/{integration => }/helper/utils.go | 0
backend/test/{integration/helper => }/init.go | 20 +---
20 files changed, 164 insertions(+), 226 deletions(-)
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml
index 56806fe58..ac54cd45b 100644
--- a/.github/workflows/test-e2e.yml
+++ b/.github/workflows/test-e2e.yml
@@ -48,13 +48,11 @@ jobs:
- name: Checkout code
uses: actions/checkout@v3
- run: git config --global --add safe.directory $(pwd)
- - name: Build Sources
+ - name: Build Python
run: |
cd backend
echo "Building Python"
make build-python
- echo "Building Go"
- make build
- name: Cache test-e2e
id: cache-test-e2e
uses: actions/cache@v3
@@ -72,9 +70,8 @@ jobs:
run: |
cp .env.example .env
cd backend
- make e2e-test
make e2e-plugins-test
- make integration-test
+ make e2e-test
e2e-postgres:
runs-on: ubuntu-latest
@@ -90,13 +87,11 @@ jobs:
- name: Checkout code
uses: actions/checkout@v3
- run: git config --global --add safe.directory $(pwd)
- - name: Build Sources
+ - name: Build Python
run: |
cd backend
echo "Building Python"
make build-python
- echo "Building Go"
- make build
- name: Cache test-e2e-pg
id: cache-test-e2e-pg
uses: actions/cache@v3
@@ -114,6 +109,5 @@ jobs:
run: |
cp .env.example .env
cd backend
- make e2e-test
make e2e-plugins-test
- make integration-test
+ make e2e-test
diff --git a/backend/Makefile b/backend/Makefile
index 1d0502aca..8b0a4d02b 100644
--- a/backend/Makefile
+++ b/backend/Makefile
@@ -93,10 +93,14 @@ e2e-plugins-test:
export ENV_PATH=$(shell readlink -f .env); set -e; for m in $$(go list
./plugins/... | egrep 'e2e'); do echo $$m; go test -timeout 300s
-gcflags=all=-l -v $$m; done
e2e-test:
- export PLUGIN_DIR=$(shell readlink -f bin/plugins); set -e; for m in
$$(go list ./test/e2e/...); do echo $$m; go test -timeout 300s -v $$m; done
+ export ENV_PATH=$(shell readlink -f .env); set -e;
+ go run ./test/init.go &&\
+ for m in $$(go list ./test/e2e/...); do echo $$m; go test -p 1 -timeout
300s -v $$m; done
integration-test:
- export ENV_PATH=$(shell readlink -f .env); set -e; for m in $$(go list
./test/integration...); do echo $$m; go test -timeout 300s -v $$m; done
+ export ENV_PATH=$(shell readlink -f .env); set -e;
+ go run ./test/init.go &&\
+ for m in $$(go list ./test/integration/...); do echo $$m; go test -p 1
-timeout 300s -v $$m; done
lint:
golangci-lint run
diff --git a/backend/core/migration/migrator.go
b/backend/core/migration/migrator.go
index de825db5a..f30a75b94 100644
--- a/backend/core/migration/migrator.go
+++ b/backend/core/migration/migrator.go
@@ -21,6 +21,7 @@ import (
"fmt"
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
+ core "github.com/apache/incubator-devlake/core/log"
"github.com/apache/incubator-devlake/core/plugin"
"sort"
"sync"
@@ -34,6 +35,7 @@ type scriptWithComment struct {
type migratorImpl struct {
sync.Mutex
basicRes context.BasicRes
+ logger core.Logger
executed map[string]bool
scripts []*scriptWithComment
pending []*scriptWithComment
@@ -54,7 +56,8 @@ func (m *migratorImpl) loadExecuted() errors.Error {
return errors.Default.Wrap(err, "error finding migration
history records")
}
for _, record := range records {
- m.executed[fmt.Sprintf("%s:%d", record.ScriptName,
record.ScriptVersion)] = true
+ scriptId := getScriptId(record.ScriptName, record.ScriptVersion)
+ m.executed[scriptId] = true
}
return nil
}
@@ -64,14 +67,16 @@ func (m *migratorImpl) Register(scripts
[]plugin.MigrationScript, comment string
m.Lock()
defer m.Unlock()
for _, script := range scripts {
- key := fmt.Sprintf("%s:%d", script.Name(), script.Version())
+ scriptId := getScriptId(script.Name(), script.Version())
swc := &scriptWithComment{
script: script,
comment: comment,
}
m.scripts = append(m.scripts, swc)
- if !m.executed[key] {
+ if !m.executed[scriptId] {
m.pending = append(m.pending, swc)
+ } else {
+ m.logger.Debug("skipping previously executed migration
script: %s", scriptId)
}
}
}
@@ -84,10 +89,9 @@ func (m *migratorImpl) Execute() errors.Error {
})
// execute them one by one
db := m.basicRes.GetDal()
- logger := m.basicRes.GetLogger().Nested("migrator")
for _, swc := range m.pending {
- scriptId := fmt.Sprintf("%d-%s", swc.script.Version(),
swc.script.Name())
- logger.Info("applying migratin script %s", scriptId)
+ scriptId := getScriptId(swc.script.Name(), swc.script.Version())
+ m.logger.Info("applying migration script %s", scriptId)
err := swc.script.Up(m.basicRes)
if err != nil {
return err
@@ -116,6 +120,7 @@ func (m *migratorImpl) HasPendingScripts() bool {
func NewMigrator(basicRes context.BasicRes) (plugin.Migrator, errors.Error) {
m := &migratorImpl{
basicRes: basicRes,
+ logger: basicRes.GetLogger().Nested("migrator"),
}
err := m.loadExecuted()
if err != nil {
@@ -123,3 +128,7 @@ func NewMigrator(basicRes context.BasicRes)
(plugin.Migrator, errors.Error) {
}
return m, nil
}
+
+func getScriptId(scriptName string, scriptVersion uint64) string {
+ return fmt.Sprintf("%s:%d", scriptName, scriptVersion)
+}
diff --git a/backend/test/Readme.md b/backend/test/Readme.md
new file mode 100644
index 000000000..03ef32470
--- /dev/null
+++ b/backend/test/Readme.md
@@ -0,0 +1,4 @@
+### Directory structure explanation
+
+* <i>e2e/</i>: Contains DevLake-server tests that interact with either fake
plugins or no plugins at all.
+* <i>integration/</i>: Contains DevLake-server tests written against real
data-sources which contain test data.
diff --git a/backend/test/e2e/api/task/task_test.go
b/backend/test/e2e/api/task/task_test.go
deleted file mode 100644
index 62644e9f8..000000000
--- a/backend/test/e2e/api/task/task_test.go
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package task
-
-import (
- "encoding/json"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/server/api"
- _ "github.com/apache/incubator-devlake/test/e2e"
- "github.com/gin-gonic/gin"
- "github.com/magiconair/properties/assert"
- "net/http"
- "net/http/httptest"
- "strings"
- "testing"
-)
-
-func TestNewTask(t *testing.T) {
- r := gin.Default()
- api.RegisterRouter(r)
-
- w := httptest.NewRecorder()
- params := strings.NewReader(`{"name": "hello", "plan": [[{ "plugin":
"jira", "options": { "host": "www.jira.com" } }]]}`)
- req, _ := http.NewRequest("POST", "/pipelines", params)
- r.ServeHTTP(w, req)
-
- assert.Equal(t, w.Code, http.StatusCreated)
- resp := w.Body.String()
- var pipeline models.Pipeline
- err := errors.Convert(json.Unmarshal([]byte(resp), &pipeline))
- if err != nil {
- t.Fatal(err)
- }
-
- assert.Equal(t, pipeline.Name, "hello")
-
- var tasks [][]*models.NewTask
- err = errors.Convert(json.Unmarshal(pipeline.Plan, &tasks))
- if err != nil {
- t.Fatal(err)
- }
- assert.Equal(t, tasks[0][0].Plugin, "jira")
-}
diff --git a/backend/test/integration/docker-compose.test.yml
b/backend/test/e2e/remote/docker-compose.test.yml
similarity index 100%
rename from backend/test/integration/docker-compose.test.yml
rename to backend/test/e2e/remote/docker-compose.test.yml
diff --git a/backend/test/integration/remote/helper.go
b/backend/test/e2e/remote/helper.go
similarity index 84%
rename from backend/test/integration/remote/helper.go
rename to backend/test/e2e/remote/helper.go
index 98bd78c11..30586c0e6 100644
--- a/backend/test/integration/remote/helper.go
+++ b/backend/test/e2e/remote/helper.go
@@ -15,7 +15,7 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package test
+package remote
import (
"fmt"
@@ -24,10 +24,7 @@ import (
"testing"
"time"
- "github.com/apache/incubator-devlake/core/config"
- "github.com/apache/incubator-devlake/core/plugin"
- orgPlugin "github.com/apache/incubator-devlake/plugins/org/impl"
- "github.com/apache/incubator-devlake/test/integration/helper"
+ "github.com/apache/incubator-devlake/test/helper"
)
const (
@@ -57,7 +54,6 @@ type (
func SetupEnv() {
fmt.Println("Setup test env")
- helper.Init()
path := filepath.Join(helper.ProjectRoot, FAKE_PLUGIN_DIR, "start.sh")
_, err := os.Stat(path)
if err != nil {
@@ -69,13 +65,7 @@ func SetupEnv() {
func ConnectLocalServer(t *testing.T) *helper.DevlakeClient {
fmt.Println("Connect to server")
- client := helper.ConnectLocalServer(t, &helper.LocalClientConfig{
- ServerPort: 8089,
- DbURL: config.GetConfig().GetString("E2E_DB_URL"),
- CreateServer: true,
- TruncateDb: true,
- Plugins: map[string]plugin.PluginMeta{"org":
orgPlugin.Org{}},
- })
+ client := helper.StartDevLakeServer(t, nil)
client.SetTimeout(30 * time.Second)
return client
}
diff --git a/backend/test/integration/remote/python_plugin_test.go
b/backend/test/e2e/remote/python_plugin_test.go
similarity index 98%
rename from backend/test/integration/remote/python_plugin_test.go
rename to backend/test/e2e/remote/python_plugin_test.go
index ae9de7a81..203149e85 100644
--- a/backend/test/integration/remote/python_plugin_test.go
+++ b/backend/test/e2e/remote/python_plugin_test.go
@@ -15,14 +15,14 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package test
+package remote
import (
"testing"
"github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/test/integration/helper"
+ "github.com/apache/incubator-devlake/test/helper"
"github.com/stretchr/testify/require"
)
diff --git a/backend/test/e2e/services/pipeline_runner_e2e_test.go
b/backend/test/e2e/services/pipeline_runner_test.go
similarity index 96%
rename from backend/test/e2e/services/pipeline_runner_e2e_test.go
rename to backend/test/e2e/services/pipeline_runner_test.go
index 97e70f75d..0cb6b9142 100644
--- a/backend/test/e2e/services/pipeline_runner_e2e_test.go
+++ b/backend/test/e2e/services/pipeline_runner_test.go
@@ -20,14 +20,15 @@ package services
import (
"github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/server/services"
- _ "github.com/apache/incubator-devlake/test/e2e"
+ "github.com/apache/incubator-devlake/test/helper"
"github.com/stretchr/testify/assert"
"testing"
)
func TestComputePipelineStatus(t *testing.T) {
- db := services.GetBasicRes().GetDal()
- // insert fake tasks to datbase
+ client := helper.StartDevLakeServer(t, nil)
+ db := client.GetDal()
+ // insert fake tasks to database
pipeline := &models.Pipeline{
TotalTasks: 3,
}
diff --git a/backend/test/integration/helper/init.go
b/backend/test/e2e/services/server_startup_test.go
similarity index 72%
copy from backend/test/integration/helper/init.go
copy to backend/test/e2e/services/server_startup_test.go
index b3e75ca29..524b21edb 100644
--- a/backend/test/integration/helper/init.go
+++ b/backend/test/e2e/services/server_startup_test.go
@@ -15,23 +15,16 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package helper
+package services
import (
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/test"
+ "github.com/apache/incubator-devlake/test/helper"
+ "github.com/stretchr/testify/require"
+ "testing"
)
-var (
- ProjectRoot = ""
- Shell = ""
-)
-
-func Init() {
- Shell = "/bin/sh"
- var err errors.Error
- ProjectRoot, err = test.NormalizeBaseDirectory()
- if err != nil {
- panic(err.Error())
- }
+func TestStartup(t *testing.T) {
+ client := helper.StartDevLakeServer(t, nil)
+ projects := client.ListProjects()
+ require.Equal(t, 0, int(projects.Count))
}
diff --git a/backend/test/integration/helper/api.go b/backend/test/helper/api.go
similarity index 100%
rename from backend/test/integration/helper/api.go
rename to backend/test/helper/api.go
diff --git a/backend/test/e2e/init.go b/backend/test/helper/bootstrap.go
similarity index 53%
copy from backend/test/e2e/init.go
copy to backend/test/helper/bootstrap.go
index 1481bc41a..6cf2da3e3 100644
--- a/backend/test/e2e/init.go
+++ b/backend/test/helper/bootstrap.go
@@ -15,33 +15,36 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package e2e
+package helper
import (
"github.com/apache/incubator-devlake/core/config"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/server/services"
- "github.com/apache/incubator-devlake/test"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/runner"
+ "github.com/apache/incubator-devlake/impls/logruslog"
)
-func init() {
- _, err := test.NormalizeBaseDirectory()
+// InitDB Bootstraps the database by getting rid of all the tables
+func InitDB(dbUrl string) {
+ logger := logruslog.Global.Nested("test-init")
+ logger.Info("Initializing database")
+ cfg := config.GetConfig()
+ cfg.Set("DB_URL", dbUrl)
+ db, err := runner.NewGormDb(cfg, logger)
if err != nil {
panic(err)
}
- v := config.GetConfig()
- encKey := v.GetString(plugin.EncodeKeyEnvStr)
- if encKey == "" {
- // Randomly generate a bunch of encryption keys and set them to
config
- encKey = plugin.RandomEncKey()
- v.Set(plugin.EncodeKeyEnvStr, encKey)
- err := config.WriteConfig(v)
- if err != nil {
- panic(err)
- }
+ migrator := db.Migrator()
+ tables, err := errors.Convert01(migrator.GetTables())
+ if err != nil {
+ panic(err)
+ }
+ logger.Info("Dropping %d existing tables", len(tables))
+ var tablesRaw []any
+ for _, table := range tables {
+ tablesRaw = append(tablesRaw, table)
}
- services.Init()
- err = services.GetMigrator().Execute()
+ err = errors.Convert(migrator.DropTable(tablesRaw...))
if err != nil {
panic(err)
}
diff --git a/backend/test/integration/helper/client.go
b/backend/test/helper/client.go
similarity index 82%
rename from backend/test/integration/helper/client.go
rename to backend/test/helper/client.go
index c05c1571c..7f7926d4a 100644
--- a/backend/test/integration/helper/client.go
+++ b/backend/test/helper/client.go
@@ -23,6 +23,10 @@ import (
"encoding/json"
goerror "errors"
"fmt"
+ "github.com/apache/incubator-devlake/core/dal"
+ dora "github.com/apache/incubator-devlake/plugins/dora/impl"
+ org "github.com/apache/incubator-devlake/plugins/org/impl"
+ remotePlugin
"github.com/apache/incubator-devlake/server/services/remote/plugin"
"io"
"math"
"net/http"
@@ -44,7 +48,6 @@ import (
"github.com/apache/incubator-devlake/impls/dalgorm"
"github.com/apache/incubator-devlake/impls/logruslog"
"github.com/apache/incubator-devlake/server/api"
- remotePlugin
"github.com/apache/incubator-devlake/server/services/remote/plugin"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
@@ -81,15 +84,14 @@ type (
pipelineTimeout time.Duration
}
LocalClientConfig struct {
- ServerPort uint
- DbURL string
- CreateServer bool
- DropDb bool
- TruncateDb bool
- Plugins map[string]plugin.PluginMeta
- AdditionalMigrations func() []plugin.MigrationScript
- Timeout time.Duration
- PipelineTimeout time.Duration
+ ServerPort uint
+ DbURL string
+ CreateServer bool
+ DropDb bool
+ TruncateDb bool
+ Plugins map[string]plugin.PluginMeta
+ Timeout time.Duration
+ PipelineTimeout time.Duration
}
RemoteClientConfig struct {
Endpoint string
@@ -97,9 +99,9 @@ type (
)
// ConnectRemoteServer returns a client to an existing server based on the
config
-func ConnectRemoteServer(t *testing.T, sbConfig *RemoteClientConfig)
*DevlakeClient {
+func ConnectRemoteServer(t *testing.T, cfg *RemoteClientConfig) *DevlakeClient
{
return &DevlakeClient{
- Endpoint: sbConfig.Endpoint,
+ Endpoint: cfg.Endpoint,
db: nil,
log: nil,
testCtx: t,
@@ -137,26 +139,26 @@ func ConnectLocalServer(t *testing.T, clientConfig
*LocalClientConfig) *DevlakeC
if d.pipelineTimeout == 0 {
d.pipelineTimeout = 30 * time.Second
}
- d.configureEncryption()
- d.initPlugins(clientConfig)
- if clientConfig.DropDb || clientConfig.TruncateDb {
- d.prepareDB(clientConfig)
- }
if clientConfig.CreateServer {
+ d.configureEncryption()
+ d.initPlugins(clientConfig)
+ if clientConfig.DropDb || clientConfig.TruncateDb {
+ d.prepareDB(clientConfig)
+ }
cfg.Set("PORT", clientConfig.ServerPort)
cfg.Set("PLUGIN_DIR", throwawayDir)
cfg.Set("LOGGING_DIR", throwawayDir)
go func() {
initService.Do(api.CreateApiService)
}()
+ req, err2 := http.NewRequest(http.MethodGet,
fmt.Sprintf("%s/proceed-db-migration", addr), nil)
+ require.NoError(t, err2)
+ d.forceSendHttpRequest(20, req, func(err errors.Error) bool {
+ e := err.Unwrap()
+ return goerror.Is(e, syscall.ECONNREFUSED)
+ })
+ logger.Info("New DevLake server initialized")
}
- req, err2 := http.NewRequest(http.MethodGet,
fmt.Sprintf("%s/proceed-db-migration", addr), nil)
- require.NoError(t, err2)
- d.forceSendHttpRequest(20, req, func(err errors.Error) bool {
- e := err.Unwrap()
- return goerror.Is(e, syscall.ECONNREFUSED)
- })
- d.runMigrations(clientConfig)
return d
}
@@ -165,6 +167,16 @@ func (d *DevlakeClient) SetTimeout(timeout time.Duration) {
d.timeout = timeout
}
+// SetTimeout override the timeout of pipeline run success expectation
+func (d *DevlakeClient) SetPipelineTimeout(timeout time.Duration) {
+ d.pipelineTimeout = timeout
+}
+
+// GetDal get a reference to the dal.Dal used by the server
+func (d *DevlakeClient) GetDal() dal.Dal {
+ return dalgorm.NewDalgorm(d.db)
+}
+
// AwaitPluginAvailability wait for this plugin to become available on the
server given a timeout. Returns false if this condition does not get met.
func (d *DevlakeClient) AwaitPluginAvailability(pluginName string, timeout
time.Duration) {
err := runWithTimeout(timeout, func() (bool, errors.Error) {
@@ -238,13 +250,18 @@ func (d *DevlakeClient) forceSendHttpRequest(retries
uint, req *http.Request, on
}
}
-func (d *DevlakeClient) initPlugins(sbConfig *LocalClientConfig) {
- remotePlugin.Init(d.basicRes)
+func (d *DevlakeClient) initPlugins(cfg *LocalClientConfig) {
d.testCtx.Helper()
- if sbConfig.Plugins != nil {
- for name, p := range sbConfig.Plugins {
- require.NoError(d.testCtx, plugin.RegisterPlugin(name,
p))
- }
+ if cfg.Plugins == nil {
+ cfg.Plugins = map[string]plugin.PluginMeta{}
+ }
+ // default plugins
+ cfg.Plugins["org"] = org.Org{}
+ cfg.Plugins["dora"] = dora.Dora{}
+
+ // register and init plugins
+ for name, p := range cfg.Plugins {
+ require.NoError(d.testCtx, plugin.RegisterPlugin(name, p))
}
for _, p := range plugin.AllPlugins() {
if pi, ok := p.(plugin.PluginInit); ok {
@@ -252,33 +269,7 @@ func (d *DevlakeClient) initPlugins(sbConfig
*LocalClientConfig) {
require.NoError(d.testCtx, err)
}
}
-}
-
-func (d *DevlakeClient) runMigrations(sbConfig *LocalClientConfig) {
- d.testCtx.Helper()
- basicRes := contextimpl.NewDefaultBasicRes(d.cfg, d.log,
dalgorm.NewDalgorm(d.db))
- getMigrator := func() plugin.Migrator {
- migrator, err := migration.NewMigrator(basicRes)
- require.NoError(d.testCtx, err)
- return migrator
- }
- {
- migrator := getMigrator()
- for pluginName, pluginInst := range sbConfig.Plugins {
- if migratable, ok :=
pluginInst.(plugin.PluginMigration); ok {
-
migrator.Register(migratable.MigrationScripts(), pluginName)
- }
- }
- require.NoError(d.testCtx, migrator.Execute())
- }
- {
- migrator := getMigrator()
- if sbConfig.AdditionalMigrations != nil {
- scripts := sbConfig.AdditionalMigrations()
- migrator.Register(scripts, "extra migrations")
- }
- require.NoError(d.testCtx, migrator.Execute())
- }
+ remotePlugin.Init(d.basicRes)
}
func (d *DevlakeClient) prepareDB(cfg *LocalClientConfig) {
@@ -286,6 +277,7 @@ func (d *DevlakeClient) prepareDB(cfg *LocalClientConfig) {
migrator := d.db.Migrator()
tables, err := migrator.GetTables()
require.NoError(d.testCtx, err)
+ d.log.Debug("Existing DB tables: %v", tables)
if cfg.DropDb {
d.log.Info("Dropping %d tables", len(tables))
var tablesRaw []any
@@ -295,7 +287,9 @@ func (d *DevlakeClient) prepareDB(cfg *LocalClientConfig) {
err = migrator.DropTable(tablesRaw...)
require.NoError(d.testCtx, err)
} else if cfg.TruncateDb {
- d.log.Info("Truncating %d tables",
len(tables)-len(dbTruncationExclusions))
+ if len(tables) > len(dbTruncationExclusions) {
+ d.log.Info("Truncating %d tables",
len(tables)-len(dbTruncationExclusions))
+ }
for _, table := range tables {
excluded := false
for _, exclusion := range dbTruncationExclusions {
diff --git a/backend/test/e2e/init.go b/backend/test/helper/client_factory.go
similarity index 59%
rename from backend/test/e2e/init.go
rename to backend/test/helper/client_factory.go
index 1481bc41a..68559f0cd 100644
--- a/backend/test/e2e/init.go
+++ b/backend/test/helper/client_factory.go
@@ -15,34 +15,23 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package e2e
+package helper
import (
"github.com/apache/incubator-devlake/core/config"
"github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/server/services"
- "github.com/apache/incubator-devlake/test"
+ "testing"
)
-func init() {
- _, err := test.NormalizeBaseDirectory()
- if err != nil {
- panic(err)
- }
- v := config.GetConfig()
- encKey := v.GetString(plugin.EncodeKeyEnvStr)
- if encKey == "" {
- // Randomly generate a bunch of encryption keys and set them to
config
- encKey = plugin.RandomEncKey()
- v.Set(plugin.EncodeKeyEnvStr, encKey)
- err := config.WriteConfig(v)
- if err != nil {
- panic(err)
- }
- }
- services.Init()
- err = services.GetMigrator().Execute()
- if err != nil {
- panic(err)
- }
+// Creates a new in-memory DevLake server with default settings and returns a
client to it
+func StartDevLakeServer(t *testing.T, loadedGoPlugins
map[string]plugin.PluginMeta) *DevlakeClient {
+ client := ConnectLocalServer(t, &LocalClientConfig{
+ ServerPort: 8089,
+ DbURL: config.GetConfig().GetString("E2E_DB_URL"),
+ CreateServer: true,
+ DropDb: false,
+ TruncateDb: true,
+ Plugins: loadedGoPlugins,
+ })
+ return client
}
diff --git a/backend/test/integration/helper/db_selector.go
b/backend/test/helper/db_selector.go
similarity index 100%
rename from backend/test/integration/helper/db_selector.go
rename to backend/test/helper/db_selector.go
diff --git a/backend/test/utils.go b/backend/test/helper/init.go
similarity index 85%
rename from backend/test/utils.go
rename to backend/test/helper/init.go
index ead1ba5e8..8d8559ac6 100644
--- a/backend/test/utils.go
+++ b/backend/test/helper/init.go
@@ -15,7 +15,7 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package test
+package helper
import (
"github.com/apache/incubator-devlake/core/errors"
@@ -23,6 +23,20 @@ import (
"path/filepath"
)
+var (
+ ProjectRoot = ""
+ Shell = ""
+)
+
+func init() {
+ Shell = "/bin/sh"
+ var err errors.Error
+ ProjectRoot, err = NormalizeBaseDirectory()
+ if err != nil {
+ panic(err.Error())
+ }
+}
+
func NormalizeBaseDirectory() (string, errors.Error) {
pwd, err := os.Getwd()
if err != nil {
diff --git a/backend/test/integration/helper/json_helper.go
b/backend/test/helper/json_helper.go
similarity index 92%
rename from backend/test/integration/helper/json_helper.go
rename to backend/test/helper/json_helper.go
index 2ce2cdd0c..780cdf0aa 100644
--- a/backend/test/integration/helper/json_helper.go
+++ b/backend/test/helper/json_helper.go
@@ -32,6 +32,18 @@ func ToJson(x any) json.RawMessage {
return b
}
+func ToMap(x any) map[string]any {
+ b, err := json.Marshal(x)
+ if err != nil {
+ panic(err)
+ }
+ m := map[string]any{}
+ if err = json.Unmarshal(b, &m); err != nil {
+ panic(err)
+ }
+ return m
+}
+
// ToCleanJson FIXME
func ToCleanJson(inline bool, x any) json.RawMessage {
j, err := json.Marshal(x)
diff --git a/backend/test/integration/helper/models.go
b/backend/test/helper/models.go
similarity index 100%
rename from backend/test/integration/helper/models.go
rename to backend/test/helper/models.go
diff --git a/backend/test/integration/helper/utils.go
b/backend/test/helper/utils.go
similarity index 100%
rename from backend/test/integration/helper/utils.go
rename to backend/test/helper/utils.go
diff --git a/backend/test/integration/helper/init.go b/backend/test/init.go
similarity index 72%
rename from backend/test/integration/helper/init.go
rename to backend/test/init.go
index b3e75ca29..cb60aef74 100644
--- a/backend/test/integration/helper/init.go
+++ b/backend/test/init.go
@@ -15,23 +15,13 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package helper
+package main
import (
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/test"
+ "github.com/apache/incubator-devlake/core/config"
+ "github.com/apache/incubator-devlake/test/helper"
)
-var (
- ProjectRoot = ""
- Shell = ""
-)
-
-func Init() {
- Shell = "/bin/sh"
- var err errors.Error
- ProjectRoot, err = test.NormalizeBaseDirectory()
- if err != nil {
- panic(err.Error())
- }
+func main() {
+ helper.InitDB(config.GetConfig().GetString("E2E_DB_URL"))
}
