This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new f2dd0d33 Issues/2038 (#2039)
f2dd0d33 is described below
commit f2dd0d33103c49acc591e879cedf2e2e5382bb0e
Author: likyh <[email protected]>
AuthorDate: Mon Jun 6 21:53:16 2022 +0800
Issues/2038 (#2039)
* add some log and ignore case of log_level
* add some log and ignore case of log_level
Co-authored-by: linyh <[email protected]>
---
logger/{log.go => init.go} | 14 ++++-----
.../helper/default_logger.go => logger/logger.go | 2 +-
plugins/gitextractor/main.go | 4 +--
plugins/gitextractor/store/database.go | 15 ++++++----
plugins/gitextractor/tasks/git_repo_collector.go | 2 +-
plugins/helper/api_async_client_test.go | 13 +++++----
plugins/helper/api_client.go | 2 +-
plugins/helper/api_collector_test.go | 33 ++++++++++------------
plugins/helper/api_extractor.go | 12 ++++++--
plugins/helper/api_extractor_test.go | 8 ++++--
plugins/helper/batch_save.go | 20 +++++++++----
plugins/helper/batch_save_divider.go | 5 ++--
plugins/helper/batch_save_divider_test.go | 7 +++--
plugins/helper/batch_save_test.go | 3 +-
plugins/helper/data_convertor.go | 2 +-
plugins/helper/data_convertor_test.go | 8 ++++--
plugins/helper/default_task_context.go | 13 +++++++--
plugins/helper/default_task_logger.go | 3 +-
runner/db.go | 8 +++---
19 files changed, 104 insertions(+), 70 deletions(-)
diff --git a/logger/log.go b/logger/init.go
similarity index 87%
rename from logger/log.go
rename to logger/init.go
index 7fc466f8..a5ff6616 100644
--- a/logger/log.go
+++ b/logger/init.go
@@ -21,10 +21,10 @@ import (
"fmt"
"github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/helper"
"github.com/sirupsen/logrus"
prefixed "github.com/x-cray/logrus-prefixed-formatter"
"os"
+ "strings"
)
var inner *logrus.Logger
@@ -33,14 +33,14 @@ var Global core.Logger
func init() {
inner = logrus.New()
logLevel := logrus.InfoLevel
- switch config.GetConfig().GetString("LOGGING_LEVEL") {
- case "Debug":
+ switch strings.ToLower(config.GetConfig().GetString("LOGGING_LEVEL")) {
+ case "debug":
logLevel = logrus.DebugLevel
- case "Info":
+ case "info":
logLevel = logrus.InfoLevel
- case "Warn":
+ case "warn":
logLevel = logrus.WarnLevel
- case "Error":
+ case "error":
logLevel = logrus.ErrorLevel
}
inner.SetLevel(logLevel)
@@ -53,5 +53,5 @@ func init() {
inner.Info(fmt.Sprintf("failed to create dir logs: %s", err))
}
loggerPool := make(map[string]*logrus.Logger)
- Global = helper.NewDefaultLogger(inner, "", loggerPool)
+ Global = NewDefaultLogger(inner, "", loggerPool)
}
diff --git a/plugins/helper/default_logger.go b/logger/logger.go
similarity index 99%
rename from plugins/helper/default_logger.go
rename to logger/logger.go
index c6afb66f..110affe1 100644
--- a/plugins/helper/default_logger.go
+++ b/logger/logger.go
@@ -15,7 +15,7 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package helper
+package logger
import (
"fmt"
diff --git a/plugins/gitextractor/main.go b/plugins/gitextractor/main.go
index 7da1f30a..300ae5b0 100644
--- a/plugins/gitextractor/main.go
+++ b/plugins/gitextractor/main.go
@@ -42,6 +42,7 @@ func main() {
output := flag.String("output", "", "-output")
db := flag.String("db", "", "-db")
flag.Parse()
+ log := logger.Global.Nested("git extractor")
var storage models.Store
var err error
if *url == "" {
@@ -60,13 +61,12 @@ func main() {
if err != nil {
panic(err)
}
- storage = store.NewDatabase(database)
+ storage = store.NewDatabase(database, log)
} else {
panic("either specify `-output` or `-db` argument as
destination")
}
defer storage.Close()
ctx := context.Background()
- log := logger.Global.Nested("git extractor")
subTaskCtx := helper.NewStandaloneSubTaskContext(
config.GetConfig(),
log,
diff --git a/plugins/gitextractor/store/database.go
b/plugins/gitextractor/store/database.go
index 7c872c8d..4e264fc6 100644
--- a/plugins/gitextractor/store/database.go
+++ b/plugins/gitextractor/store/database.go
@@ -19,6 +19,7 @@ package store
import (
"github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
"gorm.io/gorm"
"reflect"
@@ -29,16 +30,18 @@ const BathSize = 100
type Database struct {
//db *gorm.DB
driver *helper.BatchSaveDivider
+ log core.Logger
}
-func NewDatabase(db *gorm.DB) *Database {
+func NewDatabase(db *gorm.DB, log core.Logger) *Database {
database := new(Database)
database.driver = helper.NewBatchSaveDivider(db, BathSize)
+ database.log = log
return database
}
func (d *Database) RepoCommits(repoCommit *code.RepoCommit) error {
- batch, err := d.driver.ForType(reflect.TypeOf(repoCommit))
+ batch, err := d.driver.ForType(reflect.TypeOf(repoCommit), d.log)
if err != nil {
return err
}
@@ -46,7 +49,7 @@ func (d *Database) RepoCommits(repoCommit *code.RepoCommit)
error {
}
func (d *Database) Commits(commit *code.Commit) error {
- batch, err := d.driver.ForType(reflect.TypeOf(commit))
+ batch, err := d.driver.ForType(reflect.TypeOf(commit), d.log)
if err != nil {
return err
}
@@ -54,7 +57,7 @@ func (d *Database) Commits(commit *code.Commit) error {
}
func (d *Database) Refs(ref *code.Ref) error {
- batch, err := d.driver.ForType(reflect.TypeOf(ref))
+ batch, err := d.driver.ForType(reflect.TypeOf(ref), d.log)
if err != nil {
return err
}
@@ -62,7 +65,7 @@ func (d *Database) Refs(ref *code.Ref) error {
}
func (d *Database) CommitFiles(file *code.CommitFile) error {
- batch, err := d.driver.ForType(reflect.TypeOf(file))
+ batch, err := d.driver.ForType(reflect.TypeOf(file), d.log)
if err != nil {
return err
}
@@ -73,7 +76,7 @@ func (d *Database) CommitParents(pp []*code.CommitParent)
error {
if len(pp) == 0 {
return nil
}
- batch, err := d.driver.ForType(reflect.TypeOf(pp[0]))
+ batch, err := d.driver.ForType(reflect.TypeOf(pp[0]), d.log)
if err != nil {
return err
}
diff --git a/plugins/gitextractor/tasks/git_repo_collector.go
b/plugins/gitextractor/tasks/git_repo_collector.go
index 0ee6fece..4dcb8cd2 100644
--- a/plugins/gitextractor/tasks/git_repo_collector.go
+++ b/plugins/gitextractor/tasks/git_repo_collector.go
@@ -56,7 +56,7 @@ func (o GitExtractorOptions) Valid() error {
func CollectGitRepo(subTaskCtx core.SubTaskContext) error {
var err error
db := subTaskCtx.GetDb()
- storage := store.NewDatabase(db)
+ storage := store.NewDatabase(db, subTaskCtx.GetLogger())
op := subTaskCtx.GetData().(GitExtractorOptions)
p := parser.NewLibGit2(storage, subTaskCtx)
if strings.HasPrefix(op.Url, "http") {
diff --git a/plugins/helper/api_async_client_test.go
b/plugins/helper/api_async_client_test.go
index dff9c5f8..266c22b8 100644
--- a/plugins/helper/api_async_client_test.go
+++ b/plugins/helper/api_async_client_test.go
@@ -31,6 +31,7 @@ import (
"time"
"github.com/agiledragon/gomonkey/v2"
+ "github.com/apache/incubator-devlake/logger"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
@@ -103,7 +104,7 @@ func CreateTestAsyncApiClientWithRateLimitAndCtx(t
*testing.T, rateLimiter *ApiR
return &DefaultTaskContext{
&defaultExecContext{
cfg: cfg,
- logger: &DefaultLogger{},
+ logger: &logger.DefaultLogger{},
db: db,
ctx: ctx,
name: "Test",
@@ -146,7 +147,7 @@ func TestWaitAsync_EmptyWork(t *testing.T) {
func TestWaitAsync_WithWork(t *testing.T) {
asyncApiClient, _ := CreateTestAsyncApiClient(t)
- gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}),
"Info", func(_ *DefaultLogger, _ string, _ ...interface{}) {
+ gm_info :=
gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_
*logger.DefaultLogger, _ string, _ ...interface{}) {
})
defer gm_info.Reset()
@@ -187,7 +188,7 @@ func TestWaitAsync_WithWork(t *testing.T) {
func TestWaitAsync_MutiWork(t *testing.T) {
asyncApiClient, _ := CreateTestAsyncApiClient(t)
- gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}),
"Info", func(_ *DefaultLogger, _ string, _ ...interface{}) {
+ gm_info :=
gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_
*logger.DefaultLogger, _ string, _ ...interface{}) {
})
defer gm_info.Reset()
@@ -231,7 +232,7 @@ func TestWaitAsync_MutiWork(t *testing.T) {
// go test -gcflags=all=-l -run ^TestDoAsync_OnceSuceess
func TestDoAsync_OnceSuceess(t *testing.T) {
asyncApiClient, _ := CreateTestAsyncApiClient(t)
- gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}),
"Info", func(_ *DefaultLogger, _ string, _ ...interface{}) {
+ gm_info :=
gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_
*logger.DefaultLogger, _ string, _ ...interface{}) {
})
defer gm_info.Reset()
@@ -261,7 +262,7 @@ func TestDoAsync_OnceSuceess(t *testing.T) {
// go test -gcflags=all=-l -run ^TestDoAsync_TryAndFail
func TestDoAsync_TryAndFail(t *testing.T) {
asyncApiClient, _ := CreateTestAsyncApiClient(t)
- gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}),
"Info", func(_ *DefaultLogger, _ string, _ ...interface{}) {
+ gm_info :=
gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_
*logger.DefaultLogger, _ string, _ ...interface{}) {
})
defer gm_info.Reset()
@@ -294,7 +295,7 @@ func TestDoAsync_TryAndFail(t *testing.T) {
// go test -gcflags=all=-l -run ^TestDoAsync_TryAndSuceess
func TestDoAsync_TryAndSuceess(t *testing.T) {
asyncApiClient, _ := CreateTestAsyncApiClient(t)
- gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}),
"Info", func(_ *DefaultLogger, _ string, _ ...interface{}) {
+ gm_info :=
gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_
*logger.DefaultLogger, _ string, _ ...interface{}) {
})
defer gm_info.Reset()
diff --git a/plugins/helper/api_client.go b/plugins/helper/api_client.go
index 33b7ef7a..001b4c66 100644
--- a/plugins/helper/api_client.go
+++ b/plugins/helper/api_client.go
@@ -217,7 +217,7 @@ func (apiClient *ApiClient) Do(
return nil, err
}
- // after recieve
+ // after receive
if apiClient.afterReponse != nil {
err = apiClient.afterReponse(res)
if err != nil {
diff --git a/plugins/helper/api_collector_test.go
b/plugins/helper/api_collector_test.go
index 532b2293..c8f89286 100644
--- a/plugins/helper/api_collector_test.go
+++ b/plugins/helper/api_collector_test.go
@@ -33,6 +33,7 @@ import (
"time"
"github.com/agiledragon/gomonkey/v2"
+ "github.com/apache/incubator-devlake/logger"
"github.com/apache/incubator-devlake/models/common"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
@@ -176,45 +177,41 @@ func MockDB(t *testing.T) {
gt = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Table", func(db
*gorm.DB, name string, args ...interface{}) *gorm.DB {
assert.Equal(t, name, TestTableData.TableName())
return db
- },
- )
+ })
gc = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Create", func(db
*gorm.DB, value interface{}) *gorm.DB {
assert.Equal(t, TestTableData, value.(*TestTable))
return db
- },
- )
+ })
gd = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Delete", func(db
*gorm.DB, value interface{}, conds ...interface{}) (tx *gorm.DB) {
return db
- },
- )
+ })
ga = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "AutoMigrate",
func(db *gorm.DB, dst ...interface{}) error {
return nil
- },
- )
+ })
god = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Order", func(db
*gorm.DB, value interface{}) (tx *gorm.DB) {
return db
- },
- )
+ })
gw = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Where", func(db
*gorm.DB, query interface{}, args ...interface{}) (tx *gorm.DB) {
return db
- },
- )
+ })
+
+ gr = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Count", func(db
*gorm.DB, count *int64) (tx *gorm.DB) {
+ return db
+ })
gr = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Rows", func(db
*gorm.DB) (*sql.Rows, error) {
return &sql.Rows{}, nil
- },
- )
+ })
gs = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "ScanRows",
func(db *gorm.DB, rows *sql.Rows, dest interface{}) error {
dest = TestRawMessage
return nil
- },
- )
+ })
}
@@ -268,7 +265,7 @@ func CreateTestApiCollector() (*ApiCollector, error) {
return NewApiCollector(ApiCollectorArgs{
RawDataSubTaskArgs: RawDataSubTaskArgs{
Ctx: &DefaultSubTaskContext{
- defaultExecContext:
newDefaultExecContext(GetConfigForTest("../../"),
NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)), db,
ctx, "Test", nil, nil),
+ defaultExecContext:
newDefaultExecContext(GetConfigForTest("../../"),
logger.NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)),
db, ctx, "Test", nil, nil),
},
Table: TestTable{}.TableName(),
Params: &TestParam{
@@ -1096,7 +1093,7 @@ func TestExecute_Total(t *testing.T) {
})
defer gs.Reset()
- gin := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}), "Info",
func(_ *DefaultLogger, _ string, _ ...interface{}) {
+ gin := gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}),
"Info", func(_ *logger.DefaultLogger, _ string, _ ...interface{}) {
})
defer gin.Reset()
diff --git a/plugins/helper/api_extractor.go b/plugins/helper/api_extractor.go
index 8f6d6b6a..c96e33ed 100644
--- a/plugins/helper/api_extractor.go
+++ b/plugins/helper/api_extractor.go
@@ -64,10 +64,18 @@ func NewApiExtractor(args ApiExtractorArgs) (*ApiExtractor,
error) {
func (extractor *ApiExtractor) Execute() error {
// load data from database
db := extractor.args.Ctx.GetDb()
- cursor, err := db.Table(extractor.table).Order("id ASC").Where("params
= ?", extractor.params).Rows()
+ log := extractor.args.Ctx.GetLogger()
+ count := int64(0)
+ cursor, err := db.
+ Table(extractor.table).
+ Order("id ASC").
+ Where("params = ?", extractor.params).
+ Count(&count).
+ Rows()
if err != nil {
return err
}
+ log.Info("get data from %s where params=%s and got %d",
extractor.table, extractor.params, count)
defer cursor.Close()
row := &RawData{}
@@ -113,7 +121,7 @@ func (extractor *ApiExtractor) Execute() error {
for _, result := range results {
// get the batch operator for the specific type
- batch, err := divider.ForType(reflect.TypeOf(result))
+ batch, err := divider.ForType(reflect.TypeOf(result),
log)
if err != nil {
return err
}
diff --git a/plugins/helper/api_extractor_test.go
b/plugins/helper/api_extractor_test.go
index b895d2df..d225e71d 100644
--- a/plugins/helper/api_extractor_test.go
+++ b/plugins/helper/api_extractor_test.go
@@ -21,6 +21,8 @@ import (
"context"
"database/sql"
"fmt"
+ "github.com/apache/incubator-devlake/logger"
+ "github.com/apache/incubator-devlake/plugins/core"
"reflect"
"testing"
"time"
@@ -48,7 +50,7 @@ func CreateTestApiExtractor(t *testing.T) (*ApiExtractor,
error) {
return NewApiExtractor(ApiExtractorArgs{
RawDataSubTaskArgs: RawDataSubTaskArgs{
Ctx: &DefaultSubTaskContext{
- defaultExecContext:
newDefaultExecContext(GetConfigForTest("../../"),
NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)),
&gorm.DB{}, ctx, "Test", nil, nil),
+ defaultExecContext:
newDefaultExecContext(GetConfigForTest("../../"),
logger.NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)),
&gorm.DB{}, ctx, "Test", nil, nil),
},
Table: TestTable{}.TableName(),
Params: &TestParam{
@@ -107,7 +109,7 @@ func TestApiExtractorExecute(t *testing.T) {
)
fortypeTimes := 0
- gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}),
"ForType", func(d *BatchSaveDivider, rowType reflect.Type) (*BatchSave, error) {
+ gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}),
"ForType", func(d *BatchSaveDivider, rowType reflect.Type, log core.Logger)
(*BatchSave, error) {
fortypeTimes++
assert.Equal(t, rowType, reflect.TypeOf(TestTableData))
err := d.onNewBatchSave(rowType)
@@ -168,7 +170,7 @@ func TestApiExtractorExecute_Cancel(t *testing.T) {
)
fortypeTimes := 0
- gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}),
"ForType", func(d *BatchSaveDivider, rowType reflect.Type) (*BatchSave, error) {
+ gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}),
"ForType", func(d *BatchSaveDivider, rowType reflect.Type, log core.Logger)
(*BatchSave, error) {
fortypeTimes++
assert.Equal(t, rowType, reflect.TypeOf(TestTableData))
err := d.onNewBatchSave(rowType)
diff --git a/plugins/helper/batch_save.go b/plugins/helper/batch_save.go
index 2f9d20b7..a5d66a7f 100644
--- a/plugins/helper/batch_save.go
+++ b/plugins/helper/batch_save.go
@@ -19,11 +19,11 @@ package helper
import (
"fmt"
- "reflect"
- "strings"
-
+ "github.com/apache/incubator-devlake/plugins/core"
"gorm.io/gorm"
"gorm.io/gorm/clause"
+ "reflect"
+ "strings"
)
// Insert data by batch can increase database performance drastically, this
class aim to make batch-save easier,
@@ -35,20 +35,24 @@ type BatchSave struct {
// I'm guessing the reason is the type information lost when converted
to interface{}
slots reflect.Value
db *gorm.DB
+ logger core.Logger
current int
size int
valueIndex map[string]int
}
-func NewBatchSave(db *gorm.DB, slotType reflect.Type, size int) (*BatchSave,
error) {
+func NewBatchSave(db *gorm.DB, log core.Logger, slotType reflect.Type, size
int) (*BatchSave, error) {
if slotType.Kind() != reflect.Ptr {
return nil, fmt.Errorf("slotType must be a pointer")
}
if !hasPrimaryKey(slotType) {
- return nil, fmt.Errorf("no primary key")
+ return nil, fmt.Errorf("%s no primary key", slotType.String())
}
+ log = log.Nested(slotType.String())
+ log.Info("create batch save success")
return &BatchSave{
slotType: slotType,
+ logger: log,
slots: reflect.MakeSlice(reflect.SliceOf(slotType), size,
size),
db: db,
size: size,
@@ -80,15 +84,19 @@ func (c *BatchSave) Add(slot interface{}) error {
// flush out into database if max outed
if c.current == c.size {
return c.Flush()
+ } else if c.current%100 == 0 {
+ c.logger.Debug("batch save current: %d", c.current)
}
return nil
}
func (c *BatchSave) Flush() error {
- err := c.db.Clauses(clause.OnConflict{UpdateAll:
true}).Create(c.slots.Slice(0, c.current).Interface()).Error
+ result := c.db.Clauses(clause.OnConflict{UpdateAll:
true}).Create(c.slots.Slice(0, c.current).Interface())
+ err := result.Error
if err != nil {
return err
}
+ c.logger.Info("batch save flush %d and %d success", c.slots.Slice(0,
c.current).Len(), result.RowsAffected)
c.current = 0
c.valueIndex = make(map[string]int)
return nil
diff --git a/plugins/helper/batch_save_divider.go
b/plugins/helper/batch_save_divider.go
index 9a20eb42..63003e98 100644
--- a/plugins/helper/batch_save_divider.go
+++ b/plugins/helper/batch_save_divider.go
@@ -18,6 +18,7 @@ limitations under the License.
package helper
import (
+ "github.com/apache/incubator-devlake/plugins/core"
"reflect"
"gorm.io/gorm"
@@ -47,13 +48,13 @@ func (d *BatchSaveDivider) OnNewBatchSave(cb
OnNewBatchSave) {
}
// return *BatchSave for specified type
-func (d *BatchSaveDivider) ForType(rowType reflect.Type) (*BatchSave, error) {
+func (d *BatchSaveDivider) ForType(rowType reflect.Type, log core.Logger)
(*BatchSave, error) {
// get the cache for the specific type
batch := d.batches[rowType]
var err error
// create one if not exists
if batch == nil {
- batch, err = NewBatchSave(d.db, rowType, d.batchSize)
+ batch, err = NewBatchSave(d.db, log, rowType, d.batchSize)
if err != nil {
return nil, err
}
diff --git a/plugins/helper/batch_save_divider_test.go
b/plugins/helper/batch_save_divider_test.go
index 4d7a8181..42991a33 100644
--- a/plugins/helper/batch_save_divider_test.go
+++ b/plugins/helper/batch_save_divider_test.go
@@ -18,6 +18,7 @@ limitations under the License.
package helper
import (
+ "github.com/apache/incubator-devlake/logger"
"reflect"
"testing"
@@ -50,13 +51,13 @@ func TestBatchSaveDivider(t *testing.T) {
var b3 *BatchSave
// test if it saved and only saved once for one Type
- b1, err = batchSaveDivider.ForType(reflect.TypeOf(TestTableData))
+ b1, err = batchSaveDivider.ForType(reflect.TypeOf(TestTableData),
logger.Global)
assert.Equal(t, initTimes, 1)
assert.Equal(t, err, nil)
- b2, err = batchSaveDivider.ForType(reflect.TypeOf(&TestTable2{}))
+ b2, err = batchSaveDivider.ForType(reflect.TypeOf(&TestTable2{}),
logger.Global)
assert.Equal(t, initTimes, 2)
assert.Equal(t, err, nil)
- b3, err = batchSaveDivider.ForType(reflect.TypeOf(TestTableData))
+ b3, err = batchSaveDivider.ForType(reflect.TypeOf(TestTableData),
logger.Global)
assert.Equal(t, initTimes, 2)
assert.Equal(t, err, nil)
diff --git a/plugins/helper/batch_save_test.go
b/plugins/helper/batch_save_test.go
index fd6bdfad..464be8ee 100644
--- a/plugins/helper/batch_save_test.go
+++ b/plugins/helper/batch_save_test.go
@@ -22,6 +22,7 @@ import (
"testing"
"github.com/agiledragon/gomonkey/v2"
+ "github.com/apache/incubator-devlake/logger"
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/ticket"
"github.com/stretchr/testify/assert"
@@ -157,7 +158,7 @@ func TestBatchSave(t *testing.T) {
TestBatchSize = 1
rowType := reflect.TypeOf(TestTableData)
- batch, err := NewBatchSave(db, rowType, TestBatchSize)
+ batch, err := NewBatchSave(db, logger.Global, rowType, TestBatchSize)
// test diff type
assert.Equal(t, err, nil)
diff --git a/plugins/helper/data_convertor.go b/plugins/helper/data_convertor.go
index 29dc66a5..44a9affb 100644
--- a/plugins/helper/data_convertor.go
+++ b/plugins/helper/data_convertor.go
@@ -115,7 +115,7 @@ func (converter *DataConverter) Execute() error {
for _, result := range results {
// get the batch operator for the specific type
- batch, err := divider.ForType(reflect.TypeOf(result))
+ batch, err := divider.ForType(reflect.TypeOf(result),
converter.args.Ctx.GetLogger())
if err != nil {
return err
}
diff --git a/plugins/helper/data_convertor_test.go
b/plugins/helper/data_convertor_test.go
index 3a4d84f7..5791c54d 100644
--- a/plugins/helper/data_convertor_test.go
+++ b/plugins/helper/data_convertor_test.go
@@ -26,6 +26,8 @@ import (
"time"
"github.com/agiledragon/gomonkey/v2"
+ "github.com/apache/incubator-devlake/logger"
+ "github.com/apache/incubator-devlake/plugins/core"
"github.com/magiconair/properties/assert"
"github.com/sirupsen/logrus"
"gorm.io/gorm"
@@ -38,7 +40,7 @@ func CreateTestDataConverter(t *testing.T) (*DataConverter,
error) {
return NewDataConverter(DataConverterArgs{
RawDataSubTaskArgs: RawDataSubTaskArgs{
Ctx: &DefaultSubTaskContext{
- defaultExecContext:
newDefaultExecContext(GetConfigForTest("../../"),
NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)),
&gorm.DB{}, ctx, "Test", nil, nil),
+ defaultExecContext:
newDefaultExecContext(GetConfigForTest("../../"),
logger.NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)),
&gorm.DB{}, ctx, "Test", nil, nil),
},
Table: TestTable{}.TableName(),
Params: &TestParam{
@@ -99,7 +101,7 @@ func TestDataConvertorExecute(t *testing.T) {
)
fortypeTimes := 0
- gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}),
"ForType", func(d *BatchSaveDivider, rowType reflect.Type) (*BatchSave, error) {
+ gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}),
"ForType", func(d *BatchSaveDivider, rowType reflect.Type, log core.Logger)
(*BatchSave, error) {
fortypeTimes++
assert.Equal(t, rowType, reflect.TypeOf(TestTableData))
err := d.onNewBatchSave(rowType)
@@ -160,7 +162,7 @@ func TestDataConvertorExecute_Cancel(t *testing.T) {
)
fortypeTimes := 0
- gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}),
"ForType", func(d *BatchSaveDivider, rowType reflect.Type) (*BatchSave, error) {
+ gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}),
"ForType", func(d *BatchSaveDivider, rowType reflect.Type, log core.Logger)
(*BatchSave, error) {
fortypeTimes++
assert.Equal(t, rowType, reflect.TypeOf(TestTableData))
err := d.onNewBatchSave(rowType)
diff --git a/plugins/helper/default_task_context.go
b/plugins/helper/default_task_context.go
index f49e2534..3b14b0d0 100644
--- a/plugins/helper/default_task_context.go
+++ b/plugins/helper/default_task_context.go
@@ -22,6 +22,7 @@ import (
"fmt"
"sync"
"sync/atomic"
+ "time"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/spf13/viper"
@@ -152,7 +153,8 @@ func (c *DefaultTaskContext) IncProgress(quantity int) {
// SubTaskContext default implementation
type DefaultSubTaskContext struct {
*defaultExecContext
- taskCtx *DefaultTaskContext
+ taskCtx *DefaultTaskContext
+ LastProgressTime time.Time
}
func (c *DefaultSubTaskContext) SetProgress(current int, total int) {
@@ -164,7 +166,12 @@ func (c *DefaultSubTaskContext) SetProgress(current int,
total int) {
func (c *DefaultSubTaskContext) IncProgress(quantity int) {
c.defaultExecContext.IncProgress(core.SubTaskIncProgress, quantity)
- c.logger.Info("finished records: %d", c.current)
+ if c.LastProgressTime.IsZero() ||
c.LastProgressTime.Add(3*time.Second).Before(time.Now()) || c.current%1000 == 0
{
+ c.LastProgressTime = time.Now()
+ c.logger.Info("finished records: %d", c.current)
+ } else {
+ c.logger.Debug("finished records: %d", c.current)
+ }
}
func NewDefaultTaskContext(
@@ -193,6 +200,7 @@ func (c *DefaultTaskContext) SubTaskContext(subtask string)
(core.SubTaskContext
c.subtaskCtxs[subtask] = &DefaultSubTaskContext{
c.defaultExecContext.fork(subtask),
c,
+ time.Time{},
}
}
c.defaultExecContext.mu.Unlock()
@@ -220,6 +228,7 @@ func NewStandaloneSubTaskContext(
return &DefaultSubTaskContext{
newDefaultExecContext(cfg, logger, db, ctx, name, data, nil),
nil,
+ time.Time{},
}
}
diff --git a/plugins/helper/default_task_logger.go
b/plugins/helper/default_task_logger.go
index 01e72c54..2b4b20a3 100644
--- a/plugins/helper/default_task_logger.go
+++ b/plugins/helper/default_task_logger.go
@@ -18,6 +18,7 @@ limitations under the License.
package helper
import (
+ "github.com/apache/incubator-devlake/logger"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/sirupsen/logrus"
)
@@ -26,5 +27,5 @@ import (
// TODO: implement another TaskLogger for distributed runner/worker
func NewDefaultTaskLogger(log *logrus.Logger, prefix string, loggerPool
map[string]*logrus.Logger) core.Logger {
- return NewDefaultLogger(log, prefix, loggerPool)
+ return logger.NewDefaultLogger(log, prefix, loggerPool)
}
diff --git a/runner/db.go b/runner/db.go
index ecf9b379..a4d3a93b 100644
--- a/runner/db.go
+++ b/runner/db.go
@@ -33,12 +33,12 @@ import (
func NewGormDb(config *viper.Viper, logger core.Logger) (*gorm.DB, error) {
dbLoggingLevel := gormLogger.Error
- switch config.GetString("DB_LOGGING_LEVEL") {
- case "Silent":
+ switch strings.ToLower(config.GetString("DB_LOGGING_LEVEL")) {
+ case "silent":
dbLoggingLevel = gormLogger.Silent
- case "Warn":
+ case "warn":
dbLoggingLevel = gormLogger.Warn
- case "Info":
+ case "info":
dbLoggingLevel = gormLogger.Info
}
