This is an automated email from the ASF dual-hosted git repository. warren pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit fe6d26ab8f0df252e395a81fda6091a475fb77ca Author: linyh <[email protected]> AuthorDate: Wed Jun 15 12:27:55 2022 +0800 change a name --- helpers/e2ehelper/data_flow_tester.go | 20 +++++++++----------- plugins/gitlab/e2e/project_test.go | 4 ++-- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/helpers/e2ehelper/data_flow_tester.go b/helpers/e2ehelper/data_flow_tester.go index d8fc8585..1c49bbcb 100644 --- a/helpers/e2ehelper/data_flow_tester.go +++ b/helpers/e2ehelper/data_flow_tester.go @@ -147,16 +147,8 @@ func (t *DataFlowTester) Subtask(subtaskMeta core.SubTaskMeta, taskData interfac } } -// VerifyTable reads rows from csv file and compare with records from database one by one. You must specified the -// Primary Key Fields with `pkfields` so DataFlowTester could select the exact record from database, as well as which -// fields to compare with by specifying `targetfields` parameter. -func (t *DataFlowTester) CreateSnapshotOrVerify(dst schema.Tabler, csvRelPath string, pkfields []string, targetfields []string) { - _, err := os.Stat(csvRelPath) - if err == nil { - t.VerifyTable(dst, csvRelPath, pkfields, targetfields) - return - } - +// CreateSnapshot reads rows from database and write them into .csv file. +func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, csvRelPath string, pkfields []string, targetfields []string) { location, _ := time.LoadLocation(`UTC`) allFields := []string{} allFields = append(pkfields, targetfields...) @@ -222,6 +214,12 @@ func (t *DataFlowTester) CreateSnapshotOrVerify(dst schema.Tabler, csvRelPath st // Primary Key Fields with `pkfields` so DataFlowTester could select the exact record from database, as well as which // fields to compare with by specifying `targetfields` parameter. func (t *DataFlowTester) VerifyTable(dst schema.Tabler, csvRelPath string, pkfields []string, targetfields []string) { + _, err := os.Stat(csvRelPath) + if os.IsNotExist(err) { + t.CreateSnapshot(dst, csvRelPath, pkfields, targetfields) + return + } + csvIter := pluginhelper.NewCsvFileIterator(csvRelPath) location, _ := time.LoadLocation(`UTC`) defer csvIter.Close() @@ -266,7 +264,7 @@ func (t *DataFlowTester) VerifyTable(dst schema.Tabler, csvRelPath string, pkfie } var actualTotal int64 - err := t.Db.Table(dst.TableName()).Count(&actualTotal).Error + err = t.Db.Table(dst.TableName()).Count(&actualTotal).Error if err != nil { panic(err) } diff --git a/plugins/gitlab/e2e/project_test.go b/plugins/gitlab/e2e/project_test.go index e25461be..c7342ef1 100644 --- a/plugins/gitlab/e2e/project_test.go +++ b/plugins/gitlab/e2e/project_test.go @@ -45,7 +45,7 @@ func TestGitlabDataFlow(t *testing.T) { // verify extraction dataflowTester.FlushTabler(&models.GitlabProject{}) dataflowTester.Subtask(tasks.ExtractProjectMeta, taskData) - dataflowTester.CreateSnapshotOrVerify( + dataflowTester.VerifyTable( models.GitlabProject{}, "tables/_tool_gitlab_projects.csv", []string{"connection_id", "gitlab_id"}, @@ -73,7 +73,7 @@ func TestGitlabDataFlow(t *testing.T) { // verify conversion dataflowTester.FlushTabler(&code.Repo{}) dataflowTester.Subtask(tasks.ConvertProjectMeta, taskData) - dataflowTester.CreateSnapshotOrVerify( + dataflowTester.VerifyTable( code.Repo{}, "tables/repos.csv", []string{"id"},
