This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 9e702dc03 feat: add an API for uploading issue_repo_commits.csv (#4823)
9e702dc03 is described below
commit 9e702dc0390e67db9da9d53f0f971a0b395abfca
Author: Liang Zhang <[email protected]>
AuthorDate: Mon Apr 3 13:11:25 2023 +0800
feat: add an API for uploading issue_repo_commits.csv (#4823)
* feat: add an API for uploading issue_repo_commits.csv
* fix: some typo
---
backend/plugins/customize/api/csv.go | 26 ++++++++++++
.../customize/e2e/import_issue_commits_test.go | 3 +-
...s_test.go => import_issue_repo_commits_test.go} | 21 ++++++++--
.../e2e/raw_tables/issue_repo_commits.csv | 13 ++++++
.../e2e/snapshot_tables/issue_repo_commits.csv | 13 ++++++
backend/plugins/customize/impl/impl.go | 3 ++
backend/plugins/customize/service/service.go | 48 ++++++++++++++++++++++
7 files changed, 122 insertions(+), 5 deletions(-)
diff --git a/backend/plugins/customize/api/csv.go
b/backend/plugins/customize/api/csv.go
index 651808fb7..fa9ae053a 100644
--- a/backend/plugins/customize/api/csv.go
+++ b/backend/plugins/customize/api/csv.go
@@ -88,6 +88,32 @@ func (h *Handlers) ImportIssueCommit(input
*plugin.ApiResourceInput) (*plugin.Ap
return nil, h.svc.ImportIssueCommit(boardId, file)
}
+// ImportIssueRepoCommit accepts a CSV file, parses and saves it to the
database
+// @Summary Upload issue_repo_commits.csv file
+// @Description Upload issue_repo_commits.csv file
+// @Tags plugins/customize
+// @Accept multipart/form-data
+// @Param boardId formData string true "the ID of the board"
+// @Param file formData file true "select file to upload"
+// @Produce json
+// @Success 200
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/customize/csvfiles/issue_repo_commits.csv [post]
+func (h *Handlers) ImportIssueRepoCommit(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ file, err := h.extractFile(input)
+ if err != nil {
+ return nil, err
+ }
+ // nolint
+ defer file.Close()
+ boardId := strings.TrimSpace(input.Request.FormValue("boardId"))
+ if boardId == "" {
+ return nil, errors.Default.New("empty boardId")
+ }
+ return nil, h.svc.ImportIssueRepoCommit(boardId, file)
+}
+
func (h *Handlers) extractFile(input *plugin.ApiResourceInput) (io.ReadCloser,
errors.Error) {
if input.Request == nil {
return nil, errors.Default.New("request is nil")
diff --git a/backend/plugins/customize/e2e/import_issue_commits_test.go
b/backend/plugins/customize/e2e/import_issue_commits_test.go
index 3b98f6b86..033972923 100644
--- a/backend/plugins/customize/e2e/import_issue_commits_test.go
+++ b/backend/plugins/customize/e2e/import_issue_commits_test.go
@@ -30,7 +30,7 @@ func TestImportIssueCommitDataFlow(t *testing.T) {
var plugin impl.Customize
dataflowTester := e2ehelper.NewDataFlowTester(t, "customize", plugin)
- // import raw data table
+ // create table `issue_commits`
dataflowTester.FlushTabler(&crossdomain.IssueCommit{})
svc := service.NewService(dataflowTester.Dal)
@@ -39,6 +39,7 @@ func TestImportIssueCommitDataFlow(t *testing.T) {
t.Fatal(err1)
}
defer f.Close()
+ // import data
err := svc.ImportIssueCommit(`{"ConnectionId":1,"BoardId":8}`, f)
if err != nil {
t.Fatal(err)
diff --git a/backend/plugins/customize/e2e/import_issue_commits_test.go
b/backend/plugins/customize/e2e/import_issue_repo_commits_test.go
similarity index 73%
copy from backend/plugins/customize/e2e/import_issue_commits_test.go
copy to backend/plugins/customize/e2e/import_issue_repo_commits_test.go
index 3b98f6b86..2d6041b11 100644
--- a/backend/plugins/customize/e2e/import_issue_commits_test.go
+++ b/backend/plugins/customize/e2e/import_issue_repo_commits_test.go
@@ -26,23 +26,36 @@ import (
"testing"
)
-func TestImportIssueCommitDataFlow(t *testing.T) {
+func TestImportIssueRepoCommitDataFlow(t *testing.T) {
var plugin impl.Customize
dataflowTester := e2ehelper.NewDataFlowTester(t, "customize", plugin)
- // import raw data table
+ // create tables `issue_repo_commits` and `issue_commits`
+ dataflowTester.FlushTabler(&crossdomain.IssueRepoCommit{})
dataflowTester.FlushTabler(&crossdomain.IssueCommit{})
svc := service.NewService(dataflowTester.Dal)
- f, err1 := os.Open("raw_tables/issues_commits.csv")
+ f, err1 := os.Open("raw_tables/issue_repo_commits.csv")
if err1 != nil {
t.Fatal(err1)
}
defer f.Close()
- err := svc.ImportIssueCommit(`{"ConnectionId":1,"BoardId":8}`, f)
+ // import data
+ err := svc.ImportIssueRepoCommit(`{"ConnectionId":1,"BoardId":8}`, f)
if err != nil {
t.Fatal(err)
}
+ dataflowTester.VerifyTableWithRawData(
+ crossdomain.IssueRepoCommit{},
+ "snapshot_tables/issue_repo_commits.csv",
+ []string{
+ "issue_id",
+ "repo_url",
+ "commit_sha",
+ "host",
+ "namespace",
+ "repo_name",
+ })
dataflowTester.VerifyTableWithRawData(
crossdomain.IssueCommit{},
"snapshot_tables/issue_commits.csv",
diff --git a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
new file mode 100644
index 000000000..79f3dcd6e
--- /dev/null
+++ b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
@@ -0,0 +1,13 @@
+"created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","issue_id","repo_url","commit_sha","host","namespace","repo_name"
+"2023-03-30 14:06:06.831","2023-03-30
14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",1,"","jira:JiraIssue:1:10063","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf","8748a066cbaf67b15e86f2c636f9931347e987cf","example.com","PROJECTNAME","ui_jira";
+"2023-03-30 14:06:06.831","2023-03-30
14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",2,"","jira:JiraIssue:1:10064","https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca","abc0892edaee00dd7ee268dbee71620407a29bca","bitbucket.org","mynamespace","incubator-devlake";
+"2023-03-30 14:06:06.831","2023-03-30
14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",3,"","jira:JiraIssue:1:10064","https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6","e6bde456807818c5c78d7b265964d6d48b653af6","github.com","apache","incubator-devlake";
+"2023-03-30 14:06:06.831","2023-03-30
14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",4,"","jira:JiraIssue:1:10065","https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42","8f91020bcf684c6ad07adfafa3d8a2f826686c42","gitlab.com","namespace1/namespace2","murepo";
+"2023-03-30 14:06:06.831","2023-03-30
14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",5,"","jira:JiraIssue:1:10066","https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef","0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef","gitlab.com","meri.co","vdev.co";
+"2023-03-30 14:06:06.831","2023-03-30
14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",13,"","jira:JiraIssue:1:10139","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423","8993c04249e9d549e8950daec86717548c53c423","example.com","PROJECTNAME","ui_jira";
+"2023-03-30 14:06:06.831","2023-03-30
14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",19,"","jira:JiraIssue:1:10145","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74","07aa2ebed68e286dc51a7e0082031196a6135f74","example.com","PROJECTNAME","ui_jira";
+"2023-03-30 14:06:06.831","2023-03-30
14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",21,"","jira:JiraIssue:1:10145","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423","ef5ab26111744f65f5191b247767a473c70d6c95","example.com","PROJECTNAME","ui_jira";
+"2023-03-30 14:06:06.831","2023-03-30
14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",20,"","jira:JiraIssue:1:10145","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7","d70d6687e06304d9b6e0cb32b3f8c0f0928400f7","example.com","PROJECTNAME","ui_jira";
+"2023-03-30 14:06:06.831","2023-03-30
14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",36,"","jira:JiraIssue:1:10159","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da","d28785ff09229ac9e3c6734f0c97466ab00eb4da","example.com","PROJECTNAME","ui_jira";
+"2023-03-30 14:06:06.831","2023-03-30
14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",38,"","jira:JiraIssue:1:10202","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894","0ab12c4d4064003602edceed900d1456b6209894","example.com","PROJECTNAME","ui_jira";
+"2023-03-30 14:06:06.831","2023-03-30
14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",39,"","jira:JiraIssue:1:10203","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd","980e9fe7bc3e22a0409f7241a024eaf9c53680dd","example.com","PROJECTNAME","ui_jira";
diff --git
a/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv
b/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv
new file mode 100644
index 000000000..e8719548d
--- /dev/null
+++ b/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv
@@ -0,0 +1,13 @@
+issue_id,repo_url,commit_sha,host,namespace,repo_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,1,
+jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,2,
+jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,3,
+jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,4,
+jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,5,
+jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,13,
+jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,19,
+jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,21,
+jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,20,
+jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,36,
+jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,38,
+jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,39,
diff --git a/backend/plugins/customize/impl/impl.go
b/backend/plugins/customize/impl/impl.go
index 874cd300f..1c04999f5 100644
--- a/backend/plugins/customize/impl/impl.go
+++ b/backend/plugins/customize/impl/impl.go
@@ -98,5 +98,8 @@ func (p *Customize) ApiResources()
map[string]map[string]plugin.ApiResourceHandl
"csvfiles/issue_commits.csv": {
"POST": p.handlers.ImportIssueCommit,
},
+ "csvfiles/issue_repo_commits.csv": {
+ "POST": p.handlers.ImportIssueRepoCommit,
+ },
}
}
diff --git a/backend/plugins/customize/service/service.go
b/backend/plugins/customize/service/service.go
index 416e52789..4871018e0 100644
--- a/backend/plugins/customize/service/service.go
+++ b/backend/plugins/customize/service/service.go
@@ -178,6 +178,35 @@ func (s *Service) ImportIssueCommit(rawDataParams string,
file io.ReadCloser) er
return s.importCSV(file, rawDataParams, s.issueCommitHandler)
}
+// ImportIssueRepoCommit imports data to the table `issue_repo_commits` and
`issue_commits`
+func (s *Service) ImportIssueRepoCommit(rawDataParams string, file
io.ReadCloser) errors.Error {
+ fields := make(map[string]struct{})
+ // get all fields of the table `issue_repo_commit`
+ columns, err := s.dal.GetColumns(&crossdomain.IssueCommit{},
func(columnMeta dal.ColumnMeta) bool {
+ return true
+ })
+ if err != nil {
+ return err
+ }
+ for _, column := range columns {
+ fields[column.Name()] = struct{}{}
+ }
+ // delete old records of the table `issue_repo_commit` and
`issue_commit`
+ err = s.dal.Delete(&crossdomain.IssueRepoCommit{},
dal.Where("_raw_data_params = ?", rawDataParams))
+ if err != nil {
+ return err
+ }
+ err = s.dal.Delete(&crossdomain.IssueCommit{},
dal.Where("_raw_data_params = ?", rawDataParams))
+ if err != nil {
+ return err
+ }
+ return s.importCSV(file, rawDataParams,
s.issueRepoCommitHandlerFactory(fields))
+}
+
+// importCSV imports the csv file to the database,
+// the rawDataParams is used to identify the data source,
+// the recordHandler is used to handle the record, it should return an error
if the record is invalid
+// the `created_at` and `updated_at` will be set to the current time
func (s *Service) importCSV(file io.ReadCloser, rawDataParams string,
recordHandler func(map[string]interface{}) errors.Error) errors.Error {
iterator, err := pluginhelper.NewCsvFileIteratorFromFile(file)
if err != nil {
@@ -267,3 +296,22 @@ func (s *Service) issueHandlerFactory(boardId string)
func(record map[string]int
func (s *Service) issueCommitHandler(record map[string]interface{})
errors.Error {
return s.dal.CreateWithMap(&crossdomain.IssueCommit{}, record)
}
+
+// issueRepoCommitHandlerFactory returns a handler that will populate the
`issue_commits` and `issue_repo_commits` table
+// ths issueCommitsFields is used to filter the fields that should be inserted
into the `issue_commits` table
+func (s *Service) issueRepoCommitHandlerFactory(issueCommitsFields
map[string]struct{}) func(record map[string]interface{}) errors.Error {
+ return func(record map[string]interface{}) errors.Error {
+ err := s.dal.CreateWithMap(&crossdomain.IssueRepoCommit{},
record)
+ if err != nil {
+ return err
+ }
+ for head := range record {
+ if _, exists := issueCommitsFields[head]; exists {
+ continue
+ } else {
+ delete(record, head)
+ }
+ }
+ return s.dal.CreateWithMap(&crossdomain.IssueCommit{}, record)
+ }
+}
