This is an automated email from the ASF dual-hosted git repository.
lynwee pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 2b8dc3225 feat(customize): add account handling for imported issues
(#8401)
2b8dc3225 is described below
commit 2b8dc32254d61cdaa59991754de852bedaef9197
Author: NaRro <cong.w...@merico.dev>
AuthorDate: Thu Apr 24 13:03:08 2025 +0800
feat(customize): add account handling for imported issues (#8401)
* feat(customize): add account handling for imported issues
- Create or update accounts for creator and assignee
- Link accounts to issues in the database
- Add account data to issues_output.csv
- Verify account data with snapshot_tables/accounts.csv
#8400
* refactor(customize): extract getStringField function for better record
handling
- Add getStringField function to extract and validate string fields from
records
- Use getStringField to replace repetitive code in issueHandlerFactory
- Add unit tests for getStringField to ensure its correctness
#8400
---
.../plugins/customize/e2e/import_issues_test.go | 11 ++
.../customize/e2e/raw_tables/issues_input.csv | 6 +-
.../customize/e2e/snapshot_tables/accounts.csv | 5 +
.../e2e/snapshot_tables/issues_output.csv | 12 +--
backend/plugins/customize/service/service.go | 119 ++++++++++++++++++---
backend/plugins/customize/service/service_test.go | 115 ++++++++++++++++++++
6 files changed, 246 insertions(+), 22 deletions(-)
diff --git a/backend/plugins/customize/e2e/import_issues_test.go
b/backend/plugins/customize/e2e/import_issues_test.go
index aa875eb92..82de28d09 100644
--- a/backend/plugins/customize/e2e/import_issues_test.go
+++ b/backend/plugins/customize/e2e/import_issues_test.go
@@ -21,6 +21,7 @@ import (
"os"
"testing"
+
"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/customize/impl"
@@ -37,6 +38,7 @@ func TestImportIssueDataFlow(t *testing.T) {
dataflowTester.FlushTabler(&models.CustomizedField{})
dataflowTester.FlushTabler(&ticket.IssueLabel{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})
+ dataflowTester.FlushTabler(&crossdomain.Account{})
svc := service.NewService(dataflowTester.Dal)
err := svc.CreateField(&models.CustomizedField{
TbName: "issues",
@@ -172,4 +174,13 @@ func TestImportIssueDataFlow(t *testing.T) {
"board_id",
"issue_id",
})
+ dataflowTester.VerifyTableWithRawData(
+ &crossdomain.Account{},
+ "snapshot_tables/accounts.csv",
+ []string{
+ "id",
+ "full_name",
+ "user_name",
+ },
+ )
}
diff --git a/backend/plugins/customize/e2e/raw_tables/issues_input.csv
b/backend/plugins/customize/e2e/raw_tables/issues_input.csv
index 32e72055f..1eacd5fcd 100644
--- a/backend/plugins/customize/e2e/raw_tables/issues_input.csv
+++ b/backend/plugins/customize/e2e/raw_tables/issues_input.csv
@@ -1,4 +1,4 @@
id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels
-csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue
test,BUG,new,2022-07-17
07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15
15:27:56,world,8,NULL
-csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue
test007,BUG,new,2022-08-12
13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15
15:27:56,abc,24590,hello worlds
-csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,11,issue
test011,REQUIREMENT,new,2022-08-10
13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15
15:27:56,NULL,0.00014,NULL
+csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue
test,BUG,new,2022-07-17
07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,klesh,10,2022-09-15
15:27:56,world,8,NULL
+csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue
test007,BUG,new,2022-08-12
13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,warren,30,2022-09-15
15:27:56,abc,24590,hello worlds
+csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,11,issue
test011,REQUIREMENT,new,2022-08-10
13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,abeizn,1,2022-09-15
15:27:56,NULL,0.00014,NULL
diff --git a/backend/plugins/customize/e2e/snapshot_tables/accounts.csv
b/backend/plugins/customize/e2e/snapshot_tables/accounts.csv
new file mode 100644
index 000000000..5d48240be
--- /dev/null
+++ b/backend/plugins/customize/e2e/snapshot_tables/accounts.csv
@@ -0,0 +1,5 @@
+id,full_name,user_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+csv:CsvAccount:0:abeizn,abeizn,abeizn,csv-board,,0,
+csv:CsvAccount:0:klesh,klesh,klesh,csv-board,,0,
+csv:CsvAccount:0:tgp,tgp,tgp,csv-board2,,0,
+csv:CsvAccount:0:warren,warren,warren,csv-board,,0,
diff --git a/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv
b/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv
index 43d522071..ffcffa88f 100644
--- a/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv
+++ b/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv
@@ -1,7 +1,7 @@
id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,parent_issue_id,priority,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,severity,component,original_project,x_varchar,x_text,x_time,x_float,x_int,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,,1,issue
test,,,,BUG,,new,0,,2022-07-17T07:15:55.959+00:00,,,,major,0,0,,,tgp,,tgp,,,,world,,2022-09-15T15:27:56.000+00:00,8,10,csv-board2,,,
-csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue
title
edited,,,,BUG,,new,0,,2022-08-12T13:43:00.783+00:00,,,,trivial,0,0,,,tgp,,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,24590,30,csv-board2,,,
-csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,,11,issue
test011,,,,REQUIREMENT,,new,0,,2022-08-10T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
-csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,,12,issue
test012,,,,REQUIREMENT,,new,0,,2022-08-11T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
-csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue
test013,,,,REQUIREMENT,,new,0,,2022-08-12T13:44:46.508+00:00,,,,critical,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board2,,,
-csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue
test014,,,,INCIDENT,,new,0,,2022-08-12T13:45:12.810+00:00,,,,blocker,0,0,,,tgp,,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,csv-board2,,,
+csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,,1,issue
test,,,,BUG,,new,0,,2022-07-17T07:15:55.959+00:00,,,,major,0,0,,csv:CsvAccount:0:tgp,tgp,csv:CsvAccount:0:tgp,tgp,,,,world,,2022-09-15T15:27:56.000+00:00,8,10,csv-board2,,,
+csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue
title
edited,,,,BUG,,new,0,,2022-08-12T13:43:00.783+00:00,,,,trivial,0,0,,csv:CsvAccount:0:tgp,tgp,csv:CsvAccount:0:tgp,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,24590,30,csv-board2,,,
+csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,,11,issue
test011,,,,REQUIREMENT,,new,0,,2022-08-10T13:44:46.508+00:00,,,,major,0,0,,csv:CsvAccount:0:tgp,tgp,csv:CsvAccount:0:abeizn,abeizn,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
+csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,,12,issue
test012,,,,REQUIREMENT,,new,0,,2022-08-11T13:44:46.508+00:00,,,,major,0,0,,csv:CsvAccount:0:tgp,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
+csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue
test013,,,,REQUIREMENT,,new,0,,2022-08-12T13:44:46.508+00:00,,,,critical,0,0,,csv:CsvAccount:0:tgp,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board2,,,
+csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue
test014,,,,INCIDENT,,new,0,,2022-08-12T13:45:12.810+00:00,,,,blocker,0,0,,csv:CsvAccount:0:tgp,tgp,csv:CsvAccount:0:tgp,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,csv-board2,,,
diff --git a/backend/plugins/customize/service/service.go
b/backend/plugins/customize/service/service.go
index 6586dd13a..bd8b0aa9d 100644
--- a/backend/plugins/customize/service/service.go
+++ b/backend/plugins/customize/service/service.go
@@ -156,6 +156,7 @@ func (s *Service) getCustomizedFields(table string)
([]models.CustomizedField, e
// issue could exist in multiple boards, so we should only delete an old
records when it doesn't belong to another board
func (s *Service) ImportIssue(boardId string, file io.ReadCloser, incremental
bool) errors.Error {
if !incremental {
+ // not delete accounts data since account may be referenced by
others
err := s.dal.Delete(
&ticket.Issue{},
dal.Where("id IN (SELECT issue_id FROM board_issues
WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE
board_id!=?))", boardId, boardId),
@@ -262,23 +263,74 @@ func (s *Service) importCSV(file io.ReadCloser,
rawDataParams string, recordHand
}
}
+// createOrUpdateAccount creates or updates an account based on the provided
name.
+// It returns the account ID and an error if any occurred.
+func (s *Service) createOrUpdateAccount(accountName string, rawDataParams
string) (string, errors.Error) {
+ if accountName == "" {
+ return "", nil // Return empty ID if name is empty, no error
needed here.
+ }
+ now := time.Now()
+ accountId := fmt.Sprintf("csv:CsvAccount:0:%s", accountName)
+ account := &crossdomain.Account{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: accountId,
+ NoPKModel: common.NoPKModel{
+ RawDataOrigin: common.RawDataOrigin{
+ RawDataParams: rawDataParams,
+ },
+ },
+ },
+ FullName: accountName,
+ UserName: accountName,
+ CreatedDate: &now,
+ }
+ err := s.dal.CreateOrUpdate(account)
+ if err != nil {
+ return "", errors.Default.Wrap(err, fmt.Sprintf("failed to
create or update account for %s", accountName))
+ }
+ return accountId, nil
+}
+
+// getStringField extracts a string field from a record map.
+// If required is true, it returns an error if the field is missing, nil,
empty, or not a string.
+// If required is false, it returns an empty string without error if the field
is missing or nil,
+// but returns an error if the field exists and is not a string.
+func getStringField(record map[string]interface{}, fieldName string, required
bool) (string, errors.Error) {
+ value, ok := record[fieldName]
+ if !ok || value == nil {
+ if required {
+ return "", errors.Default.New(fmt.Sprintf("record
without required field %s", fieldName))
+ }
+ return "", nil // Field missing or nil, but not required
+ }
+
+ strValue, ok := value.(string)
+ if !ok {
+ return "", errors.Default.New(fmt.Sprintf("%s is not a string",
fieldName))
+ }
+
+ if required && strValue == "" {
+ return "", errors.Default.New(fmt.Sprintf("invalid or empty
required field %s", fieldName))
+ }
+
+ return strValue, nil
+}
+
// issueHandlerFactory returns a handler that save record into `issues`,
`board_issues` and `issue_labels` table
func (s *Service) issueHandlerFactory(boardId string, incremental bool)
func(record map[string]interface{}) errors.Error {
return func(record map[string]interface{}) errors.Error {
var err errors.Error
- var id string
- if record["id"] == nil {
- return errors.Default.New("record without id")
+ id, err := getStringField(record, "id", true)
+ if err != nil {
+ return err
}
- id, _ = record["id"].(string)
- if id == "" {
- return errors.Default.New("empty id")
+
+ // Handle labels
+ labels, err := getStringField(record, "labels", false)
+ if err != nil {
+ return err
}
- if record["labels"] != nil {
- labels, ok := record["labels"].(string)
- if !ok {
- return errors.Default.New("labels is not
string")
- }
+ if labels != "" {
var issueLabels []*ticket.IssueLabel
appearedLabels := make(map[string]struct{}) // record
the labels that have appeared
for _, label := range strings.Split(labels, ",") {
@@ -307,15 +359,56 @@ func (s *Service) issueHandlerFactory(boardId string,
incremental bool) func(rec
}
}
}
- delete(record, "labels")
+ delete(record, "labels") // Remove labels from record map as
it's handled
+
+ // Handle creator and assignee accounts
+ rawDataParams, err := getStringField(record,
"_raw_data_params", true)
+ if err != nil {
+ // This should ideally not happen as it's set in
importCSV, but good to check
+ return err
+ }
+
+ // Handle creator
+ creatorName, err := getStringField(record, "creator_name",
false)
+ if err != nil {
+ return err
+ }
+ creatorId, err := s.createOrUpdateAccount(creatorName,
rawDataParams)
+ if err != nil {
+ return err
+ }
+ if creatorId != "" {
+ record["creator_id"] = creatorId
+ }
+
+ // Handle assignee
+ assigneeName, err := getStringField(record, "assignee_name",
false)
+ if err != nil {
+ return err
+ }
+ assigneeId, err := s.createOrUpdateAccount(assigneeName,
rawDataParams)
+ if err != nil {
+ return err
+ }
+ if assigneeId != "" {
+ record["assignee_id"] = assigneeId
+ }
+
+ // Handle issues
err = s.dal.CreateWithMap(&ticket.Issue{}, record)
if err != nil {
return err
}
- return s.dal.CreateOrUpdate(&ticket.BoardIssue{
+
+ // Handle board_issues
+ err = s.dal.CreateOrUpdate(&ticket.BoardIssue{
BoardId: boardId,
IssueId: id,
})
+ if err != nil {
+ return err
+ }
+ return nil
}
}
diff --git a/backend/plugins/customize/service/service_test.go
b/backend/plugins/customize/service/service_test.go
index 8242e5f87..5511145f2 100644
--- a/backend/plugins/customize/service/service_test.go
+++ b/backend/plugins/customize/service/service_test.go
@@ -21,6 +21,8 @@ import (
"regexp"
"strings"
"testing"
+
+ "github.com/stretchr/testify/assert"
)
func TestService_checkFieldName(t *testing.T) {
@@ -75,3 +77,116 @@ func TestService_checkFieldName(t *testing.T) {
})
}
}
+
+func TestGetStringField(t *testing.T) {
+ testCases := []struct {
+ name string
+ record map[string]interface{}
+ fieldName string
+ required bool
+ expectValue string
+ expectError bool
+ errorMsg string
+ }{
+ // Required field tests
+ {
+ name: "Required field exists and valid",
+ record: map[string]interface{}{"id": "123"},
+ fieldName: "id",
+ required: true,
+ expectValue: "123",
+ expectError: false,
+ },
+ {
+ name: "Required field exists but empty",
+ record: map[string]interface{}{"id": ""},
+ fieldName: "id",
+ required: true,
+ expectValue: "",
+ expectError: true,
+ errorMsg: "invalid or empty required field id",
+ },
+ {
+ name: "Required field exists but wrong type",
+ record: map[string]interface{}{"id": 123},
+ fieldName: "id",
+ required: true,
+ expectValue: "",
+ expectError: true,
+ errorMsg: "id is not a string",
+ },
+ {
+ name: "Required field missing",
+ record: map[string]interface{}{"name": "test"},
+ fieldName: "id",
+ required: true,
+ expectValue: "",
+ expectError: true,
+ errorMsg: "record without required field id",
+ },
+ {
+ name: "Required field nil",
+ record: map[string]interface{}{"id": nil},
+ fieldName: "id",
+ required: true,
+ expectValue: "",
+ expectError: true,
+ errorMsg: "record without required field id",
+ },
+ // Optional field tests
+ {
+ name: "Optional field exists and valid",
+ record: map[string]interface{}{"label": "bug"},
+ fieldName: "label",
+ required: false,
+ expectValue: "bug",
+ expectError: false,
+ },
+ {
+ name: "Optional field exists but empty",
+ record: map[string]interface{}{"label": ""},
+ fieldName: "label",
+ required: false,
+ expectValue: "",
+ expectError: false,
+ },
+ {
+ name: "Optional field exists but wrong type",
+ record: map[string]interface{}{"label": 123},
+ fieldName: "label",
+ required: false,
+ expectValue: "",
+ expectError: true,
+ errorMsg: "label is not a string",
+ },
+ {
+ name: "Optional field missing",
+ record: map[string]interface{}{"name": "test"},
+ fieldName: "label",
+ required: false,
+ expectValue: "",
+ expectError: false,
+ },
+ {
+ name: "Optional field nil",
+ record: map[string]interface{}{"label": nil},
+ fieldName: "label",
+ required: false,
+ expectValue: "",
+ expectError: false,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ value, err := getStringField(tc.record, tc.fieldName,
tc.required)
+ assert.Equal(t, tc.expectValue, value)
+ if tc.expectError {
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), tc.errorMsg)
+ } else {
+ assert.NoError(t, err)
+ }
+ })
+ }
+}
