This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 8c5fbc1d9 Remove the usage of `TmpFromAccountId` and `TmpToAccountId`
in Jira issue related data. (#7740)
8c5fbc1d9 is described below
commit 8c5fbc1d90871f556e2795904c69983e6f8fc145
Author: Lynwee <[email protected]>
AuthorDate: Tue Jul 16 11:51:51 2024 +0800
Remove the usage of `TmpFromAccountId` and `TmpToAccountId` in Jira issue
related data. (#7740)
* fix(jira): don't use `tmpFromAccountId` and `tmpToAccountId` in issue
changelogs
* fix(jira): remove all usage of `tmpFromAccountId` and `tmpToAccountId`
* refactor(jira): remove some codes
* fix(jira): fix e2e test
---
backend/plugins/jira/e2e/changelog_test.go | 1 +
.../plugins/jira/tasks/apiv2models/changelog.go | 14 ++-----
backend/plugins/jira/tasks/apiv2models/issue.go | 4 +-
backend/plugins/jira/tasks/epic_extractor.go | 42 +++++++++++++++++++-
.../jira/tasks/issue_changelog_convertor.go | 45 +++++-----------------
.../jira/tasks/issue_changelog_extractor.go | 6 ++-
backend/plugins/jira/tasks/issue_extractor.go | 10 +++--
7 files changed, 68 insertions(+), 54 deletions(-)
diff --git a/backend/plugins/jira/e2e/changelog_test.go
b/backend/plugins/jira/e2e/changelog_test.go
index 670a7e12f..8f6880f00 100644
--- a/backend/plugins/jira/e2e/changelog_test.go
+++ b/backend/plugins/jira/e2e/changelog_test.go
@@ -38,6 +38,7 @@ func TestIssueChangelogDataFlow(t *testing.T) {
}
// import raw data table
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_jira_api_issue_changelogs.csv",
"_raw_jira_api_issue_changelogs")
+
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_issue_fields.csv",
&models.JiraIssueField{})
dataflowTester.FlushTabler(&models.JiraIssueChangelogs{})
dataflowTester.FlushTabler(&models.JiraIssueChangelogItems{})
dataflowTester.FlushTabler(&models.JiraAccount{})
diff --git a/backend/plugins/jira/tasks/apiv2models/changelog.go
b/backend/plugins/jira/tasks/apiv2models/changelog.go
index 5c5c2c2a6..1ad6b9825 100644
--- a/backend/plugins/jira/tasks/apiv2models/changelog.go
+++ b/backend/plugins/jira/tasks/apiv2models/changelog.go
@@ -76,23 +76,15 @@ func (c ChangelogItem) ToToolLayer(connectionId,
changelogId uint64) *models.Jir
return item
}
-func (c ChangelogItem) ExtractUser(connectionId uint64) []*models.JiraAccount {
+func (c ChangelogItem) ExtractUser(connectionId uint64, userFieldMaps
map[string]struct{}) []*models.JiraAccount {
var result []*models.JiraAccount
- // if `tmpFromAccountId` or `tmpToAccountId` is not empty, then this
change log item stands for changes about accounts.
- if c.TmpFromAccountId != "" {
- // User `from` firstly
+ _, ok := userFieldMaps[c.Field]
+ if c.Field == "assignee" || c.Field == "reporter" || ok {
if c.FromValue != "" {
result = append(result,
&models.JiraAccount{ConnectionId: connectionId, AccountId: c.FromValue})
- } else {
- result = append(result,
&models.JiraAccount{ConnectionId: connectionId, AccountId: c.TmpFromAccountId})
}
- }
- if c.TmpToAccountId != "" {
- // User `to` firstly
if c.ToValue != "" {
result = append(result,
&models.JiraAccount{ConnectionId: connectionId, AccountId: c.ToValue})
- } else {
- result = append(result,
&models.JiraAccount{ConnectionId: connectionId, AccountId: c.TmpToAccountId})
}
}
return result
diff --git a/backend/plugins/jira/tasks/apiv2models/issue.go
b/backend/plugins/jira/tasks/apiv2models/issue.go
index 8a4417a3d..68b5a26b6 100644
--- a/backend/plugins/jira/tasks/apiv2models/issue.go
+++ b/backend/plugins/jira/tasks/apiv2models/issue.go
@@ -300,7 +300,7 @@ func (i *Issue) SetAllFields(raw json.RawMessage)
errors.Error {
return nil
}
-func (i Issue) ExtractEntities(connectionId uint64) ([]uint64,
*models.JiraIssue, []*models.JiraIssueComment, []*models.JiraWorklog,
[]*models.JiraIssueChangelogs, []*models.JiraIssueChangelogItems,
[]*models.JiraAccount) {
+func (i Issue) ExtractEntities(connectionId uint64, userFieldMaps
map[string]struct{}) ([]uint64, *models.JiraIssue, []*models.JiraIssueComment,
[]*models.JiraWorklog, []*models.JiraIssueChangelogs,
[]*models.JiraIssueChangelogItems, []*models.JiraAccount) {
issue := i.toToolLayer(connectionId)
var comments []*models.JiraIssueComment
var worklogs []*models.JiraWorklog
@@ -341,7 +341,7 @@ func (i Issue) ExtractEntities(connectionId uint64)
([]uint64, *models.JiraIssue
}
for _, item := range changelog.Items {
changelogItems = append(changelogItems,
item.ToToolLayer(connectionId, changelog.ID))
- users = append(users,
item.ExtractUser(connectionId)...)
+ users = append(users,
item.ExtractUser(connectionId, userFieldMaps)...)
}
}
}
diff --git a/backend/plugins/jira/tasks/epic_extractor.go
b/backend/plugins/jira/tasks/epic_extractor.go
index 8c3c8eab3..cf1e7f28b 100644
--- a/backend/plugins/jira/tasks/epic_extractor.go
+++ b/backend/plugins/jira/tasks/epic_extractor.go
@@ -18,9 +18,12 @@ limitations under the License.
package tasks
import (
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/log"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/jira/models"
)
var _ plugin.SubTaskEntryPoint = ExtractEpics
@@ -44,6 +47,10 @@ func ExtractEpics(taskCtx plugin.SubTaskContext)
errors.Error {
if err != nil {
return err
}
+ userFieldMap, err := getUserFieldMap(db, connectionId, logger)
+ if err != nil {
+ return err
+ }
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
Ctx: taskCtx,
@@ -54,7 +61,7 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error
{
Table: RAW_EPIC_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- return extractIssues(data, mappings, row)
+ return extractIssues(data, mappings, row, userFieldMap)
},
})
if err != nil {
@@ -62,3 +69,36 @@ func ExtractEpics(taskCtx plugin.SubTaskContext)
errors.Error {
}
return extractor.Execute()
}
+
+func getIssueFieldMap(db dal.Dal, connectionId uint64, logger log.Logger)
(map[string]models.JiraIssueField, errors.Error) {
+ var allIssueFields []models.JiraIssueField
+ if err := db.All(&allIssueFields, dal.Where("connection_id = ?",
connectionId)); err != nil {
+ return nil, err
+ }
+ issueFieldMap := make(map[string]models.JiraIssueField)
+ for _, v := range allIssueFields {
+ if _, ok := issueFieldMap[v.Name]; ok {
+ logger.Warn(nil, "filed name %s is duplicated", v.Name)
+ if v.SchemaType == "user" {
+ issueFieldMap[v.Name] = v
+ }
+ } else {
+ issueFieldMap[v.Name] = v
+ }
+ }
+ return issueFieldMap, nil
+}
+
+func getUserFieldMap(db dal.Dal, connectionId uint64, logger log.Logger)
(map[string]struct{}, errors.Error) {
+ userFieldMap := make(map[string]struct{})
+ issueFieldMap, err := getIssueFieldMap(db, connectionId, logger)
+ if err != nil {
+ return nil, err
+ }
+ for filedName, issueField := range issueFieldMap {
+ if issueField.SchemaType == "user" {
+ userFieldMap[filedName] = struct{}{}
+ }
+ }
+ return userFieldMap, nil
+}
diff --git a/backend/plugins/jira/tasks/issue_changelog_convertor.go
b/backend/plugins/jira/tasks/issue_changelog_convertor.go
index 090674b13..ba8de94f3 100644
--- a/backend/plugins/jira/tasks/issue_changelog_convertor.go
+++ b/backend/plugins/jira/tasks/issue_changelog_convertor.go
@@ -68,17 +68,10 @@ func ConvertIssueChangelogs(subtaskCtx
plugin.SubTaskContext) errors.Error {
statusMap[v.ID] = v
}
- var allIssueFields []models.JiraIssueField
- if err := db.All(&allIssueFields, dal.Where("connection_id = ?",
connectionId)); err != nil {
+ issueFieldMap, err := getIssueFieldMap(db, connectionId, logger)
+ if err != nil {
return err
}
- issueFieldMap := make(map[string]models.JiraIssueField)
- for _, v := range allIssueFields {
- if _, ok := issueFieldMap[v.Name]; ok {
- logger.Warn(nil, "filed name %s is duplicated", v.Name)
- }
- issueFieldMap[v.Name] = v
- }
issueIdGenerator := didgen.NewDomainIdGenerator(&models.JiraIssue{})
sprintIdGenerator := didgen.NewDomainIdGenerator(&models.JiraSprint{})
@@ -159,16 +152,15 @@ func ConvertIssueChangelogs(subtaskCtx
plugin.SubTaskContext) errors.Error {
changelog.ToValue =
getStdStatus(toStatus.StatusCategory)
}
default:
- fromAccountId :=
tryToResolveAccountIdFromAccountLikeField(row.Field, row.TmpFromAccountId,
row.FromValue, issueFieldMap)
- if fromAccountId != "" {
- changelog.OriginalFromValue =
accountIdGen.Generate(connectionId, fromAccountId)
- }
- toAccountId :=
tryToResolveAccountIdFromAccountLikeField(row.Field, row.TmpToAccountId,
row.ToValue, issueFieldMap)
- if toAccountId != "" {
- changelog.OriginalToValue =
accountIdGen.Generate(connectionId, toAccountId)
+ if v, ok := issueFieldMap[row.FieldId]; ok &&
v.SchemaType == "user" {
+ if row.FromValue != "" {
+ changelog.OriginalFromValue =
accountIdGen.Generate(connectionId, row.FromValue)
+ }
+ if row.ToValue != "" {
+ changelog.OriginalToValue =
accountIdGen.Generate(connectionId, row.ToValue)
+ }
}
}
-
return []interface{}{changelog}, nil
},
@@ -181,25 +173,6 @@ func ConvertIssueChangelogs(subtaskCtx
plugin.SubTaskContext) errors.Error {
return converter.Execute()
}
-func tryToResolveAccountIdFromAccountLikeField(fieldName string, tmpAccountId
string, fromOrToValue string, issueFieldMap map[string]models.JiraIssueField)
string {
- if tmpAccountId != "" {
- // process other account-like fields, it works on jira9 and
jira cloud.
- if fromOrToValue != "" {
- return fromOrToValue
- } else {
- return tmpAccountId
- }
- } else {
- // it works on jira8
- // notice: field name is not unique, but we cannot fetch field
id here.
- if v, ok := issueFieldMap[fieldName]; ok && v.SchemaType ==
"user" {
- // field type is account
- return fromOrToValue
- }
- }
- return ""
-}
-
func convertIds(ids string, connectionId uint64, sprintIdGenerator
*didgen.DomainIdGenerator) (string, errors.Error) {
ss := strings.Split(ids, ",")
var resultSlice []string
diff --git a/backend/plugins/jira/tasks/issue_changelog_extractor.go
b/backend/plugins/jira/tasks/issue_changelog_extractor.go
index f333f82c2..a49142e3f 100644
--- a/backend/plugins/jira/tasks/issue_changelog_extractor.go
+++ b/backend/plugins/jira/tasks/issue_changelog_extractor.go
@@ -43,6 +43,10 @@ func ExtractIssueChangelogs(subtaskCtx
plugin.SubTaskContext) errors.Error {
return nil
}
connectionId := data.Options.ConnectionId
+ userFieldMap, err := getUserFieldMap(subtaskCtx.GetDal(), connectionId,
subtaskCtx.GetLogger())
+ if err != nil {
+ return err
+ }
extractor, err :=
api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{
SubtaskCommonArgs: &api.SubtaskCommonArgs{
SubTaskContext: subtaskCtx,
@@ -77,7 +81,7 @@ func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext)
errors.Error {
// collect changelog_items
for _, item := range changelog.Items {
result = append(result,
item.ToToolLayer(connectionId, changelog.ID))
- extractedUsersFromChangelogItem :=
item.ExtractUser(connectionId)
+ extractedUsersFromChangelogItem :=
item.ExtractUser(connectionId, userFieldMap)
for _, u := range
extractedUsersFromChangelogItem {
if u != nil && u.AccountId != "" {
result = append(result, u)
diff --git a/backend/plugins/jira/tasks/issue_extractor.go
b/backend/plugins/jira/tasks/issue_extractor.go
index 7dcb6da76..3d097d691 100644
--- a/backend/plugins/jira/tasks/issue_extractor.go
+++ b/backend/plugins/jira/tasks/issue_extractor.go
@@ -58,6 +58,10 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext)
errors.Error {
if err != nil {
return err
}
+ userFieldMap, err := getUserFieldMap(db, connectionId, logger)
+ if err != nil {
+ return err
+ }
extractor, err :=
api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{
SubtaskCommonArgs: &api.SubtaskCommonArgs{
SubTaskContext: subtaskCtx,
@@ -69,7 +73,7 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext)
errors.Error {
SubtaskConfig: mappings,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- return extractIssues(data, mappings, row)
+ return extractIssues(data, mappings, row, userFieldMap)
},
})
if err != nil {
@@ -78,7 +82,7 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext)
errors.Error {
return extractor.Execute()
}
-func extractIssues(data *JiraTaskData, mappings *typeMappings, row
*api.RawData) ([]interface{}, errors.Error) {
+func extractIssues(data *JiraTaskData, mappings *typeMappings, row
*api.RawData, userFieldMaps map[string]struct{}) ([]interface{}, errors.Error) {
var apiIssue apiv2models.Issue
err := errors.Convert(json.Unmarshal(row.Data, &apiIssue))
if err != nil {
@@ -93,7 +97,7 @@ func extractIssues(data *JiraTaskData, mappings
*typeMappings, row *api.RawData)
if apiIssue.Fields.Created == nil {
return results, nil
}
- sprints, issue, comments, worklogs, changelogs, changelogItems, users
:= apiIssue.ExtractEntities(data.Options.ConnectionId)
+ sprints, issue, comments, worklogs, changelogs, changelogItems, users
:= apiIssue.ExtractEntities(data.Options.ConnectionId, userFieldMaps)
for _, sprintId := range sprints {
sprintIssue := &models.JiraSprintIssue{
ConnectionId: data.Options.ConnectionId,
