This is an automated email from the ASF dual-hosted git repository. warren pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 0ff0746a0ac423ad4279aa14fed5f04044589644 Author: zhangliang <[email protected]> AuthorDate: Tue May 17 22:53:25 2022 +0800 fix: extract jira user from assignee changelog --- plugins/jira/jira.go | 3 ++ plugins/jira/tasks/apiv2models/changelog.go | 16 +++++++- plugins/jira/tasks/apiv2models/issue.go | 3 ++ plugins/jira/tasks/apiv2models/user_test.go | 64 +++++++++++++++++++++++++++++ plugins/jira/tasks/changelog_extractor.go | 3 ++ plugins/jira/tasks/user_collector.go | 30 ++++++++++++-- 6 files changed, 114 insertions(+), 5 deletions(-) diff --git a/plugins/jira/jira.go b/plugins/jira/jira.go index 89b16f64..31e82922 100644 --- a/plugins/jira/jira.go +++ b/plugins/jira/jira.go @@ -49,6 +49,8 @@ func (plugin Jira) SubTaskMetas() []core.SubTaskMeta { {Name: "collectChangelogs", EntryPoint: tasks.CollectChangelogs, EnabledByDefault: true, Description: "collect Jira change logs"}, {Name: "extractChangelogs", EntryPoint: tasks.ExtractChangelogs, EnabledByDefault: true, Description: "extract Jira change logs"}, + {Name: "collectUsers", EntryPoint: tasks.CollectUsers, EnabledByDefault: true, Description: "collect Jira users"}, + {Name: "collectWorklogs", EntryPoint: tasks.CollectWorklogs, EnabledByDefault: true, Description: "collect Jira work logs"}, {Name: "extractWorklogs", EntryPoint: tasks.ExtractWorklogs, EnabledByDefault: true, Description: "extract Jira work logs"}, @@ -71,6 +73,7 @@ func (plugin Jira) SubTaskMetas() []core.SubTaskMeta { {Name: "convertIssueCommits", EntryPoint: tasks.ConvertIssueCommits, EnabledByDefault: true, Description: "convert Jira issue commits"}, {Name: "convertIssueRepoCommits", EntryPoint: tasks.ConvertIssueRepoCommits, EnabledByDefault: false, Description: "convert Jira issue repo commits"}, + {Name: "extractUsers", EntryPoint: tasks.ExtractUsers, EnabledByDefault: true, Description: "extract Jira users"}, {Name: "convertUsers", EntryPoint: tasks.ConvertUsers, EnabledByDefault: true, Description: "convert Jira users"}, } } diff --git a/plugins/jira/tasks/apiv2models/changelog.go b/plugins/jira/tasks/apiv2models/changelog.go index 223ee3b6..2d5fc76f 100644 --- a/plugins/jira/tasks/apiv2models/changelog.go +++ b/plugins/jira/tasks/apiv2models/changelog.go @@ -18,7 +18,7 @@ func (c Changelog) ToToolLayer(connectionId, issueId uint64) (*models.JiraChange ConnectionId: connectionId, ChangelogId: c.ID, IssueId: issueId, - AuthorAccountId: c.Author.EmailAddress, + AuthorAccountId: c.Author.getAccountId(), AuthorDisplayName: c.Author.DisplayName, AuthorActive: c.Author.Active, Created: c.Created.ToTime(), @@ -46,3 +46,17 @@ func (c ChangelogItem) ToToolLayer(connectionId, changelogId uint64) *models.Jir ToString: c.ToString, } } + +func (c ChangelogItem) ExtractUser(connectionId uint64) []*models.JiraUser { + if c.Field != "assignee" { + return nil + } + var result []*models.JiraUser + if c.From != "" { + result = append(result, &models.JiraUser{ConnectionId: connectionId, AccountId: c.From}) + } + if c.To != "" { + result = append(result, &models.JiraUser{ConnectionId: connectionId, AccountId: c.To}) + } + return result +} diff --git a/plugins/jira/tasks/apiv2models/issue.go b/plugins/jira/tasks/apiv2models/issue.go index 41a2e502..d1001471 100644 --- a/plugins/jira/tasks/apiv2models/issue.go +++ b/plugins/jira/tasks/apiv2models/issue.go @@ -234,6 +234,9 @@ func (i Issue) ExtractEntities(connectionId uint64, epicField, storyPointField s users = append(users, user) for _, item := range changelog.Items { changelogItems = append(changelogItems, item.ToToolLayer(connectionId, changelog.ID)) + for _, u := range item.ExtractUser(connectionId) { + users = append(users, u) + } } } } diff --git a/plugins/jira/tasks/apiv2models/user_test.go b/plugins/jira/tasks/apiv2models/user_test.go new file mode 100644 index 00000000..8486b7bd --- /dev/null +++ b/plugins/jira/tasks/apiv2models/user_test.go @@ -0,0 +1,64 @@ +package apiv2models + +import "testing" + +func TestUser_getAccountId(t *testing.T) { + type fields struct { + Self string + Key string + Name string + EmailAddress string + AccountId string + AccountType string + AvatarUrls struct { + Four8X48 string `json:"48x48"` + Two4X24 string `json:"24x24"` + One6X16 string `json:"16x16"` + Three2X32 string `json:"32x32"` + } + DisplayName string + Active bool + Deleted bool + TimeZone string + Locale string + } + tests := []struct { + name string + fields fields + want string + }{ + {"", + fields{EmailAddress: "abc"}, + "abc", + }, + {"", + fields{EmailAddress: "abc", AccountId: "abcd"}, + "abcd", + }, + {"", + fields{AccountId: "abcd"}, + "abcd", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + u := &User{ + Self: tt.fields.Self, + Key: tt.fields.Key, + Name: tt.fields.Name, + EmailAddress: tt.fields.EmailAddress, + AccountId: tt.fields.AccountId, + AccountType: tt.fields.AccountType, + AvatarUrls: tt.fields.AvatarUrls, + DisplayName: tt.fields.DisplayName, + Active: tt.fields.Active, + Deleted: tt.fields.Deleted, + TimeZone: tt.fields.TimeZone, + Locale: tt.fields.Locale, + } + if got := u.getAccountId(); got != tt.want { + t.Errorf("getAccountId() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/plugins/jira/tasks/changelog_extractor.go b/plugins/jira/tasks/changelog_extractor.go index 275c4c55..8cfefa67 100644 --- a/plugins/jira/tasks/changelog_extractor.go +++ b/plugins/jira/tasks/changelog_extractor.go @@ -48,6 +48,9 @@ func ExtractChangelogs(taskCtx core.SubTaskContext) error { result = append(result, cl, user) for _, item := range changelog.Items { result = append(result, item.ToToolLayer(connectionId, changelog.ID)) + for _, u := range item.ExtractUser(connectionId) { + result = append(result, u) + } } return result, nil }, diff --git a/plugins/jira/tasks/user_collector.go b/plugins/jira/tasks/user_collector.go index b905ddb7..748f4af4 100644 --- a/plugins/jira/tasks/user_collector.go +++ b/plugins/jira/tasks/user_collector.go @@ -3,17 +3,33 @@ package tasks import ( "encoding/json" "net/http" + "net/url" + "reflect" "github.com/merico-dev/lake/plugins/core" "github.com/merico-dev/lake/plugins/helper" + "github.com/merico-dev/lake/plugins/jira/models" ) const RAW_USERS_TABLE = "jira_api_users" func CollectUsers(taskCtx core.SubTaskContext) error { data := taskCtx.GetData().(*JiraTaskData) + db := taskCtx.GetDb() logger := taskCtx.GetLogger() logger.Info("collect user") + cursor, err := db.Model(&models.JiraUser{}).Where("connection_id = ?", data.Options.ConnectionId).Rows() + if err != nil { + return err + } + iterator, err := helper.NewCursorIterator(db, cursor, reflect.TypeOf(models.JiraUser{})) + if err != nil { + return err + } + queryKey := "accountId" + if data.JiraServerInfo.DeploymentType == models.DeploymentServer { + queryKey = "username" + } collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ Ctx: taskCtx, @@ -24,16 +40,22 @@ func CollectUsers(taskCtx core.SubTaskContext) error { Table: RAW_USERS_TABLE, }, ApiClient: data.ApiClient, - UrlTemplate: "api/3/users/search", - PageSize: 50, + Input: iterator, + UrlTemplate: "api/2/user", + Query: func(reqData *helper.RequestData) (url.Values, error) { + user := reqData.Input.(*models.JiraUser) + query := url.Values{} + query.Set(queryKey, user.AccountId) + return query, nil + }, Concurrency: 10, ResponseParser: func(res *http.Response) ([]json.RawMessage, error) { - var result []json.RawMessage + var result json.RawMessage err := helper.UnmarshalResponse(res, &result) if err != nil { return nil, err } - return result, nil + return []json.RawMessage{result}, nil }, }) if err != nil {
