warren830 commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r879451887
##########
plugins/gitlab/tasks/issue_convertor.go:
##########
@@ -0,0 +1,89 @@
+package tasks
+
+import (
+ "reflect"
+ "strconv"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/helper"
+
+ "github.com/merico-dev/lake/models/domainlayer"
+ "github.com/merico-dev/lake/models/domainlayer/didgen"
+ "github.com/merico-dev/lake/models/domainlayer/ticket"
+ gitlabModels "github.com/merico-dev/lake/plugins/gitlab/models"
+)
+
+var ConvertIssuesMeta = core.SubTaskMeta{
+ Name: "convertIssues",
+ EntryPoint: ConvertIssues,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitlab_issues into domain
layer table issues",
+}
+
+func ConvertIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ data := taskCtx.GetData().(*GitlabTaskData)
+ projectId := data.ProjectCommit.GitlabProjectId
+
+ issue := &gitlabModels.GitlabIssue{}
+ cursor, err := db.Model(issue).Where("project_id = ?", projectId).Rows()
+
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabIssue{})
+ userIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabUser{})
+ boardIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_ISSUE_TABLE,
+ },
+ InputRowType: reflect.TypeOf(gitlabModels.GitlabIssue{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ issue := inputRow.(*gitlabModels.GitlabIssue)
+ domainIssue := &ticket.Issue{
+ DomainEntity: domainlayer.DomainEntity{Id:
issueIdGen.Generate(issue.GitlabId)},
+ Number: strconv.Itoa(issue.Number),
+ Title: issue.Title,
+ Description: issue.Body,
+ Priority: issue.Priority,
+ Type: issue.Type,
+ AssigneeId:
userIdGen.Generate(issue.AssigneeId),
+ AssigneeName: issue.AssigneeName,
+ LeadTimeMinutes: issue.LeadTimeMinutes,
+ Url: issue.Url,
+ CreatedDate: &issue.GitlabCreatedAt,
+ UpdatedDate: &issue.GitlabUpdatedAt,
+ ResolutionDate: issue.ClosedAt,
+ Severity: issue.Severity,
+ Component: issue.Component,
+ }
+ if issue.State == "closed" {
+ domainIssue.Status = ticket.DONE
Review Comment:
same above
##########
plugins/gitlab/tasks/issue_extractor.go:
##########
@@ -0,0 +1,192 @@
+package tasks
+
+import (
+ "encoding/json"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/gitlab/models"
+ "github.com/merico-dev/lake/plugins/helper"
+)
+
+var ExtractApiIssuesMeta = core.SubTaskMeta{
+ Name: "extractApiIssues",
+ EntryPoint: ExtractApiIssues,
+ EnabledByDefault: true,
+ Description: "Extract raw Issues data into tool layer table
gitlab_issues",
+}
+
+type IssuesResponse struct {
+ ProjectId int `json:"id"`
+ Milestone struct {
+ Due_date string
+ Project_id int
+ State string
+ Description string
+ Iid int
+ Id int
+ Title string
+ CreatedAt *core.Iso8601Time
+ UpdatedAt *core.Iso8601Time
+ }
+ Author struct {
+ State string
+ WebUrl string
+ AvatarUrl string
+ Username string
+ Id int
+ Name string
+ }
+ Description string
+ State string
+ Iid int
+ Assignees []struct {
+ AvatarUrl string
+ WebUrl string
+ State string
+ Username string
+ Id int
+ Name string
+ }
+ Assignee *struct {
+ AvatarUrl string
+ WebUrl string
+ State string
+ Username string
+ Id int
+ Name string
+ }
+ Type string
+ Labels []string `json:"labels"`
+ UpVotes int
+ DownVotes int
+ MergeRequestsCount int
+ Id int
+ Title string
+ GitlabUpdatedAt core.Iso8601Time `json:"updated_at"`
+ GitlabCreatedAt core.Iso8601Time `json:"created_at"`
+ GitlabClosedAt *core.Iso8601Time `json:"closed_at"`
+ ClosedBy struct {
+ State string
+ WebUrl string
+ AvatarUrl string
+ Username string
+ Id int
+ Name string
+ }
+ UserNotesCount int
+ DueDate *core.Iso8601Time
+ WebUrl string `json:"web_url"`
+ References struct {
+ Short string
+ Relative string
+ Full string
+ }
+ TimeStats struct {
+ TimeEstimate int
+ TotalTimeSpent int
+ HumanTimeEstimate string
+ HumanTotalTimeSpent string
+ }
+ HasTasks bool
+ TaskStatus string
+ Confidential bool
+ DiscussionLocked bool
+ IssueType string
+ Serverity string
+ Links struct {
+ Self string `json:"url"`
+ Notes string
+ AwardEmoji string
+ Project string
+ }
+ TaskCompletionStatus struct {
+ Count int
+ CompletedCount int
+ }
+}
+
+func ExtractApiIssues(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*GitlabTaskData)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ /*
+ This struct will be JSONEncoded and stored into
database along with raw data itself, to identity minimal
+ set of data to be process, for example, we
process JiraIssues by Board
+ */
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ /*
+ Table store raw data
+ */
+ Table: RAW_ISSUE_TABLE,
+ },
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ body := &IssuesResponse{}
+ err := json.Unmarshal(row.Data, body)
+ if err != nil {
+ return nil, err
+ }
+ // need to extract 2 kinds of entities here
+ if body.ProjectId == 0 {
+ return nil, nil
+ }
+ ////If this is a pr, ignore
+ //if body.PullRequest.Url != "" {
+ // return nil, nil
+ //}
+ //If this is not Issue, ignore
+ if body.IssueType != "ISSUE" {
Review Comment:
please check if there is a IssueType
##########
plugins/gitlab/tasks/issue_convertor.go:
##########
@@ -0,0 +1,89 @@
+package tasks
+
+import (
+ "reflect"
+ "strconv"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/helper"
+
+ "github.com/merico-dev/lake/models/domainlayer"
+ "github.com/merico-dev/lake/models/domainlayer/didgen"
+ "github.com/merico-dev/lake/models/domainlayer/ticket"
+ gitlabModels "github.com/merico-dev/lake/plugins/gitlab/models"
+)
+
+var ConvertIssuesMeta = core.SubTaskMeta{
+ Name: "convertIssues",
+ EntryPoint: ConvertIssues,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitlab_issues into domain
layer table issues",
+}
+
+func ConvertIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ data := taskCtx.GetData().(*GitlabTaskData)
+ projectId := data.ProjectCommit.GitlabProjectId
+
+ issue := &gitlabModels.GitlabIssue{}
+ cursor, err := db.Model(issue).Where("project_id = ?", projectId).Rows()
+
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabIssue{})
+ userIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabUser{})
+ boardIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_ISSUE_TABLE,
+ },
+ InputRowType: reflect.TypeOf(gitlabModels.GitlabIssue{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ issue := inputRow.(*gitlabModels.GitlabIssue)
+ domainIssue := &ticket.Issue{
+ DomainEntity: domainlayer.DomainEntity{Id:
issueIdGen.Generate(issue.GitlabId)},
+ Number: strconv.Itoa(issue.Number),
+ Title: issue.Title,
+ Description: issue.Body,
+ Priority: issue.Priority,
+ Type: issue.Type,
+ AssigneeId:
userIdGen.Generate(issue.AssigneeId),
+ AssigneeName: issue.AssigneeName,
+ LeadTimeMinutes: issue.LeadTimeMinutes,
+ Url: issue.Url,
+ CreatedDate: &issue.GitlabCreatedAt,
+ UpdatedDate: &issue.GitlabUpdatedAt,
+ ResolutionDate: issue.ClosedAt,
+ Severity: issue.Severity,
+ Component: issue.Component,
+ }
+ if issue.State == "closed" {
Review Comment:
and there is a OriginalStatus need to be assigned value
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
