warren830 commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r879450677
##########
plugins/gitlab/tasks/issue_collector.go:
##########
@@ -0,0 +1,127 @@
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/merico-dev/lake/plugins/helper"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/gitlab/models"
+)
+
+const RAW_ISSUE_TABLE = "gitlab_api_issues"
+
+//This Struct was declared in shared.go
+//type GitlabApiParams struct {
+// ProjectId int
+//}
+
+var CollectApiIssuesMeta = core.SubTaskMeta{
+ Name: "collectApiIssues",
+ EntryPoint: CollectApiIssues,
+ EnabledByDefault: true,
+ Description: "Collect issues data from Gitlab api",
+}
+
+func CollectApiIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ data := taskCtx.GetData().(*GitlabTaskData)
+
+ since := data.Since
+ incremental := false
+ // user didn't specify a time range to sync, try load from database
+ if since == nil {
+ var latestUpdated models.GitlabIssue
+ err := db.Model(&latestUpdated).
+ Where("project_id = ?",
data.ProjectCommit.GitlabProjectId).
+ Order("gitlab_updated_at
DESC").Limit(1).Find(&latestUpdated).Error
+
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitlab issue
record: %w", err)
+ }
+ if latestUpdated.GitlabId > 0 {
+ since = &latestUpdated.GitlabUpdatedAt
+ incremental = true
+ }
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ /*
+ This struct will be JSONEncoded and stored into
database along with raw data itself, to identity minimal
+ set of data to be process, for example, we
process JiraIssues by Board
+ */
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ /*
+ Table store raw data
+ */
+ Table: RAW_ISSUE_TABLE,
+ },
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: incremental,
+ /*
+ url may use arbitrary variables from different source
in any order, we need GoTemplate to allow more
+ flexible for all kinds of possibility.
+ Pager contains information for a particular page,
calculated by ApiCollector, and will be passed into
+ GoTemplate to generate a url for that page.
+ We want to do page-fetching in ApiCollector, because
the logic are highly similar, by doing so, we can
+ avoid duplicate logic for every tasks, and when we have
a better idea like improving performance, we can
+ do it in one place
+ */
+ UrlTemplate: "projects/{{ .Params.ProjectId }}/issues",
+ /*
+ (Optional) Return query string for request, or you can
plug them into UrlTemplate directly
+ */
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("state", "all")
+ if since != nil {
+ query.Set("since", since.String())
+ }
+ query.Set("direction", "asc")
Review Comment:
please check if gitlab use `direction` and `since`, I think it might use
other param names for this
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
