This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch fix#6075
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/fix#6075 by this push:
new 27abc2413 fix: github incremental collectors
27abc2413 is described below
commit 27abc2413c9614c6829bf72b7c25a4ff82815e1c
Author: abeizn <[email protected]>
AuthorDate: Wed Sep 13 19:08:14 2023 +0800
fix: github incremental collectors
---
backend/plugins/github/tasks/comment_collector.go | 5 ++---
backend/plugins/github/tasks/commit_collector.go | 5 ++---
backend/plugins/github/tasks/issue_collector.go | 5 ++---
backend/plugins/github/tasks/pr_review_comment_collector.go | 5 ++---
4 files changed, 8 insertions(+), 12 deletions(-)
diff --git a/backend/plugins/github/tasks/comment_collector.go
b/backend/plugins/github/tasks/comment_collector.go
index 21e57f853..f5add74bc 100644
--- a/backend/plugins/github/tasks/comment_collector.go
+++ b/backend/plugins/github/tasks/comment_collector.go
@@ -59,7 +59,6 @@ func CollectApiComments(taskCtx plugin.SubTaskContext)
errors.Error {
}
incremental := collectorWithState.IsIncremental()
- syncPolicy := taskCtx.TaskContext().SyncPolicy()
err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
ApiClient: data.ApiClient,
PageSize: 100,
@@ -69,11 +68,11 @@ func CollectApiComments(taskCtx plugin.SubTaskContext)
errors.Error {
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("state", "all")
- if syncPolicy != nil && syncPolicy.TimeAfter != nil {
+ if collectorWithState.TimeAfter != nil {
// Note that `since` is for filtering records
by the `updated` time
// which is not ideal for semantic reasons and
would result in slightly more records than expected.
// But we have no choice since it is the only
available field we could exploit from the API.
- query.Set("since",
syncPolicy.TimeAfter.String())
+ query.Set("since",
collectorWithState.TimeAfter.String())
}
// if incremental == true, we overwrite it
if incremental {
diff --git a/backend/plugins/github/tasks/commit_collector.go
b/backend/plugins/github/tasks/commit_collector.go
index c8ed10780..ca37c6227 100644
--- a/backend/plugins/github/tasks/commit_collector.go
+++ b/backend/plugins/github/tasks/commit_collector.go
@@ -59,7 +59,6 @@ func CollectApiCommits(taskCtx plugin.SubTaskContext)
errors.Error {
}
incremental := collectorWithState.IsIncremental()
- syncPolicy := taskCtx.TaskContext().SyncPolicy()
err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
ApiClient: data.ApiClient,
PageSize: 100,
@@ -80,8 +79,8 @@ func CollectApiCommits(taskCtx plugin.SubTaskContext)
errors.Error {
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("state", "all")
- if syncPolicy != nil && syncPolicy.TimeAfter != nil {
- query.Set("since",
syncPolicy.TimeAfter.String())
+ if collectorWithState.TimeAfter != nil {
+ query.Set("since",
collectorWithState.TimeAfter.String())
}
if incremental {
query.Set("since",
collectorWithState.LatestState.LatestSuccessStart.String())
diff --git a/backend/plugins/github/tasks/issue_collector.go
b/backend/plugins/github/tasks/issue_collector.go
index 551081c2d..c27eaea86 100644
--- a/backend/plugins/github/tasks/issue_collector.go
+++ b/backend/plugins/github/tasks/issue_collector.go
@@ -59,7 +59,6 @@ func CollectApiIssues(taskCtx plugin.SubTaskContext)
errors.Error {
}
incremental := collectorWithState.IsIncremental()
- syncPolicy := taskCtx.TaskContext().SyncPolicy()
err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
ApiClient: data.ApiClient,
PageSize: 100,
@@ -80,8 +79,8 @@ func CollectApiIssues(taskCtx plugin.SubTaskContext)
errors.Error {
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("state", "all")
- if syncPolicy != nil && syncPolicy.TimeAfter != nil {
- query.Set("since",
syncPolicy.TimeAfter.String())
+ if collectorWithState.TimeAfter != nil {
+ query.Set("since",
collectorWithState.TimeAfter.String())
}
if incremental {
query.Set("since",
collectorWithState.LatestState.LatestSuccessStart.String())
diff --git a/backend/plugins/github/tasks/pr_review_comment_collector.go
b/backend/plugins/github/tasks/pr_review_comment_collector.go
index bd03b032b..71c3a1904 100644
--- a/backend/plugins/github/tasks/pr_review_comment_collector.go
+++ b/backend/plugins/github/tasks/pr_review_comment_collector.go
@@ -62,7 +62,6 @@ func CollectPrReviewComments(taskCtx plugin.SubTaskContext)
errors.Error {
}
incremental := collectorWithState.IsIncremental()
- syncPolicy := taskCtx.TaskContext().SyncPolicy()
err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
ApiClient: data.ApiClient,
PageSize: 100,
@@ -77,11 +76,11 @@ func CollectPrReviewComments(taskCtx plugin.SubTaskContext)
errors.Error {
UrlTemplate: "repos/{{ .Params.Name }}/pulls/comments",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
- if syncPolicy != nil && syncPolicy.TimeAfter != nil {
+ if collectorWithState.TimeAfter != nil {
// Note that `since` is for filtering records
by the `updated` time
// which is not ideal for semantic reasons and
would result in slightly more records than expected.
// But we have no choice since it is the only
available field we could exploit from the API.
- query.Set("since",
syncPolicy.TimeAfter.String())
+ query.Set("since",
collectorWithState.TimeAfter.String())
}
// if incremental == true, we overwrite it
if incremental {
