This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new b823baba4 fix: tapd lack of pagesize conf (#4557)
b823baba4 is described below
commit b823baba42cf727fa9f099e066d9401a766a2ac8
Author: 冯智泉 <[email protected]>
AuthorDate: Tue Mar 7 12:02:55 2023 +0800
fix: tapd lack of pagesize conf (#4557)
* fix: tapd lack of pagesize conf
* fix: tapd lack of pagesize conf, fix review comments
---
backend/plugins/tapd/impl/impl.go | 3 +++
backend/plugins/tapd/tasks/bug_changelog_collector.go | 5 +++--
backend/plugins/tapd/tasks/iteration_collector.go | 7 ++++---
backend/plugins/tapd/tasks/story_changelog_collector.go | 5 +++--
backend/plugins/tapd/tasks/story_collector.go | 5 +++--
backend/plugins/tapd/tasks/story_status_last_step_collector.go | 5 +++--
backend/plugins/tapd/tasks/task_changelog_collector.go | 5 +++--
backend/plugins/tapd/tasks/task_collector.go | 5 +++--
backend/plugins/tapd/tasks/task_data.go | 1 +
backend/plugins/tapd/tasks/worklog_collector.go | 5 +++--
10 files changed, 29 insertions(+), 17 deletions(-)
diff --git a/backend/plugins/tapd/impl/impl.go
b/backend/plugins/tapd/impl/impl.go
index 05e7c20b2..f8bab51fb 100644
--- a/backend/plugins/tapd/impl/impl.go
+++ b/backend/plugins/tapd/impl/impl.go
@@ -187,6 +187,9 @@ func (p Tapd) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]int
if err != nil {
return nil, errors.Default.Wrap(err, "failed to create tapd api
client")
}
+ if op.PageSize == 0 {
+ op.PageSize = 100
+ }
cstZone, err1 := time.LoadLocation("Asia/Shanghai")
if err1 != nil {
return nil, errors.Default.Wrap(err1, "fail to get CST
Location")
diff --git a/backend/plugins/tapd/tasks/bug_changelog_collector.go
b/backend/plugins/tapd/tasks/bug_changelog_collector.go
index d47459b09..9bb39d3a0 100644
--- a/backend/plugins/tapd/tasks/bug_changelog_collector.go
+++ b/backend/plugins/tapd/tasks/bug_changelog_collector.go
@@ -19,10 +19,11 @@ package tasks
import (
"fmt"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/url"
)
const RAW_BUG_CHANGELOG_TABLE = "tapd_api_bug_changelogs"
@@ -42,7 +43,7 @@ func CollectBugChangelogs(taskCtx plugin.SubTaskContext)
errors.Error {
err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
Incremental: incremental,
ApiClient: data.ApiClient,
- PageSize: 100,
+ PageSize: int(data.Options.PageSize),
UrlTemplate: "bug_changes",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
diff --git a/backend/plugins/tapd/tasks/iteration_collector.go
b/backend/plugins/tapd/tasks/iteration_collector.go
index e5d82d04c..65d9c83dc 100644
--- a/backend/plugins/tapd/tasks/iteration_collector.go
+++ b/backend/plugins/tapd/tasks/iteration_collector.go
@@ -20,11 +20,12 @@ package tasks
import (
"encoding/json"
"fmt"
+ "net/http"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/http"
- "net/url"
)
const RAW_ITERATION_TABLE = "tapd_api_iterations"
@@ -43,7 +44,7 @@ func CollectIterations(taskCtx plugin.SubTaskContext)
errors.Error {
err = collectorWithState.InitCollector(api.ApiCollectorArgs{
Incremental: incremental,
ApiClient: data.ApiClient,
- PageSize: 100,
+ PageSize: int(data.Options.PageSize),
Concurrency: 3,
UrlTemplate: "iterations",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
diff --git a/backend/plugins/tapd/tasks/story_changelog_collector.go
b/backend/plugins/tapd/tasks/story_changelog_collector.go
index 5f492251c..16818793e 100644
--- a/backend/plugins/tapd/tasks/story_changelog_collector.go
+++ b/backend/plugins/tapd/tasks/story_changelog_collector.go
@@ -19,10 +19,11 @@ package tasks
import (
"fmt"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/url"
)
const RAW_STORY_CHANGELOG_TABLE = "tapd_api_story_changelogs"
@@ -42,7 +43,7 @@ func CollectStoryChangelogs(taskCtx plugin.SubTaskContext)
errors.Error {
err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
Incremental: incremental,
ApiClient: data.ApiClient,
- PageSize: 100,
+ PageSize: int(data.Options.PageSize),
UrlTemplate: "story_changes",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
diff --git a/backend/plugins/tapd/tasks/story_collector.go
b/backend/plugins/tapd/tasks/story_collector.go
index 7fdf6d184..00b48ac96 100644
--- a/backend/plugins/tapd/tasks/story_collector.go
+++ b/backend/plugins/tapd/tasks/story_collector.go
@@ -19,10 +19,11 @@ package tasks
import (
"fmt"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/url"
)
const RAW_STORY_TABLE = "tapd_api_stories"
@@ -41,7 +42,7 @@ func CollectStorys(taskCtx plugin.SubTaskContext)
errors.Error {
err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
Incremental: incremental,
ApiClient: data.ApiClient,
- PageSize: 100,
+ PageSize: int(data.Options.PageSize),
UrlTemplate: "stories",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
diff --git a/backend/plugins/tapd/tasks/story_status_last_step_collector.go
b/backend/plugins/tapd/tasks/story_status_last_step_collector.go
index 2709109c9..f51d9c542 100644
--- a/backend/plugins/tapd/tasks/story_status_last_step_collector.go
+++ b/backend/plugins/tapd/tasks/story_status_last_step_collector.go
@@ -19,10 +19,11 @@ package tasks
import (
"fmt"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/url"
)
const RAW_STORY_STATUS_LAST_STEP_TABLE = "tapd_api_story_status_last_steps"
@@ -37,7 +38,7 @@ func CollectStoryStatusLastStep(taskCtx
plugin.SubTaskContext) errors.Error {
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
- PageSize: 100,
+ PageSize: int(data.Options.PageSize),
UrlTemplate: "workflows/last_steps",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
diff --git a/backend/plugins/tapd/tasks/task_changelog_collector.go
b/backend/plugins/tapd/tasks/task_changelog_collector.go
index b9e256794..87200c164 100644
--- a/backend/plugins/tapd/tasks/task_changelog_collector.go
+++ b/backend/plugins/tapd/tasks/task_changelog_collector.go
@@ -19,10 +19,11 @@ package tasks
import (
"fmt"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/url"
)
const RAW_TASK_CHANGELOG_TABLE = "tapd_api_task_changelogs"
@@ -42,7 +43,7 @@ func CollectTaskChangelogs(taskCtx plugin.SubTaskContext)
errors.Error {
err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
Incremental: incremental,
ApiClient: data.ApiClient,
- PageSize: 100,
+ PageSize: int(data.Options.PageSize),
UrlTemplate: "task_changes",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
diff --git a/backend/plugins/tapd/tasks/task_collector.go
b/backend/plugins/tapd/tasks/task_collector.go
index 95d9fcc61..84f52e867 100644
--- a/backend/plugins/tapd/tasks/task_collector.go
+++ b/backend/plugins/tapd/tasks/task_collector.go
@@ -19,10 +19,11 @@ package tasks
import (
"fmt"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/url"
)
const RAW_TASK_TABLE = "tapd_api_tasks"
@@ -43,7 +44,7 @@ func CollectTasks(taskCtx plugin.SubTaskContext) errors.Error
{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Incremental: incremental,
ApiClient: data.ApiClient,
- PageSize: 100,
+ PageSize: int(data.Options.PageSize),
UrlTemplate: "tasks",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
diff --git a/backend/plugins/tapd/tasks/task_data.go
b/backend/plugins/tapd/tasks/task_data.go
index ed50375de..fc04aa875 100644
--- a/backend/plugins/tapd/tasks/task_data.go
+++ b/backend/plugins/tapd/tasks/task_data.go
@@ -28,6 +28,7 @@ type TapdOptions struct {
ConnectionId uint64 `mapstruct:"connectionId"`
WorkspaceId uint64 `mapstruct:"workspaceId"`
CompanyId uint64 `mapstruct:"companyId"`
+ PageSize uint64 `mapstruct:"pageSize"`
TimeAfter string `json:"timeAfter"
mapstructure:"timeAfter,omitempty"`
CstZone *time.Location
TransformationRules TransformationRules `json:"transformationRules"`
diff --git a/backend/plugins/tapd/tasks/worklog_collector.go
b/backend/plugins/tapd/tasks/worklog_collector.go
index 89916416b..9ac2da22b 100644
--- a/backend/plugins/tapd/tasks/worklog_collector.go
+++ b/backend/plugins/tapd/tasks/worklog_collector.go
@@ -19,10 +19,11 @@ package tasks
import (
"fmt"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/url"
)
const RAW_WORKLOG_TABLE = "tapd_api_worklogs"
@@ -42,7 +43,7 @@ func CollectWorklogs(taskCtx plugin.SubTaskContext)
errors.Error {
err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
Incremental: incremental,
ApiClient: data.ApiClient,
- PageSize: 100,
+ PageSize: int(data.Options.PageSize),
UrlTemplate: "timesheets",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
