This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/devlake.git
The following commit(s) were added to refs/heads/main by this push:
new d69bc9eb0 fix(asana): page through stories, subtasks and tags and stop
on null next_page (#9366)
d69bc9eb0 is described below
commit d69bc9eb082a8e81c1c49b9721b1b2f87d39976f
Author: Shivam Sharma <[email protected]>
AuthorDate: Thu Nov 12 07:41:14 2026 +0530
fix(asana): page through stories, subtasks and tags and stop on null
next_page (#9366)
The story, subtask and tag collectors sent limit=100 without a PageSize
or GetNextPageCustomData, so only the first 100 stories, subtasks and
tags of each task were collected. The task and user collectors returned
nil, nil when next_page was null, which made the collector request
page 1 again without an offset whenever the last page was full.
Add getNextAsanaPageOffset, which ends the collection when next_page is
null, and setAsanaPageQuery, which sets limit and offset, and use them
in all five collectors.
Closes #9365
Generated-by: Claude Code (Claude Opus 5.5)
Signed-off-by: Shivam Sharma
<[email protected]>
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
backend/plugins/asana/tasks/shared.go | 54 ++++++++++++++++++++
backend/plugins/asana/tasks/shared_test.go | 65 ++++++++++++++++++++++++
backend/plugins/asana/tasks/story_collector.go | 4 +-
backend/plugins/asana/tasks/subtask_collector.go | 4 +-
backend/plugins/asana/tasks/tag_collector.go | 4 +-
backend/plugins/asana/tasks/task_collector.go | 21 ++------
backend/plugins/asana/tasks/user_collector.go | 21 ++------
7 files changed, 134 insertions(+), 39 deletions(-)
diff --git a/backend/plugins/asana/tasks/shared.go
b/backend/plugins/asana/tasks/shared.go
new file mode 100644
index 000000000..3d70c35b1
--- /dev/null
+++ b/backend/plugins/asana/tasks/shared.go
@@ -0,0 +1,54 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/devlake/core/errors"
+ "github.com/apache/devlake/helpers/pluginhelper/api"
+)
+
+// asanaPageSize is the largest page Asana list endpoints return.
+const asanaPageSize = 100
+
+// setAsanaPageQuery sets the page size and, after the first page, the offset
+// that the previous page returned in next_page.
+func setAsanaPageQuery(query url.Values, reqData *api.RequestData) {
+ query.Set("limit", fmt.Sprintf("%d", reqData.Pager.Size))
+ if offset, ok := reqData.CustomData.(string); ok && offset != "" {
+ query.Set("offset", offset)
+ }
+}
+
+// getNextAsanaPageOffset returns the offset of the next page. Asana sets
+// next_page to null when there are no more pages, so the collection ends
+// there: requesting a page without an offset would start again at page 1.
+func getNextAsanaPageOffset(_ *api.RequestData, prevPageResponse
*http.Response) (interface{}, errors.Error) {
+ var resp asanaListResponse
+ err := api.UnmarshalResponse(prevPageResponse, &resp)
+ if err != nil {
+ return nil, err
+ }
+ if resp.NextPage == nil || resp.NextPage.Offset == "" {
+ return nil, api.ErrFinishCollect
+ }
+ return resp.NextPage.Offset, nil
+}
diff --git a/backend/plugins/asana/tasks/shared_test.go
b/backend/plugins/asana/tasks/shared_test.go
new file mode 100644
index 000000000..f1a681e74
--- /dev/null
+++ b/backend/plugins/asana/tasks/shared_test.go
@@ -0,0 +1,65 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "io"
+ "net/http"
+ "net/url"
+ "strings"
+ "testing"
+
+ "github.com/apache/devlake/helpers/pluginhelper/api"
+ "github.com/stretchr/testify/assert"
+)
+
+func asanaListPage(t *testing.T, body string) *http.Response {
+ req, err := http.NewRequest(http.MethodGet,
"https://app.asana.com/api/1.0/tasks/1/stories", nil)
+ assert.Nil(t, err)
+ return &http.Response{
+ StatusCode: http.StatusOK,
+ Body: io.NopCloser(strings.NewReader(body)),
+ Request: req,
+ }
+}
+
+func TestSetAsanaPageQuery(t *testing.T) {
+ first := url.Values{}
+ setAsanaPageQuery(first, &api.RequestData{Pager: &api.Pager{Page: 1,
Size: asanaPageSize}})
+ assert.Equal(t, "100", first.Get("limit"))
+ assert.False(t, first.Has("offset"))
+
+ next := url.Values{}
+ setAsanaPageQuery(next, &api.RequestData{Pager: &api.Pager{Page: 2,
Size: asanaPageSize}, CustomData: "eyJ0eXAiOJiKV1iQLCJhbGciOiJIUzI1NiJ9"})
+ assert.Equal(t, "100", next.Get("limit"))
+ assert.Equal(t, "eyJ0eXAiOJiKV1iQLCJhbGciOiJIUzI1NiJ9",
next.Get("offset"))
+}
+
+func TestGetNextAsanaPageOffset(t *testing.T) {
+ offset, err := getNextAsanaPageOffset(nil, asanaListPage(t, `{
+ "data": [{"gid": "1"}],
+ "next_page": {"offset": "eyJ0eXAiOJiKV1iQLCJhbGciOiJIUzI1NiJ9",
"path":
"/tasks/1/stories?limit=100&offset=eyJ0eXAiOJiKV1iQLCJhbGciOiJIUzI1NiJ9",
"uri":
"https://app.asana.com/api/1.0/tasks/1/stories?limit=100&offset=eyJ0eXAiOJiKV1iQLCJhbGciOiJIUzI1NiJ9"}
+ }`))
+ assert.Nil(t, err)
+ assert.Equal(t, "eyJ0eXAiOJiKV1iQLCJhbGciOiJIUzI1NiJ9", offset)
+
+ // A full last page has "next_page": null. Going on without an offset
+ // would request page 1 again, so the collection must end here.
+ _, err = getNextAsanaPageOffset(nil, asanaListPage(t, `{"data":
[{"gid": "1"}], "next_page": null}`))
+ assert.ErrorIs(t, err, api.ErrFinishCollect)
+}
diff --git a/backend/plugins/asana/tasks/story_collector.go
b/backend/plugins/asana/tasks/story_collector.go
index f689c9f69..780157901 100644
--- a/backend/plugins/asana/tasks/story_collector.go
+++ b/backend/plugins/asana/tasks/story_collector.go
@@ -73,13 +73,15 @@ func CollectStory(taskCtx plugin.SubTaskContext)
errors.Error {
},
ApiClient: data.ApiClient,
Input: iterator,
+ PageSize: asanaPageSize,
UrlTemplate: "tasks/{{ .Input.Gid }}/stories",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("opt_fields",
"gid,resource_type,resource_subtype,text,html_text,is_pinned,is_edited,sticker_name,created_at,created_by,target")
- query.Set("limit", "100")
+ setAsanaPageQuery(query, reqData)
return query, nil
},
+ GetNextPageCustomData: getNextAsanaPageOffset,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
var resp asanaListResponse
err := api.UnmarshalResponse(res, &resp)
diff --git a/backend/plugins/asana/tasks/subtask_collector.go
b/backend/plugins/asana/tasks/subtask_collector.go
index 73218dc2d..871ee6b05 100644
--- a/backend/plugins/asana/tasks/subtask_collector.go
+++ b/backend/plugins/asana/tasks/subtask_collector.go
@@ -74,13 +74,15 @@ func CollectSubtask(taskCtx plugin.SubTaskContext)
errors.Error {
},
ApiClient: data.ApiClient,
Input: iterator,
+ PageSize: asanaPageSize,
UrlTemplate: "tasks/{{ .Input.Gid }}/subtasks",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("opt_fields",
"gid,name,notes,resource_type,resource_subtype,completed,completed_at,due_on,created_at,modified_at,permalink_url,assignee,created_by,parent,num_subtasks,memberships.section,memberships.project")
- query.Set("limit", "100")
+ setAsanaPageQuery(query, reqData)
return query, nil
},
+ GetNextPageCustomData: getNextAsanaPageOffset,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
var resp asanaListResponse
err := api.UnmarshalResponse(res, &resp)
diff --git a/backend/plugins/asana/tasks/tag_collector.go
b/backend/plugins/asana/tasks/tag_collector.go
index 01d3b18e4..a038437a9 100644
--- a/backend/plugins/asana/tasks/tag_collector.go
+++ b/backend/plugins/asana/tasks/tag_collector.go
@@ -73,13 +73,15 @@ func CollectTag(taskCtx plugin.SubTaskContext) errors.Error
{
},
ApiClient: data.ApiClient,
Input: iterator,
+ PageSize: asanaPageSize,
UrlTemplate: "tasks/{{ .Input.Gid }}/tags",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("opt_fields",
"gid,name,resource_type,color,notes,permalink_url")
- query.Set("limit", "100")
+ setAsanaPageQuery(query, reqData)
return query, nil
},
+ GetNextPageCustomData: getNextAsanaPageOffset,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
var resp asanaListResponse
err := api.UnmarshalResponse(res, &resp)
diff --git a/backend/plugins/asana/tasks/task_collector.go
b/backend/plugins/asana/tasks/task_collector.go
index e21b30c90..8287392f2 100644
--- a/backend/plugins/asana/tasks/task_collector.go
+++ b/backend/plugins/asana/tasks/task_collector.go
@@ -61,31 +61,16 @@ func CollectTask(taskCtx plugin.SubTaskContext)
errors.Error {
Table: rawTaskTable,
},
ApiClient: data.ApiClient,
- PageSize: 100,
+ PageSize: asanaPageSize,
UrlTemplate: "projects/{{ .Params.ProjectId }}/tasks",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
- query.Set("limit", "100")
// Request all fields needed for transformation
including section name
query.Set("opt_fields",
"gid,name,notes,resource_type,resource_subtype,completed,completed_at,due_on,created_at,modified_at,permalink_url,assignee,assignee.name,created_by,created_by.name,parent,num_subtasks,memberships.section,memberships.section.name,memberships.project")
- if reqData.CustomData != nil {
- if offset, ok := reqData.CustomData.(string);
ok && offset != "" {
- query.Set("offset", offset)
- }
- }
+ setAsanaPageQuery(query, reqData)
return query, nil
},
- GetNextPageCustomData: func(prevReqData *api.RequestData,
prevPageResponse *http.Response) (interface{}, errors.Error) {
- var resp asanaTaskListResponse
- err := api.UnmarshalResponse(prevPageResponse, &resp)
- if err != nil {
- return nil, err
- }
- if resp.NextPage != nil && resp.NextPage.Offset != "" {
- return resp.NextPage.Offset, nil
- }
- return nil, nil
- },
+ GetNextPageCustomData: getNextAsanaPageOffset,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
var w asanaTaskListResponse
err := api.UnmarshalResponse(res, &w)
diff --git a/backend/plugins/asana/tasks/user_collector.go
b/backend/plugins/asana/tasks/user_collector.go
index bcfc98591..65c667028 100644
--- a/backend/plugins/asana/tasks/user_collector.go
+++ b/backend/plugins/asana/tasks/user_collector.go
@@ -52,30 +52,15 @@ func CollectUser(taskCtx plugin.SubTaskContext)
errors.Error {
Table: rawUserTable,
},
ApiClient: data.ApiClient,
- PageSize: 100,
+ PageSize: asanaPageSize,
UrlTemplate: "projects/{{ .Params.ProjectId }}/members",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("opt_fields",
"gid,name,email,resource_type,photo.image_128x128")
- query.Set("limit", "100")
- if reqData.CustomData != nil {
- if offset, ok := reqData.CustomData.(string);
ok && offset != "" {
- query.Set("offset", offset)
- }
- }
+ setAsanaPageQuery(query, reqData)
return query, nil
},
- GetNextPageCustomData: func(prevReqData *api.RequestData,
prevPageResponse *http.Response) (interface{}, errors.Error) {
- var resp asanaListResponse
- err := api.UnmarshalResponse(prevPageResponse, &resp)
- if err != nil {
- return nil, err
- }
- if resp.NextPage != nil && resp.NextPage.Offset != "" {
- return resp.NextPage.Offset, nil
- }
- return nil, nil
- },
+ GetNextPageCustomData: getNextAsanaPageOffset,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
var resp asanaListResponse
err := api.UnmarshalResponse(res, &resp)