This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 0133b9ed0 fix(taiga): page through epics, issues, tasks and user
stories (#9341)
0133b9ed0 is described below
commit 0133b9ed01960e9f9f46f254b75cbe5bd8c48773
Author: Shivam Sharma <[email protected]>
AuthorDate: Sun Nov 8 19:58:04 2026 +0530
fix(taiga): page through epics, issues, tasks and user stories (#9341)
The four collectors sent only the project filter. Taiga list endpoints
are paginated (30 items per page by default) and return page 1 when no
page is given, so only the first 30 items of a project were collected,
and the parallel page fetches stored the same page ten times. Send page
and page_size, and stop when Taiga reports no next page, since a page
past the last one returns 404.
Closes #9340
Generated-by: Claude Code (Claude Opus 5.5)
Signed-off-by: Shivam Sharma
<[email protected]>
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
backend/plugins/taiga/tasks/epic_collector.go | 15 ++----
backend/plugins/taiga/tasks/issue_collector.go | 15 ++----
backend/plugins/taiga/tasks/shared.go | 53 ++++++++++++++++++++++
backend/plugins/taiga/tasks/shared_test.go | 50 ++++++++++++++++++++
backend/plugins/taiga/tasks/task_collector.go | 15 ++----
.../plugins/taiga/tasks/user_story_collector.go | 16 ++-----
6 files changed, 123 insertions(+), 41 deletions(-)
diff --git a/backend/plugins/taiga/tasks/epic_collector.go
b/backend/plugins/taiga/tasks/epic_collector.go
index 4deec5c09..40a5445f7 100644
--- a/backend/plugins/taiga/tasks/epic_collector.go
+++ b/backend/plugins/taiga/tasks/epic_collector.go
@@ -19,9 +19,7 @@ package tasks
import (
"encoding/json"
- "fmt"
"net/http"
- "net/url"
"github.com/apache/devlake/core/errors"
"github.com/apache/devlake/core/plugin"
@@ -54,14 +52,11 @@ func CollectEpics(taskCtx plugin.SubTaskContext)
errors.Error {
},
Table: RAW_EPIC_TABLE,
},
- ApiClient: data.ApiClient,
- PageSize: 1000,
- UrlTemplate: "epics",
- Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
- query := url.Values{}
- query.Set("project", fmt.Sprintf("%d",
data.Options.ProjectId))
- return query, nil
- },
+ ApiClient: data.ApiClient,
+ PageSize: taigaPageSize,
+ UrlTemplate: "epics",
+ Query:
taigaProjectListQuery(data.Options.ProjectId),
+ GetNextPageCustomData: getNextTaigaPage,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
var result []json.RawMessage
err := api.UnmarshalResponse(res, &result)
diff --git a/backend/plugins/taiga/tasks/issue_collector.go
b/backend/plugins/taiga/tasks/issue_collector.go
index 937baeab7..99e4f2d66 100644
--- a/backend/plugins/taiga/tasks/issue_collector.go
+++ b/backend/plugins/taiga/tasks/issue_collector.go
@@ -19,9 +19,7 @@ package tasks
import (
"encoding/json"
- "fmt"
"net/http"
- "net/url"
"github.com/apache/devlake/core/errors"
"github.com/apache/devlake/core/plugin"
@@ -54,14 +52,11 @@ func CollectIssues(taskCtx plugin.SubTaskContext)
errors.Error {
},
Table: RAW_ISSUE_TABLE,
},
- ApiClient: data.ApiClient,
- PageSize: 1000,
- UrlTemplate: "issues",
- Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
- query := url.Values{}
- query.Set("project", fmt.Sprintf("%d",
data.Options.ProjectId))
- return query, nil
- },
+ ApiClient: data.ApiClient,
+ PageSize: taigaPageSize,
+ UrlTemplate: "issues",
+ Query:
taigaProjectListQuery(data.Options.ProjectId),
+ GetNextPageCustomData: getNextTaigaPage,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
var result []json.RawMessage
err := api.UnmarshalResponse(res, &result)
diff --git a/backend/plugins/taiga/tasks/shared.go
b/backend/plugins/taiga/tasks/shared.go
new file mode 100644
index 000000000..8e3a96dee
--- /dev/null
+++ b/backend/plugins/taiga/tasks/shared.go
@@ -0,0 +1,53 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/devlake/core/errors"
+ "github.com/apache/devlake/helpers/pluginhelper/api"
+)
+
+// taigaPageSize is the largest page size Taiga accepts (MAX_PAGINATE_BY).
+const taigaPageSize = 1000
+
+// taigaProjectListQuery builds the query for a Taiga list endpoint filtered by
+// project. Taiga paginates list endpoints (30 items per page by default), so
+// the page and page size are always sent.
+func taigaProjectListQuery(projectId uint64) func(reqData *api.RequestData)
(url.Values, errors.Error) {
+ return func(reqData *api.RequestData) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("project", fmt.Sprintf("%d", projectId))
+ query.Set("page", fmt.Sprintf("%d", reqData.Pager.Page))
+ query.Set("page_size", fmt.Sprintf("%d", reqData.Pager.Size))
+ return query, nil
+ }
+}
+
+// getNextTaigaPage stops the collection when Taiga reports no next page.
+// Taiga answers a page past the last one with 404, so a full last page must
+// not lead to another request.
+func getNextTaigaPage(_ *api.RequestData, prevPageResponse *http.Response)
(interface{}, errors.Error) {
+ if prevPageResponse.Header.Get("X-Pagination-Next") == "" {
+ return nil, api.ErrFinishCollect
+ }
+ return nil, nil
+}
diff --git a/backend/plugins/taiga/tasks/shared_test.go
b/backend/plugins/taiga/tasks/shared_test.go
new file mode 100644
index 000000000..2d8dd645f
--- /dev/null
+++ b/backend/plugins/taiga/tasks/shared_test.go
@@ -0,0 +1,50 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/apache/devlake/helpers/pluginhelper/api"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestTaigaProjectListQueryRequestsThePage(t *testing.T) {
+ query := taigaProjectListQuery(42)
+
+ got, err := query(&api.RequestData{Pager: &api.Pager{Page: 3, Size:
1000}})
+
+ assert.Nil(t, err)
+ assert.Equal(t, "42", got.Get("project"))
+ // Taiga paginates list endpoints (30 items by default) and only returns
+ // the page asked for, so the collector must send page and page_size.
+ assert.Equal(t, "3", got.Get("page"))
+ assert.Equal(t, "1000", got.Get("page_size"))
+}
+
+func TestGetNextTaigaPage(t *testing.T) {
+ more := &http.Response{Header: http.Header{}}
+ more.Header.Set("X-Pagination-Next",
"https://taiga.example/api/v1/issues?page=2&page_size=1000&project=42";)
+ _, err := getNextTaigaPage(nil, more)
+ assert.Nil(t, err)
+
+ last := &http.Response{Header: http.Header{}}
+ _, err = getNextTaigaPage(nil, last)
+ assert.ErrorIs(t, err, api.ErrFinishCollect)
+}
diff --git a/backend/plugins/taiga/tasks/task_collector.go
b/backend/plugins/taiga/tasks/task_collector.go
index 4d0f6a6f4..ca5a90557 100644
--- a/backend/plugins/taiga/tasks/task_collector.go
+++ b/backend/plugins/taiga/tasks/task_collector.go
@@ -19,9 +19,7 @@ package tasks
import (
"encoding/json"
- "fmt"
"net/http"
- "net/url"
"github.com/apache/devlake/core/errors"
"github.com/apache/devlake/core/plugin"
@@ -54,14 +52,11 @@ func CollectTasks(taskCtx plugin.SubTaskContext)
errors.Error {
},
Table: RAW_TASK_TABLE,
},
- ApiClient: data.ApiClient,
- PageSize: 1000,
- UrlTemplate: "tasks",
- Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
- query := url.Values{}
- query.Set("project", fmt.Sprintf("%d",
data.Options.ProjectId))
- return query, nil
- },
+ ApiClient: data.ApiClient,
+ PageSize: taigaPageSize,
+ UrlTemplate: "tasks",
+ Query:
taigaProjectListQuery(data.Options.ProjectId),
+ GetNextPageCustomData: getNextTaigaPage,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
var result []json.RawMessage
err := api.UnmarshalResponse(res, &result)
diff --git a/backend/plugins/taiga/tasks/user_story_collector.go
b/backend/plugins/taiga/tasks/user_story_collector.go
index 1fa69f86c..206ae72c4 100644
--- a/backend/plugins/taiga/tasks/user_story_collector.go
+++ b/backend/plugins/taiga/tasks/user_story_collector.go
@@ -19,9 +19,7 @@ package tasks
import (
"encoding/json"
- "fmt"
"net/http"
- "net/url"
"github.com/apache/devlake/core/errors"
"github.com/apache/devlake/core/plugin"
@@ -54,15 +52,11 @@ func CollectUserStories(taskCtx plugin.SubTaskContext)
errors.Error {
},
Table: RAW_USER_STORY_TABLE,
},
- ApiClient: data.ApiClient,
- PageSize: 1000, // Fetch all in one page - Taiga returns all
user stories for a project
- UrlTemplate: "userstories",
- Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
- query := url.Values{}
- query.Set("project", fmt.Sprintf("%d",
data.Options.ProjectId))
- // Don't specify page - get all results
- return query, nil
- },
+ ApiClient: data.ApiClient,
+ PageSize: taigaPageSize,
+ UrlTemplate: "userstories",
+ Query:
taigaProjectListQuery(data.Options.ProjectId),
+ GetNextPageCustomData: getNextTaigaPage,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
var result []json.RawMessage
err := api.UnmarshalResponse(res, &result)
