This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 15e34bb80 fix(jira): collect issues via search API to avoid kanban
sub-filter exclusion (#8931)
15e34bb80 is described below
commit 15e34bb80b5b3e35c77a88122761d507b23363e6
Author: Joshua Smith <[email protected]>
AuthorDate: Thu Jul 2 06:53:54 2026 -0600
fix(jira): collect issues via search API to avoid kanban sub-filter
exclusion (#8931)
* feat(jira): collect issues via search API to avoid kanban sub-filter
exclusion
* Switch from board Agile API to search API with saved filter JQL so
resolved issues (e.g. released fixVersions) are no longer silently
excluded by kanban sub-filters
* Add sub_query field to boards model and migration to track kanban
sub-filter changes across syncs
* Support JIRA Cloud v3 search/jql endpoint alongside v2 for Server
* fix(jira,config-ui): correct SubQuery parsing and connection form crashes
* Fix Jira board SubQuery struct to match nested API response shape
* Prevent click event from being stored as plugin state in connection list
* Default initialValues to empty object when plugin config omits it
* feat(jira): Add BoardConfiguration unmarshal tests
* Verify SubQuery parsing for kanban boards with, without, and empty
sub-filter objects
* Include a test using a full Jira Cloud response payload to validate all
fields
---
backend/plugins/jira/models/board.go | 1 +
.../20260611_add_sub_query_to_boards.go | 46 +++++++
.../jira/models/migrationscripts/register.go | 1 +
.../jira/tasks/board_filter_begin_collector.go | 46 ++++---
.../tasks/board_filter_begin_collector_test.go | 137 ++++++++++++++++++++
.../jira/tasks/board_filter_end_collector.go | 23 ++--
backend/plugins/jira/tasks/issue_collector.go | 141 +++++++++++++--------
backend/plugins/jira/tasks/issue_collector_test.go | 42 ++++++
backend/plugins/jira/tasks/task_data.go | 1 +
.../plugins/components/connection-form/index.tsx | 2 +-
10 files changed, 361 insertions(+), 79 deletions(-)
diff --git a/backend/plugins/jira/models/board.go
b/backend/plugins/jira/models/board.go
index 267c1cdc0..f76af2a52 100644
--- a/backend/plugins/jira/models/board.go
+++ b/backend/plugins/jira/models/board.go
@@ -34,6 +34,7 @@ type JiraBoard struct {
Self string `json:"self" mapstructure:"self"
gorm:"type:varchar(255)"`
Type string `json:"type" mapstructure:"type"
gorm:"type:varchar(100)"`
Jql string `json:"jql" mapstructure:"jql"`
+ SubQuery string `json:"subQuery" mapstructure:"subQuery"`
}
func (b JiraBoard) ScopeId() string {
diff --git
a/backend/plugins/jira/models/migrationscripts/20260611_add_sub_query_to_boards.go
b/backend/plugins/jira/models/migrationscripts/20260611_add_sub_query_to_boards.go
new file mode 100644
index 000000000..455be7f6d
--- /dev/null
+++
b/backend/plugins/jira/models/migrationscripts/20260611_add_sub_query_to_boards.go
@@ -0,0 +1,46 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type JiraBoard20260611 struct {
+ SubQuery string
+}
+
+func (JiraBoard20260611) TableName() string {
+ return "_tool_jira_boards"
+}
+
+type addSubQueryToBoards struct{}
+
+func (script *addSubQueryToBoards) Up(basicRes context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(basicRes, &JiraBoard20260611{})
+}
+
+func (*addSubQueryToBoards) Version() uint64 {
+ return 20260611140000
+}
+
+func (*addSubQueryToBoards) Name() string {
+ return "add sub_query to _tool_jira_boards"
+}
diff --git a/backend/plugins/jira/models/migrationscripts/register.go
b/backend/plugins/jira/models/migrationscripts/register.go
index 9c334a9ef..37fc6a5f9 100644
--- a/backend/plugins/jira/models/migrationscripts/register.go
+++ b/backend/plugins/jira/models/migrationscripts/register.go
@@ -55,5 +55,6 @@ func All() []plugin.MigrationScript {
new(flushJiraIssues),
new(updateScopeConfig),
new(addFixVersions20250619),
+ new(addSubQueryToBoards),
}
}
diff --git a/backend/plugins/jira/tasks/board_filter_begin_collector.go
b/backend/plugins/jira/tasks/board_filter_begin_collector.go
index 6c513f1f8..10dc60758 100644
--- a/backend/plugins/jira/tasks/board_filter_begin_collector.go
+++ b/backend/plugins/jira/tasks/board_filter_begin_collector.go
@@ -41,14 +41,18 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext)
errors.Error {
logger := taskCtx.GetLogger()
db := taskCtx.GetDal()
logger.Info("collect board in collectBoardFilterBegin: %d",
data.Options.BoardId)
- // get board filter id
- filterId, err := getBoardFilterId(data)
+
+ boardConfig, err := getBoardConfiguration(data)
if err != nil {
- return errors.Default.Wrap(err, fmt.Sprintf("error getting
board filter id for connection_id:%d board_id:%d", data.Options.ConnectionId,
data.Options.BoardId))
+ return errors.Default.Wrap(err, fmt.Sprintf("error getting
board configuration for connection_id:%d board_id:%d",
data.Options.ConnectionId, data.Options.BoardId))
}
+ filterId := boardConfig.Filter.ID
logger.Info("collect board filter:%s", filterId)
- // get board filter jql
+ if boardConfig.SubQuery.Query != "" {
+ logger.Warn(nil, "board %d has kanban sub-filter: %s — using
saved filter JQL for collection to avoid silent issue exclusion",
data.Options.BoardId, boardConfig.SubQuery.Query)
+ }
+
filterInfo, err := getBoardFilterJql(data, filterId)
if err != nil {
return errors.Default.Wrap(err, fmt.Sprintf("error getting
board filter jql for connection_id:%d board_id:%d", data.Options.ConnectionId,
data.Options.BoardId))
@@ -62,17 +66,21 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext)
errors.Error {
return errors.Default.Wrap(err, fmt.Sprintf("error finding
record in _tool_jira_boards table for connection_id:%d board_id:%d",
data.Options.ConnectionId, data.Options.BoardId))
}
+ // Store filter ID and sub-query on task data for downstream subtasks
+ data.FilterId = filterId
+ record.SubQuery = boardConfig.SubQuery.Query
+
// full sync
syncPolicy := taskCtx.TaskContext().SyncPolicy()
if syncPolicy != nil && syncPolicy.FullSync {
if record.Jql != jql {
record.Jql = jql
- err = db.Update(&record, dal.Where("connection_id = ?
AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId))
- if err != nil {
- return errors.Default.Wrap(err,
fmt.Sprintf("error updating record in _tool_jira_boards table for
connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId))
- }
- logger.Info("full sync mode, update jql to %s",
record.Jql)
}
+ err = db.Update(&record, dal.Where("connection_id = ? AND
board_id = ? ", data.Options.ConnectionId, data.Options.BoardId))
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error
updating record in _tool_jira_boards table for connection_id:%d board_id:%d",
data.Options.ConnectionId, data.Options.BoardId))
+ }
+ logger.Info("full sync mode, update jql to %s", record.Jql)
return nil
}
@@ -92,7 +100,6 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext)
errors.Error {
flag := cfg.GetBool("JIRA_JQL_AUTO_FULL_REFRESH")
if flag {
logger.Info("connection_id:%d board_id:%d filter jql
has changed, And the previous jql is %s, now jql is %s, run it in fullSync
mode", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql)
- // set full sync
taskCtx.TaskContext().SetSyncPolicy(&coreModels.SyncPolicy{TriggerSyncPolicy:
coreModels.TriggerSyncPolicy{FullSync: true}})
record.Jql = jql
err = db.Update(&record, dal.Where("connection_id = ?
AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId))
@@ -102,24 +109,28 @@ func CollectBoardFilterBegin(taskCtx
plugin.SubTaskContext) errors.Error {
} else {
return errors.Default.New(fmt.Sprintf("connection_id:%d
board_id:%d filter jql has changed, please use fullSync mode. And the previous
jql is %s, now jql is %s", data.Options.ConnectionId, data.Options.BoardId,
record.Jql, jql))
}
+ } else {
+ // JQL unchanged but sub-query may have changed — persist it
+ err = db.Update(&record, dal.Where("connection_id = ? AND
board_id = ? ", data.Options.ConnectionId, data.Options.BoardId))
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error
updating record in _tool_jira_boards table for connection_id:%d board_id:%d",
data.Options.ConnectionId, data.Options.BoardId))
+ }
}
- // no change
return nil
}
-func getBoardFilterId(data *JiraTaskData) (string, error) {
+func getBoardConfiguration(data *JiraTaskData) (*BoardConfiguration, error) {
url := fmt.Sprintf("agile/1.0/board/%d/configuration",
data.Options.BoardId)
boardConfiguration, err := data.ApiClient.Get(url, nil, nil)
if err != nil {
- return "", err
+ return nil, err
}
bc := &BoardConfiguration{}
err = helper.UnmarshalResponse(boardConfiguration, bc)
if err != nil {
- return "", err
+ return nil, err
}
- filterId := bc.Filter.ID
- return filterId, nil
+ return bc, nil
}
func getBoardFilterJql(data *JiraTaskData, filterId string) (*FilterInfo,
error) {
@@ -141,6 +152,9 @@ type BoardConfiguration struct {
Name string `json:"name"`
Type string `json:"type"`
Self string `json:"self"`
+ SubQuery struct {
+ Query string `json:"query"`
+ } `json:"subQuery"`
Location struct {
Type string `json:"type"`
Key string `json:"key"`
diff --git a/backend/plugins/jira/tasks/board_filter_begin_collector_test.go
b/backend/plugins/jira/tasks/board_filter_begin_collector_test.go
new file mode 100644
index 000000000..dc81004a9
--- /dev/null
+++ b/backend/plugins/jira/tasks/board_filter_begin_collector_test.go
@@ -0,0 +1,137 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "testing"
+)
+
+func Test_BoardConfiguration_UnmarshalSubQuery(t *testing.T) {
+ tests := []struct {
+ name string
+ raw string
+ wantSubQuery string
+ wantFilterID string
+ wantID int
+ wantName string
+ wantType string
+ wantColumnCount int
+ wantRankFieldID int
+ }{
+ {
+ name: "kanban board with sub-filter object",
+ raw: `{"id":1201,"name":"Squad 5","type":"kanban",` +
+
`"self":"https://example.atlassian.net/rest/agile/1.0/board/1201/configuration",`
+
+
`"filter":{"id":"17696","self":"https://example.atlassian.net/rest/api/2/filter/17696"},`
+
+ `"subQuery":{"query":"fixVersion in
unreleasedVersions() OR fixVersion is EMPTY"},` +
+
`"columnConfig":{"columns":[{"name":"Backlog","statuses":[{"id":"1","self":"https://example.atlassian.net/rest/api/2/status/1"}]},`
+
+
`{"name":"Done","statuses":[{"id":"10037","self":"https://example.atlassian.net/rest/api/2/status/10037"}]}],`
+
+
`"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}`,
+ wantSubQuery: "fixVersion in unreleasedVersions() OR
fixVersion is EMPTY",
+ wantFilterID: "17696",
+ wantID: 1201,
+ wantName: "Squad 5",
+ wantType: "kanban",
+ wantColumnCount: 2,
+ wantRankFieldID: 10019,
+ },
+ {
+ name: "board without subQuery field",
+ raw: `{"id":500,"name":"No SubFilter
Board","type":"scrum",` +
+
`"self":"https://example.atlassian.net/rest/agile/1.0/board/500/configuration",`
+
+
`"filter":{"id":"99999","self":"https://example.atlassian.net/rest/api/2/filter/99999"},`
+
+
`"columnConfig":{"columns":[],"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}`,
+ wantSubQuery: "",
+ wantFilterID: "99999",
+ wantID: 500,
+ wantName: "No SubFilter Board",
+ wantType: "scrum",
+ wantColumnCount: 0,
+ wantRankFieldID: 10019,
+ },
+ {
+ name: "board with empty subQuery object",
+ raw: `{"id":600,"name":"Empty SubQuery
Board","type":"kanban",` +
+
`"self":"https://example.atlassian.net/rest/agile/1.0/board/600/configuration",`
+
+
`"filter":{"id":"11111","self":"https://example.atlassian.net/rest/api/2/filter/11111"},`
+
+ `"subQuery":{},` +
+
`"columnConfig":{"columns":[],"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}`,
+ wantSubQuery: "",
+ wantFilterID: "11111",
+ wantID: 600,
+ wantName: "Empty SubQuery Board",
+ wantType: "kanban",
+ wantColumnCount: 0,
+ wantRankFieldID: 10019,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ var bc BoardConfiguration
+ if err := json.Unmarshal([]byte(tt.raw), &bc); err !=
nil {
+ t.Fatalf("failed to unmarshal
BoardConfiguration: %v", err)
+ }
+ if bc.SubQuery.Query != tt.wantSubQuery {
+ t.Errorf("SubQuery.Query = %q, want %q",
bc.SubQuery.Query, tt.wantSubQuery)
+ }
+ if bc.Filter.ID != tt.wantFilterID {
+ t.Errorf("Filter.ID = %q, want %q",
bc.Filter.ID, tt.wantFilterID)
+ }
+ if bc.ID != tt.wantID {
+ t.Errorf("ID = %d, want %d", bc.ID, tt.wantID)
+ }
+ if bc.Name != tt.wantName {
+ t.Errorf("Name = %q, want %q", bc.Name,
tt.wantName)
+ }
+ if bc.Type != tt.wantType {
+ t.Errorf("Type = %q, want %q", bc.Type,
tt.wantType)
+ }
+ if len(bc.ColumnConfig.Columns) != tt.wantColumnCount {
+ t.Errorf("ColumnConfig.Columns length = %d,
want %d", len(bc.ColumnConfig.Columns), tt.wantColumnCount)
+ }
+ if bc.Ranking.RankCustomFieldID != tt.wantRankFieldID {
+ t.Errorf("Ranking.RankCustomFieldID = %d, want
%d", bc.Ranking.RankCustomFieldID, tt.wantRankFieldID)
+ }
+ })
+ }
+}
+
+func Test_BoardConfiguration_FullJiraCloudResponse(t *testing.T) {
+ // Exact response payload from Jira Cloud for Board 1201 (Squad 5)
+ raw := `{"id":1201,"name":"Squad
5","type":"kanban","self":"https://rakutenadvertising.atlassian.net/rest/agile/1.0/board/1201/configuration","location":{"type":"user","id":"62d8159bb2e6b1992b5be875","self":"https://rakutenadvertising.atlassian.net/rest/api/2/user?accountId=62d8159bb2e6b1992b5be875"},"filter":{"id":"17696","self":"https://rakutenadvertising.atlassian.net/rest/api/2/filter/17696"},"subQuery":{"query":"fixVersion
in unreleasedVersions() OR fixVersion is EMPTY"},"columnCon [...]
+
+ var bc BoardConfiguration
+ if err := json.Unmarshal([]byte(raw), &bc); err != nil {
+ t.Fatalf("failed to unmarshal real Jira Cloud response: %v",
err)
+ }
+
+ if bc.SubQuery.Query != "fixVersion in unreleasedVersions() OR
fixVersion is EMPTY" {
+ t.Errorf("SubQuery.Query = %q, want the fixVersion sub-filter",
bc.SubQuery.Query)
+ }
+ if bc.Filter.ID != "17696" {
+ t.Errorf("Filter.ID = %q, want %q", bc.Filter.ID, "17696")
+ }
+ if len(bc.ColumnConfig.Columns) != 8 {
+ t.Errorf("ColumnConfig.Columns length = %d, want 8",
len(bc.ColumnConfig.Columns))
+ }
+ if bc.Location.ID != "62d8159bb2e6b1992b5be875" {
+ t.Errorf("Location.ID = %q, want %q", bc.Location.ID,
"62d8159bb2e6b1992b5be875")
+ }
+}
diff --git a/backend/plugins/jira/tasks/board_filter_end_collector.go
b/backend/plugins/jira/tasks/board_filter_end_collector.go
index 65d8eca14..4dfe06fef 100644
--- a/backend/plugins/jira/tasks/board_filter_end_collector.go
+++ b/backend/plugins/jira/tasks/board_filter_end_collector.go
@@ -40,14 +40,13 @@ func CollectBoardFilterEnd(taskCtx plugin.SubTaskContext)
errors.Error {
db := taskCtx.GetDal()
logger.Info("collect board in collectBoardFilterEnd: %d",
data.Options.BoardId)
- // get board filter id
- filterId, err := getBoardFilterId(data)
+ boardConfig, err := getBoardConfiguration(data)
if err != nil {
- return errors.Default.Wrap(err, fmt.Sprintf("error getting
board filter id for connection_id:%d board_id:%d", data.Options.ConnectionId,
data.Options.BoardId))
+ return errors.Default.Wrap(err, fmt.Sprintf("error getting
board configuration for connection_id:%d board_id:%d",
data.Options.ConnectionId, data.Options.BoardId))
}
+ filterId := boardConfig.Filter.ID
logger.Info("collect board filter:%s", filterId)
- // get board filter jql
filterInfo, err := getBoardFilterJql(data, filterId)
if err != nil {
return errors.Default.Wrap(err, fmt.Sprintf("error getting
board filter jql for connection_id:%d board_id:%d", data.Options.ConnectionId,
data.Options.BoardId))
@@ -55,7 +54,6 @@ func CollectBoardFilterEnd(taskCtx plugin.SubTaskContext)
errors.Error {
jql := filterInfo.Jql
logger.Info("collect board filter jql:%s", jql)
- // should not change
var record models.JiraBoard
err = db.First(&record, dal.Where("connection_id = ? AND board_id = ?
", data.Options.ConnectionId, data.Options.BoardId))
if err != nil {
@@ -63,12 +61,21 @@ func CollectBoardFilterEnd(taskCtx plugin.SubTaskContext)
errors.Error {
}
logger.Info("get board filter jql:%s", record.Jql)
+ cfg := taskCtx.GetConfigReader()
+ autoRefresh := cfg.GetBool("JIRA_JQL_AUTO_FULL_REFRESH")
+
if record.Jql != jql {
- cfg := taskCtx.GetConfigReader()
- flag := cfg.GetBool("JIRA_JQL_AUTO_FULL_REFRESH")
- if !flag {
+ if !autoRefresh {
return errors.Default.New(fmt.Sprintf("connection_id:%d
board_id:%d filter jql has changed, please use fullSync mode. And the previous
jql is %s, now jql is %s", data.Options.ConnectionId, data.Options.BoardId,
record.Jql, jql))
}
+ logger.Warn(nil, "connection_id:%d board_id:%d filter jql
changed during collection (previous: %s, now: %s)", data.Options.ConnectionId,
data.Options.BoardId, record.Jql, jql)
+ }
+
+ if record.SubQuery != boardConfig.SubQuery.Query {
+ logger.Warn(nil, "connection_id:%d board_id:%d board sub-filter
changed during collection (previous: %s, now: %s)", data.Options.ConnectionId,
data.Options.BoardId, record.SubQuery, boardConfig.SubQuery.Query)
+ if !autoRefresh {
+ return errors.Default.New(fmt.Sprintf("connection_id:%d
board_id:%d board sub-filter has changed during collection, please use fullSync
mode. Previous sub-filter: %s, now: %s", data.Options.ConnectionId,
data.Options.BoardId, record.SubQuery, boardConfig.SubQuery.Query))
+ }
}
return nil
diff --git a/backend/plugins/jira/tasks/issue_collector.go
b/backend/plugins/jira/tasks/issue_collector.go
index 9a361cbbc..28aa473ff 100644
--- a/backend/plugins/jira/tasks/issue_collector.go
+++ b/backend/plugins/jira/tasks/issue_collector.go
@@ -27,11 +27,10 @@ import (
"time"
"github.com/apache/incubator-devlake/core/dal"
- "github.com/apache/incubator-devlake/plugins/jira/models"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/jira/models"
)
const RAW_ISSUE_TABLE = "jira_api_issues"
@@ -51,78 +50,83 @@ func CollectIssues(taskCtx plugin.SubTaskContext)
errors.Error {
logger := taskCtx.GetLogger()
apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
Ctx: taskCtx,
- /*
- This struct will be JSONEncoded and stored into
database along with raw data itself, to identity minimal
- set of data to be process, for example, we process
JiraIssues by Board
- */
Params: JiraApiParams{
ConnectionId: data.Options.ConnectionId,
BoardId: data.Options.BoardId,
},
- /*
- Table store raw data
- */
Table: RAW_ISSUE_TABLE,
})
if err != nil {
return err
}
- // build jql
- // IMPORTANT: we have to keep paginated data in a consistence order to
avoid data-missing, if we sort issues by
- // `updated`, issue will be jumping between pages if it got updated
during the collection process
+ // IMPORTANT: we sort by `created ASC` to keep paginated data in a
consistent order.
+ // Sorting by `updated` would cause issues to jump between pages during
collection.
loc, err := getTimeZone(taskCtx)
if err != nil {
logger.Info("failed to get timezone, err: %v", err)
} else {
logger.Info("got user's timezone: %v", loc.String())
}
- jql := "ORDER BY created ASC"
+ incrementalJql := "ORDER BY created ASC"
if apiCollector.GetSince() != nil {
- jql = buildJQL(*apiCollector.GetSince(), loc)
- }
-
- err = apiCollector.InitCollector(api.ApiCollectorArgs{
- ApiClient: data.ApiClient,
- PageSize: data.Options.PageSize,
- /*
- url may use arbitrary variables from different
connection in any order, we need GoTemplate to allow more
- flexible for all kinds of possibility.
- Pager contains information for a particular page,
calculated by ApiCollector, and will be passed into
- GoTemplate to generate a url for that page.
- We want to do page-fetching in ApiCollector, because
the logic are highly similar, by doing so, we can
- avoid duplicate logic for every tasks, and when we have
a better idea like improving performance, we can
- do it in one place
- */
- UrlTemplate: "agile/1.0/board/{{ .Params.BoardId }}/issue",
- /*
- (Optional) Return query string for request, or you can
plug them into UrlTemplate directly
- */
+ incrementalJql = buildJQL(*apiCollector.GetSince(), loc)
+ }
+
+ // Use the search API with `filter = {id}` JQL instead of the board
Agile API.
+ // The board Agile API applies kanban sub-filters server-side, which
silently
+ // excludes resolved issues (e.g. those with a released fixVersion).
+ // The search API with the saved filter JQL returns all matching issues.
+ filterJql := buildFilterJQL(data.FilterId, incrementalJql)
+ logger.Info("collecting issues via search API with JQL: %s", filterJql)
+
+ pageSize := data.Options.PageSize
+ if pageSize == 0 {
+ pageSize = 100
+ }
+
+ if strings.EqualFold(string(data.JiraServerInfo.DeploymentType),
string(models.DeploymentServer)) {
+ logger.Info("Using api/2/search for JIRA Server issue
collection")
+ err = setupIssueV2Collector(apiCollector, data, filterJql,
pageSize)
+ } else {
+ logger.Info("Using api/3/search/jql for JIRA Cloud issue
collection")
+ err = setupIssueV3Collector(apiCollector, data, filterJql,
pageSize)
+ }
+ if err != nil {
+ return err
+ }
+
+ return apiCollector.Execute()
+}
+
+func buildFilterJQL(filterId string, incrementalJql string) string {
+ if filterId == "" {
+ return incrementalJql
+ }
+ // Use Jira's `filter = {id}` syntax to reference the saved filter.
+ // This avoids parenthesization bugs when composing raw JQL strings
+ // that may contain OR/AND operators.
+ if incrementalJql == "ORDER BY created ASC" {
+ return fmt.Sprintf("filter = %s ORDER BY created ASC", filterId)
+ }
+ // incrementalJql contains "updated >= '...' ORDER BY created ASC"
+ // We need to insert the filter reference before the incremental clause
+ return fmt.Sprintf("filter = %s AND %s", filterId, incrementalJql)
+}
+
+func setupIssueV2Collector(apiCollector *api.StatefulApiCollector, data
*JiraTaskData, filterJql string, pageSize int) errors.Error {
+ return apiCollector.InitCollector(api.ApiCollectorArgs{
+ ApiClient: data.ApiClient,
+ PageSize: pageSize,
+ UrlTemplate: "api/2/search",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
- query.Set("jql", jql)
+ query.Set("jql", filterJql)
query.Set("startAt", fmt.Sprintf("%v",
reqData.Pager.Skip))
query.Set("maxResults", fmt.Sprintf("%v",
reqData.Pager.Size))
query.Set("expand", "changelog")
return query, nil
},
- /*
- Some api might do pagination by http headers
- */
- //Header: func(pager *plugin.Pager) http.Header {
- //},
- /*
- Sometimes, we need to collect data based on previous
collected data, like jira changelog, it requires
- issue_id as part of the url.
- We can mimic `stdin` design, to accept a `Input`
function which produces a `Iterator`, collector
- should iterate all records, and do data-fetching for
each on, either in parallel or sequential order
- UrlTemplate: "api/3/issue/{{ Input.ID }}/changelog"
- */
- //Input: databaseIssuesIterator,
- /*
- For api endpoint that returns number of total pages,
ApiCollector can collect pages in parallel with ease,
- or other techniques are required if this information
was missing.
- */
GetTotalPages: GetTotalPagesFromResponse,
Concurrency: 10,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
@@ -140,11 +144,40 @@ func CollectIssues(taskCtx plugin.SubTaskContext)
errors.Error {
return data.Issues, nil
},
})
- if err != nil {
- return err
- }
+}
- return apiCollector.Execute()
+func setupIssueV3Collector(apiCollector *api.StatefulApiCollector, data
*JiraTaskData, filterJql string, pageSize int) errors.Error {
+ return apiCollector.InitCollector(api.ApiCollectorArgs{
+ ApiClient: data.ApiClient,
+ PageSize: pageSize,
+ UrlTemplate: "api/3/search/jql",
+ GetNextPageCustomData: getNextPageCustomDataForV3,
+ Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
+ query := url.Values{}
+ query.Set("jql", filterJql)
+ query.Set("maxResults", fmt.Sprintf("%v",
reqData.Pager.Size))
+ query.Set("expand", "changelog")
+ query.Set("fields", "*all")
+ if reqData.CustomData != nil {
+ query.Set("nextPageToken",
reqData.CustomData.(string))
+ }
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
+ var data struct {
+ Issues []json.RawMessage `json:"issues"`
+ }
+ blob, err := io.ReadAll(res.Body)
+ if err != nil {
+ return nil, errors.Convert(err)
+ }
+ err = json.Unmarshal(blob, &data)
+ if err != nil {
+ return nil, errors.Convert(err)
+ }
+ return data.Issues, nil
+ },
+ })
}
// buildJQL build jql based on timeAfter and incremental mode
diff --git a/backend/plugins/jira/tasks/issue_collector_test.go
b/backend/plugins/jira/tasks/issue_collector_test.go
index 99bf5a533..7d5bdc1c1 100644
--- a/backend/plugins/jira/tasks/issue_collector_test.go
+++ b/backend/plugins/jira/tasks/issue_collector_test.go
@@ -61,3 +61,45 @@ func Test_buildJQL(t *testing.T) {
})
}
}
+
+func Test_buildFilterJQL(t *testing.T) {
+ tests := []struct {
+ name string
+ filterId string
+ incrementalJql string
+ want string
+ }{
+ {
+ name: "full sync with filter",
+ filterId: "12345",
+ incrementalJql: "ORDER BY created ASC",
+ want: "filter = 12345 ORDER BY created ASC",
+ },
+ {
+ name: "incremental sync with filter",
+ filterId: "12345",
+ incrementalJql: "updated >= '2021/02/05 12:05' ORDER BY
created ASC",
+ want: "filter = 12345 AND updated >=
'2021/02/05 12:05' ORDER BY created ASC",
+ },
+ {
+ name: "empty filter id falls back to
incremental only",
+ filterId: "",
+ incrementalJql: "ORDER BY created ASC",
+ want: "ORDER BY created ASC",
+ },
+ {
+ name: "empty filter id with incremental
clause",
+ filterId: "",
+ incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY
created ASC",
+ want: "updated >= '2024/01/01 00:00' ORDER BY
created ASC",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := buildFilterJQL(tt.filterId,
tt.incrementalJql); got != tt.want {
+ t.Errorf("buildFilterJQL() = %v, want %v", got,
tt.want)
+ }
+ })
+ }
+}
diff --git a/backend/plugins/jira/tasks/task_data.go
b/backend/plugins/jira/tasks/task_data.go
index 1b0580396..bfab9704f 100644
--- a/backend/plugins/jira/tasks/task_data.go
+++ b/backend/plugins/jira/tasks/task_data.go
@@ -37,6 +37,7 @@ type JiraTaskData struct {
Options *JiraOptions
ApiClient *api.ApiAsyncClient
JiraServerInfo models.JiraServerInfo
+ FilterId string
}
type JiraApiParams models.JiraApiParams
diff --git a/config-ui/src/plugins/components/connection-form/index.tsx
b/config-ui/src/plugins/components/connection-form/index.tsx
index b61c1d5d0..39f4cf0df 100644
--- a/config-ui/src/plugins/components/connection-form/index.tsx
+++ b/config-ui/src/plugins/components/connection-form/index.tsx
@@ -109,7 +109,7 @@ export const ConnectionForm = ({ plugin, connectionId,
onSuccess }: Props) => {
const {
name,
- connection: { docLink, fields, initialValues },
+ connection: { docLink, fields, initialValues = {} },
} = getPluginConfig(plugin) ?? {};
const disabled = useMemo(() => {
