This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/devlake.git
The following commit(s) were added to refs/heads/main by this push:
new cd7bebf64 feat(jira): Allow extra JQL when querying JIRA board (#8972)
cd7bebf64 is described below
commit cd7bebf64db3a2e886a511cd405174978946c71d
Author: Jon Vaughan <[email protected]>
AuthorDate: Thu Jul 9 10:00:46 2026 +0100
feat(jira): Allow extra JQL when querying JIRA board (#8972)
* feat(jira): Allow extra JQL when querying JIRA board
* fix: revert code so we just do extrajql, no dynamic dev lake project name
* fix: remove reference to devlakeprojectname as a template option
---
backend/plugins/jira/impl/impl.go | 3 +-
.../20260702_add_extra_jql_to_scope_config.go | 46 +++++++++
.../jira/models/migrationscripts/register.go | 1 +
backend/plugins/jira/models/scope_config.go | 7 ++
backend/plugins/jira/tasks/issue_collector.go | 86 ++++++++++++++---
backend/plugins/jira/tasks/issue_collector_test.go | 107 ++++++++++++++++++++-
backend/plugins/jira/tasks/task_data.go | 1 +
config-ui/src/plugins/register/jira/config.tsx | 1 +
.../src/plugins/register/jira/transformation.tsx | 22 ++++-
9 files changed, 259 insertions(+), 15 deletions(-)
diff --git a/backend/plugins/jira/impl/impl.go
b/backend/plugins/jira/impl/impl.go
index 126e513e2..6fb428a92 100644
--- a/backend/plugins/jira/impl/impl.go
+++ b/backend/plugins/jira/impl/impl.go
@@ -195,8 +195,8 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]int
return nil, errors.Default.Wrap(err, "failed to create jira api
client")
}
+ var scope *models.JiraBoard
if op.BoardId != 0 {
- var scope *models.JiraBoard
// support v100 & advance mode
// If we still cannot find the record in db, we have to request
from remote server and save it to db
db := taskCtx.GetDal()
@@ -249,6 +249,7 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]int
Options: &op,
ApiClient: jiraApiClient,
JiraServerInfo: *info,
+ Board: scope,
}
return taskData, nil
diff --git
a/backend/plugins/jira/models/migrationscripts/20260702_add_extra_jql_to_scope_config.go
b/backend/plugins/jira/models/migrationscripts/20260702_add_extra_jql_to_scope_config.go
new file mode 100644
index 000000000..34966a565
--- /dev/null
+++
b/backend/plugins/jira/models/migrationscripts/20260702_add_extra_jql_to_scope_config.go
@@ -0,0 +1,46 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type JiraScopeConfig20260702 struct {
+ ExtraJQL string `gorm:"type:varchar(255)"`
+}
+
+func (JiraScopeConfig20260702) TableName() string {
+ return "_tool_jira_scope_configs"
+}
+
+type addExtraJQLToScopeConfig struct{}
+
+func (script *addExtraJQLToScopeConfig) Up(basicRes context.BasicRes)
errors.Error {
+ return migrationhelper.AutoMigrateTables(basicRes,
&JiraScopeConfig20260702{})
+}
+
+func (*addExtraJQLToScopeConfig) Version() uint64 {
+ return 20260702000000
+}
+
+func (*addExtraJQLToScopeConfig) Name() string {
+ return "add extra_jql to _tool_jira_scope_configs"
+}
diff --git a/backend/plugins/jira/models/migrationscripts/register.go
b/backend/plugins/jira/models/migrationscripts/register.go
index 37fc6a5f9..0ee1243dd 100644
--- a/backend/plugins/jira/models/migrationscripts/register.go
+++ b/backend/plugins/jira/models/migrationscripts/register.go
@@ -56,5 +56,6 @@ func All() []plugin.MigrationScript {
new(updateScopeConfig),
new(addFixVersions20250619),
new(addSubQueryToBoards),
+ new(addExtraJQLToScopeConfig),
}
}
diff --git a/backend/plugins/jira/models/scope_config.go
b/backend/plugins/jira/models/scope_config.go
index a8bd78a98..5dc6d07fe 100644
--- a/backend/plugins/jira/models/scope_config.go
+++ b/backend/plugins/jira/models/scope_config.go
@@ -19,6 +19,7 @@ package models
import (
"regexp"
+ "text/template"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
@@ -49,6 +50,7 @@ type JiraScopeConfig struct {
TypeMappings map[string]TypeMapping
`mapstructure:"typeMappings,omitempty" json:"typeMappings"
gorm:"type:json;serializer:json"`
ApplicationType string
`mapstructure:"applicationType,omitempty" json:"applicationType"
gorm:"type:varchar(255)"`
DueDateField string
`mapstructure:"dueDateField,omitempty" json:"dueDateField"
gorm:"type:varchar(255)"`
+ ExtraJQL string
`mapstructure:"extraJql,omitempty" json:"extraJql" gorm:"type:varchar(255)"`
}
func (r *JiraScopeConfig) SetConnectionId(c *JiraScopeConfig, connectionId
uint64) {
@@ -73,6 +75,11 @@ func (r *JiraScopeConfig) Validate() errors.Error {
return errors.Convert(err)
}
}
+ if r.ExtraJQL != "" {
+ if _, tmplErr :=
template.New("extraJql").Funcs(template.FuncMap{}).Option("missingkey=error").Parse(r.ExtraJQL);
tmplErr != nil {
+ return errors.BadInput.Wrap(errors.Convert(tmplErr),
"invalid ExtraJQL template")
+ }
+ }
return nil
}
diff --git a/backend/plugins/jira/tasks/issue_collector.go
b/backend/plugins/jira/tasks/issue_collector.go
index 28aa473ff..fe1939b72 100644
--- a/backend/plugins/jira/tasks/issue_collector.go
+++ b/backend/plugins/jira/tasks/issue_collector.go
@@ -18,12 +18,14 @@ limitations under the License.
package tasks
import (
+ "bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
+ "text/template"
"time"
"github.com/apache/incubator-devlake/core/dal"
@@ -77,7 +79,15 @@ func CollectIssues(taskCtx plugin.SubTaskContext)
errors.Error {
// The board Agile API applies kanban sub-filters server-side, which
silently
// excludes resolved issues (e.g. those with a released fixVersion).
// The search API with the saved filter JQL returns all matching issues.
- filterJql := buildFilterJQL(data.FilterId, incrementalJql)
+ var extraJql string
+ if data.Options.ScopeConfig != nil && data.Options.ScopeConfig.ExtraJQL
!= "" {
+ renderedJql, renderErr :=
renderExtraJQL(data.Options.ScopeConfig.ExtraJQL, data)
+ if renderErr != nil {
+ return renderErr
+ }
+ extraJql = renderedJql
+ }
+ filterJql := buildFilterJQL(data.FilterId, extraJql, incrementalJql)
logger.Info("collecting issues via search API with JQL: %s", filterJql)
pageSize := data.Options.PageSize
@@ -99,19 +109,71 @@ func CollectIssues(taskCtx plugin.SubTaskContext)
errors.Error {
return apiCollector.Execute()
}
-func buildFilterJQL(filterId string, incrementalJql string) string {
- if filterId == "" {
- return incrementalJql
+// JqlTemplateData holds the variables available inside an ExtraJQL template.
+// Users reference these with Go template syntax, e.g. `{{.BoardName}}`.
+type JqlTemplateData struct {
+ BoardId uint64 // numeric ID of the connected Jira board
+ BoardName string // display name of the connected Jira board
+}
+
+// renderExtraJQL executes the ExtraJQL scope-config field as a Go
text/template,
+// substituting board-level variables so the same scope config can produce
+// different JQL for different boards.
+//
+// The template is parsed with an empty FuncMap (no built-in helpers such as
+// printf) and missingkey=error so that typos in variable names produce an
+// explicit error rather than silently rendering "<no value>".
+func renderExtraJQL(tmplStr string, data *JiraTaskData) (string, errors.Error)
{
+ tmpl, err := template.New("extraJql").
+ Funcs(template.FuncMap{}).
+ Option("missingkey=error").
+ Parse(tmplStr)
+ if err != nil {
+ return "", errors.BadInput.Wrap(err, "invalid ExtraJQL
template")
}
- // Use Jira's `filter = {id}` syntax to reference the saved filter.
- // This avoids parenthesization bugs when composing raw JQL strings
- // that may contain OR/AND operators.
- if incrementalJql == "ORDER BY created ASC" {
- return fmt.Sprintf("filter = %s ORDER BY created ASC", filterId)
+
+ vars := JqlTemplateData{
+ BoardId: data.Options.BoardId,
+ }
+ if data.Board != nil {
+ vars.BoardName = data.Board.Name
+ }
+
+ var buf bytes.Buffer
+ if execErr := tmpl.Execute(&buf, vars); execErr != nil {
+ return "", errors.BadInput.Wrap(execErr, "failed to render
ExtraJQL template")
+ }
+ return buf.String(), nil
+}
+
+// buildFilterJQL composes a final JQL query from three inputs:
+// - filterId: a Jira saved-filter ID (referenced via `filter = {id}`)
+// - extraJql: optional user-supplied JQL fragment appended as an AND
condition
+// (e.g. `project = "MyComponent"`) to scope a large board down to one
project
+// - incrementalJql: the time-based clause generated by buildJQL, always
ending
+// with "ORDER BY created ASC"
+//
+// extraJql is wrapped in parentheses so that any OR/NOT operators inside it
+// do not interfere with the surrounding AND chain.
+func buildFilterJQL(filterId string, extraJql string, incrementalJql string)
string {
+ const orderBy = "ORDER BY created ASC"
+
+ var conditions []string
+ if filterId != "" {
+ conditions = append(conditions, fmt.Sprintf("filter = %s",
filterId))
+ }
+ if extraJql != "" {
+ conditions = append(conditions, fmt.Sprintf("(%s)", extraJql))
+ }
+ if incrementalJql != orderBy {
+ // strip the trailing " ORDER BY created ASC" to isolate the
time condition
+ conditions = append(conditions,
strings.TrimSuffix(incrementalJql, " "+orderBy))
+ }
+
+ if len(conditions) == 0 {
+ return orderBy
}
- // incrementalJql contains "updated >= '...' ORDER BY created ASC"
- // We need to insert the filter reference before the incremental clause
- return fmt.Sprintf("filter = %s AND %s", filterId, incrementalJql)
+ return strings.Join(conditions, " AND ") + " " + orderBy
}
func setupIssueV2Collector(apiCollector *api.StatefulApiCollector, data
*JiraTaskData, filterJql string, pageSize int) errors.Error {
diff --git a/backend/plugins/jira/tasks/issue_collector_test.go
b/backend/plugins/jira/tasks/issue_collector_test.go
index 7d5bdc1c1..50c66947c 100644
--- a/backend/plugins/jira/tasks/issue_collector_test.go
+++ b/backend/plugins/jira/tasks/issue_collector_test.go
@@ -20,6 +20,8 @@ package tasks
import (
"testing"
"time"
+
+ "github.com/apache/incubator-devlake/plugins/jira/models"
)
func Test_buildJQL(t *testing.T) {
@@ -66,6 +68,7 @@ func Test_buildFilterJQL(t *testing.T) {
tests := []struct {
name string
filterId string
+ extraJql string
incrementalJql string
want string
}{
@@ -93,13 +96,115 @@ func Test_buildFilterJQL(t *testing.T) {
incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY
created ASC",
want: "updated >= '2024/01/01 00:00' ORDER BY
created ASC",
},
+ {
+ name: "extra jql with filter full sync",
+ filterId: "12345",
+ extraJql: `project = "MyComponent"`,
+ incrementalJql: "ORDER BY created ASC",
+ want: `filter = 12345 AND (project =
"MyComponent") ORDER BY created ASC`,
+ },
+ {
+ name: "extra jql with filter incremental
sync",
+ filterId: "12345",
+ extraJql: `project = "MyComponent"`,
+ incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY
created ASC",
+ want: `filter = 12345 AND (project =
"MyComponent") AND updated >= '2024/01/01 00:00' ORDER BY created ASC`,
+ },
+ {
+ name: "extra jql without filter",
+ filterId: "",
+ extraJql: `project = "MyComponent"`,
+ incrementalJql: "ORDER BY created ASC",
+ want: `(project = "MyComponent") ORDER BY
created ASC`,
+ },
+ {
+ name: "extra jql without filter, incremental
sync",
+ filterId: "",
+ extraJql: `project = "MyComponent"`,
+ incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY
created ASC",
+ want: `(project = "MyComponent") AND updated
>= '2024/01/01 00:00' ORDER BY created ASC`,
+ },
+ {
+ name: "extra jql with OR operator is
parenthesized",
+ filterId: "12345",
+ extraJql: `project = "A" OR project = "B"`,
+ incrementalJql: "ORDER BY created ASC",
+ want: `filter = 12345 AND (project = "A" OR
project = "B") ORDER BY created ASC`,
+ },
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- if got := buildFilterJQL(tt.filterId,
tt.incrementalJql); got != tt.want {
+ if got := buildFilterJQL(tt.filterId, tt.extraJql,
tt.incrementalJql); got != tt.want {
t.Errorf("buildFilterJQL() = %v, want %v", got,
tt.want)
}
})
}
}
+
+func Test_renderExtraJQL(t *testing.T) {
+ makeData := func(boardId uint64, boardName string, _ string)
*JiraTaskData {
+ return &JiraTaskData{
+ Options: &JiraOptions{BoardId: boardId},
+ Board: &models.JiraBoard{BoardId: boardId, Name:
boardName},
+ }
+ }
+
+ tests := []struct {
+ name string
+ tmpl string
+ data *JiraTaskData
+ want string
+ wantErr bool
+ }{
+ {
+ name: "static JQL passes through unchanged",
+ tmpl: `project = "MyProject"`,
+ data: makeData(1, "My Board", ""),
+ want: `project = "MyProject"`,
+ },
+ {
+ name: "BoardName substitution",
+ tmpl: `project = "{{.BoardName}}"`,
+ data: makeData(42, "Team Alpha", ""),
+ want: `project = "Team Alpha"`,
+ },
+ {
+ name: "BoardId substitution",
+ tmpl: `cf[10001] = {{.BoardId}}`,
+ data: makeData(99, "Some Board", ""),
+ want: `cf[10001] = 99`,
+ },
+ {
+ name: "nil Board falls back to empty BoardName",
+ tmpl: `project = "{{.BoardName}}"`,
+ data: &JiraTaskData{Options: &JiraOptions{BoardId: 1},
Board: nil},
+ want: `project = ""`,
+ },
+ {
+ name: "invalid template returns error",
+ tmpl: `project = "{{.Unclosed"`,
+ data: makeData(1, "My Board", ""),
+ wantErr: true,
+ },
+ {
+ name: "unknown field returns error
(missingkey=error)",
+ tmpl: `component = "{{.Typo}}"`,
+ data: makeData(1, "My Board", ""),
+ wantErr: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := renderExtraJQL(tt.tmpl, tt.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("renderExtraJQL() error = %v, wantErr
%v", err, tt.wantErr)
+ return
+ }
+ if !tt.wantErr && got != tt.want {
+ t.Errorf("renderExtraJQL() = %v, want %v", got,
tt.want)
+ }
+ })
+ }
+}
diff --git a/backend/plugins/jira/tasks/task_data.go
b/backend/plugins/jira/tasks/task_data.go
index bfab9704f..3505bf162 100644
--- a/backend/plugins/jira/tasks/task_data.go
+++ b/backend/plugins/jira/tasks/task_data.go
@@ -38,6 +38,7 @@ type JiraTaskData struct {
ApiClient *api.ApiAsyncClient
JiraServerInfo models.JiraServerInfo
FilterId string
+ Board *models.JiraBoard
}
type JiraApiParams models.JiraApiParams
diff --git a/config-ui/src/plugins/register/jira/config.tsx
b/config-ui/src/plugins/register/jira/config.tsx
index d82186577..bb4996e65 100644
--- a/config-ui/src/plugins/register/jira/config.tsx
+++ b/config-ui/src/plugins/register/jira/config.tsx
@@ -64,6 +64,7 @@ export const JiraConfig: IPluginConfig = {
typeMappings: {},
remotelinkCommitShaPattern: '',
remotelinkRepoPattern: [],
+ extraJql: '',
},
},
};
diff --git a/config-ui/src/plugins/register/jira/transformation.tsx
b/config-ui/src/plugins/register/jira/transformation.tsx
index 97cd20967..bbfe760db 100644
--- a/config-ui/src/plugins/register/jira/transformation.tsx
+++ b/config-ui/src/plugins/register/jira/transformation.tsx
@@ -19,7 +19,7 @@
import { useState, useEffect } from 'react';
import { uniqWith } from 'lodash';
import { CaretRightOutlined } from '@ant-design/icons';
-import { theme, Collapse, Tag, Form, Select } from 'antd';
+import { theme, Collapse, Tag, Form, Select, Input } from 'antd';
import API from '@/api';
import { PageLoading, HelpTooltip, ExternalLink } from '@/components';
@@ -265,6 +265,26 @@ const renderCollapseItems = ({
}
/>
</Form.Item>
+ <Form.Item
+ label={
+ <>
+ <span>Extra JQL</span>
+ <HelpTooltip content="Additional JQL clause ANDed into every
issue query. Supports Go text/template syntax. Available variables:
{{.BoardName}} (Jira board display name), {{.BoardId}} (numeric board ID)." />
+ </>
+ }
+ extra='Tip: use Go template variables to make this dynamic, e.g.
owner = "{{.BoardName}}"'
+ >
+ <Input
+ placeholder='e.g. component = "my-project"'
+ value={transformation.extraJql}
+ onChange={(e) =>
+ onChangeTransformation({
+ ...transformation,
+ extraJql: e.target.value,
+ })
+ }
+ />
+ </Form.Item>
</Form>
),
},
