This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch camel-4.14.x
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/camel-4.14.x by this push:
new d4504d1833ac feat(CAMEL-17348): Jira - newIssues - use "created" field
instead of … (#19577)
d4504d1833ac is described below
commit d4504d1833acfa624b9243762e28a9ae913fa8ac
Author: Andrej Vaňo <[email protected]>
AuthorDate: Thu Oct 16 09:34:28 2025 +0200
feat(CAMEL-17348): Jira - newIssues - use "created" field instead of …
(#19577)
* feat(CAMEL-17348): Jira - newIssues - use "created" field instead of "id"
* fix(CAMEL-22558): Jira - consumer delay option was ignored
---
.../camel-jira/src/main/docs/jira-component.adoc | 6 +-
.../jira/consumer/AbstractJiraConsumer.java | 2 +-
.../component/jira/consumer/NewIssuesConsumer.java | 133 ++++++++++++++-------
.../org/apache/camel/component/jira/Utils.java | 12 +-
.../jira/consumer/NewIssuesConsumerTest.java | 116 ++++++++++++++++--
5 files changed, 209 insertions(+), 60 deletions(-)
diff --git a/components/camel-jira/src/main/docs/jira-component.adoc
b/components/camel-jira/src/main/docs/jira-component.adoc
index b3b4fc1aa792..716869b620e0 100644
--- a/components/camel-jira/src/main/docs/jira-component.adoc
+++ b/components/camel-jira/src/main/docs/jira-component.adoc
@@ -132,7 +132,7 @@ Follow the tutorial to generate the
https://confluence.atlassian.com/enterprise/
=== JQL
-The JQL URI option is used by both consumer endpoints. Theoretically,
+The JQL URI option is used by both consumer endpoints. Theoretically,
items like the "project key", etc. could be URI options themselves.
However, by requiring the use of JQL, the consumers become much more
flexible and powerful.
@@ -146,8 +146,8 @@ jira://[type]?[required options]&jql=project=[project key]
One important thing to note is that the newIssues consumer will
automatically set the JQL as:
-* append `ORDER BY key desc` to your JQL
-* prepend `id > latestIssueId` to retrieve issues added after the camel route
was started.
+* append `ORDER BY created DESC` to your JQL
+* prepend `created > latestIssueCreationDate` to retrieve issues added after
the camel route was started.
This is in order to optimize startup processing, rather than having to index
every single
issue in the project.
diff --git
a/components/camel-jira/src/main/java/org/apache/camel/component/jira/consumer/AbstractJiraConsumer.java
b/components/camel-jira/src/main/java/org/apache/camel/component/jira/consumer/AbstractJiraConsumer.java
index c9210811a5dd..a249a02c9203 100644
---
a/components/camel-jira/src/main/java/org/apache/camel/component/jira/consumer/AbstractJiraConsumer.java
+++
b/components/camel-jira/src/main/java/org/apache/camel/component/jira/consumer/AbstractJiraConsumer.java
@@ -49,7 +49,7 @@ public abstract class AbstractJiraConsumer extends
ScheduledBatchPollingConsumer
protected AbstractJiraConsumer(JiraEndpoint endpoint, Processor processor)
{
super(endpoint, processor);
this.endpoint = endpoint;
- setDelay(endpoint.getDelay());
+ this.endpoint.setDelay(endpoint.getDelay());
}
@Override
diff --git
a/components/camel-jira/src/main/java/org/apache/camel/component/jira/consumer/NewIssuesConsumer.java
b/components/camel-jira/src/main/java/org/apache/camel/component/jira/consumer/NewIssuesConsumer.java
index ea2ee3a7f9cb..32c5de9672e1 100644
---
a/components/camel-jira/src/main/java/org/apache/camel/component/jira/consumer/NewIssuesConsumer.java
+++
b/components/camel-jira/src/main/java/org/apache/camel/component/jira/consumer/NewIssuesConsumer.java
@@ -16,57 +16,102 @@
*/
package org.apache.camel.component.jira.consumer;
-import java.util.LinkedList;
+import java.net.URI;
import java.util.Queue;
-import com.atlassian.jira.rest.client.api.RestClientException;
import com.atlassian.jira.rest.client.api.domain.Issue;
+import com.atlassian.jira.rest.client.api.domain.User;
import org.apache.camel.Processor;
import org.apache.camel.component.jira.JiraEndpoint;
import org.apache.camel.util.CastUtils;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Consumes new JIRA issues.
- *
- * NOTE: We manually add "ORDER BY key desc" to the JQL in order to optimize
startup (the latest issues one at a time),
- * rather than having to index everything.
+ * <p>
+ * To correctly support getting issues across multiple projects, the issues
are ordered using created attribute, the
+ * creation date is later used in the JQL to get only the new issues. JQL does
not support timestamp or seconds and the
+ * most useful format is yyyy-MM-dd HH:mm.
+ * <p>
+ * Using the date and time in JQL is a bit tricky the creationDate of an issue
is returned in server timezone, but the
+ * date and time in the JQL must match the timezone of the user executing the
query to work properly.
+ * <p>
+ * NOTE: We manually add "ORDER BY created DESC" to the JQL in order to
optimize startup (the latest issues one at a
+ * time) and to correctly get the newest issue.
*/
public class NewIssuesConsumer extends AbstractJiraConsumer {
-
private static final Logger LOG =
LoggerFactory.getLogger(NewIssuesConsumer.class);
+ // Date format used in the JQL below
+ private static final DateTimeFormatter JIRA_DATE_FORMAT =
DateTimeFormat.forPattern("yyyy-MM-dd HH:mm");
+ // Even the operator is ">", the JQL behaves more like ">=", so it will
return all issues created at that minute and later
+ private static final String NEW_ISSUES_JQL_FORMAT = "created > \"%s\" AND
%s ORDER BY created DESC";
- private final String jql;
- private long latestIssueId = -1;
+ // Last issue that was processed by the integration
+ private Issue latestIssue;
+ // timezone of the current user authenticated
+ private DateTimeZone userTimeZone;
public NewIssuesConsumer(JiraEndpoint endpoint, Processor processor) {
super(endpoint, processor);
- jql = endpoint.getJql() + " ORDER BY key desc";
}
@Override
protected void doStart() throws Exception {
super.doStart();
- // read the actual issues, the next poll outputs only the new issues
added after the route start
- // grab only the top
- latestIssueId = findLatestIssueId();
+
+ latestIssue = findLatestIssue();
+ if (latestIssue != null) {
+ LOG.debug("Init: Latest issue: {}", latestIssue.getKey());
+ }
+
+ userTimeZone = getUserTimeZone();
+ }
+
+ /**
+ * Change the creation date of the latest issue to the timezone of the
user.
+ *
+ * @return timestamp string used in the JQL
+ */
+ private String getServerTimestamp() {
+ return
latestIssue.getCreationDate().withZone(userTimeZone).toString(JIRA_DATE_FORMAT);
+ }
+
+ /**
+ * Gets the timezone of the currently authenticated user. Basic auth
configuration has precedence over OAuth
+ * credentials as in the login process.
+ *
+ * @return {@link DateTimeZone} of the user
+ */
+ private DateTimeZone getUserTimeZone() {
+ URI userURI = URI.create(getEndpoint().getConfiguration().getJiraUrl()
+ "/rest/api/latest/myself");
+ final User user =
getEndpoint().getClient().getUserClient().getUser(userURI).claim();
+ final String timezone = user.getTimezone();
+
+ LOG.debug("Using user {} with timezone {}", user.getName(), timezone);
+ return DateTimeZone.forID(timezone);
}
- protected long findLatestIssueId() {
- // read the actual issues, the next poll outputs only the new issues
added after the route start
- // grab only the top
+ /**
+ * At the start, find the latest issue, so that only newer issues are
processed.
+ *
+ * @return latest {@link Issue} or null
+ */
+ private Issue findLatestIssue() {
try {
- Queue<Issue> issues = getIssues(jql, 1);
+ // get the issues, ordered so that the latest one is the first one
returned by the query
+ Queue<Issue> issues = getIssues(getEndpoint().getJql() + " ORDER
BY created DESC", 1);
if (!issues.isEmpty()) {
- // Issues returned are ordered descendant so this is the
newest issue
- return issues.peek().getId();
+ return issues.peek();
}
} catch (Exception e) {
// ignore
}
// in case there aren't any issues...
- return -1;
+ return null;
}
protected int doPoll() throws Exception {
@@ -78,39 +123,35 @@ public class NewIssuesConsumer extends
AbstractJiraConsumer {
return newIssues.size();
}
+ /**
+ * Get the new issues and save the latest one.
+ *
+ * @return queue of new issues
+ */
private Queue<Issue> getNewIssues() {
String jqlFilter;
- if (latestIssueId > -1) {
- // search only for issues created after the latest id
- jqlFilter = "id > " + latestIssueId + " AND " + jql;
+ // If we have processed an issue before, use its timestamp to get only
the newer ones
+ if (latestIssue != null) {
+ jqlFilter = String.format(NEW_ISSUES_JQL_FORMAT,
getServerTimestamp(), getEndpoint().getJql());
} else {
- jqlFilter = jql;
- }
- // the last issue may be deleted, so to recover we re-find it and go
from there
- Queue<Issue> issues;
- try {
- issues = getIssues(jqlFilter);
- } catch (RestClientException e) {
- if (e.getStatusCode().isPresent()) {
- int code = e.getStatusCode().get();
- if (code == 400) {
- String msg = e.getMessage();
- if (msg != null && msg.contains("does not exist for the
field 'id'")) {
- LOG.warn("Last issue id: {} no longer exists (could
have been deleted)."
- + " Will recover by fetching last issue id
from JIRA and try again on next poll",
- latestIssueId);
- latestIssueId = findLatestIssueId();
- return new LinkedList<>();
- }
- }
- }
- throw e;
+ jqlFilter = getEndpoint().getJql();
}
+ Queue<Issue> issues = getIssues(jqlFilter);
+
if (!issues.isEmpty()) {
- // remember last id we have processed
- // issues are ordered descendant so save the first issue in the
list as the newest
- latestIssueId = issues.element().getId();
+ if (latestIssue != null) {
+ // remove all issues that are older than the latestIssue
(including), because those were processed by previous polls
+ issues.removeIf(i ->
i.getCreationDate().isBefore(latestIssue.getCreationDate()) ||
i.getCreationDate().isEqual(
+ latestIssue.getCreationDate()));
+ }
+
+ // we might filter out all issues in the previous statement,
resulting in no new issues
+ if (!issues.isEmpty()) {
+ // if there are issues left in the queue, save the newest one
+ latestIssue = issues.peek();
+ LOG.debug("Latest issue: {}", latestIssue.getKey());
+ }
}
return issues;
}
diff --git
a/components/camel-jira/src/test/java/org/apache/camel/component/jira/Utils.java
b/components/camel-jira/src/test/java/org/apache/camel/component/jira/Utils.java
index d49585a1a524..a0c41dec0d46 100644
---
a/components/camel-jira/src/test/java/org/apache/camel/component/jira/Utils.java
+++
b/components/camel-jira/src/test/java/org/apache/camel/component/jira/Utils.java
@@ -100,6 +100,15 @@ public final class Utils {
return transitionIssueDone(issue, status, resolution);
}
+ public static Issue createIssueWithCreationDate(long id, DateTime
dateTime) {
+ URI selfUri = URI.create(TEST_JIRA_URL + "/rest/api/latest/issue/" +
id);
+ return new Issue(
+ "jira summary test " + id, selfUri, KEY + "-" + id, id, null,
issueType, null, "Description " + id,
+ null, null, null, null, userAssignee, dateTime, null, null,
null, null, null, null, null, null, null,
+ null,
+ null, null, null, null, null, null, null, null, null);
+ }
+
public static Issue createIssueWithAttachment(
long id, String summary, String key, IssueType issueType, String
description,
BasicPriority priority, User assignee, Collection<Attachment>
attachments) {
@@ -125,7 +134,8 @@ public final class Utils {
URI selfUri = URI.create(TEST_JIRA_URL + "/rest/api/latest/issue/" +
id);
return new Issue(
"jira summary test " + id, selfUri, KEY + "-" + id, id, null,
issueType, null, "Description " + id,
- null, null, null, null, userAssignee, null, null, null, null,
null, null, null, null, comments, null, null,
+ null, null, null, null, userAssignee, DateTime.now(), null,
null, null, null, null, null, null, comments, null,
+ null,
null, null, null, null, null, null, null, null, null);
}
diff --git
a/components/camel-jira/src/test/java/org/apache/camel/component/jira/consumer/NewIssuesConsumerTest.java
b/components/camel-jira/src/test/java/org/apache/camel/component/jira/consumer/NewIssuesConsumerTest.java
index c3931b8003aa..9f019c6eb012 100644
---
a/components/camel-jira/src/test/java/org/apache/camel/component/jira/consumer/NewIssuesConsumerTest.java
+++
b/components/camel-jira/src/test/java/org/apache/camel/component/jira/consumer/NewIssuesConsumerTest.java
@@ -16,16 +16,20 @@
*/
package org.apache.camel.component.jira.consumer;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import com.atlassian.jira.rest.client.api.JiraRestClient;
import com.atlassian.jira.rest.client.api.JiraRestClientFactory;
import com.atlassian.jira.rest.client.api.SearchRestClient;
+import com.atlassian.jira.rest.client.api.UserRestClient;
import com.atlassian.jira.rest.client.api.domain.Issue;
import com.atlassian.jira.rest.client.api.domain.SearchResult;
+import com.atlassian.jira.rest.client.api.domain.User;
import io.atlassian.util.concurrent.Promise;
import io.atlassian.util.concurrent.Promises;
import org.apache.camel.CamelContext;
@@ -35,6 +39,8 @@ import org.apache.camel.component.jira.JiraComponent;
import org.apache.camel.component.mock.MockEndpoint;
import org.apache.camel.spi.Registry;
import org.apache.camel.test.junit5.CamelTestSupport;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@ -47,6 +53,7 @@ import static
org.apache.camel.component.jira.JiraConstants.JIRA_REST_CLIENT_FAC
import static
org.apache.camel.component.jira.JiraTestConstants.JIRA_CREDENTIALS;
import static org.apache.camel.component.jira.JiraTestConstants.PROJECT;
import static org.apache.camel.component.jira.Utils.createIssue;
+import static
org.apache.camel.component.jira.Utils.createIssueWithCreationDate;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.when;
@@ -65,6 +72,9 @@ public class NewIssuesConsumerTest extends CamelTestSupport {
@Mock
private SearchRestClient searchRestClient;
+ @Mock
+ private UserRestClient userRestClient;
+
@EndpointInject("mock:result")
private MockEndpoint mockResult;
@@ -75,18 +85,24 @@ public class NewIssuesConsumerTest extends CamelTestSupport
{
@BeforeAll
public static void beforeAll() {
- ISSUES.add(createIssue(3L));
- ISSUES.add(createIssue(2L));
- ISSUES.add(createIssue(1L));
+ ISSUES.add(createIssueWithCreationDate(3L,
DateTime.now().minusMinutes(10)));
+ ISSUES.add(createIssueWithCreationDate(2L,
DateTime.now().minusMinutes(8)));
+ ISSUES.add(createIssueWithCreationDate(1L,
DateTime.now().minusMinutes(6)));
}
public void setMocks() {
- SearchResult result = new SearchResult(0, 50, 3, ISSUES);
+ SearchResult result = new SearchResult(0, 50, 0, ISSUES);
Promise<SearchResult> promiseSearchResult = Promises.promise(result);
+ User user = new User(
+ null, "admin", null, null, true, null,
+ Map.of("48x48", URI.create("")),
DateTime.now().getZone().getID());
+ Promise<User> promiseUserResult = Promises.promise(user);
when(jiraClient.getSearchClient()).thenReturn(searchRestClient);
+ when(jiraClient.getUserClient()).thenReturn(userRestClient);
when(jiraRestClientFactory.createWithBasicHttpAuthentication(any(),
any(), any())).thenReturn(jiraClient);
when(searchRestClient.searchJql(any(), any(), any(),
any())).thenReturn(promiseSearchResult);
+
when(userRestClient.getUser(any(URI.class))).thenReturn(promiseUserResult);
}
@Override
@@ -216,10 +232,11 @@ public class NewIssuesConsumerTest extends
CamelTestSupport {
@Test
public void multipleQueriesOffsetFilterTest() throws Exception {
- Issue issue1 = createIssue(51);
- Issue issue2 = createIssue(52);
- Issue issue3 = createIssue(53);
- Issue issue4 = createIssue(54);
+ DateTime now = DateTime.now();
+ Issue issue1 = createIssueWithCreationDate(51, now.minusMinutes(3));
+ Issue issue2 = createIssueWithCreationDate(52, now.minusMinutes(2));
+ Issue issue3 = createIssueWithCreationDate(53, now.minusMinutes(1));
+ Issue issue4 = createIssueWithCreationDate(54, now);
reset(searchRestClient);
when(searchRestClient.searchJql(any(), any(), any(),
any())).then(invocation -> {
@@ -230,7 +247,7 @@ public class NewIssuesConsumerTest extends CamelTestSupport
{
Assertions.assertEquals(0, startAt);
String jqlFilter = invocation.getArgument(0);
- Assertions.assertTrue(jqlFilter.startsWith("id > 53"));
+ Assertions.assertTrue(jqlFilter.startsWith("created > \"" +
now.minusMinutes(1).toString("yyyy-MM-dd HH:mm")));
SearchResult result = new SearchResult(0, 50, 1,
Collections.singletonList(issue4));
return Promises.promise(result);
});
@@ -243,4 +260,85 @@ public class NewIssuesConsumerTest extends
CamelTestSupport {
mockResult.expectedBodiesReceived(issue4);
mockResult.assertIsSatisfied();
}
+
+ @Test
+ public void shouldIgnoreOlderIssuesInPollTest() throws Exception {
+ DateTime now = DateTime.now();
+ Issue issue1 = createIssueWithCreationDate(8, now.minusSeconds(2));
+ Issue issue2 = createIssueWithCreationDate(9, now.minusSeconds(1));
+ Issue issue3 = createIssueWithCreationDate(10, now);
+
+ reset(searchRestClient);
+ when(searchRestClient.searchJql(any(), any(), any(), any()))
+ .then(invocation -> {
+ SearchResult result = new SearchResult(0, 50, 1,
List.of(issue2));
+ return Promises.promise(result);
+ }).then(invocation -> {
+ SearchResult result = new SearchResult(0, 50, 3,
List.of(issue1, issue2, issue3));
+ return Promises.promise(result);
+ });
+
+ mockResult.expectedBodiesReceived(issue2);
+ mockResult.assertIsSatisfied();
+
+ mockResult.reset();
+
+ mockResult.expectedBodiesReceived(issue3);
+ mockResult.assertIsSatisfied();
+ }
+
+ @Test
+ public void shouldUseUsersTimezoneInJQLTest() throws Exception {
+ DateTime now = DateTime.now();
+ DateTime perth = now.withZone(DateTimeZone.forID("Australia/Perth"));
+ Issue issue1 = createIssueWithCreationDate(8, perth);
+ Issue issue2 = createIssueWithCreationDate(9, perth.plusSeconds(1));
+
+ reset(searchRestClient);
+ when(searchRestClient.searchJql(any(), any(), any(), any()))
+ .then(invocation -> {
+ SearchResult result = new SearchResult(0, 50, 1,
List.of(issue1));
+ return Promises.promise(result);
+ }).then(invocation -> {
+ String jqlFilter = invocation.getArgument(0);
+ Assertions.assertTrue(
+ jqlFilter.startsWith("created > \"" +
now.toString("yyyy-MM-dd HH:mm")));
+ SearchResult result = new SearchResult(0, 50, 1,
List.of(issue2));
+ return Promises.promise(result);
+ });
+
+ mockResult.expectedBodiesReceived(issue1, issue2);
+ mockResult.assertIsSatisfied();
+ }
+
+ @Test
+ public void shouldDoEmptyPollTest() throws Exception {
+ DateTime now = DateTime.now();
+ Issue issue1 = createIssueWithCreationDate(9, now.minusSeconds(1));
+ Issue issue2 = createIssueWithCreationDate(10, now);
+
+ reset(searchRestClient);
+ when(searchRestClient.searchJql(any(), any(), any(), any()))
+ .then(invocation -> {
+ SearchResult result = new SearchResult(0, 50, 1,
List.of(issue1));
+ return Promises.promise(result);
+ }).then(invocation -> {
+ SearchResult result = new SearchResult(0, 50, 1,
List.of(issue1));
+ return Promises.promise(result);
+ }).then(invocation -> {
+ SearchResult result = new SearchResult(0, 50, 1,
List.of(issue2));
+ return Promises.promise(result);
+ });
+
+ mockResult.expectedBodiesReceived(issue1);
+ mockResult.assertIsSatisfied();
+
+ mockResult.reset();
+ mockResult.expectedMessageCount(0);
+ mockResult.assertIsSatisfied();
+
+ mockResult.reset();
+ mockResult.expectedBodiesReceived(issue2);
+ mockResult.assertIsSatisfied();
+ }
}
