gnodet-bot commented on code in PR #30481:
URL: https://github.com/apache/camel/pull/30481#discussion_r4640128466
##########
components/camel-azure/camel-azure-storage-blob/src/main/java/org/apache/camel/component/azure/storage/blob/BlobConsumer.java:
##########
@@ -125,55 +134,107 @@ private Exchange createExchangeFromBlob(final String
blobName, final BlobContain
return exchange;
}
- @SuppressWarnings("unchecked")
private Queue<Exchange> createBatchExchangesFromContainer(final
BlobContainerClient blobContainerClient)
throws IOException {
final BlobContainerClientWrapper containerClientWrapper = new
BlobContainerClientWrapper(blobContainerClient);
final BlobContainerOperations containerOperations
= new
BlobContainerOperations(getEndpoint().getConfiguration(),
containerClientWrapper);
- final List<BlobItem> blobs = (List<BlobItem>)
containerOperations.listBlobs(null).getBody();
-
- // okay we have some response from azure so lets mark the consumer as
ready
- forceConsumerAsReady();
+ // Start a new pass only after the previous listing and its pending
page are exhausted.
+ if (pendingListedBlobs.isEmpty() && listingComplete) {
+ listingContinuationToken = null;
+ listingComplete = false;
+ }
+ // Prepare the complete poll batch before processBatch routes its
first exchange.
+ // This deliberately preserves Camel's exact batch size/index/complete
contract.
+ // maxMessagesPerPoll limits successful gets (0 means unlimited); this
is bounded
+ // prefetch when a limit is set, not strict just-in-time retrieval
before each route.
final Queue<Exchange> exchanges = new LinkedList<>();
- // the listed blobs that could not be got, other than those that no
longer exist, by name
final Map<String, Exception> failures = new LinkedHashMap<>();
- // whether at least one listed blob could be got
- boolean anyGot = false;
- for (BlobItem blobItem : blobs) {
- // a failure to get one listed blob (access denied, a blob in the
archive tier) does not fail the poll as
- // long as another listed blob could be got: the blob is skipped,
the other listed blobs are consumed, and
- // a later poll tries it again
- try {
- anyGot |= addExchange(blobItem.getName(), blobContainerClient,
exchanges, true);
- } catch (Exception e) {
- failures.put(blobItem.getName(), e);
+ final int maxMessagesPerPoll = getMaxMessagesPerPoll();
+
+ try {
+ // The limit counts successful gets, not names returned by Azure.
+ // In-progress blobs, deleted blobs (404), and failed gets do not
use the limit.
+ while (maxMessagesPerPoll <= 0 || exchanges.size() <
maxMessagesPerPoll) {
+ final BlobItem blobItem = nextListedBlob(containerOperations,
maxMessagesPerPoll);
+ if (blobItem == null) {
+ break;
+ }
+
+ try {
+ addExchange(blobItem.getName(), blobContainerClient,
exchanges, true);
+ } catch (Exception e) {
+ failures.put(blobItem.getName(), e);
+ }
}
+ } catch (RuntimeException e) {
+ // A listing failure after some blobs were opened must not leak
their streams
+ // or leave them marked in progress. Restart listing on the next
poll.
+ releaseNotProcessed(exchanges);
+ pendingListedBlobs.clear();
+ listingContinuationToken = null;
+ listingComplete = false;
+ throw e;
}
if (!failures.isEmpty()) {
- if (!anyGot) {
- // none of the listed blobs could be got, so this is not about
one blob (expired credentials, no
- // permission to read blobs, no connection): the poll fails,
and the backoff of the scheduled consumer
- // applies
+ if (exchanges.isEmpty()) {
+ // None of the listed blobs could be got, so this is not about
one blob
+ // (expired credentials, no permission, no connection): fail
the poll.
releaseNotProcessed(exchanges);
final Map.Entry<String, Exception> first =
failures.entrySet().iterator().next();
LOG.warn("None of the listed blobs of container {} could be
got ({} failed, the first is {}): the poll fails",
getEndpoint().getConfiguration().getContainerName(),
failures.size(), first.getKey());
rethrow(first.getValue());
}
+
for (Map.Entry<String, Exception> failure : failures.entrySet()) {
getExceptionHandler().handleException("Error getting blob " +
failure.getKey() + " from container "
+
getEndpoint().getConfiguration().getContainerName()
+ ". The blob is skipped
for this poll.",
failure.getValue());
}
}
+
return exchanges;
}
+ /**
+ * Returns the next matching blob, fetching only one Azure page when
necessary. The unconsumed names in a page
+ * remain queued for the next poll.
+ */
+ private BlobItem nextListedBlob(
+ final BlobContainerOperations containerOperations, final int
maxMessagesPerPoll) {
+ while (pendingListedBlobs.isEmpty() && !listingComplete) {
+ final Iterator<PagedResponse<BlobItem>> pages = containerOperations
+ .listBlobsPaged(null, listingContinuationToken,
maxMessagesPerPoll)
+ .iterableByPage().iterator();
+
+ if (!pages.hasNext()) {
+ listingComplete = true;
+ forceConsumerAsReady();
+ return null;
+ }
+
+ final PagedResponse<BlobItem> page = pages.next();
+ final String regex = getEndpoint().getConfiguration().getRegex();
+
+ for (BlobItem item : page.getValue()) {
+ if (ObjectHelper.isEmpty(regex) ||
item.getName().matches(regex)) {
+ pendingListedBlobs.add(item);
+ }
+ }
+
+ listingContinuationToken = page.getContinuationToken();
+ listingComplete = listingContinuationToken == null;
Review Comment:
⚠️ **Defensive token check**: `getContinuationToken()` returns `null` in the
common case, but the Azure SDK documentation does not preclude an empty string
token in edge cases (proxy serialization, older SDK versions). Using
`ObjectHelper.isEmpty()` here costs nothing and prevents an infinite re-listing
loop if an empty token were ever returned.
```suggestion
listingContinuationToken = page.getContinuationToken();
listingComplete = ObjectHelper.isEmpty(listingContinuationToken);
```
##########
components/camel-azure/camel-azure-storage-blob/src/test/java/org/apache/camel/component/azure/storage/blob/integration/BlobConsumerIT.java:
##########
@@ -248,12 +380,21 @@ public void configure() {
// if regex is set then prefix should have no effect
from("azure-storage-blob://cameldev/" + batchContainerName
- + "?prefix=aaaa®ex=" + regex)
+ + "?prefix=aaaa®ex=" + regex +
"&maxMessagesPerPoll=2&maxResultsPerPage=3")
.idempotentConsumer(body(), new
MemoryIdempotentRepository())
.to("mock:resultRegex");
- from("azure-storage-blob://cameldev/" + prefixContainerName +
"?prefix=" + prefix)
+ from("azure-storage-blob://cameldev/" + prefixContainerName +
"?prefix=" + prefix
+ +
"&maxMessagesPerPoll=1&maxResultsPerPage=1&deleteAfterRead=true")
.to("mock:resultPrefix");
+
+ from("azure-storage-blob://cameldev/" + limitedContainerName
+ +
"?maxMessagesPerPoll=2&maxResultsPerPage=2&deleteAfterRead=true&initialDelay=5000&delay=100")
Review Comment:
⚠️ **Timing-dependent test**: `initialDelay=5000` relies on a 5-second
wall-clock gap being enough for all five blob uploads to complete before the
first poll. On a slow CI agent this can flake. The
`testRetainedBlobsAdvanceAcrossPollsAndWrapAround` test in the same class
already uses the safer `autoStartup(false)` + `startRoute()` pattern that
eliminates the timing dependency. Apply the same approach here:
```java
context().getRouteController().startRoute("limitedBlobTest");
// ... assertions ...
context().getRouteController().stopRoute("limitedBlobTest");
```
(Echoes davsclaus's suggestion #4.)
##########
components/camel-azure/camel-azure-storage-blob/src/test/java/org/apache/camel/component/azure/storage/blob/integration/BlobConsumerIT.java:
##########
@@ -201,15 +227,117 @@ void testPrefixBasedPolling() throws
InterruptedException {
templateStart.send("direct:createBlob", exchange -> {
exchange.getIn().setBody("Blob 2");
exchange.getIn().setHeader(BlobConstants.BLOB_CONTAINER_NAME,
prefixContainerName);
+ exchange.getIn().setHeader(BlobConstants.BLOB_NAME, prefix +
"/test_blob_2");
+ });
+
+ // This blob must not be returned by the server-side prefix filter.
+ templateStart.send("direct:createBlob", exchange -> {
+ exchange.getIn().setBody("Not prefixed");
+ exchange.getIn().setHeader(BlobConstants.BLOB_CONTAINER_NAME,
prefixContainerName);
exchange.getIn().setHeader(BlobConstants.BLOB_NAME,
"non_prefixed_blob");
});
mockEndpoint.assertIsSatisfied();
- String text =
mockEndpoint.getExchanges().get(0).getIn().getBody(String.class);
+ assertEquals(2, mockEndpoint.getExchanges().size());
+ for (Exchange exchange : mockEndpoint.getExchanges()) {
+ String name = exchange.getIn().getHeader(BlobConstants.BLOB_NAME,
String.class);
+ assertTrue(name.startsWith(prefix + "/"));
+ assertEquals(1,
exchange.getProperty(ExchangePropertyKey.BATCH_SIZE));
+ assertEquals(0,
exchange.getProperty(ExchangePropertyKey.BATCH_INDEX));
+ assertEquals(Boolean.TRUE,
exchange.getProperty(ExchangePropertyKey.BATCH_COMPLETE));
+ }
+ }
+
+ @Test
+ void testMaxMessagesPerPollAndBatchProperties() throws Exception {
+ final MockEndpoint mockEndpoint =
getMockEndpoint("mock:resultLimited");
+ mockEndpoint.expectedMessageCount(5);
+ mockEndpoint.setResultWaitTime(30000);
+
+ // The consumer route has an initial delay, so all five blobs are
present
+ // before its first poll begins.
+ for (int i = 0; i < 5; i++) {
+ final int index = i;
+ templateStart.send("direct:createBlob", exchange -> {
+ exchange.getIn().setBody("Limited Batch Blob " + index);
+ exchange.getIn().setHeader(BlobConstants.BLOB_CONTAINER_NAME,
limitedContainerName);
+ exchange.getIn().setHeader(BlobConstants.BLOB_NAME,
"limited_blob_" + index);
+ });
+ }
- assertEquals("Blob 1", text);
- assertEquals(1, mockEndpoint.getExchanges().size());
+ mockEndpoint.assertIsSatisfied();
+
+ java.util.List<Integer> completedBatchSizes =
mockEndpoint.getExchanges().stream()
+ .filter(exchange -> Boolean.TRUE.equals(
+
exchange.getProperty(ExchangePropertyKey.BATCH_COMPLETE)))
+ .map(exchange -> (Integer)
exchange.getProperty(ExchangePropertyKey.BATCH_SIZE))
+ .toList();
+
+ assertEquals(java.util.List.of(2, 2, 1), completedBatchSizes,
+ "five blobs with a limit of two must be routed as batches of
2, 2 and 1");
+
+ long uniqueBlobNames = mockEndpoint.getExchanges().stream()
+ .map(exchange ->
exchange.getIn().getHeader(BlobConstants.BLOB_NAME, String.class))
+ .filter(java.util.Objects::nonNull)
Review Comment:
🔧 **FQN imports**: `java.util.List` and `java.util.Objects` are used as
fully-qualified names. The file already imports `java.util.HashSet` and
`java.util.Set` — the Camel AGENTS.md requires imports rather than FQNs. Add
`import java.util.List;` and `import java.util.Objects;` to the import block.
```suggestion
List<Integer> completedBatchSizes =
mockEndpoint.getExchanges().stream()
.filter(exchange -> Boolean.TRUE.equals(
exchange.getProperty(ExchangePropertyKey.BATCH_COMPLETE)))
.map(exchange -> (Integer)
exchange.getProperty(ExchangePropertyKey.BATCH_SIZE))
.toList();
assertEquals(List.of(2, 2, 1), completedBatchSizes,
"five blobs with a limit of two must be routed as batches of
2, 2 and 1");
long uniqueBlobNames = mockEndpoint.getExchanges().stream()
.map(exchange ->
exchange.getIn().getHeader(BlobConstants.BLOB_NAME, String.class))
.filter(Objects::nonNull)
```
##########
components/camel-azure/camel-azure-storage-blob/src/main/java/org/apache/camel/component/azure/storage/blob/BlobConsumer.java:
##########
@@ -58,6 +59,14 @@ public class BlobConsumer extends
ScheduledBatchPollingConsumer {
// blobs whose exchanges are being processed, which a later poll must not
consume again
private final Set<String> inProgress = ConcurrentHashMap.newKeySet();
+ // Remaining names from the current Azure listing page. Keeping these
across polls
+ // prevents a limited consumer from repeatedly starting with the first
page.
+ private final Queue<BlobItem> pendingListedBlobs = new LinkedList<>();
+
+ // Continuation state for the next Azure listing page.
+ private String listingContinuationToken;
+ private boolean listingComplete;
+
Review Comment:
💡 **Cursor survives route stop/start**: `pendingListedBlobs`,
`listingContinuationToken`, and `listingComplete` are instance fields that
persist across a `stop()`/`start()` lifecycle. If a route is restarted (e.g.
via `RouteController`), the consumer resumes from a potentially stale cursor —
blobs may have been added, deleted, or renamed while the route was down.
Consider overriding `doStart()` to reset the cursor:
```java
@Override
protected void doStart() throws Exception {
super.doStart();
pendingListedBlobs.clear();
listingContinuationToken = null;
listingComplete = false;
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]