gnodet-bot commented on code in PR #30481:
URL: https://github.com/apache/camel/pull/30481#discussion_r4640128466


##########
components/camel-azure/camel-azure-storage-blob/src/main/java/org/apache/camel/component/azure/storage/blob/BlobConsumer.java:
##########
@@ -125,55 +134,107 @@ private Exchange createExchangeFromBlob(final String 
blobName, final BlobContain
         return exchange;
     }
 
-    @SuppressWarnings("unchecked")
     private Queue<Exchange> createBatchExchangesFromContainer(final 
BlobContainerClient blobContainerClient)
             throws IOException {
         final BlobContainerClientWrapper containerClientWrapper = new 
BlobContainerClientWrapper(blobContainerClient);
         final BlobContainerOperations containerOperations
                 = new 
BlobContainerOperations(getEndpoint().getConfiguration(), 
containerClientWrapper);
 
-        final List<BlobItem> blobs = (List<BlobItem>) 
containerOperations.listBlobs(null).getBody();
-
-        // okay we have some response from azure so lets mark the consumer as 
ready
-        forceConsumerAsReady();
+        // Start a new pass only after the previous listing and its pending 
page are exhausted.
+        if (pendingListedBlobs.isEmpty() && listingComplete) {
+            listingContinuationToken = null;
+            listingComplete = false;
+        }
 
+        // Prepare the complete poll batch before processBatch routes its 
first exchange.
+        // This deliberately preserves Camel's exact batch size/index/complete 
contract.
+        // maxMessagesPerPoll limits successful gets (0 means unlimited); this 
is bounded
+        // prefetch when a limit is set, not strict just-in-time retrieval 
before each route.
         final Queue<Exchange> exchanges = new LinkedList<>();
-        // the listed blobs that could not be got, other than those that no 
longer exist, by name
         final Map<String, Exception> failures = new LinkedHashMap<>();
-        // whether at least one listed blob could be got
-        boolean anyGot = false;
-        for (BlobItem blobItem : blobs) {
-            // a failure to get one listed blob (access denied, a blob in the 
archive tier) does not fail the poll as
-            // long as another listed blob could be got: the blob is skipped, 
the other listed blobs are consumed, and
-            // a later poll tries it again
-            try {
-                anyGot |= addExchange(blobItem.getName(), blobContainerClient, 
exchanges, true);
-            } catch (Exception e) {
-                failures.put(blobItem.getName(), e);
+        final int maxMessagesPerPoll = getMaxMessagesPerPoll();
+
+        try {
+            // The limit counts successful gets, not names returned by Azure.
+            // In-progress blobs, deleted blobs (404), and failed gets do not 
use the limit.
+            while (maxMessagesPerPoll <= 0 || exchanges.size() < 
maxMessagesPerPoll) {
+                final BlobItem blobItem = nextListedBlob(containerOperations, 
maxMessagesPerPoll);
+                if (blobItem == null) {
+                    break;
+                }
+
+                try {
+                    addExchange(blobItem.getName(), blobContainerClient, 
exchanges, true);
+                } catch (Exception e) {
+                    failures.put(blobItem.getName(), e);
+                }
             }
+        } catch (RuntimeException e) {
+            // A listing failure after some blobs were opened must not leak 
their streams
+            // or leave them marked in progress. Restart listing on the next 
poll.
+            releaseNotProcessed(exchanges);
+            pendingListedBlobs.clear();
+            listingContinuationToken = null;
+            listingComplete = false;
+            throw e;
         }
 
         if (!failures.isEmpty()) {
-            if (!anyGot) {
-                // none of the listed blobs could be got, so this is not about 
one blob (expired credentials, no
-                // permission to read blobs, no connection): the poll fails, 
and the backoff of the scheduled consumer
-                // applies
+            if (exchanges.isEmpty()) {
+                // None of the listed blobs could be got, so this is not about 
one blob
+                // (expired credentials, no permission, no connection): fail 
the poll.
                 releaseNotProcessed(exchanges);
                 final Map.Entry<String, Exception> first = 
failures.entrySet().iterator().next();
                 LOG.warn("None of the listed blobs of container {} could be 
got ({} failed, the first is {}): the poll fails",
                         getEndpoint().getConfiguration().getContainerName(), 
failures.size(), first.getKey());
                 rethrow(first.getValue());
             }
+
             for (Map.Entry<String, Exception> failure : failures.entrySet()) {
                 getExceptionHandler().handleException("Error getting blob " + 
failure.getKey() + " from container "
                                                       + 
getEndpoint().getConfiguration().getContainerName()
                                                       + ". The blob is skipped 
for this poll.",
                         failure.getValue());
             }
         }
+
         return exchanges;
     }
 
+    /**
+     * Returns the next matching blob, fetching only one Azure page when 
necessary. The unconsumed names in a page
+     * remain queued for the next poll.
+     */
+    private BlobItem nextListedBlob(
+            final BlobContainerOperations containerOperations, final int 
maxMessagesPerPoll) {
+        while (pendingListedBlobs.isEmpty() && !listingComplete) {
+            final Iterator<PagedResponse<BlobItem>> pages = containerOperations
+                    .listBlobsPaged(null, listingContinuationToken, 
maxMessagesPerPoll)
+                    .iterableByPage().iterator();
+
+            if (!pages.hasNext()) {
+                listingComplete = true;
+                forceConsumerAsReady();
+                return null;
+            }
+
+            final PagedResponse<BlobItem> page = pages.next();
+            final String regex = getEndpoint().getConfiguration().getRegex();
+
+            for (BlobItem item : page.getValue()) {
+                if (ObjectHelper.isEmpty(regex) || 
item.getName().matches(regex)) {
+                    pendingListedBlobs.add(item);
+                }
+            }
+
+            listingContinuationToken = page.getContinuationToken();
+            listingComplete = listingContinuationToken == null;

Review Comment:
   ⚠️ **Defensive token check**: `getContinuationToken()` returns `null` in the 
common case, but the Azure SDK documentation does not preclude an empty string 
token in edge cases (proxy serialization, older SDK versions). Using 
`ObjectHelper.isEmpty()` here costs nothing and prevents an infinite re-listing 
loop if an empty token were ever returned.
   
   ```suggestion
               listingContinuationToken = page.getContinuationToken();
               listingComplete = ObjectHelper.isEmpty(listingContinuationToken);
   ```



##########
components/camel-azure/camel-azure-storage-blob/src/test/java/org/apache/camel/component/azure/storage/blob/integration/BlobConsumerIT.java:
##########
@@ -248,12 +380,21 @@ public void configure() {
 
                 // if regex is set then prefix should have no effect
                 from("azure-storage-blob://cameldev/" + batchContainerName
-                     + "?prefix=aaaa&regex=" + regex)
+                     + "?prefix=aaaa&regex=" + regex + 
"&maxMessagesPerPoll=2&maxResultsPerPage=3")
                         .idempotentConsumer(body(), new 
MemoryIdempotentRepository())
                         .to("mock:resultRegex");
 
-                from("azure-storage-blob://cameldev/" + prefixContainerName + 
"?prefix=" + prefix)
+                from("azure-storage-blob://cameldev/" + prefixContainerName + 
"?prefix=" + prefix
+                     + 
"&maxMessagesPerPoll=1&maxResultsPerPage=1&deleteAfterRead=true")
                         .to("mock:resultPrefix");
+
+                from("azure-storage-blob://cameldev/" + limitedContainerName
+                     + 
"?maxMessagesPerPoll=2&maxResultsPerPage=2&deleteAfterRead=true&initialDelay=5000&delay=100")

Review Comment:
   ⚠️ **Timing-dependent test**: `initialDelay=5000` relies on a 5-second 
wall-clock gap being enough for all five blob uploads to complete before the 
first poll. On a slow CI agent this can flake. The 
`testRetainedBlobsAdvanceAcrossPollsAndWrapAround` test in the same class 
already uses the safer `autoStartup(false)` + `startRoute()` pattern that 
eliminates the timing dependency. Apply the same approach here:
   
   ```java
   context().getRouteController().startRoute("limitedBlobTest");
   // ... assertions ...
   context().getRouteController().stopRoute("limitedBlobTest");
   ```
   
   (Echoes davsclaus's suggestion #4.)



##########
components/camel-azure/camel-azure-storage-blob/src/test/java/org/apache/camel/component/azure/storage/blob/integration/BlobConsumerIT.java:
##########
@@ -201,15 +227,117 @@ void testPrefixBasedPolling() throws 
InterruptedException {
         templateStart.send("direct:createBlob", exchange -> {
             exchange.getIn().setBody("Blob 2");
             exchange.getIn().setHeader(BlobConstants.BLOB_CONTAINER_NAME, 
prefixContainerName);
+            exchange.getIn().setHeader(BlobConstants.BLOB_NAME, prefix + 
"/test_blob_2");
+        });
+
+        // This blob must not be returned by the server-side prefix filter.
+        templateStart.send("direct:createBlob", exchange -> {
+            exchange.getIn().setBody("Not prefixed");
+            exchange.getIn().setHeader(BlobConstants.BLOB_CONTAINER_NAME, 
prefixContainerName);
             exchange.getIn().setHeader(BlobConstants.BLOB_NAME, 
"non_prefixed_blob");
         });
 
         mockEndpoint.assertIsSatisfied();
 
-        String text = 
mockEndpoint.getExchanges().get(0).getIn().getBody(String.class);
+        assertEquals(2, mockEndpoint.getExchanges().size());
+        for (Exchange exchange : mockEndpoint.getExchanges()) {
+            String name = exchange.getIn().getHeader(BlobConstants.BLOB_NAME, 
String.class);
+            assertTrue(name.startsWith(prefix + "/"));
+            assertEquals(1, 
exchange.getProperty(ExchangePropertyKey.BATCH_SIZE));
+            assertEquals(0, 
exchange.getProperty(ExchangePropertyKey.BATCH_INDEX));
+            assertEquals(Boolean.TRUE, 
exchange.getProperty(ExchangePropertyKey.BATCH_COMPLETE));
+        }
+    }
+
+    @Test
+    void testMaxMessagesPerPollAndBatchProperties() throws Exception {
+        final MockEndpoint mockEndpoint = 
getMockEndpoint("mock:resultLimited");
+        mockEndpoint.expectedMessageCount(5);
+        mockEndpoint.setResultWaitTime(30000);
+
+        // The consumer route has an initial delay, so all five blobs are 
present
+        // before its first poll begins.
+        for (int i = 0; i < 5; i++) {
+            final int index = i;
+            templateStart.send("direct:createBlob", exchange -> {
+                exchange.getIn().setBody("Limited Batch Blob " + index);
+                exchange.getIn().setHeader(BlobConstants.BLOB_CONTAINER_NAME, 
limitedContainerName);
+                exchange.getIn().setHeader(BlobConstants.BLOB_NAME, 
"limited_blob_" + index);
+            });
+        }
 
-        assertEquals("Blob 1", text);
-        assertEquals(1, mockEndpoint.getExchanges().size());
+        mockEndpoint.assertIsSatisfied();
+
+        java.util.List<Integer> completedBatchSizes = 
mockEndpoint.getExchanges().stream()
+                .filter(exchange -> Boolean.TRUE.equals(
+                        
exchange.getProperty(ExchangePropertyKey.BATCH_COMPLETE)))
+                .map(exchange -> (Integer) 
exchange.getProperty(ExchangePropertyKey.BATCH_SIZE))
+                .toList();
+
+        assertEquals(java.util.List.of(2, 2, 1), completedBatchSizes,
+                "five blobs with a limit of two must be routed as batches of 
2, 2 and 1");
+
+        long uniqueBlobNames = mockEndpoint.getExchanges().stream()
+                .map(exchange -> 
exchange.getIn().getHeader(BlobConstants.BLOB_NAME, String.class))
+                .filter(java.util.Objects::nonNull)

Review Comment:
   🔧 **FQN imports**: `java.util.List` and `java.util.Objects` are used as 
fully-qualified names. The file already imports `java.util.HashSet` and 
`java.util.Set` — the Camel AGENTS.md requires imports rather than FQNs. Add 
`import java.util.List;` and `import java.util.Objects;` to the import block.
   
   ```suggestion
           List<Integer> completedBatchSizes = 
mockEndpoint.getExchanges().stream()
                   .filter(exchange -> Boolean.TRUE.equals(
                           
exchange.getProperty(ExchangePropertyKey.BATCH_COMPLETE)))
                   .map(exchange -> (Integer) 
exchange.getProperty(ExchangePropertyKey.BATCH_SIZE))
                   .toList();
   
           assertEquals(List.of(2, 2, 1), completedBatchSizes,
                   "five blobs with a limit of two must be routed as batches of 
2, 2 and 1");
   
           long uniqueBlobNames = mockEndpoint.getExchanges().stream()
                   .map(exchange -> 
exchange.getIn().getHeader(BlobConstants.BLOB_NAME, String.class))
                   .filter(Objects::nonNull)
   ```



##########
components/camel-azure/camel-azure-storage-blob/src/main/java/org/apache/camel/component/azure/storage/blob/BlobConsumer.java:
##########
@@ -58,6 +59,14 @@ public class BlobConsumer extends 
ScheduledBatchPollingConsumer {
     // blobs whose exchanges are being processed, which a later poll must not 
consume again
     private final Set<String> inProgress = ConcurrentHashMap.newKeySet();
 
+    // Remaining names from the current Azure listing page. Keeping these 
across polls
+    // prevents a limited consumer from repeatedly starting with the first 
page.
+    private final Queue<BlobItem> pendingListedBlobs = new LinkedList<>();
+
+    // Continuation state for the next Azure listing page.
+    private String listingContinuationToken;
+    private boolean listingComplete;
+

Review Comment:
   💡 **Cursor survives route stop/start**: `pendingListedBlobs`, 
`listingContinuationToken`, and `listingComplete` are instance fields that 
persist across a `stop()`/`start()` lifecycle. If a route is restarted (e.g. 
via `RouteController`), the consumer resumes from a potentially stale cursor — 
blobs may have been added, deleted, or renamed while the route was down. 
Consider overriding `doStart()` to reset the cursor:
   
   ```java
   @Override
   protected void doStart() throws Exception {
       super.doStart();
       pendingListedBlobs.clear();
       listingContinuationToken = null;
       listingComplete = false;
   }
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to