ramu11 commented on code in PR #30481:
URL: https://github.com/apache/camel/pull/30481#discussion_r4640771255


##########
components/camel-azure/camel-azure-storage-blob/src/main/java/org/apache/camel/component/azure/storage/blob/BlobConsumer.java:
##########
@@ -125,55 +134,107 @@ private Exchange createExchangeFromBlob(final String 
blobName, final BlobContain
         return exchange;
     }
 
-    @SuppressWarnings("unchecked")
     private Queue<Exchange> createBatchExchangesFromContainer(final 
BlobContainerClient blobContainerClient)
             throws IOException {
         final BlobContainerClientWrapper containerClientWrapper = new 
BlobContainerClientWrapper(blobContainerClient);
         final BlobContainerOperations containerOperations
                 = new 
BlobContainerOperations(getEndpoint().getConfiguration(), 
containerClientWrapper);
 
-        final List<BlobItem> blobs = (List<BlobItem>) 
containerOperations.listBlobs(null).getBody();
-
-        // okay we have some response from azure so lets mark the consumer as 
ready
-        forceConsumerAsReady();
+        // Start a new pass only after the previous listing and its pending 
page are exhausted.
+        if (pendingListedBlobs.isEmpty() && listingComplete) {
+            listingContinuationToken = null;
+            listingComplete = false;
+        }
 
+        // Prepare the complete poll batch before processBatch routes its 
first exchange.
+        // This deliberately preserves Camel's exact batch size/index/complete 
contract.
+        // maxMessagesPerPoll limits successful gets (0 means unlimited); this 
is bounded
+        // prefetch when a limit is set, not strict just-in-time retrieval 
before each route.
         final Queue<Exchange> exchanges = new LinkedList<>();
-        // the listed blobs that could not be got, other than those that no 
longer exist, by name
         final Map<String, Exception> failures = new LinkedHashMap<>();
-        // whether at least one listed blob could be got
-        boolean anyGot = false;
-        for (BlobItem blobItem : blobs) {
-            // a failure to get one listed blob (access denied, a blob in the 
archive tier) does not fail the poll as
-            // long as another listed blob could be got: the blob is skipped, 
the other listed blobs are consumed, and
-            // a later poll tries it again
-            try {
-                anyGot |= addExchange(blobItem.getName(), blobContainerClient, 
exchanges, true);
-            } catch (Exception e) {
-                failures.put(blobItem.getName(), e);
+        final int maxMessagesPerPoll = getMaxMessagesPerPoll();
+
+        try {
+            // The limit counts successful gets, not names returned by Azure.
+            // In-progress blobs, deleted blobs (404), and failed gets do not 
use the limit.
+            while (maxMessagesPerPoll <= 0 || exchanges.size() < 
maxMessagesPerPoll) {
+                final BlobItem blobItem = nextListedBlob(containerOperations, 
maxMessagesPerPoll);
+                if (blobItem == null) {
+                    break;
+                }
+
+                try {
+                    addExchange(blobItem.getName(), blobContainerClient, 
exchanges, true);
+                } catch (Exception e) {
+                    failures.put(blobItem.getName(), e);
+                }
             }
+        } catch (RuntimeException e) {
+            // A listing failure after some blobs were opened must not leak 
their streams
+            // or leave them marked in progress. Restart listing on the next 
poll.
+            releaseNotProcessed(exchanges);
+            pendingListedBlobs.clear();
+            listingContinuationToken = null;
+            listingComplete = false;
+            throw e;
         }
 
         if (!failures.isEmpty()) {
-            if (!anyGot) {
-                // none of the listed blobs could be got, so this is not about 
one blob (expired credentials, no
-                // permission to read blobs, no connection): the poll fails, 
and the backoff of the scheduled consumer
-                // applies
+            if (exchanges.isEmpty()) {
+                // None of the listed blobs could be got, so this is not about 
one blob
+                // (expired credentials, no permission, no connection): fail 
the poll.
                 releaseNotProcessed(exchanges);
                 final Map.Entry<String, Exception> first = 
failures.entrySet().iterator().next();
                 LOG.warn("None of the listed blobs of container {} could be 
got ({} failed, the first is {}): the poll fails",
                         getEndpoint().getConfiguration().getContainerName(), 
failures.size(), first.getKey());
                 rethrow(first.getValue());
             }
+
             for (Map.Entry<String, Exception> failure : failures.entrySet()) {
                 getExceptionHandler().handleException("Error getting blob " + 
failure.getKey() + " from container "
                                                       + 
getEndpoint().getConfiguration().getContainerName()
                                                       + ". The blob is skipped 
for this poll.",
                         failure.getValue());
             }
         }
+
         return exchanges;
     }
 
+    /**
+     * Returns the next matching blob, fetching only one Azure page when 
necessary. The unconsumed names in a page
+     * remain queued for the next poll.
+     */
+    private BlobItem nextListedBlob(
+            final BlobContainerOperations containerOperations, final int 
maxMessagesPerPoll) {
+        while (pendingListedBlobs.isEmpty() && !listingComplete) {
+            final Iterator<PagedResponse<BlobItem>> pages = containerOperations
+                    .listBlobsPaged(null, listingContinuationToken, 
maxMessagesPerPoll)
+                    .iterableByPage().iterator();
+
+            if (!pages.hasNext()) {
+                listingComplete = true;
+                forceConsumerAsReady();
+                return null;
+            }
+
+            final PagedResponse<BlobItem> page = pages.next();
+            final String regex = getEndpoint().getConfiguration().getRegex();
+
+            for (BlobItem item : page.getValue()) {
+                if (ObjectHelper.isEmpty(regex) || 
item.getName().matches(regex)) {
+                    pendingListedBlobs.add(item);
+                }
+            }
+
+            listingContinuationToken = page.getContinuationToken();
+            listingComplete = listingContinuationToken == null;

Review Comment:
   Addressed in commit `6bd2f6a6386a`. Changed the completion check to 
`ObjectHelper.isEmpty(listingContinuationToken)` to handle both null and empty 
continuation tokens and avoid accidental re-listing.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to