This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 75ddf4853b21 CAMEL-25575: camel-dropbox - search and get read the
following pages of the result (#27759)
75ddf4853b21 is described below
commit 75ddf4853b212cdacfc16d445dd904095ab9bab0
Author: allthingssecurity <[email protected]>
AuthorDate: Tue Oct 13 12:25:30 2026 +0530
CAMEL-25575: camel-dropbox - search and get read the following pages of the
result (#27759)
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
components/camel-dropbox/pom.xml | 6 +
.../component/dropbox/core/DropboxAPIFacade.java | 28 +++-
.../dropbox/core/DropboxAPIFacadePagingTest.java | 174 +++++++++++++++++++++
3 files changed, 202 insertions(+), 6 deletions(-)
diff --git a/components/camel-dropbox/pom.xml b/components/camel-dropbox/pom.xml
index eac7138d2167..db3253e147a7 100644
--- a/components/camel-dropbox/pom.xml
+++ b/components/camel-dropbox/pom.xml
@@ -78,5 +78,11 @@
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito-version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git
a/components/camel-dropbox/src/main/java/org/apache/camel/component/dropbox/core/DropboxAPIFacade.java
b/components/camel-dropbox/src/main/java/org/apache/camel/component/dropbox/core/DropboxAPIFacade.java
index 9a4da2686e2f..496afd477955 100644
---
a/components/camel-dropbox/src/main/java/org/apache/camel/component/dropbox/core/DropboxAPIFacade.java
+++
b/components/camel-dropbox/src/main/java/org/apache/camel/component/dropbox/core/DropboxAPIFacade.java
@@ -22,9 +22,11 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.AbstractMap;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import com.dropbox.core.DbxDownloader;
@@ -34,6 +36,7 @@ import com.dropbox.core.v2.files.FileMetadata;
import com.dropbox.core.v2.files.ListFolderErrorException;
import com.dropbox.core.v2.files.ListFolderResult;
import com.dropbox.core.v2.files.Metadata;
+import com.dropbox.core.v2.files.SearchMatchV2;
import com.dropbox.core.v2.files.SearchOptions;
import com.dropbox.core.v2.files.SearchV2Result;
import com.dropbox.core.v2.files.WriteMode;
@@ -252,7 +255,13 @@ public final class DropboxAPIFacade {
SearchV2Result listing = client.files().searchV2Builder(query)
.withOptions(SearchOptions.newBuilder().withPath(remotePath).build())
.start();
- return new DropboxSearchResult(listing.getMatches());
+ List<SearchMatchV2> matches = new
ArrayList<>(listing.getMatches());
+ // a response holds one page of the matches: read the following
pages with the cursor of the response
+ while (listing.getHasMore() && listing.getCursor() != null) {
+ listing = client.files().searchContinueV2(listing.getCursor());
+ matches.addAll(listing.getMatches());
+ }
+ return new DropboxSearchResult(matches);
} catch (DbxException e) {
throw new DropboxException(String.format("%s does not exist or
cannot obtain metadata", remotePath), e);
}
@@ -308,12 +317,19 @@ public final class DropboxAPIFacade {
try {
ListFolderResult folderResult =
client.files().listFolder(path.equals("/") ? "" : path);
Map<String, Object> returnMap = new LinkedHashMap<>();
- for (Metadata entry : folderResult.getEntries()) {
- Map.Entry<String, Object> singleFile =
downloadSingleFile(entry.getPathDisplay());
- if (singleFile == null) {
- continue;
+ while (true) {
+ for (Metadata entry : folderResult.getEntries()) {
+ Map.Entry<String, Object> singleFile =
downloadSingleFile(entry.getPathDisplay());
+ if (singleFile == null) {
+ continue;
+ }
+ returnMap.put(entry.getPathDisplay(),
singleFile.getValue());
+ }
+ if (!folderResult.getHasMore()) {
+ break;
}
- returnMap.put(entry.getPathDisplay(), singleFile.getValue());
+ // a response holds one page of the folder: read the following
pages with the cursor of the response
+ folderResult =
client.files().listFolderContinue(folderResult.getCursor());
}
return returnMap;
} catch (ListFolderErrorException e) {
diff --git
a/components/camel-dropbox/src/test/java/org/apache/camel/component/dropbox/core/DropboxAPIFacadePagingTest.java
b/components/camel-dropbox/src/test/java/org/apache/camel/component/dropbox/core/DropboxAPIFacadePagingTest.java
new file mode 100644
index 000000000000..40258b76aa49
--- /dev/null
+++
b/components/camel-dropbox/src/test/java/org/apache/camel/component/dropbox/core/DropboxAPIFacadePagingTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.dropbox.core;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import com.dropbox.core.DbxDownloader;
+import com.dropbox.core.v2.DbxClientV2;
+import com.dropbox.core.v2.files.DbxUserFilesRequests;
+import com.dropbox.core.v2.files.FileMetadata;
+import com.dropbox.core.v2.files.ListFolderResult;
+import com.dropbox.core.v2.files.Metadata;
+import com.dropbox.core.v2.files.MetadataV2;
+import com.dropbox.core.v2.files.SearchMatchV2;
+import com.dropbox.core.v2.files.SearchOptions;
+import com.dropbox.core.v2.files.SearchV2Builder;
+import com.dropbox.core.v2.files.SearchV2Result;
+import org.apache.camel.CamelContext;
+import org.apache.camel.component.dropbox.dto.DropboxFileDownloadResult;
+import org.apache.camel.component.dropbox.dto.DropboxSearchResult;
+import org.apache.camel.impl.DefaultCamelContext;
+import org.apache.camel.support.DefaultExchange;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * The Dropbox client is a mock that answers a search and a folder listing in
pages, as the service does: a page, the
+ * {@code has_more} flag and a cursor for the next page.
+ */
+class DropboxAPIFacadePagingTest {
+
+ private static final int MAX_REQUESTS = 10;
+
+ private CamelContext context;
+ private DbxClientV2 client;
+ private DbxUserFilesRequests files;
+ private SearchV2Builder searchBuilder;
+ private final List<String> cursors = new ArrayList<>();
+
+ @BeforeEach
+ void setUp() throws Exception {
+ context = new DefaultCamelContext();
+ client = mock(DbxClientV2.class);
+ files = mock(DbxUserFilesRequests.class);
+ searchBuilder = mock(SearchV2Builder.class);
+ when(client.files()).thenReturn(files);
+ when(files.searchV2Builder("report")).thenReturn(searchBuilder);
+
when(searchBuilder.withOptions(any(SearchOptions.class))).thenReturn(searchBuilder);
+ when(files.download(anyString())).thenAnswer(invocation -> {
+ String path = invocation.getArgument(0);
+ return new DbxDownloader<>(file(path), new
ByteArrayInputStream(path.getBytes(StandardCharsets.UTF_8)));
+ });
+ }
+
+ @AfterEach
+ void tearDown() throws Exception {
+ context.close();
+ }
+
+ private static FileMetadata file(String path) {
+ String name = path.substring(path.lastIndexOf('/') + 1);
+ return FileMetadata.newBuilder(name, "id:" + name, new Date(0), new
Date(0), "0123456789abcdef", 1)
+ .withPathDisplay(path).withPathLower(path).build();
+ }
+
+ private static SearchV2Result searchPage(boolean hasMore, String cursor,
String... paths) {
+ List<SearchMatchV2> matches = new ArrayList<>();
+ for (String path : paths) {
+ matches.add(new SearchMatchV2(MetadataV2.metadata(file(path))));
+ }
+ return new SearchV2Result(matches, hasMore, cursor);
+ }
+
+ private static ListFolderResult folderPage(boolean hasMore, String cursor,
String... paths) {
+ List<Metadata> entries = new ArrayList<>();
+ for (String path : paths) {
+ entries.add(file(path));
+ }
+ return new ListFolderResult(entries, cursor, hasMore);
+ }
+
+ /** Records the cursor of a continue request and fails the test when the
facade does not stop asking. */
+ private void request(String cursor) {
+ cursors.add(cursor);
+ if (cursors.size() > MAX_REQUESTS) {
+ throw new IllegalStateException("The facade sent more than " +
MAX_REQUESTS + " continue requests");
+ }
+ }
+
+ private static List<String> paths(DropboxSearchResult result) {
+ List<String> answer = new ArrayList<>();
+ for (SearchMatchV2 match : result.getFound()) {
+
answer.add(match.getMetadata().getMetadataValue().getPathDisplay());
+ }
+ return answer;
+ }
+
+ @Test
+ void searchReturnsTheMatchesOfEveryPage() throws Exception {
+ when(searchBuilder.start()).thenReturn(searchPage(true, "cursor-1",
"/docs/report-1.txt", "/docs/report-2.txt"));
+ when(files.searchContinueV2(anyString())).thenAnswer(invocation -> {
+ String cursor = invocation.getArgument(0);
+ request(cursor);
+ return "cursor-1".equals(cursor)
+ ? searchPage(true, "cursor-2", "/docs/report-3.txt",
"/docs/report-4.txt")
+ : searchPage(false, null, "/docs/report-5.txt");
+ });
+
+ DropboxSearchResult result = new DropboxAPIFacade(client, new
DefaultExchange(context)).search("/docs", "report");
+
+ assertEquals(List.of("/docs/report-1.txt", "/docs/report-2.txt",
"/docs/report-3.txt", "/docs/report-4.txt",
+ "/docs/report-5.txt"), paths(result), "the matches of the
three pages, each once and in order");
+ assertEquals(List.of("cursor-1", "cursor-2"), cursors, "each page is
requested with the cursor of the page before");
+ }
+
+ @Test
+ void getDownloadsTheFilesOfEveryPageOfTheFolder() throws Exception {
+ when(files.listFolder("/docs")).thenReturn(folderPage(true,
"cursor-1", "/docs/a.txt", "/docs/b.txt"));
+ when(files.listFolderContinue(anyString())).thenAnswer(invocation -> {
+ String cursor = invocation.getArgument(0);
+ request(cursor);
+ return "cursor-1".equals(cursor)
+ ? folderPage(true, "cursor-2", "/docs/c.txt")
+ : folderPage(false, "cursor-3", "/docs/d.txt");
+ });
+
+ DropboxFileDownloadResult result = new DropboxAPIFacade(client, new
DefaultExchange(context)).get("/docs");
+
+ assertEquals(List.of("/docs/a.txt", "/docs/b.txt", "/docs/c.txt",
"/docs/d.txt"),
+ new ArrayList<>(result.getEntries().keySet()), "the files of
the three pages, each once and in order");
+ assertEquals(List.of("cursor-1", "cursor-2"), cursors, "each page is
requested with the cursor of the page before");
+ }
+
+ @Test
+ void resultThatFitsInOnePageNeedsNoFurtherRequest() throws Exception {
+ // the service returns a cursor with the last page of a folder as
well; only has_more says that there is more
+ when(searchBuilder.start()).thenReturn(searchPage(false, null,
"/docs/report-1.txt"));
+ when(files.listFolder("/docs")).thenReturn(folderPage(false,
"cursor-1", "/docs/a.txt"));
+ DropboxAPIFacade facade = new DropboxAPIFacade(client, new
DefaultExchange(context));
+
+ assertEquals(List.of("/docs/report-1.txt"),
paths(facade.search("/docs", "report")));
+ assertEquals(List.of("/docs/a.txt"), new
ArrayList<>(facade.get("/docs").getEntries().keySet()));
+
+ verify(files, never()).searchContinueV2(anyString());
+ verify(files, never()).listFolderContinue(anyString());
+ }
+}
