This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 7600ba74fa Core: Add pagination when listing namespaces/tables/views
(#9782)
7600ba74fa is described below
commit 7600ba74faef15b4d5593e9ada255e6dc999d0ed
Author: Rahil C <[email protected]>
AuthorDate: Thu May 2 23:35:38 2024 -0700
Core: Add pagination when listing namespaces/tables/views (#9782)
---
.../org/apache/iceberg/rest/CatalogHandlers.java | 58 +++++++++
.../apache/iceberg/rest/RESTSessionCatalog.java | 99 ++++++++++----
.../rest/responses/ListNamespacesResponse.java | 21 ++-
.../iceberg/rest/responses/ListTablesResponse.java | 21 ++-
.../apache/iceberg/rest/RESTCatalogAdapter.java | 32 ++++-
.../org/apache/iceberg/rest/TestRESTCatalog.java | 145 +++++++++++++++++++++
.../apache/iceberg/rest/TestRESTViewCatalog.java | 85 ++++++++++++
.../rest/responses/TestListNamespacesResponse.java | 29 ++++-
.../rest/responses/TestListTablesResponse.java | 29 ++++-
9 files changed, 476 insertions(+), 43 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java
b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java
index e4e3c065fb..746da5ffca 100644
--- a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java
+++ b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java
@@ -80,6 +80,7 @@ import org.apache.iceberg.view.ViewRepresentation;
public class CatalogHandlers {
private static final Schema EMPTY_SCHEMA = new Schema();
+ private static final String INTIAL_PAGE_TOKEN = "";
private CatalogHandlers() {}
@@ -117,6 +118,29 @@ public class CatalogHandlers {
return ListNamespacesResponse.builder().addAll(results).build();
}
+ public static ListNamespacesResponse listNamespaces(
+ SupportsNamespaces catalog, Namespace parent, String pageToken, String
pageSize) {
+ List<Namespace> results;
+ List<Namespace> subResults;
+
+ if (parent.isEmpty()) {
+ results = catalog.listNamespaces();
+ } else {
+ results = catalog.listNamespaces(parent);
+ }
+
+ int start = INTIAL_PAGE_TOKEN.equals(pageToken) ? 0 :
Integer.parseInt(pageToken);
+ int end = start + Integer.parseInt(pageSize);
+ subResults = results.subList(start, end);
+ String nextToken = String.valueOf(end);
+
+ if (end >= results.size()) {
+ nextToken = null;
+ }
+
+ return
ListNamespacesResponse.builder().addAll(subResults).nextPageToken(nextToken).build();
+ }
+
public static CreateNamespaceResponse createNamespace(
SupportsNamespaces catalog, CreateNamespaceRequest request) {
Namespace namespace = request.namespace();
@@ -174,6 +198,23 @@ public class CatalogHandlers {
return ListTablesResponse.builder().addAll(idents).build();
}
+ public static ListTablesResponse listTables(
+ Catalog catalog, Namespace namespace, String pageToken, String pageSize)
{
+ List<TableIdentifier> results = catalog.listTables(namespace);
+ List<TableIdentifier> subResults;
+
+ int start = INTIAL_PAGE_TOKEN.equals(pageToken) ? 0 :
Integer.parseInt(pageToken);
+ int end = start + Integer.parseInt(pageSize);
+ subResults = results.subList(start, end);
+ String nextToken = String.valueOf(end);
+
+ if (end >= results.size()) {
+ nextToken = null;
+ }
+
+ return
ListTablesResponse.builder().addAll(subResults).nextPageToken(nextToken).build();
+ }
+
public static LoadTableResponse stageTableCreate(
Catalog catalog, Namespace namespace, CreateTableRequest request) {
request.validate();
@@ -397,6 +438,23 @@ public class CatalogHandlers {
return
ListTablesResponse.builder().addAll(catalog.listViews(namespace)).build();
}
+ public static ListTablesResponse listViews(
+ ViewCatalog catalog, Namespace namespace, String pageToken, String
pageSize) {
+ List<TableIdentifier> results = catalog.listViews(namespace);
+ List<TableIdentifier> subResults;
+
+ int start = INTIAL_PAGE_TOKEN.equals(pageToken) ? 0 :
Integer.parseInt(pageToken);
+ int end = start + Integer.parseInt(pageSize);
+ subResults = results.subList(start, end);
+ String nextToken = String.valueOf(end);
+
+ if (end >= results.size()) {
+ nextToken = null;
+ }
+
+ return
ListTablesResponse.builder().addAll(subResults).nextPageToken(nextToken).build();
+ }
+
public static LoadViewResponse createView(
ViewCatalog catalog, Namespace namespace, CreateViewRequest request) {
request.validate();
diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java
b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java
index 94f3057f9f..dcf92289df 100644
--- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java
+++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java
@@ -114,6 +114,7 @@ public class RESTSessionCatalog extends
BaseViewSessionCatalog
private static final String DEFAULT_FILE_IO_IMPL =
"org.apache.iceberg.io.ResolvingFileIO";
private static final String REST_METRICS_REPORTING_ENABLED =
"rest-metrics-reporting-enabled";
private static final String REST_SNAPSHOT_LOADING_MODE =
"snapshot-loading-mode";
+ public static final String REST_PAGE_SIZE = "rest-page-size";
private static final List<String> TOKEN_PREFERENCE_ORDER =
ImmutableList.of(
OAuth2Properties.ID_TOKEN_TYPE,
@@ -136,6 +137,7 @@ public class RESTSessionCatalog extends
BaseViewSessionCatalog
private FileIO io = null;
private MetricsReporter reporter = null;
private boolean reportingViaRestEnabled;
+ private Integer pageSize = null;
private CloseableGroup closeables = null;
// a lazy thread pool for token refresh
@@ -228,6 +230,12 @@ public class RESTSessionCatalog extends
BaseViewSessionCatalog
client, tokenRefreshExecutor(name), token,
expiresAtMillis(mergedProps), catalogAuth);
}
+ this.pageSize = PropertyUtil.propertyAsNullableInt(mergedProps,
REST_PAGE_SIZE);
+ if (pageSize != null) {
+ Preconditions.checkArgument(
+ pageSize > 0, "Invalid value for %s, must be a positive integer",
REST_PAGE_SIZE);
+ }
+
this.io = newFileIO(SessionContext.createEmpty(), mergedProps);
this.fileIOCloser = newFileIOCloser();
@@ -278,14 +286,27 @@ public class RESTSessionCatalog extends
BaseViewSessionCatalog
@Override
public List<TableIdentifier> listTables(SessionContext context, Namespace
ns) {
checkNamespaceIsValid(ns);
+ Map<String, String> queryParams = Maps.newHashMap();
+ ImmutableList.Builder<TableIdentifier> tables = ImmutableList.builder();
+ String pageToken = "";
+ if (pageSize != null) {
+ queryParams.put("pageSize", String.valueOf(pageSize));
+ }
- ListTablesResponse response =
- client.get(
- paths.tables(ns),
- ListTablesResponse.class,
- headers(context),
- ErrorHandlers.namespaceErrorHandler());
- return response.identifiers();
+ do {
+ queryParams.put("pageToken", pageToken);
+ ListTablesResponse response =
+ client.get(
+ paths.tables(ns),
+ queryParams,
+ ListTablesResponse.class,
+ headers(context),
+ ErrorHandlers.namespaceErrorHandler());
+ pageToken = response.nextPageToken();
+ tables.addAll(response.identifiers());
+ } while (pageToken != null);
+
+ return tables.build();
}
@Override
@@ -494,22 +515,31 @@ public class RESTSessionCatalog extends
BaseViewSessionCatalog
@Override
public List<Namespace> listNamespaces(SessionContext context, Namespace
namespace) {
- Map<String, String> queryParams;
- if (namespace.isEmpty()) {
- queryParams = ImmutableMap.of();
- } else {
- // query params should be unescaped
- queryParams = ImmutableMap.of("parent",
RESTUtil.NAMESPACE_JOINER.join(namespace.levels()));
+ Map<String, String> queryParams = Maps.newHashMap();
+ if (!namespace.isEmpty()) {
+ queryParams.put("parent",
RESTUtil.NAMESPACE_JOINER.join(namespace.levels()));
}
- ListNamespacesResponse response =
- client.get(
- paths.namespaces(),
- queryParams,
- ListNamespacesResponse.class,
- headers(context),
- ErrorHandlers.namespaceErrorHandler());
- return response.namespaces();
+ ImmutableList.Builder<Namespace> namespaces = ImmutableList.builder();
+ String pageToken = "";
+ if (pageSize != null) {
+ queryParams.put("pageSize", String.valueOf(pageSize));
+ }
+
+ do {
+ queryParams.put("pageToken", pageToken);
+ ListNamespacesResponse response =
+ client.get(
+ paths.namespaces(),
+ queryParams,
+ ListNamespacesResponse.class,
+ headers(context),
+ ErrorHandlers.namespaceErrorHandler());
+ pageToken = response.nextPageToken();
+ namespaces.addAll(response.namespaces());
+ } while (pageToken != null);
+
+ return namespaces.build();
}
@Override
@@ -1048,14 +1078,27 @@ public class RESTSessionCatalog extends
BaseViewSessionCatalog
@Override
public List<TableIdentifier> listViews(SessionContext context, Namespace
namespace) {
checkNamespaceIsValid(namespace);
+ Map<String, String> queryParams = Maps.newHashMap();
+ ImmutableList.Builder<TableIdentifier> views = ImmutableList.builder();
+ String pageToken = "";
+ if (pageSize != null) {
+ queryParams.put("pageSize", String.valueOf(pageSize));
+ }
- ListTablesResponse response =
- client.get(
- paths.views(namespace),
- ListTablesResponse.class,
- headers(context),
- ErrorHandlers.namespaceErrorHandler());
- return response.identifiers();
+ do {
+ queryParams.put("pageToken", pageToken);
+ ListTablesResponse response =
+ client.get(
+ paths.views(namespace),
+ queryParams,
+ ListTablesResponse.class,
+ headers(context),
+ ErrorHandlers.namespaceErrorHandler());
+ pageToken = response.nextPageToken();
+ views.addAll(response.identifiers());
+ } while (pageToken != null);
+
+ return views.build();
}
@Override
diff --git
a/core/src/main/java/org/apache/iceberg/rest/responses/ListNamespacesResponse.java
b/core/src/main/java/org/apache/iceberg/rest/responses/ListNamespacesResponse.java
index 13a599e1a7..8feeda6f2b 100644
---
a/core/src/main/java/org/apache/iceberg/rest/responses/ListNamespacesResponse.java
+++
b/core/src/main/java/org/apache/iceberg/rest/responses/ListNamespacesResponse.java
@@ -29,13 +29,15 @@ import org.apache.iceberg.rest.RESTResponse;
public class ListNamespacesResponse implements RESTResponse {
private List<Namespace> namespaces;
+ private String nextPageToken;
public ListNamespacesResponse() {
// Required for Jackson deserialization
}
- private ListNamespacesResponse(List<Namespace> namespaces) {
+ private ListNamespacesResponse(List<Namespace> namespaces, String
nextPageToken) {
this.namespaces = namespaces;
+ this.nextPageToken = nextPageToken;
validate();
}
@@ -48,9 +50,16 @@ public class ListNamespacesResponse implements RESTResponse {
return namespaces != null ? namespaces : ImmutableList.of();
}
+ public String nextPageToken() {
+ return nextPageToken;
+ }
+
@Override
public String toString() {
- return MoreObjects.toStringHelper(this).add("namespaces",
namespaces()).toString();
+ return MoreObjects.toStringHelper(this)
+ .add("namespaces", namespaces())
+ .add("next-page-token", nextPageToken())
+ .toString();
}
public static Builder builder() {
@@ -59,6 +68,7 @@ public class ListNamespacesResponse implements RESTResponse {
public static class Builder {
private final ImmutableList.Builder<Namespace> namespaces =
ImmutableList.builder();
+ private String nextPageToken;
private Builder() {}
@@ -75,8 +85,13 @@ public class ListNamespacesResponse implements RESTResponse {
return this;
}
+ public Builder nextPageToken(String pageToken) {
+ nextPageToken = pageToken;
+ return this;
+ }
+
public ListNamespacesResponse build() {
- return new ListNamespacesResponse(namespaces.build());
+ return new ListNamespacesResponse(namespaces.build(), nextPageToken);
}
}
}
diff --git
a/core/src/main/java/org/apache/iceberg/rest/responses/ListTablesResponse.java
b/core/src/main/java/org/apache/iceberg/rest/responses/ListTablesResponse.java
index 3c99c12c90..1db05709b4 100644
---
a/core/src/main/java/org/apache/iceberg/rest/responses/ListTablesResponse.java
+++
b/core/src/main/java/org/apache/iceberg/rest/responses/ListTablesResponse.java
@@ -30,13 +30,15 @@ import org.apache.iceberg.rest.RESTResponse;
public class ListTablesResponse implements RESTResponse {
private List<TableIdentifier> identifiers;
+ private String nextPageToken;
public ListTablesResponse() {
// Required for Jackson deserialization
}
- private ListTablesResponse(List<TableIdentifier> identifiers) {
+ private ListTablesResponse(List<TableIdentifier> identifiers, String
nextPageToken) {
this.identifiers = identifiers;
+ this.nextPageToken = nextPageToken;
validate();
}
@@ -49,9 +51,16 @@ public class ListTablesResponse implements RESTResponse {
return identifiers != null ? identifiers : ImmutableList.of();
}
+ public String nextPageToken() {
+ return nextPageToken;
+ }
+
@Override
public String toString() {
- return MoreObjects.toStringHelper(this).add("identifiers",
identifiers).toString();
+ return MoreObjects.toStringHelper(this)
+ .add("identifiers", identifiers)
+ .add("next-page-token", nextPageToken())
+ .toString();
}
public static Builder builder() {
@@ -60,6 +69,7 @@ public class ListTablesResponse implements RESTResponse {
public static class Builder {
private final ImmutableList.Builder<TableIdentifier> identifiers =
ImmutableList.builder();
+ private String nextPageToken;
private Builder() {}
@@ -76,8 +86,13 @@ public class ListTablesResponse implements RESTResponse {
return this;
}
+ public Builder nextPageToken(String pageToken) {
+ nextPageToken = pageToken;
+ return this;
+ }
+
public ListTablesResponse build() {
- return new ListTablesResponse(identifiers.build());
+ return new ListTablesResponse(identifiers.build(), nextPageToken);
}
}
}
diff --git a/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java
b/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java
index 7fccc4e974..357b05e85c 100644
--- a/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java
+++ b/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java
@@ -298,7 +298,17 @@ public class RESTCatalogAdapter implements RESTClient {
ns = Namespace.empty();
}
- return castResponse(responseType,
CatalogHandlers.listNamespaces(asNamespaceCatalog, ns));
+ String pageToken = PropertyUtil.propertyAsString(vars, "pageToken",
null);
+ String pageSize = PropertyUtil.propertyAsString(vars, "pageSize",
null);
+
+ if (pageSize != null) {
+ return castResponse(
+ responseType,
+ CatalogHandlers.listNamespaces(asNamespaceCatalog, ns,
pageToken, pageSize));
+ } else {
+ return castResponse(
+ responseType,
CatalogHandlers.listNamespaces(asNamespaceCatalog, ns));
+ }
}
break;
@@ -339,7 +349,14 @@ public class RESTCatalogAdapter implements RESTClient {
case LIST_TABLES:
{
Namespace namespace = namespaceFromPathVars(vars);
- return castResponse(responseType,
CatalogHandlers.listTables(catalog, namespace));
+ String pageToken = PropertyUtil.propertyAsString(vars, "pageToken",
null);
+ String pageSize = PropertyUtil.propertyAsString(vars, "pageSize",
null);
+ if (pageSize != null) {
+ return castResponse(
+ responseType, CatalogHandlers.listTables(catalog, namespace,
pageToken, pageSize));
+ } else {
+ return castResponse(responseType,
CatalogHandlers.listTables(catalog, namespace));
+ }
}
case CREATE_TABLE:
@@ -412,7 +429,16 @@ public class RESTCatalogAdapter implements RESTClient {
{
if (null != asViewCatalog) {
Namespace namespace = namespaceFromPathVars(vars);
- return castResponse(responseType,
CatalogHandlers.listViews(asViewCatalog, namespace));
+ String pageToken = PropertyUtil.propertyAsString(vars,
"pageToken", null);
+ String pageSize = PropertyUtil.propertyAsString(vars, "pageSize",
null);
+ if (pageSize != null) {
+ return castResponse(
+ responseType,
+ CatalogHandlers.listViews(asViewCatalog, namespace,
pageToken, pageSize));
+ } else {
+ return castResponse(
+ responseType, CatalogHandlers.listViews(asViewCatalog,
namespace));
+ }
}
break;
}
diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java
b/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java
index 18d832b3cd..95380424e7 100644
--- a/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java
+++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java
@@ -77,7 +77,10 @@ import org.apache.iceberg.rest.auth.OAuth2Properties;
import org.apache.iceberg.rest.auth.OAuth2Util;
import org.apache.iceberg.rest.requests.UpdateTableRequest;
import org.apache.iceberg.rest.responses.ConfigResponse;
+import org.apache.iceberg.rest.responses.CreateNamespaceResponse;
import org.apache.iceberg.rest.responses.ErrorResponse;
+import org.apache.iceberg.rest.responses.ListNamespacesResponse;
+import org.apache.iceberg.rest.responses.ListTablesResponse;
import org.apache.iceberg.rest.responses.LoadTableResponse;
import org.apache.iceberg.rest.responses.OAuthTokenResponse;
import org.apache.iceberg.types.Types;
@@ -2329,6 +2332,148 @@ public class TestRESTCatalog extends
CatalogTests<RESTCatalog> {
assertThat(schema2.columns()).hasSize(1);
}
+ @Test
+ public void testInvalidPageSize() {
+ RESTCatalogAdapter adapter = Mockito.spy(new
RESTCatalogAdapter(backendCatalog));
+ RESTCatalog catalog =
+ new RESTCatalog(SessionCatalog.SessionContext.createEmpty(), (config)
-> adapter);
+ Assertions.assertThatThrownBy(
+ () ->
+ catalog.initialize(
+ "test", ImmutableMap.of(RESTSessionCatalog.REST_PAGE_SIZE,
"-1")))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ String.format(
+ "Invalid value for %s, must be a positive integer",
+ RESTSessionCatalog.REST_PAGE_SIZE));
+ }
+
+ @Test
+ public void testPaginationForListNamespaces() {
+ RESTCatalogAdapter adapter = Mockito.spy(new
RESTCatalogAdapter(backendCatalog));
+ RESTCatalog catalog =
+ new RESTCatalog(SessionCatalog.SessionContext.createEmpty(), (config)
-> adapter);
+ catalog.initialize("test",
ImmutableMap.of(RESTSessionCatalog.REST_PAGE_SIZE, "10"));
+ int numberOfItems = 30;
+ String namespaceName = "newdb";
+
+ // create several namespaces for listing and verify
+ for (int i = 0; i < numberOfItems; i++) {
+ String nameSpaceName = namespaceName + i;
+ catalog.createNamespace(Namespace.of(nameSpaceName));
+ }
+
+ assertThat(catalog.listNamespaces()).hasSize(numberOfItems);
+
+ Mockito.verify(adapter)
+ .execute(
+ eq(HTTPMethod.GET),
+ eq("v1/config"),
+ any(),
+ any(),
+ eq(ConfigResponse.class),
+ any(),
+ any());
+
+ Mockito.verify(adapter, times(numberOfItems))
+ .execute(
+ eq(HTTPMethod.POST),
+ eq("v1/namespaces"),
+ any(),
+ any(),
+ eq(CreateNamespaceResponse.class),
+ any(),
+ any());
+
+ // verify initial request with empty pageToken
+ Mockito.verify(adapter)
+ .handleRequest(
+ eq(RESTCatalogAdapter.Route.LIST_NAMESPACES),
+ eq(ImmutableMap.of("pageToken", "", "pageSize", "10")),
+ any(),
+ eq(ListNamespacesResponse.class));
+
+ // verify second request with updated pageToken
+ Mockito.verify(adapter)
+ .handleRequest(
+ eq(RESTCatalogAdapter.Route.LIST_NAMESPACES),
+ eq(ImmutableMap.of("pageToken", "10", "pageSize", "10")),
+ any(),
+ eq(ListNamespacesResponse.class));
+
+ // verify third request with update pageToken
+ Mockito.verify(adapter)
+ .handleRequest(
+ eq(RESTCatalogAdapter.Route.LIST_NAMESPACES),
+ eq(ImmutableMap.of("pageToken", "20", "pageSize", "10")),
+ any(),
+ eq(ListNamespacesResponse.class));
+ }
+
+ @Test
+ public void testPaginationForListTables() {
+ RESTCatalogAdapter adapter = Mockito.spy(new
RESTCatalogAdapter(backendCatalog));
+ RESTCatalog catalog =
+ new RESTCatalog(SessionCatalog.SessionContext.createEmpty(), (config)
-> adapter);
+ catalog.initialize("test",
ImmutableMap.of(RESTSessionCatalog.REST_PAGE_SIZE, "10"));
+ int numberOfItems = 30;
+ String namespaceName = "newdb";
+ String tableName = "newtable";
+ catalog.createNamespace(Namespace.of(namespaceName));
+
+ // create several tables under namespace for listing and verify
+ for (int i = 0; i < numberOfItems; i++) {
+ TableIdentifier tableIdentifier = TableIdentifier.of(namespaceName,
tableName + i);
+ catalog.createTable(tableIdentifier, SCHEMA);
+ }
+
+
assertThat(catalog.listTables(Namespace.of(namespaceName))).hasSize(numberOfItems);
+
+ Mockito.verify(adapter)
+ .execute(
+ eq(HTTPMethod.GET),
+ eq("v1/config"),
+ any(),
+ any(),
+ eq(ConfigResponse.class),
+ any(),
+ any());
+
+ Mockito.verify(adapter, times(numberOfItems))
+ .execute(
+ eq(HTTPMethod.POST),
+ eq(String.format("v1/namespaces/%s/tables", namespaceName)),
+ any(),
+ any(),
+ eq(LoadTableResponse.class),
+ any(),
+ any());
+
+ // verify initial request with empty pageToken
+ Mockito.verify(adapter)
+ .handleRequest(
+ eq(RESTCatalogAdapter.Route.LIST_TABLES),
+ eq(ImmutableMap.of("pageToken", "", "pageSize", "10", "namespace",
namespaceName)),
+ any(),
+ eq(ListTablesResponse.class));
+
+ // verify second request with updated pageToken
+ Mockito.verify(adapter)
+ .handleRequest(
+ eq(RESTCatalogAdapter.Route.LIST_TABLES),
+ eq(ImmutableMap.of("pageToken", "10", "pageSize", "10",
"namespace", namespaceName)),
+ any(),
+ eq(ListTablesResponse.class));
+
+ // verify third request with update pageToken
+ Mockito.verify(adapter)
+ .handleRequest(
+ eq(RESTCatalogAdapter.Route.LIST_TABLES),
+ eq(ImmutableMap.of("pageToken", "20", "pageSize", "10",
"namespace", namespaceName)),
+ any(),
+ eq(ListTablesResponse.class));
+ }
+
@Test
public void testCleanupUncommitedFilesForCleanableFailures() {
RESTCatalogAdapter adapter = Mockito.spy(new
RESTCatalogAdapter(backendCatalog));
diff --git
a/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java
b/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java
index 0b29da7042..f67c4b078e 100644
--- a/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java
+++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java
@@ -18,20 +18,31 @@
*/
package org.apache.iceberg.rest;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.times;
+
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.File;
import java.nio.file.Path;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.function.Consumer;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.SessionCatalog;
+import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.inmemory.InMemoryCatalog;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.rest.RESTCatalogAdapter.HTTPMethod;
+import org.apache.iceberg.rest.responses.ConfigResponse;
import org.apache.iceberg.rest.responses.ErrorResponse;
+import org.apache.iceberg.rest.responses.ListTablesResponse;
+import org.apache.iceberg.rest.responses.LoadViewResponse;
import org.apache.iceberg.view.ViewCatalogTests;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.handler.gzip.GzipHandler;
@@ -39,7 +50,9 @@ import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.mockito.Mockito;
public class TestRESTViewCatalog extends ViewCatalogTests<RESTCatalog> {
private static final ObjectMapper MAPPER = RESTObjectMapper.mapper();
@@ -144,6 +157,78 @@ public class TestRESTViewCatalog extends
ViewCatalogTests<RESTCatalog> {
}
}
+ @Test
+ public void testPaginationForListViews() {
+ RESTCatalogAdapter adapter = Mockito.spy(new
RESTCatalogAdapter(backendCatalog));
+ RESTCatalog catalog =
+ new RESTCatalog(SessionCatalog.SessionContext.createEmpty(), (config)
-> adapter);
+ catalog.initialize("test",
ImmutableMap.of(RESTSessionCatalog.REST_PAGE_SIZE, "10"));
+
+ int numberOfItems = 30;
+ String namespaceName = "newdb";
+ String viewName = "newview";
+
+ // create initial namespace
+ catalog().createNamespace(Namespace.of(namespaceName));
+
+ // create several views under namespace, based off a table for listing and
verify
+ for (int i = 0; i < numberOfItems; i++) {
+ TableIdentifier viewIndentifier = TableIdentifier.of(namespaceName,
viewName + i);
+ catalog
+ .buildView(viewIndentifier)
+ .withSchema(SCHEMA)
+ .withDefaultNamespace(viewIndentifier.namespace())
+ .withQuery("spark", "select * from ns.tbl")
+ .create();
+ }
+ List<TableIdentifier> views =
catalog.listViews(Namespace.of(namespaceName));
+ assertThat(views).hasSize(numberOfItems);
+
+ Mockito.verify(adapter)
+ .execute(
+ eq(HTTPMethod.GET),
+ eq("v1/config"),
+ any(),
+ any(),
+ eq(ConfigResponse.class),
+ any(),
+ any());
+
+ Mockito.verify(adapter, times(numberOfItems))
+ .execute(
+ eq(HTTPMethod.POST),
+ eq(String.format("v1/namespaces/%s/views", namespaceName)),
+ any(),
+ any(),
+ eq(LoadViewResponse.class),
+ any(),
+ any());
+
+ // verify initial request with empty pageToken
+ Mockito.verify(adapter)
+ .handleRequest(
+ eq(RESTCatalogAdapter.Route.LIST_VIEWS),
+ eq(ImmutableMap.of("pageToken", "", "pageSize", "10", "namespace",
namespaceName)),
+ any(),
+ eq(ListTablesResponse.class));
+
+ // verify second request with update pageToken
+ Mockito.verify(adapter)
+ .handleRequest(
+ eq(RESTCatalogAdapter.Route.LIST_VIEWS),
+ eq(ImmutableMap.of("pageToken", "10", "pageSize", "10",
"namespace", namespaceName)),
+ any(),
+ eq(ListTablesResponse.class));
+
+ // verify third request with update pageToken
+ Mockito.verify(adapter)
+ .handleRequest(
+ eq(RESTCatalogAdapter.Route.LIST_VIEWS),
+ eq(ImmutableMap.of("pageToken", "20", "pageSize", "10",
"namespace", namespaceName)),
+ any(),
+ eq(ListTablesResponse.class));
+ }
+
@Override
protected RESTCatalog catalog() {
return restCatalog;
diff --git
a/core/src/test/java/org/apache/iceberg/rest/responses/TestListNamespacesResponse.java
b/core/src/test/java/org/apache/iceberg/rest/responses/TestListNamespacesResponse.java
index bfe5a662b2..d9ed801de0 100644
---
a/core/src/test/java/org/apache/iceberg/rest/responses/TestListNamespacesResponse.java
+++
b/core/src/test/java/org/apache/iceberg/rest/responses/TestListNamespacesResponse.java
@@ -34,11 +34,11 @@ public class TestListNamespacesResponse extends
RequestResponseTestBase<ListName
@Test
public void testRoundTripSerDe() throws JsonProcessingException {
- String fullJson = "{\"namespaces\":[[\"accounting\"],[\"tax\"]]}";
+ String fullJson =
"{\"namespaces\":[[\"accounting\"],[\"tax\"]],\"next-page-token\":null}";
ListNamespacesResponse fullValue =
ListNamespacesResponse.builder().addAll(NAMESPACES).build();
assertRoundTripSerializesEquallyFrom(fullJson, fullValue);
- String emptyNamespaces = "{\"namespaces\":[]}";
+ String emptyNamespaces = "{\"namespaces\":[],\"next-page-token\":null}";
assertRoundTripSerializesEquallyFrom(emptyNamespaces,
ListNamespacesResponse.builder().build());
}
@@ -83,9 +83,32 @@ public class TestListNamespacesResponse extends
RequestResponseTestBase<ListName
.hasMessage("Invalid namespace: null");
}
+ @Test
+ public void testWithNullPaginationToken() throws JsonProcessingException {
+ String jsonWithNullPageToken =
+
"{\"namespaces\":[[\"accounting\"],[\"tax\"]],\"next-page-token\":null}";
+ ListNamespacesResponse response =
+
ListNamespacesResponse.builder().addAll(NAMESPACES).nextPageToken(null).build();
+ assertRoundTripSerializesEquallyFrom(jsonWithNullPageToken, response);
+ Assertions.assertThat(response.nextPageToken()).isNull();
+ Assertions.assertThat(response.namespaces()).isEqualTo(NAMESPACES);
+ }
+
+ @Test
+ public void testWithPaginationToken() throws JsonProcessingException {
+ String pageToken = "token";
+ String jsonWithPageToken =
+
"{\"namespaces\":[[\"accounting\"],[\"tax\"]],\"next-page-token\":\"token\"}";
+ ListNamespacesResponse response =
+
ListNamespacesResponse.builder().addAll(NAMESPACES).nextPageToken(pageToken).build();
+ assertRoundTripSerializesEquallyFrom(jsonWithPageToken, response);
+ Assertions.assertThat(response.nextPageToken()).isEqualTo("token");
+ Assertions.assertThat(response.namespaces()).isEqualTo(NAMESPACES);
+ }
+
@Override
public String[] allFieldsFromSpec() {
- return new String[] {"namespaces"};
+ return new String[] {"namespaces", "next-page-token"};
}
@Override
diff --git
a/core/src/test/java/org/apache/iceberg/rest/responses/TestListTablesResponse.java
b/core/src/test/java/org/apache/iceberg/rest/responses/TestListTablesResponse.java
index 116d43a6d1..d46228f188 100644
---
a/core/src/test/java/org/apache/iceberg/rest/responses/TestListTablesResponse.java
+++
b/core/src/test/java/org/apache/iceberg/rest/responses/TestListTablesResponse.java
@@ -36,11 +36,11 @@ public class TestListTablesResponse extends
RequestResponseTestBase<ListTablesRe
@Test
public void testRoundTripSerDe() throws JsonProcessingException {
String fullJson =
-
"{\"identifiers\":[{\"namespace\":[\"accounting\",\"tax\"],\"name\":\"paid\"}]}";
+
"{\"identifiers\":[{\"namespace\":[\"accounting\",\"tax\"],\"name\":\"paid\"}],\"next-page-token\":null}";
assertRoundTripSerializesEquallyFrom(
fullJson, ListTablesResponse.builder().addAll(IDENTIFIERS).build());
- String emptyIdentifiers = "{\"identifiers\":[]}";
+ String emptyIdentifiers = "{\"identifiers\":[],\"next-page-token\":null}";
assertRoundTripSerializesEquallyFrom(emptyIdentifiers,
ListTablesResponse.builder().build());
}
@@ -105,9 +105,32 @@ public class TestListTablesResponse extends
RequestResponseTestBase<ListTablesRe
.hasMessage("Invalid table identifier: null");
}
+ @Test
+ public void testWithNullPaginationToken() throws JsonProcessingException {
+ String jsonWithNullPageToken =
+
"{\"identifiers\":[{\"namespace\":[\"accounting\",\"tax\"],\"name\":\"paid\"}],\"next-page-token\":null}";
+ ListTablesResponse response =
+
ListTablesResponse.builder().addAll(IDENTIFIERS).nextPageToken(null).build();
+ assertRoundTripSerializesEquallyFrom(jsonWithNullPageToken, response);
+ Assertions.assertThat(response.nextPageToken()).isNull();
+ Assertions.assertThat(response.identifiers()).isEqualTo(IDENTIFIERS);
+ }
+
+ @Test
+ public void testWithPaginationToken() throws JsonProcessingException {
+ String pageToken = "token";
+ String jsonWithPageToken =
+
"{\"identifiers\":[{\"namespace\":[\"accounting\",\"tax\"],\"name\":\"paid\"}],\"next-page-token\":\"token\"}";
+ ListTablesResponse response =
+
ListTablesResponse.builder().addAll(IDENTIFIERS).nextPageToken(pageToken).build();
+ assertRoundTripSerializesEquallyFrom(jsonWithPageToken, response);
+ Assertions.assertThat(response.nextPageToken()).isEqualTo("token");
+ Assertions.assertThat(response.identifiers()).isEqualTo(IDENTIFIERS);
+ }
+
@Override
public String[] allFieldsFromSpec() {
- return new String[] {"identifiers"};
+ return new String[] {"identifiers", "next-page-token"};
}
@Override
