This is an automated email from the ASF dual-hosted git repository. lzljs3620320 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push: new ffd1f9c2b9 [rest] Add tableType parameter support for listing tables (#6295) ffd1f9c2b9 is described below commit ffd1f9c2b96324a7ab87b593ecf50760943ba022 Author: Jiajia Li <plusplusjia...@alibaba-inc.com> AuthorDate: Mon Sep 22 18:07:16 2025 +0800 [rest] Add tableType parameter support for listing tables (#6295) --- docs/static/rest-catalog-open-api.yaml | 5 + .../main/java/org/apache/paimon/rest/RESTApi.java | 15 +- .../org/apache/paimon/catalog/AbstractCatalog.java | 25 +- .../java/org/apache/paimon/catalog/Catalog.java | 12 +- .../org/apache/paimon/catalog/CatalogUtils.java | 9 + .../org/apache/paimon/catalog/DelegateCatalog.java | 23 +- .../java/org/apache/paimon/rest/RESTCatalog.java | 17 +- .../org/apache/paimon/catalog/CatalogTestBase.java | 31 +- .../apache/paimon/rest/MockRESTCatalogTest.java | 8 +- .../org/apache/paimon/rest/RESTCatalogServer.java | 37 +++ .../org/apache/paimon/rest/RESTCatalogTest.java | 324 +++++++++++++++++++-- 11 files changed, 440 insertions(+), 66 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 0b85c56fcb..9bfc7cbdfb 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -350,6 +350,11 @@ paths: in: query schema: type: string + - name: tableType + description: Filter tables by table type. All table types will be returned if not set or empty. + in: query + schema: + type: string responses: "200": description: OK diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 08ed9f93d2..0a9989bdd5 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -141,6 +141,7 @@ public class RESTApi { public static final String DATABASE_NAME_PATTERN = "databaseNamePattern"; public static final String TABLE_NAME_PATTERN = "tableNamePattern"; + public static final String TABLE_TYPE = "tableType"; public static final String VIEW_NAME_PATTERN = "viewNamePattern"; public static final String FUNCTION_NAME_PATTERN = "functionNamePattern"; public static final String PARTITION_NAME_PATTERN = "partitionNamePattern"; @@ -354,14 +355,16 @@ public class RESTApi { String databaseName, @Nullable Integer maxResults, @Nullable String pageToken, - @Nullable String tableNamePattern) { + @Nullable String tableNamePattern, + @Nullable String tableType) { ListTablesResponse response = client.get( resourcePaths.tables(databaseName), buildPagedQueryParams( maxResults, pageToken, - Pair.of(TABLE_NAME_PATTERN, tableNamePattern)), + Pair.of(TABLE_NAME_PATTERN, tableNamePattern), + Pair.of(TABLE_TYPE, tableType)), ListTablesResponse.class, restAuthFunction); List<String> tables = response.getTables(); @@ -385,6 +388,8 @@ public class RESTApi { * from a specific point. * @param tableNamePattern A sql LIKE pattern (%) for table names. All tables will be returned * if not set or empty. Currently, only prefix matching is supported. + * @param tableType Optional parameter to filter tables by table type. All table types will be + * returned if not set or empty. * @return {@link PagedList}: elements and nextPageToken. * @throws NoSuchResourceException Exception thrown on HTTP 404 means the database not exists * @throws ForbiddenException Exception thrown on HTTP 403 means don't have the permission for @@ -394,14 +399,16 @@ public class RESTApi { String databaseName, @Nullable Integer maxResults, @Nullable String pageToken, - @Nullable String tableNamePattern) { + @Nullable String tableNamePattern, + @Nullable String tableType) { ListTableDetailsResponse response = client.get( resourcePaths.tableDetails(databaseName), buildPagedQueryParams( maxResults, pageToken, - Pair.of(TABLE_NAME_PATTERN, tableNamePattern)), + Pair.of(TABLE_NAME_PATTERN, tableNamePattern), + Pair.of(TABLE_TYPE, tableType)), ListTableDetailsResponse.class, restAuthFunction); List<GetTableResponse> tables = response.getTableDetails(); diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java index 841643383c..5df78212f1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java @@ -250,17 +250,27 @@ public abstract class AbstractCatalog implements Catalog { @Override public PagedList<String> listTablesPaged( - String databaseName, Integer maxResults, String pageToken, String tableNamePattern) + String databaseName, + Integer maxResults, + String pageToken, + String tableNamePattern, + String tableType) throws DatabaseNotExistException { CatalogUtils.validateNamePattern(this, tableNamePattern); + CatalogUtils.validateTableType(this, tableType); return new PagedList<>(listTables(databaseName), null); } @Override public PagedList<Table> listTableDetailsPaged( - String databaseName, Integer maxResults, String pageToken, String tableNamePattern) + String databaseName, + @Nullable Integer maxResults, + @Nullable String pageToken, + @Nullable String tableNamePattern, + @Nullable String tableType) throws DatabaseNotExistException { CatalogUtils.validateNamePattern(this, tableNamePattern); + CatalogUtils.validateTableType(this, tableType); if (isSystemDatabase(databaseName)) { List<Table> systemTables = SystemTableLoader.loadGlobalTableNames().stream() @@ -285,16 +295,21 @@ public abstract class AbstractCatalog implements Catalog { // check db exists getDatabase(databaseName); - return listTableDetailsPagedImpl(databaseName, maxResults, pageToken); + return listTableDetailsPagedImpl( + databaseName, maxResults, pageToken, tableNamePattern, tableType); } protected abstract List<String> listTablesImpl(String databaseName); protected PagedList<Table> listTableDetailsPagedImpl( - String databaseName, Integer maxResults, String pageToken) + String databaseName, + @Nullable Integer maxResults, + @Nullable String pageToken, + @Nullable String tableNamePattern, + @Nullable String tableType) throws DatabaseNotExistException { PagedList<String> pagedTableNames = - listTablesPaged(databaseName, maxResults, pageToken, null); + listTablesPaged(databaseName, maxResults, pageToken, tableNamePattern, tableType); return new PagedList<>( pagedTableNames.getElements().stream() .map( diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index 702dc155a5..557d563995 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -186,7 +186,8 @@ public interface Catalog extends AutoCloseable { String databaseName, @Nullable Integer maxResults, @Nullable String pageToken, - @Nullable String tableNamePattern) + @Nullable String tableNamePattern, + @Nullable String tableType) throws DatabaseNotExistException; /** @@ -203,6 +204,8 @@ public interface Catalog extends AutoCloseable { * from a specific point. * @param tableNamePattern A sql LIKE pattern (%) for table names. All table details will be * returned if not set or empty. Currently, only prefix matching is supported. + * @param tableType Optional parameter to filter tables by table type. All table types will be + * returned if not set or empty. * @return a list of the table details with provided page size in this database and next page * token, or a list of the details of all tables in this database if the catalog does not * {@link #supportsListObjectsPaged()}. @@ -213,7 +216,8 @@ public interface Catalog extends AutoCloseable { String databaseName, @Nullable Integer maxResults, @Nullable String pageToken, - @Nullable String tableNamePattern) + @Nullable String tableNamePattern, + @Nullable String tableType) throws DatabaseNotExistException; /** @@ -606,6 +610,10 @@ public interface Catalog extends AutoCloseable { return false; } + default boolean supportsListTableByType() { + return false; + } + // ==================== Version management methods ========================== /** diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java index fe204bfc87..c36993ff0a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java @@ -163,6 +163,15 @@ public class CatalogUtils { } } + public static void validateTableType(Catalog catalog, String tableType) { + if (Objects.nonNull(tableType) && !catalog.supportsListTableByType()) { + throw new UnsupportedOperationException( + String.format( + "Current catalog %s does not support table type filter.", + catalog.getClass().getSimpleName())); + } + } + public static List<Partition> listPartitionsFromFileSystem(Table table) { Options options = Options.fromMap(table.options()); InternalRowPartitionComputer computer = diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index 08776714a8..a4e90e5049 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -102,16 +102,26 @@ public abstract class DelegateCatalog implements Catalog { @Override public PagedList<String> listTablesPaged( - String databaseName, Integer maxResults, String pageToken, String tableNamePattern) + String databaseName, + Integer maxResults, + String pageToken, + String tableNamePattern, + String tableType) throws DatabaseNotExistException { - return wrapped.listTablesPaged(databaseName, maxResults, pageToken, tableNamePattern); + return wrapped.listTablesPaged( + databaseName, maxResults, pageToken, tableNamePattern, tableType); } @Override public PagedList<Table> listTableDetailsPaged( - String databaseName, Integer maxResults, String pageToken, String tableNamePattern) + String databaseName, + Integer maxResults, + String pageToken, + String tableNamePattern, + String tableType) throws DatabaseNotExistException { - return wrapped.listTableDetailsPaged(databaseName, maxResults, pageToken, tableNamePattern); + return wrapped.listTableDetailsPaged( + databaseName, maxResults, pageToken, tableNamePattern, tableType); } @Override @@ -165,6 +175,11 @@ public abstract class DelegateCatalog implements Catalog { return wrapped.supportsListByPattern(); } + @Override + public boolean supportsListTableByType() { + return wrapped.supportsListTableByType(); + } + @Override public boolean supportsVersionManagement() { return wrapped.supportsVersionManagement(); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 6cdb23b5a5..9179e71aa9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -224,10 +224,12 @@ public class RESTCatalog implements Catalog { String databaseName, @Nullable Integer maxResults, @Nullable String pageToken, - @Nullable String tableNamePattern) + @Nullable String tableNamePattern, + @Nullable String tableType) throws DatabaseNotExistException { try { - return api.listTablesPaged(databaseName, maxResults, pageToken, tableNamePattern); + return api.listTablesPaged( + databaseName, maxResults, pageToken, tableNamePattern, tableType); } catch (NoSuchResourceException e) { throw new DatabaseNotExistException(databaseName); } @@ -238,11 +240,13 @@ public class RESTCatalog implements Catalog { String db, @Nullable Integer maxResults, @Nullable String pageToken, - @Nullable String tableNamePattern) + @Nullable String tableNamePattern, + @Nullable String tableType) throws DatabaseNotExistException { try { PagedList<GetTableResponse> tables = - api.listTableDetailsPaged(db, maxResults, pageToken, tableNamePattern); + api.listTableDetailsPaged( + db, maxResults, pageToken, tableNamePattern, tableType); return new PagedList<>( tables.getElements().stream() .map(t -> toTable(db, t)) @@ -330,6 +334,11 @@ public class RESTCatalog implements Catalog { return true; } + @Override + public boolean supportsListTableByType() { + return true; + } + @Override public boolean supportsVersionManagement() { return true; diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java index 0e0370a5df..b1ffa865ae 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java @@ -305,7 +305,8 @@ public abstract class CatalogTestBase { // List tables paged returns an empty list when there are no tables in the database String databaseName = "tables_paged_db"; catalog.createDatabase(databaseName, false); - PagedList<String> pagedTables = catalog.listTablesPaged(databaseName, null, null, null); + PagedList<String> pagedTables = + catalog.listTablesPaged(databaseName, null, null, null, null); assertThat(pagedTables.getElements()).isEmpty(); assertNull(pagedTables.getNextPageToken()); @@ -318,22 +319,22 @@ public abstract class CatalogTestBase { // List tables paged returns a list with the names of all tables in the database in all // catalogs except RestCatalog // even if the maxResults or pageToken is not null - pagedTables = catalog.listTablesPaged(databaseName, null, null, null); + pagedTables = catalog.listTablesPaged(databaseName, null, null, null, null); assertPagedTables(pagedTables, tableNames); int maxResults = 2; - pagedTables = catalog.listTablesPaged(databaseName, maxResults, null, null); + pagedTables = catalog.listTablesPaged(databaseName, maxResults, null, null, null); assertPagedTables(pagedTables, tableNames); String pageToken = "table1"; - pagedTables = catalog.listTablesPaged(databaseName, maxResults, pageToken, null); + pagedTables = catalog.listTablesPaged(databaseName, maxResults, pageToken, null, null); assertPagedTables(pagedTables, tableNames); maxResults = 8; - pagedTables = catalog.listTablesPaged(databaseName, maxResults, null, null); + pagedTables = catalog.listTablesPaged(databaseName, maxResults, null, null, null); assertPagedTables(pagedTables, tableNames); - pagedTables = catalog.listTablesPaged(databaseName, maxResults, pageToken, null); + pagedTables = catalog.listTablesPaged(databaseName, maxResults, pageToken, null, null); assertPagedTables(pagedTables, tableNames); // List tables throws DatabaseNotExistException when the database does not exist @@ -342,7 +343,7 @@ public abstract class CatalogTestBase { .isThrownBy( () -> catalog.listTablesPaged( - "non_existing_db", finalMaxResults, pageToken, null)); + "non_existing_db", finalMaxResults, pageToken, null, null)); } @Test @@ -351,7 +352,7 @@ public abstract class CatalogTestBase { String databaseName = "table_details_paged_db"; catalog.createDatabase(databaseName, false); PagedList<Table> pagedTableDetails = - catalog.listTableDetailsPaged(databaseName, null, null, null); + catalog.listTableDetailsPaged(databaseName, null, null, null, null); assertThat(pagedTableDetails.getElements()).isEmpty(); assertNull(pagedTableDetails.getNextPageToken()); @@ -364,28 +365,30 @@ public abstract class CatalogTestBase { Identifier.create(databaseName, tableName), DEFAULT_TABLE_SCHEMA, false); } - pagedTableDetails = catalog.listTableDetailsPaged(databaseName, null, null, null); + pagedTableDetails = catalog.listTableDetailsPaged(databaseName, null, null, null, null); assertPagedTableDetails(pagedTableDetails, tableNames.length, tableNames); assertNull(pagedTableDetails.getNextPageToken()); int maxResults = 2; - pagedTableDetails = catalog.listTableDetailsPaged(databaseName, maxResults, null, null); + pagedTableDetails = + catalog.listTableDetailsPaged(databaseName, maxResults, null, null, null); assertPagedTableDetails(pagedTableDetails, tableNames.length, tableNames); assertNull(pagedTableDetails.getNextPageToken()); String pageToken = "table1"; pagedTableDetails = - catalog.listTableDetailsPaged(databaseName, maxResults, pageToken, null); + catalog.listTableDetailsPaged(databaseName, maxResults, pageToken, null, null); assertPagedTableDetails(pagedTableDetails, tableNames.length, tableNames); assertNull(pagedTableDetails.getNextPageToken()); maxResults = 8; - pagedTableDetails = catalog.listTableDetailsPaged(databaseName, maxResults, null, null); + pagedTableDetails = + catalog.listTableDetailsPaged(databaseName, maxResults, null, null, null); assertPagedTableDetails(pagedTableDetails, tableNames.length, tableNames); assertNull(pagedTableDetails.getNextPageToken()); pagedTableDetails = - catalog.listTableDetailsPaged(databaseName, maxResults, pageToken, null); + catalog.listTableDetailsPaged(databaseName, maxResults, pageToken, null, null); assertPagedTableDetails(pagedTableDetails, tableNames.length, tableNames); assertNull(pagedTableDetails.getNextPageToken()); @@ -395,7 +398,7 @@ public abstract class CatalogTestBase { .isThrownBy( () -> catalog.listTableDetailsPaged( - "non_existing_db", finalMaxResults, pageToken, null)); + "non_existing_db", finalMaxResults, pageToken, null, null)); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index a6e1936f40..bf56514b4f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -84,7 +84,9 @@ class MockRESTCatalogTest extends RESTCatalogTest { @AfterEach public void tearDown() throws Exception { - restCatalogServer.shutdown(); + if (restCatalogServer != null) { + restCatalogServer.shutdown(); + } } @Test @@ -177,10 +179,10 @@ class MockRESTCatalogTest extends RESTCatalogTest { Identifier.create(databaseName, tableName), DEFAULT_TABLE_SCHEMA, false); } PagedList<String> listTablesPaged = - restCatalog.listTablesPaged(databaseName, 1, "dt=20230101", null); + restCatalog.listTablesPaged(databaseName, 1, "dt=20230101", null, null); PagedList<String> listTablesPaged2 = restCatalog.listTablesPaged( - databaseName, 1, listTablesPaged.getNextPageToken(), null); + databaseName, 1, listTablesPaged.getNextPageToken(), null, null); assertEquals(listTablesPaged.getElements().get(0), "dt=20230102"); assertEquals(listTablesPaged2.getElements().get(0), "dt=20230103"); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index a2e1da2c40..67d4ff7e91 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -143,6 +143,7 @@ import static org.apache.paimon.rest.RESTApi.MAX_RESULTS; import static org.apache.paimon.rest.RESTApi.PAGE_TOKEN; import static org.apache.paimon.rest.RESTApi.PARTITION_NAME_PATTERN; import static org.apache.paimon.rest.RESTApi.TABLE_NAME_PATTERN; +import static org.apache.paimon.rest.RESTApi.TABLE_TYPE; import static org.apache.paimon.rest.RESTApi.VIEW_NAME_PATTERN; import static org.apache.paimon.rest.ResourcePaths.FUNCTIONS; import static org.apache.paimon.rest.ResourcePaths.FUNCTION_DETAILS; @@ -1296,12 +1297,30 @@ public class RESTCatalogServer { private List<String> listTables(String databaseName, Map<String, String> parameters) { String tableNamePattern = parameters.get(TABLE_NAME_PATTERN); + String tableType = parameters.get(TABLE_TYPE); List<String> tables = new ArrayList<>(); for (Map.Entry<String, TableMetadata> entry : tableMetadataStore.entrySet()) { Identifier identifier = Identifier.fromString(entry.getKey()); if (databaseName.equals(identifier.getDatabaseName()) && (Objects.isNull(tableNamePattern) || matchNamePattern(identifier.getTableName(), tableNamePattern))) { + + // Check table type filter if specified + if (StringUtils.isNotEmpty(tableType)) { + String actualTableType = entry.getValue().schema().options().get(TYPE.key()); + if (StringUtils.equals(tableType, "table")) { + // When filtering by "table" type, return tables with null or "table" type + if (actualTableType != null && !"table".equals(actualTableType)) { + continue; + } + } else { + // For other table types, return exact matches + if (!StringUtils.equals(tableType, actualTableType)) { + continue; + } + } + } + tables.add(identifier.getTableName()); } } @@ -1361,12 +1380,30 @@ public class RESTCatalogServer { private List<GetTableResponse> listTableDetails( String databaseName, Map<String, String> parameters) { String tableNamePattern = parameters.get(TABLE_NAME_PATTERN); + String tableType = parameters.get(TABLE_TYPE); List<GetTableResponse> tableDetails = new ArrayList<>(); for (Map.Entry<String, TableMetadata> entry : tableMetadataStore.entrySet()) { Identifier identifier = Identifier.fromString(entry.getKey()); if (databaseName.equals(identifier.getDatabaseName()) && (Objects.isNull(tableNamePattern) || matchNamePattern(identifier.getTableName(), tableNamePattern))) { + + // Check table type filter if specified + if (StringUtils.isNotEmpty(tableType)) { + String actualTableType = entry.getValue().schema().options().get(TYPE.key()); + if (StringUtils.equals(tableType, "table")) { + // When filtering by "table" type, return tables with null or "table" type + if (actualTableType != null && !"table".equals(actualTableType)) { + continue; + } + } else { + // For other table types, return exact matches + if (!StringUtils.equals(tableType, actualTableType)) { + continue; + } + } + } + GetTableResponse getTableResponse = new GetTableResponse( entry.getValue().uuid(), diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 51e2c986cf..8234cac4d3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -251,10 +251,10 @@ public abstract class RESTCatalogTest extends CatalogTestBase { false)); assertThrows( Catalog.DatabaseNotExistException.class, - () -> catalog.listTablesPaged(database, 100, null, null)); + () -> catalog.listTablesPaged(database, 100, null, null, null)); assertThrows( Catalog.DatabaseNotExistException.class, - () -> catalog.listTableDetailsPaged(database, 100, null, null)); + () -> catalog.listTableDetailsPaged(database, 100, null, null, null)); } @Test @@ -362,7 +362,8 @@ public abstract class RESTCatalogTest extends CatalogTestBase { // List tables paged returns an empty list when there are no tables in the database String databaseName = "tables_paged_db"; catalog.createDatabase(databaseName, false); - PagedList<String> pagedTables = catalog.listTablesPaged(databaseName, null, null, null); + PagedList<String> pagedTables = + catalog.listTablesPaged(databaseName, null, null, null, null); assertThat(pagedTables.getElements()).isEmpty(); assertNull(pagedTables.getNextPageToken()); @@ -374,7 +375,7 @@ public abstract class RESTCatalogTest extends CatalogTestBase { // when maxResults is null or 0, the page length is set to a server configured value String[] sortedTableNames = Arrays.stream(tableNames).sorted().toArray(String[]::new); - pagedTables = catalog.listTablesPaged(databaseName, null, null, null); + pagedTables = catalog.listTablesPaged(databaseName, null, null, null, null); List<String> tables = pagedTables.getElements(); assertThat(tables).containsExactly(sortedTableNames); assertNull(pagedTables.getNextPageToken()); @@ -383,7 +384,7 @@ public abstract class RESTCatalogTest extends CatalogTestBase { // server configured value // when pageToken is null, will list tables from the beginning int maxResults = 2; - pagedTables = catalog.listTablesPaged(databaseName, maxResults, null, null); + pagedTables = catalog.listTablesPaged(databaseName, maxResults, null, null, null); tables = pagedTables.getElements(); assertEquals(maxResults, tables.size()); assertThat(tables).containsExactly("abd", "def"); @@ -392,7 +393,7 @@ public abstract class RESTCatalogTest extends CatalogTestBase { // when pageToken is not null, will list tables from the pageToken (exclusive) pagedTables = catalog.listTablesPaged( - databaseName, maxResults, pagedTables.getNextPageToken(), null); + databaseName, maxResults, pagedTables.getNextPageToken(), null, null); tables = pagedTables.getElements(); assertEquals(maxResults, tables.size()); assertThat(tables).containsExactly("opr", "table1"); @@ -400,7 +401,7 @@ public abstract class RESTCatalogTest extends CatalogTestBase { pagedTables = catalog.listTablesPaged( - databaseName, maxResults, pagedTables.getNextPageToken(), null); + databaseName, maxResults, pagedTables.getNextPageToken(), null, null); tables = pagedTables.getElements(); assertEquals(maxResults, tables.size()); assertThat(tables).containsExactly("table2", "table3"); @@ -408,18 +409,18 @@ public abstract class RESTCatalogTest extends CatalogTestBase { pagedTables = catalog.listTablesPaged( - databaseName, maxResults, pagedTables.getNextPageToken(), null); + databaseName, maxResults, pagedTables.getNextPageToken(), null, null); tables = pagedTables.getElements(); assertEquals(1, tables.size()); assertNull(pagedTables.getNextPageToken()); maxResults = 8; - pagedTables = catalog.listTablesPaged(databaseName, maxResults, null, null); + pagedTables = catalog.listTablesPaged(databaseName, maxResults, null, null, null); tables = pagedTables.getElements(); assertThat(tables).containsExactly(sortedTableNames); assertNull(pagedTables.getNextPageToken()); - pagedTables = catalog.listTablesPaged(databaseName, maxResults, "table1", null); + pagedTables = catalog.listTablesPaged(databaseName, maxResults, "table1", null, null); tables = pagedTables.getElements(); assertEquals(3, tables.size()); assertThat(tables).containsExactly("table2", "table3", "table_name"); @@ -429,24 +430,24 @@ public abstract class RESTCatalogTest extends CatalogTestBase { assertThatExceptionOfType(Catalog.DatabaseNotExistException.class) .isThrownBy(() -> catalog.listTables("non_existing_db")); - pagedTables = catalog.listTablesPaged(databaseName, null, null, "table%"); + pagedTables = catalog.listTablesPaged(databaseName, null, null, "table%", null); tables = pagedTables.getElements(); assertEquals(4, tables.size()); assertThat(tables).containsExactly("table1", "table2", "table3", "table_name"); assertNull(pagedTables.getNextPageToken()); - pagedTables = catalog.listTablesPaged(databaseName, null, null, "table_"); + pagedTables = catalog.listTablesPaged(databaseName, null, null, "table_", null); tables = pagedTables.getElements(); assertTrue(tables.isEmpty()); assertNull(pagedTables.getNextPageToken()); - pagedTables = catalog.listTablesPaged(databaseName, null, null, "table_%"); + pagedTables = catalog.listTablesPaged(databaseName, null, null, "table_%", null); tables = pagedTables.getElements(); assertEquals(1, tables.size()); assertThat(tables).containsExactly("table_name"); assertNull(pagedTables.getNextPageToken()); - pagedTables = catalog.listTablesPaged(databaseName, null, null, "table_name"); + pagedTables = catalog.listTablesPaged(databaseName, null, null, "table_name", null); tables = pagedTables.getElements(); assertEquals(1, tables.size()); assertThat(tables).containsExactly("table_name"); @@ -454,11 +455,11 @@ public abstract class RESTCatalogTest extends CatalogTestBase { Assertions.assertThrows( BadRequestException.class, - () -> catalog.listTablesPaged(databaseName, null, null, "%table")); + () -> catalog.listTablesPaged(databaseName, null, null, "%table", null)); Assertions.assertThrows( BadRequestException.class, - () -> catalog.listTablesPaged(databaseName, null, null, "ta%le")); + () -> catalog.listTablesPaged(databaseName, null, null, "ta%le", null)); } @Test @@ -467,7 +468,7 @@ public abstract class RESTCatalogTest extends CatalogTestBase { String databaseName = "table_details_paged_db"; catalog.createDatabase(databaseName, false); PagedList<Table> pagedTableDetails = - catalog.listTableDetailsPaged(databaseName, null, null, null); + catalog.listTableDetailsPaged(databaseName, null, null, null, null); assertThat(pagedTableDetails.getElements()).isEmpty(); assertNull(pagedTableDetails.getNextPageToken()); @@ -478,42 +479,44 @@ public abstract class RESTCatalogTest extends CatalogTestBase { Identifier.create(databaseName, tableName), DEFAULT_TABLE_SCHEMA, false); } - pagedTableDetails = catalog.listTableDetailsPaged(databaseName, null, null, null); + pagedTableDetails = catalog.listTableDetailsPaged(databaseName, null, null, null, null); assertPagedTableDetails(pagedTableDetails, tableNames.length, expectedTableNames); assertNull(pagedTableDetails.getNextPageToken()); int maxResults = 2; - pagedTableDetails = catalog.listTableDetailsPaged(databaseName, maxResults, null, null); + pagedTableDetails = + catalog.listTableDetailsPaged(databaseName, maxResults, null, null, null); assertPagedTableDetails(pagedTableDetails, maxResults, "abd", "def"); assertEquals("def", pagedTableDetails.getNextPageToken()); pagedTableDetails = catalog.listTableDetailsPaged( - databaseName, maxResults, pagedTableDetails.getNextPageToken(), null); + databaseName, maxResults, pagedTableDetails.getNextPageToken(), null, null); assertPagedTableDetails(pagedTableDetails, maxResults, "opr", "table1"); assertEquals("table1", pagedTableDetails.getNextPageToken()); pagedTableDetails = catalog.listTableDetailsPaged( - databaseName, maxResults, pagedTableDetails.getNextPageToken(), null); + databaseName, maxResults, pagedTableDetails.getNextPageToken(), null, null); assertPagedTableDetails(pagedTableDetails, maxResults, "table2", "table3"); assertEquals("table3", pagedTableDetails.getNextPageToken()); pagedTableDetails = catalog.listTableDetailsPaged( - databaseName, maxResults, pagedTableDetails.getNextPageToken(), null); + databaseName, maxResults, pagedTableDetails.getNextPageToken(), null, null); assertEquals(1, pagedTableDetails.getElements().size()); assertNull(pagedTableDetails.getNextPageToken()); maxResults = 8; - pagedTableDetails = catalog.listTableDetailsPaged(databaseName, maxResults, null, null); + pagedTableDetails = + catalog.listTableDetailsPaged(databaseName, maxResults, null, null, null); assertPagedTableDetails( pagedTableDetails, Math.min(maxResults, tableNames.length), expectedTableNames); assertNull(pagedTableDetails.getNextPageToken()); String pageToken = "table1"; pagedTableDetails = - catalog.listTableDetailsPaged(databaseName, maxResults, pageToken, null); + catalog.listTableDetailsPaged(databaseName, maxResults, pageToken, null, null); assertPagedTableDetails(pagedTableDetails, 3, "table2", "table3", "table_name"); assertNull(pagedTableDetails.getNextPageToken()); @@ -523,31 +526,292 @@ public abstract class RESTCatalogTest extends CatalogTestBase { .isThrownBy( () -> catalog.listTableDetailsPaged( - "non_existing_db", finalMaxResults, pageToken, null)); + "non_existing_db", finalMaxResults, pageToken, null, null)); // List tables throws DatabaseNotExistException when the database does not exist assertThatExceptionOfType(Catalog.DatabaseNotExistException.class) .isThrownBy(() -> catalog.listTables("non_existing_db")); - pagedTableDetails = catalog.listTableDetailsPaged(databaseName, null, null, "table%"); + pagedTableDetails = catalog.listTableDetailsPaged(databaseName, null, null, "table%", null); assertPagedTableDetails(pagedTableDetails, 4, "table1", "table2", "table3", "table_name"); assertNull(pagedTableDetails.getNextPageToken()); - pagedTableDetails = catalog.listTableDetailsPaged(databaseName, null, null, "table_"); + pagedTableDetails = catalog.listTableDetailsPaged(databaseName, null, null, "table_", null); Assertions.assertTrue(pagedTableDetails.getElements().isEmpty()); assertNull(pagedTableDetails.getNextPageToken()); - pagedTableDetails = catalog.listTableDetailsPaged(databaseName, null, null, "table_%"); + pagedTableDetails = + catalog.listTableDetailsPaged(databaseName, null, null, "table_%", null); assertPagedTableDetails(pagedTableDetails, 1, "table_name"); assertNull(pagedTableDetails.getNextPageToken()); Assertions.assertThrows( BadRequestException.class, - () -> catalog.listTableDetailsPaged(databaseName, null, null, "ta%le")); + () -> catalog.listTableDetailsPaged(databaseName, null, null, "ta%le", null)); Assertions.assertThrows( BadRequestException.class, - () -> catalog.listTableDetailsPaged(databaseName, null, null, "%tale")); + () -> catalog.listTableDetailsPaged(databaseName, null, null, "%tale", null)); + } + + @Test + public void testListTableDetailsPagedWithTableType() throws Exception { + String databaseName = "table_type_filter_db"; + catalog.createDatabase(databaseName, false); + + // Create tables with different types + Schema normalTableSchema = DEFAULT_TABLE_SCHEMA; + catalog.createTable( + Identifier.create(databaseName, "normal_table"), normalTableSchema, false); + + Schema formatTableSchema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .option("type", TableType.FORMAT_TABLE.toString()) + .build(); + catalog.createTable( + Identifier.create(databaseName, "format_table"), formatTableSchema, false); + + Schema objectTableSchema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .option("type", TableType.OBJECT_TABLE.toString()) + .build(); + catalog.createTable( + Identifier.create(databaseName, "object_table"), objectTableSchema, false); + + // Test filtering by table type + PagedList<Table> allTables = + catalog.listTableDetailsPaged(databaseName, null, null, null, null); + assertThat(allTables.getElements()).hasSize(3); + + PagedList<Table> normalTables = + catalog.listTableDetailsPaged( + databaseName, null, null, null, TableType.TABLE.toString()); + assertThat(normalTables.getElements()).hasSize(1); + assertThat(normalTables.getElements().get(0).name()).isEqualTo("normal_table"); + + PagedList<Table> formatTables = + catalog.listTableDetailsPaged( + databaseName, null, null, null, TableType.FORMAT_TABLE.toString()); + assertThat(formatTables.getElements()).hasSize(1); + assertThat(formatTables.getElements().get(0).name()).isEqualTo("format_table"); + + PagedList<Table> objectTables = + catalog.listTableDetailsPaged( + databaseName, null, null, null, TableType.OBJECT_TABLE.toString()); + assertThat(objectTables.getElements()).hasSize(1); + assertThat(objectTables.getElements().get(0).name()).isEqualTo("object_table"); + + // Test with non-existent table type + PagedList<Table> nonExistentType = + catalog.listTableDetailsPaged(databaseName, null, null, null, "non-existent-type"); + assertThat(nonExistentType.getElements()).isEmpty(); + + // Test with table name pattern and table type filter combined + PagedList<Table> filteredTables = + catalog.listTableDetailsPaged( + databaseName, null, null, "format_%", TableType.FORMAT_TABLE.toString()); + assertThat(filteredTables.getElements()).hasSize(1); + assertThat(filteredTables.getElements().get(0).name()).isEqualTo("format_table"); + + // Test with table name pattern and non-existent table type filter combined + PagedList<Table> filteredNonExistentType = + catalog.listTableDetailsPaged( + databaseName, null, null, "format_%", "non-existent-type"); + assertThat(filteredNonExistentType.getElements()).isEmpty(); + + // Test maxResults parameter variations with table type filtering + // Test maxResults=1 with different table types + PagedList<Table> singleNormalTable = + catalog.listTableDetailsPaged( + databaseName, 1, null, null, TableType.TABLE.toString()); + assertThat(singleNormalTable.getElements()).hasSize(1); + assertEquals("normal_table", singleNormalTable.getElements().get(0).name()); + assertEquals("normal_table", singleNormalTable.getNextPageToken()); + + PagedList<Table> singleFormatTable = + catalog.listTableDetailsPaged( + databaseName, 1, null, null, TableType.FORMAT_TABLE.toString()); + assertThat(singleFormatTable.getElements()).hasSize(1); + assertEquals("format_table", singleFormatTable.getElements().get(0).name()); + assertEquals("format_table", singleFormatTable.getNextPageToken()); + + PagedList<Table> singleObjectTable = + catalog.listTableDetailsPaged( + databaseName, 1, null, null, TableType.OBJECT_TABLE.toString()); + assertThat(singleObjectTable.getElements()).hasSize(1); + assertEquals("object_table", singleObjectTable.getElements().get(0).name()); + assertEquals("object_table", singleObjectTable.getNextPageToken()); + + // Test maxResults=2 with all table types + PagedList<Table> allTablesWithMaxResults = + catalog.listTableDetailsPaged(databaseName, 2, null, null, null); + assertThat(allTablesWithMaxResults.getElements()).hasSize(2); + assertThat(allTablesWithMaxResults.getNextPageToken()).isNotNull(); + + // Test maxResults=2 with table name pattern and table type filter combined + PagedList<Table> filteredTablesWithMaxResults = + catalog.listTableDetailsPaged( + databaseName, 2, null, "format_%", TableType.FORMAT_TABLE.toString()); + assertThat(filteredTablesWithMaxResults.getElements()).hasSize(1); + assertEquals("format_table", filteredTablesWithMaxResults.getElements().get(0).name()); + assertThat(filteredTablesWithMaxResults.getNextPageToken()).isNull(); + + // Test maxResults=0 (should return all tables) + PagedList<Table> allTablesWithZeroMaxResults = + catalog.listTableDetailsPaged(databaseName, 0, null, null, null); + assertThat(allTablesWithZeroMaxResults.getElements()).hasSize(3); + assertThat(allTablesWithZeroMaxResults.getNextPageToken()).isNull(); + + // Test maxResults larger than total tables with table type filter + PagedList<Table> largeMaxResultsWithType = + catalog.listTableDetailsPaged( + databaseName, 10, null, null, TableType.TABLE.toString()); + assertThat(largeMaxResultsWithType.getElements()).hasSize(1); + assertEquals("normal_table", largeMaxResultsWithType.getElements().get(0).name()); + assertThat(largeMaxResultsWithType.getNextPageToken()).isNull(); + + // Test maxResults with non-existent table type + PagedList<Table> nonExistentTypeWithMaxResults = + catalog.listTableDetailsPaged(databaseName, 5, null, null, "non-existent-type"); + assertThat(nonExistentTypeWithMaxResults.getElements()).isEmpty(); + assertThat(nonExistentTypeWithMaxResults.getNextPageToken()).isNull(); + } + + @Test + public void testListTablesPagedWithTableType() throws Exception { + String databaseName = "tables_paged_table_type_db"; + catalog.createDatabase(databaseName, false); + + // Create tables with different types + Schema normalTableSchema = DEFAULT_TABLE_SCHEMA; + catalog.createTable( + Identifier.create(databaseName, "normal_table"), normalTableSchema, false); + + Schema formatTableSchema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .option("type", TableType.FORMAT_TABLE.toString()) + .build(); + catalog.createTable( + Identifier.create(databaseName, "format_table"), formatTableSchema, false); + + Schema objectTableSchema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .option("type", TableType.OBJECT_TABLE.toString()) + .build(); + catalog.createTable( + Identifier.create(databaseName, "object_table"), objectTableSchema, false); + + // Test filtering by table type + PagedList<String> allTables = catalog.listTablesPaged(databaseName, null, null, null, null); + assertThat(allTables.getElements()).hasSize(3); + + PagedList<String> normalTables = + catalog.listTablesPaged(databaseName, null, null, null, TableType.TABLE.toString()); + assertThat(normalTables.getElements()).hasSize(1); + assertThat(normalTables.getElements().get(0)).isEqualTo("normal_table"); + + PagedList<String> formatTables = + catalog.listTablesPaged( + databaseName, null, null, null, TableType.FORMAT_TABLE.toString()); + assertThat(formatTables.getElements()).hasSize(1); + assertThat(formatTables.getElements().get(0)).isEqualTo("format_table"); + + PagedList<String> objectTables = + catalog.listTablesPaged( + databaseName, null, null, null, TableType.OBJECT_TABLE.toString()); + assertThat(objectTables.getElements()).hasSize(1); + assertThat(objectTables.getElements().get(0)).isEqualTo("object_table"); + + // Test with non-existent table type + PagedList<String> nonExistentType = + catalog.listTablesPaged(databaseName, null, null, null, "non-existent-type"); + assertThat(nonExistentType.getElements()).isEmpty(); + + // Test with table name pattern and table type filter combined + PagedList<String> filteredTables = + catalog.listTablesPaged( + databaseName, null, null, "format_%", TableType.FORMAT_TABLE.toString()); + assertThat(filteredTables.getElements()).hasSize(1); + assertThat(filteredTables.getElements().get(0)).isEqualTo("format_table"); + + // Test with table name pattern and non-existent table type filter combined + PagedList<String> filteredNonExistentType = + catalog.listTablesPaged(databaseName, null, null, "format_%", "non-existent-type"); + assertThat(filteredNonExistentType.getElements()).isEmpty(); + + // Test paging with table type filter + int maxResults = 10; + PagedList<String> pagedNormalTables = + catalog.listTablesPaged( + databaseName, maxResults, null, null, TableType.TABLE.toString()); + assertThat(pagedNormalTables.getElements()).hasSize(1); + assertThat(pagedNormalTables.getElements().get(0)).isEqualTo("normal_table"); + assertNull(pagedNormalTables.getNextPageToken()); + + // Test maxResults parameter variations with table type filtering + // Test maxResults=0 (should return all tables) + PagedList<String> allTablesWithZeroMaxResults = + catalog.listTablesPaged(databaseName, 0, null, null, null); + assertThat(allTablesWithZeroMaxResults.getElements()).hasSize(3); + assertNull(allTablesWithZeroMaxResults.getNextPageToken()); + + // Test maxResults=1 with different table types + PagedList<String> singleNormalTable = + catalog.listTablesPaged(databaseName, 1, null, null, TableType.TABLE.toString()); + assertThat(singleNormalTable.getElements()).hasSize(1); + assertEquals("normal_table", singleNormalTable.getElements().get(0)); + assertEquals("normal_table", singleNormalTable.getNextPageToken()); + + PagedList<String> singleFormatTable = + catalog.listTablesPaged( + databaseName, 1, null, null, TableType.FORMAT_TABLE.toString()); + assertThat(singleFormatTable.getElements()).hasSize(1); + assertEquals("format_table", singleFormatTable.getElements().get(0)); + assertEquals("format_table", singleFormatTable.getNextPageToken()); + + PagedList<String> singleObjectTable = + catalog.listTablesPaged( + databaseName, 1, null, null, TableType.OBJECT_TABLE.toString()); + assertThat(singleObjectTable.getElements()).hasSize(1); + assertEquals("object_table", singleObjectTable.getElements().get(0)); + assertEquals("object_table", singleObjectTable.getNextPageToken()); + + // Test maxResults=2 with all table types + PagedList<String> allTablesWithMaxResults = + catalog.listTablesPaged(databaseName, 2, null, null, null); + assertThat(allTablesWithMaxResults.getElements()).hasSize(2); + assertThat(allTablesWithMaxResults.getNextPageToken()).isNotNull(); + + // Test maxResults=2 with table name pattern and table type filter combined + PagedList<String> filteredTablesWithMaxResults = + catalog.listTablesPaged( + databaseName, 2, null, "format_%", TableType.FORMAT_TABLE.toString()); + assertThat(filteredTablesWithMaxResults.getElements()).hasSize(1); + assertThat(filteredTablesWithMaxResults.getElements().get(0)).isEqualTo("format_table"); + assertNull(filteredTablesWithMaxResults.getNextPageToken()); + assertEquals("format_table", filteredTablesWithMaxResults.getElements().get(0)); + assertNull(filteredTablesWithMaxResults.getNextPageToken()); + + // Test maxResults larger than total tables with table type filter + PagedList<String> largeMaxResultsWithType = + catalog.listTablesPaged(databaseName, 10, null, null, TableType.TABLE.toString()); + assertThat(largeMaxResultsWithType.getElements()).hasSize(1); + assertEquals("normal_table", largeMaxResultsWithType.getElements().get(0)); + assertNull(largeMaxResultsWithType.getNextPageToken()); + + // Test maxResults with non-existent table type + PagedList<String> nonExistentTypeWithMaxResults = + catalog.listTablesPaged(databaseName, 5, null, null, "non-existent-type"); + assertThat(nonExistentTypeWithMaxResults.getElements()).isEmpty(); + assertNull(nonExistentTypeWithMaxResults.getNextPageToken()); } @Test