This is an automated email from the ASF dual-hosted git repository. emaynard pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push: new 07ed4bb3c Implement GenericTableCatalogAdapter; admin-related fixes (#1298) 07ed4bb3c is described below commit 07ed4bb3c554793c9a54c516d5974cdbf10c368e Author: Eric Maynard <eric.maynard+...@snowflake.com> AuthorDate: Thu Apr 10 13:23:20 2025 -0700 Implement GenericTableCatalogAdapter; admin-related fixes (#1298) * initial commit: * debugging * some polish * autolint * spec change * bugfix * bugfix * various fixes * another missing admin location * autolint * false by default * fixes per review * autolint * more fixes * DRY * revert small change for a better error * integration test * extra test * autolint * stable * wip * rework subtypes a bit * stable again * autolint * apply new lint rule * errorprone again * adjustments per review * update golden files * add another test * clean up logic in PolarisAdminService * autolint * more fixes per review * format --- integration-tests/build.gradle.kts | 1 + .../polaris/service/it/env/GenericTableApi.java | 96 +++++++++++++ .../polaris/service/it/env/PolarisClient.java | 10 ++ .../it/test/PolarisRestCatalogIntegrationTest.java | 148 +++++++++++++++++++++ .../polaris/core/config/FeatureConfiguration.java | 2 +- regtests/t_spark_sql/ref/spark_sql_basic.sh.ref | 2 +- regtests/t_spark_sql/ref/spark_sql_views.sh.ref | 2 +- .../polaris/service/admin/PolarisAdminService.java | 94 ++++++++----- .../service/catalog/common/CatalogAdapter.java | 34 +++++ .../service/catalog/common/CatalogHandler.java | 57 +++++--- .../generic/GenericTableCatalogAdapter.java | 75 ++++++++++- .../generic/GenericTableCatalogHandler.java | 4 +- .../catalog/iceberg/IcebergCatalogAdapter.java | 12 +- 13 files changed, 461 insertions(+), 76 deletions(-) diff --git a/integration-tests/build.gradle.kts b/integration-tests/build.gradle.kts index 55aafe7b0..e5018457f 100644 --- a/integration-tests/build.gradle.kts +++ b/integration-tests/build.gradle.kts @@ -22,6 +22,7 @@ plugins { id("polaris-server") } dependencies { implementation(project(":polaris-core")) implementation(project(":polaris-api-management-model")) + implementation(project(":polaris-api-catalog-service")) implementation(libs.jakarta.ws.rs.api) implementation(libs.guava) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/GenericTableApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/GenericTableApi.java new file mode 100644 index 000000000..52935a8dc --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/GenericTableApi.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static jakarta.ws.rs.core.Response.Status.NO_CONTENT; +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.polaris.service.types.CreateGenericTableRequest; +import org.apache.polaris.service.types.GenericTable; +import org.apache.polaris.service.types.ListGenericTablesResponse; +import org.apache.polaris.service.types.LoadGenericTableResponse; + +/** + * A simple, non-exhaustive set of helper methods for accessing the generic tables REST API + * + * @see PolarisClient#genericTableApi(ClientCredentials) + */ +public class GenericTableApi extends RestApi { + GenericTableApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { + super(client, endpoints, authToken, uri); + } + + public void purge(String catalog, Namespace ns) { + listGenericTables(catalog, ns).forEach(t -> dropGenericTable(catalog, t)); + } + + public List<TableIdentifier> listGenericTables(String catalog, Namespace namespace) { + String ns = RESTUtil.encodeNamespace(namespace); + try (Response res = + request("polaris/v1/{cat}/namespaces/{ns}/generic-tables", Map.of("cat", catalog, "ns", ns)) + .get()) { + assertThat(res.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + return res.readEntity(ListGenericTablesResponse.class).getIdentifiers().stream().toList(); + } + } + + public void dropGenericTable(String catalog, TableIdentifier id) { + String ns = RESTUtil.encodeNamespace(id.namespace()); + try (Response res = + request( + "polaris/v1/{cat}/namespaces/{ns}/generic-tables/{table}", + Map.of("cat", catalog, "table", id.name(), "ns", ns)) + .delete()) { + assertThat(res.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } + + public GenericTable getGenericTable(String catalog, TableIdentifier id) { + String ns = RESTUtil.encodeNamespace(id.namespace()); + try (Response res = + request( + "polaris/v1/{cat}/namespaces/{ns}/generic-tables/{table}", + Map.of("cat", catalog, "table", id.name(), "ns", ns)) + .get()) { + return res.readEntity(LoadGenericTableResponse.class).getTable(); + } + } + + public GenericTable createGenericTable( + String catalog, TableIdentifier id, String format, Map<String, String> properties) { + String ns = RESTUtil.encodeNamespace(id.namespace()); + try (Response res = + request( + "polaris/v1/{cat}/namespaces/{ns}/generic-tables/", + Map.of("cat", catalog, "ns", ns)) + .post( + Entity.json(new CreateGenericTableRequest(id.name(), format, "doc", properties)))) { + return res.readEntity(LoadGenericTableResponse.class).getTable(); + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java index 163c217c2..e18f8736b 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java @@ -106,6 +106,16 @@ public final class PolarisClient implements AutoCloseable { return new CatalogApi(client, endpoints, null, endpoints.catalogApiEndpoint()); } + public GenericTableApi genericTableApi(PrincipalWithCredentials principal) { + return new GenericTableApi( + client, endpoints, obtainToken(principal), endpoints.catalogApiEndpoint()); + } + + public GenericTableApi genericTableApi(ClientCredentials credentials) { + return new GenericTableApi( + client, endpoints, obtainToken(credentials), endpoints.catalogApiEndpoint()); + } + /** * Requests an access token from the Polaris server for the client ID/secret pair that is part of * the given principal data object. diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java index f7466b77f..c895b49cf 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java @@ -18,11 +18,13 @@ */ package org.apache.polaris.service.it.test; +import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.collect.ImmutableMap; +import jakarta.ws.rs.ProcessingException; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.HttpHeaders; @@ -32,11 +34,13 @@ import java.lang.annotation.RetentionPolicy; import java.lang.reflect.Method; import java.net.URI; import java.nio.file.Path; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; import org.apache.iceberg.BaseTransaction; @@ -57,6 +61,7 @@ import org.apache.iceberg.exceptions.ForbiddenException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.io.ResolvingFileIO; import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.RESTUtil; import org.apache.iceberg.rest.requests.CreateTableRequest; import org.apache.iceberg.rest.responses.ErrorResponse; import org.apache.iceberg.types.Types; @@ -82,12 +87,14 @@ import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.service.it.env.CatalogApi; import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.GenericTableApi; import org.apache.polaris.service.it.env.IcebergHelper; import org.apache.polaris.service.it.env.IntegrationTestsHelper; import org.apache.polaris.service.it.env.ManagementApi; import org.apache.polaris.service.it.env.PolarisApiEndpoints; import org.apache.polaris.service.it.env.PolarisClient; import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; +import org.apache.polaris.service.types.GenericTable; import org.assertj.core.api.Assertions; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.AfterAll; @@ -127,6 +134,7 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog> private PrincipalWithCredentials principalCredentials; private CatalogApi catalogApi; + private GenericTableApi genericTableApi; private RESTCatalog restCatalog; private String currentCatalogName; private Map<String, String> restCatalogConfig; @@ -178,6 +186,7 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog> principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); catalogApi = client.catalogApi(principalCredentials); + genericTableApi = client.genericTableApi(principalCredentials); Method method = testInfo.getTestMethod().orElseThrow(); currentCatalogName = client.newEntityName(method.getName()); @@ -1194,4 +1203,143 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog> assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); } } + + @Test + public void testCreateGenericTable() { + Namespace namespace = Namespace.of("ns1"); + restCatalog.createNamespace(namespace); + TableIdentifier tableIdentifier = TableIdentifier.of(namespace, "tbl1"); + + GenericTable createResponse = + genericTableApi.createGenericTable(currentCatalogName, tableIdentifier, "format", Map.of()); + Assertions.assertThat(createResponse.getFormat()).isEqualTo("format"); + + genericTableApi.purge(currentCatalogName, namespace); + } + + @Test + public void testLoadGenericTable() { + Namespace namespace = Namespace.of("ns1"); + restCatalog.createNamespace(namespace); + TableIdentifier tableIdentifier = TableIdentifier.of(namespace, "tbl1"); + + genericTableApi.createGenericTable(currentCatalogName, tableIdentifier, "format", Map.of()); + + GenericTable loadResponse = + genericTableApi.getGenericTable(currentCatalogName, tableIdentifier); + Assertions.assertThat(loadResponse.getFormat()).isEqualTo("format"); + + genericTableApi.purge(currentCatalogName, namespace); + } + + @Test + public void testListGenericTables() { + Namespace namespace = Namespace.of("ns1"); + restCatalog.createNamespace(namespace); + TableIdentifier tableIdentifier1 = TableIdentifier.of(namespace, "tbl1"); + TableIdentifier tableIdentifier2 = TableIdentifier.of(namespace, "tbl2"); + + genericTableApi.createGenericTable(currentCatalogName, tableIdentifier1, "format", Map.of()); + genericTableApi.createGenericTable(currentCatalogName, tableIdentifier2, "format", Map.of()); + + List<TableIdentifier> identifiers = + genericTableApi.listGenericTables(currentCatalogName, namespace); + + Assertions.assertThat(identifiers).hasSize(2); + Assertions.assertThat(identifiers) + .containsExactlyInAnyOrder(tableIdentifier1, tableIdentifier2); + + genericTableApi.purge(currentCatalogName, namespace); + } + + @Test + public void testDropGenericTable() { + Namespace namespace = Namespace.of("ns1"); + restCatalog.createNamespace(namespace); + TableIdentifier tableIdentifier = TableIdentifier.of(namespace, "tbl1"); + + genericTableApi.createGenericTable(currentCatalogName, tableIdentifier, "format", Map.of()); + + GenericTable loadResponse = + genericTableApi.getGenericTable(currentCatalogName, tableIdentifier); + Assertions.assertThat(loadResponse.getFormat()).isEqualTo("format"); + + genericTableApi.dropGenericTable(currentCatalogName, tableIdentifier); + + Assertions.assertThatCode( + () -> genericTableApi.getGenericTable(currentCatalogName, tableIdentifier)) + .isInstanceOf(ProcessingException.class); + + genericTableApi.purge(currentCatalogName, namespace); + } + + @Test + public void testGrantsOnGenericTable() { + Namespace namespace = Namespace.of("ns1"); + restCatalog.createNamespace(namespace); + TableIdentifier tableIdentifier = TableIdentifier.of(namespace, "tbl1"); + genericTableApi.createGenericTable(currentCatalogName, tableIdentifier, "format", Map.of()); + + managementApi.createCatalogRole(currentCatalogName, "catalogrole1"); + + Stream<TableGrant> tableGrants = + Arrays.stream(TablePrivilege.values()) + .map( + p -> { + return new TableGrant(List.of("ns1"), "tbl1", p, GrantResource.TypeEnum.TABLE); + }); + + tableGrants.forEach(g -> managementApi.addGrant(currentCatalogName, "catalogrole1", g)); + + genericTableApi.purge(currentCatalogName, namespace); + } + + @Test + public void testGrantsOnNonExistingGenericTable() { + Namespace namespace = Namespace.of("ns1"); + restCatalog.createNamespace(namespace); + + managementApi.createCatalogRole(currentCatalogName, "catalogrole1"); + + Stream<TableGrant> tableGrants = + Arrays.stream(TablePrivilege.values()) + .map( + p -> { + return new TableGrant(List.of("ns1"), "tbl1", p, GrantResource.TypeEnum.TABLE); + }); + + tableGrants.forEach( + g -> { + try (Response response = + managementApi + .request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", currentCatalogName, "role", "catalogrole1")) + .put(Entity.json(g))) { + + assertThat(response.getStatus()).isEqualTo(NOT_FOUND.getStatusCode()); + } + }); + + genericTableApi.purge(currentCatalogName, namespace); + } + + @Test + public void testDropNonExistingGenericTable() { + Namespace namespace = Namespace.of("ns1"); + restCatalog.createNamespace(namespace); + TableIdentifier tableIdentifier = TableIdentifier.of(namespace, "tbl1"); + + String ns = RESTUtil.encodeNamespace(tableIdentifier.namespace()); + try (Response res = + genericTableApi + .request( + "polaris/v1/{cat}/namespaces/{ns}/generic-tables/{table}", + Map.of("cat", currentCatalogName, "table", tableIdentifier.name(), "ns", ns)) + .delete()) { + assertThat(res.getStatus()).isEqualTo(NOT_FOUND.getStatusCode()); + } + + genericTableApi.purge(currentCatalogName, namespace); + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index cc8bae454..8e71d049c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -188,6 +188,6 @@ public class FeatureConfiguration<T> extends PolarisConfiguration<T> { PolarisConfiguration.<Boolean>builder() .key("ENABLE_GENERIC_TABLES") .description("If true, the generic-tables endpoints are enabled") - .defaultValue(false) + .defaultValue(true) .buildFeatureConfiguration(); } diff --git a/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref b/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref index 4cd8ce0f8..eaf0e18a8 100755 --- a/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref +++ b/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref @@ -1,4 +1,4 @@ -{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_basic_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD [...] +{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_basic_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD [...] Catalog created spark-sql (default)> use polaris; spark-sql ()> show namespaces; diff --git a/regtests/t_spark_sql/ref/spark_sql_views.sh.ref b/regtests/t_spark_sql/ref/spark_sql_views.sh.ref index 853c736db..ffae79311 100755 --- a/regtests/t_spark_sql/ref/spark_sql_views.sh.ref +++ b/regtests/t_spark_sql/ref/spark_sql_views.sh.ref @@ -1,4 +1,4 @@ -{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_views_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD [...] +{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_views_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD [...] Catalog created spark-sql (default)> use polaris; spark-sql ()> show namespaces; diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index effcededd..8f1c9bc92 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -40,8 +40,6 @@ import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.BadRequestException; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; -import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.exceptions.ValidationException; import org.apache.polaris.core.PolarisCallContext; @@ -95,6 +93,7 @@ import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.StorageLocation; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo; +import org.apache.polaris.service.catalog.common.CatalogHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -454,7 +453,7 @@ public class PolarisAdminService { private void authorizeGrantOnTableLikeOperationOrThrow( PolarisAuthorizableOperation op, String catalogName, - PolarisEntitySubType subType, + List<PolarisEntitySubType> subTypes, TableIdentifier identifier, String catalogRoleName) { resolutionManifest = @@ -472,18 +471,19 @@ public class PolarisAdminService { throw new NotFoundException("Catalog not found: %s", catalogName); } else if (status.getStatus() == ResolverStatus.StatusEnum.PATH_COULD_NOT_BE_FULLY_RESOLVED) { if (status.getFailedToResolvePath().getLastEntityType() == PolarisEntityType.TABLE_LIKE) { - if (subType == PolarisEntitySubType.ICEBERG_TABLE) { - throw new NoSuchTableException("Table does not exist: %s", identifier); - } else { - throw new NoSuchViewException("View does not exist: %s", identifier); - } + CatalogHandler.throwNotFoundExceptionForTableLikeEntity(identifier, subTypes); } else { throw new NotFoundException("CatalogRole not found: %s.%s", catalogName, catalogRoleName); } } PolarisResolvedPathWrapper tableLikeWrapper = - resolutionManifest.getResolvedPath(identifier, PolarisEntityType.TABLE_LIKE, subType, true); + resolutionManifest.getResolvedPath( + identifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE, true); + if (!subTypes.contains(tableLikeWrapper.getRawLeafEntity().getSubType())) { + CatalogHandler.throwNotFoundExceptionForTableLikeEntity(identifier, subTypes); + } + PolarisResolvedPathWrapper catalogRoleWrapper = resolutionManifest.getResolvedPath(catalogRoleName, true); @@ -1495,10 +1495,18 @@ public class PolarisAdminService { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.ADD_TABLE_GRANT_TO_CATALOG_ROLE; authorizeGrantOnTableLikeOperationOrThrow( - op, catalogName, PolarisEntitySubType.ICEBERG_TABLE, identifier, catalogRoleName); + op, + catalogName, + List.of(PolarisEntitySubType.GENERIC_TABLE, PolarisEntitySubType.ICEBERG_TABLE), + identifier, + catalogRoleName); return grantPrivilegeOnTableLikeToRole( - catalogName, catalogRoleName, identifier, PolarisEntitySubType.ICEBERG_TABLE, privilege); + catalogName, + catalogRoleName, + identifier, + List.of(PolarisEntitySubType.GENERIC_TABLE, PolarisEntitySubType.ICEBERG_TABLE), + privilege); } public boolean revokePrivilegeOnTableFromRole( @@ -1510,10 +1518,18 @@ public class PolarisAdminService { PolarisAuthorizableOperation.REVOKE_TABLE_GRANT_FROM_CATALOG_ROLE; authorizeGrantOnTableLikeOperationOrThrow( - op, catalogName, PolarisEntitySubType.ICEBERG_TABLE, identifier, catalogRoleName); + op, + catalogName, + List.of(PolarisEntitySubType.GENERIC_TABLE, PolarisEntitySubType.ICEBERG_TABLE), + identifier, + catalogRoleName); return revokePrivilegeOnTableLikeFromRole( - catalogName, catalogRoleName, identifier, PolarisEntitySubType.ICEBERG_TABLE, privilege); + catalogName, + catalogRoleName, + identifier, + List.of(PolarisEntitySubType.GENERIC_TABLE, PolarisEntitySubType.ICEBERG_TABLE), + privilege); } public boolean grantPrivilegeOnViewToRole( @@ -1524,10 +1540,14 @@ public class PolarisAdminService { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.ADD_VIEW_GRANT_TO_CATALOG_ROLE; authorizeGrantOnTableLikeOperationOrThrow( - op, catalogName, PolarisEntitySubType.ICEBERG_VIEW, identifier, catalogRoleName); + op, catalogName, List.of(PolarisEntitySubType.ICEBERG_VIEW), identifier, catalogRoleName); return grantPrivilegeOnTableLikeToRole( - catalogName, catalogRoleName, identifier, PolarisEntitySubType.ICEBERG_VIEW, privilege); + catalogName, + catalogRoleName, + identifier, + List.of(PolarisEntitySubType.ICEBERG_VIEW), + privilege); } public boolean revokePrivilegeOnViewFromRole( @@ -1539,10 +1559,14 @@ public class PolarisAdminService { PolarisAuthorizableOperation.REVOKE_VIEW_GRANT_FROM_CATALOG_ROLE; authorizeGrantOnTableLikeOperationOrThrow( - op, catalogName, PolarisEntitySubType.ICEBERG_VIEW, identifier, catalogRoleName); + op, catalogName, List.of(PolarisEntitySubType.ICEBERG_VIEW), identifier, catalogRoleName); return revokePrivilegeOnTableLikeFromRole( - catalogName, catalogRoleName, identifier, PolarisEntitySubType.ICEBERG_VIEW, privilege); + catalogName, + catalogRoleName, + identifier, + List.of(PolarisEntitySubType.ICEBERG_VIEW), + privilege); } public List<PolarisEntity> listAssigneePrincipalRolesForCatalogRole( @@ -1606,7 +1630,8 @@ public class PolarisAdminService { } case TABLE_LIKE: { - if (baseEntity.getSubType() == PolarisEntitySubType.ICEBERG_TABLE) { + if (baseEntity.getSubType() == PolarisEntitySubType.ICEBERG_TABLE + || baseEntity.getSubType() == PolarisEntitySubType.GENERIC_TABLE) { TableIdentifier identifier = IcebergTableLikeEntity.of(baseEntity).getTableIdentifier(); TableGrant grant = @@ -1616,7 +1641,7 @@ public class PolarisAdminService { TablePrivilege.valueOf(privilege.toString()), GrantResource.TypeEnum.TABLE); tableGrants.add(grant); - } else { + } else if (baseEntity.getSubType() == PolarisEntitySubType.ICEBERG_VIEW) { TableIdentifier identifier = IcebergTableLikeEntity.of(baseEntity).getTableIdentifier(); ViewGrant grant = @@ -1626,6 +1651,9 @@ public class PolarisAdminService { ViewPrivilege.valueOf(privilege.toString()), GrantResource.TypeEnum.VIEW); viewGrants.add(grant); + } else { + throw new IllegalStateException( + "Unrecognized entity subtype " + baseEntity.getSubType()); } break; } @@ -1694,7 +1722,7 @@ public class PolarisAdminService { String catalogName, String catalogRoleName, TableIdentifier identifier, - PolarisEntitySubType subType, + List<PolarisEntitySubType> subTypes, PolarisPrivilege privilege) { if (findCatalogByName(catalogName).isEmpty()) { throw new NotFoundException("Parent catalog %s not found", catalogName); @@ -1704,13 +1732,11 @@ public class PolarisAdminService { .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); PolarisResolvedPathWrapper resolvedPathWrapper = - resolutionManifest.getResolvedPath(identifier, PolarisEntityType.TABLE_LIKE, subType); - if (resolvedPathWrapper == null) { - if (subType == PolarisEntitySubType.ICEBERG_VIEW) { - throw new NotFoundException("View %s not found", identifier); - } else { - throw new NotFoundException("Table %s not found", identifier); - } + resolutionManifest.getResolvedPath( + identifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE); + if (resolvedPathWrapper == null + || !subTypes.contains(resolvedPathWrapper.getRawLeafEntity().getSubType())) { + CatalogHandler.throwNotFoundExceptionForTableLikeEntity(identifier, subTypes); } List<PolarisEntity> catalogPath = resolvedPathWrapper.getRawParentPath(); PolarisEntity tableLikeEntity = resolvedPathWrapper.getRawLeafEntity(); @@ -1732,7 +1758,7 @@ public class PolarisAdminService { String catalogName, String catalogRoleName, TableIdentifier identifier, - PolarisEntitySubType subType, + List<PolarisEntitySubType> subTypes, PolarisPrivilege privilege) { if (findCatalogByName(catalogName).isEmpty()) { throw new NotFoundException("Parent catalog %s not found", catalogName); @@ -1742,13 +1768,11 @@ public class PolarisAdminService { .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); PolarisResolvedPathWrapper resolvedPathWrapper = - resolutionManifest.getResolvedPath(identifier, PolarisEntityType.TABLE_LIKE, subType); - if (resolvedPathWrapper == null) { - if (subType == PolarisEntitySubType.ICEBERG_VIEW) { - throw new NotFoundException("View %s not found", identifier); - } else { - throw new NotFoundException("Table %s not found", identifier); - } + resolutionManifest.getResolvedPath( + identifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE); + if (resolvedPathWrapper == null + || !subTypes.contains(resolvedPathWrapper.getRawLeafEntity().getSubType())) { + CatalogHandler.throwNotFoundExceptionForTableLikeEntity(identifier, subTypes); } List<PolarisEntity> catalogPath = resolvedPathWrapper.getRawParentPath(); PolarisEntity tableLikeEntity = resolvedPathWrapper.getRawLeafEntity(); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/common/CatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/common/CatalogAdapter.java new file mode 100644 index 000000000..56be4d925 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/common/CatalogAdapter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.common; + +import java.net.URLEncoder; +import java.nio.charset.Charset; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.rest.RESTUtil; + +/** + * A common interface for adapters between the REST interface and {@link CatalogHandler} + * implementations + */ +public interface CatalogAdapter { + default Namespace decodeNamespace(String namespace) { + return RESTUtil.decodeNamespace(URLEncoder.encode(namespace, Charset.defaultCharset())); + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java index 293c4c488..3a902c8ed 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.service.catalog.common; +import static org.apache.polaris.core.entity.PolarisEntitySubType.ICEBERG_TABLE; + import jakarta.ws.rs.core.SecurityContext; import java.util.Arrays; import java.util.List; @@ -220,16 +222,7 @@ public abstract class CatalogHandler { PolarisResolvedPathWrapper target = resolutionManifest.getResolvedPath(identifier, PolarisEntityType.TABLE_LIKE, subType, true); if (target == null) { - switch (subType) { - case PolarisEntitySubType.ICEBERG_TABLE: - throw new NoSuchTableException("Table does not exist: %s", identifier); - - case PolarisEntitySubType.GENERIC_TABLE: - throw new NoSuchTableException("Generic table does not exist: %s", identifier); - - default: - throw new NoSuchViewException("View does not exist: %s", identifier); - } + throwNotFoundExceptionForTableLikeEntity(identifier, List.of(subType)); } authorizer.authorizeOrThrow( authenticatedPrincipal, @@ -263,11 +256,7 @@ public abstract class CatalogHandler { TableIdentifier identifier = PolarisCatalogHelpers.listToTableIdentifier( status.getFailedToResolvePath().getEntityNames()); - if (subType == PolarisEntitySubType.ICEBERG_TABLE) { - throw new NoSuchTableException("Table does not exist: %s", identifier); - } else { - throw new NoSuchViewException("View does not exist: %s", identifier); - } + throwNotFoundExceptionForTableLikeEntity(identifier, List.of(subType)); } List<PolarisResolvedPathWrapper> targets = @@ -279,7 +268,7 @@ public abstract class CatalogHandler { identifier, PolarisEntityType.TABLE_LIKE, subType, true)) .orElseThrow( () -> - subType == PolarisEntitySubType.ICEBERG_TABLE + subType == ICEBERG_TABLE ? new NoSuchTableException( "Table does not exist: %s", identifier) : new NoSuchViewException( @@ -322,11 +311,7 @@ public abstract class CatalogHandler { throw new NoSuchNamespaceException("Namespace does not exist: %s", dst.namespace()); } else if (resolutionManifest.getResolvedPath(src, PolarisEntityType.TABLE_LIKE, subType) == null) { - if (subType == PolarisEntitySubType.ICEBERG_TABLE) { - throw new NoSuchTableException("Table does not exist: %s", src); - } else { - throw new NoSuchViewException("View does not exist: %s", src); - } + throwNotFoundExceptionForTableLikeEntity(dst, List.of(subType)); } // Normally, since we added the dst as an optional path, we'd expect it to only get resolved @@ -339,7 +324,7 @@ public abstract class CatalogHandler { PolarisEntitySubType dstLeafSubType = resolutionManifest.getLeafSubType(dst); switch (dstLeafSubType) { - case PolarisEntitySubType.ICEBERG_TABLE: + case ICEBERG_TABLE: throw new AlreadyExistsException("Cannot rename %s to %s. Table already exists", src, dst); case PolarisEntitySubType.ICEBERG_VIEW: @@ -366,4 +351,32 @@ public abstract class CatalogHandler { initializeCatalog(); } + + /** + * Helper function for when a TABLE_LIKE entity is not found so we want to throw the appropriate + * exception. Used in Iceberg APIs, so the Iceberg messages cannot be changed. + * + * @param subTypes The subtypes of the entity that the exception should report doesn't exist + */ + public static void throwNotFoundExceptionForTableLikeEntity( + TableIdentifier identifier, List<PolarisEntitySubType> subTypes) { + + // In this case, we assume it's a table + if (subTypes.size() > 1) { + throw new NoSuchTableException("Table does not exist: %s", identifier); + } else { + PolarisEntitySubType subType = subTypes.getFirst(); + switch (subType) { + case ICEBERG_TABLE: + throw new NoSuchTableException("Table does not exist: %s", identifier); + case ICEBERG_VIEW: + throw new NoSuchViewException("View does not exist: %s", identifier); + case GENERIC_TABLE: + throw new NoSuchTableException("Generic table does not exist: %s", identifier); + default: + // Assume it's a table + throw new NoSuchTableException("Table does not exist: %s", identifier); + } + } + } } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogAdapter.java index cd8c2b5a6..bfd296904 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogAdapter.java @@ -19,14 +19,64 @@ package org.apache.polaris.service.catalog.generic; import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NotAuthorizedException; +import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.persistence.PolarisEntityManager; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.service.catalog.api.PolarisCatalogGenericTableApiService; +import org.apache.polaris.service.catalog.common.CatalogAdapter; import org.apache.polaris.service.types.CreateGenericTableRequest; +import org.apache.polaris.service.types.ListGenericTablesResponse; +import org.apache.polaris.service.types.LoadGenericTableResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @RequestScoped -public class GenericTableCatalogAdapter implements PolarisCatalogGenericTableApiService { +public class GenericTableCatalogAdapter + implements PolarisCatalogGenericTableApiService, CatalogAdapter { + + private static final Logger LOGGER = LoggerFactory.getLogger(GenericTableCatalogAdapter.class); + + private final CallContext callContext; + private final PolarisEntityManager entityManager; + private final PolarisMetaStoreManager metaStoreManager; + private final PolarisAuthorizer polarisAuthorizer; + + @Inject + public GenericTableCatalogAdapter( + CallContext callContext, + PolarisEntityManager entityManager, + PolarisMetaStoreManager metaStoreManager, + PolarisAuthorizer polarisAuthorizer) { + this.callContext = callContext; + this.entityManager = entityManager; + this.metaStoreManager = metaStoreManager; + this.polarisAuthorizer = polarisAuthorizer; + } + + private GenericTableCatalogHandler newHandlerWrapper( + SecurityContext securityContext, String catalogName) { + var authenticatedPrincipal = (AuthenticatedPolarisPrincipal) securityContext.getUserPrincipal(); + if (authenticatedPrincipal == null) { + throw new NotAuthorizedException("Failed to find authenticatedPrincipal in SecurityContext"); + } + + return new GenericTableCatalogHandler( + callContext, + entityManager, + metaStoreManager, + securityContext, + catalogName, + polarisAuthorizer); + } + @Override public Response createGenericTable( String prefix, @@ -34,7 +84,15 @@ public class GenericTableCatalogAdapter implements PolarisCatalogGenericTableApi CreateGenericTableRequest createGenericTableRequest, RealmContext realmContext, SecurityContext securityContext) { - return Response.status(501).build(); // not implemented + GenericTableCatalogHandler handler = newHandlerWrapper(securityContext, prefix); + LoadGenericTableResponse response = + handler.createGenericTable( + TableIdentifier.of(decodeNamespace(namespace), createGenericTableRequest.getName()), + createGenericTableRequest.getFormat(), + createGenericTableRequest.getDoc(), + createGenericTableRequest.getProperties()); + + return Response.ok(response).build(); } @Override @@ -44,7 +102,9 @@ public class GenericTableCatalogAdapter implements PolarisCatalogGenericTableApi String genericTable, RealmContext realmContext, SecurityContext securityContext) { - return Response.status(501).build(); // not implemented + GenericTableCatalogHandler handler = newHandlerWrapper(securityContext, prefix); + handler.dropGenericTable(TableIdentifier.of(decodeNamespace(namespace), genericTable)); + return Response.noContent().build(); } @Override @@ -55,7 +115,9 @@ public class GenericTableCatalogAdapter implements PolarisCatalogGenericTableApi Integer pageSize, RealmContext realmContext, SecurityContext securityContext) { - return Response.status(501).build(); // not implemented + GenericTableCatalogHandler handler = newHandlerWrapper(securityContext, prefix); + ListGenericTablesResponse response = handler.listGenericTables(decodeNamespace(namespace)); + return Response.ok(response).build(); } @Override @@ -65,6 +127,9 @@ public class GenericTableCatalogAdapter implements PolarisCatalogGenericTableApi String genericTable, RealmContext realmContext, SecurityContext securityContext) { - return Response.status(501).build(); // not implemented + GenericTableCatalogHandler handler = newHandlerWrapper(securityContext, prefix); + LoadGenericTableResponse response = + handler.loadGenericTable(TableIdentifier.of(decodeNamespace(namespace), genericTable)); + return Response.ok(response).build(); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java index 6df82dd7d..b1f2648f1 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java @@ -19,8 +19,8 @@ package org.apache.polaris.service.catalog.generic; import jakarta.ws.rs.core.SecurityContext; +import java.util.LinkedHashSet; import java.util.Map; -import java.util.TreeSet; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.auth.PolarisAuthorizableOperation; @@ -77,7 +77,7 @@ public class GenericTableCatalogHandler extends CatalogHandler { authorizeBasicNamespaceOperationOrThrow(op, parent); return ListGenericTablesResponse.builder() - .setIdentifiers(new TreeSet<>(genericTableCatalog.listGenericTables(parent))) + .setIdentifiers(new LinkedHashSet<>(genericTableCatalog.listGenericTables(parent))) .build(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index 9bab20586..8d7b49a07 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -30,8 +30,6 @@ import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; -import java.net.URLEncoder; -import java.nio.charset.Charset; import java.util.EnumSet; import java.util.Map; import java.util.Optional; @@ -72,6 +70,7 @@ import org.apache.polaris.service.catalog.AccessDelegationMode; import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.api.IcebergRestCatalogApiService; import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApiService; +import org.apache.polaris.service.catalog.common.CatalogAdapter; import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.http.IcebergHttpUtil; import org.apache.polaris.service.http.IfNoneMatch; @@ -88,7 +87,7 @@ import org.slf4j.LoggerFactory; */ @RequestScoped public class IcebergCatalogAdapter - implements IcebergRestCatalogApiService, IcebergRestConfigurationApiService { + implements IcebergRestCatalogApiService, IcebergRestConfigurationApiService, CatalogAdapter { private static final Logger LOGGER = LoggerFactory.getLogger(IcebergCatalogAdapter.class); @@ -215,8 +214,7 @@ public class IcebergCatalogAdapter String parent, RealmContext realmContext, SecurityContext securityContext) { - Optional<Namespace> namespaceOptional = - Optional.ofNullable(parent).map(IcebergCatalogAdapter::decodeNamespace); + Optional<Namespace> namespaceOptional = Optional.ofNullable(parent).map(this::decodeNamespace); return withCatalog( securityContext, prefix, @@ -232,10 +230,6 @@ public class IcebergCatalogAdapter securityContext, prefix, catalog -> Response.ok(catalog.loadNamespaceMetadata(ns)).build()); } - private static Namespace decodeNamespace(String namespace) { - return RESTUtil.decodeNamespace(URLEncoder.encode(namespace, Charset.defaultCharset())); - } - /** * For situations where we typically expect a metadataLocation to be present in the response and * so expect to insert an etag header, this helper gracefully falls back to omitting the header if