This is an automated email from the ASF dual-hosted git repository.
emaynard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 9d663b2b9 Refactor GenericTableCatalog to support generic table
federation (#1579)
9d663b2b9 is described below
commit 9d663b2b9cd74a82ef50ed160879082bc11e9a05
Author: Eric Maynard <eric.maynard+...@snowflake.com>
AuthorDate: Wed May 14 10:36:53 2025 -0700
Refactor GenericTableCatalog to support generic table federation (#1579)
* initial commit
* stable
* javadocs
* autolint
* changes per review
---
.../quarkus/admin/PolarisAuthzTestBase.java | 7 +-
...olarisGenericTableCatalogHandlerAuthzTest.java} | 2 +-
...st.java => PolarisGenericTableCatalogTest.java} | 11 +-
.../catalog/generic/GenericTableCatalog.java | 165 ++-------------------
.../generic/GenericTableCatalogHandler.java | 3 +-
...atalog.java => PolarisGenericTableCatalog.java} | 20 ++-
6 files changed, 43 insertions(+), 165 deletions(-)
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java
index b5ef68a6b..37dcc04e5 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java
@@ -80,7 +80,7 @@ import org.apache.polaris.core.secrets.UserSecretsManager;
import org.apache.polaris.core.secrets.UserSecretsManagerFactory;
import org.apache.polaris.service.admin.PolarisAdminService;
import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
-import org.apache.polaris.service.catalog.generic.GenericTableCatalog;
+import org.apache.polaris.service.catalog.generic.PolarisGenericTableCatalog;
import org.apache.polaris.service.catalog.iceberg.IcebergCatalog;
import org.apache.polaris.service.catalog.io.FileIOFactory;
import org.apache.polaris.service.catalog.policy.PolicyCatalog;
@@ -194,7 +194,7 @@ public abstract class PolarisAuthzTestBase {
@Inject protected PolarisEventListener polarisEventListener;
protected IcebergCatalog baseCatalog;
- protected GenericTableCatalog genericTableCatalog;
+ protected PolarisGenericTableCatalog genericTableCatalog;
protected PolicyCatalog policyCatalog;
protected PolarisAdminService adminService;
protected PolarisEntityManager entityManager;
@@ -481,7 +481,8 @@ public abstract class PolarisAuthzTestBase {
ImmutableMap.of(
CatalogProperties.FILE_IO_IMPL,
"org.apache.iceberg.inmemory.InMemoryFileIO"));
this.genericTableCatalog =
- new GenericTableCatalog(metaStoreManager, callContext,
passthroughView);
+ new PolarisGenericTableCatalog(metaStoreManager, callContext,
passthroughView);
+ this.genericTableCatalog.initialize(CATALOG_NAME, ImmutableMap.of());
this.policyCatalog = new PolicyCatalog(metaStoreManager, callContext,
passthroughView);
}
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogHandlerAuthzTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogHandlerAuthzTest.java
similarity index 99%
rename from
quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogHandlerAuthzTest.java
rename to
quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogHandlerAuthzTest.java
index fb3114e69..0f9fd3190 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogHandlerAuthzTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogHandlerAuthzTest.java
@@ -31,7 +31,7 @@ import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.Test;
@QuarkusTest
-public class GenericTableCatalogHandlerAuthzTest extends PolarisAuthzTestBase {
+public class PolarisGenericTableCatalogHandlerAuthzTest extends
PolarisAuthzTestBase {
private GenericTableCatalogHandler newWrapper() {
return newWrapper(Set.of());
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java
similarity index 98%
rename from
quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java
rename to
quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java
index 04c84c5f6..5f8c2d028 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java
@@ -76,7 +76,7 @@ import
org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo;
import org.apache.polaris.core.storage.cache.StorageCredentialCache;
import org.apache.polaris.service.admin.PolarisAdminService;
import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
-import org.apache.polaris.service.catalog.generic.GenericTableCatalog;
+import org.apache.polaris.service.catalog.generic.PolarisGenericTableCatalog;
import org.apache.polaris.service.catalog.iceberg.IcebergCatalog;
import org.apache.polaris.service.catalog.io.DefaultFileIOFactory;
import org.apache.polaris.service.catalog.io.FileIOFactory;
@@ -98,8 +98,8 @@ import
software.amazon.awssdk.services.sts.model.AssumeRoleResponse;
import software.amazon.awssdk.services.sts.model.Credentials;
@QuarkusTest
-@TestProfile(GenericTableCatalogTest.Profile.class)
-public class GenericTableCatalogTest {
+@TestProfile(PolarisGenericTableCatalogTest.Profile.class)
+public class PolarisGenericTableCatalogTest {
public static class Profile implements QuarkusTestProfile {
@@ -128,7 +128,7 @@ public class GenericTableCatalogTest {
@Inject PolarisStorageIntegrationProvider storageIntegrationProvider;
@Inject PolarisDiagnostics diagServices;
- private GenericTableCatalog genericTableCatalog;
+ private PolarisGenericTableCatalog genericTableCatalog;
private IcebergCatalog icebergCatalog;
private CallContext callContext;
private AwsStorageConfigInfo storageConfigModel;
@@ -262,7 +262,8 @@ public class GenericTableCatalogTest {
.thenReturn((PolarisStorageIntegration) storageIntegration);
this.genericTableCatalog =
- new GenericTableCatalog(metaStoreManager, callContext,
passthroughView);
+ new PolarisGenericTableCatalog(metaStoreManager, callContext,
passthroughView);
+ this.genericTableCatalog.initialize(CATALOG_NAME, Map.of());
this.icebergCatalog =
new IcebergCatalog(
entityManager,
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java
index 79adeaee8..197be3645 100644
---
a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java
@@ -22,163 +22,24 @@ import java.util.List;
import java.util.Map;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
-import org.apache.iceberg.exceptions.AlreadyExistsException;
-import org.apache.iceberg.exceptions.NoSuchNamespaceException;
-import org.apache.iceberg.exceptions.NoSuchTableException;
-import org.apache.polaris.core.catalog.PolarisCatalogHelpers;
-import org.apache.polaris.core.context.CallContext;
-import org.apache.polaris.core.entity.CatalogEntity;
-import org.apache.polaris.core.entity.PolarisEntity;
-import org.apache.polaris.core.entity.PolarisEntitySubType;
-import org.apache.polaris.core.entity.PolarisEntityType;
import org.apache.polaris.core.entity.table.GenericTableEntity;
-import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
-import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
-import org.apache.polaris.core.persistence.dao.entity.BaseResult;
-import org.apache.polaris.core.persistence.dao.entity.DropEntityResult;
-import org.apache.polaris.core.persistence.dao.entity.EntityResult;
-import org.apache.polaris.core.persistence.pagination.PageToken;
-import
org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-public class GenericTableCatalog {
- private static final Logger LOGGER =
LoggerFactory.getLogger(GenericTableCatalog.class);
+/** A catalog for managing `GenericTableEntity` instances */
+public interface GenericTableCatalog {
- private final CallContext callContext;
- private final PolarisResolutionManifestCatalogView resolvedEntityView;
- private final CatalogEntity catalogEntity;
- private long catalogId = -1;
- private PolarisMetaStoreManager metaStoreManager;
+ /** Should be called before other methods */
+ void initialize(String name, Map<String, String> properties);
- public GenericTableCatalog(
- PolarisMetaStoreManager metaStoreManager,
- CallContext callContext,
- PolarisResolutionManifestCatalogView resolvedEntityView) {
- this.callContext = callContext;
- this.resolvedEntityView = resolvedEntityView;
- this.catalogEntity =
-
CatalogEntity.of(resolvedEntityView.getResolvedReferenceCatalogEntity().getRawLeafEntity());
- this.catalogId = catalogEntity.getId();
- this.metaStoreManager = metaStoreManager;
- }
+ /** Create a generic table with the specified identifier */
+ GenericTableEntity createGenericTable(
+ TableIdentifier tableIdentifier, String format, String doc, Map<String,
String> properties);
- public GenericTableEntity createGenericTable(
- TableIdentifier tableIdentifier, String format, String doc, Map<String,
String> properties) {
- PolarisResolvedPathWrapper resolvedParent =
- resolvedEntityView.getResolvedPath(tableIdentifier.namespace());
- if (resolvedParent == null) {
- // Illegal state because the namespace should've already been in the
static resolution set.
- throw new IllegalStateException(
- String.format(
- "Failed to fetch resolved parent for TableIdentifier '%s'",
tableIdentifier));
- }
+ /** Retrieve a generic table entity with a given identifier */
+ GenericTableEntity loadGenericTable(TableIdentifier tableIdentifier);
- List<PolarisEntity> catalogPath = resolvedParent.getRawFullPath();
+ /** Drop a generic table entity with a given identifier */
+ boolean dropGenericTable(TableIdentifier tableIdentifier);
- PolarisResolvedPathWrapper resolvedEntities =
- resolvedEntityView.getPassthroughResolvedPath(
- tableIdentifier, PolarisEntityType.TABLE_LIKE,
PolarisEntitySubType.ANY_SUBTYPE);
- GenericTableEntity entity =
- GenericTableEntity.of(
- resolvedEntities == null ? null :
resolvedEntities.getRawLeafEntity());
- if (null == entity) {
- entity =
- new GenericTableEntity.Builder(tableIdentifier, format)
- .setCatalogId(this.catalogId)
- .setParentNamespace(tableIdentifier.namespace())
- .setParentId(resolvedParent.getRawLeafEntity().getId())
- .setId(
- this.metaStoreManager
-
.generateNewEntityId(this.callContext.getPolarisCallContext())
- .getId())
- .setProperties(properties)
- .setDoc(doc)
- .setCreateTimestamp(System.currentTimeMillis())
- .build();
- } else {
- throw new AlreadyExistsException(
- "Iceberg table, view, or generic table already exists: %s",
tableIdentifier);
- }
-
- EntityResult res =
- this.metaStoreManager.createEntityIfNotExists(
- this.callContext.getPolarisCallContext(),
- PolarisEntity.toCoreList(catalogPath),
- entity);
- if (!res.isSuccess()) {
- switch (res.getReturnStatus()) {
- case BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS:
- throw new AlreadyExistsException(
- "Iceberg table, view, or generic table already exists: %s",
tableIdentifier);
-
- default:
- throw new IllegalStateException(
- String.format(
- "Unknown error status for identifier %s: %s with extraInfo:
%s",
- tableIdentifier, res.getReturnStatus(),
res.getExtraInformation()));
- }
- }
- GenericTableEntity resultEntity = GenericTableEntity.of(res.getEntity());
- LOGGER.debug(
- "Created GenericTable entity {} with TableIdentifier {}",
resultEntity, tableIdentifier);
- return resultEntity;
- }
-
- public GenericTableEntity loadGenericTable(TableIdentifier tableIdentifier) {
- PolarisResolvedPathWrapper resolvedEntities =
- resolvedEntityView.getPassthroughResolvedPath(
- tableIdentifier, PolarisEntityType.TABLE_LIKE,
PolarisEntitySubType.GENERIC_TABLE);
- GenericTableEntity entity =
- GenericTableEntity.of(
- resolvedEntities == null ? null :
resolvedEntities.getRawLeafEntity());
- if (null == entity) {
- throw new NoSuchTableException("Generic table does not exist: %s",
tableIdentifier);
- } else {
- return entity;
- }
- }
-
- public boolean dropGenericTable(TableIdentifier tableIdentifier) {
- PolarisResolvedPathWrapper resolvedEntities =
- resolvedEntityView.getPassthroughResolvedPath(
- tableIdentifier, PolarisEntityType.TABLE_LIKE,
PolarisEntitySubType.GENERIC_TABLE);
-
- if (resolvedEntities == null) {
- throw new NoSuchTableException("Generic table does not exist: %s",
tableIdentifier);
- }
-
- List<PolarisEntity> catalogPath = resolvedEntities.getRawParentPath();
- PolarisEntity leafEntity = resolvedEntities.getRawLeafEntity();
-
- DropEntityResult dropEntityResult =
- this.metaStoreManager.dropEntityIfExists(
- this.callContext.getPolarisCallContext(),
- PolarisEntity.toCoreList(catalogPath),
- leafEntity,
- Map.of(),
- false);
-
- return dropEntityResult.isSuccess();
- }
-
- public List<TableIdentifier> listGenericTables(Namespace namespace) {
- PolarisResolvedPathWrapper resolvedEntities =
resolvedEntityView.getResolvedPath(namespace);
- if (resolvedEntities == null) {
- throw new NoSuchNamespaceException("Namespace '%s' does not exist",
namespace);
- }
-
- List<PolarisEntity> catalogPath = resolvedEntities.getRawFullPath();
- List<PolarisEntity.NameAndId> entities =
- PolarisEntity.toNameAndIdList(
- this.metaStoreManager
- .listEntities(
- this.callContext.getPolarisCallContext(),
- PolarisEntity.toCoreList(catalogPath),
- PolarisEntityType.TABLE_LIKE,
- PolarisEntitySubType.GENERIC_TABLE,
- PageToken.readEverything())
- .getEntities());
- return PolarisCatalogHelpers.nameAndIdToTableIdentifiers(catalogPath,
entities);
- }
+ /** List all generic tables under a specific namespace */
+ List<TableIdentifier> listGenericTables(Namespace namespace);
}
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java
index 126023c2b..66a2b81d1 100644
---
a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java
@@ -55,7 +55,8 @@ public class GenericTableCatalogHandler extends
CatalogHandler {
@Override
protected void initializeCatalog() {
this.genericTableCatalog =
- new GenericTableCatalog(metaStoreManager, callContext,
this.resolutionManifest);
+ new PolarisGenericTableCatalog(metaStoreManager, callContext,
this.resolutionManifest);
+ this.genericTableCatalog.initialize(catalogName, Map.of());
}
public ListGenericTablesResponse listGenericTables(Namespace parent) {
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalog.java
similarity index 93%
copy from
service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java
copy to
service/common/src/main/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalog.java
index 79adeaee8..2b884e787 100644
---
a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalog.java
@@ -42,8 +42,10 @@ import
org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCat
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class GenericTableCatalog {
- private static final Logger LOGGER =
LoggerFactory.getLogger(GenericTableCatalog.class);
+public class PolarisGenericTableCatalog implements GenericTableCatalog {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(PolarisGenericTableCatalog.class);
+
+ private String name;
private final CallContext callContext;
private final PolarisResolutionManifestCatalogView resolvedEntityView;
@@ -51,7 +53,7 @@ public class GenericTableCatalog {
private long catalogId = -1;
private PolarisMetaStoreManager metaStoreManager;
- public GenericTableCatalog(
+ public PolarisGenericTableCatalog(
PolarisMetaStoreManager metaStoreManager,
CallContext callContext,
PolarisResolutionManifestCatalogView resolvedEntityView) {
@@ -63,6 +65,15 @@ public class GenericTableCatalog {
this.metaStoreManager = metaStoreManager;
}
+ @Override
+ public void initialize(String name, Map<String, String> properties) {
+ this.name = name;
+ if (!properties.isEmpty()) {
+ throw new IllegalStateException("PolarisGenericTableCatalog does not
support properties");
+ }
+ }
+
+ @Override
public GenericTableEntity createGenericTable(
TableIdentifier tableIdentifier, String format, String doc, Map<String,
String> properties) {
PolarisResolvedPathWrapper resolvedParent =
@@ -125,6 +136,7 @@ public class GenericTableCatalog {
return resultEntity;
}
+ @Override
public GenericTableEntity loadGenericTable(TableIdentifier tableIdentifier) {
PolarisResolvedPathWrapper resolvedEntities =
resolvedEntityView.getPassthroughResolvedPath(
@@ -139,6 +151,7 @@ public class GenericTableCatalog {
}
}
+ @Override
public boolean dropGenericTable(TableIdentifier tableIdentifier) {
PolarisResolvedPathWrapper resolvedEntities =
resolvedEntityView.getPassthroughResolvedPath(
@@ -162,6 +175,7 @@ public class GenericTableCatalog {
return dropEntityResult.isSuccess();
}
+ @Override
public List<TableIdentifier> listGenericTables(Namespace namespace) {
PolarisResolvedPathWrapper resolvedEntities =
resolvedEntityView.getResolvedPath(namespace);
if (resolvedEntities == null) {
