snazy commented on code in PR #1070: URL: https://github.com/apache/polaris/pull/1070#discussion_r1973986682
########## polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalPersistence.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence.transactional; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Supplier; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.persistence.BasePersistence; +import org.apache.polaris.core.persistence.IntegrationPersistence; + +/** + * Extends BasePersistence to express a more "transaction-oriented" control flow for backing stores + * which can support a runInTransaction semantic, while providing default implementations of some of + * the BasePersistence methods in terms of lower-level methods that subclasses must implement. + */ +public abstract class TransactionalPersistence implements BasePersistence, IntegrationPersistence { Review Comment: The common denominator for atomic operations in non-transactional databases is a CAS on a single row/key. Like "fetch all things", then "check all requirements", then "persist all the changes" and eventually "perform a single CAS" to perform the atomic change. How would this be implemented for example for a "rename table" operation or a "multi table commit" involving all the changes to the individual entities and consistency guarantees? ########## polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java: ########## @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntityActiveRecord; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageIntegration; + +/** + * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like + * metadata for Polaris entities and metadata about grants between these entities which is the + * foundation of our role base access control model. + * + * <p>Note that APIs to the actual persistence store are very basic, often point read or write to + * the underlying data store. The goal is to make it really easy to back this using databases like + * Postgres or simpler KV store. + */ +public interface BasePersistence { + /** + * @param callCtx call context + * @return new unique entity identifier + */ + long generateNewId(@Nonnull PolarisCallContext callCtx); + + /** + * Write this entity to the meta store. + * + * @param callCtx call context + * @param entity entity to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntity original state of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity + */ + void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity); + + /** + * Write the specified grantRecord to the grant_records table. If there is a conflict (existing + * record with the same PK), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param grantRec entity record to write, potentially replacing an existing entity record with + * the same key + */ + void writeToGrantRecords( Review Comment: I foresee the need to have pluggable authZ. It doesn't need to be necessarily _this specific_ RBAC. It could be ABAC, another RBAC or even "noop". WDYT about making the current RBAC objects just entities? ########## polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java: ########## @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntityActiveRecord; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; + +/** + * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like + * metadata for Polaris entities and metadata about grants between these entities which is the + * foundation of our role base access control model. + * + * <p>Note that APIs to the actual persistence store are very basic, often point read or write to + * the underlying data store. The goal is to make it really easy to back this using databases like + * Postgres or simpler KV store. + */ +public interface BasePersistence { + /** + * @param callCtx call context + * @return new unique entity identifier + */ + long generateNewId(@Nonnull PolarisCallContext callCtx); + + /** + * Write this entity to the meta store. + * + * @param callCtx call context + * @param entity entity to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntity original state of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity + */ + void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity); + + /** + * Write the specified grantRecord to the grant_records table. If there is a conflict (existing + * record with the same PK), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param grantRec entity record to write, potentially replacing an existing entity record with + * the same key + */ + void writeToGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete this entity from the meta store. + * + * @param callCtx call context + * @param entity entity to delete + */ + void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + + /** + * Delete the specified grantRecord to the grant_records table. + * + * @param callCtx call context + * @param grantRec entity record to delete. + */ + void deleteFromGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete the all grant records in the grant_records table for the specified entity. This method + * will delete all grant records on that securable entity and also all grants to that grantee + * entity assuming that the entity is a grantee (catalog role, principal role or principal). + * + * @param callCtx call context + * @param entity entity whose grant records to and from should be deleted + * @param grantsOnGrantee all grants to that grantee entity. Empty list if that entity is not a + * grantee + * @param grantsOnSecurable all grants on that securable entity + */ + void deleteAllEntityGrantRecords( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore entity, + @Nonnull List<PolarisGrantRecord> grantsOnGrantee, + @Nonnull List<PolarisGrantRecord> grantsOnSecurable); + + /** + * Delete Polaris entity and grant record metadata from all tables. This is used during metadata + * bootstrap to reset all tables to their original state + * + * @param callCtx call context + */ + void deleteAll(@Nonnull PolarisCallContext callCtx); Review Comment: Practically we need two very different "purge realm" operations. "Prefix key deletion" and similar isn't supported by all databases. For databases that don't support "prefix key deletion" we'd have to scan the database and delete matching rows. For database support that, deleting a whole realm can be very time consuming (think: transaction timeouts, aborted requests, etc). What I'm thinking of is realm-state management: created (populate all the things) -> active (usable) -> inactive (just disabled) -> purging (realm can be purged) -> purged. This would give us some flexibility and I think better guarantees. When a realm's created, a bunch of things need to happen - with pluggable authZ for example even things we don't know - the "realm initial population" can be performed by the things that "core Polaris" needs plus things that plugins need. To delete and eventually purge a realm, it should be set to "inactive" first to prevent having intermediate dirty states and requests running against a realm that's being purged (or marked as such). A regularly run maintenance job could then do the actual cleanup. WDYT? ########## polaris-core/src/main/java/org/apache/polaris/core/persistence/IntegrationPersistence.java: ########## @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageIntegration; + +/** + * Interface for the necessary "peripheral integration" objects that are logically attached to core + * persistence entities but which typically involve additional separate external integrations + * related to identity/auth, kms/secrets storage, etc. + * + * <p>Implementations should orchestrate any necessary multi-phase protocols such as leasing an + * external resource before committing a reference to the external resource in the Polaris + * persistence layer, etc. + */ +public interface IntegrationPersistence { + /** + * Allows to retrieve to the secrets of a principal given its unique client id + * + * @param callCtx call context + * @param clientId principal client id + * @return the secrets + */ + @Nullable + PolarisPrincipalSecrets loadPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String clientId); + + /** + * generate and store a client id and associated secrets for a newly created principal entity + * + * @param callCtx call context + * @param principalName name of the principal + * @param principalId principal id + */ + @Nonnull + PolarisPrincipalSecrets generateNewPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String principalName, long principalId); + + /** + * Rotate the secrets of a principal entity, i.e. make the specified main secrets the secondary + * and generate a new main secret + * + * @param callCtx call context + * @param clientId principal client id + * @param principalId principal id + * @param reset true if the principal secrets should be disabled and replaced with a one-time + * password + * @param oldSecretHash the principal secret's old main secret hash + */ + @Nullable + PolarisPrincipalSecrets rotatePrincipalSecrets( Review Comment: Need to think about external IdPs as well. ########## polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java: ########## @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; + +/** + * Interface to the Polaris persistence backend, with which to persist and retrieve all the data + * defining the internal data model for Polaris, and which defines the basis for the RBAC model + * provided by Polaris. + * + * <p>Note that APIs to the actual persistence store are very basic, often point read or write to + * the underlying data store. The goal is to make it really easy to back this using databases like + * Postgres or simpler KV store. + */ +public interface BasePersistence { + /** + * The returned id must be fully unique within a realm and never reused once generated, whether or + * not anything ends up committing an entity with the generated id. + * + * @param callCtx call context + * @return new unique entity identifier + */ + long generateNewId(@Nonnull PolarisCallContext callCtx); + + /** + * Write this entity to the persistence backend. If successful, the write must be durable and + * visible to any other reader. + * + * <p>TODO: Either standardize the expected system of exceptions to throw for various concurrency + * errors (entity not found when originalEntity != null, entity changed from originalEntity, etc) + * or push down the return status enums from PolarisMetaStoreManager into this layer and document + * accordingly. + * + * @param callCtx call context + * @param entity entity to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntity original state of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity + */ + void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity); + + /** + * Atomically write a batch of entities to the persistence backend conditional on *every* member + * of originalEntities matching the existing persistent state. After this commit, *every* member + * of entities must be committed durably. + * + * <p>TODO: Push down the multi-entity commit from PolarisMetaStoreManagerImpl to use this instead + * of running single writeEntity actions within a transaction. + * + * @param callCtx call context + * @param entities entities to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntities original states of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity; must contain exactly as many elements as + * {@code entities} where each item corresponds to the element of {@code entities} in the same + * index as this list. + */ + void writeEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull List<PolarisBaseEntity> entities, + @Nullable List<PolarisBaseEntity> originalEntities); + + /** + * Write the specified grantRecord to the grant_records table. If there is a conflict (existing + * record with the same PK), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param grantRec entity record to write, potentially replacing an existing entity record with + * the same key + */ + void writeToGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete this entity from the meta store. + * + * @param callCtx call context + * @param entity entity to delete + */ + void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + + /** + * Delete the specified grantRecord to the grant_records table. + * + * @param callCtx call context + * @param grantRec entity record to delete. + */ + void deleteFromGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete the all grant records in the grant_records table for the specified entity. This method + * will delete all grant records on that securable entity and also all grants to that grantee + * entity assuming that the entity is a grantee (catalog role, principal role or principal). + * + * @param callCtx call context + * @param entity entity whose grant records to and from should be deleted + * @param grantsOnGrantee all grants to that grantee entity. Empty list if that entity is not a + * grantee + * @param grantsOnSecurable all grants on that securable entity + */ + void deleteAllEntityGrantRecords( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore entity, + @Nonnull List<PolarisGrantRecord> grantsOnGrantee, + @Nonnull List<PolarisGrantRecord> grantsOnSecurable); + + /** + * Delete Polaris entity and grant record metadata from all tables within the realm defined by the + * contents of the {@code callCtx} + * + * @param callCtx call context + */ + void deleteAll(@Nonnull PolarisCallContext callCtx); + + /** + * Lookup an entity given its catalog id (which can be {@link + * org.apache.polaris.core.entity.PolarisEntityConstants#NULL_ID} for top-level entities) and its + * entityId. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId entity id + * @return null if the entity was not found, else the retrieved entity. + */ + @Nullable + PolarisBaseEntity lookupEntity( + @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + + /** + * Lookup an entity given its catalogId, parentId, typeCode, and name. + * + * @param callCtx call context + * @param catalogId catalog id or {@link + * org.apache.polaris.core.entity.PolarisEntityConstants#NULL_ID} for top-level entities like + * CATALOG, PRINCIPAL and PRINCIPAL_ROLE. Note that by convention, a catalog itself has + * NULL_ID for its catalogId since the catalog is not "nested" under itself or any other + * catalog. + * @param parentId id of the parent + * @param typeCode the PolarisEntityType code of the entity to lookup + * @param name the name of the entity + * @return null if the specified entity does not exist + */ + @Nullable + PolarisBaseEntity lookupEntityByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name); Review Comment: The (most dominant) caller of this is when paths within a catalog are resolved. We already have all "exploded paths" available at the call size, so I guess it's more ergonomic to delegate the resolution of these "exploded paths" to something that specialized for that. The nice side effect would be that call sites wouldn't have to deal with parent-child relationships but only with paths. ########## polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java: ########## @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntityActiveRecord; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageIntegration; + +/** + * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like + * metadata for Polaris entities and metadata about grants between these entities which is the + * foundation of our role base access control model. + * + * <p>Note that APIs to the actual persistence store are very basic, often point read or write to + * the underlying data store. The goal is to make it really easy to back this using databases like + * Postgres or simpler KV store. + */ +public interface BasePersistence { + /** + * @param callCtx call context + * @return new unique entity identifier + */ + long generateNewId(@Nonnull PolarisCallContext callCtx); + + /** + * Write this entity to the meta store. + * + * @param callCtx call context + * @param entity entity to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntity original state of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity + */ + void writeEntity( + @Nonnull PolarisCallContext callCtx, Review Comment: Regarding the parameter, yea - I also think, it's not necessary. If an implementation needs it, it could take it as a constructor argument. ########## polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java: ########## @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntityActiveRecord; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; + +/** + * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like + * metadata for Polaris entities and metadata about grants between these entities which is the + * foundation of our role base access control model. + * + * <p>Note that APIs to the actual persistence store are very basic, often point read or write to + * the underlying data store. The goal is to make it really easy to back this using databases like + * Postgres or simpler KV store. + */ +public interface BasePersistence { + /** + * @param callCtx call context + * @return new unique entity identifier + */ + long generateNewId(@Nonnull PolarisCallContext callCtx); + + /** + * Write this entity to the meta store. + * + * @param callCtx call context + * @param entity entity to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntity original state of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity + */ + void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity); + + /** + * Write the specified grantRecord to the grant_records table. If there is a conflict (existing + * record with the same PK), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param grantRec entity record to write, potentially replacing an existing entity record with + * the same key + */ + void writeToGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete this entity from the meta store. + * + * @param callCtx call context + * @param entity entity to delete + */ + void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + + /** + * Delete the specified grantRecord to the grant_records table. + * + * @param callCtx call context + * @param grantRec entity record to delete. + */ + void deleteFromGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete the all grant records in the grant_records table for the specified entity. This method + * will delete all grant records on that securable entity and also all grants to that grantee + * entity assuming that the entity is a grantee (catalog role, principal role or principal). + * + * @param callCtx call context + * @param entity entity whose grant records to and from should be deleted + * @param grantsOnGrantee all grants to that grantee entity. Empty list if that entity is not a + * grantee + * @param grantsOnSecurable all grants on that securable entity + */ + void deleteAllEntityGrantRecords( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore entity, + @Nonnull List<PolarisGrantRecord> grantsOnGrantee, + @Nonnull List<PolarisGrantRecord> grantsOnSecurable); + + /** + * Delete Polaris entity and grant record metadata from all tables. This is used during metadata + * bootstrap to reset all tables to their original state + * + * @param callCtx call context + */ + void deleteAll(@Nonnull PolarisCallContext callCtx); + + /** + * Lookup an entity given its catalog id (which can be NULL_ID for top-level entities) and its + * unique id. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId unique entity id + * @return NULL if the entity was not found, else the base entity. + */ + @Nullable + PolarisBaseEntity lookupEntity( + @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + + /** + * Lookup an entity given its catalogId, parentId, typeCode, and name. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param parentId id of the parent, either a namespace or a catalog + * @param typeCode the PolarisEntityType code of the entity to lookup + * @param name the name of the entity + * @return null if the specified entity does not exist + */ + @Nullable + PolarisBaseEntity lookupEntityByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, Review Comment: Type-codes & entity-(sub)-types: It feels like we need more "levels" than just entity-type+sub-type. Concrete: we have Iceberg tables and Iceberg views - but also generic table/view/function. I think we need a way to distinguish Iceberg entities from "generic tables" - but all are "table-like". I think, we can get away with a single "entity type" and use Java's type inheritance to be more flexible, if we associate every "entity type" with a concrete Java type. With that, we could also do the type-checks in the persistence layer and removing quite some checks at the call sites. For example, there could be a Java type hierarchy like this: ``` interface Tabular interface Entity interface ContentEntity extends Entity interface Namespace extends ContentEntity interface LocalNamespace extends Namespace -> type-name "namespace-local" interface RemoteNamespace extends Namespace -> type-name "namespace-remote" interface IcebergContentEntity extends ContentEntity interface IcebergTableEntity extends IcebergContentEntity, Tabular -> type-name iceberg-table interface IcebergViewEntity extends IcebergContentEntity, Tabular -> type-name iceberg-view interface GenericContentEntity extends ContentEntity interface GenericTableEntity extends GenericContentEntity, Tabular -> type-name generic-table interface GenericFooEntity extends GenericContentEntity -> type-name generic-foo ``` From the Iceberg REST API, you'd use types that are IcebergContentEntity. For "generic tables", you'd use types that are GenericContentEntity. But all those would be ContentEntity's, which are part of a catalog. Fetching an entity could be done via some function ``` <T> T fetchEntity(EntityType entityType, long id, Class<T> clazz); // clazz only needed for "proper" Java generics ``` If that `EntityType` is a plain Java interface, we could also support pluggable entity types. WDYT? ########## polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java: ########## @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntityActiveRecord; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageIntegration; + +/** + * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like + * metadata for Polaris entities and metadata about grants between these entities which is the + * foundation of our role base access control model. + * + * <p>Note that APIs to the actual persistence store are very basic, often point read or write to + * the underlying data store. The goal is to make it really easy to back this using databases like + * Postgres or simpler KV store. + */ +public interface BasePersistence { + /** + * @param callCtx call context + * @return new unique entity identifier + */ + long generateNewId(@Nonnull PolarisCallContext callCtx); + + /** + * Write this entity to the meta store. + * + * @param callCtx call context + * @param entity entity to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntity original state of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity + */ + void writeEntity( + @Nonnull PolarisCallContext callCtx, Review Comment: Regarding tasks in general and the concrete use case of "purging tables", all that's eventually needed are some parameters. For "purge table" that's an object describing the table and some Iceberg parameters - the actual "purge" itself doesn't need persistence at all. What we eventually need is a robust tasks framework (#774) - what individual task "runnables" need and how they work is up to those. _If_ those need some persistence interaction, we can always have factories. ########## polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java: ########## @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntityActiveRecord; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; + +/** + * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like + * metadata for Polaris entities and metadata about grants between these entities which is the + * foundation of our role base access control model. + * + * <p>Note that APIs to the actual persistence store are very basic, often point read or write to + * the underlying data store. The goal is to make it really easy to back this using databases like + * Postgres or simpler KV store. + */ +public interface BasePersistence { + /** + * @param callCtx call context + * @return new unique entity identifier + */ + long generateNewId(@Nonnull PolarisCallContext callCtx); + + /** + * Write this entity to the meta store. + * + * @param callCtx call context + * @param entity entity to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntity original state of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity + */ + void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity); + + /** + * Write the specified grantRecord to the grant_records table. If there is a conflict (existing + * record with the same PK), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param grantRec entity record to write, potentially replacing an existing entity record with + * the same key + */ + void writeToGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete this entity from the meta store. + * + * @param callCtx call context + * @param entity entity to delete + */ + void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + + /** + * Delete the specified grantRecord to the grant_records table. + * + * @param callCtx call context + * @param grantRec entity record to delete. + */ + void deleteFromGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete the all grant records in the grant_records table for the specified entity. This method + * will delete all grant records on that securable entity and also all grants to that grantee + * entity assuming that the entity is a grantee (catalog role, principal role or principal). + * + * @param callCtx call context + * @param entity entity whose grant records to and from should be deleted + * @param grantsOnGrantee all grants to that grantee entity. Empty list if that entity is not a + * grantee + * @param grantsOnSecurable all grants on that securable entity + */ + void deleteAllEntityGrantRecords( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore entity, + @Nonnull List<PolarisGrantRecord> grantsOnGrantee, + @Nonnull List<PolarisGrantRecord> grantsOnSecurable); + + /** + * Delete Polaris entity and grant record metadata from all tables. This is used during metadata + * bootstrap to reset all tables to their original state + * + * @param callCtx call context + */ + void deleteAll(@Nonnull PolarisCallContext callCtx); + + /** + * Lookup an entity given its catalog id (which can be NULL_ID for top-level entities) and its + * unique id. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId unique entity id + * @return NULL if the entity was not found, else the base entity. + */ + @Nullable + PolarisBaseEntity lookupEntity( + @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); Review Comment: But isn't "catalog" a higher level use case by itself? ########## polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java: ########## @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; + +/** + * Interface to the Polaris persistence backend, with which to persist and retrieve all the data + * defining the internal data model for Polaris, and which defines the basis for the RBAC model + * provided by Polaris. + * + * <p>Note that APIs to the actual persistence store are very basic, often point read or write to + * the underlying data store. The goal is to make it really easy to back this using databases like + * Postgres or simpler KV store. + */ +public interface BasePersistence { + /** + * The returned id must be fully unique within a realm and never reused once generated, whether or + * not anything ends up committing an entity with the generated id. + * + * @param callCtx call context + * @return new unique entity identifier + */ + long generateNewId(@Nonnull PolarisCallContext callCtx); + + /** + * Write this entity to the persistence backend. If successful, the write must be durable and + * visible to any other reader. + * + * <p>TODO: Either standardize the expected system of exceptions to throw for various concurrency + * errors (entity not found when originalEntity != null, entity changed from originalEntity, etc) + * or push down the return status enums from PolarisMetaStoreManager into this layer and document + * accordingly. + * + * @param callCtx call context + * @param entity entity to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntity original state of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity + */ + void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity); + + /** + * Atomically write a batch of entities to the persistence backend conditional on *every* member + * of originalEntities matching the existing persistent state. After this commit, *every* member + * of entities must be committed durably. + * + * <p>TODO: Push down the multi-entity commit from PolarisMetaStoreManagerImpl to use this instead + * of running single writeEntity actions within a transaction. + * + * @param callCtx call context + * @param entities entities to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntities original states of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity; must contain exactly as many elements as + * {@code entities} where each item corresponds to the element of {@code entities} in the same + * index as this list. + */ + void writeEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull List<PolarisBaseEntity> entities, + @Nullable List<PolarisBaseEntity> originalEntities); + + /** + * Write the specified grantRecord to the grant_records table. If there is a conflict (existing + * record with the same PK), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param grantRec entity record to write, potentially replacing an existing entity record with + * the same key + */ + void writeToGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete this entity from the meta store. + * + * @param callCtx call context + * @param entity entity to delete + */ + void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + + /** + * Delete the specified grantRecord to the grant_records table. + * + * @param callCtx call context + * @param grantRec entity record to delete. + */ + void deleteFromGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete the all grant records in the grant_records table for the specified entity. This method + * will delete all grant records on that securable entity and also all grants to that grantee + * entity assuming that the entity is a grantee (catalog role, principal role or principal). + * + * @param callCtx call context + * @param entity entity whose grant records to and from should be deleted + * @param grantsOnGrantee all grants to that grantee entity. Empty list if that entity is not a + * grantee + * @param grantsOnSecurable all grants on that securable entity + */ + void deleteAllEntityGrantRecords( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore entity, + @Nonnull List<PolarisGrantRecord> grantsOnGrantee, + @Nonnull List<PolarisGrantRecord> grantsOnSecurable); + + /** + * Delete Polaris entity and grant record metadata from all tables within the realm defined by the + * contents of the {@code callCtx} + * + * @param callCtx call context + */ + void deleteAll(@Nonnull PolarisCallContext callCtx); + + /** + * Lookup an entity given its catalog id (which can be {@link + * org.apache.polaris.core.entity.PolarisEntityConstants#NULL_ID} for top-level entities) and its + * entityId. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId entity id + * @return null if the entity was not found, else the retrieved entity. + */ + @Nullable + PolarisBaseEntity lookupEntity( + @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + + /** + * Lookup an entity given its catalogId, parentId, typeCode, and name. + * + * @param callCtx call context + * @param catalogId catalog id or {@link + * org.apache.polaris.core.entity.PolarisEntityConstants#NULL_ID} for top-level entities like + * CATALOG, PRINCIPAL and PRINCIPAL_ROLE. Note that by convention, a catalog itself has + * NULL_ID for its catalogId since the catalog is not "nested" under itself or any other + * catalog. + * @param parentId id of the parent + * @param typeCode the PolarisEntityType code of the entity to lookup + * @param name the name of the entity + * @return null if the specified entity does not exist + */ + @Nullable + PolarisBaseEntity lookupEntityByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, Review Comment: Isn't `catalogId` superfluous? Or do multiple catalogs have overlapping catalog-object IDs? IIRC all IDs are unique? Removing this parameter would also remove the need for the special NULL_ID handling, and `PolarisEntityId` could also go away. ########## polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java: ########## @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntityActiveRecord; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; + +/** + * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like + * metadata for Polaris entities and metadata about grants between these entities which is the + * foundation of our role base access control model. + * + * <p>Note that APIs to the actual persistence store are very basic, often point read or write to + * the underlying data store. The goal is to make it really easy to back this using databases like + * Postgres or simpler KV store. + */ +public interface BasePersistence { + /** + * @param callCtx call context + * @return new unique entity identifier + */ + long generateNewId(@Nonnull PolarisCallContext callCtx); + + /** + * Write this entity to the meta store. + * + * @param callCtx call context + * @param entity entity to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntity original state of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity + */ + void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity); + + /** + * Write the specified grantRecord to the grant_records table. If there is a conflict (existing + * record with the same PK), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param grantRec entity record to write, potentially replacing an existing entity record with + * the same key + */ + void writeToGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete this entity from the meta store. + * + * @param callCtx call context + * @param entity entity to delete + */ + void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + + /** + * Delete the specified grantRecord to the grant_records table. + * + * @param callCtx call context + * @param grantRec entity record to delete. + */ + void deleteFromGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete the all grant records in the grant_records table for the specified entity. This method + * will delete all grant records on that securable entity and also all grants to that grantee + * entity assuming that the entity is a grantee (catalog role, principal role or principal). + * + * @param callCtx call context + * @param entity entity whose grant records to and from should be deleted + * @param grantsOnGrantee all grants to that grantee entity. Empty list if that entity is not a + * grantee + * @param grantsOnSecurable all grants on that securable entity + */ + void deleteAllEntityGrantRecords( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore entity, + @Nonnull List<PolarisGrantRecord> grantsOnGrantee, + @Nonnull List<PolarisGrantRecord> grantsOnSecurable); + + /** + * Delete Polaris entity and grant record metadata from all tables. This is used during metadata + * bootstrap to reset all tables to their original state + * + * @param callCtx call context + */ + void deleteAll(@Nonnull PolarisCallContext callCtx); + + /** + * Lookup an entity given its catalog id (which can be NULL_ID for top-level entities) and its + * unique id. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId unique entity id + * @return NULL if the entity was not found, else the base entity. + */ + @Nullable + PolarisBaseEntity lookupEntity( + @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + + /** + * Lookup an entity given its catalogId, parentId, typeCode, and name. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param parentId id of the parent, either a namespace or a catalog + * @param typeCode the PolarisEntityType code of the entity to lookup + * @param name the name of the entity + * @return null if the specified entity does not exist + */ + @Nullable + PolarisBaseEntity lookupEntityByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name); + + /** + * Looks up just the entity's subType and id given it catalogId, parentId, typeCode, and name. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param parentId id of the parent, either a namespace or a catalog + * @param typeCode the PolarisEntityType code of the entity to lookup + * @param name the name of the entity + * @return null if the specified entity does not exist + */ + default PolarisEntityActiveRecord lookupEntityIdAndSubTypeByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name) { + PolarisBaseEntity baseEntity = lookupEntityByName(callCtx, catalogId, parentId, typeCode, name); + if (baseEntity == null) { + return null; + } + return new PolarisEntityActiveRecord(baseEntity); + } + + /** + * Lookup a set of entities given their catalog id/entity id unique identifier + * + * @param callCtx call context + * @param entityIds list of entity ids + * @return list of polaris base entities, parallel to the input list of ids. An entity in the list + * will be null if the corresponding entity could not be found. + */ + @Nonnull + List<PolarisBaseEntity> lookupEntities( + @Nonnull PolarisCallContext callCtx, List<PolarisEntityId> entityIds); + + /** + * Lookup the current entityVersion of an entity given its catalog id (which can be NULL_ID for + * top-level entities) and its unique id. Will return 0 if the entity does not exist. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId unique entity id + * @return current version for that entity or 0 if entity was not found. + */ + int lookupEntityVersion(@Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + + /** + * Get change tracking versions for all specified entity ids. + * + * @param callCtx call context + * @param entityIds list of entity id + * @return list parallel to the input list of entity versions. If an entity cannot be found, the + * corresponding element in the list will be null + */ + @Nonnull + List<PolarisChangeTrackingVersions> lookupEntityVersions( + @Nonnull PolarisCallContext callCtx, List<PolarisEntityId> entityIds); + + /** + * List all active entities of the specified type which are child entities of the specified parent + * + * @param callCtx call context + * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level + * @param parentId id of the parent, can be the special 0 value representing the root entity + * @param entityType type of entities to list + * @return the list of entities_active records for the specified list operation + */ + @Nonnull + List<PolarisEntityActiveRecord> listActiveEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType); + + /** + * List active entities where some predicate returns true + * + * @param callCtx call context + * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level + * @param parentId id of the parent, can be the special 0 value representing the root entity + * @param entityType type of entities to list + * @param entityFilter the filter to be applied to each entity. Only entities where the predicate + * returns true are returned in the list + * @return the list of entities for which the predicate returns true + */ + @Nonnull + List<PolarisEntityActiveRecord> listActiveEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType, + @Nonnull Predicate<PolarisBaseEntity> entityFilter); + + /** + * List active entities where some predicate returns true and transform the entities with a + * function + * + * @param callCtx call context + * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level + * @param parentId id of the parent, can be the special 0 value representing the root entity + * @param entityType type of entities to list + * @param limit the max number of items to return + * @param entityFilter the filter to be applied to each entity. Only entities where the predicate + * returns true are returned in the list + * @param transformer the transformation function applied to the {@link PolarisBaseEntity} before + * returning + * @return the list of entities for which the predicate returns true + */ + @Nonnull + <T> List<T> listActiveEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType, + int limit, Review Comment: For pagination we'd need a "proper" page token that guards against changes between any two pages, to make it impossible that the same entries are returned in more than one page. For posterity, assume there are 5 tables: * table-A * table-C * table-E * table-G * table-I Assume a page-size of 2. The first request yields * table-A * table-C Then "table-B" is created The second request **should** yield * table-E * table-G * table-I However, `org.apache.iceberg.rest.CatalogHandlers#paginate` uses integer indexes, so the second request would actually yield "table-C" again: * table-C * table-E * table-G * table-I Implying that all `list*` calls return results in a deterministic order (e.g. ascending by name). ########## polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java: ########## @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntityActiveRecord; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; + +/** + * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like + * metadata for Polaris entities and metadata about grants between these entities which is the + * foundation of our role base access control model. + * + * <p>Note that APIs to the actual persistence store are very basic, often point read or write to + * the underlying data store. The goal is to make it really easy to back this using databases like + * Postgres or simpler KV store. + */ +public interface BasePersistence { + /** + * @param callCtx call context + * @return new unique entity identifier + */ + long generateNewId(@Nonnull PolarisCallContext callCtx); + + /** + * Write this entity to the meta store. + * + * @param callCtx call context + * @param entity entity to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntity original state of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity + */ + void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity); + + /** + * Write the specified grantRecord to the grant_records table. If there is a conflict (existing + * record with the same PK), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param grantRec entity record to write, potentially replacing an existing entity record with + * the same key + */ + void writeToGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete this entity from the meta store. + * + * @param callCtx call context + * @param entity entity to delete + */ + void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + + /** + * Delete the specified grantRecord to the grant_records table. + * + * @param callCtx call context + * @param grantRec entity record to delete. + */ + void deleteFromGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete the all grant records in the grant_records table for the specified entity. This method + * will delete all grant records on that securable entity and also all grants to that grantee + * entity assuming that the entity is a grantee (catalog role, principal role or principal). + * + * @param callCtx call context + * @param entity entity whose grant records to and from should be deleted + * @param grantsOnGrantee all grants to that grantee entity. Empty list if that entity is not a + * grantee + * @param grantsOnSecurable all grants on that securable entity + */ + void deleteAllEntityGrantRecords( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore entity, + @Nonnull List<PolarisGrantRecord> grantsOnGrantee, + @Nonnull List<PolarisGrantRecord> grantsOnSecurable); + + /** + * Delete Polaris entity and grant record metadata from all tables. This is used during metadata + * bootstrap to reset all tables to their original state + * + * @param callCtx call context + */ + void deleteAll(@Nonnull PolarisCallContext callCtx); + + /** + * Lookup an entity given its catalog id (which can be NULL_ID for top-level entities) and its + * unique id. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId unique entity id + * @return NULL if the entity was not found, else the base entity. + */ + @Nullable + PolarisBaseEntity lookupEntity( + @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + + /** + * Lookup an entity given its catalogId, parentId, typeCode, and name. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param parentId id of the parent, either a namespace or a catalog + * @param typeCode the PolarisEntityType code of the entity to lookup + * @param name the name of the entity + * @return null if the specified entity does not exist + */ + @Nullable + PolarisBaseEntity lookupEntityByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name); + + /** + * Looks up just the entity's subType and id given it catalogId, parentId, typeCode, and name. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param parentId id of the parent, either a namespace or a catalog + * @param typeCode the PolarisEntityType code of the entity to lookup + * @param name the name of the entity + * @return null if the specified entity does not exist + */ + default PolarisEntityActiveRecord lookupEntityIdAndSubTypeByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name) { + PolarisBaseEntity baseEntity = lookupEntityByName(callCtx, catalogId, parentId, typeCode, name); + if (baseEntity == null) { + return null; + } + return new PolarisEntityActiveRecord(baseEntity); + } + + /** + * Lookup a set of entities given their catalog id/entity id unique identifier + * + * @param callCtx call context + * @param entityIds list of entity ids + * @return list of polaris base entities, parallel to the input list of ids. An entity in the list + * will be null if the corresponding entity could not be found. + */ + @Nonnull + List<PolarisBaseEntity> lookupEntities( + @Nonnull PolarisCallContext callCtx, List<PolarisEntityId> entityIds); + + /** + * Lookup the current entityVersion of an entity given its catalog id (which can be NULL_ID for + * top-level entities) and its unique id. Will return 0 if the entity does not exist. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId unique entity id + * @return current version for that entity or 0 if entity was not found. + */ + int lookupEntityVersion(@Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + + /** + * Get change tracking versions for all specified entity ids. + * + * @param callCtx call context + * @param entityIds list of entity id + * @return list parallel to the input list of entity versions. If an entity cannot be found, the + * corresponding element in the list will be null + */ + @Nonnull + List<PolarisChangeTrackingVersions> lookupEntityVersions( + @Nonnull PolarisCallContext callCtx, List<PolarisEntityId> entityIds); + + /** + * List all active entities of the specified type which are child entities of the specified parent + * + * @param callCtx call context + * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level + * @param parentId id of the parent, can be the special 0 value representing the root entity + * @param entityType type of entities to list + * @return the list of entities_active records for the specified list operation + */ + @Nonnull + List<PolarisEntityActiveRecord> listActiveEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType); + + /** + * List active entities where some predicate returns true + * + * @param callCtx call context + * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level + * @param parentId id of the parent, can be the special 0 value representing the root entity + * @param entityType type of entities to list + * @param entityFilter the filter to be applied to each entity. Only entities where the predicate + * returns true are returned in the list + * @return the list of entities for which the predicate returns true + */ + @Nonnull + List<PolarisEntityActiveRecord> listActiveEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType, + @Nonnull Predicate<PolarisBaseEntity> entityFilter); + + /** + * List active entities where some predicate returns true and transform the entities with a + * function + * + * @param callCtx call context + * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level + * @param parentId id of the parent, can be the special 0 value representing the root entity + * @param entityType type of entities to list + * @param limit the max number of items to return + * @param entityFilter the filter to be applied to each entity. Only entities where the predicate + * returns true are returned in the list + * @param transformer the transformation function applied to the {@link PolarisBaseEntity} before + * returning + * @return the list of entities for which the predicate returns true + */ + @Nonnull + <T> List<T> listActiveEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType, + int limit, Review Comment: Looks like you can replace the last 3 parameters, if this returns a `Stream` ########## polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java: ########## @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntityActiveRecord; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; + +/** + * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like + * metadata for Polaris entities and metadata about grants between these entities which is the + * foundation of our role base access control model. + * + * <p>Note that APIs to the actual persistence store are very basic, often point read or write to + * the underlying data store. The goal is to make it really easy to back this using databases like + * Postgres or simpler KV store. + */ +public interface BasePersistence { + /** + * @param callCtx call context + * @return new unique entity identifier + */ + long generateNewId(@Nonnull PolarisCallContext callCtx); + + /** + * Write this entity to the meta store. + * + * @param callCtx call context + * @param entity entity to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntity original state of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity + */ + void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity); + + /** + * Write the specified grantRecord to the grant_records table. If there is a conflict (existing + * record with the same PK), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param grantRec entity record to write, potentially replacing an existing entity record with + * the same key + */ + void writeToGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete this entity from the meta store. + * + * @param callCtx call context + * @param entity entity to delete + */ + void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + + /** + * Delete the specified grantRecord to the grant_records table. + * + * @param callCtx call context + * @param grantRec entity record to delete. + */ + void deleteFromGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete the all grant records in the grant_records table for the specified entity. This method + * will delete all grant records on that securable entity and also all grants to that grantee + * entity assuming that the entity is a grantee (catalog role, principal role or principal). + * + * @param callCtx call context + * @param entity entity whose grant records to and from should be deleted + * @param grantsOnGrantee all grants to that grantee entity. Empty list if that entity is not a + * grantee + * @param grantsOnSecurable all grants on that securable entity + */ + void deleteAllEntityGrantRecords( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore entity, + @Nonnull List<PolarisGrantRecord> grantsOnGrantee, + @Nonnull List<PolarisGrantRecord> grantsOnSecurable); + + /** + * Delete Polaris entity and grant record metadata from all tables. This is used during metadata + * bootstrap to reset all tables to their original state + * + * @param callCtx call context + */ + void deleteAll(@Nonnull PolarisCallContext callCtx); + + /** + * Lookup an entity given its catalog id (which can be NULL_ID for top-level entities) and its + * unique id. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId unique entity id + * @return NULL if the entity was not found, else the base entity. + */ + @Nullable + PolarisBaseEntity lookupEntity( + @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + + /** + * Lookup an entity given its catalogId, parentId, typeCode, and name. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param parentId id of the parent, either a namespace or a catalog + * @param typeCode the PolarisEntityType code of the entity to lookup + * @param name the name of the entity + * @return null if the specified entity does not exist + */ + @Nullable + PolarisBaseEntity lookupEntityByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, Review Comment: > the spec does allow things of different types to have the same name I suspect you refer to the Iceberg APIs/spec. IMO it's rather a mistake in Iceberg to allow having a view and a table with the exact same name - just thinking about: "SELECT * from this_thing" ... is it a table or a view or a UDF? OTOH, there are checks that prevent creating a table using the same name as an existing view. ########## polaris-core/src/main/java/org/apache/polaris/core/persistence/IntegrationPersistence.java: ########## @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageIntegration; + +/** + * Interface for the necessary "peripheral integration" objects that are logically attached to core + * persistence entities but which typically involve additional separate external integrations + * related to identity/auth, kms/secrets storage, etc. + * + * <p>Implementations should orchestrate any necessary multi-phase protocols such as leasing an + * external resource before committing a reference to the external resource in the Polaris + * persistence layer, etc. + */ +public interface IntegrationPersistence { + /** + * Allows to retrieve to the secrets of a principal given its unique client id + * + * @param callCtx call context + * @param clientId principal client id + * @return the secrets + */ + @Nullable + PolarisPrincipalSecrets loadPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String clientId); + + /** + * generate and store a client id and associated secrets for a newly created principal entity + * + * @param callCtx call context + * @param principalName name of the principal + * @param principalId principal id + */ + @Nonnull + PolarisPrincipalSecrets generateNewPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String principalName, long principalId); + + /** + * Rotate the secrets of a principal entity, i.e. make the specified main secrets the secondary + * and generate a new main secret + * + * @param callCtx call context + * @param clientId principal client id + * @param principalId principal id + * @param reset true if the principal secrets should be disabled and replaced with a one-time + * password + * @param oldSecretHash the principal secret's old main secret hash + */ + @Nullable + PolarisPrincipalSecrets rotatePrincipalSecrets( + @Nonnull PolarisCallContext callCtx, + @Nonnull String clientId, + long principalId, + boolean reset, + @Nonnull String oldSecretHash); + + /** + * When dropping a principal, we also need to drop the secrets of that principal + * + * @param callCtx the call context + * @param clientId principal client id + * @param principalId the id of the principal whose secrets are dropped + */ + void deletePrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String clientId, long principalId); + + /** + * Create an in-memory storage integration + * + * @param callCtx the polaris calllctx + * @param catalogId the catalog id + * @param entityId the entity id + * @param polarisStorageConfigurationInfo the storage configuration information + * @return a storage integration object + */ + @Nullable + <T extends PolarisStorageConfigurationInfo> PolarisStorageIntegration<T> createStorageIntegration( Review Comment: Why does persistence need to know about object-storage at all? I think persistence and object-storage are very different things. This function and `loadPolarisStorageIntegration` are factories for object-storage specific functionality. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
