This is an automated email from the ASF dual-hosted git repository. collado pushed a commit to branch mcollado-loadentities-batch in repository https://gitbox.apache.org/repos/asf/polaris.git
commit eee219b0f00c48497d7a636d608ef713bef8a44d Author: Michael Collado <[email protected]> AuthorDate: Fri Sep 5 16:27:39 2025 -0700 Changed batch call to implement loadResolvedEntities instead --- .../AtomicOperationMetaStoreManager.java | 73 +++++++++------ .../core/persistence/PolarisMetaStoreManager.java | 31 +++---- .../TransactionWorkspaceMetaStoreManager.java | 19 ++-- .../dao/entity/ResolvedEntitiesResult.java | 44 +++++++++ .../TransactionalMetaStoreManagerImpl.java | 100 ++++++++++++--------- .../persistence/PolarisTestMetaStoreManager.java | 47 +++++++--- 6 files changed, 211 insertions(+), 103 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java index 2bf034b47..de298b70a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java @@ -64,6 +64,7 @@ import org.apache.polaris.core.persistence.dao.entity.LoadPolicyMappingsResult; import org.apache.polaris.core.persistence.dao.entity.PolicyAttachmentResult; import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; import org.apache.polaris.core.persistence.dao.entity.PrivilegeResult; +import org.apache.polaris.core.persistence.dao.entity.ResolvedEntitiesResult; import org.apache.polaris.core.persistence.dao.entity.ResolvedEntityResult; import org.apache.polaris.core.persistence.dao.entity.ScopedCredentialsResult; import org.apache.polaris.core.persistence.pagination.Page; @@ -1532,34 +1533,6 @@ public class AtomicOperationMetaStoreManager extends BaseMetaStoreManager { : new EntityResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND, null); } - @Nonnull - @Override - public EntitiesResult loadEntities( - @Nonnull PolarisCallContext callCtx, - @Nonnull List<EntityNameLookupRecord> entityLookupRecords) { - BasePersistence ms = callCtx.getMetaStore(); - List<PolarisBaseEntity> entities = - ms.lookupEntities( - callCtx, - entityLookupRecords.stream() - .map(r -> new PolarisEntityId(r.getCatalogId(), r.getId())) - .collect(Collectors.toList())); - // mimic the behavior of loadEntity above, return null if not found or type mismatch - List<PolarisBaseEntity> ret = - IntStream.range(0, entityLookupRecords.size()) - .mapToObj( - i -> { - if (entities.get(i) != null - && !entities.get(i).getType().equals(entityLookupRecords.get(i).getType())) { - return null; - } else { - return entities.get(i); - } - }) - .collect(Collectors.toList()); - return new EntitiesResult(Page.fromItems(ret)); - } - @Override public @Nonnull EntitiesResult loadTasks( @Nonnull PolarisCallContext callCtx, String executorId, PageToken pageToken) { @@ -1807,6 +1780,50 @@ public class AtomicOperationMetaStoreManager extends BaseMetaStoreManager { return result; } + @Nonnull + @Override + public ResolvedEntitiesResult loadResolvedEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull List<EntityNameLookupRecord> entityLookupRecords) { + BasePersistence ms = callCtx.getMetaStore(); + List<PolarisBaseEntity> entities = + ms.lookupEntities( + callCtx, + entityLookupRecords.stream() + .map(r -> new PolarisEntityId(r.getCatalogId(), r.getId())) + .collect(Collectors.toList())); + // mimic the behavior of loadEntity above, return null if not found or type mismatch + List<ResolvedPolarisEntity> ret = + IntStream.range(0, entityLookupRecords.size()) + .mapToObj( + i -> { + if (entities.get(i) != null + && !entities.get(i).getType().equals(entityLookupRecords.get(i).getType())) { + return null; + } else { + return entities.get(i); + } + }) + .map( + e -> { + if (e == null) { + return null; + } else { + // load the grant records + final List<PolarisGrantRecord> grantRecordsAsSecurable = + ms.loadAllGrantRecordsOnSecurable(callCtx, e.getCatalogId(), e.getId()); + final List<PolarisGrantRecord> grantRecordsAsGrantee = + e.getType().isGrantee() + ? ms.loadAllGrantRecordsOnGrantee(callCtx, e.getCatalogId(), e.getId()) + : List.of(); + return new ResolvedPolarisEntity( + PolarisEntity.of(e), grantRecordsAsGrantee, grantRecordsAsSecurable); + } + }) + .collect(Collectors.toList()); + return new ResolvedEntitiesResult(ret); + } + /** {@inheritDoc} */ @Override public @Nonnull ResolvedEntityResult refreshResolvedEntity( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java index 94a846cc2..0d2857743 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java @@ -48,6 +48,7 @@ import org.apache.polaris.core.persistence.dao.entity.EntityResult; import org.apache.polaris.core.persistence.dao.entity.EntityWithPath; import org.apache.polaris.core.persistence.dao.entity.GenerateEntityIdResult; import org.apache.polaris.core.persistence.dao.entity.ListEntitiesResult; +import org.apache.polaris.core.persistence.dao.entity.ResolvedEntitiesResult; import org.apache.polaris.core.persistence.dao.entity.ResolvedEntityResult; import org.apache.polaris.core.persistence.pagination.Page; import org.apache.polaris.core.persistence.pagination.PageToken; @@ -348,21 +349,6 @@ public interface PolarisMetaStoreManager long entityId, @Nonnull PolarisEntityType entityType); - /** - * Load a batch of entities given their {@link EntityNameLookupRecord}. Will return an empty list - * if the input list is empty. Order in that returned list is the same as the input list. Some - * elements might be NULL if the entity has been dropped. - * - * @param callCtx call context - * @param entityLookupRecords the list of entity lookup records to load - * @return a non-null list of entities corresponding to the lookup keys. Some elements might be - * NULL if the entity has been dropped. - */ - @Nonnull - EntitiesResult loadEntities( - @Nonnull PolarisCallContext callCtx, - @Nonnull List<EntityNameLookupRecord> entityLookupRecords); - /** * Fetch a list of tasks to be completed. Tasks * @@ -433,6 +419,21 @@ public interface PolarisMetaStoreManager @Nonnull PolarisEntityType entityType, @Nonnull String entityName); + /** + * Load a batch of resolved entities given their {@link EntityNameLookupRecord}. Will return an + * empty list if the input list is empty. Order in that returned list is the same as the input + * list. Some elements might be NULL if the entity has been dropped. + * + * @param callCtx call context + * @param entityLookupRecords the list of entity lookup records to load + * @return a non-null list of entities corresponding to the lookup keys. Some elements might be + * NULL if the entity has been dropped. + */ + @Nonnull + ResolvedEntitiesResult loadResolvedEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull List<EntityNameLookupRecord> entityLookupRecords); + /** * Refresh a resolved entity from the backend store. Will return NULL if the entity does not * exist, i.e. has been purged or dropped. Else, will determine what has changed based on the diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java index 43d923fa9..238d29a89 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java @@ -53,6 +53,7 @@ import org.apache.polaris.core.persistence.dao.entity.LoadPolicyMappingsResult; import org.apache.polaris.core.persistence.dao.entity.PolicyAttachmentResult; import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; import org.apache.polaris.core.persistence.dao.entity.PrivilegeResult; +import org.apache.polaris.core.persistence.dao.entity.ResolvedEntitiesResult; import org.apache.polaris.core.persistence.dao.entity.ResolvedEntityResult; import org.apache.polaris.core.persistence.dao.entity.ScopedCredentialsResult; import org.apache.polaris.core.persistence.pagination.Page; @@ -330,15 +331,6 @@ public class TransactionWorkspaceMetaStoreManager implements PolarisMetaStoreMan return null; } - @Nonnull - @Override - public EntitiesResult loadEntities( - @Nonnull PolarisCallContext callCtx, - @Nonnull List<EntityNameLookupRecord> entityLookupRecords) { - diagnostics.fail("illegal_method_in_transaction_workspace", "loadEntities"); - return null; - } - @Override public EntitiesResult loadTasks( @Nonnull PolarisCallContext callCtx, String executorId, PageToken pageToken) { @@ -388,6 +380,15 @@ public class TransactionWorkspaceMetaStoreManager implements PolarisMetaStoreMan return null; } + @Nonnull + @Override + public ResolvedEntitiesResult loadResolvedEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull List<EntityNameLookupRecord> entityLookupRecords) { + diagnostics.fail("illegal_method_in_transaction_workspace", "loadEntities"); + return null; + } + @Override public ResolvedEntityResult refreshResolvedEntity( @Nonnull PolarisCallContext callCtx, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/dao/entity/ResolvedEntitiesResult.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/dao/entity/ResolvedEntitiesResult.java new file mode 100644 index 000000000..aec18414e --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/dao/entity/ResolvedEntitiesResult.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.persistence.dao.entity; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import org.apache.polaris.core.persistence.ResolvedPolarisEntity; + +public class ResolvedEntitiesResult extends BaseResult { + private final List<ResolvedPolarisEntity> resolvedEntities; + + public ResolvedEntitiesResult(List<ResolvedPolarisEntity> resolvedEntities) { + super(ReturnStatus.SUCCESS, null); + this.resolvedEntities = resolvedEntities; + } + + public ResolvedEntitiesResult( + @Nonnull ReturnStatus returnStatus, @Nullable String extraInformation) { + super(returnStatus, extraInformation); + this.resolvedEntities = null; + } + + public List<ResolvedPolarisEntity> getResolvedEntities() { + return resolvedEntities; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java index e0a73fd24..45a02027e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java @@ -53,6 +53,7 @@ import org.apache.polaris.core.persistence.BaseMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisObjectMapperUtil; import org.apache.polaris.core.persistence.PolicyMappingAlreadyExistsException; +import org.apache.polaris.core.persistence.ResolvedPolarisEntity; import org.apache.polaris.core.persistence.RetryOnConcurrencyException; import org.apache.polaris.core.persistence.dao.entity.BaseResult; import org.apache.polaris.core.persistence.dao.entity.ChangeTrackingResult; @@ -68,6 +69,7 @@ import org.apache.polaris.core.persistence.dao.entity.LoadPolicyMappingsResult; import org.apache.polaris.core.persistence.dao.entity.PolicyAttachmentResult; import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; import org.apache.polaris.core.persistence.dao.entity.PrivilegeResult; +import org.apache.polaris.core.persistence.dao.entity.ResolvedEntitiesResult; import org.apache.polaris.core.persistence.dao.entity.ResolvedEntityResult; import org.apache.polaris.core.persistence.dao.entity.ScopedCredentialsResult; import org.apache.polaris.core.persistence.pagination.Page; @@ -2026,46 +2028,6 @@ public class TransactionalMetaStoreManagerImpl extends BaseMetaStoreManager { () -> this.loadEntity(callCtx, ms, entityCatalogId, entityId, entityType.getCode())); } - /** Refer to {@link #loadEntity(PolarisCallContext, long, long, PolarisEntityType)} */ - private @Nonnull EntitiesResult loadEntities( - @Nonnull PolarisCallContext callCtx, - @Nonnull TransactionalPersistence ms, - @Nonnull List<EntityNameLookupRecord> entityLookupRecords) { - List<PolarisBaseEntity> entities = - ms.lookupEntitiesInCurrentTxn( - callCtx, - entityLookupRecords.stream() - .map(r -> new PolarisEntityId(r.getCatalogId(), r.getId())) - .collect(Collectors.toList())); - // mimic the behavior of loadEntity above, return null if not found or type mismatch - List<PolarisBaseEntity> ret = - IntStream.range(0, entityLookupRecords.size()) - .mapToObj( - i -> { - if (entities.get(i) != null - && !entities.get(i).getType().equals(entityLookupRecords.get(i).getType())) { - return null; - } else { - return entities.get(i); - } - }) - .collect(Collectors.toList()); - return new EntitiesResult(Page.fromItems(ret)); - } - - @Nonnull - @Override - public EntitiesResult loadEntities( - @Nonnull PolarisCallContext callCtx, - @Nonnull List<EntityNameLookupRecord> entityLookupRecords) { - TransactionalPersistence ms = ((TransactionalPersistence) callCtx.getMetaStore()); - return ms.runInReadTransaction( - callCtx, - () -> - this.loadEntities( - callCtx, (TransactionalPersistence) callCtx.getMetaStore(), entityLookupRecords)); - } - /** Refer to {@link #loadTasks(PolarisCallContext, String, PageToken)} */ private @Nonnull EntitiesResult loadTasks( @Nonnull PolarisCallContext callCtx, @@ -2348,6 +2310,64 @@ public class TransactionalMetaStoreManagerImpl extends BaseMetaStoreManager { return result; } + /** Refer to {@link #loadEntity(PolarisCallContext, long, long, PolarisEntityType)} */ + private @Nonnull ResolvedEntitiesResult loadResolvedEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull TransactionalPersistence ms, + @Nonnull List<EntityNameLookupRecord> entityLookupRecords) { + List<PolarisBaseEntity> entities = + ms.lookupEntitiesInCurrentTxn( + callCtx, + entityLookupRecords.stream() + .map(r -> new PolarisEntityId(r.getCatalogId(), r.getId())) + .collect(Collectors.toList())); + // mimic the behavior of loadEntity above, return null if not found or type mismatch + List<ResolvedPolarisEntity> ret = + IntStream.range(0, entityLookupRecords.size()) + .mapToObj( + i -> { + if (entities.get(i) != null + && !entities.get(i).getType().equals(entityLookupRecords.get(i).getType())) { + return null; + } else { + return entities.get(i); + } + }) + .map( + e -> { + if (e == null) { + return null; + } else { + // load the grant records + final List<PolarisGrantRecord> grantRecordsAsSecurable = + ms.loadAllGrantRecordsOnSecurableInCurrentTxn( + callCtx, e.getCatalogId(), e.getId()); + final List<PolarisGrantRecord> grantRecordsAsGrantee = + e.getType().isGrantee() + ? ms.loadAllGrantRecordsOnGranteeInCurrentTxn( + callCtx, e.getCatalogId(), e.getId()) + : List.of(); + return new ResolvedPolarisEntity( + PolarisEntity.of(e), grantRecordsAsGrantee, grantRecordsAsSecurable); + } + }) + .collect(Collectors.toList()); + return new ResolvedEntitiesResult(ret); + } + + @Nonnull + @Override + public ResolvedEntitiesResult loadResolvedEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull List<EntityNameLookupRecord> entityLookupRecords) { + TransactionalPersistence ms = ((TransactionalPersistence) callCtx.getMetaStore()); + return ms.runInReadTransaction( + callCtx, + () -> + this.loadResolvedEntities( + callCtx, (TransactionalPersistence) callCtx.getMetaStore(), entityLookupRecords)); + } + /** {@inheritDoc} */ private @Nonnull ResolvedEntityResult refreshResolvedEntity( @Nonnull PolarisCallContext callCtx, diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java index 657a49ca5..1c36b4935 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java @@ -46,11 +46,11 @@ import org.apache.polaris.core.persistence.dao.entity.BaseResult; import org.apache.polaris.core.persistence.dao.entity.CreateCatalogResult; import org.apache.polaris.core.persistence.dao.entity.CreatePrincipalResult; import org.apache.polaris.core.persistence.dao.entity.DropEntityResult; -import org.apache.polaris.core.persistence.dao.entity.EntitiesResult; import org.apache.polaris.core.persistence.dao.entity.EntityResult; import org.apache.polaris.core.persistence.dao.entity.LoadGrantsResult; import org.apache.polaris.core.persistence.dao.entity.LoadPolicyMappingsResult; import org.apache.polaris.core.persistence.dao.entity.PolicyAttachmentResult; +import org.apache.polaris.core.persistence.dao.entity.ResolvedEntitiesResult; import org.apache.polaris.core.persistence.dao.entity.ResolvedEntityResult; import org.apache.polaris.core.persistence.pagination.PageToken; import org.apache.polaris.core.policy.PolarisPolicyMappingRecord; @@ -2757,8 +2757,8 @@ public class PolarisTestMetaStoreManager { "T1"); // batch load all entities. They should all be present and non-null - EntitiesResult entitiesResult = - polarisMetaStoreManager.loadEntities( + ResolvedEntitiesResult entitiesResult = + polarisMetaStoreManager.loadResolvedEntities( polarisCallContext, List.of( new EntityNameLookupRecord(catalog), @@ -2767,16 +2767,38 @@ public class PolarisTestMetaStoreManager { new EntityNameLookupRecord(T1))); Assertions.assertThat(entitiesResult) .isNotNull() - .returns(BaseResult.ReturnStatus.SUCCESS, EntitiesResult::getReturnStatus) + .returns(BaseResult.ReturnStatus.SUCCESS, ResolvedEntitiesResult::getReturnStatus) .extracting( - EntitiesResult::getEntities, InstanceOfAssertFactories.list(PolarisBaseEntity.class)) + ResolvedEntitiesResult::getResolvedEntities, + InstanceOfAssertFactories.list(ResolvedPolarisEntity.class)) .hasSize(4) .allSatisfy(entity -> Assertions.assertThat(entity).isNotNull()) + .extracting(r -> (PolarisBaseEntity) r.getEntity()) .containsExactly(catalog, N1, N1_N2, T1); + ResolvedPolarisEntity catalogEntity = entitiesResult.getResolvedEntities().get(0); + Assertions.assertThat(catalogEntity) + .extracting(ResolvedPolarisEntity::getAllGrantRecords) + .isNotNull() + .asInstanceOf(InstanceOfAssertFactories.list(PolarisGrantRecord.class)) + .hasSize(2) + .containsExactlyInAnyOrder( + new PolarisGrantRecord( + 0L, + catalog.getId(), + catalogCreated.getCatalogAdminRole().getCatalogId(), + catalogCreated.getCatalogAdminRole().getId(), + PolarisPrivilege.CATALOG_MANAGE_ACCESS.getCode()), + new PolarisGrantRecord( + 0L, + catalog.getId(), + catalogCreated.getCatalogAdminRole().getCatalogId(), + catalogCreated.getCatalogAdminRole().getId(), + PolarisPrivilege.CATALOG_MANAGE_METADATA.getCode())); + // try entities which do not exist entitiesResult = - polarisMetaStoreManager.loadEntities( + polarisMetaStoreManager.loadResolvedEntities( polarisCallContext, List.of( new EntityNameLookupRecord( @@ -2795,15 +2817,16 @@ public class PolarisTestMetaStoreManager { PolarisEntitySubType.NULL_SUBTYPE.getCode()))); Assertions.assertThat(entitiesResult) .isNotNull() - .returns(BaseResult.ReturnStatus.SUCCESS, EntitiesResult::getReturnStatus) + .returns(BaseResult.ReturnStatus.SUCCESS, ResolvedEntitiesResult::getReturnStatus) .extracting( - EntitiesResult::getEntities, InstanceOfAssertFactories.list(PolarisBaseEntity.class)) + ResolvedEntitiesResult::getResolvedEntities, + InstanceOfAssertFactories.list(ResolvedPolarisEntity.class)) .hasSize(2) .allSatisfy(entity -> Assertions.assertThat(entity).isNull()); // mix of existing entities and entities with wrong type entitiesResult = - polarisMetaStoreManager.loadEntities( + polarisMetaStoreManager.loadResolvedEntities( polarisCallContext, List.of( new EntityNameLookupRecord( @@ -2838,12 +2861,14 @@ public class PolarisTestMetaStoreManager { new EntityNameLookupRecord(T1))); Assertions.assertThat(entitiesResult) .isNotNull() - .returns(BaseResult.ReturnStatus.SUCCESS, EntitiesResult::getReturnStatus) + .returns(BaseResult.ReturnStatus.SUCCESS, ResolvedEntitiesResult::getReturnStatus) .extracting( - EntitiesResult::getEntities, InstanceOfAssertFactories.list(PolarisBaseEntity.class)) + ResolvedEntitiesResult::getResolvedEntities, + InstanceOfAssertFactories.list(ResolvedPolarisEntity.class)) .hasSize(6) .filteredOn(e -> e != null) .hasSize(2) + .extracting(r -> (PolarisBaseEntity) r.getEntity()) .containsExactly(catalog, T1); }
