This is an automated email from the ASF dual-hosted git repository. dhuo pushed a commit to branch persistence-poc in repository https://gitbox.apache.org/repos/asf/polaris.git
commit 54b762e985bc805200da0317de50aad51747279c Author: Dennis Huo <[email protected]> AuthorDate: Tue Feb 25 03:44:40 2025 +0000 Remove all "entitiesDropped" members; these were vestigial from trying to implement UNDROP but for now are entirely unused. We can reintroduce it with a better design against multiple backends when/if we want to implement UNDROP. --- .../PolarisEclipseLinkMetaStoreSessionImpl.java | 16 -- .../impl/eclipselink/PolarisEclipseLinkStore.java | 38 ----- .../polaris/jpa/models/ModelEntityDropped.java | 164 --------------------- .../persistence/PolarisMetaStoreManagerImpl.java | 110 ++++++-------- .../core/persistence/PolarisMetaStoreSession.java | 20 --- .../PolarisTreeMapMetaStoreSessionImpl.java | 18 --- .../core/persistence/PolarisTreeMapStore.java | 49 +----- 7 files changed, 46 insertions(+), 369 deletions(-) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index 04787d51..621993e7 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -271,14 +271,6 @@ public class PolarisEclipseLinkMetaStoreSessionImpl implements PolarisMetaStoreS this.store.writeToEntitiesActive(localSession.get(), entity); } - /** {@inheritDoc} */ - @Override - public void writeToEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { - // write it - this.store.writeToEntitiesDropped(localSession.get(), entity); - } - /** {@inheritDoc} */ @Override public void writeToEntitiesChangeTracking( @@ -312,14 +304,6 @@ public class PolarisEclipseLinkMetaStoreSessionImpl implements PolarisMetaStoreS this.store.deleteFromEntitiesActive(localSession.get(), new PolarisEntitiesActiveKey(entity)); } - /** {@inheritDoc} */ - @Override - public void deleteFromEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { - // delete it - this.store.deleteFromEntitiesDropped(localSession.get(), entity.getCatalogId(), entity.getId()); - } - /** * {@inheritDoc} * diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java index 8216bef9..59ac63f6 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java @@ -38,7 +38,6 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.jpa.models.ModelEntity; import org.apache.polaris.jpa.models.ModelEntityActive; import org.apache.polaris.jpa.models.ModelEntityChangeTracking; -import org.apache.polaris.jpa.models.ModelEntityDropped; import org.apache.polaris.jpa.models.ModelGrantRecord; import org.apache.polaris.jpa.models.ModelPrincipalSecrets; import org.slf4j.Logger; @@ -104,17 +103,6 @@ public class PolarisEclipseLinkStore { } } - void writeToEntitiesDropped(EntityManager session, PolarisBaseEntity entity) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelEntityDropped entityDropped = - lookupEntityDropped(session, entity.getCatalogId(), entity.getId()); - if (entityDropped == null) { - session.persist(ModelEntityDropped.fromEntity(entity)); - } - } - void writeToEntitiesChangeTracking(EntityManager session, PolarisBaseEntity entity) { diagnosticServices.check(session != null, "session_is_null"); checkInitialized(); @@ -158,16 +146,6 @@ public class PolarisEclipseLinkStore { session.remove(entity); } - void deleteFromEntitiesDropped(EntityManager session, long catalogId, long entityId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelEntityDropped entity = lookupEntityDropped(session, catalogId, entityId); - diagnosticServices.check(entity != null, "dropped_entity_not_found"); - - session.remove(entity); - } - void deleteFromEntitiesChangeTracking(EntityManager session, PolarisEntityCore entity) { diagnosticServices.check(session != null, "session_is_null"); checkInitialized(); @@ -216,7 +194,6 @@ public class PolarisEclipseLinkStore { session.createQuery("DELETE from ModelEntity").executeUpdate(); session.createQuery("DELETE from ModelEntityActive").executeUpdate(); - session.createQuery("DELETE from ModelEntityDropped").executeUpdate(); session.createQuery("DELETE from ModelEntityChangeTracking").executeUpdate(); session.createQuery("DELETE from ModelGrantRecord").executeUpdate(); session.createQuery("DELETE from ModelPrincipalSecrets").executeUpdate(); @@ -319,21 +296,6 @@ public class PolarisEclipseLinkStore { return query.getResultList(); } - ModelEntityDropped lookupEntityDropped(EntityManager session, long catalogId, long entityId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - return session - .createQuery( - "SELECT m from ModelEntityDropped m where m.catalogId=:catalogId and m.id=:id", - ModelEntityDropped.class) - .setParameter("catalogId", catalogId) - .setParameter("id", entityId) - .getResultStream() - .findFirst() - .orElse(null); - } - ModelEntityChangeTracking lookupEntityChangeTracking( EntityManager session, long catalogId, long entityId) { diagnosticServices.check(session != null, "session_is_null"); diff --git a/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityDropped.java b/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityDropped.java deleted file mode 100644 index 97986e79..00000000 --- a/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityDropped.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.jpa.models; - -import jakarta.persistence.Entity; -import jakarta.persistence.Id; -import jakarta.persistence.Table; -import jakarta.persistence.Version; -import org.apache.polaris.core.entity.PolarisBaseEntity; - -/** - * EntityDropped model representing some attributes of a Polaris Entity. This is used to exchange - * entity information with ENTITIES_DROPPED table - */ -@Entity -@Table(name = "ENTITIES_DROPPED") -public class ModelEntityDropped { - // the id of the catalog associated to that entity. NULL_ID if this entity is top-level like - // a catalog - @Id private long catalogId; - - // the id of the entity which was resolved - private long id; - - // the id of the parent of this entity, use 0 for a top-level entity whose parent is the account - @Id private long parentId; - - // the type of the entity when it was resolved - @Id private int typeCode; - - // the name that this entity had when it was resolved - @Id private String name; - - // the type of the entity when it was resolved - @Id private int subTypeCode; - - // when this entity was dropped. Null if was never dropped - @Id private long dropTimestamp; - - // when should we start purging this entity - private long toPurgeTimestamp; - - // Used for Optimistic Locking to handle concurrent reads and updates - @Version private long version; - - public long getCatalogId() { - return catalogId; - } - - public long getId() { - return id; - } - - public long getParentId() { - return parentId; - } - - public int getTypeCode() { - return typeCode; - } - - public String getName() { - return name; - } - - public int getSubTypeCode() { - return subTypeCode; - } - - public long getDropTimestamp() { - return dropTimestamp; - } - - public long getToPurgeTimestamp() { - return toPurgeTimestamp; - } - - public static Builder builder() { - return new Builder(); - } - - public static final class Builder { - private final ModelEntityDropped entity; - - private Builder() { - entity = new ModelEntityDropped(); - } - - public Builder catalogId(long catalogId) { - entity.catalogId = catalogId; - return this; - } - - public Builder id(long id) { - entity.id = id; - return this; - } - - public Builder parentId(long parentId) { - entity.parentId = parentId; - return this; - } - - public Builder typeCode(int typeCode) { - entity.typeCode = typeCode; - return this; - } - - public Builder name(String name) { - entity.name = name; - return this; - } - - public Builder subTypeCode(int subTypeCode) { - entity.subTypeCode = subTypeCode; - return this; - } - - public Builder dropTimestamp(long dropTimestamp) { - entity.dropTimestamp = dropTimestamp; - return this; - } - - public Builder toPurgeTimestamp(long toPurgeTimestamp) { - entity.toPurgeTimestamp = toPurgeTimestamp; - return this; - } - - public ModelEntityDropped build() { - return entity; - } - } - - public static ModelEntityDropped fromEntity(PolarisBaseEntity entity) { - if (entity == null) return null; - - return ModelEntityDropped.builder() - .catalogId(entity.getCatalogId()) - .id(entity.getId()) - .parentId(entity.getParentId()) - .typeCode(entity.getTypeCode()) - .name(entity.getName()) - .subTypeCode(entity.getSubTypeCode()) - .dropTimestamp(entity.getDropTimestamp()) - .toPurgeTimestamp(entity.getToPurgeTimestamp()) - .build(); - } -} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 6214d0f0..3eb32d07 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -69,9 +69,6 @@ public class PolarisMetaStoreManagerImpl extends BaseMetaStoreManager { /** mapper, allows to serialize/deserialize properties to/from JSON */ private static final ObjectMapper MAPPER = new ObjectMapper(); - /** use synchronous drop for entities */ - private static final boolean USE_SYNCHRONOUS_DROP = true; - /** * Lookup an entity by its name * @@ -280,74 +277,53 @@ public class PolarisMetaStoreManagerImpl extends BaseMetaStoreManager { // delete it from active slice ms.deleteFromEntitiesActive(callCtx, entity); - // for now drop all entities synchronously - if (USE_SYNCHRONOUS_DROP) { - // use synchronous drop - - // delete ALL grant records to (if the entity is a grantee) and from that entity - final List<PolarisGrantRecord> grantsOnGrantee = - (entity.getType().isGrantee()) - ? ms.loadAllGrantRecordsOnGrantee(callCtx, entity.getCatalogId(), entity.getId()) - : List.of(); - final List<PolarisGrantRecord> grantsOnSecurable = - ms.loadAllGrantRecordsOnSecurable(callCtx, entity.getCatalogId(), entity.getId()); - ms.deleteAllEntityGrantRecords(callCtx, entity, grantsOnGrantee, grantsOnSecurable); - - // Now determine the set of entities on the other side of the grants we just removed. Grants - // from/to these entities has been removed, hence we need to update the grant version of - // each entity. Collect the id of each. - Set<PolarisEntityId> entityIdsGrantChanged = new HashSet<>(); - grantsOnGrantee.forEach( - gr -> - entityIdsGrantChanged.add( - new PolarisEntityId(gr.getSecurableCatalogId(), gr.getSecurableId()))); - grantsOnSecurable.forEach( - gr -> - entityIdsGrantChanged.add( - new PolarisEntityId(gr.getGranteeCatalogId(), gr.getGranteeId()))); - - // Bump up the grant version of these entities - List<PolarisBaseEntity> entities = - ms.lookupEntities(callCtx, new ArrayList<>(entityIdsGrantChanged)); - for (PolarisBaseEntity entityGrantChanged : entities) { - entityGrantChanged.setGrantRecordsVersion(entityGrantChanged.getGrantRecordsVersion() + 1); - ms.writeToEntities(callCtx, entityGrantChanged); - ms.writeToEntitiesChangeTracking(callCtx, entityGrantChanged); - } - - // remove the entity being dropped now - ms.deleteFromEntities(callCtx, entity); - ms.deleteFromEntitiesChangeTracking(callCtx, entity); - - // if it is a principal, we also need to drop the secrets - if (entity.getType() == PolarisEntityType.PRINCIPAL) { - // get internal properties - Map<String, String> properties = - this.deserializeProperties(callCtx, entity.getInternalProperties()); - - // get client_id - String clientId = properties.get(PolarisEntityConstants.getClientIdPropertyName()); - - // delete it from the secret slice - ms.deletePrincipalSecrets(callCtx, clientId, entity.getId()); - } - } else { + // for now drop all associated grants, etc. synchronously + // delete ALL grant records to (if the entity is a grantee) and from that entity + final List<PolarisGrantRecord> grantsOnGrantee = + (entity.getType().isGrantee()) + ? ms.loadAllGrantRecordsOnGrantee(callCtx, entity.getCatalogId(), entity.getId()) + : List.of(); + final List<PolarisGrantRecord> grantsOnSecurable = + ms.loadAllGrantRecordsOnSecurable(callCtx, entity.getCatalogId(), entity.getId()); + ms.deleteAllEntityGrantRecords(callCtx, entity, grantsOnGrantee, grantsOnSecurable); + + // Now determine the set of entities on the other side of the grants we just removed. Grants + // from/to these entities has been removed, hence we need to update the grant version of + // each entity. Collect the id of each. + Set<PolarisEntityId> entityIdsGrantChanged = new HashSet<>(); + grantsOnGrantee.forEach( + gr -> + entityIdsGrantChanged.add( + new PolarisEntityId(gr.getSecurableCatalogId(), gr.getSecurableId()))); + grantsOnSecurable.forEach( + gr -> + entityIdsGrantChanged.add( + new PolarisEntityId(gr.getGranteeCatalogId(), gr.getGranteeId()))); + + // Bump up the grant version of these entities + List<PolarisBaseEntity> entities = + ms.lookupEntities(callCtx, new ArrayList<>(entityIdsGrantChanged)); + for (PolarisBaseEntity entityGrantChanged : entities) { + entityGrantChanged.setGrantRecordsVersion(entityGrantChanged.getGrantRecordsVersion() + 1); + ms.writeToEntities(callCtx, entityGrantChanged); + ms.writeToEntitiesChangeTracking(callCtx, entityGrantChanged); + } - // update the entity to indicate it has been dropped - final long now = System.currentTimeMillis(); - entity.setDropTimestamp(now); - entity.setLastUpdateTimestamp(now); + // remove the entity being dropped now + ms.deleteFromEntities(callCtx, entity); + ms.deleteFromEntitiesChangeTracking(callCtx, entity); - // schedule purge - entity.setToPurgeTimestamp(now + PolarisEntityConstants.getRetentionTimeInMs()); + // if it is a principal, we also need to drop the secrets + if (entity.getType() == PolarisEntityType.PRINCIPAL) { + // get internal properties + Map<String, String> properties = + this.deserializeProperties(callCtx, entity.getInternalProperties()); - // increment version - entity.setEntityVersion(entity.getEntityVersion() + 1); + // get client_id + String clientId = properties.get(PolarisEntityConstants.getClientIdPropertyName()); - // write to the dropped slice and to purge slice - ms.writeToEntities(callCtx, entity); - ms.writeToEntitiesDropped(callCtx, entity); - ms.writeToEntitiesChangeTracking(callCtx, entity); + // delete it from the secret slice + ms.deletePrincipalSecrets(callCtx, clientId, entity.getId()); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index 124ec1d7..ee1d4f19 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -122,17 +122,6 @@ public interface PolarisMetaStoreSession { void writeToEntitiesActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); - /** - * Write the base entity to the entities_dropped table. If there is a conflict (existing record - * with the same PK), all attributes of the new record will replace the existing one. - * - * @param callCtx call context - * @param entity entity record to write, potentially replacing an existing entity record with the - * same key - */ - void writeToEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); - /** * Write the base entity to the entities change tracking table. If there is a conflict (existing * record with the same id), all attributes of the new record will replace the existing one. @@ -172,15 +161,6 @@ public interface PolarisMetaStoreSession { void deleteFromEntitiesActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); - /** - * Delete the base entity to the entities_dropped table - * - * @param callCtx call context - * @param entity entity record to delete - */ - void deleteFromEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); - /** * Delete the base entity from the entities change tracking table * diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java index 6516b7b7..536e81c7 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java @@ -126,15 +126,6 @@ public class PolarisTreeMapMetaStoreSessionImpl implements PolarisMetaStoreSessi this.store.getSliceEntitiesActive().write(entity); } - /** {@inheritDoc} */ - @Override - public void writeToEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { - // write it - this.store.getSliceEntitiesDropped().write(entity); - this.store.getSliceEntitiesDroppedToPurge().write(entity); - } - /** {@inheritDoc} */ @Override public void writeToEntitiesChangeTracking( @@ -169,15 +160,6 @@ public class PolarisTreeMapMetaStoreSessionImpl implements PolarisMetaStoreSessi this.store.getSliceEntitiesActive().delete(this.store.buildEntitiesActiveKey(entity)); } - /** {@inheritDoc} */ - @Override - public void deleteFromEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { - // delete it - this.store.getSliceEntitiesDropped().delete(entity); - this.store.getSliceEntitiesDroppedToPurge().delete(entity); - } - /** * {@inheritDoc} * diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java index 544bcf0f..3d5f3d7f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java @@ -194,16 +194,10 @@ public class PolarisTreeMapStore { // all entities private final Slice<PolarisBaseEntity> sliceEntities; - // all entities + // all entities by-name private final Slice<PolarisBaseEntity> sliceEntitiesActive; - // all entities dropped - private final Slice<PolarisBaseEntity> sliceEntitiesDropped; - - // all entities dropped - private final Slice<PolarisBaseEntity> sliceEntitiesDroppedToPurge; - - // all entities dropped + // all entities just holding their entityVersions and grantVersions private final Slice<PolarisBaseEntity> sliceEntitiesChangeTracking; // all grant records indexed by securable @@ -231,32 +225,9 @@ public class PolarisTreeMapStore { entity -> String.format("%d::%d", entity.getCatalogId(), entity.getId()), PolarisBaseEntity::new); - // the entities active slice + // the entities active slice; simply acts as a name-based index into the entities slice this.sliceEntitiesActive = new Slice<>(this::buildEntitiesActiveKey, PolarisBaseEntity::new); - // the entities active slice - this.sliceEntitiesDropped = - new Slice<>( - entity -> - String.format( - "%d::%d::%s::%d::%d::%d", - entity.getCatalogId(), - entity.getParentId(), - entity.getName(), - entity.getTypeCode(), - entity.getSubTypeCode(), - entity.getDropTimestamp()), - PolarisBaseEntity::new); - - // the entities active slice - this.sliceEntitiesDroppedToPurge = - new Slice<>( - entity -> - String.format( - "%d::%d::%s", - entity.getToPurgeTimestamp(), entity.getCatalogId(), entity.getId()), - PolarisBaseEntity::new); - // change tracking this.sliceEntitiesChangeTracking = new Slice<>( @@ -370,8 +341,6 @@ public class PolarisTreeMapStore { this.tr = new Transaction(true); this.sliceEntities.startWriteTransaction(); this.sliceEntitiesActive.startWriteTransaction(); - this.sliceEntitiesDropped.startWriteTransaction(); - this.sliceEntitiesDroppedToPurge.startWriteTransaction(); this.sliceEntitiesChangeTracking.startWriteTransaction(); this.sliceGrantRecords.startWriteTransaction(); this.sliceGrantRecordsByGrantee.startWriteTransaction(); @@ -382,8 +351,6 @@ public class PolarisTreeMapStore { void rollback() { this.sliceEntities.rollback(); this.sliceEntitiesActive.rollback(); - this.sliceEntitiesDropped.rollback(); - this.sliceEntitiesDroppedToPurge.rollback(); this.sliceEntitiesChangeTracking.rollback(); this.sliceGrantRecords.rollback(); this.sliceGrantRecordsByGrantee.rollback(); @@ -510,14 +477,6 @@ public class PolarisTreeMapStore { return sliceEntitiesActive; } - public Slice<PolarisBaseEntity> getSliceEntitiesDropped() { - return sliceEntitiesDropped; - } - - public Slice<PolarisBaseEntity> getSliceEntitiesDroppedToPurge() { - return sliceEntitiesDroppedToPurge; - } - public Slice<PolarisBaseEntity> getSliceEntitiesChangeTracking() { return sliceEntitiesChangeTracking; } @@ -548,8 +507,6 @@ public class PolarisTreeMapStore { this.ensureReadWriteTr(); this.sliceEntities.deleteAll(); this.sliceEntitiesActive.deleteAll(); - this.sliceEntitiesDropped.deleteAll(); - this.sliceEntitiesDroppedToPurge.deleteAll(); this.sliceEntitiesChangeTracking.deleteAll(); this.sliceGrantRecordsByGrantee.deleteAll(); this.sliceGrantRecords.deleteAll();
