snazy commented on code in PR #4526:
URL: https://github.com/apache/polaris/pull/4526#discussion_r3312039989
##########
persistence/nosql/persistence/metastore-maintenance/src/main/java/org/apache/polaris/persistence/nosql/metastore/maintenance/CatalogRetainedIdentifier.java:
##########
@@ -331,4 +375,351 @@ void ignoreReferenceNotFound(Runnable runnable) {
LOGGER.debug("Reference not found: {}", e.getMessage());
}
}
+
+ private void cleanupStaleGrantRecords(Persistence persistence,
EntityLookupCache entityLookup) {
+ LOGGER.info("Cleaning stale grant records...");
+ ignoreReferenceNotFound(
+ () -> {
+ IndexKey continuation = null;
+ while (true) {
+ var refObj =
+ persistence.fetchReferenceHead(REALM_GRANTS_REF_NAME,
RealmGrantsObj.class);
+ if (refObj.isEmpty()) {
+ return;
+ }
+
+ var batch =
+ scanStaleGrantRecords(
+ persistence,
+ refObj.get().acls().indexForRead(persistence,
OBJ_REF_SERIALIZER),
+ continuation,
+ entityLookup);
+ if (batch.candidates().isEmpty()) {
+ return;
+ }
+
+ var cleaned = commitGrantCleanup(persistence, batch.candidates());
+ LOGGER.debug("Removed {} stale grant ACL entries in this batch",
cleaned);
+
+ if (batch.exhausted()) {
+ return;
+ }
+ continuation = batch.lastScannedKey();
+ }
+ });
+ }
+
+ private GrantCleanupBatch scanStaleGrantRecords(
+ Persistence persistence,
+ Index<ObjRef> index,
+ IndexKey continuation,
+ EntityLookupCache entityLookup) {
+ var candidates = new ArrayList<GrantCleanupCandidate>();
+ var iter = index.iterator(continuation, null, false);
+ var skipContinuation = continuation != null;
+ IndexKey lastScannedKey = null;
+ while (iter.hasNext()) {
+ var elem = iter.next();
+ var aclKey = elem.key();
+ if (skipContinuation && aclKey.equals(continuation)) {
+ skipContinuation = false;
+ continue;
+ }
+ skipContinuation = false;
+ lastScannedKey = aclKey;
+
+ var anchor = GrantTriplet.fromRoleName(aclKey.toString());
+ if (!entityLookup.entityExists(anchor.catalogId(), anchor.id(),
anchor.typeCode())) {
+ candidates.add(new GrantCleanupCandidate(aclKey, List.of(), true));
+ } else {
+ var aclObj = persistence.fetch(elem.value(), AclObj.class);
+ if (aclObj == null) {
+ continue;
+ }
+
+ var staleRoleIds = new ArrayList<String>();
+ aclObj
+ .acl()
+ .forEach(
+ (roleId, entry) -> {
+ var role = GrantTriplet.fromRoleName(roleId);
+ if (!entityLookup.entityExists(role.catalogId(), role.id(),
role.typeCode())) {
+ staleRoleIds.add(roleId);
+ }
+ });
+ if (!staleRoleIds.isEmpty()) {
+ candidates.add(new GrantCleanupCandidate(aclKey, staleRoleIds,
false));
+ }
+ }
+
+ if (candidates.size() >= STALE_CLEANUP_BATCH_SIZE) {
+ return new GrantCleanupBatch(candidates, lastScannedKey, false);
+ }
+ }
+ return new GrantCleanupBatch(candidates, lastScannedKey, true);
+ }
+
+ private int commitGrantCleanup(
+ Persistence persistence, List<GrantCleanupCandidate>
grantCleanupCandidates) {
+ return persistence
+ .createCommitter(REALM_GRANTS_REF_NAME, RealmGrantsObj.class,
Integer.class)
+ .synchronizingLocally()
Review Comment:
Since you asked for it here - I had the fix sitting on my disk for a while:
#4552
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
