This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a commit to branch branch-1.3
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-1.3 by this push:
new 7e63ac6775 [Cherry-pick to branch-1.3] [#12230] fix(auth): stop a lost
JCasbin role policy load from denying forever (#12231) (#12415)
7e63ac6775 is described below
commit 7e63ac6775fabd5cf91dbe12cc32dd59b3bf98e3
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Aug 11 19:40:33 2026 +0800
[Cherry-pick to branch-1.3] [#12230] fix(auth): stop a lost JCasbin role
policy load from denying forever (#12231) (#12415)
**Cherry-pick Information:**
- Original commit: e85e654d91b297b9e53c5e329717eb5efe39307c
- Target branch: `branch-1.3`
- Status: ✅ **Conflicts resolved**
The branch-1.3 conflicts were resolved in b355c0b3df.
**Validation:**
- `./gradlew spotlessApply`
- `./gradlew :server-common:test -PskipITs`
---------
Co-authored-by: Qi Yu <[email protected]>
---
.../authorization/jcasbin/JcasbinAuthorizer.java | 354 ++++++++++++++++++---
.../jcasbin/JcasbinLoadedRolesCache.java | 17 +-
.../jcasbin/ResolvedRolePolicies.java | 61 ++++
.../jcasbin/TestJcasbinAuthorizer.java | 247 +++++++++++++-
.../jcasbin/TestJcasbinLoadedRolesCache.java | 79 +++++
5 files changed, 709 insertions(+), 49 deletions(-)
diff --git
a/server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/JcasbinAuthorizer.java
b/server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/JcasbinAuthorizer.java
index 432207e146..7e68172bf3 100644
---
a/server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/JcasbinAuthorizer.java
+++
b/server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/JcasbinAuthorizer.java
@@ -34,6 +34,7 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
@@ -93,9 +94,9 @@ import org.slf4j.LoggerFactory;
* {@link #groupRoleCache}, {@link #loadedRoles}. Each cached entry
carries the {@code
* *_meta.updated_at} value it was loaded against; every read issues a
lightweight version
* probe and discards the entry if the DB sentinel has advanced. No TTL
is relied on for
- * correctness — TTL eviction only bounds memory. User/group role
snapshots use write-based
- * TTLs through {@link CaffeineGravitinoCache}; loaded role policies use
access-based TTLs
- * through {@link JcasbinLoadedRolesCache}.
+ * correctness — TTL eviction only bounds memory. User/group role
snapshots and loaded role
+ * policies both use write-based TTLs through {@link
CaffeineGravitinoCache} and {@link
+ * JcasbinLoadedRolesCache}, respectively.
* <li><b>Eventual-consistency caches</b> — {@link #metadataIdCache} and
{@link #ownerRelCache}.
* The global entity change log poller dispatches {@code
entity_change_log} batches to {@link
* #changePoller}, while {@link #changePoller} polls {@code owner_meta}.
Other Gravitino nodes
@@ -114,6 +115,43 @@ public class JcasbinAuthorizer implements
GravitinoAuthorizer {
private static final Logger LOG =
LoggerFactory.getLogger(JcasbinAuthorizer.class);
+ /**
+ * Field index of {@code sub} (the role/user/group id) in a jcasbin {@code
p} policy row. See the
+ * {@code policy_definition} in {@code jcasbin_model.conf}: {@code p = sub,
metadataType,
+ * metadataId, act, eft}.
+ */
+ private static final int POLICY_SUBJECT_FIELD_INDEX = 0;
+
+ /**
+ * How long to wait before retrying a role whose last policy load was
incomplete, i.e. at least
+ * one of its securable objects could not be resolved to a metadata id. Such
a role is
+ * deliberately not recorded in {@link #loadedRoles} (see {@link
#versionCheckAndLoadRoles}), so
+ * without this backoff every request carrying the role would re-read the
role entity and re-probe
+ * the missing object. A role can stay unresolvable indefinitely — for
example when it still
+ * references a dropped table — so the retry has to be throttled rather than
left to the version
+ * check, whose {@code role_meta.updated_at} sentinel never moves in that
case.
+ */
+ private static final long PARTIAL_ROLE_LOAD_RETRY_MS = 10_000L;
+
+ /**
+ * Serializes every mutation of role permission policies, including {@link
+ * #invalidateRolePolicies}, {@link #replaceRolePolicies}, and the policy
writes in {@link
+ * #applyRolePolicies}. Both {@code SyncedEnforcer} calls are individually
atomic, but the {@code
+ * clear -> re-add} sequence is not, and neither is it ordered against the
{@link
+ * JcasbinLoadedRolesCache} removal listener, which calls {@link
#clearRolePoliciesOnCacheRemoval}
+ * from whichever thread happens to drain Caffeine's maintenance queue.
Without this lock an
+ * eviction firing between another thread's policy writes and its {@link
#loadedRoles} update
+ * erases the rows that thread just wrote while leaving the marker saying
they are loaded — a
+ * state no subsequent version check can detect or repair.
+ *
+ * <p>The lock guards in-memory enforcer mutations only: metadata ids are
resolved before it is
+ * taken (see {@link #resolveRolePolicies}), so no DB round-trip ever runs
inside the critical
+ * section. It is reentrant because a {@link #loadedRoles} write performed
under the lock can
+ * itself trigger an eviction, and therefore a nested {@link
#clearRolePoliciesOnCacheRemoval}
+ * call.
+ */
+ private final ReentrantLock rolePolicyLock = new ReentrantLock();
+
/** Jcasbin enforcer is used for metadata authorization. */
private Enforcer allowEnforcer;
@@ -145,6 +183,14 @@ public class JcasbinAuthorizer implements
GravitinoAuthorizer {
*/
private GravitinoCache<Long, Long> loadedRoles;
+ /**
+ * partialRoleLoadBackoff: roleId -> marker present while a role whose last
load was incomplete
+ * must not be retried. Policy replacement writes this marker under {@link
#rolePolicyLock}, so a
+ * delayed loaded-role removal callback can also use its presence to
recognize a newer partial
+ * policy state. See {@link #PARTIAL_ROLE_LOAD_RETRY_MS}.
+ */
+ private GravitinoCache<Long, Boolean> partialRoleLoadBackoff;
+
// ---- Eventual consistency caches (poller-driven) ----
/** Path-based metadata object key -> entity id. Evicted by entity change
poller. */
@@ -182,13 +228,16 @@ public class JcasbinAuthorizer implements
GravitinoAuthorizer {
// Initialize enforcers before caches that reference them in removal
listeners
allowEnforcer = new SyncedEnforcer(getModel("/jcasbin_model.conf"), new
GravitinoAdapter());
- allowInternalAuthorizer = new InternalAuthorizer(allowEnforcer);
+ allowInternalAuthorizer = new InternalAuthorizer(allowEnforcer, true);
denyEnforcer = new SyncedEnforcer(getModel("/jcasbin_model.conf"), new
GravitinoAdapter());
- denyInternalAuthorizer = new InternalAuthorizer(denyEnforcer);
+ denyInternalAuthorizer = new InternalAuthorizer(denyEnforcer, false);
// loadedRoles: roleId -> updated_at.
// When evicted, we must clean up the corresponding JCasbin policies.
- loadedRoles = new JcasbinLoadedRolesCache(ttlMs, roleCacheSize,
this::clearRolePolicies);
+ loadedRoles =
+ new JcasbinLoadedRolesCache(ttlMs, roleCacheSize,
this::clearRolePoliciesOnCacheRemoval);
+ partialRoleLoadBackoff =
+ new CaffeineGravitinoCache<>(Math.min(PARTIAL_ROLE_LOAD_RETRY_MS,
ttlMs), roleCacheSize);
userRoleCache = new CaffeineGravitinoCache<>(ttlMs, roleCacheSize);
groupRoleCache = new CaffeineGravitinoCache<>(ttlMs, roleCacheSize);
@@ -561,7 +610,7 @@ public class JcasbinAuthorizer implements
GravitinoAuthorizer {
@Override
public void handleRolePrivilegeChange(Long roleId) {
- loadedRoles.invalidate(roleId);
+ invalidateRolePolicies(roleId);
}
@Override
@@ -620,6 +669,9 @@ public class JcasbinAuthorizer implements
GravitinoAuthorizer {
if (loadedRoles != null) {
loadedRoles.close();
}
+ if (partialRoleLoadBackoff != null) {
+ partialRoleLoadBackoff.close();
+ }
if (metadataIdCache != null) {
metadataIdCache.close();
}
@@ -656,10 +708,12 @@ public class JcasbinAuthorizer implements
GravitinoAuthorizer {
private class InternalAuthorizer {
- Enforcer enforcer;
+ private final Enforcer enforcer;
+ private final boolean diagnoseDenials;
- public InternalAuthorizer(Enforcer enforcer) {
+ private InternalAuthorizer(Enforcer enforcer, boolean diagnoseDenials) {
this.enforcer = enforcer;
+ this.diagnoseDenials = diagnoseDenials;
}
private boolean authorizeInternal(
@@ -771,11 +825,14 @@ public class JcasbinAuthorizer implements
GravitinoAuthorizer {
return ownerMatchesUserOrGroups(
owner, PrincipalUtils.getCurrentPrincipal(), metalake,
requestContext);
}
- return enforcer.enforce(
- String.valueOf(userId),
- String.valueOf(metadataObject.type()),
- String.valueOf(metadataId),
- privilege);
+ String metadataType = String.valueOf(metadataObject.type());
+ String metadataIdStr = String.valueOf(metadataId);
+ boolean allowed =
+ enforcer.enforce(String.valueOf(userId), metadataType,
metadataIdStr, privilege);
+ if (!allowed && diagnoseDenials) {
+ diagnoseDenial(userId, metadataType, metadataIdStr, privilege);
+ }
+ return allowed;
}
}
@@ -1127,8 +1184,7 @@ public class JcasbinAuthorizer implements
GravitinoAuthorizer {
}
for (Long roleId : uniqueRoleIds) {
if (!existingRoleIds.contains(roleId)) {
- clearRolePolicies(roleId);
- loadedRoles.invalidate(roleId);
+ invalidateRolePolicies(roleId);
}
}
@@ -1138,6 +1194,13 @@ public class JcasbinAuthorizer implements
GravitinoAuthorizer {
if (cachedUpdatedAt.isPresent() && cachedUpdatedAt.get() >=
rv.getUpdatedAt()) {
continue;
}
+ // A role missing from loadedRoles because its last load was incomplete
would otherwise be
+ // retried on every single request. Its partially loaded policies stay
in the enforcer, so
+ // the privileges that did resolve keep working while the backoff is in
effect.
+ if (!cachedUpdatedAt.isPresent()
+ && partialRoleLoadBackoff.getIfPresent(rv.getRoleId()).isPresent()) {
+ continue;
+ }
staleRoleVersions.add(rv);
}
@@ -1189,19 +1252,124 @@ public class JcasbinAuthorizer implements
GravitinoAuthorizer {
}
long roleId = rv.getRoleId();
long dbUpdatedAt = rv.getUpdatedAt();
- Optional<Long> cachedUpdatedAt = loadedRoles.getIfPresent(roleId);
+
+ // Resolve every securable object to a metadata id before touching the
enforcers, so the
+ // critical section below stays free of DB round-trips.
+ ResolvedRolePolicies resolved = resolveRolePolicies(roleEntity,
requestContext);
+
+ if (!replaceRolePolicies(roleId, dbUpdatedAt, resolved)) {
+ // Another request completed this version (or a newer one) while this
request was resolving
+ // metadata ids. Its policies and marker are authoritative, even if
this request's older
+ // resolution result was partial.
+ continue;
+ }
+
+ if (!resolved.isComplete()) {
+ LOG.warn(
+ "Loaded role {} ({}) with {} of {} securable objects; unresolved
objects: {}. "
+ + "The role is not marked as loaded and will be retried in at
most {} ms.",
+ roleId,
+ rv.getRoleName(),
+ roleEntity.securableObjects().size() -
resolved.getUnresolvedObjects().size(),
+ roleEntity.securableObjects().size(),
+ resolved.getUnresolvedObjects(),
+ PARTIAL_ROLE_LOAD_RETRY_MS);
+ }
+ }
+ }
+
+ /**
+ * Replaces a role's policies if no concurrent loader has already installed
the same or a newer
+ * role version.
+ *
+ * <p>Resolution happens outside {@link #rolePolicyLock}, so two requests
can reach this method
+ * with different results for the same role version. The version check must
therefore be repeated
+ * under the mutation lock. Otherwise, a late partial result can clear a
complete result and leave
+ * the complete loader's marker behind.
+ *
+ * @return {@code true} if {@code resolved} was applied, or {@code false} if
a same-or-newer
+ * complete load was already present
+ */
+ private boolean replaceRolePolicies(
+ long roleId, long dbUpdatedAt, ResolvedRolePolicies resolved) {
+ rolePolicyLock.lock();
+ try {
+ Optional<Long> latestLoadedAt = loadedRoles.getIfPresent(roleId);
+ if (latestLoadedAt.isPresent() && latestLoadedAt.get() >= dbUpdatedAt) {
+ partialRoleLoadBackoff.invalidate(roleId);
+ return false;
+ }
// Refresh only permission policies. deleteRole would also remove the
current user's freshly
- // bound grouping links.
- if (cachedUpdatedAt.isPresent()) {
- clearRolePolicies(roleId);
+ // bound grouping links. If a marker exists, its synchronous removal
callback clears the old
+ // policies. With no marker, rows can still remain from a previous
partial load and must be
+ // cleared explicitly.
+ loadedRoles.invalidate(roleId);
+ if (!latestLoadedAt.isPresent()) {
+ clearRolePoliciesWithoutLock(roleId);
+ }
+ applyRolePolicies(resolved);
+
+ // Only record the role as loaded when the enforcer actually received
everything the role
+ // grants. Marking a partially loaded role as loaded would pin it
forever: role_meta's
+ // updated_at does not change, so the version check above would skip the
reload on every later
+ // request and the missing privileges would never come back.
+ if (resolved.isComplete()) {
+ loadedRoles.put(roleId, dbUpdatedAt);
+ partialRoleLoadBackoff.invalidate(roleId);
+ } else {
+ partialRoleLoadBackoff.put(roleId, Boolean.TRUE);
+ }
+ return true;
+ } finally {
+ rolePolicyLock.unlock();
+ }
+ }
+
+ /** Clears a role's policies and removes its loaded marker as one serialized
operation. */
+ private void invalidateRolePolicies(long roleId) {
+ rolePolicyLock.lock();
+ try {
+ // An explicit invalidation is stronger than the retry throttle. Clear
it under the same lock
+ // before removing the loaded marker so the removal callback cannot
mistake the old partial
+ // state for a newer load and skip policy cleanup.
+ partialRoleLoadBackoff.invalidate(roleId);
+ boolean markerPresent = loadedRoles.getIfPresent(roleId).isPresent();
+ loadedRoles.invalidate(roleId);
+ if (!markerPresent) {
+ // An incomplete load has policies but deliberately has no loadedRoles
marker.
+ clearRolePoliciesWithoutLock(roleId);
}
- loadPolicyByRoleEntity(roleEntity, requestContext);
- loadedRoles.put(roleId, dbUpdatedAt);
+ } finally {
+ rolePolicyLock.unlock();
}
}
- private void clearRolePolicies(long roleId) {
+ /**
+ * Handles a loaded-role cache removal after acquiring the policy mutation
lock.
+ *
+ * <p>Caffeine removes the old marker before invoking its listener. While
the listener waits for
+ * this lock, a concurrent loader may install a new policy state: either a
complete load with a
+ * new loaded marker or a partial load with a retry-backoff marker. In
either case this is a stale
+ * removal event and must not clear the newly installed policies.
+ */
+ private void clearRolePoliciesOnCacheRemoval(long roleId) {
+ rolePolicyLock.lock();
+ try {
+ if (loadedRoles.getIfPresent(roleId).isPresent()
+ || partialRoleLoadBackoff.getIfPresent(roleId).isPresent()) {
+ LOG.debug(
+ "Skip clearing policies for role {} because it was reloaded after
cache removal",
+ roleId);
+ return;
+ }
+ clearRolePoliciesWithoutLock(roleId);
+ } finally {
+ rolePolicyLock.unlock();
+ }
+ }
+
+ private void clearRolePoliciesWithoutLock(long roleId) {
String roleIdStr = String.valueOf(roleId);
allowEnforcer.removeFilteredPolicy(0, roleIdStr);
denyEnforcer.removeFilteredPolicy(0, roleIdStr);
@@ -1218,40 +1386,146 @@ public class JcasbinAuthorizer implements
GravitinoAuthorizer {
// Policy loading from role entity
//
---------------------------------------------------------------------------
- private void loadPolicyByRoleEntity(
+ /**
+ * Resolves a role's securable objects into concrete jcasbin {@code p} rows
without touching the
+ * enforcers. Every metadata-id lookup — the only part of policy loading
that can hit the DB —
+ * happens here, so the caller can apply the result while holding {@link
#rolePolicyLock}.
+ *
+ * <p>A securable object that cannot be resolved to a metadata id is
reported in {@link
+ * ResolvedRolePolicies#getUnresolvedObjects()} rather than silently
dropped. That normally means
+ * the object has been dropped while the role still references it, but it is
indistinguishable
+ * from a transient lookup failure, and the two must not be conflated: the
caller relies on this
+ * flag to decide whether the resulting enforcer state is complete enough to
be cached.
+ */
+ private ResolvedRolePolicies resolveRolePolicies(
RoleEntity roleEntity, AuthorizationRequestContext requestContext) {
String metalake =
NameIdentifierUtil.getMetalake(roleEntity.nameIdentifier());
+ String roleIdStr = String.valueOf(roleEntity.id());
List<SecurableObject> securableObjects = roleEntity.securableObjects();
+ List<String[]> allowRows = new ArrayList<>();
+ List<String[]> denyRows = new ArrayList<>();
+ List<String> unresolvedObjects = new ArrayList<>();
+
for (SecurableObject securableObject : securableObjects) {
Optional<Long> metadataId =
lookups.resolveMetadataId(securableObject, metalake, requestContext);
- // A role may still reference a metadata object that has since been
dropped; skip it.
if (!metadataId.isPresent()) {
+ unresolvedObjects.add(securableObject.type().name() + ":" +
securableObject.fullName());
continue;
}
+ String metadataIdStr = String.valueOf(metadataId.get());
for (Privilege privilege : securableObject.privileges()) {
Privilege.Condition condition = privilege.condition();
+ String action =
+ AuthorizationUtils.replaceLegacyPrivilegeName(privilege.name())
+ .name()
+ .toUpperCase(Locale.ROOT);
if (AuthConstants.DENY.equalsIgnoreCase(condition.name())) {
- denyEnforcer.addPolicy(
- String.valueOf(roleEntity.id()),
+ denyRows.add(
+ new String[] {
+ roleIdStr, securableObject.type().name(), metadataIdStr,
action, AuthConstants.ALLOW
+ });
+ }
+
+ allowRows.add(
+ new String[] {
+ roleIdStr,
securableObject.type().name(),
- String.valueOf(metadataId.get()),
- AuthorizationUtils.replaceLegacyPrivilegeName(privilege.name())
- .name()
- .toUpperCase(Locale.ROOT),
- AuthConstants.ALLOW);
+ metadataIdStr,
+ action,
+ condition.name().toLowerCase(Locale.ROOT)
+ });
+ }
+ }
+ return new ResolvedRolePolicies(allowRows, denyRows, unresolvedObjects);
+ }
+
+ /**
+ * Writes pre-resolved policy rows into both enforcers. Must be called under
{@link
+ * #rolePolicyLock}.
+ */
+ private void applyRolePolicies(ResolvedRolePolicies resolved) {
+ for (String[] row : resolved.getDenyRows()) {
+ denyEnforcer.addPolicy(row);
+ }
+ for (String[] row : resolved.getAllowRows()) {
+ allowEnforcer.addPolicy(row);
+ }
+ }
+
+ /**
+ * Reports the enforcer state behind a denied allow-check, so that a denial
caused by missing
+ * policies can be told apart from a denial the caller genuinely earned.
+ *
+ * <p>Escalates to {@code WARN} when it finds a role that {@link
#loadedRoles} claims is loaded
+ * but that carries no {@code p} row in the allow enforcer. That combination
is the signature of a
+ * lost policy load: the version check consults {@code loadedRoles} alone,
so it would keep
+ * skipping the reload for as long as the entry lives.
+ *
+ * <p>Gated on {@code DEBUG} because it scans the enforcer's policy set per
bound role and denials
+ * are an ordinary outcome on the request path. Enable {@code DEBUG} for
this class on the node
+ * under investigation; the diagnosis itself is then reported at {@code
WARN}.
+ */
+ private void diagnoseDenial(
+ long userId, String metadataType, String metadataIdStr, String
privilege) {
+ if (!LOG.isDebugEnabled()) {
+ return;
+ }
+ try {
+ String userIdStr = String.valueOf(userId);
+ List<String> boundRoles = allowEnforcer.getRolesForUser(userIdStr);
+ if (boundRoles.isEmpty()) {
+ LOG.debug(
+ "Denied [{}, {}, {}, {}]: no role is bound to the user in the
allow enforcer",
+ userIdStr,
+ metadataType,
+ metadataIdStr,
+ privilege);
+ return;
+ }
+
+ List<String> rolesWithoutPolicies = new ArrayList<>();
+ List<String> roleStates = new ArrayList<>(boundRoles.size());
+ for (String roleIdStr : boundRoles) {
+ int policyCount =
+ allowEnforcer.getFilteredNamedPolicy("p",
POLICY_SUBJECT_FIELD_INDEX, roleIdStr).size();
+ Optional<Long> loadedAt =
loadedRoles.getIfPresent(Long.parseLong(roleIdStr));
+ roleStates.add(
+ roleIdStr
+ + "{loadedAt="
+ + (loadedAt.isPresent() ? loadedAt.get() : "absent")
+ + ", allowPolicies="
+ + policyCount
+ + "}");
+ if (loadedAt.isPresent() && policyCount == 0) {
+ rolesWithoutPolicies.add(roleIdStr);
}
+ }
- allowEnforcer.addPolicy(
- String.valueOf(roleEntity.id()),
- securableObject.type().name(),
- String.valueOf(metadataId.get()),
- AuthorizationUtils.replaceLegacyPrivilegeName(privilege.name())
- .name()
- .toUpperCase(Locale.ROOT),
- condition.name().toLowerCase(Locale.ROOT));
+ if (rolesWithoutPolicies.isEmpty()) {
+ LOG.debug(
+ "Denied [{}, {}, {}, {}]: role state {}",
+ userIdStr,
+ metadataType,
+ metadataIdStr,
+ privilege,
+ roleStates);
+ } else {
+ LOG.warn(
+ "Denied [{}, {}, {}, {}] while roles {} are recorded as loaded but
hold no allow "
+ + "policy. This node's enforcer lost their policies; the
version check cannot "
+ + "detect it. Full role state: {}",
+ userIdStr,
+ metadataType,
+ metadataIdStr,
+ privilege,
+ rolesWithoutPolicies,
+ roleStates);
}
+ } catch (Exception e) {
+ // A diagnostic must never change the outcome of an authorization check.
+ LOG.debug("Failed to diagnose denial for user {}", userId, e);
}
}
}
diff --git
a/server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/JcasbinLoadedRolesCache.java
b/server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/JcasbinLoadedRolesCache.java
index 07e730761e..3eabb0d6f6 100644
---
a/server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/JcasbinLoadedRolesCache.java
+++
b/server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/JcasbinLoadedRolesCache.java
@@ -27,14 +27,23 @@ import java.util.function.LongConsumer;
import org.apache.gravitino.cache.GravitinoCache;
/**
- * A {@link GravitinoCache} of {@code roleId -> updated_at} that synchronously
deletes the role's
- * JCasbin policies from both enforcers when a key is evicted (by TTL, size,
or explicit
- * invalidate).
+ * A {@link GravitinoCache} of {@code roleId -> updated_at} that synchronously
requests cleanup of
+ * the role's JCasbin policies when a key is evicted (by TTL, size, or
explicit invalidate). The
+ * cleaner may ignore a stale removal when the role was reloaded while the
callback waited for the
+ * authorizer's policy mutation lock.
*
* <p>This cache owns role permission policies only. Therefore, eviction must
clear only {@code
* p(roleId, ...)} policies and must not delete the role itself, because
JCasbin's {@code
* deleteRole(roleId)} also removes {@code g(user/group, roleId)} bindings
that are managed
* separately by {@link JcasbinAuthorizer}.
+ *
+ * <p>The TTL is <b>write-based</b>, matching every other authorization cache.
An access-based TTL
+ * would be renewed by the version probe that {@code versionCheckAndLoadRoles}
performs on every
+ * request, so on a node under steady traffic the entry would never expire.
That matters because the
+ * entry is only a {@code roleId -> updated_at} marker: if the enforcer ever
ends up without the
+ * policies this entry claims are loaded, an access-based TTL turns a
transient inconsistency into a
+ * permanent authorization failure, since each denied request renews the very
entry that suppresses
+ * the reload. A write-based TTL bounds any such state to one TTL.
*/
class JcasbinLoadedRolesCache implements GravitinoCache<Long, Long> {
@@ -43,7 +52,7 @@ class JcasbinLoadedRolesCache implements GravitinoCache<Long,
Long> {
JcasbinLoadedRolesCache(long ttlMs, long maxSize, LongConsumer
rolePolicyCleaner) {
this.cache =
Caffeine.newBuilder()
- .expireAfterAccess(ttlMs, TimeUnit.MILLISECONDS)
+ .expireAfterWrite(ttlMs, TimeUnit.MILLISECONDS)
.maximumSize(maxSize)
.executor(Runnable::run)
.removalListener(
diff --git
a/server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/ResolvedRolePolicies.java
b/server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/ResolvedRolePolicies.java
new file mode 100644
index 0000000000..8ef1cc5ed8
--- /dev/null
+++
b/server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/ResolvedRolePolicies.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.server.authorization.jcasbin;
+
+import java.util.List;
+
+/**
+ * A role's securable objects resolved into concrete jcasbin {@code p} rows,
together with the
+ * objects that could not be resolved to a metadata id.
+ *
+ * <p>Separating resolution from application lets {@code JcasbinAuthorizer}
keep every DB round-trip
+ * outside the lock that serializes enforcer mutations, and lets it tell a
fully loaded role apart
+ * from a partially loaded one — only the former may be recorded as loaded.
+ */
+final class ResolvedRolePolicies {
+
+ private final List<String[]> allowRows;
+ private final List<String[]> denyRows;
+ private final List<String> unresolvedObjects;
+
+ ResolvedRolePolicies(
+ List<String[]> allowRows, List<String[]> denyRows, List<String>
unresolvedObjects) {
+ this.allowRows = allowRows;
+ this.denyRows = denyRows;
+ this.unresolvedObjects = unresolvedObjects;
+ }
+
+ List<String[]> getAllowRows() {
+ return allowRows;
+ }
+
+ List<String[]> getDenyRows() {
+ return denyRows;
+ }
+
+ /** Descriptions of the securable objects whose metadata id could not be
resolved. */
+ List<String> getUnresolvedObjects() {
+ return unresolvedObjects;
+ }
+
+ /** True when every securable object of the role was resolved and turned
into policy rows. */
+ boolean isComplete() {
+ return unresolvedObjects.isEmpty();
+ }
+}
diff --git
a/server-common/src/test/java/org/apache/gravitino/server/authorization/jcasbin/TestJcasbinAuthorizer.java
b/server-common/src/test/java/org/apache/gravitino/server/authorization/jcasbin/TestJcasbinAuthorizer.java
index 740744605b..829fb99e19 100644
---
a/server-common/src/test/java/org/apache/gravitino/server/authorization/jcasbin/TestJcasbinAuthorizer.java
+++
b/server-common/src/test/java/org/apache/gravitino/server/authorization/jcasbin/TestJcasbinAuthorizer.java
@@ -19,6 +19,7 @@ package org.apache.gravitino.server.authorization.jcasbin;
import static org.apache.gravitino.authorization.Privilege.Name.USE_CATALOG;
import static org.apache.gravitino.authorization.Privilege.Name.USE_SCHEMA;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -49,7 +50,11 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.gravitino.Entity;
@@ -62,6 +67,7 @@ import org.apache.gravitino.Namespace;
import org.apache.gravitino.SupportsRelationOperations;
import org.apache.gravitino.UserGroup;
import org.apache.gravitino.UserPrincipal;
+import org.apache.gravitino.auth.AuthConstants;
import org.apache.gravitino.authorization.AuthorizationRequestContext;
import org.apache.gravitino.authorization.Privilege;
import org.apache.gravitino.authorization.SecurableObject;
@@ -543,6 +549,199 @@ public class TestJcasbinAuthorizer {
"loadedRoles entry for the deleted role must be evicted");
}
+ @Test
+ public void testPartialPolicyLoadIsNotRecordedAsLoaded() throws Exception {
+ // Regression test for the permanently-denied-role failure mode. When a
securable object cannot
+ // be resolved to a metadata id, loadPolicyByRoleEntity skips it.
Recording the role as loaded
+ // anyway pins the broken state forever: role_meta.updated_at never moves,
so the version check
+ // keeps skipping the reload and the role's privileges never come back on
that node.
+ makeCompletableFutureUseCurrentThread(jcasbinAuthorizer);
+
+ RoleEntity allowRole =
+ mockRoleInStore(ALLOW_ROLE_ID, "allowRole",
ImmutableList.of(getAllowSecurableObject()));
+ Enforcer allowEnforcer = getAllowEnforcer(jcasbinAuthorizer);
+ GravitinoCache<Long, Long> loadedRoles =
getLoadedRolesCache(jcasbinAuthorizer);
+ GravitinoCache<Long, Boolean> backoff =
getPartialRoleLoadBackoffCache(jcasbinAuthorizer);
+
+ // 1. The role's only securable object does not resolve, so nothing can be
loaded.
+ metadataIdConverterMockedStatic
+ .when(() -> MetadataIdConverter.getID(any(), eq(METALAKE)))
+ .thenReturn(Optional.empty());
+ try {
+ invokeVersionCheckAndLoadRoles(
+ jcasbinAuthorizer,
+ METALAKE,
+ ImmutableList.of(ALLOW_ROLE_ID),
+ new AuthorizationRequestContext());
+
+ assertFalse(
+ loadedRoles.getIfPresent(ALLOW_ROLE_ID).isPresent(),
+ "a role whose policies could not be loaded must not be recorded as
loaded");
+ assertTrue(
+ allowEnforcer.getFilteredPolicy(0,
String.valueOf(ALLOW_ROLE_ID)).isEmpty(),
+ "no p-row can exist when the securable object did not resolve");
+ assertTrue(
+ backoff.getIfPresent(ALLOW_ROLE_ID).isPresent(),
+ "the incomplete load must arm the retry backoff");
+ } finally {
+ metadataIdConverterMockedStatic
+ .when(() -> MetadataIdConverter.getID(any(), eq(METALAKE)))
+ .thenReturn(Optional.of(CATALOG_ID));
+ }
+
+ // 2. While the backoff is armed the role is not re-read, so a request
cannot turn into a DB
+ // round-trip per call for a role that stays unresolvable.
+ invokeVersionCheckAndLoadRoles(
+ jcasbinAuthorizer,
+ METALAKE,
+ ImmutableList.of(ALLOW_ROLE_ID),
+ new AuthorizationRequestContext());
+ assertTrue(
+ allowEnforcer.getFilteredPolicy(0,
String.valueOf(ALLOW_ROLE_ID)).isEmpty(),
+ "the backoff must suppress the immediate retry");
+
+ // 3. Once the backoff lapses the role is retried and, now that the object
resolves, loads
+ // fully and is recorded — this is the self-healing the old code could
never reach.
+ backoff.invalidate(ALLOW_ROLE_ID);
+ invokeVersionCheckAndLoadRoles(
+ jcasbinAuthorizer,
+ METALAKE,
+ ImmutableList.of(ALLOW_ROLE_ID),
+ new AuthorizationRequestContext());
+
+ assertFalse(
+ allowEnforcer.getFilteredPolicy(0,
String.valueOf(allowRole.id())).isEmpty(),
+ "the retry must load the role's p-rows");
+ assertTrue(
+ loadedRoles.getIfPresent(ALLOW_ROLE_ID).isPresent(),
+ "a fully loaded role must be recorded as loaded");
+ assertFalse(
+ backoff.getIfPresent(ALLOW_ROLE_ID).isPresent(),
+ "a successful load must disarm the retry backoff");
+ }
+
+ @Test
+ public void testStaleRemovalDoesNotClearReloadedPolicies() throws Exception {
+ Enforcer allowEnforcer = getAllowEnforcer(jcasbinAuthorizer);
+ GravitinoCache<Long, Long> loadedRoles =
getLoadedRolesCache(jcasbinAuthorizer);
+ ReentrantLock rolePolicyLock = getRolePolicyLock(jcasbinAuthorizer);
+ String roleIdStr = String.valueOf(ALLOW_ROLE_ID);
+ String[] policyRow =
+ new String[] {
+ roleIdStr,
+ MetadataObject.Type.CATALOG.name(),
+ String.valueOf(CATALOG_ID),
+ USE_CATALOG.name(),
+ AuthConstants.ALLOW
+ };
+ allowEnforcer.addPolicy(policyRow);
+ loadedRoles.put(ALLOW_ROLE_ID, 1L);
+
+ CountDownLatch invalidationStarted = new CountDownLatch(1);
+ AtomicReference<Throwable> failure = new AtomicReference<>();
+ rolePolicyLock.lock();
+ Thread invalidator =
+ new Thread(
+ () -> {
+ invalidationStarted.countDown();
+ try {
+ loadedRoles.invalidate(ALLOW_ROLE_ID);
+ } catch (Throwable t) {
+ failure.set(t);
+ }
+ });
+ invalidator.setDaemon(true);
+ try {
+ invalidator.start();
+ assertTrue(invalidationStarted.await(5, TimeUnit.SECONDS));
+
+ // Caffeine removes the marker before its listener waits for
rolePolicyLock. Wait until that
+ // ordering is visible, then simulate a loader installing a fresh policy
set and marker while
+ // the old removal callback is still blocked.
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (loadedRoles.getIfPresent(ALLOW_ROLE_ID).isPresent() &&
System.nanoTime() < deadline) {
+ Thread.yield();
+ }
+ assertFalse(
+ loadedRoles.getIfPresent(ALLOW_ROLE_ID).isPresent(),
+ "the invalidation must remove the old marker before its listener
acquires the lock");
+
+ allowEnforcer.removeFilteredPolicy(0, roleIdStr);
+ allowEnforcer.addPolicy(policyRow);
+ loadedRoles.put(ALLOW_ROLE_ID, 2L);
+ } finally {
+ rolePolicyLock.unlock();
+ }
+
+ invalidator.join(5000L);
+ assertFalse(invalidator.isAlive(), "the invalidation thread must finish");
+ Assertions.assertNull(failure.get(), "the invalidation thread must not
fail");
+ assertEquals(2L, loadedRoles.getIfPresent(ALLOW_ROLE_ID).orElse(null));
+ assertFalse(
+ allowEnforcer.getFilteredPolicy(0, roleIdStr).isEmpty(),
+ "a stale removal callback must not clear policies installed by a later
load");
+ }
+
+ @Test
+ public void testStaleRemovalDoesNotClearNewPartialPolicies() throws
Exception {
+ Enforcer allowEnforcer = getAllowEnforcer(jcasbinAuthorizer);
+ GravitinoCache<Long, Boolean> backoff =
getPartialRoleLoadBackoffCache(jcasbinAuthorizer);
+ String roleIdStr = String.valueOf(ALLOW_ROLE_ID);
+ allowEnforcer.addPolicy(
+ roleIdStr,
+ MetadataObject.Type.CATALOG.name(),
+ String.valueOf(CATALOG_ID),
+ USE_CATALOG.name(),
+ AuthConstants.ALLOW);
+ backoff.put(ALLOW_ROLE_ID, Boolean.TRUE);
+
+ // A partial loader records its backoff marker under rolePolicyLock before
an older cache
+ // removal callback can resume. The callback must treat that marker as a
newer policy state even
+ // though partial loads deliberately have no loadedRoles marker.
+ invokeClearRolePoliciesOnCacheRemoval(jcasbinAuthorizer, ALLOW_ROLE_ID);
+
+ assertFalse(
+ allowEnforcer.getFilteredPolicy(0, roleIdStr).isEmpty(),
+ "a stale removal callback must not clear a newer partial policy set");
+ }
+
+ @Test
+ public void testPartialResultCannotOverwriteCompletedConcurrentLoad() throws
Exception {
+ String roleIdStr = String.valueOf(ALLOW_ROLE_ID);
+ String[] policyRow =
+ new String[] {
+ roleIdStr,
+ MetadataObject.Type.CATALOG.name(),
+ String.valueOf(CATALOG_ID),
+ USE_CATALOG.name(),
+ AuthConstants.ALLOW
+ };
+ ResolvedRolePolicies complete =
+ new ResolvedRolePolicies(
+ ImmutableList.<String[]>of(policyRow),
+ Collections.emptyList(),
+ Collections.emptyList());
+ ResolvedRolePolicies partial =
+ new ResolvedRolePolicies(
+ Collections.emptyList(),
+ Collections.emptyList(),
+ ImmutableList.of("CATALOG:testCatalog"));
+ long roleVersion = 42L;
+
+ // Both results may be resolved before either request acquires
rolePolicyLock. Once the complete
+ // result wins, a late partial result for the same version must not clear
its policies or leave
+ // its loaded marker attached to a partial policy set.
+ assertTrue(invokeReplaceRolePolicies(jcasbinAuthorizer, ALLOW_ROLE_ID,
roleVersion, complete));
+ assertFalse(invokeReplaceRolePolicies(jcasbinAuthorizer, ALLOW_ROLE_ID,
roleVersion, partial));
+
+ assertEquals(
+ roleVersion,
+
getLoadedRolesCache(jcasbinAuthorizer).getIfPresent(ALLOW_ROLE_ID).orElse(null));
+ assertFalse(
+ getAllowEnforcer(jcasbinAuthorizer).getFilteredPolicy(0,
roleIdStr).isEmpty(),
+ "the completed load's policies must survive a late partial result");
+ }
+
/** Reflectively invoke the private versionCheckAndLoadRoles. */
private static void invokeVersionCheckAndLoadRoles(
JcasbinAuthorizer authorizer,
@@ -560,6 +759,27 @@ public class TestJcasbinAuthorizer {
m.invoke(authorizer, metalake, roleIds, requestContext);
}
+ private static boolean invokeReplaceRolePolicies(
+ JcasbinAuthorizer authorizer,
+ long roleId,
+ long updatedAt,
+ ResolvedRolePolicies resolvedRolePolicies)
+ throws Exception {
+ Method method =
+ JcasbinAuthorizer.class.getDeclaredMethod(
+ "replaceRolePolicies", long.class, long.class,
ResolvedRolePolicies.class);
+ method.setAccessible(true);
+ return (boolean) method.invoke(authorizer, roleId, updatedAt,
resolvedRolePolicies);
+ }
+
+ private static void invokeClearRolePoliciesOnCacheRemoval(
+ JcasbinAuthorizer authorizer, long roleId) throws Exception {
+ Method method =
+
JcasbinAuthorizer.class.getDeclaredMethod("clearRolePoliciesOnCacheRemoval",
long.class);
+ method.setAccessible(true);
+ method.invoke(authorizer, roleId);
+ }
+
@Test
public void testAuthorizeByOwner() throws Exception {
Principal currentPrincipal = PrincipalUtils.getCurrentPrincipal();
@@ -1613,9 +1833,10 @@ public class TestJcasbinAuthorizer {
}
@Test
- public void testClearRolePoliciesPreservesUserRoleBindings() throws
Exception {
+ public void testInvalidateRolePoliciesPreservesUserRoleBindings() throws
Exception {
Enforcer allowEnforcer = getAllowEnforcer(jcasbinAuthorizer);
Enforcer denyEnforcer = getDenyEnforcer(jcasbinAuthorizer);
+ GravitinoCache<Long, Boolean> backoff =
getPartialRoleLoadBackoffCache(jcasbinAuthorizer);
Long testRoleId = 302L;
String roleIdStr = String.valueOf(testRoleId);
@@ -1624,16 +1845,18 @@ public class TestJcasbinAuthorizer {
denyEnforcer.addRoleForUser(userIdStr, roleIdStr);
allowEnforcer.addPolicy(roleIdStr, "CATALOG", "999", "USE_CATALOG",
"allow");
denyEnforcer.addPolicy(roleIdStr, "CATALOG", "999", "USE_CATALOG",
"allow");
+ backoff.put(testRoleId, Boolean.TRUE);
- Method clearRolePolicies =
- JcasbinAuthorizer.class.getDeclaredMethod("clearRolePolicies",
long.class);
- clearRolePolicies.setAccessible(true);
- clearRolePolicies.invoke(jcasbinAuthorizer, testRoleId);
+ Method invalidateRolePolicies =
+ JcasbinAuthorizer.class.getDeclaredMethod("invalidateRolePolicies",
long.class);
+ invalidateRolePolicies.setAccessible(true);
+ invalidateRolePolicies.invoke(jcasbinAuthorizer, testRoleId);
assertFalse(allowEnforcer.hasPolicy(roleIdStr, "CATALOG", "999",
"USE_CATALOG", "allow"));
assertFalse(denyEnforcer.hasPolicy(roleIdStr, "CATALOG", "999",
"USE_CATALOG", "allow"));
assertTrue(allowEnforcer.getRolesForUser(userIdStr).contains(roleIdStr));
assertTrue(denyEnforcer.getRolesForUser(userIdStr).contains(roleIdStr));
+ assertFalse(backoff.getIfPresent(testRoleId).isPresent());
}
@Test
@@ -2066,6 +2289,20 @@ public class TestJcasbinAuthorizer {
return (GravitinoCache<Long, Long>) field.get(authorizer);
}
+ private static ReentrantLock getRolePolicyLock(JcasbinAuthorizer authorizer)
throws Exception {
+ Field field = JcasbinAuthorizer.class.getDeclaredField("rolePolicyLock");
+ field.setAccessible(true);
+ return (ReentrantLock) field.get(authorizer);
+ }
+
+ @SuppressWarnings("unchecked")
+ private static GravitinoCache<Long, Boolean> getPartialRoleLoadBackoffCache(
+ JcasbinAuthorizer authorizer) throws Exception {
+ Field field =
JcasbinAuthorizer.class.getDeclaredField("partialRoleLoadBackoff");
+ field.setAccessible(true);
+ return (GravitinoCache<Long, Boolean>) field.get(authorizer);
+ }
+
@SuppressWarnings("unchecked")
private static GravitinoCache<Long, Optional<OwnerInfo>> getOwnerRelCache(
JcasbinAuthorizer authorizer) throws Exception {
diff --git
a/server-common/src/test/java/org/apache/gravitino/server/authorization/jcasbin/TestJcasbinLoadedRolesCache.java
b/server-common/src/test/java/org/apache/gravitino/server/authorization/jcasbin/TestJcasbinLoadedRolesCache.java
new file mode 100644
index 0000000000..77ad844669
--- /dev/null
+++
b/server-common/src/test/java/org/apache/gravitino/server/authorization/jcasbin/TestJcasbinLoadedRolesCache.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.server.authorization.jcasbin;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/** Tests for the {@code roleId -> updated_at} cache that owns role permission
policies. */
+public class TestJcasbinLoadedRolesCache {
+
+ private static final long ROLE_ID = 42L;
+
+ @Test
+ public void testTtlIsWriteBasedSoReadsCannotKeepAnEntryAlive() throws
Exception {
+ // versionCheckAndLoadRoles probes this cache on every request that
carries the role. Under an
+ // access-based TTL those probes renew the entry, so on a node under
steady traffic the entry
+ // never expires. That is what turns a lost policy load into a permanent
authorization failure:
+ // each denied request renews the very entry that tells the version check
to skip the reload.
+ long ttlMs = 150L;
+ List<Long> cleaned = new ArrayList<>();
+ JcasbinLoadedRolesCache cache = new JcasbinLoadedRolesCache(ttlMs, 100L,
cleaned::add);
+ cache.put(ROLE_ID, 1L);
+
+ // Read the entry far more often than the TTL, the way a hot role is
probed in production.
+ long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(ttlMs *
4);
+ while (System.nanoTime() < deadline) {
+ cache.getIfPresent(ROLE_ID);
+ Thread.sleep(10L);
+ }
+
+ Assertions.assertFalse(
+ cache.getIfPresent(ROLE_ID).isPresent(),
+ "repeated reads must not keep the entry alive past its TTL");
+ // size() runs Caffeine's maintenance, which delivers any pending removal
notification.
+ cache.size();
+ Assertions.assertTrue(
+ cleaned.contains(ROLE_ID), "expiring the entry must clear the role's
policies");
+ cache.close();
+ }
+
+ @Test
+ public void testReplacingAnEntryDoesNotClearPolicies() {
+ // A refresh writes the new version over the old one. Treating that as a
removal would delete
+ // the policies the refresh just loaded.
+ List<Long> cleaned = new ArrayList<>();
+ JcasbinLoadedRolesCache cache = new JcasbinLoadedRolesCache(60_000L, 100L,
cleaned::add);
+
+ cache.put(ROLE_ID, 1L);
+ cache.put(ROLE_ID, 2L);
+ cache.size();
+
+ Assertions.assertTrue(cleaned.isEmpty(), "replacing a value must not clear
the role policies");
+ Assertions.assertEquals(2L, cache.getIfPresent(ROLE_ID).orElse(null));
+
+ cache.invalidate(ROLE_ID);
+ Assertions.assertTrue(
+ cleaned.contains(ROLE_ID), "explicit invalidation must clear the role
policies");
+ cache.close();
+ }
+}
