autoCAT6 commented on code in PR #14045:
URL: https://github.com/apache/gravitino/pull/14045#discussion_r4427523286
##########
server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/JcasbinAuthorizer.java:
##########
@@ -1416,102 +1437,343 @@ private Set<Long> versionCheckAndLoadRoles(
List<RoleUpdatedAt> staleRoleVersions = new ArrayList<>();
for (RoleUpdatedAt rv : roleVersions) {
- Optional<Long> cachedUpdatedAt =
loadedRoles.getIfPresent(rv.getRoleId());
- if (cachedUpdatedAt.isPresent() && cachedUpdatedAt.get() >=
rv.getUpdatedAt()) {
- continue;
- }
- // A role missing from loadedRoles because its last load was incomplete
would otherwise be
- // retried on every single request. Its partially loaded policies stay
in the enforcer, so
- // the privileges that did resolve keep working while the backoff is in
effect.
- if (!cachedUpdatedAt.isPresent()
- && partialRoleLoadBackoff.getIfPresent(rv.getRoleId()).isPresent()) {
- continue;
+ if (needsRolePolicyLoad(rv)) {
+ staleRoleVersions.add(rv);
}
- staleRoleVersions.add(rv);
}
if (staleRoleVersions.isEmpty()) {
return Collections.emptySet();
}
+ List<Long> leasedRoles = new ArrayList<>();
+ try {
+ return coalesceRolePolicyLoads(metalake, staleRoleVersions,
requestContext, leasedRoles);
+ } finally {
+ releaseRolePolicyLoadStates(leasedRoles);
+ }
+ }
+
+ private Set<Long> coalesceRolePolicyLoads(
+ String metalake,
+ List<RoleUpdatedAt> staleRoleVersions,
+ AuthorizationRequestContext requestContext,
+ List<Long> leasedRoles) {
+ Map<RolePolicyLoadKey, CompletableFuture<Boolean>> requestLoads = new
LinkedHashMap<>();
+ List<RolePolicyLoadKey> ownedLoads = new ArrayList<>();
+ try {
+ for (RolePolicyLoadKey key :
retainRolePolicyLoadStates(staleRoleVersions, leasedRoles)) {
+ CompletableFuture<Boolean> result = new CompletableFuture<>();
+ CompletableFuture<Boolean> existing = rolePolicyLoads.putIfAbsent(key,
result);
+ requestLoads.put(key, existing == null ? result : existing);
+ if (existing == null) {
+ ownedLoads.add(key);
+ }
+ }
+
+ // Claim every role before doing I/O, and finish every load we own
before waiting for another
+ // request. Waiting first could deadlock requests owning opposite halves
of a role batch.
+ List<RolePolicyLoadKey> pendingLoads = new ArrayList<>();
+ // Recheck after claiming: a request can observe a stale marker, be
delayed, and become the
+ // owner after the previous owner has already completed and removed its
flight.
+ for (RolePolicyLoadKey key : ownedLoads) {
+ if (needsRolePolicyLoad(key.roleVersion)) {
+ pendingLoads.add(key);
+ } else {
+ requestLoads.get(key).complete(true);
+ }
+ }
+ loadRolePolicies(metalake, pendingLoads, requestContext, requestLoads);
+ } catch (RuntimeException | Error e) {
+ for (RolePolicyLoadKey key : ownedLoads) {
+ requestLoads.get(key).completeExceptionally(e);
+ }
+ throw e;
+ } finally {
+ for (RolePolicyLoadKey key : ownedLoads) {
+ rolePolicyLoads.remove(key, requestLoads.get(key));
+ }
+ }
+
+ Set<Long> unreadableRoleIds = new HashSet<>();
+ for (Map.Entry<RolePolicyLoadKey, CompletableFuture<Boolean>> load :
requestLoads.entrySet()) {
+ if (!awaitRolePolicyLoad(load.getKey(), load.getValue())) {
+ unreadableRoleIds.add(load.getKey().roleVersion.getRoleId());
+ }
+ }
+ return unreadableRoleIds;
+ }
+
+ /**
+ * Captures the clear generation of every stale role and retains its version
floor, all in one
+ * critical section. The write lock is shared with every enforce() call, so
it is taken once per
+ * request rather than once per role. Pruning removes map entries before
publishing its watermark
+ * under the same lock, so a captured generation never observes the gap
between the two.
+ */
+ private List<RolePolicyLoadKey> retainRolePolicyLoadStates(
+ List<RoleUpdatedAt> staleRoleVersions, List<Long> leasedRoles) {
+ List<RolePolicyLoadKey> keys = new ArrayList<>(staleRoleVersions.size());
+ rolePolicyLock.writeLock().lock();
+ try {
+ for (RoleUpdatedAt rv : staleRoleVersions) {
+ long roleId = rv.getRoleId();
+ RolePolicyLoadKey key = new RolePolicyLoadKey(rv,
roleClearGenerationWithoutLock(roleId));
+ RolePolicyLoadState state = rolePolicyLoadStates.get(roleId);
+ if (state == null) {
+ rolePolicyLoadStates.put(roleId, new
RolePolicyLoadState(rv.getUpdatedAt()));
+ } else {
+ state.highestVersion = Math.max(state.highestVersion,
rv.getUpdatedAt());
+ state.references++;
+ }
+ leasedRoles.add(roleId);
+ keys.add(key);
+ }
+ } finally {
+ rolePolicyLock.writeLock().unlock();
+ }
+ return keys;
+ }
+
+ private void releaseRolePolicyLoadStates(List<Long> leasedRoles) {
+ rolePolicyLock.writeLock().lock();
+ try {
+ for (Long roleId : leasedRoles) {
+ RolePolicyLoadState state = rolePolicyLoadStates.get(roleId);
+ if (--state.references == 0) {
+ rolePolicyLoadStates.remove(roleId);
+ }
+ }
+ } finally {
+ rolePolicyLock.writeLock().unlock();
+ }
+ }
+
+ private boolean needsRolePolicyLoad(RoleUpdatedAt roleVersion) {
+ Optional<Long> cachedUpdatedAt =
loadedRoles.getIfPresent(roleVersion.getRoleId());
+ if (cachedUpdatedAt.isPresent() && cachedUpdatedAt.get() >=
roleVersion.getUpdatedAt()) {
+ return false;
+ }
+ // Partially loaded policies stay usable during their existing retry
backoff, even though
+ // they deliberately have no loadedRoles marker.
+ return cachedUpdatedAt.isPresent()
+ ||
!partialRoleLoadBackoff.getIfPresent(roleVersion.getRoleId()).isPresent();
+ }
+
+ private boolean awaitRolePolicyLoad(RolePolicyLoadKey key,
CompletableFuture<Boolean> result) {
+ try {
+ return result.get();
+ } catch (InterruptedException e) {
+ // Interrupting a waiter must not cancel the load other requests still
depend on.
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Interrupted while waiting for role policy load " +
key.roleVersion.getRoleId(), e);
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ }
+ if (cause instanceof Error) {
+ throw (Error) cause;
+ }
+ throw new IllegalStateException("Failed to load role policies", cause);
+ }
+ }
+
+ private void loadRolePolicies(
+ String metalake,
+ List<RolePolicyLoadKey> pendingLoads,
+ AuthorizationRequestContext requestContext,
+ Map<RolePolicyLoadKey, CompletableFuture<Boolean>> requestLoads) {
+ if (pendingLoads.isEmpty()) {
+ return;
+ }
Set<Long> unreadableRoleIds = new HashSet<>();
EntityStore entityStore = GravitinoEnv.getInstance().entityStore();
List<NameIdentifier> roleIdents =
- staleRoleVersions.stream()
- .map(rv -> NameIdentifierUtil.ofRole(metalake, rv.getRoleName()))
+ pendingLoads.stream()
+ .map(key -> NameIdentifierUtil.ofRole(metalake,
key.roleVersion.getRoleName()))
.collect(Collectors.toList());
List<RoleEntity> roleEntities;
try {
roleEntities = entityStore.batchGet(roleIdents, Entity.EntityType.ROLE,
RoleEntity.class);
} catch (Exception e) {
- LOG.warn("Failed to batch load stale role policies for roleIds {}",
staleRoleVersions, e);
- roleEntities = new ArrayList<>();
- }
- if (roleEntities == null) {
- roleEntities = new ArrayList<>();
- }
- // Some EntityStore implementations don't support batchGet for ROLE and
return empty;
- // fall back to per-role get so policies still load.
- if (roleEntities.isEmpty()) {
- roleEntities = new ArrayList<>(staleRoleVersions.size());
- for (RoleUpdatedAt rv : staleRoleVersions) {
- try {
- roleEntities.add(
- entityStore.get(
- NameIdentifierUtil.ofRole(metalake, rv.getRoleName()),
- Entity.EntityType.ROLE,
- RoleEntity.class));
- } catch (NoSuchEntityException e) {
- LOG.debug("Role {} was dropped before its policies could be loaded",
rv.getRoleId(), e);
- } catch (Exception e) {
- LOG.warn("Failed to load role policies for roleId {}",
rv.getRoleId(), e);
- unreadableRoleIds.add(rv.getRoleId());
+ LOG.warn("Failed to batch load stale role policies for roles {}",
roleIdents, e);
+ roleEntities = Collections.emptyList();
+ }
+ Map<Long, RoleEntity> roleEntitiesById = new HashMap<>();
+ if (roleEntities != null) {
+ for (RoleEntity roleEntity : roleEntities) {
+ if (roleEntity != null) {
+ roleEntitiesById.put(roleEntity.id(), roleEntity);
}
}
}
- Map<Long, RoleUpdatedAt> staleRoleVersionById =
-
staleRoleVersions.stream().collect(Collectors.toMap(RoleUpdatedAt::getRoleId,
rv -> rv));
- for (RoleEntity roleEntity : roleEntities) {
- if (roleEntity == null) {
- continue;
- }
- RoleUpdatedAt rv = staleRoleVersionById.get(roleEntity.id());
- if (rv == null) {
- continue;
+ for (RolePolicyLoadKey key : pendingLoads) {
+ CompletableFuture<Boolean> result = requestLoads.get(key);
+ try {
+ result.complete(
+ loadRolePolicy(
+ entityStore,
+ metalake,
+ key,
+ roleEntitiesById.get(key.roleVersion.getRoleId()),
+ requestContext,
+ unreadableRoleIds));
+ } catch (RuntimeException | Error e) {
+ // A failed role must not fail another role's waiters, including roles
still to be loaded.
+ // Settle each owned flight before the caller awaits results and
propagates its failure.
+ result.completeExceptionally(e);
}
- long roleId = rv.getRoleId();
- long dbUpdatedAt = rv.getUpdatedAt();
-
- // Resolve every securable object to a metadata id before touching the
enforcers, so the
- // critical section below stays free of DB round-trips.
- Optional<ResolvedRolePolicies> resolution =
resolveRolePolicies(roleEntity, requestContext);
- if (!resolution.isPresent()) {
- unreadableRoleIds.add(roleId);
- continue;
+ }
+ }
+
+ private boolean loadRolePolicy(
+ EntityStore entityStore,
+ String metalake,
+ RolePolicyLoadKey key,
+ @Nullable RoleEntity roleEntity,
+ AuthorizationRequestContext requestContext,
+ Set<Long> unreadableRoleIds) {
+ RoleUpdatedAt rv = key.roleVersion;
+ long roleId = rv.getRoleId();
+ // Stores may not support batchGet for ROLE, or may return only part of a
batch. Every owned
+ // role needs a concrete outcome so all its waiters receive the same read
failure, if any.
+ if (roleEntity == null) {
+ try {
+ roleEntity =
+ entityStore.get(
+ NameIdentifierUtil.ofRole(metalake, rv.getRoleName()),
+ Entity.EntityType.ROLE,
+ RoleEntity.class);
+ } catch (NoSuchEntityException e) {
+ LOG.debug("Role {} was dropped before its policies could be loaded",
roleId, e);
+ invalidateRolePolicies(roleId);
+ return true;
+ } catch (Exception e) {
+ LOG.warn("Failed to load role policies for roleId {}", roleId, e);
+ return false;
}
- ResolvedRolePolicies resolved = resolution.get();
+ }
+ if (roleEntity == null) {
+ return false;
+ }
+ if (roleEntity.id() != roleId) {
+ LOG.warn(
+ "Role {} ({}) was read with id {}: it was dropped and recreated
after its version was"
+ + " probed, so its policies are not loaded and requests on it
fail closed",
+ roleId,
+ rv.getRoleName(),
+ roleEntity.id());
+ return false;
+ }
- if (!replaceRolePolicies(roleId, dbUpdatedAt, resolved)) {
- // Another request completed this version (or a newer one) while this
request was resolving
- // metadata ids. Its policies and marker are authoritative, even if
this request's older
- // resolution result was partial.
- continue;
+ // Resolve every securable object to a metadata id before touching the
enforcers, so the
+ // critical section below stays free of DB round-trips.
+ Optional<ResolvedRolePolicies> resolution =
resolveRolePolicies(roleEntity, requestContext);
+ if (!resolution.isPresent()) {
+ return false;
+ }
+ ResolvedRolePolicies resolved = resolution.get();
+
+ if (!replaceUnclearedRolePolicies(key, resolved, unreadableRoleIds)) {
+ // A newer load or an invalidation superseded this work. The caller
retains its pre-load
+ // request generation, so the existing bounded evaluation loop reloads
cleared policies.
+ return !unreadableRoleIds.contains(roleId);
+ }
+
+ if (!resolved.isComplete()) {
+ LOG.warn(
+ "Loaded role {} ({}) with {} of {} securable objects; unresolved
objects: {}. "
+ + "The role is not marked as loaded and will be retried in at
most {} ms.",
+ roleId,
+ rv.getRoleName(),
+ roleEntity.securableObjects().size() -
resolved.getUnresolvedObjects().size(),
+ roleEntity.securableObjects().size(),
+ resolved.getUnresolvedObjects(),
+ PARTIAL_ROLE_LOAD_RETRY_MS);
+ }
+ return true;
+ }
+
+ private long roleClearGenerationWithoutLock(long roleId) {
+ return Math.max(roleClearGenerations.getOrDefault(roleId, 0L),
prunedRoleClearGeneration);
+ }
+
+ private boolean replaceUnclearedRolePolicies(
+ RolePolicyLoadKey key, ResolvedRolePolicies resolved, Set<Long>
unreadableRoleIds) {
+ rolePolicyLock.writeLock().lock();
+ try {
+ long roleId = key.roleVersion.getRoleId();
+ if (roleClearGenerationWithoutLock(roleId) != key.clearGeneration) {
+ return false;
+ }
+ RolePolicyLoadState state = rolePolicyLoadStates.get(roleId);
+ if (state.highestVersion > key.roleVersion.getUpdatedAt()) {
+ Optional<Long> loadedVersion = loadedRoles.getIfPresent(roleId);
+ if (!loadedVersion.isPresent() || loadedVersion.get() <
state.highestVersion) {
+ // A newer partial load (or failed read) may have no loaded marker.
Do not overwrite it
+ // with an old snapshot. Unlike an epoch change, this has no
generation repair signal,
+ // so explicitly fail this load and every waiter closed.
+ LOG.warn(
+ "Discarded the policies of role {} loaded at version {}, as
version {} was observed"
+ + " concurrently without a complete load; failing closed",
+ roleId,
+ key.roleVersion.getUpdatedAt(),
+ state.highestVersion);
+ unreadableRoleIds.add(roleId);
Review Comment:
Failing closed when a newer observed version lacks a complete replacement is
intentional. Repeated marker invalidation can also exhaust the bounded reload
loop. I added your A 6 vs 0 and B 2 vs 2 observations to the PR body, but
without the logs I cannot attribute those counts to this branch. The body names
the warnings that distinguish the two paths.
##########
server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/JcasbinAuthorizer.java:
##########
@@ -1416,102 +1437,343 @@ private Set<Long> versionCheckAndLoadRoles(
List<RoleUpdatedAt> staleRoleVersions = new ArrayList<>();
for (RoleUpdatedAt rv : roleVersions) {
- Optional<Long> cachedUpdatedAt =
loadedRoles.getIfPresent(rv.getRoleId());
- if (cachedUpdatedAt.isPresent() && cachedUpdatedAt.get() >=
rv.getUpdatedAt()) {
- continue;
- }
- // A role missing from loadedRoles because its last load was incomplete
would otherwise be
- // retried on every single request. Its partially loaded policies stay
in the enforcer, so
- // the privileges that did resolve keep working while the backoff is in
effect.
- if (!cachedUpdatedAt.isPresent()
- && partialRoleLoadBackoff.getIfPresent(rv.getRoleId()).isPresent()) {
- continue;
+ if (needsRolePolicyLoad(rv)) {
+ staleRoleVersions.add(rv);
}
- staleRoleVersions.add(rv);
}
if (staleRoleVersions.isEmpty()) {
return Collections.emptySet();
}
+ List<Long> leasedRoles = new ArrayList<>();
+ try {
+ return coalesceRolePolicyLoads(metalake, staleRoleVersions,
requestContext, leasedRoles);
+ } finally {
+ releaseRolePolicyLoadStates(leasedRoles);
+ }
+ }
+
+ private Set<Long> coalesceRolePolicyLoads(
+ String metalake,
+ List<RoleUpdatedAt> staleRoleVersions,
+ AuthorizationRequestContext requestContext,
+ List<Long> leasedRoles) {
+ Map<RolePolicyLoadKey, CompletableFuture<Boolean>> requestLoads = new
LinkedHashMap<>();
+ List<RolePolicyLoadKey> ownedLoads = new ArrayList<>();
+ try {
+ for (RolePolicyLoadKey key :
retainRolePolicyLoadStates(staleRoleVersions, leasedRoles)) {
+ CompletableFuture<Boolean> result = new CompletableFuture<>();
+ CompletableFuture<Boolean> existing = rolePolicyLoads.putIfAbsent(key,
result);
+ requestLoads.put(key, existing == null ? result : existing);
+ if (existing == null) {
+ ownedLoads.add(key);
+ }
+ }
+
+ // Claim every role before doing I/O, and finish every load we own
before waiting for another
+ // request. Waiting first could deadlock requests owning opposite halves
of a role batch.
+ List<RolePolicyLoadKey> pendingLoads = new ArrayList<>();
+ // Recheck after claiming: a request can observe a stale marker, be
delayed, and become the
+ // owner after the previous owner has already completed and removed its
flight.
+ for (RolePolicyLoadKey key : ownedLoads) {
+ if (needsRolePolicyLoad(key.roleVersion)) {
+ pendingLoads.add(key);
+ } else {
+ requestLoads.get(key).complete(true);
+ }
+ }
+ loadRolePolicies(metalake, pendingLoads, requestContext, requestLoads);
+ } catch (RuntimeException | Error e) {
+ for (RolePolicyLoadKey key : ownedLoads) {
+ requestLoads.get(key).completeExceptionally(e);
+ }
+ throw e;
+ } finally {
+ for (RolePolicyLoadKey key : ownedLoads) {
+ rolePolicyLoads.remove(key, requestLoads.get(key));
+ }
+ }
+
+ Set<Long> unreadableRoleIds = new HashSet<>();
+ for (Map.Entry<RolePolicyLoadKey, CompletableFuture<Boolean>> load :
requestLoads.entrySet()) {
+ if (!awaitRolePolicyLoad(load.getKey(), load.getValue())) {
+ unreadableRoleIds.add(load.getKey().roleVersion.getRoleId());
+ }
+ }
+ return unreadableRoleIds;
+ }
+
+ /**
+ * Captures the clear generation of every stale role and retains its version
floor, all in one
+ * critical section. The write lock is shared with every enforce() call, so
it is taken once per
+ * request rather than once per role. Pruning removes map entries before
publishing its watermark
+ * under the same lock, so a captured generation never observes the gap
between the two.
+ */
+ private List<RolePolicyLoadKey> retainRolePolicyLoadStates(
+ List<RoleUpdatedAt> staleRoleVersions, List<Long> leasedRoles) {
+ List<RolePolicyLoadKey> keys = new ArrayList<>(staleRoleVersions.size());
+ rolePolicyLock.writeLock().lock();
+ try {
+ for (RoleUpdatedAt rv : staleRoleVersions) {
+ long roleId = rv.getRoleId();
+ RolePolicyLoadKey key = new RolePolicyLoadKey(rv,
roleClearGenerationWithoutLock(roleId));
+ RolePolicyLoadState state = rolePolicyLoadStates.get(roleId);
+ if (state == null) {
+ rolePolicyLoadStates.put(roleId, new
RolePolicyLoadState(rv.getUpdatedAt()));
+ } else {
+ state.highestVersion = Math.max(state.highestVersion,
rv.getUpdatedAt());
+ state.references++;
+ }
+ leasedRoles.add(roleId);
+ keys.add(key);
+ }
+ } finally {
+ rolePolicyLock.writeLock().unlock();
+ }
+ return keys;
+ }
+
+ private void releaseRolePolicyLoadStates(List<Long> leasedRoles) {
+ rolePolicyLock.writeLock().lock();
+ try {
+ for (Long roleId : leasedRoles) {
+ RolePolicyLoadState state = rolePolicyLoadStates.get(roleId);
+ if (--state.references == 0) {
+ rolePolicyLoadStates.remove(roleId);
+ }
+ }
+ } finally {
+ rolePolicyLock.writeLock().unlock();
+ }
+ }
+
+ private boolean needsRolePolicyLoad(RoleUpdatedAt roleVersion) {
+ Optional<Long> cachedUpdatedAt =
loadedRoles.getIfPresent(roleVersion.getRoleId());
+ if (cachedUpdatedAt.isPresent() && cachedUpdatedAt.get() >=
roleVersion.getUpdatedAt()) {
+ return false;
+ }
+ // Partially loaded policies stay usable during their existing retry
backoff, even though
+ // they deliberately have no loadedRoles marker.
+ return cachedUpdatedAt.isPresent()
+ ||
!partialRoleLoadBackoff.getIfPresent(roleVersion.getRoleId()).isPresent();
+ }
+
+ private boolean awaitRolePolicyLoad(RolePolicyLoadKey key,
CompletableFuture<Boolean> result) {
+ try {
+ return result.get();
Review Comment:
Added the comment in c8b2be6e7b: there is no separate waiter timeout. The
wait depends on the owner's completion, including pool `maxWaitMillis` and
query execution time (and query timeouts, if configured); a stuck owner can
delay all its waiters.
##########
server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/JcasbinAuthorizer.java:
##########
@@ -1416,102 +1437,343 @@ private Set<Long> versionCheckAndLoadRoles(
List<RoleUpdatedAt> staleRoleVersions = new ArrayList<>();
for (RoleUpdatedAt rv : roleVersions) {
- Optional<Long> cachedUpdatedAt =
loadedRoles.getIfPresent(rv.getRoleId());
- if (cachedUpdatedAt.isPresent() && cachedUpdatedAt.get() >=
rv.getUpdatedAt()) {
- continue;
- }
- // A role missing from loadedRoles because its last load was incomplete
would otherwise be
- // retried on every single request. Its partially loaded policies stay
in the enforcer, so
- // the privileges that did resolve keep working while the backoff is in
effect.
- if (!cachedUpdatedAt.isPresent()
- && partialRoleLoadBackoff.getIfPresent(rv.getRoleId()).isPresent()) {
- continue;
+ if (needsRolePolicyLoad(rv)) {
+ staleRoleVersions.add(rv);
}
- staleRoleVersions.add(rv);
}
if (staleRoleVersions.isEmpty()) {
return Collections.emptySet();
}
+ List<Long> leasedRoles = new ArrayList<>();
+ try {
+ return coalesceRolePolicyLoads(metalake, staleRoleVersions,
requestContext, leasedRoles);
+ } finally {
+ releaseRolePolicyLoadStates(leasedRoles);
+ }
+ }
+
+ private Set<Long> coalesceRolePolicyLoads(
+ String metalake,
+ List<RoleUpdatedAt> staleRoleVersions,
+ AuthorizationRequestContext requestContext,
+ List<Long> leasedRoles) {
+ Map<RolePolicyLoadKey, CompletableFuture<Boolean>> requestLoads = new
LinkedHashMap<>();
+ List<RolePolicyLoadKey> ownedLoads = new ArrayList<>();
+ try {
+ for (RolePolicyLoadKey key :
retainRolePolicyLoadStates(staleRoleVersions, leasedRoles)) {
+ CompletableFuture<Boolean> result = new CompletableFuture<>();
+ CompletableFuture<Boolean> existing = rolePolicyLoads.putIfAbsent(key,
result);
+ requestLoads.put(key, existing == null ? result : existing);
+ if (existing == null) {
+ ownedLoads.add(key);
+ }
+ }
+
+ // Claim every role before doing I/O, and finish every load we own
before waiting for another
+ // request. Waiting first could deadlock requests owning opposite halves
of a role batch.
+ List<RolePolicyLoadKey> pendingLoads = new ArrayList<>();
+ // Recheck after claiming: a request can observe a stale marker, be
delayed, and become the
+ // owner after the previous owner has already completed and removed its
flight.
+ for (RolePolicyLoadKey key : ownedLoads) {
+ if (needsRolePolicyLoad(key.roleVersion)) {
+ pendingLoads.add(key);
+ } else {
+ requestLoads.get(key).complete(true);
+ }
+ }
+ loadRolePolicies(metalake, pendingLoads, requestContext, requestLoads);
+ } catch (RuntimeException | Error e) {
+ for (RolePolicyLoadKey key : ownedLoads) {
+ requestLoads.get(key).completeExceptionally(e);
+ }
+ throw e;
+ } finally {
+ for (RolePolicyLoadKey key : ownedLoads) {
+ rolePolicyLoads.remove(key, requestLoads.get(key));
+ }
+ }
+
+ Set<Long> unreadableRoleIds = new HashSet<>();
+ for (Map.Entry<RolePolicyLoadKey, CompletableFuture<Boolean>> load :
requestLoads.entrySet()) {
+ if (!awaitRolePolicyLoad(load.getKey(), load.getValue())) {
+ unreadableRoleIds.add(load.getKey().roleVersion.getRoleId());
+ }
+ }
+ return unreadableRoleIds;
+ }
+
+ /**
+ * Captures the clear generation of every stale role and retains its version
floor, all in one
+ * critical section. The write lock is shared with every enforce() call, so
it is taken once per
+ * request rather than once per role. Pruning removes map entries before
publishing its watermark
+ * under the same lock, so a captured generation never observes the gap
between the two.
+ */
+ private List<RolePolicyLoadKey> retainRolePolicyLoadStates(
+ List<RoleUpdatedAt> staleRoleVersions, List<Long> leasedRoles) {
+ List<RolePolicyLoadKey> keys = new ArrayList<>(staleRoleVersions.size());
+ rolePolicyLock.writeLock().lock();
+ try {
+ for (RoleUpdatedAt rv : staleRoleVersions) {
+ long roleId = rv.getRoleId();
+ RolePolicyLoadKey key = new RolePolicyLoadKey(rv,
roleClearGenerationWithoutLock(roleId));
+ RolePolicyLoadState state = rolePolicyLoadStates.get(roleId);
+ if (state == null) {
+ rolePolicyLoadStates.put(roleId, new
RolePolicyLoadState(rv.getUpdatedAt()));
+ } else {
+ state.highestVersion = Math.max(state.highestVersion,
rv.getUpdatedAt());
+ state.references++;
+ }
+ leasedRoles.add(roleId);
+ keys.add(key);
+ }
+ } finally {
+ rolePolicyLock.writeLock().unlock();
+ }
+ return keys;
+ }
+
+ private void releaseRolePolicyLoadStates(List<Long> leasedRoles) {
+ rolePolicyLock.writeLock().lock();
+ try {
+ for (Long roleId : leasedRoles) {
+ RolePolicyLoadState state = rolePolicyLoadStates.get(roleId);
+ if (--state.references == 0) {
+ rolePolicyLoadStates.remove(roleId);
+ }
+ }
+ } finally {
+ rolePolicyLock.writeLock().unlock();
+ }
+ }
+
+ private boolean needsRolePolicyLoad(RoleUpdatedAt roleVersion) {
+ Optional<Long> cachedUpdatedAt =
loadedRoles.getIfPresent(roleVersion.getRoleId());
+ if (cachedUpdatedAt.isPresent() && cachedUpdatedAt.get() >=
roleVersion.getUpdatedAt()) {
+ return false;
+ }
+ // Partially loaded policies stay usable during their existing retry
backoff, even though
+ // they deliberately have no loadedRoles marker.
+ return cachedUpdatedAt.isPresent()
+ ||
!partialRoleLoadBackoff.getIfPresent(roleVersion.getRoleId()).isPresent();
+ }
+
+ private boolean awaitRolePolicyLoad(RolePolicyLoadKey key,
CompletableFuture<Boolean> result) {
+ try {
+ return result.get();
+ } catch (InterruptedException e) {
+ // Interrupting a waiter must not cancel the load other requests still
depend on.
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Interrupted while waiting for role policy load " +
key.roleVersion.getRoleId(), e);
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ }
+ if (cause instanceof Error) {
+ throw (Error) cause;
+ }
+ throw new IllegalStateException("Failed to load role policies", cause);
+ }
+ }
+
+ private void loadRolePolicies(
+ String metalake,
+ List<RolePolicyLoadKey> pendingLoads,
+ AuthorizationRequestContext requestContext,
+ Map<RolePolicyLoadKey, CompletableFuture<Boolean>> requestLoads) {
+ if (pendingLoads.isEmpty()) {
+ return;
+ }
Set<Long> unreadableRoleIds = new HashSet<>();
EntityStore entityStore = GravitinoEnv.getInstance().entityStore();
List<NameIdentifier> roleIdents =
- staleRoleVersions.stream()
- .map(rv -> NameIdentifierUtil.ofRole(metalake, rv.getRoleName()))
+ pendingLoads.stream()
+ .map(key -> NameIdentifierUtil.ofRole(metalake,
key.roleVersion.getRoleName()))
.collect(Collectors.toList());
List<RoleEntity> roleEntities;
try {
roleEntities = entityStore.batchGet(roleIdents, Entity.EntityType.ROLE,
RoleEntity.class);
} catch (Exception e) {
- LOG.warn("Failed to batch load stale role policies for roleIds {}",
staleRoleVersions, e);
- roleEntities = new ArrayList<>();
- }
- if (roleEntities == null) {
- roleEntities = new ArrayList<>();
- }
- // Some EntityStore implementations don't support batchGet for ROLE and
return empty;
- // fall back to per-role get so policies still load.
- if (roleEntities.isEmpty()) {
- roleEntities = new ArrayList<>(staleRoleVersions.size());
- for (RoleUpdatedAt rv : staleRoleVersions) {
- try {
- roleEntities.add(
- entityStore.get(
- NameIdentifierUtil.ofRole(metalake, rv.getRoleName()),
- Entity.EntityType.ROLE,
- RoleEntity.class));
- } catch (NoSuchEntityException e) {
- LOG.debug("Role {} was dropped before its policies could be loaded",
rv.getRoleId(), e);
- } catch (Exception e) {
- LOG.warn("Failed to load role policies for roleId {}",
rv.getRoleId(), e);
- unreadableRoleIds.add(rv.getRoleId());
+ LOG.warn("Failed to batch load stale role policies for roles {}",
roleIdents, e);
+ roleEntities = Collections.emptyList();
+ }
+ Map<Long, RoleEntity> roleEntitiesById = new HashMap<>();
+ if (roleEntities != null) {
+ for (RoleEntity roleEntity : roleEntities) {
+ if (roleEntity != null) {
+ roleEntitiesById.put(roleEntity.id(), roleEntity);
}
}
}
- Map<Long, RoleUpdatedAt> staleRoleVersionById =
-
staleRoleVersions.stream().collect(Collectors.toMap(RoleUpdatedAt::getRoleId,
rv -> rv));
- for (RoleEntity roleEntity : roleEntities) {
- if (roleEntity == null) {
- continue;
- }
- RoleUpdatedAt rv = staleRoleVersionById.get(roleEntity.id());
- if (rv == null) {
- continue;
+ for (RolePolicyLoadKey key : pendingLoads) {
+ CompletableFuture<Boolean> result = requestLoads.get(key);
+ try {
+ result.complete(
+ loadRolePolicy(
+ entityStore,
+ metalake,
+ key,
+ roleEntitiesById.get(key.roleVersion.getRoleId()),
+ requestContext,
+ unreadableRoleIds));
+ } catch (RuntimeException | Error e) {
+ // A failed role must not fail another role's waiters, including roles
still to be loaded.
+ // Settle each owned flight before the caller awaits results and
propagates its failure.
+ result.completeExceptionally(e);
}
- long roleId = rv.getRoleId();
- long dbUpdatedAt = rv.getUpdatedAt();
-
- // Resolve every securable object to a metadata id before touching the
enforcers, so the
- // critical section below stays free of DB round-trips.
- Optional<ResolvedRolePolicies> resolution =
resolveRolePolicies(roleEntity, requestContext);
- if (!resolution.isPresent()) {
- unreadableRoleIds.add(roleId);
- continue;
+ }
+ }
+
+ private boolean loadRolePolicy(
+ EntityStore entityStore,
+ String metalake,
+ RolePolicyLoadKey key,
+ @Nullable RoleEntity roleEntity,
+ AuthorizationRequestContext requestContext,
+ Set<Long> unreadableRoleIds) {
+ RoleUpdatedAt rv = key.roleVersion;
+ long roleId = rv.getRoleId();
+ // Stores may not support batchGet for ROLE, or may return only part of a
batch. Every owned
+ // role needs a concrete outcome so all its waiters receive the same read
failure, if any.
+ if (roleEntity == null) {
+ try {
+ roleEntity =
+ entityStore.get(
+ NameIdentifierUtil.ofRole(metalake, rv.getRoleName()),
+ Entity.EntityType.ROLE,
+ RoleEntity.class);
+ } catch (NoSuchEntityException e) {
+ LOG.debug("Role {} was dropped before its policies could be loaded",
roleId, e);
+ invalidateRolePolicies(roleId);
+ return true;
+ } catch (Exception e) {
+ LOG.warn("Failed to load role policies for roleId {}", roleId, e);
+ return false;
}
- ResolvedRolePolicies resolved = resolution.get();
+ }
+ if (roleEntity == null) {
+ return false;
+ }
+ if (roleEntity.id() != roleId) {
+ LOG.warn(
+ "Role {} ({}) was read with id {}: it was dropped and recreated
after its version was"
+ + " probed, so its policies are not loaded and requests on it
fail closed",
+ roleId,
+ rv.getRoleName(),
+ roleEntity.id());
+ return false;
+ }
- if (!replaceRolePolicies(roleId, dbUpdatedAt, resolved)) {
- // Another request completed this version (or a newer one) while this
request was resolving
- // metadata ids. Its policies and marker are authoritative, even if
this request's older
- // resolution result was partial.
- continue;
+ // Resolve every securable object to a metadata id before touching the
enforcers, so the
+ // critical section below stays free of DB round-trips.
+ Optional<ResolvedRolePolicies> resolution =
resolveRolePolicies(roleEntity, requestContext);
+ if (!resolution.isPresent()) {
+ return false;
+ }
+ ResolvedRolePolicies resolved = resolution.get();
+
+ if (!replaceUnclearedRolePolicies(key, resolved, unreadableRoleIds)) {
+ // A newer load or an invalidation superseded this work. The caller
retains its pre-load
+ // request generation, so the existing bounded evaluation loop reloads
cleared policies.
+ return !unreadableRoleIds.contains(roleId);
+ }
+
+ if (!resolved.isComplete()) {
+ LOG.warn(
+ "Loaded role {} ({}) with {} of {} securable objects; unresolved
objects: {}. "
+ + "The role is not marked as loaded and will be retried in at
most {} ms.",
+ roleId,
+ rv.getRoleName(),
+ roleEntity.securableObjects().size() -
resolved.getUnresolvedObjects().size(),
+ roleEntity.securableObjects().size(),
+ resolved.getUnresolvedObjects(),
+ PARTIAL_ROLE_LOAD_RETRY_MS);
+ }
+ return true;
+ }
+
+ private long roleClearGenerationWithoutLock(long roleId) {
+ return Math.max(roleClearGenerations.getOrDefault(roleId, 0L),
prunedRoleClearGeneration);
+ }
+
+ private boolean replaceUnclearedRolePolicies(
+ RolePolicyLoadKey key, ResolvedRolePolicies resolved, Set<Long>
unreadableRoleIds) {
Review Comment:
Implemented in c8b2be6e7b. `replaceUnclearedRolePolicies` now returns
`APPLIED`, `SKIPPED` or `UNREADABLE`, removing the mutable output parameter and
batch-level set. `SKIPPED` still preserves generation revalidation after
invalidation. All 119 Jcasbin cases and both H2 authorization tests passed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]