jerryshao commented on code in PR #8297:
URL: https://github.com/apache/gravitino/pull/8297#discussion_r2312989382
##########
core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java:
##########
@@ -81,6 +86,16 @@ User grantRolesToUser(String metalake, List<String> roles,
String user) {
UserEntity.class,
Entity.EntityType.USER,
userEntity -> {
+ if (store instanceof EntityCache) {
+ EntityCache cache = (EntityCache) store;
+ roles.forEach(
+ roleName -> {
+ Namespace namespaceRole =
NamespaceUtil.ofRole(metalake);
+ NameIdentifier nameIdentifierRole =
NameIdentifier.of(namespaceRole, roleName);
+ cache.invalidate(nameIdentifierRole,
Entity.EntityType.ROLE);
+ });
Review Comment:
I think we should wrap this with the cache lock to keep the consistency.
##########
core/src/main/java/org/apache/gravitino/cache/CaffeineEntityCache.java:
##########
@@ -194,6 +207,7 @@ public void clear() {
withLock(
() -> {
cacheData.invalidateAll();
+ reverseIndex = new ReverseIndexCache();
cacheIndex = new ConcurrentRadixTree<>(new
DefaultCharArrayNodeFactory());
Review Comment:
I'm a little confused why do we use `new xxx` to clear the memory?
##########
core/src/main/java/org/apache/gravitino/cache/CaffeineEntityCache.java:
##########
@@ -335,16 +355,125 @@ private <KEY, VALUE> Caffeine<KEY, VALUE>
newBaseBuilder(Config cacheConfig) {
*
* @param identifier The identifier of the entity to invalidate
*/
- private boolean invalidateEntities(NameIdentifier identifier) {
+ private boolean invalidateEntitiesOld(NameIdentifier identifier) {
List<EntityCacheKey> entityKeysToRemove =
Lists.newArrayList(cacheIndex.getValuesForKeysStartingWith(identifier.toString()));
+ Map<EntityCacheRelationKey, List<Entity>> relationEnitiesMap =
+ cacheData.getAllPresent(entityKeysToRemove);
+
+ // first invalidate this entity
+ EntityCacheKey currentKey =
cacheIndex.getValueForExactKey(identifier.toString());
+ if (currentKey != null) {
+ // already removed
+ cacheData.invalidate(currentKey);
+ cacheIndex.remove(currentKey.toString());
+ }
+
+ // Remove child entities
+ relationEnitiesMap.forEach(
+ (key, entities) -> {
+ if (key.relationType() == null) {
+ // If the relation type is null, it means it's a single entity, we
can skip it.
+ return;
+ }
+ entities.forEach(
+ entity -> {
+ NameIdentifier child = ((HasIdentifier)
entity).nameIdentifier();
+ if (!child.equals(identifier)) {
+ invalidateEntitiesOld(child);
Review Comment:
Why do we need a recursive call here?
##########
core/src/main/java/org/apache/gravitino/cache/EntityCacheKey.java:
##########
@@ -23,26 +23,11 @@
import java.util.Objects;
import org.apache.gravitino.Entity;
import org.apache.gravitino.NameIdentifier;
-import org.apache.gravitino.SupportsRelationOperations;
/** Key for Entity cache. */
public class EntityCacheKey {
Review Comment:
Do we still need this class? Seems not so useful.
##########
core/src/main/java/org/apache/gravitino/cache/CaffeineEntityCache.java:
##########
@@ -335,16 +355,125 @@ private <KEY, VALUE> Caffeine<KEY, VALUE>
newBaseBuilder(Config cacheConfig) {
*
* @param identifier The identifier of the entity to invalidate
*/
- private boolean invalidateEntities(NameIdentifier identifier) {
+ private boolean invalidateEntitiesOld(NameIdentifier identifier) {
List<EntityCacheKey> entityKeysToRemove =
Lists.newArrayList(cacheIndex.getValuesForKeysStartingWith(identifier.toString()));
+ Map<EntityCacheRelationKey, List<Entity>> relationEnitiesMap =
+ cacheData.getAllPresent(entityKeysToRemove);
+
+ // first invalidate this entity
+ EntityCacheKey currentKey =
cacheIndex.getValueForExactKey(identifier.toString());
+ if (currentKey != null) {
+ // already removed
+ cacheData.invalidate(currentKey);
+ cacheIndex.remove(currentKey.toString());
+ }
+
+ // Remove child entities
+ relationEnitiesMap.forEach(
+ (key, entities) -> {
+ if (key.relationType() == null) {
+ // If the relation type is null, it means it's a single entity, we
can skip it.
+ return;
+ }
+ entities.forEach(
+ entity -> {
+ NameIdentifier child = ((HasIdentifier)
entity).nameIdentifier();
+ if (!child.equals(identifier)) {
+ invalidateEntitiesOld(child);
+ }
+ });
+ });
Review Comment:
My feeling is that the complexity of the logic is quite high, this part may
requires lots of time to execute, is there's a way to optimize this code?
##########
core/src/main/java/org/apache/gravitino/cache/CaffeineEntityCache.java:
##########
@@ -335,16 +355,125 @@ private <KEY, VALUE> Caffeine<KEY, VALUE>
newBaseBuilder(Config cacheConfig) {
*
* @param identifier The identifier of the entity to invalidate
*/
- private boolean invalidateEntities(NameIdentifier identifier) {
+ private boolean invalidateEntitiesOld(NameIdentifier identifier) {
Review Comment:
What is the meaning of `Old`?
##########
core/src/main/java/org/apache/gravitino/cache/ReverseIndexCache.java:
##########
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.cache;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.googlecode.concurrenttrees.radix.ConcurrentRadixTree;
+import com.googlecode.concurrenttrees.radix.RadixTree;
+import
com.googlecode.concurrenttrees.radix.node.concrete.DefaultCharArrayNodeFactory;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.gravitino.Entity;
+import org.apache.gravitino.HasIdentifier;
+import org.apache.gravitino.NameIdentifier;
+import org.apache.gravitino.meta.GroupEntity;
+import org.apache.gravitino.meta.RoleEntity;
+import org.apache.gravitino.meta.UserEntity;
+
+/**
+ * Reverse index cache for managing entity relationships. This cache uses a
radix tree to
+ * efficiently store and retrieve relationships between entities based on
their keys.
+ */
+public class ReverseIndexCache {
+ private final RadixTree<EntityCacheKey> reverseIndex;
+ /** Registers a reverse index processor for a specific entity class. */
+ private final Map<Class<? extends Entity>, ReverseIndexRule>
reverseIndexRules = new HashMap<>();
+
+ public ReverseIndexCache() {
+ this.reverseIndex = new ConcurrentRadixTree<>(new
DefaultCharArrayNodeFactory());
+
+ registerReverseRule(UserEntity.class, ReverseIndexRules.USER_REVERSE_RULE);
+ registerReverseRule(GroupEntity.class,
ReverseIndexRules.GROUP_REVERSE_RULE);
+ registerReverseRule(RoleEntity.class, ReverseIndexRules.ROLE_REVERSE_RULE);
+ }
+
+ public boolean remove(EntityCacheKey key) {
+ return reverseIndex.remove(key.toString());
+ }
+
+ public Iterable<EntityCacheKey> getValuesForKeysStartingWith(String
keyPrefix) {
+ return reverseIndex.getValuesForKeysStartingWith(keyPrefix);
+ }
+
+ public Iterable<CharSequence> getKeysStartingWith(String keyPrefix) {
+ return reverseIndex.getKeysStartingWith(keyPrefix);
+ }
+
+ public boolean remove(String key) {
+ return reverseIndex.remove(key);
+ }
+
+ public void put(
+ NameIdentifier nameIdentifier, Entity.EntityType type,
EntityCacheRelationKey key) {
+ EntityCacheKey entityCacheKey = EntityCacheKey.of(nameIdentifier, type);
+ String strEntityCacheKey = entityCacheKey.toString();
+ List<EntityCacheKey> entityKeys =
+
Lists.newArrayList(reverseIndex.getValuesForKeysStartingWith(strEntityCacheKey));
+ String strEntityCacheKeyNo =
+ String.format("%s-%d", strEntityCacheKey, entityKeys.size());
+ reverseIndex.put(strEntityCacheKeyNo, key);
+ }
+
+ public void put(Entity entity, EntityCacheRelationKey key) {
+ Preconditions.checkArgument(entity != null, "EntityCacheRelationKey cannot
be null");
+
+ if (entity instanceof HasIdentifier) {
+ NameIdentifier nameIdent =
+ NameIdentifier.of(((HasIdentifier) entity).namespace(),
((HasIdentifier) entity).name());
Review Comment:
Directly calling `nameIdentifiter` should be enough.
##########
core/src/main/java/org/apache/gravitino/cache/ReverseIndexCache.java:
##########
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.cache;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.googlecode.concurrenttrees.radix.ConcurrentRadixTree;
+import com.googlecode.concurrenttrees.radix.RadixTree;
+import
com.googlecode.concurrenttrees.radix.node.concrete.DefaultCharArrayNodeFactory;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.gravitino.Entity;
+import org.apache.gravitino.HasIdentifier;
+import org.apache.gravitino.NameIdentifier;
+import org.apache.gravitino.meta.GroupEntity;
+import org.apache.gravitino.meta.RoleEntity;
+import org.apache.gravitino.meta.UserEntity;
+
+/**
+ * Reverse index cache for managing entity relationships. This cache uses a
radix tree to
+ * efficiently store and retrieve relationships between entities based on
their keys.
+ */
+public class ReverseIndexCache {
+ private final RadixTree<EntityCacheKey> reverseIndex;
+ /** Registers a reverse index processor for a specific entity class. */
+ private final Map<Class<? extends Entity>, ReverseIndexRule>
reverseIndexRules = new HashMap<>();
+
+ public ReverseIndexCache() {
+ this.reverseIndex = new ConcurrentRadixTree<>(new
DefaultCharArrayNodeFactory());
+
+ registerReverseRule(UserEntity.class, ReverseIndexRules.USER_REVERSE_RULE);
+ registerReverseRule(GroupEntity.class,
ReverseIndexRules.GROUP_REVERSE_RULE);
+ registerReverseRule(RoleEntity.class, ReverseIndexRules.ROLE_REVERSE_RULE);
+ }
+
+ public boolean remove(EntityCacheKey key) {
+ return reverseIndex.remove(key.toString());
+ }
+
+ public Iterable<EntityCacheKey> getValuesForKeysStartingWith(String
keyPrefix) {
+ return reverseIndex.getValuesForKeysStartingWith(keyPrefix);
+ }
+
+ public Iterable<CharSequence> getKeysStartingWith(String keyPrefix) {
+ return reverseIndex.getKeysStartingWith(keyPrefix);
+ }
+
+ public boolean remove(String key) {
+ return reverseIndex.remove(key);
+ }
+
+ public void put(
+ NameIdentifier nameIdentifier, Entity.EntityType type,
EntityCacheRelationKey key) {
+ EntityCacheKey entityCacheKey = EntityCacheKey.of(nameIdentifier, type);
+ String strEntityCacheKey = entityCacheKey.toString();
+ List<EntityCacheKey> entityKeys =
+
Lists.newArrayList(reverseIndex.getValuesForKeysStartingWith(strEntityCacheKey));
+ String strEntityCacheKeyNo =
Review Comment:
What's the meaning of `No`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
