This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 4d01ec48f0 [#9802] feat(authz): support preload table metadata in
batch metadata authorization (#9803)
4d01ec48f0 is described below
commit 4d01ec48f0de77d102eadf83450fb9baf2f5e412
Author: yangyang zhong <[email protected]>
AuthorDate: Fri Jan 30 14:32:56 2026 +0800
[#9802] feat(authz): support preload table metadata in batch metadata
authorization (#9803)
### What changes were proposed in this pull request?
support preload table metadata in batch metadata authorization
### Why are the changes needed?
Fix: #9802
### Does this PR introduce _any_ user-facing change?
None
### How was this patch tested?
Existing IT test in
org.apache.gravitino.client.integration.test.authorization.TableAuthorizationIT
---
.../java/org/apache/gravitino/EntityStore.java | 31 ++++++++++++++++++++++
.../gravitino/storage/relational/JDBCBackend.java | 12 +++++++++
.../storage/relational/RelationalBackend.java | 12 +++++++++
.../storage/relational/RelationalEntityStore.java | 22 +++++++++++++++
.../storage/relational/mapper/TableMetaMapper.java | 4 +++
.../mapper/TableMetaSQLProviderFactory.java | 5 ++++
.../provider/base/TableMetaBaseSQLProvider.java | 24 +++++++++++++++++
.../relational/service/TableMetaService.java | 29 ++++++++++++++++++++
.../storage/memory/TestMemoryEntityStore.java | 6 +++++
.../server/authorization/MetadataAuthzHelper.java | 20 +++++++++++++-
10 files changed, 164 insertions(+), 1 deletion(-)
diff --git a/core/src/main/java/org/apache/gravitino/EntityStore.java
b/core/src/main/java/org/apache/gravitino/EntityStore.java
index c2575dc9eb..d2c19c65c6 100644
--- a/core/src/main/java/org/apache/gravitino/EntityStore.java
+++ b/core/src/main/java/org/apache/gravitino/EntityStore.java
@@ -20,6 +20,8 @@ package org.apache.gravitino;
import java.io.Closeable;
import java.io.IOException;
+import java.lang.reflect.Array;
+import java.util.Arrays;
import java.util.List;
import java.util.function.Function;
import org.apache.commons.lang3.tuple.Pair;
@@ -165,6 +167,35 @@ public interface EntityStore extends Closeable {
<E extends Entity & HasIdentifier> E get(NameIdentifier ident, EntityType
entityType, Class<E> e)
throws NoSuchEntityException, IOException;
+ /**
+ * Batch get the entity from the underlying storage.
+ *
+ * @param idents the unique identifier of the entity
+ * @param entityType the general type of the entity
+ * @param clazz the entity class instance
+ * @param <E> the class of entity
+ * @return the entity retrieved from the underlying storage
+ * @throws NoSuchEntityException if the entity does not exist
+ */
+ <E extends Entity & HasIdentifier> List<E> batchGet(
+ List<NameIdentifier> idents, EntityType entityType, Class<E> clazz);
+
+ /**
+ * Batch get the entity from the underlying storage.
+ *
+ * @param idents the unique identifier of the entity
+ * @param entityType the general type of the entity
+ * @param clazz the entity class instance
+ * @param <E> the class of entity
+ * @return the entity retrieved from the underlying storage
+ * @throws NoSuchEntityException if the entity does not exist
+ */
+ default <E extends Entity & HasIdentifier> E[] batchGet(
+ NameIdentifier[] idents, EntityType entityType, Class<E> clazz) {
+ return batchGet(Arrays.asList(idents), entityType, clazz)
+ .toArray(size -> (E[]) Array.newInstance(clazz, size));
+ }
+
/**
* Delete the entity from the underlying storage by the specified {@link
* org.apache.gravitino.NameIdentifier}.
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java
b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java
index f660653067..47c3a39f59 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java
@@ -286,6 +286,18 @@ public class JDBCBackend implements RelationalBackend {
}
}
+ @Override
+ public <E extends Entity & HasIdentifier> List<E> batchGet(
+ List<NameIdentifier> identifiers, Entity.EntityType entityType) {
+ switch (entityType) {
+ case TABLE:
+ return (List<E>)
TableMetaService.getInstance().batchGetTableByIdentifier(identifiers);
+ default:
+ throw new UnsupportedEntityTypeException(
+ "Unsupported entity type: %s for get operation", entityType);
+ }
+ }
+
@Override
public boolean delete(NameIdentifier ident, Entity.EntityType entityType,
boolean cascade)
throws IOException {
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java
b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java
index 037d5256e0..61eb9c435b 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java
@@ -113,6 +113,18 @@ public interface RelationalBackend extends Closeable,
SupportsRelationOperations
<E extends Entity & HasIdentifier> E get(NameIdentifier ident,
Entity.EntityType entityType)
throws IOException;
+ /**
+ * Batch retrieves the entities associated with the identifiers and the
entity type.
+ *
+ * @param <E> The type of the entity returned.
+ * @param identifiers The identifiers of the entities.
+ * @param entityType The type of the entity.
+ * @return The entities associated with the identifiers and the entity type,
or null if the key
+ * does not exist.
+ */
+ <E extends Entity & HasIdentifier> List<E> batchGet(
+ List<NameIdentifier> identifiers, Entity.EntityType entityType);
+
/**
* Soft deletes the entity associated with the identifier and the entity
type.
*
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java
b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java
index b4e8230e36..31183f2649 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java
@@ -23,6 +23,7 @@ import static
org.apache.gravitino.Configs.ENTITY_RELATIONAL_STORE;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
@@ -153,6 +154,27 @@ public class RelationalEntityStore implements EntityStore,
SupportsRelationOpera
});
}
+ @Override
+ public <E extends Entity & HasIdentifier> List<E> batchGet(
+ List<NameIdentifier> idents, Entity.EntityType entityType, Class<E>
clazz) {
+ List<E> allEntities = new ArrayList<>();
+ List<NameIdentifier> noCacheIdents =
+ idents.stream()
+ .filter(
+ ident -> {
+ Optional<E> entity = cache.getIfPresent(ident, entityType);
+ entity.ifPresent(allEntities::add);
+ return entity.isEmpty();
+ })
+ .toList();
+ List<E> fetchEntities = backend.batchGet(noCacheIdents, entityType);
+ for (E entity : fetchEntities) {
+ cache.put(entity);
+ allEntities.add(entity);
+ }
+ return allEntities;
+ }
+
@Override
public boolean delete(NameIdentifier ident, Entity.EntityType entityType,
boolean cascade)
throws IOException {
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java
index c284d0ce3c..786df8b5f1 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java
@@ -112,4 +112,8 @@ public interface TableMetaMapper {
method = "deleteTableMetasByLegacyTimeline")
Integer deleteTableMetasByLegacyTimeline(
@Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit);
+
+ @SelectProvider(type = TableMetaSQLProviderFactory.class, method =
"batchSelectTableByIdentifier")
+ List<TablePO> batchSelectTableByIdentifier(
+ @Param("schemaId") Long schemaId, @Param("tableNames") List<String>
tableNames);
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java
index 2a796a6e15..20859f18d7 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java
@@ -124,4 +124,9 @@ public class TableMetaSQLProviderFactory {
@Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
return getProvider().deleteTableMetasByLegacyTimeline(legacyTimeline,
limit);
}
+
+ public static String batchSelectTableByIdentifier(
+ @Param("schemaId") Long schemaId, @Param("tableNames") List<String>
tableNames) {
+ return getProvider().batchSelectTableByIdentifier(schemaId, tableNames);
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableMetaBaseSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableMetaBaseSQLProvider.java
index 4312091fca..4316f87561 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableMetaBaseSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableMetaBaseSQLProvider.java
@@ -337,4 +337,28 @@ public class TableMetaBaseSQLProvider {
TABLE_NAME,
TableVersionMapper.TABLE_NAME);
}
+
+ public String batchSelectTableByIdentifier(
+ @Param("schemaId") Long schemaId, @Param("tableNames") List<String>
tableNames) {
+ return """
+ <script>
+ SELECT
+ tm.table_id AS tableId,
+ tm.table_name AS tableName,
+ tm.metalake_id AS metalakeId,
+ tm.audit_info AS auditInfo,
+ tm.current_version AS currentVersion,
+ tm.last_version AS lastVersion,
+ tm.deleted_at AS deletedAt
+ FROM %s tm
+ WHERE schema_id = #{schemaId}
+ AND table_name IN
+ <foreach collection="tableNames" item="tableName" open="("
separator="," close=")">
+ #{tableName}
+ </foreach>
+ AND deleted_at = 0
+ </script>
+ """
+ .formatted(TABLE_NAME);
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java
b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java
index 51653f2792..7bd7cdd56a 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java
@@ -22,12 +22,15 @@ import static
org.apache.gravitino.metrics.source.MetricsSource.GRAVITINO_RELATI
import com.google.common.base.Preconditions;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
import java.util.stream.Collectors;
+import org.apache.commons.collections4.CollectionUtils;
import org.apache.gravitino.Entity;
import org.apache.gravitino.Entity.EntityType;
import org.apache.gravitino.GravitinoEnv;
@@ -290,6 +293,32 @@ public class TableMetaService {
mapper -> mapper.deleteTableVersionByLegacyTimeline(legacyTimeline,
limit));
}
+ @Monitored(
+ metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME,
+ baseMetricName = "batchGetTableByIdentifier")
+ public List<TableEntity> batchGetTableByIdentifier(List<NameIdentifier>
identifiers) {
+ if (CollectionUtils.isEmpty(identifiers)) {
+ return Collections.emptyList();
+ }
+ NameIdentifier firstIdent = identifiers.get(0);
+ NameIdentifier schemaIdent =
NameIdentifierUtil.getSchemaIdentifier(firstIdent);
+ List<String> tableNames = new ArrayList<>(identifiers.size());
+ tableNames.add(identifiers.get(0).name());
+ for (int i = 1; i < identifiers.size(); i++) {
+ NameIdentifier ident = identifiers.get(i);
+ Preconditions.checkArgument(
+ Objects.equals(schemaIdent,
NameIdentifierUtil.getSchemaIdentifier(ident)));
+ tableNames.add(ident.name());
+ }
+ Long schemaId = EntityIdService.getEntityId(schemaIdent,
Entity.EntityType.SCHEMA);
+ return SessionUtils.doWithCommitAndFetchResult(
+ TableMetaMapper.class,
+ mapper -> {
+ List<TablePO> tableList =
mapper.batchSelectTableByIdentifier(schemaId, tableNames);
+ return POConverters.fromTablePOs(tableList, firstIdent.namespace());
+ });
+ }
+
private void fillTablePOBuilderParentEntityId(TablePO.Builder builder,
Namespace namespace) {
NamespaceUtil.checkTable(namespace);
NamespacedEntityId namespacedEntityId =
diff --git
a/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java
b/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java
index 95dded4f9c..4c5bc8043e 100644
---
a/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java
+++
b/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java
@@ -146,6 +146,12 @@ public class TestMemoryEntityStore {
return e;
}
+ @Override
+ public <E extends Entity & HasIdentifier> List<E> batchGet(
+ List<NameIdentifier> idents, EntityType entityType, Class<E> e) {
+ return idents.stream().map(ident -> (E) entityMap.get(ident)).toList();
+ }
+
@Override
public boolean delete(NameIdentifier ident, EntityType entityType, boolean
cascade)
throws IOException {
diff --git
a/server-common/src/main/java/org/apache/gravitino/server/authorization/MetadataAuthzHelper.java
b/server-common/src/main/java/org/apache/gravitino/server/authorization/MetadataAuthzHelper.java
index 028747559b..451a205c4b 100644
---
a/server-common/src/main/java/org/apache/gravitino/server/authorization/MetadataAuthzHelper.java
+++
b/server-common/src/main/java/org/apache/gravitino/server/authorization/MetadataAuthzHelper.java
@@ -38,6 +38,7 @@ import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.authorization.AuthorizationRequestContext;
import org.apache.gravitino.authorization.GravitinoAuthorizer;
import org.apache.gravitino.dto.tag.MetadataObjectDTO;
+import org.apache.gravitino.meta.TableEntity;
import
org.apache.gravitino.server.authorization.expression.AuthorizationExpressionConstants;
import
org.apache.gravitino.server.authorization.expression.AuthorizationExpressionEvaluator;
import org.apache.gravitino.utils.MetadataObjectUtil;
@@ -136,6 +137,7 @@ public class MetadataAuthzHelper {
String expression,
Entity.EntityType entityType,
NameIdentifier[] nameIdentifiers) {
+ preloadToCache(entityType, nameIdentifiers);
return filterByExpression(metalake, expression, entityType,
nameIdentifiers, e -> e);
}
@@ -291,7 +293,12 @@ public class MetadataAuthzHelper {
return futures.stream()
.map(CompletableFuture::join)
.filter(Objects::nonNull)
- .toArray(size -> (E[])
Array.newInstance(entities.getClass().getComponentType(), size));
+ .toArray(size -> createArray(entities.getClass().getComponentType(),
size));
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <E> E[] createArray(Class<?> componentType, int size) {
+ return (E[]) Array.newInstance(componentType, size);
}
private static boolean enableAuthorization() {
@@ -318,4 +325,15 @@ public class MetadataAuthzHelper {
}
}
}
+
+ private static void preloadToCache(
+ Entity.EntityType entityType, NameIdentifier[] nameIdentifiers) {
+ if (GravitinoEnv.getInstance().cacheEnabled()) {
+ if (entityType == Entity.EntityType.TABLE) {
+ GravitinoEnv.getInstance()
+ .entityStore()
+ .batchGet(nameIdentifiers, entityType, TableEntity.class);
+ }
+ }
+ }
}
