This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new c54b342ac8 [#12154] fix(core): Clean up orphaned metadata object
relations (#12160)
c54b342ac8 is described below
commit c54b342ac8bb3f147ea9bbdb5aed811e58648cc0
Author: Qi Yu <[email protected]>
AuthorDate: Fri Jul 24 09:22:56 2026 +0800
[#12154] fix(core): Clean up orphaned metadata object relations (#12160)
### What changes were proposed in this pull request?
Add a garbage collection pass to soft-delete metadata object relation
rows whose referenced entity no longer has a live record.
The cleanup:
- covers owner, tag, policy, statistic, and role securable-object
relations;
- iterates over supported metadata object types;
- processes orphaned object IDs in bounded batches;
- runs on the existing relational garbage collector schedule;
- remains idempotent.
### Why are the changes needed?
Metadata object relation tables reference entities using plain object ID
and type columns without foreign keys. If an entity row is lost outside
the normal transactional deletion path, its relation rows remain
orphaned indefinitely.
This change provides background cleanup for those orphaned rows while
leaving relations belonging to live entities untouched.
Fix: #12154
### Does this PR introduce _any_ user-facing change?
No. This change only adds background cleanup for orphaned
relational-store data.
### How was this patch tested?
Added tests covering:
- cleanup of orphaned owner, tag, policy, statistic, and
securable-object relations;
- preservation of relations referencing live entities;
- idempotent repeated cleanup;
- repeated batch execution by the relational garbage collector.
Ran:
`./gradlew :core:test -PskipITs -PskipDockerTests=true`
---
.../gravitino/storage/relational/JDBCBackend.java | 10 +-
.../relational/RelationalGarbageCollector.java | 20 +++
.../SupportsOrphanedRelationCleanup.java | 38 +++++
.../OrphanedMetadataObjectRelationMapper.java | 81 +++++++++++
.../OrphanedMetadataObjectRelationSQLProvider.java | 96 +++++++++++++
.../provider/DefaultMapperPackageProvider.java | 2 +
.../OrphanedMetadataObjectRelationService.java | 153 +++++++++++++++++++++
.../relational/TestRelationalGarbageCollector.java | 72 ++++++++++
.../TestOrphanedMetadataObjectRelationService.java | 139 +++++++++++++++++++
9 files changed, 610 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java
b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java
index 922f324cd4..daef6d052e 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java
@@ -36,6 +36,7 @@ import org.apache.gravitino.Configs;
import org.apache.gravitino.Entity;
import org.apache.gravitino.EntityAlreadyExistsException;
import org.apache.gravitino.HasIdentifier;
+import org.apache.gravitino.MetadataObject;
import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.Namespace;
import org.apache.gravitino.RelationalEntity;
@@ -72,6 +73,7 @@ import
org.apache.gravitino.storage.relational.service.JobTemplateMetaService;
import org.apache.gravitino.storage.relational.service.MetalakeMetaService;
import org.apache.gravitino.storage.relational.service.ModelMetaService;
import org.apache.gravitino.storage.relational.service.ModelVersionMetaService;
+import
org.apache.gravitino.storage.relational.service.OrphanedMetadataObjectRelationService;
import org.apache.gravitino.storage.relational.service.OwnerMetaService;
import org.apache.gravitino.storage.relational.service.PolicyMetaService;
import org.apache.gravitino.storage.relational.service.RoleMetaService;
@@ -93,7 +95,7 @@ import org.slf4j.LoggerFactory;
* syntax, please implement the SQL statements and methods in MyBatis Mapper
separately and switch
* according to the {@link Configs#ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY}
parameter.
*/
-public class JDBCBackend implements RelationalBackend {
+public class JDBCBackend implements RelationalBackend,
SupportsOrphanedRelationCleanup {
private static final Logger LOG = LoggerFactory.getLogger(JDBCBackend.class);
@@ -549,6 +551,12 @@ public class JDBCBackend implements RelationalBackend {
}
}
+ @Override
+ public int softDeleteOrphanedRelations(MetadataObject.Type
metadataObjectType, int limit) {
+ return OrphanedMetadataObjectRelationService.getInstance()
+ .softDeleteOrphanedRelations(metadataObjectType, limit);
+ }
+
@Override
public int deleteOldVersionData(Entity.EntityType entityType, long
versionRetentionCount)
throws IOException {
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalGarbageCollector.java
b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalGarbageCollector.java
index d430e68dcd..efab318714 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalGarbageCollector.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalGarbageCollector.java
@@ -19,6 +19,7 @@
package org.apache.gravitino.storage.relational;
+import static
org.apache.gravitino.Configs.GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT;
import static org.apache.gravitino.Configs.STORE_DELETE_AFTER_TIME;
import static org.apache.gravitino.Configs.VERSION_RETENTION_COUNT;
@@ -31,6 +32,7 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.gravitino.Config;
import org.apache.gravitino.Entity;
+import org.apache.gravitino.MetadataObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,6 +78,24 @@ public final class RelationalGarbageCollector implements
Closeable {
try {
LOG.debug("Start to collect and delete legacy data by thread {}",
threadId);
long legacyTimeline = System.currentTimeMillis() -
storeDeleteAfterTimeMillis;
+ if (backend instanceof SupportsOrphanedRelationCleanup) {
+ SupportsOrphanedRelationCleanup orphanedRelationCleanup =
+ (SupportsOrphanedRelationCleanup) backend;
+ for (MetadataObject.Type metadataObjectType :
MetadataObject.Type.values()) {
+ long deletedCount = Long.MAX_VALUE;
+ LOG.debug("Try to softly delete orphaned {} relations",
metadataObjectType);
+ try {
+ while (deletedCount > 0) {
+ deletedCount =
+ orphanedRelationCleanup.softDeleteOrphanedRelations(
+ metadataObjectType,
GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT);
+ }
+ } catch (RuntimeException e) {
+ LOG.error("Failed to softly delete orphaned " + metadataObjectType
+ " relations: ", e);
+ }
+ }
+ }
+
for (Entity.EntityType entityType : Entity.EntityType.values()) {
long deletedCount = Long.MAX_VALUE;
LOG.debug(
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/SupportsOrphanedRelationCleanup.java
b/core/src/main/java/org/apache/gravitino/storage/relational/SupportsOrphanedRelationCleanup.java
new file mode 100644
index 0000000000..f20ad68fad
--- /dev/null
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/SupportsOrphanedRelationCleanup.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.storage.relational;
+
+import org.apache.gravitino.MetadataObject;
+
+/**
+ * An optional capability for {@link RelationalBackend} implementations that
can clean up relation
+ * rows referencing metadata objects which no longer exist. Backends without
this capability are
+ * simply skipped by the garbage collector.
+ */
+public interface SupportsOrphanedRelationCleanup {
+
+ /**
+ * Soft-deletes relation rows that reference no live metadata object of the
given type.
+ *
+ * @param metadataObjectType metadata object type to collect
+ * @param limit maximum number of orphaned object IDs processed per relation
table
+ * @return number of relation rows soft-deleted
+ */
+ int softDeleteOrphanedRelations(MetadataObject.Type metadataObjectType, int
limit);
+}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OrphanedMetadataObjectRelationMapper.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OrphanedMetadataObjectRelationMapper.java
new file mode 100644
index 0000000000..fbb63e0f58
--- /dev/null
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OrphanedMetadataObjectRelationMapper.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.storage.relational.mapper;
+
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.UpdateProvider;
+
+/** A MyBatis mapper for collecting orphaned metadata-object relations. */
+public interface OrphanedMetadataObjectRelationMapper {
+
+ /** Soft-deletes orphaned owner relations and returns the affected row
count. */
+ @UpdateProvider(
+ type = OrphanedMetadataObjectRelationSQLProvider.class,
+ method = "softDeleteOrphanedOwnerRelations")
+ int softDeleteOrphanedOwnerRelations(
+ @Param("entityTable") String entityTable,
+ @Param("entityIdColumn") String entityIdColumn,
+ @Param("metadataObjectType") String metadataObjectType,
+ @Param("deletedAt") long deletedAt,
+ @Param("limit") int limit);
+
+ /** Soft-deletes orphaned tag relations and returns the affected row count.
*/
+ @UpdateProvider(
+ type = OrphanedMetadataObjectRelationSQLProvider.class,
+ method = "softDeleteOrphanedTagRelations")
+ int softDeleteOrphanedTagRelations(
+ @Param("entityTable") String entityTable,
+ @Param("entityIdColumn") String entityIdColumn,
+ @Param("metadataObjectType") String metadataObjectType,
+ @Param("deletedAt") long deletedAt,
+ @Param("limit") int limit);
+
+ /** Soft-deletes orphaned policy relations and returns the affected row
count. */
+ @UpdateProvider(
+ type = OrphanedMetadataObjectRelationSQLProvider.class,
+ method = "softDeleteOrphanedPolicyRelations")
+ int softDeleteOrphanedPolicyRelations(
+ @Param("entityTable") String entityTable,
+ @Param("entityIdColumn") String entityIdColumn,
+ @Param("metadataObjectType") String metadataObjectType,
+ @Param("deletedAt") long deletedAt,
+ @Param("limit") int limit);
+
+ /** Soft-deletes orphaned statistics and returns the affected row count. */
+ @UpdateProvider(
+ type = OrphanedMetadataObjectRelationSQLProvider.class,
+ method = "softDeleteOrphanedStatistics")
+ int softDeleteOrphanedStatistics(
+ @Param("entityTable") String entityTable,
+ @Param("entityIdColumn") String entityIdColumn,
+ @Param("metadataObjectType") String metadataObjectType,
+ @Param("deletedAt") long deletedAt,
+ @Param("limit") int limit);
+
+ /** Soft-deletes orphaned securable objects and returns the affected row
count. */
+ @UpdateProvider(
+ type = OrphanedMetadataObjectRelationSQLProvider.class,
+ method = "softDeleteOrphanedSecurableObjects")
+ int softDeleteOrphanedSecurableObjects(
+ @Param("entityTable") String entityTable,
+ @Param("entityIdColumn") String entityIdColumn,
+ @Param("metadataObjectType") String metadataObjectType,
+ @Param("deletedAt") long deletedAt,
+ @Param("limit") int limit);
+}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OrphanedMetadataObjectRelationSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OrphanedMetadataObjectRelationSQLProvider.java
new file mode 100644
index 0000000000..3aa47e9631
--- /dev/null
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OrphanedMetadataObjectRelationSQLProvider.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.storage.relational.mapper;
+
+import static
org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper.OWNER_TABLE_NAME;
+import static
org.apache.gravitino.storage.relational.mapper.PolicyMetadataObjectRelMapper.POLICY_METADATA_OBJECT_RELATION_TABLE_NAME;
+import static
org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper.SECURABLE_OBJECT_TABLE_NAME;
+import static
org.apache.gravitino.storage.relational.mapper.StatisticMetaMapper.STATISTIC_META_TABLE_NAME;
+import static
org.apache.gravitino.storage.relational.mapper.TagMetadataObjectRelMapper.TAG_METADATA_OBJECT_RELATION_TABLE_NAME;
+
+import com.google.common.base.Preconditions;
+import org.apache.ibatis.annotations.Param;
+
+/** Provides SQL for collecting orphaned metadata-object relations. */
+public class OrphanedMetadataObjectRelationSQLProvider {
+
+ /** Returns SQL that soft-deletes orphaned owner relations. */
+ public static String softDeleteOrphanedOwnerRelations(
+ @Param("entityTable") String entityTable, @Param("entityIdColumn")
String entityIdColumn) {
+ return softDeleteOrphans(OWNER_TABLE_NAME, "metadata_object_type",
entityTable, entityIdColumn);
+ }
+
+ /** Returns SQL that soft-deletes orphaned tag relations. */
+ public static String softDeleteOrphanedTagRelations(
+ @Param("entityTable") String entityTable, @Param("entityIdColumn")
String entityIdColumn) {
+ return softDeleteOrphans(
+ TAG_METADATA_OBJECT_RELATION_TABLE_NAME,
+ "metadata_object_type",
+ entityTable,
+ entityIdColumn);
+ }
+
+ /** Returns SQL that soft-deletes orphaned policy relations. */
+ public static String softDeleteOrphanedPolicyRelations(
+ @Param("entityTable") String entityTable, @Param("entityIdColumn")
String entityIdColumn) {
+ return softDeleteOrphans(
+ POLICY_METADATA_OBJECT_RELATION_TABLE_NAME,
+ "metadata_object_type",
+ entityTable,
+ entityIdColumn);
+ }
+
+ /** Returns SQL that soft-deletes orphaned statistics. */
+ public static String softDeleteOrphanedStatistics(
+ @Param("entityTable") String entityTable, @Param("entityIdColumn")
String entityIdColumn) {
+ return softDeleteOrphans(
+ STATISTIC_META_TABLE_NAME, "metadata_object_type", entityTable,
entityIdColumn);
+ }
+
+ /** Returns SQL that soft-deletes orphaned securable objects. */
+ public static String softDeleteOrphanedSecurableObjects(
+ @Param("entityTable") String entityTable, @Param("entityIdColumn")
String entityIdColumn) {
+ return softDeleteOrphans(SECURABLE_OBJECT_TABLE_NAME, "type", entityTable,
entityIdColumn);
+ }
+
+ private static String softDeleteOrphans(
+ String relationTable, String typeColumn, String entityTable, String
entityIdColumn) {
+ Preconditions.checkArgument(
+ entityTable.matches("[a-z_]+") && entityIdColumn.matches("[a-z_]+"),
+ "Invalid entity table mapping: %s.%s",
+ entityTable,
+ entityIdColumn);
+ return "UPDATE "
+ + relationTable
+ + " SET deleted_at = #{deletedAt}"
+ + " WHERE deleted_at = 0 AND "
+ + typeColumn
+ + " = #{metadataObjectType} AND metadata_object_id IN ("
+ + "SELECT metadata_object_id FROM (SELECT DISTINCT
rel.metadata_object_id FROM "
+ + relationTable
+ + " rel WHERE rel.deleted_at = 0 AND rel."
+ + typeColumn
+ + " = #{metadataObjectType} AND NOT EXISTS (SELECT 1 FROM "
+ + entityTable
+ + " entity WHERE entity."
+ + entityIdColumn
+ + " = rel.metadata_object_id AND entity.deleted_at = 0)"
+ + " LIMIT #{limit}) orphan_ids)";
+ }
+}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/DefaultMapperPackageProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/DefaultMapperPackageProvider.java
index d4ead5df7a..8065ecfa3b 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/DefaultMapperPackageProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/DefaultMapperPackageProvider.java
@@ -34,6 +34,7 @@ import
org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper;
import org.apache.gravitino.storage.relational.mapper.ModelMetaMapper;
import
org.apache.gravitino.storage.relational.mapper.ModelVersionAliasRelMapper;
import org.apache.gravitino.storage.relational.mapper.ModelVersionMetaMapper;
+import
org.apache.gravitino.storage.relational.mapper.OrphanedMetadataObjectRelationMapper;
import org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper;
import org.apache.gravitino.storage.relational.mapper.PolicyMetaMapper;
import
org.apache.gravitino.storage.relational.mapper.PolicyMetadataObjectRelMapper;
@@ -73,6 +74,7 @@ public class DefaultMapperPackageProvider implements
MapperPackageProvider {
ModelMetaMapper.class,
ModelVersionAliasRelMapper.class,
ModelVersionMetaMapper.class,
+ OrphanedMetadataObjectRelationMapper.class,
OwnerMetaMapper.class,
PolicyMetadataObjectRelMapper.class,
PolicyMetaMapper.class,
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/service/OrphanedMetadataObjectRelationService.java
b/core/src/main/java/org/apache/gravitino/storage/relational/service/OrphanedMetadataObjectRelationService.java
new file mode 100644
index 0000000000..a0b21d3c28
--- /dev/null
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/service/OrphanedMetadataObjectRelationService.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.storage.relational.service;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.apache.gravitino.MetadataObject;
+import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.FunctionMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.JobMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.JobTemplateMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.ModelMetaMapper;
+import
org.apache.gravitino.storage.relational.mapper.OrphanedMetadataObjectRelationMapper;
+import org.apache.gravitino.storage.relational.mapper.PolicyMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.RoleMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.TableColumnMapper;
+import org.apache.gravitino.storage.relational.mapper.TableMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.TagMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.TopicMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.ViewMetaMapper;
+import org.apache.gravitino.storage.relational.utils.SessionUtils;
+
+/** Collects relation rows whose referenced metadata object is no longer live.
*/
+public class OrphanedMetadataObjectRelationService {
+ private static final OrphanedMetadataObjectRelationService INSTANCE =
+ new OrphanedMetadataObjectRelationService();
+
+ private static final Map<MetadataObject.Type, EntityTable> ENTITY_TABLES =
+ ImmutableMap.<MetadataObject.Type, EntityTable>builder()
+ .put(
+ MetadataObject.Type.METALAKE,
+ new EntityTable(MetalakeMetaMapper.TABLE_NAME, "metalake_id"))
+ .put(
+ MetadataObject.Type.CATALOG,
+ new EntityTable(CatalogMetaMapper.TABLE_NAME, "catalog_id"))
+ .put(
+ MetadataObject.Type.SCHEMA, new
EntityTable(SchemaMetaMapper.TABLE_NAME, "schema_id"))
+ .put(
+ MetadataObject.Type.FILESET,
+ new EntityTable(FilesetMetaMapper.META_TABLE_NAME, "fileset_id"))
+ .put(MetadataObject.Type.TABLE, new
EntityTable(TableMetaMapper.TABLE_NAME, "table_id"))
+ .put(MetadataObject.Type.VIEW, new
EntityTable(ViewMetaMapper.TABLE_NAME, "view_id"))
+ .put(MetadataObject.Type.TOPIC, new
EntityTable(TopicMetaMapper.TABLE_NAME, "topic_id"))
+ .put(
+ MetadataObject.Type.COLUMN,
+ new EntityTable(TableColumnMapper.COLUMN_TABLE_NAME,
"column_id"))
+ .put(MetadataObject.Type.ROLE, new
EntityTable(RoleMetaMapper.ROLE_TABLE_NAME, "role_id"))
+ .put(MetadataObject.Type.MODEL, new
EntityTable(ModelMetaMapper.TABLE_NAME, "model_id"))
+ .put(MetadataObject.Type.TAG, new
EntityTable(TagMetaMapper.TAG_TABLE_NAME, "tag_id"))
+ .put(
+ MetadataObject.Type.POLICY,
+ new EntityTable(PolicyMetaMapper.POLICY_META_TABLE_NAME,
"policy_id"))
+ .put(MetadataObject.Type.JOB, new
EntityTable(JobMetaMapper.TABLE_NAME, "job_run_id"))
+ .put(
+ MetadataObject.Type.JOB_TEMPLATE,
+ new EntityTable(JobTemplateMetaMapper.TABLE_NAME,
"job_template_id"))
+ .put(
+ MetadataObject.Type.FUNCTION,
+ new EntityTable(FunctionMetaMapper.TABLE_NAME, "function_id"))
+ .build();
+
+ private OrphanedMetadataObjectRelationService() {}
+
+ /**
+ * Returns the singleton service instance.
+ *
+ * @return singleton service instance
+ */
+ public static OrphanedMetadataObjectRelationService getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * Soft-deletes orphaned relation rows for one metadata object type.
+ *
+ * @param metadataObjectType metadata object type to collect
+ * @param limit maximum number of orphaned object IDs processed per relation
table
+ * @return number of relation rows soft-deleted
+ */
+ public int softDeleteOrphanedRelations(MetadataObject.Type
metadataObjectType, int limit) {
+ Preconditions.checkArgument(metadataObjectType != null,
"metadataObjectType cannot be null");
+ Preconditions.checkArgument(limit > 0, "limit must be positive");
+ EntityTable entityTable = ENTITY_TABLES.get(metadataObjectType);
+ if (entityTable == null) {
+ return 0;
+ }
+
+ long deletedAt = System.currentTimeMillis();
+ return SessionUtils.doWithCommitAndFetchResult(
+ OrphanedMetadataObjectRelationMapper.class,
+ mapper ->
+ mapper.softDeleteOrphanedOwnerRelations(
+ entityTable.tableName,
+ entityTable.idColumn,
+ metadataObjectType.name(),
+ deletedAt,
+ limit)
+ + mapper.softDeleteOrphanedTagRelations(
+ entityTable.tableName,
+ entityTable.idColumn,
+ metadataObjectType.name(),
+ deletedAt,
+ limit)
+ + mapper.softDeleteOrphanedPolicyRelations(
+ entityTable.tableName,
+ entityTable.idColumn,
+ metadataObjectType.name(),
+ deletedAt,
+ limit)
+ + mapper.softDeleteOrphanedStatistics(
+ entityTable.tableName,
+ entityTable.idColumn,
+ metadataObjectType.name(),
+ deletedAt,
+ limit)
+ + mapper.softDeleteOrphanedSecurableObjects(
+ entityTable.tableName,
+ entityTable.idColumn,
+ metadataObjectType.name(),
+ deletedAt,
+ limit));
+ }
+
+ private static class EntityTable {
+ private final String tableName;
+ private final String idColumn;
+
+ private EntityTable(String tableName, String idColumn) {
+ this.tableName = tableName;
+ this.idColumn = idColumn;
+ }
+ }
+}
diff --git
a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalGarbageCollector.java
b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalGarbageCollector.java
new file mode 100644
index 0000000000..51c3014500
--- /dev/null
+++
b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalGarbageCollector.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.storage.relational;
+
+import static
org.apache.gravitino.Configs.GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT;
+import static org.apache.gravitino.Configs.STORE_DELETE_AFTER_TIME;
+import static org.apache.gravitino.Configs.VERSION_RETENTION_COUNT;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
+
+import org.apache.gravitino.Config;
+import org.apache.gravitino.MetadataObject;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/** Tests relational garbage collector orchestration. */
+public class TestRelationalGarbageCollector {
+
+ @Test
+ public void testCollectOrphanedRelationsUntilBatchIsEmpty() throws Exception
{
+ RelationalBackend backend =
+ mock(
+ RelationalBackend.class,
+
withSettings().extraInterfaces(SupportsOrphanedRelationCleanup.class));
+ SupportsOrphanedRelationCleanup orphanedRelationCleanup =
+ (SupportsOrphanedRelationCleanup) backend;
+ Config config = mock(Config.class);
+ when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(1000L);
+ when(config.get(VERSION_RETENTION_COUNT)).thenReturn(1L);
+ when(orphanedRelationCleanup.softDeleteOrphanedRelations(
+ MetadataObject.Type.TABLE,
GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT))
+ .thenReturn(2, 0);
+
+ new RelationalGarbageCollector(backend, config).collectAndClean();
+
+ for (MetadataObject.Type type : MetadataObject.Type.values()) {
+ int expectedInvocations = type == MetadataObject.Type.TABLE ? 2 : 1;
+ verify(orphanedRelationCleanup, times(expectedInvocations))
+ .softDeleteOrphanedRelations(type,
GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT);
+ }
+ }
+
+ @Test
+ public void testBackendWithoutOrphanedRelationCleanupIsSkipped() {
+ RelationalBackend backend = mock(RelationalBackend.class);
+ Config config = mock(Config.class);
+ when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(1000L);
+ when(config.get(VERSION_RETENTION_COUNT)).thenReturn(1L);
+
+ Assertions.assertDoesNotThrow(
+ () -> new RelationalGarbageCollector(backend,
config).collectAndClean());
+ }
+}
diff --git
a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestOrphanedMetadataObjectRelationService.java
b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestOrphanedMetadataObjectRelationService.java
new file mode 100644
index 0000000000..e9aaf83ae4
--- /dev/null
+++
b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestOrphanedMetadataObjectRelationService.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.storage.relational.service;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import org.apache.gravitino.MetadataObject;
+import org.apache.gravitino.Namespace;
+import org.apache.gravitino.meta.TableEntity;
+import org.apache.gravitino.storage.relational.TestJDBCBackend;
+import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper;
+import org.apache.ibatis.session.SqlSession;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.TestTemplate;
+
+/** Tests orphaned metadata-object relation collection. */
+public class TestOrphanedMetadataObjectRelationService extends TestJDBCBackend
{
+ private static final long ORPHAN_ID = 987654321L;
+
+ @TestTemplate
+ public void testSoftDeleteOrphanedRelationsAndKeepLiveRelations() throws
Exception {
+ String metalake = "metalake_for_orphan_relation_test";
+ String catalog = "catalog_for_orphan_relation_test";
+ String schema = "schema_for_orphan_relation_test";
+ createAndInsertMakeLake(metalake);
+ createAndInsertCatalog(metalake, catalog);
+ createAndInsertSchema(metalake, catalog, schema);
+ TableEntity liveTable =
+ createAndInsertTableEntity(Namespace.of(metalake, catalog, schema),
"live_table");
+
+ insertRelations(liveTable.id(), ORPHAN_ID);
+
+ Assertions.assertEquals(
+ 5,
+ OrphanedMetadataObjectRelationService.getInstance()
+ .softDeleteOrphanedRelations(MetadataObject.Type.TABLE, 10));
+ Assertions.assertEquals(5, countActiveRelations(liveTable.id()));
+ Assertions.assertEquals(0, countActiveRelations(ORPHAN_ID));
+ Assertions.assertEquals(
+ 0,
+ OrphanedMetadataObjectRelationService.getInstance()
+ .softDeleteOrphanedRelations(MetadataObject.Type.TABLE, 10));
+ }
+
+ private void insertRelations(long liveId, long orphanId) throws SQLException
{
+ try (SqlSession session =
+
SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true);
+ Connection connection = session.getConnection();
+ Statement statement = connection.createStatement()) {
+ for (long objectId : new long[] {liveId, orphanId}) {
+ statement.executeUpdate(
+ "INSERT INTO owner_meta (metalake_id, metadata_object_id,
metadata_object_type,"
+ + " owner_id, owner_type, audit_info, current_version,
last_version, deleted_at,"
+ + " updated_at) VALUES (1, "
+ + objectId
+ + ", 'TABLE', 1, 'USER', '{}', 0, 0, 0, 0)");
+ statement.executeUpdate(
+ "INSERT INTO tag_relation_meta (tag_id, metadata_object_id,
metadata_object_type,"
+ + " audit_info, current_version, last_version, deleted_at)
VALUES ("
+ + objectId
+ + ", "
+ + objectId
+ + ", 'TABLE', '{}', 0, 0, 0)");
+ statement.executeUpdate(
+ "INSERT INTO policy_relation_meta (policy_id, metadata_object_id,"
+ + " metadata_object_type, audit_info, current_version,
last_version, deleted_at)"
+ + " VALUES ("
+ + objectId
+ + ", "
+ + objectId
+ + ", 'TABLE', '{}', 0, 0, 0)");
+ statement.executeUpdate(
+ "INSERT INTO statistic_meta (statistic_id, statistic_name,
statistic_value,"
+ + " metalake_id, metadata_object_id, metadata_object_type,
audit_info,"
+ + " current_version, last_version, deleted_at) VALUES ("
+ + objectId
+ + ", 'row_count_"
+ + objectId
+ + "', '{}', 1, "
+ + objectId
+ + ", 'TABLE', '{}', 0, 0, 0)");
+ statement.executeUpdate(
+ "INSERT INTO role_meta_securable_object (role_id,
metadata_object_id, type,"
+ + " privilege_names, privilege_conditions, current_version,
last_version,"
+ + " deleted_at) VALUES ("
+ + objectId
+ + ", "
+ + objectId
+ + ", 'TABLE', 'SELECT_TABLE', '', 0, 0, 0)");
+ }
+ }
+ }
+
+ private int countActiveRelations(long objectId) throws SQLException {
+ String query =
+ "SELECT SUM(relation_count) FROM ("
+ + "SELECT COUNT(*) relation_count FROM owner_meta WHERE
metadata_object_id = "
+ + objectId
+ + " AND deleted_at = 0 UNION ALL "
+ + "SELECT COUNT(*) FROM tag_relation_meta WHERE metadata_object_id
= "
+ + objectId
+ + " AND deleted_at = 0 UNION ALL "
+ + "SELECT COUNT(*) FROM policy_relation_meta WHERE
metadata_object_id = "
+ + objectId
+ + " AND deleted_at = 0 UNION ALL "
+ + "SELECT COUNT(*) FROM statistic_meta WHERE metadata_object_id = "
+ + objectId
+ + " AND deleted_at = 0 UNION ALL "
+ + "SELECT COUNT(*) FROM role_meta_securable_object WHERE
metadata_object_id = "
+ + objectId
+ + " AND deleted_at = 0) relation_counts";
+ try (SqlSession session =
+
SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true);
+ Connection connection = session.getConnection();
+ Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(query)) {
+ Assertions.assertTrue(resultSet.next());
+ return resultSet.getInt(1);
+ }
+ }
+}
