This is an automated email from the ASF dual-hosted git repository. yuqi1129 pushed a commit to branch fix/12154-orphaned-relations-gc in repository https://gitbox.apache.org/repos/asf/gravitino.git
commit cc1a3064057fe1ebca38d47ee3eeab6d7c3bb469 Author: yuqi <[email protected]> AuthorDate: Wed Jul 22 18:01:14 2026 +0800 [#12154] fix(core): Clean up orphaned metadata object relations --- .../gravitino/storage/relational/JDBCBackend.java | 8 ++ .../storage/relational/RelationalBackend.java | 12 ++ .../relational/RelationalGarbageCollector.java | 16 +++ .../OrphanedMetadataObjectRelationMapper.java | 81 +++++++++++ .../OrphanedMetadataObjectRelationSQLProvider.java | 96 +++++++++++++ .../provider/DefaultMapperPackageProvider.java | 2 + .../OrphanedMetadataObjectRelationService.java | 152 +++++++++++++++++++++ .../relational/TestRelationalGarbageCollector.java | 54 ++++++++ .../TestOrphanedMetadataObjectRelationService.java | 139 +++++++++++++++++++ 9 files changed, 560 insertions(+) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index 922f324cd4..27fa4bbfe8 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -36,6 +36,7 @@ import org.apache.gravitino.Configs; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.HasIdentifier; +import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.RelationalEntity; @@ -72,6 +73,7 @@ import org.apache.gravitino.storage.relational.service.JobTemplateMetaService; import org.apache.gravitino.storage.relational.service.MetalakeMetaService; import org.apache.gravitino.storage.relational.service.ModelMetaService; import org.apache.gravitino.storage.relational.service.ModelVersionMetaService; +import org.apache.gravitino.storage.relational.service.OrphanedMetadataObjectRelationService; import org.apache.gravitino.storage.relational.service.OwnerMetaService; import org.apache.gravitino.storage.relational.service.PolicyMetaService; import org.apache.gravitino.storage.relational.service.RoleMetaService; @@ -549,6 +551,12 @@ public class JDBCBackend implements RelationalBackend { } } + @Override + public int softDeleteOrphanedRelations(MetadataObject.Type metadataObjectType, int limit) { + return OrphanedMetadataObjectRelationService.getInstance() + .softDeleteOrphanedRelations(metadataObjectType, limit); + } + @Override public int deleteOldVersionData(Entity.EntityType entityType, long versionRetentionCount) throws IOException { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java index cffdd5e278..4e8701c54b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java @@ -27,6 +27,7 @@ import org.apache.gravitino.Config; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.HasIdentifier; +import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.SupportsRelationOperations; @@ -200,6 +201,17 @@ public interface RelationalBackend extends Closeable, SupportsRelationOperations */ int hardDeleteLegacyData(Entity.EntityType entityType, long legacyTimeline) throws IOException; + /** + * Soft-deletes relation rows that reference no live metadata object of the given type. + * + * @param metadataObjectType metadata object type to collect + * @param limit maximum number of orphaned object IDs processed per relation table + * @return number of relation rows soft-deleted + * @throws IOException if the cleanup fails + */ + int softDeleteOrphanedRelations(MetadataObject.Type metadataObjectType, int limit) + throws IOException; + /** * Soft deletes the old version data that is older than or equal to the given version retention * count. diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalGarbageCollector.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalGarbageCollector.java index 8864ae8a4d..202bddb7ca 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalGarbageCollector.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalGarbageCollector.java @@ -19,6 +19,7 @@ package org.apache.gravitino.storage.relational; +import static org.apache.gravitino.Configs.GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT; import static org.apache.gravitino.Configs.STORE_DELETE_AFTER_TIME; import static org.apache.gravitino.Configs.VERSION_RETENTION_COUNT; @@ -31,6 +32,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.gravitino.Config; import org.apache.gravitino.Entity; +import org.apache.gravitino.MetadataObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,6 +78,20 @@ public final class RelationalGarbageCollector implements Closeable { try { LOG.debug("Start to collect and delete legacy data by thread {}", threadId); long legacyTimeline = System.currentTimeMillis() - storeDeleteAfterTimeMillis; + for (MetadataObject.Type metadataObjectType : MetadataObject.Type.values()) { + long deletedCount = Long.MAX_VALUE; + LOG.debug("Try to softly delete orphaned {} relations", metadataObjectType); + try { + while (deletedCount > 0) { + deletedCount = + backend.softDeleteOrphanedRelations( + metadataObjectType, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + } + } catch (RuntimeException e) { + LOG.error("Failed to softly delete orphaned " + metadataObjectType + " relations: ", e); + } + } + for (Entity.EntityType entityType : Entity.EntityType.values()) { long deletedCount = Long.MAX_VALUE; LOG.debug( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OrphanedMetadataObjectRelationMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OrphanedMetadataObjectRelationMapper.java new file mode 100644 index 0000000000..fbb63e0f58 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OrphanedMetadataObjectRelationMapper.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.UpdateProvider; + +/** A MyBatis mapper for collecting orphaned metadata-object relations. */ +public interface OrphanedMetadataObjectRelationMapper { + + /** Soft-deletes orphaned owner relations and returns the affected row count. */ + @UpdateProvider( + type = OrphanedMetadataObjectRelationSQLProvider.class, + method = "softDeleteOrphanedOwnerRelations") + int softDeleteOrphanedOwnerRelations( + @Param("entityTable") String entityTable, + @Param("entityIdColumn") String entityIdColumn, + @Param("metadataObjectType") String metadataObjectType, + @Param("deletedAt") long deletedAt, + @Param("limit") int limit); + + /** Soft-deletes orphaned tag relations and returns the affected row count. */ + @UpdateProvider( + type = OrphanedMetadataObjectRelationSQLProvider.class, + method = "softDeleteOrphanedTagRelations") + int softDeleteOrphanedTagRelations( + @Param("entityTable") String entityTable, + @Param("entityIdColumn") String entityIdColumn, + @Param("metadataObjectType") String metadataObjectType, + @Param("deletedAt") long deletedAt, + @Param("limit") int limit); + + /** Soft-deletes orphaned policy relations and returns the affected row count. */ + @UpdateProvider( + type = OrphanedMetadataObjectRelationSQLProvider.class, + method = "softDeleteOrphanedPolicyRelations") + int softDeleteOrphanedPolicyRelations( + @Param("entityTable") String entityTable, + @Param("entityIdColumn") String entityIdColumn, + @Param("metadataObjectType") String metadataObjectType, + @Param("deletedAt") long deletedAt, + @Param("limit") int limit); + + /** Soft-deletes orphaned statistics and returns the affected row count. */ + @UpdateProvider( + type = OrphanedMetadataObjectRelationSQLProvider.class, + method = "softDeleteOrphanedStatistics") + int softDeleteOrphanedStatistics( + @Param("entityTable") String entityTable, + @Param("entityIdColumn") String entityIdColumn, + @Param("metadataObjectType") String metadataObjectType, + @Param("deletedAt") long deletedAt, + @Param("limit") int limit); + + /** Soft-deletes orphaned securable objects and returns the affected row count. */ + @UpdateProvider( + type = OrphanedMetadataObjectRelationSQLProvider.class, + method = "softDeleteOrphanedSecurableObjects") + int softDeleteOrphanedSecurableObjects( + @Param("entityTable") String entityTable, + @Param("entityIdColumn") String entityIdColumn, + @Param("metadataObjectType") String metadataObjectType, + @Param("deletedAt") long deletedAt, + @Param("limit") int limit); +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OrphanedMetadataObjectRelationSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OrphanedMetadataObjectRelationSQLProvider.java new file mode 100644 index 0000000000..3aa47e9631 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OrphanedMetadataObjectRelationSQLProvider.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper.OWNER_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.PolicyMetadataObjectRelMapper.POLICY_METADATA_OBJECT_RELATION_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper.SECURABLE_OBJECT_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.StatisticMetaMapper.STATISTIC_META_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.TagMetadataObjectRelMapper.TAG_METADATA_OBJECT_RELATION_TABLE_NAME; + +import com.google.common.base.Preconditions; +import org.apache.ibatis.annotations.Param; + +/** Provides SQL for collecting orphaned metadata-object relations. */ +public class OrphanedMetadataObjectRelationSQLProvider { + + /** Returns SQL that soft-deletes orphaned owner relations. */ + public static String softDeleteOrphanedOwnerRelations( + @Param("entityTable") String entityTable, @Param("entityIdColumn") String entityIdColumn) { + return softDeleteOrphans(OWNER_TABLE_NAME, "metadata_object_type", entityTable, entityIdColumn); + } + + /** Returns SQL that soft-deletes orphaned tag relations. */ + public static String softDeleteOrphanedTagRelations( + @Param("entityTable") String entityTable, @Param("entityIdColumn") String entityIdColumn) { + return softDeleteOrphans( + TAG_METADATA_OBJECT_RELATION_TABLE_NAME, + "metadata_object_type", + entityTable, + entityIdColumn); + } + + /** Returns SQL that soft-deletes orphaned policy relations. */ + public static String softDeleteOrphanedPolicyRelations( + @Param("entityTable") String entityTable, @Param("entityIdColumn") String entityIdColumn) { + return softDeleteOrphans( + POLICY_METADATA_OBJECT_RELATION_TABLE_NAME, + "metadata_object_type", + entityTable, + entityIdColumn); + } + + /** Returns SQL that soft-deletes orphaned statistics. */ + public static String softDeleteOrphanedStatistics( + @Param("entityTable") String entityTable, @Param("entityIdColumn") String entityIdColumn) { + return softDeleteOrphans( + STATISTIC_META_TABLE_NAME, "metadata_object_type", entityTable, entityIdColumn); + } + + /** Returns SQL that soft-deletes orphaned securable objects. */ + public static String softDeleteOrphanedSecurableObjects( + @Param("entityTable") String entityTable, @Param("entityIdColumn") String entityIdColumn) { + return softDeleteOrphans(SECURABLE_OBJECT_TABLE_NAME, "type", entityTable, entityIdColumn); + } + + private static String softDeleteOrphans( + String relationTable, String typeColumn, String entityTable, String entityIdColumn) { + Preconditions.checkArgument( + entityTable.matches("[a-z_]+") && entityIdColumn.matches("[a-z_]+"), + "Invalid entity table mapping: %s.%s", + entityTable, + entityIdColumn); + return "UPDATE " + + relationTable + + " SET deleted_at = #{deletedAt}" + + " WHERE deleted_at = 0 AND " + + typeColumn + + " = #{metadataObjectType} AND metadata_object_id IN (" + + "SELECT metadata_object_id FROM (SELECT DISTINCT rel.metadata_object_id FROM " + + relationTable + + " rel WHERE rel.deleted_at = 0 AND rel." + + typeColumn + + " = #{metadataObjectType} AND NOT EXISTS (SELECT 1 FROM " + + entityTable + + " entity WHERE entity." + + entityIdColumn + + " = rel.metadata_object_id AND entity.deleted_at = 0)" + + " LIMIT #{limit}) orphan_ids)"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/DefaultMapperPackageProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/DefaultMapperPackageProvider.java index d4ead5df7a..8065ecfa3b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/DefaultMapperPackageProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/DefaultMapperPackageProvider.java @@ -34,6 +34,7 @@ import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper; import org.apache.gravitino.storage.relational.mapper.ModelMetaMapper; import org.apache.gravitino.storage.relational.mapper.ModelVersionAliasRelMapper; import org.apache.gravitino.storage.relational.mapper.ModelVersionMetaMapper; +import org.apache.gravitino.storage.relational.mapper.OrphanedMetadataObjectRelationMapper; import org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper; import org.apache.gravitino.storage.relational.mapper.PolicyMetaMapper; import org.apache.gravitino.storage.relational.mapper.PolicyMetadataObjectRelMapper; @@ -73,6 +74,7 @@ public class DefaultMapperPackageProvider implements MapperPackageProvider { ModelMetaMapper.class, ModelVersionAliasRelMapper.class, ModelVersionMetaMapper.class, + OrphanedMetadataObjectRelationMapper.class, OwnerMetaMapper.class, PolicyMetadataObjectRelMapper.class, PolicyMetaMapper.class, diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/OrphanedMetadataObjectRelationService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/OrphanedMetadataObjectRelationService.java new file mode 100644 index 0000000000..d09b47b81c --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/OrphanedMetadataObjectRelationService.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import static org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper.TABLE_NAME; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; +import org.apache.gravitino.storage.relational.mapper.FunctionMetaMapper; +import org.apache.gravitino.storage.relational.mapper.JobMetaMapper; +import org.apache.gravitino.storage.relational.mapper.JobTemplateMetaMapper; +import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper; +import org.apache.gravitino.storage.relational.mapper.ModelMetaMapper; +import org.apache.gravitino.storage.relational.mapper.OrphanedMetadataObjectRelationMapper; +import org.apache.gravitino.storage.relational.mapper.PolicyMetaMapper; +import org.apache.gravitino.storage.relational.mapper.RoleMetaMapper; +import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TableColumnMapper; +import org.apache.gravitino.storage.relational.mapper.TableMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TagMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TopicMetaMapper; +import org.apache.gravitino.storage.relational.mapper.ViewMetaMapper; +import org.apache.gravitino.storage.relational.utils.SessionUtils; + +/** Collects relation rows whose referenced metadata object is no longer live. */ +public class OrphanedMetadataObjectRelationService { + private static final OrphanedMetadataObjectRelationService INSTANCE = + new OrphanedMetadataObjectRelationService(); + + private static final Map<MetadataObject.Type, EntityTable> ENTITY_TABLES = + ImmutableMap.<MetadataObject.Type, EntityTable>builder() + .put( + MetadataObject.Type.METALAKE, + new EntityTable(MetalakeMetaMapper.TABLE_NAME, "metalake_id")) + .put(MetadataObject.Type.CATALOG, new EntityTable(TABLE_NAME, "catalog_id")) + .put( + MetadataObject.Type.SCHEMA, new EntityTable(SchemaMetaMapper.TABLE_NAME, "schema_id")) + .put( + MetadataObject.Type.FILESET, + new EntityTable(FilesetMetaMapper.META_TABLE_NAME, "fileset_id")) + .put(MetadataObject.Type.TABLE, new EntityTable(TableMetaMapper.TABLE_NAME, "table_id")) + .put(MetadataObject.Type.VIEW, new EntityTable(ViewMetaMapper.TABLE_NAME, "view_id")) + .put(MetadataObject.Type.TOPIC, new EntityTable(TopicMetaMapper.TABLE_NAME, "topic_id")) + .put( + MetadataObject.Type.COLUMN, + new EntityTable(TableColumnMapper.COLUMN_TABLE_NAME, "column_id")) + .put(MetadataObject.Type.ROLE, new EntityTable(RoleMetaMapper.ROLE_TABLE_NAME, "role_id")) + .put(MetadataObject.Type.MODEL, new EntityTable(ModelMetaMapper.TABLE_NAME, "model_id")) + .put(MetadataObject.Type.TAG, new EntityTable(TagMetaMapper.TAG_TABLE_NAME, "tag_id")) + .put( + MetadataObject.Type.POLICY, + new EntityTable(PolicyMetaMapper.POLICY_META_TABLE_NAME, "policy_id")) + .put(MetadataObject.Type.JOB, new EntityTable(JobMetaMapper.TABLE_NAME, "job_run_id")) + .put( + MetadataObject.Type.JOB_TEMPLATE, + new EntityTable(JobTemplateMetaMapper.TABLE_NAME, "job_template_id")) + .put( + MetadataObject.Type.FUNCTION, + new EntityTable(FunctionMetaMapper.TABLE_NAME, "function_id")) + .build(); + + private OrphanedMetadataObjectRelationService() {} + + /** + * Returns the singleton service instance. + * + * @return singleton service instance + */ + public static OrphanedMetadataObjectRelationService getInstance() { + return INSTANCE; + } + + /** + * Soft-deletes orphaned relation rows for one metadata object type. + * + * @param metadataObjectType metadata object type to collect + * @param limit maximum number of orphaned object IDs processed per relation table + * @return number of relation rows soft-deleted + */ + public int softDeleteOrphanedRelations(MetadataObject.Type metadataObjectType, int limit) { + Preconditions.checkNotNull(metadataObjectType, "metadataObjectType cannot be null"); + Preconditions.checkArgument(limit > 0, "limit must be positive"); + EntityTable entityTable = ENTITY_TABLES.get(metadataObjectType); + if (entityTable == null) { + return 0; + } + + long deletedAt = System.currentTimeMillis(); + return SessionUtils.doWithCommitAndFetchResult( + OrphanedMetadataObjectRelationMapper.class, + mapper -> + mapper.softDeleteOrphanedOwnerRelations( + entityTable.tableName, + entityTable.idColumn, + metadataObjectType.name(), + deletedAt, + limit) + + mapper.softDeleteOrphanedTagRelations( + entityTable.tableName, + entityTable.idColumn, + metadataObjectType.name(), + deletedAt, + limit) + + mapper.softDeleteOrphanedPolicyRelations( + entityTable.tableName, + entityTable.idColumn, + metadataObjectType.name(), + deletedAt, + limit) + + mapper.softDeleteOrphanedStatistics( + entityTable.tableName, + entityTable.idColumn, + metadataObjectType.name(), + deletedAt, + limit) + + mapper.softDeleteOrphanedSecurableObjects( + entityTable.tableName, + entityTable.idColumn, + metadataObjectType.name(), + deletedAt, + limit)); + } + + private static class EntityTable { + private final String tableName; + private final String idColumn; + + private EntityTable(String tableName, String idColumn) { + this.tableName = tableName; + this.idColumn = idColumn; + } + } +} diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalGarbageCollector.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalGarbageCollector.java new file mode 100644 index 0000000000..b1963b3bcb --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalGarbageCollector.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational; + +import static org.apache.gravitino.Configs.GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT; +import static org.apache.gravitino.Configs.STORE_DELETE_AFTER_TIME; +import static org.apache.gravitino.Configs.VERSION_RETENTION_COUNT; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.apache.gravitino.Config; +import org.apache.gravitino.MetadataObject; +import org.junit.jupiter.api.Test; + +/** Tests relational garbage collector orchestration. */ +public class TestRelationalGarbageCollector { + + @Test + public void testCollectOrphanedRelationsUntilBatchIsEmpty() throws Exception { + RelationalBackend backend = mock(RelationalBackend.class); + Config config = mock(Config.class); + when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(1000L); + when(config.get(VERSION_RETENTION_COUNT)).thenReturn(1L); + when(backend.softDeleteOrphanedRelations( + MetadataObject.Type.TABLE, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT)) + .thenReturn(2, 0); + + new RelationalGarbageCollector(backend, config).collectAndClean(); + + for (MetadataObject.Type type : MetadataObject.Type.values()) { + int expectedInvocations = type == MetadataObject.Type.TABLE ? 2 : 1; + verify(backend, times(expectedInvocations)) + .softDeleteOrphanedRelations(type, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT); + } + } +} diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestOrphanedMetadataObjectRelationService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestOrphanedMetadataObjectRelationService.java new file mode 100644 index 0000000000..e9aaf83ae4 --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestOrphanedMetadataObjectRelationService.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.meta.TableEntity; +import org.apache.gravitino.storage.relational.TestJDBCBackend; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.session.SqlSession; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.TestTemplate; + +/** Tests orphaned metadata-object relation collection. */ +public class TestOrphanedMetadataObjectRelationService extends TestJDBCBackend { + private static final long ORPHAN_ID = 987654321L; + + @TestTemplate + public void testSoftDeleteOrphanedRelationsAndKeepLiveRelations() throws Exception { + String metalake = "metalake_for_orphan_relation_test"; + String catalog = "catalog_for_orphan_relation_test"; + String schema = "schema_for_orphan_relation_test"; + createAndInsertMakeLake(metalake); + createAndInsertCatalog(metalake, catalog); + createAndInsertSchema(metalake, catalog, schema); + TableEntity liveTable = + createAndInsertTableEntity(Namespace.of(metalake, catalog, schema), "live_table"); + + insertRelations(liveTable.id(), ORPHAN_ID); + + Assertions.assertEquals( + 5, + OrphanedMetadataObjectRelationService.getInstance() + .softDeleteOrphanedRelations(MetadataObject.Type.TABLE, 10)); + Assertions.assertEquals(5, countActiveRelations(liveTable.id())); + Assertions.assertEquals(0, countActiveRelations(ORPHAN_ID)); + Assertions.assertEquals( + 0, + OrphanedMetadataObjectRelationService.getInstance() + .softDeleteOrphanedRelations(MetadataObject.Type.TABLE, 10)); + } + + private void insertRelations(long liveId, long orphanId) throws SQLException { + try (SqlSession session = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); + Connection connection = session.getConnection(); + Statement statement = connection.createStatement()) { + for (long objectId : new long[] {liveId, orphanId}) { + statement.executeUpdate( + "INSERT INTO owner_meta (metalake_id, metadata_object_id, metadata_object_type," + + " owner_id, owner_type, audit_info, current_version, last_version, deleted_at," + + " updated_at) VALUES (1, " + + objectId + + ", 'TABLE', 1, 'USER', '{}', 0, 0, 0, 0)"); + statement.executeUpdate( + "INSERT INTO tag_relation_meta (tag_id, metadata_object_id, metadata_object_type," + + " audit_info, current_version, last_version, deleted_at) VALUES (" + + objectId + + ", " + + objectId + + ", 'TABLE', '{}', 0, 0, 0)"); + statement.executeUpdate( + "INSERT INTO policy_relation_meta (policy_id, metadata_object_id," + + " metadata_object_type, audit_info, current_version, last_version, deleted_at)" + + " VALUES (" + + objectId + + ", " + + objectId + + ", 'TABLE', '{}', 0, 0, 0)"); + statement.executeUpdate( + "INSERT INTO statistic_meta (statistic_id, statistic_name, statistic_value," + + " metalake_id, metadata_object_id, metadata_object_type, audit_info," + + " current_version, last_version, deleted_at) VALUES (" + + objectId + + ", 'row_count_" + + objectId + + "', '{}', 1, " + + objectId + + ", 'TABLE', '{}', 0, 0, 0)"); + statement.executeUpdate( + "INSERT INTO role_meta_securable_object (role_id, metadata_object_id, type," + + " privilege_names, privilege_conditions, current_version, last_version," + + " deleted_at) VALUES (" + + objectId + + ", " + + objectId + + ", 'TABLE', 'SELECT_TABLE', '', 0, 0, 0)"); + } + } + } + + private int countActiveRelations(long objectId) throws SQLException { + String query = + "SELECT SUM(relation_count) FROM (" + + "SELECT COUNT(*) relation_count FROM owner_meta WHERE metadata_object_id = " + + objectId + + " AND deleted_at = 0 UNION ALL " + + "SELECT COUNT(*) FROM tag_relation_meta WHERE metadata_object_id = " + + objectId + + " AND deleted_at = 0 UNION ALL " + + "SELECT COUNT(*) FROM policy_relation_meta WHERE metadata_object_id = " + + objectId + + " AND deleted_at = 0 UNION ALL " + + "SELECT COUNT(*) FROM statistic_meta WHERE metadata_object_id = " + + objectId + + " AND deleted_at = 0 UNION ALL " + + "SELECT COUNT(*) FROM role_meta_securable_object WHERE metadata_object_id = " + + objectId + + " AND deleted_at = 0) relation_counts"; + try (SqlSession session = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); + Connection connection = session.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(query)) { + Assertions.assertTrue(resultSet.next()); + return resultSet.getInt(1); + } + } +}
