jerryshao commented on code in PR #11299: URL: https://github.com/apache/gravitino/pull/11299#discussion_r3333096426
########## iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/cleanup/IcebergCleanupManager.java: ########## @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.service.cleanup; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Iterators; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ReachableFileUtil; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.exceptions.NotFoundException; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.FileIO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Server-wide async cleanup engine: claims {@code iceberg_cleanup_job} rows, deletes the dropped + * table's files in bulk, and renews claim heartbeats on a thread decoupled from deletion. + */ +public class IcebergCleanupManager implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergCleanupManager.class); + + private final IcebergCleanupJobStore store; + private final int workerThreads; + private final int deleteBatchSize; + private final int maxAttempts; + private final int candidateWindow; + private final long pollIntervalMs; + private final long heartbeatTimeoutMs; + private final long retentionMs; + private final ThreadPoolExecutor deleteExecutor; + private final Map<Long, Long> ownedHeartbeats = new ConcurrentHashMap<>(); + + private final AtomicBoolean running = new AtomicBoolean(false); + private final AtomicBoolean closed = new AtomicBoolean(false); + private ExecutorService workers; + private ScheduledExecutorService scheduler; + + /** + * Creates an async cleanup manager. + * + * @param store the cleanup job store backed by the entity store's relational backend + * @param config Iceberg REST server config + */ + public IcebergCleanupManager(IcebergCleanupJobStore store, IcebergConfig config) { + this.store = store; + this.workerThreads = config.get(IcebergConfig.ASYNC_CLEANUP_WORKER_THREADS); + int deleteThreads = config.get(IcebergConfig.ASYNC_CLEANUP_DELETE_THREADS); + this.deleteBatchSize = config.get(IcebergConfig.ASYNC_CLEANUP_DELETE_BATCH_SIZE); + this.pollIntervalMs = config.get(IcebergConfig.ASYNC_CLEANUP_POLL_INTERVAL_SECS) * 1000L; + this.heartbeatTimeoutMs = + config.get(IcebergConfig.ASYNC_CLEANUP_HEARTBEAT_TIMEOUT_SECS) * 1000L; + this.maxAttempts = config.get(IcebergConfig.ASYNC_CLEANUP_MAX_ATTEMPTS); + this.retentionMs = config.get(IcebergConfig.ASYNC_CLEANUP_RETENTION_HOURS) * 3_600_000L; + this.candidateWindow = Math.max(8, workerThreads * 4); + this.deleteExecutor = + new ThreadPoolExecutor( + deleteThreads, + deleteThreads, + 60L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(deleteThreads * 4), + daemon("iceberg-cleanup-delete"), + new ThreadPoolExecutor.CallerRunsPolicy()); + } + + /** + * Persists a new cleanup job. + * + * @param job job to persist + * @return generated id + */ + public long addJob(IcebergCleanupJob job) { + return store.addJob(job); + } + + /** + * Checks whether an unfinished cleanup job occupies a table identifier. + * + * @param catalogId globally unique id of the owning catalog + * @param namespace table namespace + * @param table table name + * @return true iff a PENDING or RUNNING job exists for the identifier + */ + public boolean isNameOccupied(long catalogId, String namespace, String table) { + return store.findUnfinishedJobId(catalogId, namespace, table).isPresent(); + } + + /** Starts worker threads and the heartbeat/prune scheduler. */ + public void start() { + if (closed.get()) { + throw new IllegalStateException("Iceberg cleanup manager is already closed"); + } + + // compareAndSet keeps concurrent or repeated start() calls from each allocating a pool. + if (!running.compareAndSet(false, true)) { + return; + } + + // We submit exactly workerThreads loops, so the queue is never used; it is bounded only to + // avoid Executors.newFixedThreadPool's unbounded queue. + workers = + new ThreadPoolExecutor( + workerThreads, + workerThreads, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(workerThreads), + daemon("iceberg-cleanup-worker")); + for (int i = 0; i < workerThreads; i++) { + workers.submit(this::workerLoop); + } + + // One scheduler thread runs both periodic tasks: heartbeat renewal and row pruning. + scheduler = Executors.newScheduledThreadPool(1, daemon("iceberg-cleanup-heartbeat-prune")); + long heartbeatIntervalMs = heartbeatTimeoutMs / 3L; + scheduler.scheduleAtFixedRate( + this::refreshHeartbeats, heartbeatIntervalMs, heartbeatIntervalMs, TimeUnit.MILLISECONDS); + scheduler.scheduleAtFixedRate(this::prune, 1L, 1L, TimeUnit.HOURS); + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + + running.set(false); + if (scheduler != null) { + scheduler.shutdownNow(); + } + if (workers != null) { + workers.shutdownNow(); + try { + workers.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + deleteExecutor.shutdownNow(); + // The job store is backed by the entity store's shared relational backend, which owns the + // connection pool lifecycle, so there is nothing to close here. + } + + @VisibleForTesting + void cleanupFiles(FileIO io, String metadataLocation) { + cleanupFiles(io, metadataLocation, () -> true); + } + + void cleanupFiles(FileIO io, String metadataLocation, BooleanSupplier stillOwned) { + TableMetadata metadata; + try { + metadata = TableMetadataParser.read(io, metadataLocation); + } catch (NotFoundException metadataAlreadyGone) { + // A missing root metadata.json means the table is already gone. Since we delete it last, its + // absence proves every file under it was deleted first, so this is the one NotFoundException + // we treat as success. Return and let runJob mark the job SUCCEEDED. + LOG.info("Cleanup metadata {} already absent; treating as done", metadataLocation); + return; + } + + Table table = new BaseTable(new StaticTableOperations(metadata, io), "async-cleanup"); + + // Delete children before parents, root metadata.json last. Each deleteAll blocks until its + // level is gone, so a crash always leaves the root (and the manifests above any surviving file) + // readable for a retry to rebuild from. Deleting a parent first would orphan its children. + // + // Data files are the only huge level, so they are streamed and deleted one manifest at a time + // rather than all collected first; only the smaller manifest/list/metadata paths are held. + // + // deleteAll checks ownership before each delete batch. Because each dependency level is + // deleted through deleteAll, a reclaimed worker stops before submitting more file deletes. + Set<String> manifests = new LinkedHashSet<>(); + deleteDataFiles(io, metadata, manifests, stillOwned); + deleteAll(io, manifests, stillOwned); + deleteAll(io, ReachableFileUtil.manifestListLocations(table), stillOwned); + deleteAll(io, ReachableFileUtil.statisticsFilesLocations(table), stillOwned); + + // metadataFileLocations includes the current metadata.json; drop it so it is deleted last. + Set<String> ancestorMetadata = + new LinkedHashSet<>(ReachableFileUtil.metadataFileLocations(table, true)); + ancestorMetadata.remove(metadataLocation); + deleteAll(io, ancestorMetadata, stillOwned); + deleteAll(io, Collections.singletonList(metadataLocation), stillOwned); + } + + void deleteAll(FileIO io, Iterable<String> files) { + deleteAll(io, files, () -> true); + } Review Comment: Also here, what is the purpose of this method? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
