jerryshao commented on code in PR #11299: URL: https://github.com/apache/gravitino/pull/11299#discussion_r3333105326
########## iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/cleanup/IcebergCleanupManager.java: ########## @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.service.cleanup; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Iterators; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ReachableFileUtil; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.exceptions.NotFoundException; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.FileIO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Server-wide async cleanup engine: claims {@code iceberg_cleanup_job} rows, deletes the dropped + * table's files in bulk, and renews claim heartbeats on a thread decoupled from deletion. + */ +public class IcebergCleanupManager implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergCleanupManager.class); + + private final IcebergCleanupJobStore store; + private final int workerThreads; + private final int deleteBatchSize; + private final int maxAttempts; + private final int candidateWindow; + private final long pollIntervalMs; + private final long heartbeatTimeoutMs; + private final long retentionMs; + private final ThreadPoolExecutor deleteExecutor; + private final Map<Long, Long> ownedHeartbeats = new ConcurrentHashMap<>(); + + private final AtomicBoolean running = new AtomicBoolean(false); + private final AtomicBoolean closed = new AtomicBoolean(false); + private ExecutorService workers; + private ScheduledExecutorService scheduler; + + /** + * Creates an async cleanup manager. + * + * @param store the cleanup job store backed by the entity store's relational backend + * @param config Iceberg REST server config + */ + public IcebergCleanupManager(IcebergCleanupJobStore store, IcebergConfig config) { + this.store = store; + this.workerThreads = config.get(IcebergConfig.ASYNC_CLEANUP_WORKER_THREADS); + int deleteThreads = config.get(IcebergConfig.ASYNC_CLEANUP_DELETE_THREADS); + this.deleteBatchSize = config.get(IcebergConfig.ASYNC_CLEANUP_DELETE_BATCH_SIZE); + this.pollIntervalMs = config.get(IcebergConfig.ASYNC_CLEANUP_POLL_INTERVAL_SECS) * 1000L; + this.heartbeatTimeoutMs = + config.get(IcebergConfig.ASYNC_CLEANUP_HEARTBEAT_TIMEOUT_SECS) * 1000L; + this.maxAttempts = config.get(IcebergConfig.ASYNC_CLEANUP_MAX_ATTEMPTS); + this.retentionMs = config.get(IcebergConfig.ASYNC_CLEANUP_RETENTION_HOURS) * 3_600_000L; + this.candidateWindow = Math.max(8, workerThreads * 4); + this.deleteExecutor = + new ThreadPoolExecutor( + deleteThreads, + deleteThreads, + 60L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(deleteThreads * 4), + daemon("iceberg-cleanup-delete"), + new ThreadPoolExecutor.CallerRunsPolicy()); + } + + /** + * Persists a new cleanup job. + * + * @param job job to persist + * @return generated id + */ + public long addJob(IcebergCleanupJob job) { + return store.addJob(job); + } + + /** + * Checks whether an unfinished cleanup job occupies a table identifier. + * + * @param catalogId globally unique id of the owning catalog + * @param namespace table namespace + * @param table table name + * @return true iff a PENDING or RUNNING job exists for the identifier + */ + public boolean isNameOccupied(long catalogId, String namespace, String table) { + return store.findUnfinishedJobId(catalogId, namespace, table).isPresent(); + } + + /** Starts worker threads and the heartbeat/prune scheduler. */ + public void start() { + if (closed.get()) { + throw new IllegalStateException("Iceberg cleanup manager is already closed"); + } + + // compareAndSet keeps concurrent or repeated start() calls from each allocating a pool. + if (!running.compareAndSet(false, true)) { + return; + } + + // We submit exactly workerThreads loops, so the queue is never used; it is bounded only to + // avoid Executors.newFixedThreadPool's unbounded queue. + workers = + new ThreadPoolExecutor( + workerThreads, + workerThreads, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(workerThreads), + daemon("iceberg-cleanup-worker")); + for (int i = 0; i < workerThreads; i++) { + workers.submit(this::workerLoop); + } + + // One scheduler thread runs both periodic tasks: heartbeat renewal and row pruning. + scheduler = Executors.newScheduledThreadPool(1, daemon("iceberg-cleanup-heartbeat-prune")); + long heartbeatIntervalMs = heartbeatTimeoutMs / 3L; + scheduler.scheduleAtFixedRate( + this::refreshHeartbeats, heartbeatIntervalMs, heartbeatIntervalMs, TimeUnit.MILLISECONDS); + scheduler.scheduleAtFixedRate(this::prune, 1L, 1L, TimeUnit.HOURS); + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + + running.set(false); + if (scheduler != null) { + scheduler.shutdownNow(); + } + if (workers != null) { + workers.shutdownNow(); + try { + workers.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + deleteExecutor.shutdownNow(); + // The job store is backed by the entity store's shared relational backend, which owns the Review Comment: `deleteExecutor.shutdownNow()` is fire-and-forget — in-flight delete batches are interrupted without any wait. Consider adding `deleteExecutor.awaitTermination(5, TimeUnit.SECONDS)` here, matching the pattern used for `workers` above. ########## iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/cleanup/IcebergCleanupManager.java: ########## @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.service.cleanup; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Iterators; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ReachableFileUtil; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.exceptions.NotFoundException; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.FileIO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Server-wide async cleanup engine: claims {@code iceberg_cleanup_job} rows, deletes the dropped + * table's files in bulk, and renews claim heartbeats on a thread decoupled from deletion. + */ +public class IcebergCleanupManager implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergCleanupManager.class); + + private final IcebergCleanupJobStore store; + private final int workerThreads; + private final int deleteBatchSize; + private final int maxAttempts; + private final int candidateWindow; + private final long pollIntervalMs; + private final long heartbeatTimeoutMs; + private final long retentionMs; + private final ThreadPoolExecutor deleteExecutor; + private final Map<Long, Long> ownedHeartbeats = new ConcurrentHashMap<>(); + + private final AtomicBoolean running = new AtomicBoolean(false); + private final AtomicBoolean closed = new AtomicBoolean(false); + private ExecutorService workers; + private ScheduledExecutorService scheduler; + + /** + * Creates an async cleanup manager. + * + * @param store the cleanup job store backed by the entity store's relational backend + * @param config Iceberg REST server config + */ + public IcebergCleanupManager(IcebergCleanupJobStore store, IcebergConfig config) { + this.store = store; + this.workerThreads = config.get(IcebergConfig.ASYNC_CLEANUP_WORKER_THREADS); + int deleteThreads = config.get(IcebergConfig.ASYNC_CLEANUP_DELETE_THREADS); + this.deleteBatchSize = config.get(IcebergConfig.ASYNC_CLEANUP_DELETE_BATCH_SIZE); + this.pollIntervalMs = config.get(IcebergConfig.ASYNC_CLEANUP_POLL_INTERVAL_SECS) * 1000L; + this.heartbeatTimeoutMs = + config.get(IcebergConfig.ASYNC_CLEANUP_HEARTBEAT_TIMEOUT_SECS) * 1000L; + this.maxAttempts = config.get(IcebergConfig.ASYNC_CLEANUP_MAX_ATTEMPTS); + this.retentionMs = config.get(IcebergConfig.ASYNC_CLEANUP_RETENTION_HOURS) * 3_600_000L; + this.candidateWindow = Math.max(8, workerThreads * 4); + this.deleteExecutor = + new ThreadPoolExecutor( + deleteThreads, + deleteThreads, + 60L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(deleteThreads * 4), + daemon("iceberg-cleanup-delete"), + new ThreadPoolExecutor.CallerRunsPolicy()); + } + + /** + * Persists a new cleanup job. + * + * @param job job to persist + * @return generated id + */ + public long addJob(IcebergCleanupJob job) { + return store.addJob(job); + } + + /** + * Checks whether an unfinished cleanup job occupies a table identifier. + * + * @param catalogId globally unique id of the owning catalog + * @param namespace table namespace + * @param table table name + * @return true iff a PENDING or RUNNING job exists for the identifier + */ + public boolean isNameOccupied(long catalogId, String namespace, String table) { + return store.findUnfinishedJobId(catalogId, namespace, table).isPresent(); + } + + /** Starts worker threads and the heartbeat/prune scheduler. */ + public void start() { + if (closed.get()) { + throw new IllegalStateException("Iceberg cleanup manager is already closed"); + } + + // compareAndSet keeps concurrent or repeated start() calls from each allocating a pool. + if (!running.compareAndSet(false, true)) { + return; + } + + // We submit exactly workerThreads loops, so the queue is never used; it is bounded only to + // avoid Executors.newFixedThreadPool's unbounded queue. + workers = + new ThreadPoolExecutor( + workerThreads, + workerThreads, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(workerThreads), + daemon("iceberg-cleanup-worker")); + for (int i = 0; i < workerThreads; i++) { + workers.submit(this::workerLoop); + } + + // One scheduler thread runs both periodic tasks: heartbeat renewal and row pruning. + scheduler = Executors.newScheduledThreadPool(1, daemon("iceberg-cleanup-heartbeat-prune")); + long heartbeatIntervalMs = heartbeatTimeoutMs / 3L; + scheduler.scheduleAtFixedRate( + this::refreshHeartbeats, heartbeatIntervalMs, heartbeatIntervalMs, TimeUnit.MILLISECONDS); + scheduler.scheduleAtFixedRate(this::prune, 1L, 1L, TimeUnit.HOURS); + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + + running.set(false); + if (scheduler != null) { + scheduler.shutdownNow(); + } + if (workers != null) { + workers.shutdownNow(); + try { + workers.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + deleteExecutor.shutdownNow(); + // The job store is backed by the entity store's shared relational backend, which owns the + // connection pool lifecycle, so there is nothing to close here. + } + + @VisibleForTesting + void cleanupFiles(FileIO io, String metadataLocation) { + cleanupFiles(io, metadataLocation, () -> true); + } + + void cleanupFiles(FileIO io, String metadataLocation, BooleanSupplier stillOwned) { + TableMetadata metadata; + try { + metadata = TableMetadataParser.read(io, metadataLocation); + } catch (NotFoundException metadataAlreadyGone) { + // A missing root metadata.json means the table is already gone. Since we delete it last, its + // absence proves every file under it was deleted first, so this is the one NotFoundException + // we treat as success. Return and let runJob mark the job SUCCEEDED. + LOG.info("Cleanup metadata {} already absent; treating as done", metadataLocation); + return; + } + + Table table = new BaseTable(new StaticTableOperations(metadata, io), "async-cleanup"); + + // Delete children before parents, root metadata.json last. Each deleteAll blocks until its + // level is gone, so a crash always leaves the root (and the manifests above any surviving file) + // readable for a retry to rebuild from. Deleting a parent first would orphan its children. + // + // Data files are the only huge level, so they are streamed and deleted one manifest at a time + // rather than all collected first; only the smaller manifest/list/metadata paths are held. + // + // deleteAll checks ownership before each delete batch. Because each dependency level is + // deleted through deleteAll, a reclaimed worker stops before submitting more file deletes. + Set<String> manifests = new LinkedHashSet<>(); + deleteDataFiles(io, metadata, manifests, stillOwned); + deleteAll(io, manifests, stillOwned); + deleteAll(io, ReachableFileUtil.manifestListLocations(table), stillOwned); + deleteAll(io, ReachableFileUtil.statisticsFilesLocations(table), stillOwned); + + // metadataFileLocations includes the current metadata.json; drop it so it is deleted last. + Set<String> ancestorMetadata = + new LinkedHashSet<>(ReachableFileUtil.metadataFileLocations(table, true)); + ancestorMetadata.remove(metadataLocation); + deleteAll(io, ancestorMetadata, stillOwned); + deleteAll(io, Collections.singletonList(metadataLocation), stillOwned); + } + + void deleteAll(FileIO io, Iterable<String> files) { + deleteAll(io, files, () -> true); + } + + void deleteAll(FileIO io, Iterable<String> files, BooleanSupplier stillOwned) { + // Callers pass one manifest's files at a time (or a small fixed list), so futures stay small; + // CallerRunsPolicy on deleteExecutor also throttles submission when the pool is saturated. + List<Future<?>> futures = new ArrayList<>(); + try { + Iterators.partition(files.iterator(), deleteBatchSize) + .forEachRemaining( + batch -> { + requireOwnership(stillOwned); + futures.add( + deleteExecutor.submit( + () -> CatalogUtil.deleteFiles(io, batch, "cleanup", true))); + }); + + for (Future<?> future : futures) { + requireOwnership(stillOwned); + try { + future.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted during bulk delete", e); + } catch (ExecutionException e) { + if (hasCause(e, NotFoundException.class)) { + LOG.debug("Ignoring already-deleted file during async cleanup", e); + continue; + } + throw new RuntimeException("Bulk delete batch failed", e); + } + } + } catch (OwnershipLostException e) { + futures.forEach(future -> future.cancel(true)); + throw e; + } + } + + private static boolean hasCause(Throwable throwable, Class<? extends Throwable> type) { + Throwable current = throwable; + while (current != null) { + if (type.isInstance(current)) { + return true; + } + current = current.getCause(); + } + return false; + } + + private void workerLoop() { + while (running.get()) { + try { + long now = System.currentTimeMillis(); + Optional<IcebergCleanupJob> job = + store.takePendingJob(now, heartbeatTimeoutMs, candidateWindow); + if (job.isEmpty()) { + sleep(pollIntervalMs); + continue; + } + + ownedHeartbeats.put(job.get().id(), now); + runJob(job.get()); + } catch (Throwable t) { + // The loop is submitted once, so if it exits the worker is gone for good. Catch everything + // (including Errors) so a fault only backs off instead of killing the worker. + if (t instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + LOG.warn("Cleanup worker loop hit an unexpected error; backing off", t); + sleep(pollIntervalMs); + } + } + } + + private void runJob(IcebergCleanupJob job) { + try { + FileIO io = CatalogUtil.loadFileIO(job.fileIOImpl(), job.fileIOProperties(), null); + cleanupFiles(io, job.metadataLocation(), () -> ownsJob(job.id())); + // markSucceeded/recordFailure CAS on our heartbeat token, so a worker that lost its lease + // cannot change a job a peer reclaimed. A null token means we already lost it; skip the call. + // If a heartbeat refresh bumps the token between this read and the CAS, the CAS just no-ops + // and the job is reclaimed and re-run later (which finds the files gone) -- harmless. + Long heartbeat = ownedHeartbeats.get(job.id()); + if (heartbeat != null) { Review Comment: **Race between `refreshHeartbeats` and `markSucceeded`/`recordFailure`** `refreshHeartbeats` updates the DB heartbeat first (`store.heartbeat`), then writes the new token back into `ownedHeartbeats`. If the worker reads `ownedHeartbeats.get(job.id())` in this window it gets the stale token, and the subsequent CAS in `markSucceeded`/`recordFailure` fails. The job stays RUNNING until heartbeat timeout and is re-claimed and re-run harmlessly (a missing metadata.json is treated as done), but it is unnecessary work. One fix: in `refreshHeartbeats`, update `ownedHeartbeats` *before* the DB call so the worker always sees a token that is ≥ the DB value. ########## iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/cleanup/IcebergCleanupManager.java: ########## @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.service.cleanup; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Iterators; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ReachableFileUtil; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.exceptions.NotFoundException; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.FileIO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Server-wide async cleanup engine: claims {@code iceberg_cleanup_job} rows, deletes the dropped + * table's files in bulk, and renews claim heartbeats on a thread decoupled from deletion. + */ +public class IcebergCleanupManager implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergCleanupManager.class); + + private final IcebergCleanupJobStore store; + private final int workerThreads; + private final int deleteBatchSize; + private final int maxAttempts; + private final int candidateWindow; + private final long pollIntervalMs; + private final long heartbeatTimeoutMs; + private final long retentionMs; + private final ThreadPoolExecutor deleteExecutor; + private final Map<Long, Long> ownedHeartbeats = new ConcurrentHashMap<>(); + + private final AtomicBoolean running = new AtomicBoolean(false); + private final AtomicBoolean closed = new AtomicBoolean(false); + private ExecutorService workers; + private ScheduledExecutorService scheduler; + + /** + * Creates an async cleanup manager. + * + * @param store the cleanup job store backed by the entity store's relational backend + * @param config Iceberg REST server config + */ + public IcebergCleanupManager(IcebergCleanupJobStore store, IcebergConfig config) { + this.store = store; + this.workerThreads = config.get(IcebergConfig.ASYNC_CLEANUP_WORKER_THREADS); + int deleteThreads = config.get(IcebergConfig.ASYNC_CLEANUP_DELETE_THREADS); + this.deleteBatchSize = config.get(IcebergConfig.ASYNC_CLEANUP_DELETE_BATCH_SIZE); + this.pollIntervalMs = config.get(IcebergConfig.ASYNC_CLEANUP_POLL_INTERVAL_SECS) * 1000L; + this.heartbeatTimeoutMs = + config.get(IcebergConfig.ASYNC_CLEANUP_HEARTBEAT_TIMEOUT_SECS) * 1000L; + this.maxAttempts = config.get(IcebergConfig.ASYNC_CLEANUP_MAX_ATTEMPTS); + this.retentionMs = config.get(IcebergConfig.ASYNC_CLEANUP_RETENTION_HOURS) * 3_600_000L; + this.candidateWindow = Math.max(8, workerThreads * 4); + this.deleteExecutor = + new ThreadPoolExecutor( + deleteThreads, + deleteThreads, + 60L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(deleteThreads * 4), + daemon("iceberg-cleanup-delete"), + new ThreadPoolExecutor.CallerRunsPolicy()); + } + + /** + * Persists a new cleanup job. + * + * @param job job to persist + * @return generated id + */ + public long addJob(IcebergCleanupJob job) { + return store.addJob(job); + } + + /** + * Checks whether an unfinished cleanup job occupies a table identifier. + * + * @param catalogId globally unique id of the owning catalog + * @param namespace table namespace + * @param table table name + * @return true iff a PENDING or RUNNING job exists for the identifier + */ + public boolean isNameOccupied(long catalogId, String namespace, String table) { + return store.findUnfinishedJobId(catalogId, namespace, table).isPresent(); + } + + /** Starts worker threads and the heartbeat/prune scheduler. */ + public void start() { + if (closed.get()) { + throw new IllegalStateException("Iceberg cleanup manager is already closed"); + } + + // compareAndSet keeps concurrent or repeated start() calls from each allocating a pool. + if (!running.compareAndSet(false, true)) { + return; + } + + // We submit exactly workerThreads loops, so the queue is never used; it is bounded only to + // avoid Executors.newFixedThreadPool's unbounded queue. + workers = + new ThreadPoolExecutor( + workerThreads, + workerThreads, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(workerThreads), + daemon("iceberg-cleanup-worker")); + for (int i = 0; i < workerThreads; i++) { + workers.submit(this::workerLoop); + } + + // One scheduler thread runs both periodic tasks: heartbeat renewal and row pruning. + scheduler = Executors.newScheduledThreadPool(1, daemon("iceberg-cleanup-heartbeat-prune")); + long heartbeatIntervalMs = heartbeatTimeoutMs / 3L; + scheduler.scheduleAtFixedRate( + this::refreshHeartbeats, heartbeatIntervalMs, heartbeatIntervalMs, TimeUnit.MILLISECONDS); + scheduler.scheduleAtFixedRate(this::prune, 1L, 1L, TimeUnit.HOURS); + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + + running.set(false); + if (scheduler != null) { + scheduler.shutdownNow(); + } + if (workers != null) { + workers.shutdownNow(); + try { + workers.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + deleteExecutor.shutdownNow(); + // The job store is backed by the entity store's shared relational backend, which owns the + // connection pool lifecycle, so there is nothing to close here. + } + + @VisibleForTesting + void cleanupFiles(FileIO io, String metadataLocation) { + cleanupFiles(io, metadataLocation, () -> true); + } + + void cleanupFiles(FileIO io, String metadataLocation, BooleanSupplier stillOwned) { + TableMetadata metadata; + try { + metadata = TableMetadataParser.read(io, metadataLocation); + } catch (NotFoundException metadataAlreadyGone) { + // A missing root metadata.json means the table is already gone. Since we delete it last, its + // absence proves every file under it was deleted first, so this is the one NotFoundException + // we treat as success. Return and let runJob mark the job SUCCEEDED. + LOG.info("Cleanup metadata {} already absent; treating as done", metadataLocation); + return; + } + + Table table = new BaseTable(new StaticTableOperations(metadata, io), "async-cleanup"); + + // Delete children before parents, root metadata.json last. Each deleteAll blocks until its + // level is gone, so a crash always leaves the root (and the manifests above any surviving file) + // readable for a retry to rebuild from. Deleting a parent first would orphan its children. + // + // Data files are the only huge level, so they are streamed and deleted one manifest at a time + // rather than all collected first; only the smaller manifest/list/metadata paths are held. + // + // deleteAll checks ownership before each delete batch. Because each dependency level is + // deleted through deleteAll, a reclaimed worker stops before submitting more file deletes. + Set<String> manifests = new LinkedHashSet<>(); + deleteDataFiles(io, metadata, manifests, stillOwned); + deleteAll(io, manifests, stillOwned); + deleteAll(io, ReachableFileUtil.manifestListLocations(table), stillOwned); + deleteAll(io, ReachableFileUtil.statisticsFilesLocations(table), stillOwned); + + // metadataFileLocations includes the current metadata.json; drop it so it is deleted last. + Set<String> ancestorMetadata = + new LinkedHashSet<>(ReachableFileUtil.metadataFileLocations(table, true)); + ancestorMetadata.remove(metadataLocation); + deleteAll(io, ancestorMetadata, stillOwned); + deleteAll(io, Collections.singletonList(metadataLocation), stillOwned); + } + + void deleteAll(FileIO io, Iterable<String> files) { + deleteAll(io, files, () -> true); + } + + void deleteAll(FileIO io, Iterable<String> files, BooleanSupplier stillOwned) { + // Callers pass one manifest's files at a time (or a small fixed list), so futures stay small; + // CallerRunsPolicy on deleteExecutor also throttles submission when the pool is saturated. + List<Future<?>> futures = new ArrayList<>(); + try { + Iterators.partition(files.iterator(), deleteBatchSize) + .forEachRemaining( + batch -> { + requireOwnership(stillOwned); + futures.add( + deleteExecutor.submit( + () -> CatalogUtil.deleteFiles(io, batch, "cleanup", true))); + }); + + for (Future<?> future : futures) { + requireOwnership(stillOwned); + try { + future.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted during bulk delete", e); + } catch (ExecutionException e) { + if (hasCause(e, NotFoundException.class)) { + LOG.debug("Ignoring already-deleted file during async cleanup", e); + continue; + } + throw new RuntimeException("Bulk delete batch failed", e); + } + } + } catch (OwnershipLostException e) { + futures.forEach(future -> future.cancel(true)); + throw e; + } + } + + private static boolean hasCause(Throwable throwable, Class<? extends Throwable> type) { + Throwable current = throwable; + while (current != null) { + if (type.isInstance(current)) { + return true; + } + current = current.getCause(); + } + return false; + } + + private void workerLoop() { + while (running.get()) { + try { + long now = System.currentTimeMillis(); + Optional<IcebergCleanupJob> job = + store.takePendingJob(now, heartbeatTimeoutMs, candidateWindow); + if (job.isEmpty()) { + sleep(pollIntervalMs); + continue; + } + + ownedHeartbeats.put(job.get().id(), now); + runJob(job.get()); + } catch (Throwable t) { + // The loop is submitted once, so if it exits the worker is gone for good. Catch everything + // (including Errors) so a fault only backs off instead of killing the worker. + if (t instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + LOG.warn("Cleanup worker loop hit an unexpected error; backing off", t); + sleep(pollIntervalMs); + } + } + } + + private void runJob(IcebergCleanupJob job) { + try { + FileIO io = CatalogUtil.loadFileIO(job.fileIOImpl(), job.fileIOProperties(), null); + cleanupFiles(io, job.metadataLocation(), () -> ownsJob(job.id())); + // markSucceeded/recordFailure CAS on our heartbeat token, so a worker that lost its lease + // cannot change a job a peer reclaimed. A null token means we already lost it; skip the call. + // If a heartbeat refresh bumps the token between this read and the CAS, the CAS just no-ops + // and the job is reclaimed and re-run later (which finds the files gone) -- harmless. + Long heartbeat = ownedHeartbeats.get(job.id()); + if (heartbeat != null) { + store.markSucceeded(job.id(), heartbeat); + } Review Comment: Return value of `markSucceeded` (and `recordFailure` a few lines below) is silently discarded. A failed CAS leaves the job stuck in RUNNING with no log entry. At minimum log a warning so the re-claim is observable. ########## iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/cleanup/IcebergCleanupManager.java: ########## @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.service.cleanup; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Iterators; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ReachableFileUtil; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.exceptions.NotFoundException; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.FileIO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Server-wide async cleanup engine: claims {@code iceberg_cleanup_job} rows, deletes the dropped + * table's files in bulk, and renews claim heartbeats on a thread decoupled from deletion. + */ +public class IcebergCleanupManager implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergCleanupManager.class); + + private final IcebergCleanupJobStore store; + private final int workerThreads; + private final int deleteBatchSize; + private final int maxAttempts; + private final int candidateWindow; + private final long pollIntervalMs; + private final long heartbeatTimeoutMs; + private final long retentionMs; + private final ThreadPoolExecutor deleteExecutor; + private final Map<Long, Long> ownedHeartbeats = new ConcurrentHashMap<>(); + + private final AtomicBoolean running = new AtomicBoolean(false); + private final AtomicBoolean closed = new AtomicBoolean(false); + private ExecutorService workers; + private ScheduledExecutorService scheduler; + + /** + * Creates an async cleanup manager. + * + * @param store the cleanup job store backed by the entity store's relational backend + * @param config Iceberg REST server config + */ + public IcebergCleanupManager(IcebergCleanupJobStore store, IcebergConfig config) { + this.store = store; + this.workerThreads = config.get(IcebergConfig.ASYNC_CLEANUP_WORKER_THREADS); + int deleteThreads = config.get(IcebergConfig.ASYNC_CLEANUP_DELETE_THREADS); + this.deleteBatchSize = config.get(IcebergConfig.ASYNC_CLEANUP_DELETE_BATCH_SIZE); + this.pollIntervalMs = config.get(IcebergConfig.ASYNC_CLEANUP_POLL_INTERVAL_SECS) * 1000L; + this.heartbeatTimeoutMs = + config.get(IcebergConfig.ASYNC_CLEANUP_HEARTBEAT_TIMEOUT_SECS) * 1000L; + this.maxAttempts = config.get(IcebergConfig.ASYNC_CLEANUP_MAX_ATTEMPTS); + this.retentionMs = config.get(IcebergConfig.ASYNC_CLEANUP_RETENTION_HOURS) * 3_600_000L; + this.candidateWindow = Math.max(8, workerThreads * 4); + this.deleteExecutor = Review Comment: The magic numbers `8` and `4` in `Math.max(8, workerThreads * 4)` are unexplained. A short comment on why this multiplier provides enough headroom (e.g. to absorb a full heartbeat interval of new candidates) would help future readers. ########## iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/cleanup/TestIcebergCleanupManager.java: ########## @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.service.cleanup; + +import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.gravitino.GravitinoEnv; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.storage.RandomIdGenerator; +import org.apache.gravitino.storage.relational.TestJDBCBackend; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NotFoundException; +import org.apache.iceberg.inmemory.InMemoryCatalog; +import org.apache.iceberg.inmemory.InMemoryFileIO; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.SupportsBulkOperations; +import org.apache.iceberg.types.Types; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestTemplate; + +/** + * Exercises {@link IcebergCleanupManager} against the same relational backend matrix as the cleanup + * store: {@link TestJDBCBackend} initializes H2 by default, adds MySQL and PostgreSQL when {@code + * dockerTest=true}, and truncates all backend tables before each invocation. + */ +class TestIcebergCleanupManager extends TestJDBCBackend { + + private static final long CATALOG_ID = 100L; + private static final String DATA_FILE = "memory://db/t/data/00000-0.parquet"; + + private IcebergCleanupJobStore store; + + // TestJDBCBackend's BackendTestExtension overwrites GravitinoEnv's singleton "config" and + // "idGenerator" fields with a backend-only Mockito mock and never restores them. Because the + // whole iceberg-rest-server module runs in one JVM, that mock would leak into later test classes + // (e.g. credential vending), where MetadataAuthzHelper.enableAuthorization() unboxes the + // unstubbed config.get(ENABLE_AUTHORIZATION) -> null and NPEs. Snapshot the pre-test fields and + // restore them after this class so it leaves GravitinoEnv exactly as it found it. + private Object originalConfig; + private Object originalIdGenerator; + + @BeforeAll + public void snapshotGravitinoEnv() throws IllegalAccessException { + originalConfig = FieldUtils.readField(GravitinoEnv.getInstance(), "config", true); + originalIdGenerator = FieldUtils.readField(GravitinoEnv.getInstance(), "idGenerator", true); + } + + @AfterAll + public void restoreGravitinoEnv() throws IllegalAccessException { + FieldUtils.writeField(GravitinoEnv.getInstance(), "config", originalConfig, true); + FieldUtils.writeField(GravitinoEnv.getInstance(), "idGenerator", originalIdGenerator, true); + } + + @BeforeEach + public void prepareCleanupJobStore() { + store = new IcebergCleanupJobStore(new RandomIdGenerator()); + } + + private static IcebergConfig fastPollConfig() { + Map<String, String> config = new HashMap<>(); + config.put("async-cleanup.worker-threads", "1"); + config.put("async-cleanup.poll-interval-secs", "1"); + return new IcebergConfig(config); + } + + private static IcebergCleanupJob sampleJob() { + return new IcebergCleanupJob( + 0L, + CATALOG_ID, + "db", + "t", + "s3://b/db/t/metadata/0.json", + NoopFileIO.class.getName(), + ImmutableMap.of(), + "alice"); + } + + // Builds db.t with one appended data file (so it has a manifest list, a manifest, and a data + // file) and materializes the data file in the in-memory FileIO so deletions are observable. + private static BaseTable tableWithDataFile() { + InMemoryCatalog catalog = new InMemoryCatalog(); + catalog.initialize("test", ImmutableMap.of()); + catalog.createNamespace(Namespace.of("db")); + TableIdentifier id = TableIdentifier.of(Namespace.of("db"), "t"); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table table = catalog.createTable(id, schema); + DataFile dataFile = + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath(DATA_FILE) + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build(); + table.newAppend().appendFile(dataFile).commit(); + BaseTable base = (BaseTable) catalog.loadTable(id); + ((InMemoryFileIO) base.io()).addFile(DATA_FILE, new byte[] {1}); + return base; + } + + @TestTemplate + void testDeleteAllBatches() { + CopyOnWriteArrayList<String> deleted = new CopyOnWriteArrayList<>(); + IcebergCleanupManager svc = + new IcebergCleanupManager(store, new IcebergConfig(new HashMap<>())); + try { + svc.deleteAll(new RecordingFileIO(deleted), Arrays.asList("a", "b", "c", "d", "e", "f", "g")); + Assertions.assertEquals(7, deleted.size()); + } finally { + svc.close(); + } + } + + @TestTemplate + void testDeleteAllIgnoresMissing() { + IcebergCleanupManager svc = + new IcebergCleanupManager(store, new IcebergConfig(new HashMap<>())); + try { + Assertions.assertDoesNotThrow( + () -> svc.deleteAll(new MissingFileIO(), Arrays.asList("already-gone"))); + } finally { + svc.close(); + } + } + + @TestTemplate + void testDeleteAllIgnoresMissingBulk() { + IcebergCleanupManager svc = + new IcebergCleanupManager(store, new IcebergConfig(new HashMap<>())); + try { + Assertions.assertDoesNotThrow( + () -> svc.deleteAll(new MissingBulkFileIO(), Arrays.asList("already-gone"))); + } finally { + svc.close(); + } + } + + @TestTemplate + void testDeleteAllStopsSubmittingWhenLeaseLost() { + Map<String, String> config = new HashMap<>(); + config.put("async-cleanup.delete-batch-size", "1"); + AtomicInteger ownershipChecks = new AtomicInteger(); + BooleanSupplier stillOwned = () -> ownershipChecks.incrementAndGet() == 1; + + IcebergCleanupManager svc = new IcebergCleanupManager(store, new IcebergConfig(config)); + try { + Assertions.assertThrows( + RuntimeException.class, + () -> + svc.deleteAll( + new RecordingFileIO(new CopyOnWriteArrayList<>()), + Arrays.asList("a", "b"), + stillOwned)); + Assertions.assertEquals(2, ownershipChecks.get()); + } finally { + svc.close(); + } + } + + @TestTemplate + void testCleanupDeletesAllFiles() { + BaseTable base = tableWithDataFile(); + FileIO io = base.io(); + String metadataLocation = base.operations().current().metadataFileLocation(); + + // Capture reachable files before cleanup; the manifests cannot be read once deleted. + List<String> expected = new ArrayList<>(); + expected.add(metadataLocation); + expected.add(DATA_FILE); + for (Snapshot snapshot : base.snapshots()) { + expected.add(snapshot.manifestListLocation()); + for (ManifestFile manifest : snapshot.allManifests(io)) { + expected.add(manifest.path()); + } + } + expected.forEach(file -> Assertions.assertTrue(((InMemoryFileIO) io).fileExists(file), file)); + + IcebergCleanupManager svc = + new IcebergCleanupManager(store, new IcebergConfig(new HashMap<>())); + try { + svc.cleanupFiles(io, metadataLocation); + } finally { + svc.close(); + } + + expected.forEach(file -> Assertions.assertFalse(((InMemoryFileIO) io).fileExists(file), file)); + } + + @TestTemplate + void testCleanupStopsWhenLeaseLost() { + BaseTable base = tableWithDataFile(); + FileIO io = base.io(); + String metadataLocation = base.operations().current().metadataFileLocation(); + + IcebergCleanupManager svc = + new IcebergCleanupManager(store, new IcebergConfig(new HashMap<>())); + try { + // A worker that lost its lease must stop before deleting anything. + Assertions.assertThrows( + RuntimeException.class, () -> svc.cleanupFiles(io, metadataLocation, () -> false)); + Assertions.assertTrue(((InMemoryFileIO) io).fileExists(DATA_FILE)); + Assertions.assertTrue(((InMemoryFileIO) io).fileExists(metadataLocation)); + } finally { + svc.close(); + } + } + + @TestTemplate + void testCleanupToleratesMissingMetadata() { + // A missing root metadata.json means the table is already gone, so cleanup just returns. + IcebergCleanupManager svc = + new IcebergCleanupManager(store, new IcebergConfig(new HashMap<>())); + try { + Assertions.assertDoesNotThrow( + () -> svc.cleanupFiles(new InMemoryFileIO(), "memory://db/t/metadata/missing.json")); + } finally { + svc.close(); + } + } + + @TestTemplate + void testWorkerSucceeds() { + AtomicInteger calls = new AtomicInteger(); + IcebergCleanupManager svc = + new IcebergCleanupManager(store, fastPollConfig()) { + @Override + void cleanupFiles(FileIO io, String metadataLocation, BooleanSupplier stillOwned) { + calls.incrementAndGet(); + } + }; + long id = store.addJob(sampleJob()); + svc.start(); + try { + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .until(() -> store.stateOf(id) == IcebergCleanupJob.State.SUCCEEDED); + Assertions.assertEquals(1, calls.get()); + } finally { + svc.close(); + } + } + + @TestTemplate + void testWorkerRetriesThenFails() { + Map<String, String> config = new HashMap<>(); + config.put("async-cleanup.worker-threads", "1"); + config.put("async-cleanup.poll-interval-secs", "1"); + config.put("async-cleanup.max-attempts", "3"); + IcebergCleanupManager svc = + new IcebergCleanupManager(store, new IcebergConfig(config)) { + @Override + void cleanupFiles(FileIO io, String metadataLocation, BooleanSupplier stillOwned) { + throw new RuntimeException("transient"); + } + }; + long id = store.addJob(sampleJob()); + svc.start(); + try { + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .until(() -> store.stateOf(id) == IcebergCleanupJob.State.FAILED); Review Comment: 3 retries at a 1-second poll interval needs at least ~3 s; 10 s is tight on a loaded CI machine. `atMost(15, TimeUnit.SECONDS)` would give more headroom without meaningfully slowing the suite. ########## iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/cleanup/AbstractIcebergCleanupJobStoreBackendTest.java: ########## @@ -100,36 +100,46 @@ void testAddTakeSucceedLifecycle() { Assertions.assertTrue(store.findUnfinishedJobId(CATALOG_ID, "db", "t").isPresent()); Assertions.assertFalse(store.takePendingJob(now, 300_000L, 10).isPresent()); - Assertions.assertTrue(store.markSucceeded(id)); + Assertions.assertTrue(store.markSucceeded(id, now)); Assertions.assertEquals(IcebergCleanupJob.State.SUCCEEDED, store.stateOf(id)); // A second transition no longer owns the (now terminal) row, so it reports no update. - Assertions.assertFalse(store.markSucceeded(id)); + Assertions.assertFalse(store.markSucceeded(id, now)); Assertions.assertFalse(store.findUnfinishedJobId(CATALOG_ID, "db", "t").isPresent()); Assertions.assertEquals( 1, store.deleteFinishedJobsByLegacyTimeline(System.currentTimeMillis() + 1)); } - @TestTemplate - void testMarkFailed() { - long id = store.addJob(sampleJob()); - store.takePendingJob(System.currentTimeMillis(), 300_000L, 10); - Assertions.assertTrue(store.markFailed(id, "corrupt metadata")); - Assertions.assertEquals(IcebergCleanupJob.State.FAILED, store.stateOf(id)); - } - @TestTemplate void testTransientFailureRetriesThenFailsAtCeiling() { long id = store.addJob(sampleJob()); Review Comment: The old `testMarkFailed` (direct FAILED transition) was removed. `testTransientFailureRetriesThenFailsAtCeiling` covers the retry path end-to-end, but there is no longer a store-layer test verifying that `recordFailure` at exactly `maxAttempts` produces `FAILED` (the new `testTerminalUpdateNeedsOwnership` tests ownership but not the state machine boundary). Consider keeping a targeted store-level assertion for that edge case. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
