vaijosh commented on code in PR #3081: URL: https://github.com/apache/hugegraph/pull/3081#discussion_r4452614409
########## hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListener.java: ########## @@ -0,0 +1,2949 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +import java.io.File; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import lombok.Getter; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.LiveSstFile; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.MetadataSnapshot; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.RocksdbChangedListener; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.cloud.CloudStorageProvider; + +import lombok.extern.slf4j.Slf4j; + +/** + * {@link RocksdbChangedListener} that bridges RocksDB table-file lifecycle events + * to the active {@link CloudStorageProvider}. + * + * <p>When cloud storage is enabled: + * <ul> + * <li>{@link #onDBCreated} uploads any SST files that already exist in the DB directory + * (e.g. surviving from a previous run), triggers a non-blocking MemTable flush so + * WAL-recovered or recently-written data is written to SST files (completion is signalled + * event-driven via {@link #onTableFileCreated}), then mirrors metadata inline.</li> + * <li>{@link #onTableFileCreated} pins newly created SST files via a hard link, dispatches + * upload work to a bounded background executor, and mirrors metadata after upload completes. + * If the hard link fails (e.g. cross-device mount, filesystem limits), the upload is routed + * directly to the retry queue using the original SST path — no copy is made, so no extra + * disk space is consumed.</li> + * <li>{@link #onTableFileDeleted} mirrors metadata first, then removes the superseded SST object.</li> + * </ul> + * + * <h3>Remote key construction</h3> + * The remote key is derived by stripping the {@code dataRoot} prefix from the absolute + * local file path. This keeps the object layout clean and independent of the container + * filesystem layout: + * <pre> + * dataRoot = /hugegraph-store/storage + * filePath = /hugegraph-store/storage/hgstore-metadata/000008.sst + * remoteKey = store-127.0.0.1_8501/hgstore-metadata/000008.sst + * (with path-prefix "hugegraph") → hugegraph/store-127.0.0.1_8501/hgstore-metadata/000008.sst + * </pre> + * + * This listener is registered with {@link RocksDBFactory} during application startup + * (see {@link org.apache.hugegraph.store.node.AppConfig}). + */ +@Slf4j +public class CloudStorageEventListener implements RocksdbChangedListener, AutoCloseable { + + /** Absolute, normalised paths of configured store data roots (one per configured partition root). */ + private final String primaryDataRoot; + private final List<String> allDataRoots; + + /** Optional per-store namespace prefix prepended to every remote cloud key. */ + private final String storeScopePrefix; + + static final long DEFAULT_READ_MISS_GUARD_WINDOW_MS = 3000L; + + private final boolean startupHydrationEnabled; + private final long readMissGuardWindowMs; + private final Map<String, Long> readMissAttemptTs; + + /** + * Tracks which {@code (dbName, remoteKey)} pairs are currently being restored by + * {@link #restoreMissingLiveFiles} so a thundering herd of concurrent read misses on the + * same cold SST does not download the same object many times over. This is a best-effort + * de-duplication; correctness against concurrent restores of the same file is guaranteed by + * the per-attempt unique temp name plus an atomic replace-move, not by this set. + */ + private final Set<String> inFlightRestores = ConcurrentHashMap.newKeySet(); + + /** + * Optional retry queue; when non-null, upload failures are submitted here instead + * of just being logged. When null, failures are only logged (no retry). + */ + private final CloudUploadRetryQueue retryQueue; + + /** Initialized cloud storage provider; resolved once at startup and never reconfigured. */ + private final CloudStorageProvider provider; + + /** Tracks which SST files are confirmed present in cloud (per-DB confirmed-file-number set). */ + private final CloudSyncTracker syncTracker; + + /** + * Hard health flag for pending-delete marker durability. Flips to {@code false} when a marker + * cannot be durably persisted (write + fsync) — a state in which a DB delete whose remote + * cleanup fails may be unguarded against re-hydration after a crash — and back to + * {@code true} once a marker is durably persisted again. Surfaced via + * {@link CloudStorageMetricsConst#DELETE_MARKER_HEALTHY}. {@code volatile}: written from delete + * callbacks, read from metric-scrape threads. + * -- GETTER -- + * Whether pending-delete marker persistence is currently healthy. + * indicates a + * marker could not be durably written, so a delete whose remote cleanup fails may not + * survive a crash as a hydration guard. Bound to + * <p> + * . + + */ + @Getter + private volatile boolean deleteMarkerHealthy = true; + + /** Directory fsync is not supported on Windows directory handles. */ + private static final boolean WINDOWS_OS = + System.getProperty("os.name", "").toLowerCase().contains("win"); + + /** Bounded async upload dispatcher so RocksDB callbacks return quickly. */ + private static final int ASYNC_UPLOAD_THREADS = 2; + private static final int ASYNC_UPLOAD_QUEUE_CAPACITY = 256; + /** Upload pool owned by this listener; drained by {@link #close()}. */ + private final ThreadPoolExecutor uploadExecutor; + + // ----------------------------------------------------------------------- + // Metadata-sync coalescing + // ----------------------------------------------------------------------- + // onTableFileCreated fires once per flushed/compacted SST; syncing metadata inline on every + // one triggers a full RocksDB checkpoint + S3 list/prune per SST. Instead each upload marks + // its DB dirty and a fixed-delay drain publishes each dirty DB at most once per interval. + // Event-driven callers that need an immediate publish (delete guard, onCompacted, + // onDBCreated) still call syncMetadataSnapshotInline directly and are NOT coalesced. + + /** Default drain interval for the post-upload metadata sync. */ + static final long DEFAULT_METADATA_SYNC_DEBOUNCE_MS = 1_000L; + + /** + * Scheduler for the metadata-sync drain and purge/delete retries. Owned by this listener and + * shut down by {@link #close()}; a schedule after that throws + * {@link RejectedExecutionException}, which every call site handles. + */ + private final ScheduledExecutorService metadataSyncScheduler; + + /** Drain interval (ms); {@code <= 0} publishes inline on every SST upload instead. */ + private final long metadataSyncDebounceMs; + + /** DBs with uploads not yet reflected in a published manifest; drained by the scheduler. */ + private final Set<String> dirtyMetadataDbs = ConcurrentHashMap.newKeySet(); + + // ----------------------------------------------------------------------- + // Metadata (CURRENT/MANIFEST/OPTIONS) mirroring & consistent restore + // ----------------------------------------------------------------------- + + /** Resolved DB directory -> logical DB name, so {@link #onCompacted} resolves path events. */ + private final Map<String, String> dbNameByDir = new ConcurrentHashMap<>(); + + /** + * Tracks which DBs are currently being truncated. While a DB is in this set, + * metadata sync operations are skipped to allow the purge to complete cleanly + * without new metadata files being re-uploaded. + */ + private final Set<String> truncatingDbs = ConcurrentHashMap.newKeySet(); + + /** + * Tracks the timestamp of recent truncations (DB name -> truncation time in ms). + * Used to suppress metadata syncs for a grace period after truncation, allowing + * pending RocksDB background operations and callbacks to complete without + * re-uploading metadata that was just purged. + */ + private final Map<String, Long> truncationTimes = new ConcurrentHashMap<>(); + + /** Per-DB mutexes to serialize metadata capture/publication/pruning. */ + private final Map<String, Object> metadataSyncLocks = new ConcurrentHashMap<>(); + + /** + * Per-remote-prefix mutexes serializing pending-delete cleanup (inline at open vs. the async + * retry). Ensures the purge runs at most once and only while the marker is present, so it can + * never delete data a re-created DB uploads to the same prefix after cleanup completes. + */ + private final Map<String, Object> pendingDeleteLocks = new ConcurrentHashMap<>(); + + /** + * Per-remote-prefix generation, bumped every time a DB is opened or created at that prefix. + * An asynchronous pending-delete purge captures the value when it is scheduled and aborts + * (without purging) if it changed, so a retry can never delete objects that a re-created DB + * has since taken over the prefix with. Entries are never removed: dropping one would reset + * the counter and let a stale retry match again. + */ + private final Map<String, AtomicLong> prefixEpochs = new ConcurrentHashMap<>(); + + /** + * Per-DB write generation, bumped when the DB is opened/created and on every SST creation. A + * truncate purge retry captures it when it is scheduled and aborts if the DB has resumed + * writing (new objects would be purged along with the stale ones). Never removed, see above. + */ + private final Map<String, AtomicLong> dbWriteEpochs = new ConcurrentHashMap<>(); + + /** Per-DB metadata-publish failure state used to back off a persistently failing publish. */ + private final Map<String, PublishBackoff> publishBackoffs = new ConcurrentHashMap<>(); + + /** Cap for the exponential publish-retry backoff. */ + static final long MAX_PUBLISH_BACKOFF_MS = 5 * 60 * 1000L; + /** Minimum base for the publish-retry backoff, so a tiny debounce does not hot-loop failures. */ + static final long MIN_PUBLISH_BACKOFF_BASE_MS = 1_000L; + + /** Bounded synchronous attempts at completing a pending delete while a DB is being opened. */ + private static final int OPEN_PENDING_DELETE_ATTEMPTS = 3; + private static final long OPEN_PENDING_DELETE_BACKOFF_MS = 200L; + /** Bounded attempts at flushing dirty metadata on close(). */ + private static final int CLOSE_FLUSH_ATTEMPTS = 3; + private static final long CLOSE_FLUSH_BACKOFF_MS = 200L; + + /** Consecutive publish failures and the earliest time the next attempt is allowed. */ + private static final class PublishBackoff { + + final int failures; + final long nextAttemptAtMs; + + PublishBackoff(int failures, long nextAttemptAtMs) { + this.failures = failures; + this.nextAttemptAtMs = nextAttemptAtMs; + } + } + + /** Last successfully published RocksDB generation per DB. */ + private final Map<String, Long> lastPublishedMetadataGeneration = new ConcurrentHashMap<>(); + + /** + * Grace period (ms) after truncation during which metadata syncs are suppressed. + * This allows pending RocksDB background callbacks to complete without re-uploading + * metadata that was purged during truncation. + */ + private static final long TRUNCATION_GRACE_PERIOD_MS = 5_000L; + + /** + * Suffix appended to the DB prefix (not inside it) when a database is deleted. + * Placing the tombstone as a sibling of the data prefix means the + * {@link #purgeRemotePrefix} call in {@link #onDBDeleted} cannot accidentally remove it + * while stale SST or metadata objects remain, so {@link #preHydrateDbFiles} can still + * detect the deleted generation and skip hydration. + * + * <p>Example: data prefix = {@code store-host_8500/hugegraph/db}, + * tombstone key = {@code store-host_8500/hugegraph/db_DELETED}. + */ + static final String DB_TOMBSTONE_SUFFIX = "_DELETED"; + + /** Infix of hydration temp files ({@code <name>.hyd-tmp-<threadId>-<nanoTime>}). */ + private static final String HYDRATION_TEMP_MARKER = ".hyd-tmp-"; + + /** + * Fully-parameterised constructor: the listener is completely configured the moment it is + * constructed, so there is no window in which a registered listener can be observed + * partially configured. + * + * @param provider initialized cloud storage provider + * @param dataRoots configured store data roots (typically parsed from comma-separated + * {@code app.data-path}) + * @param readMissGuardWindowMs guard window in ms for repeated read-miss hydration attempts + * for the same db/table pair (cloud.storage.read-miss-guard-window-ms) + * @param retryQueue optional {@link CloudUploadRetryQueue}; when non-null, upload failures + * are retried asynchronously and eventually given up on. + * Pass {@code null} to disable retries (failures are only logged). + * @param syncTracker tracks SST files confirmed present in cloud; the delete guard uses it + * to avoid deleting a superseded object before replacements are durable. + * Must be shared with the retry queue. Pass {@code null} for a fresh + * {@link CloudSyncTracker}. + * @param storeScopePrefix optional per-store key prefix to isolate cloud objects + * @param metadataSyncDebounceMs drain interval (ms) for the per-SST metadata sync; + * {@code <= 0} publishes inline on every SST upload + */ + public CloudStorageEventListener(CloudStorageProvider provider, + List<String> dataRoots, + boolean startupHydrationEnabled, + long readMissGuardWindowMs, + CloudUploadRetryQueue retryQueue, + CloudSyncTracker syncTracker, + String storeScopePrefix, + long metadataSyncDebounceMs) { + // Fail fast on a missing/empty data-root list: primaryDataRoot is derived from index 0 + // below, and every key<->path conversion depends on at least one root. Without this guard + // the constructor would throw an opaque IndexOutOfBoundsException at + // allDataRoots.get(0), which is far harder to diagnose than a config error. + if (dataRoots == null || dataRoots.isEmpty()) { + throw new IllegalArgumentException( + "CloudStorageEventListener requires at least one data root; none configured. " + + "Check the store data-path configuration (e.g. rocksdb.data_path / " + + "raft.path) used to derive the cloud storage data roots."); + } + // Normalize each root + this.allDataRoots = new ArrayList<>(); + for (String root : dataRoots) { + String normalised = Paths.get(root).toAbsolutePath().normalize().toString(); + // Strip trailing separator so substring arithmetic is consistent. + normalised = normalised.endsWith(File.separator) + ? normalised.substring(0, normalised.length() - 1) + : normalised; + this.allDataRoots.add(normalised); + } + this.primaryDataRoot = this.allDataRoots.get(0); + this.provider = Objects.requireNonNull(provider, "provider"); + this.startupHydrationEnabled = startupHydrationEnabled; + this.readMissGuardWindowMs = Math.max(0L, readMissGuardWindowMs); + this.readMissAttemptTs = new ConcurrentHashMap<>(); + this.retryQueue = retryQueue; + this.syncTracker = syncTracker != null ? syncTracker : new CloudSyncTracker(); + this.storeScopePrefix = normaliseKeyPrefix(storeScopePrefix); + this.uploadExecutor = new ThreadPoolExecutor( + ASYNC_UPLOAD_THREADS, + ASYNC_UPLOAD_THREADS, + 60L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(ASYNC_UPLOAD_QUEUE_CAPACITY), + newUploadThreadFactory(), + new ThreadPoolExecutor.AbortPolicy()); + ScheduledThreadPoolExecutor syncScheduler = new ScheduledThreadPoolExecutor(1, r -> { + Thread t = new Thread(r, "cloud-metadata-sync"); + t.setDaemon(true); + return t; + }); + // close() uses a graceful shutdown() so an in-flight publish completes; not-yet-due delayed + // tasks (purge/delete retries) are cancelled instead of holding close() open. Their intent + // is crash-durable (local markers) and is re-driven by the next startup scan. + syncScheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + this.metadataSyncScheduler = syncScheduler; + this.metadataSyncDebounceMs = metadataSyncDebounceMs; + if (metadataSyncDebounceMs > 0) { + this.metadataSyncScheduler.scheduleWithFixedDelay(this::drainDirtyMetadata, + metadataSyncDebounceMs, + metadataSyncDebounceMs, + TimeUnit.MILLISECONDS); + } + + // A crash may leave local pending-delete markers whose remote cleanup never completed. + // Kick off bounded async retries for each so stale remote data is eventually purged even if + // the affected DB is never re-opened (an open would otherwise be the only trigger). + scanMarkers(pendingDeleteDir(), (db, prefix) -> { + log.warn("Cloud startup: found pending remote-delete marker for prefix={} — " + + "scheduling deferred purge to prevent stale-data re-hydration.", prefix); + scheduleDeletePurgeRetry(db, prefix, 1); + }); + // Truncate purge intent is also crash-durable via local markers. + scanMarkers(pendingTruncateDir(), (db, prefix) -> { + log.warn("Cloud startup: found pending truncate marker for prefix={} — scheduling " + + "deferred purge retry.", prefix); + scheduleTruncatePurgeRetry(db, prefix); + }); + // Upload intent is not persisted across restarts, so any staging pin left by the previous + // process is orphaned; the live SSTs it pinned are re-uploaded by startup backfill. + sweepStagingDirsOnStartup(); + } + + private static ThreadFactory newUploadThreadFactory() { + return r -> { + Thread t = new Thread(r, "cloud-upload-dispatch"); + t.setDaemon(true); + return t; + }; + } + + /** Shuts down this listener's executors, waiting up to 10s for in-flight uploads. */ + @Override + public void close() { + close(10, TimeUnit.SECONDS); + } + + /** + * Shuts down the SST upload executor, waiting up to {@code timeout} for in-flight uploads to + * complete, then the metadata-sync scheduler. Called from + * {@link org.apache.hugegraph.store.node.AppConfig#onDestroy()} after the listener has been + * deregistered, so uploads started before shutdown have a chance to finish. + */ + void close(long timeout, TimeUnit unit) { + // Drain the upload executor FIRST so DBs marked dirty by in-flight uploads are flushed below. + uploadExecutor.shutdown(); + try { + if (!uploadExecutor.awaitTermination(timeout, unit)) { + log.warn("CloudStorageEventListener: upload executor did not terminate within " + + "{}{}; forcing shutdown", timeout, unit); + handoffDroppedUploadTasks(uploadExecutor.shutdownNow()); + } + } catch (InterruptedException e) { + handoffDroppedUploadTasks(uploadExecutor.shutdownNow()); + Thread.currentThread().interrupt(); + } + // Graceful stop: an in-flight drain/publish finishes instead of being interrupted mid-upload, + // and the final flush below runs only after the drain thread is gone so the two cannot + // publish the same DB concurrently. + metadataSyncScheduler.shutdown(); + try { + if (!metadataSyncScheduler.awaitTermination(timeout, unit)) { + log.warn("CloudStorageEventListener: metadata-sync scheduler did not terminate " + + "within {}{}; forcing shutdown", timeout, unit); + metadataSyncScheduler.shutdownNow(); + metadataSyncScheduler.awaitTermination(1, TimeUnit.SECONDS); + } + } catch (InterruptedException e) { + metadataSyncScheduler.shutdownNow(); + Thread.currentThread().interrupt(); + } + // Flush dirty DBs so uploads that completed during the drain are referenced by the + // mirrored CURRENT/MANIFEST. Later requests (e.g. retry-queue completions) see the + // scheduler shut down in markMetadataDirty and publish inline. + flushDirtyMetadataOnClose(); + } + + /** + * Final bounded flush of every dirty DB. A failed publish is retried a few times; DBs that + * still cannot be published (or are mid-truncate) are logged so the gap is visible. + */ + private void flushDirtyMetadataOnClose() { + List<String> remaining = new ArrayList<>(dirtyMetadataDbs); + for (int attempt = 1; attempt <= CLOSE_FLUSH_ATTEMPTS && !remaining.isEmpty(); attempt++) { + for (String dbName : new ArrayList<>(remaining)) { + if (isActivelyTruncating(dbName) || isInTruncationGracePeriod(dbName)) { + continue; // the purge supersedes it; keep it reported below + } + dirtyMetadataDbs.remove(dbName); + if (publishDirtyMetadata(dbName)) { + remaining.remove(dbName); + } else { + dirtyMetadataDbs.add(dbName); + } + } + if (remaining.isEmpty() || attempt == CLOSE_FLUSH_ATTEMPTS + || remaining.stream().allMatch(db -> isActivelyTruncating(db) + || isInTruncationGracePeriod(db))) { + break; + } + try { + Thread.sleep(CLOSE_FLUSH_BACKOFF_MS * attempt); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + if (!remaining.isEmpty()) { + log.error("CloudStorageEventListener: closed with metadata NOT published for {} DB(s): " + + "{} — the mirrored CURRENT/MANIFEST may lag the uploaded SSTs until the next " + + "start publishes them.", remaining.size(), remaining); + } + } + + /** + * Routes upload tasks discarded by a forced {@code shutdownNow()} into their retry queue + * so an SST accepted locally but never mirrored is not silently lost from the durability + * pipeline. Only {@link SstUploadTask}s carry the metadata needed for handoff; any other + * runnable (there should be none) is ignored. + */ + private static void handoffDroppedUploadTasks(java.util.List<Runnable> dropped) { + if (dropped == null || dropped.isEmpty()) { + return; + } + int handed = 0; + for (Runnable r : dropped) { + if (r instanceof SstUploadTask) { + try { + ((SstUploadTask) r).handoffOnShutdown(); + handed++; + } catch (Exception e) { + log.warn("Failed to hand off a dropped upload task to the retry queue: {}", + e.getMessage()); + } + } + } + if (handed > 0) { + log.warn("CloudStorageEventListener: handed off {} queued upload task(s) to the retry queue " + + "during shutdown so their upload intent survives.", handed); + } + } + + /** + * Finds which configured data roots contain the given file path. + * Returns the matching root, or the primary root if no exact match is found. + * + * <p>Uses {@link Path#startsWith} so that {@code /data/store1} never incorrectly + * matches {@code /data/store10/...} (a raw string prefix match would). + * + * @param filePath absolute file path + * @return the matched configured data root for this file + */ + private String findMatchingDataRoot(String filePath) { + Path fileNormalized = Paths.get(filePath).toAbsolutePath().normalize(); + for (String root : allDataRoots) { + if (fileNormalized.startsWith(Paths.get(root))) { + return root; + } + } + // Fallback to primary root (should not happen in normal operation) + return primaryDataRoot; + } + + // ----------------------------------------------------------------------- + // RocksdbChangedListener + // ----------------------------------------------------------------------- + + @Override + public boolean managesRemoteState() { + return true; + } + + /** + * Picks the directory to open when the local one holds no DB. A partition can move to a + * version-suffixed directory ({@code <name>_<v1>[_<v2>]}) and its objects then live under that + * prefix, so after a disk loss the default path would hydrate nothing. Lists the remote + * siblings that share the base name and returns the newest restorable one, matching the order + * {@code RocksDBSession} uses locally. A sibling is restorable when it has a remote + * {@code CURRENT}, no delete tombstone and no local pending-delete marker. If listing fails the + * path is kept, and the hydration of that path applies its own failure rules. + */ + @Override + public String resolveOpenPath(String dbName, String dbPath) { + if (!startupHydrationEnabled) { + return dbPath; + } + Path candidate = Paths.get(dbPath).toAbsolutePath().normalize(); + Path parent = candidate.getParent(); + if (parent == null) { + return dbPath; + } + String name = candidate.getFileName().toString(); + int sep = name.indexOf('_'); + String baseName = sep < 0 ? name : name.substring(0, sep); + String basePrefix = dbPrefix(parent.resolve(baseName).toString()); + if (!basePrefix.endsWith(baseName)) { + return dbPath; + } + String parentKeyPrefix = basePrefix.substring(0, basePrefix.length() - baseName.length()); + List<String> keys; + try { + keys = provider.listFiles(basePrefix); + } catch (IOException e) { + log.warn("Cloud open: listing version siblings of db={} failed, keeping {}: {}", + dbName, dbPath, e.getMessage()); + return dbPath; + } + String newest = newestRestorableSibling(parentKeyPrefix, baseName, keys); + if (newest == null || newest.equals(name)) { + return dbPath; + } + log.info("Cloud open: db={} has no local DB at {}; restoring newest remote sibling '{}'", + dbName, dbPath, newest); + return parent.resolve(newest).toString(); + } + + /** + * Newest restorable directory name among {@code baseName} and its version-suffixed siblings + * found in {@code keys}, or {@code null} if none (package-private for tests). + */ + String newestRestorableSibling(String parentKeyPrefix, String baseName, List<String> keys) { + Pattern dirName = Pattern.compile(Pattern.quote(baseName) + "(?:_(\\d+)(?:_(\\d+))?)?"); + Set<String> withCurrent = new HashSet<>(); + Set<String> tombstoned = new HashSet<>(); + for (String key : keys) { + if (!key.startsWith(parentKeyPrefix)) { + continue; + } + String rel = key.substring(parentKeyPrefix.length()); + int slash = rel.indexOf('/'); + if (slash < 0) { + if (rel.endsWith(DB_TOMBSTONE_SUFFIX)) { + tombstoned.add(rel.substring(0, rel.length() - DB_TOMBSTONE_SUFFIX.length())); + } + continue; + } + String dir = rel.substring(0, slash); + if ("CURRENT".equals(rel.substring(slash + 1)) && dirName.matcher(dir).matches()) { + withCurrent.add(dir); + } + } + String newest = null; + long[] newestVersion = null; + for (String dir : withCurrent) { + if (tombstoned.contains(dir) || hasPendingDeleteMarker(parentKeyPrefix + dir)) { + continue; + } + Matcher m = dirName.matcher(dir); + if (!m.matches()) { + continue; + } + long[] version; + try { + version = new long[]{m.group(1) == null ? -1L : Long.parseLong(m.group(1)), + m.group(2) == null ? -1L : Long.parseLong(m.group(2))}; + } catch (NumberFormatException e) { + continue; + } + if (newestVersion == null || version[0] > newestVersion[0] + || (version[0] == newestVersion[0] && version[1] > newestVersion[1])) { + newest = dir; + newestVersion = version; + } + } + return newest; + } + + @Override + public void onDBOpening(String dbName, String dbPath) { + // Register the dir -> logical-name mapping first: native events that arrive before + // onDBCreated (or on a disabled-hydration node) must resolve to the logical name. + recordDb(dbName, dbPath); + // Stale hydration temp files can only be left by a crashed download; sweep them from the + // local dir on every open, not just when something is about to be hydrated. + sweepStaleHydrationTemps(Paths.get(dbPath)); + + String prefix = dbPrefix(dbPath); + // Invalidate every async purge scheduled for the previous generation at this prefix BEFORE + // anything below can write to it. Retries compare against this epoch under the prefix lock. + bumpEpoch(prefixEpochs, prefix); + bumpEpoch(dbWriteEpochs, dbName); + + // Pending-delete guard, independent of hydration: a prior delete's remote cleanup may not + // be confirmed, so the reopened DB must not upload into (or hydrate from) a prefix that + // still holds the deleted generation and a delete marker. + if (hasPendingDeleteMarker(prefix)) { + completePendingDeleteOrBlockOpen(dbName, prefix); + } + boolean truncatePurgeConfirmed = true; + if (hasPendingTruncateMarker(prefix)) { + truncatePurgeConfirmed = tryCompletePendingTruncate(dbName, prefix); + } + if (!startupHydrationEnabled) { + // Nothing is hydrated, but the DB will upload into this prefix: a leftover tombstone + // must be resolved now, or a later open with hydration enabled would purge its data. + purgeTombstonedPrefix(provider, dbName, Paths.get(dbPath), prefix, false); + return; + } + if (!truncatePurgeConfirmed) { + log.warn("Cloud pre-hydration skipped for db={}: remote truncate purge not yet " + + "confirmed — opening without re-hydrating cleared data.", dbName); + return; + } + preHydrateDbFiles(provider, dbName, dbPath); + } + + /** + * Finishes a pending remote delete synchronously (bounded attempts) before the open proceeds. + * If it cannot be completed the open is BLOCKED: the marker means the prefix still holds the + * deleted generation, and opening anyway would let the new DB upload into it while the async + * retry (or a restart) later purges the new data. The async retry is re-armed so the delete + * finishes when the cloud recovers, and a subsequent open attempt simply tries again. + */ + private void completePendingDeleteOrBlockOpen(String dbName, String prefix) { + for (int attempt = 1; attempt <= OPEN_PENDING_DELETE_ATTEMPTS; attempt++) { + if (tryCompletePendingDelete(provider, dbName, prefix, null)) { + return; + } + if (attempt < OPEN_PENDING_DELETE_ATTEMPTS) { + try { + Thread.sleep(OPEN_PENDING_DELETE_BACKOFF_MS * attempt); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + } + scheduleDeletePurgeRetry(dbName, prefix, 1); + throw new IllegalStateException(String.format( + "Cloud open blocked for db=%s: remote-delete cleanup of prefix '%s' is not yet " + + "confirmed. Opening now would let the new DB upload into a prefix still holding " + + "the deleted generation; the cleanup is being retried in the background.", + dbName, prefix)); + } + + /** Best-effort inline purge of a pending truncate at open; re-arms the async retry on failure. */ + private boolean tryCompletePendingTruncate(String dbName, String prefix) { + Object lock = pendingDeleteLocks.computeIfAbsent(prefix, k -> new Object()); + synchronized (lock) { + if (!hasPendingTruncateMarker(prefix)) { + return true; + } + if (purgeRemotePrefix(provider, dbName, prefix)) { + removeMarker(pendingTruncateDir(), prefix); + return true; + } + } + scheduleTruncatePurgeRetry(dbName, prefix); + return false; + } + + /** Deletes stale {@code .hyd-tmp-} download temp files left under a local DB dir by a crash. */ + private void sweepStaleHydrationTemps(Path root) { + if (!Files.isDirectory(root)) { + return; + } + try (Stream<Path> stale = Files.walk(root, 10)) { + stale.filter(p -> p.getFileName() != null + && p.getFileName().toString().contains(HYDRATION_TEMP_MARKER)) + .forEach(p -> deleteIfExistsQuietly(p, "stale hydration temp cleanup")); + } catch (IOException e) { + log.debug("Failed to scan stale hydration temp files under {}: {}", + root, e.getMessage()); + } + } + + private static long bumpEpoch(Map<String, AtomicLong> epochs, String key) { + return epochs.computeIfAbsent(key, k -> new AtomicLong()).incrementAndGet(); + } + + private static long epochOf(Map<String, AtomicLong> epochs, String key) { + AtomicLong epoch = epochs.get(key); + return epoch == null ? 0L : epoch.get(); + } + + /** Current open/create generation of {@code prefix} (package-private for tests). */ + long prefixEpoch(String prefix) { + return epochOf(prefixEpochs, prefix); + } + + /** + * Called when a RocksDB read fails, typically because a live SST is missing on local disk. + * + * <p>We restore only the SST files that RocksDB references as <em>live</em> in its manifest but + * that are physically missing on local disk, downloading each back to its <em>exact original + * path</em> so RocksDB finds it on the next access. This deliberately avoids + * {@code ingestExternalFile}, which would (a) risk placing a file into the wrong column family + * and (b) assign a fresh sequence number that can resurrect deleted keys. Restricting to the + * live set also guarantees superseded / compacted-away objects are never resurrected. + * + * <p>If no live file is missing (the error has another cause) this returns {@code false} + * without any cloud I/O, and the original error propagates. + */ + @Override + public boolean onReadMiss(RocksDBSession session, String table, byte[] key) { + String dbName = session.getGraphName(); + if (!shouldAttemptReadMissHydration(dbName, table)) { + return false; + } + int restored = restoreMissingLiveFiles(provider, dbName, + RocksDBFactory.getInstance().getLiveSstFiles(dbName)); + if (restored > 0) { + log.info("Cloud read-miss hydration: restored {} missing live SST file(s) for db={}", + restored, dbName); + return true; + } + return false; + } + + /** + * Downloads back any SST files that are live in RocksDB's manifest but missing on local disk, + * writing each to its original path. Returns the number of files restored. + * + * <p>Package-private testable seam: caller supplies the live-file set. + */ + int restoreMissingLiveFiles(CloudStorageProvider provider, String dbName, + List<LiveSstFile> liveFiles) { + int restored = 0; + for (LiveSstFile live : liveFiles) { + Path localPath = Paths.get(live.getAbsolutePath()); + if (Files.exists(localPath)) { + continue; + } + String remoteKey = toRelativeKey(live.getAbsolutePath()); + // De-dup concurrent restores of the same object (thundering herd on a cold read + // miss). If another thread already holds the slot, skip: it will produce the file. + String restoreGuardKey = dbName + "::" + remoteKey; + if (!inFlightRestores.add(restoreGuardKey)) { + continue; + } + try { + // Re-check after acquiring the slot: a concurrent restore may have just finished. + if (Files.exists(localPath)) { + continue; + } + if (!provider.fileExists(remoteKey)) { + log.warn("Cloud read-miss: live file missing locally AND absent in cloud: " + + "db={}, key={}", dbName, remoteKey); + continue; + } + downloadAtomically(provider, remoteKey, localPath); + syncTracker.markConfirmed(dbName, live.getAbsolutePath()); + restored++; + } catch (IOException e) { + log.warn("Cloud read-miss restore failed: db={}, key={}, reason={}", + dbName, remoteKey, e.getMessage()); + } finally { + inFlightRestores.remove(restoreGuardKey); + } + } + return restored; + } + + /** + * Called when a new RocksDB instance is opened for the first time. + * + * <p>Uploads any SST files that already exist in {@code dbPath} (e.g. from a previous run) + * and then triggers a MemTable flush so that WAL-recovered data is also written to + * SST files and eventually forwarded here via {@link #onTableFileCreated}. + * + * @param dbName logical name of the graph / partition + * @param dbPath absolute path of the RocksDB directory + */ + @Override + public void onDBCreated(String dbName, String dbPath) { + recordDb(dbName, dbPath); + // The DB is live at this prefix now: cancel any purge scheduled for an earlier generation. + bumpEpoch(prefixEpochs, dbPrefix(dbPath)); + bumpEpoch(dbWriteEpochs, dbName); + // Do not propagate upload failures: the session is already live in dbSessionMap at + // this point, so throwing here would cause the createGraphDB caller to receive an + // error while the DB is actually open and usable by other threads (split-brain). + try { + uploadExistingSstFiles(provider, dbName, dbPath); + } catch (Exception e) { + log.warn("Cloud initial-upload failed for db={}: {} — DB is open, " + + "existing SSTs may not be in cloud yet", dbName, e.getMessage()); + } + flushDb(dbName); + // Mirror metadata immediately after initial upload/flush to keep cloud state recoverable. + syncMetadataSnapshotInline(provider, dbName); + } + + /** + * Called just before the local RocksDB directory is removed. Writes a small tombstone object + * (key = {@code dbPrefix + }{@value #DB_TOMBSTONE_SUFFIX}) outside the data prefix so that + * any subsequent {@link #preHydrateDbFiles} call for the same path will detect the deleted + * generation and skip hydration rather than re-ingesting stale objects. + * + * <p>This callback fires while the session is still in a pending-destroy list (refcount may + * be non-zero). The tombstone write is best-effort: a failure is logged but does not block + * the deletion. The cloud purge in {@link #onDBDeleted} provides a second line of defence. + * + * @param dbName logical graph/partition name + * @param dbPath absolute path of the RocksDB directory being destroyed + */ + @Override + public void onDBDeleteBegin(String dbName, String dbPath) { + String prefix = dbPrefix(dbPath); + // ALWAYS persist a LOCAL pending-delete marker first. This is the durable anti-resurrection + // guard that survives a failed remote cleanup: even if we cannot write the remote + // tombstone or purge below, the marker blocks hydration of this DB (see onDBOpening) and + // drives async cleanup until confirmed. It is removed once the remote purge succeeds. + // + // Marker durability is a hard precondition for a safe delete: if we cannot fsync it, a crash + // could lose the guard and let stale remote SST/metadata be re-hydrated as live data. So on a + // persistence failure we flip the health signal to degraded and HOLD delete progression + // (throw) rather than proceeding unguarded — the caller can retry once local storage recovers. + try { + writeMarker(pendingDeleteDir(), prefix); + deleteMarkerHealthy = true; + log.debug("Cloud pending-delete marker persisted (fsync'd): db={}, prefix={}", + dbName, prefix); + } catch (IOException e) { + deleteMarkerHealthy = false; + log.error("Cloud pending-delete marker could not be durably persisted for db={} " + + "prefix={}: {} — HOLDING delete to avoid an unguarded delete that a crash " + + "could let re-hydrate. Delete-marker health is DEGRADED.", + dbName, prefix, e.getMessage()); + throw new IllegalStateException( + "Cannot durably persist pending-delete marker for db=" + dbName + + "; holding delete to preserve the anti-resurrection guard", e); + } + // The delete is going ahead and the prefix will be purged, so publishing the DB's + // metadata when its session closes (onDBClosing) would be wasted work. + dirtyMetadataDbs.remove(dbName); + + // Tombstone lives OUTSIDE the data prefix (sibling, not child) so it is not + // accidentally deleted by purgeRemotePrefix when SST objects are still present. + String tombstoneKey = prefix + DB_TOMBSTONE_SUFFIX; + try { + writeTombstone(prefix); + log.info("Cloud DB tombstone written: db={}, key={}", dbName, tombstoneKey); + } catch (Exception e) { + log.warn("Cloud DB tombstone write failed (onDBDeleted will still purge): " + + "db={}, key={}, reason={}", dbName, tombstoneKey, e.getMessage()); + } + } + + /** + * Publishes a still-dirty DB's metadata while its session is open. The release path (a + * partition moving away, and {@code releaseAllGraphDB()} at shutdown, which runs before this + * listener is closed) would otherwise close the DB first, and the deferred publish could then + * never capture it, leaving the cloud CURRENT/MANIFEST behind the uploaded SSTs. + */ + @Override + public void onDBClosing(String dbName) { + if (isActivelyTruncating(dbName) || isInTruncationGracePeriod(dbName)) { + return; // the truncate purge supersedes the publish + } + if (!dirtyMetadataDbs.remove(dbName)) { + return; + } + if (!publishDirtyMetadata(dbName)) { + // Not re-marked: the DB is closing, so a deferred publish would only fail again. + log.warn("Cloud metadata publish before close failed for db={}: the mirrored " + + "CURRENT/MANIFEST may lag the uploaded SSTs until the DB is next opened.", + dbName); + } + } + + /** Uploads the sibling delete-tombstone for {@code prefix} from a short-lived temp file. */ + private void writeTombstone(String prefix) throws IOException { + Path tmp = Files.createTempFile("hgstore-tombstone-", ".tmp"); + try { + Files.writeString(tmp, "deleted"); + provider.uploadFile(tmp.toString(), prefix + DB_TOMBSTONE_SUFFIX); + } finally { + deleteIfExistsQuietly(tmp, "tombstone temp cleanup"); + } + } + + /** + * Called after the local RocksDB directory has been physically removed. Purges all cloud + * objects under the DB prefix (SSTs, metadata, tombstone) so a future creation at the same + * path starts with a clean remote state. Also clears all in-memory state for this DB. + * + * <p>The purge is best-effort: individual delete failures are logged at DEBUG level and do + * not throw. Any objects that survive the purge are neutralised by the tombstone check in + * {@link #preHydrateDbFiles}: the next open will find the tombstone (or an empty prefix if + * the purge was complete), skip hydration, and clean up any leftovers. + * + * @param dbName logical graph/partition name + * @param dbPath absolute path of the now-deleted RocksDB directory + */ + @Override + public void onDBDeleted(String dbName, String dbPath) { + String deletedDir = normaliseDir(dbPath); + dbNameByDir.remove(deletedDir); + if (dbNameByDir.containsValue(dbName)) { + // The same logical DB is live at another directory (it was re-created there while this + // one awaited destruction). Its name-keyed state belongs to the live copy, so only + // the work queued for files of the deleted directory is dropped. + deferredSstDeletes.removeIf(t -> dbName.equals(t.dbName) + && deletedDir.equals(parentDir(t.filePath))); + log.info("Cloud DB deleted at {} while db={} is live elsewhere; keeping its state", + deletedDir, dbName); + } else { + // Drop the dirty mark before the truncation guards below: a concurrent drain that + // sees the guards cleared must then fail its dirtyMetadataDbs.remove() and skip. + dirtyMetadataDbs.remove(dbName); + publishBackoffs.remove(dbName); + // Clear in-memory tracking so no stale state bleeds into a recreated DB. + syncTracker.clearDb(dbName); + readMissAttemptTs.entrySet().removeIf(e -> e.getKey().startsWith(dbName + "::")); + dbNameByDir.values().removeIf(dbName::equals); + metadataSyncLocks.remove(dbName); + lastPublishedMetadataGeneration.remove(dbName); + truncationTimes.remove(dbName); + truncatingDbs.remove(dbName); + deferredSstDeletes.removeIf(t -> dbName.equals(t.dbName)); + // Remove per-DB meters so meter cardinality does not grow without bound as DBs churn. + CloudStorageMetrics.unregisterDatabaseMetrics(dbName); + } + + String prefix = dbPrefix(dbPath); + if (purgeRemotePrefix(provider, dbName, prefix) + && deleteTombstone(provider, dbName, prefix)) { + // Remote data and the sibling tombstone are gone: drop the local marker (cleanup done). + removeMarker(pendingDeleteDir(), prefix); + } else { + // Purge or tombstone delete failed. Keep the local marker so a future onDBOpening is + // blocked until both succeed, and retry (bounded) — same safety level as the truncate + // purge. The marker must outlive the tombstone: a tombstone left next to a re-created + // DB would make a later open purge that DB's live objects. + log.warn("Cloud DB delete cleanup failed for db={}: local marker preserved to block " + + "re-opening the prefix; scheduling bounded retries.", dbName); + scheduleDeletePurgeRetry(dbName, prefix, 1); + } + } + + /** + * Deletes the sibling delete-tombstone once the prefix purge has succeeded. Returns + * {@code false} on failure: the caller must then keep the pending-delete marker (and so keep + * the prefix blocked), because a tombstone that outlives the marker would make the next open + * purge the live objects of a DB re-created at the same prefix. + */ + private boolean deleteTombstone(CloudStorageProvider provider, String dbName, String prefix) { + String tombstoneKey = prefix + DB_TOMBSTONE_SUFFIX; + try { + provider.deleteFile(tombstoneKey); + log.debug("Cloud DB tombstone cleaned up: db={}, key={}", dbName, tombstoneKey); + return true; + } catch (IOException e) { + log.warn("Cloud DB tombstone cleanup failed: db={}, key={}: {}", + dbName, tombstoneKey, e.getMessage()); + return false; + } + } + + /** + * Called after a RocksDB has been truncated (all data cleared but directory preserved). + * Purges all cloud objects under the DB prefix (SSTs, metadata) so the remote state matches + * the now-empty local state. Also clears all in-memory sync tracking for this DB. + * + * <p>This is triggered by graph.clear() operations to ensure cloud storage is cleaned up + * when the graph data is cleared. + * + * @param dbName logical graph/partition name + * @param dbPath absolute path of the RocksDB directory + */ + @Override + public void onDBTruncateBegin(String dbName, String dbPath) { + truncatingDbs.add(dbName); + syncTracker.clearDb(dbName); + readMissAttemptTs.entrySet().removeIf(e -> e.getKey().startsWith(dbName + "::")); + deferredSstDeletes.removeIf(t -> dbName.equals(t.dbName)); + } + + @Override + public void onDBTruncated(String dbName, String dbPath) { + truncatingDbs.add(dbName); + try { + String prefix = dbPrefix(dbPath); + try { + writeMarker(pendingTruncateDir(), prefix); + log.debug("Cloud pending-truncate marker persisted (fsync'd): db={}, prefix={}", + dbName, prefix); + } catch (IOException e) { + log.error("Cloud truncate marker persist failed for db={}, prefix={}: {} — " + + "remote purge intent is not crash-durable until the next successful " + + "persist.", + dbName, prefix, e.getMessage()); + } + boolean purged = purgeRemotePrefix(provider, dbName, prefix); + if (purged) { + removeMarker(pendingTruncateDir(), prefix); + } + if (!purged) { + // Do NOT ignore a failed purge: stale remote SST/metadata still describe the + // pre-clear generation and could be re-hydrated after a restart + disk loss, + // silently resurrecting data the operator explicitly cleared. A transient cloud + // error is the common cause, so retry the purge (bounded, with backoff) on the + // scheduler. If every attempt fails we log loudly for operator intervention rather + // than reporting a clean truncate. + log.error("Cloud truncate purge failed for db={} (prefix={}): stale remote objects " + + "remain; scheduling bounded retries to prevent resurrection of cleared " + + "data on a future restore.", dbName, prefix); + scheduleTruncatePurgeRetry(dbName, prefix); + } + } finally { + truncationTimes.put(dbName, System.currentTimeMillis()); + truncatingDbs.remove(dbName); + } + } + + /** Max asynchronous retries for a failed truncate purge before giving up (logging loudly). */ + private static final int MAX_TRUNCATE_PURGE_RETRIES = 5; + /** Base backoff (ms) for truncate-purge retries; doubles each attempt up to a small cap. */ + private static final long TRUNCATE_PURGE_RETRY_BASE_MS = 500L; + + /** + * Retries a failed truncate purge asynchronously with capped exponential backoff. Each attempt + * refreshes the truncation grace window so metadata mirroring stays suppressed until the remote + * prefix is confirmed clean, avoiding a re-mirror of not-yet-purged stale objects. After + * {@link #MAX_TRUNCATE_PURGE_RETRIES} failures it logs an error and stops — the stale objects + * then require manual cleanup, but the failure is at least visible rather than silent. + */ + private void scheduleTruncatePurgeRetry(String dbName, String prefix) { + // Capture the write generation now: the purge is only valid for the cleared generation. + scheduleTruncatePurgeRetry(dbName, prefix, 1, epochOf(dbWriteEpochs, dbName)); + } + + private void scheduleTruncatePurgeRetry(String dbName, String prefix, int attempt, + long writeEpoch) { + if (attempt > MAX_TRUNCATE_PURGE_RETRIES) { + log.error("Cloud truncate purge for db={} still failing after {} attempt(s) — remote " + + "prefix '{}' may retain stale objects that could be re-hydrated on restore. " + + "Manual cleanup of that prefix is required.", + dbName, MAX_TRUNCATE_PURGE_RETRIES, prefix); + return; + } + int exp = Math.min(Math.max(attempt - 1, 0), 30); + long delay = Math.min(TRUNCATE_PURGE_RETRY_BASE_MS * (1L << exp), 8_000L); + // Keep the grace window fresh so we do not mirror new metadata over the still-stale prefix. + truncationTimes.put(dbName, System.currentTimeMillis()); + try { + metadataSyncScheduler.schedule(() -> { + boolean retryAgain = false; + // Same per-prefix lock as the delete purge, so the epoch check and the purge are + // atomic with respect to an open/create at this prefix. + Object lock = pendingDeleteLocks.computeIfAbsent(prefix, k -> new Object()); + synchronized (lock) { + if (epochOf(dbWriteEpochs, dbName) != writeEpoch) { Review Comment: Addressed -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
