bitflicker64 commented on code in PR #3081:
URL: https://github.com/apache/hugegraph/pull/3081#discussion_r4578208345


##########
hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListener.java:
##########
@@ -0,0 +1,2645 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hugegraph.store.node.cloud;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiConsumer;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+import lombok.Getter;
+
+import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
+import org.apache.hugegraph.rocksdb.access.RocksDBFactory.LiveSstFile;
+import org.apache.hugegraph.rocksdb.access.RocksDBFactory.MetadataSnapshot;
+import 
org.apache.hugegraph.rocksdb.access.RocksDBFactory.RocksdbChangedListener;
+import org.apache.hugegraph.rocksdb.access.RocksDBSession;
+import org.apache.hugegraph.store.cloud.CloudStorageProvider;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * {@link RocksdbChangedListener} that bridges RocksDB table-file lifecycle 
events
+ * to the active {@link CloudStorageProvider}.
+ *
+ * <p>When cloud storage is enabled:
+ * <ul>
+ *   <li>{@link #onDBCreated} uploads any SST files that already exist in the 
DB directory
+ *       (e.g. surviving from a previous run), triggers a non-blocking 
MemTable flush so
+ *       WAL-recovered or recently-written data is written to SST files 
(completion is signalled
+ *       event-driven via {@link #onTableFileCreated}), then mirrors metadata 
inline.</li>
+ *   <li>{@link #onTableFileCreated} pins newly created SST files via a hard 
link, dispatches
+ *       upload work to a bounded background executor, and mirrors metadata 
after upload completes.
+ *       If the hard link fails (e.g. cross-device mount, filesystem limits), 
the upload is routed
+ *       directly to the retry queue using the original SST path — no copy is 
made, so no extra
+ *       disk space is consumed.</li>
+ *   <li>{@link #onTableFileDeleted} mirrors metadata first, then removes the 
superseded SST object.</li>
+ * </ul>
+ *
+ * <h3>Remote key construction</h3>
+ * The remote key is derived by stripping the {@code dataRoot} prefix from the 
absolute
+ * local file path.  This keeps the object layout clean and independent of the 
container
+ * filesystem layout:
+ * <pre>
+ *   dataRoot  = /hugegraph-store/storage
+ *   filePath  = /hugegraph-store/storage/hgstore-metadata/000008.sst
+ *   remoteKey = store-127.0.0.1_8501/hgstore-metadata/000008.sst
+ *   (with path-prefix "hugegraph") → 
hugegraph/store-127.0.0.1_8501/hgstore-metadata/000008.sst
+ * </pre>
+ *
+ * This listener is registered with {@link RocksDBFactory} during application 
startup
+ * (see {@link org.apache.hugegraph.store.node.AppConfig}).
+ */
+@Slf4j
+public class CloudStorageEventListener implements RocksdbChangedListener, 
AutoCloseable {
+
+    /** Absolute, normalised paths of configured store data roots (one per 
configured partition root). */
+    private final String primaryDataRoot;
+    private final List<String> allDataRoots;
+
+    /** Optional per-store namespace prefix prepended to every remote cloud 
key. */
+    private final String storeScopePrefix;
+
+    static final long DEFAULT_READ_MISS_GUARD_WINDOW_MS = 3000L;
+
+    private final boolean startupHydrationEnabled;
+    private final long readMissGuardWindowMs;
+    private final Map<String, Long> readMissAttemptTs;
+
+    /**
+     * Tracks which {@code (dbName, remoteKey)} pairs are currently being 
restored by
+     * {@link #restoreMissingLiveFiles} so a thundering herd of concurrent 
read misses on the
+     * same cold SST does not download the same object many times over. This 
is a best-effort
+     * de-duplication; correctness against concurrent restores of the same 
file is guaranteed by
+     * the per-attempt unique temp name plus an atomic replace-move, not by 
this set.
+     */
+    private final Set<String> inFlightRestores = ConcurrentHashMap.newKeySet();
+
+    /**
+     * Optional retry queue; when non-null, upload failures are submitted here 
instead
+     * of just being logged. When null, failures are only logged (no retry).
+     */
+    private final CloudUploadRetryQueue retryQueue;
+
+    /** Initialized cloud storage provider; resolved once at startup and never 
reconfigured. */
+    private final CloudStorageProvider provider;
+
+    /** Tracks which SST files are confirmed present in cloud (per-DB 
confirmed-file-number set). */
+    private final CloudSyncTracker syncTracker;
+
+    /**
+     * Hard health flag for pending-delete marker durability. Flips to {@code 
false} when a marker
+     * cannot be durably persisted (write + fsync) — a state in which a DB 
delete whose remote
+     * cleanup fails may be unguarded against re-hydration after a crash — and 
back to
+     * {@code true} once a marker is durably persisted again. Surfaced via
+     * {@link CloudStorageMetricsConst#DELETE_MARKER_HEALTHY}. {@code 
volatile}: written from delete
+     * callbacks, read from metric-scrape threads.
+     * -- GETTER --
+     *  Whether pending-delete marker persistence is currently healthy.
+     *  indicates a
+     *  marker could not be durably written, so a delete whose remote cleanup 
fails may not
+     *  survive a crash as a hydration guard. Bound to
+     * <p>
+     * .
+
+     */
+    @Getter
+    private volatile boolean deleteMarkerHealthy = true;
+
+    /** Directory fsync is not supported on Windows directory handles. */
+    private static final boolean WINDOWS_OS =
+            System.getProperty("os.name", "").toLowerCase().contains("win");
+
+    /** Bounded async upload dispatcher so RocksDB callbacks return quickly. */
+    private static final int ASYNC_UPLOAD_THREADS = 2;
+    private static final int ASYNC_UPLOAD_QUEUE_CAPACITY = 256;
+    /** Upload pool owned by this listener; drained by {@link #close()}. */
+    private final ThreadPoolExecutor uploadExecutor;
+
+    // -----------------------------------------------------------------------
+    // Metadata-sync coalescing
+    // -----------------------------------------------------------------------
+    // onTableFileCreated fires once per flushed/compacted SST; syncing 
metadata inline on every
+    // one triggers a full RocksDB checkpoint + S3 list/prune per SST. Instead 
each upload marks
+    // its DB dirty and a fixed-delay drain publishes each dirty DB at most 
once per interval.
+    // Event-driven callers that need an immediate publish (delete guard, 
onCompacted,
+    // onDBCreated) still call syncMetadataSnapshotInline directly and are NOT 
coalesced.
+
+    /** Default drain interval for the post-upload metadata sync. */
+    static final long DEFAULT_METADATA_SYNC_DEBOUNCE_MS = 1_000L;
+
+    /**
+     * Scheduler for the metadata-sync drain and purge/delete retries. Owned 
by this listener and
+     * shut down by {@link #close()}; a schedule after that throws
+     * {@link RejectedExecutionException}, which every call site handles.
+     */
+    private final ScheduledExecutorService metadataSyncScheduler;
+
+    /** Drain interval (ms); {@code <= 0} publishes inline on every SST upload 
instead. */
+    private final long metadataSyncDebounceMs;
+
+    /** DBs with uploads not yet reflected in a published manifest; drained by 
the scheduler. */
+    private final Set<String> dirtyMetadataDbs = ConcurrentHashMap.newKeySet();
+
+    // -----------------------------------------------------------------------
+    // Metadata (CURRENT/MANIFEST/OPTIONS) mirroring & consistent restore
+    // -----------------------------------------------------------------------
+
+    /** Resolved DB directory -> logical DB name, so {@link #onCompacted} 
resolves path events. */
+    private final Map<String, String> dbNameByDir = new ConcurrentHashMap<>();
+
+    /** Per-DB mutexes to serialize metadata capture/publication/pruning. */
+    private final Map<String, Object> metadataSyncLocks = new 
ConcurrentHashMap<>();
+
+    /**
+     * Per-remote-prefix mutexes serializing pending-delete cleanup (inline at 
open vs. the async
+     * retry). Ensures the purge runs at most once and only while the marker 
is present, so it can
+     * never delete data a re-created DB uploads to the same prefix after 
cleanup completes.
+     */
+    private final Map<String, Object> pendingDeleteLocks = new 
ConcurrentHashMap<>();
+
+    /**
+     * Per-remote-prefix generation, bumped every time a DB is opened or 
created at that prefix.
+     * An asynchronous pending-delete purge captures the value when it is 
scheduled and aborts
+     * (without purging) if it changed, so a retry can never delete objects 
that a re-created DB
+     * has since taken over the prefix with. Entries are never removed: 
dropping one would reset
+     * the counter and let a stale retry match again.
+     */
+    private final Map<String, AtomicLong> prefixEpochs = new 
ConcurrentHashMap<>();
+
+    /** Per-DB metadata-publish failure state used to back off a persistently 
failing publish. */
+    private final Map<String, PublishBackoff> publishBackoffs = new 
ConcurrentHashMap<>();
+
+    /** Cap for the exponential publish-retry backoff. */
+    static final long MAX_PUBLISH_BACKOFF_MS = 5 * 60 * 1000L;
+    /** Minimum base for the publish-retry backoff, so a tiny debounce does 
not hot-loop failures. */
+    static final long MIN_PUBLISH_BACKOFF_BASE_MS = 1_000L;
+
+    /** Bounded synchronous attempts at completing a pending delete while a DB 
is being opened. */
+    private static final int OPEN_PENDING_DELETE_ATTEMPTS = 3;
+    private static final long OPEN_PENDING_DELETE_BACKOFF_MS = 200L;
+    /** Bounded attempts at flushing dirty metadata on close(). */
+    private static final int CLOSE_FLUSH_ATTEMPTS = 3;
+    private static final long CLOSE_FLUSH_BACKOFF_MS = 200L;
+
+    /** Consecutive publish failures and the earliest time the next attempt is 
allowed. */
+    private static final class PublishBackoff {
+
+        final int failures;
+        final long nextAttemptAtMs;
+
+        PublishBackoff(int failures, long nextAttemptAtMs) {
+            this.failures = failures;
+            this.nextAttemptAtMs = nextAttemptAtMs;
+        }
+    }
+
+    /** Last successfully published RocksDB generation per DB. */
+    private final Map<String, Long> lastPublishedMetadataGeneration = new 
ConcurrentHashMap<>();
+
+    /**
+     * Suffix appended to the DB prefix (not inside it) when a database is 
deleted.
+     * Placing the tombstone as a sibling of the data prefix means the
+     * {@link #purgeRemotePrefix} call in {@link #onDBDeleted} cannot 
accidentally remove it
+     * while stale SST or metadata objects remain, so {@link 
#preHydrateDbFiles} can still
+     * detect the deleted generation and skip hydration.
+     *
+     * <p>Example: data prefix = {@code store-host_8500/hugegraph/db},
+     * tombstone key = {@code store-host_8500/hugegraph/db_DELETED}.
+     */
+    static final String DB_TOMBSTONE_SUFFIX = "_DELETED";
+
+    /** Infix of hydration temp files ({@code 
<name>.hyd-tmp-<threadId>-<nanoTime>}). */
+    private static final String HYDRATION_TEMP_MARKER = ".hyd-tmp-";
+
+    /**
+     * Fully-parameterised constructor: the listener is completely configured 
the moment it is
+     * constructed, so there is no window in which a registered listener can 
be observed
+     * partially configured.
+     *
+     * @param provider initialized cloud storage provider
+     * @param dataRoots configured store data roots (typically parsed from 
comma-separated
+     *                  {@code app.data-path})
+     * @param readMissGuardWindowMs guard window in ms for repeated read-miss 
hydration attempts
+     *                              for the same db/table pair 
(cloud.storage.read-miss-guard-window-ms)
+     * @param retryQueue optional {@link CloudUploadRetryQueue}; when 
non-null, upload failures
+     *                   are retried asynchronously and eventually given up on.
+     *                   Pass {@code null} to disable retries (failures are 
only logged).
+     * @param syncTracker tracks SST files confirmed present in cloud; the 
delete guard uses it
+     *                    to avoid deleting a superseded object before 
replacements are durable.
+     *                    Must be shared with the retry queue. Pass {@code 
null} for a fresh
+     *                    {@link CloudSyncTracker}.
+     * @param storeScopePrefix optional per-store key prefix to isolate cloud 
objects
+     * @param metadataSyncDebounceMs drain interval (ms) for the per-SST 
metadata sync;
+     *                               {@code <= 0} publishes inline on every 
SST upload
+     */
+    public CloudStorageEventListener(CloudStorageProvider provider,
+                                     List<String> dataRoots,
+                                     boolean startupHydrationEnabled,
+                                     long readMissGuardWindowMs,
+                                     CloudUploadRetryQueue retryQueue,
+                                     CloudSyncTracker syncTracker,
+                                     String storeScopePrefix,
+                                     long metadataSyncDebounceMs) {
+        // Fail fast on a missing/empty data-root list: primaryDataRoot is 
derived from index 0
+        // below, and every key<->path conversion depends on at least one 
root. Without this guard
+        // the constructor would throw an opaque IndexOutOfBoundsException at
+        // allDataRoots.get(0), which is far harder to diagnose than a config 
error.
+        if (dataRoots == null || dataRoots.isEmpty()) {
+            throw new IllegalArgumentException(
+                    "CloudStorageEventListener requires at least one data 
root; none configured. "
+                    + "Check the store data-path configuration (e.g. 
rocksdb.data_path / "
+                    + "raft.path) used to derive the cloud storage data 
roots.");
+        }
+        // Normalize each root
+        this.allDataRoots = new ArrayList<>();
+        for (String root : dataRoots) {
+            String normalised = 
Paths.get(root).toAbsolutePath().normalize().toString();
+            // Strip trailing separator so substring arithmetic is consistent.
+            normalised = normalised.endsWith(File.separator)
+                    ? normalised.substring(0, normalised.length() - 1)
+                    : normalised;
+            this.allDataRoots.add(normalised);
+        }
+        this.primaryDataRoot = this.allDataRoots.get(0);
+        this.provider = Objects.requireNonNull(provider, "provider");
+        this.startupHydrationEnabled = startupHydrationEnabled;
+        this.readMissGuardWindowMs = Math.max(0L, readMissGuardWindowMs);
+        this.readMissAttemptTs = new ConcurrentHashMap<>();
+        this.retryQueue = retryQueue;
+        this.syncTracker = syncTracker != null ? syncTracker : new 
CloudSyncTracker();
+        this.storeScopePrefix = normaliseKeyPrefix(storeScopePrefix);
+        this.uploadExecutor = new ThreadPoolExecutor(
+                ASYNC_UPLOAD_THREADS,
+                ASYNC_UPLOAD_THREADS,
+                60L,
+                TimeUnit.SECONDS,
+                new ArrayBlockingQueue<>(ASYNC_UPLOAD_QUEUE_CAPACITY),
+                newUploadThreadFactory(),
+                new ThreadPoolExecutor.AbortPolicy());
+        ScheduledThreadPoolExecutor syncScheduler = new 
ScheduledThreadPoolExecutor(1, r -> {
+            Thread t = new Thread(r, "cloud-metadata-sync");
+            t.setDaemon(true);
+            return t;
+        });
+        // close() uses a graceful shutdown() so an in-flight publish 
completes; not-yet-due delayed
+        // tasks (purge/delete retries) are cancelled instead of holding 
close() open. Their intent
+        // is crash-durable (local markers) and is re-driven by the next 
startup scan.
+        syncScheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
+        this.metadataSyncScheduler = syncScheduler;
+        this.metadataSyncDebounceMs = metadataSyncDebounceMs;
+        if (metadataSyncDebounceMs > 0) {
+            
this.metadataSyncScheduler.scheduleWithFixedDelay(this::drainDirtyMetadata,
+                                                              
metadataSyncDebounceMs,
+                                                              
metadataSyncDebounceMs,
+                                                              
TimeUnit.MILLISECONDS);
+        }
+
+        // A crash may leave local pending-delete markers whose remote cleanup 
never completed.
+        // Kick off bounded async retries for each so stale remote data is 
eventually purged even if
+        // the affected DB is never re-opened (an open would otherwise be the 
only trigger).
+        scanMarkers(pendingDeleteDir(), (db, prefix) -> {
+            log.warn("Cloud startup: found pending remote-delete marker for 
prefix={} — "
+                     + "scheduling deferred purge to prevent stale-data 
re-hydration.", prefix);
+            scheduleDeletePurgeRetry(db, prefix, 1);
+        });
+        // Upload intent is not persisted across restarts, so any staging pin 
left by the previous
+        // process is orphaned; the live SSTs it pinned are re-uploaded by 
startup backfill.
+        sweepStagingDirsOnStartup();
+    }
+
+    private static ThreadFactory newUploadThreadFactory() {
+        return r -> {
+            Thread t = new Thread(r, "cloud-upload-dispatch");
+            t.setDaemon(true);
+            return t;
+        };
+    }
+
+    /** Shuts down this listener's executors, waiting up to 10s for in-flight 
uploads. */
+    @Override
+    public void close() {
+        close(10, TimeUnit.SECONDS);
+    }
+
+    /**
+     * Shuts down the SST upload executor, waiting up to {@code timeout} for 
in-flight uploads to
+     * complete, then the metadata-sync scheduler. Called from
+     * {@link org.apache.hugegraph.store.node.AppConfig#onDestroy()} after the 
listener has been
+     * deregistered, so uploads started before shutdown have a chance to 
finish.
+     */
+    void close(long timeout, TimeUnit unit) {
+        // Drain the upload executor FIRST so DBs marked dirty by in-flight 
uploads are flushed below.
+        uploadExecutor.shutdown();
+        try {
+            if (!uploadExecutor.awaitTermination(timeout, unit)) {
+                log.warn("CloudStorageEventListener: upload executor did not 
terminate within "
+                         + "{}{}; forcing shutdown", timeout, unit);
+                handoffDroppedUploadTasks(uploadExecutor.shutdownNow());
+            }
+        } catch (InterruptedException e) {
+            handoffDroppedUploadTasks(uploadExecutor.shutdownNow());
+            Thread.currentThread().interrupt();
+        }
+        // Graceful stop: an in-flight drain/publish finishes instead of being 
interrupted mid-upload,
+        // and the final flush below runs only after the drain thread is gone 
so the two cannot
+        // publish the same DB concurrently.
+        metadataSyncScheduler.shutdown();
+        try {
+            if (!metadataSyncScheduler.awaitTermination(timeout, unit)) {
+                log.warn("CloudStorageEventListener: metadata-sync scheduler 
did not terminate "
+                         + "within {}{}; forcing shutdown", timeout, unit);
+                metadataSyncScheduler.shutdownNow();
+                metadataSyncScheduler.awaitTermination(1, TimeUnit.SECONDS);
+            }
+        } catch (InterruptedException e) {
+            metadataSyncScheduler.shutdownNow();
+            Thread.currentThread().interrupt();
+        }
+        // Flush dirty DBs so uploads that completed during the drain are 
referenced by the
+        // mirrored CURRENT/MANIFEST. Later requests (e.g. retry-queue 
completions) see the
+        // scheduler shut down in markMetadataDirty and publish inline.
+        flushDirtyMetadataOnClose();
+    }
+
+    /**
+     * Final bounded flush of every dirty DB. A failed publish is retried a 
few times; DBs that
+     * still cannot be published are logged so the gap is visible.
+     */
+    private void flushDirtyMetadataOnClose() {
+        List<String> remaining = new ArrayList<>(dirtyMetadataDbs);
+        for (int attempt = 1; attempt <= CLOSE_FLUSH_ATTEMPTS && 
!remaining.isEmpty(); attempt++) {
+            for (String dbName : new ArrayList<>(remaining)) {
+                dirtyMetadataDbs.remove(dbName);
+                if (publishDirtyMetadata(dbName)) {
+                    remaining.remove(dbName);
+                } else {
+                    dirtyMetadataDbs.add(dbName);
+                }
+            }
+            if (remaining.isEmpty() || attempt == CLOSE_FLUSH_ATTEMPTS) {
+                break;
+            }
+            try {
+                Thread.sleep(CLOSE_FLUSH_BACKOFF_MS * attempt);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break;
+            }
+        }
+        if (!remaining.isEmpty()) {
+            log.error("CloudStorageEventListener: closed with metadata NOT 
published for {} DB(s): "
+                      + "{} — the mirrored CURRENT/MANIFEST may lag the 
uploaded SSTs until the next "
+                      + "start publishes them.", remaining.size(), remaining);
+        }
+    }
+
+    /**
+     * Routes upload tasks discarded by a forced {@code shutdownNow()} into 
their retry queue
+     * so an SST accepted locally but never mirrored is not silently lost from 
the durability
+     * pipeline. Only {@link SstUploadTask}s carry the metadata needed for 
handoff; any other
+     * runnable (there should be none) is ignored.
+     */
+    private static void handoffDroppedUploadTasks(java.util.List<Runnable> 
dropped) {
+        if (dropped == null || dropped.isEmpty()) {
+            return;
+        }
+        int handed = 0;
+        for (Runnable r : dropped) {
+            if (r instanceof SstUploadTask) {
+                try {
+                    ((SstUploadTask) r).handoffOnShutdown();
+                    handed++;
+                } catch (Exception e) {
+                    log.warn("Failed to hand off a dropped upload task to the 
retry queue: {}",
+                             e.getMessage());
+                }
+            }
+        }
+        if (handed > 0) {
+            log.warn("CloudStorageEventListener: handed off {} queued upload 
task(s) to the retry queue "
+                     + "during shutdown so their upload intent survives.", 
handed);
+        }
+    }
+
+    /**
+     * Finds which configured data roots contain the given file path.
+     * Returns the matching root, or the primary root if no exact match is 
found.
+     *
+     * <p>Uses {@link Path#startsWith} so that {@code /data/store1} never 
incorrectly
+     * matches {@code /data/store10/...} (a raw string prefix match would).
+     *
+     * @param filePath absolute file path
+     * @return the matched configured data root for this file
+     */
+    private String findMatchingDataRoot(String filePath) {
+        Path fileNormalized = Paths.get(filePath).toAbsolutePath().normalize();
+        for (String root : allDataRoots) {
+            if (fileNormalized.startsWith(Paths.get(root))) {
+                return root;
+            }
+        }
+        // Fallback to primary root (should not happen in normal operation)
+        return primaryDataRoot;
+    }
+
+    // -----------------------------------------------------------------------
+    // RocksdbChangedListener
+    // -----------------------------------------------------------------------
+
+    @Override
+    public boolean managesRemoteState() {
+        return true;
+    }
+
+    /**
+     * Picks the directory to open when the local one holds no DB. A partition 
can move to a
+     * version-suffixed directory ({@code <name>_<v1>[_<v2>]}) and its objects 
then live under that
+     * prefix, so after a disk loss the default path would hydrate nothing. 
Lists the remote
+     * siblings that share the base name and returns the newest restorable 
one, matching the order
+     * {@code RocksDBSession} uses locally. A sibling is restorable when it 
has a remote
+     * {@code CURRENT}, no delete tombstone and no local pending-delete 
marker. If listing fails the
+     * path is kept, and the hydration of that path applies its own failure 
rules.
+     */
+    @Override
+    public String resolveOpenPath(String dbName, String dbPath) {
+        if (!startupHydrationEnabled) {
+            return dbPath;
+        }
+        Path candidate = Paths.get(dbPath).toAbsolutePath().normalize();
+        Path parent = candidate.getParent();
+        if (parent == null) {
+            return dbPath;
+        }
+        String name = candidate.getFileName().toString();
+        int sep = name.indexOf('_');
+        String baseName = sep < 0 ? name : name.substring(0, sep);
+        String basePrefix = dbPrefix(parent.resolve(baseName).toString());
+        if (!basePrefix.endsWith(baseName)) {
+            return dbPath;
+        }
+        String parentKeyPrefix = basePrefix.substring(0, basePrefix.length() - 
baseName.length());
+        List<String> keys;
+        try {
+            keys = provider.listFiles(basePrefix);
+        } catch (IOException e) {
+            log.warn("Cloud open: listing version siblings of db={} failed, 
keeping {}: {}",
+                     dbName, dbPath, e.getMessage());
+            return dbPath;
+        }
+        String newest = newestRestorableSibling(parentKeyPrefix, baseName, 
keys);
+        if (newest == null || newest.equals(name)) {
+            return dbPath;
+        }
+        log.info("Cloud open: db={} has no local DB at {}; restoring newest 
remote sibling '{}'",
+                 dbName, dbPath, newest);
+        return parent.resolve(newest).toString();
+    }
+
+    /**
+     * Newest restorable directory name among {@code baseName} and its 
version-suffixed siblings
+     * found in {@code keys}, or {@code null} if none (package-private for 
tests).
+     */
+    String newestRestorableSibling(String parentKeyPrefix, String baseName, 
List<String> keys) {
+        Pattern dirName = Pattern.compile(Pattern.quote(baseName) + 
"(?:_(\\d+)(?:_(\\d+))?)?");
+        Set<String> withCurrent = new HashSet<>();
+        Set<String> tombstoned = new HashSet<>();
+        for (String key : keys) {
+            if (!key.startsWith(parentKeyPrefix)) {
+                continue;
+            }
+            String rel = key.substring(parentKeyPrefix.length());
+            int slash = rel.indexOf('/');
+            if (slash < 0) {
+                if (rel.endsWith(DB_TOMBSTONE_SUFFIX)) {
+                    tombstoned.add(rel.substring(0, rel.length() - 
DB_TOMBSTONE_SUFFIX.length()));
+                }
+                continue;
+            }
+            String dir = rel.substring(0, slash);
+            if ("CURRENT".equals(rel.substring(slash + 1)) && 
dirName.matcher(dir).matches()) {
+                withCurrent.add(dir);
+            }
+        }
+        String newest = null;
+        long[] newestVersion = null;
+        for (String dir : withCurrent) {
+            if (tombstoned.contains(dir) || 
hasPendingDeleteMarker(parentKeyPrefix + dir)) {
+                continue;
+            }
+            Matcher m = dirName.matcher(dir);
+            if (!m.matches()) {
+                continue;
+            }
+            long[] version;
+            try {
+                version = new long[]{m.group(1) == null ? -1L : 
Long.parseLong(m.group(1)),
+                                     m.group(2) == null ? -1L : 
Long.parseLong(m.group(2))};
+            } catch (NumberFormatException e) {
+                continue;
+            }
+            if (newestVersion == null || version[0] > newestVersion[0]
+                || (version[0] == newestVersion[0] && version[1] > 
newestVersion[1])) {
+                newest = dir;
+                newestVersion = version;
+            }
+        }
+        return newest;
+    }
+
+    @Override
+    public void onDBOpening(String dbName, String dbPath) {
+        // Register the dir -> logical-name mapping first: native events that 
arrive before
+        // onDBCreated (or on a disabled-hydration node) must resolve to the 
logical name.
+        recordDb(dbName, dbPath);

Review Comment:
   Important: `syncTracker` confirmations and `lastPublishedMetadataGeneration` 
are keyed by DB name, so a DB reopened at a new directory (snapshot install 
into `<name>_<v>`, or destroy-then-recreate) inherits the old directory's 
state: a reused SST number counts as uploaded, and the new DB's publishes are 
rejected as stale until its sequence passes the old one. Clear both for 
`dbName` in `onDBOpening` when the name was mapped to a different directory.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to