JingsongLi commented on code in PR #11037:
URL: https://github.com/apache/paimon/pull/11037#discussion_r4522826330


##########
paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/S3FileIO.java:
##########
@@ -115,34 +125,162 @@ private Options mirrorCertainHadoopConfig(Options 
hadoopConfig) {
         return hadoopConfig;
     }
 
+    /**
+     * Overrides parent to refresh {@link #fsMap} when a cached {@link 
RecoverableS3AFileSystem} has
+     * been invalidated. Avoids introducing a generic health hook on {@link 
HadoopCompliantFileIO}.
+     */
+    @Override
+    protected FileSystem getFileSystem(org.apache.hadoop.fs.Path path) throws 
IOException {
+        if (fsMap == null) {
+            synchronized (this) {
+                if (fsMap == null) {
+                    fsMap = new ConcurrentHashMap<>();
+                }
+            }
+        }
+
+        final Map<Pair<String, String>, FileSystem> map = fsMap;
+        URI uri = path.toUri();
+        Pair<String, String> key = Pair.of(uri.getScheme(), 
uri.getAuthority());
+
+        // Create outside fsMap's ConcurrentHashMap compute lock.
+        while (true) {
+            FileSystem current = map.get(key);
+            if (isUsableFileSystem(current)) {
+                return current;
+            }
+
+            FileSystem created = createFileSystem(path);
+            if (!isUsableFileSystem(created)) {
+                continue;
+            }
+
+            if (current == null) {
+                FileSystem existing = map.putIfAbsent(key, created);
+                if (existing == null) {
+                    if (isUsableFileSystem(created)) {
+                        return created;
+                    }
+                    map.remove(key, created);
+                    continue;
+                }
+                if (isUsableFileSystem(existing)) {
+                    return existing;
+                }
+                current = existing;
+            }
+
+            if (isUsableFileSystem(current)) {
+                return current;
+            }
+            if (!map.replace(key, current, created)) {
+                continue;
+            }
+            if (isUsableFileSystem(created)) {
+                return created;
+            }
+            map.remove(key, created);
+        }
+    }
+
     @Override
-    protected FileSystem createFileSystem(org.apache.hadoop.fs.Path path) {
+    protected FileSystem createFileSystem(org.apache.hadoop.fs.Path path) 
throws IOException {
         final String scheme = path.toUri().getScheme();
         final String authority = path.toUri().getAuthority();
-        return CACHE.computeIfAbsent(
-                new CacheKey(hadoopOptions, scheme, authority),
-                key -> {
-                    Configuration hadoopConf = new Configuration();
-                    key.options.toMap().forEach(hadoopConf::set);
-                    URI fsUri = path.toUri();
-                    if (scheme == null && authority == null) {
-                        fsUri = FileSystem.getDefaultUri(hadoopConf);
-                    } else if (scheme != null && authority == null) {
-                        URI defaultUri = FileSystem.getDefaultUri(hadoopConf);
-                        if (scheme.equals(defaultUri.getScheme())
-                                && defaultUri.getAuthority() != null) {
-                            fsUri = defaultUri;
-                        }
-                    }
+        final CacheKey cacheKey = new CacheKey(hadoopOptions, scheme, 
authority);
 
-                    S3AFileSystem fs = new S3AFileSystem();
-                    try {
-                        fs.initialize(fsUri, hadoopConf);
-                    } catch (IOException e) {
-                        throw new UncheckedIOException(e);
-                    }
-                    return fs;
-                });
+        S3AFileSystem cached = CACHE.get(cacheKey);
+        if (isUsableFileSystem(cached)) {
+            return cached;
+        }
+
+        Object lock = getCacheLock(cacheKey);
+        synchronized (lock) {
+            cached = CACHE.get(cacheKey);
+            if (isUsableFileSystem(cached)) {
+                return cached;
+            }
+
+            // Only this confirmed mapping may be removed if initialize fails.
+            final S3AFileSystem expectedOld = cached;
+
+            Configuration hadoopConf = new Configuration();
+            cacheKey.options.toMap().forEach(hadoopConf::set);
+            URI fsUri = path.toUri();
+            if (scheme == null && authority == null) {
+                fsUri = FileSystem.getDefaultUri(hadoopConf);
+            } else if (scheme != null && authority == null) {
+                URI defaultUri = FileSystem.getDefaultUri(hadoopConf);
+                if (scheme.equals(defaultUri.getScheme()) && 
defaultUri.getAuthority() != null) {
+                    fsUri = defaultUri;
+                }
+            }
+
+            RecoverableS3AFileSystem fs = newRecoverableFileSystem();
+            fs.bind(() -> CACHE.remove(cacheKey, fs));
+            try {
+                fs.initialize(fsUri, hadoopConf);
+                if (!fs.isUsable()) {
+                    throw new IOException("S3A filesystem became unusable 
during initialize");
+                }
+            } catch (IOException | RuntimeException | Error e) {
+                discardFailedCreate(cacheKey, expectedOld, fs);
+                throw e;
+            }
+
+            CACHE.put(cacheKey, fs);
+            if (!fs.isUsable()) {
+                CACHE.remove(cacheKey, fs);
+                try {
+                    fs.close();

Review Comment:
   **[P2] Keep a filesystem open after another caller can borrow it**
   
   I reproduced this with a latch immediately after `CACHE.put`. A second 
`S3FileIO` took the shared instance through the `CACHE.get` fast path. A pool 
failure then marked it unusable, and the creator's post-put check called 
`close()` on that already borrowed instance. That can interrupt another 
caller's stream or transfer, which is the reason the regular eviction path 
avoids closing published clients. Could we evict this mapping without closing 
after it has become visible, or make publication atomic with the health check 
under a lease?



##########
paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/S3FileIO.java:
##########
@@ -115,34 +125,162 @@ private Options mirrorCertainHadoopConfig(Options 
hadoopConfig) {
         return hadoopConfig;
     }
 
+    /**
+     * Overrides parent to refresh {@link #fsMap} when a cached {@link 
RecoverableS3AFileSystem} has
+     * been invalidated. Avoids introducing a generic health hook on {@link 
HadoopCompliantFileIO}.
+     */
+    @Override
+    protected FileSystem getFileSystem(org.apache.hadoop.fs.Path path) throws 
IOException {
+        if (fsMap == null) {
+            synchronized (this) {
+                if (fsMap == null) {
+                    fsMap = new ConcurrentHashMap<>();
+                }
+            }
+        }
+
+        final Map<Pair<String, String>, FileSystem> map = fsMap;
+        URI uri = path.toUri();
+        Pair<String, String> key = Pair.of(uri.getScheme(), 
uri.getAuthority());
+
+        // Create outside fsMap's ConcurrentHashMap compute lock.
+        while (true) {
+            FileSystem current = map.get(key);
+            if (isUsableFileSystem(current)) {
+                return current;
+            }
+
+            FileSystem created = createFileSystem(path);
+            if (!isUsableFileSystem(created)) {
+                continue;
+            }
+
+            if (current == null) {
+                FileSystem existing = map.putIfAbsent(key, created);
+                if (existing == null) {
+                    if (isUsableFileSystem(created)) {
+                        return created;
+                    }
+                    map.remove(key, created);
+                    continue;
+                }
+                if (isUsableFileSystem(existing)) {
+                    return existing;
+                }
+                current = existing;
+            }
+
+            if (isUsableFileSystem(current)) {
+                return current;
+            }
+            if (!map.replace(key, current, created)) {
+                continue;
+            }
+            if (isUsableFileSystem(created)) {
+                return created;
+            }
+            map.remove(key, created);
+        }
+    }
+
     @Override
-    protected FileSystem createFileSystem(org.apache.hadoop.fs.Path path) {
+    protected FileSystem createFileSystem(org.apache.hadoop.fs.Path path) 
throws IOException {
         final String scheme = path.toUri().getScheme();
         final String authority = path.toUri().getAuthority();
-        return CACHE.computeIfAbsent(
-                new CacheKey(hadoopOptions, scheme, authority),
-                key -> {
-                    Configuration hadoopConf = new Configuration();
-                    key.options.toMap().forEach(hadoopConf::set);
-                    URI fsUri = path.toUri();
-                    if (scheme == null && authority == null) {
-                        fsUri = FileSystem.getDefaultUri(hadoopConf);
-                    } else if (scheme != null && authority == null) {
-                        URI defaultUri = FileSystem.getDefaultUri(hadoopConf);
-                        if (scheme.equals(defaultUri.getScheme())
-                                && defaultUri.getAuthority() != null) {
-                            fsUri = defaultUri;
-                        }
-                    }
+        final CacheKey cacheKey = new CacheKey(hadoopOptions, scheme, 
authority);
 
-                    S3AFileSystem fs = new S3AFileSystem();
-                    try {
-                        fs.initialize(fsUri, hadoopConf);
-                    } catch (IOException e) {
-                        throw new UncheckedIOException(e);
-                    }
-                    return fs;
-                });
+        S3AFileSystem cached = CACHE.get(cacheKey);
+        if (isUsableFileSystem(cached)) {
+            return cached;
+        }
+
+        Object lock = getCacheLock(cacheKey);
+        synchronized (lock) {
+            cached = CACHE.get(cacheKey);
+            if (isUsableFileSystem(cached)) {
+                return cached;
+            }
+
+            // Only this confirmed mapping may be removed if initialize fails.
+            final S3AFileSystem expectedOld = cached;
+
+            Configuration hadoopConf = new Configuration();
+            cacheKey.options.toMap().forEach(hadoopConf::set);
+            URI fsUri = path.toUri();
+            if (scheme == null && authority == null) {
+                fsUri = FileSystem.getDefaultUri(hadoopConf);
+            } else if (scheme != null && authority == null) {
+                URI defaultUri = FileSystem.getDefaultUri(hadoopConf);
+                if (scheme.equals(defaultUri.getScheme()) && 
defaultUri.getAuthority() != null) {
+                    fsUri = defaultUri;
+                }
+            }
+
+            RecoverableS3AFileSystem fs = newRecoverableFileSystem();
+            fs.bind(() -> CACHE.remove(cacheKey, fs));
+            try {
+                fs.initialize(fsUri, hadoopConf);
+                if (!fs.isUsable()) {
+                    throw new IOException("S3A filesystem became unusable 
during initialize");
+                }
+            } catch (IOException | RuntimeException | Error e) {
+                discardFailedCreate(cacheKey, expectedOld, fs);
+                throw e;
+            }
+
+            CACHE.put(cacheKey, fs);
+            if (!fs.isUsable()) {
+                CACHE.remove(cacheKey, fs);
+                try {
+                    fs.close();
+                } catch (Throwable t) {
+                    LOG.warn(
+                            "Failed to close S3AFileSystem that became 
unusable after cache publish",
+                            t);
+                }
+                throw new IOException(
+                        "S3A filesystem became unusable before cache publish 
completed");
+            }
+            return fs;
+        }
+    }
+
+    RecoverableS3AFileSystem newRecoverableFileSystem() {
+        return new RecoverableS3AFileSystem();
+    }
+
+    /**
+     * Cleans up after a failed create: remove only {@code expectedOld} if it 
is still cached, and
+     * close the unpublished filesystem. Never removes a newer published 
instance.
+     */
+    private static void discardFailedCreate(
+            CacheKey cacheKey, S3AFileSystem expectedOld, S3AFileSystem 
unpublished) {
+        if (expectedOld != null) {
+            CACHE.remove(cacheKey, expectedOld);
+        }
+        try {
+            unpublished.close();
+        } catch (Throwable t) {
+            LOG.warn("Failed to close unpublished S3AFileSystem after 
initialize failure", t);
+        }
+    }
+
+    private static Object getCacheLock(CacheKey key) {
+        return CACHE_LOCKS.computeIfAbsent(key, k -> new Object());

Review Comment:
   **[P2] Bound the lock cache for failed initializations**
   
   I ran 16 failed initializations against distinct bucket keys. `CACHE` kept 
zero entries, but `CACHE_LOCKS` grew by 16 and has no removal path. A 
long-running process that encounters new bucket/configuration keys during an 
outage will retain all of those options and locks indefinitely, even when none 
of the filesystems ever initialized. Could we use bounded striped locks or 
another lock lifecycle that accounts for waiters?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to