This is an automated email from the ASF dual-hosted git repository.

davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git


The following commit(s) were added to refs/heads/main by this push:
     new 4468687fb26d CAMEL-27322: camel-file - a cluster view no longer 
releases the file lock of another view in the same JVM (#30576)
4468687fb26d is described below

commit 4468687fb26da56fc478f86249bc442426c8eb7f
Author: allthingssecurity <[email protected]>
AuthorDate: Sun Nov 22 21:32:12 2026 +0530

    CAMEL-27322: camel-file - a cluster view no longer releases the file lock 
of another view in the same JVM (#30576)
    
    The file locks of FileLockClusterView are locks of the process (POSIX fcntl
    locks on Linux and macOS): closing any channel of the lock file releases all
    the locks of the process on it, while the FileLock of the holder still says 
it
    is valid. A follower view in the same JVM (two CamelContexts with a
    FileLockClusterService on the same root) that found the leader stale opened
    the lock file, got an OverlappingFileLockException and closed its channel,
    which dropped the lock of the leader view. A member in another JVM could 
then
    take the lock and become a second leader.
    
    The views of a JVM now register the lock files they hold or try, and a view
    does not open a lock file that another view of its JVM has registered.
    
    The key of the lock file in the JVM-wide registry resolves the real path of
    the lock file directory, which reads the file system. It ran directly on the
    scheduler thread of the service, so a hung network mount could block the
    leadership checks of all its views for good. It now runs through the cluster
    task executor with the cluster data task timeout and attempts, like the 
other
    file I/O of the leadership check (CAMEL-25089): the attempt fails and the
    check is scheduled again. The key is resolved before it is registered, so a
    failed attempt leaves nothing in the registry.
    
    If the directory cannot be resolved, the attempt now fails too, instead of
    using the unresolved path, which may differ from the key of another view of
    the JVM (opening the lock file fails without the directory anyway).
    
    New tests: a check whose key resolution hangs times out and is scheduled
    again, and the lock file is not opened while the key cannot be resolved.
    
    Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
 .../file/cluster/FileLockClusterView.java          |  78 ++++++-
 .../cluster/FileLockClusterViewSameJvmTest.java    | 258 +++++++++++++++++++++
 2 files changed, 332 insertions(+), 4 deletions(-)

diff --git 
a/components/camel-file/src/main/java/org/apache/camel/component/file/cluster/FileLockClusterView.java
 
b/components/camel-file/src/main/java/org/apache/camel/component/file/cluster/FileLockClusterView.java
index eb13f444c41b..b12af1dede9e 100644
--- 
a/components/camel-file/src/main/java/org/apache/camel/component/file/cluster/FileLockClusterView.java
+++ 
b/components/camel-file/src/main/java/org/apache/camel/component/file/cluster/FileLockClusterView.java
@@ -26,9 +26,11 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -48,6 +50,12 @@ public class FileLockClusterView extends 
AbstractCamelClusterView {
     // It isolates the critical section making sure only one service creates 
the files.
     private static final ReentrantLock contextStartLock = new ReentrantLock();
     private static final Logger LOGGER = 
LoggerFactory.getLogger(FileLockClusterView.class);
+    // The lock files that a view of this JVM has locked or is trying to lock, 
with that view. The file locks are
+    // locks of the process (POSIX fcntl locks on Linux and macOS), and 
closing any channel of a file releases all the
+    // locks of the process on it (see FileLock), while the FileLock of the 
holder still says it is valid. So a view
+    // must not open (and then close) a channel on a lock file that another 
view of the JVM has locked: it does not try
+    // the lock while the path is registered here.
+    private static final Map<Path, FileLockClusterView> LOCK_PATHS = new 
ConcurrentHashMap<>();
 
     private final ClusterMember localMember;
     private final Path leaderLockPath;
@@ -58,6 +66,8 @@ public class FileLockClusterView extends 
AbstractCamelClusterView {
     private volatile RandomAccessFile leaderLockFile;
     private volatile RandomAccessFile leaderDataFile;
     private volatile FileLock lock;
+    // the key of the lock file in LOCK_PATHS while the view holds the lock 
(written with lock, under stateLock)
+    private volatile Path lockPathKey;
     private int heartbeatTimeoutMultiplier;
     private long acquireLockIntervalMilliseconds;
     private FileLockClusterTaskExecutor clusterTaskExecutor;
@@ -154,6 +164,7 @@ public class FileLockClusterView extends 
AbstractCamelClusterView {
         final FileLock heldLock;
         final RandomAccessFile heldLockFile;
         final RandomAccessFile heldDataFile;
+        final Path heldLockPathKey;
         stateLock.lock();
         try {
             // A leadership check that is running now can no longer publish a 
lock it acquires: it re-checks the
@@ -165,9 +176,11 @@ public class FileLockClusterView extends 
AbstractCamelClusterView {
             heldLock = lock;
             heldLockFile = leaderLockFile;
             heldDataFile = leaderDataFile;
+            heldLockPathKey = lockPathKey;
             lock = null;
             leaderLockFile = null;
             leaderDataFile = null;
+            lockPathKey = null;
         } finally {
             stateLock.unlock();
         }
@@ -201,6 +214,7 @@ public class FileLockClusterView extends 
AbstractCamelClusterView {
             releaseFileLock(heldLock);
             closeFile(heldLockFile);
             closeFile(heldDataFile);
+            unregisterLockPath(heldLockPathKey);
             clusterLeaderInfoRef.set(null);
         }
     }
@@ -208,11 +222,12 @@ public class FileLockClusterView extends 
AbstractCamelClusterView {
     /**
      * The lock, lock file and data file taken over from this view by {@link 
#takeOver(long)}.
      */
-    private record HeldLock(FileLock lock, RandomAccessFile lockFile, 
RandomAccessFile dataFile) {
-        void release() {
+    private record HeldLock(FileLock lock, RandomAccessFile lockFile, 
RandomAccessFile dataFile, Path lockPathKey) {
+        void release(FileLockClusterView view) {
             releaseFileLock(lock);
             closeFile(lockFile);
             closeFile(dataFile);
+            view.unregisterLockPath(lockPathKey);
         }
     }
 
@@ -227,10 +242,11 @@ public class FileLockClusterView extends 
AbstractCamelClusterView {
             if (gen != generation) {
                 return null;
             }
-            HeldLock held = new HeldLock(lock, leaderLockFile, leaderDataFile);
+            HeldLock held = new HeldLock(lock, leaderLockFile, leaderDataFile, 
lockPathKey);
             lock = null;
             leaderLockFile = null;
             leaderDataFile = null;
+            lockPathKey = null;
             localMember.setStatus(ClusterMemberStatus.FOLLOWER);
             return held;
         } finally {
@@ -241,6 +257,44 @@ public class FileLockClusterView extends 
AbstractCamelClusterView {
     private void closeInternal() {
         releaseFileLock();
         closeLockFiles();
+        // Safe without stateLock: closeInternal() is only called from 
doStart() under contextStartLock,
+        // before the first scheduleTryLock(), so no concurrent acquireLock() 
writes lockPathKey here.
+        unregisterLockPath(lockPathKey);
+        lockPathKey = null;
+    }
+
+    /**
+     * The key of the lock file in the JVM-wide registry (see {@link 
#resolveLockPathKey()}). Resolving it reads the
+     * file system, so it runs through the cluster task executor with the 
cluster data task timeout and attempts, like
+     * the other file I/O of the leadership check: on a hung network mount the 
attempt fails instead of blocking the
+     * check (and the other checks of the service, which share its scheduler 
thread). Returns null if the check was
+     * interrupted.
+     */
+    private Path lockPathKey() throws ExecutionException, TimeoutException {
+        return clusterTaskExecutor.run(ThrowingHelper.wrapAsSupplier(new 
ThrowingSupplier<Path, Throwable>() {
+            @Override
+            public Path get() throws Throwable {
+                return resolveLockPathKey();
+            }
+        }));
+    }
+
+    /**
+     * The absolute path of the lock file, with the symbolic links of its 
directory resolved, so that two services
+     * configured with different paths to one directory share the key. If the 
directory cannot be resolved, the attempt
+     * fails rather than use a key that may differ from the key of another 
view (opening the lock file would fail
+     * without the directory anyway).
+     */
+    Path resolveLockPathKey() throws IOException {
+        Path path = leaderLockPath.toAbsolutePath().normalize();
+        Path parent = path.getParent();
+        return parent != null ? 
parent.toRealPath().resolve(path.getFileName()) : path;
+    }
+
+    private void unregisterLockPath(Path key) {
+        if (key != null) {
+            LOCK_PATHS.remove(key, this);
+        }
     }
 
     private void closeLockFiles() {
@@ -325,7 +379,7 @@ public class FileLockClusterView extends 
AbstractCamelClusterView {
                                 localMember.getUuid());
                         fireLeadershipChangedEvent((CamelClusterMember) null);
                         clusterLeaderInfoRef.set(null);
-                        held.release();
+                        held.release(this);
                     }
                     return;
                 }
@@ -394,6 +448,20 @@ public class FileLockClusterView extends 
AbstractCamelClusterView {
             // stopped while the cluster data was read: do not take the lock, 
even briefly
             return false;
         }
+        // resolved before the key is registered, so a timeout leaves nothing 
in LOCK_PATHS
+        Path key = lockPathKey();
+        if (key == null) {
+            // interrupted while resolving the key
+            return false;
+        }
+        FileLockClusterView holder = LOCK_PATHS.putIfAbsent(key, this);
+        if (holder != null) {
+            // another view of this JVM holds (or is trying) the lock: trying 
it here would fail anyway, and closing
+            // the channel afterwards would release the lock of that view at 
OS level (see LOCK_PATHS)
+            LOGGER.debug("Lock on file {} is held by another cluster view of 
this JVM (cluster-member-id={}), not trying it",
+                    leaderLockPath, holder.localMember.getUuid());
+            return false;
+        }
         RandomAccessFile newLockFile = null;
         RandomAccessFile newDataFile = null;
         FileLock newLock = null;
@@ -412,6 +480,7 @@ public class FileLockClusterView extends 
AbstractCamelClusterView {
                         lock = newLock;
                         leaderLockFile = newLockFile;
                         leaderDataFile = newDataFile;
+                        lockPathKey = key;
                         localMember.setStatus(ClusterMemberStatus.LEADER);
                         clusterLeaderInfoRef.set(null);
                         published = true;
@@ -430,6 +499,7 @@ public class FileLockClusterView extends 
AbstractCamelClusterView {
                 releaseFileLock(newLock);
                 closeFile(newLockFile);
                 closeFile(newDataFile);
+                unregisterLockPath(key);
             }
         }
         return published;
diff --git 
a/core/camel-core/src/test/java/org/apache/camel/component/file/cluster/FileLockClusterViewSameJvmTest.java
 
b/core/camel-core/src/test/java/org/apache/camel/component/file/cluster/FileLockClusterViewSameJvmTest.java
new file mode 100644
index 000000000000..edee24dc60a0
--- /dev/null
+++ 
b/core/camel-core/src/test/java/org/apache/camel/component/file/cluster/FileLockClusterViewSameJvmTest.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.file.cluster;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.camel.CamelContext;
+import org.apache.camel.cluster.CamelClusterView;
+import org.apache.camel.impl.DefaultCamelContext;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Two cluster views of one JVM on the same lock file (two CamelContexts with 
a FileLockClusterService on the same
+ * root). The file locks are locks of the process, and closing any channel of 
the lock file releases them all at OS
+ * level (see {@link java.nio.channels.FileLock}): the follower must not open 
(and close) a channel on the lock file
+ * while the leader of its JVM holds the lock, as a member in another JVM 
could then take the lock too.
+ */
+class FileLockClusterViewSameJvmTest {
+
+    private static final String NAMESPACE = "ns";
+
+    @TempDir
+    Path root;
+
+    private final List<CamelContext> contexts = new ArrayList<>();
+
+    @AfterEach
+    void stopContexts() {
+        contexts.forEach(CamelContext::stop);
+    }
+
+    @Test
+    void followerDoesNotOpenTheLockFileHeldInItsJvm() throws Exception {
+        CamelContext contextA = startContext();
+        FileLockClusterService serviceA = newService(new 
FileLockClusterService(), "A");
+        contextA.addService(serviceA);
+        CamelClusterView viewA = serviceA.getView(NAMESPACE);
+        await().atMost(10, TimeUnit.SECONDS).until(() -> 
viewA.getLocalMember().isLeader()
+                && 
FileLockClusterUtils.readClusterLeaderInfo(root.resolve(NAMESPACE + ".dat")) != 
null);
+
+        // the leadership checks of A stall (as in a long GC pause), so its 
heartbeat gets stale: this is the case where
+        // the lock must keep the members of other JVMs out
+        CountDownLatch stall = new CountDownLatch(1);
+        serviceA.getExecutor().execute(() -> {
+            try {
+                stall.await(30, TimeUnit.SECONDS);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        });
+
+        try {
+            CountingFileLockClusterService serviceB = newService(new 
CountingFileLockClusterService(), "B");
+            CamelContext contextB = startContext();
+            contextB.addService(serviceB);
+            CamelClusterView viewB = serviceB.getView(NAMESPACE);
+
+            // B finds the leader stale and tries to take the leadership. Once 
a later check has started, that attempt
+            // is over
+            await().atMost(10, TimeUnit.SECONDS).until(() -> 
serviceB.staleChecks.get() >= 2);
+
+            assertEquals(0, serviceB.lockFileOpens.get(),
+                    "the follower opened the lock file that a view of its JVM 
holds");
+            assertTrue(viewA.getLocalMember().isLeader());
+            assertFalse(viewB.getLocalMember().isLeader());
+        } finally {
+            stall.countDown();
+        }
+    }
+
+    @Test
+    void followerTakesOverWhenTheLeaderOfItsJvmStops() throws Exception {
+        CamelContext contextA = startContext();
+        FileLockClusterService serviceA = newService(new 
FileLockClusterService(), "A");
+        contextA.addService(serviceA);
+        CamelClusterView viewA = serviceA.getView(NAMESPACE);
+        await().atMost(10, TimeUnit.SECONDS).until(() -> 
viewA.getLocalMember().isLeader());
+
+        CountingFileLockClusterService serviceB = newService(new 
CountingFileLockClusterService(), "B");
+        CamelContext contextB = startContext();
+        contextB.addService(serviceB);
+        CamelClusterView viewB = serviceB.getView(NAMESPACE);
+        await().atMost(10, TimeUnit.SECONDS).until(() -> serviceB.checks.get() 
>= 2);
+        assertFalse(viewB.getLocalMember().isLeader());
+
+        // the leader releases the lock, so the follower of the same JVM can 
take it
+        contextA.stop();
+        await().atMost(10, TimeUnit.SECONDS).until(() -> 
viewB.getLocalMember().isLeader());
+        assertFalse(viewA.getLocalMember().isLeader());
+    }
+
+    @Test
+    void leadershipCheckGoesOnWhileResolvingTheLockPathHangs() throws 
Exception {
+        // resolving the real path of the lock file directory hangs, as on a 
hung NFS mount
+        CountDownLatch hang = new CountDownLatch(1);
+        AtomicInteger resolutions = new AtomicInteger();
+        FileLockClusterService service = newService(new 
FileLockClusterService() {
+            @Override
+            protected FileLockClusterView createView(String namespace) {
+                return new FileLockClusterView(this, namespace) {
+                    @Override
+                    Path resolveLockPathKey() throws IOException {
+                        resolutions.incrementAndGet();
+                        try {
+                            hang.await(30, TimeUnit.SECONDS);
+                        } catch (InterruptedException e) {
+                            Thread.currentThread().interrupt();
+                        }
+                        return super.resolveLockPathKey();
+                    }
+                };
+            }
+        }, "A");
+        service.setClusterDataTaskTimeout(200, TimeUnit.MILLISECONDS);
+        service.setClusterDataTaskMaxAttempts(1);
+        CamelContext context = startContext();
+        context.addService(service);
+        CamelClusterView view = service.getView(NAMESPACE);
+
+        try {
+            // each check times out resolving the key and schedules the next 
check, instead of blocking for good. Each
+            // hung resolution keeps one of the 5 threads of the cluster data 
task pool, so wait for fewer than 5
+            await().atMost(10, TimeUnit.SECONDS).until(() -> resolutions.get() 
>= 3);
+            assertFalse(view.getLocalMember().isLeader());
+        } finally {
+            hang.countDown();
+        }
+
+        // the checks that timed out registered nothing, so the view takes the 
lock once the key resolves
+        await().atMost(10, TimeUnit.SECONDS).until(() -> 
view.getLocalMember().isLeader());
+    }
+
+    @Test
+    void lockFileIsNotOpenedWhileTheKeyCannotBeResolved() throws Exception {
+        // a key from the unresolved path could differ from the key of another 
view of the JVM, so the view must not
+        // open the lock file without a resolved key
+        AtomicBoolean failing = new AtomicBoolean(true);
+        AtomicInteger resolutions = new AtomicInteger();
+        AtomicInteger lockFileOpens = new AtomicInteger();
+        FileLockClusterService service = newService(new 
FileLockClusterService() {
+            @Override
+            protected FileLockClusterView createView(String namespace) {
+                return new FileLockClusterView(this, namespace) {
+                    @Override
+                    Path resolveLockPathKey() throws IOException {
+                        resolutions.incrementAndGet();
+                        if (failing.get()) {
+                            throw new IOException("Simulated failure resolving 
the lock file directory");
+                        }
+                        return super.resolveLockPathKey();
+                    }
+
+                    @Override
+                    RandomAccessFile createRandomAccessFile(Path path) throws 
ExecutionException, TimeoutException {
+                        if (path.getFileName().toString().equals(NAMESPACE)) {
+                            lockFileOpens.incrementAndGet();
+                        }
+                        return super.createRandomAccessFile(path);
+                    }
+                };
+            }
+        }, "A");
+        service.setClusterDataTaskMaxAttempts(1);
+        CamelContext context = startContext();
+        context.addService(service);
+        CamelClusterView view = service.getView(NAMESPACE);
+
+        // a check that opened the lock file anyway would become leader and 
stop resolving the key
+        await().atMost(10, TimeUnit.SECONDS).until(() -> resolutions.get() >= 
2 || lockFileOpens.get() > 0);
+        assertEquals(0, lockFileOpens.get(), "the lock file was opened without 
a resolved key");
+        assertFalse(view.getLocalMember().isLeader());
+
+        failing.set(false);
+        await().atMost(10, TimeUnit.SECONDS).until(() -> 
view.getLocalMember().isLeader());
+    }
+
+    private CamelContext startContext() {
+        CamelContext context = new DefaultCamelContext();
+        context.disableJMX();
+        contexts.add(context);
+        context.start();
+        return context;
+    }
+
+    private <T extends FileLockClusterService> T newService(T service, String 
id) {
+        service.setId("node-" + id);
+        service.setRoot(root.toString());
+        service.setAcquireLockDelay(100, TimeUnit.MILLISECONDS);
+        service.setAcquireLockInterval(100, TimeUnit.MILLISECONDS);
+        service.setHeartbeatTimeoutMultiplier(2);
+        service.setClusterDataTaskTimeout(30, TimeUnit.SECONDS);
+        return service;
+    }
+
+    /**
+     * Counts the leadership checks of its view, the checks that found the 
leader stale, and the times the view opened
+     * the lock file.
+     */
+    private static final class CountingFileLockClusterService extends 
FileLockClusterService {
+        final AtomicInteger checks = new AtomicInteger();
+        final AtomicInteger staleChecks = new AtomicInteger();
+        final AtomicInteger lockFileOpens = new AtomicInteger();
+
+        @Override
+        protected FileLockClusterView createView(String namespace) {
+            return new FileLockClusterView(this, namespace) {
+                @Override
+                boolean isLeaderStale(FileLockClusterLeaderInfo info, 
FileLockClusterLeaderInfo previous) {
+                    checks.incrementAndGet();
+                    boolean stale = super.isLeaderStale(info, previous);
+                    if (stale) {
+                        staleChecks.incrementAndGet();
+                    }
+                    return stale;
+                }
+
+                @Override
+                RandomAccessFile createRandomAccessFile(Path path) throws 
ExecutionException, TimeoutException {
+                    if (path.getFileName().toString().equals(NAMESPACE)) {
+                        lockFileOpens.incrementAndGet();
+                    }
+                    return super.createRandomAccessFile(path);
+                }
+            };
+        }
+    }
+}

Reply via email to