This is an automated email from the ASF dual-hosted git repository.
Croway pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 76f305c64dc3 CAMEL-27018: camel-sftp - stopping the consumer does not
wait for a poll that hangs after the forced shutdown interrupted its download
76f305c64dc3 is described below
commit 76f305c64dc314884871a6449fbc06b97d4e569d
Author: croway <[email protected]>
AuthorDate: Mon Nov 16 10:08:32 2026 +0100
CAMEL-27018: camel-sftp - stopping the consumer does not wait for a poll
that hangs after the forced shutdown interrupted its download
The poll thread holds the SftpOperations lock for the whole retrieveFile().
When a download hangs because the server stopped answering, the forced
shutdown stops the consumer and the stop of the scheduler interrupts the
poll: the download fails, but with stepwise=true the retrieval then changes
back to the directory it started from, which waits on the same server with
the lock. RemoteFileConsumer.doStop() called disconnect(), whose
isConnected() waits for that lock without bound, so CamelContext.stop()
waited until the server answered again. A consumer on a shared scheduled
pool is interrupted once and always hung; one on its own pool is
interrupted a second time by the shutdown of the pool, which usually ends
the change of directory too.
RemoteFileConsumer.doStop() now calls
RemoteFileOperations.disconnectOnStop(),
as the producer does since CAMEL-27001: SftpOperations disconnects without
the lock once the shutdown is forced. For the other operations the default
disconnectOnStop() is what disconnect() did.
---
.../component/file/remote/RemoteFileConsumer.java | 20 +++-
.../file/remote/RemoteFileOperations.java | 13 +--
.../component/file/remote/SftpOperations.java | 6 +-
.../SftpConsumerStopDuringHungDownloadIT.java | 114 +++++++++++++++++++++
4 files changed, 144 insertions(+), 9 deletions(-)
diff --git
a/components/camel-ftp-common/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java
b/components/camel-ftp-common/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java
index 7f7d6d565a46..6b3134e8e799 100644
---
a/components/camel-ftp-common/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java
+++
b/components/camel-ftp-common/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java
@@ -214,7 +214,25 @@ public abstract class RemoteFileConsumer<T> extends
GenericFileConsumer<T> {
@Override
protected void doStop() throws Exception {
super.doStop();
- disconnect();
+ // the stop of the scheduler interrupts a poll in progress, which can
then still run an operation that the server
+ // does not answer, such as the change back to the directory it
started from
+ disconnectOnStop();
+ }
+
+ private void disconnectOnStop() {
+ // eager indicate we are no longer logged in
+ loggedIn = false;
+ // the next poll connects again, so it does not need a noop to check
the connection
+ sendNoopOnNextPoll = false;
+
+ try {
+ getOperations().disconnectOnStop();
+ } catch (Exception e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Error occurred while disconnecting from {} due: {}
This exception will be ignored.",
+ remoteServer(), e.getMessage(), e);
+ }
+ }
}
protected void disconnect() {
diff --git
a/components/camel-ftp-common/src/main/java/org/apache/camel/component/file/remote/RemoteFileOperations.java
b/components/camel-ftp-common/src/main/java/org/apache/camel/component/file/remote/RemoteFileOperations.java
index 5b297a63a525..4bbc5b361950 100644
---
a/components/camel-ftp-common/src/main/java/org/apache/camel/component/file/remote/RemoteFileOperations.java
+++
b/components/camel-ftp-common/src/main/java/org/apache/camel/component/file/remote/RemoteFileOperations.java
@@ -51,13 +51,14 @@ public interface RemoteFileOperations<T> extends
GenericFileOperations<T> {
void disconnect() throws GenericFileOperationFailedException;
/**
- * Disconnects from the remote server when the producer stops.
+ * Disconnects from the remote server when the producer or the consumer
stops.
* <p>
- * Another thread can still run an operation at that time. As long as the
shutdown of Camel is not forced, such as
- * when a producer cache evicts the producer, this method should wait for
it, as {@link #disconnect()} does: it can
- * be a healthy but slow upload. Once the shutdown is forced, the
operation is one that the shutdown gave up waiting
- * for, such as an upload that hangs as long as the server does not
answer: unlike {@link #disconnect()}, this
- * method should then not wait for it without bound, but disconnect
anyway, so that the operation fails.
+ * Another thread can still run an operation at that time: an upload of
the producer, or the poll of the consumer.
+ * As long as the shutdown of Camel is not forced, such as when a producer
cache evicts the producer, this method
+ * should wait for it, as {@link #disconnect()} does: it can be a healthy
but slow upload. Once the shutdown is
+ * forced, the operation is one that the shutdown gave up waiting for,
such as an upload that hangs as long as the
+ * server does not answer: unlike {@link #disconnect()}, this method
should then not wait for it without bound, but
+ * disconnect anyway, so that the operation fails.
* <p>
* The default implementation disconnects as {@link #disconnect()} does,
when connected.
*
diff --git
a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
index 458636b0ed91..8b2b5a012011 100644
---
a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
+++
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
@@ -645,7 +645,9 @@ public class SftpOperations
}
/**
- * Disconnects when the producer stops. Another thread can still run an
operation that holds the lock.
+ * Disconnects when the producer or the consumer stops. Another thread can
still run an operation that holds the
+ * lock: an upload of the producer, or the poll of the consumer, which the
stop of its scheduler interrupts, but
+ * which then changes back to the directory it started from (stepwise), on
a server that may not answer.
* <p>
* As long as the shutdown of Camel is not forced, this method waits for
that operation, as {@link #disconnect()}
* does: it can be a healthy but slow upload, when a producer cache evicts
the producer, when an application stops a
@@ -704,7 +706,7 @@ public class SftpOperations
}
String server = endpoint.getConfiguration().remoteServerInformation();
LOG.warn("The shutdown is forced: disconnecting from {} while another
thread still runs an operation, such as an"
- + " upload that hangs. The operation fails.",
+ + " upload or a download that hangs. The operation fails.",
server);
// JSch sends the close of the channel under the lock of the session
that a write to the socket holds: when the
// operation is blocked writing to a server that no longer reads (a
half-open connection), the disconnect waits
diff --git
a/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/sftp/integration/SftpConsumerStopDuringHungDownloadIT.java
b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/sftp/integration/SftpConsumerStopDuringHungDownloadIT.java
new file mode 100644
index 000000000000..a4083e7bd759
--- /dev/null
+++
b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/sftp/integration/SftpConsumerStopDuringHungDownloadIT.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.file.remote.sftp.integration;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.camel.builder.RouteBuilder;
+import
org.apache.camel.component.file.remote.sftp.integration.SftpProducerStopDuringHungUploadIT.HangingSftpEmbeddedService;
+import org.apache.camel.test.junit6.CamelTestSupport;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIf;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * An sftp download that hangs, because the server stopped answering, holds
the lock of {@code SftpOperations} in the
+ * thread of the poll. When Camel stops, the shutdown strategy forces the
shutdown after its timeout, and the stop of
+ * the consumer interrupts the poll: the download fails, but the poll then
changes back to the directory it started from
+ * (stepwise), which the server does not answer either, with the lock. The
stop of the consumer must then not wait for
+ * the lock, but disconnect, as the stop of the producer does (CAMEL-27001,
CAMEL-27018).
+ * <p>
+ * The stop of the scheduler interrupts the poll once when the consumer runs
on a shared scheduled thread pool (it
+ * cancels the poll), twice when the consumer has its own pool (it shuts the
pool down as well): the second interrupt
+ * often, but not always, ends the change of directory too.
+ */
+@EnabledIf(value =
"org.apache.camel.test.infra.ftp.services.embedded.SftpUtil#hasRequiredAlgorithms('src/test/resources/hostkey.pem')")
+class SftpConsumerStopDuringHungDownloadIT extends CamelTestSupport {
+
+ private static final int SHUTDOWN_TIMEOUT_SECONDS = 1;
+ // the shutdown is forced after its timeout: the stop should return well
within this limit
+ private static final int STOP_LIMIT_SECONDS = 20;
+
+ @RegisterExtension
+ HangingSftpEmbeddedService service = new HangingSftpEmbeddedService();
+
+ // threads that Camel does not manage, so that its shutdown does not
interrupt them
+ private final ExecutorService executor = Executors.newCachedThreadPool();
+ // a scheduled thread pool that the consumer shares, so its stop does not
shut it down
+ private final ScheduledExecutorService sharedPool =
Executors.newScheduledThreadPool(1);
+
+ @AfterEach
+ void releaseServer() {
+ service.release();
+ executor.shutdown();
+ sharedPool.shutdownNow();
+ }
+
+ @Test
+ void stopDoesNotWaitForDownloadWhenServerStopsAnswering() throws Exception
{
+ assertStopDoesNotWaitForDownload("");
+ }
+
+ @Test
+ void stopDoesNotWaitForDownloadOfConsumerOnSharedScheduledPool() throws
Exception {
+ context.getRegistry().bind("sharedPool", sharedPool);
+
assertStopDoesNotWaitForDownload("&scheduledExecutorService=#sharedPool");
+ }
+
+ private void assertStopDoesNotWaitForDownload(String options) throws
Exception {
+ Path download = service.getFtpRootDir().resolve("download");
+ Files.createDirectories(download);
+ Files.write(download.resolve("big.bin"), new byte[64 * 1024]);
+ service.hangTransfers();
+
+ context.getShutdownStrategy().setTimeout(SHUTDOWN_TIMEOUT_SECONDS);
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("sftp://admin@localhost:" + service.getPort() +
"/{{ftp.root.dir}}/download?password=admin"
+ + "&knownHostsFile=" + service.getKnownHostsFile() +
"&delay=100" + options)
+ .to("mock:result");
+ }
+ });
+ assertTrue(service.awaitHang(), "The download should reach the
server");
+
+ Future<?> stop = executor.submit(context::stop);
+ try {
+ stop.get(STOP_LIMIT_SECONDS, TimeUnit.SECONDS);
+ } catch (TimeoutException e) {
+ fail("CamelContext.stop() should return after the shutdown timeout
of " + SHUTDOWN_TIMEOUT_SECONDS
+ + " s, it still waits for the download after " +
STOP_LIMIT_SECONDS + " s");
+ } finally {
+ // so that a stop that waits for the download can complete
+ service.release();
+ }
+ // the download failed: the file was not routed
+ assertEquals(0, getMockEndpoint("mock:result").getReceivedCounter());
+ }
+}