This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new cab4c2a1e6ca CAMEL-27214: camel-ftp - the commit of a file on another
thread no longer runs on the connection while the consumer uses it (#30460)
cab4c2a1e6ca is described below
commit cab4c2a1e6ca5faf46dae7c9111c1af464a717af
Author: allthingssecurity <[email protected]>
AuthorDate: Fri Nov 20 13:35:10 2026 +0530
CAMEL-27214: camel-ftp - the commit of a file on another thread no longer
runs on the connection while the consumer uses it (#30460)
When the route hands the exchange over to another thread (such as seda),
the delete or
move of the file runs on that thread, on the connection of the consumer,
while the
consumer downloads the next file or walks the directories of the next poll.
FtpOperations
had no lock, so the commands and replies of the two threads mixed, and with
stepwise the
delete could run from the directory the walk was in, fail, and the file was
consumed
again.
Each operation of FtpOperations (and the connect of FtpsOperations) now
holds a lock, as
SftpOperations does, and the walk of the consumer holds it from the first
change of
directory to the change back. The disconnect on stop does not take it, so a
stop still
ends an operation the server does not answer.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../apache/camel/catalog/docs/ftp-component.adoc | 5 +
.../camel-ftp/src/main/docs/ftp-component.adoc | 5 +
.../camel/component/file/remote/FtpConsumer.java | 36 +-
.../camel/component/file/remote/FtpOperations.java | 624 ++++++++++++---------
.../component/file/remote/FtpsOperations.java | 71 +--
.../remote/FtpConsumerCommitOnOtherThreadTest.java | 173 ++++++
.../remote/RemoteFileConsumerSendNoopTest.java | 4 +-
7 files changed, 619 insertions(+), 299 deletions(-)
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/ftp-component.adoc
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/ftp-component.adoc
index ba1219fc9e16..085c526fbd12 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/ftp-component.adoc
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/ftp-component.adoc
@@ -271,6 +271,11 @@ It is only a single endpoint that does not support
concurrent consumers.
The FTP producer does *not* have this issue, it supports concurrency.
+When the route hands the file over to another thread, for example with
`to("seda:...")`, the file is deleted or
+moved (`delete`, `move`, `moveFailed`) on that thread, when it is done with
the file, on the connection of the consumer.
+The ftp and ftps consumers run one operation at a time on their connection, so
such a delete or move waits until the
+consumer is done with its current operation (such as the download of the next
file), and the other way around.
+
=== A server that stops answering
When the FTP/FTPS server stops answering on an open connection (it hangs, or a
firewall drops the connection without
diff --git a/components/camel-ftp/src/main/docs/ftp-component.adoc
b/components/camel-ftp/src/main/docs/ftp-component.adoc
index ba1219fc9e16..085c526fbd12 100644
--- a/components/camel-ftp/src/main/docs/ftp-component.adoc
+++ b/components/camel-ftp/src/main/docs/ftp-component.adoc
@@ -271,6 +271,11 @@ It is only a single endpoint that does not support
concurrent consumers.
The FTP producer does *not* have this issue, it supports concurrency.
+When the route hands the file over to another thread, for example with
`to("seda:...")`, the file is deleted or
+moved (`delete`, `move`, `moveFailed`) on that thread, when it is done with
the file, on the connection of the consumer.
+The ftp and ftps consumers run one operation at a time on their connection, so
such a delete or move waits until the
+consumer is done with its current operation (such as the download of the next
file), and the other way around.
+
=== A server that stops answering
When the FTP/FTPS server stops answering on an open connection (it hangs, or a
firewall drops the connection without
diff --git
a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
index 56ebf5d92ddb..2a7cf6dc4a68 100644
---
a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
+++
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
@@ -20,6 +20,7 @@ import java.net.SocketTimeoutException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Supplier;
import org.apache.camel.Exchange;
@@ -96,22 +97,31 @@ public class FtpConsumer extends
RemoteFileConsumer<FTPFile> {
@Override
protected boolean pollDirectory(Exchange dynamic, String fileName,
List<GenericFile<FTPFile>> fileList, int depth) {
- String currentDir = null;
- if (isStepwise()) {
- // must remember current dir so we stay in that directory after the
- // poll
- currentDir = operations.getCurrentDirectory();
- }
+ // the walk through the directories changes the current directory
(stepwise), so it holds the lock of the
+ // operations: a file of the previous poll that is committed meanwhile
on another thread (such as a seda
+ // consumer) would else be deleted or moved from the directory the
walk is in
+ ReentrantLock lock = getOperations().lock;
+ lock.lock();
+ try {
+ String currentDir = null;
+ if (isStepwise()) {
+ // must remember current dir so we stay in that directory
after the
+ // poll
+ currentDir = operations.getCurrentDirectory();
+ }
- // strip trailing slash
- fileName = FileUtil.stripTrailingSeparator(fileName);
+ // strip trailing slash
+ fileName = FileUtil.stripTrailingSeparator(fileName);
- boolean answer = doPollDirectory(dynamic, fileName, null, fileList,
depth);
- if (currentDir != null) {
- operations.changeCurrentDirectory(currentDir);
- }
+ boolean answer = doPollDirectory(dynamic, fileName, null,
fileList, depth);
+ if (currentDir != null) {
+ operations.changeCurrentDirectory(currentDir);
+ }
- return answer;
+ return answer;
+ } finally {
+ lock.unlock();
+ }
}
protected boolean pollSubDirectory(
diff --git
a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
index b49b9c98b5ef..47514dfa9b8b 100644
---
a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
+++
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
@@ -28,6 +28,7 @@ import java.time.Duration;
import java.util.Iterator;
import java.util.Objects;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.locks.ReentrantLock;
import org.apache.camel.Exchange;
import org.apache.camel.InvalidPayloadException;
@@ -68,6 +69,13 @@ public class FtpOperations implements
RemoteFileOperations<FTPFile> {
protected final FTPClientConfig clientConfig;
protected FtpEndpoint<FTPFile> endpoint;
protected FtpClientActivityListener clientActivityListener;
+ /**
+ * An FTP connection runs one command at a time, and the operations of a
consumer are not only used by the thread
+ * that polls: the commit or rollback of a file (delete, move, rename)
runs on the thread that completes the
+ * exchange, such as a seda consumer when the route sends the file to
seda, while the consumer downloads the next
+ * file. Each operation holds this lock, so that the commands and replies
of two operations do not mix.
+ */
+ protected final ReentrantLock lock = new ReentrantLock();
private static class TaskPayload {
final RemoteFileConfiguration configuration;
@@ -107,13 +115,18 @@ public class FtpOperations implements
RemoteFileOperations<FTPFile> {
@Override
public boolean connect(RemoteFileConfiguration configuration, Exchange
exchange)
throws GenericFileOperationFailedException {
- client.setCopyStreamListener(clientActivityListener);
-
+ lock.lock();
try {
- return doConnect(configuration, exchange);
- } catch (GenericFileOperationFailedException e) {
-
clientActivityListener.onGeneralError(endpoint.getConfiguration().remoteServerInformation(),
e.getMessage());
- throw e;
+ client.setCopyStreamListener(clientActivityListener);
+
+ try {
+ return doConnect(configuration, exchange);
+ } catch (GenericFileOperationFailedException e) {
+
clientActivityListener.onGeneralError(endpoint.getConfiguration().remoteServerInformation(),
e.getMessage());
+ throw e;
+ }
+ } finally {
+ lock.unlock();
}
}
@@ -314,11 +327,32 @@ public class FtpOperations implements
RemoteFileOperations<FTPFile> {
@Override
public void disconnect() throws GenericFileOperationFailedException {
+ lock.lock();
try {
- doDisconnect();
- } catch (GenericFileOperationFailedException e) {
-
clientActivityListener.onGeneralError(endpoint.getConfiguration().remoteServerInformation(),
e.getMessage());
- throw e;
+ try {
+ doDisconnect();
+ } catch (GenericFileOperationFailedException e) {
+
clientActivityListener.onGeneralError(endpoint.getConfiguration().remoteServerInformation(),
e.getMessage());
+ throw e;
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * Disconnects without the lock that the operations hold: when the stop
interrupts a poll, the poll can still run an
+ * operation that the server does not answer, and closing the connection
ends it, instead of waiting for it.
+ */
+ @Override
+ public void disconnectOnStop() throws GenericFileOperationFailedException {
+ if (isConnected()) {
+ try {
+ doDisconnect();
+ } catch (GenericFileOperationFailedException e) {
+
clientActivityListener.onGeneralError(endpoint.getConfiguration().remoteServerInformation(),
e.getMessage());
+ throw e;
+ }
}
}
@@ -373,150 +407,178 @@ public class FtpOperations implements
RemoteFileOperations<FTPFile> {
@Override
public boolean deleteFile(String name) throws
GenericFileOperationFailedException {
- log.debug("Deleting file: {}", name);
- checkName(name);
+ lock.lock();
+ try {
+ log.debug("Deleting file: {}", name);
+ checkName(name);
- boolean result;
- String target = name;
- String currentDir = null;
+ boolean result;
+ String target = name;
+ String currentDir = null;
- try {
- reconnectIfNecessary(null);
- if (endpoint.getConfiguration().isStepwise()) {
- // remember current directory
- currentDir = getCurrentDirectory();
- target = FileUtil.stripPath(name);
+ try {
+ reconnectIfNecessary(null);
+ if (endpoint.getConfiguration().isStepwise()) {
+ // remember current directory
+ currentDir = getCurrentDirectory();
+ target = FileUtil.stripPath(name);
+
+ try {
+ changeCurrentDirectory(FileUtil.onlyPath(name));
+ } catch (GenericFileOperationFailedException e) {
+ // we could not change directory, try to change back
before
+ changeCurrentDirectory(currentDir);
+ throw e;
+ }
+ }
- try {
- changeCurrentDirectory(FileUtil.onlyPath(name));
- } catch (GenericFileOperationFailedException e) {
- // we could not change directory, try to change back before
+ // delete the file
+ log.trace("Client deleteFile: {}", target);
+ result = client.deleteFile(target);
+
+ // change back to previous directory
+ if (currentDir != null) {
changeCurrentDirectory(currentDir);
- throw e;
}
- }
-
- // delete the file
- log.trace("Client deleteFile: {}", target);
- result = client.deleteFile(target);
- // change back to previous directory
- if (currentDir != null) {
- changeCurrentDirectory(currentDir);
+ } catch (IOException e) {
+ throw new GenericFileOperationFailedException(
+ client.getReplyCode(), client.getReplyString(),
e.getMessage(), e);
}
- } catch (IOException e) {
- throw new
GenericFileOperationFailedException(client.getReplyCode(),
client.getReplyString(), e.getMessage(), e);
+ return result;
+ } finally {
+ lock.unlock();
}
-
- return result;
}
@Override
public boolean renameFile(String from, String to) throws
GenericFileOperationFailedException {
- log.debug("Renaming file: {} to: {}", from, to);
- checkName(from);
- checkName(to);
+ lock.lock();
try {
- reconnectIfNecessary(null);
- return client.rename(from, to);
- } catch (IOException e) {
- throw new
GenericFileOperationFailedException(client.getReplyCode(),
client.getReplyString(), e.getMessage(), e);
+ log.debug("Renaming file: {} to: {}", from, to);
+ checkName(from);
+ checkName(to);
+ try {
+ reconnectIfNecessary(null);
+ return client.rename(from, to);
+ } catch (IOException e) {
+ throw new GenericFileOperationFailedException(
+ client.getReplyCode(), client.getReplyString(),
e.getMessage(), e);
+ }
+ } finally {
+ lock.unlock();
}
}
@Override
public boolean buildDirectory(String directory, boolean absolute) throws
GenericFileOperationFailedException {
- // must normalize directory first
- directory = endpoint.getConfiguration().normalizePath(directory);
- checkName(directory);
-
- log.trace("buildDirectory({})", directory);
+ lock.lock();
try {
- String originalDirectory = client.printWorkingDirectory();
+ // must normalize directory first
+ directory = endpoint.getConfiguration().normalizePath(directory);
+ checkName(directory);
- boolean success;
- // maybe the full directory already exists
- success = client.changeWorkingDirectory(directory);
- if (!success) {
- log.trace("Trying to build remote directory: {}", directory);
- success = client.makeDirectory(directory);
+ log.trace("buildDirectory({})", directory);
+ try {
+ String originalDirectory = client.printWorkingDirectory();
+
+ boolean success;
+ // maybe the full directory already exists
+ success = client.changeWorkingDirectory(directory);
if (!success) {
- // we are here if the server side doesn't create
- // intermediate folders so create the folder one by one
- success = buildDirectoryChunks(directory);
+ log.trace("Trying to build remote directory: {}",
directory);
+ success = client.makeDirectory(directory);
+ if (!success) {
+ // we are here if the server side doesn't create
+ // intermediate folders so create the folder one by one
+ success = buildDirectoryChunks(directory);
+ }
}
- }
- // change back to original directory
- if (originalDirectory != null) {
- changeCurrentDirectory(originalDirectory);
- }
+ // change back to original directory
+ if (originalDirectory != null) {
+ changeCurrentDirectory(originalDirectory);
+ }
- return success;
- } catch (IOException e) {
- throw new
GenericFileOperationFailedException(client.getReplyCode(),
client.getReplyString(), e.getMessage(), e);
+ return success;
+ } catch (IOException e) {
+ throw new GenericFileOperationFailedException(
+ client.getReplyCode(), client.getReplyString(),
e.getMessage(), e);
+ }
+ } finally {
+ lock.unlock();
}
}
@Override
public boolean retrieveFile(String name, Exchange exchange, long size)
throws GenericFileOperationFailedException {
- checkName(name);
- // store the name of the file to download on the listener
- clientActivityListener.setDownload(true);
- clientActivityListener.setRemoteFileName(name);
- clientActivityListener.setRemoteFileSize(size);
-
clientActivityListener.onBeginDownloading(endpoint.getConfiguration().remoteServerInformation(),
name);
-
- boolean answer;
+ lock.lock();
try {
- log.trace("retrieveFile({})", name);
- if
(org.apache.camel.util.ObjectHelper.isNotEmpty(endpoint.getLocalWorkDirectory()))
{
- // local work directory is configured so we should store file
- // content as files in this local directory
- answer = retrieveFileToFileInLocalWorkDirectory(name,
exchange, endpoint.isResumeDownload());
- } else {
- // store file content directory as stream on the body
- answer = retrieveFileToStreamInBody(name, exchange);
+ checkName(name);
+ // store the name of the file to download on the listener
+ clientActivityListener.setDownload(true);
+ clientActivityListener.setRemoteFileName(name);
+ clientActivityListener.setRemoteFileSize(size);
+
clientActivityListener.onBeginDownloading(endpoint.getConfiguration().remoteServerInformation(),
name);
+
+ boolean answer;
+ try {
+ log.trace("retrieveFile({})", name);
+ if
(org.apache.camel.util.ObjectHelper.isNotEmpty(endpoint.getLocalWorkDirectory()))
{
+ // local work directory is configured so we should store
file
+ // content as files in this local directory
+ answer = retrieveFileToFileInLocalWorkDirectory(name,
exchange, endpoint.isResumeDownload());
+ } else {
+ // store file content directory as stream on the body
+ answer = retrieveFileToStreamInBody(name, exchange);
+ }
+ } catch (GenericFileOperationFailedException e) {
+
clientActivityListener.onGeneralError(endpoint.getConfiguration().remoteServerInformation(),
e.getMessage());
+ throw e;
}
- } catch (GenericFileOperationFailedException e) {
-
clientActivityListener.onGeneralError(endpoint.getConfiguration().remoteServerInformation(),
e.getMessage());
- throw e;
- }
- if (answer) {
-
clientActivityListener.onDownloadComplete(endpoint.getConfiguration().remoteServerInformation(),
name);
+ if (answer) {
+
clientActivityListener.onDownloadComplete(endpoint.getConfiguration().remoteServerInformation(),
name);
+ }
+ return answer;
+ } finally {
+ lock.unlock();
}
- return answer;
}
@Override
public void releaseRetrievedFileResources(Exchange exchange) throws
GenericFileOperationFailedException {
- InputStream is =
exchange.getIn().getHeader(FtpConstants.REMOTE_FILE_INPUT_STREAM,
InputStream.class);
+ lock.lock();
+ try {
+ InputStream is =
exchange.getIn().getHeader(FtpConstants.REMOTE_FILE_INPUT_STREAM,
InputStream.class);
- if (is != null) {
- try {
- IOHelper.close(is);
- boolean ok = client.completePendingCommand();
- if (!ok) {
- throw new GenericFileOperationFailedException(
- client.getReplyCode(), client.getReplyString());
- }
- } catch (SocketTimeoutException e) {
- // the server did not answer in time at the end of the
streamed transfer: a reply that comes late would be
- // read as the reply to the next command, so the connection
cannot be used any more. It is closed
- // without waiting on it, and the consumer leaves the rest of
the batch to the next poll, which connects
- // again
- GenericFileOperationFailedException failure = new
GenericFileOperationFailedException(e.getMessage(), e);
+ if (is != null) {
try {
- forceDisconnect();
- } catch (GenericFileOperationFailedException fe) {
- failure.addSuppressed(fe);
+ IOHelper.close(is);
+ boolean ok = client.completePendingCommand();
+ if (!ok) {
+ throw new GenericFileOperationFailedException(
+ client.getReplyCode(),
client.getReplyString());
+ }
+ } catch (SocketTimeoutException e) {
+ // the server did not answer in time at the end of the
streamed transfer: a reply that comes late would be
+ // read as the reply to the next command, so the
connection cannot be used any more. It is closed
+ // without waiting on it, and the consumer leaves the rest
of the batch to the next poll, which connects
+ // again
+ GenericFileOperationFailedException failure = new
GenericFileOperationFailedException(e.getMessage(), e);
+ try {
+ forceDisconnect();
+ } catch (GenericFileOperationFailedException fe) {
+ failure.addSuppressed(fe);
+ }
+ throw failure;
+ } catch (IOException e) {
+ throw new
GenericFileOperationFailedException(e.getMessage(), e);
}
- throw failure;
- } catch (IOException e) {
- throw new GenericFileOperationFailedException(e.getMessage(),
e);
}
+ } finally {
+ lock.unlock();
}
}
@@ -745,63 +807,68 @@ public class FtpOperations implements
RemoteFileOperations<FTPFile> {
@Override
public boolean storeFile(String name, Exchange exchange, long size) throws
GenericFileOperationFailedException {
- // must normalize name first
- name = endpoint.getConfiguration().normalizePath(name);
- // before the listener and the client, so nothing is sent for a name
that is refused
- checkName(name);
+ lock.lock();
+ try {
+ // must normalize name first
+ name = endpoint.getConfiguration().normalizePath(name);
+ // before the listener and the client, so nothing is sent for a
name that is refused
+ checkName(name);
- // store the name of the file to upload on the listener
- clientActivityListener.setDownload(false);
- clientActivityListener.setRemoteFileName(name);
- clientActivityListener.setRemoteFileSize(size);
-
clientActivityListener.onBeginUploading(endpoint.getConfiguration().remoteServerInformation(),
name);
+ // store the name of the file to upload on the listener
+ clientActivityListener.setDownload(false);
+ clientActivityListener.setRemoteFileName(name);
+ clientActivityListener.setRemoteFileSize(size);
+
clientActivityListener.onBeginUploading(endpoint.getConfiguration().remoteServerInformation(),
name);
- log.trace("storeFile({})", name);
+ log.trace("storeFile({})", name);
- boolean answer;
- String currentDir = null;
- String path = FileUtil.onlyPath(name);
- String targetName = name;
+ boolean answer;
+ String currentDir = null;
+ String path = FileUtil.onlyPath(name);
+ String targetName = name;
- try {
- if (path != null && endpoint.getConfiguration().isStepwise()) {
- // must remember current dir so we stay in that directory after
- // the write
- currentDir = getCurrentDirectory();
+ try {
+ if (path != null && endpoint.getConfiguration().isStepwise()) {
+ // must remember current dir so we stay in that directory
after
+ // the write
+ currentDir = getCurrentDirectory();
- // change to path of name
- changeCurrentDirectory(path);
+ // change to path of name
+ changeCurrentDirectory(path);
- // the target name should be without path, as we have changed
- // directory
- targetName = FileUtil.stripPath(name);
- }
+ // the target name should be without path, as we have
changed
+ // directory
+ targetName = FileUtil.stripPath(name);
+ }
- // store the file
- answer = doStoreFile(name, targetName, exchange);
+ // store the file
+ answer = doStoreFile(name, targetName, exchange);
- // change back to current directory if we changed directory
- if (currentDir != null) {
- changeCurrentDirectory(currentDir);
+ // change back to current directory if we changed directory
+ if (currentDir != null) {
+ changeCurrentDirectory(currentDir);
+ }
+ } catch (GenericFileOperationFailedException e) {
+
clientActivityListener.onGeneralError(endpoint.getConfiguration().remoteServerInformation(),
e.getMessage());
+ throw e;
}
- } catch (GenericFileOperationFailedException e) {
-
clientActivityListener.onGeneralError(endpoint.getConfiguration().remoteServerInformation(),
e.getMessage());
- throw e;
- }
- if (answer) {
-
clientActivityListener.onUploadComplete(endpoint.getConfiguration().remoteServerInformation(),
name);
- }
+ if (answer) {
+
clientActivityListener.onUploadComplete(endpoint.getConfiguration().remoteServerInformation(),
name);
+ }
- if (!answer) {
- Integer code =
exchange.getIn().getHeader(FtpConstants.FTP_REPLY_CODE, Integer.class);
- String status =
exchange.getIn().getHeader(FtpConstants.FTP_REPLY_STRING, String.class);
- if (code != null && status != null) {
- throw new GenericFileOperationFailedException(code, status,
"Error writing file [" + targetName + "]");
+ if (!answer) {
+ Integer code =
exchange.getIn().getHeader(FtpConstants.FTP_REPLY_CODE, Integer.class);
+ String status =
exchange.getIn().getHeader(FtpConstants.FTP_REPLY_STRING, String.class);
+ if (code != null && status != null) {
+ throw new GenericFileOperationFailedException(code,
status, "Error writing file [" + targetName + "]");
+ }
}
- }
- return answer;
+ return answer;
+ } finally {
+ lock.unlock();
+ }
}
private boolean doStoreFile(String name, String targetName, Exchange
exchange) throws GenericFileOperationFailedException {
@@ -892,48 +959,60 @@ public class FtpOperations implements
RemoteFileOperations<FTPFile> {
@Override
public boolean storeFileDirectly(String name, String payload) throws
GenericFileOperationFailedException {
- checkName(name);
- ByteArrayInputStream bis = new
ByteArrayInputStream(payload.getBytes());
+ lock.lock();
try {
- return client.storeFile(name, bis);
- } catch (IOException e) {
- throw new
GenericFileOperationFailedException(client.getReplyCode(),
client.getReplyString(), e.getMessage(), e);
+ checkName(name);
+ ByteArrayInputStream bis = new
ByteArrayInputStream(payload.getBytes());
+ try {
+ return client.storeFile(name, bis);
+ } catch (IOException e) {
+ throw new GenericFileOperationFailedException(
+ client.getReplyCode(), client.getReplyString(),
e.getMessage(), e);
+ } finally {
+ IOHelper.close(bis);
+ }
} finally {
- IOHelper.close(bis);
+ lock.unlock();
}
}
@Override
public boolean existsFile(String name) throws
GenericFileOperationFailedException {
- log.trace("existsFile({})", name);
- checkName(name);
- if (endpoint.isFastExistsCheck()) {
- return fastExistsFile(name);
- }
- // check whether a file already exists
- String directory = FileUtil.onlyPath(name);
- String onlyName = FileUtil.stripPath(name);
+ lock.lock();
try {
- String[] names;
- if (directory != null) {
- names = client.listNames(directory);
- } else {
- names = client.listNames();
+ log.trace("existsFile({})", name);
+ checkName(name);
+ if (endpoint.isFastExistsCheck()) {
+ return fastExistsFile(name);
}
- // can return either null or an empty list depending on FTP servers
- if (names == null) {
- return false;
- }
- for (String existing : names) {
- log.trace("Existing file: {}, target file: {}", existing,
name);
- existing = FileUtil.stripPath(existing);
- if (existing != null && existing.equals(onlyName)) {
- return true;
+ // check whether a file already exists
+ String directory = FileUtil.onlyPath(name);
+ String onlyName = FileUtil.stripPath(name);
+ try {
+ String[] names;
+ if (directory != null) {
+ names = client.listNames(directory);
+ } else {
+ names = client.listNames();
+ }
+ // can return either null or an empty list depending on FTP
servers
+ if (names == null) {
+ return false;
}
+ for (String existing : names) {
+ log.trace("Existing file: {}, target file: {}", existing,
name);
+ existing = FileUtil.stripPath(existing);
+ if (existing != null && existing.equals(onlyName)) {
+ return true;
+ }
+ }
+ return false;
+ } catch (IOException e) {
+ throw new GenericFileOperationFailedException(
+ client.getReplyCode(), client.getReplyString(),
e.getMessage(), e);
}
- return false;
- } catch (IOException e) {
- throw new
GenericFileOperationFailedException(client.getReplyCode(),
client.getReplyString(), e.getMessage(), e);
+ } finally {
+ lock.unlock();
}
}
@@ -953,54 +1032,65 @@ public class FtpOperations implements
RemoteFileOperations<FTPFile> {
@Override
public String getCurrentDirectory() throws
GenericFileOperationFailedException {
- log.trace("getCurrentDirectory()");
+ lock.lock();
try {
- String answer = client.printWorkingDirectory();
- log.trace("Current dir: {}", answer);
- return answer;
- } catch (IOException e) {
- throw new
GenericFileOperationFailedException(client.getReplyCode(),
client.getReplyString(), e.getMessage(), e);
+ log.trace("getCurrentDirectory()");
+ try {
+ String answer = client.printWorkingDirectory();
+ log.trace("Current dir: {}", answer);
+ return answer;
+ } catch (IOException e) {
+ throw new GenericFileOperationFailedException(
+ client.getReplyCode(), client.getReplyString(),
e.getMessage(), e);
+ }
+ } finally {
+ lock.unlock();
}
}
@Override
public void changeCurrentDirectory(String path) throws
GenericFileOperationFailedException {
- log.trace("changeCurrentDirectory({})", path);
- if (org.apache.camel.util.ObjectHelper.isEmpty(path)) {
- return;
- }
- checkName(path);
+ lock.lock();
+ try {
+ log.trace("changeCurrentDirectory({})", path);
+ if (org.apache.camel.util.ObjectHelper.isEmpty(path)) {
+ return;
+ }
+ checkName(path);
- // must compact path so FTP server can traverse correctly
- // use the ftp utils implementation of the compact path
- path = FtpUtils.compactPath(path);
+ // must compact path so FTP server can traverse correctly
+ // use the ftp utils implementation of the compact path
+ path = FtpUtils.compactPath(path);
- // not stepwise should change directory in one operation
- if (!endpoint.getConfiguration().isStepwise()) {
- doChangeDirectory(path);
- return;
- }
+ // not stepwise should change directory in one operation
+ if (!endpoint.getConfiguration().isStepwise()) {
+ doChangeDirectory(path);
+ return;
+ }
- // if it starts with the root path then a little special handling for
- // that
- if (FileUtil.hasLeadingSeparator(path)) {
- // change to root path
- doChangeDirectory(path.substring(0, 1));
- path = path.substring(1);
- }
+ // if it starts with the root path then a little special handling
for
+ // that
+ if (FileUtil.hasLeadingSeparator(path)) {
+ // change to root path
+ doChangeDirectory(path.substring(0, 1));
+ path = path.substring(1);
+ }
- // split into multiple dirs
- final String[] dirs = path.split("/|\\\\");
+ // split into multiple dirs
+ final String[] dirs = path.split("/|\\\\");
- if (dirs == null || dirs.length == 0) {
- // path was just a relative single path
- doChangeDirectory(path);
- return;
- }
+ if (dirs == null || dirs.length == 0) {
+ // path was just a relative single path
+ doChangeDirectory(path);
+ return;
+ }
- // there are multiple dirs so do this in chunks
- for (String dir : dirs) {
- doChangeDirectory(dir);
+ // there are multiple dirs so do this in chunks
+ for (String dir : dirs) {
+ doChangeDirectory(dir);
+ }
+ } finally {
+ lock.unlock();
}
}
@@ -1044,41 +1134,59 @@ public class FtpOperations implements
RemoteFileOperations<FTPFile> {
@Override
public void changeToParentDirectory() throws
GenericFileOperationFailedException {
+ lock.lock();
try {
- client.changeToParentDirectory();
- } catch (IOException e) {
- throw new
GenericFileOperationFailedException(client.getReplyCode(),
client.getReplyString(), e.getMessage(), e);
+ try {
+ client.changeToParentDirectory();
+ } catch (IOException e) {
+ throw new GenericFileOperationFailedException(
+ client.getReplyCode(), client.getReplyString(),
e.getMessage(), e);
+ }
+ } finally {
+ lock.unlock();
}
}
@Override
public FTPFile[] listFiles() throws GenericFileOperationFailedException {
- log.trace("Listing remote files");
-
clientActivityListener.onScanningForFiles(endpoint.remoteServerInformation(),
null);
+ lock.lock();
try {
- return checkListing(client.listFiles(), "the current directory");
- } catch (IOException e) {
-
clientActivityListener.onGeneralError(endpoint.getConfiguration().remoteServerInformation(),
e.getMessage());
- throw new
GenericFileOperationFailedException(client.getReplyCode(),
client.getReplyString(), e.getMessage(), e);
+ log.trace("Listing remote files");
+
clientActivityListener.onScanningForFiles(endpoint.remoteServerInformation(),
null);
+ try {
+ return checkListing(client.listFiles(), "the current
directory");
+ } catch (IOException e) {
+
clientActivityListener.onGeneralError(endpoint.getConfiguration().remoteServerInformation(),
e.getMessage());
+ throw new GenericFileOperationFailedException(
+ client.getReplyCode(), client.getReplyString(),
e.getMessage(), e);
+ }
+ } finally {
+ lock.unlock();
}
}
@Override
public FTPFile[] listFiles(String path) throws
GenericFileOperationFailedException {
- log.trace("Listing remote files from path {}", path);
-
clientActivityListener.onScanningForFiles(endpoint.remoteServerInformation(),
path);
+ lock.lock();
+ try {
+ log.trace("Listing remote files from path {}", path);
+
clientActivityListener.onScanningForFiles(endpoint.remoteServerInformation(),
path);
- // use current directory if path not given
- if (org.apache.camel.util.ObjectHelper.isEmpty(path)) {
- path = ".";
- }
- checkName(path);
+ // use current directory if path not given
+ if (org.apache.camel.util.ObjectHelper.isEmpty(path)) {
+ path = ".";
+ }
+ checkName(path);
- try {
- return checkListing(client.listFiles(path), "directory: " + path);
- } catch (IOException e) {
-
clientActivityListener.onGeneralError(endpoint.getConfiguration().remoteServerInformation(),
e.getMessage());
- throw new
GenericFileOperationFailedException(client.getReplyCode(),
client.getReplyString(), e.getMessage(), e);
+ try {
+ return checkListing(client.listFiles(path), "directory: " +
path);
+ } catch (IOException e) {
+
clientActivityListener.onGeneralError(endpoint.getConfiguration().remoteServerInformation(),
e.getMessage());
+ throw new GenericFileOperationFailedException(
+ client.getReplyCode(), client.getReplyString(),
e.getMessage(), e);
+ }
+ } finally {
+ lock.unlock();
}
}
@@ -1106,21 +1214,33 @@ public class FtpOperations implements
RemoteFileOperations<FTPFile> {
@Override
public boolean sendNoop() throws GenericFileOperationFailedException {
- log.trace("sendNoOp");
+ lock.lock();
try {
- return client.sendNoOp();
- } catch (IOException e) {
- throw new
GenericFileOperationFailedException(client.getReplyCode(),
client.getReplyString(), e.getMessage(), e);
+ log.trace("sendNoOp");
+ try {
+ return client.sendNoOp();
+ } catch (IOException e) {
+ throw new GenericFileOperationFailedException(
+ client.getReplyCode(), client.getReplyString(),
e.getMessage(), e);
+ }
+ } finally {
+ lock.unlock();
}
}
@Override
public boolean sendSiteCommand(String command) throws
GenericFileOperationFailedException {
- log.trace("sendSiteCommand({})", command);
+ lock.lock();
try {
- return client.sendSiteCommand(command);
- } catch (IOException e) {
- throw new
GenericFileOperationFailedException(client.getReplyCode(),
client.getReplyString(), e.getMessage(), e);
+ log.trace("sendSiteCommand({})", command);
+ try {
+ return client.sendSiteCommand(command);
+ } catch (IOException e) {
+ throw new GenericFileOperationFailedException(
+ client.getReplyCode(), client.getReplyString(),
e.getMessage(), e);
+ }
+ } finally {
+ lock.unlock();
}
}
diff --git
a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpsOperations.java
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpsOperations.java
index 4c77fb87d00d..2eaaa5a86c1a 100644
---
a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpsOperations.java
+++
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpsOperations.java
@@ -36,45 +36,50 @@ public class FtpsOperations extends FtpOperations {
@Override
public boolean connect(RemoteFileConfiguration configuration, Exchange
exchange)
throws GenericFileOperationFailedException {
- boolean answer = super.connect(configuration, exchange);
+ lock.lock();
+ try {
+ boolean answer = super.connect(configuration, exchange);
- FtpsConfiguration config = (FtpsConfiguration) configuration;
- if (answer) {
- try {
- String execProt = config.getExecProt();
- Long execPbsz = config.getExecPbsz();
- // use default values for prop and pbsz, unless told to not do
- // so
- if (!config.isDisableSecureDataChannelDefaults()) {
- if (ObjectHelper.isEmpty(execProt)) {
- execProt = "P";
+ FtpsConfiguration config = (FtpsConfiguration) configuration;
+ if (answer) {
+ try {
+ String execProt = config.getExecProt();
+ Long execPbsz = config.getExecPbsz();
+ // use default values for prop and pbsz, unless told to
not do
+ // so
+ if (!config.isDisableSecureDataChannelDefaults()) {
+ if (ObjectHelper.isEmpty(execProt)) {
+ execProt = "P";
+ }
+ if (ObjectHelper.isEmpty(execPbsz)) {
+ execPbsz = 0L;
+ }
}
- if (ObjectHelper.isEmpty(execPbsz)) {
- execPbsz = 0L;
- }
- }
- if (execPbsz != null) {
- log.debug("FTPClient initializing with execPbsz={}",
execPbsz);
- getFtpClient().execPBSZ(execPbsz);
- }
- if (execProt != null) {
- log.debug("FTPClient initializing with execProt={}",
execProt);
- getFtpClient().execPROT(execProt);
- }
- } catch (IOException e) {
- throw new GenericFileOperationFailedException(
- client.getReplyCode(), client.getReplyString(),
e.getMessage(), e);
- } finally {
- if (exchange != null) {
- // store client reply information after the operation
- exchange.getIn().setHeader(FtpConstants.FTP_REPLY_CODE,
client.getReplyCode());
- exchange.getIn().setHeader(FtpConstants.FTP_REPLY_STRING,
client.getReplyString());
+ if (execPbsz != null) {
+ log.debug("FTPClient initializing with execPbsz={}",
execPbsz);
+ getFtpClient().execPBSZ(execPbsz);
+ }
+ if (execProt != null) {
+ log.debug("FTPClient initializing with execProt={}",
execProt);
+ getFtpClient().execPROT(execProt);
+ }
+ } catch (IOException e) {
+ throw new GenericFileOperationFailedException(
+ client.getReplyCode(), client.getReplyString(),
e.getMessage(), e);
+ } finally {
+ if (exchange != null) {
+ // store client reply information after the operation
+
exchange.getIn().setHeader(FtpConstants.FTP_REPLY_CODE, client.getReplyCode());
+
exchange.getIn().setHeader(FtpConstants.FTP_REPLY_STRING,
client.getReplyString());
+ }
}
}
- }
- return answer;
+ return answer;
+ } finally {
+ lock.unlock();
+ }
}
@Override
diff --git
a/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/FtpConsumerCommitOnOtherThreadTest.java
b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/FtpConsumerCommitOnOtherThreadTest.java
new file mode 100644
index 000000000000..bc255b216e92
--- /dev/null
+++
b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/FtpConsumerCommitOnOtherThreadTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.file.remote;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.mock.MockEndpoint;
+import org.apache.camel.test.junit6.CamelTestSupport;
+import org.apache.commons.net.ftp.FTPClient;
+import org.apache.commons.net.ftp.FTPFile;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * When the route hands the exchange over to another thread (here a seda
consumer), the delete of a consumed file runs
+ * on that thread, while the ftp consumer goes on with the next file of the
poll, or the next poll, on the same
+ * connection. The commands of the two must not mix on the one control
connection: the delete waits until the consumer
+ * is done with the download, or with the walk through the directories.
+ */
+class FtpConsumerCommitOnOtherThreadTest extends CamelTestSupport {
+
+ private final RecordingFtpClient client = new RecordingFtpClient();
+ private FakeFtpServer server;
+
+ @BeforeEach
+ void startServer() throws Exception {
+ server = new FakeFtpServer();
+ context.getRegistry().bind("client", client);
+ }
+
+ @AfterEach
+ void stopServer() throws Exception {
+ server.close();
+ }
+
+ @Test
+ void deleteOnOtherThreadWaitsForTheDownload() throws Exception {
+ for (int i = 1; i <= 3; i++) {
+ server.addFile("/inbox/f-" + i + ".txt", ("content of file " +
i).getBytes(StandardCharsets.UTF_8));
+ }
+ // the delete of the first file comes while the consumer downloads the
second
+ client.window = "RETR f-2.txt";
+ MockEndpoint mock = getMockEndpoint("mock:work");
+ mock.expectedBodiesReceived("content of file 1", "content of file 2",
"content of file 3");
+
+ addRoutes();
+
+ mock.assertIsSatisfied();
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
server.files().isEmpty());
+ assertEquals(List.of(), client.commandsInWindow, "the commands another
thread sent during the download");
+ }
+
+ @Test
+ void deleteOnOtherThreadWaitsForTheWalkOfTheNextPoll() throws Exception {
+ server.addFile("/inbox/f-1.txt", "content of file
1".getBytes(StandardCharsets.UTF_8));
+ // the delete of the file comes while the next poll lists the
directory it changed to (stepwise): the delete
+ // changes to the directory of the file from there, and fails, so the
file would be consumed again
+ client.window = "LIST 2";
+ MockEndpoint mock = getMockEndpoint("mock:work");
+ mock.expectedBodiesReceived("content of file 1");
+ // and not consumed again
+ mock.setAssertPeriod(1000);
+
+ addRoutes();
+
+ mock.assertIsSatisfied();
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
server.files().isEmpty());
+ assertEquals(List.of(), client.commandsInWindow, "the commands another
thread sent during the listing");
+ }
+
+ private void addRoutes() throws Exception {
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("ftp://[email protected]:" + server.port() +
"/inbox?password=secret&passiveMode=true"
+ +
"&delete=true&sortBy=file:name&initialDelay=0&delay=100&ftpClient=#client")
+ .to("seda:work");
+
+ // a file is done (and deleted) once the consumer is in the
window
+ from("seda:work")
+ .process(e -> client.windowStarted.await(5,
TimeUnit.SECONDS))
+ .convertBodyTo(String.class).to("mock:work");
+ }
+ });
+ }
+
+ /**
+ * Records the commands that another thread sends while the consumer is in
a window: the download of a file, or the
+ * listing of a poll. The consumer waits up to 2 seconds in the window for
such a command, so the delete that the
+ * seda consumer runs once the window has begun comes while the consumer
is in the window, when nothing keeps it
+ * out.
+ */
+ private static final class RecordingFtpClient extends FTPClient {
+
+ private final List<String> commandsInWindow = new
CopyOnWriteArrayList<>();
+ private final CountDownLatch windowStarted = new CountDownLatch(1);
+ private final CountDownLatch otherThreadCommand = new
CountDownLatch(1);
+ private final AtomicInteger listings = new AtomicInteger();
+ private volatile String window;
+ private volatile Thread inWindow;
+
+ @Override
+ public boolean retrieveFile(String remote, OutputStream local) throws
IOException {
+ if (("RETR " + remote).equals(window)) {
+ return inWindow(() -> super.retrieveFile(remote, local));
+ }
+ return super.retrieveFile(remote, local);
+ }
+
+ @Override
+ public FTPFile[] listFiles() throws IOException {
+ if (("LIST " + listings.incrementAndGet()).equals(window)) {
+ return inWindow(super::listFiles);
+ }
+ return super.listFiles();
+ }
+
+ private <T> T inWindow(IOCall<T> call) throws IOException {
+ inWindow = Thread.currentThread();
+ windowStarted.countDown();
+ try {
+ otherThreadCommand.await(2, TimeUnit.SECONDS);
+ return call.call();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(e);
+ } finally {
+ inWindow = null;
+ }
+ }
+
+ @Override
+ public int sendCommand(String command, String args) throws IOException
{
+ Thread t = inWindow;
+ if (t != null && t != Thread.currentThread()) {
+ commandsInWindow.add(args != null ? command + " " + args :
command);
+ otherThreadCommand.countDown();
+ }
+ return super.sendCommand(command, args);
+ }
+ }
+
+ @FunctionalInterface
+ private interface IOCall<T> {
+ T call() throws IOException;
+ }
+}
diff --git
a/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/RemoteFileConsumerSendNoopTest.java
b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/RemoteFileConsumerSendNoopTest.java
index c17192411fac..c10514857074 100644
---
a/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/RemoteFileConsumerSendNoopTest.java
+++
b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/RemoteFileConsumerSendNoopTest.java
@@ -36,6 +36,7 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
/**
* Unit tests for the connection check that {@link
RemoteFileConsumer#connectIfNecessary()} does before each poll: by
@@ -58,7 +59,8 @@ class RemoteFileConsumerSendNoopTest extends CamelTestSupport
{
}, sftpOperations, mock(GenericFileProcessStrategy.class));
}
// the ftp consumer (also used by ftps) needs the ftp specific
operations
- FtpOperations ftpOperations = mock(FtpOperations.class);
+ // created with its constructor, so it has the lock that the consumer
holds while it walks the directories
+ FtpOperations ftpOperations = mock(FtpOperations.class,
withSettings().useConstructor(null, null));
operations = ftpOperations;
return new FtpConsumer((RemoteFileEndpoint<FTPFile>) endpoint,
exchange -> {
}, ftpOperations, mock(GenericFileProcessStrategy.class));