This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 71090b4b0b6 CAMEL-21947: camel-file/camel-ftp: If read lock is not
acquired then … (#17731)
71090b4b0b6 is described below
commit 71090b4b0b626313f080cdfe40c61aa3d48f8a64
Author: Claus Ibsen <[email protected]>
AuthorDate: Fri Apr 11 17:30:30 2025 +0200
CAMEL-21947: camel-file/camel-ftp: If read lock is not acquired then …
(#17731)
* CAMEL-21947: camel-file/camel-ftp: If read lock is not acquired then file
is not removed from idempotent repo if eager=true
---
.../camel/component/file/GenericFileConsumer.java | 6 ++
.../integration/FtpReadLockNotStartedIT.java | 104 +++++++++++++++++++++
2 files changed, 110 insertions(+)
diff --git
a/components/camel-file/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java
b/components/camel-file/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java
index d4bece9f506..89b676b33a8 100644
---
a/components/camel-file/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java
+++
b/components/camel-file/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java
@@ -16,6 +16,7 @@
*/
package org.apache.camel.component.file;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Deque;
@@ -227,6 +228,8 @@ public abstract class GenericFileConsumer<T> extends
ScheduledBatchPollingConsum
total = maxMessagesPerPoll;
}
+ Queue<Object> notStarted = new ArrayDeque<>();
+
for (int index = 0; index < total && isBatchAllowed(); index++) {
// only loop if we are started (allowed to run)
// use poll to remove the head so it does not consume memory even
@@ -253,10 +256,13 @@ public abstract class GenericFileConsumer<T> extends
ScheduledBatchPollingConsum
// if we did not start process the file then decrement the counter
if (!started) {
answer--;
+ // this exchange was not started processing so remember to
release it afterward
+ notStarted.add(exchange);
}
}
// drain any in progress files as we are done with this batch
+ removeExcessiveInProgressFiles(CastUtils.cast((Deque<?>) notStarted,
Exchange.class), 0);
removeExcessiveInProgressFiles(CastUtils.cast((Deque<?>) exchanges,
Exchange.class), 0);
return answer;
diff --git
a/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/integration/FtpReadLockNotStartedIT.java
b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/integration/FtpReadLockNotStartedIT.java
new file mode 100644
index 00000000000..7efa581ab25
--- /dev/null
+++
b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/integration/FtpReadLockNotStartedIT.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.file.remote.integration;
+
+import java.nio.file.Path;
+
+import org.apache.camel.BindToRegistry;
+import org.apache.camel.Exchange;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.file.GenericFile;
+import org.apache.camel.component.file.GenericFileEndpoint;
+import org.apache.camel.component.file.GenericFileOperations;
+import org.apache.camel.component.file.GenericFileProcessStrategy;
+import org.apache.camel.component.file.remote.FtpConsumer;
+import org.apache.camel.component.file.strategy.GenericFileNoOpProcessStrategy;
+import org.apache.camel.component.mock.MockEndpoint;
+import
org.apache.camel.support.processor.idempotent.MemoryIdempotentRepository;
+import org.apache.camel.test.junit5.TestSupport;
+import org.apache.camel.util.FileUtil;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.testcontainers.shaded.org.awaitility.Awaitility;
+
+public class FtpReadLockNotStartedIT extends FtpServerTestSupport {
+
+ @TempDir
+ Path testDirectory;
+
+ @BindToRegistry("myLock")
+ private final GenericFileProcessStrategy lock = new MyReadLock();
+
+ protected String getFtpUrl() {
+ return "ftp://admin@localhost:{{ftp.server.port}}";
+ +
"/notstarted?password=admin&processStrategy=#myLock&idempotent=true";
+ }
+
+ @Test
+ public void testIdempotentEager() throws Exception {
+ FtpConsumer consumer = (FtpConsumer)
context.getRoute("myRoute").getConsumer();
+ Assertions.assertTrue(consumer.getEndpoint().isIdempotent());
+ Assertions.assertTrue(consumer.getEndpoint().getIdempotentEager());
+ MemoryIdempotentRepository repo = (MemoryIdempotentRepository)
consumer.getEndpoint().getIdempotentRepository();
+ Assertions.assertEquals(0, repo.getCacheSize());
+
+ context.getRouteController().startAllRoutes();
+
+ // this file is not okay and is not started
+ MockEndpoint mock = getMockEndpoint("mock:result");
+ mock.expectedMessageCount(0);
+ template.sendBodyAndHeader("file:{{ftp.root.dir}}/notstarted", "Bye
World", Exchange.FILE_NAME,
+ "bye.txt");
+ mock.assertIsSatisfied(2000);
+ Assertions.assertEquals(0, repo.getCacheSize());
+
+ // this file is okay
+ mock.reset();
+ mock.expectedMessageCount(1);
+ template.sendBodyAndHeader("file:{{ftp.root.dir}}/notstarted", "Hello
World", Exchange.FILE_NAME,
+ "hello.txt");
+ mock.assertIsSatisfied();
+
+ Awaitility.await().untilAsserted(() -> {
+ Assertions.assertEquals(1, repo.getCacheSize());
+ });
+ }
+
+ @Override
+ protected RouteBuilder createRouteBuilder() {
+ return new RouteBuilder() {
+ @Override
+ public void configure() {
+ from(getFtpUrl()).routeId("myRoute").autoStartup(false)
+ .to(TestSupport.fileUri(testDirectory, "out"),
"mock:result");
+ }
+ };
+ }
+
+ private static class MyReadLock extends GenericFileNoOpProcessStrategy {
+
+ @Override
+ public boolean begin(
+ GenericFileOperations operations, GenericFileEndpoint
endpoint, Exchange exchange, GenericFile file)
+ throws Exception {
+ String name = FileUtil.stripPath(file.getFileName());
+ return "hello.txt".equals(name);
+ }
+ }
+
+}
