This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 7d2eecd48f Issue #8800 : Do not copy files while Copy files is still
walking the source folder (#8809)
7d2eecd48f is described below
commit 7d2eecd48f0a649580d0e864e705f094cb12ef04
Author: Bart Maertens <[email protected]>
AuthorDate: Fri Oct 9 15:16:27 2026 +0200
Issue #8800 : Do not copy files while Copy files is still walking the
source folder (#8809)
---
.../main-0024-copy-files-into-own-subfolder.hwf | 263 +++++++++++++++++++++
.../actions/copyfiles/ActionCopyFiles.java | 64 ++++-
.../WorkflowActionCopyFilesLocalTest.java | 39 +++
3 files changed, 359 insertions(+), 7 deletions(-)
diff --git
a/integration-tests/actions/main-0024-copy-files-into-own-subfolder.hwf
b/integration-tests/actions/main-0024-copy-files-into-own-subfolder.hwf
new file mode 100644
index 0000000000..51261b74b0
--- /dev/null
+++ b/integration-tests/actions/main-0024-copy-files-into-own-subfolder.hwf
@@ -0,0 +1,263 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0024-copy-files-into-own-subfolder</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Issue #8800: copying a folder into a subfolder of itself copies
every file once and never copies the destination folder into
itself.</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/10/08 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/10/08 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>64</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Create source folder</name>
+ <description>Files and folders on both sides of the empty "copy" folder,
in creation and in name order, so some files are copied before the traversal
reaches "copy" whatever order the filesystem lists them in.</description>
+ <type>SHELL</type>
+ <attributes/>
+ <filename/>
+ <work_directory>${PROJECT_HOME}</work_directory>
+ <arg_from_previous>N</arg_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <set_append_logfile>N</set_append_logfile>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <insertScript>Y</insertScript>
+ <script>#!/bin/bash
+set -e
+BASE="${PROJECT_HOME}/workfiles-0024"
+SRC="$BASE/project"
+rm -rf "$BASE"
+mkdir -p "$SRC"
+for i in 0 1 2 3 4; do
+ mkdir -p "$SRC/a$i"
+ echo "a$i" > "$SRC/a$i/f.txt"
+done
+mkdir "$SRC/copy"
+for i in 0 1 2 3 4; do
+ echo "z$i" > "$SRC/z$i.txt"
+done
+ls -U "$SRC"</script>
+ <loglevel>Basic</loglevel>
+ <parallel>N</parallel>
+ <xloc>256</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Copy folder into own subfolder</name>
+ <description/>
+ <type>COPY_FILES</type>
+ <attributes/>
+ <copy_empty_folders>N</copy_empty_folders>
+ <arg_from_previous>N</arg_from_previous>
+ <overwrite_files>Y</overwrite_files>
+ <include_subfolders>Y</include_subfolders>
+ <remove_source_files>N</remove_source_files>
+ <add_result_filesname>N</add_result_filesname>
+ <destination_is_a_file>N</destination_is_a_file>
+ <create_destination_folder>Y</create_destination_folder>
+ <fields>
+ <field>
+
<source_filefolder>${PROJECT_HOME}/workfiles-0024/project</source_filefolder>
+
<destination_filefolder>${PROJECT_HOME}/workfiles-0024/project/copy</destination_filefolder>
+ <wildcard/>
+ </field>
+ </fields>
+ <parallel>N</parallel>
+ <xloc>480</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Check copied files</name>
+ <description>Every file is copied once into copy/, and there is no
copy/copy.</description>
+ <type>SHELL</type>
+ <attributes/>
+ <filename/>
+ <work_directory>${PROJECT_HOME}</work_directory>
+ <arg_from_previous>N</arg_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <set_append_logfile>N</set_append_logfile>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <insertScript>Y</insertScript>
+ <script>#!/bin/bash
+BASE="${PROJECT_HOME}/workfiles-0024"
+SRC="$BASE/project"
+DEST="$SRC/copy"
+FAILURES=0
+find "$SRC" | sort
+if [ -e "$DEST/copy" ]; then
+ echo "FAIL: the destination folder was copied into itself: $DEST/copy"
+ FAILURES=$((FAILURES + 1))
+fi
+for i in 0 1 2 3 4; do
+ if [ "$(cat "$DEST/a$i/f.txt" 2>/dev/null)" != "a$i" ]; then
+ echo "FAIL: $DEST/a$i/f.txt is missing or wrong"
+ FAILURES=$((FAILURES + 1))
+ fi
+ if [ "$(cat "$DEST/z$i.txt" 2>/dev/null)" != "z$i" ]; then
+ echo "FAIL: $DEST/z$i.txt is missing or wrong"
+ FAILURES=$((FAILURES + 1))
+ fi
+done
+rm -rf "$BASE"
+if [ "$FAILURES" -ne 0 ]; then
+ echo "$FAILURES check(s) failed"
+ exit 1
+fi
+echo "All checks passed"</script>
+ <loglevel>Basic</loglevel>
+ <parallel>N</parallel>
+ <xloc>704</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Success</name>
+ <description/>
+ <type>SUCCESS</type>
+ <attributes/>
+ <parallel>N</parallel>
+ <xloc>928</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Abort setup failed</name>
+ <description/>
+ <type>ABORT</type>
+ <attributes/>
+ <always_log_rows>N</always_log_rows>
+ <message>Could not create the source folder</message>
+ <parallel>N</parallel>
+ <xloc>256</xloc>
+ <yloc>240</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Abort copy failed</name>
+ <description/>
+ <type>ABORT</type>
+ <attributes/>
+ <always_log_rows>N</always_log_rows>
+ <message>Copy files failed</message>
+ <parallel>N</parallel>
+ <xloc>480</xloc>
+ <yloc>240</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Abort wrong copy</name>
+ <description/>
+ <type>ABORT</type>
+ <attributes/>
+ <always_log_rows>N</always_log_rows>
+ <message>Copying a folder into its own subfolder gave the wrong result,
see the Check copied files log</message>
+ <parallel>N</parallel>
+ <xloc>704</xloc>
+ <yloc>240</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Create source folder</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Create source folder</from>
+ <to>Copy folder into own subfolder</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>Create source folder</from>
+ <to>Abort setup failed</to>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>Copy folder into own subfolder</from>
+ <to>Check copied files</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>Copy folder into own subfolder</from>
+ <to>Abort copy failed</to>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>Check copied files</from>
+ <to>Success</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>Check copied files</from>
+ <to>Abort wrong copy</to>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/plugins/actions/copyfiles/src/main/java/org/apache/hop/workflow/actions/copyfiles/ActionCopyFiles.java
b/plugins/actions/copyfiles/src/main/java/org/apache/hop/workflow/actions/copyfiles/ActionCopyFiles.java
index 94926142dc..2645202cb4 100644
---
a/plugins/actions/copyfiles/src/main/java/org/apache/hop/workflow/actions/copyfiles/ActionCopyFiles.java
+++
b/plugins/actions/copyfiles/src/main/java/org/apache/hop/workflow/actions/copyfiles/ActionCopyFiles.java
@@ -424,6 +424,7 @@ public class ActionCopyFiles extends ActionBase implements
ILegacyXml {
result);
try {
destinationFileFolder.copyFrom(sourceFileFolder,
textFileSelector);
+ textFileSelector.copyPendingFiles();
} finally {
textFileSelector.shutdown();
}
@@ -746,6 +747,8 @@ public class ActionCopyFiles extends ActionBase implements
ILegacyXml {
}
}
+ private record PendingFileCopy(FileObject source, FileObject target) {}
+
private class TextFileSelector implements FileSelector {
String fileWildcard = null;
String sourceFolder = null;
@@ -759,6 +762,8 @@ public class ActionCopyFiles extends ActionBase implements
ILegacyXml {
private final Result copyResult;
+ private final List<PendingFileCopy> pendingFileCopies = new ArrayList<>();
+
/**
* @param selectedfile
* @return True if the selectedfile matches the wildcard
@@ -798,7 +803,7 @@ public class ActionCopyFiles extends ActionBase implements
ILegacyXml {
@Override
public boolean includeFile(FileSelectInfo info) {
boolean returncode = false;
- boolean streamCopied = false;
+ boolean deferred = false;
FileObject filename = null;
String addFileNameString = null;
try {
@@ -994,9 +999,11 @@ public class ActionCopyFiles extends ActionBase implements
ILegacyXml {
}
if (returncode && filename != null && info.getFile().getType() ==
FileType.FILE) {
- long copied = copyFileWithByteTracking(info.getFile(), filename,
copyResult);
- ActionCopyFiles.this.emitCopyLineage(info.getFile(), filename,
copied);
- streamCopied = true;
+ // Copy after the traversal so files written into a destination
below the source are
+ // not picked up again (see copyPendingFiles())
+ pendingFileCopies.add(new PendingFileCopy(info.getFile(), filename));
+ filename = null;
+ deferred = true;
}
} catch (Exception e) {
@@ -1022,6 +1029,12 @@ public class ActionCopyFiles extends ActionBase
implements ILegacyXml {
}
}
}
+ if (deferred) {
+ // Files are copied by copyPendingFiles(), which also registers them
for removal and in
+ // the result. Returning true would make copyFrom copy them as well.
+ return false;
+ }
+
if (returncode && removeSourceFiles) {
// add this folder/file to remove files
// This list will be fetched and all entries files
@@ -1035,9 +1048,46 @@ public class ActionCopyFiles extends ActionBase
implements ILegacyXml {
addFileNameString); // was a NPE before with the file_name=null
above in the finally
}
- // File rows: we already copied bytes above when streamCopied; true
would make copyFrom copy
- // again. Folder rows: unchanged — still return returncode so VFS
creates empty dirs.
- return streamCopied ? false : returncode;
+ // Folders: VFS copyFrom creates them
+ return returncode;
+ }
+
+ /**
+ * Copies the files selected during the traversal. Copying while VFS still
walks the source tree
+ * makes a destination folder below the source pick up the files already
copied into it (#8800).
+ */
+ public void copyPendingFiles() {
+ for (PendingFileCopy copy : pendingFileCopies) {
+ if (parentjob.isStopped()) {
+ break;
+ }
+ FileObject target = copy.target();
+ try {
+ long copied = copyFileWithByteTracking(copy.source(), target,
copyResult);
+ ActionCopyFiles.this.emitCopyLineage(copy.source(), target, copied);
+ if (removeSourceFiles) {
+ listFilesRemove.add(copy.source().toString());
+ }
+ if (addResultFilenames) {
+ listAddResult.add(target.toString());
+ }
+ } catch (Exception e) {
+ logError(
+ BaseMessages.getString(
+ PKG,
+ CONST_COPY_PROCESS,
+ HopVfs.getFriendlyURI(copy.source()),
+ HopVfs.getFriendlyURI(target),
+ e.getMessage()));
+ } finally {
+ try {
+ target.close();
+ } catch (IOException ex) {
+ /* Ignore */
+ }
+ }
+ }
+ pendingFileCopies.clear();
}
@Override
diff --git
a/plugins/actions/copyfiles/src/test/java/org/apache/hop/workflow/actions/copyfiles/WorkflowActionCopyFilesLocalTest.java
b/plugins/actions/copyfiles/src/test/java/org/apache/hop/workflow/actions/copyfiles/WorkflowActionCopyFilesLocalTest.java
index 26e62b9447..2b204beb8f 100644
---
a/plugins/actions/copyfiles/src/test/java/org/apache/hop/workflow/actions/copyfiles/WorkflowActionCopyFilesLocalTest.java
+++
b/plugins/actions/copyfiles/src/test/java/org/apache/hop/workflow/actions/copyfiles/WorkflowActionCopyFilesLocalTest.java
@@ -177,6 +177,45 @@ class WorkflowActionCopyFilesLocalTest {
assertEquals("doc", Files.readString(copied, StandardCharsets.UTF_8));
}
+ /** Issue #8800: copying a folder into a subfolder of itself must not copy
the copies again. */
+ @Test
+ void copyFolderIntoOwnSubfolderDoesNotCopyTheCopies() throws Exception {
+ Path srcRoot = tempDir.resolve("project");
+ Files.createDirectories(srcRoot);
+ // Entries on both sides of "copy" in creation and in name order, so that
whatever order the
+ // filesystem lists them in, some files are copied before the traversal
reaches "copy"
+ for (int i = 0; i < 5; i++) {
+ Files.createDirectories(srcRoot.resolve("a" + i));
+ Files.writeString(srcRoot.resolve("a" + i + "/f.txt"), "a" + i,
StandardCharsets.UTF_8);
+ }
+ Path destRoot = srcRoot.resolve("copy");
+ Files.createDirectories(destRoot);
+ for (int i = 0; i < 5; i++) {
+ Files.writeString(srcRoot.resolve("z" + i + ".txt"), "z" + i,
StandardCharsets.UTF_8);
+ }
+
+ action.setDestinationIsAFile(false);
+ action.setIncludeSubFolders(true);
+ action.setCopyEmptyFolders(false);
+ action.setOverwriteFiles(true);
+ action.setCreateDestinationFolder(true);
+ action.setFileRows(
+ List.of(
+ new CopyFilesItem(
+ srcRoot.toAbsolutePath().toString(),
destRoot.toAbsolutePath().toString(), "")));
+
+ Result result = action.execute(new Result(), 0);
+
+ assertTrue(result.isResult());
+ assertFalse(Files.exists(destRoot.resolve("copy")), "the destination must
not be copied");
+ for (int i = 0; i < 5; i++) {
+ assertEquals(
+ "a" + i, Files.readString(destRoot.resolve("a" + i + "/f.txt"),
StandardCharsets.UTF_8));
+ assertEquals(
+ "z" + i, Files.readString(destRoot.resolve("z" + i + ".txt"),
StandardCharsets.UTF_8));
+ }
+ }
+
@Test
void removeSourceFilesDeletesSourceAfterFileToFileCopy() throws Exception {
Path srcFile = tempDir.resolve("moveSrc/x.txt");
