This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new dead747486 NIFI-13686 Make TestListFile.testFilterAge() more resilient
to time delays (#9207)
dead747486 is described below
commit dead747486d6cb8b93f21fb56f94627c9397b36f
Author: jrsteinebrey <[email protected]>
AuthorDate: Thu Aug 29 09:45:44 2024 -0500
NIFI-13686 Make TestListFile.testFilterAge() more resilient to time delays
(#9207)
Signed-off-by: David Handermann <[email protected]>
---
.../nifi/processors/standard/TestListFile.java | 75 ++++++++++------------
1 file changed, 33 insertions(+), 42 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestListFile.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestListFile.java
index 014e6880c6..41e698136a 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestListFile.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestListFile.java
@@ -44,8 +44,9 @@ import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -80,8 +81,7 @@ public class TestListFile {
// age#filter are filter label strings for the filter properties
private Long syncTime = getTestModifiedTime();
private Long time0millis, time1millis, time2millis, time3millis,
time4millis, time5millis;
- private Long age0millis, age1millis, age2millis, age3millis, age4millis,
age5millis;
- private String age0, age1, age2, age3, age4, age5;
+ private String age0, age2, age4, age5;
@RegisterExtension
private final ListProcessorTestWatcher dumpState = new
ListProcessorTestWatcher(
@@ -258,27 +258,26 @@ public class TestListFile {
@Test
public void testFilterAge() throws Exception {
-
- final File file1 = new File(TESTDIR + "/age1.txt");
- assertTrue(file1.createNewFile());
+ final File file0 = new File(TESTDIR + "/age0.txt");
+ assertTrue(file0.createNewFile());
final File file2 = new File(TESTDIR + "/age2.txt");
assertTrue(file2.createNewFile());
- final File file3 = new File(TESTDIR + "/age3.txt");
- assertTrue(file3.createNewFile());
+ final File file4 = new File(TESTDIR + "/age4.txt");
+ assertTrue(file4.createNewFile());
final Function<Boolean, Object> runNext = resetAges -> {
if (resetAges) {
resetAges();
- assertTrue(file1.setLastModified(time0millis));
+ assertTrue(file0.setLastModified(time0millis));
assertTrue(file2.setLastModified(time2millis));
- assertTrue(file3.setLastModified(time4millis));
+ assertTrue(file4.setLastModified(time4millis));
}
- assertTrue(file1.lastModified() > time3millis &&
file1.lastModified() <= time0millis);
+ assertTrue(file0.lastModified() > time3millis &&
file0.lastModified() <= time0millis);
assertTrue(file2.lastModified() > time3millis &&
file2.lastModified() < time1millis);
- assertTrue(file3.lastModified() < time3millis);
+ assertTrue(file4.lastModified() < time3millis);
try {
runNext();
@@ -292,6 +291,11 @@ public class TestListFile {
runner.setProperty(ListFile.DIRECTORY, testDir.getAbsolutePath());
runNext.apply(true);
runner.assertTransferCount(ListFile.REL_SUCCESS, 3);
+ final List<MockFlowFile> successFiles1 =
runner.getFlowFilesForRelationship(ListFile.REL_SUCCESS);
+ assertEquals(3, successFiles1.size());
+ assertEquals(file4.getName(),
successFiles1.get(0).getAttribute("filename"));
+ assertEquals(file2.getName(),
successFiles1.get(1).getAttribute("filename"));
+ assertEquals(file0.getName(),
successFiles1.get(2).getAttribute("filename"));
assertVerificationOutcome(Outcome.SUCCESSFUL, "Successfully listed .*
Found 3 objects. Of those, 3 match the filter.");
// processor updates internal state, it shouldn't pick the same ones.
@@ -300,36 +304,33 @@ public class TestListFile {
// exclude oldest
runner.setProperty(ListFile.MIN_AGE, age0);
- runner.setProperty(ListFile.MAX_AGE, age3);
+ runner.setProperty(ListFile.MAX_AGE, age4);
runNext.apply(true);
runner.assertAllFlowFilesTransferred(ListFile.REL_SUCCESS);
final List<MockFlowFile> successFiles2 =
runner.getFlowFilesForRelationship(ListFile.REL_SUCCESS);
assertEquals(2, successFiles2.size());
assertEquals(file2.getName(),
successFiles2.get(0).getAttribute("filename"));
- assertEquals(file1.getName(),
successFiles2.get(1).getAttribute("filename"));
+ assertEquals(file0.getName(),
successFiles2.get(1).getAttribute("filename"));
assertVerificationOutcome(Outcome.SUCCESSFUL, "Successfully listed .*
Found 3 objects. Of those, 2 match the filter.");
// exclude newest
- runner.setProperty(ListFile.MIN_AGE, age1);
+ runner.setProperty(ListFile.MIN_AGE, age2);
runner.setProperty(ListFile.MAX_AGE, age5);
runNext.apply(true);
runner.assertAllFlowFilesTransferred(ListFile.REL_SUCCESS);
final List<MockFlowFile> successFiles3 =
runner.getFlowFilesForRelationship(ListFile.REL_SUCCESS);
assertEquals(2, successFiles3.size());
- assertEquals(file3.getName(),
successFiles3.get(0).getAttribute("filename"));
+ assertEquals(file4.getName(),
successFiles3.get(0).getAttribute("filename"));
assertEquals(file2.getName(),
successFiles3.get(1).getAttribute("filename"));
- assertVerificationOutcome(Outcome.SUCCESSFUL, "Successfully listed .*
Found 3 objects. Of those, 2 match the filter.");
// exclude oldest and newest
- runner.setProperty(ListFile.MIN_AGE, age1);
- runner.setProperty(ListFile.MAX_AGE, age3);
+ runner.setProperty(ListFile.MIN_AGE, age2);
+ runner.setProperty(ListFile.MAX_AGE, age4);
runNext.apply(true);
runner.assertAllFlowFilesTransferred(ListFile.REL_SUCCESS);
final List<MockFlowFile> successFiles4 =
runner.getFlowFilesForRelationship(ListFile.REL_SUCCESS);
assertEquals(1, successFiles4.size());
assertEquals(file2.getName(),
successFiles4.get(0).getAttribute("filename"));
- assertVerificationOutcome(Outcome.SUCCESSFUL, "Successfully listed .*
Found 3 objects. Of those, 1 matches the filter.");
-
}
@Test
@@ -787,8 +788,8 @@ public class TestListFile {
final Path absolutePath = file1.toPath().toAbsolutePath();
final String absolutePathString = absolutePath.getParent().toString()
+ File.separator;
final FileStore store = Files.getFileStore(file1Path);
- final DateFormat formatter = new
SimpleDateFormat(ListFile.FILE_MODIFY_DATE_ATTR_FORMAT, Locale.US);
- final String time3Formatted = formatter.format(time3rounded);
+ final DateTimeFormatter formatter =
DateTimeFormatter.ofPattern(ListFile.FILE_MODIFY_DATE_ATTR_FORMAT, Locale.US);
+ final String time3Formatted =
formatter.format(Instant.ofEpochMilli(time3rounded).atZone(ZoneId.systemDefault()));
// check standard attributes
MockFlowFile mock1 = successFiles1.get(0);
@@ -872,7 +873,7 @@ public class TestListFile {
* HFS+, default for OS X, only has granularity to one second,
accordingly, we go back in time to establish consistent test cases
*
* Provides "now" minus 1 second in millis
- */
+ */
private static long getTestModifiedTime() {
final long nowMillis = System.currentTimeMillis();
// Subtract a second to avoid possible rounding issues
@@ -883,20 +884,12 @@ public class TestListFile {
private void resetAges() {
syncTime = getTestModifiedTime();
- age0millis = 0L;
- age1millis = 5000L;
- age2millis = 10000L;
- age3millis = 15000L;
- age4millis = 20000L;
- age5millis = 100000L;
-
- // Allow for bigger gaps since the lag is 2s w/o milliseconds.
- if (!isMillisecondSupported) {
- age1millis *= 2;
- age2millis *= 2;
- age3millis *= 2;
- age4millis *= 2;
- }
+ final long age0millis = 0L;
+ final long age1millis = 20000L;
+ final long age2millis = 40000L;
+ final long age3millis = 60000L;
+ final long age4millis = 80000L;
+ final long age5millis = 200000L;
time0millis = syncTime - age0millis;
time1millis = syncTime - age1millis;
@@ -906,14 +899,12 @@ public class TestListFile {
time5millis = syncTime - age5millis;
age0 = Long.toString(age0millis) + " millis";
- age1 = Long.toString(age1millis) + " millis";
age2 = Long.toString(age2millis) + " millis";
- age3 = Long.toString(age3millis) + " millis";
age4 = Long.toString(age4millis) + " millis";
age5 = Long.toString(age5millis) + " millis";
}
- private void deleteDirectory(final File directory) throws IOException {
+ private void deleteDirectory(final File directory) {
if (directory.exists()) {
File[] files = directory.listFiles();
if (files != null) {
