This is an automated email from the ASF dual-hosted git repository.
dionusos pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push:
new 8abe0d8 OOZIE-3652 Oozie launcher should retry directory listing when
NoSuchFileException occurs (aajisaka via dionusos)
8abe0d8 is described below
commit 8abe0d874e9b9a2ff41bf27432178ce3712e8597
Author: Denes Bodo <[email protected]>
AuthorDate: Wed Feb 9 16:12:04 2022 +0100
OOZIE-3652 Oozie launcher should retry directory listing when
NoSuchFileException occurs (aajisaka via dionusos)
---
release-log.txt | 1 +
.../oozie/action/hadoop/LocalFsOperations.java | 30 ++++++++++++++---
.../oozie/action/hadoop/TestLocalFsOperations.java | 38 ++++++++++++++++++++++
3 files changed, 64 insertions(+), 5 deletions(-)
diff --git a/release-log.txt b/release-log.txt
index 3bbfc69..930fd31 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.3.0 release (trunk - unreleased)
+OOZIE-3652 Oozie launcher should retry directory listing when
NoSuchFileException occurs (aajisaka via dionusos)
OOZIE-3655 upgrade jdom to jdom2 2.0.6.1 (pj.fanning via dionusos)
OOZIE-3653 Upgrade commons-io to 2.11.0 (groot via dionusos)
OOZIE-3657 Upgrade Jetty to 9.4.44.v20210927 (pj.fanning via asalamon74)
diff --git
a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LocalFsOperations.java
b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LocalFsOperations.java
index 14f43ef..4277e25 100644
---
a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LocalFsOperations.java
+++
b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LocalFsOperations.java
@@ -24,11 +24,13 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.EnumSet;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
public class LocalFsOperations {
@@ -56,18 +58,36 @@ public class LocalFsOperations {
System.out.println("======================");
final Path root = folder.toPath();
- Files.walkFileTree(root, EnumSet.of(FileVisitOption.FOLLOW_LINKS),
WALK_DEPTH, new SimpleFileVisitor<Path>() {
+ // If a file is deleted between listing and printing,
NoSuchFileException is thrown.
+ // This situation can happen when an agent attaches to the JVM.
+ // Retry at most 5 times to reduce the possibility of the error.
+ final int maxRetries = 5;
+ for (int i = 0; i < maxRetries; i++) {
+ try {
+ System.out.print(listContents(root));
+ return;
+ } catch (NoSuchFileException e) {
+ System.err.println("A file or directory may be deleted when
printing the current directory. Retrying.");
+ }
+ }
+ throw new IOException("Cannot list the current dir after " +
maxRetries + " tries: " + folder.getAbsolutePath());
+ }
+
+ @VisibleForTesting
+ String listContents(Path path) throws IOException {
+ final StringBuilder builder = new StringBuilder();
+ Files.walkFileTree(path, EnumSet.of(FileVisitOption.FOLLOW_LINKS),
WALK_DEPTH, new SimpleFileVisitor<Path>() {
@Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs) throws IOException {
+ public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs) {
if (attrs.isRegularFile()) {
- System.out.println(" File: " + root.relativize(file));
+ builder.append(" File:
").append(path.relativize(file)).append("\n");
} else if (attrs.isDirectory()) {
- System.out.println(" Dir: " + root.relativize(file) +
"/");
+ builder.append(" Dir:
").append(path.relativize(file)).append("/\n");
}
-
return FileVisitResult.CONTINUE;
}
});
+ return builder.toString();
}
/**
diff --git
a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLocalFsOperations.java
b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLocalFsOperations.java
new file mode 100644
index 0000000..e968825
--- /dev/null
+++
b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLocalFsOperations.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.junit.Test;
+
+import java.io.File;
+import java.nio.file.NoSuchFileException;
+
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+public class TestLocalFsOperations {
+ @Test
+ public void testPrintContentsOfDirWithRetry() throws Exception {
+ LocalFsOperations spy = spy(new LocalFsOperations());
+ // Throw an exception on 1st call
+ when(spy.listContents(new File(".").toPath()))
+ .thenThrow(NoSuchFileException.class)
+ .thenReturn("success");
+ spy.printContentsOfDir(new File("."));
+ }
+}
