This is an automated email from the ASF dual-hosted git repository.
aajisaka pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 27621c9eef2 HDFS-13983. TestOfflineImageViewer crashes in windows.
Contributed by Vinayakumar B.
27621c9eef2 is described below
commit 27621c9eef244f6edbb26c099c575e1558efa869
Author: Inigo Goiri <[email protected]>
AuthorDate: Fri Oct 19 17:16:04 2018 -0700
HDFS-13983. TestOfflineImageViewer crashes in windows. Contributed by
Vinayakumar B.
(cherry picked from commit f069d38c8d3c0bfa91b70a60e4e556ec708fc411)
---
.../offlineImageViewer/OfflineImageViewerPB.java | 31 +++++---
.../offlineImageViewer/TestOfflineImageViewer.java | 82 ++++++++++++----------
2 files changed, 66 insertions(+), 47 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
index 34a85a6dce7..782ace818aa 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
@@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.StringUtils;
/**
@@ -135,7 +136,7 @@ public class OfflineImageViewerPB {
*/
public static void main(String[] args) throws Exception {
int status = run(args);
- System.exit(status);
+ ExitUtil.terminate(status);
}
public static int run(String[] args) throws Exception {
@@ -175,19 +176,24 @@ public class OfflineImageViewerPB {
String tempPath = cmd.getOptionValue("t", "");
Configuration conf = new Configuration();
- try (PrintStream out = outputFile.equals("-") ?
- System.out : new PrintStream(outputFile, "UTF-8")) {
+ PrintStream out = null;
+ try {
+ out = outputFile.equals("-") || "REVERSEXML".equalsIgnoreCase(processor)
?
+ System.out : new PrintStream(outputFile, "UTF-8");
switch (StringUtils.toUpperCase(processor)) {
case "FILEDISTRIBUTION":
long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
boolean formatOutput = cmd.hasOption("format");
- new FileDistributionCalculator(conf, maxSize, step, formatOutput, out)
- .visit(new RandomAccessFile(inputFile, "r"));
+ try (RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
+ new FileDistributionCalculator(conf, maxSize, step, formatOutput,
out)
+ .visit(r);
+ }
break;
case "XML":
- new PBImageXmlWriter(conf, out).visit(new RandomAccessFile(inputFile,
- "r"));
+ try (RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
+ new PBImageXmlWriter(conf, out).visit(r);
+ }
break;
case "REVERSEXML":
try {
@@ -196,7 +202,7 @@ public class OfflineImageViewerPB {
System.err.println("OfflineImageReconstructor failed: "
+ e.getMessage());
e.printStackTrace(System.err);
- System.exit(1);
+ ExitUtil.terminate(1);
}
break;
case "WEB":
@@ -208,8 +214,9 @@ public class OfflineImageViewerPB {
break;
case "DELIMITED":
try (PBImageDelimitedTextWriter writer =
- new PBImageDelimitedTextWriter(out, delimiter, tempPath)) {
- writer.visit(new RandomAccessFile(inputFile, "r"));
+ new PBImageDelimitedTextWriter(out, delimiter, tempPath);
+ RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
+ writer.visit(r);
}
break;
default:
@@ -223,6 +230,10 @@ public class OfflineImageViewerPB {
} catch (IOException e) {
System.err.println("Encountered exception. Exiting: " + e.getMessage());
e.printStackTrace(System.err);
+ } finally {
+ if (out != null && out != System.out) {
+ out.close();
+ }
}
return -1;
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index 3ec7fb30040..fa8de5602ec 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -365,8 +365,10 @@ public class TestOfflineImageViewer {
File truncatedFile = new File(tempDir, "truncatedFsImage");
PrintStream output = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM);
copyPartOfFile(originalFsimage, truncatedFile);
- new FileDistributionCalculator(new Configuration(), 0, 0, false, output)
- .visit(new RandomAccessFile(truncatedFile, "r"));
+ try (RandomAccessFile r = new RandomAccessFile(truncatedFile, "r")) {
+ new FileDistributionCalculator(new Configuration(), 0, 0, false, output)
+ .visit(r);
+ }
}
private void copyPartOfFile(File src, File dest) throws IOException {
@@ -387,38 +389,41 @@ public class TestOfflineImageViewer {
@Test
public void testFileDistributionCalculator() throws IOException {
- ByteArrayOutputStream output = new ByteArrayOutputStream();
- PrintStream o = new PrintStream(output);
- new FileDistributionCalculator(new Configuration(), 0, 0, false, o)
- .visit(new RandomAccessFile(originalFsimage, "r"));
- o.close();
-
- String outputString = output.toString();
- Pattern p = Pattern.compile("totalFiles = (\\d+)\n");
- Matcher matcher = p.matcher(outputString);
- assertTrue(matcher.find() && matcher.groupCount() == 1);
- int totalFiles = Integer.parseInt(matcher.group(1));
- assertEquals(NUM_DIRS * FILES_PER_DIR + filesECCount + 1, totalFiles);
-
- p = Pattern.compile("totalDirectories = (\\d+)\n");
- matcher = p.matcher(outputString);
- assertTrue(matcher.find() && matcher.groupCount() == 1);
- int totalDirs = Integer.parseInt(matcher.group(1));
- // totalDirs includes root directory
- assertEquals(dirCount + 1, totalDirs);
-
- FileStatus maxFile = Collections.max(writtenFiles.values(),
- new Comparator<FileStatus>() {
- @Override
- public int compare(FileStatus first, FileStatus second) {
- return first.getLen() < second.getLen() ? -1 :
- ((first.getLen() == second.getLen()) ? 0 : 1);
- }
- });
- p = Pattern.compile("maxFileSize = (\\d+)\n");
- matcher = p.matcher(output.toString("UTF-8"));
- assertTrue(matcher.find() && matcher.groupCount() == 1);
- assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1)));
+ try (ByteArrayOutputStream output = new ByteArrayOutputStream();
+ PrintStream o = new PrintStream(output);
+ RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
+ new FileDistributionCalculator(new Configuration(), 0, 0, false, o)
+ .visit(r);
+ o.close();
+
+ String outputString = output.toString();
+ Pattern p = Pattern.compile("totalFiles = (\\d+)\n");
+ Matcher matcher = p.matcher(outputString);
+ assertTrue(matcher.find() && matcher.groupCount() == 1);
+ int totalFiles = Integer.parseInt(matcher.group(1));
+ assertEquals(NUM_DIRS * FILES_PER_DIR + filesECCount + 1, totalFiles);
+
+ p = Pattern.compile("totalDirectories = (\\d+)\n");
+ matcher = p.matcher(outputString);
+ assertTrue(matcher.find() && matcher.groupCount() == 1);
+ int totalDirs = Integer.parseInt(matcher.group(1));
+ // totalDirs includes root directory
+ assertEquals(dirCount + 1, totalDirs);
+
+ FileStatus maxFile = Collections.max(writtenFiles.values(),
+ new Comparator<FileStatus>() {
+ @Override
+ public int compare(FileStatus first, FileStatus second) {
+ return first.getLen() < second.getLen() ?
+ -1 :
+ ((first.getLen() == second.getLen()) ? 0 : 1);
+ }
+ });
+ p = Pattern.compile("maxFileSize = (\\d+)\n");
+ matcher = p.matcher(output.toString("UTF-8"));
+ assertTrue(matcher.find() && matcher.groupCount() == 1);
+ assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1)));
+ }
}
@Test
@@ -524,7 +529,9 @@ public class TestOfflineImageViewer {
ByteArrayOutputStream output = new ByteArrayOutputStream();
PrintStream o = new PrintStream(output);
PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o);
- v.visit(new RandomAccessFile(originalFsimage, "r"));
+ try (RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
+ v.visit(r);
+ }
SAXParserFactory spf = SAXParserFactory.newInstance();
SAXParser parser = spf.newSAXParser();
final String xml = output.toString();
@@ -696,10 +703,11 @@ public class TestOfflineImageViewer {
final String DELIMITER = "\t";
ByteArrayOutputStream output = new ByteArrayOutputStream();
- try (PrintStream o = new PrintStream(output)) {
+ try (PrintStream o = new PrintStream(output);
+ RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
PBImageDelimitedTextWriter v =
new PBImageDelimitedTextWriter(o, DELIMITER, db);
- v.visit(new RandomAccessFile(originalFsimage, "r"));
+ v.visit(r);
}
Set<String> fileNames = new HashSet<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
