Repository: hadoop Updated Branches: refs/heads/branch-2 0f8de8cd9 -> 93fdc3d93
HADOOP-15085. Output streams closed with IOUtils suppressing write errors. Contributed by Jim Brennan Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93fdc3d9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93fdc3d9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93fdc3d9 Branch: refs/heads/branch-2 Commit: 93fdc3d935cd940d5776cd923938b0bfabcd0dcd Parents: 0f8de8c Author: Jason Lowe <[email protected]> Authored: Fri Dec 15 17:17:25 2017 -0600 Committer: Jason Lowe <[email protected]> Committed: Fri Dec 15 17:17:25 2017 -0600 ---------------------------------------------------------------------- .../java/org/apache/hadoop/fs/FileContext.java | 14 ++++------ .../java/org/apache/hadoop/fs/FileUtil.java | 13 +++------ .../apache/hadoop/fs/shell/CopyCommands.java | 11 ++------ .../main/java/org/apache/hadoop/io/MapFile.java | 16 +++++------ .../org/apache/hadoop/io/nativeio/NativeIO.java | 29 ++++++++------------ .../apache/hadoop/conf/TestConfiguration.java | 7 ++--- .../apache/hadoop/io/compress/TestCodec.java | 8 ++---- 7 files changed, 36 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/93fdc3d9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index 039dbad..7edef14 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -2102,17 +2102,13 @@ public class FileContext { content.getPath().getName())), deleteSource, overwrite); } } else { - InputStream in=null; - OutputStream out = null; - try { - in = open(qSrc); - EnumSet<CreateFlag> createFlag = overwrite ? EnumSet.of( - CreateFlag.CREATE, CreateFlag.OVERWRITE) : - EnumSet.of(CreateFlag.CREATE); - out = create(qDst, createFlag); + EnumSet<CreateFlag> createFlag = overwrite ? EnumSet.of( + CreateFlag.CREATE, CreateFlag.OVERWRITE) : + EnumSet.of(CreateFlag.CREATE); + InputStream in = open(qSrc); + try (OutputStream out = create(qDst, createFlag)) { IOUtils.copyBytes(in, out, conf, true); } finally { - IOUtils.closeStream(out); IOUtils.closeStream(in); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/93fdc3d9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index f279e5e..2584a84 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -1309,15 +1309,10 @@ public class FileUtil { // Write the manifest to output JAR file File classPathJar = File.createTempFile("classpath-", ".jar", workingDir); - FileOutputStream fos = null; - BufferedOutputStream bos = null; - JarOutputStream jos = null; - try { - fos = new FileOutputStream(classPathJar); - bos = new BufferedOutputStream(fos); - jos = new JarOutputStream(bos, jarManifest); - } finally { - IOUtils.cleanupWithLogger(LOG, jos, bos, fos); + try (FileOutputStream fos = new FileOutputStream(classPathJar); + BufferedOutputStream bos = new BufferedOutputStream(fos)) { + JarOutputStream jos = new JarOutputStream(bos, jarManifest); + jos.close(); } String[] jarCp = {classPathJar.getCanonicalPath(), unexpandedWildcardClasspath.toString()}; http://git-wip-us.apache.org/repos/asf/hadoop/blob/93fdc3d9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java index e2fad75..beab3e6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java @@ -20,7 +20,6 @@ package org.apache.hadoop.fs.shell; import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.util.Iterator; @@ -356,10 +355,8 @@ class CopyCommands { dst.fs.create(dst.path, false).close(); } - InputStream is = null; - FSDataOutputStream fos = dst.fs.append(dst.path); - - try { + FileInputStream is = null; + try (FSDataOutputStream fos = dst.fs.append(dst.path)) { if (readStdin) { if (args.size() == 0) { IOUtils.copyBytes(System.in, fos, DEFAULT_IO_LENGTH); @@ -380,10 +377,6 @@ class CopyCommands { if (is != null) { IOUtils.closeStream(is); } - - if (fos != null) { - IOUtils.closeStream(fos); - } } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/93fdc3d9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java index fde1c86..f9e0145 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java @@ -986,23 +986,23 @@ public class MapFile { Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf); MapFile.Reader reader = null; - MapFile.Writer writer = null; try { reader = new MapFile.Reader(fs, in, conf); - writer = - new MapFile.Writer(conf, fs, out, - reader.getKeyClass().asSubclass(WritableComparable.class), - reader.getValueClass()); WritableComparable key = ReflectionUtils.newInstance(reader.getKeyClass() .asSubclass(WritableComparable.class), conf); Writable value = ReflectionUtils.newInstance(reader.getValueClass() .asSubclass(Writable.class), conf); - while (reader.next(key, value)) // copy all entries - writer.append(key, value); + try (MapFile.Writer writer = new MapFile.Writer(conf, fs, out, + reader.getKeyClass().asSubclass(WritableComparable.class), + reader.getValueClass())) { + while (reader.next(key, value)) { // copy all entries + writer.append(key, value); + } + } } finally { - IOUtils.cleanupWithLogger(LOG, writer, reader); + IOUtils.cleanupWithLogger(LOG, reader); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/93fdc3d9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java index d95efb6..4111aba 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java @@ -922,28 +922,23 @@ public class NativeIO { if (nativeLoaded && Shell.WINDOWS) { copyFileUnbuffered0(src.getAbsolutePath(), dst.getAbsolutePath()); } else { - FileInputStream fis = null; - FileOutputStream fos = null; + FileInputStream fis = new FileInputStream(src); FileChannel input = null; - FileChannel output = null; try { - fis = new FileInputStream(src); - fos = new FileOutputStream(dst); input = fis.getChannel(); - output = fos.getChannel(); - long remaining = input.size(); - long position = 0; - long transferred = 0; - while (remaining > 0) { - transferred = input.transferTo(position, remaining, output); - remaining -= transferred; - position += transferred; + try (FileOutputStream fos = new FileOutputStream(dst); + FileChannel output = fos.getChannel()) { + long remaining = input.size(); + long position = 0; + long transferred = 0; + while (remaining > 0) { + transferred = input.transferTo(position, remaining, output); + remaining -= transferred; + position += transferred; + } } } finally { - IOUtils.cleanupWithLogger(LOG, output); - IOUtils.cleanupWithLogger(LOG, fos); - IOUtils.cleanupWithLogger(LOG, input); - IOUtils.cleanupWithLogger(LOG, fis); + IOUtils.cleanupWithLogger(LOG, input, fis); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/93fdc3d9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 9fda23b..f6c2d46 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -51,7 +51,6 @@ import org.apache.hadoop.conf.Configuration.IntegerRanges; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; @@ -374,11 +373,9 @@ public class TestConfiguration extends TestCase { Configuration conf = new Configuration(false); conf.addResource(new Path(CONFIG_MULTI_BYTE)); assertEquals(value, conf.get(name)); - FileOutputStream fos = new FileOutputStream(CONFIG_MULTI_BYTE_SAVED); - try { + try (FileOutputStream fos = + new FileOutputStream(CONFIG_MULTI_BYTE_SAVED)) { conf.writeXml(fos); - } finally { - IOUtils.closeStream(fos); } conf = new Configuration(false); http://git-wip-us.apache.org/repos/asf/hadoop/blob/93fdc3d9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java index 4b99def..d07d6a7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java @@ -354,11 +354,10 @@ public class TestCodec { final Path file = new Path(wd, "test" + codec.getDefaultExtension()); final byte[] b = new byte[REC_SIZE]; final Base64 b64 = new Base64(0, null); - DataOutputStream fout = null; Compressor cmp = CodecPool.getCompressor(codec); - try { - fout = new DataOutputStream(codec.createOutputStream( - fs.create(file, true), cmp)); + try (DataOutputStream fout = + new DataOutputStream(codec.createOutputStream(fs.create(file, + true), cmp))) { final DataOutputBuffer dob = new DataOutputBuffer(REC_SIZE * 4 / 3 + 4); int seq = 0; while (infLen > 0) { @@ -374,7 +373,6 @@ public class TestCodec { } LOG.info("Wrote " + seq + " records to " + file); } finally { - IOUtils.cleanupWithLogger(LOG, fout); CodecPool.returnCompressor(cmp); } return file; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
