Repository: avro Updated Branches: refs/heads/master edd8edda0 -> 528a1c8a9
AVRO-1985: Unreleased resources This closes #177 Project: http://git-wip-us.apache.org/repos/asf/avro/repo Commit: http://git-wip-us.apache.org/repos/asf/avro/commit/528a1c8a Tree: http://git-wip-us.apache.org/repos/asf/avro/tree/528a1c8a Diff: http://git-wip-us.apache.org/repos/asf/avro/diff/528a1c8a Branch: refs/heads/master Commit: 528a1c8a997a0831e83b525173c6a70b08ee681a Parents: edd8edd Author: Gabor Szadovszky <[email protected]> Authored: Thu Dec 29 12:27:38 2016 +0100 Committer: Gabor Szadovszky <[email protected]> Committed: Fri Jun 16 09:18:40 2017 +0200 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../main/java/org/apache/avro/data/Json.java | 9 ++++-- .../org/apache/avro/file/DataFileWriter.java | 14 +++++++-- .../test/java/org/apache/avro/TestDataFile.java | 32 +++++++++++--------- .../compiler/specific/SpecificCompiler.java | 19 +++++++----- .../avro/mapreduce/AvroKeyOutputFormat.java | 12 ++++++-- .../mapreduce/AvroKeyValueOutputFormat.java | 13 ++++++-- .../main/java/org/apache/avro/tool/IdlTool.java | 9 ++++-- .../main/java/org/apache/avro/tool/Main.java | 14 +++++++-- 9 files changed, 88 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index c387de6..d7b0ec1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -160,6 +160,8 @@ Trunk (not yet released) AVRO-1401: @Nullable does not work with byte[] (Nandor Kollar via gabor) + AVRO-1985: Unreleased resources (gabor) + Avro 1.8.1 (14 May 2016) INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/avro/src/main/java/org/apache/avro/data/Json.java ---------------------------------------------------------------------- diff --git a/lang/java/avro/src/main/java/org/apache/avro/data/Json.java b/lang/java/avro/src/main/java/org/apache/avro/data/Json.java index daa8482..66b92ef 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/data/Json.java +++ b/lang/java/avro/src/main/java/org/apache/avro/data/Json.java @@ -18,6 +18,7 @@ package org.apache.avro.data; import java.io.IOException; +import java.io.InputStream; import java.io.StringReader; import java.util.Iterator; @@ -55,8 +56,12 @@ public class Json { public static final Schema SCHEMA; static { try { - SCHEMA = Schema.parse - (Json.class.getResourceAsStream("/org/apache/avro/data/Json.avsc")); + InputStream in = Json.class.getResourceAsStream("/org/apache/avro/data/Json.avsc"); + try { + SCHEMA = Schema.parse(in); + } finally { + in.close(); + } } catch (IOException e) { throw new AvroRuntimeException(e); } http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java ---------------------------------------------------------------------- diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java index fe916dc..d70d4a4 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java +++ b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java @@ -190,11 +190,20 @@ public class DataFileWriter<D> implements Closeable, Flushable { /** Open a writer appending to an existing file. */ public DataFileWriter<D> appendTo(File file) throws IOException { - return appendTo(new SeekableFileInput(file), - new SyncableFileOutputStream(file, true)); + SeekableInput input = null; + try { + input = new SeekableFileInput(file); + OutputStream output = new SyncableFileOutputStream(file, true); + return appendTo(input, output); + } finally { + if (input != null) + input.close(); + // output does not need to be closed here. It will be closed by invoking close() of this writer. + } } /** Open a writer appending to an existing file. + * <strong>Since 1.9.0 this method does not close in.</strong> * @param in reading the existing file. * @param out positioned at the end of the existing file. */ @@ -213,7 +222,6 @@ public class DataFileWriter<D> implements Closeable, Flushable { } else { this.codec = CodecFactory.nullCodec().createInstance(); } - reader.close(); init(out); http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java ---------------------------------------------------------------------- diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java b/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java index e067496..608d960 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java @@ -324,19 +324,23 @@ public class TestDataFile { private void testFSync(boolean useFile) throws IOException { DataFileWriter<Object> writer = new DataFileWriter<Object>(new GenericDatumWriter<Object>()); - writer.setFlushOnEveryBlock(false); - TestingByteArrayOutputStream out = new TestingByteArrayOutputStream(); - if (useFile) { - File f = makeFile(); - SeekableFileInput in = new SeekableFileInput(f); - writer.appendTo(in, out); - } else { - writer.create(SCHEMA, out); - } - int currentCount = 0; - int syncCounter = 0; try { - for (Object datum : new RandomData(SCHEMA, COUNT, SEED+1)) { + writer.setFlushOnEveryBlock(false); + TestingByteArrayOutputStream out = new TestingByteArrayOutputStream(); + if (useFile) { + File f = makeFile(); + SeekableFileInput in = new SeekableFileInput(f); + try { + writer.appendTo(in, out); + } finally { + in.close(); + } + } else { + writer.create(SCHEMA, out); + } + int currentCount = 0; + int syncCounter = 0; + for (Object datum : new RandomData(SCHEMA, COUNT, SEED + 1)) { currentCount++; writer.append(datum); if (currentCount % 10 == 0) { @@ -344,11 +348,11 @@ public class TestDataFile { syncCounter++; } } + System.out.println("Total number of syncs: " + out.syncCount); + Assert.assertEquals(syncCounter, out.syncCount); } finally { writer.close(); } - System.out.println("Total number of syncs: " + out.syncCount); - Assert.assertEquals(syncCounter, out.syncCount); } http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java ---------------------------------------------------------------------- diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java index 2959493..7a6f5f1 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java @@ -270,17 +270,22 @@ public class SpecificCompiler { if (src != null && f.exists() && f.lastModified() >= src.lastModified()) return f; // already up to date: ignore f.getParentFile().mkdirs(); - Writer fw; - if (outputCharacterEncoding != null) { - fw = new OutputStreamWriter(new FileOutputStream(f), outputCharacterEncoding); - } else { - fw = new FileWriter(f); - } + Writer fw = null; + FileOutputStream fos = null; try { + if (outputCharacterEncoding != null) { + fos = new FileOutputStream(f); + fw = new OutputStreamWriter(fos, outputCharacterEncoding); + } else { + fw = new FileWriter(f); + } fw.write(FILE_HEADER); fw.write(contents); } finally { - fw.close(); + if (fw != null) + fw.close(); + if (fos != null) + fos.close(); } return f; } http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java ---------------------------------------------------------------------- diff --git a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java b/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java index 4c86112..125833c 100644 --- a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java +++ b/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java @@ -102,8 +102,14 @@ public class AvroKeyOutputFormat<T> extends AvroOutputFormatBase<AvroKey<T>, Nul GenericData dataModel = AvroSerialization.createDataModel(conf); - return mRecordWriterFactory.create - (writerSchema, dataModel, getCompressionCodec(context), - getAvroFileOutputStream(context), getSyncInterval(context)); + OutputStream out = getAvroFileOutputStream(context); + try { + return mRecordWriterFactory.create + (writerSchema, dataModel, getCompressionCodec(context), + out, getSyncInterval(context)); + } catch (IOException e) { + out.close(); + throw e; + } } } http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java ---------------------------------------------------------------------- diff --git a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java b/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java index b844abd..7fd5d38 100644 --- a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java +++ b/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java @@ -19,6 +19,7 @@ package org.apache.avro.mapreduce; import java.io.IOException; +import java.io.OutputStream; import org.apache.avro.generic.GenericData; import org.apache.avro.hadoop.io.AvroDatumConverter; @@ -59,8 +60,14 @@ public class AvroKeyValueOutputFormat<K, V> extends AvroOutputFormatBase<K, V> { GenericData dataModel = AvroSerialization.createDataModel(conf); - return new AvroKeyValueRecordWriter<K, V>(keyConverter, valueConverter, - dataModel, getCompressionCodec(context), getAvroFileOutputStream(context), - getSyncInterval(context)); + OutputStream out = getAvroFileOutputStream(context); + try { + return new AvroKeyValueRecordWriter<K, V>(keyConverter, valueConverter, + dataModel, getCompressionCodec(context), out, + getSyncInterval(context)); + } catch (IOException e) { + out.close(); + throw e; + } } } http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java ---------------------------------------------------------------------- diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java index f3cfe24..4095740 100644 --- a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java +++ b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java @@ -58,12 +58,17 @@ public class IdlTool implements Tool { parser = new Idl(in); } - if (args.size() == 2 && ! "-".equals(args.get(1))) { + if (args.size() == 2 && !"-".equals(args.get(1))) { parseOut = new PrintStream(new FileOutputStream(args.get(1))); } Protocol p = parser.CompilationUnit(); - parseOut.print(p.toString(true)); + try { + parseOut.print(p.toString(true)); + } finally { + if (parseOut != out) // Close only the newly created FileOutputStream + parseOut.close(); + } return 0; } http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java ---------------------------------------------------------------------- diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java b/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java index 93c827c..aa689ec 100644 --- a/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java +++ b/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java @@ -89,9 +89,19 @@ public class Main { } } System.err.print("Version "); - printStream(Main.class.getClassLoader().getResourceAsStream("VERSION.txt")); + InputStream versionInput = Main.class.getClassLoader().getResourceAsStream("VERSION.txt"); + try { + printStream(versionInput); + } finally { + versionInput.close(); + } System.err.print(" of "); - printHead(Main.class.getClassLoader().getResourceAsStream("META-INF/NOTICE"), 5); + InputStream noticeInput = Main.class.getClassLoader().getResourceAsStream("META-INF/NOTICE"); + try { + printHead(noticeInput, 5); + } finally { + noticeInput.close(); + } System.err.println("----------------"); System.err.println("Available tools:");
