Author: cutting
Date: Tue Sep 9 22:06:00 2014
New Revision: 1623881
URL: http://svn.apache.org/r1623881
Log:
AVRO-1583. Java: Add stdin support to the tojson tool. Contributed by Clément
Mahtieu.
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java
avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
Modified: avro/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1623881&r1=1623880&r2=1623881&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Tue Sep 9 22:06:00 2014
@@ -13,6 +13,9 @@ Trunk (not yet released)
AVRO-843. C#: Change Visual Studio project files to specify .NET 3.5.
(Dmitry Kovalev via cutting)
+ AVRO-1583. Java: Add stdin support to the tojson tool.
+ (Clément Mahtieu via cutting)
+
BUG FIXES
AVRO-1553. Java: MapReduce never uses MapOutputValueSchema (tomwhite)
Modified:
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java?rev=1623881&r1=1623880&r2=1623881&view=diff
==============================================================================
---
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java
(original)
+++
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java
Tue Sep 9 22:06:00 2014
@@ -17,6 +17,7 @@
*/
package org.apache.avro.tool;
+import java.io.BufferedInputStream;
import java.io.InputStream;
import java.io.PrintStream;
import java.util.List;
@@ -26,8 +27,7 @@ import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import org.apache.avro.Schema;
-import org.apache.avro.file.FileReader;
-import org.apache.avro.file.DataFileReader;
+import org.apache.avro.file.DataFileStream;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
@@ -59,30 +59,35 @@ public class DataFileReadTool implements
List<String> nargs = optionSet.nonOptionArguments();
if (nargs.size() != 1) {
- // Unlike other commands, "-" can't be used for stdin, because
- // we can only use seekable files.
- err.println("tojson --pretty input-file");
- err.println(" converts Avro data file to JSON.");
+ printHelp(err);
+ err.println();
optionParser.printHelpOn(err);
return 1;
}
+ BufferedInputStream inStream = Util.fileOrStdin(nargs.get(0), stdin);
+
GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
- FileReader<Object> fileReader =
- DataFileReader.openReader(Util.openSeekableFromFS(nargs.get(0)), reader);
+ DataFileStream<Object> streamReader = new DataFileStream<Object>(inStream,
reader);
try {
- Schema schema = fileReader.getSchema();
+ Schema schema = streamReader.getSchema();
DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, out,
pretty);
- for (Object datum : fileReader)
+ for (Object datum : streamReader)
writer.write(datum, encoder);
encoder.flush();
out.println();
out.flush();
} finally {
- fileReader.close();
+ streamReader.close();
}
return 0;
}
+ private void printHelp(PrintStream ps) {
+ ps.println("tojson --pretty input-file");
+ ps.println();
+ ps.println(getShortDescription());
+ ps.println("A dash ('-') can be given as an input file to use stdin");
+ }
}
Modified:
avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java?rev=1623881&r1=1623880&r2=1623881&view=diff
==============================================================================
---
avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
(original)
+++
avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
Tue Sep 9 22:06:00 2014
@@ -23,9 +23,11 @@ import static org.junit.Assert.fail;
import java.io.ByteArrayOutputStream;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.InputStream;
import java.io.PrintStream;
import java.io.StringBufferInputStream;
import java.util.ArrayList;
@@ -83,25 +85,35 @@ public class TestDataFileTools {
}
private String run(Tool tool, String... args) throws Exception {
+ return run(tool, null, args);
+ }
+
+ private String run(Tool tool, InputStream stdin, String... args) throws
Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream p = new PrintStream(baos);
tool.run(
- null, // stdin
+ stdin,
p, // stdout
null, // stderr
Arrays.asList(args));
return baos.toString("UTF-8").replace("\r", "");
}
-
+
@Test
public void testRead() throws Exception {
- assertEquals(jsonData.toString(),
+ assertEquals(jsonData,
run(new DataFileReadTool(), sampleFile.getPath()));
}
+
+ @Test
+ public void testReadStdin() throws Exception {
+ FileInputStream stdin = new FileInputStream(sampleFile);
+ assertEquals(jsonData, run(new DataFileReadTool(), stdin, "-"));
+ }
@Test
public void testReadToJsonPretty() throws Exception {
- assertEquals(jsonData.toString(),
+ assertEquals(jsonData,
run(new DataFileReadTool(), "--pretty", sampleFile.getPath()));
}
