http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java ---------------------------------------------------------------------- diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java index 9661e4e..0270b71 100644 --- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java +++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java @@ -52,7 +52,7 @@ public class TestDataFileTools { static String jsonData; static Schema schema; static File schemaFile; - + private static final String KEY_NEEDING_ESCAPES = "trn\\\r\t\n"; private static final String ESCAPED_KEY = "trn\\\\\\r\\t\\n"; @@ -65,7 +65,7 @@ public class TestDataFileTools { FileWriter fw = new FileWriter(schemaFile); fw.append(schema.toString()); fw.close(); - + DataFileWriter<Object> writer = new DataFileWriter<Object>(new GenericDatumWriter<Object>(schema)) .setMeta(KEY_NEEDING_ESCAPES, "") @@ -80,10 +80,10 @@ public class TestDataFileTools { writer.flush(); writer.close(); - + jsonData = builder.toString(); } - + private String run(Tool tool, String... args) throws Exception { return run(tool, null, args); } @@ -110,49 +110,49 @@ public class TestDataFileTools { FileInputStream stdin = new FileInputStream(sampleFile); assertEquals(jsonData, run(new DataFileReadTool(), stdin, "-")); } - + @Test public void testReadToJsonPretty() throws Exception { assertEquals(jsonData, run(new DataFileReadTool(), "--pretty", sampleFile.getPath())); } - + @Test public void testGetMeta() throws Exception { String output = run(new DataFileGetMetaTool(), sampleFile.getPath()); assertTrue(output, output.contains("avro.schema\t"+schema.toString()+"\n")); assertTrue(output, output.contains(ESCAPED_KEY+"\t\n")); } - + @Test public void testGetMetaForSingleKey() throws Exception { assertEquals(schema.toString() + "\n", run(new DataFileGetMetaTool(), sampleFile.getPath(), "--key", "avro.schema")); } - + @Test public void testGetSchema() throws Exception { assertEquals(schema.toString() + "\n", run(new DataFileGetSchemaTool(), sampleFile.getPath())); } - + @Test public void testWriteWithDeflate() throws Exception { testWrite("deflate", Arrays.asList("--codec", "deflate"), "deflate"); } - + @Test public void testWrite() throws Exception { testWrite("plain", Collections.<String>emptyList(), "null"); } - - public void testWrite(String name, List<String> extra, String expectedCodec) + + public void testWrite(String name, List<String> extra, String expectedCodec) throws Exception { testWrite(name, extra, expectedCodec, "-schema", schema.toString()); testWrite(name, extra, expectedCodec, "-schema-file", schemaFile.toString()); } - public void testWrite(String name, List<String> extra, String expectedCodec, String... extraArgs) + public void testWrite(String name, List<String> extra, String expectedCodec, String... extraArgs) throws Exception { File outFile = AvroTestUtil.tempFile(getClass(), TestDataFileTools.class + ".testWrite." + name + ".avro"); @@ -171,7 +171,7 @@ public class TestDataFileTools { args); out.close(); fout.close(); - + // Read it back, and make sure it's valid. GenericDatumReader<Object> reader = new GenericDatumReader<Object>(); DataFileReader<Object> fileReader = new DataFileReader<Object>(outFile,reader); @@ -188,7 +188,7 @@ public class TestDataFileTools { } assertEquals(expectedCodec, codecStr); } - + @Test public void testFailureOnWritingPartialJSONValues() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -197,7 +197,7 @@ public class TestDataFileTools { new DataFileWriteTool().run( new StringBufferInputStream("{"), new PrintStream(out), // stdout - null, // stderr + null, // stderr Arrays.asList("-schema", "{ \"type\":\"record\", \"fields\":" + "[{\"name\":\"foo\", \"type\":\"string\"}], " + "\"name\":\"boring\" }", "-")); @@ -206,7 +206,7 @@ public class TestDataFileTools { // expected } } - + @Test public void testWritingZeroJsonValues() throws Exception { File outFile = writeToAvroFile("zerojsonvalues", @@ -214,10 +214,10 @@ public class TestDataFileTools { ""); assertEquals(0, countRecords(outFile)); } - + private int countRecords(File outFile) throws IOException { GenericDatumReader<Object> reader = new GenericDatumReader<Object>(); - DataFileReader<Object> fileReader = + DataFileReader<Object> fileReader = new DataFileReader<Object>(outFile,reader); int i = 0; for (@SuppressWarnings("unused") Object datum : fileReader) { @@ -229,12 +229,12 @@ public class TestDataFileTools { @Test public void testDifferentSeparatorsBetweenJsonRecords() throws Exception { File outFile = writeToAvroFile( - "seperators", - "{ \"type\":\"array\", \"items\":\"int\" }", + "seperators", + "{ \"type\":\"array\", \"items\":\"int\" }", "[] [] []\n[][3] "); assertEquals(5, countRecords(outFile)); } - + public File writeToAvroFile(String testName, String schema, String json) throws Exception { File outFile = AvroTestUtil.tempFile(getClass(), TestDataFileTools.class + "." + testName + ".avro"); @@ -249,5 +249,5 @@ public class TestDataFileTools { fout.close(); return outFile; } - + }
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java ---------------------------------------------------------------------- diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java index 557cac9..c5e3c97 100644 --- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java +++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java @@ -37,22 +37,22 @@ import org.junit.BeforeClass; import org.junit.Test; /** - * Tests both {@link JsonToBinaryFragmentTool} + * Tests both {@link JsonToBinaryFragmentTool} * and {@link BinaryFragmentToJsonTool}. */ public class TestJsonToFromBinaryFragmentTools { private static final String STRING_SCHEMA = Schema.create(Type.STRING).toString(); private static final String UTF8 = "utf-8"; - private static final String AVRO = + private static final String AVRO = "ZLong string implies readable length encoding."; - private static final String JSON = + private static final String JSON = "\"Long string implies readable length encoding.\"\n"; @Test public void testBinaryToJson() throws Exception { binaryToJson(AVRO, JSON, STRING_SCHEMA); } - + @Test public void testJsonToBinary() throws Exception { jsonToBinary(JSON, AVRO, STRING_SCHEMA); @@ -82,7 +82,7 @@ public class TestJsonToFromBinaryFragmentTools { public void testBinaryToJsonSchemaFile() throws Exception { binaryToJson(AVRO, JSON, "--schema-file", schemaFile()); } - + @Test public void testJsonToBinarySchemaFile() throws Exception { jsonToBinary(JSON, AVRO, "--schema-file", schemaFile()); @@ -91,7 +91,7 @@ public class TestJsonToFromBinaryFragmentTools { private void binaryToJson(String avro, String json, String... options) throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintStream p = new PrintStream(new BufferedOutputStream(baos)); - + List<String> args = new ArrayList<String>(); args.addAll(Arrays.asList(options)); args.add("-"); http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/tools/src/test/java/org/apache/avro/tool/TestRpcReceiveAndSendTools.java ---------------------------------------------------------------------- diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestRpcReceiveAndSendTools.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestRpcReceiveAndSendTools.java index 97e527d..7e38198 100644 --- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestRpcReceiveAndSendTools.java +++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestRpcReceiveAndSendTools.java @@ -27,7 +27,7 @@ import java.util.Arrays; import org.junit.Test; public class TestRpcReceiveAndSendTools { - + /** * Starts a server (using the tool) and sends a single message to it. */ @@ -38,7 +38,7 @@ public class TestRpcReceiveAndSendTools { ByteArrayOutputStream baos1 = new ByteArrayOutputStream(); PrintStream p1 = new PrintStream(baos1); RpcReceiveTool receive = new RpcReceiveTool(); - receive.run1(null, p1, System.err, + receive.run1(null, p1, System.err, Arrays.asList("http://0.0.0.0:0/";, protocolFile, "hello", "-data", "\"Hello!\"")); @@ -47,10 +47,10 @@ public class TestRpcReceiveAndSendTools { RpcSendTool send = new RpcSendTool(); send.run(null, p2, System.err, Arrays.asList("http://127.0.0.1:"+receive.server.getPort()+"/", - protocolFile, "hello", + protocolFile, "hello", "-data", "{ \"greeting\": \"Hi!\" }")); receive.run2(System.err); - + assertTrue(baos1.toString("UTF-8").replace("\r", "") .endsWith("hello\t{\"greeting\":\"Hi!\"}\n")); assertEquals("\"Hello!\"\n", baos2.toString("UTF-8").replace("\r", "")); http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/tools/src/test/java/org/apache/avro/tool/TestTextFileTools.java ---------------------------------------------------------------------- diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestTextFileTools.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestTextFileTools.java index eee4027..0e3c5e8 100644 --- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestTextFileTools.java +++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestTextFileTools.java @@ -57,7 +57,7 @@ public class TestTextFileTools { static ByteBuffer[] lines; static Schema schema; static File schemaFile; - + @BeforeClass public static void writeRandomFile() throws IOException { schema = Schema.create(Type.BYTES); @@ -82,7 +82,7 @@ public class TestTextFileTools { } out.close(); } - + private void fromText(String name, String... args) throws Exception { File avroFile = AvroTestUtil.tempFile(getClass(), name + ".avro"); @@ -92,7 +92,7 @@ public class TestTextFileTools { arglist.add(avroFile.toString()); new FromTextTool().run(null, null, null, arglist); - + // Read it back, and make sure it's valid. DataFileReader<ByteBuffer> file = new DataFileReader<ByteBuffer> (avroFile, new GenericDatumReader<ByteBuffer>()); @@ -104,7 +104,7 @@ public class TestTextFileTools { } assertEquals(COUNT, i); } - + @Test public void testFromText() throws Exception { fromText("null", "--codec", "null"); @@ -128,7 +128,7 @@ public class TestTextFileTools { arglist.add(outFile.toString()); new ToTextTool().run(null, null, null, arglist); - + // Read it back, and make sure it's valid. InputStream orig = new BufferedInputStream(new FileInputStream(linesFile)); InputStream after = new BufferedInputStream(new FileInputStream(outFile)); http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/tools/src/test/java/org/apache/avro/tool/TestToTrevniTool.java ---------------------------------------------------------------------- diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestToTrevniTool.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestToTrevniTool.java index 31f1ab6..164acaf 100644 --- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestToTrevniTool.java +++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestToTrevniTool.java @@ -50,7 +50,7 @@ public class TestToTrevniTool { new ToTrevniTool().run(null, p, null, Arrays.asList(args)); return baos.toString("UTF-8").replace("\r", ""); } - + @Test public void test() throws Exception { Schema schema = Schema.parse(SCHEMA_FILE); http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnReader.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnReader.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnReader.java index f7514db..97ee485 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnReader.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnReader.java @@ -50,7 +50,7 @@ public class AvroColumnReader<D> private GenericData model; private Schema fileSchema; private Schema readSchema; - + private ColumnValues[] values; private int[] arrayWidths; private int column; // current index in values @@ -125,16 +125,16 @@ public class AvroColumnReader<D> switch (read.getType()) { case NULL: case BOOLEAN: case INT: case LONG: - case FLOAT: case DOUBLE: - case BYTES: case STRING: + case FLOAT: case DOUBLE: + case BYTES: case STRING: case ENUM: case FIXED: if (read.getType() != write.getType()) throw new TrevniRuntimeException("Type mismatch: "+read+" & "+write); break; - case MAP: + case MAP: findDefaults(read.getValueType(), write.getValueType()); break; - case ARRAY: + case ARRAY: findDefaults(read.getElementType(), write.getElementType()); break; case UNION: @@ -145,7 +145,7 @@ public class AvroColumnReader<D> findDefaults(s, write.getTypes().get(index)); } break; - case RECORD: + case RECORD: for (Field f : read.getFields()) { Field g = write.getField(f.name()); if (g == null) @@ -200,7 +200,7 @@ public class AvroColumnReader<D> final int startColumn = column; switch (s.getType()) { - case MAP: + case MAP: int size = values[column].nextLength(); Map map = (Map)new HashMap(size); for (int i = 0; i < size; i++) { @@ -211,7 +211,7 @@ public class AvroColumnReader<D> } column = startColumn + arrayWidths[startColumn]; return map; - case RECORD: + case RECORD: Object record = model.newRecord(null, s); Map<String,Object> rDefaults = defaults.get(s.getFullName()); for (Field f : s.getFields()) { @@ -221,7 +221,7 @@ public class AvroColumnReader<D> model.setField(record, f.name(), f.pos(), value); } return record; - case ARRAY: + case ARRAY: int length = values[column].nextLength(); List elements = (List)new GenericData.Array(length, s); for (int i = 0; i < length; i++) { @@ -254,7 +254,7 @@ public class AvroColumnReader<D> private Object nextValue(Schema s, int column) throws IOException { Object v = values[column].nextValue(); - + switch (s.getType()) { case ENUM: return model.createEnum(s.getEnumSymbols().get((Integer)v), s); http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnWriter.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnWriter.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnWriter.java index abb5682..caf7fd5 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnWriter.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnWriter.java @@ -89,14 +89,14 @@ public class AvroColumnWriter<D> { assert(count == writer.getColumnCount()); writer.endRow(); } - + private int write(Object o, Schema s, int column) throws IOException { if (isSimple(s)) { writeValue(o, s, column); return column+1; } switch (s.getType()) { - case MAP: + case MAP: Map<?,?> map = (Map)o; writer.writeLength(map.size(), column); for (Map.Entry e : map.entrySet()) { @@ -106,11 +106,11 @@ public class AvroColumnWriter<D> { assert(c == column+arrayWidths[column]); } return column+arrayWidths[column]; - case RECORD: + case RECORD: for (Field f : s.getFields()) column = write(model.getField(o,f.name(),f.pos()), f.schema(), column); return column; - case ARRAY: + case ARRAY: Collection elements = (Collection)o; writer.writeLength(elements.size(), column); if (isSimple(s.getElementType())) { // optimize simple arrays @@ -151,7 +151,7 @@ public class AvroColumnWriter<D> { private void writeValue(Object value, Schema s, int column) throws IOException { - + switch (s.getType()) { case STRING: if (value instanceof Utf8) // convert Utf8 to String @@ -160,7 +160,7 @@ public class AvroColumnWriter<D> { case ENUM: if (value instanceof Enum) value = ((Enum)value).ordinal(); - else + else value = s.getEnumOrdinal(value.toString()); break; case FIXED: http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java index 2f9a3ef..a546c14 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java @@ -71,9 +71,9 @@ class AvroColumnator { if (seen.containsKey(s)) // catch recursion throw new TrevniRuntimeException("Cannot shred recursive schemas: "+s); seen.put(s, s); - + switch (s.getType()) { - case MAP: + case MAP: path = path == null ? ">" : path+">"; int start = columns.size(); ColumnMetaData p = addColumn(path, ValueType.NULL, parent, true); @@ -85,7 +85,7 @@ class AvroColumnator { for (Field field : s.getFields()) // flatten fields to columns columnize(p(path, field.name(), "#"), field.schema(), parent, isArray); break; - case ARRAY: + case ARRAY: path = path == null ? "[]" : path+"[]"; addArrayColumn(path, s.getElementType(), parent); break; @@ -131,7 +131,7 @@ class AvroColumnator { // complex array: insert a parent column with lengths int start = columns.size(); ColumnMetaData array = addColumn(path, ValueType.NULL, parent, true); - columnize(path, element, array, false); + columnize(path, element, array, false); arrayWidths.set(start, columns.size()-start); // fixup with actual width } @@ -139,8 +139,8 @@ class AvroColumnator { switch (s.getType()) { case NULL: case BOOLEAN: case INT: case LONG: - case FLOAT: case DOUBLE: - case BYTES: case STRING: + case FLOAT: case DOUBLE: + case BYTES: case STRING: case ENUM: case FIXED: return true; default: @@ -165,4 +165,4 @@ class AvroColumnator { } } -} +} http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniInputFormat.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniInputFormat.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniInputFormat.java index 47bec01..003c266 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniInputFormat.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniInputFormat.java @@ -79,9 +79,9 @@ public class AvroTrevniInputFormat<T> private long row; public AvroWrapper<T> createKey() { return new AvroWrapper<T>(null); } - + public NullWritable createValue() { return NullWritable.get(); } - + public boolean next(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException { if (!reader.hasNext()) @@ -90,13 +90,13 @@ public class AvroTrevniInputFormat<T> row++; return true; } - + public float getProgress() throws IOException { return row / rows; } - + public long getPos() throws IOException { return row; } public void close() throws IOException { reader.close(); } - + }; } http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniOutputFormat.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniOutputFormat.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniOutputFormat.java index 60b432b..ca71107 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniOutputFormat.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniOutputFormat.java @@ -52,7 +52,7 @@ public class AvroTrevniOutputFormat <T> /** The file name extension for trevni files. */ public final static String EXT = ".trv"; - + public static final String META_PREFIX = "trevni.meta."; /** Add metadata to job output files.*/ @@ -84,7 +84,7 @@ public class AvroTrevniOutputFormat <T> private AvroColumnWriter<T> writer = new AvroColumnWriter<T>(schema, meta, ReflectData.get()); - + private void flush() throws IOException { OutputStream out = fs.create(new Path(dir, "part-"+(part++)+EXT)); try { http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyInputFormat.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyInputFormat.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyInputFormat.java index 89287e6..30d1682 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyInputFormat.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyInputFormat.java @@ -29,28 +29,28 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; /** * An {@link org.apache.hadoop.mapreduce.InputFormat} for Trevni files. - * + * * This implement was modeled off * {@link org.apache.avro.mapreduce.AvroKeyInputFormat} to allow for easy * transition - * + * * A MapReduce InputFormat that can handle Trevni container files. * * <p>Keys are AvroKey wrapper objects that contain the Trevni data. Since Trevni * container files store only records (not key/value pairs), the value from * this InputFormat is a NullWritable.</p> - * + * * <p> * A subset schema to be read may be specified with * {@link org.apache.avro.mapreduce.AvroJob#setInputKeySchema}. */ public class AvroTrevniKeyInputFormat<T> extends FileInputFormat<AvroKey<T>, NullWritable> { - + @Override public RecordReader<AvroKey<T>, NullWritable> createRecordReader( InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { - + return new AvroTrevniKeyRecordReader<T>(); } http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyOutputFormat.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyOutputFormat.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyOutputFormat.java index 34354f7..cc1f48f 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyOutputFormat.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyOutputFormat.java @@ -29,17 +29,17 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; /** An {@link org.apache.hadoop.mapreduce.OutputFormat} that writes Avro data to * Trevni files. * - * This implement was modeled off + * This implement was modeled off * {@link org.apache.avro.mapreduce.AvroKeyOutputFormat} to allow for easy * transition - * + * * FileOutputFormat for writing Trevni container files. * * <p>Since Trevni container files only contain records (not key/value pairs), this output * format ignores the value.</p> * * @param <T> The (java) type of the Trevni data to write. - * + * * <p>Writes a directory of files per task, each comprising a single filesystem * block. To reduce the number of files, increase the default filesystem block * size for the job. Each task also requires enough memory to buffer a @@ -50,7 +50,7 @@ public class AvroTrevniKeyOutputFormat <T> extends FileOutputFormat<AvroKey<T>, @Override public RecordWriter<AvroKey<T>, NullWritable> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { - + return new AvroTrevniKeyRecordWriter<T>(context ); } } http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyRecordReader.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyRecordReader.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyRecordReader.java index 88f2410..4751983 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyRecordReader.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyRecordReader.java @@ -29,10 +29,10 @@ import org.apache.hadoop.io.NullWritable; * @param <T> The (java) type of data in Trevni container file. */ public class AvroTrevniKeyRecordReader<T> extends AvroTrevniRecordReaderBase<AvroKey<T>, NullWritable, T> { - + /** A reusable object to hold records of the Avro container file. */ private final AvroKey<T> mCurrentKey = new AvroKey<T>(); - + /** {@inheritDoc} */ @Override public AvroKey<T> getCurrentKey() throws IOException, http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueInputFormat.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueInputFormat.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueInputFormat.java index c16e381..2fb1b33 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueInputFormat.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueInputFormat.java @@ -28,11 +28,11 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; /** * An {@link org.apache.hadoop.mapreduce.InputFormat} for Trevni files. - * + * * This implement was modeled off * {@link org.apache.avro.mapreduce.AvroKeyValueInputFormat} to allow for easy * transition - * + * * <p> * A MapReduce InputFormat that reads from Trevni container files of key/value generic records. * @@ -43,7 +43,7 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; * * @param <K> The type of the Trevni key to read. * @param <V> The type of the Trevni value to read. - * + * * <p> * A subset schema to be read may be specified with * {@link org.apache.avro.mapreduce.AvroJob#setInputKeySchema} and @@ -56,7 +56,7 @@ public class AvroTrevniKeyValueInputFormat<K, V> extends FileInputFormat<AvroKe public RecordReader<AvroKey<K>, AvroValue<V>> createRecordReader( InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { - + return new AvroTrevniKeyValueRecordReader<K, V>(); } http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueOutputFormat.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueOutputFormat.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueOutputFormat.java index c508df3..9f6b23f 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueOutputFormat.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueOutputFormat.java @@ -28,11 +28,11 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; /** An {@link org.apache.hadoop.mapreduce.OutputFormat} that writes Avro data to * Trevni files. - * - * This implement was modeled off + * + * This implement was modeled off * {@link org.apache.avro.mapreduce.AvroKeyValueOutputFormat} to allow for easy - * transition - * + * transition + * * * FileOutputFormat for writing Trevni container files of key/value pairs. * * <p>Since Trevni container files can only contain records (not key/value pairs), this @@ -46,19 +46,19 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; * * @param <K> The type of key. If an Avro type, it must be wrapped in an <code>AvroKey</code>. * @param <V> The type of value. If an Avro type, it must be wrapped in an <code>AvroValue</code>. - * + * * <p>Writes a directory of files per task, each comprising a single filesystem * block. To reduce the number of files, increase the default filesystem block * size for the job. Each task also requires enough memory to buffer a * filesystem block. */ -public class AvroTrevniKeyValueOutputFormat <K, V> extends FileOutputFormat<AvroKey<K>, AvroValue<V>> { - +public class AvroTrevniKeyValueOutputFormat <K, V> extends FileOutputFormat<AvroKey<K>, AvroValue<V>> { + /** {@inheritDoc} */ @Override public RecordWriter<AvroKey<K>, AvroValue<V>> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { - + return new AvroTrevniKeyValueRecordWriter<K, V>(context ); } } http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordReader.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordReader.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordReader.java index 3aa956f..31fe3f4 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordReader.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordReader.java @@ -41,7 +41,7 @@ public class AvroTrevniKeyValueRecordReader<K, V> extends AvroTrevniRecordReader private final AvroKey<K> mCurrentKey = new AvroKey<K>(); /** The current value the reader is on. */ private final AvroValue<V> mCurrentValue = new AvroValue<V>(); - + /** {@inheritDoc} */ @Override public AvroKey<K> getCurrentKey() throws IOException, http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordWriter.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordWriter.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordWriter.java index 136ef06..9038028 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordWriter.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordWriter.java @@ -46,16 +46,16 @@ public class AvroTrevniKeyValueRecordWriter <K, V> extends AvroTrevniRecordWrite /** The writer schema for the generic record entries of the Trevni container file. */ Schema mKeyValuePairSchema; - + /** A reusable Avro generic record for writing key/value pairs to the file. */ AvroKeyValue<Object, Object> keyValueRecord; - + /** A helper object that converts the input key to an Avro datum. */ AvroDatumConverter<K, ?> keyConverter; - + /** A helper object that converts the input value to an Avro datum. */ AvroDatumConverter<V, ?> valueConverter; - + /** * Constructor. * @param context The TaskAttempContext to supply the writer with information form the job configuration @@ -63,30 +63,30 @@ public class AvroTrevniKeyValueRecordWriter <K, V> extends AvroTrevniRecordWrite public AvroTrevniKeyValueRecordWriter(TaskAttemptContext context) throws IOException { super(context); - + mKeyValuePairSchema = initSchema(context); keyValueRecord = new AvroKeyValue<Object, Object>(new GenericData.Record(mKeyValuePairSchema)); } - + /** {@inheritDoc} */ @Override public void write(AvroKey<K> key, AvroValue<V> value) throws IOException, InterruptedException { - + keyValueRecord.setKey(key.datum()); keyValueRecord.setValue(value.datum()); writer.write(keyValueRecord.get()); if (writer.sizeEstimate() >= blockSize) // block full flush(); } - + /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override protected Schema initSchema(TaskAttemptContext context) { AvroDatumConverterFactory converterFactory = new AvroDatumConverterFactory( context.getConfiguration()); - + keyConverter = converterFactory.create((Class<K>) context .getOutputKeyClass()); valueConverter = converterFactory.create((Class<V>) context @@ -95,7 +95,7 @@ public class AvroTrevniKeyValueRecordWriter <K, V> extends AvroTrevniRecordWrite // Create the generic record schema for the key/value pair. return AvroKeyValue.getSchema( keyConverter.getWriterSchema(), valueConverter.getWriterSchema()); - + } - + } http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordReaderBase.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordReaderBase.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordReaderBase.java index b68669f..1a3114f 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordReaderBase.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordReaderBase.java @@ -37,16 +37,16 @@ import org.apache.trevni.avro.HadoopInput; * @param <T> The type of the entries within the Trevni container file being read. */ public abstract class AvroTrevniRecordReaderBase<K, V, T> extends RecordReader<K, V> { - + /** The Trevni file reader */ private AvroColumnReader<T> reader; - + /** Number of rows in the Trevni file */ private float rows; - + /** The current row number being read in */ private long row; - + /** A reusable object to hold records of the Avro container file. */ private T mCurrentRecord; @@ -60,11 +60,11 @@ public abstract class AvroTrevniRecordReaderBase<K, V, T> extends RecordReader<K final AvroColumnReader.Params params = new AvroColumnReader.Params(new HadoopInput(file.getPath(), context.getConfiguration())); params.setModel(ReflectData.get()); - + if (AvroJob.getInputKeySchema(context.getConfiguration()) != null) { params.setSchema(AvroJob.getInputKeySchema(context.getConfiguration())); } - + reader = new AvroColumnReader<T>(params); rows = reader.getRowCount(); } @@ -78,7 +78,7 @@ public abstract class AvroTrevniRecordReaderBase<K, V, T> extends RecordReader<K row++; return true; } - + /** * Gets the current record read from the Trevni container file. * @@ -93,7 +93,7 @@ public abstract class AvroTrevniRecordReaderBase<K, V, T> extends RecordReader<K /** {@inheritDoc} */ @Override public void close() throws IOException { - reader.close(); + reader.close(); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordWriterBase.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordWriterBase.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordWriterBase.java index 94a332d..7f1d57b 100644 --- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordWriterBase.java +++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordWriterBase.java @@ -43,15 +43,15 @@ import org.apache.trevni.avro.AvroColumnWriter; * @param <T> The type of the entries within the Trevni container file being writen. */ public abstract class AvroTrevniRecordWriterBase<K,V, T> extends RecordWriter<K, V> { - + /** trevni file extension */ public final static String EXT = ".trv"; - + /** prefix of job configs that we care about */ public static final String META_PREFIX = "trevni.meta."; - - /** Counter that increments as new trevni files are create because the current file - * has exceeded the block size + + /** Counter that increments as new trevni files are create because the current file + * has exceeded the block size * */ protected int part = 0; @@ -60,31 +60,31 @@ public abstract class AvroTrevniRecordWriterBase<K,V, T> extends RecordWriter<K, /** This will be a unique directory linked to the task */ final Path dirPath; - + /** HDFS object */ final FileSystem fs; /** Current configured blocksize */ final long blockSize; - + /** Provided avro schema from the context */ protected Schema schema; - + /** meta data to be stored in the output file. */ protected ColumnFileMetaData meta; - + /** * Constructor. * @param context The TaskAttempContext to supply the writer with information form the job configuration */ public AvroTrevniRecordWriterBase(TaskAttemptContext context) throws IOException { - + schema = initSchema(context); meta = filterMetadata(context.getConfiguration()); writer = new AvroColumnWriter<T>(schema, meta, ReflectData.get()); Path outputPath = FileOutputFormat.getOutputPath(context); - + String dir = FileOutputFormat.getUniqueFile(context, "part", ""); dirPath = new Path(outputPath.toString() + "/" + dir); fs = dirPath.getFileSystem(context.getConfiguration()); @@ -97,8 +97,8 @@ public abstract class AvroTrevniRecordWriterBase<K,V, T> extends RecordWriter<K, * Use the task context to construct a schema for writing * @throws IOException */ - abstract protected Schema initSchema(TaskAttemptContext context); - + abstract protected Schema initSchema(TaskAttemptContext context); + /** * A Trevni flush will close the current file and prep a new writer * @throws IOException @@ -112,14 +112,14 @@ public abstract class AvroTrevniRecordWriterBase<K,V, T> extends RecordWriter<K, } writer = new AvroColumnWriter<T>(schema, meta, ReflectData.get()); } - + /** {@inheritDoc} */ @Override public void close(TaskAttemptContext arg0) throws IOException, InterruptedException { flush(); } - + static ColumnFileMetaData filterMetadata(final Configuration configuration) { final ColumnFileMetaData meta = new ColumnFileMetaData(); Iterator<Entry<String, String>> keyIterator = configuration.iterator(); http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/RandomData.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/RandomData.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/RandomData.java index 8f5976a..5d43aae 100644 --- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/RandomData.java +++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/RandomData.java @@ -45,7 +45,7 @@ public class RandomData implements Iterable<Object> { this.root = schema; this.count = count; } - + public Iterator<Object> iterator() { return new Iterator<Object>() { private int n; @@ -58,14 +58,14 @@ public class RandomData implements Iterable<Object> { public void remove() { throw new UnsupportedOperationException(); } }; } - + @SuppressWarnings(value="unchecked") private static Object generate(Schema schema, Random random, int d) { switch (schema.getType()) { case RECORD: GenericRecord record = new GenericData.Record(schema); for (Schema.Field field : schema.getFields()) { - Object value = (field.getJsonProp(USE_DEFAULT) == null) + Object value = (field.getJsonProp(USE_DEFAULT) == null) ? generate(field.schema(), random, d+1) : GenericData.get().getDefaultValue(field); record.put(field.name(), value); http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestMetadataFiltering.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestMetadataFiltering.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestMetadataFiltering.java index 920eb89..e3b9507 100644 --- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestMetadataFiltering.java +++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestMetadataFiltering.java @@ -27,15 +27,15 @@ public class TestMetadataFiltering { @Test public void testMetadataFiltering() throws Exception { JobConf job = new JobConf(); - + job.set(AvroTrevniOutputFormat.META_PREFIX + "test1", "1"); job.set(AvroTrevniOutputFormat.META_PREFIX + "test2", "2"); job.set("test3", "3"); job.set(AvroJob.TEXT_PREFIX + "test4", "4"); job.set(AvroTrevniOutputFormat.META_PREFIX + "test5", "5"); - + ColumnFileMetaData metadata = AvroTrevniOutputFormat.filterMetadata(job); - + assertTrue(metadata.get("test1") != null); assertTrue(new String(metadata.get("test1")).equals("1")); assertTrue(metadata.get("test2") != null); @@ -44,7 +44,7 @@ public class TestMetadataFiltering { assertTrue(new String(metadata.get("test5")).equals("5")); assertTrue(metadata.get("test3") == null); assertTrue(metadata.get("test4") == null); - + } - + } http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java index 06fdd09..9ca7344 100644 --- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java +++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java @@ -78,7 +78,7 @@ public class TestShredder { } @Test public void testDefaultValue() throws Exception { - String s = + String s = "{\"type\":\"record\",\"name\":\"R\",\"fields\":[" +SIMPLE_FIELDS+"," +"{\"name\":\"z\",\"type\":\"int\"," @@ -89,7 +89,7 @@ public class TestShredder { } @Test public void testNestedRecord() throws Exception { - String s = + String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"x\",\"type\":\"int\"}," +"{\"name\":\"R\",\"type\":"+SIMPLE_RECORD+"}," @@ -103,25 +103,25 @@ public class TestShredder { } @Test public void testNamedRecord() throws Exception { - String s = - "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" - +"{\"name\":\"R1\",\"type\":"+SIMPLE_RECORD+"}," - +"{\"name\":\"R2\",\"type\":\"R\"}" - +"]}"; - check(Schema.parse(s), - new ColumnMetaData("R1#x", ValueType.INT), - new ColumnMetaData("R1#y", ValueType.STRING), - new ColumnMetaData("R2#x", ValueType.INT), - new ColumnMetaData("R2#y", ValueType.STRING)); - } - + String s = + "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" + +"{\"name\":\"R1\",\"type\":"+SIMPLE_RECORD+"}," + +"{\"name\":\"R2\",\"type\":\"R\"}" + +"]}"; + check(Schema.parse(s), + new ColumnMetaData("R1#x", ValueType.INT), + new ColumnMetaData("R1#y", ValueType.STRING), + new ColumnMetaData("R2#x", ValueType.INT), + new ColumnMetaData("R2#y", ValueType.STRING)); + } + @Test public void testSimpleArray() throws Exception { String s = "{\"type\":\"array\",\"items\":\"long\"}"; check(Schema.parse(s), new ColumnMetaData("[]", ValueType.LONG).isArray(true)); } - private static final String RECORD_ARRAY = + private static final String RECORD_ARRAY = "{\"type\":\"array\",\"items\":"+SIMPLE_RECORD+"}"; @Test public void testArray() throws Exception { @@ -157,7 +157,7 @@ public class TestShredder { } @Test public void testNestedArray() throws Exception { - String s = + String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"x\",\"type\":\"int\"}," +"{\"name\":\"A\",\"type\":"+RECORD_ARRAY+"}," @@ -173,7 +173,7 @@ public class TestShredder { } @Test public void testNestedUnion() throws Exception { - String s = + String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"x\",\"type\":\"int\"}," +"{\"name\":\"u\",\"type\":"+UNION+"}," @@ -190,7 +190,7 @@ public class TestShredder { } @Test public void testUnionInArray() throws Exception { - String s = + String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"a\",\"type\":{\"type\":\"array\",\"items\":"+UNION+"}}" +"]}"; @@ -209,7 +209,7 @@ public class TestShredder { } @Test public void testArrayInUnion() throws Exception { - String s = + String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"a\",\"type\":[\"int\","+RECORD_ARRAY+"]}]}"; ColumnMetaData q = new ColumnMetaData("a/array",ValueType.NULL) http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestWordCount.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestWordCount.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestWordCount.java index d928a9f..79a8fff 100644 --- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestWordCount.java +++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestWordCount.java @@ -62,7 +62,7 @@ public class TestWordCount { collector.collect(new Pair<String,Long>(tokens.nextToken(),1L)); } } - + public static class ReduceImpl extends AvroReducer<String, Long, Pair<String, Long> > { @Override @@ -74,7 +74,7 @@ public class TestWordCount { sum += count; collector.collect(new Pair<String,Long>(word, sum)); } - } + } @Test public void runTestsInOrder() throws Exception { testOutputFormat(); @@ -87,26 +87,26 @@ public class TestWordCount { public void testOutputFormat() throws Exception { JobConf job = new JobConf(); - + WordCountUtil wordCountUtil = new WordCountUtil("trevniMapredTest"); - + wordCountUtil.writeLinesFile(); - + AvroJob.setInputSchema(job, STRING); AvroJob.setOutputSchema(job, Pair.getPairSchema(STRING,LONG)); - - AvroJob.setMapperClass(job, MapImpl.class); + + AvroJob.setMapperClass(job, MapImpl.class); AvroJob.setCombinerClass(job, ReduceImpl.class); AvroJob.setReducerClass(job, ReduceImpl.class); - + FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/in")); FileOutputFormat.setOutputPath(job, new Path(wordCountUtil.getDir().toString() + "/out")); FileOutputFormat.setCompressOutput(job, true); - + job.setOutputFormat(AvroTrevniOutputFormat.class); JobClient.runJob(job); - + wordCountUtil.validateCountsFile(); } @@ -118,20 +118,20 @@ public class TestWordCount { total += (Long)r.get("value"); } } - + public void testInputFormat() throws Exception { JobConf job = new JobConf(); WordCountUtil wordCountUtil = new WordCountUtil("trevniMapredTest"); - - + + Schema subSchema = Schema.parse("{\"type\":\"record\"," + "\"name\":\"PairValue\","+ - "\"fields\": [ " + - "{\"name\":\"value\", \"type\":\"long\"}" + + "\"fields\": [ " + + "{\"name\":\"value\", \"type\":\"long\"}" + "]}"); AvroJob.setInputSchema(job, subSchema); - AvroJob.setMapperClass(job, Counter.class); + AvroJob.setMapperClass(job, Counter.class); FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/out/*")); job.setInputFormat(AvroTrevniInputFormat.class); http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/WordCountUtil.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/WordCountUtil.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/WordCountUtil.java index 68af7a3..775992a 100644 --- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/WordCountUtil.java +++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/WordCountUtil.java @@ -60,13 +60,13 @@ public class WordCountUtil { public WordCountUtil (String testName) { this(testName, "part-00000"); } - + public WordCountUtil (String testName, String partDirName) { dir = new File("target/wc", testName); linesFiles = new File(new File(dir, "in"), "lines.avro"); countFiles = new File(new File(dir, "out"), partDirName + "/part-0.trv"); } - + public static final String[] LINES = new String[] { "the quick brown fox jumps over the lazy dog", "the cow jumps over the moon", @@ -93,7 +93,7 @@ public class WordCountUtil { public File getDir() { return dir; } - + public void writeLinesFile() throws IOException { FileUtil.fullyDelete(dir); DatumWriter<String> writer = new GenericDatumWriter<String>(); @@ -117,7 +117,7 @@ public class WordCountUtil { reader.close(); assertEquals(COUNTS.size(), numWords); } - + public void validateCountsFileGenericRecord() throws Exception { AvroColumnReader<GenericRecord > reader = new AvroColumnReader<GenericRecord > http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyValueWordCount.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyValueWordCount.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyValueWordCount.java index deea1ca..47cbee4 100644 --- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyValueWordCount.java +++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyValueWordCount.java @@ -43,13 +43,13 @@ import org.apache.trevni.avro.WordCountUtil; import org.junit.Test; public class TestKeyValueWordCount { - + private static long total = 0; static final Schema STRING = Schema.create(Schema.Type.STRING); static { GenericData.setStringType(STRING, GenericData.StringType.String); } static final Schema LONG = Schema.create(Schema.Type.LONG); - + private static class WordCountMapper extends Mapper<AvroKey<String>, NullWritable, Text, LongWritable> { private LongWritable mCount = new LongWritable(); @@ -76,12 +76,12 @@ public class TestKeyValueWordCount { } } - + private static class WordCountReducer extends Reducer< Text, LongWritable, AvroKey<String>, AvroValue<Long>> { - + AvroKey<String> resultKey = new AvroKey<String>(); AvroValue<Long> resultValue = new AvroValue<Long>(); - + @Override protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { long sum = 0; @@ -90,11 +90,11 @@ public class TestKeyValueWordCount { } resultKey.datum(key.toString()); resultValue.datum(sum); - + context.write(resultKey, resultValue); } } - + public static class Counter extends Mapper<AvroKey<String>, AvroValue<Long>, NullWritable, NullWritable> { @Override @@ -102,8 +102,8 @@ public class TestKeyValueWordCount { throws IOException, InterruptedException { total += value.datum(); } - } - + } + @Test public void testIOFormat() throws Exception { checkOutputFormat(); checkInputFormat(); @@ -111,49 +111,49 @@ public class TestKeyValueWordCount { public void checkOutputFormat() throws Exception { Job job = new Job(); - + WordCountUtil wordCountUtil = new WordCountUtil("trevniMapReduceKeyValueTest", "part-r-00000"); - + wordCountUtil.writeLinesFile(); - + AvroJob.setInputKeySchema(job, STRING); AvroJob.setOutputKeySchema(job, STRING); AvroJob.setOutputValueSchema(job, LONG); - + job.setMapperClass(WordCountMapper.class); job.setReducerClass(WordCountReducer.class); - + job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); - + FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/in")); FileOutputFormat.setOutputPath(job, new Path(wordCountUtil.getDir().toString() + "/out")); FileOutputFormat.setCompressOutput(job, true); - + job.setInputFormatClass(AvroKeyInputFormat.class); job.setOutputFormatClass(AvroTrevniKeyValueOutputFormat.class); job.waitForCompletion(true); - + wordCountUtil.validateCountsFileGenericRecord(); } - + public void checkInputFormat() throws Exception { Job job = new Job(); - + WordCountUtil wordCountUtil = new WordCountUtil("trevniMapReduceKeyValueTest"); - + job.setMapperClass(Counter.class); - + FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/out/*")); job.setInputFormatClass(AvroTrevniKeyValueInputFormat.class); - + job.setNumReduceTasks(0); job.setOutputFormatClass(NullOutputFormat.class); - + total = 0; job.waitForCompletion(true); assertEquals(WordCountUtil.TOTAL, total); - + } } http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyWordCount.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyWordCount.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyWordCount.java index 8623fb4..1ba50ff 100644 --- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyWordCount.java +++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyWordCount.java @@ -45,14 +45,14 @@ import org.apache.trevni.avro.mapreduce.AvroTrevniKeyOutputFormat; import org.junit.Test; public class TestKeyWordCount { - + private static long total = 0; static final Schema STRING = Schema.create(Schema.Type.STRING); static { GenericData.setStringType(STRING, GenericData.StringType.String); } static final Schema LONG = Schema.create(Schema.Type.LONG); - - + + private static class WordCountMapper extends Mapper<AvroKey<String>, NullWritable, Text, LongWritable> { private LongWritable mCount = new LongWritable(); @@ -79,33 +79,33 @@ public class TestKeyWordCount { } } - + private static class WordCountReducer extends Reducer< Text, LongWritable, AvroKey<GenericData.Record>, NullWritable> { - + private AvroKey<GenericData.Record> result ; - + @Override protected void setup(Context context) { result = new AvroKey<GenericData.Record>(); result.datum(new Record(Pair.getPairSchema(STRING,LONG))); } - + @Override protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { long count = 0; for (LongWritable value: values) { count += value.get(); } - + result.datum().put("key", key.toString()); result.datum().put("value", count); - + context.write(result, NullWritable.get()); } } - - + + public static class Counter extends Mapper<AvroKey<GenericData.Record>, NullWritable, NullWritable, NullWritable> { @Override @@ -114,8 +114,8 @@ public class TestKeyWordCount { total += (Long)key.datum().get("value"); } } - - + + @Test public void testIOFormat() throws Exception { checkOutputFormat(); checkInputFormat(); @@ -123,56 +123,56 @@ public class TestKeyWordCount { public void checkOutputFormat() throws Exception { Job job = new Job(); - + WordCountUtil wordCountUtil = new WordCountUtil("trevniMapReduceKeyTest", "part-r-00000"); - + wordCountUtil.writeLinesFile(); - + AvroJob.setInputKeySchema(job, STRING); AvroJob.setOutputKeySchema(job, Pair.getPairSchema(STRING,LONG)); - + job.setMapperClass(WordCountMapper.class); job.setReducerClass(WordCountReducer.class); - + job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); - + FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/in")); FileOutputFormat.setOutputPath(job, new Path(wordCountUtil.getDir().toString() + "/out")); FileOutputFormat.setCompressOutput(job, true); - + job.setInputFormatClass(AvroKeyInputFormat.class); job.setOutputFormatClass(AvroTrevniKeyOutputFormat.class); job.waitForCompletion(true); - + wordCountUtil.validateCountsFile(); } - + public void checkInputFormat() throws Exception { Job job = new Job(); - + WordCountUtil wordCountUtil = new WordCountUtil("trevniMapReduceKeyTest"); - + job.setMapperClass(Counter.class); Schema subSchema = Schema.parse("{\"type\":\"record\"," + "\"name\":\"PairValue\","+ - "\"fields\": [ " + - "{\"name\":\"value\", \"type\":\"long\"}" + + "\"fields\": [ " + + "{\"name\":\"value\", \"type\":\"long\"}" + "]}"); AvroJob.setInputKeySchema(job, subSchema); - + FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/out/*")); job.setInputFormatClass(AvroTrevniKeyInputFormat.class); - + job.setNumReduceTasks(0); job.setOutputFormatClass(NullOutputFormat.class); - + total = 0; job.waitForCompletion(true); assertEquals(WordCountUtil.TOTAL, total); - + } - + } http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/pom.xml ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/pom.xml b/lang/java/trevni/core/pom.xml index 8e96aa9..084b167 100644 --- a/lang/java/trevni/core/pom.xml +++ b/lang/java/trevni/core/pom.xml @@ -37,7 +37,7 @@ <artifactId>snappy-java</artifactId> <version>${snappy.version}</version> <scope>compile</scope> - </dependency> + </dependency> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-compress</artifactId> http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/BZip2Codec.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/BZip2Codec.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/BZip2Codec.java index 1c0d64b..9513ed4 100644 --- a/lang/java/trevni/core/src/main/java/org/apache/trevni/BZip2Codec.java +++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/BZip2Codec.java @@ -29,7 +29,7 @@ public class BZip2Codec extends Codec { private ByteArrayOutputStream outputBuffer; public static final int DEFAULT_BUFFER_SIZE = 64 * 1024; - + @Override ByteBuffer compress(ByteBuffer uncompressedData) throws IOException { ByteArrayOutputStream baos = getOutputBuffer(uncompressedData.remaining()); @@ -55,23 +55,23 @@ public class BZip2Codec extends Codec { byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; int readCount = -1; - + while ( (readCount = inputStream.read(buffer, compressedData.position(), buffer.length))> 0) { baos.write(buffer, 0, readCount); } - + ByteBuffer result = ByteBuffer.wrap(baos.toByteArray()); return result; } finally { inputStream.close(); } } - + private ByteArrayOutputStream getOutputBuffer(int suggestedLength) { if (null == outputBuffer) outputBuffer = new ByteArrayOutputStream(suggestedLength); outputBuffer.reset(); return outputBuffer; } - + } http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/BlockDescriptor.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/BlockDescriptor.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/BlockDescriptor.java index 5f2c0f3..cbb15bd 100644 --- a/lang/java/trevni/core/src/main/java/org/apache/trevni/BlockDescriptor.java +++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/BlockDescriptor.java @@ -25,13 +25,13 @@ class BlockDescriptor { int compressedSize; BlockDescriptor() {} - + BlockDescriptor(int rowCount, int uncompressedSize, int compressedSize) { this.rowCount = rowCount; this.uncompressedSize = uncompressedSize; this.compressedSize = compressedSize; } - + public void writeTo(OutputBuffer out) throws IOException { out.writeFixed32(rowCount); out.writeFixed32(uncompressedSize); http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnDescriptor.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnDescriptor.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnDescriptor.java index 6a052d1..3cbabac 100644 --- a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnDescriptor.java +++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnDescriptor.java @@ -76,7 +76,7 @@ class ColumnDescriptor<T extends Comparable> { firstValues[i] = in.<T>readValue(metaData.getType()); } dataStart = in.tell(); - + // compute blockStarts and firstRows Checksum checksum = Checksum.get(metaData); blockStarts = new long[blocks.length]; http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileReader.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileReader.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileReader.java index ec080b8..cb475f5 100644 --- a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileReader.java +++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileReader.java @@ -131,7 +131,7 @@ public class ColumnFileReader implements Closeable { for (int i = 0; i < columnCount; i++) columns[i].start = in.readFixed64(); } - + /** Return an iterator over values in the named column. */ public <T extends Comparable> ColumnValues<T> getValues(String columnName) throws IOException { http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileWriter.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileWriter.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileWriter.java index 0f4a21e..5eb92aa 100644 --- a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileWriter.java +++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileWriter.java @@ -68,7 +68,7 @@ public class ColumnFileWriter { if (parent != null && !seen.contains(parent.getName())) throw new TrevniRuntimeException("Parent must precede child: "+name); seen.add(name); - } + } } void incrementSize(int n) { size += n; } @@ -133,7 +133,7 @@ public class ColumnFileWriter { /** Write all rows added to the named output stream. */ public void writeTo(OutputStream out) throws IOException { writeHeader(out); - + for (int column = 0; column < columnCount; column++) columns[column].writeTo(out); } http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnOutputBuffer.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnOutputBuffer.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnOutputBuffer.java index b689915..3217753 100644 --- a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnOutputBuffer.java +++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnOutputBuffer.java @@ -90,9 +90,9 @@ class ColumnOutputBuffer { ? firstValues.get(firstValues.size()-1).length : 0) + data.position(); // data - + writer.incrementSize(sizeIncrement); - size += sizeIncrement; + size += sizeIncrement; buffer = new OutputBuffer(); rowCount = 0; http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java index a47fc85..9df153f 100644 --- a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java +++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java @@ -34,7 +34,7 @@ public class ColumnValues<T extends Comparable> private InputBuffer values; private int block = -1; private long row = 0; - private T previous; + private T previous; private int arrayLength; http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/DeflateCodec.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/DeflateCodec.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/DeflateCodec.java index 7e9a0be..96077bb 100644 --- a/lang/java/trevni/core/src/main/java/org/apache/trevni/DeflateCodec.java +++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/DeflateCodec.java @@ -45,13 +45,13 @@ class DeflateCodec extends Codec { writeAndClose(data, new InflaterOutputStream(baos, getInflater())); return ByteBuffer.wrap(baos.toByteArray()); } - + private void writeAndClose(ByteBuffer data, OutputStream out) throws IOException { out.write(data.array(), data.position(), data.remaining()); out.close(); } - + private Inflater getInflater() { if (null == inflater) inflater = new Inflater(true); @@ -65,7 +65,7 @@ class DeflateCodec extends Codec { deflater.reset(); return deflater; } - + private ByteArrayOutputStream getOutputBuffer(int suggestedLength) { if (null == outputBuffer) outputBuffer = new ByteArrayOutputStream(suggestedLength); http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java index e3bd415..4023731 100644 --- a/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java +++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java @@ -35,7 +35,7 @@ class InputBuffer { private int limit; // end of valid buffer data private CharsetDecoder utf8 = Charset.forName("UTF-8").newDecoder(); - + private int bitCount; // position in booleans private int runLength; // length of run @@ -233,7 +233,7 @@ class InputBuffer { } return (l >>> 1) ^ -(l & 1); // back to two's-complement } - + // splitting readLong up makes it faster because of the JVM does more // optimizations on small methods private long innerLongDecode(long l) throws IOException { @@ -302,7 +302,7 @@ class InputBuffer { byte[] bytes = new byte[length]; readFully(bytes, 0, length); return utf8.decode(ByteBuffer.wrap(bytes, 0, length)).toString(); - } + } public byte[] readBytes() throws IOException { byte[] result = new byte[readInt()]; http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/MetaData.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/MetaData.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/MetaData.java index 648d8a1..08d0072 100644 --- a/lang/java/trevni/core/src/main/java/org/apache/trevni/MetaData.java +++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/MetaData.java @@ -44,7 +44,7 @@ public class MetaData<T extends MetaData> extends LinkedHashMap<String,byte[]> { setReserved(CODEC_KEY, codec); return (T)this; } - + /** Return the checksum algorithm name. */ public String getChecksum() { return getString(CHECKSUM_KEY); } http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java index 41174fb..0bba8ac 100644 --- a/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java +++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java @@ -98,7 +98,7 @@ class OutputBuffer extends ByteArrayOutputStream { int len = bytes.limit() - pos; writeBytes(bytes.array(), start, len); } - + public void writeBytes(byte[] bytes) throws IOException { writeBytes(bytes, 0, bytes.length); } @@ -158,7 +158,7 @@ class OutputBuffer extends ByteArrayOutputStream { } } } - } + } buf[count++] = (byte) n; } @@ -203,7 +203,7 @@ class OutputBuffer extends ByteArrayOutputStream { } buf[count++] = (byte) n; } - + private void ensure(int n) { if (count + n > buf.length) buf = Arrays.copyOf(buf, Math.max(buf.length << 1, count + n)); http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java index 712a7d9..dddcb5a 100644 --- a/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java +++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java @@ -30,5 +30,5 @@ public enum ValueType { public static ValueType forName(String name) { return valueOf(name.toUpperCase()); } - + } http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/test/java/org/apache/trevni/TestBZip2Codec.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestBZip2Codec.java b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestBZip2Codec.java index c4a39de..ac7c81b 100644 --- a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestBZip2Codec.java +++ b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestBZip2Codec.java @@ -25,42 +25,42 @@ import java.nio.ByteBuffer; import org.junit.Test; public class TestBZip2Codec { - + @Test public void testBZip2CompressionAndDecompression() throws IOException { - + MetaData meta = new MetaData(); meta.setCodec("bzip2"); Codec codec = Codec.get(meta); - + //Confirm that the right codec Came back assertTrue(codec instanceof BZip2Codec); - + //This is 3 times the byte buffer on the BZip2 decompress plus some extra final int inputByteSize = BZip2Codec.DEFAULT_BUFFER_SIZE * 3 + 42; - + byte[] inputByteArray = new byte[inputByteSize]; - + //Generate something that will compress well for (int i = 0; i < inputByteSize; i++) { inputByteArray[i] = (byte)(65 + i % 10); } - + ByteBuffer inputByteBuffer = ByteBuffer.wrap(inputByteArray); - + ByteBuffer compressedBuffer = codec.compress(inputByteBuffer); - + //Make sure something returned assertTrue(compressedBuffer.array().length > 0); //Make sure the compressed output is smaller then the original assertTrue(compressedBuffer.array().length < inputByteArray.length); - + ByteBuffer decompressedBuffer = codec.decompress(compressedBuffer); - + //The original array should be the same length as the decompressed array assertTrue(decompressedBuffer.array().length == inputByteArray.length); - - //Every byte in the outputByteArray should equal every byte in the input array + + //Every byte in the outputByteArray should equal every byte in the input array byte[] outputByteArray = decompressedBuffer.array(); for (int i = 0; i < inputByteSize; i++) { inputByteArray[i] = outputByteArray[i]; http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java index 707848e..a0b1068 100644 --- a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java +++ b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java @@ -52,7 +52,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeValue(random.nextBoolean(), ValueType.BOOLEAN); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) @@ -65,7 +65,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeInt(random.nextInt()); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) @@ -77,7 +77,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeLong(random.nextLong()); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) @@ -89,7 +89,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeFixed32(random.nextInt()); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) @@ -101,41 +101,41 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeFixed64(random.nextLong()); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(random.nextLong(), in.readFixed64()); } - + @Test public void testFloat() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeFloat(random.nextFloat()); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(random.nextFloat(), in.readFloat(), 0); } - + @Test public void testDouble() throws Exception { OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeDouble(Double.MIN_VALUE); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); for (int i = 0; i < COUNT; i++) Assert.assertEquals(Double.MIN_VALUE, in.readDouble(), 0); } - + @Test public void testBytes() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeBytes(TestUtil.randomBytes(random)); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) @@ -147,7 +147,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeString(TestUtil.randomString(random)); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) @@ -158,7 +158,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); out.writeValue(null, ValueType.NULL); out.writeLong(sentinel); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); in.skipValue(ValueType.NULL); Assert.assertEquals(sentinel, in.readLong()); @@ -168,7 +168,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); out.writeValue(false, ValueType.BOOLEAN); out.writeLong(sentinel); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); in.skipValue(ValueType.BOOLEAN); Assert.assertEquals(sentinel, in.readLong()); @@ -178,7 +178,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); out.writeValue(Integer.MAX_VALUE, ValueType.INT); out.writeLong(sentinel); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); in.skipValue(ValueType.INT); Assert.assertEquals(sentinel, in.readLong()); @@ -188,7 +188,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); out.writeValue(Long.MAX_VALUE, ValueType.LONG); out.writeLong(sentinel); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); in.skipValue(ValueType.LONG); Assert.assertEquals(sentinel, in.readLong()); @@ -198,7 +198,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); out.writeValue(Integer.MAX_VALUE, ValueType.FIXED32); out.writeLong(sentinel); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); in.skipValue(ValueType.LONG); Assert.assertEquals(sentinel, in.readLong()); @@ -208,7 +208,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); out.writeValue(Long.MAX_VALUE, ValueType.FIXED64); out.writeLong(sentinel); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); in.skipValue(ValueType.LONG); Assert.assertEquals(sentinel, in.readLong()); @@ -218,7 +218,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); out.writeValue(Float.MAX_VALUE, ValueType.FLOAT); out.writeLong(sentinel); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); in.skipValue(ValueType.FLOAT); Assert.assertEquals(sentinel, in.readLong()); @@ -228,7 +228,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); out.writeValue(Double.MAX_VALUE, ValueType.DOUBLE); out.writeLong(sentinel); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); in.skipValue(ValueType.DOUBLE); Assert.assertEquals(sentinel, in.readLong()); @@ -238,7 +238,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); out.writeValue("trevni", ValueType.STRING); out.writeLong(sentinel); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); in.skipValue(ValueType.STRING); Assert.assertEquals(sentinel, in.readLong()); @@ -248,7 +248,7 @@ public class TestIOBuffers { OutputBuffer out = new OutputBuffer(); out.writeValue("trevni".getBytes(), ValueType.BYTES); out.writeLong(sentinel); - + InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); in.skipValue(ValueType.BYTES); Assert.assertEquals(sentinel, in.readLong()); http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/test/java/org/apache/trevni/TestInputBytes.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestInputBytes.java b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestInputBytes.java index c4cb2aa..c55b532 100644 --- a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestInputBytes.java +++ b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestInputBytes.java @@ -38,7 +38,7 @@ public class TestInputBytes { random.nextBytes(data); Input in = new InputBytes(data); - + for (int i = 0; i < COUNT; i++) { int p = random.nextInt(length); int l = Math.min(random.nextInt(SIZE/10), length-p); http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/test/java/org/apache/trevni/TestUtil.java ---------------------------------------------------------------------- diff --git a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestUtil.java b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestUtil.java index ab4796d..22bbae9 100644 --- a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestUtil.java +++ b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestUtil.java @@ -39,7 +39,7 @@ public class TestUtil { String configured = System.getProperty("test.seed"); if (configured != null) seed = Long.valueOf(configured); - else + else seed = System.currentTimeMillis(); System.err.println("test.seed="+seed); seedSet = true; http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/py/build.xml ---------------------------------------------------------------------- diff --git a/lang/py/build.xml b/lang/py/build.xml index 61c3f4c..5ef5214 100644 --- a/lang/py/build.xml +++ b/lang/py/build.xml @@ -17,7 +17,7 @@ --> <project name="Avro" default="dist" xmlns:ivy="antlib:org.apache.ivy.ant"> - + <!-- Load user's default properties. --> <property file="${user.home}/build.properties"/> @@ -66,7 +66,7 @@ </classpath> </typedef> </target> - + <target name="ivy-download" unless="ivy.jar.found" > <get src="http://repo2.maven.org/maven2/org/apache/ivy/ivy/${ivy.version}/ivy-${ivy.version}.jar"; dest="${ivy.jar}" usetimestamp="true" /> </target> @@ -79,13 +79,13 @@ <fileset dir="${src.dir}"> <exclude name="**/*.pyc"/> <exclude name="**/*.py~"/> - </fileset> + </fileset> </copy> <copy todir="${build.dir}/test"> <fileset dir="${test.dir}"> <exclude name="**/*.pyc"/> <exclude name="**/*.py~"/> - </fileset> + </fileset> </copy> <copy todir="${build.dir}/lib"> <fileset dir="${lib.dir}" /> @@ -102,9 +102,9 @@ toFile="${build.dir}/src/avro/ipc.py" overwrite="true"> <filterset> - <filter token="HANDSHAKE_REQUEST_SCHEMA" + <filter token="HANDSHAKE_REQUEST_SCHEMA" value="${handshake.request.json}"/> - <filter token="HANDSHAKE_RESPONSE_SCHEMA" + <filter token="HANDSHAKE_RESPONSE_SCHEMA" value="${handshake.response.json}"/> </filterset> </copy> http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/py/ivysettings.xml ---------------------------------------------------------------------- diff --git a/lang/py/ivysettings.xml b/lang/py/ivysettings.xml index 31de16e..22104c7 100644 --- a/lang/py/ivysettings.xml +++ b/lang/py/ivysettings.xml @@ -19,8 +19,8 @@ <property name="m2-pattern" value="${user.home}/.m2/repository/[organisation]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]" override="false" /> <resolvers> <chain name="repos"> - <ibiblio name="central" m2compatible="true"/> - <ibiblio name="apache-snapshots" m2compatible="true" root="https://repository.apache.org/content/groups/snapshots"/> + <ibiblio name="central" m2compatible="true"/> + <ibiblio name="apache-snapshots" m2compatible="true" root="https://repository.apache.org/content/groups/snapshots"/> <filesystem name="local-maven2" m2compatible="true"> <!-- needed when building non-snapshot version for release --> <artifact pattern="${m2-pattern}"/> <ivy pattern="${m2-pattern}"/> http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/py3/scripts/avro ---------------------------------------------------------------------- diff --git a/lang/py3/scripts/avro b/lang/py3/scripts/avro old mode 100644 new mode 100755
