Author: cutting
Date: Fri Jan 18 21:32:55 2013
New Revision: 1435355
URL: http://svn.apache.org/viewvc?rev=1435355&view=rev
Log:
AVRO-1232. Java: Add a toString() method to AvroWrapper so that it works with
TextOutputFormat. Contributed by Garrett Wu.
Added:
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroWrapper.java
(with props)
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroWrapper.java
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapreduce/TestWordCount.java
Modified: avro/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1435355&r1=1435354&r2=1435355&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Jan 18 21:32:55 2013
@@ -22,6 +22,9 @@ Trunk (not yet released)
AVRO-1223. Java: Add a static method to generated classes that
returns its schema, getClassSchema(). (cutting)
+ AVRO-1232. Java: Add a toString() method to AvroWrapper so that it
+ works with TextOutputFormat. (Garrett Wu via cutting)
+
BUG FIXES
AVRO-1231. Java: Fix Trevni shredder to work on non-recursive
Modified:
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroWrapper.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroWrapper.java?rev=1435355&r1=1435354&r2=1435355&view=diff
==============================================================================
---
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroWrapper.java
(original)
+++
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroWrapper.java
Fri Jan 18 21:32:55 2013
@@ -54,4 +54,9 @@ public class AvroWrapper<T> {
return true;
}
+ /** Get the wrapped datum as JSON. */
+ @Override
+ public String toString() {
+ return datum.toString();
+ }
}
Added:
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroWrapper.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroWrapper.java?rev=1435355&view=auto
==============================================================================
---
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroWrapper.java
(added)
+++
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroWrapper.java
Fri Jan 18 21:32:55 2013
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.avro.mapred;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class TestAvroWrapper {
+ @Test
+ public void testToString() {
+ String datum = "my string";
+ AvroWrapper<CharSequence> wrapper = new AvroWrapper<CharSequence>(datum);
+ assertEquals(datum, wrapper.toString());
+ }
+}
Propchange:
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroWrapper.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapreduce/TestWordCount.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapreduce/TestWordCount.java?rev=1435355&r1=1435354&r2=1435355&view=diff
==============================================================================
---
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapreduce/TestWordCount.java
(original)
+++
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapreduce/TestWordCount.java
Fri Jan 18 21:32:55 2013
@@ -18,7 +18,10 @@
package org.apache.avro.mapreduce;
+import java.io.BufferedReader;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.Map;
@@ -27,6 +30,7 @@ import org.apache.avro.file.DataFileRead
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.mapred.AvroKey;
+import org.apache.avro.mapred.AvroValue;
import org.apache.avro.mapred.FsInput;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.reflect.ReflectData;
@@ -45,6 +49,7 @@ import org.apache.hadoop.mapreduce.Reduc
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.junit.Assert;
import org.junit.Rule;
@@ -126,6 +131,19 @@ public class TestWordCount {
}
}
+ private static class AvroSumReducer
+ extends Reducer<Text, IntWritable, AvroKey<CharSequence>,
AvroValue<Integer>> {
+ @Override
+ protected void reduce(Text key, Iterable<IntWritable> counts, Context
context)
+ throws IOException, InterruptedException {
+ int sum = 0;
+ for (IntWritable count : counts) {
+ sum += count.get();
+ }
+ context.write(new AvroKey<CharSequence>(key.toString()), new
AvroValue<Integer>(sum));
+ }
+ }
+
private static class GenericStatsReducer
extends Reducer<Text, IntWritable, AvroKey<GenericData.Record>,
NullWritable> {
private AvroKey<GenericData.Record> mStats;
@@ -455,4 +473,49 @@ public class TestWordCount {
Assert.assertEquals(2, counts.get("banana").intValue());
Assert.assertEquals(1, counts.get("carrot").intValue());
}
+
+ /**
+ * Tests the MR output to text files when using AvroKey and AvroValue
records.
+ */
+ @Test
+ public void testAvroUsingTextFileOutput() throws Exception {
+ Job job = new Job();
+
+ FileInputFormat.setInputPaths(job, new Path(getClass()
+ .getResource("/org/apache/avro/mapreduce/mapreduce-test-input.txt")
+ .toURI().toString()));
+ job.setInputFormatClass(TextInputFormat.class);
+
+ job.setMapperClass(LineCountMapper.class);
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(IntWritable.class);
+
+ job.setReducerClass(AvroSumReducer.class);
+ AvroJob.setOutputKeySchema(job, Schema.create(Schema.Type.STRING));
+ AvroJob.setOutputValueSchema(job, Schema.create(Schema.Type.INT));
+
+ job.setOutputFormatClass(TextOutputFormat.class);
+ Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-text");
+ FileOutputFormat.setOutputPath(job, outputPath);
+
+ Assert.assertTrue(job.waitForCompletion(true));
+
+ // Check that the results from the MapReduce were as expected.
+ FileSystem fileSystem = FileSystem.get(job.getConfiguration());
+ FileStatus[] outputFiles =
fileSystem.globStatus(outputPath.suffix("/part-*"));
+ Assert.assertEquals(1, outputFiles.length);
+ Path filePath = outputFiles[0].getPath();
+ InputStream inputStream =
filePath.getFileSystem(job.getConfiguration()).open(filePath);
+ Assert.assertNotNull(inputStream);
+ BufferedReader reader = new BufferedReader(new
InputStreamReader(inputStream));
+ try {
+ Assert.assertTrue(reader.ready());
+ Assert.assertEquals("apple\t3", reader.readLine());
+ Assert.assertEquals("banana\t2", reader.readLine());
+ Assert.assertEquals("carrot\t1", reader.readLine());
+ Assert.assertFalse(reader.ready());
+ } finally {
+ reader.close();
+ }
+ }
}
