Author: cutting
Date: Tue Apr 26 16:34:50 2011
New Revision: 1096800
URL: http://svn.apache.org/viewvc?rev=1096800&view=rev
Log:
Merge -c 1096798 from trunk to 1.5 branch. Fixes AVRO-808.
Added:
avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroAsTextInputFormat.java
- copied unchanged from r1096798,
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroAsTextInputFormat.java
avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroAsTextRecordReader.java
- copied unchanged from r1096798,
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroAsTextRecordReader.java
avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroAsTextInputFormat.java
- copied unchanged from r1096798,
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroAsTextInputFormat.java
Modified:
avro/branches/branch-1.5/ (props changed)
avro/branches/branch-1.5/CHANGES.txt
avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/WordCountUtil.java
Propchange: avro/branches/branch-1.5/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 26 16:34:50 2011
@@ -1 +1 @@
-/avro/trunk:1075938,1075993,1078917,1079055,1079060,1079063,1083246,1085921,1086727,1086730,1086866,1087076,1087129,1087136,1087439-1087440,1087463,1087472,1087792,1089128,1089131,1089550,1094812,1095206-1095208,1095493,1095529,1095548,1095550
+/avro/trunk:1075938,1075993,1078917,1079055,1079060,1079063,1083246,1085921,1086727,1086730,1086866,1087076,1087129,1087136,1087439-1087440,1087463,1087472,1087792,1089128,1089131,1089550,1094812,1095206-1095208,1095493,1095529,1095548,1095550,1096798
Modified: avro/branches/branch-1.5/CHANGES.txt
URL:
http://svn.apache.org/viewvc/avro/branches/branch-1.5/CHANGES.txt?rev=1096800&r1=1096799&r2=1096800&view=diff
==============================================================================
--- avro/branches/branch-1.5/CHANGES.txt (original)
+++ avro/branches/branch-1.5/CHANGES.txt Tue Apr 26 16:34:50 2011
@@ -10,6 +10,9 @@ Avro 1.5.1 (unreleased)
AVRO-788. Java: Add Snappy compression for data files, including
MapReduce API support. (cutting)
+ AVRO-808. Java: Add AvroAsTextInputFormat for use with streaming.
+ (Tom White via cutting)
+
IMPROVEMENTS
AVRO-785. Java: Squash a Velocity warning by upgrading to Velocity 1.7.
Modified:
avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/WordCountUtil.java
URL:
http://svn.apache.org/viewvc/avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/WordCountUtil.java?rev=1096800&r1=1096799&r2=1096800&view=diff
==============================================================================
---
avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/WordCountUtil.java
(original)
+++
avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/WordCountUtil.java
Tue Apr 26 16:34:50 2011
@@ -20,13 +20,19 @@ package org.apache.avro.mapred;
import static org.junit.Assert.*;
+import java.io.BufferedReader;
+import java.io.FileReader;
import java.io.IOException;
import java.io.File;
import java.io.InputStream;
import java.io.FileInputStream;
import java.io.BufferedInputStream;
import java.io.PrintStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import java.util.StringTokenizer;
import java.util.Map;
import java.util.TreeMap;
@@ -53,6 +59,8 @@ class WordCountUtil {
= new File(new File(DIR, "in"), "lines.txt");
private static final File COUNTS_FILE
= new File(new File(DIR, "out"), "part-00000.avro");
+ private static final File SORTED_FILE
+ = new File(new File(DIR, "out"), "part-00000");
public static final String[] LINES = new String[] {
"the quick brown fox jumps over the lazy dog",
@@ -84,6 +92,17 @@ class WordCountUtil {
out.append(new Utf8(line));
out.close();
}
+
+ public static void writeLinesBytesFile() throws IOException {
+ FileUtil.fullyDelete(DIR);
+ DatumWriter<ByteBuffer> writer = new GenericDatumWriter<ByteBuffer>();
+ DataFileWriter<ByteBuffer> out = new DataFileWriter<ByteBuffer>(writer);
+ LINES_FILE.getParentFile().mkdirs();
+ out.create(Schema.create(Schema.Type.BYTES), LINES_FILE);
+ for (String line : LINES)
+ out.append(ByteBuffer.wrap(line.getBytes("UTF-8")));
+ out.close();
+ }
public static void writeLinesTextFile() throws IOException {
FileUtil.fullyDelete(DIR);
@@ -110,6 +129,19 @@ class WordCountUtil {
in.close();
assertEquals(COUNTS.size(), numWords);
}
+
+ public static void validateSortedFile() throws Exception {
+ BufferedReader reader = new BufferedReader(new FileReader(SORTED_FILE));
+ List<String> sortedLines = new ArrayList<String>();
+ for (String line : LINES) {
+ sortedLines.add(line);
+ }
+ Collections.sort(sortedLines);
+ for (String expectedLine : sortedLines) {
+ assertEquals(expectedLine, reader.readLine().trim());
+ }
+ assertNull(reader.readLine());
+ }
// metadata tests
private static final String STRING_KEY = "string-key";
