Author: suresh
Date: Thu Jan 31 17:27:56 2013
New Revision: 1441079
URL: http://svn.apache.org/viewvc?rev=1441079&view=rev
Log:
MAPREDUCE-4969. Merge change 1441078 from branch-1
Modified:
hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapreduce/lib/input/TestKeyValueTextInputFormat.java
Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1441079&r1=1441078&r2=1441079&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Thu Jan 31
17:27:56 2013
@@ -340,3 +340,6 @@ Branch-hadoop-1-win (branched from branc
HADOOP-9177. Address issues that reported by static code analysis on
winutils. (Ivan Mitic via suresh)
+
+ MAPREDUCE-4969. TestKeyValueTextInputFormat test fails with Open JDK 7.
+ (Arpit Agarwal via suresh)
Modified:
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java?rev=1441079&r1=1441078&r2=1441079&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
(original)
+++
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
Thu Jan 31 17:27:56 2013
@@ -47,14 +47,10 @@ public class TestKeyValueTextInputFormat
private static Path workDir =
new Path(new Path(System.getProperty("test.build.data", "."), "data"),
"TestKeyValueTextInputFormat");
-
- private static Path workDirFormat =
- new Path(new Path(System.getProperty("test.build.data", "."), "data"),
- "TestKeyValueTextInputFormat-testFormat");
public void testFormat() throws Exception {
JobConf job = new JobConf();
- Path file = new Path(workDirFormat, "test.txt");
+ Path file = new Path(workDir, "test.txt");
// A reporter that does nothing
Reporter reporter = Reporter.NULL;
@@ -63,8 +59,8 @@ public class TestKeyValueTextInputFormat
LOG.info("seed = "+seed);
Random random = new Random(seed);
- localFs.delete(workDirFormat, true);
- FileInputFormat.setInputPaths(job, workDirFormat);
+ localFs.delete(workDir, true);
+ FileInputFormat.setInputPaths(job, workDir);
// for a variety of lengths
for (int length = 0; length < MAX_LENGTH;
@@ -140,32 +136,47 @@ public class TestKeyValueTextInputFormat
}
public void testUTF8() throws Exception {
- LineReader in = makeStream("abcd\u20acbdcd\u20ac");
- Text line = new Text();
- in.readLine(line);
- assertEquals("readLine changed utf8 characters",
- "abcd\u20acbdcd\u20ac", line.toString());
- in = makeStream("abc\u200axyz");
- in.readLine(line);
- assertEquals("split on fake newline", "abc\u200axyz", line.toString());
+ LineReader in = null;
+
+ try {
+ in = makeStream("abcd\u20acbdcd\u20ac");
+ Text line = new Text();
+ in.readLine(line);
+ assertEquals("readLine changed utf8 characters",
+ "abcd\u20acbdcd\u20ac", line.toString());
+ in = makeStream("abc\u200axyz");
+ in.readLine(line);
+ assertEquals("split on fake newline", "abc\u200axyz", line.toString());
+ } finally {
+ if (in != null) {
+ in.close();
+ }
+ }
}
public void testNewLines() throws Exception {
- LineReader in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
- Text out = new Text();
- in.readLine(out);
- assertEquals("line1 length", 1, out.getLength());
- in.readLine(out);
- assertEquals("line2 length", 2, out.getLength());
- in.readLine(out);
- assertEquals("line3 length", 0, out.getLength());
- in.readLine(out);
- assertEquals("line4 length", 3, out.getLength());
- in.readLine(out);
- assertEquals("line5 length", 4, out.getLength());
- in.readLine(out);
- assertEquals("line5 length", 5, out.getLength());
- assertEquals("end of file", 0, in.readLine(out));
+ LineReader in = null;
+ try {
+ in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
+ Text out = new Text();
+ in.readLine(out);
+ assertEquals("line1 length", 1, out.getLength());
+ in.readLine(out);
+ assertEquals("line2 length", 2, out.getLength());
+ in.readLine(out);
+ assertEquals("line3 length", 0, out.getLength());
+ in.readLine(out);
+ assertEquals("line4 length", 3, out.getLength());
+ in.readLine(out);
+ assertEquals("line5 length", 4, out.getLength());
+ in.readLine(out);
+ assertEquals("line5 length", 5, out.getLength());
+ assertEquals("end of file", 0, in.readLine(out));
+ } finally {
+ if (in != null) {
+ in.close();
+ }
+ }
}
private static void writeFile(FileSystem fs, Path name,
@@ -187,15 +198,23 @@ public class TestKeyValueTextInputFormat
InputSplit split,
JobConf job) throws IOException {
List<Text> result = new ArrayList<Text>();
- RecordReader<Text, Text> reader = format.getRecordReader(split, job,
- voidReporter);
- Text key = reader.createKey();
- Text value = reader.createValue();
- while (reader.next(key, value)) {
- result.add(value);
- value = (Text) reader.createValue();
+ RecordReader<Text, Text> reader = null;
+
+ try {
+ reader = format.getRecordReader(split, job, voidReporter);
+ Text key = reader.createKey();
+ Text value = reader.createValue();
+ while (reader.next(key, value)) {
+ result.add(value);
+ value = (Text) reader.createValue();
+ }
+
+ return result;
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
}
- return result;
}
/**
Modified:
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapreduce/lib/input/TestKeyValueTextInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapreduce/lib/input/TestKeyValueTextInputFormat.java?rev=1441079&r1=1441078&r2=1441079&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapreduce/lib/input/TestKeyValueTextInputFormat.java
(original)
+++
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapreduce/lib/input/TestKeyValueTextInputFormat.java
Thu Jan 31 17:27:56 2013
@@ -56,20 +56,16 @@ public class TestKeyValueTextInputFormat
new Path(new Path(System.getProperty("test.build.data", "."), "data"),
"TestKeyValueTextInputFormat");
- private static Path workDirFormat =
- new Path(new Path(System.getProperty("test.build.data", "."), "data"),
- "TestKeyValueTextInputFormat-testFormat");
-
public void testFormat() throws Exception {
Job job = new Job(defaultConf);
- Path file = new Path(workDirFormat, "test.txt");
+ Path file = new Path(workDir, "test.txt");
int seed = new Random().nextInt();
LOG.info("seed = "+seed);
Random random = new Random(seed);
- localFs.delete(workDirFormat, true);
- FileInputFormat.setInputPaths(job, workDirFormat);
+ localFs.delete(workDir, true);
+ FileInputFormat.setInputPaths(job, workDir);
// for a variety of lengths
for (int length = 0; length < MAX_LENGTH;
@@ -137,32 +133,46 @@ public class TestKeyValueTextInputFormat
}
public void testUTF8() throws Exception {
- LineReader in = makeStream("abcd\u20acbdcd\u20ac");
- Text line = new Text();
- in.readLine(line);
- assertEquals("readLine changed utf8 characters",
- "abcd\u20acbdcd\u20ac", line.toString());
- in = makeStream("abc\u200axyz");
- in.readLine(line);
- assertEquals("split on fake newline", "abc\u200axyz", line.toString());
+ LineReader in = null;
+ try {
+ in = makeStream("abcd\u20acbdcd\u20ac");
+ Text line = new Text();
+ in.readLine(line);
+ assertEquals("readLine changed utf8 characters",
+ "abcd\u20acbdcd\u20ac", line.toString());
+ in = makeStream("abc\u200axyz");
+ in.readLine(line);
+ assertEquals("split on fake newline", "abc\u200axyz", line.toString());
+ } finally {
+ if (in != null) {
+ in.close();
+ }
+ }
}
public void testNewLines() throws Exception {
- LineReader in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
- Text out = new Text();
- in.readLine(out);
- assertEquals("line1 length", 1, out.getLength());
- in.readLine(out);
- assertEquals("line2 length", 2, out.getLength());
- in.readLine(out);
- assertEquals("line3 length", 0, out.getLength());
- in.readLine(out);
- assertEquals("line4 length", 3, out.getLength());
- in.readLine(out);
- assertEquals("line5 length", 4, out.getLength());
- in.readLine(out);
- assertEquals("line5 length", 5, out.getLength());
- assertEquals("end of file", 0, in.readLine(out));
+ LineReader in = null;
+ try {
+ in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
+ Text out = new Text();
+ in.readLine(out);
+ assertEquals("line1 length", 1, out.getLength());
+ in.readLine(out);
+ assertEquals("line2 length", 2, out.getLength());
+ in.readLine(out);
+ assertEquals("line3 length", 0, out.getLength());
+ in.readLine(out);
+ assertEquals("line4 length", 3, out.getLength());
+ in.readLine(out);
+ assertEquals("line5 length", 4, out.getLength());
+ in.readLine(out);
+ assertEquals("line5 length", 5, out.getLength());
+ assertEquals("end of file", 0, in.readLine(out));
+ } finally {
+ if (in != null) {
+ in.close();
+ }
+ }
}
private static void writeFile(FileSystem fs, Path name,
@@ -182,12 +192,19 @@ public class TestKeyValueTextInputFormat
InputSplit split,
TaskAttemptContext context) throws
IOException, InterruptedException {
List<Text> result = new ArrayList<Text>();
- RecordReader<Text, Text> reader = format.createRecordReader(split,
context);
- reader.initialize(split, context);
- while (reader.nextKeyValue()) {
- result.add(new Text(reader.getCurrentValue()));
+ RecordReader<Text, Text> reader = null;
+ try {
+ reader = format.createRecordReader(split, context);
+ reader.initialize(split, context);
+ while (reader.nextKeyValue()) {
+ result.add(new Text(reader.getCurrentValue()));
+ }
+ return result;
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
}
- return result;
}
/**
