Author: eli
Date: Tue Sep 11 21:28:53 2012
New Revision: 1383619
URL: http://svn.apache.org/viewvc?rev=1383619&view=rev
Log:
HDFS-3910. DFSTestUtil#waitReplication should timeout. Contributed by Eli
Collins
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java?rev=1383619&r1=1383618&r2=1383619&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
Tue Sep 11 21:28:53 2012
@@ -23,6 +23,7 @@ import static org.mockito.Mockito.when;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.util.concurrent.TimeoutException;
import junit.framework.TestCase;
@@ -95,7 +96,7 @@ public class TestFileInputFormat extends
}
private void createInputs(FileSystem fs, Path inDir, String fileName)
- throws IOException {
+ throws IOException, TimeoutException, InterruptedException {
// create a multi-block file on hdfs
Path path = new Path(inDir, fileName);
final short replication = 2;
@@ -157,7 +158,7 @@ public class TestFileInputFormat extends
}
}
- public void testMultiLevelInput() throws IOException {
+ public void testMultiLevelInput() throws Exception {
JobConf job = new JobConf(conf);
job.setBoolean("dfs.replication.considerLoad", false);
@@ -291,7 +292,8 @@ public class TestFileInputFormat extends
}
static void writeFile(Configuration conf, Path name,
- short replication, int numBlocks) throws IOException {
+ short replication, int numBlocks)
+ throws IOException, TimeoutException, InterruptedException {
FileSystem fileSys = FileSystem.get(conf);
FSDataOutputStream stm = fileSys.create(name, true,
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java?rev=1383619&r1=1383618&r2=1383619&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java
Tue Sep 11 21:28:53 2012
@@ -71,13 +71,13 @@ public class TestMultipleLevelCaching ex
return rack.toString();
}
- public void testMultiLevelCaching() throws IOException {
+ public void testMultiLevelCaching() throws Exception {
for (int i = 1 ; i <= MAX_LEVEL; ++i) {
testCachingAtLevel(i);
}
}
- private void testCachingAtLevel(int level) throws IOException {
+ private void testCachingAtLevel(int level) throws Exception {
String namenode = null;
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java?rev=1383619&r1=1383618&r2=1383619&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
Tue Sep 11 21:28:53 2012
@@ -31,6 +31,7 @@ import java.util.Enumeration;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
+import java.util.concurrent.TimeoutException;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -449,11 +450,14 @@ public class UtilsForTests {
static void signalTasks(MiniDFSCluster dfs, FileSystem fileSys,
String mapSignalFile,
String reduceSignalFile, int replication)
- throws IOException {
- writeFile(dfs.getNameNode(), fileSys.getConf(), new Path(mapSignalFile),
- (short)replication);
- writeFile(dfs.getNameNode(), fileSys.getConf(), new
Path(reduceSignalFile),
- (short)replication);
+ throws IOException, TimeoutException {
+ try {
+ writeFile(dfs.getNameNode(), fileSys.getConf(), new Path(mapSignalFile),
+ (short)replication);
+ writeFile(dfs.getNameNode(), fileSys.getConf(), new
Path(reduceSignalFile), (short)replication);
+ } catch (InterruptedException ie) {
+ // Ignore
+ }
}
/**
@@ -462,12 +466,16 @@ public class UtilsForTests {
static void signalTasks(MiniDFSCluster dfs, FileSystem fileSys,
boolean isMap, String mapSignalFile,
String reduceSignalFile)
- throws IOException {
- // signal the maps to complete
- writeFile(dfs.getNameNode(), fileSys.getConf(),
- isMap
- ? new Path(mapSignalFile)
- : new Path(reduceSignalFile), (short)1);
+ throws IOException, TimeoutException {
+ try {
+ // signal the maps to complete
+ writeFile(dfs.getNameNode(), fileSys.getConf(),
+ isMap
+ ? new Path(mapSignalFile)
+ : new Path(reduceSignalFile), (short)1);
+ } catch (InterruptedException ie) {
+ // Ignore
+ }
}
static String getSignalFile(Path dir) {
@@ -483,7 +491,8 @@ public class UtilsForTests {
}
static void writeFile(NameNode namenode, Configuration conf, Path name,
- short replication) throws IOException {
+ short replication)
+ throws IOException, TimeoutException, InterruptedException {
FileSystem fileSys = FileSystem.get(conf);
SequenceFile.Writer writer =
SequenceFile.createWriter(fileSys, conf, name,
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java?rev=1383619&r1=1383618&r2=1383619&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
Tue Sep 11 21:28:53 2012
@@ -23,6 +23,7 @@ import java.net.URI;
import java.util.List;
import java.util.ArrayList;
import java.util.zip.GZIPOutputStream;
+import java.util.concurrent.TimeoutException;
import junit.framework.TestCase;
@@ -278,7 +279,7 @@ public class TestCombineFileInputFormat
assertFalse(rr.nextKeyValue());
}
- public void testSplitPlacement() throws IOException {
+ public void testSplitPlacement() throws Exception {
MiniDFSCluster dfs = null;
FileSystem fileSys = null;
try {
@@ -678,7 +679,8 @@ public class TestCombineFileInputFormat
}
static void writeFile(Configuration conf, Path name,
- short replication, int numBlocks) throws IOException {
+ short replication, int numBlocks)
+ throws IOException, TimeoutException, InterruptedException {
FileSystem fileSys = FileSystem.get(conf);
FSDataOutputStream stm = fileSys.create(name, true,
@@ -689,7 +691,8 @@ public class TestCombineFileInputFormat
// Creates the gzip file and return the FileStatus
static FileStatus writeGzipFile(Configuration conf, Path name,
- short replication, int numBlocks) throws IOException {
+ short replication, int numBlocks)
+ throws IOException, TimeoutException, InterruptedException {
FileSystem fileSys = FileSystem.get(conf);
GZIPOutputStream out = new GZIPOutputStream(fileSys.create(name, true, conf
@@ -699,7 +702,8 @@ public class TestCombineFileInputFormat
}
private static void writeDataAndSetReplication(FileSystem fileSys, Path name,
- OutputStream out, short replication, int numBlocks) throws IOException {
+ OutputStream out, short replication, int numBlocks)
+ throws IOException, TimeoutException, InterruptedException {
for (int i = 0; i < numBlocks; i++) {
out.write(databuf);
}
@@ -707,7 +711,7 @@ public class TestCombineFileInputFormat
DFSTestUtil.waitReplication(fileSys, name, replication);
}
- public void testSplitPlacementForCompressedFiles() throws IOException {
+ public void testSplitPlacementForCompressedFiles() throws Exception {
MiniDFSCluster dfs = null;
FileSystem fileSys = null;
try {
@@ -1058,7 +1062,7 @@ public class TestCombineFileInputFormat
/**
* Test that CFIF can handle missing blocks.
*/
- public void testMissingBlocks() throws IOException {
+ public void testMissingBlocks() throws Exception {
String namenode = null;
MiniDFSCluster dfs = null;
FileSystem fileSys = null;
