Author: suresh
Date: Fri May 24 07:12:07 2013
New Revision: 1485951
URL: http://svn.apache.org/r1485951
Log:
MAPREDUCE-4885. Merge r1467158 from trunk
Added:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/bin/
- copied from r1467158,
hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/bin/
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/bin/cat.cmd
- copied unchanged from r1467158,
hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/bin/cat.cmd
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/bin/xargs_cat.cmd
- copied unchanged from r1467158,
hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/bin/xargs_cat.cmd
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreaming.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingKeyValue.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml?rev=1485951&r1=1485950&r2=1485951&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml Fri
May 24 07:12:07 2013
@@ -127,6 +127,20 @@
</target>
</configuration>
</execution>
+ <execution>
+ <id>copy-test-bin</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <copy todir="target/bin">
+ <fileset dir="src/test/bin" />
+ </copy>
+ </target>
+ </configuration>
+ </execution>
</executions>
</plugin>
<plugin>
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1485951&r1=1485950&r2=1485951&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Fri May 24 07:12:07 2013
@@ -294,8 +294,7 @@ public class StreamJob implements Tool {
for (String file : values) {
packageFiles_.add(file);
try {
- URI pathURI = new URI(file);
- Path path = new Path(pathURI);
+ Path path = new Path(file);
FileSystem localFs = FileSystem.getLocal(config_);
String finalPath = path.makeQualified(localFs).toString();
if(fileList.length() > 0) {
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java?rev=1485951&r1=1485950&r2=1485951&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java
Fri May 24 07:12:07 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.streaming;
import java.io.DataOutputStream;
+import java.io.File;
import java.io.IOException;
import java.util.Map;
@@ -27,6 +28,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.util.Shell;
import org.junit.After;
import org.junit.Before;
@@ -45,7 +47,8 @@ public class TestFileArgs extends TestSt
private static final String EXPECTED_OUTPUT =
"job.jar\t\nsidefile\t\n";
- private static final String LS_PATH = "/bin/ls";
+ private static final String LS_PATH = Shell.WINDOWS ? "cmd /c dir /B" :
+ "/bin/ls";
public TestFileArgs() throws IOException
{
@@ -58,6 +61,7 @@ public class TestFileArgs extends TestSt
map = LS_PATH;
FileSystem.setDefaultUri(conf, "hdfs://" + namenode);
+ setTestDir(new File("/tmp/TestFileArgs"));
}
@Before
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java?rev=1485951&r1=1485950&r2=1485951&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java
Fri May 24 07:12:07 2013
@@ -70,8 +70,8 @@ public class TestMultipleArchiveFiles ex
namenode = fileSys.getUri().getAuthority();
mr = new MiniMRCluster(1, namenode, 1);
- map = "xargs cat";
- reduce = "cat";
+ map = XARGS_CAT;
+ reduce = CAT;
}
@Override
@@ -84,7 +84,8 @@ public class TestMultipleArchiveFiles ex
{
fileSys.delete(new Path(INPUT_DIR), true);
DataOutputStream dos = fileSys.create(new Path(INPUT_FILE));
- String inputFileString = "symlink1/cacheArchive1\nsymlink2/cacheArchive2";
+ String inputFileString = "symlink1" + File.separator
+ + "cacheArchive1\nsymlink2" + File.separator + "cacheArchive2";
dos.write(inputFileString.getBytes("UTF-8"));
dos.close();
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java?rev=1485951&r1=1485950&r2=1485951&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java
Fri May 24 07:12:07 2013
@@ -49,8 +49,8 @@ public class TestMultipleCachefiles
String CACHE_FILE = "/testing-streaming/cache.txt";
String CACHE_FILE_2 = "/testing-streaming/cache2.txt";
String input = "check to see if we can read this none reduce";
- String map = "xargs cat ";
- String reduce = "cat";
+ String map = TestStreaming.XARGS_CAT;
+ String reduce = TestStreaming.CAT;
String mapString = "testlink";
String mapString2 = "testlink2";
String cacheString = "This is just the cache string";
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java?rev=1485951&r1=1485950&r2=1485951&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java
Fri May 24 07:12:07 2013
@@ -33,7 +33,7 @@ public class TestStreamXmlRecordReader e
INPUT_FILE = new File("target/input.xml");
input = "<xmltag>\t\nroses.are.red\t\nviolets.are.blue\t\n" +
"bunnies.are.pink\t\n</xmltag>\t\n";
- map = "cat";
+ map = CAT;
reduce = "NONE";
outputExpect = input;
}
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreaming.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreaming.java?rev=1485951&r1=1485950&r2=1485951&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreaming.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreaming.java
Fri May 24 07:12:07 2013
@@ -33,7 +33,7 @@ import static org.junit.Assert.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.Configuration;
-
+import org.apache.hadoop.util.Shell;
/**
* This class tests hadoopStreaming in MapReduce local mode.
@@ -43,6 +43,22 @@ public class TestStreaming
public static final String STREAMING_JAR = JarFinder.getJar(StreamJob.class);
+ /**
+ * cat command used for copying stdin to stdout as mapper or reducer
function.
+ * On Windows, use a cmd script that approximates the functionality of cat.
+ */
+ static final String CAT = Shell.WINDOWS ?
+ "cmd /c " + new File("target/bin/cat.cmd").getAbsolutePath() : "cat";
+
+ /**
+ * Command used for iterating through file names on stdin and copying each
+ * file's contents to stdout, used as mapper or reducer function. On
Windows,
+ * use a cmd script that approximates the functionality of xargs cat.
+ */
+ static final String XARGS_CAT = Shell.WINDOWS ?
+ "cmd /c " + new File("target/bin/xargs_cat.cmd").getAbsolutePath() :
+ "xargs cat";
+
// "map" command: grep -E (red|green|blue)
// reduce command: uniq
protected File TEST_DIR;
@@ -66,9 +82,22 @@ public class TestStreaming
UtilTest utilTest = new UtilTest(getClass().getName());
utilTest.checkUserDir();
utilTest.redirectIfAntJunit();
- TEST_DIR = new File("target/TestStreaming").getAbsoluteFile();
- OUTPUT_DIR = new File(TEST_DIR, "out");
- INPUT_FILE = new File(TEST_DIR, "input.txt");
+ setTestDir(new File("target/TestStreaming").getAbsoluteFile());
+ }
+
+ /**
+ * Sets root of test working directory and resets any other paths that must
be
+ * children of the test working directory. Typical usage is for subclasses
+ * that use HDFS to override the test directory to the form "/tmp/<test
name>"
+ * so that on Windows, tests won't attempt to use paths containing a ':' from
+ * the drive specifier. The ':' character is considered invalid by HDFS.
+ *
+ * @param testDir File to set
+ */
+ protected void setTestDir(File testDir) {
+ TEST_DIR = testDir;
+ OUTPUT_DIR = new File(testDir, "out");
+ INPUT_FILE = new File(testDir, "input.txt");
}
@Before
@@ -89,19 +118,18 @@ public class TestStreaming
protected void createInput() throws IOException
{
- DataOutputStream out = getFileSystem().create(
- new Path(INPUT_FILE.getAbsolutePath()));
+ DataOutputStream out = getFileSystem().create(new Path(
+ INPUT_FILE.getPath()));
out.write(getInputData().getBytes("UTF-8"));
out.close();
}
protected void setInputOutput() {
- inputFile = INPUT_FILE.getAbsolutePath();
- outDir = OUTPUT_DIR.getAbsolutePath();
+ inputFile = INPUT_FILE.getPath();
+ outDir = OUTPUT_DIR.getPath();
}
protected String[] genArgs() {
- setInputOutput();
args.add("-input");args.add(inputFile);
args.add("-output");args.add(outDir);
args.add("-mapper");args.add(map);
@@ -129,7 +157,7 @@ public class TestStreaming
}
protected void checkOutput() throws IOException {
- Path outPath = new Path(OUTPUT_DIR.getAbsolutePath(), "part-00000");
+ Path outPath = new Path(OUTPUT_DIR.getPath(), "part-00000");
FileSystem fs = getFileSystem();
String output = StreamUtil.slurpHadoop(outPath, fs);
fs.delete(outPath, true);
@@ -155,6 +183,7 @@ public class TestStreaming
* @throws IOException
*/
protected int runStreamJob() throws IOException {
+ setInputOutput();
createInput();
boolean mayExit = false;
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingKeyValue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingKeyValue.java?rev=1485951&r1=1485950&r2=1485951&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingKeyValue.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingKeyValue.java
Fri May 24 07:12:07 2013
@@ -76,7 +76,7 @@ public class TestStreamingKeyValue
return new String[] {
"-input", INPUT_FILE.getAbsolutePath(),
"-output", OUTPUT_DIR.getAbsolutePath(),
- "-mapper", "cat",
+ "-mapper", TestStreaming.CAT,
"-jobconf", MRJobConfig.PRESERVE_FAILED_TASK_FILES + "=true",
"-jobconf", "stream.non.zero.exit.is.failure=true",
"-jobconf",
"stream.tmpdir="+System.getProperty("test.build.data","/tmp"),
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java?rev=1485951&r1=1485950&r2=1485951&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
Fri May 24 07:12:07 2013
@@ -120,7 +120,7 @@ public class TestStreamingOutputKeyValue
@Test
public void testJavaMapperAndCommandReducer() throws Exception {
map = "org.apache.hadoop.mapred.lib.IdentityMapper";
- reduce = "cat";
+ reduce = CAT;
super.testCommandLine();
}
@@ -128,7 +128,7 @@ public class TestStreamingOutputKeyValue
@Test
public void testJavaMapperAndCommandReducerAndZeroReduces() throws Exception
{
map = "org.apache.hadoop.mapred.lib.IdentityMapper";
- reduce = "cat";
+ reduce = CAT;
args.add("-numReduceTasks");
args.add("0");
super.testCommandLine();
@@ -137,7 +137,7 @@ public class TestStreamingOutputKeyValue
// Check with Command Mapper, Java Reducer
@Test
public void testCommandMapperAndJavaReducer() throws Exception {
- map = "cat";
+ map = CAT;
reduce = MyReducer.class.getName();
super.testCommandLine();
}
@@ -145,7 +145,7 @@ public class TestStreamingOutputKeyValue
// Check with Command Mapper, Java Reducer and -numReduceTasks 0
@Test
public void testCommandMapperAndJavaReducerAndZeroReduces() throws Exception
{
- map = "cat";
+ map = CAT;
reduce = MyReducer.class.getName();
args.add("-numReduceTasks");
args.add("0");
@@ -155,7 +155,7 @@ public class TestStreamingOutputKeyValue
// Check with Command Mapper, Reducer = "NONE"
@Test
public void testCommandMapperWithReduceNone() throws Exception {
- map = "cat";
+ map = CAT;
reduce = "NONE";
super.testCommandLine();
}
@@ -163,8 +163,8 @@ public class TestStreamingOutputKeyValue
// Check with Command Mapper, Command Reducer
@Test
public void testCommandMapperAndCommandReducer() throws Exception {
- map = "cat";
- reduce = "cat";
+ map = CAT;
+ reduce = CAT;
super.testCommandLine();
}
@@ -172,8 +172,8 @@ public class TestStreamingOutputKeyValue
@Test
public void testCommandMapperAndCommandReducerAndZeroReduces()
throws Exception {
- map = "cat";
- reduce = "cat";
+ map = CAT;
+ reduce = CAT;
args.add("-numReduceTasks");
args.add("0");
super.testCommandLine();
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java?rev=1485951&r1=1485950&r2=1485951&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
Fri May 24 07:12:07 2013
@@ -83,7 +83,7 @@ public class TestStreamingTaskLog {
* (b) hadoop.tasklog.totalLogFileSize
* for the children of java tasks in streaming jobs.
*/
- @Test (timeout = 30000)
+ @Test (timeout = 120000)
public void testStreamingTaskLogWithHadoopCmd() {
try {
final int numSlaves = 1;
@@ -95,13 +95,14 @@ public class TestStreamingTaskLog {
fs.delete(testDir, true);
}
fs.mkdirs(testDir);
- File scriptFile = createScript(
- testDir.toString() + "/testTaskLog.sh");
+ File scriptFile = createScript(testDir.toString() +
+ (Shell.WINDOWS ? "/testTaskLog.cmd" : "/testTaskLog.sh"));
conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, false);
mr = new MiniMRCluster(numSlaves, fs.getUri().toString(), 1, null, null,
conf);
writeInputFile(fs, inputPath);
- map = scriptFile.getAbsolutePath();
+ map = Shell.WINDOWS ? "cmd /c " + scriptFile.getAbsolutePath() :
+ scriptFile.getAbsolutePath();
runStreamJobAndValidateEnv();
@@ -120,8 +121,12 @@ public class TestStreamingTaskLog {
File scriptFile = new File(script);
UtilTest.recursiveDelete(scriptFile);
FileOutputStream in = new FileOutputStream(scriptFile);
- in.write(("cat > /dev/null 2>&1\n" +
- "echo $HADOOP_ROOT_LOGGER $HADOOP_CLIENT_OPTS").getBytes());
+ if (Shell.WINDOWS) {
+ in.write("@echo %HADOOP_ROOT_LOGGER% %HADOOP_CLIENT_OPTS%".getBytes());
+ } else {
+ in.write(("cat > /dev/null 2>&1\n" +
+ "echo $HADOOP_ROOT_LOGGER $HADOOP_CLIENT_OPTS").getBytes());
+ }
in.close();
Shell.execCommand(Shell.getSetPermissionCommand("+x", false,
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java?rev=1485951&r1=1485950&r2=1485951&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
Fri May 24 07:12:07 2013
@@ -47,13 +47,13 @@ public class TestSymLink
String OUTPUT_DIR = "/testing-streaming/out";
String CACHE_FILE = "/testing-streaming/cache.txt";
String input = "check to see if we can read this none reduce";
- String map = "xargs cat ";
- String reduce = "cat";
+ String map = TestStreaming.XARGS_CAT;
+ String reduce = TestStreaming.CAT;
String mapString = "testlink\n";
String cacheString = "This is just the cache string";
StreamJob job;
- @Test (timeout = 60000)
+ @Test (timeout = 120000)
public void testSymLink() throws Exception
{
boolean mayExit = false;
