Author: omalley
Date: Tue Apr 24 15:21:12 2012
New Revision: 1329779
URL: http://svn.apache.org/viewvc?rev=1329779&view=rev
Log:
MAPREDUCE-4154. streaming MR job succeeds even if the streaming command
fails. (Devaraj Das via tgraves)
Modified:
hadoop/common/branches/branch-1.0/CHANGES.txt
hadoop/common/branches/branch-1.0/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
hadoop/common/branches/branch-1.0/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java
Modified: hadoop/common/branches/branch-1.0/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/CHANGES.txt?rev=1329779&r1=1329778&r2=1329779&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1.0/CHANGES.txt Tue Apr 24 15:21:12 2012
@@ -36,6 +36,9 @@ Release 1.0.3 - unreleased
HADOOP-8293. Fix the Makefile.am for the native library to include the
JNI path. (omalley)
+ MAPREDUCE-4154. streaming MR job succeeds even if the streaming command
+ fails. (Devaraj Das via tgraves)
+
Release 1.0.2 - 2012.03.24
NEW FEATURES
Modified:
hadoop/common/branches/branch-1.0/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java?rev=1329779&r1=1329778&r2=1329779&view=diff
==============================================================================
---
hadoop/common/branches/branch-1.0/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
(original)
+++
hadoop/common/branches/branch-1.0/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
Tue Apr 24 15:21:12 2012
@@ -569,6 +569,10 @@ public abstract class PipeMapRed {
clientOut_.flush();
clientOut_.close();
}
+ } catch (IOException io) {
+ LOG.warn(StringUtils.stringifyException(io));
+ }
+ try {
waitOutputThreads();
} catch (IOException io) {
LOG.warn(StringUtils.stringifyException(io));
Modified:
hadoop/common/branches/branch-1.0/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java?rev=1329779&r1=1329778&r2=1329779&view=diff
==============================================================================
---
hadoop/common/branches/branch-1.0/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java
(original)
+++
hadoop/common/branches/branch-1.0/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingFailure.java
Tue Apr 24 15:21:12 2012
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.ToolRunner;
/**
* This class tests if hadoopStreaming returns Exception
@@ -82,6 +83,25 @@ public class TestStreamingFailure extend
}
}
}
+
+ public void testStreamingFailureForFailedProcess() throws Exception {
+ int ret = 0;
+ try {
+ createInput();
+ String[] args = {
+ "-input", INPUT_FILE.getAbsolutePath(),
+ "-output", OUTPUT_DIR.getAbsolutePath(),
+ "-mapper", "/bin/ls dsdsdsds-does-not-exist",
+ "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data",
+ "/tmp"),
+ };
+ ret = ToolRunner.run(new StreamJob(), args);
+ } finally {
+ INPUT_FILE.delete();
+ FileUtil.fullyDelete(OUTPUT_DIR.getAbsoluteFile());
+ }
+ assertEquals("Streaming job failure code expected", 1, ret);
+ }
public static void main(String[]args) throws Exception
{
