Repository: sqoop Updated Branches: refs/heads/sqoop2 a389c2c7c -> 0e4f46b44
SQOOP-1696: Sqoop2: TestMapReduce.testOutputFormat is failing on Hadoop 1 profile (Jarek Jarcec Cecho via Abraham Elmahrek) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/0e4f46b4 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/0e4f46b4 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/0e4f46b4 Branch: refs/heads/sqoop2 Commit: 0e4f46b44d66e9418e3e81614b688f7352f026b3 Parents: a389c2c Author: Abraham Elmahrek <[email protected]> Authored: Sat Nov 8 23:26:08 2014 -0800 Committer: Abraham Elmahrek <[email protected]> Committed: Sat Nov 8 23:26:35 2014 -0800 ---------------------------------------------------------------------- .../java/org/apache/sqoop/job/JobUtils.java | 41 +++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/0e4f46b4/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java index 7ce8011..dafdeb4 100644 --- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java +++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java @@ -23,10 +23,16 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.OutputFormat; import org.apache.sqoop.job.io.SqoopWritable; import org.apache.sqoop.job.mr.SqoopSplit; +import org.apache.sqoop.utils.ClassUtils; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class JobUtils { @@ -44,7 +50,40 @@ public class JobUtils { job.setOutputKeyClass(SqoopWritable.class); job.setOutputValueClass(NullWritable.class); - return job.waitForCompletion(true); + boolean ret = job.waitForCompletion(true); + + // Hadoop 1.0 (and 0.20) have nasty bug when job committer is not called in LocalJobRuner + if (isHadoop1()) { + callOutputCommitter(job, output); + } + + return ret; + } + + /** + * Call output format on given job manually. + */ + private static void callOutputCommitter(Job job, Class<? extends OutputFormat<SqoopWritable, NullWritable>> outputFormat) throws IOException, InterruptedException { + OutputCommitter committer = ((OutputFormat)ClassUtils.instantiate(outputFormat)).getOutputCommitter(null); + + JobContext jobContext = mock(JobContext.class); + when(jobContext.getConfiguration()).thenReturn(job.getConfiguration()); + + committer.commitJob(jobContext); + } + + /** + * Detect Hadoop 1.0 installation + * + * @return True if and only if this is Hadoop 1 and below + */ + public static boolean isHadoop1() { + String version = org.apache.hadoop.util.VersionInfo.getVersion(); + if (version.matches("\\b0\\.20\\..+\\b") + || version.matches("\\b1\\.\\d\\.\\d")) { + return true; + } + return false; } private JobUtils() {
