Repository: oozie Updated Branches: refs/heads/master 0da31f47e -> 4483f6dd9
OOZIE-2792 Hive2 action is not parsing Spark application ID from log file properly when Hive is on Spark (zhengxb2005 via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/4483f6dd Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/4483f6dd Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/4483f6dd Branch: refs/heads/master Commit: 4483f6dd94daa0aee1bdaf19cbb4b4a68cbefa8c Parents: 0da31f4 Author: Robert Kanter <rkan...@apache.org> Authored: Tue Feb 14 16:40:46 2017 -0800 Committer: Robert Kanter <rkan...@apache.org> Committed: Tue Feb 14 16:40:46 2017 -0800 ---------------------------------------------------------------------- .../oozie/action/hadoop/TestDistcpMain.java | 26 +++++++++- release-log.txt | 1 + .../apache/oozie/action/hadoop/DistcpMain.java | 6 ++- .../apache/oozie/action/hadoop/HiveMain.java | 5 +- .../oozie/action/hadoop/TestHiveMain.java | 35 +++++++++++-- .../apache/oozie/action/hadoop/Hive2Main.java | 8 ++- .../oozie/action/hadoop/TestHive2Main.java | 52 ++++++++++++++++++++ .../oozie/action/hadoop/LauncherMain.java | 32 +++++++----- .../org/apache/oozie/action/hadoop/PigMain.java | 6 ++- .../apache/oozie/action/hadoop/TestPigMain.java | 23 +++++++++ .../apache/oozie/action/hadoop/SparkMain.java | 3 +- .../oozie/action/hadoop/TestSparkMain.java | 18 +++++++ .../apache/oozie/action/hadoop/SqoopMain.java | 6 ++- .../oozie/action/hadoop/TestSqoopMain.java | 49 ++++++++++++++++++ 14 files changed, 247 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java index bf08ccb..199962e 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java @@ -23,10 +23,13 @@ import org.apache.oozie.util.XConfiguration; import java.io.File; import java.io.FileOutputStream; -import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; import org.apache.oozie.action.hadoop.DistcpMain; @@ -95,4 +98,25 @@ public class TestDistcpMain extends MainTestCase { new File(getTestCaseDir(), "action.xml").delete(); return null; } + + public void testJobIDPattern() { + List<String> lines = new ArrayList<String>(); + lines.add("Job complete: job_001"); + lines.add("Job job_002 completed successfully"); + lines.add("Submitted application application_003"); + // Non-matching ones + lines.add("Job complete: job004"); + lines.add("Job complete: (job_005"); + lines.add("Job abc job_006 completed successfully"); + lines.add("Submitted application. application_007"); + Set<String> jobIds = new LinkedHashSet<String>(); + for (String line : lines) { + LauncherMain.extractJobIDs(line, DistcpMain.DISTCP_JOB_IDS_PATTERNS, jobIds); + } + Set<String> expected = new LinkedHashSet<String>(); + expected.add("job_001"); + expected.add("job_002"); + expected.add("job_003"); + assertEquals(expected, jobIds); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 32e3648..4562524 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.4.0 release (trunk - unreleased) +OOZIE-2792 Hive2 action is not parsing Spark application ID from log file properly when Hive is on Spark (zhengxb2005 via rkanter) OOZIE-2788 Fix jobs API servlet mapping for EmbeddedOozieServer (abhishekbafna via rkanter) OOZIE-2778 Copy only jetty.version related server dependencies to distro (asasvari via abhishekbafna) OOZIE-2786 Pass Oozie workflow ID and settings to Spark application configuration (zhengxb2005 via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java ---------------------------------------------------------------------- diff --git a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java index 2b84f92..fcaadef 100644 --- a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java +++ b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java @@ -33,12 +33,16 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.apache.log4j.PropertyConfigurator; +import com.google.common.annotations.VisibleForTesting; + public class DistcpMain extends JavaMain { private Constructor<?> construct; private Object[] constArgs; private static final String DISTCP_LOG4J_PROPS = "distcp-log4j.properties"; - private static final Pattern[] DISTCP_JOB_IDS_PATTERNS = { + + @VisibleForTesting + static final Pattern[] DISTCP_JOB_IDS_PATTERNS = { Pattern.compile("Job complete: (job_\\S*)"), Pattern.compile("Job (job_\\S*) completed successfully"), Pattern.compile("Submitted application (application[0-9_]*)") http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java index e1c7caa..bc8ab29 100644 --- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java +++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java @@ -40,8 +40,11 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.cli.CliDriver; import org.apache.hadoop.hive.conf.HiveConf; +import com.google.common.annotations.VisibleForTesting; + public class HiveMain extends LauncherMain { - private static final Pattern[] HIVE_JOB_IDS_PATTERNS = { + @VisibleForTesting + static final Pattern[] HIVE_JOB_IDS_PATTERNS = { Pattern.compile("Ended Job = (job_\\S*)"), Pattern.compile("Submitted application (application[0-9_]*)") }; http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java index 3026311..2ba0da7 100644 --- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java +++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java @@ -18,10 +18,6 @@ package org.apache.oozie.action.hadoop; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.oozie.test.MiniHCatServer; -import org.apache.oozie.util.XConfiguration; import java.io.File; import java.io.FileOutputStream; import java.io.FileWriter; @@ -29,16 +25,27 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.net.URL; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Properties; +import java.util.Set; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.test.MiniHCatServer; +import org.apache.oozie.util.XConfiguration; public class TestHiveMain extends MainTestCase { private SecurityManager SECURITY_MANAGER; + @Override protected void setUp() throws Exception { super.setUp(); SECURITY_MANAGER = System.getSecurityManager(); } + @Override protected void tearDown() throws Exception { System.setSecurityManager(SECURITY_MANAGER); super.tearDown(); @@ -61,6 +68,7 @@ public class TestHiveMain extends MainTestCase { return buffer.toString(); } + @Override public Void call() throws Exception { if (System.getenv("HADOOP_HOME") == null) { System.out.println("WARNING: 'HADOOP_HOME' env var not defined, TestHiveMain test is not running"); @@ -165,4 +173,23 @@ public class TestHiveMain extends MainTestCase { } return null; } + + public void testJobIDPattern() { + List<String> lines = new ArrayList<String>(); + lines.add("Ended Job = job_001"); + lines.add("Submitted application application_002"); + // Non-matching ones + lines.add("Ended Job = . job_003"); + lines.add("Ended Job = abc004"); + lines.add("Submitted application = job_005"); + lines.add("Submitted application. job_006"); + Set<String> jobIds = new LinkedHashSet<String>(); + for (String line : lines) { + LauncherMain.extractJobIDs(line, HiveMain.HIVE_JOB_IDS_PATTERNS, jobIds); + } + Set<String> expected = new LinkedHashSet<String>(); + expected.add("job_001"); + expected.add("job_002"); + assertEquals(expected, jobIds); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java ---------------------------------------------------------------------- diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java index 54f7039..8e214fa 100644 --- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java +++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java @@ -37,10 +37,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hive.beeline.BeeLine; +import com.google.common.annotations.VisibleForTesting; + public class Hive2Main extends LauncherMain { - private static final Pattern[] HIVE2_JOB_IDS_PATTERNS = { + @VisibleForTesting + static final Pattern[] HIVE2_JOB_IDS_PATTERNS = { Pattern.compile("Ended Job = (job_\\S*)"), - Pattern.compile("Submitted application (application[0-9_]*)") + Pattern.compile("Submitted application (application[0-9_]*)"), + Pattern.compile("Running with YARN Application = (application_[0-9]+)") }; private static final Set<String> DISALLOWED_BEELINE_OPTIONS = new HashSet<String>(); http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java ---------------------------------------------------------------------- diff --git a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java new file mode 100644 index 0000000..b94b7c3 --- /dev/null +++ b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.action.hadoop; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.apache.oozie.test.XTestCase; + +public class TestHive2Main extends XTestCase { + + public void testJobIDPattern() { + List<String> lines = new ArrayList<String>(); + lines.add("Ended Job = job_001"); + lines.add("Running with YARN Application = application_002"); + lines.add("Submitted application application_003"); + // Non-matching ones + lines.add("Ended Job = "); + lines.add("Ended Job = abc"); + lines.add("Running with YARN Application = job_004"); + lines.add("Running with YARN Application = application005"); + lines.add("Running with YARN Application = application_abc"); + lines.add("Submitted application job_006"); + Set<String> jobIds = new LinkedHashSet<String>(); + for (String line : lines) { + LauncherMain.extractJobIDs(line, Hive2Main.HIVE2_JOB_IDS_PATTERNS, + jobIds); + } + Set<String> expected = new LinkedHashSet<String>(); + expected.add("job_001"); + expected.add("job_002"); + expected.add("job_003"); + assertEquals(expected, jobIds); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index 785ca5e..8087178 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@ -38,6 +38,10 @@ import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.Shell; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; + import org.apache.commons.lang.StringUtils; import org.apache.hadoop.mapred.JobConf; @@ -69,17 +73,7 @@ public abstract class LauncherMain { try (BufferedReader br = new BufferedReader(new FileReader(logFile))) { String line = br.readLine(); while (line != null) { - for (Pattern pattern : patterns) { - Matcher matcher = pattern.matcher(line); - if (matcher.find()) { - String jobId = matcher.group(1); - if (StringUtils.isEmpty(jobId) || jobId.equalsIgnoreCase("NULL")) { - continue; - } - jobId = jobId.replaceAll("application", "job"); - jobIds.add(jobId); - } - } + extractJobIDs(line, patterns, jobIds); line = br.readLine(); } } catch (IOException e) { @@ -90,6 +84,22 @@ public abstract class LauncherMain { return jobIds.isEmpty() ? null : StringUtils.join(jobIds, ","); } + @VisibleForTesting + protected static void extractJobIDs(String line, Pattern[] patterns, Set<String> jobIds) { + Preconditions.checkNotNull(line); + for (Pattern pattern : patterns) { + Matcher matcher = pattern.matcher(line); + if (matcher.find()) { + String jobId = matcher.group(1); + if (StringUtils.isEmpty(jobId) || jobId.equalsIgnoreCase("NULL")) { + continue; + } + jobId = jobId.replaceAll("application", "job"); + jobIds.add(jobId); + } + } + } + protected static void writeExternalChildIDs(String logFile, Pattern[] patterns, String name) { // Harvesting and recording Hadoop Job IDs String jobIds = getHadoopJobIds(logFile, patterns); http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java index 7de2e45..0c47ac8 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java @@ -23,6 +23,9 @@ import org.apache.pig.PigRunner; import org.apache.pig.scripting.ScriptEngine; import org.apache.pig.tools.pigstats.JobStats; import org.apache.pig.tools.pigstats.PigStats; + +import com.google.common.annotations.VisibleForTesting; + import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -54,7 +57,8 @@ public class PigMain extends LauncherMain { public static final int STRING_BUFFER_SIZE = 100; public static final String LOG_EXPANDED_PIG_SCRIPT = LauncherMapper.ACTION_PREFIX + "pig.log.expandedscript"; - private static final Pattern[] PIG_JOB_IDS_PATTERNS = { + @VisibleForTesting + static final Pattern[] PIG_JOB_IDS_PATTERNS = { Pattern.compile("HadoopJobId: (job_\\S*)"), Pattern.compile("Submitted application (application[0-9_]*)") }; http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java index 081d319..22e7952 100644 --- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java +++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java @@ -31,8 +31,12 @@ import java.io.OutputStreamWriter; import java.io.Writer; import java.io.FileWriter; import java.io.FileReader; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.net.URL; public class TestPigMain extends PigTestCase { @@ -157,4 +161,23 @@ public class TestPigMain extends PigTestCase { return null; } + public void testJobIDPattern() { + List<String> lines = new ArrayList<String>(); + lines.add("HadoopJobId: job_001"); + lines.add("Submitted application application_002"); + // Non-matching ones + lines.add("HadoopJobId is set. job_003"); + lines.add("HadoopJobId: abc004"); + lines.add("Submitted application = job_005"); + lines.add("Submitted application. job_006"); + Set<String> jobIds = new LinkedHashSet<String>(); + for (String line : lines) { + LauncherMain.extractJobIDs(line, PigMain.PIG_JOB_IDS_PATTERNS, jobIds); + } + Set<String> expected = new LinkedHashSet<String>(); + expected.add("job_001"); + expected.add("job_002"); + assertEquals(expected, jobIds); + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java index 12eb61a..3cc6ccb 100644 --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@ -66,7 +66,8 @@ public class SparkMain extends LauncherMain { Pattern.compile("pyspark.zip") }; private static final Pattern SPARK_DEFAULTS_FILE_PATTERN = Pattern.compile("spark-defaults.conf"); private static final String SPARK_LOG4J_PROPS = "spark-log4j.properties"; - private static final Pattern[] SPARK_JOB_IDS_PATTERNS = { + @VisibleForTesting + static final Pattern[] SPARK_JOB_IDS_PATTERNS = { Pattern.compile("Submitted application (application[0-9_]*)") }; public static final Pattern SPARK_ASSEMBLY_JAR_PATTERN = Pattern .compile("^spark-assembly((?:(-|_|(\\d+\\.))\\d+(?:\\.\\d+)*))*\\.jar$"); http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java index bf4adf9..5f6a0cd 100644 --- a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java +++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java @@ -26,7 +26,9 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; @@ -137,4 +139,20 @@ public class TestSparkMain extends MainTestCase { instance.appendOoziePropertiesToSparkConf(sparkArgs, actionConf); assertEquals(Lists.newArrayList("--conf", "spark.oozie.bar=bar"), sparkArgs); } + + public void testJobIDPattern() { + List<String> lines = new ArrayList<String>(); + lines.add("Submitted application application_001"); + // Non-matching ones + lines.add("Submitted application job_002"); + lines.add("HadoopJobId: application_003"); + lines.add("Submitted application = application_004"); + Set<String> jobIds = new LinkedHashSet<String>(); + for (String line : lines) { + LauncherMain.extractJobIDs(line, SparkMain.SPARK_JOB_IDS_PATTERNS, jobIds); + } + Set<String> expected = new LinkedHashSet<String>(); + expected.add("job_001"); + assertEquals(expected, jobIds); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java ---------------------------------------------------------------------- diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java index cb65eb8..cb8f56e 100644 --- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java +++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java @@ -32,11 +32,14 @@ import org.apache.hadoop.fs.Path; import org.apache.log4j.PropertyConfigurator; import org.apache.sqoop.Sqoop; +import com.google.common.annotations.VisibleForTesting; + public class SqoopMain extends LauncherMain { public static final String SQOOP_SITE_CONF = "sqoop-site.xml"; - private static final Pattern[] SQOOP_JOB_IDS_PATTERNS = { + @VisibleForTesting + static final Pattern[] SQOOP_JOB_IDS_PATTERNS = { Pattern.compile("Job complete: (job_\\S*)"), Pattern.compile("Job (job_\\S*) has completed successfully"), Pattern.compile("Submitted application (application[0-9_]*)") @@ -153,6 +156,7 @@ public class SqoopMain extends LauncherMain { return logFile; } + @Override protected void run(String[] args) throws Exception { System.out.println(); System.out.println("Oozie Sqoop action configuration"); http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java ---------------------------------------------------------------------- diff --git a/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java new file mode 100644 index 0000000..d6f96d5 --- /dev/null +++ b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.action.hadoop; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.apache.oozie.test.XTestCase; + +public class TestSqoopMain extends XTestCase { + + public void testJobIDPattern() { + List<String> lines = new ArrayList<String>(); + lines.add("Job complete: job_001"); + lines.add("Job job_002 has completed successfully"); + lines.add("Submitted application application_003"); + // Non-matching ones + lines.add("Job complete: application_004"); + lines.add("Job application_005 has completed successfully"); + lines.add("Submitted application job_006"); + Set<String> jobIds = new LinkedHashSet<String>(); + for (String line : lines) { + LauncherMain.extractJobIDs(line, SqoopMain.SQOOP_JOB_IDS_PATTERNS, + jobIds); + } + Set<String> expected = new LinkedHashSet<String>(); + expected.add("job_001"); + expected.add("job_002"); + expected.add("job_003"); + assertEquals(expected, jobIds); + } +} \ No newline at end of file