Merge branch 'master' into oya
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/cb6be836 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/cb6be836 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/cb6be836 Branch: refs/heads/oya Commit: cb6be8369d1bd80c71c3b92707e71e4e88625364 Parents: 09c5389 f669852 Author: Peter Bacsko <[email protected]> Authored: Mon May 8 12:39:50 2017 +0200 Committer: Peter Bacsko <[email protected]> Committed: Mon May 8 12:39:50 2017 +0200 ---------------------------------------------------------------------- bin/test-patch | 65 +++--- bin/test-patch-11-findbugs-diff | 79 +++++-- bin/test-patch-20-tests | 31 ++- .../java/org/apache/oozie/cli/OozieCLI.java | 7 + .../org/apache/oozie/cli/ValidationUtil.java | 30 +++ .../oozie/client/CoordinatorWfAction.java | 48 ++++ .../org/apache/oozie/client/OozieClient.java | 69 ++++++ .../org/apache/oozie/client/rest/JsonTags.java | 6 + .../apache/oozie/client/rest/RestConstants.java | 10 + .../apache/oozie/client/rest/TestJsonUtils.java | 21 +- core/pom.xml | 12 +- core/src/main/conf/oozie-env.cmd | 2 +- core/src/main/conf/oozie-env.sh | 28 --- core/src/main/conf/oozie-log4j.properties | 13 ++ .../main/java/org/apache/oozie/BaseEngine.java | 85 ++++--- .../java/org/apache/oozie/BundleEngine.java | 50 +--- .../org/apache/oozie/CoordinatorEngine.java | 85 +++---- .../apache/oozie/CoordinatorWfActionBean.java | 86 +++++++ .../main/java/org/apache/oozie/DagEngine.java | 83 +------ .../main/java/org/apache/oozie/ErrorCode.java | 1 + .../oozie/action/hadoop/JavaActionExecutor.java | 14 +- .../action/hadoop/SqoopActionExecutor.java | 29 ++- .../oozie/action/ssh/SshActionExecutor.java | 2 +- .../org/apache/oozie/command/PurgeXCommand.java | 6 +- .../command/coord/CoordChangeXCommand.java | 4 +- .../command/coord/CoordSubmitXCommand.java | 25 ++ .../coord/CoordWfActionInfoXCommand.java | 144 ++++++++++++ .../org/apache/oozie/coord/HCatELFunctions.java | 15 +- .../logic/CoordInputLogicEvaluatorPhaseOne.java | 2 +- .../apache/oozie/dependency/HCatURIHandler.java | 41 +++- .../jpa/WorkflowActionGetJPAExecutor.java | 16 +- .../oozie/service/MemoryLocksService.java | 9 +- .../org/apache/oozie/service/PurgeService.java | 1 + .../org/apache/oozie/service/XLogService.java | 12 +- .../oozie/service/XLogStreamingService.java | 80 ++----- .../apache/oozie/service/ZKLocksService.java | 53 +++-- .../oozie/service/ZKXLogStreamingService.java | 138 +++-------- .../apache/oozie/servlet/BaseAdminServlet.java | 73 +++++- .../apache/oozie/servlet/BaseJobServlet.java | 22 ++ .../java/org/apache/oozie/servlet/JVMInfo.java | 2 +- .../apache/oozie/servlet/V0AdminServlet.java | 16 +- .../org/apache/oozie/servlet/V0JobServlet.java | 3 +- .../apache/oozie/servlet/V1AdminServlet.java | 41 ++-- .../org/apache/oozie/servlet/V1JobServlet.java | 4 +- .../org/apache/oozie/servlet/V2JobServlet.java | 45 ++++ .../org/apache/oozie/sla/SLACalcStatus.java | 65 ++++-- .../apache/oozie/sla/SLACalculatorMemory.java | 192 +++++++-------- .../sla/listener/SLAEmailEventListener.java | 2 +- .../apache/oozie/sla/service/SLAService.java | 5 + .../org/apache/oozie/util/AuthUrlClient.java | 3 +- .../java/org/apache/oozie/util/LogLine.java | 64 +++++ .../util/SimpleTimestampedMessageParser.java | 15 +- .../oozie/util/TimestampedMessageParser.java | 105 +++++---- .../main/java/org/apache/oozie/util/XLog.java | 12 + .../org/apache/oozie/util/XLogAuditFilter.java | 16 -- .../apache/oozie/util/XLogAuditStreamer.java | 79 +++++++ .../apache/oozie/util/XLogErrorStreamer.java | 67 ++++++ .../java/org/apache/oozie/util/XLogFilter.java | 152 +++++++++--- .../org/apache/oozie/util/XLogStreamer.java | 117 +++++++++- .../org/apache/oozie/util/db/SqlStatement.java | 4 +- .../src/main/resources/jetty-logging.properties | 26 +++ core/src/main/resources/oozie-default.xml | 232 +++++++++++-------- core/src/main/resources/oozie-log4j.properties | 7 +- .../oozie/TestCoordinatorEngineStreamLog.java | 9 +- .../action/hadoop/TestFsActionExecutor.java | 14 ++ .../action/hadoop/TestJavaActionExecutor.java | 14 ++ .../hadoop/TestLauncherHCatURIHandler.java | 28 ++- .../TestScriptLanguageActionExecutor.java | 5 - .../action/hadoop/TestShellActionExecutor.java | 18 +- .../org/apache/oozie/client/TestOozieCLI.java | 93 ++++++-- .../command/coord/TestCoordChangeXCommand.java | 40 +++- .../command/coord/TestCoordSubmitXCommand.java | 30 +++ .../coord/TestCoordWfActionInfoXCommand.java | 174 ++++++++++++++ .../apache/oozie/coord/TestHCatELFunctions.java | 72 +++++- .../oozie/dependency/TestHCatURIHandler.java | 21 +- .../oozie/service/TestConfigurationService.java | 7 +- .../apache/oozie/service/TestHASLAService.java | 3 + .../service/TestSparkConfigurationService.java | 6 +- .../oozie/service/TestXLogStreamingService.java | 57 ++++- .../service/TestZKXLogStreamingService.java | 46 +++- .../servlet/MockCoordinatorEngineService.java | 55 ++++- .../apache/oozie/servlet/TestAdminServlet.java | 32 ++- .../oozie/servlet/TestV1AdminServlet.java | 13 ++ .../apache/oozie/servlet/TestV2JobServlet.java | 189 +++++++++++++++ .../oozie/sla/TestSLACalculatorMemory.java | 22 +- .../org/apache/oozie/test/MiniHCatServer.java | 30 ++- .../org/apache/oozie/test/XHCatTestCase.java | 8 + .../org/apache/oozie/util/TestClassUtils.java | 7 +- .../apache/oozie/util/TestHCatURIParser.java | 87 +++++++ .../java/org/apache/oozie/util/TestIOUtils.java | 17 +- .../org/apache/oozie/util/TestLogStreamer.java | 47 +++- .../TestSimplifiedTimestampedMessageParser.java | 15 +- .../util/TestTimestampedMessageParser.java | 9 +- .../apache/oozie/util/TestWritableUtils.java | 9 +- .../oozie/util/TestXLogUserFilterParam.java | 34 +-- .../org/apache/oozie/util/TestXmlUtils.java | 8 +- .../resources/coord-invalid-el-function.xml | 35 +++ .../resources/coord-invalid-output-instance.xml | 58 +++++ .../coord-multiple-input-instance4.xml | 2 +- .../coord-multiple-input-start-instance2.xml | 2 +- .../coord-multiple-output-instance4.xml | 2 +- distro/pom.xml | 6 + distro/src/main/bin/oozie-setup.sh | 4 - distro/src/main/bin/oozie-sys.sh | 56 ----- docs/src/site/twiki/AG_Install.twiki | 68 ++---- .../site/twiki/CoordinatorFunctionalSpec.twiki | 6 +- docs/src/site/twiki/DG_CommandLineTool.twiki | 56 +++-- .../site/twiki/DG_CustomActionExecutor.twiki | 2 +- docs/src/site/twiki/DG_Examples.twiki | 10 +- .../src/site/twiki/DG_HCatalogIntegration.twiki | 42 +++- docs/src/site/twiki/DG_QuickStart.twiki | 10 +- docs/src/site/twiki/ENG_Building.twiki | 5 +- docs/src/site/twiki/WebServicesAPI.twiki | 28 +++ .../src/site/twiki/WorkflowFunctionalSpec.twiki | 121 +++------- docs/src/site/twiki/index.twiki | 2 +- pom.xml | 16 +- release-log.txt | 33 +++ .../oozie/server/EmbeddedOozieServer.java | 20 +- .../ConstraintSecurityHandlerProvider.java | 15 +- .../oozie/server/TestEmbeddedOozieServer.java | 2 + .../TestConstraintSecurityHandlerProvider.java | 66 ++++++ .../apache/oozie/action/hadoop/DistcpMain.java | 49 ++-- .../action/hadoop/HCatLauncherURIHandler.java | 6 +- .../java/org/apache/oozie/util/HCatURI.java | 55 +++-- .../org/apache/oozie/util/HCatURIParser.java | 48 ++++ .../java/org/apache/oozie/util/TestHCatURI.java | 27 ++- .../apache/oozie/action/hadoop/HiveMain.java | 82 +++---- .../oozie/action/hadoop/LauncherMain.java | 50 ++++ .../src/main/resources/default-log4j.properties | 32 +++ .../oozie/action/hadoop/TestLauncherMain.java | 57 +++++ .../org/apache/oozie/action/hadoop/PigMain.java | 62 +++-- .../oozie/action/hadoop/PigMainWithOldAPI.java | 58 ++--- .../apache/oozie/action/hadoop/SparkMain.java | 56 ++--- .../apache/oozie/action/hadoop/SqoopMain.java | 47 ++-- .../action/hadoop/TestSqoopActionExecutor.java | 106 ++++++++- webapp/pom.xml | 4 + webapp/src/main/webapp/oozie-console.js | 4 +- 137 files changed, 3844 insertions(+), 1556 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/client/src/main/java/org/apache/oozie/cli/OozieCLI.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 2256586,f62c997..048b75c --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@@ -36,11 -35,13 +36,13 @@@ import java.util.Iterator import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Properties; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import org.apache.commons.io.IOUtils; + import com.google.common.annotations.VisibleForTesting; + import com.google.common.primitives.Ints; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.FileStatus; @@@ -838,11 -983,12 +840,11 @@@ public class JavaActionExecutor extend prepareXML); // Set the launcher Main Class - LauncherMapperHelper.setupMainClass(launcherJobConf, getLauncherMain(launcherJobConf, actionXml)); - LauncherMapperHelper.setupLauncherURIHandlerConf(launcherJobConf); - - LauncherMapperHelper.setupMaxOutputData(launcherJobConf, getMaxOutputData(actionConf)); - LauncherMapperHelper.setupMaxExternalStatsSize(launcherJobConf, maxExternalStatsSize); - LauncherMapperHelper.setupMaxFSGlob(launcherJobConf, maxFSGlobMax); + LauncherHelper.setupMainClass(launcherJobConf, getLauncherMain(launcherJobConf, actionXml)); + LauncherHelper.setupLauncherURIHandlerConf(launcherJobConf); - LauncherHelper.setupMaxOutputData(launcherJobConf, maxActionOutputLen); ++ LauncherHelper.setupMaxOutputData(launcherJobConf, getMaxOutputData(actionConf)); + LauncherHelper.setupMaxExternalStatsSize(launcherJobConf, maxExternalStatsSize); + LauncherHelper.setupMaxFSGlob(launcherJobConf, maxFSGlobMax); List<Element> list = actionXml.getChildren("arg", ns); String[] args = new String[list.size()]; @@@ -882,9 -1037,33 +884,19 @@@ } } + @VisibleForTesting + protected static int getMaxOutputData(Configuration actionConf) { + String userMaxActionOutputLen = actionConf.get("oozie.action.max.output.data"); + if (userMaxActionOutputLen != null) { + Integer i = Ints.tryParse(userMaxActionOutputLen); + return i != null ? i : maxActionOutputLen; + } + return maxActionOutputLen; + } + - private boolean checkPropertiesToDisableUber(Configuration launcherConf) { - boolean disable = false; - if (launcherConf.getBoolean(HADOOP_JOB_CLASSLOADER, false)) { - disable = true; - } - else if (launcherConf.getBoolean(HADOOP_USER_CLASSPATH_FIRST, false)) { - disable = true; - } - return disable; - } - protected void injectCallback(Context context, Configuration conf) { - String callback = context.getCallbackUrl("$jobStatus"); - if (conf.get("job.end.notification.url") != null) { - LOG.warn("Overriding the action job end notification URI"); - } - conf.set("job.end.notification.url", callback); + String callback = context.getCallbackUrl(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_JOBSTATUS_TOKEN); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_URL, callback); } void injectActionCallback(Context context, Configuration actionConf) { http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java index 1541a42,22e2874..73cc190 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java @@@ -88,22 -91,30 +90,31 @@@ public class SqoopActionExecutor extend if (actionXml.getChild("command", ns) != null) { String command = actionXml.getChild("command", ns).getTextTrim(); StringTokenizer st = new StringTokenizer(command, " "); - List<String> l = new ArrayList<String>(); while (st.hasMoreTokens()) { - l.add(st.nextToken()); + argList.add(st.nextToken()); } - args = l.toArray(new String[l.size()]); } else { + @SuppressWarnings("unchecked") List<Element> eArgs = (List<Element>) actionXml.getChildren("arg", ns); - args = new String[eArgs.size()]; - for (int i = 0; i < eArgs.size(); i++) { - args[i] = eArgs.get(i).getTextTrim(); + for (Element elem : eArgs) { + argList.add(elem.getTextTrim()); } } + // If the command is given accidentally as "sqoop import --option" + // instead of "import --option" we can make a user's life easier + // by removing away the unnecessary "sqoop" token. + // However, we do not do this if the command looks like + // "sqoop --option", as that's entirely invalid. + if (argList.size() > 1 && + argList.get(0).equalsIgnoreCase(SQOOP) && + !argList.get(1).startsWith("-")) { + XLog.getLog(getClass()).info( + "Found a redundant 'sqoop' prefixing the command. Removing it."); + argList.remove(0); + } - setSqoopCommand(actionConf, args); + setSqoopCommand(actionConf, argList.toArray(new String[argList.size()])); return actionConf; } http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/main/resources/oozie-log4j.properties ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 3d423b9,d1f53fe..c3ff968 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@@ -2281,6 -2965,20 +2281,20 @@@ public class TestJavaActionExecutor ext assertNull(context.getAction().getData()); ae.end(context, context.getAction()); - assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); + assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); } + + public void testMaxOutputDataSetByUser() { + Configuration conf = new Configuration(false); + conf.set("oozie.action.max.output.data", "10000"); + + assertEquals(10000, JavaActionExecutor.getMaxOutputData(conf)); + } + + public void testInvalidMaxOutputDataSetByUser() { + Configuration conf = new Configuration(false); + conf.set("oozie.action.max.output.data", "sdasda"); + + assertEquals(2048, JavaActionExecutor.getMaxOutputData(conf)); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/test/java/org/apache/oozie/command/coord/TestCoordChangeXCommand.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/oozie/service/TestConfigurationService.java index 2d7b3b4,3c6525d..5957ad6 --- a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java +++ b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java @@@ -208,7 -209,10 +209,7 @@@ public class TestConfigurationService e assertEquals("10", ConfigurationService.get(JPAService.CONF_VALIDATE_DB_CONN_EVICTION_NUM).trim()); assertEquals(2048, ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA)); - assertEquals("http://localhost:8080/oozie?job=";, ConfigurationService.get(JobXCommand.CONF_CONSOLE_URL)); + assertEquals("http://0.0.0.0:11000/oozie?job=";, ConfigurationService.get(JobXCommand.CONF_CONSOLE_URL)); - assertEquals(true, ConfigurationService.getBoolean(JavaActionExecutor.CONF_HADOOP_YARN_UBER_MODE)); - assertEquals(false, ConfigurationService.getBoolean( - "oozie.action.shell.launcher." + JavaActionExecutor.HADOOP_YARN_UBER_MODE)); assertEquals(false, ConfigurationService.getBoolean(HadoopAccessorService.KERBEROS_AUTH_ENABLED)); assertEquals(0, ConfigurationService.getStrings("no.defined").length); http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/docs/src/site/twiki/DG_QuickStart.twiki ---------------------------------------------------------------------- diff --cc docs/src/site/twiki/DG_QuickStart.twiki index 37cb53c,fdc8374..607fd7f --- a/docs/src/site/twiki/DG_QuickStart.twiki +++ b/docs/src/site/twiki/DG_QuickStart.twiki @@@ -45,10 -49,10 +45,10 @@@ available to customise the versions of -Dhadoop.auth.version=<version> - defaults to hadoop version -Ddistcp.version=<version> - defaults to hadoop version -Dpig.version=<version> - default 0.16.0 --Dpig.classifier=<classifier> - default none +-Dpig.classifier=<classifier> - default h2 -Dsqoop.version=<version> - default 1.4.3 -Dsqoop.classifier=<classifier> - default hadoop100 - -Dtomcat.version=<version> - default 6.0.47 + -Djetty.version=<version> - default 9.2.19.v20160908 -Dopenjpa.version=<version> - default 2.2.2 -Dxerces.version=<version> - default 2.10.0 -Dcurator.version=<version> - default 2.5.0 http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/docs/src/site/twiki/ENG_Building.twiki ---------------------------------------------------------------------- diff --cc docs/src/site/twiki/ENG_Building.twiki index f0bbe71,019f8f1..67540dd --- a/docs/src/site/twiki/ENG_Building.twiki +++ b/docs/src/site/twiki/ENG_Building.twiki @@@ -216,10 -219,10 +215,10 @@@ the versions of the dependencies -Dhadoop.auth.version=<version> - defaults to hadoop version -Ddistcp.version=<version> - defaults to hadoop version -Dpig.version=<version> - default 0.16.0 --Dpig.classifier=<classifier> - default none +-Dpig.classifier=<classifier> - default h2 -Dsqoop.version=<version> - default 1.4.3 -Dsqoop.classifier=<classifier> - default hadoop100 - -Dtomcat.version=<version> - default 6.0.47 + -jetty.version=<version> - default 9.2.19.v20160908 -Dopenjpa.version=<version> - default 2.2.2 -Dxerces.version=<version> - default 2.10.0 -Dcurator.version=<version> - default 2.5.0 http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/release-log.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java ---------------------------------------------------------------------- diff --cc sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index 58051d1,9a411ac..5ec509c --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@@ -27,12 -27,10 +27,13 @@@ import java.io.IOException import java.io.InputStream; import java.io.OutputStream; import java.io.StringWriter; + import java.net.URL; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java ---------------------------------------------------------------------- diff --cc sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java index 41dc9f0,3dfd606..f3f729f --- a/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java +++ b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java @@@ -164,15 -197,68 +187,57 @@@ public class TestSqoopActionExecutor ex conn.close(); } + /** + * Tests a bad command of 'sqoop --username ...' style. + * Test asserts that the job will fail. + */ + public void testSqoopActionWithBadCommand() throws Exception { + runSqoopActionWithBadCommand(getBadCommandActionXml()); + } + + private void runSqoopActionWithBadCommand(String actionXml) throws Exception { + createDB(); + - Context context = createContext(actionXml); - final RunningJob launcherJob = submitAction(context); - String launcherId = context.getAction().getExternalId(); - waitFor(120 * 1000, new Predicate() { - public boolean evaluate() throws Exception { - return launcherJob.isComplete(); - } - }); - assertTrue(launcherJob.isSuccessful()); - Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), ++ Context context = createContext(getActionXml()); ++ final String launcherId = submitAction(context); ++ waitUntilYarnAppDoneAndAssertSuccess(launcherId); ++ ++ Map<String, String> actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(), + context.getProtoActionConf()); - assertFalse(LauncherMapperHelper.hasIdSwap(actionData)); ++ assertFalse(LauncherHelper.hasIdSwap(actionData)); + + SqoopActionExecutor ae = new SqoopActionExecutor(); + ae.check(context, context.getAction()); + assertTrue(launcherId.equals(context.getAction().getExternalId())); + assertEquals("FAILED/KILLED", context.getAction().getExternalStatus()); + ae.end(context, context.getAction()); + assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); + } + + /** + * Tests a normal command of 'import --username ...'. + */ public void testSqoopAction() throws Exception { + runSqoopAction(getActionXml()); + } + + /** + * Tests a redundant command of 'sqoop import --username ...'. + * The test guarantees a success, since the redundant 'sqoop' must get removed. + */ + public void testSqoopActionWithRedundantPrefix() throws Exception { + runSqoopAction(getRedundantCommandActionXml()); + } + + private void runSqoopAction(String actionXml) throws Exception { createDB(); - Context context = createContext(actionXml); - final RunningJob launcherJob = submitAction(context); - String launcherId = context.getAction().getExternalId(); - waitFor(120 * 1000, new Predicate() { - public boolean evaluate() throws Exception { - return launcherJob.isComplete(); - } - }); - assertTrue(launcherJob.isSuccessful()); - Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), + Context context = createContext(getActionXml()); + final String launcherId = submitAction(context); + waitUntilYarnAppDoneAndAssertSuccess(launcherId); + Map<String, String> actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(), context.getProtoActionConf()); - assertFalse(LauncherMapperHelper.hasIdSwap(actionData)); + assertFalse(LauncherHelper.hasIdSwap(actionData)); SqoopActionExecutor ae = new SqoopActionExecutor(); ae.check(context, context.getAction()); @@@ -222,15 -314,44 +287,38 @@@ assertTrue(hadoopCounters.isEmpty()); } - public void testSqoopActionFreeFormQuery() throws Exception { + /** + * Runs a job with arg-style command of 'sqoop --username ...' form that's invalid. + * The test ensures it fails. + */ + public void testSqoopActionWithBadRedundantArgsAndFreeFormQuery() throws Exception { + runSqoopActionWithBadCommand(getBadArgsActionXml()); + } + + /** + * Runs a job with the arg-style command of 'sqoop import --username ...'. + * The test guarantees that the redundant 'sqoop' is auto-removed (job passes). + */ + public void testSqoopActionWithRedundantArgsAndFreeFormQuery() throws Exception { + runSqoopActionFreeFormQuery(getArgsActionXmlFreeFromQuery(true)); + } + + /** + * Runs a job with the normal arg-style command of 'import --username ...'. + */ + public void testSqoopActionWithArgsAndFreeFormQuery() throws Exception { + runSqoopActionFreeFormQuery(getArgsActionXmlFreeFromQuery(false)); + } + + private void runSqoopActionFreeFormQuery(String actionXml) throws Exception { createDB(); - Context context = createContext(getActionXmlFreeFromQuery()); + Context context = createContext(actionXml); - final RunningJob launcherJob = submitAction(context); - String launcherId = context.getAction().getExternalId(); - waitFor(120 * 1000, new Predicate() { - public boolean evaluate() throws Exception { - return launcherJob.isComplete(); - } - }); - assertTrue(launcherJob.isSuccessful()); - Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), + final String launcherId = submitAction(context); + waitUntilYarnAppDoneAndAssertSuccess(launcherId); + Map<String, String> actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(), context.getProtoActionConf()); - assertFalse(LauncherMapperHelper.hasIdSwap(actionData)); + assertFalse(LauncherHelper.hasIdSwap(actionData)); SqoopActionExecutor ae = new SqoopActionExecutor(); ae.check(context, context.getAction()); http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/webapp/pom.xml ---------------------------------------------------------------------- diff --cc webapp/pom.xml index 36cde07,e4fdfb7..4dc0c30 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@@ -37,9 -37,13 +37,13 @@@ <artifactId>oozie-client</artifactId> <exclusions> <exclusion> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-auth</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-auth</artifactId> </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + </exclusion> </exclusions> </dependency>
