Merge branch 'apache_forked_master' into apache_forked_oya Change-Id: Ice9eb475ea2ebfd1ffbbd54b59d23665efd2a498
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/8315fb74 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/8315fb74 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/8315fb74 Branch: refs/heads/oya Commit: 8315fb749944695fc3f589f4511c22b1354ba4cd Parents: e19dd19 d638386 Author: Peter Bacsko <[email protected]> Authored: Tue Mar 21 12:22:17 2017 +0100 Committer: Peter Bacsko <[email protected]> Committed: Tue Mar 21 12:22:17 2017 +0100 ---------------------------------------------------------------------- .../java/org/apache/oozie/cli/OozieCLI.java | 21 +- .../org/apache/oozie/client/OozieClient.java | 100 ++++- .../client/event/jms/JMSMessagingUtils.java | 2 +- .../org/apache/oozie/client/rest/JsonTags.java | 5 + .../apache/oozie/client/rest/RestConstants.java | 2 + core/src/main/conf/action-conf/hive.xml | 41 +- .../org/apache/oozie/CoordinatorEngine.java | 14 + .../main/java/org/apache/oozie/ErrorCode.java | 7 +- .../oozie/action/email/EmailActionExecutor.java | 4 +- .../hadoop/ScriptLanguageActionExecutor.java | 3 +- .../action/hadoop/SparkActionExecutor.java | 15 +- .../oozie/command/SchemaCheckXCommand.java | 8 +- .../command/coord/CoordActionCheckXCommand.java | 2 +- .../CoordActionMissingDependenciesXCommand.java | 132 ++++++ .../command/coord/CoordActionStartXCommand.java | 2 +- .../oozie/command/coord/CoordCommandUtils.java | 157 +++++-- .../command/coord/CoordSLAAlertsXCommand.java | 15 +- .../apache/oozie/coord/CoordELFunctions.java | 68 ++- .../java/org/apache/oozie/coord/CoordUtils.java | 30 +- .../java/org/apache/oozie/coord/TimeUnit.java | 5 +- .../AbstractCoordInputDependency.java | 15 +- .../input/dependency/CoordInputDependency.java | 20 + .../dependency/CoordOldInputDependency.java | 125 +++++- .../dependency/CoordPullInputDependency.java | 19 + .../oozie/dependency/ActionDependency.java | 5 + .../hcat/EhcacheHCatDependencyCache.java | 42 +- .../hcat/SimpleHCatDependencyCache.java | 63 ++- .../oozie/executor/jpa/BatchQueryExecutor.java | 2 +- .../executor/jpa/CoordActionQueryExecutor.java | 22 +- .../oozie/service/HCatAccessorService.java | 4 +- .../oozie/service/HadoopAccessorService.java | 2 - .../oozie/service/JMSAccessorService.java | 2 +- .../service/SparkConfigurationService.java | 144 ++++--- .../apache/oozie/servlet/BaseJobServlet.java | 21 +- .../org/apache/oozie/servlet/V0JobServlet.java | 26 +- .../org/apache/oozie/servlet/V1JobServlet.java | 29 +- .../org/apache/oozie/servlet/V2JobServlet.java | 44 ++ .../apache/oozie/sla/SLACalculatorMemory.java | 2 +- .../java/org/apache/oozie/util/DateUtils.java | 21 +- .../org/apache/oozie/util/Instrumentation.java | 1 + .../main/java/org/apache/oozie/util/Pair.java | 6 +- .../org/apache/oozie/util/PasswordMasker.java | 121 ------ .../oozie/util/StringSerializationUtil.java | 72 ++++ .../oozie/workflow/lite/LiteWorkflowApp.java | 61 +-- .../workflow/lite/LiteWorkflowInstance.java | 33 +- .../org/apache/oozie/workflow/lite/NodeDef.java | 21 +- core/src/main/resources/oozie-default.xml | 39 +- .../oozie/TestCoordinatorEngineSimple.java | 4 +- .../oozie/action/hadoop/TestDistcpMain.java | 26 +- .../TestScriptLanguageActionExecutor.java | 101 +++++ .../org/apache/oozie/client/TestOozieCLI.java | 16 + ...tCoordActionMissingDependenciesXCommand.java | 288 +++++++++++++ .../command/coord/TestCoordCommandUtils.java | 303 +++++++++++++- .../TestCoordMaterializeTransitionXCommand.java | 47 +++ .../oozie/command/wf/TestSubmitXCommand.java | 47 ++- .../oozie/coord/TestCoordELFunctions.java | 8 + .../input/logic/TestCoordInputLogicPush.java | 59 +-- .../TestPartitionDependencyManagerService.java | 55 ++- .../service/TestSparkConfigurationService.java | 130 +++--- .../servlet/MockCoordinatorEngineService.java | 10 + .../apache/oozie/util/TestPasswordMasker.java | 92 ---- .../oozie/util/TestStringSerializationUtil.java | 53 +++ .../workflow/lite/TestLiteWorkflowApp.java | 78 ++++ .../test/resources/coord-dataset-endOfDays.xml | 48 +++ .../resources/coord-dataset-endOfMonths.xml | 48 +++ .../test/resources/coord-dataset-endOfWeeks.xml | 48 +++ .../coord-multiple-output-instance5.xml | 108 +++++ .../test/resources/instrumentation-os-env.json | 47 --- .../instrumentation-system-properties.json | 88 ---- .../test/resources/oldWorkFlowApp.serialized | Bin 0 -> 68945 bytes .../test/resources/out-of-phase-coordinator.xml | 35 ++ distro/pom.xml | 1 + .../site/twiki/CoordinatorFunctionalSpec.twiki | 365 +++++++++++++++- docs/src/site/twiki/DG_CommandLineTool.twiki | 26 ++ .../site/twiki/DG_SparkActionExtension.twiki | 36 ++ docs/src/site/twiki/WebServicesAPI.twiki | 49 +++ .../src/site/twiki/WorkflowFunctionalSpec.twiki | 2 + examples/src/main/apps/pyspark/job.properties | 26 ++ examples/src/main/apps/pyspark/lib/pi.py | 41 ++ examples/src/main/apps/pyspark/workflow.xml | 41 ++ pom.xml | 3 +- release-log.txt | 26 ++ server/pom.xml | 8 + .../oozie/server/SSLServerConnectorFactory.java | 7 +- .../org/apache/oozie/server/ServletMapper.java | 19 +- .../apache/oozie/action/hadoop/DistcpMain.java | 6 +- .../apache/oozie/action/hadoop/HiveMain.java | 8 +- .../oozie/action/hadoop/TestHiveMain.java | 35 +- .../apache/oozie/action/hadoop/Hive2Main.java | 8 +- .../oozie/action/hadoop/TestHive2Main.java | 54 +++ .../oozie/action/hadoop/LauncherMain.java | 32 +- .../oozie/action/hadoop/MapReduceMain.java | 13 +- .../oozie/action/hadoop/PasswordMasker.java | 144 +++++++ .../oozie/action/hadoop/TestPasswordMasker.java | 140 +++++++ .../test/resources/instrumentation-os-env.json | 47 +++ .../instrumentation-system-properties.json | 88 ++++ .../org/apache/oozie/action/hadoop/PigMain.java | 8 +- .../oozie/action/hadoop/PigMainWithOldAPI.java | 2 +- .../apache/oozie/action/hadoop/TestPigMain.java | 23 + .../apache/oozie/action/hadoop/SparkMain.java | 312 ++++++++++---- .../TestDuplicateFilteringInSparkMain.java | 87 ++++ .../oozie/action/hadoop/TestJarFilter.java | 109 +++++ .../oozie/action/hadoop/TestSparkMain.java | 35 ++ .../apache/oozie/action/hadoop/SqoopMain.java | 6 +- .../oozie/action/hadoop/TestSqoopMain.java | 49 +++ .../hadoop/TestMapReduceActionExecutor.java | 2 +- src/main/assemblies/distro.xml | 3 +- .../java/org/apache/oozie/tools/OozieDBCLI.java | 4 +- utils/dbutils/updatescripts/dbscripts-2.2.5.sql | 330 --------------- utils/dbutils/updatescripts/dbscripts-3.0.0.sql | 409 ------------------ utils/dbutils/updatescripts/dbscripts-3.0.1.sql | 414 ------------------ utils/dbutils/updatescripts/dbscripts-3.1.0.sql | 417 ------------------- utils/dbutils/updatescripts/readme.txt | 7 - .../updatescripts-2.x-to-3.0.0.sql | 101 ----- .../updatescripts-2.x-to-3.0.1.sql | 104 ----- .../updatescripts-3.0.0-to-3.0.1.sql | 23 - .../updatescripts-3.x-to-3.1.0.sql | 20 - .../upgradescript-2.x-to-3.0.0.sql | 37 -- webapp/src/main/webapp/oozie-console.js | 127 +++++- 119 files changed, 4456 insertions(+), 2732 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/client/src/main/java/org/apache/oozie/cli/OozieCLI.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/core/src/main/java/org/apache/oozie/action/hadoop/ScriptLanguageActionExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java index b411c4c,1a3197a..b237702 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java @@@ -31,7 -32,7 +31,8 @@@ import org.jdom.Namespace import java.util.ArrayList; import java.util.List; +import java.util.Map; + import java.util.Properties; public class SparkActionExecutor extends JavaActionExecutor { public static final String SPARK_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.SparkMain"; @@@ -76,10 -78,12 +77,12 @@@ StringBuilder sparkOptsSb = new StringBuilder(); if (master.startsWith("yarn")) { - String resourceManager = actionConf.get(HADOOP_JOB_TRACKER); + String resourceManager = actionConf.get(HADOOP_YARN_RM); - Map<String, String> sparkConfig = Services.get().get(SparkConfigurationService.class).getSparkConfig(resourceManager); - for (Map.Entry<String, String> entry : sparkConfig.entrySet()) { - sparkOptsSb.append("--conf ").append(entry.getKey()).append("=").append(entry.getValue()).append(" "); + Properties sparkConfig = + Services.get().get(SparkConfigurationService.class).getSparkConfig(resourceManager); + for (String property : sparkConfig.stringPropertyNames()) { + sparkOptsSb.append("--conf ") + .append(property).append("=").append(sparkConfig.getProperty(property)).append(" "); } } String sparkOpts = actionXml.getChildTextTrim("spark-opts", ns); http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/core/src/main/java/org/apache/oozie/dependency/hcat/EhcacheHCatDependencyCache.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/core/src/main/java/org/apache/oozie/util/Instrumentation.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/release-log.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java ---------------------------------------------------------------------- diff --cc sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index 148bed6,7ae48e1..d21b49a --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@@ -42,16 -38,12 +42,20 @@@ import java.util.regex.Pattern import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.Shell; + + import com.google.common.annotations.VisibleForTesting; + import com.google.common.base.Preconditions; + import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.client.ClientRMProxy; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.exceptions.YarnException; public abstract class LauncherMain { http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java ---------------------------------------------------------------------- diff --cc sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java index a1881a2,c24d95c..309071c --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@@ -85,8 -94,9 +94,9 @@@ public class SparkMain extends Launcher prepareHadoopConfig(actionConf); setYarnTag(actionConf); - LauncherMainHadoopUtils.killChildYarnJobs(actionConf); + LauncherMain.killChildYarnJobs(actionConf); String logFile = setUpSparkLog4J(actionConf); + setHiveSite(actionConf); List<String> sparkArgs = new ArrayList<String>(); sparkArgs.add(MASTER_OPTION); @@@ -319,18 -362,13 +362,18 @@@ * @param fileNamePattern the pattern to look for * @return the file if there is one else it returns null */ - private File getMatchingFile(Pattern fileNamePattern) throws OozieActionConfiguratorException { + private static File getMatchingFile(Pattern fileNamePattern) throws OozieActionConfiguratorException { File localDir = new File("."); - for(String fileName : localDir.list()){ - if(fileNamePattern.matcher(fileName).find()){ - return new File(fileName); + String[] files = localDir.list(); + + if (files != null) { + for(String fileName : files){ + if(fileNamePattern.matcher(fileName).find()){ + return new File(fileName); + } } } + return null; } http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8315fb74/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java ----------------------------------------------------------------------
