OOZIE-2339 [fluent-job] Minimum Viable Fluent Job API (daniel.becker, andras.piros via rkanter, gezapeti, pbacsko)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/8a0a6487 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/8a0a6487 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/8a0a6487 Branch: refs/heads/master Commit: 8a0a6487dcf9ef2e4489b5bdeea2360e12d91b14 Parents: f8cbce6 Author: Andras Piros <[email protected]> Authored: Tue Jun 19 11:11:06 2018 +0200 Committer: Andras Piros <[email protected]> Committed: Tue Jun 19 11:13:28 2018 +0200 ---------------------------------------------------------------------- client/pom.xml | 9 + .../java/org/apache/oozie/cli/OozieCLI.java | 115 ++- .../org/apache/oozie/client/ApiJarFactory.java | 117 +++ .../org/apache/oozie/client/ApiJarLoader.java | 78 ++ .../org/apache/oozie/client/OozieClient.java | 82 +- core/pom.xml | 11 + .../org/apache/oozie/BaseLocalOozieClient.java | 13 +- .../java/org/apache/oozie/local/LocalOozie.java | 4 + .../apache/oozie/servlet/BaseJobServlet.java | 3 +- .../apache/oozie/servlet/BaseJobsServlet.java | 43 + .../apache/oozie/servlet/ServletUtilities.java | 4 +- .../org/apache/oozie/servlet/V0JobsServlet.java | 5 + .../org/apache/oozie/servlet/V1JobsServlet.java | 141 ++- .../apache/oozie/servlet/V2ValidateServlet.java | 5 +- core/src/main/resources/oozie-default.xml | 11 + .../org/apache/oozie/client/TestOozieCLI.java | 191 +++- .../apache/oozie/servlet/TestV1JobsServlet.java | 12 +- .../apache/oozie/test/MiniOozieTestCase.java | 8 +- docs/src/site/twiki/DG_CommandLineTool.twiki | 139 +++ .../site/twiki/DG_CustomActionExecutor.twiki | 3 + docs/src/site/twiki/DG_Examples.twiki | 7 + docs/src/site/twiki/DG_FluentJobAPI.twiki | 376 ++++++++ examples/pom.xml | 6 + .../example/fluentjob/CredentialsRetrying.java | 93 ++ .../apache/oozie/example/fluentjob/Global.java | 49 + .../oozie/example/fluentjob/JavaMain.java | 56 ++ .../example/fluentjob/MultipleShellActions.java | 79 ++ .../oozie/example/fluentjob/Parameters.java | 39 + .../apache/oozie/example/fluentjob/Shell.java | 67 ++ .../apache/oozie/example/fluentjob/Spark.java | 61 ++ findbugs-filter.xml | 12 + fluent-job/fluent-job-api/pom.xml | 212 +++++ .../apache/oozie/fluentjob/api/Condition.java | 102 ++ .../oozie/fluentjob/api/GraphVisualization.java | 117 +++ .../apache/oozie/fluentjob/api/ModifyOnce.java | 67 ++ .../fluentjob/api/action/ActionAttributes.java | 258 +++++ .../api/action/ActionAttributesBuilder.java | 567 +++++++++++ .../oozie/fluentjob/api/action/Builder.java | 31 + .../oozie/fluentjob/api/action/ChFSBase.java | 81 ++ .../fluentjob/api/action/ChFSBaseBuilder.java | 95 ++ .../oozie/fluentjob/api/action/Chgrp.java | 53 ++ .../fluentjob/api/action/ChgrpBuilder.java | 68 ++ .../oozie/fluentjob/api/action/Chmod.java | 52 + .../fluentjob/api/action/ChmodBuilder.java | 68 ++ .../oozie/fluentjob/api/action/Delete.java | 60 ++ .../fluentjob/api/action/DistcpAction.java | 80 ++ .../api/action/DistcpActionBuilder.java | 118 +++ .../oozie/fluentjob/api/action/EmailAction.java | 118 +++ .../api/action/EmailActionBuilder.java | 234 +++++ .../fluentjob/api/action/ErrorHandler.java | 78 ++ .../oozie/fluentjob/api/action/FSAction.java | 137 +++ .../fluentjob/api/action/FSActionBuilder.java | 323 +++++++ .../fluentjob/api/action/HasAttributes.java | 31 + .../oozie/fluentjob/api/action/Hive2Action.java | 65 ++ .../api/action/Hive2ActionBuilder.java | 259 +++++ .../oozie/fluentjob/api/action/HiveAction.java | 56 ++ .../fluentjob/api/action/HiveActionBuilder.java | 242 +++++ .../oozie/fluentjob/api/action/JavaAction.java | 119 +++ .../fluentjob/api/action/JavaActionBuilder.java | 235 +++++ .../oozie/fluentjob/api/action/Launcher.java | 81 ++ .../fluentjob/api/action/LauncherBuilder.java | 92 ++ .../fluentjob/api/action/MapReduceAction.java | 138 +++ .../api/action/MapReduceActionBuilder.java | 256 +++++ .../oozie/fluentjob/api/action/Mkdir.java | 48 + .../apache/oozie/fluentjob/api/action/Move.java | 58 ++ .../apache/oozie/fluentjob/api/action/Node.java | 282 ++++++ .../api/action/NodeBuilderBaseImpl.java | 340 +++++++ .../oozie/fluentjob/api/action/PigAction.java | 107 +++ .../fluentjob/api/action/PigActionBuilder.java | 215 +++++ .../oozie/fluentjob/api/action/Pipes.java | 106 +++ .../fluentjob/api/action/PipesBuilder.java | 127 +++ .../oozie/fluentjob/api/action/Prepare.java | 58 ++ .../fluentjob/api/action/PrepareBuilder.java | 85 ++ .../oozie/fluentjob/api/action/ShellAction.java | 111 +++ .../api/action/ShellActionBuilder.java | 220 +++++ .../oozie/fluentjob/api/action/SparkAction.java | 134 +++ .../api/action/SparkActionBuilder.java | 261 +++++ .../oozie/fluentjob/api/action/SqoopAction.java | 99 ++ .../api/action/SqoopActionBuilder.java | 186 ++++ .../oozie/fluentjob/api/action/SshAction.java | 74 ++ .../fluentjob/api/action/SshActionBuilder.java | 136 +++ .../oozie/fluentjob/api/action/Streaming.java | 98 ++ .../fluentjob/api/action/StreamingBuilder.java | 117 +++ .../fluentjob/api/action/SubWorkflowAction.java | 88 ++ .../api/action/SubWorkflowActionBuilder.java | 159 ++++ .../oozie/fluentjob/api/action/Touchz.java | 47 + .../fluentjob/api/dag/DagNodeWithCondition.java | 108 +++ .../oozie/fluentjob/api/dag/Decision.java | 190 ++++ .../oozie/fluentjob/api/dag/DecisionJoin.java | 51 + .../org/apache/oozie/fluentjob/api/dag/End.java | 121 +++ .../oozie/fluentjob/api/dag/ExplicitNode.java | 150 +++ .../apache/oozie/fluentjob/api/dag/Fork.java | 142 +++ .../apache/oozie/fluentjob/api/dag/Graph.java | 872 +++++++++++++++++ .../apache/oozie/fluentjob/api/dag/Join.java | 36 + .../fluentjob/api/dag/JoiningNodeBase.java | 126 +++ .../oozie/fluentjob/api/dag/NodeBase.java | 104 ++ .../apache/oozie/fluentjob/api/dag/Start.java | 120 +++ .../fluentjob/api/factory/WorkflowFactory.java | 35 + .../api/mapping/BooleanToFLAGConverter.java | 50 + .../mapping/BooleanToShellFLAGConverter.java | 50 + .../api/mapping/BooleanToSshFLAGConverter.java | 50 + .../api/mapping/CredentialsConverter.java | 94 ++ .../api/mapping/DecisionConverter.java | 122 +++ .../mapping/DistcpConfigurationConverter.java | 81 ++ .../api/mapping/DistcpPrepareConverter.java | 97 ++ .../api/mapping/DozerBeanMapperSingleton.java | 50 + .../api/mapping/ExplicitNodeConverter.java | 296 ++++++ .../fluentjob/api/mapping/ForkConverter.java | 76 ++ .../fluentjob/api/mapping/GlobalConverter.java | 100 ++ .../oozie/fluentjob/api/mapping/GraphNodes.java | 86 ++ .../GraphNodesToWORKFLOWAPPConverter.java | 257 +++++ .../mapping/GraphToWORKFLOWAPPConverter.java | 67 ++ .../mapping/Hive2ConfigurationConverter.java | 81 ++ .../api/mapping/Hive2LauncherConverter.java | 74 ++ .../api/mapping/Hive2PrepareConverter.java | 97 ++ .../api/mapping/HiveConfigurationConverter.java | 81 ++ .../api/mapping/HiveLauncherConverter.java | 74 ++ .../api/mapping/HivePrepareConverter.java | 97 ++ .../InlineWorkflowConfigurationConverter.java | 81 ++ .../InlineWorkflowLauncherConverter.java | 74 ++ .../mapping/InlineWorkflowPrepareConverter.java | 97 ++ .../fluentjob/api/mapping/JoinConverter.java | 63 ++ .../MapToConfigurationPropertyConverter.java | 75 ++ .../api/mapping/ParametersConverter.java | 73 ++ .../fluentjob/api/mapping/RealChildLocator.java | 35 + .../mapping/ShellConfigurationConverter.java | 81 ++ .../api/mapping/ShellLauncherConverter.java | 74 ++ .../api/mapping/ShellPrepareConverter.java | 97 ++ .../mapping/SparkConfigurationConverter.java | 81 ++ .../api/mapping/SparkLauncherConverter.java | 74 ++ .../api/mapping/SparkPrepareConverter.java | 97 ++ .../mapping/SqoopConfigurationConverter.java | 81 ++ .../api/mapping/SqoopLauncherConverter.java | 74 ++ .../api/mapping/SqoopPrepareConverter.java | 97 ++ .../fluentjob/api/mapping/StartConverter.java | 50 + .../api/serialization/WorkflowMarshaller.java | 105 +++ .../api/workflow/ConfigurationEntry.java | 61 ++ .../fluentjob/api/workflow/Credential.java | 60 ++ .../api/workflow/CredentialBuilder.java | 89 ++ .../fluentjob/api/workflow/Credentials.java | 48 + .../api/workflow/CredentialsBuilder.java | 77 ++ .../oozie/fluentjob/api/workflow/Global.java | 67 ++ .../fluentjob/api/workflow/GlobalBuilder.java | 94 ++ .../oozie/fluentjob/api/workflow/Parameter.java | 57 ++ .../fluentjob/api/workflow/Parameters.java | 48 + .../api/workflow/ParametersBuilder.java | 66 ++ .../oozie/fluentjob/api/workflow/Workflow.java | 141 +++ .../fluentjob/api/workflow/WorkflowBuilder.java | 204 ++++ .../src/main/resources/action_mappings.xml | 821 ++++++++++++++++ .../src/main/resources/checkstyle-header.txt | 17 + .../src/main/resources/checkstyle.xml | 41 + .../src/main/resources/dozer_config.xml | 94 ++ .../resources/mappingGraphToWORKFLOWAPP.xml | 50 + .../fluent-job-api/src/main/xjb/bindings.xml | 166 ++++ .../apache/oozie/fluentjob/api/NodesToPng.java | 52 + .../oozie/fluentjob/api/TestCondition.java | 43 + .../oozie/fluentjob/api/TestModifyOnce.java | 52 + .../api/action/TestActionAttributesBuilder.java | 713 ++++++++++++++ .../fluentjob/api/action/TestChBaseBuilder.java | 127 +++ .../fluentjob/api/action/TestChgrpBuilder.java | 53 ++ .../fluentjob/api/action/TestChmodBuilder.java | 53 ++ .../oozie/fluentjob/api/action/TestDelete.java | 36 + .../api/action/TestDistcpActionBuilder.java | 213 +++++ .../api/action/TestEmailActionBuilder.java | 197 ++++ .../fluentjob/api/action/TestErrorHandler.java | 51 + .../api/action/TestFSActionBuilder.java | 469 +++++++++ .../api/action/TestHive2ActionBuilder.java | 224 +++++ .../api/action/TestHiveActionBuilder.java | 220 +++++ .../api/action/TestJavaActionBuilder.java | 225 +++++ .../api/action/TestLauncherBuilder.java | 44 + .../api/action/TestMapReduceActionBuilder.java | 392 ++++++++ .../oozie/fluentjob/api/action/TestMove.java | 36 + .../api/action/TestNodeBuilderBaseImpl.java | 523 +++++++++++ .../api/action/TestPigActionBuilder.java | 218 +++++ .../fluentjob/api/action/TestPipesBuilder.java | 168 ++++ .../api/action/TestPrepareBuilder.java | 105 +++ .../api/action/TestShellActionBuilder.java | 223 +++++ .../api/action/TestSparkActionBuilder.java | 229 +++++ .../api/action/TestSqoopActionBuilder.java | 219 +++++ .../api/action/TestSshActionBuilder.java | 147 +++ .../api/action/TestStreamingBuilder.java | 127 +++ .../api/action/TestSubWorkflowBuilder.java | 185 ++++ .../oozie/fluentjob/api/action/TestTouchz.java | 33 + .../oozie/fluentjob/api/dag/TestDecision.java | 210 +++++ .../fluentjob/api/dag/TestDecisionJoin.java | 36 + .../apache/oozie/fluentjob/api/dag/TestEnd.java | 117 +++ .../fluentjob/api/dag/TestExplicitNode.java | 156 +++ .../oozie/fluentjob/api/dag/TestFork.java | 156 +++ .../oozie/fluentjob/api/dag/TestGraph.java | 941 +++++++++++++++++++ .../oozie/fluentjob/api/dag/TestJoin.java | 52 + .../fluentjob/api/dag/TestJoiningNodeBase.java | 157 ++++ .../oozie/fluentjob/api/dag/TestNodeBase.java | 40 + .../oozie/fluentjob/api/dag/TestStart.java | 113 +++ .../api/factory/SimpleWorkflowFactory.java | 65 ++ .../api/mapping/SourceDataFactory.java | 48 + .../mapping/TestActionAttributesMapping.java | 100 ++ .../api/mapping/TestConfigurationMapping.java | 45 + .../api/mapping/TestControlNodeMappingBase.java | 27 + .../api/mapping/TestCredentialsMapping.java | 37 + .../api/mapping/TestDecisionMapping.java | 113 +++ .../api/mapping/TestDeleteMapping.java | 38 + .../api/mapping/TestDistcpActionMapping.java | 66 ++ .../api/mapping/TestEmailActionMapping.java | 60 ++ .../fluentjob/api/mapping/TestEndMapping.java | 37 + .../api/mapping/TestExplicitNodeMapping.java | 47 + .../api/mapping/TestFSActionMapping.java | 77 ++ .../fluentjob/api/mapping/TestForkMapping.java | 70 ++ .../api/mapping/TestGlobalMapping.java | 56 ++ .../fluentjob/api/mapping/TestGraphMapping.java | 255 +++++ .../api/mapping/TestHive2ActionMapping.java | 84 ++ .../api/mapping/TestHiveActionMapping.java | 84 ++ .../api/mapping/TestJavaActionMapping.java | 95 ++ .../fluentjob/api/mapping/TestJoinMapping.java | 64 ++ .../api/mapping/TestMapReduceActionMapping.java | 96 ++ .../fluentjob/api/mapping/TestMappings.java | 59 ++ .../fluentjob/api/mapping/TestMkdirMapping.java | 38 + .../api/mapping/TestParametersMapping.java | 47 + .../api/mapping/TestPigActionMapping.java | 84 ++ .../fluentjob/api/mapping/TestPipesMapping.java | 57 ++ .../api/mapping/TestPrepareMapping.java | 65 ++ .../api/mapping/TestShellActionMapping.java | 92 ++ .../api/mapping/TestSparkActionMapping.java | 98 ++ .../api/mapping/TestSqoopActionMapping.java | 89 ++ .../api/mapping/TestSshActionMapping.java | 59 ++ .../fluentjob/api/mapping/TestStartMapping.java | 61 ++ .../api/mapping/TestStreamingMapping.java | 64 ++ .../mapping/TestSubWorkflowActionMapping.java | 47 + .../mapping/TestWorkflowAttributesMapping.java | 53 ++ .../api/workflow/TestCredentialBuilder.java | 56 ++ .../api/workflow/TestCredentialsBuilder.java | 66 ++ .../api/workflow/TestGlobalBuilder.java | 63 ++ .../api/workflow/TestParametersBuilder.java | 80 ++ .../api/workflow/TestWorkflowBuilder.java | 258 +++++ fluent-job/fluent-job-client/pom.xml | 99 ++ .../jobs/client/jaxb/TestJAXBWorkflow.java | 435 +++++++++ .../jobs/client/minitest/TestDistcpAction.java | 89 ++ .../jobs/client/minitest/TestEmailAction.java | 73 ++ .../jobs/client/minitest/TestFSAction.java | 72 ++ .../jobs/client/minitest/TestHive2Action.java | 83 ++ .../jobs/client/minitest/TestHiveAction.java | 81 ++ .../jobs/client/minitest/TestJavaAction.java | 92 ++ .../client/minitest/TestMapReduceAction.java | 76 ++ .../jobs/client/minitest/TestPigAction.java | 81 ++ .../jobs/client/minitest/TestShellAction.java | 90 ++ .../jobs/client/minitest/TestSparkAction.java | 87 ++ .../jobs/client/minitest/TestSqoopAction.java | 80 ++ .../jobs/client/minitest/TestSshAction.java | 74 ++ .../src/test/resources/workflow-all-actions.xml | 234 +++++ .../resources/workflow-mapreduce-action.xml | 63 ++ fluent-job/pom.xml | 62 ++ minitest/pom.xml | 18 +- .../org/apache/oozie/test/TestWorkflow.java | 159 +--- .../org/apache/oozie/test/WorkflowTestCase.java | 259 +++++ pom.xml | 72 +- release-log.txt | 1 + 255 files changed, 29670 insertions(+), 198 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/client/pom.xml ---------------------------------------------------------------------- diff --git a/client/pom.xml b/client/pom.xml index 7d36e2d..118f1c3 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -95,6 +95,15 @@ <artifactId>xercesImpl</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>oozie-fluent-job-api</artifactId> + </dependency> + <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>annotations</artifactId> + <scope>provided</scope> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/client/src/main/java/org/apache/oozie/cli/OozieCLI.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java index 08e2b91..bc234e3 100644 --- a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java +++ b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java @@ -19,13 +19,16 @@ package org.apache.oozie.cli; import com.google.common.annotations.VisibleForTesting; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.OptionGroup; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.commons.io.FilenameUtils; import org.apache.oozie.BuildInfo; +import org.apache.oozie.client.ApiJarLoader; import org.apache.oozie.client.AuthOozieClient; import org.apache.oozie.client.BulkResponse; import org.apache.oozie.client.BundleJob; @@ -40,6 +43,8 @@ import org.apache.oozie.client.XOozieClient; import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.client.rest.JsonToBean; import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.fluentjob.api.serialization.WorkflowMarshaller; +import org.apache.oozie.fluentjob.api.workflow.Workflow; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.w3c.dom.DOMException; @@ -50,6 +55,7 @@ import org.w3c.dom.NodeList; import org.w3c.dom.Text; import org.xml.sax.SAXException; +import javax.xml.bind.JAXBException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -59,6 +65,10 @@ import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; +import java.lang.reflect.InvocationTargetException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -113,6 +123,9 @@ public class OozieCLI { public static final String LOG_OPTION = "log"; public static final String ERROR_LOG_OPTION = "errorlog"; public static final String AUDIT_LOG_OPTION = "auditlog"; + public static final String VALIDATE_JAR_OPTION = "validatejar"; + public static final String SUBMIT_JAR_OPTION = "submitjar"; + public static final String RUN_JAR_OPTION = "runjar"; public static final String ACTION_OPTION = "action"; public static final String DEFINITION_OPTION = "definition"; @@ -344,6 +357,9 @@ public class OozieCLI { Option log = new Option(LOG_OPTION, true, "job log"); Option errorlog = new Option(ERROR_LOG_OPTION, true, "job error log"); Option auditlog = new Option(AUDIT_LOG_OPTION, true, "job audit log"); + final Option generateAndCheck = new Option(VALIDATE_JAR_OPTION, true, "generate and check job definition"); + final Option generateAndSubmit = new Option(SUBMIT_JAR_OPTION, true, "generate and submit job definition"); + final Option generateAndRun = new Option(RUN_JAR_OPTION, true, "generate and run job definition"); Option logFilter = new Option( RestConstants.LOG_FILTER_OPTION, true, "job log search parameter. Can be specified as -logfilter opt1=val1;opt2=val1;opt3=val1. " @@ -407,6 +423,9 @@ public class OozieCLI { actions.addOption(log); actions.addOption(errorlog); actions.addOption(auditlog); + actions.addOption(generateAndCheck); + actions.addOption(generateAndSubmit); + actions.addOption(generateAndRun); actions.addOption(definition); actions.addOption(config_content); actions.addOption(ignore); @@ -1348,13 +1367,105 @@ public class OozieCLI { wc.getCoordActionMissingDependencies(commandLine.getOptionValue(COORD_ACTION_MISSING_DEPENDENCIES), actions, dates, System.out); } - + else if (options.contains(VALIDATE_JAR_OPTION)) { + checkApiJar(wc, commandLine, options.contains(VERBOSE_OPTION)); + } + else if (options.contains(SUBMIT_JAR_OPTION)) { + submitApiJar(wc, commandLine, options.contains(VERBOSE_OPTION)); + } + else if (options.contains(RUN_JAR_OPTION)) { + runApiJar(wc, commandLine, options.contains(VERBOSE_OPTION)); + } } - catch (OozieClientException ex) { + catch (final OozieClientException ex) { throw new OozieCLIException(ex.toString(), ex); } } + private void checkApiJar(final XOozieClient wc, final CommandLine commandLine, final boolean verbose) + throws OozieClientException { + final String apiJarPath = commandLine.getOptionValue(VALIDATE_JAR_OPTION); + logIfVerbose(verbose, "Checking API jar: " + apiJarPath); + + final String generatedXml = loadApiJarAndGenerateXml(apiJarPath, verbose); + + final Path workflowXml; + try { + workflowXml = Files.createTempFile("workflow", ".xml"); + Files.write(workflowXml, generatedXml.getBytes(StandardCharsets.UTF_8)); + + logIfVerbose(verbose, "API jar was written to " + workflowXml.toString()); + } + catch (final IOException e) { + throw new OozieClientException(e.getMessage(), e); + } + + logIfVerbose(verbose, "Servlet response is: "); + System.out.println(wc.validateXML(workflowXml.toString())); + + logIfVerbose(verbose, "API jar is valid."); + } + + private void logIfVerbose(final boolean verbose, final String message) { + if (verbose) { + System.out.println(message); + } + } + + @SuppressFBWarnings(value = {"PATH_TRAVERSAL_IN", "WEAK_FILENAMEUTILS"}, + justification = "FilenameUtils is used to filter user input. JDK8+ is used.") + private String loadApiJarAndGenerateXml(final String apiJarPath, final boolean verbose) throws OozieClientException { + final String generatedXml; + try { + logIfVerbose(verbose, "Loading API jar " + apiJarPath); + + final Workflow generatedWorkflow = new ApiJarLoader(new File( + FilenameUtils.getFullPath(apiJarPath) + FilenameUtils.getName(apiJarPath))) + .loadAndGenerate(); + generatedXml = WorkflowMarshaller.marshal(generatedWorkflow); + + logIfVerbose(verbose, "Workflow job definition generated from API jar: \n" + generatedXml); + } + catch (final IOException | ClassNotFoundException | IllegalAccessException | NoSuchMethodException | + InvocationTargetException | InstantiationException | JAXBException e) { + throw new OozieClientException(e.getMessage(), e); + } + + return generatedXml; + } + + private void submitApiJar(final XOozieClient wc, final CommandLine commandLine, final boolean verbose) + throws OozieClientException { + final String apiJarPath = commandLine.getOptionValue(SUBMIT_JAR_OPTION); + logIfVerbose(verbose, "Submitting a job based on API jar: " + apiJarPath); + + try { + System.out.println(JOB_ID_PREFIX + wc.submit(getConfiguration(wc, commandLine), + loadApiJarAndGenerateXml(apiJarPath, verbose))); + } + catch (final IOException e) { + throw new OozieClientException(e.getMessage(), e); + } + + logIfVerbose(verbose, "Job based on API jar submitted successfully."); + } + + private void runApiJar(final XOozieClient wc, final CommandLine commandLine, final boolean verbose) + throws OozieClientException { + final String apiJarPath = commandLine.getOptionValue(RUN_JAR_OPTION); + logIfVerbose(verbose, "Running a job based on API jar: " + apiJarPath); + + try { + System.out.println(JOB_ID_PREFIX + wc.run(getConfiguration(wc, commandLine), + loadApiJarAndGenerateXml(apiJarPath, verbose))); + } + catch (final IOException e) { + throw new OozieClientException(e.getMessage(), e); + } + + logIfVerbose(verbose, "Job based on API jar run successfully."); + } + @VisibleForTesting void printCoordJob(CoordinatorJob coordJob, String timeZoneId, boolean verbose) { System.out.println("Job ID : " + coordJob.getId()); http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/client/src/main/java/org/apache/oozie/client/ApiJarFactory.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/ApiJarFactory.java b/client/src/main/java/org/apache/oozie/client/ApiJarFactory.java new file mode 100644 index 0000000..bb85d67 --- /dev/null +++ b/client/src/main/java/org/apache/oozie/client/ApiJarFactory.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.client; + +import com.google.common.base.Preconditions; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.apache.oozie.fluentjob.api.factory.WorkflowFactory; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; + +/** + * Given a folder where the {@code .class} file containing a {@link WorkflowFactory} subclass persist, a name, and the + * exact {@code Class} reference, creates a {@code .jar} file containing {@code Main-Class} manifest attribute + * pointing to {@code apiFactoryClass}. + * <p> + * Used mainly by unit tests, this class helps to dynamically assemble Fluent Job API {@code .jar} files. + */ +class ApiJarFactory { + private final File classFolder; + private final File jarFolder; + private final String apiJarName; + private final Class<? extends WorkflowFactory> apiFactoryClass; + + ApiJarFactory(final File classFolder, + File jarFolder, final Class<? extends WorkflowFactory> apiFactoryClass, final String apiJarName) { + Preconditions.checkNotNull(classFolder, "classFolder should be set"); + Preconditions.checkNotNull(jarFolder, "jarFolder should be set"); + Preconditions.checkNotNull(apiJarName, "apiJarName should be set"); + Preconditions.checkNotNull(apiFactoryClass, "apiFactoryClass should be set"); + Preconditions.checkState(WorkflowFactory.class.isAssignableFrom(apiFactoryClass), + String.format("%s should be a %s", apiFactoryClass.getName(), WorkflowFactory.class.getName())); + + this.classFolder = classFolder; + this.jarFolder = jarFolder; + this.apiJarName = apiJarName; + this.apiFactoryClass = apiFactoryClass; + } + + @SuppressFBWarnings(value = {"PATH_TRAVERSAL_OUT", "WEAK_FILENAMEUTILS"}, + justification = "FilenameUtils is used to filter user output. JDK8+ is used.") + JarFile create() throws IOException { + final Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + manifest.getMainAttributes().put(Attributes.Name.MAIN_CLASS, apiFactoryClass.getName()); + + final String apiJarPath = jarFolder + File.separator + FilenameUtils.getName(apiJarName); + try (final JarOutputStream target = new JarOutputStream( + new FileOutputStream(FilenameUtils.getFullPath(apiJarPath) + FilenameUtils.getName(apiJarPath)), manifest)) { + addWorkflowJarEntry(classFolder, target); + } + + return new JarFile(apiJarPath); + } + + private void addWorkflowJarEntry(final File source, final JarOutputStream target) throws IOException { + if (source.isDirectory()) { + String name = source.getPath().replace("\\", "/"); + if (!name.isEmpty()) { + if (!name.endsWith("/")) { + name += "/"; + } + final JarEntry entry = new JarEntry(name); + entry.setTime(source.lastModified()); + target.putNextEntry(entry); + target.closeEntry(); + } + + final File[] nestedFiles = source.listFiles(); + if (nestedFiles == null) { + return; + } + + for (final File nestedFile : nestedFiles) { + addWorkflowJarEntry(nestedFile, target); + } + + return; + } + + try (final BufferedInputStream in = new BufferedInputStream(new FileInputStream(source))) { + + final JarEntry entry = new JarEntry(source.getPath().replace("\\", "/")); + entry.setTime(source.lastModified()); + target.putNextEntry(entry); + + IOUtils.copy(in, target); + target.closeEntry(); + } + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/client/src/main/java/org/apache/oozie/client/ApiJarLoader.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/ApiJarLoader.java b/client/src/main/java/org/apache/oozie/client/ApiJarLoader.java new file mode 100644 index 0000000..6ecf485 --- /dev/null +++ b/client/src/main/java/org/apache/oozie/client/ApiJarLoader.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.client; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import org.apache.oozie.fluentjob.api.factory.WorkflowFactory; +import org.apache.oozie.fluentjob.api.workflow.Workflow; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.jar.Attributes; +import java.util.jar.JarFile; + +/** + * Given a Fluent Job API {@code .jar} file compiled by the user this class loads the {@code Class} instance defined by + * {@code Main-Class} manifest attribute, calls its {@code Workflow create()} method, and returns its output to the caller. + */ +public class ApiJarLoader { + private final File apiJarFile; + + public ApiJarLoader(final File apiJarFile) { + Preconditions.checkArgument(apiJarFile.isFile(), "Fluent Job API JAR [%s] should be a file", apiJarFile.toString()); + Preconditions.checkArgument(apiJarFile.getName().endsWith(".jar"), "Fluent Job API JAR [%s] should be a JAR file", + apiJarFile.toString()); + + this.apiJarFile = apiJarFile; + } + + public Workflow loadAndGenerate() throws IOException, ClassNotFoundException, NoSuchMethodException, IllegalAccessException, + InstantiationException, InvocationTargetException { + final String mainClassName = getMainClassName(); + Preconditions.checkState(!Strings.isNullOrEmpty(mainClassName), + "Fluent Job API JAR should have a Main-Class defined in MANIFEST.MF"); + + final URLClassLoader workflowFactoryClassLoader = URLClassLoader.newInstance(new URL[]{apiJarFile.toURI().toURL()}); + final Class mainClass = workflowFactoryClassLoader.loadClass(mainClassName); + + Preconditions.checkNotNull(mainClass, "Fluent Job API JAR file should have a main class"); + Preconditions.checkState(WorkflowFactory.class.isAssignableFrom(mainClass), + "Fluent Job API JAR main class should be an " + WorkflowFactory.class.getName()); + + @SuppressWarnings("unchecked") + final Method mainMethod = mainClass.getMethod("create"); + Preconditions.checkState(Workflow.class.isAssignableFrom(mainMethod.getReturnType()), + "Fluent Job API JAR file's main class's create() method should return a " + Workflow.class.getName()); + + return (Workflow) mainMethod.invoke(mainClass.newInstance()); + } + + private String getMainClassName() throws IOException { + try (final JarFile apiJar = new JarFile(apiJarFile)) { + Preconditions.checkNotNull(apiJar.getManifest(), "Fluent Job API JAR doesn't have MANIFEST.MF"); + + return apiJar.getManifest().getMainAttributes().getValue(Attributes.Name.MAIN_CLASS); + } + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/client/src/main/java/org/apache/oozie/client/OozieClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java index 4c81756..2cc1692 100644 --- a/client/src/main/java/org/apache/oozie/client/OozieClient.java +++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java @@ -20,6 +20,9 @@ package org.apache.oozie.client; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import org.apache.commons.io.IOUtils; import com.google.common.collect.Lists; import org.apache.oozie.BuildInfo; import org.apache.oozie.cli.ValidationUtil; @@ -195,6 +198,8 @@ public class OozieClient { public static final String FILTER_SORT_BY = "sortby"; + public static final String CONFIG_KEY_GENERATED_XML = "oozie.jobs.api.generated.xml"; + public enum SORT_BY { createdTime("createdTimestamp"), lastModifiedTime("lastModifiedTimestamp"); private final String fullname; @@ -424,11 +429,7 @@ public class OozieClient { */ public Properties createConfiguration() { Properties conf = new Properties(); - String userName = USER_NAME_TL.get(); - if (userName == null) { - userName = System.getProperty("user.name"); - } - conf.setProperty(USER_NAME, userName); + conf.setProperty(USER_NAME, getUserName()); return conf; } @@ -690,36 +691,54 @@ public class OozieClient { private class JobSubmit extends ClientCallable<String> { private final Properties conf; - - JobSubmit(Properties conf, boolean start) { - super("POST", RestConstants.JOBS, "", (start) ? prepareParams(RestConstants.ACTION_PARAM, - RestConstants.JOB_ACTION_START) : prepareParams()); + private final String generatedXml; + + JobSubmit(final Properties conf, final boolean start, final String generatedXml) { + super("POST", RestConstants.JOBS, "", + (start) + ? prepareParams(RestConstants.ACTION_PARAM, + RestConstants.JOB_ACTION_START, + RestConstants.USER_PARAM, + getUserName()) + : prepareParams(RestConstants.USER_PARAM, + getUserName())); this.conf = notNull(conf, "conf"); + this.generatedXml = generatedXml; } JobSubmit(String jobId, Properties conf) { super("PUT", RestConstants.JOB, notEmpty(jobId, "jobId"), prepareParams(RestConstants.ACTION_PARAM, - RestConstants.JOB_ACTION_RERUN)); + RestConstants.JOB_ACTION_RERUN, RestConstants.USER_PARAM, getUserName())); this.conf = notNull(conf, "conf"); + this.generatedXml = null; } public JobSubmit(Properties conf, String jobActionDryrun) { super("POST", RestConstants.JOBS, "", prepareParams(RestConstants.ACTION_PARAM, - RestConstants.JOB_ACTION_DRYRUN)); + RestConstants.JOB_ACTION_DRYRUN, RestConstants.USER_PARAM, getUserName())); this.conf = notNull(conf, "conf"); + this.generatedXml = null; } @Override protected String call(HttpURLConnection conn) throws IOException, OozieClientException { conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); + + if (!Strings.isNullOrEmpty(generatedXml)) { + conf.setProperty(CONFIG_KEY_GENERATED_XML, generatedXml); + } + writeToXml(conf, conn.getOutputStream()); + if (conn.getResponseCode() == HttpURLConnection.HTTP_CREATED) { JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); return (String) json.get(JsonTags.JOB_ID); } + if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) { handleError(conn); } + return null; } } @@ -732,7 +751,11 @@ public class OozieClient { * @throws OozieClientException thrown if the job could not be submitted. */ public String submit(Properties conf) throws OozieClientException { - return (new JobSubmit(conf, false)).call(); + return (new JobSubmit(conf, false, null)).call(); + } + + public String submit(final Properties conf, final String generatedXml) throws OozieClientException { + return (new JobSubmit(conf, false, generatedXml)).call(); } private class JobAction extends ClientCallable<Void> { @@ -875,7 +898,11 @@ public class OozieClient { * @throws OozieClientException thrown if the job could not be submitted. */ public String run(Properties conf) throws OozieClientException { - return (new JobSubmit(conf, true)).call(); + return (new JobSubmit(conf, true, null)).call(); + } + + public String run(final Properties conf, final String generatedXml) throws OozieClientException { + return (new JobSubmit(conf, true, generatedXml)).call(); } /** @@ -2188,7 +2215,7 @@ public class OozieClient { String file = null; ValidateXML(String file, String user) { - super("POST", RestConstants.VALIDATE, "", + super("POST", WS_PROTOCOL_VERSION, RestConstants.VALIDATE, "", prepareParams(RestConstants.FILE_PARAM, file, RestConstants.USER_PARAM, user)); this.file = file; } @@ -2219,6 +2246,21 @@ public class OozieClient { } } + private class SubmitXML extends ClientCallable<String> { + SubmitXML(final String xml, final String user) { + super("POST", + WS_PROTOCOL_VERSION, + RestConstants.ACTION_PARAM, + RestConstants.JOB_ACTION_SUBMIT, + prepareParams()); + } + + @Override + protected String call(final HttpURLConnection conn) throws IOException, OozieClientException { + conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); + return null; + } + } private class UpdateSharelib extends ClientCallable<String> { @@ -2405,11 +2447,15 @@ public class OozieClient { } fileName = f.getAbsolutePath(); } - String user = USER_NAME_TL.get(); - if (user == null) { - user = System.getProperty("user.name"); + return new ValidateXML(fileName, getUserName()).call(); + } + + private String getUserName() { + String userName = USER_NAME_TL.get(); + if (userName == null) { + userName = System.getProperty("user.name"); } - return new ValidateXML(fileName, user).call(); + return userName; } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index 45880a6..a5a776c 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -404,6 +404,17 @@ </dependency> <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-fluent-job-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-fluent-job-api</artifactId> + <classifier>tests</classifier> + <scope>test</scope> + </dependency> + + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/core/src/main/java/org/apache/oozie/BaseLocalOozieClient.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/BaseLocalOozieClient.java b/core/src/main/java/org/apache/oozie/BaseLocalOozieClient.java index 38fb006..376ea11 100644 --- a/core/src/main/java/org/apache/oozie/BaseLocalOozieClient.java +++ b/core/src/main/java/org/apache/oozie/BaseLocalOozieClient.java @@ -29,6 +29,7 @@ import org.apache.oozie.client.OozieClientException; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.client.WorkflowJob; import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.servlet.V2ValidateServlet; import org.apache.oozie.util.XConfiguration; import org.json.simple.JSONObject; @@ -555,8 +556,16 @@ abstract class BaseLocalOozieClient extends OozieClient { } @Override - public String validateXML(String file) throws OozieClientException { - return throwNoOp(); + public String validateXML(final String xmlContent) throws OozieClientException { + final V2ValidateServlet validateServlet = new V2ValidateServlet(); + + try { + validateServlet.validate(xmlContent); + return V2ValidateServlet.VALID_WORKFLOW_APP; + } + catch (final Exception e) { + throw new OozieClientException("Cannot validate XML.", e); + } } @Override http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/core/src/main/java/org/apache/oozie/local/LocalOozie.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/local/LocalOozie.java b/core/src/main/java/org/apache/oozie/local/LocalOozie.java index 9ab646c..6475f33 100644 --- a/core/src/main/java/org/apache/oozie/local/LocalOozie.java +++ b/core/src/main/java/org/apache/oozie/local/LocalOozie.java @@ -87,6 +87,10 @@ public class LocalOozie { XLog.getLog(LocalOozie.class).info("LocalOozie started callback set to [{0}]", callbackUrl); } + public static boolean isStarted() { + return localOozieActive; + } + /** * Stop LocalOozie. */ http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java b/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java index e1bd3cf..3145650 100644 --- a/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java @@ -257,7 +257,8 @@ public abstract class BaseJobServlet extends JsonRestServlet { throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0405); } } - ServletUtilities.ValidateAppPath(wfPath, coordPath, bundlePath); + + ServletUtilities.validateAppPath(wfPath, coordPath, bundlePath); if (wfPath != null) { auth.authorizeForApp(user, acl, wfPath, "workflow.xml", conf); http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/core/src/main/java/org/apache/oozie/servlet/BaseJobsServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/BaseJobsServlet.java b/core/src/main/java/org/apache/oozie/servlet/BaseJobsServlet.java index d4b0871..8df1443 100644 --- a/core/src/main/java/org/apache/oozie/servlet/BaseJobsServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/BaseJobsServlet.java @@ -18,26 +18,33 @@ package org.apache.oozie.servlet; +import java.io.File; import java.io.IOException; import java.util.Arrays; +import java.util.Date; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.google.common.base.Strings; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.ErrorCode; import org.apache.oozie.client.OozieClient; +import org.apache.oozie.client.XOozieClient; import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.Services; import org.apache.oozie.service.AuthorizationException; import org.apache.oozie.service.AuthorizationService; import org.apache.oozie.util.JobUtils; import org.apache.oozie.util.JobsFilterUtils; import org.apache.oozie.util.XConfiguration; +import org.apache.oozie.util.XLog; import org.json.simple.JSONObject; public abstract class BaseJobsServlet extends JsonRestServlet { + private static final XLog LOG = XLog.getLog(BaseJobsServlet.class); private static final JsonRestServlet.ResourceInfo RESOURCES_INFO[] = new JsonRestServlet.ResourceInfo[1]; @@ -96,7 +103,15 @@ public abstract class BaseJobsServlet extends JsonRestServlet { if (!requestUser.equals(UNDEF)) { conf.set(OozieClient.USER_NAME, requestUser); } + + final String fsUser = request.getParameter(RestConstants.USER_PARAM) == null + ? conf.get(OozieClient.USER_NAME) + : request.getParameter(RestConstants.USER_PARAM); + + checkAndWriteApplicationXMLToHDFS(fsUser, ensureJobApplicationPath(conf)); + BaseJobServlet.checkAuthorizationForApp(conf); + JobUtils.normalizeAppPath(conf.get(OozieClient.USER_NAME), conf.get(OozieClient.GROUP_NAME), conf); JSONObject json = submitJob(request, conf); @@ -104,6 +119,34 @@ public abstract class BaseJobsServlet extends JsonRestServlet { sendJsonResponse(response, HttpServletResponse.SC_CREATED, json); } + private XConfiguration ensureJobApplicationPath(final XConfiguration configuration) { + if (!Strings.isNullOrEmpty(configuration.get(XOozieClient.IS_PROXY_SUBMISSION)) + && Boolean.valueOf(configuration.get(XOozieClient.IS_PROXY_SUBMISSION))) { + LOG.debug("Proxy submission in progress, no need to set application path."); + return configuration; + } + + if (Strings.isNullOrEmpty(configuration.get(OozieClient.APP_PATH)) + && Strings.isNullOrEmpty(configuration.get(OozieClient.LIBPATH)) + && Strings.isNullOrEmpty(configuration.get(OozieClient.COORDINATOR_APP_PATH)) + && Strings.isNullOrEmpty(configuration.get(OozieClient.BUNDLE_APP_PATH))) { + final String generatedJobApplicationPath = ConfigurationService.get("oozie.fluent-job-api.generated.path") + + File.separator + "gen_app_" + new Date().getTime(); + LOG.debug("Parameters [{0}], [{1}], [{2}], and [{3}] were all missing, setting to generated path [{4}]", + OozieClient.APP_PATH, + OozieClient.LIBPATH, + OozieClient.COORDINATOR_APP_PATH, + OozieClient.BUNDLE_APP_PATH, + generatedJobApplicationPath); + configuration.set(OozieClient.APP_PATH, generatedJobApplicationPath); + } + + return configuration; + } + + protected abstract void checkAndWriteApplicationXMLToHDFS(final String requestUser, final Configuration conf) + throws XServletException; + /** * Return information about jobs. */ http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/core/src/main/java/org/apache/oozie/servlet/ServletUtilities.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/ServletUtilities.java b/core/src/main/java/org/apache/oozie/servlet/ServletUtilities.java index 3eb32d5..e6c2670 100644 --- a/core/src/main/java/org/apache/oozie/servlet/ServletUtilities.java +++ b/core/src/main/java/org/apache/oozie/servlet/ServletUtilities.java @@ -32,7 +32,7 @@ public class ServletUtilities { * @param coordPath coordinator app path * @throws XServletException if either path is not valid */ - protected static void ValidateAppPath(String wfPath, String coordPath) throws XServletException { + protected static void validateAppPath(String wfPath, String coordPath) throws XServletException { if (wfPath != null && coordPath != null) { throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0302, "multiple app paths specified, only one is allowed"); @@ -55,7 +55,7 @@ public class ServletUtilities { * @param bundlePath bundle app path * @throws XServletException if either path is not valid */ - protected static void ValidateAppPath(String wfPath, String coordPath, String bundlePath) throws XServletException { + protected static void validateAppPath(String wfPath, String coordPath, String bundlePath) throws XServletException { int n = 0; if (wfPath != null) { http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/core/src/main/java/org/apache/oozie/servlet/V0JobsServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/V0JobsServlet.java b/core/src/main/java/org/apache/oozie/servlet/V0JobsServlet.java index 1d80094..0ff9c6a 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V0JobsServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V0JobsServlet.java @@ -119,6 +119,11 @@ public class V0JobsServlet extends BaseJobsServlet { } + @Override + protected void checkAndWriteApplicationXMLToHDFS(String requestUser, Configuration conf) throws XServletException { + // NOP + } + /** * service implementation to bulk kill jobs * @param request http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java b/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java index c1ca65f..5cfcf57 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java @@ -18,7 +18,13 @@ package org.apache.oozie.servlet; +import java.io.File; import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.StringReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -27,7 +33,12 @@ import java.util.Set; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.oozie.BaseEngineException; import org.apache.oozie.BulkResponseInfo; import org.apache.oozie.BundleEngine; @@ -43,19 +54,24 @@ import org.apache.oozie.OozieJsonFactory; import org.apache.oozie.WorkflowsInfo; import org.apache.oozie.cli.OozieCLI; import org.apache.oozie.client.OozieClient; +import org.apache.oozie.client.XOozieClient; import org.apache.oozie.client.rest.BulkResponseImpl; import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.service.BundleEngineService; import org.apache.oozie.service.CoordinatorEngineService; import org.apache.oozie.service.DagEngineService; +import org.apache.oozie.service.HadoopAccessorException; +import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.Services; +import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.XLog; import org.apache.oozie.util.XmlUtils; import org.json.simple.JSONArray; import org.json.simple.JSONObject; public class V1JobsServlet extends BaseJobsServlet { + private static final XLog LOG = XLog.getLog(V1JobsServlet.class); private static final String INSTRUMENTATION_NAME = "v1jobs"; private static final Set<String> httpJobType = new HashSet<String>(){{ @@ -84,7 +100,7 @@ public class V1JobsServlet extends BaseJobsServlet { String coordPath = conf.get(OozieClient.COORDINATOR_APP_PATH); String bundlePath = conf.get(OozieClient.BUNDLE_APP_PATH); - ServletUtilities.ValidateAppPath(wfPath, coordPath, bundlePath); + ServletUtilities.validateAppPath(wfPath, coordPath, bundlePath); if (wfPath != null) { json = submitWorkflowJob(request, conf); @@ -108,6 +124,129 @@ public class V1JobsServlet extends BaseJobsServlet { return json; } + @Override + protected void checkAndWriteApplicationXMLToHDFS(final String userName, final Configuration conf) throws XServletException { + if (!Strings.isNullOrEmpty(conf.get(XOozieClient.IS_PROXY_SUBMISSION)) + && Boolean.valueOf(conf.get(XOozieClient.IS_PROXY_SUBMISSION))) { + LOG.debug("Proxy submission in progress, no need to write application XML."); + return; + } + + final List<String> appPathsWithFileNames; + if (!findAppPathsWithFileNames(conf.get(OozieClient.APP_PATH), "workflow.xml").isEmpty()) { + appPathsWithFileNames = findAppPathsWithFileNames(conf.get(OozieClient.APP_PATH), "workflow.xml"); + } + else if (!findAppPathsWithFileNames(conf.get(OozieClient.LIBPATH), "workflow.xml").isEmpty()) { + appPathsWithFileNames = findAppPathsWithFileNames(conf.get(OozieClient.LIBPATH), "workflow.xml"); + } + else if (!findAppPathsWithFileNames(conf.get(OozieClient.COORDINATOR_APP_PATH), "coordinator.xml").isEmpty()) { + appPathsWithFileNames = findAppPathsWithFileNames(conf.get(OozieClient.COORDINATOR_APP_PATH), "coordinator.xml"); + } + else { + appPathsWithFileNames = findAppPathsWithFileNames(conf.get(OozieClient.BUNDLE_APP_PATH), "bundle.xml"); + } + + LOG.debug("Checking whether XML exists on HDFS. [appPathsWithFileNames={0}]", appPathsWithFileNames); + + for (final String appPathWithFileName : appPathsWithFileNames) { + if (existsOnDFS(userName, appPathWithFileName)) { + return; + } + } + + for (final String appPathWithFileName : appPathsWithFileNames) { + final String sourceContent = conf.get(OozieClient.CONFIG_KEY_GENERATED_XML); + if (sourceContent == null) { + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0307, + String.format("Configuration entry %s not present", OozieClient.CONFIG_KEY_GENERATED_XML)); + } + + if (tryCreateOnDFS(userName, appPathWithFileName, sourceContent)) { + return; + } + } + + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0307, + String.format("Could not create on HDFS any of the missing application XMLs [%s]", + appPathsWithFileNames)); + } + + private List<String> findAppPathsWithFileNames(final String appPaths, final String defaultFileName) { + final List<String> appPathsWithFileNames = Lists.newArrayList(); + + if (Strings.isNullOrEmpty(appPaths)) { + return appPathsWithFileNames; + } + + for (final String appPath : appPaths.split(",")) { + if (appPath.endsWith(".xml")) { + appPathsWithFileNames.add(appPath); + } + else { + appPathsWithFileNames.add(appPath + File.separator + defaultFileName); + } + } + + return appPathsWithFileNames; + } + + private boolean existsOnDFS(final String userName, final String appPathWithFileName) throws XServletException { + try { + final URI uri = new URI(appPathWithFileName); + final HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); + final Configuration fsConf = has.createConfiguration(uri.getAuthority()); + final FileSystem dfs = has.createFileSystem(userName, uri, fsConf); + + final Path path = new Path(uri.getPath()); + + if (dfs.exists(path)) { + if (!dfs.isFile(path)) { + final String errorMessage = String.format("HDFS path [%s] exists but is not a file.", path.toString()); + LOG.error(errorMessage); + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0307, errorMessage); + } + + LOG.debug("HDFS path [{0}] is an existing file, no need to create.", path.toString()); + return true; + } + + LOG.debug("HDFS path [{0}] is not an existing file.", path.toString()); + return false; + } + catch (final URISyntaxException | IOException | HadoopAccessorException e) { + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0307, + String.format("Could not check whether file [%s] exists on HDFS. Error message: %s", + appPathWithFileName, e.getMessage())); + } + } + + private boolean tryCreateOnDFS(final String userName, final String appPathWithFileName, final String sourceContent) { + try { + final URI uri = new URI(appPathWithFileName); + final HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); + final Configuration fsConf = has.createConfiguration(uri.getAuthority()); + final FileSystem dfs = has.createFileSystem(userName, uri, fsConf); + + final Path path = new Path(uri.getPath()); + + LOG.debug("HDFS path [{0}] does not exist, will try to create.", path.toString()); + + try (final FSDataOutputStream target = dfs.create(path)) { + LOG.debug("HDFS path [{0}] created.", path.toString()); + + IOUtils.copyCharStream(new StringReader(sourceContent), new OutputStreamWriter(target, StandardCharsets.UTF_8)); + } + + LOG.debug("XML written to HDFS file [{0}].", path.toString()); + + return true; + } + catch (final URISyntaxException | IOException | HadoopAccessorException e) { + LOG.warn("Could not write XML [%s] to HDFS. Error message: %s", appPathWithFileName, e.getMessage()); + return false; + } + } + /** * v1 service implementation to get a JSONObject representation of a job from its external ID */ http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java b/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java index b86fa6a..36a9de2 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java @@ -53,6 +53,7 @@ public class V2ValidateServlet extends JsonRestServlet { new ResourceInfo("", Arrays.asList("POST"), Arrays.asList( new ParameterInfo(RestConstants.FILE_PARAM, String.class, true, Arrays.asList("POST")), new ParameterInfo(RestConstants.USER_PARAM, String.class, true, Arrays.asList("POST")))); + public static final String VALID_WORKFLOW_APP = "Valid workflow-app"; public V2ValidateServlet() { @@ -98,12 +99,12 @@ public class V2ValidateServlet extends JsonRestServlet { file + ", " + e.toString()); } - JSONObject json = createJSON("Valid workflow-app"); + JSONObject json = createJSON(VALID_WORKFLOW_APP); startCron(); sendJsonResponse(response, HttpServletResponse.SC_OK, json); } - private void validate(String xml) throws Exception{ + public void validate(String xml) throws Exception { SchemaService schemaService = Services.get().get(SchemaService.class); Schema[] schemas = {schemaService.getSchema(SchemaService.SchemaName.WORKFLOW), schemaService.getSchema(SchemaService.SchemaName.COORDINATOR), http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 8d7465c..ff1820c 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -3377,5 +3377,16 @@ will be the requeue interval for the actions which are waiting for a long time w </description> </property> + <property> + <name>oozie.fluent-job-api.generated.path</name> + <value>/user/${user.name}/oozie-fluent-job-api-generated</value> + <description> + HDFS path to store workflow / coordinator / bundle definitions generated by fluent-job-api artifact. + The XML files are first generated out of the fluent-job-api JARs submitted by the user at command line, then stored + under this HDFS folder structure for later retrieval / resubmit / check. + Note that the submitting user needs r/w permissions under this HDFS folder. + Note further that this folder structure, when does not exist, will be created. + </description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java index 3395cc2..56639b0 100644 --- a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java +++ b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java @@ -23,18 +23,30 @@ import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.io.PrintStream; import java.io.StringReader; +import java.util.Enumeration; import java.util.Properties; import java.util.concurrent.Callable; - +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.io.Files; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.oozie.BuildInfo; import org.apache.oozie.cli.CLIParser; import org.apache.oozie.cli.OozieCLI; import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.fluentjob.api.factory.SimpleWorkflowFactory; +import org.apache.oozie.fluentjob.api.factory.WorkflowFactory; import org.apache.oozie.service.InstrumentationService; import org.apache.oozie.service.MetricsInstrumentationService; import org.apache.oozie.service.Services; @@ -51,7 +63,6 @@ import org.apache.oozie.servlet.V2JobServlet; import org.apache.oozie.servlet.V2ValidateServlet; import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.XConfiguration; -import org.json.simple.JSONValue; //hardcoding options instead using constants on purpose, to detect changes to option names if any and correct docs. public class TestOozieCLI extends DagServletTestCase { @@ -84,7 +95,9 @@ public class TestOozieCLI extends DagServletTestCase { private String createConfigFile(String appPath) throws Exception { String path = getTestCaseDir() + "/" + getName() + ".xml"; Configuration conf = new Configuration(false); - conf.set(OozieClient.APP_PATH, appPath); + if (!Strings.isNullOrEmpty(appPath)) { + conf.set(OozieClient.APP_PATH, appPath); + } conf.set(OozieClient.RERUN_SKIP_NODES, "node"); OutputStream os = new FileOutputStream(path); @@ -1711,6 +1724,178 @@ public class TestOozieCLI extends DagServletTestCase { }); } + public void testValidateJar() throws Exception { + final JarFile workflowApiJar = createWorkflowApiJar(); + + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + String oozieUrl = getContextURL(); + + String[] args = new String[]{"job", + "-validatejar", workflowApiJar.getName(), + "-oozie", oozieUrl, + "-verbose"}; + assertEquals(0, new OozieCLI().run(args)); + + return null; + } + }); + } + + public void testSubmitJar() throws Exception { + final JarFile workflowApiJar = createWorkflowApiJar(); + + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + final int wfCount = MockDagEngineService.INIT_WF_COUNT; + + final String oozieUrl = getContextURL(); + final Path appPath = new Path(getFsTestCaseDir(), "app"); + getFileSystem().mkdirs(appPath); + + final String[] args = new String[]{"job", + "-submitjar", workflowApiJar.getName(), + "-oozie", oozieUrl, + "-config", createConfigFile(appPath.toString()), + "-verbose"}; + assertEquals(0, new OozieCLI().run(args)); + assertEquals("submit", MockDagEngineService.did); + assertFalse(MockDagEngineService.started.get(wfCount)); + + return null; + } + }); + } + + public void testSubmitJarWithoutAppPath() throws Exception { + final JarFile workflowApiJar = createWorkflowApiJar(); + + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + final int wfCount = MockDagEngineService.INIT_WF_COUNT; + + final String oozieUrl = getContextURL(); + final Path appPath = new Path(getFsTestCaseDir(), "app"); + getFileSystem().mkdirs(appPath); + + final String[] args = new String[]{"job", + "-submitjar", workflowApiJar.getName(), + "-oozie", oozieUrl, + "-config", createConfigFile(null), + "-verbose"}; + assertEquals(0, new OozieCLI().run(args)); + assertEquals("submit", MockDagEngineService.did); + assertFalse(MockDagEngineService.started.get(wfCount)); + + return null; + } + }); + } + + public void testRunJar() throws Exception { + final JarFile workflowApiJar = createWorkflowApiJar(); + + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + final int wfCount = MockDagEngineService.INIT_WF_COUNT; + + final String oozieUrl = getContextURL(); + final Path appPath = new Path(getFsTestCaseDir(), "app"); + getFileSystem().mkdirs(appPath); + + final String[] args = new String[]{"job", + "-runjar", workflowApiJar.getName(), + "-oozie", oozieUrl, + "-config", createConfigFile(appPath.toString()), + "-verbose"}; + assertEquals(0, new OozieCLI().run(args)); + assertEquals("submit", MockDagEngineService.did); + assertTrue(MockDagEngineService.started.get(wfCount)); + + return null; + } + }); + } + + public void testRunJarWithoutAppPath() throws Exception { + final JarFile workflowApiJar = createWorkflowApiJar(); + + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + final int wfCount = MockDagEngineService.INIT_WF_COUNT; + + final String oozieUrl = getContextURL(); + final Path appPath = new Path(getFsTestCaseDir(), "app"); + getFileSystem().mkdirs(appPath); + + final String[] args = new String[]{"job", + "-runjar", workflowApiJar.getName(), + "-oozie", oozieUrl, + "-config", createConfigFile(null), + "-verbose"}; + assertEquals(0, new OozieCLI().run(args)); + assertEquals("submit", MockDagEngineService.did); + assertTrue(MockDagEngineService.started.get(wfCount)); + + return null; + } + }); + } + + private JarFile createWorkflowApiJar() throws IOException { + final String workflowApiJarName = "workflow-api-jar.jar"; + final File targetFolder = Files.createTempDir(); + final File classFolder = new File(targetFolder.getPath() + File.separator + "classes"); + Preconditions.checkState(classFolder.mkdir(), "could not create classFolder %s", classFolder); + final File jarFolder = new File(targetFolder.getPath() + File.separator + "jar"); + Preconditions.checkState(jarFolder.mkdir(), "could not create jarFolder %s", jarFolder); + final Class<? extends WorkflowFactory> workflowFactoryClass = SimpleWorkflowFactory.class; + final String workflowFactoryClassName = workflowFactoryClass.getSimpleName() + ".class"; + final String workflowFactoryClassLocation = + workflowFactoryClass.getProtectionDomain().getCodeSource().getLocation().getFile(); + + if (workflowFactoryClassLocation.endsWith(".jar")) { + // Maven / JAR file based test execution + extractClassFromJar(new JarFile(workflowFactoryClassLocation), workflowFactoryClassName, classFolder); + } + else { + // IDE / class file based test execution + FileUtils.copyDirectory( + new File(workflowFactoryClassLocation), + classFolder, + FileFilterUtils.or(FileFilterUtils.directoryFileFilter(), + FileFilterUtils.nameFileFilter(workflowFactoryClassName))); + } + + return new ApiJarFactory(targetFolder, jarFolder, workflowFactoryClass, workflowApiJarName).create(); + } + + private void extractClassFromJar(final JarFile jarFile, final String filterClassName, final File targetFolder) + throws IOException { + final Enumeration<JarEntry> entries = jarFile.entries(); + while (entries.hasMoreElements()) { + final JarEntry entry = entries.nextElement(); + if (entry.getName().endsWith(File.separator + filterClassName)) { + final String relativeClassFolderName = entry.getName().substring(0, entry.getName().lastIndexOf(File.separator)); + final String classFileName = + entry.getName().substring(entry.getName().lastIndexOf(File.separator) + File.separator.length()); + final File classFolder = new File(targetFolder + File.separator + relativeClassFolderName); + Preconditions.checkState(classFolder.mkdirs(), "could not create classFolder %s", classFolder); + + try (final InputStream entryStream = jarFile.getInputStream(entry); + final FileOutputStream filterClassStream = new FileOutputStream( + new File(classFolder + File.separator + classFileName))) { + IOUtils.copyStream(entryStream, filterClassStream); + } + } + } + } + private String runOozieCLIAndGetStdout(String[] args) { PrintStream original = System.out; ByteArrayOutputStream baos = new ByteArrayOutputStream(); http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/core/src/test/java/org/apache/oozie/servlet/TestV1JobsServlet.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/servlet/TestV1JobsServlet.java b/core/src/test/java/org/apache/oozie/servlet/TestV1JobsServlet.java index fb481e3..589d859 100644 --- a/core/src/test/java/org/apache/oozie/servlet/TestV1JobsServlet.java +++ b/core/src/test/java/org/apache/oozie/servlet/TestV1JobsServlet.java @@ -72,7 +72,7 @@ public class TestV1JobsServlet extends DagServletTestCase { jobConf.set(OozieClient.USER_NAME, getTestUser()); jobConf.set(OozieClient.APP_PATH, appPath); - Map<String, String> params = new HashMap<String, String>(); + Map<String, String> params = new HashMap<>(); URL url = createURL("", params); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("POST"); @@ -90,7 +90,7 @@ public class TestV1JobsServlet extends DagServletTestCase { jobConf.set(OozieClient.USER_NAME, getTestUser()); jobConf.set(OozieClient.APP_PATH, appPath); - params = new HashMap<String, String>(); + params = new HashMap<>(); params.put(RestConstants.ACTION_PARAM, RestConstants.JOB_ACTION_START); url = createURL("", params); conn = (HttpURLConnection) url.openConnection(); @@ -112,7 +112,7 @@ public class TestV1JobsServlet extends DagServletTestCase { jobConf = new XConfiguration(); jobConf.set(OozieClient.USER_NAME, getTestUser()); - params = new HashMap<String, String>(); + params = new HashMap<>(); url = createURL("", params); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("POST"); @@ -129,7 +129,7 @@ public class TestV1JobsServlet extends DagServletTestCase { jobConf.set(OozieClient.USER_NAME, getTestUser()); jobConf.set(OozieClient.LIBPATH, libPath1.toString()); - params = new HashMap<String, String>(); + params = new HashMap<>(); url = createURL("", params); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("POST"); @@ -137,7 +137,6 @@ public class TestV1JobsServlet extends DagServletTestCase { conn.setDoOutput(true); jobConf.writeXml(conn.getOutputStream()); assertEquals(HttpServletResponse.SC_CREATED, conn.getResponseCode()); - assertEquals(HttpServletResponse.SC_CREATED, conn.getResponseCode()); obj = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); assertEquals(MockDagEngineService.JOB_ID + wfCount + MockDagEngineService.JOB_ID_END, obj.get(JsonTags.JOB_ID)); @@ -150,7 +149,7 @@ public class TestV1JobsServlet extends DagServletTestCase { jobConf.set(OozieClient.USER_NAME, getTestUser()); jobConf.set(OozieClient.LIBPATH, libPath1.toString() + "," + libPath2.toString()); - params = new HashMap<String, String>(); + params = new HashMap<>(); url = createURL("", params); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("POST"); @@ -158,7 +157,6 @@ public class TestV1JobsServlet extends DagServletTestCase { conn.setDoOutput(true); jobConf.writeXml(conn.getOutputStream()); assertEquals(HttpServletResponse.SC_CREATED, conn.getResponseCode()); - assertEquals(HttpServletResponse.SC_CREATED, conn.getResponseCode()); obj = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); assertEquals(MockDagEngineService.JOB_ID + wfCount + MockDagEngineService.JOB_ID_END, obj.get(JsonTags.JOB_ID)); http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/core/src/test/java/org/apache/oozie/test/MiniOozieTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/MiniOozieTestCase.java b/core/src/test/java/org/apache/oozie/test/MiniOozieTestCase.java index 0a030d6..46698d1 100644 --- a/core/src/test/java/org/apache/oozie/test/MiniOozieTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/MiniOozieTestCase.java @@ -61,12 +61,16 @@ public abstract class MiniOozieTestCase extends XFsTestCase { protected void setUp() throws Exception { System.setProperty("hadoop20", "true"); super.setUp(); - LocalOozie.start(); + if (!LocalOozie.isStarted()) { + LocalOozie.start(); + } } @Override protected void tearDown() throws Exception { - LocalOozie.stop(); + if (LocalOozie.isStarted()) { + LocalOozie.stop(); + } super.tearDown(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/docs/src/site/twiki/DG_CommandLineTool.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_CommandLineTool.twiki b/docs/src/site/twiki/DG_CommandLineTool.twiki index 9a17459..d1d302a 100644 --- a/docs/src/site/twiki/DG_CommandLineTool.twiki +++ b/docs/src/site/twiki/DG_CommandLineTool.twiki @@ -106,6 +106,16 @@ oozie job <OPTIONS> : job operations -slaenable enables sla alerts for the job and its children -slachange Update sla param for jobs, supported param are should-start, should-end and max-duration -retries Get information of the retry attempts for a given workflow action. + -apijarcheck <jar> based on the supplied Fluent Job API jar, a workflow definition XML is generated and checked whether + it's a valid Oozie workflow. Output is whether the generated workflow is a valid one + -apijarsubmit <jar> based on the supplied Fluent Job API jar, a workflow definition XML is generated and submitted. When + the parameter =oozie.wf.application.path= isn't supplied on the command line, an HDFS location with + the prefix defined in + =oozie-site.xml#oozie.client.jobs.application.generated.path= is used. Output is the workflow ID + -apijarrun <jar> based on the supplied Fluent Job API jar, a workflow definition XML is generated and run. When the + parameter =oozie.wf.application.path= isn't supplied on the command line, an HDFS location with the + prefix defined in + =oozie-site.xml#oozie.client.jobs.application.generated.path= is used. Output is the workflow ID </verbatim> @@ -1033,6 +1043,135 @@ Pending Dependencies : $ </verbatim> +---+++ Checking a workflow definition generated by a Fluent Job API jar file + +Since Oozie 5.1.0. + +Generate a workflow definition given the Fluent Job API jar file supplied at command line, and check for its correctness. + +*Preconditions:* + * the Fluent Job API jar file has to be present and readable by the current user at the local path provided + * the folder containing the Fluent Job API jar file provided has to be writable by the current user, since the generated workflow + definition is written there + +If the =-verbose= option is provided, a lot more debugging output, including the generated workflow definition, is given. + +For more information what an Fluent Job API jar file is, how to build it etc., +refer to [[DG_FluentJobAPI#AE.A_Appendix_A_API_JAR_format][Fluent Job API - API JAR format]]. + +*Example:* + +<verbatim> +$ oozie job -oozie http://localhost:11000/oozie -validatejar /tmp/workflow-api-jar.jar +Valid workflow-app +</verbatim> + +*Example (verbose):* + +<verbatim> +$ oozie job -oozie http://localhost:11000/oozie -validatejar /tmp/workflow-api-jar.jar -verbose +Checking API jar:/tmp/workflow-api-jar.jar +Loading API jar /tmp/workflow-api-jar.jar +Workflow job definition generated from API jar: +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<workflow:workflow-app xmlns:workflow="uri:oozie:workflow:1.0" ... name="shell-example"> +... +</workflow:workflow-app> +. +API jar is written to /tmp/workflow1876390751841950810.xml +Servlet response is: +Valid workflow-app +API jar is valid. +</verbatim> + +---+++ Submitting a workflow definition generated by a Fluent Job API jar file + +Since Oozie 5.1.0. + +Generate a workflow definition given the Fluent Job API jar file supplied at command line, write it to a provided or generated HDFS +location, and submit. + +*Preconditions:* + * all the parameters that are present in the workflow definition have to be supplied either as command line parameters or via + =job.properties= passed along with the =-config= option + * the Fluent Job API jar file has to be present and readable by the current user at the local path provided + * the folder containing the Fluent Job API jar file provided has to be writable by the current user, since the generated workflow + definition is written there + * the HDFS folder either given by =-Doozie.wf.application.path= command line parameter, or in its absence contained by + =oozie-site.xml#oozie.client.jobs.application.generated.path= has to be writable by the current user + +If the =-verbose= option is provided, a lot more debugging output, including the generated workflow definition, is given. + +For more information what an Fluent Job API jar file is, how to build it etc., refer to +[[DG_FluentJobAPI#AE.A_Appendix_A_API_JAR_format][Fluent Job API - API JAR format]]. + +*Example:* + +<verbatim> +$ oozie job -oozie http://localhost:11000/oozie -submitjar /tmp/workflow-api-jar.jar -config /tmp/job.properties +job: 0000009-180107110323219-oozie-oozi-W +</verbatim> + +*Example (verbose):* + +<verbatim> +$ oozie job -oozie http://localhost:11000/oozie -submitjar /tmp/workflow-api-jar.jar -config /tmp/job.properties -verbose +Submitting a job based on API jar: /tmp/workflow-api-jar.jar +Loading API jar /tmp/workflow-api-jar.jar +Workflow job definition generated from API jar: +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<workflow:workflow-app xmlns:workflow="uri:oozie:workflow:1.0" ... name="shell-example"> +... +</workflow:workflow-app> +. +job: 0000010-180107110323219-oozie-oozi-W +Job based on API jar submitted successfully. +</verbatim> + +---+++ Running a workflow definition generated by a Fluent Job API jar file + +Since Oozie 5.1.0. + +Generate a workflow definition given the Fluent Job API jar file supplied at command line, write it to a provided or generated HDFS +location, submit and run. + +*Preconditions:* + * all the parameters that are present in the workflow definition have to be supplied either as command line parameters or via + =job.properties= passed along with the =-config= option + * the Fluent Job API jar file has to be present and readable by the current user at the local path provided + * the folder containing the Fluent Job API jar file provided has to be writable by the current user, since the generated workflow + definition is written there + * the HDFS folder either given by =-Doozie.wf.application.path= command line parameter, or in its absence contained by + =oozie-site.xml#oozie.client.jobs.application.generated.path= has to be writable by the current user + +If the =-verbose= option is provided, a lot more debugging output, including the generated workflow definition, is given. + +For more information what an Fluent Job API jar file is, how to build it etc., refer to +[[DG_FluentJobAPI#AE.A_Appendix_A_API_JAR_format][Fluent Job API - API JAR format]]. + +*Example:* + +<verbatim> +$ oozie job -oozie http://localhost:11000/oozie -runjar /tmp/workflow-api-jar.jar -config /tmp/job.properties +job: 0000011-180107110323219-oozie-oozi-W +</verbatim> + +*Example (verbose):* + +<verbatim> +$ oozie job -oozie http://localhost:11000/oozie -runjar /tmp/workflow-api-jar.jar -config /tmp/job.properties -verbose +Submitting a job based on API jar: /tmp/workflow-api-jar.jar +Loading API jar /tmp/workflow-api-jar.jar +Workflow job definition generated from API jar: +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<workflow:workflow-app xmlns:workflow="uri:oozie:workflow:1.0" ... name="shell-example"> +... +</workflow:workflow-app> +. +job: 0000010-180107110323219-oozie-oozi-W +Job based on API jar run successfully. +</verbatim> + ---++ Jobs Operations ---+++ Checking the Status of multiple Workflow Jobs
