Repository: oozie Updated Branches: refs/heads/master 3cbb64882 -> f21074a0e
OOZIE-3346 [examples] [action] Fix Git example. PrepareActionsHandler should support XML namespace prefixes (asalamon74, andras.piros) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f21074a0 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f21074a0 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f21074a0 Branch: refs/heads/master Commit: f21074a0ef9eae9becec67fd90035d73df1b5195 Parents: 3cbb648 Author: Andras Piros <[email protected]> Authored: Mon Sep 24 09:58:37 2018 +0200 Committer: Andras Piros <[email protected]> Committed: Mon Sep 24 09:58:37 2018 +0200 ---------------------------------------------------------------------- examples/src/main/apps/git/workflow.xml | 7 +- .../org/apache/oozie/example/fluentjob/Git.java | 12 ++- release-log.txt | 1 + .../apache/oozie/action/hadoop/LauncherAM.java | 2 +- .../hadoop/LauncherURIHandlerFactory.java | 6 ++ .../action/hadoop/PrepareActionsDriver.java | 6 +- .../action/hadoop/PrepareActionsHandler.java | 14 +++- .../hadoop/TestPrepareActionsHandler.java | 81 ++++++++++++++++++++ 8 files changed, 115 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/f21074a0/examples/src/main/apps/git/workflow.xml ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/git/workflow.xml b/examples/src/main/apps/git/workflow.xml index b1b0858..bbdac95 100644 --- a/examples/src/main/apps/git/workflow.xml +++ b/examples/src/main/apps/git/workflow.xml @@ -18,11 +18,14 @@ <workflow-app xmlns="uri:oozie:workflow:1.0" name="git-wf"> <start to="git-node"/> <action name="git-node"> - <git xmlns="uri:oozie:git-action:0.1"> + <git xmlns="uri:oozie:git-action:1.0"> <resource-manager>${resourceManager}</resource-manager> <name-node>${nameNode}</name-node> + <prepare> + <delete path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/git/oozie"/> + </prepare> <git-uri>https://github.com/apache/oozie</git-uri> - <destination-uri>/user/${wf:user()}/${examplesRoot}/output-data/git/oozie</destination-uri> + <destination-uri>${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/git/oozie</destination-uri> </git> <ok to="end"/> <error to="fail"/> http://git-wip-us.apache.org/repos/asf/oozie/blob/f21074a0/examples/src/main/java/org/apache/oozie/example/fluentjob/Git.java ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/oozie/example/fluentjob/Git.java b/examples/src/main/java/org/apache/oozie/example/fluentjob/Git.java index 2548d39..8d036cf 100644 --- a/examples/src/main/java/org/apache/oozie/example/fluentjob/Git.java +++ b/examples/src/main/java/org/apache/oozie/example/fluentjob/Git.java @@ -18,7 +18,10 @@ package org.apache.oozie.example.fluentjob; -import org.apache.oozie.fluentjob.api.action.*; +import org.apache.oozie.fluentjob.api.action.GitAction; +import org.apache.oozie.fluentjob.api.action.GitActionBuilder; +import org.apache.oozie.fluentjob.api.action.Prepare; +import org.apache.oozie.fluentjob.api.action.PrepareBuilder; import org.apache.oozie.fluentjob.api.factory.WorkflowFactory; import org.apache.oozie.fluentjob.api.workflow.Workflow; import org.apache.oozie.fluentjob.api.workflow.WorkflowBuilder; @@ -29,10 +32,15 @@ import org.apache.oozie.fluentjob.api.workflow.WorkflowBuilder; public class Git implements WorkflowFactory { @Override public Workflow create() { + final Prepare prepare = new PrepareBuilder() + .withDelete("${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/git/oozie") + .build(); + final GitAction parent = GitActionBuilder.create() .withResourceManager("${resourceManager}") .withNameNode("${nameNode}") - .withDestinationUri("/user/${wf:user()}/${examplesRoot}/output-data/git/oozie") + .withPrepare(prepare) + .withDestinationUri("${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/git/oozie") .withGitUri("https://github.com/apache/oozie";) .build(); http://git-wip-us.apache.org/repos/asf/oozie/blob/f21074a0/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 37ad2ac..330cee6 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.1.0 release (trunk - unreleased) +OOZIE-3346 [examples] [action] Fix Git example. PrepareActionsHandler should support XML namespace prefixes (asalamon74, andras.piros) OOZIE-3347 [examples] Fix Fluent Job global example (asalamon74 via andras.piros) OOZIE-3160 amend PriorityDelayQueue put()/take() can cause significant CPU load due to busy waiting (pbacsko) OOZIE-3342 [examples] Missing archive tag from Fluent Job examples (asalamon74 via andras.piros) http://git-wip-us.apache.org/repos/asf/oozie/blob/f21074a0/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java index 4a41941..63afd91 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java @@ -145,7 +145,7 @@ public class LauncherAM { new AMRMCallBackHandler(), new HdfsOperations(new SequenceFileWriterFactory()), new LocalFsOperations(), - new PrepareActionsHandler(), + new PrepareActionsHandler(new LauncherURIHandlerFactory(null)), new LauncherAMCallbackNotifierFactory(), new LauncherSecurityManager(), System.getenv(ApplicationConstants.Environment.CONTAINER_ID.name()), http://git-wip-us.apache.org/repos/asf/oozie/blob/f21074a0/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherURIHandlerFactory.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherURIHandlerFactory.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherURIHandlerFactory.java index 04c9175..bb12bfa 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherURIHandlerFactory.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherURIHandlerFactory.java @@ -19,6 +19,8 @@ package org.apache.oozie.action.hadoop; import java.net.URI; + +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ReflectionUtils; @@ -62,4 +64,8 @@ public class LauncherURIHandlerFactory { return handler; } + @VisibleForTesting + LauncherURIHandler getURIHandler(final URI uri, final Configuration configuration) throws LauncherException { + return new LauncherURIHandlerFactory(configuration).getURIHandler(uri); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/f21074a0/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java index cb5b1ac..e730f0f 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java @@ -21,14 +21,10 @@ package org.apache.oozie.action.hadoop; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.net.URI; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.xml.sax.SAXException; import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.DocumentBuilder; @@ -40,7 +36,7 @@ import javax.xml.parsers.ParserConfigurationException; */ @Deprecated public class PrepareActionsDriver { - private static final PrepareActionsHandler prepareHandler = new PrepareActionsHandler(); + private static final PrepareActionsHandler prepareHandler = new PrepareActionsHandler(new LauncherURIHandlerFactory(null)); /** * Method to parse the prepare XML and execute the corresponding prepare actions http://git-wip-us.apache.org/repos/asf/oozie/blob/f21074a0/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsHandler.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsHandler.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsHandler.java index 923b085..93731ad 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsHandler.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsHandler.java @@ -26,6 +26,7 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.w3c.dom.Document; @@ -34,6 +35,12 @@ import org.w3c.dom.NodeList; import org.xml.sax.SAXException; public class PrepareActionsHandler { + private final LauncherURIHandlerFactory factory; + + @VisibleForTesting + PrepareActionsHandler(final LauncherURIHandlerFactory factory) { + this.factory = factory; + } /** * Method to parse the prepare XML and execute the corresponding prepare actions @@ -45,25 +52,24 @@ public class PrepareActionsHandler { * @throws ParserConfigurationException if the parser is not well configured * @throws LauncherException in case of error */ - public void prepareAction(String prepareXML, Configuration conf) + void prepareAction(String prepareXML, Configuration conf) throws IOException, SAXException, ParserConfigurationException, LauncherException { Document doc = getDocumentFromXML(prepareXML); doc.getDocumentElement().normalize(); // Get the list of child nodes, basically, each one corresponding to a separate action NodeList nl = doc.getDocumentElement().getChildNodes(); - LauncherURIHandlerFactory factory = new LauncherURIHandlerFactory(conf); for (int i = 0; i < nl.getLength(); ++i) { Node n = nl.item(i); - String operation = n.getNodeName(); + String operation = n.getLocalName(); if (n.getAttributes() == null || n.getAttributes().getNamedItem("path") == null) { continue; } String pathStr = n.getAttributes().getNamedItem("path").getNodeValue().trim(); // use Path to avoid URIsyntax error caused by square bracket in glob URI uri = new Path(pathStr).toUri(); - LauncherURIHandler handler = factory.getURIHandler(uri); + LauncherURIHandler handler = factory.getURIHandler(uri, conf); execute(operation, uri, handler, conf); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/f21074a0/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsHandler.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsHandler.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsHandler.java new file mode 100644 index 0000000..2547350 --- /dev/null +++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsHandler.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.action.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.net.URI; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class TestPrepareActionsHandler { + private PrepareActionsHandler prepareActionsHandler; + + @Mock + private LauncherURIHandlerFactory launcherURIHandlerFactory; + + @Mock + private LauncherURIHandler launcherURIHandler; + + @Before + public void setUp() throws LauncherException { + prepareActionsHandler = new PrepareActionsHandler(launcherURIHandlerFactory); + + when(launcherURIHandlerFactory.getURIHandler(any(URI.class), any(Configuration.class))).thenReturn(launcherURIHandler); + } + + @Test + public void whenPrepareActionWithoutPrefixExecutedSuccessfully() + throws SAXException, ParserConfigurationException, LauncherException, IOException { + final String prepareXmlWithoutPrefix = "<prepare>\n" + + " <delete path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/git/oozie\"/>\n" + + "</prepare>"; + + callAndVerifyPrepareAction(prepareXmlWithoutPrefix); + } + + private void callAndVerifyPrepareAction(final String prepareXml) + throws IOException, SAXException, ParserConfigurationException, LauncherException { + prepareActionsHandler.prepareAction(prepareXml, null); + + verify(launcherURIHandler).delete(any(), any()); + verifyNoMoreInteractions(launcherURIHandler); + } + + @Test + public void whenPrepareActionWithGitPrefixExecutedSuccessfully() + throws SAXException, ParserConfigurationException, LauncherException, IOException { + final String prepareXmlWithGitPrefix = "<git:prepare xmlns:git=\"uri:oozie:git-action:1.0\">\n" + + " <git:delete path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/git/oozie\"/>\n" + + "</git:prepare>"; + + callAndVerifyPrepareAction(prepareXmlWithGitPrefix); + } +} \ No newline at end of file
