Author: angeloh
Date: Sat Feb 25 00:50:36 2012
New Revision: 1293497
URL: http://svn.apache.org/viewvc?rev=1293497&view=rev
Log:
OOZIE-616 Moving action prepare FS execution to LauncherMapper ( Kiran
Nagasubramanian via Angelo)
Added:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FileSystemActions.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
incubator/oozie/trunk/release-log.txt
Added:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FileSystemActions.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FileSystemActions.java?rev=1293497&view=auto
==============================================================================
---
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FileSystemActions.java
(added)
+++
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FileSystemActions.java
Sat Feb 25 00:50:36 2012
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.w3c.dom.Node;
+/**
+ * Class to perform file system operations specified in the prepare block of
Workflow
+ *
+ */
+public class FileSystemActions {
+ private static Set<String> supportedFileSystems = new HashSet<String>();
+
+ public FileSystemActions() {
+ supportedFileSystems.add("hdfs");
+ }
+
+ public FileSystemActions(Set<String> fileSystems) {
+ supportedFileSystems = fileSystems;
+ }
+
+ /**
+ * Method to execute the prepare actions based on the command
+ *
+ * @param n Child node of the prepare XML
+ * @throws LauncherException
+ */
+ public void execute(Node n) throws LauncherException {
+ String command = n.getNodeName();
+ if (command.equals("delete")) {
+ delete(new
Path(n.getAttributes().getNamedItem("path").getNodeValue().trim()));
+ } else if (command.equals("mkdir")) {
+ mkdir(new
Path(n.getAttributes().getNamedItem("path").getNodeValue().trim()));
+ }
+ }
+
+ // Method to delete the specified file based on the path
+ private void delete(Path path) throws LauncherException {
+ try {
+ validatePath(path, true);
+ FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
+ if (fs.exists(path)) {
+ if (!fs.delete(path, true)) {
+ String deleteFailed = "Deletion of path " +
path.toString() + " failed.";
+ System.out.println(deleteFailed);
+ throw new LauncherException(deleteFailed);
+ } else {
+ System.out.println("Deletion of path " + path.toString() +
" was successful.");
+ }
+ }
+ } catch (IOException ex) {
+ throw new LauncherException(ex.getMessage(), ex);
+ }
+
+ }
+
+ // Method to create a directory based on the path
+ private void mkdir(Path path) throws LauncherException {
+ try {
+ validatePath(path, true);
+ FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
+ if (!fs.exists(path)) {
+ if (!fs.mkdirs(path)) {
+ String mkdirFailed = "Creating directory at " + path + "
failed.";
+ System.out.println(mkdirFailed);
+ throw new LauncherException(mkdirFailed);
+ } else {
+ System.out.println("Creating directory at path " + path +
" was successful.");
+ }
+ }
+ } catch (IOException ex) {
+ throw new LauncherException(ex.getMessage(), ex);
+ }
+ }
+
+ // Method to validate the path provided for the prepare action
+ private void validatePath(Path path, boolean withScheme) throws
LauncherException {
+ String scheme = path.toUri().getScheme();
+ if (withScheme) {
+ if (scheme == null) {
+ String nullScheme = "Scheme of the path " + path + " is null";
+ System.out.println(nullScheme);
+ throw new LauncherException(nullScheme);
+ } else if (!supportedFileSystems.contains(scheme.toLowerCase())) {
+ String unsupportedScheme = "Scheme of the provided path " +
path + " is of type not supported.";
+ System.out.println(unsupportedScheme);
+ throw new LauncherException(unsupportedScheme);
+ }
+ } else if (scheme != null) {
+ String notNullScheme = "Scheme of the path " + path + " is not
null as specified.";
+ System.out.println(notNullScheme);
+ throw new LauncherException(notNullScheme);
+ }
+ }
+}
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1293497&r1=1293496&r2=1293497&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
Sat Feb 25 00:50:36 2012
@@ -117,6 +117,8 @@ public class JavaActionExecutor extends
classes.add(LauncherSecurityManager.class);
classes.add(LauncherException.class);
classes.add(LauncherMainException.class);
+ classes.add(FileSystemActions.class);
+ classes.add(PrepareActionsDriver.class);
classes.add(ActionStats.class);
classes.add(ActionType.class);
return classes;
@@ -505,14 +507,23 @@ public class JavaActionExecutor extends
Path actionDir = context.getActionDir();
String recoveryId = context.getRecoveryId();
- LauncherMapper.setupLauncherInfo(launcherJobConf, jobId, actionId,
actionDir, recoveryId, actionConf);
+ // Getting the prepare XML from the action XML
+ Namespace ns = actionXml.getNamespace();
+ Element prepareElement = actionXml.getChild("prepare", ns);
+ String prepareXML = "";
+ if (prepareElement != null) {
+ if (prepareElement.getChildren().size() > 0) {
+ prepareXML =
XmlUtils.prettyPrint(prepareElement).toString().trim();
+ }
+ }
+ LauncherMapper.setupLauncherInfo(launcherJobConf, jobId, actionId,
actionDir, recoveryId, actionConf,
+ prepareXML);
LauncherMapper.setupMainClass(launcherJobConf,
getLauncherMain(launcherConf, actionXml));
LauncherMapper.setupMaxOutputData(launcherJobConf,
maxActionOutputLen);
LauncherMapper.setupMaxExternalStatsSize(launcherJobConf,
maxExternalStatsSize);
- Namespace ns = actionXml.getNamespace();
List<Element> list = actionXml.getChildren("arg", ns);
String[] args = new String[list.size()];
for (int i = 0; i < list.size(); i++) {
@@ -634,7 +645,6 @@ public class JavaActionExecutor extends
}
}
else {
- prepare(context, actionXml);
XLog.getLog(getClass()).debug("Submitting the job through Job
Client for action " + action.getId());
// setting up propagation of the delegation token.
@@ -809,17 +819,6 @@ public class JavaActionExecutor extends
return credProp;
}
- void prepare(Context context, Element actionXml) throws
ActionExecutorException {
- Namespace ns = actionXml.getNamespace();
- Element prepare = actionXml.getChild("prepare", ns);
- if (prepare != null) {
- XLog.getLog(getClass()).debug("Preparing the action with
FileSystem operation");
- FsActionExecutor fsAe = new FsActionExecutor();
- fsAe.doOperations(context, prepare);
- XLog.getLog(getClass()).debug("FS Operation is completed");
- }
- }
-
@Override
public void start(Context context, WorkflowAction action) throws
ActionExecutorException {
try {
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java?rev=1293497&r1=1293496&r2=1293497&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
(original)
+++
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
Sat Feb 25 00:50:36 2012
@@ -77,6 +77,7 @@ public class LauncherMapper<K1, V1, K2,
public static final String EXTERNAL_ACTION_STATS = ACTION_PREFIX +
"stats.properties";
static final String ACTION_CONF_XML = "action.xml";
+ public static final String ACTION_PREPARE_XML = "oozie.action.prepare.xml";
private static final String ACTION_OUTPUT_PROPS = "output.properties";
private static final String ACTION_STATS_PROPS = "stats.properties";
private static final String ACTION_EXTERNAL_CHILD_IDS_PROPS =
"externalChildIds.properties";
@@ -179,7 +180,7 @@ public class LauncherMapper<K1, V1, K2,
* @throws HadoopAccessorException
*/
public static void setupLauncherInfo(JobConf launcherConf, String jobId,
String actionId, Path actionDir,
- String recoveryId, Configuration actionConf) throws IOException,
HadoopAccessorException {
+ String recoveryId, Configuration actionConf, String prepareXML)
throws IOException, HadoopAccessorException {
launcherConf.setMapperClass(LauncherMapper.class);
launcherConf.setSpeculativeExecution(false);
@@ -190,6 +191,7 @@ public class LauncherMapper<K1, V1, K2,
launcherConf.set(OOZIE_ACTION_ID, actionId);
launcherConf.set(OOZIE_ACTION_DIR_PATH, actionDir.toString());
launcherConf.set(OOZIE_ACTION_RECOVERY_ID, recoveryId);
+ launcherConf.set(ACTION_PREPARE_XML, prepareXML);
actionConf.set(OOZIE_JOB_ID, jobId);
actionConf.set(OOZIE_ACTION_ID, actionId);
@@ -359,7 +361,7 @@ public class LauncherMapper<K1, V1, K2,
private ScheduledThreadPoolExecutor timer;
private boolean configFailure = false;
-
+ private LauncherException configureFailureEx;
public LauncherMapper() {
}
@@ -375,6 +377,8 @@ public class LauncherMapper<K1, V1, K2,
setRecoveryId(jobConf, actionDir, recoveryId);
}
catch (LauncherException ex) {
+ System.out.println("Launcher config error "+ex.getMessage());
+ configureFailureEx = ex;
configFailure = true;
}
}
@@ -383,7 +387,7 @@ public class LauncherMapper<K1, V1, K2,
public void map(K1 key, V1 value, OutputCollector<K2, V2> collector,
Reporter reporter) throws IOException {
try {
if (configFailure) {
- throw new LauncherException();
+ throw configureFailureEx;
}
else {
String mainClass =
getJobConf().get(CONF_OOZIE_ACTION_MAIN_CLASS);
@@ -405,6 +409,15 @@ public class LauncherMapper<K1, V1, K2,
setupMainConfiguration();
+ try {
+ System.out.println("Starting the execution of prepare
actions");
+ executePrepare();
+ System.out.println("Completed the execution of prepare
actions successfully");
+ } catch (Exception ex) {
+ System.out.println("Prepare execution in the Launcher
Mapper has failed");
+ throw new LauncherException(ex.getMessage(), ex);
+ }
+
String[] args = getMainArguments(getJobConf());
printContentsOfCurrentDir();
@@ -615,6 +628,18 @@ public class LauncherMapper<K1, V1, K2,
System.setProperty("oozie.action.newId.properties", new
File(ACTION_NEW_ID_PROPS).getAbsolutePath());
}
+ // Method to execute the prepare actions
+ private void executePrepare() throws IOException, LauncherException {
+ String prepareXML = getJobConf().get(ACTION_PREPARE_XML);
+ if (prepareXML != null) {
+ if (!prepareXML.equals("")) {
+ PrepareActionsDriver.doOperations(prepareXML);
+ } else {
+ System.out.println("There are no prepare actions to
execute.");
+ }
+ }
+ }
+
public static String[] getMainArguments(Configuration conf) {
String[] args = new
String[conf.getInt(CONF_OOZIE_ACTION_MAIN_ARG_COUNT, 0)];
for (int i = 0; i < args.length; i++) {
@@ -673,7 +698,7 @@ public class LauncherMapper<K1, V1, K2,
ex.printStackTrace(System.out);
ex.printStackTrace(System.err);
}
- throw new LauncherException();
+ throw new LauncherException(reason, ex);
}
catch (IOException rex) {
throw new RuntimeException("Error while failing launcher, " +
rex.getMessage(), rex);
@@ -761,4 +786,12 @@ class LauncherSecurityManager extends Se
}
class LauncherException extends Exception {
+
+ LauncherException(String message) {
+ super(message);
+ }
+
+ LauncherException(String message, Throwable cause) {
+ super(message, cause);
+ }
}
Added:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java?rev=1293497&view=auto
==============================================================================
---
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java
(added)
+++
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java
Sat Feb 25 00:50:36 2012
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.xml.sax.SAXException;
+import org.apache.oozie.action.hadoop.FileSystemActions;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.ParserConfigurationException;
+
+/**
+ * Utility class to perform operations on the prepare block of Workflow
+ *
+ */
+public class PrepareActionsDriver {
+
+ /**
+ * Method to parse the prepare XML and execute the corresponding prepare
actions
+ *
+ * @param prepareXML Prepare XML block in string format
+ * @throws LauncherException
+ */
+ static void doOperations(String prepareXML) throws LauncherException {
+ try {
+ Document doc = getDocumentFromXML(prepareXML);
+ doc.getDocumentElement().normalize();
+
+ // Get the list of child nodes, basically, each one corresponding
to a separate action
+ NodeList nl = doc.getDocumentElement().getChildNodes();
+ FileSystemActions fsActions = new FileSystemActions();
+
+ for (int i = 0; i < nl.getLength(); ++i) {
+ String commandType = "";
+ /* Logic to find the command type goes here
+ commandType = ..........;
+ */
+ // As of now, the available prepare action is of type hdfs.
Hence, assigning the value directly
+ commandType = "hdfs";
+ if (commandType.equalsIgnoreCase("hdfs")) {
+ fsActions.execute(nl.item(i));
+ } /*else if(commandType.equalsIgnoreCase("hcat")) {
//Other command types go here
+ hCatActions.execute(nl.item(i));
+ }*/
+ }
+ } catch (IOException ioe) {
+ throw new LauncherException(ioe.getMessage(), ioe);
+ } catch (SAXException saxe) {
+ throw new LauncherException(saxe.getMessage(), saxe);
+ } catch (ParserConfigurationException pce) {
+ throw new LauncherException(pce.getMessage(), pce);
+ }
+ }
+
+ // Method to return the document from the prepare XML block
+ static Document getDocumentFromXML(String prepareXML) throws
ParserConfigurationException, SAXException,
+ IOException {
+ DocumentBuilderFactory docBuilderFactory =
DocumentBuilderFactory.newInstance();
+ DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
+ InputStream is = new
ByteArrayInputStream(prepareXML.getBytes("UTF-8"));
+ return docBuilder.parse(is);
+ }
+}
Added:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java?rev=1293497&view=auto
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java
(added)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java
Sat Feb 25 00:50:36 2012
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.oozie.test.XFsTestCase;
+import org.apache.oozie.service.Services;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+public class TestFileSystemActions extends XFsTestCase {
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ new Services().init();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ Services.get().destroy();
+ super.tearDown();
+ }
+
+ // Test for delete as prepare action
+ public void testDelete() throws Exception {
+ Path actionDir = getFsTestCaseDir();
+ FileSystem fs = getFileSystem();
+ Path newDir = new Path(actionDir, "newDir");
+ // Delete the file if it is already there
+ if (fs.exists(newDir)) {
+ fs.delete(newDir, true);
+ }
+ fs.mkdirs(newDir);
+ // Prepare block that contains delete action
+ String prepareXML = "<prepare>" + "<delete path='" + newDir + "'/>" +
"</prepare>";
+
+ // Parse the XML to get the node
+ Document doc = PrepareActionsDriver.getDocumentFromXML(prepareXML);
+ Node n = doc.getDocumentElement().getChildNodes().item(0);
+
+ new FileSystemActions().execute(n);
+ assertFalse(fs.exists(newDir));
+ }
+
+ // Test for mkdir as prepare action
+ public void testMkdir() throws Exception {
+ Path actionDir = getFsTestCaseDir();
+ FileSystem fs = getFileSystem();
+ Path newDir = new Path(actionDir, "newDir");
+ // Delete the file if it is already there
+ if (fs.exists(newDir)) {
+ fs.delete(newDir, true);
+ }
+
+ // Prepare block that contains mkdir action
+ String prepareXML = "<prepare>" + "<mkdir path='" + newDir + "'/>" +
"</prepare>";
+
+ // Parse the XML to get the node
+ Document doc = PrepareActionsDriver.getDocumentFromXML(prepareXML);
+ Node n = doc.getDocumentElement().getChildNodes().item(0);
+
+ new FileSystemActions().execute(n);
+ assertTrue(fs.exists(newDir));
+ }
+
+ // Test for invalid scheme value in the path for action
+ public void testForInvalidScheme() throws Exception {
+ Path actionDir = getFsTestCaseDir();
+ // Construct a path with invalid scheme
+ Path newDir = new Path("http" + actionDir.toString().substring(4) +
"/delete");
+ // Construct prepare XML block with the path
+ String prepareXML = "<prepare>" + "<delete path='" + newDir + "'/>" +
"</prepare>";
+ // Parse the XML to get the node
+ Document doc = PrepareActionsDriver.getDocumentFromXML(prepareXML);
+ Node n = doc.getDocumentElement().getChildNodes().item(0);
+
+ try {
+ new FileSystemActions().execute(n);
+ fail("Expected to catch an exception but did not encounter any");
+ } catch (LauncherException le) {
+ Path path = new
Path(n.getAttributes().getNamedItem("path").getNodeValue().trim());
+ assertEquals(le.getMessage(), "Scheme of the provided path " +
path + " is of type not supported.");
+ } catch(Exception ex){
+ fail("Expected a LauncherException but received an Exception");
+ }
+ }
+
+ // Test for null scheme value in the path for action
+ public void testForNullScheme() throws Exception {
+ // Construct a path without scheme
+ Path newDir = new Path("test/oozietests/testDelete/delete");
+ FileSystem fs = getFileSystem();
+ // Delete the file if it is already there
+ if (fs.exists(newDir)) {
+ fs.delete(newDir, true);
+ }
+ // Construct prepare XML block with the path
+ String prepareXML = "<prepare>" + "<delete path='" + newDir + "'/>" +
"</prepare>";
+
+ // Parse the XML to get the node
+ Document doc = PrepareActionsDriver.getDocumentFromXML(prepareXML);
+ Node n = doc.getDocumentElement().getChildNodes().item(0);
+
+ try {
+ new FileSystemActions().execute(n);
+ fail("Expected to catch an exception but did not encounter any");
+ } catch (LauncherException le) {
+ Path path = new
Path(n.getAttributes().getNamedItem("path").getNodeValue().trim());
+ assertEquals(le.getMessage(), "Scheme of the path " + path + " is
null");
+ } catch(Exception ex) {
+ fail("Expected a LauncherException but received an Exception");
+ }
+ }
+
+}
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java?rev=1293497&r1=1293496&r2=1293497&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
Sat Feb 25 00:50:36 2012
@@ -79,16 +79,15 @@ public class TestJavaActionExecutor exte
public void testSetupMethods() throws Exception {
JavaActionExecutor ae = new JavaActionExecutor();
-
assertEquals("java", ae.getType());
-
assertEquals("java-launcher.jar", ae.getLauncherJarName());
-
List<Class> classes = new ArrayList<Class>();
classes.add(LauncherMapper.class);
classes.add(LauncherSecurityManager.class);
classes.add(LauncherException.class);
classes.add(LauncherMainException.class);
+ classes.add(FileSystemActions.class);
+ classes.add(PrepareActionsDriver.class);
classes.add(ActionStats.class);
classes.add(ActionType.class);
assertEquals(classes, ae.getLauncherClasses());
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java?rev=1293497&r1=1293496&r2=1293497&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
Sat Feb 25 00:50:36 2012
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -27,9 +27,11 @@ import org.apache.hadoop.mapred.RunningJ
import org.apache.oozie.test.XFsTestCase;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XConfiguration;
+import org.apache.oozie.service.HadoopAccessorException;
import org.apache.oozie.service.Services;
import java.io.File;
+import java.io.IOException;
public class TestLauncher extends XFsTestCase {
@@ -75,7 +77,7 @@ public class TestLauncher extends XFsTes
lm.setupMainArguments(jobConf, arg);
Configuration actionConf = new XConfiguration();
- lm.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0",
actionConf);
+ lm.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0",
actionConf, "");
assertEquals("1", actionConf.get("oozie.job.id"));
assertEquals("1@a", actionConf.get("oozie.action.id"));
@@ -94,6 +96,7 @@ public class TestLauncher extends XFsTes
return runningJob.isComplete();
}
});
+ assertTrue(jobConf.get("oozie.action.prepare.xml").equals(""));
return runningJob;
}
@@ -217,4 +220,43 @@ public class TestLauncher extends XFsTes
assertFalse(fs.exists(LauncherMapper.getOutputDataPath(actionDir)));
}
+ // Test to ensure that the property value "oozie.action.prepare.xml" in
the configuration of the job is an empty
+ // string when there is no prepare block in workflow XML or there is one
with no prepare actions in it
+ public void testSetupLauncherInfoWithEmptyPrepareXML() throws
HadoopAccessorException, IOException {
+ Path actionDir = getFsTestCaseDir();
+
+ // Setting up the job configuration
+ JobConf jobConf = new JobConf();
+ jobConf.set("user.name", getTestUser());
+ jobConf.set("group.name", getTestGroup());
+ jobConf.set("fs.default.name", getNameNodeUri());
+ injectKerberosInfo(jobConf);
+
+ LauncherMapper lm = new LauncherMapper();
+ Configuration actionConf = new XConfiguration();
+ String prepareBlock = "";
+ lm.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0",
actionConf, prepareBlock);
+ assertTrue(jobConf.get("oozie.action.prepare.xml").equals(""));
+ }
+
+ // Test to ensure that the property value "oozie.action.prepare.xml" in
the configuration of the job is properly set
+ // when there is prepare block in workflow XML
+ public void testSetupLauncherInfoWithNonEmptyPrepareXML() throws
HadoopAccessorException, IOException {
+ Path actionDir = getFsTestCaseDir();
+ FileSystem fs = getFileSystem();
+ Path newDir = new Path(actionDir, "newDir");
+
+ // Setting up the job configuration
+ JobConf jobConf = new JobConf();
+ jobConf.set("user.name", getTestUser());
+ jobConf.set("group.name", getTestGroup());
+ jobConf.set("fs.default.name", getNameNodeUri());
+ injectKerberosInfo(jobConf);
+
+ LauncherMapper lm = new LauncherMapper();
+ Configuration actionConf = new XConfiguration();
+ String prepareBlock = "<prepare>" + "<mkdir path='" + newDir + "'/>" +
"</prepare>";
+ lm.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0",
actionConf, prepareBlock);
+
assertTrue(jobConf.get("oozie.action.prepare.xml").equals(prepareBlock));
+ }
}
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java?rev=1293497&r1=1293496&r2=1293497&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java
Sat Feb 25 00:50:36 2012
@@ -72,6 +72,8 @@ public class TestMapReduceActionError ex
classes.add(LauncherSecurityManager.class);
classes.add(LauncherException.class);
classes.add(LauncherMainException.class);
+ classes.add(FileSystemActions.class);
+ classes.add(PrepareActionsDriver.class);
classes.add(ActionStats.class);
classes.add(ActionType.class);
classes.add(LauncherMain.class);
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java?rev=1293497&r1=1293496&r2=1293497&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
Sat Feb 25 00:50:36 2012
@@ -75,6 +75,8 @@ public class TestMapReduceActionExecutor
classes.add(LauncherSecurityManager.class);
classes.add(LauncherException.class);
classes.add(LauncherMainException.class);
+ classes.add(FileSystemActions.class);
+ classes.add(PrepareActionsDriver.class);
classes.add(ActionStats.class);
classes.add(ActionType.class);
classes.add(LauncherMain.class);
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java?rev=1293497&r1=1293496&r2=1293497&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
Sat Feb 25 00:50:36 2012
@@ -91,6 +91,8 @@ public class TestPigActionExecutor exten
classes.add(LauncherSecurityManager.class);
classes.add(LauncherException.class);
classes.add(LauncherMainException.class);
+ classes.add(FileSystemActions.class);
+ classes.add(PrepareActionsDriver.class);
classes.add(ActionStats.class);
classes.add(ActionType.class);
classes.add(LauncherMain.class);
Added:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java?rev=1293497&view=auto
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java
(added)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java
Sat Feb 25 00:50:36 2012
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.test.XFsTestCase;
+import org.apache.hadoop.fs.FileSystem;
+
+public class TestPrepareActionsDriver extends XFsTestCase {
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ new Services().init();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ Services.get().destroy();
+ super.tearDown();
+ }
+
+ // Test to check if prepare action is performed as expected when the
prepare XML block is a valid one
+ public void testDoOperationsWithValidXML() throws LauncherException,
IOException {
+ Path actionDir = getFsTestCaseDir();
+ FileSystem fs = getFileSystem();
+ Path newDir = new Path(actionDir, "newDir");
+ String prepareXML = "<prepare>" + "<mkdir path='" + newDir + "'/>" +
"</prepare>";
+
+ // Delete the file if it is already there
+ if (fs.exists(newDir)) {
+ fs.delete(newDir, true);
+ }
+
+ PrepareActionsDriver.doOperations(prepareXML);
+ assertTrue(fs.exists(actionDir));
+ }
+
+ // Test to check if LauncherException is thrown when the prepare XML block
is invalid
+ public void testDoOperationsWithInvalidXML() throws LauncherException,
IOException {
+ Path actionDir = getFsTestCaseDir();
+ FileSystem fs = getFileSystem();
+ Path newDir = new Path(actionDir, "newDir");
+ String prepareXML = "";
+
+ // Delete the file if it is already there
+ if (fs.exists(newDir)) {
+ fs.delete(newDir, true);
+ }
+
+ try {
+ prepareXML = "prepare>" + "<mkdir path='" + newDir + "'/>" +
"</prepare>";
+ PrepareActionsDriver.doOperations(prepareXML);
+ fail("Expected to catch an exception but did not encounter any");
+ } catch (LauncherException le) {
+ assertEquals(le.getCause().getClass(),
org.xml.sax.SAXParseException.class);
+ assertEquals(le.getMessage(), "Content is not allowed in prolog.");
+ } catch(Exception ex){
+ fail("Expected a LauncherException but received an Exception");
+ }
+ }
+}
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java?rev=1293497&r1=1293496&r2=1293497&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
Sat Feb 25 00:50:36 2012
@@ -82,6 +82,8 @@ public class TestShellActionExecutor ext
classes.add(LauncherSecurityManager.class);
classes.add(LauncherException.class);
classes.add(LauncherMainException.class);
+ classes.add(FileSystemActions.class);
+ classes.add(PrepareActionsDriver.class);
classes.add(ActionStats.class);
classes.add(ActionType.class);
classes.add(LauncherMain.class);
Modified: incubator/oozie/trunk/release-log.txt
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1293497&r1=1293496&r2=1293497&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Sat Feb 25 00:50:36 2012
@@ -1,5 +1,6 @@
-- Oozie 3.2.0 release
+OOZIE-616 Moving action prepare FS execution to LauncherMapper ( Kiran
Nagasubramanian via Angelo)
OOZIE-687 The start time of the action is set after the job is already
submitted to hadoop (Mohamed Battisha vis Angelo)
OOZIE-712 Hive testcases fail with 0.23 (tucu)
OOZIE-710 add a timeout to the surefire configuration (tucu)
