Author: rkanter
Date: Sat Dec 7 02:18:17 2013
New Revision: 1548778
URL: http://svn.apache.org/r1548778
Log:
OOZIE-1575 Add functionality to submit sqoop jobs through http on oozie server
side (bowenzhangusa via rkanter)
Added:
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java
oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java
Modified:
oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java
oozie/trunk/core/src/main/java/org/apache/oozie/DagEngine.java
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHiveXCommand.java
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitPigXCommand.java
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
oozie/trunk/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java
oozie/trunk/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
oozie/trunk/core/src/test/java/org/apache/oozie/servlet/MockDagEngineService.java
oozie/trunk/release-log.txt
Modified: oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java?rev=1548778&r1=1548777&r2=1548778&view=diff
==============================================================================
--- oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
(original)
+++ oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java Sat Dec
7 02:18:17 2013
@@ -91,6 +91,7 @@ public class OozieCLI {
public static final String SLA_CMD = "sla";
public static final String PIG_CMD = "pig";
public static final String HIVE_CMD = "hive";
+ public static final String SQOOP_CMD = "sqoop";
public static final String MR_CMD = "mapreduce";
public static final String INFO_CMD = "info";
@@ -112,6 +113,7 @@ public class OozieCLI {
public static final String ACTION_OPTION = "action";
public static final String DEFINITION_OPTION = "definition";
public static final String CONFIG_CONTENT_OPTION = "configcontent";
+ public static final String SQOOP_COMMAND_OPTION = "command";
public static final String DO_AS_OPTION = "doas";
@@ -427,6 +429,29 @@ public class OozieCLI {
}
/**
+ * Create option for command line option 'sqoop'
+ * @return sqoop options
+ */
+ @SuppressWarnings("static-access")
+ protected Options createSqoopCLIOptions() {
+ Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL");
+ Option config = new Option(CONFIG_OPTION, true, "job configuration
file '.properties'");
+ Option command =
OptionBuilder.withArgName(SQOOP_COMMAND_OPTION).hasArgs().withValueSeparator().withDescription(
+ "sqoop command").create(SQOOP_COMMAND_OPTION);
+ Option property =
OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription(
+ "set/override value for given property").create("D");
+ Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates
as the specified user");
+ Options Options = new Options();
+ Options.addOption(oozie);
+ Options.addOption(doAs);
+ Options.addOption(config);
+ Options.addOption(property);
+ Options.addOption(command);
+ addAuthOptions(Options);
+ return Options;
+ }
+
+ /**
* Create option for command line option 'info'
* @return info options
*/
@@ -485,6 +510,8 @@ public class OozieCLI {
+ "arguments after '-X' are put in <configuration>",
createScriptLanguageOptions(PIG_CMD), true);
parser.addCommand(HIVE_CMD, "-X ", "submit a hive job, everything
after '-X' are pass-through parameters to hive, any '-D' "
+ "arguments after '-X' are put in <configuration>",
createScriptLanguageOptions(HIVE_CMD), true);
+ parser.addCommand(SQOOP_CMD, "-X ", "submit a sqoop job, everything
after '-X' are pass-through parameters " +
+ "to sqoop, any '-D' arguments after '-X' are put in
<configuration>", createSqoopCLIOptions(), true);
parser.addCommand(INFO_CMD, "", "get more detailed info about specific
topics", createInfoOptions(), false);
parser.addCommand(MR_CMD, "", "submit a mapreduce job",
createMROptions(), false);
@@ -553,6 +580,9 @@ public class OozieCLI {
else if (command.getName().equals(HIVE_CMD)) {
scriptLanguageCommand(command.getCommandLine(), HIVE_CMD);
}
+ else if (command.getName().equals(SQOOP_CMD)) {
+ sqoopCommand(command.getCommandLine());
+ }
else if (command.getName().equals(INFO_CMD)) {
infoCommand(command.getCommandLine());
}
@@ -1696,6 +1726,35 @@ public class OozieCLI {
}
}
+ private void sqoopCommand(CommandLine commandLine) throws IOException,
OozieCLIException {
+ List<String> args = commandLine.getArgList();
+ if (args.size() > 0) {
+ // checking if args starts with -X (because CLIParser cannot check
this)
+ if (!args.get(0).equals("-X")) {
+ throw new OozieCLIException("Unrecognized option: " +
args.get(0) + " Expecting -X");
+ }
+ args.remove(0);
+ }
+
+ if (!commandLine.hasOption(SQOOP_COMMAND_OPTION)) {
+ throw new OozieCLIException("Need to specify -command");
+ }
+
+ if (!commandLine.hasOption(CONFIG_OPTION)) {
+ throw new OozieCLIException("Need to specify -config
<configfile>");
+ }
+
+ try {
+ XOozieClient wc = createXOozieClient(commandLine);
+ Properties conf = getConfiguration(wc, commandLine);
+ String[] command =
commandLine.getOptionValues(SQOOP_COMMAND_OPTION);
+ System.out.println(JOB_ID_PREFIX + wc.submitSqoop(conf, command,
args.toArray(new String[args.size()])));
+ }
+ catch (OozieClientException ex) {
+ throw new OozieCLIException(ex.toString(), ex);
+ }
+ }
+
private void infoCommand(CommandLine commandLine) throws OozieCLIException
{
for (Option option : commandLine.getOptions()) {
String opt = option.getOpt();
Modified:
oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java?rev=1548778&r1=1548777&r2=1548778&view=diff
==============================================================================
--- oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java
(original)
+++ oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java
Sat Dec 7 02:18:17 2013
@@ -57,6 +57,10 @@ public class XOozieClient extends OozieC
public static final String HIVE_SCRIPT_PARAMS = "oozie.hive.script.params";
+ public static final String SQOOP_COMMAND = "oozie.sqoop.command";
+
+ public static final String SQOOP_OPTIONS = "oozie.sqoop.options";
+
public static final String FILES = "oozie.files";
public static final String ARCHIVES = "oozie.archives";
@@ -100,6 +104,14 @@ public class XOozieClient extends OozieC
}
}
+ private String serializeSqoopCommand(String[] command) {
+ StringBuilder sb = new StringBuilder();
+ for (String arg : command) {
+ sb.append(arg).append("\n");
+ }
+ return sb.toString();
+ }
+
static void setStrings(Properties conf, String key, String[] values) {
if (values != null) {
conf.setProperty(key + ".size", (new
Integer(values.length)).toString());
@@ -224,6 +236,32 @@ public class XOozieClient extends OozieC
}
/**
+ * Submit a Sqoop job via HTTP.
+ *
+ * @param conf job configuration.
+ * @param command sqoop command to run.
+ * @param args arguments string.
+ * @return the job Id.
+ * @throws OozieClientException thrown if the job could not be submitted.
+ */
+ public String submitSqoop(Properties conf, String[] command, String[] args)
+ throws OozieClientException {
+ if (conf == null) {
+ throw new IllegalArgumentException("conf cannot be null");
+ }
+ if (command == null) {
+ throw new IllegalArgumentException("command cannot be null");
+ }
+
+ validateHttpSubmitConf(conf);
+
+ conf.setProperty(XOozieClient.SQOOP_COMMAND,
serializeSqoopCommand(command));
+ setStrings(conf, XOozieClient.SQOOP_OPTIONS, args);
+
+ return (new HttpJobSubmit(conf, OozieCLI.SQOOP_CMD)).call();
+ }
+
+ /**
* Submit a Map/Reduce job via HTTP.
*
* @param conf job configuration.
Modified: oozie/trunk/core/src/main/java/org/apache/oozie/DagEngine.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/DagEngine.java?rev=1548778&r1=1548777&r2=1548778&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/DagEngine.java (original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/DagEngine.java Sat Dec 7
02:18:17 2013
@@ -38,6 +38,7 @@ import org.apache.oozie.command.wf.Submi
import org.apache.oozie.command.wf.SubmitHttpXCommand;
import org.apache.oozie.command.wf.SubmitMRXCommand;
import org.apache.oozie.command.wf.SubmitPigXCommand;
+import org.apache.oozie.command.wf.SubmitSqoopXCommand;
import org.apache.oozie.command.wf.SubmitXCommand;
import org.apache.oozie.command.wf.SuspendXCommand;
import org.apache.oozie.command.wf.WorkflowActionInfoXCommand;
@@ -144,7 +145,7 @@ public class DagEngine extends BaseEngin
* It validates configuration properties.
*
* @param conf job configuration.
- * @param jobType job type - can be "pig", "hive, or "mapreduce".
+ * @param jobType job type - can be "pig", "hive", "sqoop" or "mapreduce".
* @return the job Id.
* @throws DagEngineException thrown if the job could not be created.
*/
@@ -163,6 +164,9 @@ public class DagEngine extends BaseEngin
else if (jobType.equals("hive")) {
submit = new SubmitHiveXCommand(conf);
}
+ else if (jobType.equals("sqoop")) {
+ submit = new SubmitSqoopXCommand(conf);
+ }
jobId = submit.call();
start(jobId);
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHiveXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHiveXCommand.java?rev=1548778&r1=1548777&r2=1548778&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHiveXCommand.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHiveXCommand.java
Sat Dec 7 02:18:17 2013
@@ -26,18 +26,22 @@ public class SubmitHiveXCommand extends
super("submitHive", "submitHive", conf);
}
- protected String getLanguageName(){
+ @Override
+ protected String getWorkflowName(){
return "hive";
}
+ @Override
protected String getOptions(){
return XOozieClient.HIVE_OPTIONS;
}
+ @Override
protected String getScriptParamters() {
return XOozieClient.HIVE_SCRIPT_PARAMS;
}
+ @Override
protected Namespace getSectionNamespace(){
return Namespace.getNamespace("uri:oozie:hive-action:0.5");
}
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java?rev=1548778&r1=1548777&r2=1548778&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java
Sat Dec 7 02:18:17 2013
@@ -46,6 +46,7 @@ import org.jdom.Element;
import org.jdom.Namespace;
import java.util.Date;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.HashSet;
@@ -86,13 +87,99 @@ public abstract class SubmitHttpXCommand
PropertiesUtils.createPropertySet(badDefaultProps,
DISALLOWED_DEFAULT_PROPERTIES);
}
+ abstract protected Element generateSection(Configuration conf, Namespace
ns);
+
+ abstract protected Namespace getSectionNamespace();
+
+ abstract protected String getWorkflowName();
+
+ protected void checkMandatoryConf(Configuration conf) {
+ for (String key : MANDATORY_OOZIE_CONFS) {
+ String value = conf.get(key);
+ if (value == null) {
+ throw new RuntimeException(key + " is not specified");
+ }
+ }
+ }
+
+ protected Namespace getWorkflowNamespace() {
+ return Namespace.getNamespace("uri:oozie:workflow:0.2");
+ }
/**
* Generate workflow xml from conf object
*
* @param conf the configuration object
* @return workflow xml def string representation
*/
- abstract protected String getWorkflowXml(Configuration conf);
+ protected String getWorkflowXml(Configuration conf) {
+ checkMandatoryConf(conf);
+
+ Namespace ns = getWorkflowNamespace();
+ Element root = new Element("workflow-app", ns);
+ String name = getWorkflowName();
+ root.setAttribute("name", "oozie-" + name);
+
+ Element start = new Element("start", ns);
+ String nodeName = name + "1";
+ start.setAttribute("to", nodeName);
+ root.addContent(start);
+
+ Element action = new Element("action", ns);
+ action.setAttribute("name", nodeName);
+
+ Element ele = generateSection(conf, getSectionNamespace());
+ action.addContent(ele);
+
+ Element ok = new Element("ok", ns);
+ ok.setAttribute("to", "end");
+ action.addContent(ok);
+
+ Element error = new Element("error", ns);
+ error.setAttribute("to", "fail");
+ action.addContent(error);
+
+ root.addContent(action);
+
+ Element kill = new Element("kill", ns);
+ kill.setAttribute("name", "fail");
+ Element message = new Element("message", ns);
+ message.addContent(name + " failed, error
message[${wf:errorMessage(wf:lastErrorNode())}]");
+ kill.addContent(message);
+ root.addContent(kill);
+
+ Element end = new Element("end", ns);
+ end.setAttribute("name", "end");
+ root.addContent(end);
+
+ return XmlUtils.prettyPrint(root).toString();
+ };
+
+ protected Element generateConfigurationSection(List<String> Dargs,
Namespace ns) {
+ Element configuration = new Element("configuration", ns);
+ for (String arg : Dargs) {
+ String name = null, value = null;
+ int pos = arg.indexOf("=");
+ if (pos == -1) { // "-D<name>" or "-D" only
+ name = arg.substring(2, arg.length());
+ value = "";
+ }
+ else { // "-D<name>=<value>"
+ name = arg.substring(2, pos);
+ value = arg.substring(pos + 1, arg.length());
+ }
+
+ Element property = new Element("property", ns);
+ Element nameElement = new Element("name", ns);
+ nameElement.addContent(name);
+ property.addContent(nameElement);
+ Element valueElement = new Element("value", ns);
+ valueElement.addContent(value);
+ property.addContent(valueElement);
+ configuration.addContent(property);
+ }
+
+ return configuration;
+ }
/* (non-Javadoc)
* @see org.apache.oozie.command.XCommand#execute()
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java?rev=1548778&r1=1548777&r2=1548778&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
Sat Dec 7 02:18:17 2013
@@ -19,7 +19,6 @@ package org.apache.oozie.command.wf;
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.service.WorkflowAppService;
-import org.apache.oozie.util.XmlUtils;
import org.jdom.Element;
import org.jdom.Namespace;
import org.apache.oozie.client.XOozieClient;
@@ -52,6 +51,16 @@ public class SubmitMRXCommand extends Su
DEPRECATE_MAP.put(WorkflowAppService.HADOOP_USER,
"mapreduce.job.user.name");
}
+ @Override
+ protected Namespace getSectionNamespace(){
+ return Namespace.getNamespace("uri:oozie:workflow:0.2");
+ }
+
+ @Override
+ protected String getWorkflowName(){
+ return "mapreduce";
+ }
+
private Element generateConfigurationSection(Configuration conf, Namespace
ns) {
Element configuration = null;
Iterator<Map.Entry<String, String>> iter = conf.iterator();
@@ -82,7 +91,8 @@ public class SubmitMRXCommand extends Su
return configuration;
}
- private Element generateMRSection(Configuration conf, Namespace ns) {
+ @Override
+ protected Element generateSection(Configuration conf, Namespace ns) {
Element mapreduce = new Element("map-reduce", ns);
Element jt = new Element("job-tracker", ns);
String newJTVal = conf.get(DEPRECATE_MAP.get(XOozieClient.JT));
@@ -111,15 +121,8 @@ public class SubmitMRXCommand extends Su
return mapreduce;
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.oozie.command.wf.SubmitHttpCommand#getWorkflowXml(org.apache
- * .hadoop.conf.Configuration)
- */
@Override
- protected String getWorkflowXml(Configuration conf) {
+ protected void checkMandatoryConf(Configuration conf) {
for (String key : MANDATORY_OOZIE_CONFS) {
String value = conf.get(key);
if(value == null) {
@@ -133,43 +136,6 @@ public class SubmitMRXCommand extends Su
}
}
}
-
- Namespace ns = Namespace.getNamespace("uri:oozie:workflow:0.2");
- Element root = new Element("workflow-app", ns);
- root.setAttribute("name", "oozie-mapreduce");
-
- Element start = new Element("start", ns);
- start.setAttribute("to", "hadoop1");
- root.addContent(start);
-
- Element action = new Element("action", ns);
- action.setAttribute("name", "hadoop1");
-
- Element mapreduce = generateMRSection(conf, ns);
- action.addContent(mapreduce);
-
- Element ok = new Element("ok", ns);
- ok.setAttribute("to", "end");
- action.addContent(ok);
-
- Element error = new Element("error", ns);
- error.setAttribute("to", "fail");
- action.addContent(error);
-
- root.addContent(action);
-
- Element kill = new Element("kill", ns);
- kill.setAttribute("name", "fail");
- Element message = new Element("message", ns);
- message.addContent("Map/Reduce failed, error
message[${wf:errorMessage(wf:lastErrorNode())}]");
- kill.addContent(message);
- root.addContent(kill);
-
- Element end = new Element("end", ns);
- end.setAttribute("name", "end");
- root.addContent(end);
-
- return XmlUtils.prettyPrint(root).toString();
}
@Override
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitPigXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitPigXCommand.java?rev=1548778&r1=1548777&r2=1548778&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitPigXCommand.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitPigXCommand.java
Sat Dec 7 02:18:17 2013
@@ -25,14 +25,17 @@ public class SubmitPigXCommand extends S
super("submitPig", "submitPig", conf);
}
- protected String getLanguageName(){
+ @Override
+ protected String getWorkflowName(){
return "pig";
}
+ @Override
protected String getOptions(){
return XOozieClient.PIG_OPTIONS;
}
+ @Override
protected String getScriptParamters() {
return XOozieClient.PIG_SCRIPT_PARAMS;
}
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java?rev=1548778&r1=1548777&r2=1548778&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
Sat Dec 7 02:18:17 2013
@@ -21,7 +21,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.oozie.action.hadoop.MapReduceMain;
import org.apache.oozie.client.XOozieClient;
import org.apache.oozie.command.CommandException;
-import org.apache.oozie.util.XmlUtils;
import org.jdom.Element;
import org.jdom.Namespace;
@@ -33,18 +32,21 @@ public abstract class SubmitScriptLangua
super(name, type, conf);
}
- protected abstract String getLanguageName();
+ @Override
+ protected abstract String getWorkflowName();
protected abstract String getOptions();
protected abstract String getScriptParamters();
+ @Override
protected Namespace getSectionNamespace() {
return Namespace.getNamespace("uri:oozie:workflow:0.2");
}
- private Element generateSection(Configuration conf, Namespace ns) {
- String name = getLanguageName();
+ @Override
+ protected Element generateSection(Configuration conf, Namespace ns) {
+ String name = getWorkflowName();
Element ele = new Element(name, ns);
Element jt = new Element("job-tracker", ns);
jt.addContent(conf.get(XOozieClient.JT));
@@ -99,89 +101,6 @@ public abstract class SubmitScriptLangua
return ele;
}
- private Element generateConfigurationSection(List<String> Dargs, Namespace
ns) {
- Element configuration = new Element("configuration", ns);
- for (String arg : Dargs) {
- String name = null, value = null;
- int pos = arg.indexOf("=");
- if (pos == -1) { // "-D<name>" or "-D" only
- name = arg.substring(2, arg.length());
- value = "";
- }
- else { // "-D<name>=<value>"
- name = arg.substring(2, pos);
- value = arg.substring(pos + 1, arg.length());
- }
-
- Element property = new Element("property", ns);
- Element nameElement = new Element("name", ns);
- nameElement.addContent(name);
- property.addContent(nameElement);
- Element valueElement = new Element("value", ns);
- valueElement.addContent(value);
- property.addContent(valueElement);
- configuration.addContent(property);
- }
-
- return configuration;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.oozie.command.wf.SubmitHttpCommand#getWorkflowXml(org.apache
- * .hadoop.conf.Configuration)
- */
- @Override
- protected String getWorkflowXml(Configuration conf) {
- for (String key : MANDATORY_OOZIE_CONFS) {
- String value = conf.get(key);
- if (value == null) {
- throw new RuntimeException(key + " is not specified");
- }
- }
-
- Namespace ns = Namespace.getNamespace("uri:oozie:workflow:0.2");
- Element root = new Element("workflow-app", ns);
- root.setAttribute("name", "oozie-" + getLanguageName());
-
- Element start = new Element("start", ns);
- String name = getLanguageName();
- String nodeName = name + "1";
- start.setAttribute("to", nodeName);
- root.addContent(start);
-
- Element action = new Element("action", ns);
- action.setAttribute("name", nodeName);
-
- Element ele = generateSection(conf, getSectionNamespace());
- action.addContent(ele);
-
- Element ok = new Element("ok", ns);
- ok.setAttribute("to", "end");
- action.addContent(ok);
-
- Element error = new Element("error", ns);
- error.setAttribute("to", "fail");
- action.addContent(error);
-
- root.addContent(action);
-
- Element kill = new Element("kill", ns);
- kill.setAttribute("name", "fail");
- Element message = new Element("message", ns);
- message.addContent(name + " failed, error
message[${wf:errorMessage(wf:lastErrorNode())}]");
- kill.addContent(message);
- root.addContent(kill);
-
- Element end = new Element("end", ns);
- end.setAttribute("name", "end");
- root.addContent(end);
-
- return XmlUtils.prettyPrint(root).toString();
- }
-
@Override
public String getEntityKey() {
return null;
Added:
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java?rev=1548778&view=auto
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java
(added)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java
Sat Dec 7 02:18:17 2013
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.command.wf;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.client.XOozieClient;
+import org.apache.oozie.command.CommandException;
+import org.apache.oozie.action.hadoop.MapReduceMain;
+import org.jdom.Namespace;
+import org.jdom.Element;
+
+public class SubmitSqoopXCommand extends SubmitHttpXCommand {
+ public SubmitSqoopXCommand(Configuration conf) {
+ super("submitSqoop", "submitSqoop", conf);
+ }
+
+ protected String getOptions(){
+ return XOozieClient.SQOOP_OPTIONS;
+ }
+
+ @Override
+ protected Namespace getSectionNamespace(){
+ return Namespace.getNamespace("uri:oozie:sqoop-action:0.4");
+ }
+
+ @Override
+ protected String getWorkflowName(){
+ return "sqoop";
+ }
+
+ @Override
+ protected Element generateSection(Configuration conf, Namespace ns) {
+ String name = "sqoop";
+ Element ele = new Element(name, ns);
+ Element jt = new Element("job-tracker", ns);
+ jt.addContent(conf.get(XOozieClient.JT));
+ ele.addContent(jt);
+ Element nn = new Element("name-node", ns);
+ nn.addContent(conf.get(XOozieClient.NN));
+ ele.addContent(nn);
+
+ List<String> Dargs = new ArrayList<String>();
+ String[] args = MapReduceMain.getStrings(conf, getOptions());
+ for (String arg : args) {
+ if (arg.startsWith("-D")) {
+ Dargs.add(arg);
+ }
+ }
+
+ // configuration section
+ if (Dargs.size() > 0) {
+ Element configuration = generateConfigurationSection(Dargs, ns);
+ ele.addContent(configuration);
+ }
+
+ String[] sqoopArgs = conf.get(XOozieClient.SQOOP_COMMAND).split("\n");
+ for (String arg : sqoopArgs) {
+ Element eArg = new Element("arg", ns);
+ eArg.addContent(arg);
+ ele.addContent(eArg);
+ }
+
+ // file section
+ addFileSection(ele, conf, ns);
+
+ // archive section
+ addArchiveSection(ele, conf, ns);
+
+ return ele;
+ }
+
+ @Override
+ protected void verifyPrecondition() throws CommandException {
+ }
+
+ @Override
+ protected boolean isLockRequired() {
+ return false;
+ }
+
+ @Override
+ protected void loadState() {
+ }
+
+ @Override
+ public String getEntityKey() {
+ return null;
+ }
+}
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java?rev=1548778&r1=1548777&r2=1548778&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java
(original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java
Sat Dec 7 02:18:17 2013
@@ -59,6 +59,7 @@ public class V1JobsServlet extends BaseJ
private static final String INSTRUMENTATION_NAME = "v1jobs";
private static final Set<String> httpJobType = new HashSet<String>(){{
this.add(OozieCLI.HIVE_CMD);
+ this.add(OozieCLI.SQOOP_CMD);
this.add(OozieCLI.PIG_CMD);
this.add(OozieCLI.MR_CMD);
}};
Modified:
oozie/trunk/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java?rev=1548778&r1=1548777&r2=1548778&view=diff
==============================================================================
---
oozie/trunk/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
(original)
+++
oozie/trunk/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
Sat Dec 7 02:18:17 2013
@@ -108,6 +108,31 @@ public class TestWorkflowXClient extends
});
}
+ public void testSubmitSqoop() throws Exception {
+ runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new
Callable<Void>() {
+ public Void call() throws Exception {
+ String oozieUrl = getContextURL();
+ int wfCount = MockDagEngineService.INIT_WF_COUNT;
+ XOozieClient wc = new XOozieClient(oozieUrl);
+ Properties conf = wc.createConfiguration();
+ Path libPath = new Path(getFsTestCaseDir(), "lib");
+ getFileSystem().mkdirs(libPath);
+ System.out.println(libPath.toString());
+ conf.setProperty(OozieClient.LIBPATH, libPath.toString());
+ conf.setProperty(XOozieClient.JT, "localhost:9001");
+ conf.setProperty(XOozieClient.NN, "hdfs://localhost:9000");
+
+ assertEquals(MockDagEngineService.JOB_ID + wfCount +
MockDagEngineService.JOB_ID_END,
+ wc.submitSqoop(conf, new String[] {"import",
"--connect",
+ "jdbc:mysql://localhost:3306/oozie"},
+ null));
+
+ assertTrue(MockDagEngineService.started.get(wfCount));
+ return null;
+ }
+ });
+ }
+
public void testSubmitMR() throws Exception {
runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new
Callable<Void>() {
public Void call() throws Exception {
Added:
oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java?rev=1548778&view=auto
==============================================================================
---
oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java
(added)
+++
oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java
Sat Dec 7 02:18:17 2013
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.command.wf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.local.LocalOozie;
+import org.apache.oozie.action.hadoop.MapReduceMain;
+import org.apache.oozie.client.XOozieClient;
+import org.apache.oozie.test.XFsTestCase;
+import org.apache.oozie.util.XLog;
+import org.apache.oozie.util.XmlUtils;
+import org.apache.oozie.service.XLogService;
+import org.jdom.Element;
+
+public class TestSubmitSqoopXCommand extends XFsTestCase {
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ setSystemProperty(XLogService.LOG4J_FILE, "oozie-log4j.properties");
+ LocalOozie.start();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ LocalOozie.stop();
+ super.tearDown();
+ }
+
+ public void testWFXmlGeneration() throws Exception {
+ Configuration conf = new Configuration();
+
+ conf.set(XOozieClient.JT, "jobtracker");
+ conf.set(XOozieClient.NN, "namenode");
+ conf.set(OozieClient.LIBPATH, "libpath");
+
+ conf.set(XOozieClient.SQOOP_COMMAND,
"import\n--connect\njdbc:mysql://localhost:3306/oozie");
+
+ String sqoopArgsStr = "-Da=aaa -Db=bbb";
+ String[] args = sqoopArgsStr.split(" ");
+ MapReduceMain.setStrings(conf, XOozieClient.SQOOP_OPTIONS, args);
+
+ SubmitSqoopXCommand submitSqoopCmd = new SubmitSqoopXCommand(conf);
+ String xml = submitSqoopCmd.getWorkflowXml(conf);
+
+ XLog.getLog(getClass()).info("xml = " + xml);
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("<workflow-app xmlns=\"uri:oozie:workflow:0.2\"
name=\"oozie-sqoop\">");
+ sb.append("<start to=\"sqoop1\" />");
+ sb.append("<action name=\"sqoop1\">");
+ sb.append("<sqoop xmlns=\"uri:oozie:sqoop-action:0.4\">");
+ sb.append("<job-tracker>jobtracker</job-tracker>");
+ sb.append("<name-node>namenode</name-node>");
+ sb.append("<configuration>");
+ sb.append("<property>");
+ sb.append("<name>a</name>");
+ sb.append("<value>aaa</value>");
+ sb.append("</property>");
+ sb.append("<property>");
+ sb.append("<name>b</name>");
+ sb.append("<value>bbb</value>");
+ sb.append("</property>");
+ sb.append("</configuration>");
+ sb.append("<arg>import</arg>");
+ sb.append("<arg>--connect</arg>");
+ sb.append("<arg>jdbc:mysql://localhost:3306/oozie</arg>");
+ sb.append("</sqoop>");
+ sb.append("<ok to=\"end\" />");
+ sb.append("<error to=\"fail\" />");
+ sb.append("</action>");
+ sb.append("<kill name=\"fail\">");
+ sb.append("<message>sqoop failed, error
message[${wf:errorMessage(wf:lastErrorNode())}]</message>");
+ sb.append("</kill>");
+ sb.append("<end name=\"end\" />");
+ sb.append("</workflow-app>");
+
+ Element root = XmlUtils.parseXml(sb.toString());
+ String reference = XmlUtils.prettyPrint(root).toString();
+
+ XLog.getLog(getClass()).info("reference xml = " + reference);
+ assertTrue(xml.equals(reference));
+ }
+}
Modified:
oozie/trunk/core/src/test/java/org/apache/oozie/servlet/MockDagEngineService.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/servlet/MockDagEngineService.java?rev=1548778&r1=1548777&r2=1548778&view=diff
==============================================================================
---
oozie/trunk/core/src/test/java/org/apache/oozie/servlet/MockDagEngineService.java
(original)
+++
oozie/trunk/core/src/test/java/org/apache/oozie/servlet/MockDagEngineService.java
Sat Dec 7 02:18:17 2013
@@ -114,6 +114,9 @@ public class MockDagEngineService extend
else if (jobType.equals("mapreduce")) {
did = "submitMR";
}
+ else if (jobType.equals("sqoop")) {
+ did = "submitSqoop";
+ }
int idx = workflows.size();
WorkflowJob job = createDummyWorkflow(idx,
XmlUtils.prettyPrint(conf).toString());
workflows.add(job);
Modified: oozie/trunk/release-log.txt
URL:
http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1548778&r1=1548777&r2=1548778&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Sat Dec 7 02:18:17 2013
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1575 Add functionality to submit sqoop jobs through http on oozie server
side (bowenzhangusa via rkanter)
OOZIE-1634 TestJavaActionExecutor#testUpdateConfForUberMode fails against
Hadoop 2 (rkanter)
OOZIE-1633 Test failures related to sharelib when running against Hadoop 2
(rkanter)
OOZIE-1598 enable html email in email action (puru via ryota)
