Author: tucu
Date: Fri Dec 9 14:34:18 2011
New Revision: 1212444
URL: http://svn.apache.org/viewvc?rev=1212444&view=rev
Log:
OOZIE-617 Add back Ssh action as extension. (tucu)
Added:
incubator/oozie/trunk/client/src/main/resources/ssh-action-0.1.xsd
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutorAsExtension.java
incubator/oozie/trunk/docs/src/site/twiki/DG_SshActionExtension.twiki
incubator/oozie/trunk/examples/src/main/apps/ssh/job.properties
incubator/oozie/trunk/examples/src/main/apps/ssh/workflow.xml
Modified:
incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
incubator/oozie/trunk/core/src/main/conf/oozie-site.xml
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java
incubator/oozie/trunk/docs/src/site/twiki/index.twiki
incubator/oozie/trunk/pom.xml
incubator/oozie/trunk/release-log.txt
Modified:
incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java?rev=1212444&r1=1212443&r2=1212444&view=diff
==============================================================================
---
incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
(original)
+++
incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
Fri Dec 9 14:34:18 2011
@@ -1246,6 +1246,8 @@ public class OozieCLI {
"hive-action-0.2.xsd")));
sources.add(new
StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
"sqoop-action-0.2.xsd")));
+ sources.add(new
StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
+ "ssh-action-0.1.xsd")));
SchemaFactory factory =
SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
Schema schema = factory.newSchema(sources.toArray(new
StreamSource[sources.size()]));
Validator validator = schema.newValidator();
Added: incubator/oozie/trunk/client/src/main/resources/ssh-action-0.1.xsd
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/client/src/main/resources/ssh-action-0.1.xsd?rev=1212444&view=auto
==============================================================================
--- incubator/oozie/trunk/client/src/main/resources/ssh-action-0.1.xsd (added)
+++ incubator/oozie/trunk/client/src/main/resources/ssh-action-0.1.xsd Fri Dec
9 14:34:18 2011
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema";
+ xmlns:ssh="uri:oozie:ssh-action:0.1" elementFormDefault="qualified"
+ targetNamespace="uri:oozie:ssh-action:0.1">
+
+ <xs:element name="ssh" type="ssh:ACTION"/>
+
+ <xs:complexType name="ACTION">
+ <xs:sequence>
+ <xs:element name="host" type="xs:string" minOccurs="1"
maxOccurs="1"/>
+ <xs:element name="command" type="xs:string" minOccurs="1"
maxOccurs="1"/>
+ <xs:element name="args" type="xs:string" minOccurs="0"
maxOccurs="unbounded"/>
+ <xs:element name="capture-output" type="ssh:FLAG" minOccurs="0"
maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="FLAG"/>
+
+</xs:schema>
Modified: incubator/oozie/trunk/core/src/main/conf/oozie-site.xml
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/conf/oozie-site.xml?rev=1212444&r1=1212443&r2=1212444&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/conf/oozie-site.xml (original)
+++ incubator/oozie/trunk/core/src/main/conf/oozie-site.xml Fri Dec 9 14:34:18
2011
@@ -37,7 +37,7 @@
<property>
<name>oozie.service.SchemaService.wf.ext.schemas</name>
-
<value>shell-action-0.1.xsd,email-action-0.1.xsd,hive-action-0.2.xsd,sqoop-action-0.2.xsd</value>
+
<value>shell-action-0.1.xsd,email-action-0.1.xsd,hive-action-0.2.xsd,sqoop-action-0.2.xsd,ssh-action-0.1.xsd</value>
</property>
-->
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java?rev=1212444&r1=1212443&r2=1212444&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java
Fri Dec 9 14:34:18 2011
@@ -40,6 +40,7 @@ import org.apache.oozie.service.UUIDServ
import org.apache.oozie.service.UUIDService.ApplicationType;
import org.apache.oozie.test.XFsTestCase;
import org.apache.oozie.util.ELEvaluator;
+import org.apache.oozie.util.PropertiesUtils;
import org.apache.oozie.util.XConfiguration;
public class TestSshActionExecutor extends XFsTestCase {
@@ -147,7 +148,10 @@ public class TestSshActionExecutor exten
fs.delete(path, true);
}
-/*
+ protected String getActionXMLSchema() {
+ return "uri:oozie-workflow:0.1";
+ }
+
public void testJobStart() throws ActionExecutorException {
String baseDir = getTestCaseDir();
Path appPath = new Path(getNameNodeUri(), baseDir);
@@ -168,7 +172,7 @@ public class TestSshActionExecutor exten
final WorkflowActionBean action = new WorkflowActionBean();
action.setId("actionId");
- action.setConf("<ssh xmlns='uri:oozie-workflow:0.1'>" +
+ action.setConf("<ssh xmlns='" + getActionXMLSchema() + "'>" +
"<host>localhost</host>" +
"<command>echo</command>" +
"<capture-output/>" +
@@ -189,9 +193,7 @@ public class TestSshActionExecutor exten
assertEquals(Status.OK, action.getStatus());
assertEquals("something",
PropertiesUtils.stringToProperties(action.getData()).getProperty("prop1"));
}
-*/
-/*
public void testJobRecover() throws ActionExecutorException,
InterruptedException {
String baseDir = getTestCaseDir();
Path appPath = new Path(getNameNodeUri(), baseDir);
@@ -212,7 +214,7 @@ public class TestSshActionExecutor exten
final WorkflowActionBean action = new WorkflowActionBean();
action.setId("actionId");
- action.setConf("<ssh xmlns='uri:oozie-workflow:0.1'>" +
+ action.setConf("<ssh xmlns='" + getActionXMLSchema() + "'>" +
"<host>localhost</host>" +
"<command>echo</command>" +
"<capture-output/>" +
@@ -226,7 +228,7 @@ public class TestSshActionExecutor exten
Thread.sleep(200);
final WorkflowActionBean action1 = new WorkflowActionBean();
action1.setId("actionId");
- action1.setConf("<ssh xmlns='uri:oozie-workflow:0.1'>" +
+ action1.setConf("<ssh xmlns='" + getActionXMLSchema() + "'>" +
"<host>localhost</host>" +
"<command>echo</command>" +
"<capture-output/>" +
@@ -249,7 +251,7 @@ public class TestSshActionExecutor exten
assertEquals(Status.OK, action1.getStatus());
assertEquals("something",
PropertiesUtils.stringToProperties(action1.getData()).getProperty("prop1"));
}
-*/
+
// TODO Move this test case over to a new class. Conflict between this one
// and testConnectionErrors. The property to replace the ssh user cannot be
@@ -275,7 +277,7 @@ public class TestSshActionExecutor exten
//
// final WorkflowActionBean action = new WorkflowActionBean();
// action.setId("actionId_" + System.currentTimeMillis());
-// action.setConf("<ssh xmlns='uri:oozie-workflow:0.1'>" +
+// action.setConf("<ssh xmlns='" + getActionXMLSchema() + "'>" +
// "<host>invalid@localhost</host>" +
// "<command>echo</command>" +
// "<capture-output/>" +
@@ -315,7 +317,7 @@ public class TestSshActionExecutor exten
final WorkflowActionBean action = new WorkflowActionBean();
action.setId("actionId");
- action.setConf("<ssh xmlns='uri:oozie-workflow:0.1'>" +
+ action.setConf("<ssh xmlns='" + getActionXMLSchema() + "'>" +
"<host>blabla</host>" +
"<command>echo</command>" +
"<args>\"prop1=something\"</args>" +
@@ -332,7 +334,7 @@ public class TestSshActionExecutor exten
assertEquals("COULD_NOT_RESOLVE_HOST", ex.getErrorCode());
assertEquals(ActionExecutorException.ErrorType.TRANSIENT,
ex.getErrorType());
}
- action.setConf("<ssh xmlns='uri:oozie-workflow:0.1'>" +
+ action.setConf("<ssh xmlns='" + getActionXMLSchema() + "'>" +
"<host>11.11.11.11</host>" +
"<command>echo</command>" +
"<args>\"prop1=something\"</args>" +
@@ -346,7 +348,7 @@ public class TestSshActionExecutor exten
assertEquals("COULD_NOT_CONNECT", ex.getErrorCode());
assertEquals(ActionExecutorException.ErrorType.TRANSIENT,
ex.getErrorType());
}
- action.setConf("<ssh xmlns='uri:oozie-workflow:0.1'>" +
+ action.setConf("<ssh xmlns='" + getActionXMLSchema() + "'>" +
"<host>y@localhost</host>" +
"<command>echo</command>" +
"<args>\"prop1=something\"</args>" +
Added:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutorAsExtension.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutorAsExtension.java?rev=1212444&view=auto
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutorAsExtension.java
(added)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutorAsExtension.java
Fri Dec 9 14:34:18 2011
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.ssh;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.WorkflowActionBean;
+import org.apache.oozie.WorkflowJobBean;
+import org.apache.oozie.action.ActionExecutor;
+import org.apache.oozie.action.ActionExecutorException;
+import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.client.WorkflowAction.Status;
+import org.apache.oozie.client.WorkflowJob;
+import org.apache.oozie.service.CallbackService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.UUIDService;
+import org.apache.oozie.service.UUIDService.ApplicationType;
+import org.apache.oozie.service.WorkflowAppService;
+import org.apache.oozie.test.XFsTestCase;
+import org.apache.oozie.util.ELEvaluator;
+import org.apache.oozie.util.PropertiesUtils;
+import org.apache.oozie.util.XConfiguration;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.net.URISyntaxException;
+import java.util.Properties;
+
+public class TestSshActionExecutorAsExtension extends TestSshActionExecutor {
+
+ @Override
+ protected String getActionXMLSchema() {
+ return "uri:oozie:ssh-action:0.1";
+ }
+
+}
Added: incubator/oozie/trunk/docs/src/site/twiki/DG_SshActionExtension.twiki
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/DG_SshActionExtension.twiki?rev=1212444&view=auto
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/DG_SshActionExtension.twiki
(added)
+++ incubator/oozie/trunk/docs/src/site/twiki/DG_SshActionExtension.twiki Fri
Dec 9 14:34:18 2011
@@ -0,0 +1,118 @@
+<noautolink>
+
+[[index][::Go back to Oozie Documentation Index::]]
+
+-----
+
+---+!! Oozie Ssh Action Extension
+
+%TOC%
+
+---++ Ssh Action
+
+The =ssh= action starts a shell command on a remote machine as a remote secure
shell in background. The workflow job
+will wait until the remote shell command completes before continuing to the
next action.
+
+The shell command must be present in the remote machine and it must be
available for execution via the command path.
+
+The shell command is executed in the home directory of the specified user in
the remote host.
+
+The output (STDOUT) of the ssh job can be made available to the workflow job
after the ssh job ends. This information
+could be used from within decision nodes. If the output of the ssh job is made
available to the workflow job the shell
+command must follow the following requirements:
+
+ * The format of the output must be a valid Java Properties file.
+ * The size of the output must not exceed 2KB.
+
+*Syntax:*
+
+<verbatim>
+<workflow-app name="[WF-DEF-NAME]" xmlns="uri:oozie:workflow:0.1">
+ ...
+ <action name="[NODE-NAME]">
+ <ssh xmlns="uri:oozie:ssh-action:0.1">
+ <host>[USER]@[HOST]</host>
+ <command>[SHELL]</command>
+ <args>[ARGUMENTS]</args>
+ ...
+ <capture-output/>
+ </ssh>
+ <ok to="[NODE-NAME]"/>
+ <error to="[NODE-NAME]"/>
+ </action>
+ ...
+</workflow-app>
+</verbatim>
+
+The =host= indicates the user and host where the shell will be executed.
+
+*IMPORTANT:* The =oozie.action.ssh.allow.user.at.host= property, in the
=oozie-site.xml= configuration, indicates if
+an alternate user than the one submitting the job can be used for the ssh
invocation. By default this property is set
+tot =true=.
+
+The =command= element indicates the shell command to execute.
+
+The =args= element, if present, contains parameters to be passed to the shell
command. If more than one =args= element
+is present they are concatenated in order.
+
+If the =capture-output= element is present, it indicates Oozie to capture
output of the STDOUT of the ssh command
+execution. The ssh command output must be in Java Properties file format and
it must not exceed 2KB. From within the
+workflow definition, the output of an ssh action node is accessible via the
=String action:output(String node,
+String key)= function (Refer to section '4.2.6 Action EL Functions').
+
+The configuration of the =ssh= action can be parameterized (templatized) using
EL expressions.
+
+*Example:*
+
+<verbatim>
+<workflow-app name="sample-wf" xmlns="uri:oozie:workflow:0.1">
+ ...
+ <action name="myssjob">
+ <ssh xmlns="uri:oozie:ssh-action:0.1">
+ <host>[email protected]<host>
+ <command>uploaddata</command>
+ <args>jdbc:derby://bar.com:1527/myDB</args>
+ <args>hdfs://foobar.com:9000/usr/tucu/myData</args>
+ </ssh>
+ <ok to="myotherjob"/>
+ <error to="errorcleanup"/>
+ </action>
+ ...
+</workflow-app>
+</verbatim>
+
+In the above example, the =uploaddata= shell command is executed with two
arguments, =jdbc:derby://foo.com:1527/myDB=
+and =hdfs://foobar.com:9000/usr/tucu/myData=.
+
+The =uploaddata= shell must be available in the remote host and available in
the command path.
+
+The output of the command will be ignored because the =capture-output= element
is not present.
+
+---++ Appendix, Ssh XML-Schema
+
+---+++ AE.A Appendix A, Ssh XML-Schema
+
+<verbatim>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema";
+ xmlns:ssh="uri:oozie:ssh-action:0.1" elementFormDefault="qualified"
+ targetNamespace="uri:oozie:ssh-action:0.1">
+.
+ <xs:element name="ssh" type="ssh:ACTION"/>
+.
+ <xs:complexType name="ACTION">
+ <xs:sequence>
+ <xs:element name="host" type="xs:string" minOccurs="1"
maxOccurs="1"/>
+ <xs:element name="command" type="xs:string" minOccurs="1"
maxOccurs="1"/>
+ <xs:element name="args" type="xs:string" minOccurs="0"
maxOccurs="unbounded"/>
+ <xs:element name="capture-output" type="ssh:FLAG" minOccurs="0"
maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+.
+ <xs:complexType name="FLAG"/>
+.
+</xs:schema>
+</verbatim>
+
+[[index][::Go back to Oozie Documentation Index::]]
+
+</noautolink>
Modified: incubator/oozie/trunk/docs/src/site/twiki/index.twiki
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/index.twiki?rev=1212444&r1=1212443&r2=1212444&view=diff
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/index.twiki (original)
+++ incubator/oozie/trunk/docs/src/site/twiki/index.twiki Fri Dec 9 14:34:18
2011
@@ -56,6 +56,7 @@ Enough reading already? Follow the steps
* [[DG_ShellActionExtension][Shell Action]]
* [[DG_HiveActionExtension][Hive Action]]
* [[DG_SqoopActionExtension][Sqoop Action]]
+ * [[DG_SshActionExtension][Ssh Action]]
* [[DG_CustomActionExecutor][Writing a Custom Action Executor]]
---++ Administrator Documentation
Added: incubator/oozie/trunk/examples/src/main/apps/ssh/job.properties
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/examples/src/main/apps/ssh/job.properties?rev=1212444&view=auto
==============================================================================
--- incubator/oozie/trunk/examples/src/main/apps/ssh/job.properties (added)
+++ incubator/oozie/trunk/examples/src/main/apps/ssh/job.properties Fri Dec 9
14:34:18 2011
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+nameNode=hdfs://localhost:8020
+jobTracker=localhost:8021
+queueName=default
+examplesRoot=examples
+
+oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/ssh
Added: incubator/oozie/trunk/examples/src/main/apps/ssh/workflow.xml
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/examples/src/main/apps/ssh/workflow.xml?rev=1212444&view=auto
==============================================================================
--- incubator/oozie/trunk/examples/src/main/apps/ssh/workflow.xml (added)
+++ incubator/oozie/trunk/examples/src/main/apps/ssh/workflow.xml Fri Dec 9
14:34:18 2011
@@ -0,0 +1,36 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<workflow-app xmlns="uri:oozie:workflow:0.2" name="ssh-wf">
+ <start to="ssh"/>
+
+ <action name="ssh">
+ <ssh xmlns="uri:oozie:ssh-action:0.1">
+ <host>localhost</host>
+ <command>echo</command>
+ <args>"Hello Oozie!"</args>
+ </ssh>
+ <ok to="end"/>
+ <error to="fail"/>
+ </action>
+
+ <kill name="fail">
+ <message>SSH action failed, error
message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+ </kill>
+
+ <end name="end"/>
+</workflow-app>
Modified: incubator/oozie/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/pom.xml?rev=1212444&r1=1212443&r2=1212444&view=diff
==============================================================================
--- incubator/oozie/trunk/pom.xml (original)
+++ incubator/oozie/trunk/pom.xml Fri Dec 9 14:34:18 2011
@@ -707,6 +707,11 @@
<exclude>**/${test.exclude}.java</exclude>
<exclude>${test.exclude.pattern}</exclude>
+ <!-- Explictly use
-Dtest=TestSshActionExecutor,TestSshActionExecutorExtension
+ to test the SSH action
+ -->
+ <exclude>**/TestSsh*.java</exclude>
+
<!-- See 'testHive' profile in core/pom.xml and the
Building doc-->
<exclude>**/TestHive*.java</exclude>
Modified: incubator/oozie/trunk/release-log.txt
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1212444&r1=1212443&r2=1212444&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Fri Dec 9 14:34:18 2011
@@ -1,5 +1,6 @@
-- Oozie 3.2.0 release
+OOZIE-617 Add back Ssh action as extension. (tucu)
OOZIE-578 Support shell action in Oozie WF (Mohammad)
OOZIE-620 POMs clean/tune up before 3.2 release. (tucu)
OOZIE-613 Update documentation for execution order.(Mohammad)
