Author: rohini
Date: Tue Nov 12 18:58:12 2013
New Revision: 1541196
URL: http://svn.apache.org/r1541196
Log:
OOZIE-1544 Support variables for coord data-in/data-out dataset (puru via
rohini)
Added:
oozie/trunk/core/src/test/resources/coord-el-dataset-4.xml
Modified:
oozie/trunk/client/src/main/resources/oozie-coordinator-0.1.xsd
oozie/trunk/client/src/main/resources/oozie-coordinator-0.2.xsd
oozie/trunk/client/src/main/resources/oozie-coordinator-0.3.xsd
oozie/trunk/client/src/main/resources/oozie-coordinator-0.4.xsd
oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
oozie/trunk/release-log.txt
Modified: oozie/trunk/client/src/main/resources/oozie-coordinator-0.1.xsd
URL:
http://svn.apache.org/viewvc/oozie/trunk/client/src/main/resources/oozie-coordinator-0.1.xsd?rev=1541196&r1=1541195&r2=1541196&view=diff
==============================================================================
--- oozie/trunk/client/src/main/resources/oozie-coordinator-0.1.xsd (original)
+++ oozie/trunk/client/src/main/resources/oozie-coordinator-0.1.xsd Tue Nov 12
18:58:12 2013
@@ -88,7 +88,7 @@
</xs:sequence>
</xs:choice>
<xs:attribute name="name" type="coordinator:IDENTIFIER"
use="required"/>
- <xs:attribute name="dataset" type="coordinator:IDENTIFIER"
use="required"/>
+ <xs:attribute name="dataset" type="xs:string" use="required"/>
</xs:complexType>
<xs:complexType name="OUTPUTEVENTS">
<xs:sequence minOccurs="1" maxOccurs="1">
@@ -100,7 +100,7 @@
<xs:element name="instance" type="xs:string" minOccurs="1"
maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="name" type="coordinator:IDENTIFIER"
use="required"/>
- <xs:attribute name="dataset" type="coordinator:IDENTIFIER"
use="required"/>
+ <xs:attribute name="dataset" type="xs:string" use="required"/>
</xs:complexType>
<xs:complexType name="ACTION">
<xs:sequence minOccurs="1" maxOccurs="1">
Modified: oozie/trunk/client/src/main/resources/oozie-coordinator-0.2.xsd
URL:
http://svn.apache.org/viewvc/oozie/trunk/client/src/main/resources/oozie-coordinator-0.2.xsd?rev=1541196&r1=1541195&r2=1541196&view=diff
==============================================================================
--- oozie/trunk/client/src/main/resources/oozie-coordinator-0.2.xsd (original)
+++ oozie/trunk/client/src/main/resources/oozie-coordinator-0.2.xsd Tue Nov 12
18:58:12 2013
@@ -89,7 +89,7 @@
</xs:sequence>
</xs:choice>
<xs:attribute name="name" type="coordinator:IDENTIFIER"
use="required"/>
- <xs:attribute name="dataset" type="coordinator:IDENTIFIER"
use="required"/>
+ <xs:attribute name="dataset" type="xs:string" use="required"/>
</xs:complexType>
<xs:complexType name="OUTPUTEVENTS">
<xs:sequence minOccurs="1" maxOccurs="1">
@@ -101,7 +101,7 @@
<xs:element name="instance" type="xs:string" minOccurs="1"
maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="name" type="coordinator:IDENTIFIER"
use="required"/>
- <xs:attribute name="dataset" type="coordinator:IDENTIFIER"
use="required"/>
+ <xs:attribute name="dataset" type="xs:string" use="required"/>
</xs:complexType>
<xs:complexType name="ACTION">
<xs:sequence minOccurs="1" maxOccurs="1">
Modified: oozie/trunk/client/src/main/resources/oozie-coordinator-0.3.xsd
URL:
http://svn.apache.org/viewvc/oozie/trunk/client/src/main/resources/oozie-coordinator-0.3.xsd?rev=1541196&r1=1541195&r2=1541196&view=diff
==============================================================================
--- oozie/trunk/client/src/main/resources/oozie-coordinator-0.3.xsd (original)
+++ oozie/trunk/client/src/main/resources/oozie-coordinator-0.3.xsd Tue Nov 12
18:58:12 2013
@@ -89,7 +89,7 @@
</xs:sequence>
</xs:choice>
<xs:attribute name="name" type="coordinator:IDENTIFIER"
use="required"/>
- <xs:attribute name="dataset" type="coordinator:IDENTIFIER"
use="required"/>
+ <xs:attribute name="dataset" type="xs:string" use="required"/>
</xs:complexType>
<xs:complexType name="OUTPUTEVENTS">
<xs:sequence minOccurs="1" maxOccurs="1">
@@ -101,7 +101,7 @@
<xs:element name="instance" type="xs:string" minOccurs="1"
maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="name" type="coordinator:IDENTIFIER"
use="required"/>
- <xs:attribute name="dataset" type="coordinator:IDENTIFIER"
use="required"/>
+ <xs:attribute name="dataset" type="xs:string" use="required"/>
</xs:complexType>
<xs:complexType name="ACTION">
<xs:sequence minOccurs="1" maxOccurs="1">
Modified: oozie/trunk/client/src/main/resources/oozie-coordinator-0.4.xsd
URL:
http://svn.apache.org/viewvc/oozie/trunk/client/src/main/resources/oozie-coordinator-0.4.xsd?rev=1541196&r1=1541195&r2=1541196&view=diff
==============================================================================
--- oozie/trunk/client/src/main/resources/oozie-coordinator-0.4.xsd (original)
+++ oozie/trunk/client/src/main/resources/oozie-coordinator-0.4.xsd Tue Nov 12
18:58:12 2013
@@ -103,7 +103,7 @@
</xs:sequence>
</xs:choice>
<xs:attribute name="name" type="coordinator:IDENTIFIER"
use="required"/>
- <xs:attribute name="dataset" type="coordinator:IDENTIFIER"
use="required"/>
+ <xs:attribute name="dataset" type="xs:string" use="required"/>
</xs:complexType>
<xs:complexType name="OUTPUTEVENTS">
<xs:sequence minOccurs="1" maxOccurs="1">
@@ -115,7 +115,7 @@
<xs:element name="instance" type="xs:string" minOccurs="1"
maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="name" type="coordinator:IDENTIFIER"
use="required"/>
- <xs:attribute name="dataset" type="coordinator:IDENTIFIER"
use="required"/>
+ <xs:attribute name="dataset" type="xs:string" use="required"/>
</xs:complexType>
<xs:complexType name="ACTION">
<xs:sequence minOccurs="1" maxOccurs="1">
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java?rev=1541196&r1=1541195&r2=1541196&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
Tue Nov 12 18:58:12 2013
@@ -456,6 +456,7 @@ public class CoordSubmitXCommand extends
String appPath =
ParamChecker.notEmpty(conf.get(OozieClient.COORDINATOR_APP_PATH),
OozieClient.COORDINATOR_APP_PATH);
String coordXml = readDefinition(appPath);
+
validateXml(coordXml);
return coordXml;
}
@@ -730,6 +731,7 @@ public class CoordSubmitXCommand extends
// for each data set
resolveDataSets(eAppXml);
HashMap<String, String> dataNameList = new HashMap<String, String>();
+ resolveIODataset(eAppXml);
resolveIOEvents(eAppXml, dataNameList);
resolveTagContents("app-path", eAppXml.getChild("action",
eAppXml.getNamespace()).getChild("workflow",
@@ -804,6 +806,7 @@ public class CoordSubmitXCommand extends
resolveTagContents("instance", dataIn, evalInst);
resolveTagContents("start-instance", dataIn, evalInst);
resolveTagContents("end-instance", dataIn, evalInst);
+
}
}
// Resolving output-events/data-out
@@ -821,12 +824,41 @@ public class CoordSubmitXCommand extends
eventNameSet.add(dataOutName);
}
resolveTagContents("instance", dataOut, evalInst);
+
}
}
}
/**
+ * Resolve input-events/dataset and output-events/dataset tags.
+ *
+ * @param eJob : Job element
+ * @throws CoordinatorJobException thrown if failed to resolve input and
output events
+ */
+ @SuppressWarnings("unchecked")
+ private void resolveIODataset(Element eAppXml) throws
CoordinatorJobException {
+ // Resolving input-events/data-in
+ Element inputList = eAppXml.getChild("input-events",
eAppXml.getNamespace());
+ if (inputList != null) {
+ for (Element dataIn : (List<Element>)
inputList.getChildren("data-in", eAppXml.getNamespace())) {
+ resolveAttribute("dataset", dataIn, evalInst);
+
+ }
+ }
+ // Resolving output-events/data-out
+ Element outputList = eAppXml.getChild("output-events",
eAppXml.getNamespace());
+ if (outputList != null) {
+ for (Element dataOut : (List<Element>)
outputList.getChildren("data-out", eAppXml.getNamespace())) {
+ resolveAttribute("dataset", dataOut, evalInst);
+
+ }
+ }
+
+ }
+
+
+ /**
* Add an attribute into XML element.
*
* @param attrName :attribute name
Modified:
oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java?rev=1541196&r1=1541195&r2=1541196&view=diff
==============================================================================
---
oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
(original)
+++
oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
Tue Nov 12 18:58:12 2013
@@ -479,6 +479,41 @@ public class TestCoordSubmitXCommand ext
}
}
+ public void testELdataIO_xsd_4() throws Exception {
+ Configuration conf = new XConfiguration();
+ String appPath = "file://" + getTestCaseDir() + File.separator +
"coordinator.xml";
+
+
+ CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
+ Reader reader = IOUtils.getResourceAsReader("coord-el-dataset-4.xml",
-1);
+ Writer writer = new FileWriter(new URI(appPath).getPath());
+
+ IOUtils.copyCharStream(reader, writer);
+ conf.set(OozieClient.COORDINATOR_APP_PATH, appPath);
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ conf.set("nameNode", "hdfs://localhost:9000");
+ conf.set("queueName", "default");
+ conf.set("jobTracker", "localhost:9001");
+
+ conf.set("examplesRoot", "examples");
+ conf.set("aggregated_logs", "aggregated-logs");
+ conf.set("raw_logs", "raw-logs");
+
+
+ sc = new CoordSubmitXCommand(conf);
+
+ try {
+ sc.call();
+ }
+ catch (CommandException e) {
+ e.printStackTrace();
+ fail("should not throw exception " + e.getMessage());
+ }
+ }
+
+
+
+
/**
* Basic coordinator submit test with bundleId
*
Added: oozie/trunk/core/src/test/resources/coord-el-dataset-4.xml
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/test/resources/coord-el-dataset-4.xml?rev=1541196&view=auto
==============================================================================
--- oozie/trunk/core/src/test/resources/coord-el-dataset-4.xml (added)
+++ oozie/trunk/core/src/test/resources/coord-el-dataset-4.xml Tue Nov 12
18:58:12 2013
@@ -0,0 +1,79 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<coordinator-app name="aggregator-coord" frequency="${coord:hours(1)}"
start="2009-02-01T01:00Z" end="2009-02-03T23:59Z"
+ timezone="UTC"
+ xmlns="uri:oozie:coordinator:0.4">
+ <controls>
+ <concurrency>1</concurrency>
+ </controls>
+
+ <datasets>
+ <dataset name="raw-logs" frequency="${coord:minutes(20)}"
initial-instance="2010-01-01T00:00Z" timezone="UTC">
+ <uri-template>
+
${nameNode}/user/${coord:user()}/${examplesRoot}/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}
+ </uri-template>
+ </dataset>
+ <dataset name="aggregated-logs" frequency="${coord:hours(1)}"
initial-instance="2010-01-01T01:00Z" timezone="UTC">
+ <uri-template>
+
${nameNode}/user/${coord:user()}/${examplesRoot}/output-data/aggregator/aggregatedLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}
+ </uri-template>
+ </dataset>
+ </datasets>
+
+ <input-events>
+ <data-in name="input" dataset="${raw_logs}">
+ <start-instance>${coord:current(-2)}</start-instance>
+ <end-instance>${coord:current(0)}</end-instance>
+ </data-in>
+ </input-events>
+
+ <output-events>
+ <data-out name="output" dataset="${aggregated_logs}">
+ <instance>${coord:current(0)}</instance>
+ </data-out>
+ </output-events>
+
+ <action>
+ <workflow>
+
<app-path>${nameNode}/user/${coord:user()}/${examplesRoot}/apps/aggregator</app-path>
+ <configuration>
+ <property>
+ <name>jobTracker</name>
+ <value>${jobTracker}</value>
+ </property>
+ <property>
+ <name>nameNode</name>
+ <value>${nameNode}</value>
+ </property>
+ <property>
+ <name>queueName</name>
+ <value>${queueName}</value>
+ </property>
+ <property>
+ <name>inputData</name>
+ <value>${coord:dataIn('input')}</value>
+ </property>
+ <property>
+ <name>outputData</name>
+ <value>${coord:dataOut('output')}</value>
+ </property>
+ </configuration>
+ </workflow>
+ </action>
+</coordinator-app>
+
Modified: oozie/trunk/release-log.txt
URL:
http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1541196&r1=1541195&r2=1541196&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Tue Nov 12 18:58:12 2013
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1544 Support variables for coord data-in/data-out dataset (puru via
rohini)
OOZIE-1603 cannot submit job to oozie on mysql (virag)
OOZIE-1600 map-reduce actions without configuration section in workflow.xml
throws "IllegalArgumentException: element cannot be null" (bowenzhangusa via
rkanter)
OOZIE-1580 EL variables don't get resolved in configurations imported from a
<job-xml> (bowenzhangusa via rkanter)
