Thank you Virag!
On Mon, Feb 11, 2013 at 3:10 PM, <[email protected]> wrote: > Author: virag > Date: Mon Feb 11 23:10:20 2013 > New Revision: 1444990 > > URL: http://svn.apache.org/r1444990 > Log: > OOZIE-1211 oozie does not support duplicated dataset (jaoki via virag) > > Modified: > > oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java > > oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java > oozie/trunk/release-log.txt > > Modified: > oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java > URL: > http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java?rev=1444990&r1=1444989&r2=1444990&view=diff > > ============================================================================== > --- > oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java > (original) > +++ > oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java > Mon Feb 11 23:10:20 2013 > @@ -948,7 +948,7 @@ public class CoordSubmitXCommand extends > for (Element e : (List<Element>) > datasets.getChildren("dataset", datasets.getNamespace())) { > String dsName = e.getAttributeValue("name"); > if (dsList.contains(dsName)) {// Override with this DS > - // Remove old DS > + // Remove duplicate > removeDataSet(allDataSets, dsName); > } > else { > @@ -1017,6 +1017,7 @@ public class CoordSubmitXCommand extends > for (Element eDataset : (List<Element>) > eDatasets.getChildren("dataset", eDatasets.getNamespace())) { > if (eDataset.getAttributeValue("name").equals(name)) { > eDataset.detach(); > + return; > } > } > throw new RuntimeException("undefined dataset: " + name); > > Modified: > oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java > URL: > http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java?rev=1444990&r1=1444989&r2=1444990&view=diff > > ============================================================================== > --- > oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java > (original) > +++ > oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java > Mon Feb 11 23:10:20 2013 > @@ -24,6 +24,7 @@ import java.io.PrintWriter; > import java.io.Reader; > import java.io.Writer; > import java.net.URI; > +import java.util.List; > > import org.apache.hadoop.conf.Configuration; > import org.apache.oozie.BundleJobBean; > @@ -39,6 +40,9 @@ import org.apache.oozie.service.Services > import org.apache.oozie.test.XDataTestCase; > import org.apache.oozie.util.IOUtils; > import org.apache.oozie.util.XConfiguration; > +import org.apache.oozie.util.XmlUtils; > +import org.jdom.Element; > +import org.jdom.Namespace; > > public class TestCoordSubmitXCommand extends XDataTestCase { > > @@ -779,6 +783,142 @@ public class TestCoordSubmitXCommand ext > } > } > > + /** > + * Basic submit with include file > + * @throws Exception > + */ > + public void testBasicSubmitWithIncludeFile() throws Exception { > + Configuration conf = new XConfiguration(); > + final String includePath = "file://" + getTestCaseDir() + > File.separator + "include1.xml"; > + final String URI_TEMPLATE_INCLUDE_XML = > "file:///tmp/include_xml/workflows/${YEAR}/${DAY}"; > + final String URI_TEMPLATE_COORD_XML = > "file:///tmp/coord_xml/workflows/${YEAR}/${DAY}"; > + String includeXml = > + "<datasets> " > + + "<dataset name=\"A\" frequency=\"${coord:days(7)}\" > initial-instance=\"2009-02-01T01:00Z\" timezone=\"UTC\">" > + + "<uri-template>" + URI_TEMPLATE_INCLUDE_XML + > "</uri-template>" > + + "</dataset> " > + + "</datasets>"; > + writeToFile(includeXml, includePath); > + > + String appPath = "file://" + getTestCaseDir() + File.separator + > "coordinator.xml"; > + String appXml = > + "<coordinator-app name=\"${appName}-foo\" > frequency=\"${coord:days(1)}\" start=\"2009-02-01T01:00Z\" " > + + "end=\"2009-02-03T23:59Z\" timezone=\"UTC\" > xmlns=\"uri:oozie:coordinator:0.2\">" > + + "<controls> " > + + "<execution>LIFO</execution>" > + + "</controls>" > + + "<datasets> " > + + "<include>" + includePath + "</include>" > + + "<dataset name=\"B\" frequency=\"${coord:days(7)}\" > initial-instance=\"2009-02-01T01:00Z\" timezone=\"UTC\">" > + + "<uri-template>" + URI_TEMPLATE_COORD_XML + > "</uri-template>" > + + "</dataset> " > + + "</datasets>" > + + " <input-events> " > + + "<data-in name=\"inputA\" dataset=\"A\"> > <instance>${coord:latest(0)}</instance> </data-in> " > + + "<data-in name=\"inputB\" dataset=\"B\"> > <instance>${coord:latest(0)}</instance> </data-in> " > + + "</input-events> " > + + "<action>" > + + "<workflow>" > + + "<app-path>hdfs:///tmp/workflows/</app-path> " > + + "<configuration>" > + + "<property> <name>inputA</name> > <value>${coord:dataIn('inputB')}</value> </property> " > + + "</configuration>" > + + "</workflow>" > + + "</action>" > + + " </coordinator-app>"; > + writeToFile(appXml, appPath); > + conf.set(OozieClient.COORDINATOR_APP_PATH, appPath); > + conf.set(OozieClient.USER_NAME, getTestUser()); > + conf.set("appName", "var-app-name"); > + CoordSubmitXCommand sc = new CoordSubmitXCommand(conf, > "UNIT_TESTING"); > + String jobId = sc.call(); > + > + assertEquals(jobId.substring(jobId.length() - 2), "-C"); > + CoordinatorJobBean job = checkCoordJobs(jobId); > + assertNotNull(job); > + Element processedJobXml = XmlUtils.parseXml(job.getJobXml()); > + > + Namespace namespace = processedJobXml.getNamespace(); > + @SuppressWarnings("unchecked") > + List<Element> datainElements = > processedJobXml.getChild("input-events", namespace).getChildren("data-in", > namespace); > + assertTrue("<data-in> should be 2. One from coordinator.xml and > the other from the include file" > + , datainElements.size() == 2); > + > + assertEquals(URI_TEMPLATE_INCLUDE_XML > + , datainElements.get(0).getChild("dataset", > namespace).getChildText("uri-template", namespace)); > + assertEquals(URI_TEMPLATE_COORD_XML > + , datainElements.get(1).getChild("dataset", > namespace).getChildText("uri-template", namespace)); > + } > + > + /** > + * https://issues.apache.org/jira/browse/OOZIE-1211 > + * If a datasets include file has a dataset name as in one defined in > coordinator.xml, > + * the one in coordinator.xml should be honored. > + * > http://oozie.apache.org/docs/3.3.1/CoordinatorFunctionalSpec.html#a10.1.1._Dataset_Names_Collision_Resolution > + * > + * @throws Exception > + */ > + public void testDuplicateDatasetNameInIncludeFile() throws Exception { > + Configuration conf = new XConfiguration(); > + final String includePath = "file://" + getTestCaseDir() + > File.separator + "include1.xml"; > + final String URI_TEMPLATE_INCLUDE_XML = > "file:///tmp/include_xml/workflows/${YEAR}/${DAY}"; > + final String URI_TEMPLATE_COORD_XML = > "file:///tmp/coord_xml/workflows/${YEAR}/${DAY}"; > + String includeXml = > + "<datasets> " > + + "<dataset name=\"B\" frequency=\"${coord:days(7)}\" > initial-instance=\"2009-02-01T01:00Z\" timezone=\"UTC\">" > + + "<uri-template>" + URI_TEMPLATE_INCLUDE_XML + > "</uri-template>" > + + "</dataset> " > + + "</datasets>"; > + writeToFile(includeXml, includePath); > + > + String appPath = "file://" + getTestCaseDir() + File.separator + > "coordinator.xml"; > + String appXml = > + "<coordinator-app name=\"${appName}-foo\" > frequency=\"${coord:days(1)}\" start=\"2009-02-01T01:00Z\" " > + + "end=\"2009-02-03T23:59Z\" timezone=\"UTC\" > xmlns=\"uri:oozie:coordinator:0.2\">" > + + "<controls> " > + + "<execution>LIFO</execution>" > + + "</controls>" > + + "<datasets> " > + + "<include>" + includePath + "</include>" > + + "<dataset name=\"B\" frequency=\"${coord:days(7)}\" > initial-instance=\"2009-02-01T01:00Z\" timezone=\"UTC\">" > + + "<uri-template>" + URI_TEMPLATE_COORD_XML + > "</uri-template>" > + + "</dataset> " > + + "</datasets>" > + + " <input-events> " > + + "<data-in name=\"inputB\" dataset=\"B\"> > <instance>${coord:latest(0)}</instance> </data-in> " > + + "</input-events> " > + + "<action>" > + + "<workflow>" > + + "<app-path>hdfs:///tmp/workflows/</app-path> " > + + "<configuration>" > + + "<property> <name>inputB</name> > <value>${coord:dataIn('inputB')}</value> </property> " > + + "</configuration>" > + + "</workflow>" > + + "</action>" > + + " </coordinator-app>"; > + writeToFile(appXml, appPath); > + conf.set(OozieClient.COORDINATOR_APP_PATH, appPath); > + conf.set(OozieClient.USER_NAME, getTestUser()); > + conf.set("appName", "var-app-name"); > + CoordSubmitXCommand sc = new CoordSubmitXCommand(conf, > "UNIT_TESTING"); > + String jobId = sc.call(); > + > + assertEquals(jobId.substring(jobId.length() - 2), "-C"); > + CoordinatorJobBean job = checkCoordJobs(jobId); > + assertNotNull(job); > + Element processedJobXml = XmlUtils.parseXml(job.getJobXml()); > + > + Namespace namespace = processedJobXml.getNamespace(); > + @SuppressWarnings("unchecked") > + List<Element> datasetElements = > processedJobXml.getChild("input-events", namespace).getChild("data-in", > namespace) > + .getChildren("dataset", namespace); > + assertTrue("<dataset> should not be duplicate", > datasetElements.size() == 1); > + > + assertEquals(URI_TEMPLATE_COORD_XML, > datasetElements.get(0).getChildText("uri-template", namespace)); > + assertFalse("<uri-template> should not contain one from the > include file" > + , job.getJobXml().contains(URI_TEMPLATE_INCLUDE_XML)); > + } > + > private void _testConfigDefaults(boolean withDefaults) throws > Exception { > Configuration conf = new XConfiguration(); > String appPath = "file://" + getTestCaseDir() + File.separator + > "coordinator.xml"; > > Modified: oozie/trunk/release-log.txt > URL: > http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1444990&r1=1444989&r2=1444990&view=diff > > ============================================================================== > --- oozie/trunk/release-log.txt (original) > +++ oozie/trunk/release-log.txt Mon Feb 11 23:10:20 2013 > @@ -1,5 +1,6 @@ > -- Oozie 3.4.0 release (trunk - unreleased) > > +OOZIE-1211 oozie does not support duplicated dataset (jaoki via virag) > OOZIE-1187 reduce memory usage of SLA query (invoked by CLI command) to > avoid OOM (egashira via virag) > OOZIE-1096 Update wfgen README.txt to have the TLP mailing list (jun aoki > via rkanter) > OOZIE-1015 HadoopAccessorService jobtracker validation should not have > hardcoded conf key (mona) > > >
