Repository: oozie Updated Branches: refs/heads/master 5abd3e6a5 -> 81ce22b6f
http://git-wip-us.apache.org/repos/asf/oozie/blob/81ce22b6/core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java b/core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java new file mode 100644 index 0000000..622e57f --- /dev/null +++ b/core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java @@ -0,0 +1,367 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.coord.input.logic; + + +import junit.framework.TestCase; + +import org.apache.oozie.coord.input.logic.InputLogicParser; +import org.apache.oozie.util.XmlUtils; +import org.jdom.Element; +import org.jdom.JDOMException; + +public class TestInputLogicParser extends TestCase { + + public void testAndOr() throws JDOMException { + //@formatter:off + String xml = + "<input-logic>" + + "<and>" + + "<or>" + + "<data-in dataset=\"A\"/> " + + "<data-in dataset=\"B\"/> " + + "</or>" + + "<or>" + + "<data-in dataset=\"C\"/>" + + "<data-in dataset=\"D\"/>" + + "</or>" + + "</and>" + + "</input-logic>"; + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals("((dependencyBuilder.input(\"A\").build() || dependencyBuilder.input(\"B\").build()) && " + + "(dependencyBuilder.input(\"C\").build() || dependencyBuilder.input(\"D\").build()))", + inputLogicParser.parse(root)); + + } + + public void testAnd() throws JDOMException { + //@formatter:off + String xml = + "<input-logic>" + + "<and>" + + "<data-in dataset=\"A\"/> " + + "<data-in dataset=\"B\"/>" + + "</and>" + + "</input-logic>"; + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals("(dependencyBuilder.input(\"A\").build() && dependencyBuilder.input(\"B\").build())", + inputLogicParser.parse(root)); + + } + + public void testOr() throws JDOMException { + //@formatter:off + String xml = + "<input-logic>" + + "<or>" + + "<data-in dataset=\"A\"/> " + + "<data-in dataset=\"B\"/>" + + "</or>" + + "</input-logic>"; + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals("(dependencyBuilder.input(\"A\").build() || dependencyBuilder.input(\"B\").build())", + inputLogicParser.parse(root)); + + } + + public void testOrWithMin() throws JDOMException { + //@formatter:off + String xml = "<input-logic>" + "<or>" + "<data-in dataset=\"A\" min=\"3\"/> " + "<data-in dataset=\"B\"/>" + "</or>" + + "</input-logic>"; + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals("(dependencyBuilder.input(\"A\").min(3).build() || dependencyBuilder.input(\"B\").build())", + inputLogicParser.parse(root)); + } + + public void testOrWithMinAtOr() throws JDOMException { + //@formatter:off + String xml = + "<input-logic>" + + "<or min=\"10\">" + + "<data-in dataset=\"A\"/> " + + "<data-in dataset=\"B\"/>" + + "</or>" + + "</input-logic>"; + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals( + "(dependencyBuilder.input(\"A\").min(10).build() || dependencyBuilder.input(\"B\").min(10).build())", + inputLogicParser.parse(root)); + } + + public void testWithName() throws JDOMException { + //@formatter:off + String xml = + "<input-logic>" + + "<or name =\"test\" min=\"10\">" + + "<data-in dataset=\"A\"/> " + + "<data-in dataset=\"B\"/>" + + "</or>" + + "</input-logic>"; + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals( + "(dependencyBuilder.input(\"A\").min(10).build() || dependencyBuilder.input(\"B\").min(10).build())", + inputLogicParser.parseWithName(root, "test")); + } + + public void testCombine() throws JDOMException { + //@formatter:off + String xml = + "<input-logic>" + + "<combine name =\"test\" min=\"10\">" + + "<data-in dataset=\"A\"/> " + + "<data-in dataset=\"B\"/>" + + "</combine>" + + "</input-logic>"; + + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals("(dependencyBuilder.combine(\"A\",\"B\").min(10).build())", + inputLogicParser.parseWithName(root, "test")); + } + + public void testWithNameNested() throws JDOMException { + //@formatter:off + String xml = + "<input-logic>" + + "<and>" + + "<or>" + + "<data-in dataset=\"A\"/> " + + "<data-in dataset=\"B\"/> " + + "</or>" + + "<or name=\"test\">" + + "<data-in dataset=\"C\"/>" + + "<data-in dataset=\"D\"/>" + + "</or>" + + "</and>" + + "</input-logic>"; + //@formatter:on + + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals("(dependencyBuilder.input(\"C\").build() || dependencyBuilder.input(\"D\").build())", + inputLogicParser.parseWithName(root, "test")); + + } + + public void testDepth2() throws JDOMException { + //@formatter:off + String xml = + "<input-logic>" + + "<and>" + + "<and>" + + "<or>" + + "<data-in dataset=\"A\"/>" + + "<data-in dataset=\"B\"/>" + + "</or>" + + "<or>" + + "<data-in dataset=\"C\"/>" + + "<data-in dataset=\"D\"/>" + + "</or>" + + "</and>" + + "<and>" + + "<data-in dataset=\"E\"/>" + + "<data-in dataset=\"F\"/>" + + "</and>" + + "</and>" + + "</input-logic>"; + //@formatter:on + + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals("(((dependencyBuilder.input(\"A\").build() || dependencyBuilder.input(\"B\").build())" + + " && (dependencyBuilder.input(\"C\").build() || dependencyBuilder.input(\"D\").build()))" + + " && (dependencyBuilder.input(\"E\").build() && dependencyBuilder.input(\"F\").build()))", + inputLogicParser.parse(root)); + + } + + public void testDepth2WithCombine() throws JDOMException { + //@formatter:off + String xml = + "<input-logic>" + + "<and>" + + "<and>" + + "<combine>" + + "<data-in dataset=\"A\" />" + + "<data-in dataset=\"B\" />" + + "</combine>" + + "<or>" + + "<data-in dataset=\"C\" />" + + "<data-in dataset=\"D\" />" + + "</or>" + + "</and>" + + "<combine>" + + "<data-in dataset=\"E\" />" + + "<data-in dataset=\"F\" />" + + "</combine>" + + "</and>" + + "</input-logic>"; + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals("(((dependencyBuilder.combine(\"A\",\"B\").build()) && (dependencyBuilder.input(\"C\").build()" + + " || dependencyBuilder.input(\"D\").build())) && (dependencyBuilder.combine(\"E\",\"F\").build()))", + inputLogicParser.parse(root)); + } + + public void testAndCombine() throws JDOMException { + //@formatter:off + String xml = + "<input-logic>" + + "<and>" + + "<combine>" + + "<data-in dataset=\"A\" />" + + "<data-in dataset=\"B\" />"+ + "</combine>" + + "<combine>" + + "<data-in dataset=\"C\" />" + + "<data-in dataset=\"D\" />" + + "</combine>" + + "</and>" + + "</input-logic>"; + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals( + "((dependencyBuilder.combine(\"A\",\"B\").build()) && (dependencyBuilder.combine(\"C\",\"D\").build()))", + inputLogicParser.parse(root)); + } + + public void testComplex1() throws JDOMException { + //@formatter:off + String xml= + "<input-logic>"+ + "<and name=\"test\">"+ + "<or>"+ + "<and>" + + "<data-in dataset=\"A\" />"+ + "<data-in dataset=\"B\" />"+ + "</and>" + + "<and>"+ + "<data-in dataset=\"C\" />"+ + "<data-in dataset=\"D\" />"+ + "</and>"+ + "</or>"+ + "<and>"+ + "<data-in dataset=\"A\" />"+ + "<data-in dataset=\"B\" />"+ + "</and>"+ + "</and>"+ + "</input-logic>"; + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals("(((dependencyBuilder.input(\"A\").build() && dependencyBuilder.input(\"B\").build())" + + " || (dependencyBuilder.input(\"C\").build() && dependencyBuilder.input(\"D\").build()))" + + " && (dependencyBuilder.input(\"A\").build() && dependencyBuilder.input(\"B\").build()))", + inputLogicParser.parse(root)); + } + + public void testAllAnd() throws JDOMException { + //@formatter:off + String xml= + "<input-logic>"+ + "<and name=\"test\">"+ + "<data-in dataset=\"A\" />"+ + "<data-in dataset=\"B\" />"+ + "<data-in dataset=\"C\" />"+ + "<data-in dataset=\"D\" />"+ + "<data-in dataset=\"E\" />"+ + "<data-in dataset=\"F\" />"+ + "</and>"+ + "</input-logic>"; + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals("(dependencyBuilder.input(\"A\").build() && dependencyBuilder.input(\"B\").build() && " + + "dependencyBuilder.input(\"C\").build() && dependencyBuilder.input(\"D\").build() && " + + "dependencyBuilder.input(\"E\").build() && dependencyBuilder.input(\"F\").build())", + inputLogicParser.parse(root)); + } + + public void testDataIn() throws JDOMException { + //@formatter:off + String xml= + "<input-logic>"+ + "<data-in dataset=\"A\" />"+ + "</input-logic>"; + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals("dependencyBuilder.input(\"A\").build()", inputLogicParser.parse(root)); + } + + public void testMinWait() throws JDOMException { + //@formatter:off + String xml = + "<input-logic>" + + "<and name=\"test\" min=\"3\" wait=\"10\">" + + "<data-in dataset=\"A\"/> " + + "<data-in dataset=\"B\"/>" + + "</and>" + + "</input-logic>"; + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals("(dependencyBuilder.input(\"A\").min(3).inputWait(10).build() " + + "&& dependencyBuilder.input(\"B\").min(3).inputWait(10).build())", + inputLogicParser.parseWithName(root, "test")); + + assertEquals("(dependencyBuilder.input(\"A\").min(3).inputWait(10).build() " + + "&& dependencyBuilder.input(\"B\").min(3).inputWait(10).build())", inputLogicParser.parse(root)); + } + + public void testOrAndDataIn() throws JDOMException { + //@formatter:off + String xml = + "<input-logic>" + + "<or>" + + "<and>" + + "<data-in dataset=\"A\"/> " + + "<data-in dataset=\"B\"/>" + + "</and>" + + "<data-in dataset=\"C\"/>" + + "</or>"+ + "</input-logic>"; + //@formatter:on + Element root = XmlUtils.parseXml(xml); + InputLogicParser inputLogicParser = new InputLogicParser(); + assertEquals( + "((dependencyBuilder.input(\"A\").build() && dependencyBuilder.input(\"B\").build()) || " + + "dependencyBuilder.input(\"C\").build())", + inputLogicParser.parse(root)); + + } + + + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/81ce22b6/core/src/test/resources/coord-action-sla.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-action-sla.xml b/core/src/test/resources/coord-action-sla.xml index f3f1bc0..7df8c80 100644 --- a/core/src/test/resources/coord-action-sla.xml +++ b/core/src/test/resources/coord-action-sla.xml @@ -23,7 +23,7 @@ <datasets> <dataset name="a" frequency="7" initial-instance="2009-01-01T01:00Z" timezone="UTC"> - <uri-template>file://#testDir/${YEAR}/${DAY}</uri-template> + <uri-template>file:///testDir/${YEAR}/${DAY}</uri-template> </dataset> </datasets> <input-events> http://git-wip-us.apache.org/repos/asf/oozie/blob/81ce22b6/core/src/test/resources/coord-inputlogic-combine.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-inputlogic-combine.xml b/core/src/test/resources/coord-inputlogic-combine.xml new file mode 100644 index 0000000..2cd4bd5 --- /dev/null +++ b/core/src/test/resources/coord-inputlogic-combine.xml @@ -0,0 +1,119 @@ +<!-- /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ --> + +<coordinator-app name="aggregator-coord" frequency="${coord:days(1)}" + start="${start_time}" end="${end_time}" timezone="UTC" + xmlns="uri:oozie:coordinator:0.5"> + <controls> + <concurrency>1</concurrency> + <throttle>1</throttle> + </controls> + + <datasets> + <dataset name="a" frequency="${coord:days(1)}" + initial-instance="${initial_instance}" timezone="UTC"> + <uri-template>${data_set}/${db_a}/${table}/dt=${YEAR}${MONTH}${DAY};country=usa + </uri-template> + </dataset> + <dataset name="b" frequency="${coord:days(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_b}/${YEAR}/${MONTH}/${DAY} + </uri-template> + </dataset> + <dataset name="c" frequency="${coord:days(1)}" + initial-instance="${initial_instance}" timezone="UTC"> + <uri-template>${data_set}/${db_c}/${table}/dt=${YEAR}${MONTH}${DAY};country=usa + </uri-template> + </dataset> + <dataset name="d" frequency="${coord:days(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_d}/${YEAR}/${MONTH}/${DAY} + </uri-template> + </dataset> + <dataset name="e" frequency="${coord:days(1)}" + initial-instance="${initial_instance}" timezone="UTC"> + <uri-template>${data_set}/${db_e}/${table}/dt=${YEAR}${MONTH}${DAY};country=usa + </uri-template> + </dataset> + <dataset name="f" frequency="${coord:days(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_f}/${YEAR}/${MONTH}/${DAY} + </uri-template> + </dataset> + + <dataset name="aggregated-logs" frequency="${coord:days(10)}" + initial-instance="2014-10-08T00:00Z" timezone="UTC"> + <uri-template>file:///output-data/aggregator/aggregatedLogs/${YEAR}/${MONTH}/${DAY} + </uri-template> + </dataset> + </datasets> + + <input-events> + <data-in name="A" dataset="a"> + =data-in-param-1= + </data-in> + <data-in name="B" dataset="b"> + =data-in-param-2= + </data-in> + <data-in name="C" dataset="c"> + =data-in-param-3= + </data-in> + <data-in name="D" dataset="d"> + =data-in-param-4= + </data-in> + <data-in name="E" dataset="e"> + =data-in-param-5= + </data-in> + <data-in name="F" dataset="f"> + =data-in-param-6= + </data-in> + </input-events> + + <input-logic> + =input-logic= + </input-logic> + + <output-events> + <data-out name="output" dataset="aggregated-logs"> + <instance>${coord:current(0)}</instance> + </data-out> + </output-events> + <action> + <workflow> + <app-path>hdfs:///tmp/workflows</app-path> + <configuration> + <property> + <name>jobTracker</name> + <value>${jobTracker}</value> + </property> + <property> + <name>nameNode</name> + <value>${nameNode}</value> + </property> + <property> + <name>queueName</name> + <value>${queueName}</value> + </property> + <property> + <name>inputLogicData</name> + <value>${coord:dataIn(partitionName)}</value> + </property> + </configuration> + </workflow> + </action> +</coordinator-app> http://git-wip-us.apache.org/repos/asf/oozie/blob/81ce22b6/core/src/test/resources/coord-inputlogic-hcat.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-inputlogic-hcat.xml b/core/src/test/resources/coord-inputlogic-hcat.xml new file mode 100644 index 0000000..ff260d3 --- /dev/null +++ b/core/src/test/resources/coord-inputlogic-hcat.xml @@ -0,0 +1,119 @@ +<!-- /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ --> + <coordinator-app name="aggregator-coord" frequency="${coord:days(1)}" + start="${start_time}" end="${end_time}" timezone="UTC" + xmlns="uri:oozie:coordinator:0.5"> + <controls> + <concurrency>1</concurrency> + <throttle>1</throttle> + </controls> + + <datasets> + <dataset name="a" frequency="${coord:days(1)}" + initial-instance="${initial_instance}" timezone="UTC"> + <uri-template>${data_set}/${db_a}/${table}/dt=${YEAR}${MONTH}${DAY};country=usa + </uri-template> + </dataset> + <dataset name="b" frequency="${coord:days(1)}" + initial-instance="${initial_instance}" timezone="UTC"> + <uri-template>${data_set}/${db_b}/${table}/dt=${YEAR}${MONTH}${DAY};country=usa + </uri-template> + </dataset> + <dataset name="c" frequency="${coord:days(1)}" + initial-instance="${initial_instance}" timezone="UTC"> + <uri-template>${data_set}/${db_c}/${table}/dt=${YEAR}${MONTH}${DAY};country=usa + </uri-template> + </dataset> + <dataset name="d" frequency="${coord:days(1)}" + initial-instance="${initial_instance}" timezone="UTC"> + <uri-template>${data_set}/${db_d}/${table}/dt=${YEAR}${MONTH}${DAY};country=usa + </uri-template> + </dataset> + <dataset name="e" frequency="${coord:days(1)}" + initial-instance="${initial_instance}" timezone="UTC"> + <uri-template>${data_set}/${db_e}/${table}/dt=${YEAR}${MONTH}${DAY};country=usa + </uri-template> + </dataset> + <dataset name="f" frequency="${coord:days(1)}" + initial-instance="${initial_instance}" timezone="UTC"> + <uri-template>${data_set}/${db_f}/${table}/dt=${YEAR}${MONTH}${DAY};country=usa + </uri-template> + </dataset> + + <dataset name="aggregated-logs" frequency="${coord:days(10)}" + initial-instance="2014-10-08T00:00Z" timezone="UTC"> + <uri-template>file:///output-data/aggregator/aggregatedLogs/${YEAR}/${MONTH}/${DAY} + </uri-template> + </dataset> + </datasets> + + <input-events> + <data-in name="A" dataset="a"> + =data-in-param-1= + </data-in> + <data-in name="B" dataset="b"> + =data-in-param-2= + </data-in> + <data-in name="C" dataset="c"> + =data-in-param-3= + </data-in> + <data-in name="D" dataset="d"> + =data-in-param-4= + </data-in> + <data-in name="E" dataset="e"> + =data-in-param-5= + </data-in> + <data-in name="F" dataset="f"> + =data-in-param-6= + </data-in> + </input-events> + + <input-logic> + =input-logic= + </input-logic> + + <output-events> + <data-out name="output" dataset="aggregated-logs"> + <instance>${coord:current(0)}</instance> + </data-out> + </output-events> + <action> + <workflow> + <app-path>hdfs:///tmp/workflows</app-path> + <configuration> + <property> + <name>jobTracker</name> + <value>${jobTracker}</value> + </property> + <property> + <name>nameNode</name> + <value>${nameNode}</value> + </property> + <property> + <name>queueName</name> + <value>${queueName}</value> + </property> + <property> + <name>inputLogicData</name> + <value>${coord:dataIn(partitionName)}</value> + </property> + + </configuration> + </workflow> + </action> +</coordinator-app> http://git-wip-us.apache.org/repos/asf/oozie/blob/81ce22b6/core/src/test/resources/coord-inputlogic-latest.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-inputlogic-latest.xml b/core/src/test/resources/coord-inputlogic-latest.xml new file mode 100644 index 0000000..576f00d --- /dev/null +++ b/core/src/test/resources/coord-inputlogic-latest.xml @@ -0,0 +1,124 @@ +<!-- /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ --> + +<coordinator-app name="aggregator-coord" frequency="${coord:hours(1)}" + start="${start_time}" end="${end_time}" timezone="UTC" + xmlns="uri:oozie:coordinator:0.5"> + <controls> + <concurrency>1</concurrency> + <throttle>1</throttle> + </controls> + + <datasets> + <dataset name="a" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_a}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + <dataset name="b" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_b}" timezone="UTC"> + <uri-template>${data_set_b}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + <dataset name="c" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_c}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + <dataset name="d" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_d}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + <dataset name="e" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_e}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + <dataset name="f" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_f}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + + <dataset name="aggregated-logs" frequency="${coord:hours(10)}" + initial-instance="2014-10-08T00:00Z" timezone="UTC"> + <uri-template>file:///output-data/aggregator/aggregatedLogs/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + </datasets> + + <input-events> + <data-in name="A" dataset="a"> + <instance>${coord:latest(0)}</instance> + </data-in> + <data-in name="B" dataset="b"> + <instance>${coord:latest(0)}</instance> + </data-in> + <data-in name="C" dataset="c"> + <instance>${coord:latest(0)}</instance> + </data-in> + <data-in name="D" dataset="d"> + <instance>${coord:latest(0)}</instance> + </data-in> + <data-in name="E" dataset="e"> + <instance>${coord:latest(0)}</instance> + </data-in> + <data-in name="F" dataset="f"> + <instance>${coord:latest(0)}</instance> + </data-in> + </input-events> + + <input-logic> + =input-logic= + </input-logic> + + <output-events> + <data-out name="output" dataset="aggregated-logs"> + <instance>${coord:current(0)}</instance> + </data-out> + </output-events> + + <action> + <workflow> + <app-path>${wfPath}</app-path> + <configuration> + <property> + <name>jobTracker</name> + <value>${jobTracker}</value> + </property> + <property> + <name>nameNode</name> + <value>${nameNode}</value> + </property> + <property> + <name>queueName</name> + <value>${queueName}</value> + </property> + <property> + <name>inputLogicData</name> + <value>${coord:dataIn(partitionName)}</value> + </property> + </configuration> + </workflow> + </action> +</coordinator-app> http://git-wip-us.apache.org/repos/asf/oozie/blob/81ce22b6/core/src/test/resources/coord-inputlogic-range-latest.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-inputlogic-range-latest.xml b/core/src/test/resources/coord-inputlogic-range-latest.xml new file mode 100644 index 0000000..f9d79c2 --- /dev/null +++ b/core/src/test/resources/coord-inputlogic-range-latest.xml @@ -0,0 +1,130 @@ +<!-- /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ --> + +<coordinator-app name="aggregator-coord" frequency="${coord:hours(1)}" + start="${start_time}" end="${end_time}" timezone="UTC" + xmlns="uri:oozie:coordinator:0.5"> + <controls> + <concurrency>1</concurrency> + <throttle>1</throttle> + </controls> + + <datasets> + <dataset name="a" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_a}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + <dataset name="b" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_b}" timezone="UTC"> + <uri-template>${data_set_b}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + <dataset name="c" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_c}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + <dataset name="d" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_d}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + <dataset name="e" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_e}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + <dataset name="f" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_f}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + + <dataset name="aggregated-logs" frequency="${coord:hours(10)}" + initial-instance="2014-10-08T00:00Z" timezone="UTC"> + <uri-template>file:///output-data/aggregator/aggregatedLogs/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + </datasets> + + <input-events> + <data-in name="A" dataset="a"> + <start-instance>${coord:latest(-5)}</start-instance> + <end-instance>${coord:latest(0)}</end-instance> + </data-in> + <data-in name="B" dataset="b"> + <start-instance>${coord:latest(-5)}</start-instance> + <end-instance>${coord:latest(0)}</end-instance> + </data-in> + <data-in name="C" dataset="c"> + <start-instance>${coord:latest(-5)}</start-instance> + <end-instance>${coord:latest(0)}</end-instance> + </data-in> + <data-in name="D" dataset="d"> + <start-instance>${coord:latest(-5)}</start-instance> + <end-instance>${coord:latest(0)}</end-instance> + </data-in> + <data-in name="E" dataset="e"> + <start-instance>${coord:latest(-5)}</start-instance> + <end-instance>${coord:latest(0)}</end-instance> + </data-in> + <data-in name="F" dataset="f"> + <start-instance>${coord:latest(-5)}</start-instance> + <end-instance>${coord:latest(0)}</end-instance> + </data-in> + </input-events> + + <input-logic> + =input-logic= + </input-logic> + + <output-events> + <data-out name="output" dataset="aggregated-logs"> + <instance>${coord:current(0)}</instance> + </data-out> + </output-events> + + <action> + <workflow> + <app-path>${wfPath}</app-path> + <configuration> + <property> + <name>jobTracker</name> + <value>${jobTracker}</value> + </property> + <property> + <name>nameNode</name> + <value>${nameNode}</value> + </property> + <property> + <name>queueName</name> + <value>${queueName}</value> + </property> + <property> + <name>inputLogicData</name> + <value>${coord:dataIn(partitionName)}</value> + </property> + </configuration> + </workflow> + </action> +</coordinator-app> http://git-wip-us.apache.org/repos/asf/oozie/blob/81ce22b6/core/src/test/resources/coord-inputlogic-range.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-inputlogic-range.xml b/core/src/test/resources/coord-inputlogic-range.xml new file mode 100644 index 0000000..d6581e7 --- /dev/null +++ b/core/src/test/resources/coord-inputlogic-range.xml @@ -0,0 +1,107 @@ +<!-- /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ --> + +<coordinator-app name="aggregator-coord" frequency="${coord:hours(1)}" + start="${start_time}" end="${end_time}" timezone="UTC" + xmlns="uri:oozie:coordinator:0.5"> + <controls> + <concurrency>1</concurrency> + <throttle>1</throttle> + </controls> + + <datasets> + <dataset name="a" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_a}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + <dataset name="b" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_b}" timezone="UTC"> + <uri-template>${data_set_b}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + <dataset name="c" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_c}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + <dataset name="d" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_d}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + <dataset name="e" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_e}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + <dataset name="f" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_f}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + + <dataset name="aggregated-logs" frequency="${coord:hours(10)}" + initial-instance="2014-10-08T00:00Z" timezone="UTC"> + <uri-template>file:///output-data/aggregator/aggregatedLogs/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + </datasets> + + <input-events> + =input-events= + </input-events> + + <input-logic> + =input-logic= + </input-logic> + + <output-events> + <data-out name="output" dataset="aggregated-logs"> + <instance>${coord:current(0)}</instance> + </data-out> + </output-events> + + <action> + <workflow> + <app-path>${wfPath}</app-path> + <configuration> + <property> + <name>jobTracker</name> + <value>${jobTracker}</value> + </property> + <property> + <name>nameNode</name> + <value>${nameNode}</value> + </property> + <property> + <name>queueName</name> + <value>${queueName}</value> + </property> + <property> + <name>inputLogicData</name> + <value>${coord:dataIn(partitionName)}</value> + </property> + </configuration> + </workflow> + </action> +</coordinator-app> http://git-wip-us.apache.org/repos/asf/oozie/blob/81ce22b6/core/src/test/resources/coord-inputlogic.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-inputlogic.xml b/core/src/test/resources/coord-inputlogic.xml new file mode 100644 index 0000000..51b67ac --- /dev/null +++ b/core/src/test/resources/coord-inputlogic.xml @@ -0,0 +1,126 @@ +<!-- /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ --> + +<coordinator-app name="aggregator-coord" frequency="${coord:hours(1)}" + start="${start_time}" end="${end_time}" timezone="UTC" + xmlns="uri:oozie:coordinator:0.5"> + <controls> + <concurrency>1</concurrency> + <throttle>1</throttle> + </controls> + + <datasets> + <dataset name="a" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_a}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + <dataset name="b" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_b}" timezone="UTC"> + <uri-template>${data_set_b}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + <dataset name="c" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_c}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + <dataset name="d" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_d}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + <dataset name="e" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_e}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + <dataset name="f" frequency="${coord:hours(1)}" + initial-instance="${initial_instance_a}" timezone="UTC"> + <uri-template>${data_set_f}/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + + + <dataset name="aggregated-logs" frequency="${coord:hours(10)}" + initial-instance="2014-10-08T00:00Z" timezone="UTC"> + <uri-template>file:///output-data/aggregator/aggregatedLogs/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + </datasets> + + <input-events> + <data-in name="A" dataset="a"> + <instance>${coord:current(0)}</instance> + </data-in> + <data-in name="B" dataset="b"> + <instance>${coord:current(0)}</instance> + </data-in> + <data-in name="C" dataset="c"> + <instance>${coord:current(0)}</instance> + </data-in> + <data-in name="D" dataset="d"> + <instance>${coord:current(0)}</instance> + </data-in> + <data-in name="E" dataset="e"> + <instance>${coord:current(0)}</instance> + </data-in> + <data-in name="F" dataset="f"> + <instance>${coord:current(0)}</instance> + </data-in> + + + </input-events> + + <input-logic> + =input-logic= + </input-logic> + + <output-events> + <data-out name="output" dataset="aggregated-logs"> + <instance>${coord:current(0)}</instance> + </data-out> + </output-events> + + <action> + <workflow> + <app-path>${wfPath}</app-path> + <configuration> + <property> + <name>jobTracker</name> + <value>${jobTracker}</value> + </property> + <property> + <name>nameNode</name> + <value>${nameNode}</value> + </property> + <property> + <name>queueName</name> + <value>${queueName}</value> + </property> + <property> + <name>inputLogicData</name> + <value>${coord:dataIn(partitionName)}</value> + </property> + </configuration> + </workflow> + </action> +</coordinator-app> http://git-wip-us.apache.org/repos/asf/oozie/blob/81ce22b6/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index dc519cb..26f10a3 100644 --- a/pom.xml +++ b/pom.xml @@ -1267,6 +1267,13 @@ </dependency> <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-jexl</artifactId> + <version>2.1.1</version> + <scope>compile</scope> + </dependency> + + <dependency> <groupId>javax.mail</groupId> <artifactId>mail</artifactId> <version>1.4</version> http://git-wip-us.apache.org/repos/asf/oozie/blob/81ce22b6/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 5c2ee5b..9639b0c 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-1976 Specifying coordinator input datasets in more logical ways (puru) OOZIE-2444 Need conditional logic in bundles (satishsaley via puru) OOZIE-2394 Oozie can execute command without holding lock (puru) OOZIE-1922 MemoryLocksService fails if lock is acquired multiple times in same thread and released (puru)
