Repository: oozie Updated Branches: refs/heads/master b5c1137ba -> 2d18d935d
OOZIE-2308 Add support for bundle:conf() function (kailongs via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/2d18d935 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/2d18d935 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/2d18d935 Branch: refs/heads/master Commit: 2d18d935d6661219613e49dc781a56da1de930b1 Parents: b5c1137 Author: Rohini Palaniswamy <[email protected]> Authored: Tue Jul 28 10:17:43 2015 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Tue Jul 28 10:17:43 2015 -0700 ---------------------------------------------------------------------- .../apache/oozie/bundle/BundleELFunctions.java | 29 ++++++++++ .../command/bundle/BundleSubmitXCommand.java | 24 ++++++-- core/src/main/resources/oozie-default.xml | 7 ++- .../bundle/TestBundleSubmitXCommand.java | 3 +- .../org/apache/oozie/test/XDataTestCase.java | 1 + .../bundle-submit-job-with-functions.xml | 60 ++++++++++++++++++++ docs/src/site/twiki/BundleFunctionalSpec.twiki | 15 ++--- release-log.txt | 1 + 8 files changed, 126 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/core/src/main/java/org/apache/oozie/bundle/BundleELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/bundle/BundleELFunctions.java b/core/src/main/java/org/apache/oozie/bundle/BundleELFunctions.java new file mode 100644 index 0000000..2c31ffa --- /dev/null +++ b/core/src/main/java/org/apache/oozie/bundle/BundleELFunctions.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.bundle; + +import org.apache.oozie.util.ELEvaluator; + + +public class BundleELFunctions { + public static String bundle_conf(String property) { + ELEvaluator eval = ELEvaluator.getCurrent(); + return (String) eval.getVariable(property); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java index 5d4e0f0..3df37fa 100644 --- a/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java @@ -45,12 +45,13 @@ import org.apache.oozie.command.CommandException; import org.apache.oozie.command.PreconditionException; import org.apache.oozie.command.SubmitTransitionXCommand; import org.apache.oozie.executor.jpa.BundleJobQueryExecutor; +import org.apache.oozie.service.ELService; import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.SchemaService; +import org.apache.oozie.service.SchemaService.SchemaName; import org.apache.oozie.service.Services; import org.apache.oozie.service.UUIDService; -import org.apache.oozie.service.SchemaService.SchemaName; import org.apache.oozie.service.UUIDService.ApplicationType; import org.apache.oozie.util.ConfigUtils; import org.apache.oozie.util.DateUtils; @@ -60,10 +61,10 @@ import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.InstrumentUtils; import org.apache.oozie.util.LogUtils; import org.apache.oozie.util.ParamChecker; +import org.apache.oozie.util.ParameterVerifier; import org.apache.oozie.util.PropertiesUtils; import org.apache.oozie.util.XConfiguration; import org.apache.oozie.util.XmlUtils; -import org.apache.oozie.util.ParameterVerifier; import org.jdom.Attribute; import org.jdom.Element; import org.jdom.JDOMException; @@ -136,7 +137,7 @@ public class BundleSubmitXCommand extends SubmitTransitionXCommand { } conf = resolvedVarsConf; - String resolvedJobXml = resolvedVars(jobXmlWithNoComment, conf); + String resolvedJobXml = resolvedVarsandFunctions(jobXmlWithNoComment, conf); //verify the uniqueness of coord names verifyCoordNameUnique(resolvedJobXml); @@ -414,6 +415,18 @@ public class BundleSubmitXCommand extends SubmitTransitionXCommand { return bundleBean; } + public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) { + ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group); + setConfigToEval(eval, conf); + return eval; + } + + private static void setConfigToEval(ELEvaluator eval, Configuration conf) { + for (Map.Entry<String, String> entry : conf) { + eval.setVariable(entry.getKey(), entry.getValue().trim()); + } + } + /** * Resolve job xml with conf * @@ -422,9 +435,10 @@ public class BundleSubmitXCommand extends SubmitTransitionXCommand { * @return resolved job xml * @throws BundleJobException thrown if failed to resolve variables */ - private String resolvedVars(String bundleXml, Configuration conf) throws BundleJobException { + private String resolvedVarsandFunctions(String bundleXml, Configuration conf) throws BundleJobException { + ELEvaluator eval; try { - ELEvaluator eval = createEvaluator(conf); + eval = createELEvaluatorForGroup(conf, "bundle-submit"); return eval.evaluate(bundleXml, String.class); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 2f44827..12c5aa0 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -549,7 +549,7 @@ <!-- List of supported groups for ELService --> <property> <name>oozie.service.ELService.groups</name> - <value>job-submit,workflow,wf-sla-submit,coord-job-submit-freq,coord-job-submit-nofuncs,coord-job-submit-data,coord-job-submit-instances,coord-sla-submit,coord-action-create,coord-action-create-inst,coord-sla-create,coord-action-start,coord-job-wait-timeout</value> + <value>job-submit,workflow,wf-sla-submit,coord-job-submit-freq,coord-job-submit-nofuncs,coord-job-submit-data,coord-job-submit-instances,coord-sla-submit,coord-action-create,coord-action-create-inst,coord-sla-create,coord-action-start,coord-job-wait-timeout,bundle-submit</value> <description>List of groups for different ELServices</description> </property> @@ -1003,6 +1003,11 @@ </property> <property> + <name>oozie.service.ELService.functions.bundle-submit</name> + <value>bundle:conf=org.apache.oozie.bundle.BundleELFunctions#bundle_conf</value> + </property> + + <property> <name>oozie.service.ELService.functions.coord-sla-submit</name> <value> coord:dataOut=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dataOut_echo, http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java index f793601..b883dc2 100644 --- a/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java @@ -99,6 +99,7 @@ public class TestBundleSubmitXCommand extends XDataTestCase { final XConfiguration jobConf = setUpBundle(); jobConf.set("coordName1", "coord1"); jobConf.set("coordName2", "coord2"); + jobConf.set("coord1.starttime","2009-02-01T00:00Z"); BundleSubmitXCommand command = new BundleSubmitXCommand(jobConf); final BundleJobBean bundleBean = (BundleJobBean) command.getJob(); @@ -149,7 +150,7 @@ public class TestBundleSubmitXCommand extends XDataTestCase { writeCoordXml(coordPath2, "coord-job-bundle.xml"); Path bundleAppPath = new Path(getFsTestCaseDir(), "bundle"); - String bundleAppXml = getBundleXml("bundle-submit-job.xml"); + String bundleAppXml = getBundleXml("bundle-submit-job-with-functions.xml"); assertNotNull(bundleAppXml); assertTrue(bundleAppXml.length() > 0); http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/core/src/test/java/org/apache/oozie/test/XDataTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XDataTestCase.java b/core/src/test/java/org/apache/oozie/test/XDataTestCase.java index ac8f1f9..e6a7d9c 100644 --- a/core/src/test/java/org/apache/oozie/test/XDataTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XDataTestCase.java @@ -1477,6 +1477,7 @@ public abstract class XDataTestCase extends XHCatTestCase { conf.set("nameNode", getNameNodeUri()); conf.set("coordName1", "coord1"); conf.set("coordName2", "coord2"); + conf.set("coord1.starttime","2009-02-01T00:00Z"); BundleJobBean bundle = new BundleJobBean(); bundle.setId(Services.get().get(UUIDService.class).generateId(ApplicationType.BUNDLE)); http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/core/src/test/resources/bundle-submit-job-with-functions.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/bundle-submit-job-with-functions.xml b/core/src/test/resources/bundle-submit-job-with-functions.xml new file mode 100644 index 0000000..094a06a --- /dev/null +++ b/core/src/test/resources/bundle-submit-job-with-functions.xml @@ -0,0 +1,60 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<bundle-app name='${appName}' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:bundle:0.2'> + <controls> + <kick-off-time>2099-02-01T00:00Z</kick-off-time> + </controls> + <coordinator name="${bundle:conf('coordName1')}" critical='true'> + <app-path>#app_path1</app-path> + <configuration> + <property> + <name>COORD_NAME</name> + <value>coord1</value> + </property> + <property> + <name>START_TIME</name> + <value>${bundle:conf('coord1.starttime')}</value> + </property> + <property> + <name>END_TIME</name> + <value>2009-02-20T23:59Z</value> + </property> + <property> + <name>testProperty</name> + <value>abc</value> + </property> + </configuration> + </coordinator> + <coordinator name='${coordName2}' critical='false'> + <app-path>#app_path2</app-path> + <configuration> + <property> + <name>COORD_NAME</name> + <value>coord2</value> + </property> + <property> + <name>START_TIME</name> + <value>2009-02-02T00:00Z</value> + </property> + <property> + <name>END_TIME</name> + <value>2009-02-23T23:59Z</value> + </property> + </configuration> + </coordinator> +</bundle-app> http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/docs/src/site/twiki/BundleFunctionalSpec.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/BundleFunctionalSpec.twiki b/docs/src/site/twiki/BundleFunctionalSpec.twiki index b400c19..c9252c0 100644 --- a/docs/src/site/twiki/BundleFunctionalSpec.twiki +++ b/docs/src/site/twiki/BundleFunctionalSpec.twiki @@ -91,12 +91,13 @@ When all the coordinator jobs finish, oozie updates the bundle status accordingl A bundle definition is defined in XML by a name, controls and one or more coordinator application specifications: * *%BLUE% name: %ENDCOLOR%* The name for the bundle job. - * *%BLUE% controls: %ENDCOLOR%* The control specification for the bundle. - * *%BLUE% kick-off-time: %ENDCOLOR%* It defines when the bundle job should start and submit the coordinator applications. This field is optional and the default is *NOW* that means the job should start right-a-way. - * *%BLUE% coordinator: %ENDCOLOR%* Coordinator application specification. There should be at least one coordinator application in any bundle. - * *%BLUE% name: %ENDCOLOR%* Name of the coordinator application. It can be used for referring this application through bundle to control such as kill, suspend, rerun. - * *%BLUE% app-path: %ENDCOLOR%* Path of the coordinator application definition in hdfs. This is a mandatory element. + * *%BLUE% controls: %ENDCOLOR%* The control specification for the bundle. + * *%BLUE% kick-off-time: %ENDCOLOR%* It defines when the bundle job should start and submit the coordinator applications. This field is optional and the default is *NOW* that means the job should start right-a-way. + * *%BLUE% coordinator: %ENDCOLOR%* Coordinator application specification. There should be at least one coordinator application in any bundle. + * *%BLUE% name: %ENDCOLOR%* Name of the coordinator application. It can be used for referring this application through bundle to control such as kill, suspend, rerun. + * *%BLUE% app-path: %ENDCOLOR%* Path of the coordinator application definition in hdfs. This is a mandatory element. * *%BLUE% configuration: %ENDCOLOR%* A hadoop like configuration to parameterize corresponding coordinator application. This is optional. + * *%BLUE% Parameterization: %ENDCOLOR%* Configuration properties that are a valid Java identifier, [A-Za-z_][0-9A-Za-z_]*, are available as =${NAME}= variables within the bundle application definition. Configuration properties that are not a valid Java identifier, for example =job.tracker=, are available via the =${bundle:conf(String name)}= function. Valid Java identifier properties are available via this function as well. *%PURPLE% Syntax: %ENDCOLOR%* @@ -130,12 +131,12 @@ A bundle definition is defined in XML by a name, controls and one or more coordi <controls> <kick-off-time>${kickOffTime}</kick-off-time> </controls> - <coordinator name='coordJobFromBundle1' > + <coordinator name="${bundle:conf('coordName1')}" > <app-path>${appPath}</app-path> <configuration> <property> <name>startTime1</name> - <value>${START_TIME}</value> + <value>${bundle:conf('coord1.startTime1')}</value> </property> <property> <name>endTime1</name> http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index fcf6d9d..89735d6 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2308 Add support for bundle:conf() function (kailongs via rohini) OOZIE-2315 TestOozieCLI.testshareLibUpdate_withSecurity fails with Hadoop 2 (rkanter) OOZIE-2291 Hive2 workflow.xml.security should have "cred" in action tag instead of "hive2" tag (bzhang via rkanter) OOZIE-2289 hive-jdbc dependency in core/pom.xml should be compile (bzhang)
