Repository: oozie Updated Branches: refs/heads/master 50e6c35b4 -> 07deb2b80
OOZIE-3056 Implement new mechanism to specify ShareLibs for workflow actions (gezapeti via andras.piros) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/07deb2b8 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/07deb2b8 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/07deb2b8 Branch: refs/heads/master Commit: 07deb2b80bf707f9b7841c4c4ca6e677f6c9367f Parents: 50e6c35 Author: Andras Piros <[email protected]> Authored: Mon Mar 26 14:46:39 2018 +0200 Committer: Andras Piros <[email protected]> Committed: Mon Mar 26 14:46:39 2018 +0200 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 54 ++------ .../oozie/action/hadoop/SharelibResolver.java | 96 ++++++++++++++ .../action/hadoop/TestJavaActionExecutor.java | 72 +++-------- .../action/hadoop/TestSharelibConfigs.java | 126 +++++++++++++++++++ .../action/hadoop/TestSharelibResolver.java | 72 +++++++++++ .../oozie/action/hadoop/TestWorkflowHelper.java | 103 +++++++++++++++ release-log.txt | 1 + 7 files changed, 431 insertions(+), 93 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 122dfd0..b9c89b2 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -55,7 +55,6 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.TaskLog; -import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.AccessControlException; @@ -164,6 +163,7 @@ public class JavaActionExecutor extends ActionExecutor { private static final String HADOOP_JOB_NAME = "mapred.job.name"; private static final Set<String> DISALLOWED_PROPERTIES = new HashSet<String>(); private static final String OOZIE_ACTION_NAME = "oozie.action.name"; + private final static String ACTION_SHARELIB_FOR = "oozie.action.sharelib.for."; private static int maxActionOutputLen; private static int maxExternalStatsSize; @@ -1831,54 +1831,26 @@ public class JavaActionExecutor extends ActionExecutor { /** * Return the sharelib names for the action. - * <p> - * If <code>NULL</code> or empty, it means that the action does not use the action - * sharelib. - * <p> - * If a non-empty string, i.e. <code>foo</code>, it means the action uses the - * action sharelib sub-directory <code>foo</code> and all JARs in the sharelib - * <code>foo</code> directory will be in the action classpath. Multiple sharelib - * sub-directories can be specified as a comma separated list. - * <p> - * The resolution is done using the following precedence order: - * <ul> - * <li><b>action.sharelib.for.#ACTIONTYPE#</b> in the action configuration</li> - * <li><b>action.sharelib.for.#ACTIONTYPE#</b> in the job configuration</li> - * <li><b>action.sharelib.for.#ACTIONTYPE#</b> in the oozie configuration</li> - * <li>Action Executor <code>getDefaultShareLibName()</code> method</li> - * </ul> - * + * See {@link SharelibResolver} for details. * * @param context executor context. - * @param actionXml + * @param actionXml the action xml. * @param conf action configuration. * @return the action sharelib names. */ - protected String[] getShareLibNames(Context context, Element actionXml, Configuration conf) { - String[] names = conf.getStrings(ACTION_SHARELIB_FOR + getType()); - if (names == null || names.length == 0) { - try { - XConfiguration jobConf = getWorkflowConf(context); - names = jobConf.getStrings(ACTION_SHARELIB_FOR + getType()); - if (names == null || names.length == 0) { - names = Services.get().getConf().getStrings(ACTION_SHARELIB_FOR + getType()); - if (names == null || names.length == 0) { - String name = getDefaultShareLibName(actionXml); - if (name != null) { - names = new String[] { name }; - } - } - } - } - catch (IOException ex) { - throw new RuntimeException("It cannot happen, " + ex.toString(), ex); - } + protected String[] getShareLibNames(final Context context, final Element actionXml, Configuration conf) { + final String sharelibFromConfigurationProperty = ACTION_SHARELIB_FOR + getType(); + try { + final String defaultShareLibName = getDefaultShareLibName(actionXml); + final Configuration oozieServerConfiguration = Services.get().get(ConfigurationService.class).getConf(); + final XConfiguration workflowConfiguration = getWorkflowConf(context); + return new SharelibResolver(sharelibFromConfigurationProperty, conf, workflowConfiguration, + oozieServerConfiguration, defaultShareLibName).resolve(); + } catch (IOException ex) { + throw new RuntimeException("Can't get workflow configuration: " + ex.toString(), ex); } - return names; } - private final static String ACTION_SHARELIB_FOR = "oozie.action.sharelib.for."; - /** * Returns the default sharelib name for the action if any. http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/core/src/main/java/org/apache/oozie/action/hadoop/SharelibResolver.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SharelibResolver.java b/core/src/main/java/org/apache/oozie/action/hadoop/SharelibResolver.java new file mode 100644 index 0000000..18c50cb --- /dev/null +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SharelibResolver.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.action.hadoop; + +import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.util.XConfiguration; + +import java.io.IOException; +import java.util.List; + +class SharelibResolver { + private final String[] defaultValue; + private final List<SharelibNameProvider> configProviders; + + + /** + * Return the sharelib names for the action based on the given configurations. + * <p> + * If <code>NULL</code> or empty, it means that the action does not use the action + * sharelib. + * <p> + * If a non-empty string, i.e. <code>foo</code>, it means the action uses the + * action sharelib sub-directory <code>foo</code> and all JARs in the sharelib + * <code>foo</code> directory will be in the action classpath. Multiple sharelib + * sub-directories can be specified as a comma separated list. + * <p> + * The resolution is done using the following precedence order: + * <ul> + * <li><b><sharelib></b> tag in the action's launcher configuration</li> + * <li><b>oozie.action.sharelib.for.#ACTIONTYPE#</b> in the action configuration</li> + * <li><b><sharelib></b> tag in the workflow's launcher configuration</li> + * <li><b>oozie.action.sharelib.for.#ACTIONTYPE#</b> in the workflow configuration</li> + * <li><b>oozie.action.sharelib.for.#ACTIONTYPE#</b> in the oozie configuration</li> + * <li>Action Executor <code>getDefaultShareLibName()</code> method</li> + * </ul> + * + * @param sharelibPropertyName the property for the current sharelib. E.g. oozie.action.sharelib.for.java + * @param actionConf the configuration of the current action + * @param workflowConf the global configuration for the workflow + * @param oozieServerConfiguration the Oozie server's configuration + * @param defaultValue the default value to use if there is no sharelib definition in the configs + */ + SharelibResolver(final String sharelibPropertyName, final Configuration actionConf, + final XConfiguration workflowConf, final Configuration oozieServerConfiguration, + final String defaultValue) { + if (defaultValue == null) { + this.defaultValue = new String[0]; + } else { + this.defaultValue = new String[] { defaultValue }; + } + configProviders = Lists.newArrayList( + () -> actionConf.getStrings(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY), + () -> actionConf.getStrings(sharelibPropertyName), + () -> workflowConf.getStrings(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY), + () -> workflowConf.getStrings(sharelibPropertyName), + () -> oozieServerConfiguration.getStrings(sharelibPropertyName) + ); + } + + /** + * Return the sharelib names for the action. + * @return the sharelib names + */ + public String[] resolve() { + for (SharelibNameProvider cp : configProviders) { + if (isValidSharelibProperty(cp.getSharelibNames())) { + return cp.getSharelibNames(); + } + } + return defaultValue; + } + + private boolean isValidSharelibProperty(String[] value){ + return value != null && value.length > 0; + } +} + +interface SharelibNameProvider { + String[] getSharelibNames(); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index b55a3cd..2bd6a3f 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -89,12 +89,15 @@ import org.jdom.Element; import org.junit.Assert; import org.junit.Test; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + public class TestJavaActionExecutor extends ActionExecutorTestCase { private static final String YARN_RESOURCEMANAGER_ADDRESS = "yarn.resourcemanager.address"; private static final String MAPRED_CHILD_JAVA_OPTS = "mapred.child.java.opts"; private static final String MAPREDUCE_MAP_JAVA_OPTS = "mapreduce.map.java.opts"; - + private TestWorkflowHelper helper; @Override protected void beforeSetUp() throws Exception { super.beforeSetUp(); @@ -102,6 +105,12 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { } @Override + public void setUp() throws Exception { + super.setUp(); + helper = new TestWorkflowHelper(getJobTrackerUri(), getNameNodeUri(), getTestCaseDir()); + + } + @Override protected void setSystemProps() throws Exception { super.setSystemProps(); @@ -2247,7 +2256,10 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { public void testDefaultConfigurationInActionConf() throws Exception { JavaActionExecutor ae = new JavaActionExecutor(); - String xmlStr = getJavaActionXml(true); + final String dummyConfiguration = "<configuration>" + + "<property><name>action.foo</name><value>AA</value></property>" + + "</configuration>"; + String xmlStr = helper.getJavaActionXml(dummyConfiguration); Element actionXml = XmlUtils.parseXml(xmlStr); Context context = createContext(xmlStr, getTestGroup()); Configuration conf = new Configuration(true); @@ -2261,43 +2273,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { conf.get("mapreduce.map.maxattempts")); } - private String getJavaActionXml(boolean addConfig) { - String config = addConfig ? "<configuration>" + - "<property><name>action.foo</name><value>AA</value></property>" + - "</configuration>" : ""; - return "<java>" + - "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + - "<name-node>" + getNameNodeUri() + "</name-node>" + - config + - "<main-class>MAIN-CLASS</main-class>" + - "</java>"; - } - - private String createTestWorkflowXml(final String globalXml, final String actionXml) throws IOException { - String workflowUri = getTestCaseFileUri("workflow.xml"); - String appXml = "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" name=\"workflow\">" + - globalXml + - "<start to=\"java\"/>" + - "<action name=\"java\">" + - actionXml + - " <ok to=\"end\"/>" + - " <error to=\"fail\"/>" + - "</action>" + - "<kill name=\"fail\">" + - " <message>Sub workflow failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>" + - "</kill>" + - "<end name=\"end\"/>" + - "</workflow-app>"; - - final File f = new File(URI.create(workflowUri)); - final ByteArrayInputStream inputStream = new ByteArrayInputStream(appXml.getBytes("UTF-8")); - IOUtils.copyStream(inputStream, new FileOutputStream(f)); - return workflowUri; - } - public void testGlobalConfigurationWithActionDefaults() throws Exception { try { - String workflowUri = createTestWorkflowXml(getWorkflowGlobalXml(), getJavaActionXml(false)); + String workflowUri = helper.createTestWorkflowXml(getWorkflowGlobalXml(), helper.getJavaActionXml("")); LocalOozie.start(); final OozieClient wfClient = LocalOozie.getClient(); Properties conf = wfClient.createConfiguration(); @@ -2310,11 +2288,11 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { waitFor(20 * 1000, new Predicate() { @Override public boolean evaluate() throws Exception { - WorkflowAction javaAction = getJavaAction(wfClient.getJobInfo(jobId)); + WorkflowAction javaAction = helper.getJavaAction(wfClient.getJobInfo(jobId)); return javaAction != null && !javaAction.getStatus().equals("PREP"); } }); - final WorkflowAction workflowAction = getJavaAction(workflow); + final WorkflowAction workflowAction = helper.getJavaAction(workflow); Element eConf = XmlUtils.parseXml(workflowAction.getConf()); Element element = eConf.getChild("configuration", eConf.getNamespace()); Configuration actionConf = new XConfiguration(new StringReader(XmlUtils.prettyPrint(element).toString())); @@ -2332,7 +2310,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { "<resource-manager>RM</resource-manager>"+ "</global>"; - final String workflowUri = createTestWorkflowXml(global, getJavaActionXml(false)); + final String workflowUri = helper.createTestWorkflowXml(global, helper.getJavaActionXml("")); LocalOozie.start(); final OozieClient wfClient = LocalOozie.getClient(); final Properties conf = wfClient.createConfiguration(); @@ -2345,11 +2323,11 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { waitFor(20 * 1000, new Predicate() { @Override public boolean evaluate() throws Exception { - WorkflowAction javaAction = getJavaAction(wfClient.getJobInfo(jobId)); + WorkflowAction javaAction = helper.getJavaAction(wfClient.getJobInfo(jobId)); return javaAction != null && !javaAction.getStatus().equals("PREP"); } }); - final WorkflowAction workflowAction = getJavaAction(workflow); + final WorkflowAction workflowAction = helper.getJavaAction(workflow); final String actualConfig = workflowAction.getConf(); final String actualJobTrackerURI = XmlUtils.parseXml(actualConfig).getChildTextNormalize("job-tracker", null); assertEquals(getJobTrackerUri(), actualJobTrackerURI); @@ -2359,16 +2337,6 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { } } - private WorkflowAction getJavaAction(WorkflowJob workflowJob){ - List<WorkflowAction> actions = workflowJob.getActions(); - for(WorkflowAction wa : actions){ - if(wa.getType().equals("java")){ - return wa; - } - } - return null; - } - public void testSetRootLoggerLevel() throws Exception { String oozieActionRootLogger = "oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL; String oozieActionHiveRootLogger = "oozie.action.hive" + LauncherAMUtils.ROOT_LOGGER_LEVEL; http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibConfigs.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibConfigs.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibConfigs.java new file mode 100644 index 0000000..bda5ea7 --- /dev/null +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibConfigs.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.action.hadoop; + +import org.apache.oozie.client.OozieClient; +import org.apache.oozie.client.WorkflowAction; +import org.apache.oozie.client.WorkflowJob; +import org.apache.oozie.local.LocalOozie; +import org.apache.oozie.test.MiniOozieTestCase; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.XConfiguration; +import org.apache.oozie.util.XmlUtils; +import org.jdom.Element; +import org.junit.Before; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.StringReader; +import java.net.URI; +import java.util.List; +import java.util.Properties; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +public class TestSharelibConfigs extends MiniOozieTestCase { + + private TestWorkflowHelper helper; + + @Override + protected void setUp() throws Exception { + super.setUp(); + helper = new TestWorkflowHelper(getJobTrackerUri(), getNameNodeUri(), getTestCaseDir()); + } + + public void testActionSharelibConfigPropagation() throws Exception { + final String globalConfigForJavaSharelib = "<global><configuration><property>" + + "<name>oozie.action.sharelib.for.java</name><value>globalJavaConfigSharelib</value>" + + "</property></configuration></global>"; + final String globalLauncherWithSharelib = "<global><launcher>" + + "<sharelib>globalLauncherSharelib</sharelib>" + + "</launcher></global>"; + final String globalLauncherAndConfigWithSharelib = "<global><launcher>" + + "<sharelib>globalLauncherSharelib</sharelib>" + + "</launcher><configuration><property>" + + "<name>oozie.action.sharelib.for.java</name><value>globalJavaConfigSharelib</value>" + + "</property></configuration></global>"; + final String localConfigForJavaSharelib = "<configuration><property>" + + "<name>oozie.action.sharelib.for.java</name><value>localJavaConfigSharelib</value>" + + "</property></configuration>"; + final String localLauncherWithSharelib = "<launcher>" + + "<sharelib>localLauncherSharelib</sharelib>" + + "</launcher>"; + + launchJavaActionAndValidateSharelibValues("", "", + null, null); + launchJavaActionAndValidateSharelibValues(globalConfigForJavaSharelib, "", + "globalJavaConfigSharelib", null); + launchJavaActionAndValidateSharelibValues(globalLauncherWithSharelib, "", + null, "globalLauncherSharelib"); + launchJavaActionAndValidateSharelibValues(globalLauncherAndConfigWithSharelib, localConfigForJavaSharelib, + "localJavaConfigSharelib", "globalLauncherSharelib"); + launchJavaActionAndValidateSharelibValues(globalLauncherAndConfigWithSharelib, localLauncherWithSharelib, + "globalJavaConfigSharelib", "localLauncherSharelib"); + launchJavaActionAndValidateSharelibValues(globalLauncherAndConfigWithSharelib, + localLauncherWithSharelib + localConfigForJavaSharelib, + "localJavaConfigSharelib", "localLauncherSharelib"); + + } + + private void launchJavaActionAndValidateSharelibValues(String globalConfig, + String localConfig, + String oozieSharelibForJavaPropertyValue, + String oozieSharelibPropertyValue) + throws Exception { + final String workflowUri = helper.createTestWorkflowXml(globalConfig, + helper.getJavaActionXml(localConfig)); + final OozieClient wfClient = LocalOozie.getClient(); + final Properties conf = wfClient.createConfiguration(); + conf.setProperty(OozieClient.APP_PATH, workflowUri); + conf.setProperty(OozieClient.USER_NAME, getTestUser()); + conf.setProperty("appName", "var-app-name"); + conf.setProperty(OozieClient.USE_SYSTEM_LIBPATH, "true"); + final String jobId = wfClient.submit(conf); + wfClient.start(jobId); + WorkflowJob workflow = wfClient.getJobInfo(jobId); + waitFor(20 * 1000, new Predicate() { + @Override + public boolean evaluate() throws Exception { + WorkflowAction javaAction = helper.getJavaAction(wfClient.getJobInfo(jobId)); + return javaAction != null && !javaAction.getStatus().equals("PREP"); + } + }); + final XConfiguration actionConf = getJavaActionConfiguration(workflow); + assertThat("Configuration priorities are incorrect! Global/local configs are not overwriting each other.", + actionConf.get(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY), is(oozieSharelibPropertyValue)); + assertThat("Configuration priorities are incorrect! Global/local configs are not overwriting each other.", + actionConf.get("oozie.action.sharelib.for.java"), is(oozieSharelibForJavaPropertyValue)); + } + + private XConfiguration getJavaActionConfiguration(WorkflowJob workflow) throws Exception { + final WorkflowAction workflowAction = helper.getJavaAction(workflow); + final Element element = XmlUtils.parseXml(workflowAction.getConf()); + final String configuration = XmlUtils.prettyPrint(element.getChild("configuration", + element.getNamespace())).toString(); + return new XConfiguration(new StringReader(configuration)); + } + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibResolver.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibResolver.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibResolver.java new file mode 100644 index 0000000..985a250 --- /dev/null +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibResolver.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.action.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.util.XConfiguration; +import org.junit.Test; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +public class TestSharelibResolver { + public final String defaultValue = "default"; + private final String serverConfigSharelib = "serverConfigSharelib"; + private final String globalConfigSharelib = "globalConfigSharelib"; + private final String globalLauncherSharelib = "globalLauncherSharelib"; + private final String localConfigSharelib = "localConfigSharelib"; + private final String localLauncherSharelib = "localLauncherSharelib"; + private final String sharelibProperty = "sharelibProperty"; + + /** + * Add sharelib values in reverse priority order and check that the last added value is the one that gets resolved. + */ + @Test + public void testResolvingOrder() { + Configuration oozieServerConfiguration = new Configuration(false); + XConfiguration workflowConf = new XConfiguration(); + Configuration actionConf = new Configuration(false); + + SharelibResolver resolver = + new SharelibResolver(sharelibProperty, actionConf, workflowConf, oozieServerConfiguration, defaultValue); + + assertThat("Without setting anything we should've got the default value.", + resolver.resolve(), is(new String[]{defaultValue})); + + oozieServerConfiguration.set(sharelibProperty, serverConfigSharelib); + assertThat("Server-level sharelib configuration is not processed", + resolver.resolve(), is(new String[]{serverConfigSharelib})); + + workflowConf.set(sharelibProperty, globalConfigSharelib); + assertThat("Global workflow-level sharelib configuration is not processed", + resolver.resolve(), is(new String[]{globalConfigSharelib})); + + workflowConf.set(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY, globalLauncherSharelib); + assertThat("Global workflow-level sharelib configuration is not processed", + resolver.resolve(), is(new String[]{globalLauncherSharelib})); + + actionConf.set(sharelibProperty, localConfigSharelib); + assertThat("Local action-level sharelib configuration is not processed", + resolver.resolve(), is(new String[]{localConfigSharelib})); + + actionConf.set(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY, localLauncherSharelib); + assertThat("Local action-level sharelib configuration is not processed", + resolver.resolve(), is(new String[]{localLauncherSharelib})); + } + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/core/src/test/java/org/apache/oozie/action/hadoop/TestWorkflowHelper.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestWorkflowHelper.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestWorkflowHelper.java new file mode 100644 index 0000000..d6bebe4 --- /dev/null +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestWorkflowHelper.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.action.hadoop; + +import org.apache.oozie.client.WorkflowAction; +import org.apache.oozie.client.WorkflowJob; +import org.apache.oozie.test.MiniOozieTestCase; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.IOUtils; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URI; +import java.util.List; + +public class TestWorkflowHelper { + + private final String jobTrackerUri; + private final String nameNodeUri; + private final String testCaseDir; + + + public TestWorkflowHelper(String jobTrackerUri, String nameNodeUri, String testCaseDir) { + this.jobTrackerUri = jobTrackerUri; + this.nameNodeUri = nameNodeUri; + this.testCaseDir = testCaseDir; + } + + protected String getJavaActionXml(String configToAdd) { + return "<java>" + + "<job-tracker>" + jobTrackerUri + "</job-tracker>" + + "<name-node>" + nameNodeUri + "</name-node>" + + configToAdd + + "<main-class>MAIN-CLASS</main-class>" + + "</java>"; + } + + protected String createTestWorkflowXml(final String globalXml, final String actionXml) throws IOException { + String workflowUri = getTestCaseFileUri("workflow.xml"); + String appXml = "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" name=\"workflow\">" + + globalXml + + "<start to=\"java\"/>" + + "<action name=\"java\">" + + actionXml + + " <ok to=\"end\"/>" + + " <error to=\"fail\"/>" + + "</action>" + + "<kill name=\"fail\">" + + " <message>Sub workflow failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>" + + "</kill>" + + "<end name=\"end\"/>" + + "</workflow-app>"; + final File f = new File(URI.create(workflowUri)); + final ByteArrayInputStream inputStream = new ByteArrayInputStream(appXml.getBytes("UTF-8")); + IOUtils.copyStream(inputStream, new FileOutputStream(f)); + return workflowUri; + } + + protected WorkflowAction getJavaAction(WorkflowJob workflowJob){ + List<WorkflowAction> actions = workflowJob.getActions(); + for(WorkflowAction wa : actions){ + if(wa.getType().equals("java")){ + return wa; + } + } + return null; + } + /** + * Return the URI for a test file. The returned value is the testDir + concatenated URI. + * + * @return the test working directory path, it is always an absolute path and appends the relative path. The + * reason for the manual parsing instead of an actual File.toURI is because Oozie tests use tokens ${} + * frequently. Something like URI("c:/temp/${HOUR}").toString() will generate escaped values that will break tests + */ + private String getTestCaseFileUri(String relativeUri) { + String uri = new File(testCaseDir).toURI().toString(); + + // truncates '/' if the testCaseDir was provided with a fullpath ended with separator + if (uri.endsWith("/")){ + uri = uri.substring(0, uri.length() -1); + } + + return uri + "/" + relativeUri; + } + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 76401f5..d126338 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-3056 Implement new mechanism to specify ShareLibs for workflow actions (gezapeti via andras.piros) OOZIE-2600 OYA: Update Documentation (andras.piros via gezapeti) OOZIE-3189 Update the release script and wiki page to use sha512 instead of md5 (rkanter via gezapeti) OOZIE-3195 Typo in WebServicesAPI.twiki: Proxy Hive Job Submission (kmarton via andras.piros)
