This is an automated email from the ASF dual-hosted git repository.
asalamon74 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push:
new 7c2e80a OOZIE-1624 Exclusion pattern for sharelib JARs (puru, matijhs
via asalamon74, andras.piros)
7c2e80a is described below
commit 7c2e80afc552e5cafe38b70aabcb765bc28121a4
Author: Andras Salamon <[email protected]>
AuthorDate: Fri Mar 8 15:19:26 2019 +0100
OOZIE-1624 Exclusion pattern for sharelib JARs (puru, matijhs via
asalamon74, andras.piros)
---
.../oozie/action/hadoop/JavaActionExecutor.java | 78 +++-
.../oozie/action/hadoop/ShareLibExcluder.java | 102 +++++
.../org/apache/oozie/service/ShareLibService.java | 14 +
.../action/hadoop/ActionExecutorTestCase.java | 66 +++
.../action/hadoop/TestJavaActionExecutor.java | 425 +------------------
.../hadoop/TestJavaActionExecutorLibAddition.java | 448 +++++++++++++++++++++
.../oozie/action/hadoop/TestShareLibExcluder.java | 233 +++++++++++
docs/src/site/markdown/WorkflowFunctionalSpec.md | 54 +++
release-log.txt | 1 +
9 files changed, 1002 insertions(+), 419 deletions(-)
diff --git
a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 0e76a7b..e70e90b 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -171,8 +171,10 @@ public class JavaActionExecutor extends ActionExecutor {
OozieClient.USER_NAME, MRJobConfig.USER_NAME, HADOOP_NAME_NODE,
HADOOP_YARN_RM
);
private static final String OOZIE_ACTION_NAME = "oozie.action.name";
- private final static String ACTION_SHARELIB_FOR =
"oozie.action.sharelib.for.";
- public static final String OOZIE_ACTION_DEPENDENCY_DEDUPLICATE =
"oozie.action.dependency.deduplicate";
+ private static final String OOZIE_ACTION_DEPENDENCY_DEDUPLICATE =
"oozie.action.dependency.deduplicate";
+
+ static final String ACTION_SHARELIB_FOR = "oozie.action.sharelib.for.";
+ static final String SHARELIB_EXCLUDE_SUFFIX = ".exclude";
/**
* Heap to physical memory ration for {@link LauncherAM}, in order its
YARN container doesn't get killed before physical memory
@@ -209,6 +211,7 @@ public class JavaActionExecutor extends ActionExecutor {
private static final String JAVA_TMP_DIR_SETTINGS = "-Djava.io.tmpdir=";
private static DependencyDeduplicator dependencyDeduplicator = new
DependencyDeduplicator();
+ private ShareLibExcluder shareLibExcluder = null;
public XConfiguration workflowConf = null;
@@ -695,7 +698,7 @@ public class JavaActionExecutor extends ActionExecutor {
if (FSUtils.isLocalFile(libPath.toString())) {
conf = ClasspathUtils.addToClasspathFromLocalShareLib(conf,
libPath);
}
- else {
+ else if (!shareLibExcluder.shouldExclude(libPath.toUri())) {
if (isAddToCache) {
addToCache(conf, libPath, libPath.toUri().getPath(),
false);
}
@@ -708,7 +711,16 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
- protected void addActionLibs(Path appPath, Configuration conf) throws
ActionExecutorException {
+ private void addFilesToCacheIfNotExcluded(Path appPath, Configuration
conf, FileStatus[] files) throws ActionExecutorException {
+ if (files == null) return;
+ for (FileStatus file : files) {
+ if (!shareLibExcluder.shouldExclude(file.getPath().toUri())) {
+ addToCache(conf, appPath, file.getPath().toUri().getPath(),
false);
+ }
+ }
+ }
+
+ private void addActionLibs(Path appPath, Configuration conf) throws
ActionExecutorException {
String[] actionLibsStrArr =
conf.getStrings("oozie.launcher.oozie.libpath");
if (actionLibsStrArr != null) {
try {
@@ -721,10 +733,7 @@ public class JavaActionExecutor extends ActionExecutor {
FileSystem fs =
Services.get().get(HadoopAccessorService.class).createFileSystem(user,
appPath.toUri(), conf);
if (fs.exists(actionLibsPath)) {
- FileStatus[] files = fs.listStatus(actionLibsPath);
- for (FileStatus file : files) {
- addToCache(conf, appPath,
file.getPath().toUri().getPath(), false);
- }
+ addFilesToCacheIfNotExcluded(appPath, conf,
fs.listStatus(actionLibsPath));
}
}
}
@@ -740,22 +749,42 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
+ private void initShareLibExcluder(Configuration conf, Context context)
throws ActionExecutorException {
+ XConfiguration wfJobConf = null;
+ try {
+ wfJobConf = getWorkflowConf(context);
+ }
+ catch (IOException e) {
+ throw new
ActionExecutorException(ActionExecutorException.ErrorType.FAILED,
+ "Failed to acquire workflow configuration from context.",
e.getMessage());
+ }
+ LOG.info("Initializing sharelib excluder for action if specified.");
+ shareLibExcluder = new ShareLibExcluder(conf,
Services.get().getConf(), wfJobConf, getType(), getSharelibRoot());
+ }
+
@SuppressWarnings("unchecked")
public void setLibFilesArchives(Context context, Element actionXml, Path
appPath, Configuration conf)
throws ActionExecutorException {
- Configuration proto = context.getProtoActionConf();
- // Workflow lib/
+ addWfApplicationLibs(appPath, conf, context.getProtoActionConf());
+ addActionXmlFilesAndArchives(actionXml, appPath, conf);
+
+ initShareLibExcluder(conf, context);
+
+ addActionLibs(appPath, conf);
+ addAllShareLibs(appPath, conf, context, actionXml);
+ }
+
+ private void addWfApplicationLibs(Path appPath, Configuration conf,
Configuration proto) throws ActionExecutorException {
String[] paths =
proto.getStrings(WorkflowAppService.APP_LIB_PATH_LIST);
if (paths != null) {
for (String path : paths) {
addToCache(conf, appPath, path, false);
}
}
+ }
- // Action libs
- addActionLibs(appPath, conf);
-
+ private void addActionXmlFilesAndArchives(Element actionXml, Path appPath,
Configuration conf) throws ActionExecutorException {
// files and archives defined in the action
for (Element eProp : (List<Element>) actionXml.getChildren()) {
if (eProp.getName().equals("file")) {
@@ -771,8 +800,6 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
}
-
- addAllShareLibs(appPath, conf, context, actionXml);
}
@VisibleForTesting
@@ -1131,6 +1158,27 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
+ private URI getSharelibRoot() {
+ ShareLibService shareLibService =
Services.get().get(ShareLibService.class);
+ if (shareLibService == null) {
+ LOG.warn("ShareLibService is not configured, no root can be
found");
+ return null;
+ }
+ try {
+ Path shareLibRootPath = shareLibService.getShareLibRootPath();
+ if (shareLibRootPath != null) {
+ return shareLibRootPath.toUri();
+ }
+ else {
+ LOG.warn("ShareLib root not found");
+ }
+ }
+ catch (IOException e) {
+ LOG.warn("ShareLib root not found", e);
+ }
+ return null;
+ }
+
private void addAppNameContext(final Context context, final WorkflowAction
action) {
final String oozieActionName = getYarnApplicationName(context, action,
"oozie:launcher");
context.setVar(OOZIE_ACTION_NAME, oozieActionName);
diff --git
a/core/src/main/java/org/apache/oozie/action/hadoop/ShareLibExcluder.java
b/core/src/main/java/org/apache/oozie/action/hadoop/ShareLibExcluder.java
new file mode 100644
index 0000000..9c45e53
--- /dev/null
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/ShareLibExcluder.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.util.XLog;
+
+import java.net.URI;
+import java.util.Optional;
+import java.util.regex.Pattern;
+import com.google.common.base.Preconditions;
+
+import static
org.apache.oozie.action.hadoop.JavaActionExecutor.ACTION_SHARELIB_FOR;
+import static
org.apache.oozie.action.hadoop.JavaActionExecutor.SHARELIB_EXCLUDE_SUFFIX;
+
+class ShareLibExcluder {
+
+ private final URI shareLibRoot;
+ private final Pattern configuredExcludePattern;
+ private static XLog LOG = XLog.getLog(ShareLibExcluder.class);
+ @VisibleForTesting
+ static final String VALUE_NULL_MSG = "The value of %s cannot be null.";
+
+ ShareLibExcluder(final Configuration actionConf, final Configuration
servicesConf, final Configuration jobConf,
+ final String executorType, final URI shareLibRoot) {
+ this.shareLibRoot = shareLibRoot;
+ this.configuredExcludePattern = loadAndBuildPattern(actionConf,
servicesConf, jobConf, executorType);
+ }
+
+ private Pattern loadAndBuildPattern(final Configuration actionConf, final
Configuration servicesConf,
+ final Configuration jobConf, final
String executorType) {
+ if(shareLibRoot == null) {
+ return null;
+ }
+
+ Preconditions.checkNotNull(actionConf, VALUE_NULL_MSG, "actionConf");
+ Preconditions.checkNotNull(servicesConf, VALUE_NULL_MSG,
"servicesConf");
+ Preconditions.checkNotNull(jobConf, VALUE_NULL_MSG, "jobConf");
+
+ final String excludeProperty = ACTION_SHARELIB_FOR + executorType +
SHARELIB_EXCLUDE_SUFFIX;
+
+ final Optional<String> maybeExcludePattern =
tryFindExcludePattern(actionConf, servicesConf, jobConf, excludeProperty);
+ if (maybeExcludePattern.isPresent()) {
+ final String excludePattern = maybeExcludePattern.get();
+ actionConf.set(excludeProperty, excludePattern);
+ LOG.debug("Setting action configuration property: {0}={1}",
excludeProperty, excludePattern);
+ LOG.info("The following sharelib exclude pattern will be used:
{0}", excludePattern);
+
+ return Pattern.compile(excludePattern);
+ }
+ return null;
+ }
+
+ private Optional<String> tryFindExcludePattern(final Configuration
actionConf, final Configuration servicesConf,
+ final Configuration
jobConf, final String excludeProperty) {
+ String excludePattern = actionConf.get(excludeProperty);
+
+ if (excludePattern == null) {
+ excludePattern = jobConf.get(excludeProperty);
+ }
+
+ if (excludePattern == null) {
+ excludePattern = servicesConf.get(excludeProperty);
+ }
+
+ if (excludePattern == null) {
+ LOG.info("Sharelib exclude pattern not configured, skipping.");
+ return Optional.empty();
+ }
+ return Optional.of(excludePattern);
+ }
+
+ boolean shouldExclude(final URI actionLibURI) {
+ Preconditions.checkNotNull(actionLibURI, VALUE_NULL_MSG,
"actionLibURI");
+
+ if (configuredExcludePattern != null && shareLibRoot != null) {
+ if
(configuredExcludePattern.matcher(shareLibRoot.relativize(actionLibURI).getPath()).matches())
{
+ LOG.info("Mark file for excluding from distributed cache:
{0}", actionLibURI.getPath());
+ return true;
+ }
+ }
+ LOG.debug("Mark file for adding to distributed cache: {0}",
actionLibURI.getPath());
+ return false;
+ }
+}
diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java
b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
index b88dab3..fc2c29b 100644
--- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java
+++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
@@ -618,6 +618,20 @@ public class ShareLibService implements Service,
Instrumentable {
}
/**
+ * Get the latest share lib root path
+ *
+ * @return share lib root Path
+ * @throws IOException Signals that the Oozie share lib root path could
not be reached.
+ */
+ public Path getShareLibRootPath() throws IOException {
+ Path shareLibpath =
getLatestLibPath(Services.get().get(WorkflowAppService.class).getSystemLibPath(),
SHARE_LIB_PREFIX);
+ if (shareLibpath == null){
+ LOG.info("No share lib directory found");
+ }
+ return shareLibpath;
+ }
+
+ /**
* Update share lib cache. Parse the share lib directory and each sub
directory is a action key
*
* @param shareLibMap the share lib jar map
diff --git
a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java
b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java
index 05511e4..eca4013 100644
---
a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java
+++
b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java
@@ -43,6 +43,7 @@ import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.LiteWorkflowStoreService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.UUIDService;
+import org.apache.oozie.service.ShareLibService;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.service.WorkflowStoreService;
import org.apache.oozie.test.XHCatTestCase;
@@ -75,6 +76,12 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Collection;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.FileInputStream;
+
+import java.text.SimpleDateFormat;
public abstract class ActionExecutorTestCase extends XHCatTestCase {
protected static final int JOB_TIMEOUT = 100_000;
@@ -495,4 +502,63 @@ public abstract class ActionExecutorTestCase extends
XHCatTestCase {
HadoopAccessorService getHadoopAccessorService() {
return Services.get().get(HadoopAccessorService.class);
}
+
+ void createFiles(Collection<Path> paths) throws Exception{
+ for(Path p : paths){
+ getFileSystem().create(p);
+ }
+ }
+
+ void makeDirs(Path... dirs) throws Exception{
+ for(Path p : dirs){
+ getFileSystem().mkdirs(p);
+ }
+ }
+
+ private void assertContainsJarsOrNot(boolean contains, String
cacheFilesStr, Collection<Path> jars) {
+ for (Path jar : jars) {
+ assertEquals("Unexpected distributed cache file content",
contains, cacheFilesStr.contains(jar.toString()));
+ }
+ }
+
+ void assertContainsJars(String cacheFilesStr, Collection<Path> jars) {
+ assertContainsJarsOrNot(true, cacheFilesStr, jars);
+ }
+
+ void assertNotContainsJars(String cacheFilesStr, Collection<Path> jars) {
+ assertContainsJarsOrNot(false, cacheFilesStr, jars);
+ }
+
+ protected Context createContext(String actionXml, String group) throws
Exception {
+ JavaActionExecutor ae = new JavaActionExecutor();
+
+ Path appJarPath = new Path("lib/test.jar");
+ File jarFile = IOUtils.createJar(new File(getTestCaseDir()),
"test.jar", LauncherMainTester.class);
+ InputStream is = new FileInputStream(jarFile);
+ OutputStream os = getFileSystem().create(new Path(getAppPath(),
"lib/test.jar"));
+ IOUtils.copyStream(is, os);
+
+ Path appSoPath = new Path("lib/test.so");
+ getFileSystem().create(new Path(getAppPath(), appSoPath)).close();
+
+ XConfiguration protoConf = new XConfiguration();
+ protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+ protoConf.setStrings(WorkflowAppService.APP_LIB_PATH_LIST,
appJarPath.toString(), appSoPath.toString());
+
+ WorkflowJobBean wf = createBaseWorkflow(protoConf, "action");
+ if(group != null) {
+ wf.setGroup(group);
+ }
+ WorkflowActionBean action = (WorkflowActionBean)
wf.getActions().get(0);
+ action.setType(ae.getType());
+ action.setConf(actionXml);
+
+ return new Context(wf, action);
+ }
+
+ Path getNewSystemLibPath() {
+ WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
+ return new Path(wps.getSystemLibPath(),
ShareLibService.SHARE_LIB_PREFIX
+ + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()));
+ }
}
diff --git
a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 6383e81..4d3e3d7 100644
---
a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++
b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -18,6 +18,7 @@
package org.apache.oozie.action.hadoop;
+import java.io.IOException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -26,9 +27,6 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.StringReader;
import java.io.Writer;
-import java.net.URI;
-import java.security.PrivilegedExceptionAction;
-import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
@@ -37,6 +35,11 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.net.URI;
+import java.security.PrivilegedExceptionAction;
+
+import org.jdom.Element;
+import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.examples.SleepJob;
@@ -82,12 +85,6 @@ import org.apache.oozie.workflow.WorkflowLib;
import org.apache.oozie.workflow.lite.EndNodeDef;
import org.apache.oozie.workflow.lite.LiteWorkflowApp;
import org.apache.oozie.workflow.lite.StartNodeDef;
-import org.jdom.Element;
-import org.junit.Assert;
-import org.junit.Test;
-
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
public class TestJavaActionExecutor extends ActionExecutorTestCase {
@@ -105,21 +102,27 @@ public class TestJavaActionExecutor extends
ActionExecutorTestCase {
public void setUp() throws Exception {
super.setUp();
helper = new TestWorkflowHelper(getJobTrackerUri(), getNameNodeUri(),
getTestCaseDir());
-
}
+
@Override
protected void setSystemProps() throws Exception {
super.setSystemProps();
+ setHadoopSystemProps();
+ createActionConfDirFiles();
+ }
- setSystemProperty("oozie.service.ActionService.executor.classes",
JavaActionExecutor.class.getName());
-
setSystemProperty("oozie.service.HadoopAccessorService.action.configurations",
- "*=hadoop-conf," + getJobTrackerUri() +
"=action-conf");
- setSystemProperty(WorkflowAppService.SYSTEM_LIB_PATH,
getFsTestCaseDir().toUri().getPath() + "/systemlib");
+ private void createActionConfDirFiles() throws IOException {
new File(getTestCaseConfDir(), "action-conf").mkdir();
InputStream is =
Thread.currentThread().getContextClassLoader().getResourceAsStream("test-action-config.xml");
OutputStream os = new FileOutputStream(new File(getTestCaseConfDir() +
"/action-conf", "java.xml"));
IOUtils.copyStream(is, os);
+ }
+ private void setHadoopSystemProps() {
+ setSystemProperty("oozie.service.ActionService.executor.classes",
JavaActionExecutor.class.getName());
+
setSystemProperty("oozie.service.HadoopAccessorService.action.configurations",
+ "*=hadoop-conf," + getJobTrackerUri() +
"=action-conf");
+ setSystemProperty(WorkflowAppService.SYSTEM_LIB_PATH,
getFsTestCaseDir().toUri().getPath() + "/systemlib");
}
public void testSetupMethods() throws Exception {
@@ -625,218 +628,6 @@ public class TestJavaActionExecutor extends
ActionExecutorTestCase {
assertEquals(WorkflowAction.Status.OK,
context.getAction().getStatus());
}
- public void testLibFileArchives() throws Exception {
- Path root = new Path(getFsTestCaseDir(), "root");
-
- Path jar = new Path("jar.jar");
- getFileSystem().create(new Path(getAppPath(), jar)).close();
- Path rootJar = new Path(root, "rootJar.jar");
- getFileSystem().create(rootJar).close();
-
- Path file = new Path("file");
- getFileSystem().create(new Path(getAppPath(), file)).close();
- Path rootFile = new Path(root, "rootFile");
- getFileSystem().create(rootFile).close();
-
- Path so = new Path("soFile.so");
- getFileSystem().create(new Path(getAppPath(), so)).close();
- Path rootSo = new Path(root, "rootSoFile.so");
- getFileSystem().create(rootSo).close();
-
- Path so1 = new Path("soFile.so.1");
- getFileSystem().create(new Path(getAppPath(), so1)).close();
- Path rootSo1 = new Path(root, "rootSoFile.so.1");
- getFileSystem().create(rootSo1).close();
-
- Path archive = new Path("archive.tar");
- getFileSystem().create(new Path(getAppPath(), archive)).close();
- Path rootArchive = new Path(root, "rootArchive.tar");
- getFileSystem().create(rootArchive).close();
-
- String actionXml = "<java>" +
- " <job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
- " <name-node>" + getNameNodeUri() + "</name-node>" +
- " <main-class>CLASS</main-class>" +
- " <file>" + jar.toString() + "</file>\n" +
- " <file>" + rootJar.toString() + "</file>\n" +
- " <file>" + file.toString() + "</file>\n" +
- " <file>" + rootFile.toString() + "</file>\n" +
- " <file>" + so.toString() + "</file>\n" +
- " <file>" + rootSo.toString() + "</file>\n" +
- " <file>" + so1.toString() + "</file>\n" +
- " <file>" + rootSo1.toString() + "</file>\n" +
- " <archive>" + archive.toString() + "</archive>\n" +
- " <archive>" + rootArchive.toString() + "</archive>\n" +
- "</java>";
-
- Element eActionXml = XmlUtils.parseXml(actionXml);
-
- Context context = createContext(actionXml, null);
-
- Path appPath = getAppPath();
-
- JavaActionExecutor ae = new JavaActionExecutor();
-
- Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
- ae.setupActionConf(jobConf, context, eActionXml, appPath);
- ae.setLibFilesArchives(context, eActionXml, appPath, jobConf);
-
-
- assertTrue(DistributedCache.getSymlink(jobConf));
-
- Path[] filesInClasspath = DistributedCache.getFileClassPaths(jobConf);
- for (Path p : new Path[]{new Path(getAppPath(), jar), rootJar}) {
- boolean found = false;
- for (Path c : filesInClasspath) {
- if (!found && p.toUri().getPath().equals(c.toUri().getPath()))
{
- found = true;
- }
- }
- assertTrue("file " + p.toUri().getPath() + " not found in
classpath", found);
- }
- for (Path p : new Path[]{new Path(getAppPath(), file), rootFile, new
Path(getAppPath(), so), rootSo,
- new Path(getAppPath(), so1), rootSo1}) {
- boolean found = false;
- for (Path c : filesInClasspath) {
- if (!found && p.toUri().getPath().equals(c.toUri().getPath()))
{
- found = true;
- }
- }
- assertFalse("file " + p.toUri().getPath() + " found in classpath",
found);
- }
-
- URI[] filesInCache = DistributedCache.getCacheFiles(jobConf);
- for (Path p : new Path[]{new Path(getAppPath(), jar), rootJar, new
Path(getAppPath(), file), rootFile,
- new Path(getAppPath(), so), rootSo, new
Path(getAppPath(), so1), rootSo1}) {
- boolean found = false;
- for (URI c : filesInCache) {
- if (!found && p.toUri().getPath().equals(c.getPath())) {
- found = true;
- }
- }
- assertTrue("file " + p.toUri().getPath() + " not found in cache",
found);
- }
-
- URI[] archivesInCache = DistributedCache.getCacheArchives(jobConf);
- for (Path p : new Path[]{new Path(getAppPath(), archive),
rootArchive}) {
- boolean found = false;
- for (URI c : archivesInCache) {
- if (!found && p.toUri().getPath().equals(c.getPath())) {
- found = true;
- }
- }
- assertTrue("archive " + p.toUri().getPath() + " not found in
cache", found);
- }
- }
-
- /**
- * https://issues.apache.org/jira/browse/OOZIE-87
- * @throws Exception
- */
- public void testCommaSeparatedFilesAndArchives() throws Exception {
- Path root = new Path(getFsTestCaseDir(), "root");
-
- Path jar = new Path("jar.jar");
- getFileSystem().create(new Path(getAppPath(), jar)).close();
- Path rootJar = new Path(root, "rootJar.jar");
- getFileSystem().create(rootJar).close();
-
- Path file = new Path("file");
- getFileSystem().create(new Path(getAppPath(), file)).close();
- Path rootFile = new Path(root, "rootFile");
- getFileSystem().create(rootFile).close();
-
- Path so = new Path("soFile.so");
- getFileSystem().create(new Path(getAppPath(), so)).close();
- Path rootSo = new Path(root, "rootSoFile.so");
- getFileSystem().create(rootSo).close();
-
- Path so1 = new Path("soFile.so.1");
- getFileSystem().create(new Path(getAppPath(), so1)).close();
- Path rootSo1 = new Path(root, "rootSoFile.so.1");
- getFileSystem().create(rootSo1).close();
-
- Path archive = new Path("archive.tar");
- getFileSystem().create(new Path(getAppPath(), archive)).close();
- Path rootArchive = new Path(root, "rootArchive.tar");
- getFileSystem().create(rootArchive).close();
-
- String actionXml = "<java>" +
- " <job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
- " <name-node>" + getNameNodeUri() + "</name-node>" +
- " <main-class>CLASS</main-class>" +
- " <file>" + jar.toString() +
- "," + rootJar.toString() +
- "," + file.toString() +
- ", " + rootFile.toString() + // with leading and
trailing spaces
- " ," + so.toString() +
- "," + rootSo.toString() +
- "," + so1.toString() +
- "," + rootSo1.toString() + "</file>\n" +
- " <archive>" + archive.toString() + ", "
- + rootArchive.toString() + " </archive>\n" + //
with leading and trailing spaces
- "</java>";
-
- Element eActionXml = XmlUtils.parseXml(actionXml);
-
- Context context = createContext(actionXml, null);
-
- Path appPath = getAppPath();
-
- JavaActionExecutor ae = new JavaActionExecutor();
-
- Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
- ae.setupActionConf(jobConf, context, eActionXml, appPath);
- ae.setLibFilesArchives(context, eActionXml, appPath, jobConf);
-
-
- assertTrue(DistributedCache.getSymlink(jobConf));
-
- Path[] filesInClasspath = DistributedCache.getFileClassPaths(jobConf);
- for (Path p : new Path[]{new Path(getAppPath(), jar), rootJar}) {
- boolean found = false;
- for (Path c : filesInClasspath) {
- if (!found && p.toUri().getPath().equals(c.toUri().getPath()))
{
- found = true;
- }
- }
- assertTrue("file " + p.toUri().getPath() + " not found in
classpath", found);
- }
- for (Path p : new Path[]{new Path(getAppPath(), file), rootFile, new
Path(getAppPath(), so), rootSo,
- new Path(getAppPath(), so1), rootSo1}) {
- boolean found = false;
- for (Path c : filesInClasspath) {
- if (!found && p.toUri().getPath().equals(c.toUri().getPath()))
{
- found = true;
- }
- }
- assertFalse("file " + p.toUri().getPath() + " found in classpath",
found);
- }
-
- URI[] filesInCache = DistributedCache.getCacheFiles(jobConf);
- for (Path p : new Path[]{new Path(getAppPath(), jar), rootJar, new
Path(getAppPath(), file), rootFile,
- new Path(getAppPath(), so), rootSo, new
Path(getAppPath(), so1), rootSo1}) {
- boolean found = false;
- for (URI c : filesInCache) {
- if (!found && p.toUri().getPath().equals(c.getPath())) {
- found = true;
- }
- }
- assertTrue("file " + p.toUri().getPath() + " not found in cache",
found);
- }
-
- URI[] archivesInCache = DistributedCache.getCacheArchives(jobConf);
- for (Path p : new Path[]{new Path(getAppPath(), archive),
rootArchive}) {
- boolean found = false;
- for (URI c : archivesInCache) {
- if (!found && p.toUri().getPath().equals(c.getPath())) {
- found = true;
- }
- }
- assertTrue("archive " + p.toUri().getPath() + " not found in
cache", found);
- }
- }
-
public void testPrepare() throws Exception {
FileSystem fs = getFileSystem();
Path mkdir = new Path(getFsTestCaseDir(), "mkdir");
@@ -1371,179 +1162,6 @@ public class TestJavaActionExecutor extends
ActionExecutorTestCase {
assertEquals("JAVA-OPT3 JAVA-OPT4 JAVA-OPT1 JAVA-OPT2",
conf.get(MAPREDUCE_MAP_JAVA_OPTS));
}
- public void testActionLibsPath() throws Exception {
- // Test adding a directory
- Path actionLibPath = new Path(getFsTestCaseDir(), "actionlibs");
- getFileSystem().mkdirs(actionLibPath);
- Path jar1Path = new Path(actionLibPath, "jar1.jar");
- getFileSystem().create(jar1Path).close();
- Path jar2Path = new Path(actionLibPath, "jar2.jar");
- getFileSystem().create(jar2Path).close();
-
- String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() +
"</job-tracker>" +
- "<name-node>" + getNameNodeUri() + "</name-node>" +
"<configuration>" +
- "<property><name>oozie.launcher.oozie.libpath</name><value>" +
actionLibPath + "</value></property>" +
- "</configuration>" + "<main-class>MAIN-CLASS</main-class>" +
- "</java>";
- Element eActionXml = XmlUtils.parseXml(actionXml);
- Context context = createContext(actionXml, null);
-
- JavaActionExecutor ae = new JavaActionExecutor();
-
- Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
- ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
- ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
-
- URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
- String cacheFilesStr = Arrays.toString(cacheFiles);
- assertTrue(cacheFilesStr.contains(jar1Path.toString()));
- assertTrue(cacheFilesStr.contains(jar2Path.toString()));
-
- // Test adding a file
- Path jar3Path = new Path(getFsTestCaseDir(), "jar3.jar");
- getFileSystem().create(jar3Path).close();
-
- actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() +
"</job-tracker>" +
- "<name-node>" + getNameNodeUri() + "</name-node>" +
"<configuration>" +
- "<property><name>oozie.launcher.oozie.libpath</name><value>" +
jar3Path + "</value></property>" +
- "</configuration>" + "<main-class>MAIN-CLASS</main-class>" +
- "</java>";
- eActionXml = XmlUtils.parseXml(actionXml);
- context = createContext(actionXml, null);
-
- ae = new JavaActionExecutor();
-
- jobConf = ae.createBaseHadoopConf(context, eActionXml);
- ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
- ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
-
- cacheFiles = DistributedCache.getCacheFiles(jobConf);
- cacheFilesStr = Arrays.toString(cacheFiles);
- assertTrue(cacheFilesStr.contains(jar3Path.toString()));
-
- // Test adding a directory and a file (comma separated)
- actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() +
"</job-tracker>" +
- "<name-node>" + getNameNodeUri() + "</name-node>" +
"<configuration>" +
- "<property><name>oozie.launcher.oozie.libpath</name><value>" +
actionLibPath + "," + jar3Path +
- "</value></property>" +
- "</configuration>" + "<main-class>MAIN-CLASS</main-class>" +
- "</java>";
- eActionXml = XmlUtils.parseXml(actionXml);
- context = createContext(actionXml, null);
-
- ae = new JavaActionExecutor();
-
- jobConf = ae.createBaseHadoopConf(context, eActionXml);
- ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
- ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
-
- cacheFiles = DistributedCache.getCacheFiles(jobConf);
- cacheFilesStr = Arrays.toString(cacheFiles);
- assertTrue(cacheFilesStr.contains(jar1Path.toString()));
- assertTrue(cacheFilesStr.contains(jar2Path.toString()));
- assertTrue(cacheFilesStr.contains(jar3Path.toString()));
- }
-
- @Test
- public void testAddActionShareLib() throws Exception {
-
- WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
-
- Path systemLibPath = new Path(wps.getSystemLibPath(),
ShareLibService.SHARE_LIB_PREFIX
- + new SimpleDateFormat("yyyyMMddHHmmss").format(new
Date()).toString());
-
- Path javaShareLibPath = new Path(systemLibPath, "java");
- getFileSystem().mkdirs(javaShareLibPath);
- Path jar1Path = new Path(javaShareLibPath, "jar1.jar");
- getFileSystem().create(jar1Path).close();
- Path jar2Path = new Path(javaShareLibPath, "jar2.jar");
- getFileSystem().create(jar2Path).close();
-
- Path hcatShareLibPath = new Path(systemLibPath, "hcat");
- getFileSystem().mkdirs(hcatShareLibPath);
- Path jar3Path = new Path(hcatShareLibPath, "jar3.jar");
- getFileSystem().create(jar3Path).close();
- Path jar4Path = new Path(hcatShareLibPath, "jar4.jar");
- getFileSystem().create(jar4Path).close();
-
- Path otherShareLibPath = new Path(systemLibPath, "other");
- getFileSystem().mkdirs(otherShareLibPath);
- Path jar5Path = new Path(otherShareLibPath, "jar5.jar");
- getFileSystem().create(jar5Path).close();
-
- String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() +
"</job-tracker>" +
- "<name-node>" + getNameNodeUri() + "</name-node>" +
- "<main-class>MAIN-CLASS</main-class>" +
- "</java>";
- Element eActionXml = XmlUtils.parseXml(actionXml);
- Context context = createContext(actionXml, null);
-
- Services.get().setService(ShareLibService.class);
-
- // Test oozie server action sharelib setting
- WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow();
- XConfiguration wfConf = new XConfiguration();
- wfConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
- wfConf.set(OozieClient.APP_PATH, new Path(getAppPath(),
"workflow.xml").toString());
- wfConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
- workflow.setConf(XmlUtils.prettyPrint(wfConf).toString());
-
- ConfigurationService.set("oozie.action.sharelib.for.java",
"java,hcat");
-
- JavaActionExecutor ae = new JavaActionExecutor();
-
- Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
- ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
- try {
- ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
- fail();
- } catch (ActionExecutorException aee) {
- assertEquals("EJ001", aee.getErrorCode());
- assertEquals("Could not locate Oozie sharelib", aee.getMessage());
- }
- Path launcherPath = new Path(systemLibPath, "oozie");
- getFileSystem().mkdirs(launcherPath);
- Path jar6Path = new Path(launcherPath, "jar6.jar");
- getFileSystem().create(jar6Path).close();
- Services.get().get(ShareLibService.class).updateShareLib();
- ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
-
- URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
- String cacheFilesStr = Arrays.toString(cacheFiles);
- assertTrue(cacheFilesStr.contains(jar1Path.toString()));
- assertTrue(cacheFilesStr.contains(jar2Path.toString()));
- assertTrue(cacheFilesStr.contains(jar3Path.toString()));
- assertTrue(cacheFilesStr.contains(jar4Path.toString()));
- assertFalse(cacheFilesStr.contains(jar5Path.toString()));
- assertTrue(cacheFilesStr.contains(jar6Path.toString()));
-
- // Test per workflow action sharelib setting
- workflow = (WorkflowJobBean) context.getWorkflow();
- wfConf = new XConfiguration();
- wfConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
- wfConf.set(OozieClient.APP_PATH, new Path(getAppPath(),
"workflow.xml").toString());
- wfConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
- wfConf.set("oozie.action.sharelib.for.java", "other,hcat");
- workflow.setConf(XmlUtils.prettyPrint(wfConf).toString());
-
- ConfigurationService.set("oozie.action.sharelib.for.java", "java");
- ae = new JavaActionExecutor();
-
- jobConf = ae.createBaseHadoopConf(context, eActionXml);
- ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
- ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
-
- cacheFiles = DistributedCache.getCacheFiles(jobConf);
- cacheFilesStr = Arrays.toString(cacheFiles);
- // The oozie server setting should have been overridden by workflow
setting
- assertFalse(cacheFilesStr.contains(jar1Path.toString()));
- assertFalse(cacheFilesStr.contains(jar2Path.toString()));
- assertTrue(cacheFilesStr.contains(jar3Path.toString()));
- assertTrue(cacheFilesStr.contains(jar4Path.toString()));
- assertTrue(cacheFilesStr.contains(jar5Path.toString()));
- assertTrue(cacheFilesStr.contains(jar6Path.toString()));
- }
-
public void testAddShareLibSchemeAndAuthority() throws Exception {
JavaActionExecutor ae = new JavaActionExecutor() {
@Override
@@ -1563,8 +1181,7 @@ public class TestJavaActionExecutor extends
ActionExecutorTestCase {
new Services().init();
// Create the dir
WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
- Path systemLibPath = new Path(wps.getSystemLibPath(),
ShareLibService.SHARE_LIB_PREFIX
- + new SimpleDateFormat("yyyyMMddHHmmss").format(new
Date()).toString());
+ Path systemLibPath = getNewSystemLibPath();
Path javaShareLibPath = new Path(systemLibPath,
"java-action-executor");
getFileSystem().mkdirs(javaShareLibPath);
Services.get().setService(ShareLibService.class);
@@ -1851,7 +1468,8 @@ public class TestJavaActionExecutor extends
ActionExecutorTestCase {
// Test when server side setting is not enabled
Configuration launcherConf = ae.createLauncherConf(getFileSystem(),
context, action, actionXml, actionConf);
- assertEquals("false",
launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED)); //
disabled by default
+ // disabled by default
+ assertEquals("false",
launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED));
ConfigurationService.set("oozie.action.launcher." +
JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED, "true");
@@ -2129,8 +1747,7 @@ public class TestJavaActionExecutor extends
ActionExecutorTestCase {
WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
- Path systemLibPath = new Path(wps.getSystemLibPath(),
ShareLibService.SHARE_LIB_PREFIX
- + new SimpleDateFormat("yyyyMMddHHmmss").format(new
Date()).toString());
+ Path systemLibPath = getNewSystemLibPath();
File jarFile = IOUtils.createJar(new File(getTestCaseDir()),
"sourcejar.jar", LauncherMainTester.class);
InputStream is = new FileInputStream(jarFile);
diff --git
a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutorLibAddition.java
b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutorLibAddition.java
new file mode 100644
index 0000000..00e7777
--- /dev/null
+++
b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutorLibAddition.java
@@ -0,0 +1,448 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.WorkflowJobBean;
+import org.apache.oozie.action.ActionExecutorException;
+import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.service.ConfigurationService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.ShareLibService;
+import org.apache.oozie.service.WorkflowAppService;
+import org.apache.oozie.util.IOUtils;
+import org.apache.oozie.util.XConfiguration;
+import org.apache.oozie.util.XmlUtils;
+import org.jdom.Element;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+
+import static
org.apache.oozie.action.hadoop.JavaActionExecutor.ACTION_SHARELIB_FOR;
+import static
org.apache.oozie.action.hadoop.JavaActionExecutor.SHARELIB_EXCLUDE_SUFFIX;
+
+public class TestJavaActionExecutorLibAddition extends ActionExecutorTestCase {
+
+ private static final String[] TEST_SHARELIB_OOZIE_FILES = {
+ "jackson-core-2.3.jar",
+ "jackson-databind-2.3.jar",
+ "other-lib.jar",
+ "oozie-library.jar",
+ };
+
+ private static final String[] TEST_SHARELIB_JAVA_FILES = {
+ "jackson-core-2.6.5.jar",
+ "jackson-databind-2.6.5.jar",
+ "some-lib.jar",
+ "another-lib.jar",
+ };
+
+ private static final String[] TEST_SHARELIB_PIG_FILES = {
+ "jackson-pig-0.3.3.jar",
+ "jackson-datapig-0.3.5.jar",
+ "pig_data.txt",
+ };
+
+ private static final String[] TEST_SHARELIB_USER_FILES = {
+ "jackson-user-3.3.jar",
+ "jackson-workflow-app.jar",
+ "user-job-utils.jar",
+ "jackson-files.zip"
+ };
+
+ private static final String[] TEST_SHARELIB_FILES = {
+ "soFile.so",
+ "soFile.so.1",
+ "file",
+ "jar.jar"
+ };
+
+ private static final String[] TEST_SHARELIB_ROOT_FILES = {
+ "rootSoFile.so",
+ "rootSoFile.so.1",
+ "rootFile",
+ "rootJar.jar"
+ };
+
+ private static final String[] TEST_SHARELIB_ARCHIVES = {
+ "archive.tar",
+ "rootArchive.tar",
+ };
+
+ @Override
+ protected void setSystemProps() throws Exception {
+ super.setSystemProps();
+ setHadoopSystemProps();
+ createActionConfDirFiles();
+ }
+
+ private void createActionConfDirFiles() throws IOException {
+ new File(getTestCaseConfDir(), "action-conf").mkdir();
+ InputStream is =
Thread.currentThread().getContextClassLoader().getResourceAsStream("test-action-config.xml");
+ OutputStream os = new FileOutputStream(new File(getTestCaseConfDir() +
"/action-conf", "java.xml"));
+ IOUtils.copyStream(is, os);
+ }
+
+ private void setHadoopSystemProps() {
+ setSystemProperty("oozie.service.ActionService.executor.classes",
JavaActionExecutor.class.getName());
+
setSystemProperty("oozie.service.HadoopAccessorService.action.configurations",
+ "*=hadoop-conf," + getJobTrackerUri() + "=action-conf");
+ setSystemProperty(WorkflowAppService.SYSTEM_LIB_PATH,
getFsTestCaseDir().toUri().getPath() + "/systemlib");
+ }
+
+ private Map<String, Path> setupTestShareLibExcludeTestJars(final Path
systemLibPath) throws Exception {
+ final Path oozieShareLibPath = new Path(systemLibPath, "oozie");
+ final Path javaShareLibPath = new Path(systemLibPath, "java");
+ final Path pigShareLibPath = new Path(systemLibPath, new Path("pig",
"lib"));
+ final Path actionLibPath = new Path(getAppPath(), "lib");
+ makeDirs(oozieShareLibPath, javaShareLibPath, pigShareLibPath,
actionLibPath);
+
+ final Map<String, Path> libs = new LinkedHashMap<>();
+ loadLibPathsToMap(libs, oozieShareLibPath, TEST_SHARELIB_OOZIE_FILES);
+ loadLibPathsToMap(libs, javaShareLibPath, TEST_SHARELIB_JAVA_FILES);
+ loadLibPathsToMap(libs, pigShareLibPath, TEST_SHARELIB_PIG_FILES);
+ loadLibPathsToMap(libs, actionLibPath, TEST_SHARELIB_USER_FILES);
+ createFiles(libs.values());
+
+ return libs;
+ }
+
+ private void loadLibPathsToMap(Map<String, Path> libMap, Path root,
String... files) {
+ for (String f : files) {
+ libMap.put(f, new Path(root, f));
+ }
+ }
+
+ private void checkLibExclude(final String... libsToBeExcluded) throws
Exception {
+ final int LIBS_ADDED_BY_TESTS = 6;
+ final Map<String, Path> libs =
setupTestShareLibExcludeTestJars(getNewSystemLibPath());
+ final Path actionLibPath = new Path(getAppPath(), "lib");
+ final Context context = createContextUsingSharelib(actionLibPath);
+
+ createWorkflowJobUsingSharelib(context);
+ setSharelibForActionInConfiguration("java,pig");
+
+ final Configuration jobConf =
createActionExecutorAndSetupServices(context);
+ final URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
+ final String cacheFilesStr = getDistributedCacheFilesStr(jobConf);
+
+ for (final String lib : libsToBeExcluded) {
+ assertFalse(lib + " should have been excluded from distributed
cache",
+ cacheFilesStr.contains(libs.get(lib).toString()));
+ }
+ assertEquals("The number of files on distributed cache is not what
expected.",
+ libs.size() - libsToBeExcluded.length + LIBS_ADDED_BY_TESTS,
cacheFiles.length);
+ }
+
+ private Context createContextUsingSharelib(final Path actionLibPath)
throws Exception {
+ final String actionXml = String.format("<java>" +
+ "<job-tracker>%s</job-tracker>" +
+ "<name-node>%s</name-node>" +
+ "<configuration>" +
+ "<property>" +
+ "<name>oozie.launcher.oozie.libpath</name>" +
+ "<value>%s</value>" +
+ "</property>" +
+ "</configuration>" +
+ "<main-class>MAIN-CLASS</main-class>" +
+ "</java>", getJobTrackerUri(), getNameNodeUri(),
actionLibPath);
+
+ return createContext(actionXml, null);
+ }
+
+ private void createWorkflowJobUsingSharelib(final Context context, final
XConfiguration wfConf) {
+ final WorkflowJobBean workflow = (WorkflowJobBean)
context.getWorkflow();
+ wfConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+ wfConf.set(OozieClient.APP_PATH, new Path(getAppPath(),
"workflow.xml").toString());
+ wfConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
+ workflow.setConf(XmlUtils.prettyPrint(wfConf).toString());
+ }
+
+ private void createWorkflowJobUsingSharelib(final Context context) {
+ createWorkflowJobUsingSharelib(context, new XConfiguration());
+ }
+
+ private Configuration createActionExecutorAndSetupServices(final Context
context) throws Exception {
+ Services.get().setService(ShareLibService.class);
+ final Element eActionXml =
XmlUtils.parseXml(context.getAction().getConf());
+ final JavaActionExecutor ae = new JavaActionExecutor();
+ final Configuration jobConf = ae.createBaseHadoopConf(context,
eActionXml);
+ ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
+ Services.get().get(ShareLibService.class).updateShareLib();
+ ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
+ return jobConf;
+ }
+
+ private Configuration createActionExecutorAndSetLibFilesArchives(final
Context context) throws Exception {
+ final Element eActionXml =
XmlUtils.parseXml(context.getAction().getConf());
+ final JavaActionExecutor ae = new JavaActionExecutor();
+ final Configuration jobConf = ae.createBaseHadoopConf(context,
eActionXml);
+ ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
+ ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
+ return jobConf;
+ }
+
+ private void setExcludePatternInConfiguration(String pattern) {
+ ConfigurationService.set(ACTION_SHARELIB_FOR + "java" +
SHARELIB_EXCLUDE_SUFFIX, pattern);
+ }
+
+ private void setSharelibForActionInConfiguration(String libs) {
+ ConfigurationService.set(ACTION_SHARELIB_FOR + "java", libs);
+ }
+
+ public void testExcludeFilesFromAllSharelibLocation() throws Exception {
+ setExcludePatternInConfiguration(".*jackson.*");
+ checkLibExclude(TEST_SHARELIB_OOZIE_FILES[0],
TEST_SHARELIB_OOZIE_FILES[1],
+ TEST_SHARELIB_JAVA_FILES[0], TEST_SHARELIB_JAVA_FILES[1],
+ TEST_SHARELIB_PIG_FILES[0], TEST_SHARELIB_PIG_FILES[1],
+ TEST_SHARELIB_USER_FILES[0], TEST_SHARELIB_USER_FILES[1],
TEST_SHARELIB_USER_FILES[3]);
+ }
+
+ public void testExcludeFilesFromAllOozieSharelibFolder() throws Exception {
+ setExcludePatternInConfiguration("oozie/jackson.*");
+ checkLibExclude(TEST_SHARELIB_OOZIE_FILES[0],
TEST_SHARELIB_OOZIE_FILES[1]);
+ }
+
+ public void testExcludeFilesFromMultipleLocations() throws Exception {
+
setExcludePatternInConfiguration("pig/lib/jackson.*|java/jackson.*|oozie/jackson.*");
+ checkLibExclude(TEST_SHARELIB_OOZIE_FILES[0],
TEST_SHARELIB_OOZIE_FILES[1],
+ TEST_SHARELIB_JAVA_FILES[0], TEST_SHARELIB_JAVA_FILES[1],
+ TEST_SHARELIB_PIG_FILES[0], TEST_SHARELIB_PIG_FILES[1]);
+ }
+
+ public void testExcludeUserProvidedFiles() throws Exception {
+ setExcludePatternInConfiguration(".*/app/lib/jackson.*");
+ checkLibExclude(TEST_SHARELIB_USER_FILES[0],
TEST_SHARELIB_USER_FILES[1], TEST_SHARELIB_USER_FILES[3]);
+ }
+
+ public void testAddActionShareLib() throws Exception {
+ final Path systemLibPath = getNewSystemLibPath();
+ final Path actionLibPath = new Path(getAppPath(), "lib");
+
+ final Path javaShareLibPath = new Path(systemLibPath, "java");
+ final Path jar1Path = new Path(javaShareLibPath, "jar1.jar");
+ final Path jar2Path = new Path(javaShareLibPath, "jar2.jar");
+
+ final Path hcatShareLibPath = new Path(systemLibPath, "hcat");
+ final Path jar3Path = new Path(hcatShareLibPath, "jar3.jar");
+ final Path jar4Path = new Path(hcatShareLibPath, "jar4.jar");
+
+ final Path otherShareLibPath = new Path(systemLibPath, "other");
+ final Path jar5Path = new Path(otherShareLibPath, "jar5.jar");
+
+ makeDirs(javaShareLibPath, hcatShareLibPath, otherShareLibPath);
+ createFiles(Arrays.asList(jar1Path, jar2Path, jar3Path, jar4Path,
jar5Path));
+
+ final Context context = createContextUsingSharelib(actionLibPath);
+ createWorkflowJobUsingSharelib(context);
+ setSharelibForActionInConfiguration("java,hcat");
+
+ try {
+ createActionExecutorAndSetupServices(context);
+ fail("Expected ActionExecutorException to be thrown, but got
nothing.");
+ }
+ catch (ActionExecutorException aee) {
+ assertEquals("Unexpected error code. Message: " +
aee.getMessage(), "EJ001", aee.getErrorCode());
+ assertEquals("Unexpected error message","Could not locate Oozie
sharelib", aee.getMessage());
+ }
+
+ final Path launcherPath = new Path(systemLibPath, "oozie");
+ final Path jar6Path = new Path(launcherPath, "jar6.jar");
+ makeDirs(launcherPath);
+ getFileSystem().create(jar6Path).close();
+
+ Configuration jobConf = createActionExecutorAndSetupServices(context);
+ String cacheFilesStr = getDistributedCacheFilesStr(jobConf);
+
+ assertContainsJars( cacheFilesStr, Arrays.asList(jar1Path, jar2Path,
jar3Path, jar4Path, jar6Path));
+ assertNotContainsJars( cacheFilesStr,
Collections.singletonList(jar5Path));
+
+ final XConfiguration wfConf = new XConfiguration();
+ wfConf.set(ACTION_SHARELIB_FOR + "java", "other,hcat");
+ createWorkflowJobUsingSharelib(context, wfConf);
+ setSharelibForActionInConfiguration("java");
+ jobConf = createActionExecutorAndSetupServices(context);
+
+ // The oozie server setting should have been overridden by workflow
setting
+ cacheFilesStr = getDistributedCacheFilesStr(jobConf);
+ assertContainsJars(cacheFilesStr, Arrays.asList(jar3Path, jar4Path,
jar5Path, jar6Path));
+ assertNotContainsJars(cacheFilesStr, Arrays.asList(jar1Path,
jar2Path));
+ }
+
+ private Path getActionLibPath() throws Exception {
+ Path actionLibPath = new Path(getFsTestCaseDir(), "actionlibs");
+ makeDirs(actionLibPath);
+ return actionLibPath;
+ }
+
+ private List<Path> createTestActionLibPaths(Path... paths) throws
Exception{
+ final Path actionLibPath = new Path(getFsTestCaseDir(), "actionlibs");
+ makeDirs(actionLibPath);
+ createFiles(Arrays.asList(paths));
+ return Arrays.asList(paths);
+ }
+
+ public void testAddingActionLibDir() throws Exception{
+ makeDirs(getActionLibPath());
+ List<Path> expectedJars = createTestActionLibPaths(
+ new Path(getActionLibPath(), "jar1.jar"),
+ new Path(getActionLibPath(), "jar2.jar"));
+
+ Context context = createContextUsingSharelib(getActionLibPath());
+ Configuration jobConf =
createActionExecutorAndSetLibFilesArchives(context);
+ assertContainsJars(getDistributedCacheFilesStr(jobConf), expectedJars);
+ }
+
+ public void testAddingActionLibFile() throws Exception{
+ List<Path> expectedJars = createTestActionLibPaths(new
Path(getFsTestCaseDir(), "jar3.jar"));
+
+ String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() +
"</job-tracker>" +
+ "<name-node>" + getNameNodeUri() + "</name-node>" +
"<configuration>" +
+ "<property><name>oozie.launcher.oozie.libpath</name><value>" +
expectedJars.get(0) + "</value></property>" +
+ "</configuration>" + "<main-class>MAIN-CLASS</main-class>" +
+ "</java>";
+ Context context = createContext(actionXml, null);
+ Configuration jobConf =
createActionExecutorAndSetLibFilesArchives(context);
+ assertContainsJars(getDistributedCacheFilesStr(jobConf), expectedJars);
+ }
+
+ public void testActionLibFileAndDir() throws Exception {
+ makeDirs(getActionLibPath());
+ List<Path> expectedJars = createTestActionLibPaths(
+ new Path(getActionLibPath(), "jar1.jar"),
+ new Path(getActionLibPath(), "jar2.jar"),
+ new Path(getFsTestCaseDir(), "jar3.jar"));
+
+ String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() +
"</job-tracker>" +
+ "<name-node>" + getNameNodeUri() + "</name-node>" +
"<configuration>" +
+ "<property><name>oozie.launcher.oozie.libpath</name><value>" +
getActionLibPath() + "," + expectedJars.get(2) +
+ "</value></property>" +
+ "</configuration>" + "<main-class>MAIN-CLASS</main-class>" +
+ "</java>";
+ Context context = createContext(actionXml, null);
+ Configuration jobConf =
createActionExecutorAndSetLibFilesArchives(context);
+ assertContainsJars(getDistributedCacheFilesStr(jobConf), expectedJars);
+ }
+
+ private String getDistributedCacheFilesStr(final Configuration jobConf)
throws IOException {
+ return Arrays.toString(DistributedCache.getCacheFiles(jobConf));
+ }
+
+ private Map<String, Path> createAndGetSharelibTestFiles() throws Exception
{
+ Path rootPath = new Path(getFsTestCaseDir(), "root");
+ Map<String, Path> libs = new LinkedHashMap<>();
+ loadLibPathsToMap(libs, getAppPath(), TEST_SHARELIB_FILES);
+ loadLibPathsToMap(libs, rootPath, TEST_SHARELIB_ROOT_FILES);
+ loadLibPathsToMap(libs, getAppPath(), TEST_SHARELIB_ARCHIVES[0]);
+ loadLibPathsToMap(libs, rootPath, TEST_SHARELIB_ARCHIVES[1]);
+ createFiles(libs.values());
+ return libs;
+ }
+
+ public void testLibFileArchives() throws Exception {
+ Map<String, Path> libs = createAndGetSharelibTestFiles();
+
+ final String actionXml = "<java>" +
+ " <job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
+ " <name-node>" + getNameNodeUri() + "</name-node>" +
+ " <main-class>CLASS</main-class>" +
+ " <file>" + libs.get(TEST_SHARELIB_FILES[0]).toString() +
"</file>\n" +
+ " <file>" + libs.get(TEST_SHARELIB_FILES[1]).toString() +
"</file>\n" +
+ " <file>" + libs.get(TEST_SHARELIB_FILES[2]).toString() +
"</file>\n" +
+ " <file>" + libs.get(TEST_SHARELIB_FILES[3]).toString() +
"</file>\n" +
+ " <file>" +
libs.get(TEST_SHARELIB_ROOT_FILES[0]).toString() + "</file>\n" +
+ " <file>" +
libs.get(TEST_SHARELIB_ROOT_FILES[1]).toString() + "</file>\n" +
+ " <file>" +
libs.get(TEST_SHARELIB_ROOT_FILES[2]).toString() + "</file>\n" +
+ " <file>" +
libs.get(TEST_SHARELIB_ROOT_FILES[3]).toString() + "</file>\n" +
+ " <archive>" +
libs.get(TEST_SHARELIB_ARCHIVES[0]).toString() + "</archive>\n" +
+ " <archive>" +
libs.get(TEST_SHARELIB_ARCHIVES[1]).toString() + "</archive>\n" +
+ "</java>";
+
+ final Configuration jobConf =
createActionExecutorAndSetLibFilesArchives(createContext(actionXml, null));
+ verifyFilesInDistributedCache(libs, jobConf);
+ }
+
+ public void testCommaSeparatedFilesAndArchives() throws Exception {
+ Map<String, Path> libs = createAndGetSharelibTestFiles();
+
+ final String actionXml = "<java>" +
+ " <job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
+ " <name-node>" + getNameNodeUri() + "</name-node>" +
+ " <main-class>CLASS</main-class>" +
+ " <file>" + libs.get(TEST_SHARELIB_FILES[3]).toString() +
+ "," + libs.get(TEST_SHARELIB_ROOT_FILES[3]).toString() +
+ "," + libs.get(TEST_SHARELIB_FILES[2]).toString() +
+ ", " + libs.get(TEST_SHARELIB_ROOT_FILES[2]).toString() + //
with leading and trailing spaces
+ " ," + libs.get(TEST_SHARELIB_FILES[0]).toString() +
+ "," + libs.get(TEST_SHARELIB_ROOT_FILES[0]).toString() +
+ "," + libs.get(TEST_SHARELIB_FILES[1]).toString() +
+ "," + libs.get(TEST_SHARELIB_ROOT_FILES[1]).toString() +
"</file>\n" +
+ " <archive>" +
libs.get(TEST_SHARELIB_ARCHIVES[0]).toString() + ", "
+ + libs.get(TEST_SHARELIB_ARCHIVES[1]).toString() + "
</archive>\n" + // with leading and trailing spaces
+ "</java>";
+
+ final Configuration jobConf =
createActionExecutorAndSetLibFilesArchives(createContext(actionXml, null));
+ verifyFilesInDistributedCache(libs, jobConf);
+ }
+
+ private void verifyFilesInDistributedCache(Map<String, Path> libs, final
Configuration jobConf) throws Exception {
+ assertTrue(DistributedCache.getSymlink(jobConf));
+
+ String filesInClassPathString =
Arrays.toString(DistributedCache.getFileClassPaths(jobConf));
+ assertContainsJars(filesInClassPathString, Arrays.asList(
+ libs.get(TEST_SHARELIB_FILES[3]),
+ libs.get(TEST_SHARELIB_ROOT_FILES[3])));
+
+ assertNotContainsJars(filesInClassPathString, Arrays.asList(
+ libs.get(TEST_SHARELIB_FILES[0]),
+ libs.get(TEST_SHARELIB_FILES[1]),
+ libs.get(TEST_SHARELIB_FILES[2]),
+ libs.get(TEST_SHARELIB_ROOT_FILES[0]),
+ libs.get(TEST_SHARELIB_ROOT_FILES[1]),
+ libs.get(TEST_SHARELIB_ROOT_FILES[2])));
+
+ filesInClassPathString =
Arrays.toString(DistributedCache.getCacheFiles(jobConf));
+ assertContainsJars(filesInClassPathString, Arrays.asList(
+ libs.get(TEST_SHARELIB_FILES[0]),
+ libs.get(TEST_SHARELIB_FILES[1]),
+ libs.get(TEST_SHARELIB_FILES[2]),
+ libs.get(TEST_SHARELIB_FILES[3]),
+ libs.get(TEST_SHARELIB_ROOT_FILES[0]),
+ libs.get(TEST_SHARELIB_ROOT_FILES[1]),
+ libs.get(TEST_SHARELIB_ROOT_FILES[2]),
+ libs.get(TEST_SHARELIB_ROOT_FILES[3])));
+
+ filesInClassPathString =
Arrays.toString(DistributedCache.getCacheArchives(jobConf));
+ assertContainsJars(filesInClassPathString, Arrays.asList(
+ libs.get(TEST_SHARELIB_ARCHIVES[0]),
+ libs.get(TEST_SHARELIB_ARCHIVES[1])));
+ }
+}
diff --git
a/core/src/test/java/org/apache/oozie/action/hadoop/TestShareLibExcluder.java
b/core/src/test/java/org/apache/oozie/action/hadoop/TestShareLibExcluder.java
new file mode 100644
index 0000000..e92bd3f
--- /dev/null
+++
b/core/src/test/java/org/apache/oozie/action/hadoop/TestShareLibExcluder.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.oozie.action.hadoop;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.util.XConfiguration;
+
+import static
org.apache.oozie.action.hadoop.JavaActionExecutor.ACTION_SHARELIB_FOR;
+import static
org.apache.oozie.action.hadoop.JavaActionExecutor.SHARELIB_EXCLUDE_SUFFIX;
+import static org.apache.oozie.action.hadoop.ShareLibExcluder.VALUE_NULL_MSG;
+
+public class TestShareLibExcluder {
+
+ private static final String[] EMPTY_ARRAY = {};
+ private ShareLibExcluder shareLibExcluder;
+ private Configuration actionConf;
+ private Configuration servicesConf;
+ private XConfiguration jobConf;
+ private static final String executorType = "spark";
+ private static final String excludeProperty = ACTION_SHARELIB_FOR +
executorType + SHARELIB_EXCLUDE_SUFFIX;
+ private static URI sharelibRootURI;
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ private static final String SHARELIB_ROOT =
"/user/oozie/share/lib/lib20180612/";
+ private static final String[] OOZIE_SHARELIB_JARS = {
+ SHARELIB_ROOT + "oozie/jackson-core-2.3.jar",
+ SHARELIB_ROOT + "oozie/jackson-databind-2.3.jar",
+ SHARELIB_ROOT + "oozie/other-lib.jar",
+ SHARELIB_ROOT + "oozie/oozie-library.jar"
+ };
+ private static final String[] SPARK_SHARELIB_JARS = {
+ SHARELIB_ROOT + "spark/jackson-core-2.6.5.jar",
+ SHARELIB_ROOT + "spark/jackson-databind-2.6.5.jar",
+ SHARELIB_ROOT + "spark/spark-lib.jar",
+ SHARELIB_ROOT + "spark/lib/another-lib.jar"
+ };
+ private static final String[] PIG_SHARELIB_JARS = {
+ SHARELIB_ROOT + "pig/lib/jackson-pig-0.3.3.jar",
+ SHARELIB_ROOT + "pig/lib/jackson-datapig-0.3.5.jar",
+ SHARELIB_ROOT + "pig/temp/pig_data.txt"
+ };
+
+ private List<String> libs;
+
+ private static final String PIG_LIB_PATTERN = "pig/lib.*";
+ private static final String ALL_JARS_PATTERN = ".*";
+ private static final String ALL_JACKSON_PATTERN = ".*jackson.*";
+ private static final String OOZIE_JACKSON_PATTERN = "oozie/jackson.*";
+ private static final String SPARK_JACKSON_PATTERN = "spark/jackson.*";
+ private static final String PIG_LIB_JACKSON_SPARK_JACKSON_PATTERN =
"pig/lib/jackson.*|spark/jackson.*";
+ private static final String ALL_EXCEPT_OOZIE_PATTERN = "^(?!.*oozie).*$";
+
+ @Before
+ public void setUp() {
+ libs = new LinkedList<>();
+ libs.addAll(Arrays.asList(OOZIE_SHARELIB_JARS));
+ libs.addAll(Arrays.asList(SPARK_SHARELIB_JARS));
+ libs.addAll(Arrays.asList(PIG_SHARELIB_JARS));
+
+ sharelibRootURI = URI.create(SHARELIB_ROOT);
+ actionConf = new Configuration();
+ servicesConf = new Configuration();
+ jobConf = new XConfiguration();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ shareLibExcluder = null;
+ }
+
+ private void doShareLibExclude(final String excludePattern, final
String... libsExpectedToExclude) {
+ doShareLibExclude(excludePattern, sharelibRootURI,
libsExpectedToExclude);
+ }
+
+ private void doShareLibExclude(final String excludePattern, URI rootURI,
final String... libsExpectedToExclude) {
+ jobConf.set(excludeProperty, excludePattern);
+ shareLibExcluder = new ShareLibExcluder(actionConf, servicesConf,
jobConf, executorType, rootURI);
+ checkShareLibExclude(libsExpectedToExclude);
+ }
+
+ private void checkShareLibExclude(final String... libsExpectedToExclude) {
+ libs.removeAll(Arrays.asList(libsExpectedToExclude));
+
+ for (String excludeLibPath : libsExpectedToExclude) {
+ Assert.assertTrue("Lib path is not excluded, but should be",
+
shareLibExcluder.shouldExclude(URI.create(excludeLibPath)));
+ }
+
+ for (String dontExcludeLibPath : libs) {
+ Assert.assertFalse("Lib path is excluded, but should not be",
+
shareLibExcluder.shouldExclude(URI.create(dontExcludeLibPath)));
+ }
+ }
+
+ @Test
+ public void testExcludeLibsFromEverywhere() {
+ doShareLibExclude(ALL_JACKSON_PATTERN,
+ OOZIE_SHARELIB_JARS[0], OOZIE_SHARELIB_JARS[1],
+ SPARK_SHARELIB_JARS[0], SPARK_SHARELIB_JARS[1],
+ PIG_SHARELIB_JARS[0], PIG_SHARELIB_JARS[1]);
+ }
+
+ @Test
+ public void testExcludeLibsFromOozieDirOnly() {
+ doShareLibExclude(OOZIE_JACKSON_PATTERN, OOZIE_SHARELIB_JARS[0],
OOZIE_SHARELIB_JARS[1]);
+ }
+
+ @Test
+ public void testExcludeLibsFromSparkDirOnly() {
+ doShareLibExclude(SPARK_JACKSON_PATTERN, SPARK_SHARELIB_JARS[0],
SPARK_SHARELIB_JARS[1]);
+ }
+
+ @Test
+ public void testExcludeLibsFromPigAndSparkDirOnly() {
+ doShareLibExclude(PIG_LIB_JACKSON_SPARK_JACKSON_PATTERN,
+ PIG_SHARELIB_JARS[0], PIG_SHARELIB_JARS[1],
+ SPARK_SHARELIB_JARS[0], SPARK_SHARELIB_JARS[1]);
+ }
+
+ @Test
+ public void testDontExcludeSharelibRoot() {
+ doShareLibExclude(SHARELIB_ROOT + PIG_LIB_PATTERN, EMPTY_ARRAY);
+ }
+
+ @Test
+ public void testDontExcludeWithNotSetExcludeConfiguration() {
+ shareLibExcluder = new ShareLibExcluder(actionConf, servicesConf,
jobConf, executorType, sharelibRootURI);
+ checkShareLibExclude(EMPTY_ARRAY);
+ }
+
+ @Test
+ public void testDontExcludeWithNullRootURIValue() {
+ doShareLibExclude(PIG_LIB_PATTERN, null, EMPTY_ARRAY);
+ }
+
+ @Test
+ public void testDontExcludeWithNullExecutorType() {
+ jobConf.set(excludeProperty, PIG_LIB_PATTERN);
+ shareLibExcluder = new ShareLibExcluder(actionConf, servicesConf,
jobConf, null, sharelibRootURI);
+ checkShareLibExclude(EMPTY_ARRAY);
+ }
+
+ @Test
+ public void testForExceptionsWhenIllegalActionConfParameterPassed() {
+ thrown.expect(NullPointerException.class);
+ thrown.expectMessage(String.format(VALUE_NULL_MSG, "actionConf"));
+ shareLibExcluder = new ShareLibExcluder(null, servicesConf, jobConf,
executorType, sharelibRootURI);
+ }
+
+ @Test
+ public void testForExceptionsWhenIllegalServicesConfParameterPassed() {
+ thrown.expect(NullPointerException.class);
+ thrown.expectMessage(String.format(VALUE_NULL_MSG, "servicesConf"));
+ shareLibExcluder = new ShareLibExcluder(actionConf, null, jobConf,
executorType, sharelibRootURI);
+ }
+
+ @Test
+ public void testForExceptionsWhenIllegalJobConfParameterPassed() {
+ thrown.expect(NullPointerException.class);
+ thrown.expectMessage(String.format(VALUE_NULL_MSG, "jobConf"));
+ shareLibExcluder = new ShareLibExcluder(actionConf, servicesConf,
null, executorType, sharelibRootURI);
+ }
+
+ @Test
+ public void testGetExcludePropertyFromConfigurations() {
+ actionConf.set(excludeProperty, ALL_JARS_PATTERN);
+ jobConf.set(excludeProperty, ALL_JARS_PATTERN);
+
+ shareLibExcluder = new ShareLibExcluder(actionConf, servicesConf,
jobConf, executorType, sharelibRootURI);
+ jobConf.unset(excludeProperty);
+ Assert.assertNotNull(actionConf.get(excludeProperty, null));
+
+ actionConf.unset(excludeProperty);
+ servicesConf.set(excludeProperty, ALL_JARS_PATTERN);
+ shareLibExcluder = new ShareLibExcluder(actionConf, servicesConf,
jobConf, executorType, sharelibRootURI);
+ Assert.assertNotNull(actionConf.get(excludeProperty, null));
+ }
+
+ @Test
+ public void testExcludeAllExceptOozie() {
+ doShareLibExclude(ALL_EXCEPT_OOZIE_PATTERN,
+ SPARK_SHARELIB_JARS[0], SPARK_SHARELIB_JARS[1],
SPARK_SHARELIB_JARS[2], SPARK_SHARELIB_JARS[3],
+ PIG_SHARELIB_JARS[0], PIG_SHARELIB_JARS[1],
PIG_SHARELIB_JARS[2]);
+ }
+
+ @Test
+ public void testExcludeAll() {
+ String[] libArray = new String[libs.size()];
+ doShareLibExclude(ALL_JARS_PATTERN, libs.toArray(libArray));
+ }
+
+ @Test
+ public void testWithNullLibPathArgShouldThrowException() {
+ servicesConf.set(excludeProperty, ALL_JACKSON_PATTERN);
+ shareLibExcluder = new ShareLibExcluder(actionConf, servicesConf,
jobConf, executorType, sharelibRootURI);
+
+ // expect exception when illegal parameters passed
+ thrown.expect(NullPointerException.class);
+ thrown.expectMessage(String.format(VALUE_NULL_MSG, "actionLibURI"));
+ shareLibExcluder.shouldExclude(null);
+ }
+}
diff --git a/docs/src/site/markdown/WorkflowFunctionalSpec.md
b/docs/src/site/markdown/WorkflowFunctionalSpec.md
index 7d6a31b..0eae0b5 100644
--- a/docs/src/site/markdown/WorkflowFunctionalSpec.md
+++ b/docs/src/site/markdown/WorkflowFunctionalSpec.md
@@ -2633,6 +2633,60 @@ For example: When using HCatLoader and HCatStorer in
pig, `oozie.action.sharelib
both pig and hcatalog jars.
<a name="UserRetryWFActions"></a>
+
+### 17.2 Action Share Library Exclude (since Oozie 5.2)
+
+Oozie allows to exclude files on its share library directory from being added
to the Distributed Cache. This feature is useful to
+prevent the submitted applications from runtime jar conflict issues.
+
+The unwanted files can be left out by setting
`oozie.action.sharelib.for.#ACTIONTYPE#.exclude` configuration to a regex
pattern.
+Libraries matching the pattern will not be added to the Distributed Cache.
+
+The configuration is supported on action, job, and oozie server configuration
level.
+The precedence order is the same as in the previous 17.1 paragraph.
+
+*One should be very careful as it is easy to exclude very basic oozie jars
with a wrongly set exclude pattern.*
+
+Examples for using sharelib exclude on a java action:
+Actual share library content:
+
+```
+ /user/oozie/share/lib/lib20180701/oozie/lib-one-1.5.jar
+ /user/oozie/share/lib/lib20180701/oozie/lib-two-1.5.jar
+ /user/oozie/share/lib/lib20180701/java/lib-one-2.6.jar
+ /user/oozie/share/lib/lib20180701/java/lib-two-2.6.jar
+ /user/oozie/share/lib/lib20180701/java/component-connector.jar
+```
+
+If not setting `oozie.action.sharelib.for.java.exclude`, none of the jars will
be excluded.
+
+By setting `oozie.action.sharelib.for.java.exclude` to `oozie/lib-one.*` the
expected Distributed Cache content is:
+
+```
+ /user/oozie/share/lib/lib20180701/oozie/lib-two-1.5.jar
+ /user/oozie/share/lib/lib20180701/java/lib-one-2.6.jar
+ /user/oozie/share/lib/lib20180701/java/lib-two-2.6.jar
+ /user/oozie/share/lib/lib20180701/java/component-connector.jar
+```
+
+By setting `oozie.action.sharelib.for.java.exclude` to
`oozie/lib-one.*|component-connector.jar`
+the expected Distributed Cache content is:
+
+```
+ /user/oozie/share/lib/lib20180701/oozie/lib-two-1.5.jar
+ /user/oozie/share/lib/lib20180701/java/lib-one-2.6.jar
+ /user/oozie/share/lib/lib20180701/java/lib-two-2.6.jar
+```
+
+By setting `oozie.action.sharelib.for.java.exclude` to `oozie/lib-*2.6.*`
+the expected Distributed Cache content is:
+
+```
+ /user/oozie/share/lib/lib20180701/oozie/lib-one-1.5.jar
+ /user/oozie/share/lib/lib20180701/oozie/lib-two-1.5.jar
+ /user/oozie/share/lib/lib20180701/java/component-connector.jar
+```
+
## 18 User-Retry for Workflow Actions (since Oozie 3.1)
Oozie provides User-Retry capabilities when an action is in `ERROR` or
`FAILED` state.
diff --git a/release-log.txt b/release-log.txt
index cc8bffc..2302a02 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.2.0 release (trunk - unreleased)
+OOZIE-1624 Exclusion pattern for sharelib JARs (puru, matijhs via asalamon74,
andras.piros)
OOZIE-2693 SimpleHCatDependencyCache.removeMissingDependency can throw NPE
(puru via asalamon74)
OOZIE-3442 Check and eliminate builds/runHudsonCIBuild.sh (nobigo via
asalamon74)
OOZIE-1702 [build] Fix Javadoc warnings (dionusos via kmarton)
