Repository: oozie
Updated Branches:
  refs/heads/master 2dd10ff0f -> 3c03d03fe


OOZIE-2829 Improve sharelib upload to accept multiple source folders (kmarton 
via gezapeti)


Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3c03d03f
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3c03d03f
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3c03d03f

Branch: refs/heads/master
Commit: 3c03d03fe606659fd80c430b13f413fbdfa21faa
Parents: 2dd10ff
Author: Gezapeti Cseh <gezap...@apache.org>
Authored: Mon Jul 30 14:42:51 2018 +0200
Committer: Gezapeti Cseh <gezap...@apache.org>
Committed: Mon Jul 30 14:42:51 2018 +0200

----------------------------------------------------------------------
 docs/src/site/twiki/AG_Install.twiki            |   8 +-
 release-log.txt                                 |   1 +
 .../apache/oozie/tools/OozieSharelibCLI.java    | 113 +++++++--
 .../tools/IntegrationTestOozieSharelibCLI.java  | 199 ++++++++++++++++
 .../tools/OozieSharelibFileOperations.java      |   7 +-
 .../tools/TestConcurrentCopyFromLocal.java      |   4 +-
 .../oozie/tools/TestCopyTaskCallable.java       |   3 +-
 .../oozie/tools/TestOozieSharelibCLI.java       | 237 +++++++------------
 .../TestOozieSharelibCLIExtraArgsParser.java    |  66 ++++++
 9 files changed, 463 insertions(+), 175 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/docs/src/site/twiki/AG_Install.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/AG_Install.twiki 
b/docs/src/site/twiki/AG_Install.twiki
index 46363a3..b8031c8 100644
--- a/docs/src/site/twiki/AG_Install.twiki
+++ b/docs/src/site/twiki/AG_Install.twiki
@@ -47,7 +47,7 @@ The =oozie-setup.sh= script options are:
 
 <verbatim>
 Usage  : oozie-setup.sh <Command and OPTIONS>
-          sharelib create -fs FS_URI [-locallib SHARED_LIBRARY] [-concurrency 
CONCURRENCY]
+          sharelib create -fs FS_URI [-locallib SHARED_LIBRARY] [-extralib 
EXTRA_SHARED_LIBRARY] [-concurrency CONCURRENCY]
                                                                 (create 
sharelib for oozie,
                                                                 FS_URI is the 
fs.default.name
                                                                 for hdfs uri; 
SHARED_LIBRARY, path to the
@@ -55,6 +55,12 @@ Usage  : oozie-setup.sh <Command and OPTIONS>
                                                                 or an expanded 
version of it. If omitted,
                                                                 the Oozie 
sharelib tarball from the Oozie
                                                                 installation 
directory will be used.
+                                                                
EXTRA_SHARED_LIBRARY represents extra sharelib resources.
+                                                                This option 
requires a pair of sharelibname
+                                                                and 
comma-separated list of pathnames in the following format:
+                                                                
sharelib-name=path-name-1,path-name-2
+                                                                In case of 
more than one sharelib, this option can be specified
+                                                                multiple times.
                                                                 CONCURRENCY is 
a number of threads to be used
                                                                 for copy 
operations.
                                                                 By default 1 
thread will be used)

http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 733ea3d..9096017 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
 -- Oozie 5.1.0 release (trunk - unreleased)
 
+OOZIE-2829 Improve sharelib upload to accept multiple source folders (kmarton 
via gezapeti)
 OOZIE-3309 Runtime error during /v2/sla filtering for bundle (asalamon74 via 
andras.piros)
 OOZIE-3306 Make it possible to override maven dependency plugin version number 
(asalamon74 via andras.piros)
 OOZIE-3208 "It should never happen" error messages should be more specific to 
root cause (kmarton via andras.piros)

http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java 
b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
index 75e932c..312770c 100644
--- a/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
+++ b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
@@ -26,7 +26,9 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CancellationException;
@@ -37,10 +39,12 @@ import java.util.concurrent.Future;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.filefilter.WildcardFileFilter;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -64,10 +68,29 @@ public class OozieSharelibCLI {
     public static final String CREATE_CMD = "create";
     public static final String UPGRADE_CMD = "upgrade";
     public static final String LIB_OPT = "locallib";
+    public static final String EXTRALIBS = "extralib";
     public static final String FS_OPT = "fs";
     public static final String CONCURRENCY_OPT = "concurrency";
     public static final String OOZIE_HOME = "oozie.home.dir";
     public static final String SHARE_LIB_PREFIX = "lib_";
+    public static final String NEW_LINE = System.lineSeparator();
+    public static final String EXTRALIBS_USAGE = "Extra sharelib resources. " +
+            "This option requires a pair of sharelibname and coma-separated 
list of pathnames" +
+            " in the following format:" + NEW_LINE +
+            "\"sharelib_name=pathname[,pathname...]\"" + NEW_LINE +
+            "Caveats:" + NEW_LINE +
+            "* Each pathname is either a directory or a regular file 
(compressed files are not extracted prior to " +
+            "the upload operation)." + NEW_LINE +
+            "* Sharelibname shall be specified only once." + NEW_LINE + 
NEW_LINE +
+            "* Do not upload multiple conflicting library versions for an 
extra sharelib directory as it may " +
+            "cause runtime issues." + NEW_LINE +
+            "This option can be present multiple times, in case of more than 
one sharelib" + NEW_LINE +
+            "Example command:" + NEW_LINE + NEW_LINE +
+            "$ oozie-setup.sh sharelib create -fs hdfs://localhost:9000 
-locallib oozie-sharelib.tar.gz " +
+            "-extralib share2=dir2,file2 -extralib share3=file3";
+    public static final String EXTRALIBS_PATH_SEPARATOR = ",";
+    public static final String EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR = "=";
+
     private boolean used;
 
     public static void main(String[] args) throws Exception{
@@ -86,9 +109,12 @@ public class OozieSharelibCLI {
         options.addOption(sharelib);
         options.addOption(uri);
         options.addOption(concurrency);
+        Option addLibsOption = new Option(EXTRALIBS, true, EXTRALIBS_USAGE);
+        options.addOption(addLibsOption);
         return options;
     }
 
+    @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "False 
positive")
     public synchronized int run(String[] args) throws Exception{
         if (used) {
             throw new IllegalStateException("CLI instance already used");
@@ -141,6 +167,12 @@ public class OozieSharelibCLI {
                 srcFile = files.iterator().next();
             }
 
+            Map<String, String> extraLibs = new HashMap<>();
+            if (command.getCommandLine().hasOption(EXTRALIBS)) {
+                String[] param = 
command.getCommandLine().getOptionValues(EXTRALIBS);
+                extraLibs = getExtraLibs(param);
+            }
+
             File temp = File.createTempFile("oozie", ".dir");
             temp.delete();
             temp.mkdir();
@@ -183,21 +215,9 @@ public class OozieSharelibCLI {
 
             System.out.println("the destination path for sharelib is: " + 
dstPath);
 
-            if (!srcFile.exists()){
-                throw new IOException(srcPath + " cannot be found");
-            }
-
-            if (threadPoolSize > 1) {
-                long fsLimitsMinBlockSize = fs.getConf()
-                        
.getLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 
DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT);
-                long bytesPerChecksum = fs.getConf()
-                        .getLong(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 
DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT);
-                new ConcurrentCopyFromLocal(threadPoolSize, 
fsLimitsMinBlockSize, bytesPerChecksum)
-                        .concurrentCopyFromLocal(fs, srcFile, dstPath);
-
-            } else {
-                fs.copyFromLocalFile(false, srcPath, dstPath);
-            }
+            checkIfSourceFilesExist(srcFile);
+            copyToSharelib(threadPoolSize, srcFile, srcPath, dstPath, fs);
+            copyExtraLibs(threadPoolSize, extraLibs, dstPath, fs);
 
             services.destroy();
             FileUtils.deleteDirectory(temp);
@@ -220,6 +240,69 @@ public class OozieSharelibCLI {
         }
     }
 
+    @VisibleForTesting
+    static Map<String,String> getExtraLibs(String[] param) {
+        Map<String, String> extraLibs = new HashMap<>();
+
+        for (String lib : param) {
+            String[] addLibParts = 
lib.split(EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR);
+            if (addLibParts.length != 2) {
+                printExtraSharelibUsage();
+                throw new IllegalArgumentException(String
+                        .format("Argument of extralibs '%s' is in a wrong 
format. Exiting.", param));
+            }
+            String sharelibName = addLibParts[0];
+            String sharelibPaths = addLibParts[1];
+            if (extraLibs.containsKey(sharelibName)) {
+                printExtraSharelibUsage();
+                throw new IllegalArgumentException(String
+                        .format("Extra sharelib, '%s', has been specified 
multiple times. " + "Exiting.", param));
+            }
+            extraLibs.put(sharelibName, sharelibPaths);
+        }
+        return extraLibs;
+    }
+
+    private static void printExtraSharelibUsage() {
+        System.err.println(EXTRALIBS_USAGE);
+    }
+
+
+    @VisibleForTesting
+    @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = 
"FilenameUtils is used to filter user input. JDK8+ is used.")
+    void copyExtraLibs(int threadPoolSize, Map<String, String> extraLibs, Path 
dstPath, FileSystem fs) throws IOException {
+        for (Map.Entry<String, String> sharelib : extraLibs.entrySet()) {
+            Path libDestPath = new Path(dstPath.toString() + Path.SEPARATOR + 
sharelib.getKey());
+            for (String libPath : 
sharelib.getValue().split(EXTRALIBS_PATH_SEPARATOR)) {
+                File srcFile = new File(FilenameUtils.getFullPath(libPath) + 
FilenameUtils.getName(libPath));
+                Path srcPath = new Path(FilenameUtils.getFullPath(libPath) + 
FilenameUtils.getName(libPath));
+                checkIfSourceFilesExist(srcFile);
+                copyToSharelib(threadPoolSize, srcFile, srcPath, libDestPath, 
fs);
+            }
+        }
+    }
+
+    @VisibleForTesting
+    protected void copyToSharelib(int threadPoolSize, File srcFile, Path 
srcPath, Path dstPath, FileSystem fs) throws IOException {
+        if (threadPoolSize > 1) {
+            long fsLimitsMinBlockSize = fs.getConf()
+                    .getLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 
DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT);
+            long bytesPerChecksum = fs.getConf()
+                    .getLong(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 
DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT);
+            new ConcurrentCopyFromLocal(threadPoolSize, fsLimitsMinBlockSize, 
bytesPerChecksum)
+                    .concurrentCopyFromLocal(fs, srcFile, dstPath);
+
+        } else {
+            fs.copyFromLocalFile(false, srcPath, dstPath);
+        }
+    }
+
+    @VisibleForTesting
+    protected void checkIfSourceFilesExist(File srcFile) throws IOException {
+        if (!srcFile.exists()){
+            throw new IOException(srcFile + " cannot be found");
+        }
+    }
 
 
     private static void logError(String errorMessage, Throwable ex) {

http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/IntegrationTestOozieSharelibCLI.java
----------------------------------------------------------------------
diff --git 
a/tools/src/test/java/org/apache/oozie/tools/IntegrationTestOozieSharelibCLI.java
 
b/tools/src/test/java/org/apache/oozie/tools/IntegrationTestOozieSharelibCLI.java
new file mode 100644
index 0000000..12cb665
--- /dev/null
+++ 
b/tools/src/test/java/org/apache/oozie/tools/IntegrationTestOozieSharelibCLI.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.tools;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.action.hadoop.security.LauncherSecurityManager;
+import org.apache.oozie.service.ConfigurationService;
+import org.apache.oozie.service.HadoopAccessorService;
+import org.apache.oozie.service.ServiceException;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.ShareLibService;
+import org.apache.oozie.service.WorkflowAppService;
+import org.apache.oozie.test.XTestCase;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.List;
+
+import static 
org.apache.oozie.tools.OozieSharelibCLI.EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR;
+
+
+public class IntegrationTestOozieSharelibCLI extends XTestCase {
+
+    private final TemporaryFolder tmpFolder = new TemporaryFolder();
+    private File libDirectory;
+    private Services services = null;
+    private Path dstPath = null;
+    private FileSystem fs;
+    private LauncherSecurityManager launcherSecurityManager;
+    public final String outPath = "outFolder";
+
+    @Override
+    protected void setUp() throws Exception {
+        launcherSecurityManager = new LauncherSecurityManager();
+        launcherSecurityManager.enable();
+        tmpFolder.create();
+        libDirectory = tmpFolder.newFolder("lib");
+        super.setUp(false);
+    }
+
+    @Override
+    protected void tearDown() throws Exception {
+        launcherSecurityManager.disable();
+        if (services != null) {
+            services.destroy();
+        }
+        super.tearDown();
+    }
+
+
+    /**
+     * test copy libraries
+     */
+    public void testOozieSharelibCLICreate() throws Exception {
+
+        final int fileNr = 2;
+        List<File> sharelibFiles = 
OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, fileNr);
+
+        String[] argsCreate = { "create", "-fs", outPath, "-locallib", 
libDirectory.getParentFile().getAbsolutePath() };
+        assertEquals("Exit code mismatch", 0, 
execOozieSharelibCLICommands(argsCreate));
+
+        ShareLibService sharelibService = 
getServices().get(ShareLibService.class);
+        Path latestLibPath = sharelibService.getLatestLibPath(getDistPath(),
+                ShareLibService.SHARE_LIB_PREFIX);
+
+        checkCopiedSharelibFiles(sharelibFiles, latestLibPath);
+    }
+
+    /**
+     * test parallel copy libraries
+     */
+    public void testOozieSharelibCLICreateConcurrent() throws Exception {
+
+        final int testFiles = 7;
+        final int concurrency = 5;
+
+        List<File> sharelibFiles = 
OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, testFiles);
+        String[] argsCreate = {"create", "-fs", outPath, "-locallib", 
libDirectory.getParentFile().getAbsolutePath(),
+                "-concurrency", String.valueOf(concurrency)};
+        assertEquals("Exit code mismatch",0, 
execOozieSharelibCLICommands(argsCreate));
+
+        getTargetFileSysyem();
+        ShareLibService sharelibService = 
getServices().get(ShareLibService.class);
+        Path latestLibPath = sharelibService.getLatestLibPath(getDistPath(),
+                ShareLibService.SHARE_LIB_PREFIX);
+
+        checkCopiedSharelibFiles(sharelibFiles, latestLibPath);
+    }
+
+    public void testOozieSharelibCreateExtraLibs () throws Exception {
+        File extraLibBirectory1 = tmpFolder.newFolder("extralib1");
+        File extraLibBirectory2 = tmpFolder.newFolder("extralib2");
+        final int sharelibFileNr = 3;
+        final int extraSharelib1FileNr = 4;
+        final int extraSharelib2FileNr = 4;
+        List<File> shareLibFiles = 
OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, sharelibFileNr);
+        List<File> extraShareLibFiles1 = OozieSharelibFileOperations
+                .generateAndWriteFiles(extraLibBirectory1, 
extraSharelib1FileNr);
+        List<File> extraShareLibFiles2 = OozieSharelibFileOperations
+                .generateAndWriteFiles(extraLibBirectory2, 
extraSharelib2FileNr);
+
+        String extraLib1 = extraLibBirectory1.getName() + 
EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR
+                + extraLibBirectory1.getAbsolutePath();
+        String extraLib2 = extraLibBirectory2.getName() + 
EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR
+                + extraLibBirectory2.getAbsolutePath();
+        String[] argsCreate = { "create", "-fs", outPath, "-locallib", 
libDirectory.getParentFile().getAbsolutePath(),
+                "-" + OozieSharelibCLI.EXTRALIBS, extraLib1, "-" + 
OozieSharelibCLI.EXTRALIBS, extraLib2};
+
+        assertEquals("Exit code mismatch",0, 
execOozieSharelibCLICommands(argsCreate));
+
+        ShareLibService sharelibService = 
getServices().get(ShareLibService.class);
+        Path latestLibPath = sharelibService.getLatestLibPath(getDistPath(), 
ShareLibService.SHARE_LIB_PREFIX);
+        Path extraSharelibPath1 = new Path(latestLibPath + Path.SEPARATOR + 
extraLibBirectory1.getName());
+        Path extraSharelibPath2 = new Path(latestLibPath + Path.SEPARATOR + 
extraLibBirectory2.getName());
+
+        checkCopiedSharelibFiles(shareLibFiles, latestLibPath);
+        checkCopiedSharelibFiles(extraShareLibFiles1, extraSharelibPath1);
+        checkCopiedSharelibFiles(extraShareLibFiles2, extraSharelibPath2);
+    }
+
+    private void checkCopiedSharelibFiles(List<File> fileList, Path libPath) 
throws Exception {
+        FileSystem fileSystem = getTargetFileSysyem();
+        for (File f: fileList) {
+            try (InputStream originalFileStream = new FileInputStream(f);
+                 InputStream copiedFileStream = fileSystem.open(new 
Path(libPath, f.getName()))) {
+                assertTrue("The content of the files must be equal", 
IOUtils.contentEquals(originalFileStream, copiedFileStream));
+            }
+        }
+    }
+
+    private FileSystem getTargetFileSysyem() throws Exception {
+        if (fs == null) {
+            HadoopAccessorService has = 
getServices().get(HadoopAccessorService.class);
+            URI uri = new Path(outPath).toUri();
+            Configuration fsConf = has.createConfiguration(uri.getAuthority());
+            fs = has.createFileSystem(System.getProperty("user.name"), uri, 
fsConf);
+        }
+        return fs;
+
+    }
+
+    private Services getServices() throws ServiceException {
+        if (services == null) {
+            services = new Services();
+            services.get(ConfigurationService.class).getConf()
+                    
.set(Services.CONF_SERVICE_CLASSES,"org.apache.oozie.service.LiteWorkflowAppService,"
+                            + "org.apache.oozie.service.SchedulerService,"
+                            + "org.apache.oozie.service.HadoopAccessorService,"
+                            + "org.apache.oozie.service.ShareLibService");
+            services.init();
+        }
+        return services;
+    }
+
+    private Path getDistPath() throws Exception {
+        if (dstPath == null) {
+            WorkflowAppService lwas = 
getServices().get(WorkflowAppService.class);
+            dstPath = lwas.getSystemLibPath();
+        }
+        return dstPath;
+    }
+
+    private int execOozieSharelibCLICommands(String[] args) throws Exception {
+        try {
+            OozieSharelibCLI.main(args);
+        } catch (SecurityException ex) {
+            if (launcherSecurityManager.getExitInvoked()) {
+                System.out.println("Intercepting System.exit(" + 
launcherSecurityManager.getExitCode() + ")");
+                System.err.println("Intercepting System.exit(" + 
launcherSecurityManager.getExitCode() + ")");
+                return launcherSecurityManager.getExitCode();
+            } else {
+                throw ex;
+            }
+        }
+        return 1;
+    }
+}

http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java
----------------------------------------------------------------------
diff --git 
a/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java 
b/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java
index d344300..ca31e96 100644
--- 
a/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java
+++ 
b/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java
@@ -22,6 +22,7 @@ import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.Writer;
+import java.util.ArrayList;
 import java.util.List;
 
 public final class OozieSharelibFileOperations {
@@ -36,15 +37,17 @@ public final class OozieSharelibFileOperations {
     /**
      * generate a number of files equals with fileNr, and save the fileList 
parameter
      * @param fileNr number of files to be generated
-     * @param fileList a list of the generated files
+     * @return a list of the generated files
      * @throws Exception
      */
-    public static void generateAndWriteFiles(File libDirectory, int fileNr, 
List<File> fileList) throws IOException {
+    public static List<File> generateAndWriteFiles(File libDirectory, int 
fileNr) throws IOException {
+        List<File> fileList = new ArrayList<>();
         for (int i=0; i<fileNr; i++) {
             String fileName = generateFileName(i);
             String fileContent = generateFileContent(i);
             fileList.add(writeFile(libDirectory, fileName, fileContent));
         }
+        return fileList;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java
----------------------------------------------------------------------
diff --git 
a/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java 
b/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java
index d77eba6..51cf93d 100644
--- 
a/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java
+++ 
b/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java
@@ -100,9 +100,7 @@ public class TestConcurrentCopyFromLocal extends XTestCase {
 
     private void performAndCheckConcurrentCopy(final int testFiles, final int 
threadPoolSize, final long fsLimitsMinBlockSize,
                                                final long bytesPerChecksum) 
throws Exception {
-        List<File> fileList = new ArrayList<>();
-
-        OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, 
testFiles, fileList);
+        List<File> fileList = 
OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, testFiles);
         File srcFile = new 
File(libDirectory.getParentFile().getAbsolutePath());
         OozieSharelibCLI.ConcurrentCopyFromLocal concurrentCopy = new 
OozieSharelibCLI
                 .ConcurrentCopyFromLocal(threadPoolSize, fsLimitsMinBlockSize, 
bytesPerChecksum);

http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java
----------------------------------------------------------------------
diff --git 
a/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java 
b/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java
index bce0433..02f937a 100644
--- a/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java
+++ b/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java
@@ -110,8 +110,7 @@ public class TestCopyTaskCallable extends XTestCase {
     private void performAndCheckCallCopyTask(final long blockSize, final int 
poolSize, final int testFiles) throws Exception {
         Set<OozieSharelibCLI.CopyTaskConfiguration> failedCopyTasks = new 
ConcurrentHashSet<>();
 
-        List<File> fileList = new ArrayList<>();
-        OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, 
testFiles, fileList);
+        List<File> fileList = 
OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, testFiles);
 
         File srcFile = new 
File(libDirectory.getParentFile().getAbsolutePath());
 

http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java
----------------------------------------------------------------------
diff --git 
a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java 
b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java
index 5929e5c..18ab8f2 100644
--- a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java
+++ b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java
@@ -19,206 +19,139 @@
 
 package org.apache.oozie.tools;
 
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.oozie.action.hadoop.security.LauncherSecurityManager;
-import org.apache.oozie.service.ConfigurationService;
-import org.apache.oozie.service.HadoopAccessorService;
-import org.apache.oozie.service.ServiceException;
-import org.apache.oozie.service.Services;
-import org.apache.oozie.service.ShareLibService;
-import org.apache.oozie.service.WorkflowAppService;
-import org.apache.oozie.test.XTestCase;
-import org.junit.rules.TemporaryFolder;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.InputStream;
+import java.io.IOException;
 import java.io.PrintStream;
-import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.oozie.tools.OozieSharelibCLI.getExtraLibs;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyMap;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 
 /**
  * Test OozieSharelibCLI
  */
-public class TestOozieSharelibCLI extends XTestCase {
-    public final String outPath = "outFolder";
-    private final TemporaryFolder tmpFolder = new TemporaryFolder();
-    private File libDirectory;
-    private Services services = null;
-    private Path dstPath = null;
-    private FileSystem fs;
+public class TestOozieSharelibCLI {
+
+    private final static String TEST_SHAERELIBNAME1 = "sharelibName";
+    private final static String TEST_SHAERELIBNAME2 = "sharelibName2";
+    private final static String TEST_EXTRALIBS_PATHS1 = 
"/path/to/source/,/path/to/some/file";
+    private final static String TEST_EXTRALIBS_PATHS2 = 
"hdfs://my/jar.jar#myjar.jar";
+
     private LauncherSecurityManager launcherSecurityManager;
-    @Override
-    protected void setUp() throws Exception {
+
+    @Before
+    public void setUp() {
         launcherSecurityManager = new LauncherSecurityManager();
         launcherSecurityManager.enable();
-        tmpFolder.create();
-        libDirectory = tmpFolder.newFolder("lib");
-        super.setUp(false);
-
     }
 
-    @Override
-    protected void tearDown() throws Exception {
+    @After
+    public void tearDown() {
         launcherSecurityManager.disable();
-        if (services != null) {
-            services.destroy();
-        }
-        super.tearDown();
     }
 
-    /**
-     * Test help command
-     */
-    public void testHelp() throws Exception {
+    @Test
+    public void testHelpCommand() throws Exception {
         ByteArrayOutputStream data = new ByteArrayOutputStream();
         PrintStream oldPrintStream = System.out;
         System.setOut(new PrintStream(data));
         try {
-            String[] argsHelp = { "help" };
-            assertEquals(0, execOozieSharelibCLICommands(argsHelp));
+            String[] argsHelp = {"help"};
+            assertEquals("Exit code mismatch",0, 
execOozieSharelibCLICommands(argsHelp));
             String helpMessage = data.toString();
-            assertTrue(helpMessage.contains(
+            assertTrue("Missing create <OPTIONS> description from help 
message", helpMessage.contains(
                     "oozie-setup.sh create <OPTIONS> : create a new 
timestamped version of oozie sharelib"));
-            assertTrue(helpMessage.contains(
-                    "oozie-setup.sh upgrade <OPTIONS> : [deprecated][use 
command \"create\" "
-                            + "to create new version]   upgrade oozie sharelib 
"));
-            assertTrue(helpMessage.contains(" oozie-setup.sh help"));
-            assertTrue(helpMessage.contains(
-                    "-concurrency <arg>   Number of threads to be used for 
copy operations."));
-        }
-        finally {
+            assertTrue("Missing  upgrade <OPTIONS> description from help 
message",
+                    helpMessage.contains("oozie-setup.sh upgrade <OPTIONS> : 
[deprecated][use command \"create\" "
+                    + "to create new version]   upgrade oozie sharelib "));
+            assertTrue("Help message mismatch", helpMessage.contains(" 
oozie-setup.sh help"));
+            assertTrue("Missing -concurrency from help message",
+                    helpMessage.contains("-concurrency <arg>   Number of 
threads to be used for copy operations."));
+        } finally {
             System.setOut(oldPrintStream);
         }
-
-    }
-
-    /**
-     * test copy libraries
-     */
-    public void testOozieSharelibCLICreate() throws Exception {
-
-        OozieSharelibFileOperations.writeFile(libDirectory, "file1", "test 
File");
-        OozieSharelibFileOperations.writeFile(libDirectory, "file2", "test 
File2");
-
-        String[] argsCreate = { "create", "-fs", outPath, "-locallib", 
libDirectory.getParentFile().getAbsolutePath() };
-        assertEquals(0, execOozieSharelibCLICommands(argsCreate));
-
-        FileSystem fs = getTargetFileSysyem();
-        ShareLibService sharelibService = 
getServices().get(ShareLibService.class);
-
-        // test files in new folder
-        assertEquals(9, fs.getFileStatus(new 
Path(sharelibService.getLatestLibPath(getDistPath(),
-                ShareLibService.SHARE_LIB_PREFIX), "file1")).getLen());
-        assertEquals(10, fs.getFileStatus(new 
Path(sharelibService.getLatestLibPath(getDistPath(),
-                ShareLibService.SHARE_LIB_PREFIX), "file2")).getLen());
-
-    }
-
-    /**
-     * test parallel copy libraries
-     */
-    public void testOozieSharelibCLICreateConcurrent() throws Exception {
-
-        final int testFiles = 7;
-        final int concurrency = 5;
-
-        for (int i = 0; i < testFiles; i++) {
-            OozieSharelibFileOperations.writeFile(libDirectory, 
OozieSharelibFileOperations.generateFileName(i),
-                    OozieSharelibFileOperations.generateFileContent(i));
-        }
-
-        String[] argsCreate = {"create", "-fs", outPath, "-locallib", 
libDirectory.getParentFile().getAbsolutePath(),
-            "-concurrency", String.valueOf(concurrency)};
-        assertEquals(0, execOozieSharelibCLICommands(argsCreate));
-
-        getTargetFileSysyem();
-        ShareLibService sharelibService = 
getServices().get(ShareLibService.class);
-        Path latestLibPath = sharelibService.getLatestLibPath(getDistPath(),
-                ShareLibService.SHARE_LIB_PREFIX);
-
-        // test files in new folder
-
-        for (int i = 0; i < testFiles; i++) {
-            String fileName = OozieSharelibFileOperations.generateFileName(i);
-            String expectedFileContent = 
OozieSharelibFileOperations.generateFileContent(i);
-            InputStream in = null;
-            try {
-                in = getTargetFileSysyem().open(new Path(latestLibPath, 
fileName));
-                String actualFileContent = IOUtils.toString(in);
-                assertEquals(fileName, expectedFileContent, actualFileContent);
-            } finally {
-                IOUtils.closeQuietly(in);
-            }
-        }
-
     }
 
-    /**
-     * test fake command
-     */
+    @Test
     public void testFakeCommand() throws Exception {
 
         ByteArrayOutputStream data = new ByteArrayOutputStream();
         PrintStream oldPrintStream = System.err;
         System.setErr(new PrintStream(data));
         try {
-            String[] argsFake = { "fakeCommand" };
-            assertEquals(1, execOozieSharelibCLICommands(argsFake));
-            assertTrue(data.toString().contains("Invalid sub-command: invalid 
sub-command [fakeCommand]"));
-            assertTrue(data.toString().contains("use 'help [sub-command]' for 
help details"));
-        }
-        finally {
+            String[] argsFake = {"fakeCommand"};
+            assertEquals("Exit code mismatch", 1, 
execOozieSharelibCLICommands(argsFake));
+            assertTrue("Error message missing",
+                    data.toString().contains("Invalid sub-command: invalid 
sub-command [fakeCommand]"));
+            assertTrue("Help message missing", data.toString().contains("use 
'help [sub-command]' for help details"));
+        } finally {
             System.setErr(oldPrintStream);
         }
-
-    }
-
-    private FileSystem getTargetFileSysyem() throws Exception {
-        if (fs == null) {
-            HadoopAccessorService has = 
getServices().get(HadoopAccessorService.class);
-            URI uri = new Path(outPath).toUri();
-            Configuration fsConf = has.createConfiguration(uri.getAuthority());
-            fs = has.createFileSystem(System.getProperty("user.name"), uri, 
fsConf);
-        }
-        return fs;
-
-    }
-
-    private Services getServices() throws ServiceException {
-        if (services == null) {
-            services = new Services();
-            services.get(ConfigurationService.class).getConf()
-                    
.set(Services.CONF_SERVICE_CLASSES,"org.apache.oozie.service.LiteWorkflowAppService,"
-                            + "org.apache.oozie.service.SchedulerService,"
-                            + "org.apache.oozie.service.HadoopAccessorService,"
-                            + "org.apache.oozie.service.ShareLibService");
-            services.init();
-        }
-        return services;
     }
 
-    private Path getDistPath() throws Exception {
-        if (dstPath == null) {
-            WorkflowAppService lwas = 
getServices().get(WorkflowAppService.class);
-            dstPath = lwas.getSystemLibPath();
-        }
-        return dstPath;
+    @Test
+    public void testCopyingExtraSharelibs() throws IOException {
+        OozieSharelibCLI oozieSharelibCLI = spy(new OozieSharelibCLI() {
+            @Override
+            protected void checkIfSourceFilesExist(File srcFile) { }
+
+            @Override
+            protected void copyToSharelib(int threadPoolSize, File srcFile, 
Path srcPath, Path dstPath, FileSystem fs) {
+                String dstStr = dstPath.toString();
+                String actualSharelibName = 
dstStr.substring(dstStr.lastIndexOf(Path.SEPARATOR) + 1);
+                Assert.assertTrue("Expected sharelib name missing",
+                        Arrays.asList(TEST_SHAERELIBNAME1, 
TEST_SHAERELIBNAME2).contains(actualSharelibName));
+
+                List<File> expectedTestFiles = new ArrayList<>();
+                for (String s : 
Arrays.asList(TEST_EXTRALIBS_PATHS1.split(","))) {
+                    expectedTestFiles.add(new File(s));
+                }
+                for (String s : 
Arrays.asList(TEST_EXTRALIBS_PATHS2.split(","))) {
+                    expectedTestFiles.add(new File(s));
+                }
+                Assert.assertTrue("Expected file missing", 
expectedTestFiles.contains(srcFile));
+            }
+        });
+
+        final FileSystem fs = mock(FileSystem.class);
+        final Path dst = new Path("/share/lib");
+        String extraLibOption1 = TEST_SHAERELIBNAME1 + "=" + 
TEST_EXTRALIBS_PATHS1;
+        String extraLibOption2 = TEST_SHAERELIBNAME2 + "=" + 
TEST_EXTRALIBS_PATHS2;
+        Map<String, String> addLibs = getExtraLibs(new String[] 
{extraLibOption1, extraLibOption2});
+        oozieSharelibCLI.copyExtraLibs(1, addLibs, dst, fs);
+        verify(oozieSharelibCLI, times(1))
+                .copyExtraLibs(anyInt(), anyMap(), any(Path.class), 
any(FileSystem.class));
     }
 
     private int execOozieSharelibCLICommands(String[] args) throws Exception {
         try {
             OozieSharelibCLI.main(args);
-        }
-        catch (SecurityException ex) {
+        } catch (SecurityException ex) {
             if (launcherSecurityManager.getExitInvoked()) {
                 System.out.println("Intercepting System.exit(" + 
launcherSecurityManager.getExitCode() + ")");
                 System.err.println("Intercepting System.exit(" + 
launcherSecurityManager.getExitCode() + ")");
                 return launcherSecurityManager.getExitCode();
-            }
-            else {
+            } else {
                 throw ex;
             }
         }

http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLIExtraArgsParser.java
----------------------------------------------------------------------
diff --git 
a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLIExtraArgsParser.java
 
b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLIExtraArgsParser.java
new file mode 100644
index 0000000..f18db4d
--- /dev/null
+++ 
b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLIExtraArgsParser.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.tools;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Map;
+
+import static org.apache.oozie.tools.OozieSharelibCLI.getExtraLibs;
+
+public class TestOozieSharelibCLIExtraArgsParser {
+
+    private final static String TEST_SHAERELIBNAME1 = "sharelibName";
+    private final static String TEST_SHAERELIBNAME2 = "sharelibName2";
+    private final static String TEST_EXTRALIBS_PATHS1 = 
"/path/to/source/,/path/to/some/file";
+    private final static String TEST_EXTRALIBS_PATHS2 = 
"hdfs://my/jar.jar#myjar.jar,path/to/source/";
+
+    @Test
+    public void testParsingExtraSharelibs() {
+        String extraLibOption1 = TEST_SHAERELIBNAME1 + "=" + 
TEST_EXTRALIBS_PATHS1;
+        String extraLibOption2 = TEST_SHAERELIBNAME2 + "=" + 
TEST_EXTRALIBS_PATHS2;
+        Map<String, String> addLibs = getExtraLibs(new String[] 
{extraLibOption1, extraLibOption2});
+
+        Assert.assertEquals("Extra libs count mismatch", 2, addLibs.size());
+        Assert.assertEquals("Missing extra lib", TEST_EXTRALIBS_PATHS1, 
addLibs.get(TEST_SHAERELIBNAME1));
+        Assert.assertEquals("Missing extra lib", TEST_EXTRALIBS_PATHS2, 
addLibs.get(TEST_SHAERELIBNAME2));
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void testParsingExtraSharelibsMissingValue() {
+        String extraLibOption1 = TEST_SHAERELIBNAME1 + "=";
+        String extraLibOption2 = TEST_SHAERELIBNAME2 + "=" + 
TEST_EXTRALIBS_PATHS2;
+        getExtraLibs(new String[] {extraLibOption1, extraLibOption2});
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void testParsingExtraSharelibsMissingKey() {
+        String extraLibOption1 = TEST_EXTRALIBS_PATHS1;
+        String extraLibOption2 = TEST_SHAERELIBNAME2 + "=" + 
TEST_EXTRALIBS_PATHS2;
+        getExtraLibs(new String[] {extraLibOption1, extraLibOption2});
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testParsingSharelibNamePresentMultipleTimes() {
+        String extraLibOption1 = TEST_SHAERELIBNAME1 + "=" + 
TEST_EXTRALIBS_PATHS1;
+        String extraLibOption2 = TEST_SHAERELIBNAME1 + "=" + 
TEST_EXTRALIBS_PATHS2;
+        getExtraLibs(new String[] {extraLibOption1, extraLibOption2});
+    }
+}
\ No newline at end of file

Reply via email to