IGNITE-3906: Hadoop: implemented additional user libs facility.

Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0dc9713a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0dc9713a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0dc9713a

Branch: refs/heads/ignite-3199-1
Commit: 0dc9713ac3124c0ea34d949271a813c992873656
Parents: 3a94f27
Author: vozerov-gridgain <voze...@gridgain.com>
Authored: Thu Sep 15 14:01:19 2016 +0300
Committer: vozerov-gridgain <voze...@gridgain.com>
Committed: Thu Sep 15 14:01:19 2016 +0300

----------------------------------------------------------------------
 .../processors/hadoop/HadoopClassLoader.java    |   6 +-
 .../processors/hadoop/HadoopClasspathMain.java  |   2 +-
 .../processors/hadoop/HadoopClasspathUtils.java | 230 +++++++++++++---
 .../processors/hadoop/HadoopTestUtils.java      |  73 +++++-
 .../hadoop/HadoopUserLibsSelfTest.java          | 260 +++++++++++++++++++
 .../testsuites/IgniteHadoopTestSuite.java       |   3 +
 6 files changed, 536 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/0dc9713a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 389de8c..2e0e271 100644
--- 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -61,10 +61,8 @@ import org.objectweb.asm.commons.RemappingClassAdapter;
  * unavailable for parent.
  */
 public class HadoopClassLoader extends URLClassLoader implements ClassCache {
-    /**
-     * We are very parallel capable.
-     */
     static {
+        // We are very parallel capable.
         registerAsParallelCapable();
     }
 
@@ -498,7 +496,7 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
                 return hadoopUrls;
 
             try {
-                hadoopUrls = HadoopClasspathUtils.classpathUrls();
+                hadoopUrls = HadoopClasspathUtils.classpathForClassLoader();
             }
             catch (IOException e) {
                 throw new IgniteCheckedException("Failed to resolve Hadoop JAR 
locations: " + e.getMessage(), e);

http://git-wip-us.apache.org/repos/asf/ignite/blob/0dc9713a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
----------------------------------------------------------------------
diff --git 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
index 5279b7d..4069496 100644
--- 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
+++ 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
@@ -36,7 +36,7 @@ public class HadoopClasspathMain {
 
         StringBuilder sb = new StringBuilder();
 
-        for (String path : HadoopClasspathUtils.classpathForJavaProcess())
+        for (String path : HadoopClasspathUtils.classpathForProcess())
             sb.append(path).append(separator);
 
         System.out.println(sb);

http://git-wip-us.apache.org/repos/asf/ignite/blob/0dc9713a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index 121fcab..f5c2814 100644
--- 
a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ 
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -27,6 +27,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.LinkedList;
 import java.util.List;
 
 /**
@@ -36,18 +37,21 @@ public class HadoopClasspathUtils {
     /** Prefix directory. */
     public static final String PREFIX = "HADOOP_PREFIX";
 
-    /** Home directory. */
+    /** Hadoop home directory. */
     public static final String HOME = "HADOOP_HOME";
 
-    /** Home directory. */
+    /** Hadoop common directory. */
     public static final String COMMON_HOME = "HADOOP_COMMON_HOME";
 
-    /** Home directory. */
+    /** Hadoop HDFS directory. */
     public static final String HDFS_HOME = "HADOOP_HDFS_HOME";
 
-    /** Home directory. */
+    /** Hadoop mapred directory. */
     public static final String MAPRED_HOME = "HADOOP_MAPRED_HOME";
 
+    /** Arbitrary additional dependencies. Compliant with standard Java 
classpath resolution. */
+    public static final String HADOOP_USER_LIBS = "HADOOP_USER_LIBS";
+
     /** Empty string. */
     private static final String EMPTY_STR = "";
 
@@ -57,16 +61,20 @@ public class HadoopClasspathUtils {
      * @return List of the class path elements.
      * @throws IOException If failed.
      */
-    public static List<String> classpathForJavaProcess() throws IOException {
+    public static List<String> classpathForProcess() throws IOException {
         List<String> res = new ArrayList<>();
 
         for (final SearchDirectory dir : classpathDirectories()) {
-            if (dir.hasFilter()) {
-                for (File file : dir.files())
+            File[] files = dir.files();
+
+            if (dir.useWildcard()) {
+                if (files.length > 0)
+                    res.add(dir.absolutePath() + File.separator + '*');
+            }
+            else {
+                for (File file : files)
                     res.add(file.getAbsolutePath());
             }
-            else
-                res.add(dir.absolutePath() + File.separator + '*');
         }
 
         return res;
@@ -78,7 +86,7 @@ public class HadoopClasspathUtils {
      * @return List of class path URLs.
      * @throws IOException If failed.
      */
-    public static List<URL> classpathUrls() throws IOException {
+    public static List<URL> classpathForClassLoader() throws IOException {
         List<URL> res = new ArrayList<>();
 
         for (SearchDirectory dir : classpathDirectories()) {
@@ -182,17 +190,70 @@ public class HadoopClasspathUtils {
 
         Collection<SearchDirectory> res = new ArrayList<>();
 
-        res.add(new SearchDirectory(new File(loc.common(), "lib"), null));
-        res.add(new SearchDirectory(new File(loc.hdfs(), "lib"), null));
-        res.add(new SearchDirectory(new File(loc.mapred(), "lib"), null));
+        res.add(new SearchDirectory(new File(loc.common(), "lib"), 
AcceptAllDirectoryFilter.INSTANCE));
+        res.add(new SearchDirectory(new File(loc.hdfs(), "lib"), 
AcceptAllDirectoryFilter.INSTANCE));
+        res.add(new SearchDirectory(new File(loc.mapred(), "lib"), 
AcceptAllDirectoryFilter.INSTANCE));
+
+        res.add(new SearchDirectory(new File(loc.common()), new 
PrefixDirectoryFilter("hadoop-common-")));
+        res.add(new SearchDirectory(new File(loc.common()), new 
PrefixDirectoryFilter("hadoop-auth-")));
 
-        res.add(new SearchDirectory(new File(loc.common()), "hadoop-common-"));
-        res.add(new SearchDirectory(new File(loc.common()), "hadoop-auth-"));
+        res.add(new SearchDirectory(new File(loc.hdfs()), new 
PrefixDirectoryFilter("hadoop-hdfs-")));
 
-        res.add(new SearchDirectory(new File(loc.hdfs()), "hadoop-hdfs-"));
+        res.add(new SearchDirectory(new File(loc.mapred()),
+            new PrefixDirectoryFilter("hadoop-mapreduce-client-common")));
+        res.add(new SearchDirectory(new File(loc.mapred()),
+            new PrefixDirectoryFilter("hadoop-mapreduce-client-core")));
 
-        res.add(new SearchDirectory(new File(loc.mapred()), 
"hadoop-mapreduce-client-common"));
-        res.add(new SearchDirectory(new File(loc.mapred()), 
"hadoop-mapreduce-client-core"));
+        res.addAll(parseUserLibs());
+
+        return res;
+    }
+
+    /**
+     * Parse user libs.
+     *
+     * @return Parsed libs search patterns.
+     * @throws IOException If failed.
+     */
+    static Collection<SearchDirectory> parseUserLibs() throws IOException {
+        return parseUserLibs(systemOrEnv(HADOOP_USER_LIBS, null));
+    }
+
+    /**
+     * Parse user libs.
+     *
+     * @param str String.
+     * @return Result.
+     * @throws IOException If failed.
+     */
+    static Collection<SearchDirectory> parseUserLibs(String str) throws 
IOException {
+        Collection<SearchDirectory> res = new LinkedList<>();
+
+        if (!isEmpty(str)) {
+            String[] tokens = normalize(str).split(File.pathSeparator);
+
+            for (String token : tokens) {
+                // Skip empty tokens.
+                if (isEmpty(token))
+                    continue;
+
+                File file = new File(token);
+                File dir = file.getParentFile();
+
+                if (token.endsWith("*")) {
+                    assert dir != null;
+
+                    res.add(new SearchDirectory(dir, 
AcceptAllDirectoryFilter.INSTANCE, false));
+                }
+                else {
+                    // Met "/" or "C:\" pattern, nothing to do with it.
+                    if (dir == null)
+                        continue;
+
+                    res.add(new SearchDirectory(dir, new 
ExactDirectoryFilter(file.getName()), false));
+                }
+            }
+        }
 
         return res;
     }
@@ -239,57 +300,162 @@ public class HadoopClasspathUtils {
     }
 
     /**
+     * NOramlize the string.
+     *
+     * @param str String.
+     * @return Normalized string.
+     */
+    private static String normalize(String str) {
+        assert str != null;
+
+        return str.trim().toLowerCase();
+    }
+
+    /**
      * Simple pair-like structure to hold directory name and a mask assigned 
to it.
      */
-    private static class SearchDirectory {
+    static class SearchDirectory {
         /** File. */
         private final File dir;
 
-        /** The mask. */
-        private final String filter;
+        /** Filter. */
+        private final DirectoryFilter filter;
+
+        /** Whether directory must exist. */
+        private final boolean strict;
+
+        /**
+         * Constructor for directory search with strict rule.
+         *
+         * @param dir Directory.
+         * @param filter Filter.
+         * @throws IOException If failed.
+         */
+        private SearchDirectory(File dir, DirectoryFilter filter) throws 
IOException {
+            this(dir, filter, true);
+        }
 
         /**
          * Constructor.
          *
          * @param dir Directory.
          * @param filter Filter.
+         * @param strict Whether directory must exist.
+         * @throws IOException If failed.
          */
-        private SearchDirectory(File dir, String filter) throws IOException {
+        private SearchDirectory(File dir, DirectoryFilter filter, boolean 
strict) throws IOException {
             this.dir = dir;
             this.filter = filter;
+            this.strict = strict;
 
-            if (!exists(dir.getAbsolutePath()))
+            if (strict && !exists(dir.getAbsolutePath()))
                 throw new IOException("Directory cannot be read: " + 
dir.getAbsolutePath());
         }
 
         /**
          * @return Absolute path.
          */
-        private String absolutePath() {
+        String absolutePath() {
             return dir.getAbsolutePath();
         }
 
         /**
          * @return Child files.
          */
-        private File[] files() throws IOException {
+        File[] files() throws IOException {
             File[] files = dir.listFiles(new FilenameFilter() {
                 @Override public boolean accept(File dir, String name) {
-                    return filter == null || name.startsWith(filter);
+                    return filter.test(name);
                 }
             });
 
-            if (files == null)
-                throw new IOException("Path is not a directory. [dir=" + dir + 
']');
+            if (files == null) {
+                if (strict)
+                    throw new IOException("Failed to get directory files 
[dir=" + dir + ']');
+                else
+                    return new File[0];
+            }
+            else
+                return files;
+        }
 
-            return files;
+        /**
+         * @return {@code True} if wildcard can be used.
+         */
+        boolean useWildcard() {
+            return filter instanceof AcceptAllDirectoryFilter;
         }
+    }
 
+    /**
+     * Directory filter interface.
+     */
+    static interface DirectoryFilter {
         /**
-         * @return {@code True} if filter exists.
+         * Test if file with this name should be included.
+         *
+         * @param name File name.
+         * @return {@code True} if passed.
          */
-        private boolean hasFilter() {
-            return filter != null;
+        public boolean test(String name);
+    }
+
+    /**
+     * Filter to accept all files.
+     */
+    static class AcceptAllDirectoryFilter implements DirectoryFilter {
+        /** Singleton instance. */
+        public static final AcceptAllDirectoryFilter INSTANCE = new 
AcceptAllDirectoryFilter();
+
+        /** {@inheritDoc} */
+        @Override public boolean test(String name) {
+            return true;
+        }
+    }
+
+    /**
+     * Filter which uses prefix to filter files.
+     */
+    static class PrefixDirectoryFilter implements DirectoryFilter {
+        /** Prefix. */
+        private final String prefix;
+
+        /**
+         * Constructor.
+         *
+         * @param prefix Prefix.
+         */
+        public PrefixDirectoryFilter(String prefix) {
+            assert prefix != null;
+
+            this.prefix = normalize(prefix);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean test(String name) {
+            return normalize(name).startsWith(prefix);
+        }
+    }
+
+    /**
+     * Filter which uses exact comparison.
+     */
+    static class ExactDirectoryFilter implements DirectoryFilter {
+        /** Name. */
+        private final String name;
+
+        /**
+         * Constructor.
+         *
+         * @param name Name.
+         */
+        public ExactDirectoryFilter(String name) {
+            this.name = normalize(name);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean test(String name) {
+            return normalize(name).equals(this.name);
         }
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/0dc9713a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java
----------------------------------------------------------------------
diff --git 
a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java
 
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java
index 9ebad78..da0d922 100644
--- 
a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java
+++ 
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java
@@ -17,14 +17,18 @@
 
 package org.apache.ignite.internal.processors.hadoop;
 
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
-import org.apache.ignite.internal.util.typedef.F;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -33,6 +37,41 @@ import static org.junit.Assert.assertTrue;
  * Utility class for tests.
  */
 public class HadoopTestUtils {
+    /** Base test directory. */
+    private static final File BASE_TEST_DIR = new File(U.getIgniteHome() + 
"/work/test/hadoop/");
+
+    /**
+     * @return Base directory for tests.
+     */
+    public static File baseTestDir() {
+        return BASE_TEST_DIR;
+    }
+
+    /**
+     * Get test directory.
+     *
+     * @param parts Parts.
+     * @return Directory.
+     */
+    public static File testDir(String... parts) {
+        File res = BASE_TEST_DIR;
+
+        if (parts != null) {
+            for (String part : parts)
+                res = new File(res, part);
+        }
+
+        return res;
+    }
+
+    /**
+     * Clear base test directory.
+     */
+    public static void clearBaseTestDir() {
+        if (baseTestDir().exists())
+            assert delete(baseTestDir());
+    }
+
     /**
      * Checks that job statistics file contains valid strings only.
      *
@@ -40,6 +79,7 @@ public class HadoopTestUtils {
      * @return Amount of events.
      * @throws IOException If failed.
      */
+    @SuppressWarnings("ResultOfMethodCallIgnored")
     public static long simpleCheckJobStatFile(BufferedReader reader) throws 
IOException {
         Collection<String> phases = new HashSet<>();
 
@@ -104,4 +144,35 @@ public class HadoopTestUtils {
 
         return evtCnt;
     }
+
+    /**
+     * Deletes file or directory with all sub-directories and files.
+     *
+     * @param file File or directory to delete.
+     * @return {@code true} if and only if the file or directory is 
successfully deleted,
+     *      {@code false} otherwise
+     */
+    public static boolean delete(@Nullable File file) {
+        if (file == null)
+            return false;
+
+        boolean res = true;
+
+        if (file.isDirectory()) {
+            File[] files = file.listFiles();
+
+            if (files != null && files.length > 0)
+                for (File file1 : files)
+                    if (file1.isDirectory())
+                        res &= delete(file1);
+                    else
+                        res &= file1.delete();
+
+            res &= file.delete();
+        }
+        else
+            res = file.delete();
+
+        return res;
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/0dc9713a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopUserLibsSelfTest.java
----------------------------------------------------------------------
diff --git 
a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopUserLibsSelfTest.java
 
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopUserLibsSelfTest.java
new file mode 100644
index 0000000..9e3c8f4
--- /dev/null
+++ 
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopUserLibsSelfTest.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+
+/**
+ * Tests for user libs parsing.
+ */
+public class HadoopUserLibsSelfTest extends GridCommonAbstractTest {
+    /** Directory 1. */
+    private static final File DIR_1 = HadoopTestUtils.testDir("dir1");
+
+    /** File 1 in directory 1. */
+    private static final File FILE_1_1 = new File(DIR_1, "file1.jar");
+
+    /** File 2 in directory 1. */
+    private static final File FILE_1_2 = new File(DIR_1, "file2.jar");
+
+    /** Directory 2. */
+    private static final File DIR_2 = HadoopTestUtils.testDir("dir2");
+
+    /** File 1 in directory 2. */
+    private static final File FILE_2_1 = new File(DIR_2, "file1.jar");
+
+    /** File 2 in directory 2. */
+    private static final File FILE_2_2 = new File(DIR_2, "file2.jar");
+
+    /** Missing directory. */
+    private static final File MISSING_DIR = 
HadoopTestUtils.testDir("missing_dir");
+
+    /** Missing file. */
+    private static final File MISSING_FILE = new File(MISSING_DIR, "file.jar");
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        HadoopTestUtils.clearBaseTestDir();
+
+        assert DIR_1.mkdirs();
+        assert DIR_2.mkdirs();
+
+        assert FILE_1_1.createNewFile();
+        assert FILE_1_2.createNewFile();
+        assert FILE_2_1.createNewFile();
+        assert FILE_2_2.createNewFile();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        // Sanity checks before test start.
+        ensureExists(FILE_1_1);
+        ensureExists(FILE_1_2);
+        ensureExists(FILE_2_1);
+        ensureExists(FILE_2_2);
+
+        ensureNotExists(MISSING_DIR);
+        ensureNotExists(MISSING_FILE);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        HadoopTestUtils.clearBaseTestDir();
+    }
+
+    /**
+     * Test null or empty user libs.
+     *
+     * @throws Exception If failed.
+     */
+    public void testNullOrEmptyUserLibs() throws Exception {
+        assert parse(null).isEmpty();
+        assert parse("").isEmpty();
+    }
+
+    /**
+     * Test single file.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSingle() throws Exception {
+        Collection<File> res = parse(single(FILE_1_1));
+
+        assert res.size() == 1;
+        assert res.contains(FILE_1_1);
+
+        res = parse(single(MISSING_FILE));
+
+        assert res.size() == 0;
+    }
+
+    /**
+     * Test multiple files.
+     *
+     * @throws Exception If failed.
+     */
+    public void testMultiple() throws Exception {
+        Collection<File> res =
+            parse(merge(single(FILE_1_1), single(FILE_1_2), single(FILE_2_1), 
single(FILE_2_2), single(MISSING_FILE)));
+
+        assert res.size() == 4;
+        assert res.contains(FILE_1_1);
+        assert res.contains(FILE_1_2);
+        assert res.contains(FILE_2_1);
+        assert res.contains(FILE_2_2);
+    }
+
+    /**
+     * Test single wildcard.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSingleWildcard() throws Exception {
+        Collection<File> res = parse(wildcard(DIR_1));
+
+        assert res.size() == 2;
+        assert res.contains(FILE_1_1);
+        assert res.contains(FILE_1_2);
+
+        res = parse(wildcard(MISSING_DIR));
+
+        assert res.size() == 0;
+    }
+
+    /**
+     * Test multiple wildcards.
+     *
+     * @throws Exception If failed.
+     */
+    public void testMultipleWildcards() throws Exception {
+        Collection<File> res = parse(merge(wildcard(DIR_1), wildcard(DIR_2), 
wildcard(MISSING_DIR)));
+
+        assert res.size() == 4;
+        assert res.contains(FILE_1_1);
+        assert res.contains(FILE_1_2);
+        assert res.contains(FILE_2_1);
+        assert res.contains(FILE_2_2);
+    }
+
+    /**
+     * Test mixed tokens.
+     *
+     * @throws Exception If failed.
+     */
+    public void testMixed() throws Exception {
+        String str = merge(
+            single(FILE_1_1),
+            wildcard(DIR_2),
+            single(MISSING_FILE),
+            wildcard(MISSING_DIR)
+        );
+
+        Collection<File> res = parse(str);
+
+        assert res.size() == 3;
+        assert res.contains(FILE_1_1);
+        assert res.contains(FILE_2_1);
+        assert res.contains(FILE_2_2);
+    }
+    /**
+     * Ensure provided file exists.
+     *
+     * @param file File.
+     */
+    private static void ensureExists(File file) {
+        assert file.exists();
+    }
+
+    /**
+     * Ensure provided file doesn't exist.
+     *
+     * @param file File.
+     */
+    private static void ensureNotExists(File file) {
+        assert !file.exists();
+    }
+
+    /**
+     * Merge string using path separator.
+     *
+     * @param vals Values.
+     * @return Result.
+     */
+    private static String merge(String... vals) {
+        StringBuilder res = new StringBuilder();
+
+        if (vals != null) {
+            boolean first = true;
+
+            for (String val : vals) {
+                if (first)
+                    first = false;
+                else
+                    res.append(File.pathSeparatorChar);
+
+                res.append(val);
+            }
+        }
+
+        return res.toString();
+    }
+
+    /**
+     * Parse string.
+     *
+     * @param str String.
+     * @return Files.
+     * @throws IOException If failed.
+     */
+    Collection<File> parse(String str) throws IOException {
+        Collection<HadoopClasspathUtils.SearchDirectory> dirs = 
HadoopClasspathUtils.parseUserLibs(str);
+
+        Collection<File> res = new HashSet<>();
+
+        for (HadoopClasspathUtils.SearchDirectory dir : dirs)
+            Collections.addAll(res, dir.files());
+
+        return res;
+    }
+
+    /**
+     * Get absolute path to a single file.
+     *
+     * @param file File.
+     * @return Path.
+     */
+    private static String single(File file) {
+        return file.getAbsolutePath();
+    }
+
+    /**
+     * Create a wildcard.
+     *
+     * @param file File.
+     * @return Wildcard.
+     */
+    private static String wildcard(File file) {
+        return file.getAbsolutePath() + File.separatorChar + "*";
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/0dc9713a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
----------------------------------------------------------------------
diff --git 
a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
 
b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
index 3374547..603fd5b 100644
--- 
a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
+++ 
b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
@@ -68,6 +68,7 @@ import 
org.apache.ignite.internal.processors.hadoop.HadoopSplitWrapperSelfTest;
 import 
org.apache.ignite.internal.processors.hadoop.HadoopTaskExecutionSelfTest;
 import org.apache.ignite.internal.processors.hadoop.HadoopTasksV1Test;
 import org.apache.ignite.internal.processors.hadoop.HadoopTasksV2Test;
+import org.apache.ignite.internal.processors.hadoop.HadoopUserLibsSelfTest;
 import org.apache.ignite.internal.processors.hadoop.HadoopV2JobSelfTest;
 import org.apache.ignite.internal.processors.hadoop.HadoopValidationSelfTest;
 import 
org.apache.ignite.internal.processors.hadoop.HadoopWeightedMapReducePlannerTest;
@@ -110,6 +111,8 @@ public class IgniteHadoopTestSuite extends TestSuite {
 
         TestSuite suite = new TestSuite("Ignite Hadoop MR Test Suite");
 
+        suite.addTest(new 
TestSuite(ldr.loadClass(HadoopUserLibsSelfTest.class.getName())));
+
         suite.addTest(new 
TestSuite(ldr.loadClass(HadoopDefaultMapReducePlannerSelfTest.class.getName())));
         suite.addTest(new 
TestSuite(ldr.loadClass(HadoopWeightedMapReducePlannerTest.class.getName())));
 

Reply via email to