Tests.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/86579fed Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/86579fed Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/86579fed Branch: refs/heads/ignite-3906 Commit: 86579fed9005940d2b09da61045f7e2fc045a482 Parents: 268392d Author: vozerov-gridgain <voze...@gridgain.com> Authored: Thu Sep 15 13:13:28 2016 +0300 Committer: vozerov-gridgain <voze...@gridgain.com> Committed: Thu Sep 15 13:13:28 2016 +0300 ---------------------------------------------------------------------- .../processors/hadoop/HadoopClasspathUtils.java | 40 ++- .../processors/hadoop/HadoopLocations.java | 14 +- .../processors/hadoop/HadoopTestUtils.java | 73 +++++- .../hadoop/HadoopUserLibsSelfTest.java | 260 +++++++++++++++++++ .../testsuites/IgniteHadoopTestSuite.java | 3 + 5 files changed, 355 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/86579fed/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java index 71acf16..10f901b 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java @@ -114,11 +114,9 @@ public class HadoopClasspathUtils { String hdfsHome = systemOrEnv(HDFS_HOME, EMPTY_STR); String mapredHome = systemOrEnv(MAPRED_HOME, EMPTY_STR); - String userLibs = systemOrEnv(HADOOP_USER_LIBS, null); - // If any composite location is defined, use only them. if (!isEmpty(commonHome) || !isEmpty(hdfsHome) || !isEmpty(mapredHome)) { - HadoopLocations res = new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome, userLibs); + HadoopLocations res = new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome); if (res.valid()) return res; @@ -140,8 +138,7 @@ public class HadoopClasspathUtils { hadoopHome, hadoopHome + "/share/hadoop/common", hadoopHome + "/share/hadoop/hdfs", - hadoopHome + "/share/hadoop/mapreduce", - userLibs + hadoopHome + "/share/hadoop/mapreduce" ); if (res.valid()) @@ -152,8 +149,7 @@ public class HadoopClasspathUtils { hadoopHome, hadoopHome, hadoopHome + "/../hadoop-hdfs", - hadoopHome + "/../hadoop-mapreduce", - userLibs + hadoopHome + "/../hadoop-mapreduce" ); if (res.valid()) @@ -164,8 +160,7 @@ public class HadoopClasspathUtils { hadoopHome, hadoopHome, hadoopHome + "/../hadoop-hdfs-client", - hadoopHome + "/../hadoop-mapreduce-client", - userLibs + hadoopHome + "/../hadoop-mapreduce-client" ); if (res.valid()) @@ -207,7 +202,7 @@ public class HadoopClasspathUtils { res.add(new SearchDirectory(new File(loc.mapred()), new PrefixDirectoryFilter("hadoop-mapreduce-client-core"))); - res.addAll(parseUserLibs(loc.userLibs())); + res.addAll(parseUserLibs()); return res; } @@ -215,11 +210,14 @@ public class HadoopClasspathUtils { /** * Parse user libs. * - * @param str Original string. * @return Parsed libs search patterns. * @throws IOException If failed. */ - private static Collection<SearchDirectory> parseUserLibs(String str) throws IOException { + static Collection<SearchDirectory> parseUserLibs() throws IOException { + return parseUserLibs(systemOrEnv(HADOOP_USER_LIBS, null)); + } + + static Collection<SearchDirectory> parseUserLibs(String str) throws IOException { Collection<SearchDirectory> res = new LinkedList<>(); if (!isEmpty(str)) { @@ -232,7 +230,7 @@ public class HadoopClasspathUtils { if (token.endsWith("*")) { // Wildcard. - File dir = new File(token.substring(0, token.length() - 1)).getParentFile(); + File dir = new File(token).getParentFile(); assert dir != null; @@ -311,7 +309,7 @@ public class HadoopClasspathUtils { /** * Simple pair-like structure to hold directory name and a mask assigned to it. */ - private static class SearchDirectory { + static class SearchDirectory { /** File. */ private final File dir; @@ -352,14 +350,14 @@ public class HadoopClasspathUtils { /** * @return Absolute path. */ - private String absolutePath() { + String absolutePath() { return dir.getAbsolutePath(); } /** * @return Child files. */ - private File[] files() throws IOException { + File[] files() throws IOException { File[] files = dir.listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { return filter.test(name); @@ -379,7 +377,7 @@ public class HadoopClasspathUtils { /** * @return {@code True} if wildcard can be used. */ - private boolean useWildcard() { + boolean useWildcard() { return filter instanceof AcceptAllDirectoryFilter; } } @@ -387,7 +385,7 @@ public class HadoopClasspathUtils { /** * Directory filter interface. */ - private static interface DirectoryFilter { + static interface DirectoryFilter { /** * Test if file with this name should be included. * @@ -400,7 +398,7 @@ public class HadoopClasspathUtils { /** * Filter to accept all files. */ - public static class AcceptAllDirectoryFilter implements DirectoryFilter { + static class AcceptAllDirectoryFilter implements DirectoryFilter { /** Singleton instance. */ public static final AcceptAllDirectoryFilter INSTANCE = new AcceptAllDirectoryFilter(); @@ -413,7 +411,7 @@ public class HadoopClasspathUtils { /** * Filter which uses prefix to filter files. */ - public static class PrefixDirectoryFilter implements DirectoryFilter { + static class PrefixDirectoryFilter implements DirectoryFilter { /** Prefix. */ private final String prefix; @@ -437,7 +435,7 @@ public class HadoopClasspathUtils { /** * Filter which uses exact comparison. */ - public static class ExactDirectoryFilter implements DirectoryFilter { + static class ExactDirectoryFilter implements DirectoryFilter { /** Name. */ private final String name; http://git-wip-us.apache.org/repos/asf/ignite/blob/86579fed/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java index 066cf7c..a90007f 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java @@ -42,9 +42,6 @@ public class HadoopLocations { /** Whether mapred home exists. */ private final boolean mapredExists; - /** User libs. */ - private final String userLibs; - /** * Constructor. * @@ -52,16 +49,14 @@ public class HadoopLocations { * @param common Common home. * @param hdfs HDFS home. * @param mapred Mapred home. - * @param userLibs User libs. */ - public HadoopLocations(String home, String common, String hdfs, String mapred, String userLibs) { + public HadoopLocations(String home, String common, String hdfs, String mapred) { assert common != null && hdfs != null && mapred != null; this.home = home; this.common = common; this.hdfs = hdfs; this.mapred = mapred; - this.userLibs = userLibs; commonExists = HadoopClasspathUtils.exists(common); hdfsExists = HadoopClasspathUtils.exists(hdfs); @@ -97,13 +92,6 @@ public class HadoopLocations { } /** - * @return User libs. - */ - public String userLibs() { - return userLibs; - } - - /** * @return Whether common home exists. */ public boolean commonExists() { http://git-wip-us.apache.org/repos/asf/ignite/blob/86579fed/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java index 9ebad78..da0d922 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTestUtils.java @@ -17,14 +17,18 @@ package org.apache.ignite.internal.processors.hadoop; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.jetbrains.annotations.Nullable; + import java.io.BufferedReader; +import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import org.apache.ignite.internal.util.typedef.F; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -33,6 +37,41 @@ import static org.junit.Assert.assertTrue; * Utility class for tests. */ public class HadoopTestUtils { + /** Base test directory. */ + private static final File BASE_TEST_DIR = new File(U.getIgniteHome() + "/work/test/hadoop/"); + + /** + * @return Base directory for tests. + */ + public static File baseTestDir() { + return BASE_TEST_DIR; + } + + /** + * Get test directory. + * + * @param parts Parts. + * @return Directory. + */ + public static File testDir(String... parts) { + File res = BASE_TEST_DIR; + + if (parts != null) { + for (String part : parts) + res = new File(res, part); + } + + return res; + } + + /** + * Clear base test directory. + */ + public static void clearBaseTestDir() { + if (baseTestDir().exists()) + assert delete(baseTestDir()); + } + /** * Checks that job statistics file contains valid strings only. * @@ -40,6 +79,7 @@ public class HadoopTestUtils { * @return Amount of events. * @throws IOException If failed. */ + @SuppressWarnings("ResultOfMethodCallIgnored") public static long simpleCheckJobStatFile(BufferedReader reader) throws IOException { Collection<String> phases = new HashSet<>(); @@ -104,4 +144,35 @@ public class HadoopTestUtils { return evtCnt; } + + /** + * Deletes file or directory with all sub-directories and files. + * + * @param file File or directory to delete. + * @return {@code true} if and only if the file or directory is successfully deleted, + * {@code false} otherwise + */ + public static boolean delete(@Nullable File file) { + if (file == null) + return false; + + boolean res = true; + + if (file.isDirectory()) { + File[] files = file.listFiles(); + + if (files != null && files.length > 0) + for (File file1 : files) + if (file1.isDirectory()) + res &= delete(file1); + else + res &= file1.delete(); + + res &= file.delete(); + } + else + res = file.delete(); + + return res; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/86579fed/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopUserLibsSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopUserLibsSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopUserLibsSelfTest.java new file mode 100644 index 0000000..9e3c8f4 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopUserLibsSelfTest.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop; + +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; + +/** + * Tests for user libs parsing. + */ +public class HadoopUserLibsSelfTest extends GridCommonAbstractTest { + /** Directory 1. */ + private static final File DIR_1 = HadoopTestUtils.testDir("dir1"); + + /** File 1 in directory 1. */ + private static final File FILE_1_1 = new File(DIR_1, "file1.jar"); + + /** File 2 in directory 1. */ + private static final File FILE_1_2 = new File(DIR_1, "file2.jar"); + + /** Directory 2. */ + private static final File DIR_2 = HadoopTestUtils.testDir("dir2"); + + /** File 1 in directory 2. */ + private static final File FILE_2_1 = new File(DIR_2, "file1.jar"); + + /** File 2 in directory 2. */ + private static final File FILE_2_2 = new File(DIR_2, "file2.jar"); + + /** Missing directory. */ + private static final File MISSING_DIR = HadoopTestUtils.testDir("missing_dir"); + + /** Missing file. */ + private static final File MISSING_FILE = new File(MISSING_DIR, "file.jar"); + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + HadoopTestUtils.clearBaseTestDir(); + + assert DIR_1.mkdirs(); + assert DIR_2.mkdirs(); + + assert FILE_1_1.createNewFile(); + assert FILE_1_2.createNewFile(); + assert FILE_2_1.createNewFile(); + assert FILE_2_2.createNewFile(); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + // Sanity checks before test start. + ensureExists(FILE_1_1); + ensureExists(FILE_1_2); + ensureExists(FILE_2_1); + ensureExists(FILE_2_2); + + ensureNotExists(MISSING_DIR); + ensureNotExists(MISSING_FILE); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + HadoopTestUtils.clearBaseTestDir(); + } + + /** + * Test null or empty user libs. + * + * @throws Exception If failed. + */ + public void testNullOrEmptyUserLibs() throws Exception { + assert parse(null).isEmpty(); + assert parse("").isEmpty(); + } + + /** + * Test single file. + * + * @throws Exception If failed. + */ + public void testSingle() throws Exception { + Collection<File> res = parse(single(FILE_1_1)); + + assert res.size() == 1; + assert res.contains(FILE_1_1); + + res = parse(single(MISSING_FILE)); + + assert res.size() == 0; + } + + /** + * Test multiple files. + * + * @throws Exception If failed. + */ + public void testMultiple() throws Exception { + Collection<File> res = + parse(merge(single(FILE_1_1), single(FILE_1_2), single(FILE_2_1), single(FILE_2_2), single(MISSING_FILE))); + + assert res.size() == 4; + assert res.contains(FILE_1_1); + assert res.contains(FILE_1_2); + assert res.contains(FILE_2_1); + assert res.contains(FILE_2_2); + } + + /** + * Test single wildcard. + * + * @throws Exception If failed. + */ + public void testSingleWildcard() throws Exception { + Collection<File> res = parse(wildcard(DIR_1)); + + assert res.size() == 2; + assert res.contains(FILE_1_1); + assert res.contains(FILE_1_2); + + res = parse(wildcard(MISSING_DIR)); + + assert res.size() == 0; + } + + /** + * Test multiple wildcards. + * + * @throws Exception If failed. + */ + public void testMultipleWildcards() throws Exception { + Collection<File> res = parse(merge(wildcard(DIR_1), wildcard(DIR_2), wildcard(MISSING_DIR))); + + assert res.size() == 4; + assert res.contains(FILE_1_1); + assert res.contains(FILE_1_2); + assert res.contains(FILE_2_1); + assert res.contains(FILE_2_2); + } + + /** + * Test mixed tokens. + * + * @throws Exception If failed. + */ + public void testMixed() throws Exception { + String str = merge( + single(FILE_1_1), + wildcard(DIR_2), + single(MISSING_FILE), + wildcard(MISSING_DIR) + ); + + Collection<File> res = parse(str); + + assert res.size() == 3; + assert res.contains(FILE_1_1); + assert res.contains(FILE_2_1); + assert res.contains(FILE_2_2); + } + /** + * Ensure provided file exists. + * + * @param file File. + */ + private static void ensureExists(File file) { + assert file.exists(); + } + + /** + * Ensure provided file doesn't exist. + * + * @param file File. + */ + private static void ensureNotExists(File file) { + assert !file.exists(); + } + + /** + * Merge string using path separator. + * + * @param vals Values. + * @return Result. + */ + private static String merge(String... vals) { + StringBuilder res = new StringBuilder(); + + if (vals != null) { + boolean first = true; + + for (String val : vals) { + if (first) + first = false; + else + res.append(File.pathSeparatorChar); + + res.append(val); + } + } + + return res.toString(); + } + + /** + * Parse string. + * + * @param str String. + * @return Files. + * @throws IOException If failed. + */ + Collection<File> parse(String str) throws IOException { + Collection<HadoopClasspathUtils.SearchDirectory> dirs = HadoopClasspathUtils.parseUserLibs(str); + + Collection<File> res = new HashSet<>(); + + for (HadoopClasspathUtils.SearchDirectory dir : dirs) + Collections.addAll(res, dir.files()); + + return res; + } + + /** + * Get absolute path to a single file. + * + * @param file File. + * @return Path. + */ + private static String single(File file) { + return file.getAbsolutePath(); + } + + /** + * Create a wildcard. + * + * @param file File. + * @return Wildcard. + */ + private static String wildcard(File file) { + return file.getAbsolutePath() + File.separatorChar + "*"; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/86579fed/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java index 3374547..603fd5b 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java @@ -68,6 +68,7 @@ import org.apache.ignite.internal.processors.hadoop.HadoopSplitWrapperSelfTest; import org.apache.ignite.internal.processors.hadoop.HadoopTaskExecutionSelfTest; import org.apache.ignite.internal.processors.hadoop.HadoopTasksV1Test; import org.apache.ignite.internal.processors.hadoop.HadoopTasksV2Test; +import org.apache.ignite.internal.processors.hadoop.HadoopUserLibsSelfTest; import org.apache.ignite.internal.processors.hadoop.HadoopV2JobSelfTest; import org.apache.ignite.internal.processors.hadoop.HadoopValidationSelfTest; import org.apache.ignite.internal.processors.hadoop.HadoopWeightedMapReducePlannerTest; @@ -110,6 +111,8 @@ public class IgniteHadoopTestSuite extends TestSuite { TestSuite suite = new TestSuite("Ignite Hadoop MR Test Suite"); + suite.addTest(new TestSuite(ldr.loadClass(HadoopUserLibsSelfTest.class.getName()))); + suite.addTest(new TestSuite(ldr.loadClass(HadoopDefaultMapReducePlannerSelfTest.class.getName()))); suite.addTest(new TestSuite(ldr.loadClass(HadoopWeightedMapReducePlannerTest.class.getName())));
