Repository: ignite Updated Branches: refs/heads/ignite-1.6.8-hadoop 778878a03 -> 87981a9e2
IGNITE-3943: Hadoop: fixed scrips to match new class loader architecture. This closes #1098. Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/87981a9e Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/87981a9e Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/87981a9e Branch: refs/heads/ignite-1.6.8-hadoop Commit: 87981a9e247577b1c63768e216ca04383f4bbb33 Parents: 778878a Author: vozerov-gridgain <[email protected]> Authored: Thu Sep 22 15:40:11 2016 +0300 Committer: vozerov-gridgain <[email protected]> Committed: Thu Sep 22 15:40:11 2016 +0300 ---------------------------------------------------------------------- bin/include/setenv.bat | 11 +--- bin/include/setenv.sh | 35 ++----------- .../processors/hadoop/HadoopClassLoader.java | 6 ++- .../processors/hadoop/HadoopClasspathMain.java | 44 ---------------- .../processors/hadoop/HadoopClasspathUtils.java | 54 ++++++-------------- .../hadoop/impl/HadoopCommandLineTest.java | 3 +- 6 files changed, 27 insertions(+), 126 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/bin/include/setenv.bat ---------------------------------------------------------------------- diff --git a/bin/include/setenv.bat b/bin/include/setenv.bat index 2e8eac6..6b9abf5 100644 --- a/bin/include/setenv.bat +++ b/bin/include/setenv.bat @@ -41,18 +41,11 @@ goto :eof :: The following libraries are required for Ignite. set IGNITE_LIBS=%IGNITE_HOME%\libs\* -for /D %%F in (%IGNITE_HOME%\libs\*) do if not "%%F" == "%IGNITE_HOME%\libs\optional" call :concat %%F\* - -if exist %IGNITE_HOME%\libs\ignite-hadoop set HADOOP_EDITION=1 +for /D %%F in (%IGNITE_HOME%\libs\*) do if not "%%F" == "%IGNITE_HOME%\libs\optional" ^ + and not "%%F" == "%IGNITE_HOME%\libs\ignite-hadoop-impl" call :concat %%F\* if defined USER_LIBS set IGNITE_LIBS=%USER_LIBS%;%IGNITE_LIBS% -if "%HADOOP_EDITION%" == "1" FOR /F "delims=" %%i IN ('%JAVA_HOME%\bin\java.exe -cp %IGNITE_HOME%\libs\* org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ";"' ) DO set IGNITE_HADOOP_CLASSPATH=%%i - -if "%IGNITE_HADOOP_CLASSPATH%" == "" goto :eof - -set IGNITE_LIBS=%IGNITE_LIBS%;%IGNITE_HADOOP_CLASSPATH% - goto :eof :concat http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/bin/include/setenv.sh ---------------------------------------------------------------------- diff --git a/bin/include/setenv.sh b/bin/include/setenv.sh index 54f5831..c6004e5 100755 --- a/bin/include/setenv.sh +++ b/bin/include/setenv.sh @@ -58,42 +58,13 @@ IGNITE_LIBS="${IGNITE_HOME}/libs/*" for file in ${IGNITE_HOME}/libs/* do - if [ -d ${file} ] && [ "${file}" != "${IGNITE_HOME}"/libs/optional ]; then + if [ -d ${file} ] \ + && [ "${file}" != "${IGNITE_HOME}"/libs/optional ] \ + && [ "${file}" != "${IGNITE_HOME}"/libs/ignite-hadoop-impl ]; then IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/* fi - - if [ -d ${file} ] && [ "${file}" == "${IGNITE_HOME}"/libs/ignite-hadoop ]; then - HADOOP_EDITION=1 - fi done if [ "${USER_LIBS}" != "" ]; then IGNITE_LIBS=${USER_LIBS}${SEP}${IGNITE_LIBS} fi - -if [ "${HADOOP_EDITION}" == "1" ]; then - # Resolve constants. - HADOOP_DEFAULTS="/etc/default/hadoop" - - # - # Resolve the rest of Hadoop environment variables. - # - if [[ -z "${HADOOP_COMMON_HOME}" || -z "${HADOOP_HDFS_HOME}" || -z "${HADOOP_MAPRED_HOME}" ]]; then - if [ -f "$HADOOP_DEFAULTS" ]; then - source "$HADOOP_DEFAULTS" - fi - fi - - IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/'*' \ - org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ":" ) - - statusCode=${?} - - if [ "${statusCode}" -ne 0 ]; then - exit ${statusCode} - fi - - unset statusCode - - IGNITE_LIBS=${IGNITE_LIBS}${SEP}${IGNITE_HADOOP_CLASSPATH} -fi http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java index 60f5f55..4751283 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java @@ -311,7 +311,11 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { byte[] bytes = bytesCache.get(originalName); if (bytes == null) { - InputStream in = helper.loadClassBytes(getParent(), replaceName); + InputStream in = helper.loadClassBytes(this, replaceName); + + if (in == null) + throw new IgniteException("Failed to replace class [originalName=" + originalName + + ", replaceName=" + replaceName + ']'); bytes = helper.loadReplace(in, originalName, replaceName); http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java deleted file mode 100644 index 4069496..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -/** - * Main class to compose Hadoop classpath depending on the environment. - * This class is designed to be independent on any Ignite classes as possible. - * Please make sure to pass the path separator character as the 1st parameter to the main method. - */ -public class HadoopClasspathMain { - /** - * Main method to be executed from scripts. It prints the classpath to the standard output. - * - * @param args The 1st argument should be the path separator character (":" on Linux, ";" on Windows). - */ - public static void main(String[] args) throws Exception { - if (args.length < 1) - throw new IllegalArgumentException("Path separator must be passed as the first argument."); - - String separator = args[0]; - - StringBuilder sb = new StringBuilder(); - - for (String path : HadoopClasspathUtils.classpathForProcess()) - sb.append(path).append(separator); - - System.out.println(sb); - } -} http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java index a0ef7d7..163ddfa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java @@ -34,6 +34,9 @@ import java.util.List; * Hadoop classpath utilities. */ public class HadoopClasspathUtils { + /** Ignite home. */ + private static final String IGNITE_HOME = "IGNITE_HOME"; + /** Prefix directory. */ public static final String PREFIX = "HADOOP_PREFIX"; @@ -56,31 +59,6 @@ public class HadoopClasspathUtils { private static final String EMPTY_STR = ""; /** - * Gets Hadoop class path as list of classpath elements for process. - * - * @return List of the class path elements. - * @throws IOException If failed. - */ - public static List<String> classpathForProcess() throws IOException { - List<String> res = new ArrayList<>(); - - for (final SearchDirectory dir : classpathDirectories()) { - File[] files = dir.files(); - - if (dir.useWildcard()) { - if (files.length > 0) - res.add(dir.absolutePath() + File.separator + '*'); - } - else { - for (File file : files) - res.add(file.getAbsolutePath()); - } - } - - return res; - } - - /** * Gets Hadoop class path as a list of URLs (for in-process class loader usage). * * @return List of class path URLs. @@ -190,6 +168,7 @@ public class HadoopClasspathUtils { Collection<SearchDirectory> res = new ArrayList<>(); + // 1. Add libraries from Hadoop distribution: res.add(new SearchDirectory(new File(loc.common(), "lib"), AcceptAllDirectoryFilter.INSTANCE)); res.add(new SearchDirectory(new File(loc.hdfs(), "lib"), AcceptAllDirectoryFilter.INSTANCE)); res.add(new SearchDirectory(new File(loc.mapred(), "lib"), AcceptAllDirectoryFilter.INSTANCE)); @@ -204,8 +183,19 @@ public class HadoopClasspathUtils { res.add(new SearchDirectory(new File(loc.mapred()), new PrefixDirectoryFilter("hadoop-mapreduce-client-core"))); + // 2. Add user provided libs: res.addAll(parseUserLibs()); + // 3. Add hadoop-impl module. + String home = systemOrEnv(IGNITE_HOME, null); + + if (home != null) { + File dir = new File(home, "libs/ignite-hadoop-impl"); + + if (dir.exists()) + res.add(new SearchDirectory(dir, AcceptAllDirectoryFilter.INSTANCE, true)); + } + return res; } @@ -353,13 +343,6 @@ public class HadoopClasspathUtils { } /** - * @return Absolute path. - */ - public String absolutePath() { - return dir.getAbsolutePath(); - } - - /** * @return Child files. */ public File[] files() throws IOException { @@ -378,13 +361,6 @@ public class HadoopClasspathUtils { else return files; } - - /** - * @return {@code True} if wildcard can be used. - */ - public boolean useWildcard() { - return filter instanceof AcceptAllDirectoryFilter; - } } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java index 0be8bf9..0ca2851 100644 --- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java +++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java @@ -247,7 +247,8 @@ public class HadoopCommandLineTest extends GridCommonAbstractTest { String ggClsPath = HadoopJob.class.getProtectionDomain().getCodeSource().getLocation().getPath() + sep + HadoopJobTracker.class.getProtectionDomain().getCodeSource().getLocation().getPath() + sep + - ConcurrentHashMap8.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + ConcurrentHashMap8.class.getProtectionDomain().getCodeSource().getLocation().getPath() + + sep + U.getIgniteHome() + "/modules/hadoop-impl/target/classes"; ProcessBuilder res = new ProcessBuilder();
