Repository: ignite
Updated Branches:
  refs/heads/ignite-1.6.8-hadoop 778878a03 -> 87981a9e2


IGNITE-3943: Hadoop: fixed scrips to match new class loader architecture. This 
closes #1098.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/87981a9e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/87981a9e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/87981a9e

Branch: refs/heads/ignite-1.6.8-hadoop
Commit: 87981a9e247577b1c63768e216ca04383f4bbb33
Parents: 778878a
Author: vozerov-gridgain <voze...@gridgain.com>
Authored: Thu Sep 22 15:40:11 2016 +0300
Committer: vozerov-gridgain <voze...@gridgain.com>
Committed: Thu Sep 22 15:40:11 2016 +0300

----------------------------------------------------------------------
 bin/include/setenv.bat                          | 11 +---
 bin/include/setenv.sh                           | 35 ++-----------
 .../processors/hadoop/HadoopClassLoader.java    |  6 ++-
 .../processors/hadoop/HadoopClasspathMain.java  | 44 ----------------
 .../processors/hadoop/HadoopClasspathUtils.java | 54 ++++++--------------
 .../hadoop/impl/HadoopCommandLineTest.java      |  3 +-
 6 files changed, 27 insertions(+), 126 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/bin/include/setenv.bat
----------------------------------------------------------------------
diff --git a/bin/include/setenv.bat b/bin/include/setenv.bat
index 2e8eac6..6b9abf5 100644
--- a/bin/include/setenv.bat
+++ b/bin/include/setenv.bat
@@ -41,18 +41,11 @@ goto :eof
 :: The following libraries are required for Ignite.
 set IGNITE_LIBS=%IGNITE_HOME%\libs\*
 
-for /D %%F in (%IGNITE_HOME%\libs\*) do if not "%%F" == 
"%IGNITE_HOME%\libs\optional" call :concat %%F\*
-
-if exist %IGNITE_HOME%\libs\ignite-hadoop set HADOOP_EDITION=1
+for /D %%F in (%IGNITE_HOME%\libs\*) do if not "%%F" == 
"%IGNITE_HOME%\libs\optional" ^
+    and not "%%F" == "%IGNITE_HOME%\libs\ignite-hadoop-impl" call :concat %%F\*
 
 if defined USER_LIBS set IGNITE_LIBS=%USER_LIBS%;%IGNITE_LIBS%
 
-if "%HADOOP_EDITION%" == "1" FOR /F "delims=" %%i IN 
('%JAVA_HOME%\bin\java.exe -cp %IGNITE_HOME%\libs\* 
org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ";"' ) DO set 
IGNITE_HADOOP_CLASSPATH=%%i
-
-if "%IGNITE_HADOOP_CLASSPATH%" == "" goto :eof
-
-set IGNITE_LIBS=%IGNITE_LIBS%;%IGNITE_HADOOP_CLASSPATH%
-
 goto :eof
 
 :concat

http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/bin/include/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/include/setenv.sh b/bin/include/setenv.sh
index 54f5831..c6004e5 100755
--- a/bin/include/setenv.sh
+++ b/bin/include/setenv.sh
@@ -58,42 +58,13 @@ IGNITE_LIBS="${IGNITE_HOME}/libs/*"
 
 for file in ${IGNITE_HOME}/libs/*
 do
-    if [ -d ${file} ] && [ "${file}" != "${IGNITE_HOME}"/libs/optional ]; then
+    if [ -d ${file} ] \
+        && [ "${file}" != "${IGNITE_HOME}"/libs/optional ] \
+        && [ "${file}" != "${IGNITE_HOME}"/libs/ignite-hadoop-impl ]; then
         IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/*
     fi
-
-    if [ -d ${file} ] && [ "${file}" == "${IGNITE_HOME}"/libs/ignite-hadoop ]; 
then
-        HADOOP_EDITION=1
-    fi
 done
 
 if [ "${USER_LIBS}" != "" ]; then
     IGNITE_LIBS=${USER_LIBS}${SEP}${IGNITE_LIBS}
 fi
-
-if [ "${HADOOP_EDITION}" == "1" ]; then
-    # Resolve constants.
-    HADOOP_DEFAULTS="/etc/default/hadoop"
-
-    #
-    # Resolve the rest of Hadoop environment variables.
-    #
-    if [[ -z "${HADOOP_COMMON_HOME}" || -z "${HADOOP_HDFS_HOME}" || -z 
"${HADOOP_MAPRED_HOME}" ]]; then
-        if [ -f "$HADOOP_DEFAULTS" ]; then
-            source "$HADOOP_DEFAULTS"
-        fi
-    fi
-
-    IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/'*' \
-        org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ":" )
-
-    statusCode=${?}
-
-    if [ "${statusCode}" -ne 0 ]; then
-       exit ${statusCode}
-    fi
-
-    unset statusCode
-
-    IGNITE_LIBS=${IGNITE_LIBS}${SEP}${IGNITE_HADOOP_CLASSPATH}
-fi

http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 60f5f55..4751283 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -311,7 +311,11 @@ public class HadoopClassLoader extends URLClassLoader 
implements ClassCache {
             byte[] bytes = bytesCache.get(originalName);
 
             if (bytes == null) {
-                InputStream in = helper.loadClassBytes(getParent(), 
replaceName);
+                InputStream in = helper.loadClassBytes(this, replaceName);
+
+                if (in == null)
+                    throw new IgniteException("Failed to replace class 
[originalName=" + originalName +
+                        ", replaceName=" +  replaceName + ']');
 
                 bytes = helper.loadReplace(in, originalName, replaceName);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
deleted file mode 100644
index 4069496..0000000
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop;
-
-/**
- * Main class to compose Hadoop classpath depending on the environment.
- * This class is designed to be independent on any Ignite classes as possible.
- * Please make sure to pass the path separator character as the 1st parameter 
to the main method.
- */
-public class HadoopClasspathMain {
-    /**
-     * Main method to be executed from scripts. It prints the classpath to the 
standard output.
-     *
-     * @param args The 1st argument should be the path separator character 
(":" on Linux, ";" on Windows).
-     */
-    public static void main(String[] args) throws Exception {
-        if (args.length < 1)
-            throw new IllegalArgumentException("Path separator must be passed 
as the first argument.");
-
-        String separator = args[0];
-
-        StringBuilder sb = new StringBuilder();
-
-        for (String path : HadoopClasspathUtils.classpathForProcess())
-            sb.append(path).append(separator);
-
-        System.out.println(sb);
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index a0ef7d7..163ddfa 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -34,6 +34,9 @@ import java.util.List;
  * Hadoop classpath utilities.
  */
 public class HadoopClasspathUtils {
+    /** Ignite home. */
+    private static final String IGNITE_HOME = "IGNITE_HOME";
+
     /** Prefix directory. */
     public static final String PREFIX = "HADOOP_PREFIX";
 
@@ -56,31 +59,6 @@ public class HadoopClasspathUtils {
     private static final String EMPTY_STR = "";
 
     /**
-     * Gets Hadoop class path as list of classpath elements for process.
-     *
-     * @return List of the class path elements.
-     * @throws IOException If failed.
-     */
-    public static List<String> classpathForProcess() throws IOException {
-        List<String> res = new ArrayList<>();
-
-        for (final SearchDirectory dir : classpathDirectories()) {
-            File[] files = dir.files();
-
-            if (dir.useWildcard()) {
-                if (files.length > 0)
-                    res.add(dir.absolutePath() + File.separator + '*');
-            }
-            else {
-                for (File file : files)
-                    res.add(file.getAbsolutePath());
-            }
-        }
-
-        return res;
-    }
-
-    /**
      * Gets Hadoop class path as a list of URLs (for in-process class loader 
usage).
      *
      * @return List of class path URLs.
@@ -190,6 +168,7 @@ public class HadoopClasspathUtils {
 
         Collection<SearchDirectory> res = new ArrayList<>();
 
+        // 1. Add libraries from Hadoop distribution:
         res.add(new SearchDirectory(new File(loc.common(), "lib"), 
AcceptAllDirectoryFilter.INSTANCE));
         res.add(new SearchDirectory(new File(loc.hdfs(), "lib"), 
AcceptAllDirectoryFilter.INSTANCE));
         res.add(new SearchDirectory(new File(loc.mapred(), "lib"), 
AcceptAllDirectoryFilter.INSTANCE));
@@ -204,8 +183,19 @@ public class HadoopClasspathUtils {
         res.add(new SearchDirectory(new File(loc.mapred()),
             new PrefixDirectoryFilter("hadoop-mapreduce-client-core")));
 
+        // 2. Add user provided libs:
         res.addAll(parseUserLibs());
 
+        // 3. Add hadoop-impl module.
+        String home = systemOrEnv(IGNITE_HOME, null);
+
+        if (home != null) {
+            File dir = new File(home, "libs/ignite-hadoop-impl");
+
+            if (dir.exists())
+                res.add(new SearchDirectory(dir, 
AcceptAllDirectoryFilter.INSTANCE, true));
+        }
+
         return res;
     }
 
@@ -353,13 +343,6 @@ public class HadoopClasspathUtils {
         }
 
         /**
-         * @return Absolute path.
-         */
-        public String absolutePath() {
-            return dir.getAbsolutePath();
-        }
-
-        /**
          * @return Child files.
          */
         public File[] files() throws IOException {
@@ -378,13 +361,6 @@ public class HadoopClasspathUtils {
             else
                 return files;
         }
-
-        /**
-         * @return {@code True} if wildcard can be used.
-         */
-        public boolean useWildcard() {
-            return filter instanceof AcceptAllDirectoryFilter;
-        }
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/87981a9e/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java
----------------------------------------------------------------------
diff --git 
a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java
 
b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java
index 0be8bf9..0ca2851 100644
--- 
a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java
+++ 
b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopCommandLineTest.java
@@ -247,7 +247,8 @@ public class HadoopCommandLineTest extends 
GridCommonAbstractTest {
 
         String ggClsPath = 
HadoopJob.class.getProtectionDomain().getCodeSource().getLocation().getPath() + 
sep +
             
HadoopJobTracker.class.getProtectionDomain().getCodeSource().getLocation().getPath()
 + sep +
-            
ConcurrentHashMap8.class.getProtectionDomain().getCodeSource().getLocation().getPath();
+            
ConcurrentHashMap8.class.getProtectionDomain().getCodeSource().getLocation().getPath()
+            + sep + U.getIgniteHome() + "/modules/hadoop-impl/target/classes";
 
         ProcessBuilder res = new ProcessBuilder();
 

Reply via email to