Repository: incubator-slider Updated Branches: refs/heads/develop fbeb02e64 -> 06ff4c2ea
Revert "SLIDER-515. Ensure Slider clients uploads all jars needed by AppMaster and construct own class path" This reverts commit eb790e780d262eb29b5551d21956faac78347145. Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/06ff4c2e Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/06ff4c2e Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/06ff4c2e Branch: refs/heads/develop Commit: 06ff4c2ea51acb44f308a0ce52b5a50e85442bfa Parents: fbeb02e Author: Sumit Mohanty <[email protected]> Authored: Sat Oct 25 07:31:18 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Sat Oct 25 07:31:18 2014 -0700 ---------------------------------------------------------------------- slider-assembly/src/main/scripts/slider | 4 +- slider-assembly/src/main/scripts/slider.py | 2 - .../org/apache/slider/common/SliderKeys.java | 6 --- .../apache/slider/common/tools/SliderUtils.java | 40 +------------------- .../apache/slider/providers/ProviderUtils.java | 28 -------------- .../slideram/SliderAMClientProvider.java | 37 ++++++++++++++---- 6 files changed, 31 insertions(+), 86 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ff4c2e/slider-assembly/src/main/scripts/slider ---------------------------------------------------------------------- diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider index 80b5f32..4228843 100755 --- a/slider-assembly/src/main/scripts/slider +++ b/slider-assembly/src/main/scripts/slider @@ -56,9 +56,7 @@ if [ -f "${confdir}/slider-env.sh" ]; then fi slider_confdir_opts="-Dslider.confdir=${confdir}" -libdir_jvm_opts="-Dslider.libdir=${libdir}" - -slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true ${libdir_jvm_opts} -Xmx256m" +slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m" slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts} # allow for an extra classpath http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ff4c2e/slider-assembly/src/main/scripts/slider.py ---------------------------------------------------------------------- diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py index c72fc0d..a6dd90e 100644 --- a/slider-assembly/src/main/scripts/slider.py +++ b/slider-assembly/src/main/scripts/slider.py @@ -247,8 +247,6 @@ def main(): executeEnvSh(confdir) jvm_opts_list = (SLIDER_CONFDIR_OPTS % confdir).split() default_jvm_opts = DEFAULT_JVM_OPTS - libdir_jvm_opts = "-Dslider.libdir={0}".format(libdir) - default_jvm_opts = "{0} {1}".format(default_jvm_opts, libdir_jvm_opts) slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts) jvm_opts_list.extend(slider_jvm_opts.split()) slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ff4c2e/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 048dfa7..ae58ef3 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -122,12 +122,6 @@ public interface SliderKeys extends SliderXmlConfKeys { String PROPERTY_CONF_DIR = "slider.confdir"; /** - * JVM property to define the slider lib directory; - * this is set by the slider script: {@value} - */ - String PROPERTY_LIB_DIR = "slider.libdir"; - - /** * name of generated dir for this conf: {@value} */ String SUBMITTED_CONF_DIR = "confdir"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ff4c2e/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 43dfbd0..33e028f 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -40,7 +40,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.VersionInfo; -import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -71,7 +70,6 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; -import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; @@ -1172,42 +1170,6 @@ public final class SliderUtils { return res; } - /** - * Submit a JAR containing and map it - * @param providerResources provider map to build up - * @param sliderFileSystem remote fs - * @param libDir lib directory - * @param srcPath copy jars from - * @throws IOException, SliderException trouble copying to HDFS - */ - public static void putAllJars(Map<String, LocalResource> providerResources, - SliderFileSystem sliderFileSystem, - Path tempPath, - String libDir, - String srcPath - ) - throws IOException, SliderException { - log.info("Loading all dependencies from {}", srcPath); - if (SliderUtils.isSet(srcPath)) { - File srcFolder = new File(srcPath); - FilenameFilter jarFilter = new FilenameFilter() { - public boolean accept(File dir, String name) { - String lowercaseName = name.toLowerCase(); - if (lowercaseName.endsWith(".jar")) { - return true; - } else { - return false; - } - } - }; - File[] listOfJars = srcFolder.listFiles(jarFilter); - for (File jarFile : listOfJars) { - LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName()); - providerResources.put(libDir + "/" + jarFile.getName(), res); - } - } - } - public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) { Map<String, Map<String, String>> dest = new HashMap<String, Map<String, String>>(); @@ -1339,7 +1301,7 @@ public final class SliderUtils { } classpath.addLibDir(libdir); classpath.addRemoteClasspathEnvVar(); - classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$()); + classpath.appendAll(classpath.yarnApplicationClasspath(config)); } return classpath; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ff4c2e/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java index e26105e..8e77a9c 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java @@ -175,34 +175,6 @@ public class ProviderUtils implements RoleKeys { } /** - * Loads all dependency jars from the default path - * @param providerResources map of provider resources to add these entries to - * @param sliderFileSystem target filesystem - * @param tempPath path in the cluster FS for temp files - * @param libDir relative directory to place resources - * @param libLocalSrcDir explicitly supplied local libs dir - * @throws IOException - * @throws SliderException - */ - public static void addAllDependencyJars(Map<String, LocalResource> providerResources, - SliderFileSystem sliderFileSystem, - Path tempPath, - String libDir, - String libLocalSrcDir - ) throws - IOException, - SliderException { - String libSrcToUse = libLocalSrcDir; - if (SliderUtils.isSet(libLocalSrcDir)) { - File file = new File(libLocalSrcDir); - if (!file.exists() || !file.isDirectory()) { - throw new BadCommandArgumentsException("Supplied lib src dir %s is not valid", libLocalSrcDir); - } - } - SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath, libDir, libSrcToUse); - } - - /** * build the log directory * @return the log dir */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ff4c2e/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java index 60edce7..5edf1bf 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java @@ -18,6 +18,11 @@ package org.apache.slider.providers.slideram; +import com.beust.jcommander.JCommander; +import com.codahale.metrics.MetricRegistry; +import com.google.gson.GsonBuilder; +import org.apache.curator.CuratorZookeeperClient; +import org.apache.curator.framework.CuratorFramework; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; @@ -42,6 +47,7 @@ import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderUtils; +import org.mortbay.jetty.security.SslSelectChannelConnector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -181,14 +187,29 @@ public class SliderAMClientProvider extends AbstractClientProvider libdir, miniClusterTestRun); - String libDirProp = - System.getProperty(SliderKeys.PROPERTY_LIB_DIR); - log.info("Loading all dependencies for AM."); - ProviderUtils.addAllDependencyJars(providerResources, - fileSystem, - tempPath, - libdir, - libDirProp); + Class<?>[] classes = { + JCommander.class, + GsonBuilder.class, + SslSelectChannelConnector.class, + + CuratorFramework.class, + CuratorZookeeperClient.class, + MetricRegistry.class + }; + String[] jars = + { + JCOMMANDER_JAR, + GSON_JAR, + "jetty-sslengine.jar", + + "curator-framework.jar", + "curator-client.jar", + "metrics-core.jar" + }; + ProviderUtils.addDependencyJars(providerResources, fileSystem, tempPath, + libdir, jars, + classes); + addKeytabResourceIfNecessary(fileSystem, launcher, instanceDescription,
