Repository: incubator-slider Updated Branches: refs/heads/develop a8d31ee27 -> eb790e780
SLIDER-515. Ensure Slider clients uploads all jars needed by AppMaster and construct own class path Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/eb790e78 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/eb790e78 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/eb790e78 Branch: refs/heads/develop Commit: eb790e780d262eb29b5551d21956faac78347145 Parents: a8d31ee Author: Sumit Mohanty <[email protected]> Authored: Fri Oct 24 15:35:18 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Fri Oct 24 15:35:18 2014 -0700 ---------------------------------------------------------------------- slider-assembly/src/main/scripts/slider | 4 +- slider-assembly/src/main/scripts/slider.py | 2 + .../org/apache/slider/common/SliderKeys.java | 6 +++ .../apache/slider/common/tools/SliderUtils.java | 40 +++++++++++++++++++- .../apache/slider/providers/ProviderUtils.java | 28 ++++++++++++++ .../slideram/SliderAMClientProvider.java | 37 ++++-------------- 6 files changed, 86 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb790e78/slider-assembly/src/main/scripts/slider ---------------------------------------------------------------------- diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider index 4228843..80b5f32 100755 --- a/slider-assembly/src/main/scripts/slider +++ b/slider-assembly/src/main/scripts/slider @@ -56,7 +56,9 @@ if [ -f "${confdir}/slider-env.sh" ]; then fi slider_confdir_opts="-Dslider.confdir=${confdir}" -slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m" +libdir_jvm_opts="-Dslider.libdir=${libdir}" + +slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true ${libdir_jvm_opts} -Xmx256m" slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts} # allow for an extra classpath http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb790e78/slider-assembly/src/main/scripts/slider.py ---------------------------------------------------------------------- diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py index a6dd90e..c72fc0d 100644 --- a/slider-assembly/src/main/scripts/slider.py +++ b/slider-assembly/src/main/scripts/slider.py @@ -247,6 +247,8 @@ def main(): executeEnvSh(confdir) jvm_opts_list = (SLIDER_CONFDIR_OPTS % confdir).split() default_jvm_opts = DEFAULT_JVM_OPTS + libdir_jvm_opts = "-Dslider.libdir={0}".format(libdir) + default_jvm_opts = "{0} {1}".format(default_jvm_opts, libdir_jvm_opts) slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts) jvm_opts_list.extend(slider_jvm_opts.split()) slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb790e78/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index ae58ef3..048dfa7 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -122,6 +122,12 @@ public interface SliderKeys extends SliderXmlConfKeys { String PROPERTY_CONF_DIR = "slider.confdir"; /** + * JVM property to define the slider lib directory; + * this is set by the slider script: {@value} + */ + String PROPERTY_LIB_DIR = "slider.libdir"; + + /** * name of generated dir for this conf: {@value} */ String SUBMITTED_CONF_DIR = "confdir"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb790e78/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 33e028f..43dfbd0 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -40,6 +40,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -70,6 +71,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; +import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; @@ -1170,6 +1172,42 @@ public final class SliderUtils { return res; } + /** + * Submit a JAR containing and map it + * @param providerResources provider map to build up + * @param sliderFileSystem remote fs + * @param libDir lib directory + * @param srcPath copy jars from + * @throws IOException, SliderException trouble copying to HDFS + */ + public static void putAllJars(Map<String, LocalResource> providerResources, + SliderFileSystem sliderFileSystem, + Path tempPath, + String libDir, + String srcPath + ) + throws IOException, SliderException { + log.info("Loading all dependencies from {}", srcPath); + if (SliderUtils.isSet(srcPath)) { + File srcFolder = new File(srcPath); + FilenameFilter jarFilter = new FilenameFilter() { + public boolean accept(File dir, String name) { + String lowercaseName = name.toLowerCase(); + if (lowercaseName.endsWith(".jar")) { + return true; + } else { + return false; + } + } + }; + File[] listOfJars = srcFolder.listFiles(jarFilter); + for (File jarFile : listOfJars) { + LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName()); + providerResources.put(libDir + "/" + jarFile.getName(), res); + } + } + } + public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) { Map<String, Map<String, String>> dest = new HashMap<String, Map<String, String>>(); @@ -1301,7 +1339,7 @@ public final class SliderUtils { } classpath.addLibDir(libdir); classpath.addRemoteClasspathEnvVar(); - classpath.appendAll(classpath.yarnApplicationClasspath(config)); + classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$()); } return classpath; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb790e78/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java index 8e77a9c..e26105e 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java @@ -175,6 +175,34 @@ public class ProviderUtils implements RoleKeys { } /** + * Loads all dependency jars from the default path + * @param providerResources map of provider resources to add these entries to + * @param sliderFileSystem target filesystem + * @param tempPath path in the cluster FS for temp files + * @param libDir relative directory to place resources + * @param libLocalSrcDir explicitly supplied local libs dir + * @throws IOException + * @throws SliderException + */ + public static void addAllDependencyJars(Map<String, LocalResource> providerResources, + SliderFileSystem sliderFileSystem, + Path tempPath, + String libDir, + String libLocalSrcDir + ) throws + IOException, + SliderException { + String libSrcToUse = libLocalSrcDir; + if (SliderUtils.isSet(libLocalSrcDir)) { + File file = new File(libLocalSrcDir); + if (!file.exists() || !file.isDirectory()) { + throw new BadCommandArgumentsException("Supplied lib src dir %s is not valid", libLocalSrcDir); + } + } + SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath, libDir, libSrcToUse); + } + + /** * build the log directory * @return the log dir */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb790e78/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java index 5edf1bf..60edce7 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java @@ -18,11 +18,6 @@ package org.apache.slider.providers.slideram; -import com.beust.jcommander.JCommander; -import com.codahale.metrics.MetricRegistry; -import com.google.gson.GsonBuilder; -import org.apache.curator.CuratorZookeeperClient; -import org.apache.curator.framework.CuratorFramework; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; @@ -47,7 +42,6 @@ import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderUtils; -import org.mortbay.jetty.security.SslSelectChannelConnector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -187,29 +181,14 @@ public class SliderAMClientProvider extends AbstractClientProvider libdir, miniClusterTestRun); - Class<?>[] classes = { - JCommander.class, - GsonBuilder.class, - SslSelectChannelConnector.class, - - CuratorFramework.class, - CuratorZookeeperClient.class, - MetricRegistry.class - }; - String[] jars = - { - JCOMMANDER_JAR, - GSON_JAR, - "jetty-sslengine.jar", - - "curator-framework.jar", - "curator-client.jar", - "metrics-core.jar" - }; - ProviderUtils.addDependencyJars(providerResources, fileSystem, tempPath, - libdir, jars, - classes); - + String libDirProp = + System.getProperty(SliderKeys.PROPERTY_LIB_DIR); + log.info("Loading all dependencies for AM."); + ProviderUtils.addAllDependencyJars(providerResources, + fileSystem, + tempPath, + libdir, + libDirProp); addKeytabResourceIfNecessary(fileSystem, launcher, instanceDescription,
