Repository: incubator-slider Updated Branches: refs/heads/develop 06ff4c2ea -> 773c78420
SLIDER-515. Ensure Slider clients uploads all jars needed by AppMaster and construct own class path Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/773c7842 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/773c7842 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/773c7842 Branch: refs/heads/develop Commit: 773c78420c8c609f0c676637f0d3e71d30466480 Parents: 06ff4c2 Author: Sumit Mohanty <[email protected]> Authored: Sat Oct 25 10:30:31 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Sat Oct 25 10:30:31 2014 -0700 ---------------------------------------------------------------------- slider-assembly/src/main/scripts/slider | 4 +- slider-assembly/src/main/scripts/slider.py | 10 +++++ .../org/apache/slider/common/SliderKeys.java | 6 +++ .../apache/slider/common/tools/SliderUtils.java | 40 +++++++++++++++++++- .../apache/slider/providers/ProviderUtils.java | 28 ++++++++++++++ .../slideram/SliderAMClientProvider.java | 37 ++++-------------- 6 files changed, 94 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/773c7842/slider-assembly/src/main/scripts/slider ---------------------------------------------------------------------- diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider index 4228843..51ec060 100755 --- a/slider-assembly/src/main/scripts/slider +++ b/slider-assembly/src/main/scripts/slider @@ -56,6 +56,8 @@ if [ -f "${confdir}/slider-env.sh" ]; then fi slider_confdir_opts="-Dslider.confdir=${confdir}" +libdir_jvm_opts="-Dslider.libdir=${libdir}" + slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m" slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts} @@ -88,4 +90,4 @@ fi #echo $javabin #type ${javabin}java 2>&1 -exec ${javabin}java ${slider_jvm_opts} ${slider_confdir_opts} ${launcher} $@ +exec ${javabin}java ${slider_jvm_opts} ${slider_confdir_opts} ${libdir_jvm_opts} ${launcher} $@ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/773c7842/slider-assembly/src/main/scripts/slider.py ---------------------------------------------------------------------- diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py index a6dd90e..969f972 100644 --- a/slider-assembly/src/main/scripts/slider.py +++ b/slider-assembly/src/main/scripts/slider.py @@ -34,6 +34,7 @@ HADOOP_CONF_DIR = "HADOOP_CONF_DIR" SLIDER_CLASSNAME = "org.apache.slider.Slider" SLIDER_CONFDIR_OPTS ="-Dslider.confdir=%s" +SLIDER_LIBDIR_OPTS ="-Dslider.libdir=%s" DEFAULT_JVM_OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m" ON_POSIX = 'posix' in sys.builtin_module_names @@ -245,10 +246,19 @@ def main(): libdir = dirMustExist(libDir(slider_home)) confdir = dirMustExist(confDir(slider_home)) executeEnvSh(confdir) + + #create sys property for conf dirs jvm_opts_list = (SLIDER_CONFDIR_OPTS % confdir).split() + + #extend with libdir + libdir_jvm_opts = (SLIDER_LIBDIR_OPTS % libdir) + jvm_opts_list.extend(libdir_jvm_opts.split()) + + #append user specified additional properties default_jvm_opts = DEFAULT_JVM_OPTS slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts) jvm_opts_list.extend(slider_jvm_opts.split()) + slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "") hadoop_conf_dir = os.environ.get(HADOOP_CONF_DIR, "") p = os.pathsep # path separator http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/773c7842/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index ae58ef3..048dfa7 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -122,6 +122,12 @@ public interface SliderKeys extends SliderXmlConfKeys { String PROPERTY_CONF_DIR = "slider.confdir"; /** + * JVM property to define the slider lib directory; + * this is set by the slider script: {@value} + */ + String PROPERTY_LIB_DIR = "slider.libdir"; + + /** * name of generated dir for this conf: {@value} */ String SUBMITTED_CONF_DIR = "confdir"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/773c7842/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 33e028f..43dfbd0 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -40,6 +40,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -70,6 +71,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; +import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; @@ -1170,6 +1172,42 @@ public final class SliderUtils { return res; } + /** + * Submit a JAR containing and map it + * @param providerResources provider map to build up + * @param sliderFileSystem remote fs + * @param libDir lib directory + * @param srcPath copy jars from + * @throws IOException, SliderException trouble copying to HDFS + */ + public static void putAllJars(Map<String, LocalResource> providerResources, + SliderFileSystem sliderFileSystem, + Path tempPath, + String libDir, + String srcPath + ) + throws IOException, SliderException { + log.info("Loading all dependencies from {}", srcPath); + if (SliderUtils.isSet(srcPath)) { + File srcFolder = new File(srcPath); + FilenameFilter jarFilter = new FilenameFilter() { + public boolean accept(File dir, String name) { + String lowercaseName = name.toLowerCase(); + if (lowercaseName.endsWith(".jar")) { + return true; + } else { + return false; + } + } + }; + File[] listOfJars = srcFolder.listFiles(jarFilter); + for (File jarFile : listOfJars) { + LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName()); + providerResources.put(libDir + "/" + jarFile.getName(), res); + } + } + } + public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) { Map<String, Map<String, String>> dest = new HashMap<String, Map<String, String>>(); @@ -1301,7 +1339,7 @@ public final class SliderUtils { } classpath.addLibDir(libdir); classpath.addRemoteClasspathEnvVar(); - classpath.appendAll(classpath.yarnApplicationClasspath(config)); + classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$()); } return classpath; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/773c7842/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java index 8e77a9c..e26105e 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java @@ -175,6 +175,34 @@ public class ProviderUtils implements RoleKeys { } /** + * Loads all dependency jars from the default path + * @param providerResources map of provider resources to add these entries to + * @param sliderFileSystem target filesystem + * @param tempPath path in the cluster FS for temp files + * @param libDir relative directory to place resources + * @param libLocalSrcDir explicitly supplied local libs dir + * @throws IOException + * @throws SliderException + */ + public static void addAllDependencyJars(Map<String, LocalResource> providerResources, + SliderFileSystem sliderFileSystem, + Path tempPath, + String libDir, + String libLocalSrcDir + ) throws + IOException, + SliderException { + String libSrcToUse = libLocalSrcDir; + if (SliderUtils.isSet(libLocalSrcDir)) { + File file = new File(libLocalSrcDir); + if (!file.exists() || !file.isDirectory()) { + throw new BadCommandArgumentsException("Supplied lib src dir %s is not valid", libLocalSrcDir); + } + } + SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath, libDir, libSrcToUse); + } + + /** * build the log directory * @return the log dir */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/773c7842/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java index 5edf1bf..60edce7 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java @@ -18,11 +18,6 @@ package org.apache.slider.providers.slideram; -import com.beust.jcommander.JCommander; -import com.codahale.metrics.MetricRegistry; -import com.google.gson.GsonBuilder; -import org.apache.curator.CuratorZookeeperClient; -import org.apache.curator.framework.CuratorFramework; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; @@ -47,7 +42,6 @@ import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderUtils; -import org.mortbay.jetty.security.SslSelectChannelConnector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -187,29 +181,14 @@ public class SliderAMClientProvider extends AbstractClientProvider libdir, miniClusterTestRun); - Class<?>[] classes = { - JCommander.class, - GsonBuilder.class, - SslSelectChannelConnector.class, - - CuratorFramework.class, - CuratorZookeeperClient.class, - MetricRegistry.class - }; - String[] jars = - { - JCOMMANDER_JAR, - GSON_JAR, - "jetty-sslengine.jar", - - "curator-framework.jar", - "curator-client.jar", - "metrics-core.jar" - }; - ProviderUtils.addDependencyJars(providerResources, fileSystem, tempPath, - libdir, jars, - classes); - + String libDirProp = + System.getProperty(SliderKeys.PROPERTY_LIB_DIR); + log.info("Loading all dependencies for AM."); + ProviderUtils.addAllDependencyJars(providerResources, + fileSystem, + tempPath, + libdir, + libDirProp); addKeytabResourceIfNecessary(fileSystem, launcher, instanceDescription,
