Repository: incubator-slider Updated Branches: refs/heads/develop 38b0f2349 -> 7c0766274
SLIDER-777 Provide slider dependencies as a self contained versioned tarball Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7c076627 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7c076627 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7c076627 Branch: refs/heads/develop Commit: 7c0766274e3d102ec7a27e84a388ae3b0fac10b4 Parents: 38b0f23 Author: Gour Saha <[email protected]> Authored: Fri Oct 9 08:15:56 2015 -0700 Committer: Gour Saha <[email protected]> Committed: Fri Oct 9 08:15:56 2015 -0700 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 69 ++++++++++ .../apache/slider/client/SliderClientAPI.java | 13 ++ .../org/apache/slider/common/SliderKeys.java | 15 ++ .../common/params/ActionDependencyArgs.java | 65 +++++++++ .../apache/slider/common/params/Arguments.java | 1 + .../apache/slider/common/params/ClientArgs.java | 9 ++ .../slider/common/params/SliderActions.java | 3 + .../slider/common/tools/CoreFileSystem.java | 136 ++++++++++++++++++- .../apache/slider/common/tools/SliderUtils.java | 119 +++++++++++++++- .../slideram/SliderAMClientProvider.java | 8 +- .../slider/common/tools/TestSliderUtils.java | 14 ++ 11 files changed, 446 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c076627/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 0ad3418..16e5c9a 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -83,6 +83,7 @@ import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionClientArgs; import org.apache.slider.common.params.ActionCreateArgs; +import org.apache.slider.common.params.ActionDependencyArgs; import org.apache.slider.common.params.ActionDiagnosticArgs; import org.apache.slider.common.params.ActionEchoArgs; import org.apache.slider.common.params.ActionExistsArgs; @@ -172,6 +173,7 @@ import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.File; +import java.io.FilenameFilter; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -211,6 +213,7 @@ import static org.apache.slider.common.params.SliderActions.ACTION_AM_SUICIDE; import static org.apache.slider.common.params.SliderActions.ACTION_BUILD; import static org.apache.slider.common.params.SliderActions.ACTION_CLIENT; import static org.apache.slider.common.params.SliderActions.ACTION_CREATE; +import static org.apache.slider.common.params.SliderActions.ACTION_DEPENDENCY; import static org.apache.slider.common.params.SliderActions.ACTION_DESTROY; import static org.apache.slider.common.params.SliderActions.ACTION_DIAGNOSTICS; import static org.apache.slider.common.params.SliderActions.ACTION_EXISTS; @@ -385,6 +388,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs()); break; + case ACTION_DEPENDENCY: + exitCode = actionDependency(serviceArgs.getActionDependencyArgs()); + break; + case ACTION_DESTROY: exitCode = actionDestroy(clusterName); break; @@ -2150,6 +2157,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe ClasspathConstructor classpath = SliderUtils.buildClasspath(relativeConfDir, libdir, getConfig(), + sliderFileSystem, usingMiniMRCluster); amLauncher.setClasspath(classpath); //add english env @@ -4365,9 +4373,70 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return EXIT_SUCCESS; } + @Override + public int actionDependency(ActionDependencyArgs args) throws IOException, + YarnException { + // check to ensure if the current user is hdfs + String currentUser = getUsername(); + String hdfsUser = "hdfs"; + if (!hdfsUser.equalsIgnoreCase(currentUser)) { + log.error("Please run this command as user {}", hdfsUser); + return EXIT_FALSE; + } + + String version = SliderUtils.getSliderVersion(); + Path dependencyLibTarGzip = sliderFileSystem.getDependencyTarGzip(); + + // Check if dependency has already been uploaded, in which case log + // appropriately and exit success (unless overwrite has been requested) + if (sliderFileSystem.isFile(dependencyLibTarGzip) && !args.overwrite) { + println(String.format( + "Dependency libs are already uploaded to %s. Use %s " + + "if you want to re-upload", dependencyLibTarGzip.toUri(), + Arguments.ARG_OVERWRITE)); + return EXIT_SUCCESS; + } + + String libDir = System.getProperty(SliderKeys.PROPERTY_LIB_DIR); + if (SliderUtils.isSet(libDir)) { + File srcFolder = new File(libDir); + File tempLibTarGzipFile = File.createTempFile( + SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + "_", + SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT); + // copy all jars except slider-core-<version>.jar + FilenameFilter jarFilter = new FilenameFilter() { + public boolean accept(File dir, String name) { + String lowercaseName = name.toLowerCase(); + if (lowercaseName.endsWith(".jar") + && !lowercaseName.startsWith("slider-core")) { + return true; + } else { + return false; + } + } + }; + SliderUtils.tarGzipFolder(srcFolder, tempLibTarGzipFile, jarFilter); + + log.info("Uploading dependency for AM (version {}) from {} to {}", + version, tempLibTarGzipFile.toURI(), dependencyLibTarGzip.toUri()); + sliderFileSystem.copyLocalFileToHdfs(tempLibTarGzipFile, + dependencyLibTarGzip, new FsPermission( + SliderKeys.SLIDER_DEPENDENCY_DIR_PERMISSIONS)); + return EXIT_SUCCESS; + } else { + return EXIT_FALSE; + } + } + private int actionHelp(String actionName) throws YarnException, IOException { throw new UsageException(CommonArgs.usage(serviceArgs, actionName)); } + + private int actionHelp(String errMsg, String actionName) + throws YarnException, IOException { + throw new UsageException("%s %s", errMsg, CommonArgs.usage(serviceArgs, + actionName)); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c076627/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index 8a6e49a..bab451f 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -27,6 +27,7 @@ import org.apache.slider.api.types.SliderInstanceDescription; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionClientArgs; +import org.apache.slider.common.params.ActionDependencyArgs; import org.apache.slider.common.params.ActionDiagnosticArgs; import org.apache.slider.common.params.ActionEchoArgs; import org.apache.slider.common.params.ActionFlexArgs; @@ -325,4 +326,16 @@ public interface SliderClientAPI extends Service { */ RegistryOperations getRegistryOperations() throws SliderException, IOException; + + /** + * Upload all Slider AM and agent dependency libraries to HDFS, so that they + * do not need to be uploaded with every create call. This operation is + * Slider version specific. So it needs to be invoked for every single + * version of slider/slider-client. + * + * @throws SliderException + * @throws IOException + */ + int actionDependency(ActionDependencyArgs dependencyArgs) throws IOException, + YarnException; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c076627/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index a5aad37..22798e3 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -42,6 +42,21 @@ public interface SliderKeys extends SliderXmlConfKeys { String SLIDER_BASE_DIRECTORY = ".slider"; /** + * The paths under which Slider AM dependency libraries are stored + */ + String SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK = "slider_dep"; + String SLIDER_DEPENDENCY_HDP_PARENT_DIR = "/hdp"; + String SLIDER_DEPENDENCY_DIR = "/apps/%s/slider"; + String SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME = "slider"; + String SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz"; + String SLIDER_DEPENDENCY_DIR_PERMISSIONS = "755"; + + /** + * + */ + String HDP_VERSION_PROP_NAME = "HDP_VERSION"; + + /** * name of the relative path to expaned an image into: {@value}. * The title of this path is to help people understand it when * they see it in their error messages http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c076627/slider-core/src/main/java/org/apache/slider/common/params/ActionDependencyArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionDependencyArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionDependencyArgs.java new file mode 100644 index 0000000..87f9f0d --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionDependencyArgs.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.common.params; + +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.UsageException; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +@Parameters(commandNames = { SliderActions.ACTION_DEPENDENCY }, + commandDescription = SliderActions.DESCRIBE_ACTION_DEPENDENCY) +public class ActionDependencyArgs extends AbstractActionArgs { + + @Override + public String getActionName() { + return SliderActions.ACTION_DEPENDENCY; + } + + @Parameter(names = { ARG_UPLOAD }, + description = "Upload AM and agent libraries to HDFS for this client") + public boolean upload; + + @Parameter(names = { ARG_OVERWRITE }, + description = "Overwrite current uploaded dependency libs") + public boolean overwrite = false; + + /** + * Get the min #of params expected + * + * @return the min number of params in the {@link #parameters} field + */ + public int getMinParams() { + return 0; + } + + @Override + public int getMaxParams() { + return 1; + } + + @Override + public void validate() throws BadCommandArgumentsException, UsageException { + super.validate(); + + if (!upload) { + throw new UsageException("Option " + ARG_UPLOAD + " is mandatory"); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c076627/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index 7dda57b..c2fa09c 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -110,6 +110,7 @@ public interface Arguments { String ARG_TEMPLATE = "--template"; String ARG_TRUSTSTORE = "--truststore"; String ARG_USER = "--user"; + String ARG_UPLOAD = "--upload"; String ARG_VERBOSE = "--verbose"; String ARG_VERSION = "--version"; String ARG_WAIT = "--wait"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c076627/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java index cbe80e7..3c430f3 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java @@ -50,6 +50,7 @@ public class ClientArgs extends CommonArgs { private final ActionAMSuicideArgs actionAMSuicideArgs = new ActionAMSuicideArgs(); private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs(); private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs(); + private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs(); private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs(); private final ActionDiagnosticArgs actionDiagnosticArgs = new ActionDiagnosticArgs(); private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs(); @@ -88,6 +89,7 @@ public class ClientArgs extends CommonArgs { actionPackageArgs, actionKeytabArgs, actionBuildArgs, + actionDependencyArgs, actionCreateArgs, actionListArgs, actionStatusArgs, @@ -166,6 +168,10 @@ public class ClientArgs extends CommonArgs { return actionCreateArgs; } + public ActionDependencyArgs getActionDependencyArgs() { + return actionDependencyArgs; + } + public ActionDestroyArgs getActionDestroyArgs() { return actionDestroyArgs; } @@ -238,6 +244,9 @@ public class ClientArgs extends CommonArgs { } else if (SliderActions.ACTION_AM_SUICIDE.equals(action)) { bindCoreAction(actionAMSuicideArgs); + } else if (SliderActions.ACTION_DEPENDENCY.equals(action)) { + bindCoreAction(actionDependencyArgs); + } else if (SliderActions.ACTION_DESTROY.equals(action)) { bindCoreAction(actionDestroyArgs); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c076627/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java index cdde8f1..4d0e04d 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java @@ -27,6 +27,7 @@ public interface SliderActions { String ACTION_AM_SUICIDE = "am-suicide"; String ACTION_BUILD = "build"; String ACTION_CREATE = "create"; + String ACTION_DEPENDENCY = "dependency"; String ACTION_UPDATE = "update"; String ACTION_UPGRADE = "upgrade"; String ACTION_DESTROY = "destroy"; @@ -57,6 +58,8 @@ public interface SliderActions { "Build a Slider cluster specification, but do not start it"; String DESCRIBE_ACTION_CREATE = "Create a live Slider application"; + String DESCRIBE_ACTION_DEPENDENCY = + "Slider AM and agent dependency (libraries) management"; String DESCRIBE_ACTION_UPDATE = "Update template for a Slider application"; String DESCRIBE_ACTION_UPGRADE = http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c076627/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index c290e0f..d34dd2a 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -47,6 +47,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FilenameFilter; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.charset.Charset; @@ -390,6 +391,27 @@ public class CoreFileSystem { } /** + * Given a path, check if it exists and is a file + * + * @param path + * absolute path to the file to check + * @returns true if and only if path exists and is a file, false for all other + * reasons including if file check throws IOException + */ + public boolean isFile(Path path) { + boolean isFile = false; + try { + FileStatus status = fileSystem.getFileStatus(path); + if (status.isFile()) { + isFile = true; + } + } catch (IOException e) { + // ignore, isFile is already set to false + } + return isFile; + } + + /** * Verify that a file exists in the zip file given by path * @param path path to zip file * @param file file expected to be in zip @@ -462,6 +484,34 @@ public class CoreFileSystem { new Path(getHomeDirectory(), SliderKeys.SLIDER_BASE_DIRECTORY); } + /** + * Get slider dependency parent dir in HDFS + * + * @return the parent dir path of slider.tar.gz in HDFS + */ + public Path getDependencyPath() { + String parentDir = (SliderUtils.isHdp()) ? SliderKeys.SLIDER_DEPENDENCY_HDP_PARENT_DIR + + SliderKeys.SLIDER_DEPENDENCY_DIR + : SliderKeys.SLIDER_DEPENDENCY_DIR; + Path dependencyPath = new Path(String.format(parentDir, + SliderUtils.getSliderVersion())); + return dependencyPath; + } + + /** + * Get slider.tar.gz absolute filepath in HDFS + * + * @return the absolute path to slider.tar.gz in HDFS + */ + public Path getDependencyTarGzip() { + Path dependencyLibAmPath = getDependencyPath(); + Path dependencyLibTarGzip = new Path( + dependencyLibAmPath.toUri().toString(), + SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + + SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT); + return dependencyLibTarGzip; + } + public Path getHomeDirectory() { return fileSystem.getHomeDirectory(); } @@ -504,7 +554,8 @@ public class CoreFileSystem { // Setting to most private option amResource.setVisibility(LocalResourceVisibility.APPLICATION); // Set the resource to be copied over - amResource.setResource(ConverterUtils.getYarnUrlFromPath(destPath)); + amResource.setResource(ConverterUtils.getYarnUrlFromPath(fileSystem + .resolvePath(destStatus.getPath()))); // Set timestamp and length of file so that the framework // can do basic sanity checks for the local resource // after it has been copied over to ensure it is the same @@ -579,6 +630,89 @@ public class CoreFileSystem { } /** + * Submit the AM tar.gz resource referenced by the instance's cluster + * filesystem. Also, update the providerResources object with the new + * resource. + * + * @param providerResources + * the provider resource map to be updated + * @throws IOException + * trouble copying to HDFS + */ + public void submitTarGzipAndUpdate( + Map<String, LocalResource> providerResources) throws IOException, + BadClusterStateException { + Path dependencyLibTarGzip = getDependencyTarGzip(); + LocalResource lc = createAmResource(dependencyLibTarGzip, + LocalResourceType.ARCHIVE); + providerResources.put(SliderKeys.SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK, lc); + } + + /** + * Copy local file(s) to destination HDFS directory. If {@code localPath} is a + * local directory then all files matching the {@code filenameFilter} + * (optional) are copied, otherwise {@code filenameFilter} is ignored. + * + * @param localPath + * a local file or directory path + * @param filenameFilter + * if {@code localPath} is a directory then filenameFilter is used as + * a filter (if specified) + * @param destDir + * the destination HDFS directory where the file(s) should be copied + * @param fp + * file permissions of all the directories and files that will be + * created in this api + * @throws IOException + */ + public void copyLocalFilesToHdfs(File localPath, + FilenameFilter filenameFilter, Path destDir, FsPermission fp) + throws IOException { + if (localPath == null || destDir == null) { + throw new IOException("Either localPath or destDir is null"); + } + fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, + "000"); + fileSystem.mkdirs(destDir, fp); + if (localPath.isDirectory()) { + // copy all local files under localPath to destDir (honoring filename + // filter if provided + File[] localFiles = localPath.listFiles(filenameFilter); + Path[] localFilePaths = new Path[localFiles.length]; + int i = 0; + for (File localFile : localFiles) { + localFilePaths[i++] = new Path(localFile.getPath()); + } + log.info("Copying {} files from {} to {}", i, localPath.toURI(), + destDir.toUri()); + fileSystem.copyFromLocalFile(false, true, localFilePaths, destDir); + } else { + log.info("Copying file {} to {}", localPath.toURI(), destDir.toUri()); + fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()), + destDir); + } + // set permissions for all the files created in the destDir + fileSystem.setPermission(destDir, fp); + } + + public void copyLocalFileToHdfs(File localPath, + Path destPath, FsPermission fp) + throws IOException { + if (localPath == null || destPath == null) { + throw new IOException("Either localPath or destPath is null"); + } + fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, + "000"); + fileSystem.mkdirs(destPath.getParent(), fp); + log.info("Copying file {} to {}", localPath.toURI(), destPath.toUri()); + + fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()), + destPath); + // set file permissions of the destPath + fileSystem.setPermission(destPath, fp); + } + + /** * list entries in a filesystem directory * * @param path directory http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c076627/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 4ff8d11..cc19052 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -20,6 +20,8 @@ package org.apache.slider.common.tools; import com.google.common.base.Preconditions; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.io.output.ByteArrayOutputStream; @@ -55,6 +57,8 @@ import org.apache.slider.api.RoleKeys; import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.slider.common.params.Arguments; +import org.apache.slider.common.params.SliderActions; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.BadClusterStateException; @@ -71,6 +75,7 @@ import org.apache.zookeeper.server.util.KerberosUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; @@ -112,6 +117,7 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -1365,6 +1371,22 @@ public final class SliderUtils { } } + /** + * Submit the AM tar.gz containing all dependencies and map it + * @param providerResources provider map to build up + * @param sliderFileSystem remote fs + * @throws IOException, SliderException trouble copying to HDFS + */ + public static void putAmTarGzipAndUpdate( + Map<String, LocalResource> providerResources, + SliderFileSystem sliderFileSystem + ) throws IOException, SliderException { + log.info("Loading all dependencies from {}{}", + SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME, + SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT); + sliderFileSystem.submitTarGzipAndUpdate(providerResources); + } + public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) { Map<String, Map<String, String>> dest = new HashMap<String, Map<String, String>>(); @@ -1480,6 +1502,7 @@ public final class SliderUtils { public static ClasspathConstructor buildClasspath(String sliderConfDir, String libdir, Configuration config, + SliderFileSystem sliderFileSystem, boolean usingMiniMRCluster) { ClasspathConstructor classpath = new ClasspathConstructor(); @@ -1494,6 +1517,13 @@ public final class SliderUtils { classpath.addClassDirectory(sliderConfDir); } classpath.addLibDir(libdir); + if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) { + classpath.addLibDir(SliderKeys.SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK); + } else { + log.info( + "For faster submission of apps, upload dependencies using cmd {} {}", + SliderActions.ACTION_DEPENDENCY, Arguments.ARG_UPLOAD); + } classpath.addRemoteClasspathEnvVar(); classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$()); } @@ -1819,21 +1849,102 @@ public final class SliderUtils { } } + /** + * Given a source folder create a tar.gz file + * + * @param srcFolder + * @param tarGzipFile + * + * @throws IOException + */ + public static void tarGzipFolder(File srcFolder, File tarGzipFile, + FilenameFilter filter) throws IOException { + log.info("Tar-gzipping folder {} to {}", srcFolder.getAbsolutePath(), + tarGzipFile.getAbsolutePath()); + List<String> files = new ArrayList<>(); + generateFileList(files, srcFolder, srcFolder, true, filter); + + TarArchiveOutputStream taos = null; + try { + taos = new TarArchiveOutputStream(new GZIPOutputStream( + new BufferedOutputStream(new FileOutputStream(tarGzipFile)))); + for (String file : files) { + File srcFile = new File(srcFolder, file); + TarArchiveEntry tarEntry = new TarArchiveEntry( + srcFile, file); + taos.putArchiveEntry(tarEntry); + FileInputStream in = new FileInputStream(srcFile); + try { + org.apache.commons.io.IOUtils.copy(in, taos); + } finally { + if (in != null) { + in.close(); + } + } + taos.flush(); + taos.closeArchiveEntry(); + } + } finally { + if (taos != null) { + taos.close(); + } + } + } + + /** + * Retrieve the HDP version if it is an HDP cluster, or null otherwise + * + * @return HDP version + */ + public static String getHdpVersion() { + return System.getenv(SliderKeys.HDP_VERSION_PROP_NAME); + } + + /** + * Query to find if it is an HDP cluster + * + * @return true if this is invoked in an HDP cluster or false otherwise + */ + public static boolean isHdp() { + return StringUtils.isNotEmpty(getHdpVersion()) ? true : false; + } + + /** + * Retrieve the version of the current Slider install + * + * @return the version string of the Slider release + */ + public static String getSliderVersion() { + if (isHdp()) { + return getHdpVersion(); + } else { + Properties props = SliderVersionInfo.loadVersionProperties(); + return props.getProperty(SliderVersionInfo.APP_VERSION); + } + } + + private static void generateFileList(List<String> fileList, File node, + File rootFolder, Boolean relative) { + generateFileList(fileList, node, rootFolder, relative, null); + } - private static void generateFileList(List<String> fileList, File node, File rootFolder, Boolean relative) { + private static void generateFileList(List<String> fileList, File node, + File rootFolder, Boolean relative, FilenameFilter filter) { if (node.isFile()) { String fileFullPath = node.toString(); if (relative) { - fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1, fileFullPath.length())); + fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1, + fileFullPath.length())); } else { fileList.add(fileFullPath); } } if (node.isDirectory()) { - String[] subNode = node.list(); + String[] subNode = node.list(filter); for (String filename : subNode) { - generateFileList(fileList, new File(node, filename), rootFolder, relative); + generateFileList(fileList, new File(node, filename), rootFolder, + relative, filter); } } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c076627/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java index abb9648..e1dc4f9 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java @@ -187,12 +187,18 @@ public class SliderAMClientProvider extends AbstractClientProvider String libDirProp = System.getProperty(SliderKeys.PROPERTY_LIB_DIR); - log.info("Loading all dependencies for AM."); + log.info("Loading all dependencies for AM."); + // If slider.tar.gz is available in hdfs use it, else upload all jars + Path dependencyLibTarGzip = fileSystem.getDependencyTarGzip(); + if (fileSystem.isFile(dependencyLibTarGzip)) { + SliderUtils.putAmTarGzipAndUpdate(providerResources, fileSystem); + } else { ProviderUtils.addAllDependencyJars(providerResources, fileSystem, tempPath, libdir, libDirProp); + } addKeytabResourceIfNecessary(fileSystem, instanceDescription, providerResources); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c076627/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java b/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java index c1e0940..46fe638 100644 --- a/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java +++ b/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java @@ -133,4 +133,18 @@ public class TestSliderUtils { ar.setYarnApplicationState(state); return ar; } + + + @Test + public void testGetHdpVersion() { + String hdpVersion = "2.3.2.0-2766"; + Assert.assertEquals("Version should be empty", null, + SliderUtils.getHdpVersion()); + } + + @Test + public void testIsHdp() { + Assert.assertFalse("Should be false", SliderUtils.isHdp()); + } + }
