SLIDER-961 clean up SliderClient code
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/227339e5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/227339e5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/227339e5 Branch: refs/heads/develop Commit: 227339e5c661571a989aa3f5e2d7ff5ff9ac667d Parents: 88f881b Author: Steve Loughran <[email protected]> Authored: Wed Nov 4 21:36:47 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Wed Nov 4 21:36:47 2015 +0000 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 717 +++++++------------ .../slider/common/tools/CoreFileSystem.java | 2 +- .../apache/slider/common/tools/SliderUtils.java | 25 +- .../slider/core/launch/AppMasterLauncher.java | 5 +- .../core/launch/ClasspathConstructor.java | 1 - .../slider/core/launch/CommandLineBuilder.java | 55 ++ .../utility/AbstractSliderLaunchedService.java | 20 +- .../agent/actions/TestActionPackage.groovy | 7 +- 8 files changed, 333 insertions(+), 499 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index de0e07e..07c915c 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -19,7 +19,6 @@ package org.apache.slider.client; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; import com.google.common.io.Files; import org.apache.commons.collections.CollectionUtils; @@ -65,9 +64,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; -import org.apache.slider.api.InternalKeys; -import org.apache.slider.api.OptionKeys; -import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.StateValues; import org.apache.slider.api.proto.Messages; @@ -77,7 +73,6 @@ import org.apache.slider.client.ipc.SliderClusterOperations; import org.apache.slider.common.Constants; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.params.AbstractActionArgs; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; @@ -106,11 +101,9 @@ import org.apache.slider.common.params.Arguments; import org.apache.slider.common.params.ClientArgs; import org.apache.slider.common.params.CommonArgs; import org.apache.slider.common.params.LaunchArgsAccessor; -import org.apache.slider.common.params.SliderActions; import org.apache.slider.common.tools.ConfigHelper; import org.apache.slider.common.tools.Duration; import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.common.tools.SliderVersionInfo; import org.apache.slider.core.build.InstanceBuilder; import org.apache.slider.core.build.InstanceIO; @@ -132,7 +125,6 @@ import org.apache.slider.core.exceptions.UsageException; import org.apache.slider.core.exceptions.WaitTimeoutException; import org.apache.slider.core.launch.AppMasterLauncher; import org.apache.slider.core.launch.ClasspathConstructor; -import org.apache.slider.core.launch.CommandLineBuilder; import org.apache.slider.core.launch.JavaCommandLineBuilder; import org.apache.slider.core.launch.LaunchedApplication; import org.apache.slider.core.launch.RunningApplication; @@ -203,38 +195,13 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.apache.hadoop.registry.client.binding.RegistryUtils.ServiceRecordMarshal; -import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser; -import static org.apache.hadoop.registry.client.binding.RegistryUtils.extractServiceRecords; -import static org.apache.hadoop.registry.client.binding.RegistryUtils.listServiceRecords; -import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath; -import static org.apache.hadoop.registry.client.binding.RegistryUtils.serviceclassPath; -import static org.apache.hadoop.registry.client.binding.RegistryUtils.statChildren; -import static org.apache.slider.common.params.SliderActions.ACTION_AM_SUICIDE; -import static org.apache.slider.common.params.SliderActions.ACTION_BUILD; -import static org.apache.slider.common.params.SliderActions.ACTION_CLIENT; -import static org.apache.slider.common.params.SliderActions.ACTION_CREATE; -import static org.apache.slider.common.params.SliderActions.ACTION_DEPENDENCY; -import static org.apache.slider.common.params.SliderActions.ACTION_DESTROY; -import static org.apache.slider.common.params.SliderActions.ACTION_DIAGNOSTICS; -import static org.apache.slider.common.params.SliderActions.ACTION_EXISTS; -import static org.apache.slider.common.params.SliderActions.ACTION_FLEX; -import static org.apache.slider.common.params.SliderActions.ACTION_FREEZE; -import static org.apache.slider.common.params.SliderActions.ACTION_HELP; -import static org.apache.slider.common.params.SliderActions.ACTION_INSTALL_KEYTAB; -import static org.apache.slider.common.params.SliderActions.ACTION_INSTALL_PACKAGE; -import static org.apache.slider.common.params.SliderActions.ACTION_KEYTAB; -import static org.apache.slider.common.params.SliderActions.ACTION_KILL_CONTAINER; -import static org.apache.slider.common.params.SliderActions.ACTION_LIST; -import static org.apache.slider.common.params.SliderActions.ACTION_LOOKUP; -import static org.apache.slider.common.params.SliderActions.ACTION_PACKAGE; -import static org.apache.slider.common.params.SliderActions.ACTION_REGISTRY; -import static org.apache.slider.common.params.SliderActions.ACTION_RESOLVE; -import static org.apache.slider.common.params.SliderActions.ACTION_STATUS; -import static org.apache.slider.common.params.SliderActions.ACTION_THAW; -import static org.apache.slider.common.params.SliderActions.ACTION_UPDATE; -import static org.apache.slider.common.params.SliderActions.ACTION_UPGRADE; -import static org.apache.slider.common.params.SliderActions.ACTION_VERSION; +import static org.apache.hadoop.registry.client.binding.RegistryUtils.*; +import static org.apache.slider.api.InternalKeys.*; +import static org.apache.slider.api.OptionKeys.*; +import static org.apache.slider.api.ResourceKeys.*; +import static org.apache.slider.common.params.SliderActions.*; +import static org.apache.slider.common.tools.SliderUtils.*; + /** * Client service for Slider @@ -287,6 +254,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe /** * The YARN registry service */ + @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized") private RegistryOperations registryOperations; /** @@ -317,19 +285,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe ConfigHelper.injectSliderXMLResource(); // yarn-ify YarnConfiguration yarnConfiguration = new YarnConfiguration(config); - return SliderUtils.patchConfiguration(yarnConfiguration); + return patchConfiguration(yarnConfiguration); } @Override protected void serviceInit(Configuration conf) throws Exception { - Configuration clientConf = SliderUtils.loadSliderClientXML(); + Configuration clientConf = loadSliderClientXML(); ConfigHelper.mergeConfigurations(conf, clientConf, SLIDER_CLIENT_XML, true); serviceArgs.applyDefinitions(conf); serviceArgs.applyFileSystemBinding(conf); // init security with our conf - if (SliderUtils.isHadoopClusterSecure(conf)) { - SliderUtils.forceLogin(); - SliderUtils.initProcessSecurity(conf); + if (isHadoopClusterSecure(conf)) { + forceLogin(); + initProcessSecurity(conf); } AbstractActionArgs coreAction = serviceArgs.getCoreAction(); if (coreAction.getHadoopServicesRequired()) { @@ -362,9 +330,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // choose the action String action = serviceArgs.getAction(); - if (SliderUtils.isUnset(action)) { - throw new SliderException(EXIT_USAGE, - serviceArgs.usage()); + if (isUnset(action)) { + throw new SliderException(EXIT_USAGE, serviceArgs.usage()); } int exitCode = EXIT_SUCCESS; @@ -483,7 +450,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return exitCode; } -/** + /** * Perform everything needed to init the hadoop binding. * This assumes that the service is already in inited or started state * @throws IOException @@ -491,7 +458,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe */ protected void initHadoopBinding() throws IOException, SliderException { // validate the client - SliderUtils.validateSliderClientEnvironment(null); + validateSliderClientEnvironment(null); //create the YARN client yarnClient = new SliderYarnClientImpl(); yarnClient.init(getConfig()); @@ -626,7 +593,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe public int actionDestroy(String clustername, ActionDestroyArgs destroyArgs) throws YarnException, IOException { // verify that a live cluster isn't there - SliderUtils.validateClusterName(clustername); + validateClusterName(clustername); //no=op, it is now mandatory. verifyBindingsDefined(); verifyNoLiveClusters(clustername, "Destroy"); @@ -645,9 +612,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe throw new UsageException("Destroy will permanently delete directories and registries. " + "Reissue this command with the --force option if you want to proceed."); } - boolean deleted = - fs.delete(clusterDirectory, true); - if (!deleted) { + if (!fs.delete(clusterDirectory, true)) { log.warn("Filesystem returned false from delete() operation"); } @@ -743,33 +708,26 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // Otherwise the internal app config and resources states of the app will be // unwantedly modified and the change will take effect to the running app // immediately. - if (template != null && resources == null) { - throw new BadCommandArgumentsException( + require(!(template != null && resources == null), "Option %s must be specified with option %s", Arguments.ARG_RESOURCES, Arguments.ARG_TEMPLATE); - } - if (resources != null && template == null) { - throw new BadCommandArgumentsException( + + require(!(resources != null && template == null), "Option %s must be specified with option %s", Arguments.ARG_TEMPLATE, Arguments.ARG_RESOURCES); - } // For upgrade spec, both --template and --resources should be specified // and neither of --containers or --components should be used if (template != null && resources != null) { - if (CollectionUtils.isNotEmpty(containers)) { - throw new BadCommandArgumentsException( + require(CollectionUtils.isEmpty(containers), "Option %s cannot be specified with %s or %s", Arguments.ARG_CONTAINERS, Arguments.ARG_TEMPLATE, Arguments.ARG_RESOURCES); - } - if (CollectionUtils.isNotEmpty(components)) { - throw new BadCommandArgumentsException( - "Option %s cannot be specified with %s or %s", - Arguments.ARG_COMPONENTS, Arguments.ARG_TEMPLATE, - Arguments.ARG_RESOURCES); - } - + require(CollectionUtils.isEmpty(components), + "Option %s cannot be specified with %s or %s", + Arguments.ARG_COMPONENTS, Arguments.ARG_TEMPLATE, + Arguments.ARG_RESOURCES); + // not an error to try to upgrade a stopped cluster, just return success // code, appropriate log messages have already been dumped if (!isAppInRunningState(clustername)) { @@ -794,7 +752,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private int actionUpgradeContainers(String clustername, ActionUpgradeArgs upgradeArgs) throws YarnException, IOException { verifyBindingsDefined(); - SliderUtils.validateClusterName(clustername); + validateClusterName(clustername); int waittime = upgradeArgs.getWaittime(); // ignored for now String text = "Upgrade containers"; log.debug("actionUpgradeContainers({}, reason={}, wait={})", clustername, @@ -892,13 +850,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return false; } log.debug("App to upgrade was found: {}:\n{}", clustername, - new SliderUtils.OnDemandReportStringifier(app)); - if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED - .ordinal()) { - log.info( - "Cluster {} is in a terminated state {}. Use command '{}' instead.", - clustername, app.getYarnApplicationState(), - SliderActions.ACTION_UPDATE); + new OnDemandReportStringifier(app)); + if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED.ordinal()) { + log.info("Cluster {} is in a terminated state {}. Use command '{}' instead.", + clustername, app.getYarnApplicationState(), ACTION_UPDATE); return false; } @@ -915,7 +870,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private static void checkForCredentials(Configuration conf, ConfTree tree) throws IOException { - if (tree.credentials == null || tree.credentials.size()==0) { + if (tree.credentials == null || tree.credentials.isEmpty()) { log.info("No credentials requested"); return; } @@ -930,10 +885,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } Configuration c = new Configuration(conf); c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider); - CredentialProvider credentialProvider = - CredentialProviderFactory.getProviders(c).get(0); - Set<String> existingAliases = - new HashSet<>(credentialProvider.getAliases()); + CredentialProvider credentialProvider = CredentialProviderFactory.getProviders(c).get(0); + Set<String> existingAliases = new HashSet<>(credentialProvider.getAliases()); for (String alias : aliases) { if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) { log.info("Credentials for " + alias + " found in " + provider); @@ -949,21 +902,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } } } finally { - if (br != null) { - br.close(); - } + org.apache.hadoop.io.IOUtils.closeStream(br); } } private static char[] readOnePassword(String alias) throws IOException { - BufferedReader br = null; - try { - br = new BufferedReader(new InputStreamReader(System.in)); + try(BufferedReader br = new BufferedReader(new InputStreamReader(System.in))) { return readPassword(alias, br); - } finally { - if (br != null) { - br.close(); - } } } @@ -1049,12 +994,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe Path fileInFs = new Path(pkgPath, keytabInfo.keytab ); log.info("Deleting keytab {}", fileInFs); - if (!sliderFileSystem.getFileSystem().exists(fileInFs)) { - throw new BadCommandArgumentsException("No keytab to delete found at " + - fileInFs.toUri().toString()); - } - - sliderFileSystem.getFileSystem().delete(fileInFs, false); + FileSystem sfs = sliderFileSystem.getFileSystem(); + require(sfs.exists(fileInFs), "No keytab to delete found at %s", fileInFs.toUri()); + sfs.delete(fileInFs, false); return EXIT_SUCCESS; } @@ -1062,40 +1004,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private int actionInstallKeytab(ActionKeytabArgs keytabInfo) throws BadCommandArgumentsException, IOException { Path srcFile = null; - if (StringUtils.isEmpty(keytabInfo.folder)) { - throw new BadCommandArgumentsException( - "A valid destination keytab sub-folder name is required (e.g. 'security').\n" - + CommonArgs.usage(serviceArgs, ACTION_KEYTAB)); - } + require(isSet(keytabInfo.folder), + "A valid destination keytab sub-folder name is required (e.g. 'security').\n" + + CommonArgs.usage(serviceArgs, ACTION_KEYTAB)); - if (StringUtils.isEmpty(keytabInfo.keytab)) { - throw new BadCommandArgumentsException("A valid local keytab location is required."); - } else { - File keytabFile = new File(keytabInfo.keytab); - if (!keytabFile.exists() || keytabFile.isDirectory()) { - throw new BadCommandArgumentsException("Unable to access supplied keytab file at " + - keytabFile.getAbsolutePath()); - } else { - srcFile = new Path(keytabFile.toURI()); - } - } + requireArgumentSet(Arguments.ARG_KEYTAB, keytabInfo.keytab); + File keytabFile = new File(keytabInfo.keytab); + require(keytabFile.isFile(), + "Unable to access supplied keytab file at %s", keytabFile.getAbsolutePath()); + srcFile = new Path(keytabFile.toURI()); Path pkgPath = sliderFileSystem.buildKeytabInstallationDirPath(keytabInfo.folder); - sliderFileSystem.getFileSystem().mkdirs(pkgPath); - sliderFileSystem.getFileSystem().setPermission(pkgPath, new FsPermission( + FileSystem sfs = sliderFileSystem.getFileSystem(); + sfs.mkdirs(pkgPath); + sfs.setPermission(pkgPath, new FsPermission( FsAction.ALL, FsAction.NONE, FsAction.NONE)); Path fileInFs = new Path(pkgPath, srcFile.getName()); - log.info("Installing keytab {} at {} and overwrite is {}.", srcFile, fileInFs, keytabInfo.overwrite); - if (sliderFileSystem.getFileSystem().exists(fileInFs) && !keytabInfo.overwrite) { - throw new BadCommandArgumentsException("Keytab exists at " + - fileInFs.toUri().toString() + - ". Use --overwrite to overwrite."); - } + log.info("Installing keytab {} at {} and overwrite is {}.", + srcFile, fileInFs, keytabInfo.overwrite); + require(!(sfs.exists(fileInFs) && !keytabInfo.overwrite), + "Keytab exists at %s. Use --overwrite to overwrite.", fileInFs.toUri()); - sliderFileSystem.getFileSystem().copyFromLocalFile(false, keytabInfo.overwrite, srcFile, fileInFs); - sliderFileSystem.getFileSystem().setPermission(fileInFs, new FsPermission( - FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE)); + sfs.copyFromLocalFile(false, keytabInfo.overwrite, srcFile, fileInFs); + sfs.setPermission(fileInFs, + new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE)); return EXIT_SUCCESS; } @@ -1111,44 +1044,29 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws YarnException, IOException { - log.warn("The " + SliderActions.ACTION_INSTALL_PACKAGE + log.warn("The " + ACTION_INSTALL_PACKAGE + " option has been deprecated. Please use '" - + SliderActions.ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'."); - Path srcFile = null; + + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'."); if (StringUtils.isEmpty(installPkgInfo.name)) { throw new BadCommandArgumentsException( - E_INVALID_APPLICATION_TYPE_NAME +"\n" + E_INVALID_APPLICATION_TYPE_NAME + "\n" + CommonArgs.usage(serviceArgs, ACTION_INSTALL_PACKAGE)); } - - if (StringUtils.isEmpty(installPkgInfo.packageURI)) { - throw new BadCommandArgumentsException(E_INVALID_APPLICATION_PACKAGE_LOCATION); - } else { - File pkgFile = new File(installPkgInfo.packageURI); - if (!pkgFile.exists() || pkgFile.isDirectory()) { - throw new BadCommandArgumentsException( - E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE +": " - + pkgFile.getAbsolutePath()); - } else { - srcFile = new Path(pkgFile.toURI()); - } - } + Path srcFile = extractPackagePath(installPkgInfo.packageURI); // Do not provide new options to install-package command as it is in // deprecated mode. So version is kept null here. Use package --install. Path pkgPath = sliderFileSystem.buildPackageDirPath(installPkgInfo.name, null); - sliderFileSystem.getFileSystem().mkdirs(pkgPath); + FileSystem sfs = sliderFileSystem.getFileSystem(); + sfs.mkdirs(pkgPath); Path fileInFs = new Path(pkgPath, srcFile.getName()); - log.info("Installing package {} at {} and overwrite is {}.", srcFile, fileInFs, installPkgInfo.replacePkg); - if (sliderFileSystem.getFileSystem().exists(fileInFs) && !installPkgInfo.replacePkg) { - throw new BadCommandArgumentsException( - "Package exists at " + fileInFs.toUri().toString() +"." - + E_USE_REPLACEPKG_TO_OVERWRITE); - } - - sliderFileSystem.getFileSystem().copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs); + log.info("Installing package {} at {} and overwrite is {}.", + srcFile, fileInFs, installPkgInfo.replacePkg); + require(!(sfs.exists(fileInFs) && !installPkgInfo.replacePkg), + "Package exists at %s. : %s", fileInFs.toUri(), E_USE_REPLACEPKG_TO_OVERWRITE); + sfs.copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs); return EXIT_SUCCESS; } @@ -1177,11 +1095,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe + CommonArgs.usage(serviceArgs, ACTION_CLIENT)); } - if (clientInfo.name == null) { - throw new BadCommandArgumentsException("No application name specified\n" - + CommonArgs.usage(serviceArgs, - ACTION_CLIENT)); - } + requireArgumentSet(Arguments.ARG_NAME, clientInfo.name); File storeFile = null; SecurityStore.StoreType type; @@ -1193,11 +1107,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe type = SecurityStore.StoreType.truststore; } - if (storeFile.exists()) { - throw new BadCommandArgumentsException("File %s already exists. " - + "Please remove that file or select a different file name.", - storeFile.getAbsolutePath()); - } + require (!storeFile.exists(), + "File %s already exists. Please remove that file or select a different file name.", + storeFile.getAbsolutePath()); String hostname = null; if (type == SecurityStore.StoreType.keystore) { hostname = clientInfo.hostname; @@ -1232,10 +1144,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } } - byte[] - keystore = - createClusterOperations(clientInfo.name).getClientCertificateStore( - hostname, "client", password, type.name()); + byte[] keystore = createClusterOperations(clientInfo.name) + .getClientCertificateStore(hostname, "client", password, type.name()); // persist to file IOUtils.write(keystore, new FileOutputStream(storeFile)); @@ -1245,31 +1155,20 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private int doClientInstall(ActionClientArgs clientInfo) throws IOException, SliderException { - if (clientInfo.installLocation == null) { - throw new BadCommandArgumentsException( + require(clientInfo.installLocation != null, E_INVALID_INSTALL_LOCATION +"\n" + CommonArgs.usage(serviceArgs, ACTION_CLIENT)); - } else { - if (!clientInfo.installLocation.exists()) { - throw new BadCommandArgumentsException(E_INSTALL_PATH_DOES_NOT_EXIST - +": " + clientInfo.installLocation.getAbsolutePath()); - } - if (!clientInfo.installLocation.isDirectory()) { - throw new BadCommandArgumentsException(E_INVALID_INSTALL_PATH - +": " + clientInfo.installLocation.getAbsolutePath()); - } - } + require(clientInfo.installLocation.exists(), + E_INSTALL_PATH_DOES_NOT_EXIST + ": " + clientInfo.installLocation.getAbsolutePath()); + + require(clientInfo.installLocation.isFile(), + E_INVALID_INSTALL_PATH + ": " + clientInfo.installLocation.getAbsolutePath()); File pkgFile; - if (StringUtils.isEmpty(clientInfo.packageURI)) { - throw new BadCommandArgumentsException(E_INVALID_APPLICATION_PACKAGE_LOCATION); - } else { - pkgFile = new File(clientInfo.packageURI); - if (!pkgFile.exists() || pkgFile.isDirectory()) { - throw new BadCommandArgumentsException(E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE - +" at " + pkgFile.getAbsolutePath()); - } - } + requireArgumentSet(Arguments.ARG_PACKAGE, clientInfo.packageURI); + pkgFile = new File(clientInfo.packageURI); + require(pkgFile.isFile(), + E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + " at %s", pkgFile.getAbsolutePath()); JSONObject config = null; if(clientInfo.clientConfig != null) { @@ -1368,7 +1267,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe Path appDefPath = null; try { appDefPath = new Path( - SliderUtils.getApplicationDefinitionPath(instanceDefinition + getApplicationDefinitionPath(instanceDefinition .getAppConfOperations())); } catch (BadConfigException e) { // Invalid cluster state, so move on to next. No need to log anything @@ -1390,11 +1289,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe println("%-25s %15s %30s %s", clusterName, packageName, packageVersion, appDefPathStr); } - } catch(IOException e) { - if(log.isDebugEnabled()) { - log.debug(clusterName + " application definition path " - + appDefPathStr + " is not found."); - } + } catch (IOException e) { + log.debug("{} application definition path {} is not found.", clusterName, appDefPathStr); } } return EXIT_SUCCESS; @@ -1403,19 +1299,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private int actionPackageList() throws IOException { Path pkgPath = sliderFileSystem.buildPackageDirPath(StringUtils.EMPTY, StringUtils.EMPTY); - log.info("Package install path : " + pkgPath); - if (!sliderFileSystem.getFileSystem().isDirectory(pkgPath)) { + log.info("Package install path : {}", pkgPath); + FileSystem sfs = sliderFileSystem.getFileSystem(); + if (!sfs.isDirectory(pkgPath)) { log.info("No package(s) installed"); return EXIT_SUCCESS; } - FileStatus[] fileStatus = sliderFileSystem.getFileSystem().listStatus( - pkgPath); + FileStatus[] fileStatus = sfs.listStatus(pkgPath); boolean hasPackage = false; StringBuilder sb = new StringBuilder(); sb.append("List of installed packages:\n"); for (FileStatus fstat : fileStatus) { if (fstat.isDirectory()) { - sb.append("\t" + fstat.getPath().getName()); + sb.append("\t").append(fstat.getPath().getName()); sb.append("\n"); hasPackage = true; } @@ -1428,53 +1324,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return EXIT_SUCCESS; } - private int actionPackageInstall(ActionPackageArgs actionPackageArgs) throws - YarnException, - IOException { - - Path srcFile = null; - if (StringUtils.isEmpty(actionPackageArgs.name)) { - throw new BadCommandArgumentsException( - "A valid application type name is required (e.g. HBASE).\n" - + CommonArgs.usage(serviceArgs, ACTION_PACKAGE)); - } + private int actionPackageInstall(ActionPackageArgs actionPackageArgs) + throws YarnException, IOException { + requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name); - if (StringUtils.isEmpty(actionPackageArgs.packageURI)) { - throw new BadCommandArgumentsException( - E_INVALID_APPLICATION_PACKAGE_LOCATION); - } else { - File pkgFile = new File(actionPackageArgs.packageURI); - if (!pkgFile.exists() || pkgFile.isDirectory()) { - throw new BadCommandArgumentsException( - E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE - + ": " + pkgFile.getAbsolutePath()); - } else { - srcFile = new Path(pkgFile.toURI()); - } - } + Path srcFile = extractPackagePath(actionPackageArgs.packageURI); Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name, actionPackageArgs.version); - if (!sliderFileSystem.getFileSystem().exists(pkgPath)) { - sliderFileSystem.getFileSystem().mkdirs(pkgPath); + FileSystem fs = sliderFileSystem.getFileSystem(); + if (!fs.exists(pkgPath)) { + fs.mkdirs(pkgPath); } Path fileInFs = new Path(pkgPath, srcFile.getName()); - if (sliderFileSystem.getFileSystem().exists(fileInFs) - && !actionPackageArgs.replacePkg) { - throw new BadCommandArgumentsException(E_PACKAGE_EXISTS +" at " + - fileInFs.toUri() + ". Use --replacepkg to overwrite."); - } + require(actionPackageArgs.replacePkg || !fs.exists(fileInFs), + E_PACKAGE_EXISTS +" at %s. Use --replacepkg to overwrite.", fileInFs.toUri()); log.info("Installing package {} to {} (overwrite set to {})", srcFile, fileInFs, actionPackageArgs.replacePkg); - sliderFileSystem.getFileSystem().copyFromLocalFile(false, - actionPackageArgs.replacePkg, srcFile, fileInFs); + fs.copyFromLocalFile(false, actionPackageArgs.replacePkg, srcFile, fileInFs); String destPathWithHomeDir = Path .getPathWithoutSchemeAndAuthority(fileInFs).toString(); String destHomeDir = Path.getPathWithoutSchemeAndAuthority( - sliderFileSystem.getFileSystem().getHomeDirectory()).toString(); + fs.getHomeDirectory()).toString(); // a somewhat contrived approach to stripping out the home directory and any trailing // separator; designed to work on windows and unix String destPathWithoutHomeDir; @@ -1492,23 +1366,26 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return EXIT_SUCCESS; } + private Path extractPackagePath(String packageURI) + throws BadCommandArgumentsException { + require(isSet(packageURI), E_INVALID_APPLICATION_PACKAGE_LOCATION); + File pkgFile = new File(packageURI); + require(pkgFile.isFile(), + E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + ": " + pkgFile.getAbsolutePath()); + return new Path(pkgFile.toURI()); + } + private int actionPackageDelete(ActionPackageArgs actionPackageArgs) throws YarnException, IOException { - if (StringUtils.isEmpty(actionPackageArgs.name)) { - throw new BadCommandArgumentsException( - "A valid application type name is required (e.g. HBASE).\n" - + CommonArgs.usage(serviceArgs, ACTION_PACKAGE)); - } + requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name); Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name, actionPackageArgs.version); - if (!sliderFileSystem.getFileSystem().exists(pkgPath)) { - throw new BadCommandArgumentsException(E_PACKAGE_DOES_NOT_EXIST +": " - + pkgPath.toUri().toString()); - } + FileSystem fs = sliderFileSystem.getFileSystem(); + require(fs.exists(pkgPath), E_PACKAGE_DOES_NOT_EXIST +": %s ", pkgPath.toUri()); log.info("Deleting package {} at {}.", actionPackageArgs.name, pkgPath); - if(sliderFileSystem.getFileSystem().delete(pkgPath, true)) { + if(fs.delete(pkgPath, true)) { log.info("Deleted package {} " + actionPackageArgs.name); return EXIT_SUCCESS; } else { @@ -1548,7 +1425,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe boolean liveClusterAllowed, boolean isUpgradeFlow) throws YarnException, IOException { // verify that a live cluster isn't there - SliderUtils.validateClusterName(clustername); + validateClusterName(clustername); verifyBindingsDefined(); if (!liveClusterAllowed) { verifyNoLiveClusters(clustername, "Create"); @@ -1627,8 +1504,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe String count = roleEntry.getValue(); String key = roleEntry.getKey(); log.info("{} => {}", key, count); - resources.getOrAddComponent(key) - .put(ResourceKeys.COMPONENT_INSTANCES, count); + resources.getOrAddComponent(key).put(COMPONENT_INSTANCES, count); } //all CLI role options @@ -1663,7 +1539,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe builder.setQueue(buildInfo.queue); String quorum = buildInfo.getZKhosts(); - if (SliderUtils.isUnset(quorum)) { + if (isUnset(quorum)) { quorum = registryQuorum; } if (isUnset(quorum)) { @@ -1735,7 +1611,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe if (!SliderKeys.COMPONENT_AM.equals(componentName)) { clientComponentInstances.put(componentName, clientResources .getComponentOptInt(componentName, - ResourceKeys.COMPONENT_INSTANCES, -1)); + COMPONENT_INSTANCES, -1)); } } @@ -1745,8 +1621,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } catch (LockAcquireFailedException e) { log.warn("Failed to get a Lock on cluster resource : {}", e, e); throw new BadClusterStateException( - "Failed to load client resource definition " + clustername + ": " - + e); + "Failed to load client resource definition " + clustername + ": " + e, e); } Map<String, Integer> clusterComponentInstances = new HashMap<>(); for (Map.Entry<String, Map<String, String>> component : clusterConf @@ -1755,7 +1630,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe clusterComponentInstances.put( component.getKey(), Integer.decode(component.getValue().get( - ResourceKeys.COMPONENT_INSTANCES))); + COMPONENT_INSTANCES))); } } @@ -1763,16 +1638,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe Iterator<Map.Entry<String, Integer>> clientComponentInstanceIt = clientComponentInstances .entrySet().iterator(); while (clientComponentInstanceIt.hasNext()) { - Map.Entry<String, Integer> clientComponentInstanceEntry = clientComponentInstanceIt - .next(); - if (clusterComponentInstances - .containsKey(clientComponentInstanceEntry.getKey())) { + Map.Entry<String, Integer> clientComponentInstanceEntry = clientComponentInstanceIt.next(); + if (clusterComponentInstances.containsKey(clientComponentInstanceEntry.getKey())) { // compare instance count now and remove from both maps if they match if (clusterComponentInstances .get(clientComponentInstanceEntry.getKey()) == clientComponentInstanceEntry .getValue()) { - clusterComponentInstances - .remove(clientComponentInstanceEntry.getKey()); + clusterComponentInstances.remove(clientComponentInstanceEntry.getKey()); clientComponentInstanceIt.remove(); } } @@ -1846,8 +1718,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe public FsPermission getClusterDirectoryPermissions(Configuration conf) { String clusterDirPermsOct = - conf.get(CLUSTER_DIRECTORY_PERMISSIONS, - DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS); + conf.get(CLUSTER_DIRECTORY_PERMISSIONS, DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS); return new FsPermission(clusterDirPermsOct); } @@ -1857,9 +1728,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @throws BadCommandArgumentsException the exception raised on an invalid config */ public void verifyBindingsDefined() throws BadCommandArgumentsException { - InetSocketAddress rmAddr = SliderUtils.getRmAddress(getConfig()); + InetSocketAddress rmAddr = getRmAddress(getConfig()); if (!getConfig().getBoolean(YarnConfiguration.RM_HA_ENABLED, false) - && !SliderUtils.isAddressDefined(rmAddr)) { + && !isAddressDefined(rmAddr)) { throw new BadCommandArgumentsException( E_NO_RESOURCE_MANAGER + " in the argument " @@ -1923,9 +1794,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @throws UnknownApplicationInstanceException if the file is not found */ public AggregateConf loadInstanceDefinitionUnresolved(String name, - Path clusterDirectory) throws - IOException, - SliderException { + Path clusterDirectory) throws IOException, SliderException { try { AggregateConf definition = @@ -1937,7 +1806,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe throw UnknownApplicationInstanceException.unknownInstance(name, e); } } - /** + + /** * Load the instance definition. * @param name cluster name * @param resolved flag to indicate the cluster should be resolved @@ -1980,11 +1850,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe deployedClusterName = clustername; - SliderUtils.validateClusterName(clustername); + validateClusterName(clustername); verifyNoLiveClusters(clustername, "Launch"); Configuration config = getConfig(); lookupZKQuorum(); - boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config); + boolean clusterSecure = isHadoopClusterSecure(config); //create the Slider AM provider -this helps set up the AM SliderAMClientProvider sliderAM = new SliderAMClientProvider(config); @@ -1997,18 +1867,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations(); Path generatedConfDirPath = createPathThatMustExist(internalOptions.getMandatoryOption( - InternalKeys.INTERNAL_GENERATED_CONF_PATH)); + INTERNAL_GENERATED_CONF_PATH)); Path snapshotConfPath = createPathThatMustExist(internalOptions.getMandatoryOption( - InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH)); + INTERNAL_SNAPSHOT_CONF_PATH)); // cluster Provider AbstractClientProvider provider = createClientProvider( - internalOptions.getMandatoryOption( - InternalKeys.INTERNAL_PROVIDER_NAME)); - // make sure the conf dir is valid; - + internalOptions.getMandatoryOption(INTERNAL_PROVIDER_NAME)); if (log.isDebugEnabled()) { log.debug(instanceDefinition.toString()); } @@ -2018,7 +1885,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // add the tags if available Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem, - SliderUtils.getApplicationDefinitionPath(appOperations)); + getApplicationDefinitionPath(appOperations)); AppMasterLauncher amLauncher = new AppMasterLauncher(clustername, SliderKeys.APP_TYPE, config, @@ -2043,8 +1910,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe String libdir = "lib"; Path libPath = new Path(tempPath, libdir); sliderFileSystem.getFileSystem().mkdirs(libPath); - log.debug("FS={}, tempPath={}, libdir={}", - sliderFileSystem, tempPath, libPath); + log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem, tempPath, libPath); // set local resources for the application master // local files or archives as needed @@ -2055,9 +1921,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe boolean hasServerLog4jProperties = false; Path remoteConfPath = null; String relativeConfDir = null; - String confdirProp = - System.getProperty(SliderKeys.PROPERTY_CONF_DIR); - if (confdirProp == null || confdirProp.isEmpty()) { + String confdirProp = System.getProperty(SliderKeys.PROPERTY_CONF_DIR); + if (isUnset(confdirProp)) { log.debug("No local configuration directory provided as system property"); } else { File confDir = new File(confdirProp); @@ -2065,11 +1930,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe throw new BadConfigException(E_CONFIGURATION_DIRECTORY_NOT_FOUND, confDir); } - Path localConfDirPath = SliderUtils.createLocalPath(confDir); + Path localConfDirPath = createLocalPath(confDir); remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR); log.debug("Slider configuration directory is {}; remote to be {}", localConfDirPath, remoteConfPath); - SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null); + copyDirectory(config, localConfDirPath, remoteConfPath, null); File log4jserver = new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME); @@ -2089,7 +1954,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe Map<String, LocalResource> submittedConfDir = sliderFileSystem.submitDirectory(remoteConfPath, relativeConfDir); - SliderUtils.mergeMaps(localResources, submittedConfDir); + mergeMaps(localResources, submittedConfDir); } } // build up the configuration @@ -2109,7 +1974,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe Configuration clientConfExtras = new Configuration(false); // then build up the generated path. FsPermission clusterPerms = getClusterDirectoryPermissions(config); - SliderUtils.copyDirectory(config, snapshotConfPath, generatedConfDirPath, + copyDirectory(config, snapshotConfPath, generatedConfDirPath, clusterPerms); @@ -2162,15 +2027,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // TODO: consider supporting apps that don't have an image path Path imagePath = - SliderUtils.extractImagePath(sliderFileSystem, internalOptions); + extractImagePath(sliderFileSystem, internalOptions); if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) { log.debug("Registered image path {}", imagePath); } // build the environment amLauncher.putEnv( - SliderUtils.buildEnvMap(sliderAMResourceComponent)); - ClasspathConstructor classpath = SliderUtils.buildClasspath(relativeConfDir, + buildEnvMap(sliderAMResourceComponent)); + ClasspathConstructor classpath = buildClasspath(relativeConfDir, libdir, getConfig(), sliderFileSystem, @@ -2182,13 +2047,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe amLauncher.setEnv("LANGUAGE", "en_US.UTF-8"); amLauncher.putEnv(getAmLaunchEnv(config)); - for (Map.Entry<String, String> envs : SliderUtils.getSystemEnv().entrySet()) { + for (Map.Entry<String, String> envs : getSystemEnv().entrySet()) { log.debug("System env {}={}", envs.getKey(), envs.getValue()); } if (log.isDebugEnabled()) { log.debug("AM classpath={}", classpath); log.debug("Environment Map:\n{}", - SliderUtils.stringifyMap(amLauncher.getEnv())); + stringifyMap(amLauncher.getEnv())); log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath)); } @@ -2196,14 +2061,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe InetSocketAddress rmSchedulerAddress; try { - rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(config); + rmSchedulerAddress = getRmSchedulerAddress(config); } catch (IllegalArgumentException e) { throw new BadConfigException("%s Address invalid: %s", - YarnConfiguration.RM_SCHEDULER_ADDRESS, - config.get( - YarnConfiguration.RM_SCHEDULER_ADDRESS) - ); - + YarnConfiguration.RM_SCHEDULER_ADDRESS, + config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS)); } String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress); @@ -2244,18 +2106,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe /** * pass the registry binding */ - addConfOptionToCLI(commandLine, config, - RegistryConstants.KEY_REGISTRY_ZK_ROOT, + commandLine.addConfOptionToCLI(config, RegistryConstants.KEY_REGISTRY_ZK_ROOT, RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT); - addMandatoryConfOptionToCLI(commandLine, config, - RegistryConstants.KEY_REGISTRY_ZK_QUORUM); + commandLine.addMandatoryConfOption(config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM); if (clusterSecure) { // if the cluster is secure, make sure that // the relevant security settings go over - addConfOptionToCLI(commandLine, - config, - DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); + commandLine.addConfOption(config, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); } // write out the path output commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM); @@ -2281,8 +2139,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // Set the queue to which this application is to be submitted in the RM // Queue for App master String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE); - String suppliedQueue = internalOperations.getGlobalOptions().get(InternalKeys.INTERNAL_QUEUE); - if(!SliderUtils.isUnset(suppliedQueue)) { + String suppliedQueue = internalOperations.getGlobalOptions().get(INTERNAL_QUEUE); + if(!isUnset(suppliedQueue)) { amQueue = suppliedQueue; log.info("Using queue {} for the application instance.", amQueue); } @@ -2297,8 +2155,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } protected Map<String, String> getAmLaunchEnv(Configuration config) { - String sliderAmLaunchEnv = config.get(SliderXmlConfKeys.KEY_AM_LAUNCH_ENV); - log.debug("{} = {}", SliderXmlConfKeys.KEY_AM_LAUNCH_ENV, sliderAmLaunchEnv); + String sliderAmLaunchEnv = config.get(KEY_AM_LAUNCH_ENV); + log.debug("{} = {}", KEY_AM_LAUNCH_ENV, sliderAmLaunchEnv); // Multiple env variables can be specified with a comma (,) separator String[] envs = StringUtils.isEmpty(sliderAmLaunchEnv) ? null : sliderAmLaunchEnv.split(","); @@ -2343,7 +2201,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe String systemKey = placeholderKey .substring(2, placeholderKey.length() - 1).toUpperCase(Locale.ENGLISH) .replaceAll("\\.", "_"); - String placeholderValue = SliderUtils.getSystemEnv(systemKey); + String placeholderValue = getSystemEnv(systemKey); log.debug("Placeholder {}={}", placeholderKey, placeholderValue); placeholderKeyValueMap.put(placeholderKey, placeholderValue); } @@ -2353,10 +2211,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private void propagatePythonExecutable(Configuration config, AggregateConf instanceDefinition) { String pythonExec = config.get( - SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH); + PYTHON_EXECUTABLE_PATH); if (pythonExec != null) { instanceDefinition.getAppConfOperations().getGlobalOptions().putIfUnset( - SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH, + PYTHON_EXECUTABLE_PATH, pythonExec); } } @@ -2386,7 +2244,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe new Duration(acceptWaitMillis)); // may have failed, so check that - if (SliderUtils.hasAppFinished(report)) { + if (hasAppFinished(report)) { exitCode = buildExitCode(report); } else { // exit unless there is a wait @@ -2421,68 +2279,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe AggregateConf clusterSpec) { String dfsPrincipal = config.get(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); if (dfsPrincipal != null) { - String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX + - DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; + String siteDfsPrincipal = SITE_XML_PREFIX + DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; clusterSpec.getAppConfOperations().getGlobalOptions().putIfUnset( siteDfsPrincipal, dfsPrincipal); } } - - private boolean addConfOptionToCLI(CommandLineBuilder cmdLine, - Configuration conf, - String key) { - String val = conf.get(key); - return defineIfSet(cmdLine, key, val); - } - - private String addConfOptionToCLI(CommandLineBuilder cmdLine, - Configuration conf, - String key, - String defVal) { - String val = conf.get(key, defVal); - define(cmdLine, key, val); - return val; - } - - /** - * Add a <code>-D key=val</code> command to the CLI - * @param cmdLine command line - * @param key key - * @param val value - */ - private void define(CommandLineBuilder cmdLine, String key, String val) { - Preconditions.checkArgument(key != null, "null key"); - Preconditions.checkArgument(val != null, "null value"); - cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val); - } - - /** - * Add a <code>-D key=val</code> command to the CLI if <code>val</code> - * is not null - * @param cmdLine command line - * @param key key - * @param val value - */ - private boolean defineIfSet(CommandLineBuilder cmdLine, String key, String val) { - Preconditions.checkArgument(key != null, "null key"); - if (val != null) { - define(cmdLine, key, val); - return true; - } else { - return false; - } - } - - private void addMandatoryConfOptionToCLI(CommandLineBuilder cmdLine, - Configuration conf, - String key) throws BadConfigException { - if (!addConfOptionToCLI(cmdLine, conf, key)) { - throw new BadConfigException("Missing configuration option: " + key); - } - } - /** * Create a path that must exist in the cluster fs * @param uri uri to create @@ -2781,9 +2584,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // and those the RM knows about List<ApplicationReport> instances = listSliderInstances(null); - SliderUtils.sortApplicationsByMostRecent(instances); + sortApplicationsByMostRecent(instances); Map<String, ApplicationReport> reportMap = - SliderUtils.buildApplicationReportMap(instances, min, max); + buildApplicationReportMap(instances, min, max); log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}", persistentInstances.size(), instances.size(), @@ -2816,7 +2619,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // a report listedInstances.add(name); // containers will be non-null when only one instance is requested - String details = SliderUtils.instanceDetailsToString(name, report, + String details = instanceDetailsToString(name, report, containers, version, components, verbose); print(details); } @@ -2899,7 +2702,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @VisibleForTesting public int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException { verifyBindingsDefined(); - SliderUtils.validateClusterName(name); + validateClusterName(name); log.debug("actionFlex({})", name); Map<String, Integer> roleInstances = new HashMap<>(); Map<String, String> roleMap = args.getComponentMap(); @@ -2926,7 +2729,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe public int actionExists(String name, ActionExistsArgs args) throws YarnException, IOException { verifyBindingsDefined(); - SliderUtils.validateClusterName(name); + validateClusterName(name); boolean checkLive = args.live; log.debug("actionExists({}, {}, {})", name, checkLive, args.state); @@ -2936,7 +2739,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe throw unknownClusterException(name); } String state = args.state; - if (!checkLive && SliderUtils.isUnset(state)) { + if (!checkLive && isUnset(state)) { log.info("Application {} exists", name); return EXIT_SUCCESS; } @@ -2970,8 +2773,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } } - SliderUtils.OnDemandReportStringifier report = - new SliderUtils.OnDemandReportStringifier(instance); + OnDemandReportStringifier report = + new OnDemandReportStringifier(instance); if (!inDesiredState) { //cluster in the list of apps but not running log.info("Application {} found but is in wrong state {}", name, @@ -2991,7 +2794,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe public int actionKillContainer(String name, ActionKillContainerArgs args) throws YarnException, IOException { String id = args.id; - if (SliderUtils.isUnset(id)) { + if (isUnset(id)) { throw new BadCommandArgumentsException("Missing container id"); } log.info("killingContainer {}:{}", name, id); @@ -3087,7 +2890,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe YarnException, IOException { verifyBindingsDefined(); - SliderUtils.validateClusterName(clustername); + validateClusterName(clustername); String outfile = statusArgs.getOutput(); ClusterDescription status = getClusterDescription(clustername); String text = status.toJsonString(); @@ -3109,7 +2912,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe public int actionFreeze(String clustername, ActionFreezeArgs freezeArgs) throws YarnException, IOException { verifyBindingsDefined(); - SliderUtils.validateClusterName(clustername); + validateClusterName(clustername); int waittime = freezeArgs.getWaittime(); String text = freezeArgs.message; boolean forcekill = freezeArgs.force; @@ -3128,7 +2931,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return EXIT_SUCCESS; } log.debug("App to stop was found: {}:\n{}", clustername, - new SliderUtils.OnDemandReportStringifier(app)); + new OnDemandReportStringifier(app)); if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED.ordinal()) { log.info("Cluster {} is in a terminated state {}", clustername, @@ -3195,7 +2998,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @Override public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException { - SliderUtils.validateClusterName(clustername); + validateClusterName(clustername); verifyBindingsDefined(); // see if it is actually running and bail out; verifyNoLiveClusters(clustername, "Start"); @@ -3215,7 +3018,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe public int flex(String clustername, Map<String, Integer> roleInstances) throws YarnException, IOException { verifyBindingsDefined(); - SliderUtils.validateClusterName(clustername); + validateClusterName(clustername); Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( clustername, @@ -3226,7 +3029,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe for (Map.Entry<String, Integer> entry : roleInstances.entrySet()) { String role = entry.getKey(); int count = entry.getValue(); - resources.getOrAddComponent(role).put(ResourceKeys.COMPONENT_INSTANCES, + resources.getOrAddComponent(role).put(COMPONENT_INSTANCES, Integer.toString(count)); log.debug("Flexed cluster specification ( {} -> {}) : \n{}", @@ -3237,7 +3040,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig()); AbstractClientProvider provider = createClientProvider( instanceDefinition.getInternalOperations().getGlobalOptions().getMandatoryOption( - InternalKeys.INTERNAL_PROVIDER_NAME)); + INTERNAL_PROVIDER_NAME)); // slider provider to validate what there is validateInstanceDefinition(sliderAM, instanceDefinition, sliderFileSystem); validateInstanceDefinition(provider, instanceDefinition, sliderFileSystem); @@ -3548,7 +3351,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } else { String filename = RegistryPathUtils.lastPathEntry(name) + ".json"; File jsonFile = new File(destDir, filename); - SliderUtils.write(jsonFile, + write(jsonFile, serviceRecordMarshal.toBytes(instance), true); } @@ -3561,7 +3364,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe outFile = new File(args.destdir, RegistryPathUtils.lastPathEntry(path)); } if (outFile != null) { - SliderUtils.write(outFile, serviceRecordMarshal.toBytes(instance), true); + write(outFile, serviceRecordMarshal.toBytes(instance), true); } else { println(serviceRecordMarshal.toJson(instance)); } @@ -3592,12 +3395,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } else if (registryArgs.listExports) { // list the exports actionRegistryListExports(registryArgs); - } else if (SliderUtils.isSet(registryArgs.getConf)) { + } else if (isSet(registryArgs.getConf)) { // get a configuration PublishedConfiguration publishedConfiguration = actionRegistryGetConfig(registryArgs); outputConfig(publishedConfiguration, registryArgs); - } else if (SliderUtils.isSet(registryArgs.getExport)) { + } else if (isSet(registryArgs.getExport)) { // get a export group PublishedExports publishedExports = actionRegistryGetExport(registryArgs); @@ -3609,11 +3412,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } // JDK7 } catch (FileNotFoundException e) { - log.info("{}", e); + log.info("{}", e.toString()); log.debug("{}", e, e); return EXIT_NOT_FOUND; } catch (PathNotFoundException e) { - log.info("{}", e); + log.info("{}", e.toString()); log.debug("{}", e, e); return EXIT_NOT_FOUND; } @@ -3637,10 +3440,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe RegistryOperations operations = getRegistryOperations(); Collection<ServiceRecord> serviceRecords; if (StringUtils.isEmpty(name)) { - String path = - serviceclassPath( - currentUser(), - serviceType); + String path = serviceclassPath(currentUser(), serviceType); try { Map<String, ServiceRecord> recordMap = @@ -3698,18 +3498,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // application name after --application option and member variable // cluster name has to be put behind action String clusterName = diagnosticArgs.name; - if(SliderUtils.isUnset(clusterName)){ - throw new BadCommandArgumentsException("application name must be provided with --name option"); - } - + requireArgumentSet(Arguments.ARG_NAME, clusterName); + try { - SliderUtils.validateClientConfigFile(); + validateClientConfigFile(); log.info("Slider-client.xml is accessible"); } catch (IOException e) { // we are catching exceptions here because those are indication of // validation result, and we need to print them here - log.error( - "validation of slider-client.xml fails because: " + e.toString(), e); + log.error("validation of slider-client.xml fails because: " + e, e); return; } SliderClusterOperations clusterOperations = createClusterOperations(clusterName); @@ -3722,7 +3519,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe AggregateConf instanceDefinition = clusterOperations .getInstanceDefinition(); String imagePath = instanceDefinition.getInternalOperations().get( - InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); + INTERNAL_APPLICATION_IMAGE_PATH); // if null, that means slider uploaded the agent tarball for the user // and we need to use where slider has put if (imagePath == null) { @@ -3733,26 +3530,27 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe + "/agent"); imagePath = subPath.toString(); } + String pathStr = imagePath + "/" + AGENT_TAR; try { - SliderUtils.validateHDFSFile(sliderFileSystem, imagePath + "/" + AGENT_TAR); + validateHDFSFile(sliderFileSystem, pathStr); log.info("Slider agent package is properly installed"); } catch (FileNotFoundException e) { - log.error("can not find agent package: " + e.toString()); + log.error("can not find agent package: {}", pathStr); + log.debug("can not find agent package: {}", pathStr, e); return; } catch (IOException e) { - log.error("can not open agent package: " + e.toString()); + log.error("can not open agent package: {}", pathStr, e); return; } - String pkgTarballPath = SliderUtils - .getApplicationDefinitionPath(instanceDefinition + String pkgTarballPath = getApplicationDefinitionPath(instanceDefinition .getAppConfOperations()); try { - SliderUtils.validateHDFSFile(sliderFileSystem, pkgTarballPath); + validateHDFSFile(sliderFileSystem, pkgTarballPath); log.info("Application package is properly installed"); } catch (FileNotFoundException e) { - log.error("can not find application package: {}", e); + log.error("can not find application package: {}", pkgTarballPath, e); } catch (IOException e) { - log.error("can not open application package: {} ", e); + log.error("can not open application package: {} ", pkgTarballPath, e); } } } @@ -3769,11 +3567,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private void actionDiagnosticCredentials() throws BadConfigException, IOException { - if (SliderUtils.isHadoopClusterSecure(SliderUtils - .loadSliderClientXML())) { + if (isHadoopClusterSecure(loadSliderClientXML())) { String credentialCacheFileDescription = null; try { - credentialCacheFileDescription = SliderUtils.checkCredentialCacheFile(); + credentialCacheFileDescription = checkCredentialCacheFile(); } catch (BadConfigException e) { log.error("The credential config is not valid: " + e.toString()); throw e; @@ -3835,25 +3632,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // application name after --application option and member variable // cluster name has to be put behind action String clusterName = diagnosticArgs.name; - if(SliderUtils.isUnset(clusterName)){ + if(isUnset(clusterName)){ throw new BadCommandArgumentsException("application name must be provided with --name option"); } - SliderClusterOperations clusterOperations; - AggregateConf instanceDefinition = null; - try { - clusterOperations = createClusterOperations(clusterName); - instanceDefinition = clusterOperations.getInstanceDefinition(); - } catch (YarnException e) { - log.error("Exception happened when retrieving instance definition from YARN: " - + e.toString()); - throw e; - } catch (IOException e) { - log.error("Network problem happened when retrieving instance definition from YARN: " - + e.toString()); - throw e; - } + AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName); String imagePath = instanceDefinition.getInternalOperations().get( - InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); + INTERNAL_APPLICATION_IMAGE_PATH); // if null, it will be uploaded by Slider and thus at slider's path if (imagePath == null) { ApplicationReport appReport = findInstance(clusterName); @@ -3865,58 +3649,54 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe log.info("The path of slider agent tarball on HDFS is: " + imagePath); } - private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs) + private AggregateConf fetchInstanceDefinition(String clusterName) throws YarnException, IOException { - // not using member variable clustername because we want to place - // application name after --application option and member variable - // cluster name has to be put behind action - String clusterName = diagnosticArgs.name; - if(SliderUtils.isUnset(clusterName)){ - throw new BadCommandArgumentsException("application name must be provided with --name option"); - } SliderClusterOperations clusterOperations; AggregateConf instanceDefinition = null; try { clusterOperations = createClusterOperations(clusterName); instanceDefinition = clusterOperations.getInstanceDefinition(); - } catch (YarnException e) { - log.error("Exception happened when retrieving instance definition from YARN: " - + e.toString()); - throw e; - } catch (IOException e) { - log.error("Network problem happened when retrieving instance definition from YARN: " + } catch (YarnException | IOException e) { + log.error("Failed to retrieve instance definition from YARN: " + e.toString()); throw e; } + return instanceDefinition; + } + + private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs) + throws YarnException, IOException { + // not using member variable clustername because we want to place + // application name after --application option and member variable + // cluster name has to be put behind action + String clusterName = diagnosticArgs.name; + requireArgumentSet(Arguments.ARG_NAME, clusterName); + AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName); String clusterDir = instanceDefinition.getAppConfOperations() .getGlobalOptions().get(AgentKeys.APP_ROOT); - String pkgTarball = SliderUtils - .getApplicationDefinitionPath(instanceDefinition.getAppConfOperations()); + String pkgTarball = getApplicationDefinitionPath(instanceDefinition.getAppConfOperations()); String runAsUser = instanceDefinition.getAppConfOperations() .getGlobalOptions().get(AgentKeys.RUNAS_USER); - log.info("The location of the cluster instance directory in HDFS is: " - + clusterDir); - log.info("The name of the application package tarball on HDFS is: " - + pkgTarball); - log.info("The runas user of the application in the cluster is: " - + runAsUser); + log.info("The location of the cluster instance directory in HDFS is: {}", clusterDir); + log.info("The name of the application package tarball on HDFS is: {}",pkgTarball); + log.info("The runas user of the application in the cluster is: {}",runAsUser); if (diagnosticArgs.verbose) { - log.info("App config of the application: " - + instanceDefinition.getAppConf().toJson()); - log.info("Resource config of the application: " - + instanceDefinition.getResources().toJson()); + log.info("App config of the application:\n{}", + instanceDefinition.getAppConf().toJson()); + log.info("Resource config of the application:\n{}", + instanceDefinition.getResources().toJson()); } } private void actionDiagnosticClient(ActionDiagnosticArgs diagnosticArgs) throws SliderException, IOException { try { - String currentCommandPath = SliderUtils.getCurrentCommandPath(); + String currentCommandPath = getCurrentCommandPath(); SliderVersionInfo.loadAndPrintVersionInfo(log); - String clientConfigPath = SliderUtils.getClientConfigPath(); - String jdkInfo = SliderUtils.getJDKInfo(); + String clientConfigPath = getClientConfigPath(); + String jdkInfo = getJDKInfo(); println("The slider command path: %s", currentCommandPath); println("The slider-client.xml used by current running command path: %s", clientConfigPath); @@ -3924,7 +3704,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // security info Configuration config = getConfig(); - if (SliderUtils.isHadoopClusterSecure(config)) { + if (isHadoopClusterSecure(config)) { println("Hadoop Cluster is secure"); println("Login user is %s", UserGroupInformation.getLoginUser()); println("Current user is %s", UserGroupInformation.getCurrentUser()); @@ -3936,7 +3716,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // verbose? if (diagnosticArgs.verbose) { // do the environment - Map<String, String> env = SliderUtils.getSystemEnv(); + Map<String, String> env = getSystemEnv(); Set<String> envList = ConfigHelper.sortedConfigKeys(env.entrySet()); StringBuilder builder = new StringBuilder("Environment variables:\n"); for (String key : envList) { @@ -3947,7 +3727,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // Java properties builder = new StringBuilder("JVM Properties\n"); Map<String, String> props = - SliderUtils.sortedMap(SliderUtils.toMap(System.getProperties())); + sortedMap(toMap(System.getProperties())); for (Entry<String, String> entry : props.entrySet()) { builder.append(entry.getKey()).append("=") .append(entry.getValue()).append("\n"); @@ -3956,12 +3736,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe println(builder.toString()); // then the config - println("Slider client configuration:\n" - + ConfigHelper.dumpConfigToString(config)); - + println("Slider client configuration:\n" + ConfigHelper.dumpConfigToString(config)); } - SliderUtils.validateSliderClientEnvironment(log); + validateSliderClientEnvironment(log); } catch (SliderException | IOException e) { log.error(e.toString()); throw e; @@ -4122,8 +3900,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe RegistryRetriever retriever = new RegistryRetriever(getConfig(), instance); boolean external = !registryArgs.internal; - PublishedExportsSet exports = - retriever.getExports(external); + PublishedExportsSet exports = retriever.getExports(external); PublishedExports published = retriever.retrieveExports(exports, registryArgs.getExport, @@ -4395,7 +4172,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return EXIT_FALSE; } - String version = SliderUtils.getSliderVersion(); + String version = getSliderVersion(); Path dependencyLibTarGzip = sliderFileSystem.getDependencyTarGzip(); // Check if dependency has already been uploaded, in which case log @@ -4409,23 +4186,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } String libDir = System.getProperty(SliderKeys.PROPERTY_LIB_DIR); - if (SliderUtils.isSet(libDir)) { + if (isSet(libDir)) { File srcFolder = new File(libDir); File tempLibTarGzipFile = File.createTempFile( SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + "_", SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT); // copy all jars except slider-core-<version>.jar - FilenameFilter jarFilter = new FilenameFilter() { - public boolean accept(File dir, String name) { - String lowercaseName = name.toLowerCase(); - if (lowercaseName.endsWith(".jar")) { - return true; - } else { - return false; - } - } - }; - SliderUtils.tarGzipFolder(srcFolder, tempLibTarGzipFile, jarFilter); + tarGzipFolder(srcFolder, tempLibTarGzipFile, createJarFilter()); log.info("Uploading dependency for AM (version {}) from {} to {}", version, tempLibTarGzipFile.toURI(), dependencyLibTarGzip.toUri()); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index d34dd2a..a1ad690 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -159,7 +159,7 @@ public class CoreFileSystem { Preconditions.checkNotNull(packageName); Path path = getBaseApplicationPath(); path = new Path(path, SliderKeys.PACKAGE_DIRECTORY + "/" + packageName); - if (StringUtils.isNotEmpty(packageVersion)) { + if (SliderUtils.isSet(packageVersion)) { path = new Path(path, packageVersion); } return path; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index f380ec6..1f97982 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -1034,7 +1034,7 @@ public final class SliderUtils { * something other than 0.0.0.0 */ public static boolean isAddressDefined(InetSocketAddress address) { - return !(address.getHostName().equals("0.0.0.0")); + return !(address.getHostString().equals("0.0.0.0")); } public static void setRmAddress(Configuration conf, String rmAddr) { @@ -1352,16 +1352,7 @@ public final class SliderUtils { log.info("Loading all dependencies from {}", srcPath); if (SliderUtils.isSet(srcPath)) { File srcFolder = new File(srcPath); - FilenameFilter jarFilter = new FilenameFilter() { - public boolean accept(File dir, String name) { - String lowercaseName = name.toLowerCase(); - if (lowercaseName.endsWith(".jar")) { - return true; - } else { - return false; - } - } - }; + FilenameFilter jarFilter = createJarFilter(); File[] listOfJars = srcFolder.listFiles(jarFilter); for (File jarFile : listOfJars) { LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName()); @@ -1371,6 +1362,18 @@ public final class SliderUtils { } /** + * Accept all filenames ending with {@code .jar} + * @return a filename filter + */ + public static FilenameFilter createJarFilter() { + return new FilenameFilter() { + public boolean accept(File dir, String name) { + return name.toLowerCase(Locale.ENGLISH).endsWith(".jar"); + } + }; + } + + /** * Submit the AM tar.gz containing all dependencies and map it * @param providerResources provider map to build up * @param sliderFileSystem remote fs http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java index b4ca791..06dbfea 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java @@ -220,8 +220,7 @@ public class AppMasterLauncher extends AbstractLauncher { } Token<? extends TokenIdentifier>[] tokens = null; - boolean tokensProvided = - this.getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) != null; + boolean tokensProvided = getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) != null; if (!tokensProvided) { // For now, only getting tokens for the default file-system. FileSystem fs = coreFileSystem.getFileSystem(); @@ -232,7 +231,7 @@ public class AppMasterLauncher extends AbstractLauncher { if (tokens != null && tokens.length > 0) { AbstractDelegationTokenIdentifier id = (AbstractDelegationTokenIdentifier)tokens[0].decodeIdentifier(); - Date d = new Date(id.getIssueDate() + 24*60*60*1000); + Date d = new Date(id.getIssueDate() + 24 * 60 * 60 * 1000); log.info("HDFS delegation tokens for AM launch context require renewal by {}", DateFormat.getDateTimeInstance().format(d)); } else { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java index 3ba0181..6eb4058 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java @@ -38,7 +38,6 @@ import java.util.List; public class ClasspathConstructor { public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR; -// public static final String CLASS_PATH_SEPARATOR = File.pathSeparator; private final List<String> pathElements = new ArrayList<>(); public ClasspathConstructor() { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java index dbaa981..57b8965 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java @@ -19,8 +19,11 @@ package org.apache.slider.core.launch; import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.slider.common.params.Arguments; import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.BadConfigException; import java.util.ArrayList; import java.util.List; @@ -101,4 +104,56 @@ public class CommandLineBuilder { return argumentList; } + public boolean addConfOption(Configuration conf, String key) { + String val = conf.get(key); + return defineIfSet(key, val); + } + + public String addConfOptionToCLI(Configuration conf, + String key, + String defVal) { + String val = conf.get(key, defVal); + define(key, val); + return val; + } + + /** + * Add a <code>-D key=val</code> command to the CLI + * @param key key + * @param val value + */ + public void define(String key, String val) { + Preconditions.checkArgument(key != null, "null key"); + Preconditions.checkArgument(val != null, "null value"); + add(Arguments.ARG_DEFINE, key + "=" + val); + } + + /** + * Add a <code>-D key=val</code> command to the CLI if <code>val</code> + * is not null + * @param key key + * @param val value + */ + public boolean defineIfSet(String key, String val) { + Preconditions.checkArgument(key != null, "null key"); + if (val != null) { + define(key, val); + return true; + } else { + return false; + } + } + + /** + * Add a mandatory config option + * @param conf configuration + * @param key key + * @throws BadConfigException if the key is missing + */ + public void addMandatoryConfOption(Configuration conf, + String key) throws BadConfigException { + if (!addConfOption(conf, key)) { + throw new BadConfigException("Missing configuration option: " + key); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java index 7f3931b..1622309 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java @@ -98,11 +98,23 @@ public abstract class AbstractSliderLaunchedService extends */ protected static void requireArgumentSet(String argname, String value) throws BadCommandArgumentsException { - if (isUnset(value)) { - throw new BadCommandArgumentsException( - "Required argument " + argname + " missing"); - } + require(isSet(value), "Required argument %s missing", argname ); } + /** + * Require a condition to hold; throw {@link BadCommandArgumentsException} if not. + * The exception text is the formatted message. + * @param condition condition + * @param message string to format + * @param args list of arguments to format. + * @throws BadCommandArgumentsException + */ + protected static void require(boolean condition, String message, + Object... args) + throws BadCommandArgumentsException { + if (!condition) { + throw new BadCommandArgumentsException(message, args); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy index 8245dae..7e3f55a 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy @@ -41,8 +41,7 @@ import org.junit.Test class TestActionPackage extends AgentMiniClusterTestBase { - public static final String E_INVALID_APP_TYPE = - "A valid application type name is required (e.g. HBASE)" + public static final String E_NO_NAME = "Required argument --name missing" String s = File.separator File packageFile = new File("src${s}test${s}resources${s}log4j.properties") @@ -70,7 +69,7 @@ class TestActionPackage extends AgentMiniClusterTestBase { } catch (BadCommandArgumentsException e) { assertExceptionDetails(e, LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR, - E_INVALID_APP_TYPE) + E_NO_NAME) } } @@ -285,7 +284,7 @@ class TestActionPackage extends AgentMiniClusterTestBase { } catch (BadCommandArgumentsException e) { assertExceptionDetails(e, LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR, - E_INVALID_APP_TYPE) + E_NO_NAME) } } }
