http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/client/ClientUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/ClientUtils.java b/slider-core/src/main/java/org/apache/slider/client/ClientUtils.java new file mode 100644 index 0000000..c3ccb1d --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/client/ClientUtils.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.client; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.PathNotFoundException; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; +import org.apache.hadoop.registry.client.exceptions.NoRecordException; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.NotFoundException; +import org.apache.slider.core.exceptions.SliderException; +import org.apache.slider.core.registry.docstore.ConfigFormat; +import org.apache.slider.core.registry.docstore.PublishedConfigSet; +import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; +import org.apache.slider.core.registry.retrieve.RegistryRetriever; + +import java.io.File; +import java.io.IOException; + +import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser; +import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath; + +public class ClientUtils { + public static ServiceRecord lookupServiceRecord(RegistryOperations rops, + String user, String name) throws IOException, SliderException { + return lookupServiceRecord(rops, user, null, name); + } + + public static ServiceRecord lookupServiceRecord(RegistryOperations rops, + String user, String type, String name) throws IOException, + SliderException { + if (StringUtils.isEmpty(user)) { + user = currentUser(); + } else { + user = RegistryPathUtils.encodeForRegistry(user); + } + if (StringUtils.isEmpty(type)) { + type = SliderKeys.APP_TYPE; + } + + String path = servicePath(user, type, name); + return resolve(rops, path); + } + + public static ServiceRecord resolve(RegistryOperations rops, String path) + throws IOException, SliderException { + try { + return rops.resolve(path); + } catch (PathNotFoundException | NoRecordException e) { + throw new NotFoundException(e.getPath().toString(), e); + } + } + + public static PublishedConfiguration getConfigFromRegistry( + RegistryOperations rops, Configuration configuration, + String configName, String appName, String user, boolean external) + throws IOException, SliderException { + ServiceRecord instance = lookupServiceRecord(rops, user, appName); + + RegistryRetriever retriever = new RegistryRetriever(configuration, instance); + PublishedConfigSet configurations = retriever.getConfigurations(external); + + PublishedConfiguration published = retriever.retrieveConfiguration( + configurations, configName, external); + return published; + } + + public static String saveOrReturnConfig(PublishedConfiguration published, + String format, File destPath, String fileName) + throws BadCommandArgumentsException, IOException { + ConfigFormat configFormat = ConfigFormat.resolve(format); + if (configFormat == null) { + throw new BadCommandArgumentsException( + "Unknown/Unsupported format %s ", format); + } + PublishedConfigurationOutputter outputter = + PublishedConfigurationOutputter.createOutputter(configFormat, + published); + boolean print = destPath == null; + if (!print) { + if (destPath.isDirectory()) { + // creating it under a directory + destPath = new File(destPath, fileName); + } + outputter.save(destPath); + return null; + } else { + return outputter.asString(); + } + } +}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index f3dcea3..8210f4d 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -101,6 +101,7 @@ import org.apache.slider.common.params.ActionNodesArgs; import org.apache.slider.common.params.ActionPackageArgs; import org.apache.slider.common.params.ActionRegistryArgs; import org.apache.slider.common.params.ActionResolveArgs; +import org.apache.slider.common.params.ActionResourceArgs; import org.apache.slider.common.params.ActionStatusArgs; import org.apache.slider.common.params.ActionThawArgs; import org.apache.slider.common.params.ActionTokensArgs; @@ -177,7 +178,6 @@ import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.File; -import java.io.FilenameFilter; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -415,7 +415,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe case ACTION_INSTALL_PACKAGE: exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs()); break; - + case ACTION_KEYTAB: exitCode = actionKeytab(serviceArgs.getActionKeytabArgs()); break; @@ -443,7 +443,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe case ACTION_RESOLVE: exitCode = actionResolve(serviceArgs.getActionResolveArgs()); break; - + + case ACTION_RESOURCE: + exitCode = actionResource(serviceArgs.getActionResourceArgs()); + break; + case ACTION_STATUS: exitCode = actionStatus(clusterName, serviceArgs.getActionStatusArgs()); break; @@ -1029,7 +1033,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe Path fileInFs = new Path(pkgPath, keytabInfo.keytab ); log.info("Deleting keytab {}", fileInFs); FileSystem sfs = sliderFileSystem.getFileSystem(); - require(sfs.exists(fileInFs), "No keytab to delete found at %s", fileInFs.toUri()); + require(sfs.exists(fileInFs), "No keytab to delete found at %s", + fileInFs.toUri()); sfs.delete(fileInFs, false); return EXIT_SUCCESS; @@ -1105,6 +1110,106 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } @Override + public int actionResource(ActionResourceArgs resourceInfo) + throws YarnException, IOException { + if (resourceInfo.help) { + actionHelp(ACTION_RESOURCE); + return EXIT_SUCCESS; + } else if (resourceInfo.install) { + return actionInstallResource(resourceInfo); + } else if (resourceInfo.delete) { + return actionDeleteResource(resourceInfo); + } else if (resourceInfo.list) { + return actionListResource(resourceInfo); + } else { + throw new BadCommandArgumentsException( + "Resource option specified not found.\n" + + CommonArgs.usage(serviceArgs, ACTION_RESOURCE)); + } + } + + private int actionListResource(ActionResourceArgs resourceInfo) throws IOException { + String folder = resourceInfo.folder != null ? resourceInfo.folder : StringUtils.EMPTY; + Path path = sliderFileSystem.buildResourcePath(folder); + RemoteIterator<LocatedFileStatus> files = + sliderFileSystem.getFileSystem().listFiles(path, true); + log.info("Resources:"); + while (files.hasNext()) { + log.info("\t" + files.next().getPath().toString()); + } + + return EXIT_SUCCESS; + } + + private int actionDeleteResource(ActionResourceArgs resourceInfo) + throws BadCommandArgumentsException, IOException { + if (StringUtils.isEmpty(resourceInfo.resource)) { + throw new BadCommandArgumentsException("A file name is required."); + } + + Path fileInFs; + if (resourceInfo.folder == null) { + fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.resource); + } else { + fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.folder, + resourceInfo.resource); + } + + log.info("Deleting resource {}", fileInFs); + FileSystem sfs = sliderFileSystem.getFileSystem(); + require(sfs.exists(fileInFs), "No resource to delete found at %s", fileInFs.toUri()); + sfs.delete(fileInFs, true); + + return EXIT_SUCCESS; + } + + private int actionInstallResource(ActionResourceArgs resourceInfo) + throws BadCommandArgumentsException, IOException { + Path srcFile = null; + String folder = resourceInfo.folder != null ? resourceInfo.folder : StringUtils.EMPTY; + + requireArgumentSet(Arguments.ARG_RESOURCE, resourceInfo.resource); + File file = new File(resourceInfo.resource); + require(file.isFile() || file.isDirectory(), + "Unable to access supplied file at %s", file.getAbsolutePath()); + + File[] files; + if (file.isDirectory()) { + files = file.listFiles(); + } else { + files = new File[] { file }; + } + + Path pkgPath = sliderFileSystem.buildResourcePath(folder); + FileSystem sfs = sliderFileSystem.getFileSystem(); + + if (!sfs.exists(pkgPath)) { + sfs.mkdirs(pkgPath); + sfs.setPermission(pkgPath, new FsPermission( + FsAction.ALL, FsAction.NONE, FsAction.NONE)); + } else { + require(sfs.isDirectory(pkgPath), "Specified folder %s exists and is " + + "not a directory", folder); + } + + for (File f : files) { + srcFile = new Path(f.toURI()); + + Path fileInFs = new Path(pkgPath, srcFile.getName()); + log.info("Installing file {} at {} and overwrite is {}.", + srcFile, fileInFs, resourceInfo.overwrite); + require(!(sfs.exists(fileInFs) && !resourceInfo.overwrite), + "File exists at %s. Use --overwrite to overwrite.", fileInFs.toUri()); + + sfs.copyFromLocalFile(false, resourceInfo.overwrite, srcFile, fileInFs); + sfs.setPermission(fileInFs, + new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE)); + } + + return EXIT_SUCCESS; + } + + @Override public int actionClient(ActionClientArgs clientInfo) throws YarnException, IOException { @@ -1210,8 +1315,21 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe E_INVALID_INSTALL_PATH + ": " + clientInfo.installLocation.getAbsolutePath()); File pkgFile; - require(isSet(clientInfo.packageURI), E_INVALID_APPLICATION_PACKAGE_LOCATION); - pkgFile = new File(clientInfo.packageURI); + File tmpDir = null; + + require(isSet(clientInfo.packageURI) || isSet(clientInfo.name), + E_INVALID_APPLICATION_PACKAGE_LOCATION); + if (isSet(clientInfo.packageURI)) { + pkgFile = new File(clientInfo.packageURI); + } else { + Path appDirPath = sliderFileSystem.buildAppDefDirPath(clientInfo.name); + Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG); + require(sliderFileSystem.isFile(appDefPath), + E_INVALID_APPLICATION_PACKAGE_LOCATION); + tmpDir = Files.createTempDir(); + pkgFile = new File(tmpDir, SliderKeys.DEFAULT_APP_PKG); + sliderFileSystem.copyHdfsFileToLocal(appDefPath, pkgFile); + } require(pkgFile.isFile(), E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + " at %s", pkgFile.getAbsolutePath()); @@ -1233,6 +1351,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe AbstractClientProvider provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE); provider.processClientOperation(sliderFileSystem, + getRegistryOperations(), + getConfig(), "INSTALL", clientInfo.installLocation, pkgFile, @@ -4077,17 +4197,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @VisibleForTesting public PublishedConfiguration actionRegistryGetConfig(ActionRegistryArgs registryArgs) throws YarnException, IOException { - ServiceRecord instance = lookupServiceRecord(registryArgs); - - RegistryRetriever retriever = new RegistryRetriever(getConfig(), instance); - boolean external = !registryArgs.internal; - PublishedConfigSet configurations = - retriever.getConfigurations(external); - - PublishedConfiguration published = retriever.retrieveConfiguration(configurations, - registryArgs.getConf, - external); - return published; + return ClientUtils.getConfigFromRegistry(getRegistryOperations(), + getConfig(), registryArgs.getConf, registryArgs.name, registryArgs.user, + !registryArgs.internal); } /** @@ -4130,27 +4242,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // decide whether or not to print String entry = registryArgs.getConf; String format = registryArgs.format; - ConfigFormat configFormat = ConfigFormat.resolve(format); - if (configFormat == null) { - throw new BadCommandArgumentsException( - "Unknown/Unsupported format %s ", format); - } - PublishedConfigurationOutputter outputter = - PublishedConfigurationOutputter.createOutputter(configFormat, - published); - boolean print = registryArgs.out == null; - if (!print) { - File outputPath = registryArgs.out; - if (outputPath.isDirectory()) { - // creating it under a directory - outputPath = new File(outputPath, entry + "." + format); - } - log.debug("Destination path: {}", outputPath); - outputter.save(outputPath); - } else { - print(outputter.asString()); + String output = ClientUtils.saveOrReturnConfig(published, + registryArgs.format, registryArgs.out, entry + "." + format); + if (output != null) { + print(output); } - } /** @@ -4200,32 +4296,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private ServiceRecord lookupServiceRecord(ActionRegistryArgs registryArgs) throws SliderException, IOException { - String user; - if (StringUtils.isNotEmpty(registryArgs.user)) { - user = RegistryPathUtils.encodeForRegistry(registryArgs.user); - } else { - user = currentUser(); - } - - String path = servicePath(user, registryArgs.serviceType, - registryArgs.name); - return resolve(path); - } - - /** - * Look up a service record of the current user - * @param serviceType service type - * @param id instance ID - * @return instance data - * @throws UnknownApplicationInstanceException no path or service record - * at the end of the path - * @throws SliderException other failures - * @throws IOException IO problems or wrapped exceptions - */ - public ServiceRecord lookupServiceRecord(String serviceType, String id) - throws IOException, SliderException { - String path = servicePath(currentUser(), serviceType, id); - return resolve(path); + return ClientUtils.lookupServiceRecord(getRegistryOperations(), + registryArgs.user, registryArgs.serviceType, registryArgs.name); } /** @@ -4240,11 +4312,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe */ public ServiceRecord resolve(String path) throws IOException, SliderException { - try { - return getRegistryOperations().resolve(path); - } catch (PathNotFoundException | NoRecordException e) { - throw new NotFoundException(e.getPath().toString(), e); - } + return ClientUtils.resolve(getRegistryOperations(), path); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index 5c5d96b..30f6ba9 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -43,6 +43,7 @@ import org.apache.slider.common.params.ActionKillContainerArgs; import org.apache.slider.common.params.ActionListArgs; import org.apache.slider.common.params.ActionRegistryArgs; import org.apache.slider.common.params.ActionResolveArgs; +import org.apache.slider.common.params.ActionResourceArgs; import org.apache.slider.common.params.ActionStatusArgs; import org.apache.slider.common.params.ActionThawArgs; import org.apache.slider.common.params.ActionUpgradeArgs; @@ -117,7 +118,6 @@ public interface SliderClientAPI extends Service { * @throws YarnException Yarn problems * @throws IOException other problems * @throws BadCommandArgumentsException bad arguments. - * @deprecated use #actionKeytab */ int actionKeytab(ActionKeytabArgs keytabInfo) throws YarnException, IOException; @@ -134,6 +134,17 @@ public interface SliderClientAPI extends Service { throws YarnException, IOException; /** + * Manage file resources leveraged by slider + * + * @param resourceInfo the arguments needed to manage the resource + * @throws YarnException Yarn problems + * @throws IOException other problems + * @throws BadCommandArgumentsException bad arguments. + */ + int actionResource(ActionResourceArgs resourceInfo) + throws YarnException, IOException; + + /** * Perform client operations such as install or configure * * @param clientInfo the arguments needed for client operations http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 05c7048..ba3effc 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -116,7 +116,8 @@ public interface SliderKeys extends SliderXmlConfKeys { String HISTORY_FILENAME_SUFFIX = "json"; String HISTORY_FILENAME_PREFIX = "rolehistory-"; String KEYTAB_DIR = "keytabs"; - + String RESOURCE_DIR = "resources"; + /** * Filename pattern is required to save in strict temporal order. * Important: older files must sort less-than newer files when using http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java new file mode 100644 index 0000000..60fcc87 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +@Parameters(commandNames = {SliderActions.ACTION_RESOURCE}, + commandDescription = SliderActions.DESCRIBE_ACTION_RESOURCE) + +public class ActionResourceArgs extends AbstractActionArgs { + + @Override + public String getActionName() { + return SliderActions.ACTION_RESOURCE; + } + + @Parameter(names = {ARG_INSTALL}, + description = "Install the resource(s)") + public boolean install; + + @Parameter(names = {ARG_DELETE}, + description = "Delete the file") + public boolean delete; + + @Parameter(names = {ARG_LIST}, + description = "List of installed files") + public boolean list; + + @Parameter(names = {ARG_RESOURCE}, + description = "Name of the file or directory") + public String resource; + + @Parameter(names = {ARG_DESTDIR}, + description = "The name of the folder in which to store the resources") + public String folder; + + @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing resource(s)") + public boolean overwrite = false; + + /** + * Get the min #of params expected + * @return the min number of params in the {@link #parameters} field + */ + public int getMinParams() { + return 0; + } + + @Override + public int getMaxParams() { + return 3; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index 0a8388d..aec4e26 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -104,6 +104,7 @@ public interface Arguments { String ARG_PROVIDER = "--provider"; String ARG_QUEUE = "--queue"; String ARG_REPLACE_PKG = "--replacepkg"; + String ARG_RESOURCE = "--resource"; String ARG_RESOURCES = "--resources"; String ARG_RES_COMP_OPT = "--rescompopt"; String ARG_RES_COMP_OPT_SHORT = "--rco"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java index 0a658ea..4016cc9 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java @@ -75,6 +75,7 @@ public class ClientArgs extends CommonArgs { private final ActionPackageArgs actionPackageArgs = new ActionPackageArgs(); private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs(); private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs(); + private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs(); private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs(); private final ActionThawArgs actionThawArgs = new ActionThawArgs(); private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs(); @@ -116,6 +117,7 @@ public class ClientArgs extends CommonArgs { actionPackageArgs, actionRegistryArgs, actionResolveArgs, + actionResourceArgs, actionStatusArgs, actionThawArgs, actionTokenArgs, @@ -227,6 +229,10 @@ public class ClientArgs extends CommonArgs { return actionResolveArgs; } + public ActionResourceArgs getActionResourceArgs() { + return actionResourceArgs; + } + public ActionStatusArgs getActionStatusArgs() { return actionStatusArgs; } @@ -346,6 +352,10 @@ public class ClientArgs extends CommonArgs { bindCoreAction(actionResolveArgs); break; + case ACTION_RESOURCE: + bindCoreAction(actionResourceArgs); + break; + case ACTION_STATUS: bindCoreAction(actionStatusArgs); break; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java index aab7c98..204ad9a 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java @@ -51,6 +51,7 @@ public interface SliderActions { String ACTION_RECONFIGURE = "reconfigure"; String ACTION_REGISTRY = "registry"; String ACTION_RESOLVE = "resolve"; + String ACTION_RESOURCE = "resource"; String ACTION_STATUS = "status"; String ACTION_THAW = "start"; String ACTION_TOKENS = "tokens"; @@ -106,6 +107,7 @@ public interface SliderActions { " Deprecated, use '" + ACTION_KEYTAB + " " + ClientArgs.ARG_INSTALL + "'."; String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory"; String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client"; + String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index 6a02367..aa5edf1 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -23,6 +23,7 @@ import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -64,6 +65,8 @@ public class CoreFileSystem { private static final Logger log = LoggerFactory.getLogger(CoreFileSystem.class); + private static final String UTF_8 = "UTF-8"; + protected final FileSystem fileSystem; protected final Configuration configuration; @@ -209,6 +212,55 @@ public class CoreFileSystem { } /** + * Build up the path string for resource install location -no attempt to + * create the directory is made + * + * @return the path for resource + */ + public Path buildResourcePath(String resourceFolder) { + Preconditions.checkNotNull(resourceFolder); + Path path = getBaseApplicationPath(); + return new Path(path, SliderKeys.RESOURCE_DIR + "/" + resourceFolder); + } + + /** + * Build up the path string for resource install location -no attempt to + * create the directory is made + * + * @return the path for resource + */ + public Path buildResourcePath(String dirName, String fileName) { + Preconditions.checkNotNull(dirName); + Preconditions.checkNotNull(fileName); + Path path = getBaseApplicationPath(); + return new Path(path, SliderKeys.RESOURCE_DIR + "/" + dirName + "/" + fileName); + } + + /** + * Build up the path string for cluster resource install location -no + * attempt to create the directory is made + * + * @return the path for resource + */ + public Path buildClusterResourcePath(String clusterName, String component) { + Preconditions.checkNotNull(clusterName); + Path path = buildClusterDirPath(clusterName); + return new Path(path, SliderKeys.RESOURCE_DIR + "/" + component); + } + + /** + * Build up the path string for cluster resource install location -no + * attempt to create the directory is made + * + * @return the path for resource + */ + public Path buildClusterResourcePath(String clusterName) { + Preconditions.checkNotNull(clusterName); + Path path = buildClusterDirPath(clusterName); + return new Path(path, SliderKeys.RESOURCE_DIR); + } + + /** * Create the Slider cluster path for a named cluster and all its subdirs * This is a directory; a mkdirs() operation is executed * to ensure that it is there. @@ -713,6 +765,17 @@ public class CoreFileSystem { fileSystem.setPermission(destPath, fp); } + public void copyHdfsFileToLocal(Path hdfsPath, File destFile) + throws IOException { + if (hdfsPath == null || destFile == null) { + throw new IOException("Either hdfsPath or destPath is null"); + } + log.info("Copying file {} to {}", hdfsPath.toUri(), destFile.toURI()); + + Path destPath = new Path(destFile.getPath()); + fileSystem.copyToLocalFile(hdfsPath, destPath); + } + /** * list entries in a filesystem directory * @@ -767,15 +830,36 @@ public class CoreFileSystem { } public void touch(Path path, boolean overwrite) throws IOException { - FSDataOutputStream out = fileSystem.create(path, overwrite); - out.close(); + FSDataOutputStream out = null; + try { + out = fileSystem.create(path, overwrite); + } finally { + IOUtils.closeStream(out); + } } public void cat(Path path, boolean overwrite, String data) throws IOException { - FSDataOutputStream out = fileSystem.create(path, overwrite); - byte[] bytes = data.getBytes(Charset.forName("UTF-8")); - out.write(bytes); - out.close(); + FSDataOutputStream out = null; + try { + out = fileSystem.create(path, overwrite); + byte[] bytes = data.getBytes(Charset.forName("UTF-8")); + out.write(bytes); + } finally { + IOUtils.closeStream(out); + } + } + + public String cat(Path path) throws IOException { + FileStatus status = fileSystem.getFileStatus(path); + byte[] b = new byte[(int) status.getLen()]; + FSDataInputStream in = null; + try { + in = fileSystem.open(path); + int count = in.read(b); + return new String(b, 0, count, UTF_8); + } finally { + IOUtils.closeStream(in); + } } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java index 9013edb..d24a158 100644 --- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java +++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java @@ -345,7 +345,7 @@ public class ConfTreeOperations { confTreeSerDeser.fromFile(resource) ); return ops; } - + /** * Build from an existing instance -which is cloned via JSON ser/deser * @param instance the source instance @@ -431,6 +431,20 @@ public class ConfTreeOperations { } /** + * Get a component opt as a boolean using {@link Boolean#valueOf(String)}. + * + * @param name component name + * @param option option name + * @param defVal default value + * @return parsed value + * @throws NumberFormatException if the role could not be parsed. + */ + public boolean getComponentOptBool(String name, String option, boolean defVal) { + String val = getComponentOpt(name, option, Boolean.toString(defVal)); + return Boolean.valueOf(val); + } + + /** * Set a component option, creating the component if necessary * @param component component name * @param option option name http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java index 0348828..5a3eb3d 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java @@ -50,6 +50,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; /** * Launcher of applications: base class @@ -71,6 +72,7 @@ public abstract class AbstractLauncher extends Configured { Records.newRecord(ContainerLaunchContext.class); protected final List<String> commands = new ArrayList<>(20); protected final Map<String, LocalResource> localResources = new HashMap<>(); + protected final Map<String, String> mountPaths = new HashMap<>(); private final Map<String, ByteBuffer> serviceData = new HashMap<>(); // security protected final Credentials credentials; @@ -131,8 +133,13 @@ public abstract class AbstractLauncher extends Configured { return localResources; } - public void addLocalResource(String subpath, LocalResource resource) { - localResources.put(subpath, resource); + public void addLocalResource(String subPath, LocalResource resource) { + localResources.put(subPath, resource); + } + + public void addLocalResource(String subPath, LocalResource resource, String mountPath) { + localResources.put(subPath, resource); + mountPaths.put(subPath, mountPath); } /** @@ -227,6 +234,16 @@ public abstract class AbstractLauncher extends Configured { env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker"); env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);//if yarnDockerMode, then dockerImage is set env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer); + StringBuilder sb = new StringBuilder(); + for (Entry<String,String> mount : mountPaths.entrySet()) { + if (sb.length() > 0) { + sb.append(","); + } + sb.append(mount.getKey()); + sb.append(":"); + sb.append(mount.getValue()); + } + env.put("YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS", sb.toString()); log.info("yarn docker env var has been set {}", containerLaunchContext.getEnvironment().toString()); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java index 8efaa5b..7fb3158 100644 --- a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java +++ b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java @@ -150,7 +150,11 @@ public class AppDefinitionPersister { pkgSrcDir.mkdirs(); File destMetaInfo = new File(pkgSrcDir, "metainfo.json"); if (isFileUsed) { - Files.copy(buildInfo.appMetaInfo, destMetaInfo); + if (buildInfo.appMetaInfo.getName().endsWith(".xml")) { + Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.xml")); + } else { + Files.copy(buildInfo.appMetaInfo, destMetaInfo); + } } else { Files.write( buildInfo.appMetaInfoJson.getBytes(Charset.forName("UTF-8")), http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java index 12581d7..ddab606 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java @@ -23,8 +23,10 @@ public enum ConfigFormat { JSON("json"), PROPERTIES("properties"), XML("xml"), + HADOOP_XML("hadoop-xml"), ENV("env"), -// YAML("yaml"); + TEMPLATE("template"), + YAML("yaml"), ; ConfigFormat(String suffix) { this.suffix = suffix; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java new file mode 100644 index 0000000..2e1615b --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.core.registry.docstore; + +import org.apache.hadoop.fs.Path; +import org.apache.slider.common.tools.SliderFileSystem; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ConfigUtils { + public static final String TEMPLATE_FILE = "template.file"; + + public static String replaceProps(Map<String, String> config, String content) { + Map<String, String> tokens = new HashMap<>(); + for (Entry<String, String> entry : config.entrySet()) { + tokens.put("${" + entry.getKey() + "}", entry.getValue()); + tokens.put("{{" + entry.getKey() + "}}", entry.getValue()); + } + String value = content; + for (Map.Entry<String,String> token : tokens.entrySet()) { + value = value.replaceAll(Pattern.quote(token.getKey()), + Matcher.quoteReplacement(token.getValue())); + } + return value; + } + + public static Map<String, String> replacePropsInConfig( + Map<String, String> config, Map<String, String> env) { + Map<String, String> tokens = new HashMap<>(); + for (Entry<String, String> entry : env.entrySet()) { + tokens.put("${" + entry.getKey() + "}", entry.getValue()); + } + Map<String, String> newConfig = new HashMap<>(); + for (Entry<String, String> entry : config.entrySet()) { + String value = entry.getValue(); + for (Map.Entry<String,String> token : tokens.entrySet()) { + value = value.replaceAll(Pattern.quote(token.getKey()), + Matcher.quoteReplacement(token.getValue())); + } + newConfig.put(entry.getKey(), entry.getValue()); + } + return newConfig; + } + + public static void prepConfigForTemplateOutputter(ConfigFormat configFormat, + Map<String, String> config, SliderFileSystem fileSystem, + String clusterName, String fileName) throws IOException { + if (!configFormat.equals(ConfigFormat.TEMPLATE)) { + return; + } + Path templateFile = null; + if (config.containsKey(TEMPLATE_FILE)) { + templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE)); + if (!fileSystem.isFile(templateFile)) { + templateFile = fileSystem.buildResourcePath(clusterName, + config.get(TEMPLATE_FILE)); + } + if (!fileSystem.isFile(templateFile)) { + throw new IOException("config specified template file " + config + .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist"); + } + } + if (templateFile == null && fileName != null) { + templateFile = fileSystem.buildResourcePath(fileName); + if (!fileSystem.isFile(templateFile)) { + templateFile = fileSystem.buildResourcePath(clusterName, + fileName); + } + } + if (fileSystem.isFile(templateFile)) { + config.put("content", fileSystem.cat(templateFile)); + } else { + config.put("content", ""); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java index 15ac207..9bdcfcb 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java @@ -24,9 +24,11 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.slider.common.tools.ConfigHelper; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.DumperOptions.FlowStyle; +import org.yaml.snakeyaml.Yaml; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.StringWriter; @@ -57,14 +59,7 @@ public abstract class PublishedConfigurationOutputter { } */ public void save(File dest) throws IOException { - FileOutputStream out = null; - try { - out = new FileOutputStream(dest); - save(out); - out.close(); - } finally { - org.apache.hadoop.io.IOUtils.closeStream(out); - } + FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8); } /** @@ -89,12 +84,13 @@ public abstract class PublishedConfigurationOutputter { * @param owner owning config * @return the outputter */ - + public static PublishedConfigurationOutputter createOutputter(ConfigFormat format, PublishedConfiguration owner) { Preconditions.checkNotNull(owner); switch (format) { case XML: + case HADOOP_XML: return new XmlOutputter(owner); case PROPERTIES: return new PropertiesOutputter(owner); @@ -102,11 +98,15 @@ public abstract class PublishedConfigurationOutputter { return new JsonOutputter(owner); case ENV: return new EnvOutputter(owner); + case TEMPLATE: + return new TemplateOutputter(owner); + case YAML: + return new YamlOutputter(owner); default: throw new RuntimeException("Unsupported format :" + format); } } - + public static class XmlOutputter extends PublishedConfigurationOutputter { @@ -131,7 +131,7 @@ public abstract class PublishedConfigurationOutputter { return configuration; } } - + public static class PropertiesOutputter extends PublishedConfigurationOutputter { private final Properties properties; @@ -146,15 +146,15 @@ public abstract class PublishedConfigurationOutputter { properties.store(out, ""); } - + public String asString() throws IOException { StringWriter sw = new StringWriter(); properties.store(sw, ""); return sw.toString(); } } - - + + public static class JsonOutputter extends PublishedConfigurationOutputter { public JsonOutputter(PublishedConfiguration owner) { @@ -162,11 +162,6 @@ public abstract class PublishedConfigurationOutputter { } @Override - public void save(File dest) throws IOException { - FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8); - } - - @Override public String asString() throws IOException { return owner.asJson(); } @@ -180,19 +175,36 @@ public abstract class PublishedConfigurationOutputter { } @Override - public void save(File dest) throws IOException { - FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8); - } - - @Override public String asString() throws IOException { if (!owner.entries.containsKey("content")) { throw new IOException("Configuration has no content field and cannot " + "be retrieved as type 'env'"); } - return owner.entries.get("content"); + String content = owner.entries.get("content"); + return ConfigUtils.replaceProps(owner.entries, content); } } + public static class TemplateOutputter extends EnvOutputter { + public TemplateOutputter(PublishedConfiguration owner) { + super(owner); + } + } + + public static class YamlOutputter extends PublishedConfigurationOutputter { + + private final Yaml yaml; + + public YamlOutputter(PublishedConfiguration owner) { + super(owner); + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(FlowStyle.BLOCK); + yaml = new Yaml(options); + } + + public String asString() throws IOException { + return yaml.dump(owner.entries); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java index fcab65e..510de5d 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java @@ -21,6 +21,7 @@ package org.apache.slider.providers; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTreeOperations; @@ -223,18 +224,22 @@ public abstract class AbstractClientProvider extends Configured { /** * Process client operations for applications such as install, configure * @param fileSystem + * @param registryOperations + * @param configuration * @param operation * @param clientInstallPath * @param clientPackage - * @param config + * @param clientConfig * @param name * @throws SliderException */ public void processClientOperation(SliderFileSystem fileSystem, + RegistryOperations registryOperations, + Configuration configuration, String operation, File clientInstallPath, File clientPackage, - JSONObject config, + JSONObject clientConfig, String name) throws SliderException { throw new SliderException("Provider does not support client operations."); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index f3dcd1d..4c6a50b 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -22,13 +22,17 @@ import com.google.common.io.Files; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.registry.client.binding.RegistryUtils; import org.apache.slider.api.InternalKeys; import org.apache.slider.api.ResourceKeys; +import org.apache.slider.client.ClientUtils; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; @@ -38,13 +42,18 @@ import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.AbstractLauncher; +import org.apache.slider.core.registry.docstore.PublishedConfiguration; import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderUtils; import org.apache.slider.providers.agent.application.metadata.Application; import org.apache.slider.providers.agent.application.metadata.Component; +import org.apache.slider.providers.agent.application.metadata.ConfigFile; import org.apache.slider.providers.agent.application.metadata.Metainfo; import org.apache.slider.providers.agent.application.metadata.MetainfoParser; +import org.apache.slider.providers.agent.application.metadata.OSPackage; +import org.apache.slider.providers.agent.application.metadata.OSSpecific; +import org.apache.slider.providers.agent.application.metadata.Package; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; @@ -53,7 +62,6 @@ import org.slf4j.LoggerFactory; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -291,6 +299,8 @@ public class AgentClientProvider extends AbstractClientProvider @Override public void processClientOperation(SliderFileSystem fileSystem, + RegistryOperations rops, + Configuration configuration, String operation, File clientInstallPath, File appPackage, @@ -319,51 +329,37 @@ public class AgentClientProvider extends AbstractClientProvider { ZipEntry zipEntry = zipInputStream.getNextEntry(); while (zipEntry != null) { - if ("metainfo.xml".equals(zipEntry.getName())) { - int size = (int) zipEntry.getSize(); - if (size != -1) { - log.info("Reading {} of size {}", zipEntry.getName(), - zipEntry.getSize()); - byte[] content = new byte[size]; - int offset = 0; - while (offset < size) { - offset += zipInputStream.read(content, offset, size - offset); - } - metaInfo = new MetainfoParser().fromXmlStream(new ByteArrayInputStream(content)); - } - } else if ("metainfo.json".equals(zipEntry.getName())) { - int size = (int) zipEntry.getSize(); - if (size != -1) { - log.info("Reading {} of size {}", zipEntry.getName(), - zipEntry.getSize()); - byte[] content = new byte[size]; - int offset = 0; - while (offset < size) { - offset += zipInputStream.read(content, offset, size - offset); + log.info("Processing {}", zipEntry.getName()); + String filePath = appPkgDir + File.separator + zipEntry.getName(); + if (!zipEntry.isDirectory()) { + log.info("Extracting file {}", filePath); + extractFile(zipInputStream, filePath); + + if ("metainfo.xml".equals(zipEntry.getName())) { + FileInputStream input = null; + try { + input = new FileInputStream(filePath); + metaInfo = new MetainfoParser().fromXmlStream(input); + } finally { + IOUtils.closeStream(input); } - metaInfo = new MetainfoParser().fromJsonStream(new ByteArrayInputStream(content)); - } - } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) { - int size = (int) zipEntry.getSize(); - if (size != -1) { - log.info("Reading {} of size {}", zipEntry.getName(), - zipEntry.getSize()); - byte[] content = new byte[size]; - int offset = 0; - while (offset < size) { - offset += zipInputStream.read(content, offset, size - offset); + } else if ("metainfo.json".equals(zipEntry.getName())) { + FileInputStream input = null; + try { + input = new FileInputStream(filePath); + metaInfo = new MetainfoParser().fromJsonStream(input); + } finally { + IOUtils.closeStream(input); } + } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) { try { - defaultConfig = new JSONObject(new String(content, Charset.defaultCharset())); + defaultConfig = new JSONObject(FileUtils.readFileToString(new File(filePath), Charset.defaultCharset())); } catch (JSONException jex) { throw new SliderException("Unable to read default client config.", jex); } } - } - String filePath = appPkgDir + File.separator + zipEntry.getName(); - if (!zipEntry.isDirectory()) { - extractFile(zipInputStream, filePath); } else { + log.info("Creating dir {}", filePath); File dir = new File(filePath); dir.mkdir(); } @@ -379,34 +375,109 @@ public class AgentClientProvider extends AbstractClientProvider throw new BadConfigException(E_COULD_NOT_READ_METAINFO); } - expandAgentTar(agentPkgDir); - - JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath, name); - FileWriter file = new FileWriter(new File(cmdDir, "command.json")); - try { - file.write(commandJson.toString()); - - } catch (IOException e) { - e.printStackTrace(); - } finally { - file.flush(); - file.close(); - } - - String client_script = null; + String clientScript = null; + String clientComponent = null; for (Component component : metaInfo.getApplication().getComponents()) { if (component.getCategory().equals("CLIENT")) { - client_script = component.getCommandScript().getScript(); - log.info("Installing CLIENT {} using script {}", component.getName(), client_script); + clientComponent = component.getName(); + if (component.getCommandScript() != null) { + clientScript = component.getCommandScript().getScript(); + } break; } } - if (SliderUtils.isUnset(client_script)) { - throw new SliderException("No valid CLIENT component found. Aborting install."); - } + if (SliderUtils.isUnset(clientScript)) { + log.info("Installing CLIENT without script"); + List<Package> packages = metaInfo.getApplication().getPackages(); + if (packages.size() > 0) { + // retrieve package resources from HDFS and extract + for (Package pkg : packages) { + Path pkgPath = fileSystem.buildResourcePath(pkg.getName()); + if (!fileSystem.isFile(pkgPath) && name != null) { + pkgPath = fileSystem.buildResourcePath(name, pkg.getName()); + } + if (!fileSystem.isFile(pkgPath)) { + throw new IOException("Package doesn't exist as a resource: " + + pkg.getName()); + } + if ("archive".equals(pkg.getType())) { + File pkgFile = new File(tmpDir, pkg.getName()); + fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile); + expandTar(pkgFile, clientInstallPath); + } else { + File pkgFile = new File(clientInstallPath, pkg.getName()); + fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile); + } + } + } else { + // extract tarball from app def + for (OSSpecific osSpecific : metaInfo.getApplication() + .getOSSpecifics()) { + for (OSPackage pkg : osSpecific.getPackages()) { + if ("tarball".equals(pkg.getType())) { + File pkgFile = new File(appPkgDir, pkg.getName()); + expandTar(pkgFile, clientInstallPath); + } + } + } + } + if (name == null) { + log.warn("Conf files not being generated because no app name was " + + "provided"); + return; + } + File confInstallDir; + String clientRoot = null; + if (config != null) { + try { + clientRoot = config.getJSONObject("global") + .getString(AgentKeys.APP_CLIENT_ROOT); + } catch (JSONException e) { + log.info("Couldn't read {} from provided client config, falling " + + "back on default", AgentKeys.APP_CLIENT_ROOT); + } + } + if (clientRoot == null && defaultConfig != null) { + try { + clientRoot = defaultConfig.getJSONObject("global") + .getString(AgentKeys.APP_CLIENT_ROOT); + } catch (JSONException e) { + log.info("Couldn't read {} from default client config, using {}", + AgentKeys.APP_CLIENT_ROOT, clientInstallPath); + } + } + if (clientRoot == null) { + confInstallDir = clientInstallPath; + } else { + confInstallDir = new File(new File(clientInstallPath, clientRoot), "conf"); + if (!confInstallDir.exists()) { + confInstallDir.mkdirs(); + } + } + String user = RegistryUtils.currentUser(); + for (ConfigFile configFile : metaInfo.getComponentConfigFiles(clientComponent)) { + retrieveConfigFile(rops, configuration, configFile, name, user, + confInstallDir); + } + } else { + log.info("Installing CLIENT using script {}", clientScript); + expandAgentTar(agentPkgDir); + + JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath, name); + FileWriter file = new FileWriter(new File(cmdDir, "command.json")); + try { + file.write(commandJson.toString()); + + } catch (IOException e) { + log.error("Couldn't write command.json to file"); + } finally { + file.flush(); + file.close(); + } - runCommand(appPkgDir, agentPkgDir, cmdDir, client_script); + runCommand(appPkgDir, agentPkgDir, cmdDir, clientScript); + } } catch (IOException ioex) { log.warn("Error while executing INSTALL command {}", ioex.getMessage()); @@ -481,6 +552,11 @@ public class AgentClientProvider extends AbstractClientProvider String libDirProp = System.getProperty(SliderKeys.PROPERTY_LIB_DIR); File tarFile = new File(libDirProp, SliderKeys.AGENT_TAR); + expandTar(tarFile, agentPkgDir); + } + + private void expandTar(File tarFile, File destDir) throws IOException { + log.info("Expanding tar {} to {}", tarFile, destDir); TarArchiveInputStream tarIn = new TarArchiveInputStream( new GzipCompressorInputStream( new BufferedInputStream( @@ -491,11 +567,14 @@ public class AgentClientProvider extends AbstractClientProvider try { TarArchiveEntry tarEntry = tarIn.getNextTarEntry(); while (tarEntry != null) { - File destPath = new File(agentPkgDir, tarEntry.getName()); + File destPath = new File(destDir, tarEntry.getName()); + File parent = destPath.getParentFile(); + if (!parent.exists()) { + parent.mkdirs(); + } if (tarEntry.isDirectory()) { destPath.mkdirs(); } else { - destPath.createNewFile(); byte[] byteToRead = new byte[1024]; BufferedOutputStream buffOut = new BufferedOutputStream(new FileOutputStream(destPath)); @@ -508,6 +587,9 @@ public class AgentClientProvider extends AbstractClientProvider buffOut.close(); } } + if ((tarEntry.getMode() & 0100) != 0) { + destPath.setExecutable(true); + } tarEntry = tarIn.getNextTarEntry(); } } finally { @@ -515,6 +597,17 @@ public class AgentClientProvider extends AbstractClientProvider } } + private void retrieveConfigFile(RegistryOperations rops, + Configuration configuration, ConfigFile configFile, String name, + String user, File destDir) throws IOException, SliderException { + log.info("Retrieving config {} to {}", configFile.getDictionaryName(), + destDir); + PublishedConfiguration published = ClientUtils.getConfigFromRegistry(rops, + configuration, configFile.getDictionaryName(), name, user, true); + ClientUtils.saveOrReturnConfig(published, configFile.getType(), + destDir, configFile.getFileName()); + } + protected JSONObject getCommandJson(JSONObject defaultConfig, JSONObject inputConfig, Metainfo metainfo, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index b027939..01a3f1a 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -48,6 +48,7 @@ public interface AgentKeys { */ String APP_HOME = "app.home"; String APP_ROOT = "site.global.app_root"; + String APP_CLIENT_ROOT = "client_root"; /** * Runas user of the application */ @@ -77,11 +78,16 @@ public interface AgentKeys { String APP_RESOURCES = "application.resources"; String APP_RESOURCES_DIR = "app/resources"; + String APP_CONF_DIR = "app/conf"; + String AGENT_INSTALL_DIR = "infra/agent"; String APP_DEFINITION_DIR = "app/definition"; String ADDON_DEFINITION_DIR = "addon/definition"; String AGENT_CONFIG_FILE = "infra/conf/agent.ini"; String AGENT_VERSION_FILE = "infra/version"; + String APP_PACKAGES_DIR = "app/packages"; + String PER_COMPONENT = "per.component"; + String PER_GROUP = "per.group"; String JAVA_HOME = "java_home"; String PACKAGE_LIST = "package_list"; @@ -97,6 +103,7 @@ public interface AgentKeys { String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt"; String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec"; String TEST_RELAX_VERIFICATION = "test.relax.validation"; + String AM_CONFIG_GENERATION = "am.config.generation"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index f20757a..4ffae7c 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ProtocolTypes; @@ -60,8 +61,11 @@ import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.CommandLineBuilder; import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.core.registry.docstore.ConfigFormat; +import org.apache.slider.core.registry.docstore.ConfigUtils; import org.apache.slider.core.registry.docstore.ExportEntry; import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; import org.apache.slider.core.registry.docstore.PublishedExports; import org.apache.slider.core.registry.info.CustomRegistryConstants; import org.apache.slider.providers.AbstractProviderService; @@ -124,7 +128,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Map.Entry; import java.util.Scanner; import java.util.Set; import java.util.TreeMap; @@ -132,7 +135,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.regex.Pattern; import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS; @@ -169,6 +171,7 @@ public class AgentProviderService extends AbstractProviderService implements private AgentClientProvider clientProvider; private AtomicInteger taskId = new AtomicInteger(0); private volatile Metainfo metaInfo = null; + private SliderFileSystem fileSystem = null; private Map<String, DefaultConfig> defaultConfigs = null; private ComponentCommandOrder commandOrder = null; private HeartbeatMonitor monitor; @@ -281,6 +284,7 @@ public class AgentProviderService extends AbstractProviderService implements if (metaInfo == null) { synchronized (syncLock) { if (metaInfo == null) { + this.fileSystem = fileSystem; readAndSetHeartbeatMonitoringInterval(instanceDefinition); initializeAgentDebugCommands(instanceDefinition); @@ -435,6 +439,26 @@ public class AgentProviderService extends AbstractProviderService implements LocalResourceType.ARCHIVE); launcher.addLocalResource(AgentKeys.APP_DEFINITION_DIR, appDefRes); + for (Package pkg : getMetaInfo().getApplication().getPackages()) { + Path pkgPath = fileSystem.buildResourcePath(pkg.getName()); + if (!fileSystem.isFile(pkgPath)) { + pkgPath = fileSystem.buildResourcePath(getClusterName(), + pkg.getName()); + } + if (!fileSystem.isFile(pkgPath)) { + throw new IOException("Package doesn't exist as a resource: " + + pkg.getName()); + } + log.info("Adding resource {}", pkg.getName()); + LocalResourceType type = LocalResourceType.FILE; + if ("archive".equals(pkg.getType())) { + type = LocalResourceType.ARCHIVE; + } + LocalResource packageResource = fileSystem.createAmResource( + pkgPath, type); + launcher.addLocalResource(AgentKeys.APP_PACKAGES_DIR, packageResource); + } + String agentConf = instanceDefinition.getAppConfOperations(). getGlobalOptions().getOption(AgentKeys.AGENT_CONF, ""); if (SliderUtils.isSet(agentConf)) { @@ -476,6 +500,15 @@ public class AgentProviderService extends AbstractProviderService implements generatedConfPath, SliderKeys.PROPAGATED_CONF_DIR_NAME)); + if (appComponent.getOptionBool(AgentKeys.AM_CONFIG_GENERATION, false)) { + // build and localize configuration files + Map<String, Map<String, String>> configurations = + buildCommandConfigurations(instanceDefinition.getAppConfOperations(), + container.getId().toString(), roleName, roleGroup); + localizeConfigFiles(launcher, roleName, roleGroup, getMetaInfo(), + configurations, launcher.getEnv(), fileSystem); + } + String label = getContainerLabel(container, roleName, roleGroup); CommandLineBuilder operation = new CommandLineBuilder(); @@ -646,19 +679,41 @@ public class AgentProviderService extends AbstractProviderService implements private Path uploadSecurityResource(File resource, SliderFileSystem fileSystem) throws IOException { Path certsDir = fileSystem.buildClusterSecurityDirPath(getClusterName()); - if (!fileSystem.getFileSystem().exists(certsDir)) { - fileSystem.getFileSystem().mkdirs(certsDir, + return uploadResource(resource, fileSystem, certsDir); + } + + private Path uploadResource(File resource, SliderFileSystem fileSystem, + String roleName) throws IOException { + Path dir; + if (roleName == null) { + dir = fileSystem.buildClusterResourcePath(getClusterName()); + } else { + dir = fileSystem.buildClusterResourcePath(getClusterName(), roleName); + } + return uploadResource(resource, fileSystem, dir); + } + + private static synchronized Path uploadResource(File resource, + SliderFileSystem fileSystem, Path parentDir) throws IOException { + if (!fileSystem.getFileSystem().exists(parentDir)) { + fileSystem.getFileSystem().mkdirs(parentDir, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); } - Path destPath = new Path(certsDir, resource.getName()); + Path destPath = new Path(parentDir, resource.getName()); if (!fileSystem.getFileSystem().exists(destPath)) { - FSDataOutputStream os = fileSystem.getFileSystem().create(destPath); - byte[] contents = FileUtils.readFileToByteArray(resource); - os.write(contents, 0, contents.length); - - os.flush(); - os.close(); + FSDataOutputStream os = null; + try { + os = fileSystem.getFileSystem().create(destPath); + byte[] contents = FileUtils.readFileToByteArray(resource); + os.write(contents, 0, contents.length); + os.flush(); + } finally { + IOUtils.closeStream(os); + } log.info("Uploaded {} to localization path {}", resource, destPath); + } else { + log.info("Resource {} already existed at localization path {}", resource, + destPath); } while (!fileSystem.getFileSystem().exists(destPath)) { @@ -718,6 +773,69 @@ public class AgentProviderService extends AbstractProviderService implements } } + private void createConfigFile(SliderFileSystem fileSystem, File file, + ConfigFile configFile, Map<String, String> config) + throws IOException { + ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType()); + log.info("Writing {} file {}", configFormat, file); + + ConfigUtils.prepConfigForTemplateOutputter(configFormat, config, + fileSystem, getClusterName(), file.getName()); + PublishedConfiguration publishedConfiguration = + new PublishedConfiguration(configFile.getDictionaryName(), + config.entrySet()); + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(configFormat, + publishedConfiguration); + configurationOutputter.save(file); + } + + @VisibleForTesting + protected void localizeConfigFiles(ContainerLauncher launcher, + String roleName, String roleGroup, + Metainfo metainfo, + Map<String, Map<String, String>> configs, + MapOperations env, + SliderFileSystem fileSystem) + throws IOException { + for (ConfigFile configFile : metainfo.getComponentConfigFiles(roleGroup)) { + Map<String, String> config = ConfigUtils.replacePropsInConfig( + configs.get(configFile.getDictionaryName()), env.options); + String fileName = ConfigUtils.replaceProps(config, + configFile.getFileName()); + File localFile = new File(SliderKeys.RESOURCE_DIR); + if (!localFile.exists()) { + localFile.mkdir(); + } + localFile = new File(localFile, new File(fileName).getName()); + + String folder = null; + if ("true".equals(config.get(PER_COMPONENT))) { + folder = roleName; + } else if ("true".equals(config.get(PER_GROUP))) { + folder = roleGroup; + } + + log.info("Localizing {} configs to config file {} (destination {}) " + + "based on {} configs", config.size(), localFile, fileName, + configFile.getDictionaryName()); + createConfigFile(fileSystem, localFile, configFile, config); + Path destPath = uploadResource(localFile, fileSystem, folder); + LocalResource configResource = fileSystem.createAmResource(destPath, + LocalResourceType.FILE); + + File destFile = new File(fileName); + if (destFile.isAbsolute()) { + launcher.addLocalResource( + SliderKeys.RESOURCE_DIR + "/" + destFile.getName(), + configResource, fileName); + } else { + launcher.addLocalResource(AgentKeys.APP_CONF_DIR + "/" + fileName, + configResource); + } + } + } + /** * build the zookeeper registry path. * @@ -1242,6 +1360,69 @@ public class AgentProviderService extends AbstractProviderService implements } catch (URISyntaxException e) { throw new IOException(e); } + + // identify client component + Component client = null; + for (Component component : getMetaInfo().getApplication().getComponents()) { + if (component.getCategory().equals("CLIENT")) { + client = component; + break; + } + } + if (client == null) { + log.info("No client component specified, not publishing client configs"); + return; + } + + // register AM-generated client configs + ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); + MapOperations clientOperations = appConf.getOrAddComponent(client.getName()); + appConf.resolve(); + if (!clientOperations.getOptionBool(AgentKeys.AM_CONFIG_GENERATION, + false)) { + log.info("AM config generation is false, not publishing client configs"); + return; + } + + // build and localize configuration files + Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); + Map<String, String> tokens = null; + try { + tokens = getStandardTokenMap(appConf, client.getName(), client.getName()); + } catch (SliderException e) { + throw new IOException(e); + } + + for (ConfigFile configFile : getMetaInfo() + .getComponentConfigFiles(client.getName())) { + addNamedConfiguration(configFile.getDictionaryName(), + appConf.getGlobalOptions().options, configurations, tokens, null, + client.getName()); + if (appConf.getComponent(client.getName()) != null) { + addNamedConfiguration(configFile.getDictionaryName(), + appConf.getComponent(client.getName()).options, configurations, + tokens, null, client.getName()); + } + } + + //do a final replacement of re-used configs + dereferenceAllConfigs(configurations); + + for (ConfigFile configFile : getMetaInfo() + .getComponentConfigFiles(client.getName())) { + ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType()); + + Map<String, String> config = configurations.get(configFile.getDictionaryName()); + ConfigUtils.prepConfigForTemplateOutputter(configFormat, config, + fileSystem, getClusterName(), + new File(configFile.getFileName()).getName()); + PublishedConfiguration publishedConfiguration = + new PublishedConfiguration(configFile.getDictionaryName(), + config.entrySet()); + getAmState().getPublishedSliderConfigurations().put( + configFile.getDictionaryName(), publishedConfiguration); + log.info("Publishing AM configuration {}", configFile.getDictionaryName()); + } } @Override @@ -1585,7 +1766,9 @@ public class AgentProviderService extends AbstractProviderService implements if (status.getConfigs() != null) { Application application = getMetaInfo().getApplication(); - if (canAnyMasterPublishConfig() == false || canPublishConfig(componentGroup)) { + if ((!canAnyMasterPublishConfig() || canPublishConfig(componentGroup)) && + !getAmState().getAppConfSnapshot().getComponentOptBool( + componentGroup, AgentKeys.AM_CONFIG_GENERATION, false)) { // If no Master can explicitly publish then publish if its a master // Otherwise, wait till the master that can publish is ready @@ -1709,7 +1892,11 @@ public class AgentProviderService extends AbstractProviderService implements simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue()); } } - publishApplicationInstanceData(groupName, groupName, simpleEntries.entrySet()); + if (!getAmState().getAppConfSnapshot().getComponentOptBool( + groupName, AgentKeys.AM_CONFIG_GENERATION, false)) { + publishApplicationInstanceData(groupName, groupName, + simpleEntries.entrySet()); + } PublishedExports exports = new PublishedExports(groupName); exports.setUpdated(new Date().getTime()); @@ -2036,7 +2223,7 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setConfigurations(configurations); Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf); cmd.setComponentConfigurations(componentConfigurations); - + if (SliderUtils.isSet(scriptPath)) { cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false)); } else { @@ -2154,10 +2341,10 @@ public class AgentProviderService extends AbstractProviderService implements List<String> packages = new ArrayList<>(); if (application != null) { if (application.getPackages().size() > 0) { - List<Package> appPackages = application.getPackages(); - for (Package appPackage : appPackages) { - packages.add(String.format(pkgFormatString, appPackage.getType(), appPackage.getName())); - } + // no-op if there are packages that are not OS-specific, as these + // will be localized by AM rather than the Agent + // this should be backwards compatible, as there was previously an + // XML parsing bug that ensured non-OS-specific packages did not exist } else { List<OSSpecific> osSpecifics = application.getOSSpecifics(); if (osSpecifics != null && osSpecifics.size() > 0) { @@ -2821,14 +3008,41 @@ public class AgentProviderService extends AbstractProviderService implements } } + boolean finished = false; + while (!finished) { + finished = true; + for (Map.Entry<String, String> entry : allConfigs.entrySet()) { + String configValue = entry.getValue(); + for (Map.Entry<String, String> lookUpEntry : allConfigs.entrySet()) { + String lookUpValue = lookUpEntry.getValue(); + if (lookUpValue.contains("${@//site/")) { + continue; + } + String lookUpKey = lookUpEntry.getKey(); + if (configValue != null && configValue.contains(lookUpKey)) { + configValue = configValue.replace(lookUpKey, lookUpValue); + } + } + if (!configValue.equals(entry.getValue())) { + finished = false; + allConfigs.put(entry.getKey(), configValue); + } + } + } + for (String configType : configurations.keySet()) { Map<String, String> configBucket = configurations.get(configType); for (Map.Entry<String, String> entry: configBucket.entrySet()) { String configName = entry.getKey(); String configValue = entry.getValue(); - for (String lookUpKey : allConfigs.keySet()) { + for (Map.Entry<String, String> lookUpEntry : allConfigs.entrySet()) { + String lookUpValue = lookUpEntry.getValue(); + if (lookUpValue.contains("${@//site/")) { + continue; + } + String lookUpKey = lookUpEntry.getKey(); if (configValue != null && configValue.contains(lookUpKey)) { - configValue = configValue.replace(lookUpKey, allConfigs.get(lookUpKey)); + configValue = configValue.replace(lookUpKey, lookUpValue); } } configBucket.put(configName, configValue); @@ -2974,6 +3188,7 @@ public class AgentProviderService extends AbstractProviderService implements config.put("app_log_dir", "${AGENT_LOG_ROOT}"); config.put("app_pid_dir", "${AGENT_WORK_ROOT}/app/run"); config.put("app_install_dir", "${AGENT_WORK_ROOT}/app/install"); + config.put("app_conf_dir", "${AGENT_WORK_ROOT}/" + AgentKeys.APP_CONF_DIR); config.put("app_input_conf_dir", "${AGENT_WORK_ROOT}/" + SliderKeys.PROPAGATED_CONF_DIR_NAME); config.put("app_container_id", containerId); config.put("app_container_tag", tags.getTag(roleName, containerId)); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java index 1b63b58..b6ae4de 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java @@ -65,6 +65,10 @@ public abstract class AbstractComponent implements Validate { this.commands = commands; } + public void addCommand(ComponentCommand command) { + commands.add(command); + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("{"); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java index 63546a4..5556c7f 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java @@ -116,6 +116,10 @@ public class Application extends AbstractMetainfoSchema { return commandOrders; } + public void addPackage(Package pkg) { + packages.add(pkg); + } + @JsonProperty("packages") public List<Package> getPackages() { return packages; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java index 3f23455..78bb8c1 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java @@ -38,7 +38,8 @@ public class Component extends AbstractComponent { String type = TYPE_STANDARD; List<ComponentExport> componentExports = new ArrayList<>(); List<DockerContainer> dockerContainers = new ArrayList<>(); - + List<ConfigFile> configFiles = new ArrayList<>(); + public Component() { } @@ -155,6 +156,15 @@ public class Component extends AbstractComponent { return Boolean.parseBoolean(this.autoStartOnFailure); } + public void addConfigFile(ConfigFile configFile) { + this.configFiles.add(configFile); + } + + @JsonProperty("configFiles") + public List<ConfigFile> getConfigFiles() { + return configFiles; + } + @Override public String toString() { final StringBuilder sb =
