Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-1107_AM_config_generation [created] b1cc78a63
SLIDER-1107 initial implementation of AM config generation Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/84fee8a3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/84fee8a3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/84fee8a3 Branch: refs/heads/feature/SLIDER-1107_AM_config_generation Commit: 84fee8a30e4626ef50ba7a44fc4283d12603c389 Parents: 1b833e5 Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Fri Apr 29 12:59:25 2016 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Fri Apr 29 12:59:25 2016 -0700 ---------------------------------------------------------------------- .../src/main/python/agent/YarnDockerManager.py | 37 +--- .../python/resource_management/core/shell.py | 20 +-- .../python/scripts/shell_cmd/basic_installer.py | 11 +- .../org/apache/slider/client/SliderClient.java | 110 +++++++++++- .../apache/slider/client/SliderClientAPI.java | 13 +- .../org/apache/slider/common/SliderKeys.java | 3 +- .../common/params/ActionResourceArgs.java | 68 +++++++ .../apache/slider/common/params/Arguments.java | 1 + .../apache/slider/common/params/ClientArgs.java | 10 ++ .../slider/common/params/SliderActions.java | 2 + .../slider/common/tools/CoreFileSystem.java | 60 +++++++ .../slider/core/launch/AbstractLauncher.java | 17 ++ .../core/persist/AppDefinitionPersister.java | 6 +- .../core/registry/docstore/ConfigFormat.java | 2 + .../core/registry/docstore/ConfigUtils.java | 59 +++++++ .../docstore/PublishedConfiguration.java | 23 ++- .../PublishedConfigurationOutputter.java | 75 +++++++- .../slider/providers/agent/AgentKeys.java | 3 + .../providers/agent/AgentProviderService.java | 175 +++++++++++++++++-- .../application/metadata/AbstractComponent.java | 4 + .../agent/application/metadata/Application.java | 4 + .../agent/application/metadata/Component.java | 12 +- .../agent/application/metadata/Metainfo.java | 13 ++ .../application/metadata/MetainfoParser.java | 19 +- .../agent/TestAgentProviderService.java | 62 +++++-- .../metadata/TestMetainfoParser.java | 13 +- .../agent/application/metadata/metainfo.xml | 21 +++ 27 files changed, 739 insertions(+), 104 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-agent/src/main/python/agent/YarnDockerManager.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/YarnDockerManager.py b/slider-agent/src/main/python/agent/YarnDockerManager.py index abdb2fb..4ae19ab 100644 --- a/slider-agent/src/main/python/agent/YarnDockerManager.py +++ b/slider-agent/src/main/python/agent/YarnDockerManager.py @@ -55,8 +55,7 @@ class YarnDockerManager(Script): self.container_id = command['hostLevelParams']['container_id'] if command['roleCommand'] == 'INSTALL': - with Environment(self.workroot) as env: - self.install_container(command, env) + # do nothing returncode = 0; out = ''; err = '' if command['roleCommand'] == 'START': returncode, out, err = self.start_container(command) @@ -130,40 +129,6 @@ class YarnDockerManager(Script): return command['configurations']['global'][keyName] return None - def install_container(self, command, env): - try: - configFiles = self.extract_config_files_from_command(command) - for configFile in configFiles: - properties = self.extract_config_file_properties_from_command(command, configFile) - fileName = configFile['fileName'] - dir = self.get_config_file_global(command, configFile, 'destDir') - if dir is None: - dir = self.workroot - logger.info("creating config file " + str(configFile) + " in directory "+str(dir)) - Directory(dir, recursive = True) - if configFile['type'] == 'properties': - PropertiesFile(fileName, - dir=dir, - properties=properties) - elif configFile['type'] == 'env': - content = self.get_config_file_global(command, configFile, 'content', useEnv=False) - if content is not None: - File(os.path.join(dir, fileName), - content=InlineTemplate(content, **properties)) - elif configFile['type'] == 'template': - templateFile = self.get_config_file_global(command, configFile, 'templateFile') - if templateFile is not None: - with open(templateFile,"r") as fp: - fileContent = fp.read() - File(os.path.join(dir, fileName), - content=InlineTemplate(fileContent, **properties)) - elif configFile['type'] == 'xml': - XmlConfig(fileName, - conf_dir=dir, - configurations=properties) - except: - traceback.print_exc() - def start_container(self, command): #extracting param needed by docker run from the command passed from AM startCommand = self.extract_config_from_command(command, 'docker.startCommand') http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-agent/src/main/python/resource_management/core/shell.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/resource_management/core/shell.py b/slider-agent/src/main/python/resource_management/core/shell.py index f21dbbf..e717ae4 100644 --- a/slider-agent/src/main/python/resource_management/core/shell.py +++ b/slider-agent/src/main/python/resource_management/core/shell.py @@ -31,10 +31,8 @@ from exceptions import ExecuteTimeoutException from resource_management.core.logger import Logger import time -APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application-' -APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = '.log' -APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application-' -APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = '.err' +APPLICATION_STD_OUTPUT_LOG_FILE = 'application.log' +APPLICATION_STD_ERROR_LOG_FILE = 'application.err' def checked_call(command, logoutput=False, cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file=None, poll_after=None): @@ -69,15 +67,9 @@ def _call(command, logoutput=False, throw_on_failure=True, """ command = ["/bin/bash","--login","-c", command] #adding redirecting stdout stderr to file - outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \ - str(pid_file_name) + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE - - errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \ - str(pid_file_name) + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE - - stdoutFile = open(outfilename, 'w') - stderrFile = open(errfilename, 'w') - + stdoutFile = open(APPLICATION_STD_OUTPUT_LOG_FILE, 'w+') + stderrFile = open(APPLICATION_STD_ERROR_LOG_FILE, 'w+') + proc = subprocess.Popen(command, stdout = stdoutFile, stderr = stderrFile, universal_newlines = True, cwd=cwd, env=env, shell=False, preexec_fn=preexec_fn) @@ -142,4 +134,4 @@ def quote_bash_args(command): for char in command: if char not in valid: return "'" + command.replace("'", "'\"'\"'") + "'" - return command \ No newline at end of file + return command http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py b/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py index 561fd6c..df9b6f0 100644 --- a/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py +++ b/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py @@ -35,16 +35,15 @@ class BasicInstaller(Script): self.configure(env) process_cmd = format("{cmd}") - Execute(process_cmd, - logoutput=False, - wait_for_finish=False, - pid_file=params.pid_file, - poll_after = 5 - ) + Execute(process_cmd) def stop(self, env): import params env.set_params(params) + self.configure(env) + process_cmd = format("{cmd}") + + Execute(process_cmd) def status(self, env): import params http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index dd90e46..f332a66 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -101,6 +101,7 @@ import org.apache.slider.common.params.ActionNodesArgs; import org.apache.slider.common.params.ActionPackageArgs; import org.apache.slider.common.params.ActionRegistryArgs; import org.apache.slider.common.params.ActionResolveArgs; +import org.apache.slider.common.params.ActionResourceArgs; import org.apache.slider.common.params.ActionStatusArgs; import org.apache.slider.common.params.ActionThawArgs; import org.apache.slider.common.params.ActionTokensArgs; @@ -177,7 +178,6 @@ import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.File; -import java.io.FilenameFilter; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -415,7 +415,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe case ACTION_INSTALL_PACKAGE: exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs()); break; - + case ACTION_KEYTAB: exitCode = actionKeytab(serviceArgs.getActionKeytabArgs()); break; @@ -443,7 +443,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe case ACTION_RESOLVE: exitCode = actionResolve(serviceArgs.getActionResolveArgs()); break; - + + case ACTION_RESOURCE: + exitCode = actionResource(serviceArgs.getActionResourceArgs()); + break; + case ACTION_STATUS: exitCode = actionStatus(clusterName, serviceArgs.getActionStatusArgs()); break; @@ -1029,7 +1033,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe Path fileInFs = new Path(pkgPath, keytabInfo.keytab ); log.info("Deleting keytab {}", fileInFs); FileSystem sfs = sliderFileSystem.getFileSystem(); - require(sfs.exists(fileInFs), "No keytab to delete found at %s", fileInFs.toUri()); + require(sfs.exists(fileInFs), "No keytab to delete found at %s", + fileInFs.toUri()); sfs.delete(fileInFs, false); return EXIT_SUCCESS; @@ -1105,6 +1110,103 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } @Override + public int actionResource(ActionResourceArgs resourceInfo) + throws YarnException, IOException { + if (resourceInfo.install) { + return actionInstallResource(resourceInfo); + } else if (resourceInfo.delete) { + return actionDeleteResource(resourceInfo); + } else if (resourceInfo.list) { + return actionListResource(resourceInfo); + } else { + throw new BadCommandArgumentsException( + "Resource option specified not found.\n" + + CommonArgs.usage(serviceArgs, ACTION_RESOURCE)); + } + } + + private int actionListResource(ActionResourceArgs resourceInfo) throws IOException { + String folder = resourceInfo.folder != null ? resourceInfo.folder : StringUtils.EMPTY; + Path path = sliderFileSystem.buildResourcePath(folder); + RemoteIterator<LocatedFileStatus> files = + sliderFileSystem.getFileSystem().listFiles(path, true); + log.info("Resources:"); + while (files.hasNext()) { + log.info("\t" + files.next().getPath().toString()); + } + + return EXIT_SUCCESS; + } + + private int actionDeleteResource(ActionResourceArgs resourceInfo) + throws BadCommandArgumentsException, IOException { + if (StringUtils.isEmpty(resourceInfo.resource)) { + throw new BadCommandArgumentsException("A file name is required."); + } + + Path fileInFs; + if (resourceInfo.folder == null) { + fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.resource); + } else { + fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.folder, + resourceInfo.resource); + } + + log.info("Deleting resource {}", fileInFs); + FileSystem sfs = sliderFileSystem.getFileSystem(); + require(sfs.exists(fileInFs), "No resource to delete found at %s", fileInFs.toUri()); + sfs.delete(fileInFs, true); + + return EXIT_SUCCESS; + } + + private int actionInstallResource(ActionResourceArgs resourceInfo) + throws BadCommandArgumentsException, IOException { + Path srcFile = null; + String folder = resourceInfo.folder != null ? resourceInfo.folder : StringUtils.EMPTY; + + requireArgumentSet(Arguments.ARG_RESOURCE, resourceInfo.resource); + File file = new File(resourceInfo.resource); + require(file.isFile() || file.isDirectory(), + "Unable to access supplied file at %s", file.getAbsolutePath()); + + File[] files; + if (file.isDirectory()) { + files = file.listFiles(); + } else { + files = new File[] { file }; + } + + Path pkgPath = sliderFileSystem.buildResourcePath(folder); + FileSystem sfs = sliderFileSystem.getFileSystem(); + + if (!sfs.exists(pkgPath)) { + sfs.mkdirs(pkgPath); + sfs.setPermission(pkgPath, new FsPermission( + FsAction.ALL, FsAction.NONE, FsAction.NONE)); + } else { + require(sfs.isDirectory(pkgPath), "Specified folder %s exists and is " + + "not a directory", folder); + } + + for (File f : files) { + srcFile = new Path(f.toURI()); + + Path fileInFs = new Path(pkgPath, srcFile.getName()); + log.info("Installing file {} at {} and overwrite is {}.", + srcFile, fileInFs, resourceInfo.overwrite); + require(!(sfs.exists(fileInFs) && !resourceInfo.overwrite), + "File exists at %s. Use --overwrite to overwrite.", fileInFs.toUri()); + + sfs.copyFromLocalFile(false, resourceInfo.overwrite, srcFile, fileInFs); + sfs.setPermission(fileInFs, + new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE)); + } + + return EXIT_SUCCESS; + } + + @Override public int actionClient(ActionClientArgs clientInfo) throws YarnException, IOException { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index 5c5d96b..30f6ba9 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -43,6 +43,7 @@ import org.apache.slider.common.params.ActionKillContainerArgs; import org.apache.slider.common.params.ActionListArgs; import org.apache.slider.common.params.ActionRegistryArgs; import org.apache.slider.common.params.ActionResolveArgs; +import org.apache.slider.common.params.ActionResourceArgs; import org.apache.slider.common.params.ActionStatusArgs; import org.apache.slider.common.params.ActionThawArgs; import org.apache.slider.common.params.ActionUpgradeArgs; @@ -117,7 +118,6 @@ public interface SliderClientAPI extends Service { * @throws YarnException Yarn problems * @throws IOException other problems * @throws BadCommandArgumentsException bad arguments. - * @deprecated use #actionKeytab */ int actionKeytab(ActionKeytabArgs keytabInfo) throws YarnException, IOException; @@ -134,6 +134,17 @@ public interface SliderClientAPI extends Service { throws YarnException, IOException; /** + * Manage file resources leveraged by slider + * + * @param resourceInfo the arguments needed to manage the resource + * @throws YarnException Yarn problems + * @throws IOException other problems + * @throws BadCommandArgumentsException bad arguments. + */ + int actionResource(ActionResourceArgs resourceInfo) + throws YarnException, IOException; + + /** * Perform client operations such as install or configure * * @param clientInfo the arguments needed for client operations http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 1d2d5f8..e06c243 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -116,7 +116,8 @@ public interface SliderKeys extends SliderXmlConfKeys { String HISTORY_FILENAME_SUFFIX = "json"; String HISTORY_FILENAME_PREFIX = "rolehistory-"; String KEYTAB_DIR = "keytabs"; - + String RESOURCE_DIR = "resources"; + /** * Filename pattern is required to save in strict temporal order. * Important: older files must sort less-than newer files when using http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java new file mode 100644 index 0000000..60fcc87 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionResourceArgs.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +@Parameters(commandNames = {SliderActions.ACTION_RESOURCE}, + commandDescription = SliderActions.DESCRIBE_ACTION_RESOURCE) + +public class ActionResourceArgs extends AbstractActionArgs { + + @Override + public String getActionName() { + return SliderActions.ACTION_RESOURCE; + } + + @Parameter(names = {ARG_INSTALL}, + description = "Install the resource(s)") + public boolean install; + + @Parameter(names = {ARG_DELETE}, + description = "Delete the file") + public boolean delete; + + @Parameter(names = {ARG_LIST}, + description = "List of installed files") + public boolean list; + + @Parameter(names = {ARG_RESOURCE}, + description = "Name of the file or directory") + public String resource; + + @Parameter(names = {ARG_DESTDIR}, + description = "The name of the folder in which to store the resources") + public String folder; + + @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing resource(s)") + public boolean overwrite = false; + + /** + * Get the min #of params expected + * @return the min number of params in the {@link #parameters} field + */ + public int getMinParams() { + return 0; + } + + @Override + public int getMaxParams() { + return 3; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index bac20d7..63c75ee 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -103,6 +103,7 @@ public interface Arguments { String ARG_PROVIDER = "--provider"; String ARG_QUEUE = "--queue"; String ARG_REPLACE_PKG = "--replacepkg"; + String ARG_RESOURCE = "--resource"; String ARG_RESOURCES = "--resources"; String ARG_RES_COMP_OPT = "--rescompopt"; String ARG_RES_COMP_OPT_SHORT = "--rco"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java index 0a658ea..4016cc9 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java @@ -75,6 +75,7 @@ public class ClientArgs extends CommonArgs { private final ActionPackageArgs actionPackageArgs = new ActionPackageArgs(); private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs(); private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs(); + private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs(); private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs(); private final ActionThawArgs actionThawArgs = new ActionThawArgs(); private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs(); @@ -116,6 +117,7 @@ public class ClientArgs extends CommonArgs { actionPackageArgs, actionRegistryArgs, actionResolveArgs, + actionResourceArgs, actionStatusArgs, actionThawArgs, actionTokenArgs, @@ -227,6 +229,10 @@ public class ClientArgs extends CommonArgs { return actionResolveArgs; } + public ActionResourceArgs getActionResourceArgs() { + return actionResourceArgs; + } + public ActionStatusArgs getActionStatusArgs() { return actionStatusArgs; } @@ -346,6 +352,10 @@ public class ClientArgs extends CommonArgs { bindCoreAction(actionResolveArgs); break; + case ACTION_RESOURCE: + bindCoreAction(actionResourceArgs); + break; + case ACTION_STATUS: bindCoreAction(actionStatusArgs); break; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java index aab7c98..e209cdc 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java @@ -51,6 +51,7 @@ public interface SliderActions { String ACTION_RECONFIGURE = "reconfigure"; String ACTION_REGISTRY = "registry"; String ACTION_RESOLVE = "resolve"; + String ACTION_RESOURCE = "resource"; String ACTION_STATUS = "status"; String ACTION_THAW = "start"; String ACTION_TOKENS = "tokens"; @@ -106,6 +107,7 @@ public interface SliderActions { " Deprecated, use '" + ACTION_KEYTAB + " " + ClientArgs.ARG_INSTALL + "'."; String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory"; String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client"; + String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the sub-folder 'resources' of the user's Slider base directory"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index 6a02367..0d3436b 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -23,6 +23,7 @@ import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -64,6 +65,8 @@ public class CoreFileSystem { private static final Logger log = LoggerFactory.getLogger(CoreFileSystem.class); + private static final String UTF_8 = "UTF-8"; + protected final FileSystem fileSystem; protected final Configuration configuration; @@ -209,6 +212,55 @@ public class CoreFileSystem { } /** + * Build up the path string for resource install location -no attempt to + * create the directory is made + * + * @return the path for resource + */ + public Path buildResourcePath(String resourceFolder) { + Preconditions.checkNotNull(resourceFolder); + Path path = getBaseApplicationPath(); + return new Path(path, SliderKeys.RESOURCE_DIR + "/" + resourceFolder); + } + + /** + * Build up the path string for resource install location -no attempt to + * create the directory is made + * + * @return the path for resource + */ + public Path buildResourcePath(String dirName, String fileName) { + Preconditions.checkNotNull(dirName); + Preconditions.checkNotNull(fileName); + Path path = getBaseApplicationPath(); + return new Path(path, SliderKeys.RESOURCE_DIR + "/" + dirName + "/" + fileName); + } + + /** + * Build up the path string for cluster resource install location -no + * attempt to create the directory is made + * + * @return the path for resource + */ + public Path buildClusterResourcePath(String clusterName, String component) { + Preconditions.checkNotNull(clusterName); + Path path = buildClusterDirPath(clusterName); + return new Path(path, SliderKeys.RESOURCE_DIR + "/" + component); + } + + /** + * Build up the path string for cluster resource install location -no + * attempt to create the directory is made + * + * @return the path for resource + */ + public Path buildClusterResourcePath(String clusterName) { + Preconditions.checkNotNull(clusterName); + Path path = buildClusterDirPath(clusterName); + return new Path(path, SliderKeys.RESOURCE_DIR); + } + + /** * Create the Slider cluster path for a named cluster and all its subdirs * This is a directory; a mkdirs() operation is executed * to ensure that it is there. @@ -778,6 +830,14 @@ public class CoreFileSystem { out.close(); } + public String cat(Path path) throws IOException { + FileStatus status = fileSystem.getFileStatus(path); + byte[] b = new byte[(int) status.getLen()]; + FSDataInputStream in = fileSystem.open(path); + int count = in.read(b); + return new String(b, 0, count, UTF_8); + } + /** * Create a path that must exist in the cluster fs * @param uri uri to create http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java index 0348828..efc58b4 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java @@ -50,6 +50,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; /** * Launcher of applications: base class @@ -71,6 +72,7 @@ public abstract class AbstractLauncher extends Configured { Records.newRecord(ContainerLaunchContext.class); protected final List<String> commands = new ArrayList<>(20); protected final Map<String, LocalResource> localResources = new HashMap<>(); + protected final Map<String, String> mountPaths = new HashMap<>(); private final Map<String, ByteBuffer> serviceData = new HashMap<>(); // security protected final Credentials credentials; @@ -135,6 +137,11 @@ public abstract class AbstractLauncher extends Configured { localResources.put(subpath, resource); } + public void addLocalResource(String subpath, LocalResource resource, String mountPath) { + localResources.put(subpath, resource); + mountPaths.put(subpath, mountPath); + } + /** * Add a set of local resources * @param resourceMap map of name:resource to add @@ -227,6 +234,16 @@ public abstract class AbstractLauncher extends Configured { env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker"); env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);//if yarnDockerMode, then dockerImage is set env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer); + StringBuilder sb = new StringBuilder(); + for (Entry<String,String> mount : mountPaths.entrySet()) { + if (sb.length() > 0) { + sb.append(","); + } + sb.append(mount.getKey()); + sb.append(":"); + sb.append(mount.getValue()); + } + env.put("YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS", sb.toString()); log.info("yarn docker env var has been set {}", containerLaunchContext.getEnvironment().toString()); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java index 2448c72..e96baef 100644 --- a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java +++ b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java @@ -125,7 +125,11 @@ public class AppDefinitionPersister { File tempDir = Files.createTempDir(); File pkgSrcDir = new File(tempDir, "default"); pkgSrcDir.mkdirs(); - Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.json")); + if (buildInfo.appMetaInfo.getName().endsWith(".xml")) { + Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.xml")); + } else { + Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.json")); + } Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername); log.info("Using default app def path {}", appDirPath.toString()); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java index 12581d7..9e0c095 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java @@ -23,7 +23,9 @@ public enum ConfigFormat { JSON("json"), PROPERTIES("properties"), XML("xml"), + HADOOP_XML("hadoop-xml"), ENV("env"), + TEMPLATE("template"), // YAML("yaml"); ; ConfigFormat(String suffix) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java new file mode 100644 index 0000000..7e6ee5a --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.core.registry.docstore; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ConfigUtils { + public static String replaceProps(Map<String, String> config, String content) { + Map<String, String> tokens = new HashMap<>(); + for (Entry<String, String> entry : config.entrySet()) { + tokens.put("${" + entry.getKey() + "}", entry.getValue()); + tokens.put("{{" + entry.getKey() + "}}", entry.getValue()); + } + String value = content; + for (Map.Entry<String,String> token : tokens.entrySet()) { + value = value.replaceAll(Pattern.quote(token.getKey()), + Matcher.quoteReplacement(token.getValue())); + } + return value; + } + + public static Map<String, String> replacePropsInConfig( + Map<String, String> config, Map<String, String> env) { + Map<String, String> tokens = new HashMap<>(); + for (Entry<String, String> entry : env.entrySet()) { + tokens.put("${" + entry.getKey() + "}", entry.getValue()); + } + Map<String, String> newConfig = new HashMap<>(); + for (Entry<String, String> entry : config.entrySet()) { + String value = entry.getValue(); + for (Map.Entry<String,String> token : tokens.entrySet()) { + value = value.replaceAll(Pattern.quote(token.getKey()), + Matcher.quoteReplacement(token.getValue())); + } + newConfig.put(entry.getKey(), entry.getValue()); + } + return newConfig; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java index 50b522f..cabdce6 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java @@ -20,6 +20,7 @@ package org.apache.slider.core.registry.docstore; import org.apache.hadoop.conf.Configuration; import org.apache.slider.common.tools.ConfigHelper; +import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.exceptions.BadConfigException; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.ObjectMapper; @@ -49,6 +50,9 @@ public class PublishedConfiguration { public Map<String, String> entries = new HashMap<>(); + public SliderFileSystem fileSystem; + public String clusterName; + public PublishedConfiguration() { } @@ -85,7 +89,24 @@ public class PublishedConfiguration { putValues(ConfigHelper.resolveConfiguration(keysource, valuesource)); } - + /** + * Build a configuration from the entries + * @param description configuration description + * @param entries entries to put + * @param fileSystem Slider file system (source of configuration templates) + * @param clusterName cluster name + */ + public PublishedConfiguration(String description, + Iterable<Map.Entry<String, String>> entries, + SliderFileSystem fileSystem, + String clusterName) { + this.description = description; + putValues(entries); + this.fileSystem = fileSystem; + this.clusterName = clusterName; + } + + /** * Is the configuration empty. This means either that it has not * been given any values, or it is stripped down copy set down over the http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java index 15ac207..bae9674 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java @@ -23,13 +23,17 @@ import com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.slider.common.tools.ConfigHelper; +import org.apache.slider.common.tools.SliderFileSystem; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.StringWriter; +import java.util.Collections; +import java.util.Map; import java.util.Properties; /** @@ -89,12 +93,13 @@ public abstract class PublishedConfigurationOutputter { * @param owner owning config * @return the outputter */ - + public static PublishedConfigurationOutputter createOutputter(ConfigFormat format, PublishedConfiguration owner) { Preconditions.checkNotNull(owner); switch (format) { case XML: + case HADOOP_XML: return new XmlOutputter(owner); case PROPERTIES: return new PropertiesOutputter(owner); @@ -102,11 +107,13 @@ public abstract class PublishedConfigurationOutputter { return new JsonOutputter(owner); case ENV: return new EnvOutputter(owner); + case TEMPLATE: + return new TemplateOutputter(owner); default: throw new RuntimeException("Unsupported format :" + format); } } - + public static class XmlOutputter extends PublishedConfigurationOutputter { @@ -131,7 +138,7 @@ public abstract class PublishedConfigurationOutputter { return configuration; } } - + public static class PropertiesOutputter extends PublishedConfigurationOutputter { private final Properties properties; @@ -146,15 +153,15 @@ public abstract class PublishedConfigurationOutputter { properties.store(out, ""); } - + public String asString() throws IOException { StringWriter sw = new StringWriter(); properties.store(sw, ""); return sw.toString(); } } - - + + public static class JsonOutputter extends PublishedConfigurationOutputter { public JsonOutputter(PublishedConfiguration owner) { @@ -190,9 +197,63 @@ public abstract class PublishedConfigurationOutputter { throw new IOException("Configuration has no content field and cannot " + "be retrieved as type 'env'"); } - return owner.entries.get("content"); + String content = owner.entries.get("content"); + return ConfigUtils.replaceProps(owner.entries, content); } } + public static class TemplateOutputter extends PublishedConfigurationOutputter { + + public static final String TEMPLATE_FILE = "template.file"; + + public TemplateOutputter(PublishedConfiguration owner) { + super(owner); + } + + @Override + public void save(File dest) throws IOException { + FileUtils.writeStringToFile(dest, asString(dest.getName()), + Charsets.UTF_8); + } + + public String asString(String fileName) throws IOException { + if (owner.fileSystem == null) { + throw new IOException("File system not specified for template " + + "configuration"); + } + Map<String,String> config = owner.entries; + SliderFileSystem fileSystem = owner.fileSystem; + Path templateFile = null; + if (config.containsKey(TEMPLATE_FILE)) { + templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE)); + if (!fileSystem.isFile(templateFile)) { + templateFile = fileSystem.buildResourcePath(owner.clusterName, + config.get(TEMPLATE_FILE)); + } + if (!fileSystem.isFile(templateFile)) { + throw new IOException("config specified template file " + config + .get(TEMPLATE_FILE) + " for config " + owner.description + + " but " + templateFile + " doesn't exist"); + } + } + if (templateFile == null && fileName != null) { + templateFile = fileSystem.buildResourcePath(fileName); + if (!fileSystem.isFile(templateFile)) { + templateFile = fileSystem.buildResourcePath(owner.clusterName, + fileName); + } + } + if (fileSystem.isFile(templateFile)) { + return ConfigUtils.replaceProps(config, fileSystem.cat(templateFile)); + } else { + return ""; + } + } + + @Override + public String asString() throws IOException { + return asString(null); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index b027939..063ee77 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -82,6 +82,9 @@ public interface AgentKeys { String ADDON_DEFINITION_DIR = "addon/definition"; String AGENT_CONFIG_FILE = "infra/conf/agent.ini"; String AGENT_VERSION_FILE = "infra/version"; + String APP_PACKAGES_DIR = "app/packages"; + String PER_COMPONENT = "per.component"; + String PER_GROUP = "per.group"; String JAVA_HOME = "java_home"; String PACKAGE_LIST = "package_list"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index d5fae19..d2a7e2c 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -60,8 +60,11 @@ import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.CommandLineBuilder; import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.core.registry.docstore.ConfigFormat; +import org.apache.slider.core.registry.docstore.ConfigUtils; import org.apache.slider.core.registry.docstore.ExportEntry; import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; import org.apache.slider.core.registry.docstore.PublishedExports; import org.apache.slider.core.registry.info.CustomRegistryConstants; import org.apache.slider.providers.AbstractProviderService; @@ -124,7 +127,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Map.Entry; import java.util.Scanner; import java.util.Set; import java.util.TreeMap; @@ -132,7 +134,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.regex.Pattern; import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS; @@ -435,6 +436,26 @@ public class AgentProviderService extends AbstractProviderService implements LocalResourceType.ARCHIVE); launcher.addLocalResource(AgentKeys.APP_DEFINITION_DIR, appDefRes); + for (Package pkg : getMetaInfo().getApplication().getPackages()) { + Path pkgPath = fileSystem.buildResourcePath(pkg.getName()); + if (!fileSystem.isFile(pkgPath)) { + pkgPath = fileSystem.buildResourcePath(getClusterName(), + pkg.getName()); + } + if (!fileSystem.isFile(pkgPath)) { + throw new IOException("Package doesn't exist as a resource: " + + pkg.getName()); + } + log.info("Adding resource {}", pkg.getName()); + LocalResourceType type = LocalResourceType.FILE; + if ("archive".equals(pkg.getType())) { + type = LocalResourceType.ARCHIVE; + } + LocalResource packageResource = fileSystem.createAmResource( + pkgPath, type); + launcher.addLocalResource(AgentKeys.APP_PACKAGES_DIR, packageResource); + } + String agentConf = instanceDefinition.getAppConfOperations(). getGlobalOptions().getOption(AgentKeys.AGENT_CONF, ""); if (SliderUtils.isSet(agentConf)) { @@ -476,6 +497,13 @@ public class AgentProviderService extends AbstractProviderService implements generatedConfPath, SliderKeys.PROPAGATED_CONF_DIR_NAME)); + // build and localize configuration files + Map<String, Map<String, String>> configurations = + buildCommandConfigurations(instanceDefinition.getAppConfOperations(), + container.getId().toString(), roleName, roleGroup); + localizeConfigFiles(launcher, roleName, roleGroup, getMetaInfo(), + configurations, launcher.getEnv(), fileSystem); + String label = getContainerLabel(container, roleName, roleGroup); CommandLineBuilder operation = new CommandLineBuilder(); @@ -646,11 +674,42 @@ public class AgentProviderService extends AbstractProviderService implements private Path uploadSecurityResource(File resource, SliderFileSystem fileSystem) throws IOException { Path certsDir = fileSystem.buildClusterSecurityDirPath(getClusterName()); - if (!fileSystem.getFileSystem().exists(certsDir)) { - fileSystem.getFileSystem().mkdirs(certsDir, + return uploadResource(resource, fileSystem, certsDir); + } + + private Path checkResourceExists(File resource, SliderFileSystem + fileSystem, String roleName) throws IOException { + Path dir; + if (roleName == null) { + dir = fileSystem.buildClusterResourcePath(getClusterName()); + } else { + dir = fileSystem.buildClusterResourcePath(getClusterName(), roleName); + } + Path destPath = new Path(dir, resource.getName()); + if (fileSystem.getFileSystem().exists(destPath)) { + return destPath; + } + return null; + } + + private Path uploadResource(File resource, SliderFileSystem fileSystem, + String roleName) throws IOException { + Path dir; + if (roleName == null) { + dir = fileSystem.buildClusterResourcePath(getClusterName()); + } else { + dir = fileSystem.buildClusterResourcePath(getClusterName(), roleName); + } + return uploadResource(resource, fileSystem, dir); + } + + private static Path uploadResource(File resource, SliderFileSystem fileSystem, + Path parentDir) throws IOException { + if (!fileSystem.getFileSystem().exists(parentDir)) { + fileSystem.getFileSystem().mkdirs(parentDir, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); } - Path destPath = new Path(certsDir, resource.getName()); + Path destPath = new Path(parentDir, resource.getName()); if (!fileSystem.getFileSystem().exists(destPath)) { FSDataOutputStream os = fileSystem.getFileSystem().create(destPath); byte[] contents = FileUtils.readFileToByteArray(resource); @@ -659,6 +718,9 @@ public class AgentProviderService extends AbstractProviderService implements os.flush(); os.close(); log.info("Uploaded {} to localization path {}", resource, destPath); + } else { + log.info("Resource {} already existed at localization path {}", resource, + destPath); } while (!fileSystem.getFileSystem().exists(destPath)) { @@ -718,6 +780,68 @@ public class AgentProviderService extends AbstractProviderService implements } } + private void createConfigFile(SliderFileSystem fileSystem, File file, + ConfigFile configFile, Map<String, String> config) throws IOException { + ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType()); + log.info("Writing {} file {}", configFormat, file); + + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(configFormat, + new PublishedConfiguration(configFile.getDictionaryName(), + config.entrySet(), fileSystem, getClusterName())); + configurationOutputter.save(file); + } + + @VisibleForTesting + protected void localizeConfigFiles(ContainerLauncher launcher, + String roleName, String roleGroup, + Metainfo metainfo, + Map<String, Map<String, String>> configs, + MapOperations env, + SliderFileSystem fileSystem) + throws IOException { + for (ConfigFile configFile : metainfo.getComponentConfigFiles(roleGroup)) { + Map<String, String> config = ConfigUtils.replacePropsInConfig( + configs.get(configFile.getDictionaryName()), env.options); + String fileName = ConfigUtils.replaceProps(config, + configFile.getFileName()); + File localFile = new File(SliderKeys.RESOURCE_DIR); + if (!localFile.exists()) { + localFile.mkdir(); + } + localFile = new File(localFile, new File(fileName).getName()); + + String folder = null; + if ("true".equals(config.get(PER_COMPONENT))) { + folder = roleName; + } else if ("true".equals(config.get(PER_GROUP))) { + folder = roleGroup; + } + + Path destPath = checkResourceExists(localFile, fileSystem, folder); + if (destPath == null) { + log.info("Localizing {} configs to config file {} (destination {}) " + + "based on {} configs", + config.size(), localFile, fileName, configFile.getDictionaryName()); + createConfigFile(fileSystem, localFile, configFile, config); + destPath = uploadResource(localFile, fileSystem, folder); + } else { + log.info("Config already exists at {}, not recreating it", destPath); + } + LocalResource configResource = fileSystem.createAmResource(destPath, + LocalResourceType.FILE); + + File destFile = new File(fileName); + if (destFile.isAbsolute()) { + launcher.addLocalResource( + SliderKeys.RESOURCE_DIR + "/" + destFile.getName(), + configResource, fileName); + } else { + launcher.addLocalResource(fileName, configResource); + } + } + } + /** * build the zookeeper registry path. * @@ -2036,7 +2160,7 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setConfigurations(configurations); Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf); cmd.setComponentConfigurations(componentConfigurations); - + if (SliderUtils.isSet(scriptPath)) { cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false)); } else { @@ -2154,10 +2278,10 @@ public class AgentProviderService extends AbstractProviderService implements List<String> packages = new ArrayList<>(); if (application != null) { if (application.getPackages().size() > 0) { - List<Package> appPackages = application.getPackages(); - for (Package appPackage : appPackages) { - packages.add(String.format(pkgFormatString, appPackage.getType(), appPackage.getName())); - } + // no-op if there are packages that are not OS-specific, as these + // will be localized by AM rather than the Agent + // this should be backwards compatible, as there was previously an + // XML parsing bug that ensured non-OS-specific packages did not exist } else { List<OSSpecific> osSpecifics = application.getOSSpecifics(); if (osSpecifics != null && osSpecifics.size() > 0) { @@ -2821,14 +2945,41 @@ public class AgentProviderService extends AbstractProviderService implements } } + boolean finished = false; + while (!finished) { + finished = true; + for (Map.Entry<String, String> entry : allConfigs.entrySet()) { + String configValue = entry.getValue(); + for (Map.Entry<String, String> lookUpEntry : allConfigs.entrySet()) { + String lookUpValue = lookUpEntry.getValue(); + if (lookUpValue.contains("${@//site/")) { + continue; + } + String lookUpKey = lookUpEntry.getKey(); + if (configValue != null && configValue.contains(lookUpKey)) { + configValue = configValue.replace(lookUpKey, lookUpValue); + } + } + if (!configValue.equals(entry.getValue())) { + finished = false; + allConfigs.put(entry.getKey(), configValue); + } + } + } + for (String configType : configurations.keySet()) { Map<String, String> configBucket = configurations.get(configType); for (Map.Entry<String, String> entry: configBucket.entrySet()) { String configName = entry.getKey(); String configValue = entry.getValue(); - for (String lookUpKey : allConfigs.keySet()) { + for (Map.Entry<String, String> lookUpEntry : allConfigs.entrySet()) { + String lookUpValue = lookUpEntry.getValue(); + if (lookUpValue.contains("${@//site/")) { + continue; + } + String lookUpKey = lookUpEntry.getKey(); if (configValue != null && configValue.contains(lookUpKey)) { - configValue = configValue.replace(lookUpKey, allConfigs.get(lookUpKey)); + configValue = configValue.replace(lookUpKey, lookUpValue); } } configBucket.put(configName, configValue); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java index 1b63b58..b6ae4de 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java @@ -65,6 +65,10 @@ public abstract class AbstractComponent implements Validate { this.commands = commands; } + public void addCommand(ComponentCommand command) { + commands.add(command); + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("{"); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java index 63546a4..5556c7f 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java @@ -116,6 +116,10 @@ public class Application extends AbstractMetainfoSchema { return commandOrders; } + public void addPackage(Package pkg) { + packages.add(pkg); + } + @JsonProperty("packages") public List<Package> getPackages() { return packages; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java index 3f23455..78bb8c1 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java @@ -38,7 +38,8 @@ public class Component extends AbstractComponent { String type = TYPE_STANDARD; List<ComponentExport> componentExports = new ArrayList<>(); List<DockerContainer> dockerContainers = new ArrayList<>(); - + List<ConfigFile> configFiles = new ArrayList<>(); + public Component() { } @@ -155,6 +156,15 @@ public class Component extends AbstractComponent { return Boolean.parseBoolean(this.autoStartOnFailure); } + public void addConfigFile(ConfigFile configFile) { + this.configFiles.add(configFile); + } + + @JsonProperty("configFiles") + public List<ConfigFile> getConfigFiles() { + return configFiles; + } + @Override public String toString() { final StringBuilder sb = http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java index 036d98e..10c497f 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java @@ -21,6 +21,9 @@ import org.apache.slider.core.exceptions.SliderException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; + /** * Application metainfo uber class */ @@ -71,6 +74,16 @@ public class Metainfo { return null; } + public List<ConfigFile> getComponentConfigFiles(String roleGroup) { + List<ConfigFile> componentConfigFiles = new ArrayList<>(); + componentConfigFiles.addAll(application.getConfigFiles()); + Component component = getApplicationComponent(roleGroup); + if (component != null) { + componentConfigFiles.addAll(component.getConfigFiles()); + } + return componentConfigFiles; + } + public void validate() throws SliderException { if (!VERSION_TWO_ONE.equals(schemaVersion) && !VERSION_TWO_ZERO.equals(schemaVersion)) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java index a6f0e9d..8b520eb 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java @@ -67,14 +67,25 @@ public class MetainfoParser extends AbstractMetainfoParser{ digester.addBeanPropertySetter("*/commandScript/timeout"); digester.addSetNext("*/commandScript", "addCommandScript"); + digester.addObjectCreate("*/command", ComponentCommand.class); + digester.addBeanPropertySetter("*/command/exec"); + digester.addBeanPropertySetter("*/command/name"); + digester.addBeanPropertySetter("*/command/type"); + digester.addSetNext("*/command", "addCommand"); + digester.addObjectCreate("*/osSpecific", OSSpecific.class); digester.addBeanPropertySetter("*/osSpecific/osType"); - digester.addObjectCreate("*/package", OSPackage.class); - digester.addBeanPropertySetter("*/package/type"); - digester.addBeanPropertySetter("*/package/name"); - digester.addSetNext("*/package", "addOSPackage"); + digester.addObjectCreate("*/osSpecific/packages/package", OSPackage.class); + digester.addBeanPropertySetter("*/osSpecific/packages/package/type"); + digester.addBeanPropertySetter("*/osSpecific/packages/package/name"); + digester.addSetNext("*/osSpecific/packages/package", "addOSPackage"); digester.addSetNext("*/osSpecific", "addOSSpecific"); + digester.addObjectCreate("*/application/packages/package", Package.class); + digester.addBeanPropertySetter("*/application/packages/package/type"); + digester.addBeanPropertySetter("*/application/packages/package/name"); + digester.addSetNext("*/application/packages/package", "addPackage"); + digester.addObjectCreate("*/configFile", ConfigFile.class); digester.addBeanPropertySetter("*/configFile/type"); digester.addBeanPropertySetter("*/configFile/fileName"); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index 9fbb3d0..af1c61f 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -88,11 +88,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -285,6 +287,7 @@ public class TestAgentProviderService { ProviderRole role = new ProviderRole("HBASE_MASTER", 1); SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class); ContainerLauncher launcher = createNiceMock(ContainerLauncher.class); + expect(launcher.getEnv()).andReturn(new MapOperations()).anyTimes(); Path generatedConfPath = new Path(".", "test"); MapOperations resourceComponent = new MapOperations(); MapOperations appComponent = new MapOperations(); @@ -347,6 +350,9 @@ public class TestAgentProviderService { anyString(), anyString() ); + + doReturn(Collections.emptyMap()).when(mockAps).getRoleClusterNodeMapping(); + expect(access.isApplicationLive()).andReturn(true).anyTimes(); ClusterDescription desc = new ClusterDescription(); desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181"); @@ -357,16 +363,15 @@ public class TestAgentProviderService { expect(access.getRoleClusterNodeMapping()).andReturn(cnMap).anyTimes(); - AggregateConf aggConf = new AggregateConf(); - ConfTreeOperations treeOps = aggConf.getAppConfOperations(); + ConfTreeOperations treeOps = instanceDefinition.getAppConfOperations(); treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0"); treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE"); treeOps.set("site.fs.defaultFS", "hdfs://HOST1:8020/"); treeOps.set("internal.data.dir.path", "hdfs://HOST1:8020/database"); - expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf); + expect(access.getInstanceDefinitionSnapshot()).andReturn(instanceDefinition); expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes(); expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes(); - replay(access, ctx, container, sliderFileSystem, mockFs); + replay(access, ctx, container, sliderFileSystem, mockFs, launcher); try { mockAps.buildContainerLaunchContext(launcher, @@ -1225,6 +1230,8 @@ public class TestAgentProviderService { SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class); ContainerLauncher launcher = createNiceMock(ContainerLauncher.class); ContainerLauncher launcher2 = createNiceMock(ContainerLauncher.class); + expect(launcher.getEnv()).andReturn(new MapOperations()).anyTimes(); + expect(launcher2.getEnv()).andReturn(new MapOperations()).anyTimes(); Path generatedConfPath = new Path(".", "test"); MapOperations resourceComponent = new MapOperations(); MapOperations appComponent = new MapOperations(); @@ -1291,18 +1298,28 @@ public class TestAgentProviderService { desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE"); expect(access.getClusterStatus()).andReturn(desc).anyTimes(); - AggregateConf aggConf = new AggregateConf(); - ConfTreeOperations treeOps = aggConf.getAppConfOperations(); + ConfTreeOperations treeOps = instanceDefinition.getAppConfOperations(); treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0"); - treeOps.getOrAddComponent("HBASE_REGIONSERVER").put(AgentKeys.WAIT_HEARTBEAT, "0"); + treeOps.getOrAddComponent("HBASE_REGIONSERVER").put( + AgentKeys.WAIT_HEARTBEAT, "0"); treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE"); treeOps.set("site.fs.defaultFS", "hdfs://HOST1:8020/"); treeOps.set("internal.data.dir.path", "hdfs://HOST1:8020/database"); - expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf).anyTimes(); + expect(access.getInstanceDefinitionSnapshot()).andReturn(instanceDefinition).anyTimes(); expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes(); expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes(); - doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection()); - replay(access, ctx, container, sliderFileSystem, mockFs); + doNothing().when(mockAps).publishApplicationInstanceData(anyString(), + anyString(), anyCollection()); + doNothing().when(mockAps).localizeConfigFiles( + (ContainerLauncher)Matchers.anyObject(), + anyString(), + anyString(), + (Metainfo)Matchers.anyObject(), + anyMap(), + (MapOperations)Matchers.anyObject(), + (SliderFileSystem)Matchers.anyObject()); + doReturn(Collections.emptyMap()).when(mockAps).getRoleClusterNodeMapping(); + replay(access, ctx, container, sliderFileSystem, mockFs, launcher, launcher2); // build two containers try { @@ -1849,12 +1866,12 @@ public class TestAgentProviderService { AgentProviderService aps = createAgentProviderService(new Configuration()); Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>(); Map<String, String> cfg1 = new HashMap<String, String>(); - cfg1.put("a1", "${@//site/cfg-2/A1}"); + cfg1.put("a1", "0${@//site/cfg-2/A1}"); cfg1.put("b1", "22"); cfg1.put("c1", "33"); cfg1.put("d1", "${@//site/cfg1/c1}AA"); Map<String, String> cfg2 = new HashMap<String, String>(); - cfg2.put("A1", "11"); + cfg2.put("A1", "11${@//site/cfg1/b1}"); cfg2.put("B1", "${@//site/cfg-2/A1},${@//site/cfg-2/A1},AA,${@//site/cfg1/c1}"); cfg2.put("C1", "DD${@//site/cfg1/c1}"); cfg2.put("D1", "${14}"); @@ -1862,15 +1879,30 @@ public class TestAgentProviderService { allConfigs.put("cfg1", cfg1); allConfigs.put("cfg-2", cfg2); aps.dereferenceAllConfigs(allConfigs); - Assert.assertEquals("11", cfg1.get("a1")); + Assert.assertEquals("01122", cfg1.get("a1")); Assert.assertEquals("22", cfg1.get("b1")); Assert.assertEquals("33", cfg1.get("c1")); Assert.assertEquals("33AA", cfg1.get("d1")); - Assert.assertEquals("11", cfg2.get("A1")); - Assert.assertEquals("11,11,AA,33", cfg2.get("B1")); + Assert.assertEquals("1122", cfg2.get("A1")); + Assert.assertEquals("1122,1122,AA,33", cfg2.get("B1")); Assert.assertEquals("DD33", cfg2.get("C1")); Assert.assertEquals("${14}", cfg2.get("D1")); } + @Test + public void testDereferenceAllConfigLoop() throws IOException { + AgentProviderService aps = createAgentProviderService(new Configuration()); + Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>(); + Map<String, String> cfg1 = new HashMap<String, String>(); + cfg1.put("a1", "0${@//site/cfg-2/A1}"); + Map<String, String> cfg2 = new HashMap<String, String>(); + cfg2.put("A1", "11${@//site/cfg1/a1}"); + + allConfigs.put("cfg1", cfg1); + allConfigs.put("cfg-2", cfg2); + aps.dereferenceAllConfigs(allConfigs); + Assert.assertEquals("0${@//site/cfg-2/A1}", cfg1.get("a1")); + Assert.assertEquals("11${@//site/cfg1/a1}", cfg2.get("A1")); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java index 30283d1..ba1912a 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java @@ -50,7 +50,7 @@ public class TestMetainfoParser { Assert.assertNotNull(metainfo.getApplication()); Application application = metainfo.getApplication(); assert "STORM".equals(application.getName()); - assert 5 == application.getComponents().size(); + assert 6 == application.getComponents().size(); OSPackage pkg = application.getOSSpecifics().get(0).getPackages().get(0); assert "tarball".equals(pkg.getType()); assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName()); @@ -63,9 +63,20 @@ public class TestMetainfoParser { if (comp != null && comp.getName().equals("SUPERVISOR")) { Assert.assertEquals(1, comp.getComponentExports().size()); } + if (comp != null && comp.getName().equals("ANOTHER_COMPONENT")) { + assert 2 == comp.getCommands().size(); + assert "start command".equals(comp.getCommands().get(0).getExec()); + assert "START".equals(comp.getCommands().get(0).getName()); + assert "stop command".equals(comp.getCommands().get(1).getExec()); + assert "STOP".equals(comp.getCommands().get(1).getName()); + } } assert found; Assert.assertEquals(0, application.getConfigFiles().size()); + assert 1 == application.getPackages().size(); + Package p = application.getPackages().get(0); + assert "tarball".equals(p.getType()); + assert "test-tarball-name.tgz".equals(p.getName()); } @Test http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/84fee8a3/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml ---------------------------------------------------------------------- diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml index d9004ad..fbe9299 100644 --- a/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml +++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml @@ -142,6 +142,20 @@ <timeout>600</timeout> </commandScript> </component> + + <component> + <name>ANOTHER_COMPONENT</name> + <category>MASTER</category> + <commands> + <command> + <exec>start command</exec> + </command> + <command> + <exec>stop command</exec> + <name>STOP</name> + </command> + </commands> + </component> </components> <osSpecifics> @@ -155,5 +169,12 @@ </packages> </osSpecific> </osSpecifics> + + <packages> + <package> + <type>tarball</type> + <name>test-tarball-name.tgz</name> + </package> + </packages> </application> </metainfo>
