SLIDER-906 Support for Docker based application packaging with first class YARN support - Phase 2 [contributed by Thomas (Yu) Liu, Billie Rinaldi and Gour Saha]
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fffd4e63 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fffd4e63 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fffd4e63 Branch: refs/heads/feature/SLIDER-906_docker_support Commit: fffd4e63673c521d7c943d69d4c65f2ee55a6bfd Parents: ef8204d Author: Gour Saha <gourks...@apache.org> Authored: Tue Mar 22 18:00:26 2016 -0700 Committer: Gour Saha <gourks...@apache.org> Committed: Tue Mar 22 18:00:26 2016 -0700 ---------------------------------------------------------------------- .../src/main/python/agent/ActionQueue.py | 15 +- .../src/main/python/agent/CommandStatusDict.py | 2 + .../src/main/python/agent/Controller.py | 2 +- .../src/main/python/agent/YarnDockerManager.py | 248 +++++++++++++++++ slider-agent/src/main/python/agent/main.py | 2 +- .../core/providers/windows/system.py | 1 + .../java/org/apache/slider/api/ClusterNode.java | 2 + .../org/apache/slider/api/ResourceKeys.java | 14 + .../org/apache/slider/client/SliderClient.java | 10 + .../apache/slider/common/tools/SliderUtils.java | 22 +- .../slider/core/launch/AbstractLauncher.java | 31 ++- .../slider/providers/agent/AgentKeys.java | 1 + .../providers/agent/AgentProviderService.java | 264 ++++++++++++++----- .../providers/agent/ComponentInstanceState.java | 20 ++ .../agent/application/metadata/Application.java | 2 +- .../agent/application/metadata/Component.java | 3 + .../application/metadata/DockerContainer.java | 65 ++++- .../server/appmaster/SliderAppMaster.java | 1 + .../slider/server/appmaster/state/AppState.java | 1 + .../server/appmaster/state/RoleInstance.java | 4 + .../web/rest/agent/ComponentStatus.java | 20 ++ .../web/rest/agent/ExecutionCommand.java | 21 +- .../appmaster/web/rest/agent/HeartBeat.java | 4 +- .../web/rest/agent/HeartBeatResponse.java | 1 + .../web/rest/agent/RegistrationResponse.java | 20 +- .../appmaster/web/rest/agent/StatusCommand.java | 11 + .../allCompFailed/appConfig.json | 10 + .../allCompFailed/metainfo.json | 44 ++++ .../allCompFailed/resources.json | 20 ++ .../basic/appConfig.json | 10 + .../test_docker_on_yarn_pkg/basic/metainfo.json | 26 ++ .../basic/resources.json | 14 + .../multiComp/appConfig.json | 10 + .../multiComp/metainfo.json | 44 ++++ .../multiComp/resources.json | 20 ++ .../multiCompMultiContainer/appConfig.json | 10 + .../multiCompMultiContainer/metainfo.json | 44 ++++ .../multiCompMultiContainer/resources.json | 20 ++ .../oneCompFailed/appConfig.json | 10 + .../oneCompFailed/metainfo.json | 44 ++++ .../oneCompFailed/resources.json | 20 ++ .../agent/TestAgentProviderService.java | 10 +- .../DockerAppLaunchedOnYarnIT.groovy | 259 ++++++++++++++++++ 43 files changed, 1321 insertions(+), 81 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-agent/src/main/python/agent/ActionQueue.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py index 497d4f4..107d6c3 100644 --- a/slider-agent/src/main/python/agent/ActionQueue.py +++ b/slider-agent/src/main/python/agent/ActionQueue.py @@ -31,6 +31,7 @@ from AgentToggleLogger import AgentToggleLogger from CommandStatusDict import CommandStatusDict from CustomServiceOrchestrator import CustomServiceOrchestrator from DockerManager import DockerManager +from YarnDockerManager import YarnDockerManager import Constants @@ -54,6 +55,7 @@ class ActionQueue(threading.Thread): AUTO_RESTART = 'auto_restart' docker_mode = False + yarn_docker_mode = False def __init__(self, config, controller, agentToggleLogger): super(ActionQueue, self).__init__() @@ -71,6 +73,7 @@ class ActionQueue(threading.Thread): controller, self.queueOutAgentToggleLogger) self.dockerManager = DockerManager(self.tmpdir, config.getWorkRootPath(), self.customServiceOrchestrator) + self.yarnDockerManager = YarnDockerManager(self.tmpdir, config.getWorkRootPath(), self.customServiceOrchestrator) def stop(self): @@ -169,7 +172,10 @@ class ActionQueue(threading.Thread): logger.info("Running command: " + str(command)) - if 'configurations' in command and 'docker' in command['configurations']: + if 'configurations' in command and 'yarnDockerMode' in command and command['yarnDockerMode'] == True: + self.yarn_docker_mode = True + commandresult = self.yarnDockerManager.execute_command(command, store_config or store_command) + elif 'configurations' in command and 'docker' in command['configurations']: self.docker_mode = True commandresult = self.dockerManager.execute_command(command, store_config or store_command) else: @@ -235,6 +241,8 @@ class ActionQueue(threading.Thread): component_status = None if self.docker_mode: component_status = self.dockerManager.query_status(command) + elif self.yarn_docker_mode: + component_status = self.yarnDockerManager.query_status(command) else: component_status = self.customServiceOrchestrator.requestComponentStatus(command) @@ -254,8 +262,13 @@ class ActionQueue(threading.Thread): " of service " + str(service) + \ " of cluster " + str(cluster)) logger.debug(pprint.pformat(result)) + if 'ip' in component_status: + result['ip'] = component_status['ip'] + if 'hostname' in component_status: + result['hostname'] = component_status['hostname'] if result is not None: + logger.debug("result of execute_status_command: " + str(result)) self.commandStatuses.put_command_status(command, result, reportResult) except Exception, err: traceback.print_exc() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-agent/src/main/python/agent/CommandStatusDict.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/CommandStatusDict.py b/slider-agent/src/main/python/agent/CommandStatusDict.py index bcbce9e..5af75fb 100644 --- a/slider-agent/src/main/python/agent/CommandStatusDict.py +++ b/slider-agent/src/main/python/agent/CommandStatusDict.py @@ -73,6 +73,7 @@ class CommandStatusDict(): for key, item in self.current_state.items(): command = item[0] report = item[1] + logger.debug("report inside generate report is: " + str(report)) if command ['commandType'] == ActionQueue.EXECUTION_COMMAND: if (report['status']) != ActionQueue.IN_PROGRESS_STATUS: resultReports.append(report) @@ -90,6 +91,7 @@ class CommandStatusDict(): 'reports': resultReports, 'componentStatus': resultComponentStatus } + logger.debug("result from generate report: " + str(result)) return result http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-agent/src/main/python/agent/Controller.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py index ec3bed7..8c01315 100644 --- a/slider-agent/src/main/python/agent/Controller.py +++ b/slider-agent/src/main/python/agent/Controller.py @@ -165,7 +165,7 @@ class Controller(threading.Thread): print("Registered with the server") self.responseId = int(regResp['responseId']) self.isRegistered = True - if 'statusCommands' in regResp.keys(): + if ('statusCommands' in regResp.keys()) and (not regResp['statusCommands'] == None): logger.info("Got status commands on registration " + pprint.pformat( regResp['statusCommands'])) self.addToQueue(regResp['statusCommands']) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-agent/src/main/python/agent/YarnDockerManager.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/YarnDockerManager.py b/slider-agent/src/main/python/agent/YarnDockerManager.py new file mode 100644 index 0000000..abdb2fb --- /dev/null +++ b/slider-agent/src/main/python/agent/YarnDockerManager.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import logging +import os +import subprocess +import Constants +import time +import traceback +from resource_management import * + +logger = logging.getLogger() + +class YarnDockerManager(Script): + stored_status_command = '' + stored_command = '' + container_id = '' + + def __init__(self, tmpdir, workroot, customServiceOrchestrator): + self.tmpdir = tmpdir + self.workroot = workroot + self.customServiceOrchestrator = customServiceOrchestrator + + def execute_command(self, command, store_command=False): + returncode = '' + out = '' + err = '' + + if store_command: + logger.info("Storing applied config: " + str(command['configurations'])) + self.stored_command = command + status_command_str = self.extract_config_from_command(command, 'docker.status_command') + if status_command_str: + self.stored_status_command = status_command_str + logger.info("status command" + self.stored_status_command) + if 'hostLevelParams' in command: + if 'container_id' in command['hostLevelParams']: + self.container_id = command['hostLevelParams']['container_id'] + + if command['roleCommand'] == 'INSTALL': + with Environment(self.workroot) as env: + self.install_container(command, env) + returncode = 0; out = ''; err = '' + if command['roleCommand'] == 'START': + returncode, out, err = self.start_container(command) + # need check + return {Constants.EXIT_CODE:returncode, 'stdout':out, 'stderr':err} + + + def extract_config_files_from_command(self, command): + if 'containers' in command and len(command['containers']) > 0: + if 'configFiles' in command['containers'][0]: + return command['containers'][0]['configFiles'] + return [] + + def extract_config_file_properties_from_command(self, command, file): + if 'configurations' in command: + if 'dictionaryName' in file and file['dictionaryName'] in command['configurations']: + properties = {} + for k,v in command['configurations'][file['dictionaryName']].iteritems(): + properties[k] = format(v, **os.environ) + return properties + return {} + + def extract_config_from_command(self, command, field): + value = '' + if 'configurations' in command: + if 'docker' in command['configurations']: + if field in command['configurations']['docker']: + logger.info(field + ': ' + str( command['configurations']['docker'][field])) + value = command['configurations']['docker'][field] + return value + + + # will evolve into a class hierarch, linux and windows + def execute_command_on_linux(self, docker_command, blocking, stdoutFile=None, stderrFile=None): + + logger.info("command str: " + docker_command) + logger.info("command env: " + str(os.environ)) + if stdoutFile != None or stderrFile != None: + proc = subprocess.Popen(docker_command, + stdout = stdoutFile, + stderr = stderrFile, universal_newlines = True, shell=True) + else: + proc = subprocess.Popen(docker_command, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True) + + returncode = 0 + out = '' + err = '' + if blocking is True: + out, err = proc.communicate() + returncode = proc.returncode + else: + time.sleep(5) + if proc.returncode is not None: + # this means the command has already returned + returncode = proc.returncode + out, err = proc.communicate() + logger.info("returncode: " + str(returncode) + " out: " + str(out) + " err: " + str(err)) + return returncode, out, err + + def enqueue_output(out, queue): + for line in iter(out.readline, b''): + queue.put(line) + out.close() + + def get_config_file_global(self, command, file, property, useEnv=True): + keyName = file['dictionaryName'] + '.' + property + if keyName in command['configurations']['global']: + if useEnv: + return format(command['configurations']['global'][keyName], **os.environ) + else: + return command['configurations']['global'][keyName] + return None + + def install_container(self, command, env): + try: + configFiles = self.extract_config_files_from_command(command) + for configFile in configFiles: + properties = self.extract_config_file_properties_from_command(command, configFile) + fileName = configFile['fileName'] + dir = self.get_config_file_global(command, configFile, 'destDir') + if dir is None: + dir = self.workroot + logger.info("creating config file " + str(configFile) + " in directory "+str(dir)) + Directory(dir, recursive = True) + if configFile['type'] == 'properties': + PropertiesFile(fileName, + dir=dir, + properties=properties) + elif configFile['type'] == 'env': + content = self.get_config_file_global(command, configFile, 'content', useEnv=False) + if content is not None: + File(os.path.join(dir, fileName), + content=InlineTemplate(content, **properties)) + elif configFile['type'] == 'template': + templateFile = self.get_config_file_global(command, configFile, 'templateFile') + if templateFile is not None: + with open(templateFile,"r") as fp: + fileContent = fp.read() + File(os.path.join(dir, fileName), + content=InlineTemplate(fileContent, **properties)) + elif configFile['type'] == 'xml': + XmlConfig(fileName, + conf_dir=dir, + configurations=properties) + except: + traceback.print_exc() + + def start_container(self, command): + #extracting param needed by docker run from the command passed from AM + startCommand = self.extract_config_from_command(command, 'docker.startCommand') + #adding redirecting stdout stderr to file + outfilename = Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \ + self.container_id + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE + + errfilename = Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \ + self.container_id + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE + + stdoutFile = open(outfilename, 'w') + stderrFile = open(errfilename, 'w') + returncode,out,err = self.execute_command_on_linux(startCommand, False, + stdoutFile, stderrFile) + return returncode,out,err + + def query_status(self, command): + if command['roleCommand'] == "GET_CONFIG": + return self.getConfig(command) + else: + returncode = '' + out = '' + err = '' + status_command_str = self.extract_config_from_command(command, 'docker.status_command') + if status_command_str: + self.stored_status_command = status_command_str + logger.info("in query_status, got stored status command" + self.stored_status_command) + if self.stored_status_command: + logger.info("stored status command to run: " + self.stored_status_command) + returncode, out, err = self.execute_command_on_linux(self.stored_status_command, True) + logger.info("status of the app in docker container: " + str(returncode) + ";" + str(out) + ";" + str(err)) + + return {Constants.EXIT_CODE:returncode, 'stdout':out, 'stderr':err} + + def getConfig(self, command): + logger.info("get config command: " + str(command)) + config = {} + + if 'configurations' in self.stored_command: + if 'commandParams' in command and 'config_type' in command['commandParams']: + config_type = command['commandParams']['config_type'] + logger.info("Requesting applied config for type {0}".format(config_type)) + if config_type in self.stored_command['configurations']: + logger.info("get config result: " + self.stored_command['configurations'][config_type]) + config = { + 'configurations': {config_type: self.stored_command['configurations'][config_type]} + } + else: + config = { + 'configurations': {} + } + pass + else: + logger.info("Requesting all applied config." + str(self.stored_command['configurations'])) + config = { + 'configurations': self.stored_command['configurations'] + } + pass + else: + config = { + 'configurations': {} + } + + #query the ip and hostname of the docker container where the agent is running + ip_query_command = "ip addr show dev eth0 | grep \'inet \' | awk \'{print $2}\' | cut -d / -f 1" + proc = subprocess.Popen(ip_query_command, stdout = subprocess.PIPE, shell=True) + ip, err = proc.communicate() + if err is not None: + logger.error("error when retrieving ip: " + err) + + hostname_query_command = "hostname" + proc = subprocess.Popen(hostname_query_command, stdout = subprocess.PIPE, shell=True) + hostname, err = proc.communicate() + if err is not None: + logger.error("error when retrieving hostname: " + err) + + config['ip'] = ip.rstrip() + config['hostname'] = hostname.rstrip() + + logger.info('response from getconfig: ' + str(config)) + return config + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-agent/src/main/python/agent/main.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py index 36aa4f4..39de1ae 100644 --- a/slider-agent/src/main/python/agent/main.py +++ b/slider-agent/src/main/python/agent/main.py @@ -55,7 +55,7 @@ def signal_handler(signum, frame): os._exit(0) logger.info('signal received, exiting.') global controller - if controller is not None and hasattr(controller, 'actionQueue'): + if controller is not None and hasattr(controller, 'actionQueue') and controller.actionQueue.yarn_docker_mode == False: docker_mode = controller.actionQueue.docker_mode if docker_mode: tmpdir = controller.actionQueue.dockerManager.stop_container() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-agent/src/main/python/resource_management/core/providers/windows/system.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py index af513a1..b85c020 100644 --- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py +++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py @@ -89,6 +89,7 @@ def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish= if not wait_for_finish: Logger.debug("No need to wait for the process to exit. Will leave the process running ...") code = 0 + logAnyway = False if pid_file_name: Logger.debug("Writing the process id %s to file %s" % (str(proc.pid), pid_file_name)) pidfile = open(pid_file_name, 'w') http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java b/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java index e05a2ef..8b0a563 100644 --- a/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java +++ b/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java @@ -66,6 +66,8 @@ public final class ClusterNode implements Cloneable { */ public boolean released; public String host; + public String ip; + public String hostname; public String hostUrl; /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java index f92a58d..aba544b 100644 --- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java +++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java @@ -66,6 +66,18 @@ public interface ResourceKeys { */ String YARN_CORES = "yarn.vcores"; + /** + * Number of disks per instance to ask YARN for + * {@value} + */ + String YARN_DISKS = "yarn.disks.count-per-instance"; + + /** + * Disk size per disk to ask YARN for + * {@value} + */ + String YARN_DISK_SIZE = "yarn.disk.size"; + /** {@value} */ int DEF_YARN_CORES = 1; @@ -168,6 +180,8 @@ public interface ResourceKeys { String YARN_LOG_INCLUDE_PATTERNS = "yarn.log.include.patterns"; String YARN_LOG_EXCLUDE_PATTERNS = "yarn.log.exclude.patterns"; + String YARN_PROFILE_NAME = "yarn.resource-profile-name"; + /** * Window of time where application master's failure count * can be reset to 0. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index cf15dfc..7baf659 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -176,6 +176,7 @@ import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.File; +import java.io.FilenameFilter; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -1727,6 +1728,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } conf.global.putAll(newglobal); + for (String component : conf.components.keySet()) { + Map<String,String> newComponent = new HashMap<>(); + for (Entry<String,String> entry : conf.components.get(component).entrySet()) { + newComponent.put(entry.getKey(), replaceTokens(entry.getValue(), + userName, clusterName)); + } + conf.components.get(component).putAll(newComponent); + } + Map<String,List<String>> newcred = new HashMap<>(); for (Entry<String,List<String>> entry : conf.credentials.entrySet()) { List<String> resultList = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 5f5599c..18cc779 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -117,6 +117,7 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Pattern; import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -153,9 +154,14 @@ public final class SliderUtils { public static final String PYTHON = "python"; /** - * name of docker program + * type of docker standalone application */ public static final String DOCKER = "docker"; + /** + * type of docker on yarn application + */ + public static final String DOCKER_YARN = "yarn_docker"; + public static final int NODE_LIST_LIMIT = 10; private SliderUtils() { @@ -1132,6 +1138,11 @@ public final class SliderUtils { * @return a possibly empty map of environment variables. */ public static Map<String, String> buildEnvMap(Map<String, String> roleOpts) { + return buildEnvMap(roleOpts, null); + } + + public static Map<String, String> buildEnvMap(Map<String, String> roleOpts, + Map<String,String> tokenMap) { Map<String, String> env = new HashMap<>(); if (roleOpts != null) { for (Map.Entry<String, String> entry : roleOpts.entrySet()) { @@ -1139,7 +1150,14 @@ public final class SliderUtils { if (key.startsWith(RoleKeys.ENV_PREFIX)) { String envName = key.substring(RoleKeys.ENV_PREFIX.length()); if (!envName.isEmpty()) { - env.put(envName, entry.getValue()); + String value = entry.getValue(); + if (tokenMap != null) { + for (Map.Entry<String,String> token : tokenMap.entrySet()) { + value = value.replaceAll(Pattern.quote(token.getKey()), + token.getValue()); + } + } + env.put(envName, value); } } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java index 2ec3cda..0348828 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java @@ -75,6 +75,11 @@ public abstract class AbstractLauncher extends Configured { // security protected final Credentials credentials; protected LogAggregationContext logAggregationContext; + protected boolean yarnDockerMode = false; + protected String dockerImage; + protected String yarnContainerMountPoints; + protected String runPrivilegedContainer; + /** * Create instance. @@ -97,6 +102,10 @@ public abstract class AbstractLauncher extends Configured { public ContainerLaunchContext getContainerLaunchContext() { return containerLaunchContext; } + + public void setYarnDockerMode(boolean yarnDockerMode){ + this.yarnDockerMode = yarnDockerMode; + } /** * Get the env vars to work on @@ -181,7 +190,6 @@ public abstract class AbstractLauncher extends Configured { */ public ContainerLaunchContext completeContainerLaunch() throws IOException { - String cmdStr = SliderUtils.join(commands, " ", false); log.debug("Completed setting up container command {}", cmdStr); containerLaunchContext.setCommands(commands); @@ -192,7 +200,7 @@ public abstract class AbstractLauncher extends Configured { for (Map.Entry<String, String> envPair : envVars.entrySet()) { log.debug(" \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue()); } - } + } containerLaunchContext.setEnvironment(env); //service data @@ -214,6 +222,14 @@ public abstract class AbstractLauncher extends Configured { containerLaunchContext.setTokens(CredentialUtils.marshallCredentials( credentials)); + if(yarnDockerMode){ + Map<String, String> env = containerLaunchContext.getEnvironment(); + env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker"); + env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);//if yarnDockerMode, then dockerImage is set + env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer); + log.info("yarn docker env var has been set {}", containerLaunchContext.getEnvironment().toString()); + } + return containerLaunchContext; } @@ -480,5 +496,16 @@ public abstract class AbstractLauncher extends Configured { return null; } + public void setDockerImage(String dockerImage) { + this.dockerImage = dockerImage; + } + + public void setYarnContainerMountPoints(String yarnContainerMountPoints) { + this.yarnContainerMountPoints = yarnContainerMountPoints; + } + + public void setRunPrivilegedContainer(String runPrivilegedContainer) { + this.runPrivilegedContainer = runPrivilegedContainer; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index 5ea95df..b027939 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -64,6 +64,7 @@ public interface AgentKeys { String ARG_ZOOKEEPER_REGISTRY_PATH = "--zk-reg-path"; String ARG_DEBUG = "--debug"; String AGENT_MAIN_SCRIPT_ROOT = "./infra/agent/slider-agent/"; + String AGENT_JINJA2_ROOT = "./infra/agent/slider-agent/jinja2"; String AGENT_MAIN_SCRIPT = "agent/main.py"; String APP_DEF = "application.def"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 0a1a98f..9d12f1d 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -28,9 +28,12 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ProtocolTypes; import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; +import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.Container; @@ -338,7 +341,7 @@ public class AgentProviderService extends AbstractProviderService implements Path containerTmpDirPath) throws IOException, SliderException { - + String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition .getAppConfOperations()); @@ -346,9 +349,20 @@ public class AgentProviderService extends AbstractProviderService implements log.info("Build launch context for Agent"); log.debug(instanceDefinition.toString()); + + //if we are launching docker based app on yarn, then we need to pass docker image + if (isYarnDockerContainer(role)) { + launcher.setYarnDockerMode(true); + launcher.setDockerImage(getConfigFromMetaInfo(role, "image")); + launcher.setRunPrivilegedContainer(getConfigFromMetaInfo(role, "runPriviledgedContainer")); + launcher + .setYarnContainerMountPoints(getConfigFromMetaInfoWithAppConfigOverriding( + role, "yarn.container.mount.points")); + } // Set the environment - launcher.putEnv(SliderUtils.buildEnvMap(appComponent)); + launcher.putEnv(SliderUtils.buildEnvMap(appComponent, + getStandardTokenMap(getAmState().getAppConfSnapshot(), role))); String workDir = ApplicationConstants.Environment.PWD.$(); launcher.setEnv("AGENT_WORK_ROOT", workDir); @@ -379,6 +393,7 @@ public class AgentProviderService extends AbstractProviderService implements // set PYTHONPATH List<String> pythonPaths = new ArrayList<String>(); pythonPaths.add(AgentKeys.AGENT_MAIN_SCRIPT_ROOT); + pythonPaths.add(AgentKeys.AGENT_JINJA2_ROOT); String pythonPath = StringUtils.join(File.pathSeparator, pythonPaths); launcher.setEnv(PYTHONPATH, pythonPath); log.info("PYTHONPATH set to {}", pythonPath); @@ -836,7 +851,7 @@ public class AgentProviderService extends AbstractProviderService implements response.setResponseId(id + 1L); String label = heartBeat.getHostname(); - String pkg = heartBeat.getPkg(); + String pkg = heartBeat.getPackage(); log.debug("package received: " + pkg); @@ -851,7 +866,7 @@ public class AgentProviderService extends AbstractProviderService implements CommandScript cmdScript = getScriptPathForMasterPackage(roleName); List<ComponentCommand> commands = getMetaInfo().getApplicationComponent(roleName).getCommands(); - if (!isDockerContainer(roleName) + if (!isDockerContainer(roleName) && !isYarnDockerContainer(roleName) && (cmdScript == null || cmdScript.getScript() == null) && commands.size() == 0) { log.error( @@ -880,6 +895,51 @@ public class AgentProviderService extends AbstractProviderService implements return response; } + List<ComponentStatus> statuses = heartBeat.getComponentStatus(); + if (statuses != null && !statuses.isEmpty()) { + log.info("status from agent: " + statuses.toString()); + try { + for(ComponentStatus status : statuses){ + RoleInstance role = null; + if(status.getIp() != null && !status.getIp().isEmpty()){ + role = amState.getOwnedContainer(containerId); + role.ip = status.getIp(); + } + if(status.getHostname() != null & !status.getHostname().isEmpty()){ + role = amState.getOwnedContainer(containerId); + role.hostname = status.getHostname(); + } + if (role != null) { + // create an updated service record (including hostname and ip) and publish... + ServiceRecord record = new ServiceRecord(); + record.set(YarnRegistryAttributes.YARN_ID, containerId); + record.description = roleName; + record.set(YarnRegistryAttributes.YARN_PERSISTENCE, + PersistencePolicies.CONTAINER); + // TODO: switch record attributes to use constants from YarnRegistryAttributes + // when it's been updated. + if (role.ip != null) { + record.set("yarn:ip", role.ip); + } + if (role.hostname != null) { + record.set("yarn:hostname", role.hostname); + } + yarnRegistry.putComponent( + RegistryPathUtils.encodeYarnID(containerId), record); + + } + } + + + } catch (NoSuchNodeException e) { + // ignore - there is nothing to do if we don't find a container + log.warn("Owned container {} not found - {}", containerId, e); + } catch (IOException e) { + log.warn("Error updating container {} service record in registry", + containerId, e); + } + } + Boolean isMaster = isMaster(roleName); ComponentInstanceState componentStatus = getComponentStatuses().get(label); componentStatus.heartbeat(System.currentTimeMillis()); @@ -944,7 +1004,7 @@ public class AgentProviderService extends AbstractProviderService implements componentStatus.getNextPkgToInstall(), command.toString()); if (command == Command.INSTALL) { log.info("Installing {} on {}.", roleName, containerId); - if (isDockerContainer(roleName)){ + if (isDockerContainer(roleName) || isYarnDockerContainer(roleName)){ addInstallDockerCommand(roleName, containerId, response, null, timeout); } else if (scriptPath != null) { addInstallCommand(roleName, containerId, response, scriptPath, @@ -994,7 +1054,7 @@ public class AgentProviderService extends AbstractProviderService implements boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values()); if (canExecute) { log.info("Starting {} on {}.", roleName, containerId); - if (isDockerContainer(roleName)){ + if (isDockerContainer(roleName) || isYarnDockerContainer(roleName)){ addStartDockerCommand(roleName, containerId, response, null, timeout, false); } else if (scriptPath != null) { addStartCommand(roleName, @@ -1046,7 +1106,7 @@ public class AgentProviderService extends AbstractProviderService implements && command == Command.NOP) { if (!componentStatus.getConfigReported()) { log.info("Requesting applied config for {} on {}.", roleName, containerId); - if (isDockerContainer(roleName)){ + if (isDockerContainer(roleName) || isYarnDockerContainer(roleName)){ addGetConfigDockerCommand(roleName, containerId, response); } else { addGetConfigCommand(roleName, containerId, response); @@ -1080,7 +1140,15 @@ public class AgentProviderService extends AbstractProviderService implements private boolean isDockerContainer(String roleName) { String type = getMetaInfo().getApplicationComponent(roleName).getType(); if (SliderUtils.isSet(type)) { - return type.toLowerCase().equals(SliderUtils.DOCKER); + return type.toLowerCase().equals(SliderUtils.DOCKER) || type.toLowerCase().equals(SliderUtils.DOCKER_YARN); + } + return false; + } + + private boolean isYarnDockerContainer(String roleName) { + String type = getMetaInfo().getApplicationComponent(roleName).getType(); + if (SliderUtils.isSet(type)) { + return type.toLowerCase().equals(SliderUtils.DOCKER_YARN); } return false; } @@ -2005,12 +2073,17 @@ public class AgentProviderService extends AbstractProviderService implements cmd.addContainerDetails(componentName, getMetaInfo()); Map<String, String> dockerConfig = new HashMap<String, String>(); - dockerConfig.put( - "docker.command_path", - getConfigFromMetaInfoWithAppConfigOverriding(componentName, - "commandPath")); - dockerConfig.put("docker.image_name", - getConfigFromMetaInfo(componentName, "image")); + if(isYarnDockerContainer(componentName)){ + //put nothing + cmd.setYarnDockerMode(true); + } else { + dockerConfig.put( + "docker.command_path", + getConfigFromMetaInfoWithAppConfigOverriding(componentName, + "commandPath")); + dockerConfig.put("docker.image_name", + getConfigFromMetaInfo(componentName, "image")); + } configurations.put("docker", dockerConfig); log.debug("Docker- command: {}", cmd.toString()); @@ -2122,7 +2195,7 @@ public class AgentProviderService extends AbstractProviderService implements throws SliderException { assert getAmState().isApplicationLive(); ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - if (isDockerContainer(componentName)) { + if (isDockerContainer(componentName) || isYarnDockerContainer(componentName)) { addStatusDockerCommand(componentName, containerId, response, scriptPath, timeout); return; @@ -2182,9 +2255,14 @@ public class AgentProviderService extends AbstractProviderService implements Map<String, String> dockerConfig = new HashMap<String, String>(); String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(componentName, "statusCommand"); if (statusCommand == null) { - statusCommand = "docker top " + if(isYarnDockerContainer(componentName)){ + //should complain the required field is null + cmd.setYarnDockerMode(true); + } else { + statusCommand = "docker top " + containerId + " | grep \"\"";// default value + } } dockerConfig.put("docker.status_command",statusCommand); configurations.put("docker", dockerConfig); @@ -2218,16 +2296,21 @@ public class AgentProviderService extends AbstractProviderService implements Map<String, String> dockerConfig = new HashMap<String, String>(); String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(componentName, "statusCommand"); if (statusCommand == null) { - statusCommand = "docker top " + if(isYarnDockerContainer(componentName)){ + //should complain the required field is null + cmd.setYarnDockerMode(true); + } else { + statusCommand = "docker top " + containerId + " | grep \"\"";// default value + } } dockerConfig.put("docker.status_command",statusCommand); configurations.put("docker", dockerConfig); cmd.setConfigurations(configurations); log.debug("Docker- getconfig command {}", cmd); - + response.addStatusCommand(cmd); } @@ -2309,53 +2392,70 @@ public class AgentProviderService extends AbstractProviderService implements log.debug("after resolution: " + appConf.toString()); Map<String, String> dockerConfig = new HashMap<String, String>(); - dockerConfig.put( + if (isYarnDockerContainer(componentName)) { + dockerConfig.put( + "docker.startCommand", + getConfigFromMetaInfoWithAppConfigOverriding(componentName, + "start_command")); + cmd.setYarnDockerMode(true); + } else { + dockerConfig.put( "docker.command_path", getConfigFromMetaInfoWithAppConfigOverriding(componentName, "commandPath")); - dockerConfig.put("docker.image_name", - getConfigFromMetaInfo(componentName, "image")); - // options should always have -d - String options = getConfigFromMetaInfoWithAppConfigOverriding( - componentName, "options"); - if(options != null && !options.isEmpty()){ - options = options + " -d"; - } else { - options = "-d"; + + dockerConfig.put("docker.image_name", + getConfigFromMetaInfo(componentName, "image")); + // options should always have -d + String options = getConfigFromMetaInfoWithAppConfigOverriding( + componentName, "options"); + if(options != null && !options.isEmpty()){ + options = options + " -d"; + } else { + options = "-d"; + } + dockerConfig.put("docker.options", options); + // options should always have -d + dockerConfig.put( + "docker.containerPort", + getConfigFromMetaInfoWithAppConfigOverriding(componentName, + "containerPort")); + dockerConfig + .put( + "docker.hostPort", + getConfigFromMetaInfoWithAppConfigOverriding(componentName, + "hostPort")); + + dockerConfig.put( + "docker.mounting_directory", + getConfigFromMetaInfoWithAppConfigOverriding(componentName, + "containerMount")); + dockerConfig + .put( + "docker.host_mounting_directory", + getConfigFromMetaInfoWithAppConfigOverriding(componentName, + "hostMount")); + + dockerConfig.put("docker.additional_param", + getConfigFromMetaInfoWithAppConfigOverriding(componentName, "additionalParam")); + + dockerConfig.put("docker.input_file.mount_path", getConfigFromMetaInfo( + componentName, "containerPath")); } - dockerConfig.put("docker.options", options); - // options should always have -d - dockerConfig.put( - "docker.containerPort", - getConfigFromMetaInfoWithAppConfigOverriding(componentName, - "containerPort")); - dockerConfig - .put( - "docker.hostPort", - getConfigFromMetaInfoWithAppConfigOverriding(componentName, - "hostPort")); - - dockerConfig.put( - "docker.mounting_directory", - getConfigFromMetaInfoWithAppConfigOverriding(componentName, - "containerMount")); - dockerConfig - .put( - "docker.host_mounting_directory", - getConfigFromMetaInfoWithAppConfigOverriding(componentName, - "hostMount")); - - dockerConfig.put("docker.additional_param", - getConfigFromMetaInfoWithAppConfigOverriding(componentName, "additionalParam")); - - dockerConfig.put("docker.input_file.mount_path", getConfigFromMetaInfo( - componentName, "containerPath")); + + String lifetime = getConfigFromMetaInfoWithAppConfigOverriding( + componentName, "lifetime"); + dockerConfig.put("docker.lifetime", lifetime); configurations.put("docker", dockerConfig); String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding( componentName, "statusCommand"); if (statusCommand == null) { - statusCommand = "docker top " - + containerId + " | grep \"\"";// default value + if(isYarnDockerContainer(componentName)){ + //should complain the required field is null + } else { + statusCommand = "docker top " + + containerId + " | grep \"\""; + } } dockerConfig.put("docker.status_command",statusCommand); @@ -2413,11 +2513,27 @@ public class AgentProviderService extends AbstractProviderService implements log.debug("Docker- containers metainfo: {}", containers.toString()); if (containers.size() > 0) { DockerContainer container = containers.get(0); - switch (configName) { + case "start_command": + result = container.getStartCommand(); + break; case "image": result = container.getImage(); break; + case "network": + if (container.getNetwork() == null || container.getNetwork().isEmpty()) { + result = "none"; + } else { + result = container.getNetwork(); + } + break; + case "useNetworkScript": + if (container.getUseNetworkScript() == null || container.getUseNetworkScript().isEmpty()) { + result = "yes"; + } else { + result = container.getUseNetworkScript(); + } + break; case "statusCommand": result = container.getStatusCommand(); break; @@ -2452,6 +2568,13 @@ public class AgentProviderService extends AbstractProviderService implements case "additionalParam": result = container.getAdditionalParam();// to support multi port later break; + case "runPriviledgedContainer": + if (container.getRunPrivilegedContainer() == null) { + result = "false"; + } else { + result = container.getRunPrivilegedContainer(); + } + break; default: break; } @@ -2679,12 +2802,16 @@ public class AgentProviderService extends AbstractProviderService implements Map<String, String> tokens = getStandardTokenMap(appConf, componentName); Set<String> configs = new HashSet<String>(); - configs.addAll(getApplicationConfigurationTypes()); + configs.addAll(getApplicationConfigurationTypes(componentName)); configs.addAll(getSystemConfigurationsRequested(appConf)); for (String configType : configs) { addNamedConfiguration(configType, appConf.getGlobalOptions().options, configurations, tokens, containerId, componentName); + if (appConf.getComponent(componentName) != null) { + addNamedConfiguration(configType, appConf.getComponent(componentName).options, + configurations, tokens, containerId, componentName); + } } //do a final replacement of re-used configs @@ -2752,7 +2879,7 @@ public class AgentProviderService extends AbstractProviderService implements @VisibleForTesting - protected List<String> getApplicationConfigurationTypes() { + protected List<String> getApplicationConfigurationTypes(String componentName) { List<String> configList = new ArrayList<String>(); configList.add(GLOBAL_CONFIG_TAG); @@ -2761,6 +2888,23 @@ public class AgentProviderService extends AbstractProviderService implements log.info("Expecting config type {}.", configFile.getDictionaryName()); configList.add(configFile.getDictionaryName()); } + for (Component component : getMetaInfo().getApplication().getComponents()) { + if (!component.getName().equals(componentName)) { + continue; + } + if (component.getDockerContainers() == null) { + continue; + } + for (DockerContainer container : component.getDockerContainers()) { + if (container.getConfigFiles() == null) { + continue; + } + for (ConfigFile configFile : container.getConfigFiles()) { + log.info("Expecting config type {}.", configFile.getDictionaryName()); + configList.add(configFile.getDictionaryName()); + } + } + } // remove duplicates. mostly worried about 'global' being listed return new ArrayList<String>(new HashSet<String>(configList)); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java index 55fdba6..6ee0ebb 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java @@ -46,6 +46,8 @@ public class ComponentInstanceState { private int failuresSeen = 0; private Boolean configReported = false; private long lastHeartbeat = 0; + private String ip; + private String hostname; private ContainerState containerState; private Map<String, State> pkgStatuses; @@ -314,7 +316,25 @@ public class ComponentInstanceState { sb.append(", lastHeartbeat=").append(lastHeartbeat); sb.append(", containerState=").append(containerState); sb.append(", componentName='").append(componentName).append('\''); + sb.append(", ip=").append(ip); + sb.append(", hostname='").append(hostname).append('\''); sb.append('}'); return sb.toString(); } + + public String getIp() { + return ip; + } + + public void setIp(String ip) { + this.ip = ip; + } + + public String getHostname() { + return hostname; + } + + public void setHostname(String hostname) { + this.hostname = hostname; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java index 3a445f4..63546a4 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java @@ -130,7 +130,7 @@ public class Application extends AbstractMetainfoSchema { sb.append(",\n\"version\" :").append(version); sb.append(",\n\"components\" : {"); for (Component component : components) { - sb.append("\n").append(component); + sb.append("\n").append(component.toString()); } sb.append("\n},"); sb.append('}'); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java index 7099448..3f23455 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java @@ -162,6 +162,9 @@ public class Component extends AbstractComponent { sb.append("\n\"name\": ").append(name); sb.append(",\n\"category\": ").append(category); sb.append(",\n\"commandScript\" :").append(commandScript); + for(DockerContainer dc : dockerContainers){ + sb.append(",\n\"container\" :").append(dc.toString()); + } sb.append('}'); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java index ecd0166..4c61e7f 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java @@ -35,13 +35,18 @@ public class DockerContainer implements Validate { private String name; private String image; + private String network; + private String useNetworkScript; private String options; private List<DockerContainerMount> mounts = new ArrayList<>(); private List<DockerContainerPort> ports = new ArrayList<>(); private String statusCommand; + private String startCommand; private String commandPath; private String additionalParam; + private String runPrivilegedContainer; private List<DockerContainerInputFile> inputFiles = new ArrayList<>(); + private List<ConfigFile> configFiles = new ArrayList<>(); public DockerContainer() { } @@ -75,6 +80,22 @@ public class DockerContainer implements Validate { this.image = image; } + public String getNetwork() { + return network; + } + + public void setNetwork(String network) { + this.network = network; + } + + public String getUseNetworkScript() { + return useNetworkScript; + } + + public void setUseNetworkScript(String useNetworkScript) { + this.useNetworkScript = useNetworkScript; + } + public String getOptions() { return options; } @@ -95,10 +116,12 @@ public class DockerContainer implements Validate { } } + @JsonProperty("statusCommand") public String getStatusCommand() { return statusCommand; } + @JsonProperty("statusCommand") public void setStatusCommand(String statusCommand) { this.statusCommand = statusCommand; } @@ -119,14 +142,46 @@ public class DockerContainer implements Validate { this.additionalParam = additionalParam; } + @JsonProperty("startCommand") + public String getStartCommand() { + return startCommand; + } + + @JsonProperty("startCommand") + public void setStartCommand(String startCommand) { + this.startCommand = startCommand; + } + + @JsonProperty("runPrivilegedContainer") + public String getRunPrivilegedContainer() { + return runPrivilegedContainer; + } + + @JsonProperty("runPrivilegedContainer") + public void setRunPrivilegedContainer(String runPrivilegedContainer) { + this.runPrivilegedContainer = runPrivilegedContainer; + } + + public List<ConfigFile> getConfigFiles() { + return configFiles; + } + + public void setConfigFiles(List<ConfigFile> configFiles) { + this.configFiles = configFiles; + } + @Override public String toString() { StringBuilder result = new StringBuilder("DockerContainer [name=") - .append(name).append(", image=").append(image).append(", options=") - .append(options).append(", mounts=").append(mounts).append(", ports=") - .append(ports).append(", statusCommand=").append(statusCommand) - .append(", commandPath=").append(commandPath).append(", additionalParam=") - .append(additionalParam).append(", inputFiles=").append(inputFiles).append("]"); + .append(name).append(", image=").append(image).append(", options=") + .append(options).append(", mounts=").append(mounts).append(", ports=") + .append(ports).append(", statusCommand=").append(statusCommand) + .append(", commandPath=").append(commandPath) + .append(", additionalParam=").append(additionalParam) + .append(", inputFiles=").append(inputFiles).append(", startCommand=") + .append(startCommand).append(", runPriviledgedContainer=") + .append(runPrivilegedContainer).append(", configFiles=") + .append(configFiles).append("]"); return result.toString(); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index ba5a060..0759c5a 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -2227,6 +2227,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService ContainerLaunchContext ctx, RoleInstance instance) throws IOException { appState.containerStartSubmitted(container, instance); + nmClientAsync.startContainerAsync(container, ctx); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index c8369c1..a11ce56 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -1703,6 +1703,7 @@ public class AppState { //build the map of node -> containers Map<String, Map<String, ClusterNode>> clusterNodes = createRoleToClusterNodeMap(); + log.info("app state clusterNodes {} ", clusterNodes.toString()); cd.status = new HashMap<>(); cd.status.put(ClusterDescriptionKeys.KEY_CLUSTER_LIVE, clusterNodes); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java index 30a2bb0..345c67e 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java @@ -98,6 +98,8 @@ public final class RoleInstance implements Cloneable { */ public String[] environment; + public String ip; + public String hostname; public String host; public String hostURL; public ContainerAllocationOutcome placement; @@ -231,6 +233,8 @@ public final class RoleInstance implements Cloneable { node.environment = Arrays.copyOf(environment, environment.length); } node.exitCode = exitCode; + node.ip = ip; + node.hostname = hostname; node.host = host; node.hostUrl = hostURL; if (output != null) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java index 63bdd0c..acdc234 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java @@ -34,6 +34,8 @@ public class ComponentStatus { String serviceName; String clusterName; String roleCommand; + String ip; + String hostname; @JsonProperty("configurations") private Map<String, Map<String, String>> configurations; @@ -104,6 +106,24 @@ public class ComponentStatus { ", serviceName='" + serviceName + '\'' + ", clusterName='" + clusterName + '\'' + ", roleCommand='" + roleCommand + '\'' + + ", ip='" + ip + '\'' + + ", hostname='" + hostname + '\'' + '}'; } + + public String getIp() { + return ip; + } + + public void setIp(String ip) { + this.ip = ip; + } + + public String getHostname() { + return hostname; + } + + public void setHostname(String hostname) { + this.hostname = hostname; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java index e0d3a75..eb97a55 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java @@ -59,6 +59,7 @@ public class ExecutionCommand { private String componentType; private List<DockerContainer> containers = new ArrayList<>(); private String pkg; + private boolean yarnDockerMode = false; public ExecutionCommand(AgentCommandType commandType) { this.commandType = commandType; @@ -229,7 +230,16 @@ public class ExecutionCommand { public List<DockerContainer> getContainers() { return containers; } - + + @JsonProperty("yarnDockerMode") + public boolean isYarnDockerMode() { + return yarnDockerMode ; + } + + @JsonProperty("yarnDockerMode") + public void setYarnDockerMode(boolean yarnDockerMode) { + this.yarnDockerMode = yarnDockerMode; + } @Override public String toString() { StringBuilder builder = new StringBuilder(); @@ -243,7 +253,8 @@ public class ExecutionCommand { .append(", commandParams=").append(commandParams) .append(", serviceName=").append(serviceName) .append(", componentName=").append(componentName) - .append(", componentType=").append(componentType).append(", pkg=") + .append(", componentType=").append(componentType) + .append(", yarnDockerMode=").append(yarnDockerMode).append(", pkg=") .append(pkg).append("]"); return builder.toString(); } @@ -256,11 +267,14 @@ public class ExecutionCommand { for (DockerContainer metaContainer : component.getDockerContainers()) { DockerContainer container = new DockerContainer(); container.setImage(metaContainer.getImage()); + container.setNetwork(metaContainer.getNetwork()); + container.setUseNetworkScript(metaContainer.getUseNetworkScript()); container.setName(metaContainer.getName()); container.setOptions(metaContainer.getOptions()); container.setAdditionalParam(metaContainer.getAdditionalParam()); container.setCommandPath(metaContainer.getAdditionalParam()); container.setStatusCommand(metaContainer.getStatusCommand()); + container.setStartCommand(metaContainer.getStartCommand()); if (metaContainer.getMounts().size() > 0) { for (DockerContainerMount metaContMount : metaContainer.getMounts()) { DockerContainerMount contMnt = new DockerContainerMount(); @@ -286,6 +300,9 @@ public class ExecutionCommand { container.getInputFiles().add(inpFile); } } + if (metaContainer.getConfigFiles() != null) { + container.setConfigFiles(metaContainer.getConfigFiles()); + } log.info("Docker container meta info ready: " + container.toString()); this.getContainers().add(container); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java index b7172c2..d17c465 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java @@ -125,12 +125,12 @@ public class HeartBeat { } @JsonProperty("package") - public String getPkg() { + public String getPackage() { return pkg; } @JsonProperty("package") - public void setPkg(String pkg) { + public void setPackage(String pkg) { this.pkg = pkg; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java index c118840..b500d67 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java @@ -39,6 +39,7 @@ public class HeartBeatResponse { RegistrationCommand registrationCommand; + boolean yarnDockerMode = false; boolean restartAgent = false; boolean restartEnabled = true; boolean hasMappedComponents = false; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java index 50a3c87..80b7a5e 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java @@ -58,6 +58,22 @@ public class RegistrationResponse { public RegistrationResponse() { } + public RegistrationStatus getResponse() { + return response; + } + + public void setResponse(RegistrationStatus response) { + this.response = response; + } + + public int getExitstatus() { + return exitstatus; + } + + public void setExitstatus(int exitstatus) { + this.exitstatus = exitstatus; + } + public RegistrationStatus getResponseStatus() { return response; } @@ -90,8 +106,8 @@ public class RegistrationResponse { this.tags = tags; } - public void setExitstatus(int exitstatus) { - this.exitstatus = exitstatus; + public String getLog() { + return log; } public void setLog(String log) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java index 52fb8dd..5b205b5 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java @@ -41,6 +41,7 @@ public class StatusCommand { private Map<String, String> commandParams = new HashMap<String, String>(); private Map<String, String> hostLevelParams = new HashMap<String, String>(); private String roleCommand; + private boolean yarnDockerMode; public StatusCommand() { this.agentCommandType = AgentCommandType.STATUS_COMMAND; @@ -125,6 +126,16 @@ public class StatusCommand { public void setRoleCommand(String roleCommand) { this.roleCommand = roleCommand; } + + @JsonProperty("yarnDockerMode") + public boolean isYarnDockerMode() { + return yarnDockerMode; + } + + @JsonProperty("yarnDockerMode") + public void setYarnDockerMode(boolean yarnDockerMode) { + this.yarnDockerMode = yarnDockerMode; + } @Override public String toString() { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/appConfig.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/appConfig.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/appConfig.json new file mode 100644 index 0000000..8839efc --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/appConfig.json @@ -0,0 +1,10 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": {}, + "global": {}, + "components": { + "YCLOUD": { + "ycloud.statusCommand": "ls" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/metainfo.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/metainfo.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/metainfo.json new file mode 100644 index 0000000..79c80e9 --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/metainfo.json @@ -0,0 +1,44 @@ +{ + "schemaVersion": "2.1", + "application": { + "name": "MYYCLOUD", + "components": [ + { + "name": "YCLOUD1", + "type": "yarn_docker", + "dockerContainers": [ + { + "name": "ycloud1", + "image": "doesnotexist", + "startCommand": "ls", + "yarnContainerMountPoint": "null", + "ports": [ + { + "containerPort": "8080", + "hostPort": "49160" + } + ] + } + ] + }, + { + "name": "YCLOUD2", + "type": "yarn_docker", + "dockerContainers": [ + { + "name": "ycloud2", + "image": "doesnotexist", + "startCommand": "ls", + "yarnContainerMountPoint": "null", + "ports": [ + { + "containerPort": "8080", + "hostPort": "49161" + } + ] + } + ] + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/resources.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/resources.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/resources.json new file mode 100644 index 0000000..985d8e3 --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/resources.json @@ -0,0 +1,20 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": {}, + "global": {}, + "components": { + "slider-appmaster": {}, + "YCLOUD1": { + "yarn.role.priority": "1", + "yarn.component.instances": 2, + "yarn.memory": "256", + "yarn.vcores": "1" + }, + "YCLOUD2": { + "yarn.role.priority": "2", + "yarn.component.instances": 2, + "yarn.memory": "256", + "yarn.vcores": "1" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/appConfig.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/appConfig.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/appConfig.json new file mode 100644 index 0000000..8839efc --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/appConfig.json @@ -0,0 +1,10 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": {}, + "global": {}, + "components": { + "YCLOUD": { + "ycloud.statusCommand": "ls" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/metainfo.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/metainfo.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/metainfo.json new file mode 100644 index 0000000..67d6171 --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/metainfo.json @@ -0,0 +1,26 @@ +{ + "schemaVersion": "2.1", + "application": { + "name": "MYYCLOUD", + "components": [ + { + "name": "YCLOUD", + "type": "yarn_docker", + "dockerContainers": [ + { + "name": "ycloud", + "image": "ycloud-centos6:latest", + "startCommand": "ls", + "yarnContainerMountPoint": "null", + "ports": [ + { + "containerPort": "8080", + "hostPort": "49160" + } + ] + } + ] + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/resources.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/resources.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/resources.json new file mode 100644 index 0000000..22ff65e --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/resources.json @@ -0,0 +1,14 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": {}, + "global": {}, + "components": { + "slider-appmaster": {}, + "YCLOUD": { + "yarn.role.priority": "1", + "yarn.component.instances": 1, + "yarn.memory": "256", + "yarn.vcores": "1" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/appConfig.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/appConfig.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/appConfig.json new file mode 100644 index 0000000..8839efc --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/appConfig.json @@ -0,0 +1,10 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": {}, + "global": {}, + "components": { + "YCLOUD": { + "ycloud.statusCommand": "ls" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/metainfo.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/metainfo.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/metainfo.json new file mode 100644 index 0000000..efbb638 --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/metainfo.json @@ -0,0 +1,44 @@ +{ + "schemaVersion": "2.1", + "application": { + "name": "MYYCLOUD", + "components": [ + { + "name": "YCLOUD1", + "type": "yarn_docker", + "dockerContainers": [ + { + "name": "ycloud1", + "image": "ycloud-centos6:latest", + "startCommand": "ls", + "yarnContainerMountPoint": "null", + "ports": [ + { + "containerPort": "8080", + "hostPort": "49160" + } + ] + } + ] + }, + { + "name": "YCLOUD2", + "type": "yarn_docker", + "dockerContainers": [ + { + "name": "ycloud2", + "image": "ycloud-centos6:latest", + "startCommand": "ls", + "yarnContainerMountPoint": "null", + "ports": [ + { + "containerPort": "8080", + "hostPort": "49161" + } + ] + } + ] + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/resources.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/resources.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/resources.json new file mode 100644 index 0000000..0cbb88a --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/resources.json @@ -0,0 +1,20 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": {}, + "global": {}, + "components": { + "slider-appmaster": {}, + "YCLOUD1": { + "yarn.role.priority": "1", + "yarn.component.instances": 1, + "yarn.memory": "256", + "yarn.vcores": "1" + }, + "YCLOUD2": { + "yarn.role.priority": "2", + "yarn.component.instances": 1, + "yarn.memory": "256", + "yarn.vcores": "1" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/appConfig.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/appConfig.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/appConfig.json new file mode 100644 index 0000000..8839efc --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/appConfig.json @@ -0,0 +1,10 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": {}, + "global": {}, + "components": { + "YCLOUD": { + "ycloud.statusCommand": "ls" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/metainfo.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/metainfo.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/metainfo.json new file mode 100644 index 0000000..efbb638 --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/metainfo.json @@ -0,0 +1,44 @@ +{ + "schemaVersion": "2.1", + "application": { + "name": "MYYCLOUD", + "components": [ + { + "name": "YCLOUD1", + "type": "yarn_docker", + "dockerContainers": [ + { + "name": "ycloud1", + "image": "ycloud-centos6:latest", + "startCommand": "ls", + "yarnContainerMountPoint": "null", + "ports": [ + { + "containerPort": "8080", + "hostPort": "49160" + } + ] + } + ] + }, + { + "name": "YCLOUD2", + "type": "yarn_docker", + "dockerContainers": [ + { + "name": "ycloud2", + "image": "ycloud-centos6:latest", + "startCommand": "ls", + "yarnContainerMountPoint": "null", + "ports": [ + { + "containerPort": "8080", + "hostPort": "49161" + } + ] + } + ] + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/resources.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/resources.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/resources.json new file mode 100644 index 0000000..399a301 --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/resources.json @@ -0,0 +1,20 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": {}, + "global": {}, + "components": { + "slider-appmaster": {}, + "YCLOUD1": { + "yarn.role.priority": "1", + "yarn.component.instances": 2, + "yarn.memory": "256", + "yarn.vcores": "1" + }, + "YCLOUD2": { + "yarn.role.priority": "2", + "yarn.component.instances": 3, + "yarn.memory": "256", + "yarn.vcores": "1" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/appConfig.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/appConfig.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/appConfig.json new file mode 100644 index 0000000..8839efc --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/appConfig.json @@ -0,0 +1,10 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": {}, + "global": {}, + "components": { + "YCLOUD": { + "ycloud.statusCommand": "ls" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/metainfo.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/metainfo.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/metainfo.json new file mode 100644 index 0000000..2befb20 --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/metainfo.json @@ -0,0 +1,44 @@ +{ + "schemaVersion": "2.1", + "application": { + "name": "MYYCLOUD", + "components": [ + { + "name": "YCLOUD1", + "type": "yarn_docker", + "dockerContainers": [ + { + "name": "ycloud1", + "image": "doesnotexist", + "startCommand": "ls", + "yarnContainerMountPoint": "null", + "ports": [ + { + "containerPort": "8080", + "hostPort": "49160" + } + ] + } + ] + }, + { + "name": "YCLOUD2", + "type": "yarn_docker", + "dockerContainers": [ + { + "name": "ycloud2", + "image": "ycloud-centos6:latest", + "startCommand": "ls", + "yarnContainerMountPoint": "null", + "ports": [ + { + "containerPort": "8080", + "hostPort": "49161" + } + ] + } + ] + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fffd4e63/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/resources.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/resources.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/resources.json new file mode 100644 index 0000000..985d8e3 --- /dev/null +++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/resources.json @@ -0,0 +1,20 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": {}, + "global": {}, + "components": { + "slider-appmaster": {}, + "YCLOUD1": { + "yarn.role.priority": "1", + "yarn.component.instances": 2, + "yarn.memory": "256", + "yarn.vcores": "1" + }, + "YCLOUD2": { + "yarn.role.priority": "2", + "yarn.component.instances": 2, + "yarn.memory": "256", + "yarn.vcores": "1" + } + } +} \ No newline at end of file
