SLIDER-773. Add co-processor support for app packages (Thomas Liu via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/016cd66a Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/016cd66a Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/016cd66a Branch: refs/heads/develop Commit: 016cd66aee97fe9752f78f9f8eb1283c162cc4d3 Parents: 4270267 Author: Sumit Mohanty <[email protected]> Authored: Sat Apr 11 20:21:50 2015 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Sat Apr 11 20:22:10 2015 -0700 ---------------------------------------------------------------------- .../src/main/python/agent/ActionQueue.py | 13 +- .../src/main/python/agent/AgentConfig.py | 2 + .../src/main/python/agent/Controller.py | 7 + .../python/agent/CustomServiceOrchestrator.py | 12 +- slider-agent/src/main/python/agent/Heartbeat.py | 3 +- slider-agent/src/main/python/agent/main.py | 3 +- .../src/test/python/agent/TestHeartbeat.py | 6 +- slider-core/pom.xml | 10 ++ .../core/persist/AppDefinitionPersister.java | 14 +- .../providers/agent/AgentClientProvider.java | 4 +- .../slider/providers/agent/AgentKeys.java | 2 + .../providers/agent/AgentProviderService.java | 152 ++++++++++++++++--- .../slider/providers/agent/AgentUtils.java | 17 ++- .../apache/slider/providers/agent/Command.java | 1 + .../providers/agent/ComponentCommandOrder.java | 2 + .../providers/agent/ComponentInstanceState.java | 132 +++++++++++++--- .../providers/agent/HeartbeatMonitor.java | 2 + .../apache/slider/providers/agent/State.java | 7 + .../application/metadata/AbstractComponent.java | 77 ++++++++++ .../metadata/AbstractMetainfoParser.java | 117 ++++++++++++++ .../metadata/AbstractMetainfoSchema.java | 72 +++++++++ .../metadata/AddoPackageMetainfoParser.java | 53 +++++++ .../agent/application/metadata/Application.java | 10 +- .../metadata/ApplicationPackage.java | 51 +++++++ .../agent/application/metadata/Component.java | 19 +-- .../metadata/ComponentsInAddonPackage.java | 13 ++ .../agent/application/metadata/Metainfo.java | 27 ++++ .../application/metadata/MetainfoParser.java | 85 +---------- .../web/rest/agent/ExecutionCommand.java | 11 ++ .../appmaster/web/rest/agent/HeartBeat.java | 12 ++ .../appmaster/web/rest/agent/Register.java | 9 ++ .../web/rest/agent/RegistrationResponse.java | 11 ++ .../agent/AgentMiniClusterTestBase.groovy | 25 +++ .../common/tools/TestWindowsSupport.groovy | 2 +- .../agent/TestAgentProviderService.java | 41 +++-- .../TestApplicationWithAddonPackagesIT.groovy | 63 ++++++++ 36 files changed, 911 insertions(+), 176 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-agent/src/main/python/agent/ActionQueue.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py index 4cb5de7..6e3c3e2 100644 --- a/slider-agent/src/main/python/agent/ActionQueue.py +++ b/slider-agent/src/main/python/agent/ActionQueue.py @@ -63,10 +63,11 @@ class ActionQueue(threading.Thread): self.controller = controller self._stop = threading.Event() self.tmpdir = config.getResolvedPath(AgentConfig.APP_TASK_DIR) + self.componentPackage = '' self.customServiceOrchestrator = CustomServiceOrchestrator(config, controller, self.queueOutAgentToggleLogger) - + def stop(self): self._stop.set() @@ -121,6 +122,12 @@ class ActionQueue(threading.Thread): ''' clusterName = command['clusterName'] commandId = command['commandId'] + if 'package' in command: + self.componentPackage = command['package'] + else: + self.componentPackage = '' + + logger.info("package received: " + self.componentPackage + ";") message = "Executing command with id = {commandId} for role = {role} of " \ "cluster {cluster}".format( @@ -155,7 +162,9 @@ class ActionQueue(threading.Thread): if store_command: logger.info("Component has indicated auto-restart. Saving details from START command.") - + + logger.info("will run: " + str(command)) + # running command commandresult = self.customServiceOrchestrator.runCommand(command, in_progress_status[ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-agent/src/main/python/agent/AgentConfig.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/AgentConfig.py b/slider-agent/src/main/python/agent/AgentConfig.py index 86925b1..a78c22b 100644 --- a/slider-agent/src/main/python/agent/AgentConfig.py +++ b/slider-agent/src/main/python/agent/AgentConfig.py @@ -87,6 +87,8 @@ class AgentConfig: # the location of the app package APP_PACKAGE_DIR = "app_pkg_dir" + + ADDON_PKG_ROOT_DIR = "addon/definition" # the location where the app component is installed APP_INSTALL_DIR = "app_install_dir" # the location to store component instance PID directories http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-agent/src/main/python/agent/Controller.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py index 2d823b5..128b226 100644 --- a/slider-agent/src/main/python/agent/Controller.py +++ b/slider-agent/src/main/python/agent/Controller.py @@ -448,11 +448,18 @@ class Controller(threading.Thread): index = 0 deleteIndex = 0 delete = False + ''' Do not break for START command, since we might get a STOP command (used during failure scenarios to gracefully attempt stop) ''' + for command in commands: + + if "package" in command and command["package"] != "MASTER": + # we do not update component state upon add on package command + continue + if command["roleCommand"] == "START": self.componentExpectedState = State.STARTED self.componentActualState = State.STARTING http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py index ce0787f..71b9799 100644 --- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py +++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py @@ -94,7 +94,17 @@ class CustomServiceOrchestrator(): if script_type.upper() == self.SCRIPT_TYPE_PYTHON: script = command['commandParams']['script'] timeout = int(command['commandParams']['command_timeout']) - script_path = self.resolve_script_path(self.base_dir, script, script_type) + script_path = '' + if 'package' in command: + add_on_dir_str = self.config.getWorkRootPath() + "/" + AgentConfig.ADDON_PKG_ROOT_DIR + "/application.addon." + command['package'] + add_on_base_dir = os.path.realpath(posixpath.join(add_on_dir_str, "package")) + logger.info("add on: " + command['package'] + " add on base dir is: " + str(add_on_base_dir)) + script_path = self.resolve_script_path(add_on_base_dir, script, script_type) + + else: + self.base_dir = os.path.realpath(posixpath.join(self.config.getResolvedPath(AgentConfig.APP_PACKAGE_DIR), "package")) + logger.info("base dir is: " + str(self.base_dir)) + script_path = self.resolve_script_path(self.base_dir, script, script_type) script_tuple = (script_path, self.base_dir) py_file_list = [script_tuple] http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-agent/src/main/python/agent/Heartbeat.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/Heartbeat.py b/slider-agent/src/main/python/agent/Heartbeat.py index e157ce3..db9c2d5 100644 --- a/slider-agent/src/main/python/agent/Heartbeat.py +++ b/slider-agent/src/main/python/agent/Heartbeat.py @@ -45,11 +45,12 @@ class Heartbeat: nodeStatus = {"status": "HEALTHY", "cause": "NONE"} - + logger.info("package to heartbeat: " + self.actionQueue.componentPackage) heartbeat = {'responseId': int(id), 'timestamp': timestamp, 'hostname': self.config.getLabel(), 'nodeStatus': nodeStatus, + 'package': self.actionQueue.componentPackage, 'fqdn': hostname.public_hostname() } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-agent/src/main/python/agent/main.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py index 3fdd938..a706e70 100644 --- a/slider-agent/src/main/python/agent/main.py +++ b/slider-agent/src/main/python/agent/main.py @@ -113,6 +113,7 @@ def update_config_from_file(agentConfig): try: configFile = posixpath.join(agentConfig.getWorkRootPath(), configFileRelPath) if os.path.exists(configFile): + logger.info("config file is: " + configFile) agentConfig.setConfig(configFile) else: logger.warn("No config found, using default") @@ -212,7 +213,7 @@ def main(): # Check for configuration file. agentConfig = AgentConfig(options.root_folder, options.log_folder, options.label) update_config_from_file(agentConfig) - + # update configurations if needed if options.zk_quorum: agentConfig.set(AgentConfig.SERVER_SECTION, Constants.ZK_QUORUM, options.zk_quorum) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-agent/src/test/python/agent/TestHeartbeat.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/test/python/agent/TestHeartbeat.py b/slider-agent/src/test/python/agent/TestHeartbeat.py index 9898444..769e235 100644 --- a/slider-agent/src/test/python/agent/TestHeartbeat.py +++ b/slider-agent/src/test/python/agent/TestHeartbeat.py @@ -153,7 +153,7 @@ class TestHeartbeat(TestCase): hb['hostname'] = 'hostname' hb['timestamp'] = 'timestamp' hb['fqdn'] = 'fqdn' - expected = {'nodeStatus': + expected = {'package': '', 'nodeStatus': {'status': 'HEALTHY', 'cause': 'NONE'}, 'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn', @@ -208,7 +208,7 @@ class TestHeartbeat(TestCase): hb['hostname'] = 'hostname' hb['timestamp'] = 'timestamp' hb['fqdn'] = 'fqdn' - expected = {'nodeStatus': + expected = {'package': '', 'nodeStatus': {'status': 'HEALTHY', 'cause': 'NONE'}, 'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn', @@ -245,7 +245,7 @@ class TestHeartbeat(TestCase): hb['hostname'] = 'hostname' hb['timestamp'] = 'timestamp' hb['fqdn'] = 'fqdn' - expected = {'nodeStatus': + expected = {'package': '', 'nodeStatus': {'status': 'HEALTHY', 'cause': 'NONE'}, 'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn', http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/pom.xml ---------------------------------------------------------------------- diff --git a/slider-core/pom.xml b/slider-core/pom.xml index 22bc868..e84fe0e 100644 --- a/slider-core/pom.xml +++ b/slider-core/pom.xml @@ -508,6 +508,16 @@ <scope>test</scope> </dependency> +<dependency> + + <groupId>asm</groupId> + + <artifactId>asm</artifactId> + + <version>3.3.1</version> + +</dependency> + </dependencies> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java index f394542..5a3d393 100644 --- a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java +++ b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java @@ -79,8 +79,8 @@ public class AppDefinitionPersister { } File src = appDefinition.appDefPkgOrFolder; - String targetName = appDefinition.appDefPkgOrFolder.getName(); - + String targetName = appDefinition.pkgName; + log.debug("first targetName: " + targetName); if (appDefinition.appDefPkgOrFolder.isDirectory()) { log.info("Processing app package/folder {} for {}", appDefinition.appDefPkgOrFolder.getAbsolutePath(), @@ -92,7 +92,8 @@ public class AppDefinitionPersister { src = zipFile; targetName = appDefinition.pkgName; } - + log.debug("final targetName: " + targetName); + sliderFileSystem.getFileSystem().copyFromLocalFile( false, false, @@ -173,6 +174,13 @@ public class AppDefinitionPersister { Path addonPath = sliderFileSystem.buildAddonDirPath(clustername, key); String addonPkgName = "addon_" + key + ".zip"; + + log.debug("addonMap.get(key): " + addonMap.get(key) + + " addonPath: " + addonPath + + " defPath: " + defPath + + " addonPkgName: " + addonPkgName); + + //String addonPkgName = key + ".zip"; appDefinitions.add(new AppDefinition(addonPath, defPath, addonPkgName)); String addOnKey = AgentKeys.ADDON_PREFIX + key; Path addonPkgPath = new Path(addonPath, addonPkgName); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index 5cd7ead..0c1389f 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -169,7 +169,7 @@ public class AgentClientProvider extends AbstractClientProvider Metainfo metaInfo = null; if (fs != null) { try { - metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef); + metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef, false); } catch (IOException ioe) { // Ignore missing metainfo file for now log.info("Missing metainfo {}", ioe.getMessage()); @@ -281,7 +281,7 @@ public class AgentClientProvider extends AbstractClientProvider Set<String> tags; Metainfo metainfo; try { - metainfo = AgentUtils.getApplicationMetainfo(fileSystem, appDef); + metainfo = AgentUtils.getApplicationMetainfo(fileSystem, appDef, false); } catch (IOException e) { log.error("Error retrieving metainfo from {}", appDef, e); throw new SliderException("Error retrieving metainfo", e); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index 963dadc..710ced1 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -71,9 +71,11 @@ public interface AgentKeys { String ADDONS = "application.addons"; String AGENT_VERSION = "agent.version"; String AGENT_CONF = "agent.conf"; + String ADDON_FOR_ALL_COMPONENTS = "ALL"; String AGENT_INSTALL_DIR = "infra/agent"; String APP_DEFINITION_DIR = "app/definition"; + String ADDON_DEFINITION_DIR = "addon/definition"; String AGENT_CONFIG_FILE = "infra/conf/agent.ini"; String AGENT_VERSION_FILE = "infra/version"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 76353ff..832d149 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -70,6 +70,7 @@ import org.apache.slider.providers.agent.application.metadata.CommandScript; import org.apache.slider.providers.agent.application.metadata.Component; import org.apache.slider.providers.agent.application.metadata.ComponentCommand; import org.apache.slider.providers.agent.application.metadata.ComponentExport; +import org.apache.slider.providers.agent.application.metadata.ComponentsInAddonPackage; import org.apache.slider.providers.agent.application.metadata.ConfigFile; import org.apache.slider.providers.agent.application.metadata.DefaultConfig; import org.apache.slider.providers.agent.application.metadata.Export; @@ -117,6 +118,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Scanner; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; @@ -176,6 +178,8 @@ public class AgentProviderService extends AbstractProviderService implements new ConcurrentHashMap<String, Map<String, List<ExportEntry>>>(); private final Map<String, Map<String, String>> allocatedPorts = new ConcurrentHashMap<String, Map<String, String>>(); + private final Map<String, Metainfo> packageMetainfo = + new ConcurrentHashMap<String, Metainfo>(); private final Map<String, ExportEntry> logFolderExports = Collections.synchronizedMap(new LinkedHashMap<String, ExportEntry>(MAX_LOG_ENTRIES, 0.75f, false) { @@ -265,13 +269,15 @@ public class AgentProviderService extends AbstractProviderService implements String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition .getAppConfOperations()); + if (metaInfo == null) { synchronized (syncLock) { if (metaInfo == null) { readAndSetHeartbeatMonitoringInterval(instanceDefinition); initializeAgentDebugCommands(instanceDefinition); - metaInfo = getApplicationMetainfo(fileSystem, appDef); + metaInfo = getApplicationMetainfo(fileSystem, appDef, false); + log.info("master package metainfo: {}", metaInfo.toString()); if (metaInfo == null || metaInfo.getApplication() == null) { log.error("metainfo.xml is unavailable or malformed at {}.", appDef); throw new SliderException( @@ -281,6 +287,29 @@ public class AgentProviderService extends AbstractProviderService implements defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metaInfo); monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval()); monitor.start(); + + //build a map from component to metainfo + String addonAppDefString = instanceDefinition.getAppConfOperations() + .getGlobalOptions().getOption(AgentKeys.ADDONS, null); + log.debug("all addon appdef are: {}", addonAppDefString); + if (addonAppDefString != null) { + Scanner scanner = new Scanner(addonAppDefString).useDelimiter(","); + while (scanner.hasNext()) { + String addonAppDef = scanner.next(); + String addonAppDefPath = instanceDefinition + .getAppConfOperations().getGlobalOptions().get(addonAppDef); + log.debug("this addon package {} is stored at: {}", addonAppDef + + addonAppDefPath); + Metainfo addonMetaInfo = getApplicationMetainfo(fileSystem, + addonAppDefPath, true); + addonMetaInfo.validate(); + log.debug("this addon pkg metainfo: {}", addonMetaInfo.toString()); + packageMetainfo.put(addonMetaInfo.getApplicationPackage() + .getName(), addonMetaInfo); + } + log.debug("metainfo map for master and addon is: {}", + packageMetainfo.toString()); + } } } } @@ -454,14 +483,47 @@ public class AgentProviderService extends AbstractProviderService implements launcher.addCommand(operation.build()); + //localize addon package + String addonAppDefString = instanceDefinition.getAppConfOperations() + .getGlobalOptions().getOption(AgentKeys.ADDONS, null); + log.debug("all addon appdef are: {}", addonAppDefString); + if (addonAppDefString != null) { + Scanner scanner = new Scanner(addonAppDefString).useDelimiter(","); + while (scanner.hasNext()) { + String addonAppDef = scanner.next(); + String addonAppDefPath = instanceDefinition + .getAppConfOperations().getGlobalOptions().get(addonAppDef); + log.debug("this addon package {} is stored at: {}", addonAppDef, addonAppDefPath); + LocalResource addonPkgRes = fileSystem.createAmResource( + fileSystem.getFileSystem().resolvePath(new Path(addonAppDefPath)), + LocalResourceType.ARCHIVE); + launcher.addLocalResource(AgentKeys.ADDON_DEFINITION_DIR + "/" + addonAppDef, addonPkgRes); + } + log.debug("metainfo map for master and addon is: {}", + packageMetainfo.toString()); + } + + //initialize addon pkg states for all componentInstanceStatus + TreeMap<String, State> pkgStatuses = new TreeMap<String, State>(); + for(Metainfo appPkg : packageMetainfo.values()){ + //check each component of that addon to see if they apply to this component 'role' + for(ComponentsInAddonPackage comp : appPkg.getApplicationPackage().getComponents()){ + log.debug("current component: {} component in metainfo {}", role, comp.getName()); + if(comp.getName().equals(role) || comp.getName().equals(AgentKeys.ADDON_FOR_ALL_COMPONENTS)){ + pkgStatuses.put(appPkg.getApplicationPackage().getName(), State.INIT); + } + } + } + log.debug("for component: {} pkg status map: {}", role, pkgStatuses.toString()); + // initialize the component instance state getComponentStatuses().put(label, new ComponentInstanceState( role, container.getId(), - getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME))); + getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME), pkgStatuses)); } - + private void localizeContainerSecurityStores(ContainerLauncher launcher, Container container, String role, @@ -636,8 +698,7 @@ public class AgentProviderService extends AbstractProviderService implements String label = getContainerLabel(container, roleName); log.info("Rebuilding in-memory: container {} in role {} in cluster {}", container.getId(), roleName, applicationId); - getComponentStatuses().put( - label, + getComponentStatuses().put(label, new ComponentInstanceState(roleName, container.getId(), applicationId)); } else { @@ -664,7 +725,11 @@ public class AgentProviderService extends AbstractProviderService implements log.info("Handling registration: {}", registration); RegistrationResponse response = new RegistrationResponse(); String label = registration.getLabel(); + String pkg = registration.getPkg(); State agentState = registration.getActualState(); + + log.info("label: " + label + " pkg: " + pkg); + if (getComponentStatuses().containsKey(label)) { response.setResponseStatus(RegistrationStatus.OK); ComponentInstanceState componentStatus = getComponentStatuses().get(label); @@ -714,6 +779,10 @@ public class AgentProviderService extends AbstractProviderService implements response.setResponseId(id + 1L); String label = heartBeat.getHostname(); + String pkg = heartBeat.getPkg(); + + log.debug("package received: " + pkg); + String roleName = getRoleName(label); String containerId = getContainerId(label); boolean doUpgrade = false; @@ -722,7 +791,7 @@ public class AgentProviderService extends AbstractProviderService implements } StateAccessForProviders accessor = getAmState(); - CommandScript cmdScript = getScriptPathFromMetainfo(roleName); + CommandScript cmdScript = getScriptPathForMasterPackage(roleName); List<ComponentCommand> commands = getMetaInfo().getApplicationComponent(roleName).getCommands(); if ((cmdScript == null || cmdScript.getScript() == null) && commands.size() == 0) { @@ -773,7 +842,7 @@ public class AgentProviderService extends AbstractProviderService implements } publishConfigAndExportGroups(heartBeat, componentStatus, roleName); - + CommandResult result = null; List<CommandReport> reports = heartBeat.getReports(); if (SliderUtils.isNotEmpty(reports)) { CommandReport report = reports.get(0); @@ -781,13 +850,14 @@ public class AgentProviderService extends AbstractProviderService implements if (SliderUtils.isNotEmpty(ports)) { processAllocatedPorts(heartBeat.getFqdn(), roleName, containerId, ports); } - CommandResult result = CommandResult.getCommandResult(report.getStatus()); + result = CommandResult.getCommandResult(report.getStatus()); + Command command = Command.getCommand(report.getRoleCommand()); - componentStatus.applyCommandResult(result, command); - log.info("Component operation. Status: {}; new container state: {};" - + " new component state: {}", result, - componentStatus.getContainerState(), componentStatus.getState()); + componentStatus.applyCommandResult(result, command, pkg); + log.info("Component operation. Status: {}; new container state: {}", + result, componentStatus.getContainerState()); + if (command == Command.INSTALL && SliderUtils.isNotEmpty(report.getFolders())) { publishFolderPaths(report.getFolders(), containerId, roleName, heartBeat.getFqdn()); } @@ -798,17 +868,20 @@ public class AgentProviderService extends AbstractProviderService implements if (id < waitForCount) { log.info("Waiting until heartbeat count {}. Current val: {}", waitForCount, id); - getComponentStatuses().put(roleName, componentStatus); + getComponentStatuses().put(label, componentStatus); return response; } - + Command command = componentStatus.getNextCommand(doUpgrade); + log.debug("for comp " + roleName + " pkg " + componentStatus.getNextPkgToInstall() + " issuing " + command.toString()); + try { if (Command.NOP != command) { + if (command == Command.INSTALL) { log.info("Installing {} on {}.", roleName, containerId); if (scriptPath != null) { - addInstallCommand(roleName, containerId, response, scriptPath, null, timeout); + addInstallCommand(roleName, containerId, response, scriptPath, null, timeout, null); } else { // commands ComponentCommand installCmd = null; @@ -817,7 +890,36 @@ public class AgentProviderService extends AbstractProviderService implements installCmd = compCmd; } } - addInstallCommand(roleName, containerId, response, null, installCmd, timeout); + addInstallCommand(roleName, containerId, response, null, installCmd, timeout, null); + } + componentStatus.commandIssued(command); + } else if (command == Command.INSTALL_ADDON) { + String nextPkgToInstall = componentStatus.getNextPkgToInstall(); + // retrieve scriptPath or command of that package for the component + for (ComponentsInAddonPackage comp : packageMetainfo + .get(nextPkgToInstall).getApplicationPackage().getComponents()) { + // given nextPkgToInstall and roleName is determined, the if below + // should only execute once per heartbeat + log.debug("this component:" + comp.getName() + " this pkg:" + + nextPkgToInstall + " script: " + + comp.getCommandScript().getScript()); + if (comp.getName().equals(roleName) + || comp.getName().equals(AgentKeys.ADDON_FOR_ALL_COMPONENTS)) { + scriptPath = comp.getCommandScript().getScript(); + if (scriptPath != null) { + addInstallCommand(roleName, containerId, response, scriptPath, + null, timeout, nextPkgToInstall); + } else { + ComponentCommand installCmd = null; + for (ComponentCommand compCmd : comp.getCommands()) { + if (compCmd.getName().equals("INSTALL")) { + installCmd = compCmd; + } + } + addInstallCommand(roleName, containerId, response, null, + installCmd, timeout, nextPkgToInstall); + } + } } componentStatus.commandIssued(command); } else if (command == Command.START) { @@ -894,7 +996,7 @@ public class AgentProviderService extends AbstractProviderService implements } catch (SliderException e) { log.warn("Component instance failed operation.", e); - componentStatus.applyCommandResult(CommandResult.FAILED, command); + componentStatus.applyCommandResult(CommandResult.FAILED, command, null); } log.debug("Heartbeat response: " + response); @@ -1098,8 +1200,14 @@ public class AgentProviderService extends AbstractProviderService implements @VisibleForTesting protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, + String appDef, boolean addonPackage) throws IOException, BadConfigException { + return AgentUtils.getApplicationMetainfo(fileSystem, appDef, addonPackage); + } + + @VisibleForTesting + protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, String appDef) throws IOException, BadConfigException { - return AgentUtils.getApplicationMetainfo(fileSystem, appDef); + return AgentUtils.getApplicationMetainfo(fileSystem, appDef, false); } @VisibleForTesting @@ -1613,10 +1721,9 @@ public class AgentProviderService extends AbstractProviderService implements * Extract script path from the application metainfo * * @param roleName component name - * * @return the script path or null for no match */ - protected CommandScript getScriptPathFromMetainfo(String roleName) { + protected CommandScript getScriptPathForMasterPackage(String roleName) { Component component = getApplicationComponent(roleName); if (component != null) { return component.getCommandScript(); @@ -1712,7 +1819,7 @@ public class AgentProviderService extends AbstractProviderService implements * @param containerId * @param response * @param scriptPath - * + * @param pkg TODO * @throws SliderException */ @VisibleForTesting @@ -1721,7 +1828,7 @@ public class AgentProviderService extends AbstractProviderService implements HeartBeatResponse response, String scriptPath, ComponentCommand compCmd, - long timeout) + long timeout, String pkg) throws SliderException { assert getAmState().isApplicationLive(); ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); @@ -1734,6 +1841,7 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setServiceName(clusterName); cmd.setComponentName(componentName); cmd.setRole(componentName); + cmd.setPkg(pkg); Map<String, String> hostLevelParams = new TreeMap<String, String>(); hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir())); hostLevelParams.put(PACKAGE_LIST, getPackageList()); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java index 6fe1161..1cbc550 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java @@ -21,6 +21,8 @@ import org.apache.hadoop.fs.Path; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.providers.agent.application.metadata.AbstractMetainfoParser; +import org.apache.slider.providers.agent.application.metadata.AddoPackageMetainfoParser; import org.apache.slider.providers.agent.application.metadata.DefaultConfig; import org.apache.slider.providers.agent.application.metadata.DefaultConfigParser; import org.apache.slider.providers.agent.application.metadata.Metainfo; @@ -39,13 +41,18 @@ public class AgentUtils { private static final Logger log = LoggerFactory.getLogger(AgentUtils.class); public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, - String appDef) throws IOException, BadConfigException { - log.info("Reading metainfo at {}", appDef); + String metainfoPath, boolean metainfoForAddon) throws IOException, BadConfigException { + log.info("Reading metainfo at {}", metainfoPath); FileSystem fs = fileSystem.getFileSystem(); - Path appPath = new Path(appDef); + Path appPath = new Path(metainfoPath); Metainfo metainfo = null; - MetainfoParser metainfoParser = new MetainfoParser(); + AbstractMetainfoParser metainfoParser = null; + if(metainfoForAddon){ + metainfoParser = new AddoPackageMetainfoParser(); + } else { + metainfoParser = new MetainfoParser(); + } InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream( fs, appPath, "metainfo.json"); if (metainfoJsonStream == null) { @@ -59,7 +66,7 @@ public class AgentUtils { } if (metainfo == null) { - log.error("metainfo is unavailable at {}.", appDef); + log.error("metainfo is unavailable at {}.", metainfoPath); throw new FileNotFoundException("metainfo.xml/json is required in app package. " + appPath); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java b/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java index 7d13a8f..d0fdbe6 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java @@ -22,6 +22,7 @@ package org.apache.slider.providers.agent; public enum Command { NOP, // do nothing INSTALL, // Install the component + INSTALL_ADDON, //Install add on packages if any START, // Start the component STOP, // Stop the component UPGRADE, // The component will undergo upgrade http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java index 194d6ff..2e03e82 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java @@ -120,6 +120,8 @@ public class ComponentCommandOrder { return new ComponentState(compStr, state); } + //dependency is still on component level, but not package level + //so use component name to check dependency, not component-package public boolean canExecute(String component, Command command, Collection<ComponentInstanceState> currentStates) { boolean canExecute = true; if (dependencies.containsKey(command) && dependencies.get(command).containsKey(component)) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java index dd78278..fe051cf 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java @@ -18,8 +18,12 @@ package org.apache.slider.providers.agent; +import java.util.TreeMap; + import com.google.common.annotations.VisibleForTesting; + import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.slider.providers.agent.application.metadata.Component; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,6 +46,9 @@ public class ComponentInstanceState { private long lastHeartbeat = 0; private ContainerState containerState; + private TreeMap<String, State> pkgStatuses = new TreeMap<String, State>(); + private String nextPkgToInstall; + public ComponentInstanceState(String componentName, ContainerId containerId, String applicationId) { @@ -53,6 +60,18 @@ public class ComponentInstanceState { this.lastHeartbeat = System.currentTimeMillis(); } + public ComponentInstanceState(String componentName, + ContainerId containerId, + String applicationId, TreeMap<String, State> pkgStatuses) { + this.componentName = componentName; + this.containerId = containerId; + this.containerIdAsString = containerId.toString(); + this.applicationId = applicationId; + this.containerState = ContainerState.INIT; + this.lastHeartbeat = System.currentTimeMillis(); + this.pkgStatuses = pkgStatuses; + } + public String getComponentName() { return componentName; } @@ -106,32 +125,76 @@ public class ComponentInstanceState { if (expected != command) { throw new IllegalArgumentException("Command " + command + " is not allowed in state " + state); } - state = state.getNextState(command); + if (expected == Command.INSTALL_ADDON){ + //for add on packages. the pkg must be nextPkgToInstall + State currentState = pkgStatuses.get(nextPkgToInstall); + log.debug("commandissued: component: {} is in {}", componentName, currentState); + State nextState = currentState.getNextState(command); + pkgStatuses.put(nextPkgToInstall, nextState); + log.debug("commandissued: component: {} is now in {}", componentName, nextState); + } else { + //for master package + state = state.getNextState(command); + } } - public void applyCommandResult(CommandResult result, Command command) { - if (!this.state.couldHaveIssued(command)) { - throw new IllegalStateException("Invalid command " + command + " for state " + this.state); - } + public void applyCommandResult(CommandResult result, Command command, String pkg) { + // if the heartbeat is for a package + // update that package's state in the component status + // and don't bother with the master pkg + if (pkg != null && !pkg.isEmpty() + && !pkg.equals(Component.MASTER_PACKAGE_NAME)) { + log.debug("this result is for component: {} pkg: {}", componentName, pkg); + State previousPkgState = pkgStatuses.get(pkg); + log.debug("currently component: {} pkg: {} is in state: {}", componentName, pkg, previousPkgState.toString()); + State nextPkgState = previousPkgState.getNextState(result); + pkgStatuses.put(pkg, nextPkgState); + log.debug("component: {} pkg: {} next state: {}", componentName, pkg, nextPkgState); + } else { + log.debug("this result is for component: {} master package", componentName); + if (!this.state.couldHaveIssued(command)) { + throw new IllegalStateException("Invalid command " + command + " for state " + this.state); + } - try { - if (result == CommandResult.FAILED) { - failuresSeen++; - } else if (result == CommandResult.COMPLETED) { - failuresSeen = 0; + try { + if (result == CommandResult.FAILED) { + failuresSeen++; + } else if (result == CommandResult.COMPLETED) { + failuresSeen = 0; + } + state = state.getNextState(result); + } catch (IllegalArgumentException e) { + String message = String.format(INVALID_TRANSITION_ERROR, + result.toString(), command.toString(), componentName, + state.toString()); + log.warn(message); + throw new IllegalStateException(message); } - state = state.getNextState(result); - } catch (IllegalArgumentException e) { - String message = String.format(INVALID_TRANSITION_ERROR, - result.toString(), - command.toString(), - componentName, - state.toString()); - log.warn(message); - throw new IllegalStateException(message); } } + public void applyCommandResult(CommandResult result, Command command) { + + if (!this.state.couldHaveIssued(command)) { + throw new IllegalStateException("Invalid command " + command + " for state " + this.state); + } + + try { + if (result == CommandResult.FAILED) { + failuresSeen++; + } else if (result == CommandResult.COMPLETED) { + failuresSeen = 0; + } + state = state.getNextState(result); + } catch (IllegalArgumentException e) { + String message = String.format(INVALID_TRANSITION_ERROR, + result.toString(), command.toString(), componentName, + state.toString()); + log.warn(message); + throw new IllegalStateException(message); + } + } + public boolean hasPendingCommand() { if (state.canIssueCommands() && state != targetState && @@ -148,9 +211,36 @@ public class ComponentInstanceState { public Command getNextCommand(boolean isInUpgradeMode) { if (!hasPendingCommand()) { + nextPkgToInstall = null; return Command.NOP; } + log.debug("start checking for component: {} ", componentName); + // if the master pkg is just installed, check if any add on pkg need to be + // installed + if (state == State.INSTALLED) { + // first check if any pkg is install in progress, if so, wait + for (String pkg : pkgStatuses.keySet()) { + State pkgState = pkgStatuses.get(pkg); + log.debug("in getNextCommand, pkg: {} is in {}", pkg, pkgState); + if (pkgState == State.INSTALLING) { + log.debug("in getNextCommand, pkg: {} we are issuing NOP", pkg); + nextPkgToInstall = pkg; + return Command.NOP; + } + } + // then check if any pkg is in init, if so, install it + for (String pkg : pkgStatuses.keySet()) { + State pkgState = pkgStatuses.get(pkg); + log.debug("in getNextCommand, pkg: {} is in {}", pkg, pkgState); + if (pkgState == State.INIT) { + log.debug("in getNextCommand, pkg: {} we are issuing install addon", pkg); + nextPkgToInstall = pkg; + return Command.INSTALL_ADDON; + } + } + } + nextPkgToInstall = null; return this.state.getSupportedCommand(isInUpgradeMode); } @@ -220,4 +310,8 @@ public class ComponentInstanceState { sb.append('}'); return sb.toString(); } + + public String getNextPkgToInstall() { + return nextPkgToInstall; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java b/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java index 0a1beca..1b7177b 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java @@ -18,7 +18,9 @@ package org.apache.slider.providers.agent; import com.google.common.annotations.VisibleForTesting; + import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.slider.providers.agent.application.metadata.Component; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/State.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/State.java b/slider-core/src/main/java/org/apache/slider/providers/agent/State.java index 91aee87..f8ea822 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/State.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/State.java @@ -141,6 +141,12 @@ public enum State { } else { throw new IllegalArgumentException(command + " is not valid for " + this); } + case INSTALL_ADDON: + if (this == State.INIT || this == State.INSTALL_FAILED) { + return State.INSTALLING; + } else { + throw new IllegalArgumentException(command + " is not valid for " + this); + } case START: if (this == State.INSTALLED) { return State.STARTING; @@ -184,4 +190,5 @@ public enum State { } return false; } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java new file mode 100644 index 0000000..cd11193 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Component defined in master package metainfo.json + */ +public abstract class AbstractComponent implements Validate { + + public static String TYPE_STANDARD = "STANDARD"; + public static String TYPE_DOCKER = "DOCKER"; + public static String CATEGORY_MASTER = "MASTER"; + public static String CATEGORY_SLAVE = "SLAVE"; + public static String CATEGORY_CLIENT = "CLIENT"; + public static final String MASTER_PACKAGE_NAME = "MASTER"; + + protected String name; + protected CommandScript commandScript; + protected List<ComponentCommand> commands = new ArrayList<>(); + + public AbstractComponent() { + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public CommandScript getCommandScript() { + return commandScript; + } + + public void addCommandScript(CommandScript commandScript) { + this.commandScript = commandScript; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("{"); + sb.append(",\n\"name\": ").append(name); + sb.append(",\n\"commandScript\" :").append(commandScript); + sb.append('}'); + return sb.toString(); + } + + public List<ComponentCommand> getCommands() { + return commands; + } + + public void setCommands(List<ComponentCommand> commands) { + this.commands = commands; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java new file mode 100644 index 0000000..557284e --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import org.apache.commons.digester.Digester; +import org.apache.commons.io.IOUtils; +import org.apache.slider.client.SliderClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; + +/** + * + */ +public abstract class AbstractMetainfoParser { + protected final GsonBuilder gsonBuilder = new GsonBuilder(); + protected final Gson gson; + private static final Logger log = LoggerFactory.getLogger(AbstractMetainfoParser.class); + + public AbstractMetainfoParser() { + gson = gsonBuilder.create(); + } + + /** + * Convert to a JSON string + * + * @return a JSON string description + * + * @throws IOException Problems mapping/writing the object + */ + public String toJsonString(Metainfo metaInfo) throws IOException { + return gson.toJson(metaInfo); + } + + /** + * Convert from JSON + * + * @param json input + * + * @return the parsed JSON + * + * @throws IOException IO + */ + public Metainfo fromJsonString(String json) + throws IOException { + return gson.fromJson(json, Metainfo.class); + } + + /** + * Parse metainfo from an IOStream + * + * @param is + * + * @return + * + * @throws IOException + */ + public Metainfo fromJsonStream(InputStream is) throws IOException { + log.debug("loading from xml stream"); + StringWriter writer = new StringWriter(); + IOUtils.copy(is, writer); + return fromJsonString(writer.toString()); + } + + + /** + * Parse metainfo from an XML formatted IOStream + * + * @param metainfoStream + * + * @return + * + * @throws IOException + */ + public Metainfo fromXmlStream(InputStream metainfoStream) throws IOException { + log.debug("loading from xml stream"); + Digester digester = new Digester(); + digester.setValidating(false); + + composeSchema(digester); + + try { + return (Metainfo) digester.parse(metainfoStream); + } catch (IOException e) { + log.debug("IOException in metainfoparser during fromXmlStream: " + e.getMessage()); + } catch (SAXException e) { + log.debug("SAXException in metainfoparser during fromXmlStream: " + e.getMessage()); + } finally { + metainfoStream.close(); + } + + return null; + } + + abstract protected void composeSchema(Digester digester); +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java new file mode 100644 index 0000000..430b34c --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.SliderException; +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.ArrayList; +import java.util.List; + +/** + * Application type defined in the metainfo + */ +public abstract class AbstractMetainfoSchema implements Validate { + protected String name; + protected String comment; + protected String version; + protected List<ConfigFile> configFiles = new ArrayList<>(); + + public AbstractMetainfoSchema() { + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public void addConfigFile(ConfigFile configFile) { + this.configFiles.add(configFile); + } + + @JsonProperty("configFiles") + public List<ConfigFile> getConfigFiles() { + return configFiles; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddoPackageMetainfoParser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddoPackageMetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddoPackageMetainfoParser.java new file mode 100644 index 0000000..4fcadf5 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddoPackageMetainfoParser.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.commons.digester.Digester; + +/** + * + */ +public class AddoPackageMetainfoParser extends AbstractMetainfoParser { + + protected void composeSchema(Digester digester){ + digester.addObjectCreate("metainfo", Metainfo.class); + digester.addBeanPropertySetter("metainfo/schemaVersion"); + + digester.addObjectCreate("*/applicationPackage", ApplicationPackage.class); + digester.addBeanPropertySetter("*/applicationPackage/name"); + digester.addBeanPropertySetter("*/applicationPackage/comment"); + digester.addBeanPropertySetter("*/applicationPackage/version"); + + digester.addObjectCreate("*/component", ComponentsInAddonPackage.class); + digester.addBeanPropertySetter("*/component/name"); + digester.addSetNext("*/component", "addComponent"); + + digester.addObjectCreate("*/commandScript", CommandScript.class); + digester.addBeanPropertySetter("*/commandScript/script"); + digester.addBeanPropertySetter("*/commandScript/scriptType"); + digester.addBeanPropertySetter("*/commandScript/timeout"); + digester.addSetNext("*/commandScript", "addCommandScript"); + + digester.addObjectCreate("*/configFile", ConfigFile.class); + digester.addBeanPropertySetter("*/configFile/type"); + digester.addBeanPropertySetter("*/configFile/fileName"); + digester.addBeanPropertySetter("*/configFile/dictionaryName"); + digester.addSetNext("*/configFile", "addConfigFile"); + + digester.addSetRoot("*/applicationPackage", "setApplicationPackage"); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java index cd5555f..8384e44 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java @@ -28,17 +28,13 @@ import java.util.List; /** * Application type defined in the metainfo */ -public class Application implements Validate { - String name; - String comment; - String version; +public class Application extends AbstractMetainfoSchema{ String exportedConfigs; - List<Component> components = new ArrayList<>(); List<ExportGroup> exportGroups = new ArrayList<>(); List<OSSpecific> osSpecifics = new ArrayList<>(); List<CommandOrder> commandOrders = new ArrayList<>(); - List<ConfigFile> configFiles = new ArrayList<>(); List<Package> packages = new ArrayList<>(); + private List<Component> components = new ArrayList<>(); public Application() { } @@ -90,7 +86,7 @@ public class Application implements Validate { @JsonProperty("components") public List<Component> getComponents() { - return components; + return components ; } public void addExportGroup(ExportGroup exportGroup) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java new file mode 100644 index 0000000..cce5a35 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java @@ -0,0 +1,51 @@ +package org.apache.slider.providers.agent.application.metadata; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.slider.core.exceptions.SliderException; + +public class ApplicationPackage extends AbstractMetainfoSchema{ + private List<ComponentsInAddonPackage> components = new ArrayList<ComponentsInAddonPackage>(); + + public void addComponent(ComponentsInAddonPackage component) { + components.add(component); + } + + //we must override getcomponent() as well. otherwise it is pointing to the overriden components of type List<Componet> + public List<ComponentsInAddonPackage> getComponents(){ + return this.components; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("{"); + sb.append(",\n\"name\": ").append(name); + sb.append(",\n\"comment\": ").append(comment); + sb.append(",\n\"version\" :").append(version); + sb.append(",\n\"components\" : {"); + for (ComponentsInAddonPackage component : components) { + sb.append("\n").append(component); + } + sb.append("\n},"); + sb.append('}'); + return sb.toString(); + } + + @Override + public void validate(String version) throws SliderException { + if(name == null || name.isEmpty()){ + throw new SliderException("Missing name in metainfo.json for add on packages"); + } + if(components.isEmpty()){ + throw new SliderException("Missing components in metainfo.json for add on packages"); + } + for (ComponentsInAddonPackage component : components) { + if(component.name == null || component.name.isEmpty()){ + throw new SliderException("Missing name of components in metainfo.json for add on packages"); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java index 3f1b7fe..84f2700 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java @@ -23,19 +23,16 @@ import org.codehaus.jackson.annotate.JsonProperty; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; /** - * + * Component defined in master package metainfo.json */ -public class Component implements Validate { - - public static String TYPE_STANDARD = "STANDARD"; - public static String TYPE_DOCKER = "DOCKER"; - public static String CATEGORY_MASTER = "MASTER"; - public static String CATEGORY_SLAVE = "SLAVE"; - public static String CATEGORY_CLIENT = "CLIENT"; +public class Component extends AbstractComponent { - String name; String category = CATEGORY_MASTER; String publishConfig = Boolean.FALSE.toString(); String minInstanceCount = "0"; @@ -43,11 +40,9 @@ public class Component implements Validate { String autoStartOnFailure = Boolean.FALSE.toString(); String appExports; String compExports; - CommandScript commandScript; String type = TYPE_STANDARD; List<ComponentExport> componentExports = new ArrayList<>(); - List<ComponentCommand> commands = new ArrayList<>(); - + public Component() { } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java new file mode 100644 index 0000000..9927400 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java @@ -0,0 +1,13 @@ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.core.exceptions.SliderException; + +public class ComponentsInAddonPackage extends AbstractComponent { + + @Override + public void validate(String version) throws SliderException { + // TODO Auto-generated method stub + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java index f89846e..88359e5 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java @@ -31,6 +31,7 @@ public class Metainfo { public static String VERSION_TWO_ONE = "2.1"; String schemaVersion; + ApplicationPackage applicationPackage; Application application; public String getSchemaVersion() { @@ -41,6 +42,14 @@ public class Metainfo { this.schemaVersion = schemaVersion; } + public ApplicationPackage getApplicationPackage() { + return applicationPackage; + } + + public void setApplicationPackage(ApplicationPackage pkg) { + this.applicationPackage = pkg; + } + public Application getApplication() { return application; } @@ -69,6 +78,9 @@ public class Metainfo { } application.validate(schemaVersion); + if(applicationPackage != null){ + applicationPackage.validate(schemaVersion); + } } public static void checkNonNull(String value, String field, String type) throws SliderException { @@ -76,4 +88,19 @@ public class Metainfo { throw new SliderException(type + "." + field + " cannot be null"); } } + + @Override + public String toString() { + + String result = "Metainfo [schemaVersion=" + schemaVersion; + if(applicationPackage != null){ + result += ", pkg=" + applicationPackage.toString(); + } + if(application != null){ + result += ", application=" + application.toString(); + } + return result; + //return "Metainfo [schemaVersion=" + schemaVersion + ", pkg=" + pkg==null? "" : pkg.toString() + // + ", application=" + application==null? "" : application.toString() + "]"; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java index 12af586..a6f0e9d 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java @@ -16,81 +16,14 @@ */ package org.apache.slider.providers.agent.application.metadata; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import org.apache.commons.digester.Digester; -import org.apache.commons.io.IOUtils; -import org.xml.sax.SAXException; - -import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; /** * */ -public class MetainfoParser { - private final GsonBuilder gsonBuilder = new GsonBuilder(); - private final Gson gson; - - public MetainfoParser() { - gson = gsonBuilder.create(); - } - - /** - * Convert to a JSON string - * - * @return a JSON string description - * - * @throws IOException Problems mapping/writing the object - */ - public String toJsonString(Metainfo metaInfo) throws IOException { - return gson.toJson(metaInfo); - } - - /** - * Convert from JSON - * - * @param json input - * - * @return the parsed JSON - * - * @throws IOException IO - */ - public Metainfo fromJsonString(String json) - throws IOException { - return gson.fromJson(json, Metainfo.class); - } - - /** - * Parse metainfo from an IOStream - * - * @param is - * - * @return - * - * @throws IOException - */ - public Metainfo fromJsonStream(InputStream is) throws IOException { - StringWriter writer = new StringWriter(); - IOUtils.copy(is, writer); - return fromJsonString(writer.toString()); - } - - - /** - * Parse metainfo from an XML formatted IOStream - * - * @param metainfoStream - * - * @return - * - * @throws IOException - */ - public Metainfo fromXmlStream(InputStream metainfoStream) throws IOException { - Digester digester = new Digester(); - digester.setValidating(false); - +public class MetainfoParser extends AbstractMetainfoParser{ + + protected void composeSchema(Digester digester){ digester.addObjectCreate("metainfo", Metainfo.class); digester.addBeanPropertySetter("metainfo/schemaVersion"); @@ -149,17 +82,5 @@ public class MetainfoParser { digester.addSetNext("*/configFile", "addConfigFile"); digester.addSetRoot("*/application", "setApplication"); - - try { - return (Metainfo) digester.parse(metainfoStream); - } catch (IOException e) { - - } catch (SAXException e) { - - } finally { - metainfoStream.close(); - } - - return null; } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java index ea6e74d..89ee6c3 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java @@ -40,6 +40,7 @@ public class ExecutionCommand { //TODO Remove hostname from being set in the command private String hostname; private String role; + private String pkg; private Map<String, String> hostLevelParams = new HashMap<String, String>(); private Map<String, String> roleParams = null; private String roleCommand; @@ -218,4 +219,14 @@ public class ExecutionCommand { .append(", componentName=").append(componentName).append("]"); return builder.toString(); } + + @JsonProperty("package") + public String getPkg() { + return pkg; + } + + @JsonProperty("package") + public void setPkg(String pkg) { + this.pkg = pkg; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java index a08d46d..a7a14f6 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java @@ -39,6 +39,7 @@ public class HeartBeat { private long responseId = -1; private long timestamp; private String hostname; + private String pkg; List<CommandReport> reports = new ArrayList<CommandReport>(); List<ComponentStatus> componentStatus = new ArrayList<ComponentStatus>(); private List<DiskInfo> mounts = new ArrayList<DiskInfo>(); @@ -132,7 +133,18 @@ public class HeartBeat { ", hostname='" + hostname + '\'' + ", reports=" + reports + ", componentStatus=" + componentStatus + + ", package=" + pkg + ", nodeStatus=" + nodeStatus + '}'; } + + @JsonProperty("package") + public String getPkg() { + return pkg; + } + + @JsonProperty("package") + public void setPkg(String pkg) { + this.pkg = pkg; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java index 842e5a9..4f7d4de 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java @@ -30,6 +30,7 @@ public class Register { private int responseId = -1; private long timestamp; private String label; + private String pkg; private int currentPingPort; private HostInfo hardwareProfile; private String publicHostname; @@ -169,4 +170,12 @@ public class Register { } return ret; } + + public String getPkg() { + return pkg; + } + + public void setPkg(String pkg) { + this.pkg = pkg; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java index fd852e2..ae2ac55 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java @@ -44,6 +44,9 @@ public class RegistrationResponse { /** tags - tags associated with the container */ @JsonProperty("tags") private String tags; + + @JsonProperty("package") + private String pkg; //Response id to start with, usually zero. @JsonProperty("responseId") @@ -103,4 +106,12 @@ public class RegistrationResponse { ", statusCommands=" + statusCommands + '}'; } + + public String getPkg() { + return pkg; + } + + public void setPkg(String pkg) { + this.pkg = pkg; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy index 0854b79..1775298 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy @@ -103,6 +103,31 @@ extends YarnZKMiniClusterTestBase { log.warn("Temp folder deletion failed: $e") } } + + public static String createAddOnPackageFiles() { + + File destDir = new File("target/agent_minicluster_testbase_addon") + destDir.mkdirs() + File addonAgentConf = new File(destDir, "addon1.zip") + addonAgentConf.createNewFile() + + // dynamically create the app package for the test + TemporaryFolder addonTempFolder = new TemporaryFolder(); + addonTempFolder.create() + def pkgPath = addonTempFolder.newFolder("testpkg") + File imagePath = new File(pkgPath, "appdef_1.zip").canonicalFile + File metainfo = new File(new File(".").absoluteFile, "src/test/python/metainfo.xml"); + ZipArchiveOutputStream zipFile = new ZipArchiveOutputStream(new FileOutputStream(imagePath)); + try { + zipFile.putArchiveEntry(new ZipArchiveEntry(metainfo.name)); + IOUtils.copy(new FileInputStream(metainfo), zipFile); + zipFile.closeArchiveEntry(); + } + finally { + zipFile.close(); + } + return addonAgentConf.toURI().toString() + } @Override public String getTestConfigurationPath() { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/016cd66a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy index d96886a..230d519 100644 --- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy @@ -97,7 +97,7 @@ class TestWindowsSupport extends YarnMiniClusterTestBase { assumeWindows() SliderFileSystem sfs = new SliderFileSystem(new Configuration()) try { - def metainfo = AgentUtils.getApplicationMetainfo(sfs, windowsFile) + def metainfo = AgentUtils.getApplicationMetainfo(sfs, windowsFile, false) } catch (FileNotFoundException fnfe) { // expected }
