Repository: incubator-slider Updated Branches: refs/heads/develop 6ae5ea328 -> 211cb28c5
SLIDER-773. Add co-processor support for app packages (code review changes - Thomas Liu via gourksaha) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/211cb28c Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/211cb28c Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/211cb28c Branch: refs/heads/develop Commit: 211cb28c5ee231bb9899db9d0c85da60aab88a21 Parents: 6ae5ea3 Author: Gour Saha <[email protected]> Authored: Fri Apr 17 11:04:31 2015 -0700 Committer: Gour Saha <[email protected]> Committed: Fri Apr 17 11:26:22 2015 -0700 ---------------------------------------------------------------------- .../src/main/python/agent/ActionQueue.py | 6 +- .../src/main/python/agent/AgentConfig.py | 1 - .../src/main/python/agent/Controller.py | 5 +- .../python/agent/CustomServiceOrchestrator.py | 17 +++- slider-agent/src/main/python/agent/Heartbeat.py | 4 +- slider-agent/src/main/python/agent/main.py | 4 +- slider-core/pom.xml | 14 +-- .../core/persist/AppDefinitionPersister.java | 19 ++-- .../providers/agent/AgentProviderService.java | 87 ++++++++-------- .../slider/providers/agent/AgentUtils.java | 5 +- .../apache/slider/providers/agent/Command.java | 14 +-- .../providers/agent/ComponentCommandOrder.java | 4 +- .../providers/agent/ComponentInstanceState.java | 100 ++++++++++--------- .../providers/agent/HeartbeatMonitor.java | 1 - .../apache/slider/providers/agent/State.java | 1 - .../application/metadata/AbstractComponent.java | 36 ++++--- .../metadata/AbstractMetainfoParser.java | 13 +-- .../metadata/AbstractMetainfoSchema.java | 5 +- .../metadata/AddonPackageMetainfoParser.java | 4 +- .../agent/application/metadata/Application.java | 4 +- .../metadata/ApplicationPackage.java | 24 +++-- .../agent/application/metadata/Component.java | 32 +----- .../metadata/ComponentsInAddonPackage.java | 3 - .../agent/application/metadata/Metainfo.java | 24 ++--- .../web/rest/agent/ExecutionCommand.java | 26 ++--- .../appmaster/web/rest/agent/HeartBeat.java | 23 ++--- .../appmaster/web/rest/agent/Register.java | 18 ++-- .../web/rest/agent/RegistrationResponse.java | 16 +-- .../agent/AgentMiniClusterTestBase.groovy | 11 +- .../agent/TestAgentProviderService.java | 15 ++- 30 files changed, 254 insertions(+), 282 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-agent/src/main/python/agent/ActionQueue.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py index 6e3c3e2..e5e2d1a 100644 --- a/slider-agent/src/main/python/agent/ActionQueue.py +++ b/slider-agent/src/main/python/agent/ActionQueue.py @@ -67,7 +67,7 @@ class ActionQueue(threading.Thread): self.customServiceOrchestrator = CustomServiceOrchestrator(config, controller, self.queueOutAgentToggleLogger) - + def stop(self): self._stop.set() @@ -127,7 +127,7 @@ class ActionQueue(threading.Thread): else: self.componentPackage = '' - logger.info("package received: " + self.componentPackage + ";") + logger.info("Package received: " + self.componentPackage) message = "Executing command with id = {commandId} for role = {role} of " \ "cluster {cluster}".format( @@ -163,7 +163,7 @@ class ActionQueue(threading.Thread): if store_command: logger.info("Component has indicated auto-restart. Saving details from START command.") - logger.info("will run: " + str(command)) + logger.info("Running command: " + str(command)) # running command commandresult = self.customServiceOrchestrator.runCommand(command, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-agent/src/main/python/agent/AgentConfig.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/AgentConfig.py b/slider-agent/src/main/python/agent/AgentConfig.py index a78c22b..7d0fc1e 100644 --- a/slider-agent/src/main/python/agent/AgentConfig.py +++ b/slider-agent/src/main/python/agent/AgentConfig.py @@ -87,7 +87,6 @@ class AgentConfig: # the location of the app package APP_PACKAGE_DIR = "app_pkg_dir" - ADDON_PKG_ROOT_DIR = "addon/definition" # the location where the app component is installed APP_INSTALL_DIR = "app_install_dir" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-agent/src/main/python/agent/Controller.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py index 128b226..cbe2c6a 100644 --- a/slider-agent/src/main/python/agent/Controller.py +++ b/slider-agent/src/main/python/agent/Controller.py @@ -448,18 +448,15 @@ class Controller(threading.Thread): index = 0 deleteIndex = 0 delete = False - ''' Do not break for START command, since we might get a STOP command (used during failure scenarios to gracefully attempt stop) ''' - for command in commands: - if "package" in command and command["package"] != "MASTER": # we do not update component state upon add on package command continue - + if command["roleCommand"] == "START": self.componentExpectedState = State.STARTED self.componentActualState = State.STARTING http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py index 71b9799..b4a1e21 100644 --- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py +++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py @@ -96,14 +96,21 @@ class CustomServiceOrchestrator(): timeout = int(command['commandParams']['command_timeout']) script_path = '' if 'package' in command: - add_on_dir_str = self.config.getWorkRootPath() + "/" + AgentConfig.ADDON_PKG_ROOT_DIR + "/application.addon." + command['package'] + add_on_dir_str = (self.config.getWorkRootPath() + + "/" + + AgentConfig.ADDON_PKG_ROOT_DIR + + "/application.addon." + + command['package'] + ) add_on_base_dir = os.path.realpath(posixpath.join(add_on_dir_str, "package")) - logger.info("add on: " + command['package'] + " add on base dir is: " + str(add_on_base_dir)) + logger.info("Add on package: %s, add on base dir: %s" + % (command['package'], str(add_on_base_dir))) script_path = self.resolve_script_path(add_on_base_dir, script, script_type) - else: - self.base_dir = os.path.realpath(posixpath.join(self.config.getResolvedPath(AgentConfig.APP_PACKAGE_DIR), "package")) - logger.info("base dir is: " + str(self.base_dir)) + self.base_dir = os.path.realpath(posixpath.join( + self.config.getResolvedPath(AgentConfig.APP_PACKAGE_DIR), + "package")) + logger.debug("Base dir: " + str(self.base_dir)) script_path = self.resolve_script_path(self.base_dir, script, script_type) script_tuple = (script_path, self.base_dir) py_file_list = [script_tuple] http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-agent/src/main/python/agent/Heartbeat.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/Heartbeat.py b/slider-agent/src/main/python/agent/Heartbeat.py index db9c2d5..5a114b1 100644 --- a/slider-agent/src/main/python/agent/Heartbeat.py +++ b/slider-agent/src/main/python/agent/Heartbeat.py @@ -45,7 +45,9 @@ class Heartbeat: nodeStatus = {"status": "HEALTHY", "cause": "NONE"} - logger.info("package to heartbeat: " + self.actionQueue.componentPackage) + if not self.actionQueue.componentPackage == '': + logger.info("Add package to heartbeat: " + + self.actionQueue.componentPackage) heartbeat = {'responseId': int(id), 'timestamp': timestamp, 'hostname': self.config.getLabel(), http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-agent/src/main/python/agent/main.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py index a706e70..f592d08 100644 --- a/slider-agent/src/main/python/agent/main.py +++ b/slider-agent/src/main/python/agent/main.py @@ -113,7 +113,7 @@ def update_config_from_file(agentConfig): try: configFile = posixpath.join(agentConfig.getWorkRootPath(), configFileRelPath) if os.path.exists(configFile): - logger.info("config file is: " + configFile) + logger.info("Config file: " + configFile) agentConfig.setConfig(configFile) else: logger.warn("No config found, using default") @@ -213,7 +213,7 @@ def main(): # Check for configuration file. agentConfig = AgentConfig(options.root_folder, options.log_folder, options.label) update_config_from_file(agentConfig) - + # update configurations if needed if options.zk_quorum: agentConfig.set(AgentConfig.SERVER_SECTION, Constants.ZK_QUORUM, options.zk_quorum) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/pom.xml ---------------------------------------------------------------------- diff --git a/slider-core/pom.xml b/slider-core/pom.xml index e84fe0e..19c3253 100644 --- a/slider-core/pom.xml +++ b/slider-core/pom.xml @@ -508,15 +508,11 @@ <scope>test</scope> </dependency> -<dependency> - - <groupId>asm</groupId> - - <artifactId>asm</artifactId> - - <version>3.3.1</version> - -</dependency> + <dependency> + <groupId>asm</groupId> + <artifactId>asm</artifactId> + <version>3.3.1</version> + </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java index 5a3d393..2448c72 100644 --- a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java +++ b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java @@ -80,7 +80,7 @@ public class AppDefinitionPersister { File src = appDefinition.appDefPkgOrFolder; String targetName = appDefinition.pkgName; - log.debug("first targetName: " + targetName); + log.debug("Package name: " + targetName); if (appDefinition.appDefPkgOrFolder.isDirectory()) { log.info("Processing app package/folder {} for {}", appDefinition.appDefPkgOrFolder.getAbsolutePath(), @@ -88,12 +88,9 @@ public class AppDefinitionPersister { File tmpDir = Files.createTempDir(); File zipFile = new File(tmpDir.getCanonicalPath(), File.separator + appDefinition.pkgName); SliderUtils.zipFolder(appDefinition.appDefPkgOrFolder, zipFile); - src = zipFile; - targetName = appDefinition.pkgName; } - log.debug("final targetName: " + targetName); - + sliderFileSystem.getFileSystem().copyFromLocalFile( false, false, @@ -174,13 +171,11 @@ public class AppDefinitionPersister { Path addonPath = sliderFileSystem.buildAddonDirPath(clustername, key); String addonPkgName = "addon_" + key + ".zip"; - - log.debug("addonMap.get(key): " + addonMap.get(key) - + " addonPath: " + addonPath - + " defPath: " + defPath - + " addonPkgName: " + addonPkgName); - - //String addonPkgName = key + ".zip"; + + log.debug( + "addonMap.get(key): {} addonPath: {} defPath: {} addonPkgName: {}", + addonMap.get(key), addonPath, defPath, addonPkgName); + appDefinitions.add(new AppDefinition(addonPath, defPath, addonPkgName)); String addOnKey = AgentKeys.ADDON_PREFIX + key; Path addonPkgPath = new Path(addonPath, addonPkgName); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index a076606..a3d29df 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -269,7 +269,6 @@ public class AgentProviderService extends AbstractProviderService implements String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition .getAppConfOperations()); - if (metaInfo == null) { synchronized (syncLock) { if (metaInfo == null) { @@ -277,7 +276,7 @@ public class AgentProviderService extends AbstractProviderService implements initializeAgentDebugCommands(instanceDefinition); metaInfo = getApplicationMetainfo(fileSystem, appDef, false); - log.info("master package metainfo: {}", metaInfo.toString()); + log.info("Master package metainfo: {}", metaInfo.toString()); if (metaInfo == null || metaInfo.getApplication() == null) { log.error("metainfo.xml is unavailable or malformed at {}.", appDef); throw new SliderException( @@ -287,18 +286,18 @@ public class AgentProviderService extends AbstractProviderService implements defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metaInfo); monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval()); monitor.start(); - - //build a map from component to metainfo + + // build a map from component to metainfo String addonAppDefString = instanceDefinition.getAppConfOperations() .getGlobalOptions().getOption(AgentKeys.ADDONS, null); - log.debug("all addon appdef are: {}", addonAppDefString); + log.debug("All addon appdefs: {}", addonAppDefString); if (addonAppDefString != null) { Scanner scanner = new Scanner(addonAppDefString).useDelimiter(","); while (scanner.hasNext()) { String addonAppDef = scanner.next(); String addonAppDefPath = instanceDefinition .getAppConfOperations().getGlobalOptions().get(addonAppDef); - log.debug("this addon package {} is stored at: {}", addonAppDef + log.debug("Addon package {} is stored at: {}", addonAppDef + addonAppDefPath); Metainfo addonMetaInfo = getApplicationMetainfo(fileSystem, addonAppDefPath, true); @@ -306,7 +305,7 @@ public class AgentProviderService extends AbstractProviderService implements packageMetainfo.put(addonMetaInfo.getApplicationPackage() .getName(), addonMetaInfo); } - log.info("metainfo map for master and addon is: {}", + log.info("Metainfo map for master and addon: {}", packageMetainfo.toString()); } } @@ -482,47 +481,53 @@ public class AgentProviderService extends AbstractProviderService implements launcher.addCommand(operation.build()); - //localize addon package + // localize addon package String addonAppDefString = instanceDefinition.getAppConfOperations() .getGlobalOptions().getOption(AgentKeys.ADDONS, null); - log.debug("all addon appdef are: {}", addonAppDefString); + log.debug("All addon appdefs: {}", addonAppDefString); if (addonAppDefString != null) { Scanner scanner = new Scanner(addonAppDefString).useDelimiter(","); while (scanner.hasNext()) { String addonAppDef = scanner.next(); String addonAppDefPath = instanceDefinition .getAppConfOperations().getGlobalOptions().get(addonAppDef); - log.debug("this addon package {} is stored at: {}", addonAppDef, addonAppDefPath); + log.debug("Addon package {} is stored at: {}", addonAppDef, addonAppDefPath); LocalResource addonPkgRes = fileSystem.createAmResource( fileSystem.getFileSystem().resolvePath(new Path(addonAppDefPath)), LocalResourceType.ARCHIVE); launcher.addLocalResource(AgentKeys.ADDON_DEFINITION_DIR + "/" + addonAppDef, addonPkgRes); } - log.debug("metainfo map for master and addon is: {}", + log.debug("Metainfo map for master and addon: {}", packageMetainfo.toString()); } - //initialize addon pkg states for all componentInstanceStatus - TreeMap<String, State> pkgStatuses = new TreeMap<String, State>(); - for(Metainfo appPkg : packageMetainfo.values()){ - //check each component of that addon to see if they apply to this component 'role' - for(ComponentsInAddonPackage comp : appPkg.getApplicationPackage().getComponents()){ - log.debug("current component: {} component in metainfo {}", role, comp.getName()); - if(comp.getName().equals(role) || comp.getName().equals(AgentKeys.ADDON_FOR_ALL_COMPONENTS)){ + // initialize addon pkg states for all componentInstanceStatus + Map<String, State> pkgStatuses = new TreeMap<>(); + for (Metainfo appPkg : packageMetainfo.values()) { + // check each component of that addon to see if they apply to this + // component 'role' + for (ComponentsInAddonPackage comp : appPkg.getApplicationPackage() + .getComponents()) { + log.debug("Current component: {} component in metainfo: {}", role, + comp.getName()); + if (comp.getName().equals(role) + || comp.getName().equals(AgentKeys.ADDON_FOR_ALL_COMPONENTS)) { pkgStatuses.put(appPkg.getApplicationPackage().getName(), State.INIT); } } } - log.debug("for component: {} pkg status map: {}", role, pkgStatuses.toString()); + log.debug("For component: {} pkg status map: {}", role, + pkgStatuses.toString()); // initialize the component instance state getComponentStatuses().put(label, new ComponentInstanceState( role, container.getId(), - getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME), pkgStatuses)); + getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME), + pkgStatuses)); } - + private void localizeContainerSecurityStores(ContainerLauncher launcher, Container container, String role, @@ -727,8 +732,8 @@ public class AgentProviderService extends AbstractProviderService implements String pkg = registration.getPkg(); State agentState = registration.getActualState(); - log.info("label: " + label + " pkg: " + pkg); - + log.info("label: {} pkg: {}", label, pkg); + if (getComponentStatuses().containsKey(label)) { response.setResponseStatus(RegistrationStatus.OK); ComponentInstanceState componentStatus = getComponentStatuses().get(label); @@ -850,13 +855,12 @@ public class AgentProviderService extends AbstractProviderService implements processAllocatedPorts(heartBeat.getFqdn(), roleName, containerId, ports); } result = CommandResult.getCommandResult(report.getStatus()); - Command command = Command.getCommand(report.getRoleCommand()); - componentStatus.applyCommandResult(result, command, pkg); - log.info("Component operation. Status: {}; new container state: {}", - result, componentStatus.getContainerState()); - + log.info("Component operation. Status: {}; new container state: {};" + + " new component state: {}", result, + componentStatus.getContainerState(), componentStatus.getState()); + if (command == Command.INSTALL && SliderUtils.isNotEmpty(report.getFolders())) { publishFolderPaths(report.getFolders(), containerId, roleName, heartBeat.getFqdn()); } @@ -870,19 +874,17 @@ public class AgentProviderService extends AbstractProviderService implements getComponentStatuses().put(label, componentStatus); return response; } - - Command command = componentStatus.getNextCommand(doUpgrade); - log.info("for comp " + roleName + " pkg " - + componentStatus.getNextPkgToInstall() + " issuing " - + command.toString()); + Command command = componentStatus.getNextCommand(doUpgrade); try { if (Command.NOP != command) { - + log.debug("For comp {} pkg {} issuing {}", roleName, + componentStatus.getNextPkgToInstall(), command.toString()); if (command == Command.INSTALL) { log.info("Installing {} on {}.", roleName, containerId); if (scriptPath != null) { - addInstallCommand(roleName, containerId, response, scriptPath, null, timeout, null); + addInstallCommand(roleName, containerId, response, scriptPath, + null, timeout, null); } else { // commands ComponentCommand installCmd = null; @@ -891,7 +893,8 @@ public class AgentProviderService extends AbstractProviderService implements installCmd = compCmd; } } - addInstallCommand(roleName, containerId, response, null, installCmd, timeout, null); + addInstallCommand(roleName, containerId, response, null, + installCmd, timeout, null); } componentStatus.commandIssued(command); } else if (command == Command.INSTALL_ADDON) { @@ -901,9 +904,8 @@ public class AgentProviderService extends AbstractProviderService implements .get(nextPkgToInstall).getApplicationPackage().getComponents()) { // given nextPkgToInstall and roleName is determined, the if below // should only execute once per heartbeat - log.debug("this component:" + comp.getName() + " this pkg:" - + nextPkgToInstall + " script: " - + comp.getCommandScript().getScript()); + log.debug("Addon component: {} pkg: {} script: {}", comp.getName(), + nextPkgToInstall, comp.getCommandScript().getScript()); if (comp.getName().equals(roleName) || comp.getName().equals(AgentKeys.ADDON_FOR_ALL_COMPONENTS)) { scriptPath = comp.getCommandScript().getScript(); @@ -1209,7 +1211,7 @@ public class AgentProviderService extends AbstractProviderService implements @VisibleForTesting protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, String appDef) throws IOException, BadConfigException { - return AgentUtils.getApplicationMetainfo(fileSystem, appDef, false); + return getApplicationMetainfo(fileSystem, appDef, false); } @VisibleForTesting @@ -1822,7 +1824,7 @@ public class AgentProviderService extends AbstractProviderService implements * @param response * @param scriptPath * @param pkg - * - when this field is null, it indicates the command is for the + * when this field is null, it indicates the command is for the * master package; while not null, for the package named by this * field * @throws SliderException @@ -1833,7 +1835,8 @@ public class AgentProviderService extends AbstractProviderService implements HeartBeatResponse response, String scriptPath, ComponentCommand compCmd, - long timeout, String pkg) + long timeout, + String pkg) throws SliderException { assert getAmState().isApplicationLive(); ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java index 071194a..b38dc6e 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java @@ -41,14 +41,15 @@ public class AgentUtils { private static final Logger log = LoggerFactory.getLogger(AgentUtils.class); public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, - String metainfoPath, boolean metainfoForAddon) throws IOException, BadConfigException { + String metainfoPath, boolean metainfoForAddon) throws IOException, + BadConfigException { log.info("Reading metainfo at {}", metainfoPath); FileSystem fs = fileSystem.getFileSystem(); Path appPath = new Path(metainfoPath); Metainfo metainfo = null; AbstractMetainfoParser metainfoParser = null; - if(metainfoForAddon){ + if (metainfoForAddon) { metainfoParser = new AddonPackageMetainfoParser(); } else { metainfoParser = new MetainfoParser(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java b/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java index d0fdbe6..647cb86 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java @@ -20,13 +20,13 @@ package org.apache.slider.providers.agent; /** The states a component instance can be. */ public enum Command { - NOP, // do nothing - INSTALL, // Install the component - INSTALL_ADDON, //Install add on packages if any - START, // Start the component - STOP, // Stop the component - UPGRADE, // The component will undergo upgrade - TERMINATE;// Send terminate signal to agent + NOP, // do nothing + INSTALL, // Install the component + INSTALL_ADDON, // Install add on packages if any + START, // Start the component + STOP, // Stop the component + UPGRADE, // The component will undergo upgrade + TERMINATE; // Send terminate signal to agent public static Command getCommand(String commandVal) { if (commandVal.equals(Command.START.toString())) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java index 2e03e82..91f1259 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java @@ -120,8 +120,8 @@ public class ComponentCommandOrder { return new ComponentState(compStr, state); } - //dependency is still on component level, but not package level - //so use component name to check dependency, not component-package + // dependency is still on component level, but not package level + // so use component name to check dependency, not component-package public boolean canExecute(String component, Command command, Collection<ComponentInstanceState> currentStates) { boolean canExecute = true; if (dependencies.containsKey(command) && dependencies.get(command).containsKey(component)) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java index bed8e54..c4a694e 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java @@ -18,10 +18,12 @@ package org.apache.slider.providers.agent; +import java.util.Map; import java.util.TreeMap; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.slider.providers.agent.application.metadata.Component; import org.slf4j.Logger; @@ -46,23 +48,19 @@ public class ComponentInstanceState { private long lastHeartbeat = 0; private ContainerState containerState; - private TreeMap<String, State> pkgStatuses = new TreeMap<String, State>(); + private Map<String, State> pkgStatuses; private String nextPkgToInstall; public ComponentInstanceState(String componentName, ContainerId containerId, String applicationId) { - this.componentName = componentName; - this.containerId = containerId; - this.containerIdAsString = containerId.toString(); - this.applicationId = applicationId; - this.containerState = ContainerState.INIT; - this.lastHeartbeat = System.currentTimeMillis(); + this(componentName, containerId, applicationId, + new TreeMap<String, State>()); } public ComponentInstanceState(String componentName, ContainerId containerId, - String applicationId, TreeMap<String, State> pkgStatuses) { + String applicationId, Map<String, State> pkgStatuses) { this.componentName = componentName; this.containerId = containerId; this.containerIdAsString = containerId.toString(); @@ -125,56 +123,64 @@ public class ComponentInstanceState { if (expected != command) { throw new IllegalArgumentException("Command " + command + " is not allowed in state " + state); } - if (expected == Command.INSTALL_ADDON){ - //for add on packages. the pkg must be nextPkgToInstall + if (expected == Command.INSTALL_ADDON) { + // for add on packages, the pkg must be nextPkgToInstall State currentState = pkgStatuses.get(nextPkgToInstall); - log.debug("command issued: component: {} is in {}", componentName, currentState); + log.debug("Command issued: component: {} is in {}", componentName, + currentState); State nextState = currentState.getNextState(command); pkgStatuses.put(nextPkgToInstall, nextState); - log.debug("command issued: component: {} is now in {}", componentName, nextState); + log.debug("Command issued: component: {} is now in {}", componentName, + nextState); } else { - //for master package + // for master package state = state.getNextState(command); } } - public void applyCommandResult(CommandResult result, Command command, String pkg) { - // if the heartbeat is for a package + public void applyCommandResult(CommandResult result, Command command, + String pkg) { + // if the heartbeat is for a package // update that package's state in the component status // and don't bother with the master pkg - if (pkg != null && !pkg.isEmpty() && !pkg.equals(Component.MASTER_PACKAGE_NAME)) { - log.debug("this result is for component: {} pkg: {}", componentName, pkg); + if (StringUtils.isNotEmpty(pkg) + && !Component.MASTER_PACKAGE_NAME.equals(pkg)) { + log.debug("This result is for component: {} pkg: {}", componentName, pkg); State previousPkgState = pkgStatuses.get(pkg); - log.debug("currently component: {} pkg: {} is in state: {}", componentName, pkg, previousPkgState.toString()); + log.debug("Currently component: {} pkg: {} is in state: {}", + componentName, pkg, previousPkgState.toString()); State nextPkgState = previousPkgState.getNextState(result); pkgStatuses.put(pkg, nextPkgState); - log.debug("component: {} pkg: {} next state: {}", componentName, pkg, nextPkgState); + log.debug("Component: {} pkg: {} next state: {}", componentName, pkg, + nextPkgState); } else { - log.debug("this result is for component: {} master package", componentName); + log.debug("This result is for component: {} master package", + componentName); applyCommandResult(result, command); } } public void applyCommandResult(CommandResult result, Command command) { - - if (!this.state.couldHaveIssued(command)) { - throw new IllegalStateException("Invalid command " + command + " for state " + this.state); - } + if (!this.state.couldHaveIssued(command)) { + throw new IllegalStateException("Invalid command " + command + " for state " + this.state); + } - try { - if (result == CommandResult.FAILED) { - failuresSeen++; - } else if (result == CommandResult.COMPLETED) { - failuresSeen = 0; - } - state = state.getNextState(result); - } catch (IllegalArgumentException e) { - String message = String.format(INVALID_TRANSITION_ERROR, - result.toString(), command.toString(), componentName, - state.toString()); - log.warn(message); - throw new IllegalStateException(message); + try { + if (result == CommandResult.FAILED) { + failuresSeen++; + } else if (result == CommandResult.COMPLETED) { + failuresSeen = 0; } + state = state.getNextState(result); + } catch (IllegalArgumentException e) { + String message = String.format(INVALID_TRANSITION_ERROR, + result.toString(), + command.toString(), + componentName, + state.toString()); + log.warn(message); + throw new IllegalStateException(message); + } } public boolean hasPendingCommand() { @@ -197,24 +203,25 @@ public class ComponentInstanceState { return Command.NOP; } - log.debug("in getNextCommand, checking for component: {} ", componentName); + log.debug("In getNextCommand, checking for component: {} ", componentName); // if the master pkg is just installed, check if any add on pkg need to be // installed nextPkgToInstall = null; if (state == State.INSTALLED) { - for (String pkg : pkgStatuses.keySet()) { - State pkgState = pkgStatuses.get(pkg); - log.debug("in getNextCommand, pkg: {} is in {}", pkg, pkgState); + for (Map.Entry<String, State> pkgStatus : pkgStatuses.entrySet()) { + String pkg = pkgStatus.getKey(); + State pkgState = pkgStatus.getValue(); + log.debug("In getNextCommand, pkg: {} is in {}", pkg, pkgState); if (pkgState == State.INSTALLING) { // first check if any pkg is install in progress, if so, wait // so we don't need to do anything, just return NOP - log.debug("in getNextCommand, pkg: {} we are issuing NOP", pkg); + log.debug("In getNextCommand, pkg: {} we are issuing NOP", pkg); nextPkgToInstall = pkg; return Command.NOP; } else if (pkgState == State.INIT) { - //temporarily storing pkg here - //in case no pkg in 'installing' state - //will return the package to install + // temporarily storing pkg here + // in case no pkg in 'installing' state + // will return the package to install nextPkgToInstall = pkg; } } @@ -222,12 +229,11 @@ public class ComponentInstanceState { if (nextPkgToInstall != null) { // nextPkgToInstall != null means some pkg is in INIT state // issue 'install' to the pkg we have stored in nextPkgToInstall - log.debug("in getNextCommand, pkg: {} we are issuing install addon", + log.debug("In getNextCommand, pkg: {} we are issuing install addon", nextPkgToInstall); return Command.INSTALL_ADDON; } } - nextPkgToInstall = null; return this.state.getSupportedCommand(isInUpgradeMode); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java b/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java index 1b7177b..4293916 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java @@ -20,7 +20,6 @@ package org.apache.slider.providers.agent; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.slider.providers.agent.application.metadata.Component; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/State.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/State.java b/slider-core/src/main/java/org/apache/slider/providers/agent/State.java index f8ea822..11105fb 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/State.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/State.java @@ -190,5 +190,4 @@ public enum State { } return false; } - } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java index cd11193..be3a26c 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java @@ -18,19 +18,18 @@ package org.apache.slider.providers.agent.application.metadata; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.TreeMap; + +import org.codehaus.jackson.annotate.JsonProperty; /** * Component defined in master package metainfo.json */ public abstract class AbstractComponent implements Validate { - - public static String TYPE_STANDARD = "STANDARD"; - public static String TYPE_DOCKER = "DOCKER"; - public static String CATEGORY_MASTER = "MASTER"; - public static String CATEGORY_SLAVE = "SLAVE"; - public static String CATEGORY_CLIENT = "CLIENT"; + public static final String TYPE_STANDARD = "STANDARD"; + public static final String TYPE_DOCKER = "DOCKER"; + public static final String CATEGORY_MASTER = "MASTER"; + public static final String CATEGORY_SLAVE = "SLAVE"; + public static final String CATEGORY_CLIENT = "CLIENT"; public static final String MASTER_PACKAGE_NAME = "MASTER"; protected String name; @@ -39,7 +38,7 @@ public abstract class AbstractComponent implements Validate { public AbstractComponent() { } - + public String getName() { return name; } @@ -56,16 +55,7 @@ public abstract class AbstractComponent implements Validate { this.commandScript = commandScript; } - @Override - public String toString() { - final StringBuilder sb = - new StringBuilder("{"); - sb.append(",\n\"name\": ").append(name); - sb.append(",\n\"commandScript\" :").append(commandScript); - sb.append('}'); - return sb.toString(); - } - + @JsonProperty("commands") public List<ComponentCommand> getCommands() { return commands; } @@ -74,4 +64,12 @@ public abstract class AbstractComponent implements Validate { this.commands = commands; } + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("{"); + sb.append("\n\"name\": ").append(name); + sb.append(",\n\"commandScript\" :").append(commandScript); + sb.append('}'); + return sb.toString(); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java index ca7b286..bce53b8 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java @@ -21,7 +21,6 @@ import com.google.gson.GsonBuilder; import org.apache.commons.digester.Digester; import org.apache.commons.io.IOUtils; -import org.apache.slider.client.SliderClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.xml.sax.SAXException; @@ -37,7 +36,8 @@ import java.io.StringWriter; public abstract class AbstractMetainfoParser { protected final GsonBuilder gsonBuilder = new GsonBuilder(); protected final Gson gson; - private static final Logger log = LoggerFactory.getLogger(AbstractMetainfoParser.class); + private static final Logger log = LoggerFactory + .getLogger(AbstractMetainfoParser.class); public AbstractMetainfoParser() { gson = gsonBuilder.create(); @@ -84,7 +84,6 @@ public abstract class AbstractMetainfoParser { return fromJsonString(writer.toString()); } - /** * Parse metainfo from an XML formatted IOStream * @@ -100,13 +99,15 @@ public abstract class AbstractMetainfoParser { digester.setValidating(false); composeSchema(digester); - + try { return (Metainfo) digester.parse(metainfoStream); } catch (IOException e) { - log.debug("IOException in metainfoparser during fromXmlStream: " + e.getMessage()); + log.debug("IOException in metainfoparser during fromXmlStream: " + + e.getMessage()); } catch (SAXException e) { - log.debug("SAXException in metainfoparser during fromXmlStream: " + e.getMessage()); + log.debug("SAXException in metainfoparser during fromXmlStream: " + + e.getMessage()); } finally { metainfoStream.close(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java index 430b34c..cfa2895 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java @@ -16,9 +16,6 @@ */ package org.apache.slider.providers.agent.application.metadata; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.SliderException; import org.codehaus.jackson.annotate.JsonProperty; import java.util.ArrayList; @@ -32,7 +29,7 @@ public abstract class AbstractMetainfoSchema implements Validate { protected String comment; protected String version; protected List<ConfigFile> configFiles = new ArrayList<>(); - + public AbstractMetainfoSchema() { } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java index c025658..c75837f 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java @@ -22,8 +22,8 @@ import org.apache.commons.digester.Digester; * */ public class AddonPackageMetainfoParser extends AbstractMetainfoParser { - - protected void composeSchema(Digester digester){ + + protected void composeSchema(Digester digester) { digester.addObjectCreate("metainfo", Metainfo.class); digester.addBeanPropertySetter("metainfo/schemaVersion"); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java index 8384e44..5967a3a 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java @@ -28,7 +28,7 @@ import java.util.List; /** * Application type defined in the metainfo */ -public class Application extends AbstractMetainfoSchema{ +public class Application extends AbstractMetainfoSchema { String exportedConfigs; List<ExportGroup> exportGroups = new ArrayList<>(); List<OSSpecific> osSpecifics = new ArrayList<>(); @@ -86,7 +86,7 @@ public class Application extends AbstractMetainfoSchema{ @JsonProperty("components") public List<Component> getComponents() { - return components ; + return components; } public void addExportGroup(ExportGroup exportGroup) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java index 2ddcb71..a94a213 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java @@ -28,16 +28,16 @@ public class ApplicationPackage extends AbstractMetainfoSchema{ components.add(component); } - //we must override getcomponent() as well. otherwise it is pointing to the overriden components of type List<Componet> + // we must override getcomponent() as well. otherwise it is pointing to the + // overriden components of type List<Component> public List<ComponentsInAddonPackage> getComponents(){ return this.components; } @Override public String toString() { - final StringBuilder sb = - new StringBuilder("{"); - sb.append(",\n\"name\": ").append(name); + final StringBuilder sb = new StringBuilder("{"); + sb.append("\n\"name\": ").append(name); sb.append(",\n\"comment\": ").append(comment); sb.append(",\n\"version\" :").append(version); sb.append(",\n\"components\" : {"); @@ -51,17 +51,19 @@ public class ApplicationPackage extends AbstractMetainfoSchema{ @Override public void validate(String version) throws SliderException { - if(name == null || name.isEmpty()){ - throw new SliderException("Missing name in metainfo.json for add on packages"); + if (name == null || name.isEmpty()) { + throw new SliderException( + "Missing name in metainfo.json for add on packages"); } - if(components.isEmpty()){ - throw new SliderException("Missing components in metainfo.json for add on packages"); + if (components.isEmpty()) { + throw new SliderException( + "Missing components in metainfo.json for add on packages"); } for (ComponentsInAddonPackage component : components) { - if(component.name == null || component.name.isEmpty()){ - throw new SliderException("Missing name of components in metainfo.json for add on packages"); + if (component.name == null || component.name.isEmpty()) { + throw new SliderException( + "Missing name of components in metainfo.json for add on packages"); } } } - } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java index 84f2700..609ffa1 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java @@ -19,17 +19,12 @@ package org.apache.slider.providers.agent.application.metadata; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.core.exceptions.SliderException; -import org.codehaus.jackson.annotate.JsonProperty; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; /** - * Component defined in master package metainfo.json + * Component defined in master package metainfo.json */ public class Component extends AbstractComponent { @@ -42,16 +37,8 @@ public class Component extends AbstractComponent { String compExports; String type = TYPE_STANDARD; List<ComponentExport> componentExports = new ArrayList<>(); - - public Component() { - } - - public String getName() { - return name; - } - public void setName(String name) { - this.name = name; + public Component() { } public String getType() { @@ -150,14 +137,6 @@ public class Component extends AbstractComponent { this.maxInstanceCount = maxInstanceCount; } - public CommandScript getCommandScript() { - return commandScript; - } - - public void addCommandScript(CommandScript commandScript) { - this.commandScript = commandScript; - } - public void addComponentExport(ComponentExport export) { componentExports.add(export); } @@ -170,16 +149,11 @@ public class Component extends AbstractComponent { return Boolean.parseBoolean(this.autoStartOnFailure); } - @JsonProperty("commands") - public List<ComponentCommand> getCommands() { - return this.commands; - } - @Override public String toString() { final StringBuilder sb = new StringBuilder("{"); - sb.append(",\n\"name\": ").append(name); + sb.append("\n\"name\": ").append(name); sb.append(",\n\"category\": ").append(category); sb.append(",\n\"commandScript\" :").append(commandScript); sb.append('}'); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java index 3f62343..855e5b6 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java @@ -22,8 +22,5 @@ public class ComponentsInAddonPackage extends AbstractComponent { @Override public void validate(String version) throws SliderException { - // TODO Auto-generated method stub - } - } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java index 696e6f3..f89a8a6 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java @@ -76,10 +76,10 @@ public class Metainfo { !VERSION_TWO_ZERO.equals(schemaVersion)) { throw new SliderException("Unsupported version " + getSchemaVersion()); } - if(application != null){ + if (application != null) { application.validate(schemaVersion); } - if(applicationPackage != null){ + if (applicationPackage != null) { applicationPackage.validate(schemaVersion); } } @@ -92,16 +92,14 @@ public class Metainfo { @Override public String toString() { - - String result = "Metainfo [schemaVersion=" + schemaVersion; - if(applicationPackage != null){ - result += ", pkg=" + applicationPackage.toString(); - } - if(application != null){ - result += ", application=" + application.toString(); - } - return result; - //return "Metainfo [schemaVersion=" + schemaVersion + ", pkg=" + pkg==null? "" : pkg.toString() - // + ", application=" + application==null? "" : application.toString() + "]"; + StringBuilder builder = new StringBuilder(); + builder.append("Metainfo [schemaVersion="); + builder.append(schemaVersion); + builder.append(", applicationPackage="); + builder.append(applicationPackage); + builder.append(", application="); + builder.append(application); + builder.append("]"); + return builder.toString(); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java index 89ee6c3..d8a4dbc 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java @@ -40,7 +40,6 @@ public class ExecutionCommand { //TODO Remove hostname from being set in the command private String hostname; private String role; - private String pkg; private Map<String, String> hostLevelParams = new HashMap<String, String>(); private Map<String, String> roleParams = null; private String roleCommand; @@ -50,6 +49,7 @@ public class ExecutionCommand { private String serviceName; private String componentName; private String componentType; + private String pkg; public ExecutionCommand(AgentCommandType commandType) { this.commandType = commandType; @@ -195,6 +195,16 @@ public class ExecutionCommand { this.componentName = componentName; } + @JsonProperty("package") + public String getPkg() { + return pkg; + } + + @JsonProperty("package") + public void setPkg(String pkg) { + this.pkg = pkg; + } + public Map<String, Map<String, String>> getComponentConfigurations() { return componentConfigurations; } @@ -216,17 +226,9 @@ public class ExecutionCommand { .append(roleCommand).append(", configurations=").append(configurations) .append(", commandParams=").append(commandParams) .append(", serviceName=").append(serviceName) - .append(", componentName=").append(componentName).append("]"); + .append(", componentName=").append(componentName) + .append(", componentType=").append(componentType).append(", pkg=") + .append(pkg).append("]"); return builder.toString(); } - - @JsonProperty("package") - public String getPkg() { - return pkg; - } - - @JsonProperty("package") - public void setPkg(String pkg) { - this.pkg = pkg; - } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java index a7a14f6..b7172c2 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java @@ -18,7 +18,6 @@ package org.apache.slider.server.appmaster.web.rest.agent; -import org.apache.slider.providers.agent.State; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.annotate.JsonProperty; import org.codehaus.jackson.map.annotate.JsonSerialize; @@ -39,13 +38,13 @@ public class HeartBeat { private long responseId = -1; private long timestamp; private String hostname; - private String pkg; List<CommandReport> reports = new ArrayList<CommandReport>(); List<ComponentStatus> componentStatus = new ArrayList<ComponentStatus>(); private List<DiskInfo> mounts = new ArrayList<DiskInfo>(); HostStatus nodeStatus; private AgentEnv agentEnv = null; private String fqdn; + private String pkg; public long getResponseId() { return responseId; @@ -125,6 +124,16 @@ public class HeartBeat { this.mounts = mounts; } + @JsonProperty("package") + public String getPkg() { + return pkg; + } + + @JsonProperty("package") + public void setPkg(String pkg) { + this.pkg = pkg; + } + @Override public String toString() { return "HeartBeat{" + @@ -137,14 +146,4 @@ public class HeartBeat { ", nodeStatus=" + nodeStatus + '}'; } - - @JsonProperty("package") - public String getPkg() { - return pkg; - } - - @JsonProperty("package") - public void setPkg(String pkg) { - this.pkg = pkg; - } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java index 4f7d4de..b1366c2 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java @@ -30,7 +30,6 @@ public class Register { private int responseId = -1; private long timestamp; private String label; - private String pkg; private int currentPingPort; private HostInfo hardwareProfile; private String publicHostname; @@ -41,6 +40,7 @@ public class Register { private State expectedState; private Map<String, String> allocatedPorts; private Map<String, String> logFolders; + private String pkg; @JsonProperty("responseId") public int getResponseId() { @@ -156,6 +156,14 @@ public class Register { this.logFolders = logFolders; } + public String getPkg() { + return pkg; + } + + public void setPkg(String pkg) { + this.pkg = pkg; + } + @Override public String toString() { String ret = "responseId=" + responseId + "\n" + @@ -170,12 +178,4 @@ public class Register { } return ret; } - - public String getPkg() { - return pkg; - } - - public void setPkg(String pkg) { - this.pkg = pkg; - } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java index ae2ac55..50a3c87 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java @@ -98,6 +98,14 @@ public class RegistrationResponse { this.log = log; } + public String getPkg() { + return pkg; + } + + public void setPkg(String pkg) { + this.pkg = pkg; + } + @Override public String toString() { return "RegistrationResponse{" + @@ -106,12 +114,4 @@ public class RegistrationResponse { ", statusCommands=" + statusCommands + '}'; } - - public String getPkg() { - return pkg; - } - - public void setPkg(String pkg) { - this.pkg = pkg; - } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy index 1775298..b4abf6f 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy @@ -103,21 +103,22 @@ extends YarnZKMiniClusterTestBase { log.warn("Temp folder deletion failed: $e") } } - - public static String createAddOnPackageFiles() { + public static String createAddOnPackageFiles() { File destDir = new File("target/agent_minicluster_testbase_addon") destDir.mkdirs() File addonAgentConf = new File(destDir, "addon1.zip") addonAgentConf.createNewFile() - + // dynamically create the app package for the test TemporaryFolder addonTempFolder = new TemporaryFolder(); addonTempFolder.create() def pkgPath = addonTempFolder.newFolder("testpkg") File imagePath = new File(pkgPath, "appdef_1.zip").canonicalFile - File metainfo = new File(new File(".").absoluteFile, "src/test/python/metainfo.xml"); - ZipArchiveOutputStream zipFile = new ZipArchiveOutputStream(new FileOutputStream(imagePath)); + File metainfo = new File(new File(".").absoluteFile, + "src/test/python/metainfo.xml"); + ZipArchiveOutputStream zipFile = new ZipArchiveOutputStream( + new FileOutputStream(imagePath)); try { zipFile.putArchiveEntry(new ZipArchiveEntry(metainfo.name)); IOUtils.copy(new FileInputStream(metainfo), zipFile); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/211cb28c/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index aa64ff6..9e1c135 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -98,7 +98,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static org.easymock.EasyMock.anyBoolean; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.expect; @@ -106,6 +105,7 @@ import static org.easymock.EasyMock.replay; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyCollection; import static org.mockito.Matchers.anyMap; import static org.mockito.Matchers.anyString; @@ -313,7 +313,7 @@ public class TestAgentProviderService { metainfo.getApplication().addComponent(hm); doReturn(metainfo).when(mockAps).getApplicationMetainfo( - any(SliderFileSystem.class), Matchers.anyString(), Matchers.anyBoolean()); + any(SliderFileSystem.class), anyString(), anyBoolean()); Configuration conf = new Configuration(); conf.set(RegistryConstants.KEY_REGISTRY_ZK_ROOT, @@ -657,7 +657,7 @@ public class TestAgentProviderService { metainfo.setApplication(application); doReturn(metainfo).when(mockAps).getMetaInfo(); doReturn(metainfo).when(mockAps).getApplicationMetainfo( - any(SliderFileSystem.class), Matchers.anyString(), Matchers.anyBoolean()); + any(SliderFileSystem.class), anyString(), anyBoolean()); Register reg = new Register(); reg.setResponseId(0); @@ -724,7 +724,7 @@ public class TestAgentProviderService { application.addComponent(hbaseMaster); metainfo.setApplication(application); doReturn(metainfo).when(mockAps).getApplicationMetainfo( - any(SliderFileSystem.class), Matchers.anyString(), Matchers.anyBoolean()); + any(SliderFileSystem.class), anyString(), anyBoolean()); doReturn(metainfo).when(mockAps).getMetaInfo(); doNothing().when(mockAps).addRoleRelatedTokens(anyMap()); @@ -738,8 +738,7 @@ public class TestAgentProviderService { // Simulating agent in INSTALLED state reg.setActualState(State.INSTALLED); - mockAps.initializeApplicationConfiguration(instanceDefinition, - null); + mockAps.initializeApplicationConfiguration(instanceDefinition, null); RegistrationResponse resp = mockAps.handleRegistration(reg); Assert.assertEquals(0, resp.getResponseId()); @@ -1229,8 +1228,8 @@ public class TestAgentProviderService { AgentProviderService mockAps = Mockito.spy(aps); doReturn(access).when(mockAps).getAmState(); - doReturn(metainfo).when(mockAps).getApplicationMetainfo(any( - SliderFileSystem.class), Matchers.anyString(), Matchers.anyBoolean()); + doReturn(metainfo).when(mockAps).getApplicationMetainfo( + any(SliderFileSystem.class), anyString(), anyBoolean()); doReturn(new HashMap<String, DefaultConfig>()).when(mockAps). initializeDefaultConfigs(any(SliderFileSystem.class), anyString(), any(Metainfo.class));
