Repository: incubator-slider Updated Branches: refs/heads/develop a063aa14a -> 9cb584820
SLIDER-787 App Upgrade/Reconfig support in Slider (fix version missing issue post upgrade spec) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9cb58482 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9cb58482 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9cb58482 Branch: refs/heads/develop Commit: 9cb58482060e24aa68c5abfa70dbb1245d09df92 Parents: a063aa1 Author: Gour Saha <[email protected]> Authored: Sun Apr 26 03:11:01 2015 -0700 Committer: Gour Saha <[email protected]> Committed: Sun Apr 26 03:11:01 2015 -0700 ---------------------------------------------------------------------- .../src/main/python/agent/Controller.py | 1 + slider-agent/src/main/python/agent/Register.py | 4 ++- .../src/test/python/agent/TestRegistration.py | 6 +++-- .../org/apache/slider/common/SliderKeys.java | 2 ++ .../apache/slider/common/tools/SliderUtils.java | 7 ++++-- .../providers/agent/AgentProviderService.java | 26 ++++++++++++++++++++ .../slider/server/appmaster/state/AppState.java | 1 + .../appmaster/web/rest/agent/Register.java | 14 ++++++++++- 8 files changed, 55 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cb58482/slider-agent/src/main/python/agent/Controller.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py index cbe2c6a..e99b78c 100644 --- a/slider-agent/src/main/python/agent/Controller.py +++ b/slider-agent/src/main/python/agent/Controller.py @@ -133,6 +133,7 @@ class Controller(threading.Thread): self.componentExpectedState, self.actionQueue.customServiceOrchestrator.allocated_ports, self.actionQueue.customServiceOrchestrator.log_folders, + self.appVersion, self.tags, id)) logger.info("Registering with the server at " + self.registerUrl + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cb58482/slider-agent/src/main/python/agent/Register.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/Register.py b/slider-agent/src/main/python/agent/Register.py index c5197fd..d73fa18 100644 --- a/slider-agent/src/main/python/agent/Register.py +++ b/slider-agent/src/main/python/agent/Register.py @@ -29,7 +29,8 @@ class Register: def __init__(self, config): self.config = config - def build(self, actualState, expectedState, allocated_ports, log_folders, tags="", id='-1'): + def build(self, actualState, expectedState, allocated_ports, log_folders, + appVersion, tags="", id='-1'): timestamp = int(time.time() * 1000) version = self.read_agent_version() @@ -43,6 +44,7 @@ class Register: 'expectedState': expectedState, 'allocatedPorts': allocated_ports, 'logFolders': log_folders, + 'appVersion': appVersion, 'tags': tags } return register http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cb58482/slider-agent/src/test/python/agent/TestRegistration.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/test/python/agent/TestRegistration.py b/slider-agent/src/test/python/agent/TestRegistration.py index 735fcc3..2bc6460 100644 --- a/slider-agent/src/test/python/agent/TestRegistration.py +++ b/slider-agent/src/test/python/agent/TestRegistration.py @@ -35,12 +35,13 @@ class TestRegistration(TestCase): def test_registration_build(self): tmpdir = tempfile.gettempdir() ver_dir = os.path.join(tmpdir, "infra") + app_ver = "1.0.0" config = AgentConfig(tmpdir, ver_dir) config.set('agent', 'prefix', tmpdir) config.set('agent', 'current_ping_port', '33777') register = Register(config) - data = register.build(State.INIT, State.INIT, {}, {}, "tag", 1) + data = register.build(State.INIT, State.INIT, {}, {}, app_ver, "tag", 1) #print ("Register: " + pprint.pformat(data)) self.assertEquals(data['label'] != "", True, "hostname should not be empty") self.assertEquals(data['publicHostname'] != "", True, "publicHostname should not be empty") @@ -51,8 +52,9 @@ class TestRegistration(TestCase): self.assertEquals(data['expectedState'], State.INIT, "expectedState should not be empty") self.assertEquals(data['allocatedPorts'], {}, "allocatedPorts should be empty") self.assertEquals(data['logFolders'], {}, "allocated log should be empty") + self.assertEquals(data['appVersion'], app_ver, "app version should match") self.assertEquals(data['tags'], "tag", "tags should be tag") - self.assertEquals(len(data), 10) + self.assertEquals(len(data), 11) self.assertEquals(posixpath.join(tmpdir, "app/definition"), config.getResolvedPath("app_pkg_dir")) self.assertEquals(posixpath.join(tmpdir, "app/install"), config.getResolvedPath("app_install_dir")) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cb58482/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 56ea022..a5aad37 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -58,6 +58,7 @@ public interface SliderKeys extends SliderXmlConfKeys { * Key for application version. This must be set in app_config/global {@value} */ String APP_VERSION = "site.global.app_version"; + String APP_VERSION_UNKNOWN = "awaiting heartbeat..."; /** * JVM arg to force IPv4 {@value} @@ -256,4 +257,5 @@ public interface SliderKeys extends SliderXmlConfKeys { String SLIDER_JVM_OPTS = "SLIDER_JVM_OPTS"; String SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"; + String YARN_CONTAINER_PATH = "/node/container/"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cb58482/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index dbac1e1..4fcfed1 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -708,16 +708,19 @@ public final class SliderUtils { public static String containersToString( List<ContainerInformation> containers, String version, Set<String> components) { - String containerf = " %-28s %30s %40s %s\n"; + String containerf = " %-28s %30s %45s %s\n"; StringBuilder builder = new StringBuilder(512); builder.append("Containers:\n"); + builder.append(String.format(" %-28s %30s %45s %s\n", "Component Name", + "App Version", "Container Id", "Container Info/Logs")); for (ContainerInformation container : containers) { if (filter(container.appVersion, version) || filter(container.component, components)) { continue; } builder.append(String.format(containerf, container.component, - container.appVersion, container.containerId, container.host)); + container.appVersion, container.containerId, container.host + + SliderKeys.YARN_CONTAINER_PATH + container.containerId)); } return builder.toString(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cb58482/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index a3d29df..c804aad 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -731,6 +731,7 @@ public class AgentProviderService extends AbstractProviderService implements String label = registration.getLabel(); String pkg = registration.getPkg(); State agentState = registration.getActualState(); + String appVersion = registration.getAppVersion(); log.info("label: {} pkg: {}", label, pkg); @@ -759,6 +760,9 @@ public class AgentProviderService extends AbstractProviderService implements if (folders != null && !folders.isEmpty()) { publishFolderPaths(folders, containerId, roleName, hostFqdn); } + + // Set app version if empty. It gets unset during upgrade - why? + checkAndSetContainerAppVersion(containerId, appVersion); } else { response.setResponseStatus(RegistrationStatus.FAILED); response.setLog("Label not recognized."); @@ -768,6 +772,28 @@ public class AgentProviderService extends AbstractProviderService implements return response; } + // Checks if app version is empty. Sets it to the version as reported by the + // container during registration phase. + private void checkAndSetContainerAppVersion(String containerId, + String appVersion) { + StateAccessForProviders amState = getAmState(); + try { + RoleInstance role = amState.getOwnedContainer(containerId); + if (role != null) { + String currentAppVersion = role.appVersion; + log.debug("Container = {}, app version current = {} new = {}", + containerId, currentAppVersion, appVersion); + if (currentAppVersion == null + || currentAppVersion.equals(APP_VERSION_UNKNOWN)) { + amState.getOwnedContainer(containerId).appVersion = appVersion; + } + } + } catch (NoSuchNodeException e) { + // ignore - there is nothing to do if we don't find a container + log.warn("Owned container {} not found - {}", containerId, e); + } + } + /** * Handle heartbeat response from agents * http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cb58482/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index c68b2a9..5428a34 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -2240,6 +2240,7 @@ public class AppState { instance.container = container; instance.createTime = now(); instance.state = STATE_LIVE; + instance.appVersion = SliderKeys.APP_VERSION_UNKNOWN; putOwnedContainer(cid, instance); //role history gets told roleHistory.onContainerAssigned(container); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9cb58482/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java index b1366c2..0150079 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java @@ -41,6 +41,7 @@ public class Register { private Map<String, String> allocatedPorts; private Map<String, String> logFolders; private String pkg; + private String appVersion; @JsonProperty("responseId") public int getResponseId() { @@ -164,6 +165,16 @@ public class Register { this.pkg = pkg; } + @JsonProperty("appVersion") + public String getAppVersion() { + return appVersion; + } + + @JsonProperty("appVersion") + public void setAppVersion(String appVersion) { + this.appVersion = appVersion; + } + @Override public String toString() { String ret = "responseId=" + responseId + "\n" + @@ -171,7 +182,8 @@ public class Register { "label=" + label + "\n" + "hostname=" + publicHostname + "\n" + "expectedState=" + expectedState + "\n" + - "actualState=" + actualState + "\n"; + "actualState=" + actualState + "\n" + + "appVersion=" + appVersion + "\n"; if (hardwareProfile != null) { ret = ret + "hardwareprofile=" + this.hardwareProfile.toString();
