SLIDER-755. AgentProvider doesn't raise and exception when the agent tar isn't there SLIDER-723. Memcached component launch fail does not propagate
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/657f734c Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/657f734c Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/657f734c Branch: refs/heads/develop Commit: 657f734c1104be650318241c5c0bdbcd0d61533e Parents: 98d676b Author: Sumit Mohanty <[email protected]> Authored: Mon Jan 26 17:18:02 2015 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Mon Jan 26 17:18:10 2015 -0800 ---------------------------------------------------------------------- .../slider/providers/agent/AgentKeys.java | 16 +-- .../providers/agent/AgentProviderService.java | 21 ++- .../slider/providers/agent/TestAgentEcho.groovy | 4 +- .../appmaster/TestDelayInContainerLaunch.groovy | 1 + .../agent/TestAgentProviderService.java | 132 ++++++++++++++++++- 5 files changed, 155 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index e682b13..7977181 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -25,20 +25,7 @@ public interface AgentKeys { String PROVIDER_AGENT = "agent"; String ROLE_NODE = "echo"; - /** - * {@value} - */ - String REGION_SERVER = "regionserver"; - /** - * What is the command for hbase to print a version: {@value} - */ - String COMMAND_VERSION = "version"; - String ACTION_START = "start"; - String ACTION_STOP = "stop"; - /** - * Config directory : {@value} - */ - String ARG_CONFIG = "--config"; + /** * Template stored in the slider classpath -to use if there is * no site-specific template @@ -100,6 +87,7 @@ public interface AgentKeys { String KEY_AGENT_TWO_WAY_SSL_ENABLED = "ssl.server.client.auth"; String CERT_FILE_LOCALIZATION_PATH = "certs/ca.crt"; String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec"; + String TEST_RELAX_VERIFICATION = "test.relax.validation"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index e4680e2..439b5b6 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -353,13 +353,21 @@ public class AgentProviderService extends AbstractProviderService implements agentImagePath = new Path(agentImage); } - // TODO: throw exception when agent tarball is not available - if (fileSystem.getFileSystem().exists(agentImagePath)) { LocalResource agentImageRes = fileSystem.createAmResource(agentImagePath, LocalResourceType.ARCHIVE); launcher.addLocalResource(AgentKeys.AGENT_INSTALL_DIR, agentImageRes); } else { - log.error("Required agent image slider-agent.tar.gz is unavailable."); + String msg = + String.format("Required agent image slider-agent.tar.gz is unavailable at %s", agentImagePath.toString()); + MapOperations compOps = instanceDefinition. + getAppConfOperations().getComponent(role); + boolean relaxVerificationForTest = compOps != null ? Boolean.valueOf(compOps. + getOptionBool(AgentKeys.TEST_RELAX_VERIFICATION, false)) : false; + log.error(msg); + + if (!relaxVerificationForTest) { + throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, msg); + } } log.info("Using {} for agent.", scriptPath); @@ -750,6 +758,13 @@ public class AgentProviderService extends AbstractProviderService implements && command == Command.NOP && isMarkedAutoRestart(roleName)) { response.setRestartEnabled(true); } + + //If INSTALL_FAILED and no INSTALL is scheduled let the agent fail + if(componentStatus.getState() == State.INSTALL_FAILED && command == command.NOP) { + log.warn("Sending terminate signal to container that failed installation: {}", label); + response.setTerminateAgent(true); + } + } catch (SliderException e) { log.warn("Component instance failed operation.", e); componentStatus.applyCommandResult(CommandResult.FAILED, command); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy index 2eb39e3..973114d 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy @@ -99,7 +99,9 @@ class TestAgentEcho extends AgentTestBase { ARG_COMP_OPT, role, SCRIPT_PATH, echo_py, ARG_COMP_OPT, role, SERVICE_NAME, "Agent", ARG_DEFINE, - SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED + "=false" + SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED + "=false", + ARG_COMP_OPT, role, TEST_RELAX_VERIFICATION, "true", + ], true, true, true) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy index 26a2c95..51e01d5 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy @@ -107,6 +107,7 @@ class TestDelayInContainerLaunch extends AgentTestBase { ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1", ARG_COMP_OPT, role, SCRIPT_PATH, echo_py, ARG_COMP_OPT, role, SERVICE_NAME, "Agent", + ARG_COMP_OPT, role, TEST_RELAX_VERIFICATION, "true", ], true, true, true) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index dbc1b0a..3b78fb8 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -496,6 +496,136 @@ public class TestAgentProviderService { } @Test + public void testThreeInstallFailures() throws IOException, SliderException { + InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes()); + Metainfo metainfo = new MetainfoParser().parse(metainfo_1); + ConfTree tree = new ConfTree(); + tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, "."); + + Configuration conf = new Configuration(); + AgentProviderService aps = createAgentProviderService(conf); + YarnRegistryViewForProviders registryViewForProviders = aps.getYarnRegistry(); + assertNotNull(registryViewForProviders); + + ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class); + AggregateConf instanceDefinition = new AggregateConf(); + + instanceDefinition.setInternal(tree); + instanceDefinition.setAppConf(tree); + instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.APP_DEF, "."); + instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_CONF, "."); + instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, "."); + + Container container = createNiceMock(Container.class); + String role_hm = "HBASE_MASTER"; + SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class); + ContainerLauncher launcher = createNiceMock(ContainerLauncher.class); + Path generatedConfPath = new Path(".", "test"); + MapOperations resourceComponent = new MapOperations(); + MapOperations appComponent = new MapOperations(); + Path containerTmpDirPath = new Path(".", "test"); + FilterFileSystem mockFs = createNiceMock(FilterFileSystem.class); + expect(sliderFileSystem.getFileSystem()) + .andReturn(mockFs).anyTimes(); + expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes(); + expect(sliderFileSystem.createAmResource(anyObject(Path.class), + anyObject(LocalResourceType.class))) + .andReturn(createNiceMock(LocalResource.class)).anyTimes(); + expect(container.getId()).andReturn(new MockContainerId(1)).anyTimes(); + expect(container.getNodeId()).andReturn(new MockNodeId("localhost")).anyTimes(); + StateAccessForProviders access = createNiceMock(StateAccessForProviders.class); + + AgentProviderService mockAps = Mockito.spy(aps); + doReturn(access).when(mockAps).getAmState(); + doReturn(metainfo).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString()); + doReturn(new HashMap<String, DefaultConfig>()).when(mockAps). + initializeDefaultConfigs(any(SliderFileSystem.class), anyString(), any(Metainfo.class)); + + + try { + doReturn(true).when(mockAps).isMaster(anyString()); + doNothing().when(mockAps).addInstallCommand( + anyString(), + anyString(), + any(HeartBeatResponse.class), + anyString(), + Mockito.anyLong()); + doReturn(conf).when(mockAps).getConfig(); + } catch (SliderException e) { + } + + expect(access.isApplicationLive()).andReturn(true).anyTimes(); + ClusterDescription desc = new ClusterDescription(); + desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181"); + desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE"); + expect(access.getClusterStatus()).andReturn(desc).anyTimes(); + + AggregateConf aggConf = new AggregateConf(); + ConfTreeOperations treeOps = aggConf.getAppConfOperations(); + treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0"); + treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE"); + expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf).anyTimes(); + expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes(); + replay(access, ctx, container, sliderFileSystem, mockFs); + + try { + mockAps.buildContainerLaunchContext(launcher, + instanceDefinition, + container, + role_hm, + sliderFileSystem, + generatedConfPath, + resourceComponent, + appComponent, + containerTmpDirPath); + + Register reg = new Register(); + reg.setResponseId(0); + reg.setLabel("mockcontainer_1___HBASE_MASTER"); + RegistrationResponse resp = mockAps.handleRegistration(reg); + Assert.assertEquals(0, resp.getResponseId()); + Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus()); + + HeartBeat hb = new HeartBeat(); + hb.setResponseId(1); + hb.setHostname("mockcontainer_1___HBASE_MASTER"); + HeartBeatResponse hbr = mockAps.handleHeartBeat(hb); + Assert.assertEquals(2, hbr.getResponseId()); + Assert.assertFalse(hbr.isTerminateAgent()); + + hb.setResponseId(2); + CommandReport cr = new CommandReport(); + cr.setRole("HBASE_MASTER"); + cr.setRoleCommand("INSTALL"); + cr.setStatus("FAILED"); + hb.setReports(Arrays.asList(cr)); + hbr = mockAps.handleHeartBeat(hb); + Assert.assertEquals(3, hbr.getResponseId()); + Assert.assertFalse(hbr.isTerminateAgent()); + + hb.setResponseId(3); + hbr = mockAps.handleHeartBeat(hb); + Assert.assertEquals(4, hbr.getResponseId()); + Assert.assertFalse(hbr.isTerminateAgent()); + + //Third failure triggers a stop to the agent + hb.setResponseId(4); + hbr = mockAps.handleHeartBeat(hb); + Assert.assertEquals(5, hbr.getResponseId()); + Assert.assertTrue(hbr.isTerminateAgent()); + Mockito.verify(mockAps, Mockito.times(3)).addInstallCommand(anyString(), + anyString(), + any(HeartBeatResponse.class), + anyString(), + Mockito.anyLong()); + } catch (SliderException he) { + log.warn(he.getMessage()); + } catch (IOException he) { + log.warn(he.getMessage()); + } + } + + @Test public void testAgentStateStarted() throws IOException, SliderException { AggregateConf instanceDefinition = prepareConfForAgentStateTests(); AgentProviderService mockAps = prepareProviderServiceForAgentStateTests(); @@ -1056,7 +1186,7 @@ public class TestAgentProviderService { FilterFileSystem mockFs = createNiceMock(FilterFileSystem.class); expect(sliderFileSystem.getFileSystem()) .andReturn(mockFs).anyTimes(); - expect(mockFs.exists(anyObject(Path.class))).andReturn(true); + expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes(); expect(sliderFileSystem.createAmResource(anyObject(Path.class), anyObject(LocalResourceType.class))) .andReturn(createNiceMock(LocalResource.class)).anyTimes();
