AMBARI-13420: RU should check for INSTALL_FAILED components before starting (jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/520e41f8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/520e41f8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/520e41f8 Branch: refs/heads/branch-dev-patch-upgrade Commit: 520e41f8c4a51cfc238f9cac7619e78cff191ad0 Parents: 5c800d2 Author: Jayush Luniya <[email protected]> Authored: Thu Oct 15 12:25:47 2015 -0700 Committer: Jayush Luniya <[email protected]> Committed: Thu Oct 15 12:25:47 2015 -0700 ---------------------------------------------------------------------- .../ambari/server/checks/CheckDescription.java | 7 + .../checks/ComponentsInstallationCheck.java | 100 ++++++++ .../checks/ComponentsInstallationCheckTest.java | 245 +++++++++++++++++++ .../checks/ConfigurationMergeCheckTest.java | 2 +- .../ServicesNamenodeTruncateCheckTest.java | 2 +- .../server/checks/ServicesUpCheckTest.java | 1 + 6 files changed, 355 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/520e41f8/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java index 7151b0e..fef4f7e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java @@ -133,6 +133,13 @@ public enum CheckDescription { "The following Services must be started: {{fails}}. Try to do a Stop & Start in case they were started outside of Ambari."); }}), + COMPONENTS_INSTALLATION(PrereqCheckType.SERVICE, + "All service components must be installed", + new HashMap<String, String>() {{ + put(AbstractCheckDescriptor.DEFAULT, + "The following Services must be reinstalled: {{fails}}. Try to reinstall the service components in INSTALL_FAILED state."); + }}), + SERVICES_YARN_WP(PrereqCheckType.SERVICE, "YARN work preserving restart should be enabled", new HashMap<String, String>() {{ http://git-wip-us.apache.org/repos/asf/ambari/blob/520e41f8/ambari-server/src/main/java/org/apache/ambari/server/checks/ComponentsInstallationCheck.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/ComponentsInstallationCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/ComponentsInstallationCheck.java new file mode 100644 index 0000000..07f4d05 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/ComponentsInstallationCheck.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.checks; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.PrereqCheckRequest; +import org.apache.ambari.server.orm.models.HostComponentSummary; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.ComponentInfo; +import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.ServiceComponent; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.State; +import org.apache.ambari.server.state.stack.PrereqCheckStatus; +import org.apache.ambari.server.state.stack.PrerequisiteCheck; +import org.apache.commons.lang.StringUtils; + +import com.google.inject.Singleton; + +/** + * Checks that services are up. + */ +@Singleton +@UpgradeCheck(group = UpgradeCheckGroup.LIVELINESS, order = 2.0f, required = true) +public class ComponentsInstallationCheck extends AbstractCheckDescriptor { + + /** + * Constructor. + */ + public ComponentsInstallationCheck() { + super(CheckDescription.COMPONENTS_INSTALLATION); + } + + @Override + public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request) throws AmbariException { + final String clusterName = request.getClusterName(); + final Cluster cluster = clustersProvider.get().getCluster(clusterName); + List<String> errorMessages = new ArrayList<String>(); + Set<String> failedServiceNames = new HashSet<String>(); + + StackId stackId = cluster.getCurrentStackVersion(); + + // Preq-req check should fail if any service component is in INSTALL_FAILED state + Set<String> installFailedHostComponents = new HashSet<String>(); + + for (Map.Entry<String, Service> serviceEntry : cluster.getServices().entrySet()) { + final Service service = serviceEntry.getValue(); + Map<String, ServiceComponent> serviceComponents = service.getServiceComponents(); + + for (Map.Entry<String, ServiceComponent> component : serviceComponents.entrySet()) { + ServiceComponent serviceComponent = component.getValue(); + + if (serviceComponent.isVersionAdvertised()) { + List<HostComponentSummary> hostComponentSummaries = HostComponentSummary.getHostComponentSummaries( + service.getName(), serviceComponent.getName()); + + for(HostComponentSummary hcs : hostComponentSummaries) { + if (State.INSTALL_FAILED == hcs.getCurrentState()) { + failedServiceNames.add(service.getName()); + installFailedHostComponents.add(MessageFormat.format( + "[{0}:{1} on {2}]", service.getName(), serviceComponent.getName(), hcs.getHostName())); + } + } + } + } + } + + if(!installFailedHostComponents.isEmpty()) { + String message = MessageFormat.format("Service components in INSTALL_FAILED state: {0}.", + StringUtils.join(installFailedHostComponents, ", ")); + prerequisiteCheck.setFailedOn(new LinkedHashSet<String>(failedServiceNames)); + prerequisiteCheck.setStatus(PrereqCheckStatus.FAIL); + prerequisiteCheck.setFailReason( + "Found service components in INSTALL_FAILED state. Please re-install these components. " + message); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/520e41f8/ambari-server/src/test/java/org/apache/ambari/server/checks/ComponentsInstallationCheckTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/ComponentsInstallationCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/ComponentsInstallationCheckTest.java new file mode 100644 index 0000000..e5a999a --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/ComponentsInstallationCheckTest.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.checks; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.PrereqCheckRequest; +import org.apache.ambari.server.orm.models.HostComponentSummary; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.ComponentInfo; +import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.ServiceComponent; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.State; +import org.apache.ambari.server.state.stack.PrereqCheckStatus; +import org.apache.ambari.server.state.stack.PrerequisiteCheck; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.google.inject.Provider; + + +/** + * Unit tests for ComponentsInstallationCheck + * + */ +@RunWith(PowerMockRunner.class) // Allow mocking static methods +@PrepareForTest(HostComponentSummary.class) // This class has a static method that will be mocked +public class ComponentsInstallationCheckTest { + private final Clusters clusters = Mockito.mock(Clusters.class); + private AmbariMetaInfo ambariMetaInfo = Mockito.mock(AmbariMetaInfo.class); + + @Test + public void testIsApplicable() throws Exception { + PrereqCheckRequest checkRequest = new PrereqCheckRequest("c1"); + checkRequest.setRepositoryVersion("HDP-2.2.0.0"); + checkRequest.setSourceStackId(new StackId("HDP", "2.2")); + checkRequest.setTargetStackId(new StackId("HDP", "2.2")); + ComponentsInstallationCheck cic = new ComponentsInstallationCheck(); + Configuration config = Mockito.mock(Configuration.class); + Mockito.when(config.getRollingUpgradeMinStack()).thenReturn("HDP-2.2"); + Mockito.when(config.getRollingUpgradeMaxStack()).thenReturn(""); + cic.config = config; + + Assert.assertTrue(cic.isApplicable(checkRequest)); + } + + @Test + public void testPerform() throws Exception { + PowerMockito.mockStatic(HostComponentSummary.class); + + final ComponentsInstallationCheck componentsInstallationCheck = new ComponentsInstallationCheck(); + componentsInstallationCheck.clustersProvider = new Provider<Clusters>() { + + @Override + public Clusters get() { + return clusters; + } + }; + + componentsInstallationCheck.ambariMetaInfo = new Provider<AmbariMetaInfo>() { + @Override + public AmbariMetaInfo get() { + return ambariMetaInfo; + } + }; + + final Cluster cluster = Mockito.mock(Cluster.class); + Mockito.when(cluster.getClusterId()).thenReturn(1L); + Mockito.when(cluster.getCurrentStackVersion()).thenReturn(new StackId("HDP", "2.2")); + Mockito.when(clusters.getCluster("cluster")).thenReturn(cluster); + + final Service hdfsService = Mockito.mock(Service.class); + final Service tezService = Mockito.mock(Service.class); + final Service amsService = Mockito.mock(Service.class); + + HashMap<String, Service> clusterServices = new HashMap<String, Service>(); + clusterServices.put("HDFS", hdfsService); + clusterServices.put("TEZ", tezService); + clusterServices.put("AMBARI_METRICS", amsService); + + Mockito.when(hdfsService.getName()).thenReturn("HDFS"); + Mockito.when(tezService.getName()).thenReturn("TEZ"); + Mockito.when(amsService.getName()).thenReturn("AMBARI_METRICS"); + + Mockito.when(hdfsService.isClientOnlyService()).thenReturn(false); + Mockito.when(tezService.isClientOnlyService()).thenReturn(true); + Mockito.when(amsService.isClientOnlyService()).thenReturn(false); + + Mockito.when(cluster.getServices()).thenReturn(clusterServices); + + Mockito.when(ambariMetaInfo.getComponent(Mockito.anyString(), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString())).thenAnswer(new Answer<ComponentInfo>() { + @Override + public ComponentInfo answer(InvocationOnMock invocation) throws Throwable { + ComponentInfo anyInfo = Mockito.mock(ComponentInfo.class); + if (invocation.getArguments().length > 3 && "DATANODE".equals(invocation.getArguments()[3])) { + Mockito.when(anyInfo.getCardinality()).thenReturn("1+"); + } else { + Mockito.when(anyInfo.getCardinality()).thenReturn(null); + } + + return anyInfo; + } + }); + + // Put Components inside Services + // HDFS + Map<String, ServiceComponent> hdfsComponents = new HashMap<String, ServiceComponent>(); + + ServiceComponent nameNode = Mockito.mock(ServiceComponent.class); + Mockito.when(nameNode.getName()).thenReturn("NAMENODE"); + Mockito.when(nameNode.isClientComponent()).thenReturn(false); + Mockito.when(nameNode.isVersionAdvertised()).thenReturn(true); + Mockito.when(nameNode.isMasterComponent()).thenReturn(true); + + ServiceComponent dataNode = Mockito.mock(ServiceComponent.class); + Mockito.when(dataNode.getName()).thenReturn("DATANODE"); + Mockito.when(dataNode.isClientComponent()).thenReturn(false); + Mockito.when(dataNode.isVersionAdvertised()).thenReturn(true); + Mockito.when(dataNode.isMasterComponent()).thenReturn(false); + + ServiceComponent zkfc = Mockito.mock(ServiceComponent.class); + Mockito.when(zkfc.getName()).thenReturn("ZKFC"); + Mockito.when(zkfc.isClientComponent()).thenReturn(false); + Mockito.when(zkfc.isVersionAdvertised()).thenReturn(false); + Mockito.when(zkfc.isMasterComponent()).thenReturn(false); + + hdfsComponents.put("NAMENODE", nameNode); + hdfsComponents.put("DATANODE", dataNode); + hdfsComponents.put("ZKFC", zkfc); + + Mockito.when(hdfsService.getServiceComponents()).thenReturn(hdfsComponents); + + // TEZ + Map<String, ServiceComponent> tezComponents = new HashMap<String, ServiceComponent>(); + + ServiceComponent tezClient = Mockito.mock(ServiceComponent.class); + Mockito.when(tezClient.getName()).thenReturn("TEZ_CLIENT"); + Mockito.when(tezClient.isClientComponent()).thenReturn(true); + Mockito.when(tezClient.isVersionAdvertised()).thenReturn(true); + + tezComponents.put("TEZ_CLIENT", tezClient); + + Mockito.when(tezService.getServiceComponents()).thenReturn(tezComponents); + + // AMS + Map<String, ServiceComponent> amsComponents = new HashMap<String, ServiceComponent>(); + + ServiceComponent metricsCollector = Mockito.mock(ServiceComponent.class); + Mockito.when(metricsCollector.getName()).thenReturn("METRICS_COLLECTOR"); + Mockito.when(metricsCollector.isClientComponent()).thenReturn(false); + Mockito.when(metricsCollector.isVersionAdvertised()).thenReturn(false); + + ServiceComponent metricsMonitor = Mockito.mock(ServiceComponent.class); + Mockito.when(metricsMonitor.getName()).thenReturn("METRICS_MONITOR"); + Mockito.when(metricsMonitor.isClientComponent()).thenReturn(false); + Mockito.when(metricsMonitor.isVersionAdvertised()).thenReturn(false); + + amsComponents.put("METRICS_COLLECTOR", metricsCollector); + amsComponents.put("METRICS_MONITOR", metricsMonitor); + + Mockito.when(amsService.getServiceComponents()).thenReturn(amsComponents); + + final HostComponentSummary hcsNameNode = Mockito.mock(HostComponentSummary.class); + final HostComponentSummary hcsDataNode1 = Mockito.mock(HostComponentSummary.class); + final HostComponentSummary hcsDataNode2 = Mockito.mock(HostComponentSummary.class); + final HostComponentSummary hcsDataNode3 = Mockito.mock(HostComponentSummary.class); + final HostComponentSummary hcsZKFC = Mockito.mock(HostComponentSummary.class); + final HostComponentSummary hcsTezClient = Mockito.mock(HostComponentSummary.class); + final HostComponentSummary hcsMetricsCollector = Mockito.mock(HostComponentSummary.class); + final HostComponentSummary hcsMetricsMonitor = Mockito.mock(HostComponentSummary.class); + + List<HostComponentSummary> allHostComponentSummaries = new ArrayList<HostComponentSummary>(); + allHostComponentSummaries.add(hcsNameNode); + allHostComponentSummaries.add(hcsDataNode1); + allHostComponentSummaries.add(hcsDataNode2); + allHostComponentSummaries.add(hcsDataNode3); + allHostComponentSummaries.add(hcsZKFC); + allHostComponentSummaries.add(hcsTezClient); + allHostComponentSummaries.add(hcsMetricsCollector); + allHostComponentSummaries.add(hcsMetricsMonitor); + + // Mock the static method + Mockito.when(HostComponentSummary.getHostComponentSummaries("HDFS", "NAMENODE")).thenReturn(Arrays.asList(hcsNameNode)); + Mockito.when(HostComponentSummary.getHostComponentSummaries("HDFS", "DATANODE")).thenReturn(Arrays.asList(hcsDataNode1, hcsDataNode2, hcsDataNode3)); + Mockito.when(HostComponentSummary.getHostComponentSummaries("HDFS", "ZKFC")).thenReturn(Arrays.asList(hcsZKFC)); + Mockito.when(HostComponentSummary.getHostComponentSummaries("TEZ", "TEZ_CLIENT")).thenReturn(Arrays.asList(hcsTezClient)); + Mockito.when(HostComponentSummary.getHostComponentSummaries("AMBARI_METRICS", "METRICS_COLLECTOR")).thenReturn(Arrays.asList(hcsMetricsCollector)); + Mockito.when(HostComponentSummary.getHostComponentSummaries("AMBARI_METRICS", "METRICS_MONITOR")).thenReturn(Arrays.asList(hcsMetricsMonitor)); + + // Case 1. Initialize with good values + for (HostComponentSummary hcs : allHostComponentSummaries) { + Mockito.when(hcs.getDesiredState()).thenReturn(State.INSTALLED); + Mockito.when(hcs.getCurrentState()).thenReturn(State.STARTED); + } + Mockito.when(hcsTezClient.getCurrentState()).thenReturn(State.INSTALLED); + PrerequisiteCheck check = new PrerequisiteCheck(null, null); + componentsInstallationCheck.perform(check, new PrereqCheckRequest("cluster")); + Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus()); + + // Case 2. Ensure that AMS is ignored even if their current state is not INSTALLED + Mockito.when(hcsMetricsCollector.getCurrentState()).thenReturn(State.INSTALL_FAILED); + Mockito.when(hcsMetricsMonitor.getCurrentState()).thenReturn(State.INSTALL_FAILED); + check = new PrerequisiteCheck(null, null); + componentsInstallationCheck.perform(check, new PrereqCheckRequest("cluster")); + Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus()); + + // Case 3: Change TEZ client state to INSTALL_FAILED, should fail + Mockito.when(hcsTezClient.getCurrentState()).thenReturn(State.INSTALL_FAILED); + check = new PrerequisiteCheck(null, null); + componentsInstallationCheck.perform(check, new PrereqCheckRequest("cluster")); + Assert.assertEquals(PrereqCheckStatus.FAIL, check.getStatus()); + Assert.assertTrue(check.getFailReason().indexOf("Service components in INSTALL_FAILED state") > -1); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/520e41f8/ambari-server/src/test/java/org/apache/ambari/server/checks/ConfigurationMergeCheckTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/ConfigurationMergeCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/ConfigurationMergeCheckTest.java index 85a2f02..dae0a58 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/checks/ConfigurationMergeCheckTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/ConfigurationMergeCheckTest.java @@ -48,7 +48,7 @@ import org.junit.Test; import com.google.inject.Provider; // /** - * Unit tests for ServicesUpCheck + * Unit tests for ConfigurationMergeCheck */ public class ConfigurationMergeCheckTest { http://git-wip-us.apache.org/repos/asf/ambari/blob/520e41f8/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java index ef39e9e..9fec6f1 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java @@ -43,7 +43,7 @@ import com.google.inject.Provider; import org.mockito.Mockito; /** - * Unit tests for ServicesUpCheck + * Unit tests for ServicesNamenodeTruncateCheck * */ public class ServicesNamenodeTruncateCheckTest { http://git-wip-us.apache.org/repos/asf/ambari/blob/520e41f8/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesUpCheckTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesUpCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesUpCheckTest.java index 38b4374..67b801c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesUpCheckTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesUpCheckTest.java @@ -226,6 +226,7 @@ public class ServicesUpCheckTest { } PrerequisiteCheck check = new PrerequisiteCheck(null, null); servicesUpCheck.perform(check, new PrereqCheckRequest("cluster")); + Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus()); // Case 2. Change some desired states to STARTED, should still pass Mockito.when(hcsNameNode.getDesiredState()).thenReturn(State.STARTED);
