AMBARI-22387. Create a Pre-Upgrade Check Warning About LZO (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/76349ac2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/76349ac2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/76349ac2 Branch: refs/heads/branch-feature-AMBARI-22008 Commit: 76349ac20d8955b3f25383b6f4f209a690b38bef Parents: 5122671 Author: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Authored: Mon Nov 13 13:29:52 2017 +0200 Committer: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Committed: Mon Nov 13 13:30:26 2017 +0200 ---------------------------------------------------------------------- .../ambari/server/checks/CheckDescription.java | 9 ++ .../apache/ambari/server/checks/LZOCheck.java | 77 ++++++++++ .../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 1 + .../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 1 + .../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 1 + .../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 1 + .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 1 + .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 1 + .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 1 + .../stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 1 + .../ambari/server/checks/LZOCheckTest.java | 145 +++++++++++++++++++ 11 files changed, 239 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/76349ac2/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java index 3caac14..20bb1b0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java @@ -327,6 +327,15 @@ public class CheckDescription { "As Ranger is SSL enabled, Ranger SSL configurations will need to be changed from default value of /etc/ranger/*/conf folder to /etc/ranger/security. " + "Since the certificates/keystores/truststores in this path may affect the upgrade/downgrade process, it is recommended to manually move the certificates/keystores/truststores out of the conf folders and change the appropriate config values before proceeding.").build()); + public static CheckDescription LZO_CONFIG_CHECK = new CheckDescription("LZO_CONFIG_CHECK", + PrereqCheckType.CLUSTER, + "LZO Codec Check", + new ImmutableMap.Builder<String, String>() + .put(AbstractCheckDescriptor.DEFAULT, + "You have LZO codec enabled in the core-site config of your cluster. LZO is no longer installed automatically. " + + "If any hosts require LZO, it should be installed before starting the upgrade. " + + "Consult Ambari documentation for instructions on how to do this.").build()); + public static CheckDescription JAVA_VERSION = new CheckDescription("JAVA_VERSION", PrereqCheckType.CLUSTER, "Verify Java version requirement", http://git-wip-us.apache.org/repos/asf/ambari/blob/76349ac2/ambari-server/src/main/java/org/apache/ambari/server/checks/LZOCheck.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/LZOCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/LZOCheck.java new file mode 100644 index 0000000..3000b79 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/LZOCheck.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.checks; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.PrereqCheckRequest; +import org.apache.ambari.server.state.stack.PrereqCheckStatus; +import org.apache.ambari.server.state.stack.PrerequisiteCheck; +import org.apache.commons.lang.StringUtils; + +import com.google.inject.Singleton; + +/** + * The {@link LZOCheck} + * is used to check that the LZO codec enabled in the core-site config fnd warning if any hosts require LZO, it should be installed before starting the upgrade. + */ +@Singleton +@UpgradeCheck(group = UpgradeCheckGroup.INFORMATIONAL_WARNING) +public class LZOCheck extends AbstractCheckDescriptor { + + final static String IO_COMPRESSION_CODECS = "io.compression.codecs"; + final static String LZO_ENABLE_KEY = "io.compression.codec.lzo.class"; + final static String LZO_ENABLE_VALUE = "com.hadoop.compression.lzo.LzoCodec"; + + /** + * Constructor. + */ + public LZOCheck() { + super(CheckDescription.LZO_CONFIG_CHECK); + } + + /** + * {@inheritDoc} + */ + @Override + public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request) throws AmbariException { + List<String> errorMessages = new ArrayList<>(); + PrereqCheckStatus checkStatus = PrereqCheckStatus.WARNING; + + String codecs = getProperty(request, "core-site", IO_COMPRESSION_CODECS); + if (codecs!= null && codecs.contains(LZO_ENABLE_VALUE)) { + errorMessages.add(getFailReason(IO_COMPRESSION_CODECS, prerequisiteCheck, request)); + } + String classValue = getProperty(request, "core-site", LZO_ENABLE_KEY); + + if (LZO_ENABLE_VALUE.equals(classValue)) { + errorMessages.add(getFailReason(LZO_ENABLE_KEY, prerequisiteCheck, request)); + } + + if (!errorMessages.isEmpty()) { + prerequisiteCheck.setFailReason(StringUtils.join(errorMessages, "You have LZO codec enabled in the core-site config of your cluster. LZO is no longer installed automatically. " + + "If any hosts require LZO, it should be installed before starting the upgrade. " + + "Consult Ambari documentation for instructions on how to do this.")); + prerequisiteCheck.getFailedOn().add("LZO"); + prerequisiteCheck.setStatus(checkStatus); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/76349ac2/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml index 907626d..bb22c1e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml @@ -24,6 +24,7 @@ <check>org.apache.ambari.server.checks.RangerAuditDbCheck</check> <check>org.apache.ambari.server.checks.ServicePresenceCheck</check> <check>org.apache.ambari.server.checks.RangerSSLConfigCheck</check> + <check>org.apache.ambari.server.checks.LZOCheck</check> <configuration> <!-- Configuration properties for all pre-reqs including required pre-reqs --> <check-properties name="org.apache.ambari.server.checks.HiveDynamicServiceDiscoveryCheck"> http://git-wip-us.apache.org/repos/asf/ambari/blob/76349ac2/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml index 567e6e1..faf8cbc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml @@ -36,6 +36,7 @@ <check>org.apache.ambari.server.checks.ServicePresenceCheck</check> <check>org.apache.ambari.server.checks.RangerAuditDbCheck</check> <check>org.apache.ambari.server.checks.RangerSSLConfigCheck</check> + <check>org.apache.ambari.server.checks.LZOCheck</check> <!-- Specific to HDP 2.5, Storm is not rolling --> <check>org.apache.ambari.server.checks.StormShutdownWarning</check> http://git-wip-us.apache.org/repos/asf/ambari/blob/76349ac2/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml index faf5b76..97824f2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml @@ -24,6 +24,7 @@ <check>org.apache.ambari.server.checks.RangerAuditDbCheck</check> <check>org.apache.ambari.server.checks.ServicePresenceCheck</check> <check>org.apache.ambari.server.checks.RangerSSLConfigCheck</check> + <check>org.apache.ambari.server.checks.LZOCheck</check> <configuration> <!-- Configuration properties for all pre-reqs including required pre-reqs --> http://git-wip-us.apache.org/repos/asf/ambari/blob/76349ac2/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml index 572a259..9e56d97 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml @@ -37,6 +37,7 @@ <check>org.apache.ambari.server.checks.ServicePresenceCheck</check> <check>org.apache.ambari.server.checks.RangerAuditDbCheck</check> <check>org.apache.ambari.server.checks.RangerSSLConfigCheck</check> + <check>org.apache.ambari.server.checks.LZOCheck</check> <!-- Specific to HDP 2.5, Storm is not rolling --> <check>org.apache.ambari.server.checks.StormShutdownWarning</check> http://git-wip-us.apache.org/repos/asf/ambari/blob/76349ac2/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 8012c90..ace9542 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -22,6 +22,7 @@ <type>NON_ROLLING</type> <prerequisite-checks> <check>org.apache.ambari.server.checks.RangerSSLConfigCheck</check> + <check>org.apache.ambari.server.checks.LZOCheck</check> <configuration> <!-- Configuration properties for all pre-reqs including required pre-reqs --> <check-properties name="org.apache.ambari.server.checks.HiveDynamicServiceDiscoveryCheck"> http://git-wip-us.apache.org/repos/asf/ambari/blob/76349ac2/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index 7c43948..df11ae1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -35,6 +35,7 @@ <check>org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck</check> <check>org.apache.ambari.server.checks.RangerSSLConfigCheck</check> <check>org.apache.ambari.server.checks.DruidHighAvailabilityCheck</check> + <check>org.apache.ambari.server.checks.LZOCheck</check> <configuration> <!-- Configuration properties for all pre-reqs including required pre-reqs --> http://git-wip-us.apache.org/repos/asf/ambari/blob/76349ac2/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml index ae2a855..0355362 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml @@ -21,6 +21,7 @@ <target-stack>HDP-2.6</target-stack> <type>NON_ROLLING</type> <prerequisite-checks> + <check>org.apache.ambari.server.checks.LZOCheck</check> <configuration> <!-- Configuration properties for all pre-reqs including required pre-reqs --> <check-properties name="org.apache.ambari.server.checks.HiveDynamicServiceDiscoveryCheck"> http://git-wip-us.apache.org/repos/asf/ambari/blob/76349ac2/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml index 37847a2..5aa2d20 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml @@ -34,6 +34,7 @@ <check>org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck</check> <check>org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck</check> <check>org.apache.ambari.server.checks.DruidHighAvailabilityCheck</check> + <check>org.apache.ambari.server.checks.LZOCheck</check> <configuration> <!-- Configuration properties for all pre-reqs including required pre-reqs --> http://git-wip-us.apache.org/repos/asf/ambari/blob/76349ac2/ambari-server/src/test/java/org/apache/ambari/server/checks/LZOCheckTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/LZOCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/LZOCheckTest.java new file mode 100644 index 0000000..e50e936 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/LZOCheckTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.checks; + + import java.util.HashMap; + import java.util.Map; + + import org.apache.ambari.server.configuration.Configuration; + import org.apache.ambari.server.controller.PrereqCheckRequest; + import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; + import org.apache.ambari.server.state.Cluster; + import org.apache.ambari.server.state.Clusters; + import org.apache.ambari.server.state.Config; + import org.apache.ambari.server.state.DesiredConfig; + import org.apache.ambari.server.state.RepositoryType; + import org.apache.ambari.server.state.Service; + import org.apache.ambari.server.state.repository.ClusterVersionSummary; + import org.apache.ambari.server.state.repository.VersionDefinitionXml; + import org.apache.ambari.server.state.stack.PrereqCheckStatus; + import org.apache.ambari.server.state.stack.PrerequisiteCheck; + import org.junit.Assert; + import org.junit.Before; + import org.junit.Test; + import org.junit.runner.RunWith; + import org.mockito.Mock; + import org.mockito.Mockito; + import org.mockito.runners.MockitoJUnitRunner; + + import com.google.inject.Provider; + + +/* Test for LZOCheck */ +@RunWith(MockitoJUnitRunner.class) +public class LZOCheckTest { + + private final Clusters clusters = Mockito.mock(Clusters.class); + private final LZOCheck lZOCheck = new LZOCheck(); + + @Mock + private ClusterVersionSummary m_clusterVersionSummary; + + @Mock + private VersionDefinitionXml m_vdfXml; + + @Mock + private RepositoryVersionEntity m_repositoryVersion; + + final Map<String, Service> m_services = new HashMap<>(); + + @Before + public void setup() throws Exception { + lZOCheck.clustersProvider = new Provider<Clusters>() { + @Override + public Clusters get() { + return clusters; + } + }; + Configuration config = Mockito.mock(Configuration.class); + lZOCheck.config = config; + + m_services.clear(); + + Mockito.when(m_repositoryVersion.getType()).thenReturn(RepositoryType.STANDARD); + Mockito.when(m_repositoryVersion.getRepositoryXml()).thenReturn(m_vdfXml); + Mockito.when(m_vdfXml.getClusterSummary(Mockito.any(Cluster.class))).thenReturn(m_clusterVersionSummary); + Mockito.when(m_clusterVersionSummary.getAvailableServiceNames()).thenReturn(m_services.keySet()); + } + + @Test + public void testIsApplicable() throws Exception { + final Cluster cluster = Mockito.mock(Cluster.class); + + Mockito.when(cluster.getServices()).thenReturn(m_services); + Mockito.when(cluster.getClusterId()).thenReturn(1L); + Mockito.when(clusters.getCluster("cluster")).thenReturn(cluster); + + PrereqCheckRequest request = new PrereqCheckRequest("cluster"); + request.setTargetRepositoryVersion(m_repositoryVersion); + + Assert.assertTrue(lZOCheck.isApplicable(request)); + } + + @Test + public void testPerform() throws Exception { + final Cluster cluster = Mockito.mock(Cluster.class); + final Map<String, Service> services = new HashMap<>(); + final Service service = Mockito.mock(Service.class); + + Mockito.when(cluster.getServices()).thenReturn(services); + Mockito.when(cluster.getClusterId()).thenReturn(1L); + Mockito.when(clusters.getCluster("cluster")).thenReturn(cluster); + + final DesiredConfig desiredConfig = Mockito.mock(DesiredConfig.class); + Mockito.when(desiredConfig.getTag()).thenReturn("tag"); + Map<String, DesiredConfig> configMap = new HashMap<>(); + configMap.put("core-site", desiredConfig); + + Mockito.when(cluster.getDesiredConfigs()).thenReturn(configMap); + final Config config = Mockito.mock(Config.class); + Mockito.when(cluster.getConfig(Mockito.anyString(), Mockito.anyString())).thenReturn(config); + final Map<String, String> properties = new HashMap<>(); + Mockito.when(config.getProperties()).thenReturn(properties); + + PrerequisiteCheck check = new PrerequisiteCheck(null, null); + lZOCheck.perform(check, new PrereqCheckRequest("cluster")); + Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus()); + + + properties.put(LZOCheck.IO_COMPRESSION_CODECS,"test," + LZOCheck.LZO_ENABLE_VALUE); + check = new PrerequisiteCheck(null, null); + lZOCheck.perform(check, new PrereqCheckRequest("cluster")); + Assert.assertEquals(PrereqCheckStatus.WARNING, check.getStatus()); + + properties.put(LZOCheck.IO_COMPRESSION_CODECS,"test"); + check = new PrerequisiteCheck(null, null); + lZOCheck.perform(check, new PrereqCheckRequest("cluster")); + Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus()); + + properties.put(LZOCheck.LZO_ENABLE_KEY, LZOCheck.LZO_ENABLE_VALUE); + check = new PrerequisiteCheck(null, null); + lZOCheck.perform(check, new PrereqCheckRequest("cluster")); + Assert.assertEquals(PrereqCheckStatus.WARNING, check.getStatus()); + + properties.put(LZOCheck.LZO_ENABLE_KEY, LZOCheck.LZO_ENABLE_VALUE); + properties.put(LZOCheck.IO_COMPRESSION_CODECS,"test," + LZOCheck.LZO_ENABLE_VALUE); + check = new PrerequisiteCheck(null, null); + lZOCheck.perform(check, new PrereqCheckRequest("cluster")); + Assert.assertEquals(PrereqCheckStatus.WARNING, check.getStatus()); + } +}