AMBARI-18132 : Remove FIFO and Normalizer in ams-env config. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0e343585 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0e343585 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0e343585 Branch: refs/heads/branch-dev-patch-upgrade Commit: 0e3435856a2aee7951f765e90e55e027b6455c1a Parents: 9455b52 Author: Aravindan Vijayan <avija...@hortonworks.com> Authored: Tue Sep 13 20:32:07 2016 -0700 Committer: Aravindan Vijayan <avija...@hortonworks.com> Committed: Wed Sep 14 15:03:34 2016 -0700 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog250.java | 62 ++++++++++- .../0.1.0/configuration/ams-env.xml | 6 - .../0.1.0/package/scripts/params.py | 2 - .../server/upgrade/UpgradeCatalog250Test.java | 110 +++++++++++++++++++ 4 files changed, 171 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0e343585/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index 43f489b..35c773a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -18,9 +18,18 @@ package org.apache.ambari.server.upgrade; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.orm.dao.DaoUtils; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,7 +42,7 @@ import com.google.inject.Injector; public class UpgradeCatalog250 extends AbstractUpgradeCatalog { protected static final String HOST_VERSION_TABLE = "host_version"; - + private static final String AMS_ENV = "ams-env"; /** * Logger. */ @@ -97,6 +106,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { */ @Override protected void executeDMLUpdates() throws AmbariException, SQLException { + updateAMSConfigs(); } protected void updateHostVersionTable() throws SQLException { @@ -106,6 +116,56 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { dbAccessor.addUniqueConstraint(HOST_VERSION_TABLE, "UQ_host_repo", "repo_version_id", "host_id"); } + protected void updateAMSConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + + if (clusters != null) { + Map<String, Cluster> clusterMap = clusters.getClusters(); + + if (clusterMap != null && !clusterMap.isEmpty()) { + for (final Cluster cluster : clusterMap.values()) { + + Config amsEnv = cluster.getDesiredConfigByType(AMS_ENV); + if (amsEnv != null) { + Map<String, String> amsEnvProperties = amsEnv.getProperties(); + String content = amsEnvProperties.get("content"); + Map<String, String> newProperties = new HashMap<>(); + newProperties.put("content", updateAmsEnvContent(content)); + updateConfigurationPropertiesForCluster(cluster, AMS_ENV, newProperties, true, true); + } + + } + } + } + } + + + protected String updateAmsEnvContent(String content) { + if (content == null) { + return null; + } + + List<String> toReplaceList = new ArrayList<>(); + toReplaceList.add("\n# HBase normalizer enabled\n"); + toReplaceList.add("\n# HBase compaction policy enabled\n"); + toReplaceList.add("export AMS_HBASE_NORMALIZER_ENABLED={{ams_hbase_normalizer_enabled}}\n"); + toReplaceList.add("export AMS_HBASE_FIFO_COMPACTION_ENABLED={{ams_hbase_fifo_compaction_enabled}}\n"); + + //Because of AMBARI-15331 : AMS HBase FIFO compaction policy and Normalizer settings are not handled correctly + toReplaceList.add("export HBASE_NORMALIZATION_ENABLED={{ams_hbase_normalizer_enabled}}\n"); + toReplaceList.add("export HBASE_FIFO_COMPACTION_POLICY_ENABLED={{ams_hbase_fifo_compaction_policy_enabled}}\n"); + + + for (String toReplace : toReplaceList) { + if (content.contains(toReplace)) { + content = content.replace(toReplace, StringUtils.EMPTY); + } + } + + return content; + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/0e343585/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml index a52cb82..48232af 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml @@ -130,12 +130,6 @@ export AMS_HBASE_PID_DIR={{hbase_pid_dir}} # AMS Collector heapsize export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}} -# HBase normalizer enabled -export AMS_HBASE_NORMALIZER_ENABLED={{ams_hbase_normalizer_enabled}} - -# HBase compaction policy enabled -export AMS_HBASE_FIFO_COMPACTION_ENABLED={{ams_hbase_fifo_compaction_enabled}} - # HBase Tables Initialization check enabled export AMS_HBASE_INIT_CHECK_ENABLED={{ams_hbase_init_check_enabled}} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e343585/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index 61b89f8..22024bb 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -159,8 +159,6 @@ ams_grafana_cert_key = default("/configurations/ams-grafana-ini/cert_key", '/etc ams_hbase_home_dir = "/usr/lib/ams-hbase/" -ams_hbase_normalizer_enabled = default("/configurations/ams-hbase-site/hbase.normalizer.enabled", None) -ams_hbase_fifo_compaction_enabled = default("/configurations/ams-site/timeline.metrics.hbase.fifo.compaction.enabled", None) ams_hbase_init_check_enabled = default("/configurations/ams-site/timeline.metrics.hbase.init.check.enabled", True) #hadoop params http://git-wip-us.apache.org/repos/asf/ambari/blob/0e343585/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java index 6dadb22..c4e0a7c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java @@ -28,8 +28,21 @@ import static org.easymock.EasyMock.verify; import javax.persistence.EntityManager; +import com.google.common.collect.Maps; +import com.google.gson.Gson; +import org.apache.ambari.server.actionmanager.ActionManager; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.AmbariManagementControllerImpl; +import org.apache.ambari.server.controller.KerberosHelper; +import org.apache.ambari.server.controller.MaintenanceStateHelper; import org.apache.ambari.server.orm.DBAccessor; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; import org.apache.ambari.server.state.stack.OsFamily; +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.easymock.EasyMockSupport; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -40,6 +53,15 @@ import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.Provider; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Map; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; +import static org.easymock.EasyMock.capture; +import static org.easymock.EasyMock.createMockBuilder; +import static org.junit.Assert.assertTrue; /** * {@link UpgradeCatalog250} unit tests. */ @@ -86,4 +108,92 @@ public class UpgradeCatalog250Test { verify(dbAccessor); } + @Test + public void testExecuteDMLUpdates() throws Exception { + Method updateAmsConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateAMSConfigs"); + + UpgradeCatalog250 upgradeCatalog250 = createMockBuilder(UpgradeCatalog250.class) + .addMockedMethod(updateAmsConfigs) + .createMock(); + + upgradeCatalog250.updateAMSConfigs(); + expectLastCall().once(); + + replay(upgradeCatalog250); + + upgradeCatalog250.executeDMLUpdates(); + + verify(upgradeCatalog250); + } + + @Test + public void testAmsEnvUpdateConfigs() throws Exception{ + + Map<String, String> oldPropertiesAmsEnv = new HashMap<String, String>() { + { + put("content", "\n" + + "# AMS Collector heapsize\n" + + "export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}\n" + + "\n" + + "# HBase normalizer enabled\n" + + "export AMS_HBASE_NORMALIZER_ENABLED={{ams_hbase_normalizer_enabled}}\n" + + "\n" + + "# HBase compaction policy enabled\n" + + "export HBASE_FIFO_COMPACTION_POLICY_ENABLED={{ams_hbase_fifo_compaction_policy_enabled}}\n" + + "\n" + + "# HBase Tables Initialization check enabled\n" + + "export AMS_HBASE_INIT_CHECK_ENABLED={{ams_hbase_init_check_enabled}}\n"); + } + }; + Map<String, String> newPropertiesAmsEnv = new HashMap<String, String>() { + { + put("content", "\n" + + "# AMS Collector heapsize\n" + + "export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}\n" + + "\n" + + "# HBase Tables Initialization check enabled\n" + + "export AMS_HBASE_INIT_CHECK_ENABLED={{ams_hbase_init_check_enabled}}\n"); + } + }; + EasyMockSupport easyMockSupport = new EasyMockSupport(); + + Clusters clusters = easyMockSupport.createNiceMock(Clusters.class); + final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class); + Config mockAmsEnv = easyMockSupport.createNiceMock(Config.class); + + expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", cluster); + }}).once(); + expect(cluster.getDesiredConfigByType("ams-env")).andReturn(mockAmsEnv).atLeastOnce(); + expect(mockAmsEnv.getProperties()).andReturn(oldPropertiesAmsEnv).anyTimes(); + + Injector injector = easyMockSupport.createNiceMock(Injector.class); + expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes(); + expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null).anyTimes(); + expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class)).anyTimes(); + + replay(injector, clusters, mockAmsEnv, cluster); + + AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) + .addMockedMethod("createConfiguration") + .addMockedMethod("getClusters", new Class[] { }) + .addMockedMethod("createConfig") + .withConstructor(createNiceMock(ActionManager.class), clusters, injector) + .createNiceMock(); + + Injector injector2 = easyMockSupport.createNiceMock(Injector.class); + Capture<Map> propertiesCapture = EasyMock.newCapture(); + + expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); + expect(controller.getClusters()).andReturn(clusters).anyTimes(); + expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(propertiesCapture), anyString(), + anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); + + replay(controller, injector2); + new UpgradeCatalog250(injector2).updateAMSConfigs(); + easyMockSupport.verifyAll(); + + Map<String, String> updatedProperties = propertiesCapture.getValue(); + assertTrue(Maps.difference(newPropertiesAmsEnv, updatedProperties).areEqual()); + } }