Repository: ambari Updated Branches: refs/heads/branch-2.5 6632caf53 -> bbbe15d96
AMBARI-19933. HSI is not started with configured Hiveserver2 heap size, upgrade changes (smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bbbe15d9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bbbe15d9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bbbe15d9 Branch: refs/heads/branch-2.5 Commit: bbbe15d96136c5153e3dc3e0388e447717cd0779 Parents: 6632caf Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Thu Feb 9 10:55:18 2017 -0800 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Thu Feb 9 10:55:23 2017 -0800 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog250.java | 28 ++++++-- .../server/upgrade/UpgradeCatalog250Test.java | 69 ++++++++++++++++++++ 2 files changed, 90 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/bbbe15d9/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index 3ea9f9e..dd0d048 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -681,6 +681,8 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { * @throws AmbariException */ private static final String HIVE_INTERACTIVE_SITE = "hive-interactive-site"; + private static final String HIVE_INTERACTIVE_ENV = "hive-interactive-env"; + private static final String HIVE_ENV = "hive-env"; protected void updateHIVEInteractiveConfigs() throws AmbariException { AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); Clusters clusters = ambariManagementController.getClusters(); @@ -691,24 +693,36 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { for (final Cluster cluster : clusterMap.values()) { Config hiveInteractiveSite = cluster.getDesiredConfigByType(HIVE_INTERACTIVE_SITE); if (hiveInteractiveSite != null) { - updateConfigurationProperties(HIVE_INTERACTIVE_SITE, Collections.singletonMap("hive.tez.container.size", - "SET_ON_FIRST_INVOCATION"), true, true); + Map<String, String> newProperties = new HashMap<>(); + newProperties.put("hive.auto.convert.join.noconditionaltask.size", "1000000000"); - updateConfigurationProperties(HIVE_INTERACTIVE_SITE, Collections.singletonMap("hive.auto.convert.join.noconditionaltask.size", - "1000000000"), true, true); String llapRpcPortString = hiveInteractiveSite.getProperties().get("hive.llap.daemon.rpc.port"); if (StringUtils.isNotBlank(llapRpcPortString)) { try { int llapRpcPort = Integer.parseInt(llapRpcPortString); if (llapRpcPort == 15001) { - updateConfigurationProperties(HIVE_INTERACTIVE_SITE, - Collections.singletonMap("hive.llap.daemon.rpc.port", "0"), - true, true); + newProperties.put("hive.llap.daemon.rpc.port", "0"); + LOG.info("Updating HSI hive.llap.daemon.rpc.port to: 0"); } } catch (NumberFormatException e) { LOG.warn("Unable to parse llap.rpc.port as integer: " + llapRpcPortString); } } + updateConfigurationProperties(HIVE_INTERACTIVE_SITE, newProperties, true, true); + } + + Config hiveInteractiveEnv = cluster.getDesiredConfigByType(HIVE_INTERACTIVE_ENV); + Config hiveEnv = cluster.getDesiredConfigByType(HIVE_ENV); + if (hiveInteractiveEnv != null) { + String hsiHeapSize = "512"; + if (hiveEnv != null) { + if (hiveEnv.getProperties().containsKey("hive.heapsize")) { + hsiHeapSize = hiveEnv.getProperties().get("hive.heapsize"); + LOG.info("Updating HSI heap size to: " + hsiHeapSize); + } + } + updateConfigurationProperties(HIVE_INTERACTIVE_ENV, Collections.singletonMap("hive_heapsize", + hsiHeapSize), true, true); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/bbbe15d9/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java index ab9094f..844bb4e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java @@ -1428,6 +1428,75 @@ public class UpgradeCatalog250Test { } @Test + public void testUpdateHiveConfigs() throws Exception { + reset(clusters, cluster); + expect(clusters.getClusters()).andReturn(ImmutableMap.of("normal", cluster)).anyTimes(); + + EasyMockSupport easyMockSupport = new EasyMockSupport(); + + Injector injector2 = easyMockSupport.createNiceMock(Injector.class); + AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) + .addMockedMethod("createConfiguration") + .addMockedMethod("getClusters", new Class[] {}) + .addMockedMethod("createConfig") + .withConstructor(actionManager, clusters, injector) + .createNiceMock(); + + expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); + expect(controller.getClusters()).andReturn(clusters).anyTimes(); + + Map<String, String> oldHsiEnv = ImmutableMap.of( + "llap_app_name", "llap0"); + + Map<String, String> expectedHsiEnv = ImmutableMap.of( + "llap_app_name", "llap0", + "hive_heapsize", "1082"); + + Map<String, String> oldHiveEnv = ImmutableMap.of( + "hive.client.heapsize", "1024", + "hive.heapsize", "1082", + "hive.metastore.heapsize", "512", + "hive_ambari_database", "MySQL"); + + Map<String, String> oldHiveIntSite = ImmutableMap.of( + "hive.llap.daemon.rpc.port","15001"); + + Map<String, String> expectedHiveIntSite = ImmutableMap.of( + "hive.llap.daemon.rpc.port","0", + "hive.auto.convert.join.noconditionaltask.size", "1000000000"); + + Config mockHsiSite = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("hive-interactive-site")).andReturn(mockHsiSite).atLeastOnce(); + expect(mockHsiSite.getProperties()).andReturn(oldHiveIntSite).anyTimes(); + Capture<Map<String, String>> hsiSiteCapture = EasyMock.newCapture(); + expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(hsiSiteCapture), anyString(), + EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once(); + + Config mockHiveEnv = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("hive-env")).andReturn(mockHiveEnv).atLeastOnce(); + expect(mockHiveEnv.getProperties()).andReturn(oldHiveEnv).anyTimes(); + + Config mockHsiEnv = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("hive-interactive-env")).andReturn(mockHsiEnv).atLeastOnce(); + expect(mockHsiEnv.getProperties()).andReturn(oldHsiEnv).anyTimes(); + Capture<Map<String, String>> hsiEnvCapture = EasyMock.newCapture(); + expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(hsiEnvCapture), anyString(), + EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once(); + + replay(clusters, cluster); + replay(controller, injector2); + replay(mockHsiEnv, mockHiveEnv, mockHsiSite); + new UpgradeCatalog250(injector2).updateHIVEInteractiveConfigs(); + easyMockSupport.verifyAll(); + + Map<String, String> updatedHsiSite = hsiSiteCapture.getValue(); + assertTrue(Maps.difference(expectedHiveIntSite, updatedHsiSite).areEqual()); + + Map<String, String> updatedHsiEnv = hsiEnvCapture.getValue(); + assertTrue(Maps.difference(expectedHsiEnv, updatedHsiEnv).areEqual()); + } + + @Test public void testUpdateAtlasConfigs() throws Exception { Map<String, String> oldHiveProperties = new HashMap<String, String>();
