Repository: ambari Updated Branches: refs/heads/branch-2.5 ef6c72da3 -> 7f2f40e33
AMBARI-20999. More updates to llap configs for hdp stack (Siddharth via smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7f2f40e3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7f2f40e3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7f2f40e3 Branch: refs/heads/branch-2.5 Commit: 7f2f40e33dd66e49192efd46c6e7a11c31b3c643 Parents: ef6c72d Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Thu May 11 17:03:27 2017 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Thu May 11 17:03:27 2017 -0700 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog250.java | 73 -------------------- .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 15 ++++ .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 12 ++++ .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 2 + .../configuration/hive-interactive-site.xml | 11 ++- .../HIVE/configuration/tez-interactive-site.xml | 13 ++++ .../server/upgrade/UpgradeCatalog250Test.java | 23 +----- 7 files changed, 52 insertions(+), 97 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7f2f40e3/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index c415a2b..53abe01 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -200,7 +200,6 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { updateHadoopEnvConfigs(); updateKafkaConfigs(); updateHIVEInteractiveConfigs(); - updateHiveLlapConfigs(); unInstallAllZeppelinViews(); updateZeppelinConfigs(); updateAtlasConfigs(); @@ -762,57 +761,6 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { return content; } - protected void updateHiveLlapConfigs() throws AmbariException { - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - Clusters clusters = ambariManagementController.getClusters(); - - if (clusters != null) { - Map<String, Cluster> clusterMap = clusters.getClusters(); - - if (clusterMap != null && !clusterMap.isEmpty()) { - for (final Cluster cluster : clusterMap.values()) { - Set<String> installedServices = cluster.getServices().keySet(); - - if (installedServices.contains("HIVE")) { - Config hiveSite = cluster.getDesiredConfigByType(HIVE_INTERACTIVE_SITE); - if (hiveSite != null) { - Map<String, String> hiveSiteProperties = hiveSite.getProperties(); - String schedulerDelay = hiveSiteProperties.get("hive.llap.task.scheduler.locality.delay"); - if (schedulerDelay != null) { - // Property exists. Change to new default if set to -1. - if (schedulerDelay.length() != 0) { - try { - int schedulerDelayInt = Integer.parseInt(schedulerDelay); - if (schedulerDelayInt == -1) { - // Old default. Set to new default. - updateConfigurationProperties(HIVE_INTERACTIVE_SITE, Collections - .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true, - false); - } - } catch (NumberFormatException e) { - // Invalid existing value. Set to new default. - updateConfigurationProperties(HIVE_INTERACTIVE_SITE, Collections - .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true, - false); - } - } - } - updateConfigurationProperties(HIVE_INTERACTIVE_SITE, - Collections.singletonMap("hive.mapjoin.hybridgrace.hashtable", "true"), true, - false); - updateConfigurationProperties("tez-interactive-site", - Collections.singletonMap("tez.session.am.dag.submit.timeout.secs", "1209600"), true, - false); - // Explicitly skipping hive.llap.allow.permanent.fns during upgrades, since it's related to security, - // and we don't know if the value is set by the user or as a result of the previous default. - } - } - } - } - } - } - - protected void updateHadoopEnvConfigs() throws AmbariException { AmbariManagementController ambariManagementController = injector.getInstance( AmbariManagementController.class); @@ -1045,7 +993,6 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { * * @throws AmbariException */ - private static final String HIVE_INTERACTIVE_SITE = "hive-interactive-site"; private static final String HIVE_INTERACTIVE_ENV = "hive-interactive-env"; private static final String HIVE_ENV = "hive-env"; protected void updateHIVEInteractiveConfigs() throws AmbariException { @@ -1056,26 +1003,6 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { if (clusterMap != null && !clusterMap.isEmpty()) { for (final Cluster cluster : clusterMap.values()) { - Config hiveInteractiveSite = cluster.getDesiredConfigByType(HIVE_INTERACTIVE_SITE); - if (hiveInteractiveSite != null) { - Map<String, String> newProperties = new HashMap<>(); - newProperties.put("hive.auto.convert.join.noconditionaltask.size", "1000000000"); - - String llapRpcPortString = hiveInteractiveSite.getProperties().get("hive.llap.daemon.rpc.port"); - if (StringUtils.isNotBlank(llapRpcPortString)) { - try { - int llapRpcPort = Integer.parseInt(llapRpcPortString); - if (llapRpcPort == 15001) { - newProperties.put("hive.llap.daemon.rpc.port", "0"); - LOG.info("Updating HSI hive.llap.daemon.rpc.port to: 0"); - } - } catch (NumberFormatException e) { - LOG.warn("Unable to parse llap.rpc.port as integer: " + llapRpcPortString); - } - } - updateConfigurationProperties(HIVE_INTERACTIVE_SITE, newProperties, true, true); - } - Config hiveInteractiveEnv = cluster.getDesiredConfigByType(HIVE_INTERACTIVE_ENV); Config hiveEnv = cluster.getDesiredConfigByType(HIVE_ENV); if (hiveInteractiveEnv != null) { http://git-wip-us.apache.org/repos/asf/ambari/blob/7f2f40e3/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 15a4f55..aadf6eb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -481,6 +481,21 @@ <set key="hive.vectorized.groupby.maxentries" value="1000000"/> </definition> + <definition xsi:type="configure" id="llap_update_settings" summary="Update additional LLAP settings"> + <type>hive-interactive-site</type> + <set key="hive.llap.task.scheduler.locality.delay" value="-1"/> + <set key="hive.mapjoin.hybridgrace.hashtable" value="false"/> + <set key="hive.merge.nway.joins" value="false"/> + <set key="hive.llap.daemon.rpc.port" value="0"/> + </definition> + + <definition xsi:type="configure" id="llap_update_tez_settings" summary="Update additional LLAP-Tez settings"> + <type>tez-interactive-site</type> + <set key="tez.runtime.shuffle.keep-alive.enabled" value="true"/> + <set key="tez.am.am-rm.heartbeat.interval-ms.max" value="30000"/> + <set key="tez.session.am.dag.submit.timeout.secs" value="1209600"/> + </definition> + <definition xsi:type="configure" id="hdp_2_6_0_0_copy_hive_tez_container_size_to_hiveInteractive"> <type>hive-interactive-site</type> <transfer operation="copy" from-type="hive-site" from-key="hive.tez.container.size" to-key="hive.tez.container.size" default-value="682" if-type="hive-interactive-site" if-key="hive.tez.container.size" if-key-state="absent"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/7f2f40e3/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 1886d93..befc38b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -558,6 +558,18 @@ </task> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Update additional LLAP settings"> + <task xsi:type="configure" id="llap_update_settings"> + <summary>Update additional LLAP settings</summary> + </task> + </execute-stage> + + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Update additional LLAP-Tez settings"> + <task xsi:type="configure" id="llap_update_tez_settings"> + <summary>Update additional LLAP-Tez settings</summary> + </task> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Update JVM stack size for LLAP"> <task xsi:type="configure" id="llap_append_stack_size_java_opts"> <summary>Updating JVM stack size for LLAP</summary> http://git-wip-us.apache.org/repos/asf/ambari/blob/7f2f40e3/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index 0470b7a..3054ca3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -845,6 +845,8 @@ <task xsi:type="configure" id="llap_update_hashaggregation"/> <task xsi:type="configure" id="llap_append_stack_size_java_opts"/> <task xsi:type="configure" id="llap_update_shuffle_parallel_copies"/> + <task xsi:type="configure" id="llap_update_settings"/> + <task xsi:type="configure" id="llap_update_tez_settings"/> </pre-upgrade> <pre-downgrade /> http://git-wip-us.apache.org/repos/asf/ambari/blob/7f2f40e3/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml index 0610833..cb4f947 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml @@ -30,7 +30,7 @@ limitations under the License. --> <property> <name>hive.llap.task.scheduler.locality.delay</name> - <value>8000</value> + <value>-1</value> <description> Amount of time to wait (in ms) before allocating a request which contains location information, to a location other than the ones requested. Set to -1 for an infinite delay, 0 @@ -41,7 +41,7 @@ limitations under the License. <property> <name>hive.mapjoin.hybridgrace.hashtable</name> - <value>true</value> + <value>false</value> <description>Whether to use hybrid grace hash join as the join method for mapjoin. Applies to dynamically partitioned joins when running in LLAP, but not to regular broadcast(map) joins. hive.llap.enable.grace.join.in.llap is used for this. @@ -103,4 +103,11 @@ limitations under the License. <on-ambari-upgrade add="false"/> </property> + <property> + <name>hive.merge.nway.joins</name> + <value>false</value> + <description>Merge adjacent joins into a single n-way join</description> + <on-ambari-upgrade add="false"/> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/7f2f40e3/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml index ab1202a..09416e3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml @@ -102,4 +102,17 @@ </description> <on-ambari-upgrade add="false"/> </property> + <property> + <name>tez.runtime.shuffle.keep-alive.enabled</name> + <value>true</value> + <description>Connection keep-alive for shuffle</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.am-rm.heartbeat.interval-ms.max</name> + <value>10000</value> + <description>The heartbeat interval between the tez AM and YARN RM</description> + <on-ambari-upgrade add="false"/> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/7f2f40e3/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java index 042c958..2590d1c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java @@ -288,7 +288,6 @@ public class UpgradeCatalog250Test { Method updateAmsConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateAMSConfigs"); Method updateHadoopEnvConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateHadoopEnvConfigs"); Method updateKafkaConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateKafkaConfigs"); - Method updateHiveLlapConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateHiveLlapConfigs"); Method updateTablesForZeppelinViewRemoval = UpgradeCatalog250.class.getDeclaredMethod("unInstallAllZeppelinViews"); Method updateZeppelinConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateZeppelinConfigs"); Method updateAtlasConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateAtlasConfigs"); @@ -312,7 +311,6 @@ public class UpgradeCatalog250Test { .addMockedMethod(updateHadoopEnvConfigs) .addMockedMethod(updateKafkaConfigs) .addMockedMethod(updateHIVEInteractiveConfigs) - .addMockedMethod(updateHiveLlapConfigs) .addMockedMethod(updateTablesForZeppelinViewRemoval) .addMockedMethod(updateZeppelinConfigs) .addMockedMethod(updateAtlasConfigs) @@ -347,9 +345,6 @@ public class UpgradeCatalog250Test { upgradeCatalog250.updateHIVEInteractiveConfigs(); expectLastCall().once(); - upgradeCatalog250.updateHiveLlapConfigs(); - expectLastCall().once(); - upgradeCatalog250.unInstallAllZeppelinViews(); expectLastCall().once(); @@ -1704,19 +1699,6 @@ public class UpgradeCatalog250Test { "hive.metastore.heapsize", "512", "hive_ambari_database", "MySQL"); - Map<String, String> oldHiveIntSite = ImmutableMap.of( - "hive.llap.daemon.rpc.port", "15001"); - - Map<String, String> expectedHiveIntSite = ImmutableMap.of( - "hive.llap.daemon.rpc.port", "0", - "hive.auto.convert.join.noconditionaltask.size", "1000000000"); - - Config mockHsiSite = easyMockSupport.createNiceMock(Config.class); - expect(cluster.getDesiredConfigByType("hive-interactive-site")).andReturn(mockHsiSite).atLeastOnce(); - expect(mockHsiSite.getProperties()).andReturn(oldHiveIntSite).anyTimes(); - Capture<Map<String, String>> hsiSiteCapture = EasyMock.newCapture(); - expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(hsiSiteCapture), anyString(), - EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once(); Config mockHiveEnv = easyMockSupport.createNiceMock(Config.class); expect(cluster.getDesiredConfigByType("hive-env")).andReturn(mockHiveEnv).atLeastOnce(); @@ -1731,13 +1713,10 @@ public class UpgradeCatalog250Test { replay(clusters, cluster); replay(controller, injector2); - replay(mockHsiEnv, mockHiveEnv, mockHsiSite); + replay(mockHsiEnv, mockHiveEnv); new UpgradeCatalog250(injector2).updateHIVEInteractiveConfigs(); easyMockSupport.verifyAll(); - Map<String, String> updatedHsiSite = hsiSiteCapture.getValue(); - assertTrue(Maps.difference(expectedHiveIntSite, updatedHsiSite).areEqual()); - Map<String, String> updatedHsiEnv = hsiEnvCapture.getValue(); assertTrue(Maps.difference(expectedHsiEnv, updatedHsiEnv).areEqual()); }
