AMBARI-18792. Update some configuration properties for hive interactive for the HDP 2.6 stack (Siddharth Seth via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ca7e071f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ca7e071f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ca7e071f Branch: refs/heads/branch-feature-AMBARI-18456 Commit: ca7e071f044d70186c7bf9d18bf24c29f3767730 Parents: 2cc4c9e Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Tue Nov 29 11:39:53 2016 -0800 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Tue Nov 29 15:28:46 2016 -0800 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog250.java | 52 +++++++++++++ .../configuration/hive-interactive-site.xml | 58 +++++++++++++++ .../HIVE/configuration/tez-interactive-site.xml | 78 ++++++++++++++++++++ 3 files changed, 188 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ca7e071f/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index e81568c..3b2cdd3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.CommandExecutionType; @@ -134,6 +135,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { addNewConfigurationsFromXml(); updateAMSConfigs(); updateKafkaConfigs(); + updateHiveLlapConfigs(); } protected void updateHostVersionTable() throws SQLException { @@ -151,6 +153,56 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { dbAccessor.addUniqueConstraint(GROUPS_TABLE, "UNQ_groups_0", "group_name", "group_type"); } + protected void updateHiveLlapConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + + if (clusters != null) { + Map<String, Cluster> clusterMap = clusters.getClusters(); + + if (clusterMap != null && !clusterMap.isEmpty()) { + for (final Cluster cluster : clusterMap.values()) { + Set<String> installedServices = cluster.getServices().keySet(); + + if (installedServices.contains("HIVE")) { + Config hiveSite = cluster.getDesiredConfigByType("hive-interactive-site"); + if (hiveSite != null) { + Map<String, String> hiveSiteProperties = hiveSite.getProperties(); + String schedulerDelay = hiveSiteProperties.get("hive.llap.task.scheduler.locality.delay"); + if (schedulerDelay != null) { + // Property exists. Change to new default if set to -1. + if (schedulerDelay.length() != 0) { + try { + int schedulerDelayInt = Integer.parseInt(schedulerDelay); + if (schedulerDelayInt == -1) { + // Old default. Set to new default. + updateConfigurationProperties("hive-interactive-site", Collections + .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true, + false); + } + } catch (NumberFormatException e) { + // Invalid existing value. Set to new default. + updateConfigurationProperties("hive-interactive-site", Collections + .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true, + false); + } + } + } + updateConfigurationProperties("hive-interactive-site", + Collections.singletonMap("hive.mapjoin.hybridgrace.hashtable", "true"), true, + false); + updateConfigurationProperties("tez-interactive-site", + Collections.singletonMap("tez.session.am.dag.submit.timeout.secs", "1209600"), true, + false); + // Explicitly skipping hive.llap.allow.permanent.fns during upgrades, since it's related to security, + // and we don't know if the value is set by the user or as a result of the previous default. + } + } + } + } + } + } + protected void updateAMSConfigs() throws AmbariException { AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); Clusters clusters = ambariManagementController.getClusters(); http://git-wip-us.apache.org/repos/asf/ambari/blob/ca7e071f/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml new file mode 100644 index 0000000..f33ade7 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml @@ -0,0 +1,58 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<configuration supports_final="true"> + <!-- + Note: This file includes only those configs which are supposed to have different value from the parent hive/hive-site. + It inherits the other required configs from hive/hive-site. + The inheritance logic in order to get hive2/hive-site goes like this : + + 1. We read the hive/hive-site which includes the stack defaults and Stack Advisor recommended values. + 2. We take the read hive/hive-site (step 1), and on top of it apply the hive-interactive-site to get + hive2/hive-site. + + Therefore, any config defined here will override the config value read from hive2/hive-site (step 1). + --> + <property> + <name>hive.llap.task.scheduler.locality.delay</name> + <value>8000</value> + <description> + Amount of time to wait (in ms) before allocating a request which contains location information, + to a location other than the ones requested. Set to -1 for an infinite delay, 0 + for no delay. + </description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>hive.mapjoin.hybridgrace.hashtable</name> + <value>true</value> + <description>Whether to use hybrid grace hash join as the join method for mapjoin. Tez only.</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>hive.llap.enable.grace.join.in.llap</name> + <value>false</value> + <description>Override if grace join should be allowed to run in llap for regular map joins. + Dynamic partitioned joins will honor the hive.mapjoin.hybridgrace.hashtable property in LLAP + </description> + <on-ambari-upgrade add="true"/> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/ca7e071f/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml new file mode 100644 index 0000000..f7de121 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml @@ -0,0 +1,78 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration supports_final="true"> + <!-- + Note: This file includes only those configs which are supposed to have different value from the parent tez/tez-site. + It inherits the other required configs from tez/tez-site. + The inheritance logic in order to get tez_hive2/tez-site goes like this : + + 1. We read the tez/tez-site which includes the stack defaults and Stack Advisor recommended values. + 2. We take the read tez/tez-site (step 1), and on top of it apply the tez-interactive-site to get tez_hive2/tez-site. + + Therefore, any config defined here will override the config value read from tez/tez-site (step 1). + --> + <property> + <name>tez.task.heartbeat.timeout.check-ms</name> + <value>11000</value> + <description>The time interval, in milliseconds, at which the AM will check + for timed out tasks + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>tez.task.timeout-ms</name> + <value>20000</value> + <description>Amount of time the Tez AM waits before marking a task which + has not sent in a heartbeat, as timed out + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>tez.am.client.heartbeat.timeout.secs</name> + <value>60</value> + <description>The time interval, after which an AM will kill itself, if it + does not receive a heartbeat from the client. + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>tez.am.client.heartbeat.poll.interval.millis</name> + <value>6000</value> + <description>The interval at which the AM checks for a client heartbeat + timeout + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>tez.am.node-blacklisting.enabled</name> + <value>false</value> + <description>Whether to enable blacklisting in Tez AMs. Disable for LLAP + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>tez.session.am.dag.submit.timeout.secs</name> + <value>1209600</value> + <description>The amoung of time an AM will wait, before killing itself, + if not DAG is submitted. + </description> + <on-ambari-upgrade add="true"/> + </property> + +</configuration>
