Repository: ambari Updated Branches: refs/heads/trunk 426468881 -> 03f6398fc
AMBARI-18792. Update som configuration properties for hive interactive for the HDP 2.6 stack (sseth via Swapan Shridhar) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/03f6398f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/03f6398f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/03f6398f Branch: refs/heads/trunk Commit: 03f6398fcfa7d2e8accd72013d834815ba8755c5 Parents: 4264688 Author: Swapan Shridhar <[email protected]> Authored: Thu Nov 10 02:06:37 2016 -0800 Committer: Swapan Shridhar <[email protected]> Committed: Thu Nov 10 02:06:37 2016 -0800 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog250.java | 67 +++++++++++++++++--- .../configuration/hive-interactive-site.xml | 57 +++++++++++++++++ .../HIVE/configuration/tez-interactive-site.xml | 47 ++++++++++++++ 3 files changed, 161 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/03f6398f/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index 3425dd7..e679ada 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -17,13 +17,8 @@ */ package org.apache.ambari.server.upgrade; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - +import com.google.inject.Inject; +import com.google.inject.Injector; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.CommandExecutionType; import org.apache.ambari.server.controller.AmbariManagementController; @@ -37,8 +32,13 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.inject.Inject; -import com.google.inject.Injector; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * Upgrade catalog for version 2.5.0. @@ -120,7 +120,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { */ @Override protected void executePreDMLUpdates() throws AmbariException, SQLException { - + updateHiveLlapConfigs(); } /** @@ -214,6 +214,53 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { } } + protected void updateHiveLlapConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + + if (clusters != null) { + Map<String, Cluster> clusterMap = clusters.getClusters(); + + if (clusterMap != null && !clusterMap.isEmpty()) { + for (final Cluster cluster : clusterMap.values()) { + Set<String> installedServices = cluster.getServices().keySet(); + + if (installedServices.contains("HIVE")) { + Config hiveSite = cluster.getDesiredConfigByType("hive-interactive-site"); + if (hiveSite != null) { + Map<String, String> hiveSiteProperties = hiveSite.getProperties(); + String schedulerDelay = hiveSiteProperties.get("hive.llap.task.scheduler.locality.delay"); + if (schedulerDelay != null) { + // Property exists. Change to new default if set to -1. + if (schedulerDelay.length() != 0) { + try { + int schedulerDelayInt = Integer.parseInt(schedulerDelay); + if (schedulerDelayInt == -1) { + // Old default. Set to new default. + updateConfigurationProperties("hive-interactive-site", Collections + .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true, + false); + } + } catch (NumberFormatException e) { + // Invalid existing value. Set to new default. + updateConfigurationProperties("hive-interactive-site", Collections + .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true, + false); + } + } + } + updateConfigurationProperties("hive-interactive-site", + Collections.singletonMap("hive.mapjoin.hybridgrace.hashtable", "true"), true, + false); + // Explicitly skipping hive.llap.allow.permanent.fns during upgrades, since it's related to security, + // and we don't know if the value is set by the user or as a result of the previous default. + } + } + } + } + } + } + /** * Creates the servicecomponent_version table * @throws SQLException http://git-wip-us.apache.org/repos/asf/ambari/blob/03f6398f/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml new file mode 100644 index 0000000..2c18bcc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml @@ -0,0 +1,57 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<configuration supports_final="true"> + <!-- + Note: This file includes only those configs which are supposed to have different value from the parent hive/hive-site. + It inherits the other required configs from hive/hive-site. + The inheritance logic in order to get hive2/hive-site goes like this : + + 1. We read the hive/hive-site which includes the stack defaults and Stack Advisor recommended values. + 2. We take the read hive/hive-site (step 1), and on top of it apply the hive-interactive-site to get + hive2/hive-site. + + Therefore, any config defined here will override the config value read from hive2/hive-site (step 1). + --> + <property> + <name>hive.llap.task.scheduler.locality.delay</name> + <value>8000</value> + <description> + Amount of time to wait (in ms) before allocating a request which contains location information, + to a location other than the ones requested. Set to -1 for an infinite delay, 0 + for no delay. + </description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>hive.mapjoin.hybridgrace.hashtable</name> + <value>true</value> + <description>Whether to use hybrid grace hash join as the join method for mapjoin. Tez only.</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>hive.llap.enable.grace.join.in.llap</name> + <value>false</value> + <description>Override if grace join should be allowed to run in llap for regular map joins. + Dynamic partitioned joins will honor the hive.mapjoin.hybridgrace.hashtable property in LLAP + </description> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/03f6398f/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml new file mode 100644 index 0000000..2b80980 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml @@ -0,0 +1,47 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration supports_final="true"> + <!-- + Note: This file includes only those configs which are supposed to have different value from the parent tez/tez-site. + It inherits the other required configs from tez/tez-site. + The inheritance logic in order to get tez_hive2/tez-site goes like this : + + 1. We read the tez/tez-site which includes the stack defaults and Stack Advisor recommended values. + 2. We take the read tez/tez-site (step 1), and on top of it apply the tez-interactive-site to get tez_hive2/tez-site. + + Therefore, any config defined here will override the config value read from tez/tez-site (step 1). + --> + <property> + <name>tez.task.heartbeat.timeout.check-ms</name> + <value>11000</value> + <description>The time interval, in milliseconds, at which the AM will check + for timed out tasks + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>tez.task.timeout-ms</name> + <value>20000</value> + <description>Amount of time the Tez AM waits before marking a task which + has not sent in a heartbeat, as timed out + </description> + <on-ambari-upgrade add="true"/> + </property> + +</configuration>
