AMBARI-18954. Use 'Number of LLAP Nodes' selected as the driver for LLAP config calculations.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4942deaf Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4942deaf Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4942deaf Branch: refs/heads/branch-feature-AMBARI-18901 Commit: 4942deaf5bd79f35beeaeb6f47844a93881502ea Parents: 4fcc493 Author: Swapan Shridhar <[email protected]> Authored: Mon Nov 21 12:37:04 2016 -0800 Committer: Swapan Shridhar <[email protected]> Committed: Mon Nov 21 12:37:04 2016 -0800 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog250.java | 71 +- .../HIVE/configuration/hive-interactive-env.xml | 42 +- .../configuration/hive-interactive-site.xml | 68 +- .../HIVE/configuration/tez-interactive-site.xml | 71 +- .../HDP/2.5/services/HIVE/themes/theme.json | 27 +- .../YARN/configuration/capacity-scheduler.xml | 2 +- .../stacks/HDP/2.5/services/stack_advisor.py | 665 +++++++++++-------- .../server/upgrade/UpgradeCatalog250Test.java | 287 +++++++- 8 files changed, 854 insertions(+), 379 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4942deaf/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index bc3c120..6de48c1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -17,13 +17,8 @@ */ package org.apache.ambari.server.upgrade; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - +import com.google.inject.Inject; +import com.google.inject.Injector; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.CommandExecutionType; import org.apache.ambari.server.controller.AmbariManagementController; @@ -37,8 +32,12 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.inject.Inject; -import com.google.inject.Injector; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * Upgrade catalog for version 2.5.0. @@ -134,6 +133,8 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { addNewConfigurationsFromXml(); updateAMSConfigs(); updateKafkaConfigs(); + updateHIVEInteractiveConfigs(); + updateTEZInteractiveConfigs(); } protected void updateHostVersionTable() throws SQLException { @@ -269,5 +270,57 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { dbAccessor.addColumn(SERVICE_DESIRED_STATE_TABLE, new DBColumnInfo(CREDENTIAL_STORE_ENABLED_COL, Short.class, null, 0, false)); } + + /** + * Updates Hive Interactive's config in hive-interactive-site. + * + * @throws AmbariException + */ + protected void updateHIVEInteractiveConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + if (clusters != null) { + Map<String, Cluster> clusterMap = clusters.getClusters(); + + if (clusterMap != null && !clusterMap.isEmpty()) { + for (final Cluster cluster : clusterMap.values()) { + Config hiveInteractiveSite = cluster.getDesiredConfigByType("hive-interactive-site"); + if (hiveInteractiveSite != null) { + updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.tez.container.size", + "SET_ON_FIRST_INVOCATION"), true, true); + + updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.auto.convert.join.noconditionaltask.size", + "1000000000"), true, true); + } + } + } + } + } + + /** + * Updates Tez for Hive2 Interactive's config in tez-interactive-site. + * + * @throws AmbariException + */ + protected void updateTEZInteractiveConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + if (clusters != null) { + Map<String, Cluster> clusterMap = clusters.getClusters(); + + if (clusterMap != null && !clusterMap.isEmpty()) { + for (final Cluster cluster : clusterMap.values()) { + Config tezInteractiveSite = cluster.getDesiredConfigByType("tez-interactive-site"); + if (tezInteractiveSite != null) { + + updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.io.sort.mb", "512"), true, true); + + updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.unordered.output.buffer.size-mb", + "100"), true, true); + } + } + } + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/4942deaf/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml index 1fd72eb..5a186b1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml @@ -53,40 +53,14 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>llap_queue_capacity</name> - <value>0</value> - <description>Percentage of the cluster dedicated to interactive query.</description> - <display-name>% of Cluster Capacity</display-name> - <value-attributes> - <type>int</type> - <minimum>20</minimum> - <maximum>100</maximum> - <increment-step>1</increment-step> - </value-attributes> - <depends-on> - <property> - <type>hive-interactive-env</type> - <name>enable_hive_interactive</name> - </property> - <property> - <type>hive-interactive-site</type> - <name>hive.llap.daemon.queue.name</name> - </property> - <property> - <type>capacity-scheduler</type> - <name>yarn.scheduler.capacity.root.queues</name> - </property> - </depends-on> - <on-ambari-upgrade add="true"/> - </property> - <property> <name>num_llap_nodes</name> <value>1</value> <description>The number of Hive LLAP daemons to run.</description> - <display-name>Number of LLAP Daemons</display-name> + <display-name>Number of LLAP Nodes</display-name> <value-attributes> <type>int</type> <minimum>1</minimum> + <increment-step>1</increment-step> </value-attributes> <depends-on> <property> @@ -94,10 +68,6 @@ <name>enable_hive_interactive</name> </property> <property> - <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> - </property> - <property> <type>hive-interactive-site</type> <name>hive.llap.daemon.queue.name</name> </property> @@ -105,10 +75,6 @@ <type>capacity-scheduler</type> <name>yarn.scheduler.capacity.root.queues</name> </property> - <property> - <type>hive-interactive-site</type> - <name>hive.server2.tez.sessions.per.default.queue</name> - </property> </depends-on> <on-ambari-upgrade add="true"/> </property> @@ -140,7 +106,7 @@ </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> @@ -177,7 +143,7 @@ </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/4942deaf/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml index 0207e49..2fb1553 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml @@ -67,7 +67,7 @@ limitations under the License. </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> @@ -411,7 +411,7 @@ limitations under the License. </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> @@ -447,7 +447,7 @@ limitations under the License. </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> @@ -478,7 +478,7 @@ limitations under the License. <name>hive.llap.daemon.yarn.container.mb</name> <display-name>Memory per daemon</display-name> <description>Total memory used by individual LLAP daemons. This includes memory for the cache as well as for the query execution.</description> - <value>341</value> + <value>0</value> <value-attributes> <type>int</type> <unit>MB</unit> @@ -491,7 +491,7 @@ limitations under the License. </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> @@ -646,4 +646,62 @@ limitations under the License. </value-attributes> <on-ambari-upgrade add="false"/> </property> + <property> + <name>hive.tez.container.size</name> + <value>SET_ON_FIRST_INVOCATION</value> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>hive.auto.convert.join.noconditionaltask.size</name> + <value>1000000000</value> + <description>If hive.auto.convert.join.noconditionaltask is off, this parameter does not take affect. However, if it + is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than this size, the join is directly + converted to a mapjoin(there is no conditional task). The default is 10MB. + </description> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/4942deaf/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml index 9e588e9..b331736 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml @@ -83,11 +83,8 @@ </property> <property> <name>tez.am.resource.memory.mb</name> - <value>1536</value> + <value>0</value> <description>The amount of memory to be used by the AppMaster</description> - <value-attributes> - <type>int</type> - </value-attributes> <depends-on> <property> <type>hive-interactive-env</type> @@ -95,7 +92,71 @@ </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>tez.runtime.io.sort.mb</name> + <value>512</value> + <description>The size of the sort buffer when output needs to be sorted</description> + <depends-on> + <property> + <type>tez-site</type> + <name>tez.task.resource.memory.mb</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>tez.runtime.unordered.output.buffer.size-mb</name> + <value>100</value> + <description>The size of the buffer when output does not require to be sorted</description> + <depends-on> + <property> + <type>tez-site</type> + <name>tez.task.resource.memory.mb</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> </property> <property> <type>hive-interactive-site</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/4942deaf/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json index 452537d..1d50b6a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json @@ -63,11 +63,7 @@ ] }, { - "config": "hive-interactive-env/llap_queue_capacity", - "subsection-name": "interactive-query-row1-col1" - }, - { - "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue", + "config": "hive-interactive-env/num_llap_nodes", "subsection-name": "interactive-query-row1-col1", "depends-on": [ { @@ -89,12 +85,8 @@ ] }, { - "config": "hive-interactive-env/copy_num_llap_nodes", + "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue", "subsection-name": "interactive-query-row1-col1", - "property_value_attributes": { - "ui_only_property": true, - "copy": "hive-interactive-env/num_llap_nodes" - }, "depends-on": [ { "configs":[ @@ -226,23 +218,12 @@ } }, { - "config": "hive-interactive-env/copy_num_llap_nodes", - "widget": { - "type": "label", - "units": [ - { - "unit-name": "int" - } - ] - } - }, - { - "config": "hive-interactive-env/llap_queue_capacity", + "config": "hive-interactive-env/num_llap_nodes", "widget": { "type": "slider", "units": [ { - "unit-name": "percent" + "unit-name": "int" } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/4942deaf/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml index 9ff8484..bc0ecc4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml @@ -26,7 +26,7 @@ </property> <property> <type>hive-interactive-env</type> - <name>llap_queue_capacity</name> + <name>num_llap_nodes</name> </property> </depends-on> <on-ambari-upgrade add="false"/>
