Revert "AMBARI-18954. Use 'Number of LLAP Nodes' selected as the driver for LLAP config calculations."
This reverts commit 1162778e36f5b4e5a0f0d912a431559b049684e1. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4fcc4938 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4fcc4938 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4fcc4938 Branch: refs/heads/branch-feature-AMBARI-18901 Commit: 4fcc493834d2cdf4ff7edbf7e402e856a8447e93 Parents: 8be71ba Author: Swapan Shridhar <[email protected]> Authored: Mon Nov 21 12:28:44 2016 -0800 Committer: Swapan Shridhar <[email protected]> Committed: Mon Nov 21 12:32:31 2016 -0800 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog250.java | 71 +- .../HIVE/configuration/hive-interactive-env.xml | 42 +- .../configuration/hive-interactive-site.xml | 68 +- .../HIVE/configuration/tez-interactive-site.xml | 71 +- .../HDP/2.5/services/HIVE/themes/theme.json | 27 +- .../YARN/configuration/capacity-scheduler.xml | 2 +- .../stacks/HDP/2.5/services/stack_advisor.py | 665 ++++++++----------- .../server/upgrade/UpgradeCatalog250Test.java | 287 +------- 8 files changed, 379 insertions(+), 854 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4fcc4938/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index 6de48c1..bc3c120 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -17,8 +17,13 @@ */ package org.apache.ambari.server.upgrade; -import com.google.inject.Inject; -import com.google.inject.Injector; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.CommandExecutionType; import org.apache.ambari.server.controller.AmbariManagementController; @@ -32,12 +37,8 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import com.google.inject.Inject; +import com.google.inject.Injector; /** * Upgrade catalog for version 2.5.0. @@ -133,8 +134,6 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { addNewConfigurationsFromXml(); updateAMSConfigs(); updateKafkaConfigs(); - updateHIVEInteractiveConfigs(); - updateTEZInteractiveConfigs(); } protected void updateHostVersionTable() throws SQLException { @@ -270,57 +269,5 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { dbAccessor.addColumn(SERVICE_DESIRED_STATE_TABLE, new DBColumnInfo(CREDENTIAL_STORE_ENABLED_COL, Short.class, null, 0, false)); } - - /** - * Updates Hive Interactive's config in hive-interactive-site. - * - * @throws AmbariException - */ - protected void updateHIVEInteractiveConfigs() throws AmbariException { - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - Clusters clusters = ambariManagementController.getClusters(); - if (clusters != null) { - Map<String, Cluster> clusterMap = clusters.getClusters(); - - if (clusterMap != null && !clusterMap.isEmpty()) { - for (final Cluster cluster : clusterMap.values()) { - Config hiveInteractiveSite = cluster.getDesiredConfigByType("hive-interactive-site"); - if (hiveInteractiveSite != null) { - updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.tez.container.size", - "SET_ON_FIRST_INVOCATION"), true, true); - - updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.auto.convert.join.noconditionaltask.size", - "1000000000"), true, true); - } - } - } - } - } - - /** - * Updates Tez for Hive2 Interactive's config in tez-interactive-site. - * - * @throws AmbariException - */ - protected void updateTEZInteractiveConfigs() throws AmbariException { - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - Clusters clusters = ambariManagementController.getClusters(); - if (clusters != null) { - Map<String, Cluster> clusterMap = clusters.getClusters(); - - if (clusterMap != null && !clusterMap.isEmpty()) { - for (final Cluster cluster : clusterMap.values()) { - Config tezInteractiveSite = cluster.getDesiredConfigByType("tez-interactive-site"); - if (tezInteractiveSite != null) { - - updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.io.sort.mb", "512"), true, true); - - updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.unordered.output.buffer.size-mb", - "100"), true, true); - } - } - } - } - } } http://git-wip-us.apache.org/repos/asf/ambari/blob/4fcc4938/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml index 5a186b1..1fd72eb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml @@ -53,14 +53,40 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>llap_queue_capacity</name> + <value>0</value> + <description>Percentage of the cluster dedicated to interactive query.</description> + <display-name>% of Cluster Capacity</display-name> + <value-attributes> + <type>int</type> + <minimum>20</minimum> + <maximum>100</maximum> + <increment-step>1</increment-step> + </value-attributes> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> + <property> <name>num_llap_nodes</name> <value>1</value> <description>The number of Hive LLAP daemons to run.</description> - <display-name>Number of LLAP Nodes</display-name> + <display-name>Number of LLAP Daemons</display-name> <value-attributes> <type>int</type> <minimum>1</minimum> - <increment-step>1</increment-step> </value-attributes> <depends-on> <property> @@ -68,6 +94,10 @@ <name>enable_hive_interactive</name> </property> <property> + <type>hive-interactive-env</type> + <name>llap_queue_capacity</name> + </property> + <property> <type>hive-interactive-site</type> <name>hive.llap.daemon.queue.name</name> </property> @@ -75,6 +105,10 @@ <type>capacity-scheduler</type> <name>yarn.scheduler.capacity.root.queues</name> </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> </depends-on> <on-ambari-upgrade add="true"/> </property> @@ -106,7 +140,7 @@ </property> <property> <type>hive-interactive-env</type> - <name>num_llap_nodes</name> + <name>llap_queue_capacity</name> </property> <property> <type>hive-interactive-site</type> @@ -143,7 +177,7 @@ </property> <property> <type>hive-interactive-env</type> - <name>num_llap_nodes</name> + <name>llap_queue_capacity</name> </property> <property> <type>hive-interactive-site</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/4fcc4938/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml index 2fb1553..0207e49 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml @@ -67,7 +67,7 @@ limitations under the License. </property> <property> <type>hive-interactive-env</type> - <name>num_llap_nodes</name> + <name>llap_queue_capacity</name> </property> <property> <type>hive-interactive-site</type> @@ -411,7 +411,7 @@ limitations under the License. </property> <property> <type>hive-interactive-env</type> - <name>num_llap_nodes</name> + <name>llap_queue_capacity</name> </property> <property> <type>hive-interactive-site</type> @@ -447,7 +447,7 @@ limitations under the License. </property> <property> <type>hive-interactive-env</type> - <name>num_llap_nodes</name> + <name>llap_queue_capacity</name> </property> <property> <type>hive-interactive-site</type> @@ -478,7 +478,7 @@ limitations under the License. <name>hive.llap.daemon.yarn.container.mb</name> <display-name>Memory per daemon</display-name> <description>Total memory used by individual LLAP daemons. This includes memory for the cache as well as for the query execution.</description> - <value>0</value> + <value>341</value> <value-attributes> <type>int</type> <unit>MB</unit> @@ -491,7 +491,7 @@ limitations under the License. </property> <property> <type>hive-interactive-env</type> - <name>num_llap_nodes</name> + <name>llap_queue_capacity</name> </property> <property> <type>hive-interactive-site</type> @@ -646,62 +646,4 @@ limitations under the License. </value-attributes> <on-ambari-upgrade add="false"/> </property> - <property> - <name>hive.tez.container.size</name> - <value>SET_ON_FIRST_INVOCATION</value> - <depends-on> - <property> - <type>hive-interactive-env</type> - <name>enable_hive_interactive</name> - </property> - <property> - <type>hive-interactive-env</type> - <name>num_llap_nodes</name> - </property> - <property> - <type>hive-interactive-site</type> - <name>hive.llap.daemon.queue.name</name> - </property> - <property> - <type>capacity-scheduler</type> - <name>yarn.scheduler.capacity.root.queues</name> - </property> - <property> - <type>hive-interactive-site</type> - <name>hive.server2.tez.sessions.per.default.queue</name> - </property> - </depends-on> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>hive.auto.convert.join.noconditionaltask.size</name> - <value>1000000000</value> - <description>If hive.auto.convert.join.noconditionaltask is off, this parameter does not take affect. However, if it - is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than this size, the join is directly - converted to a mapjoin(there is no conditional task). The default is 10MB. - </description> - <depends-on> - <property> - <type>hive-interactive-env</type> - <name>enable_hive_interactive</name> - </property> - <property> - <type>hive-interactive-env</type> - <name>num_llap_nodes</name> - </property> - <property> - <type>hive-interactive-site</type> - <name>hive.llap.daemon.queue.name</name> - </property> - <property> - <type>capacity-scheduler</type> - <name>yarn.scheduler.capacity.root.queues</name> - </property> - <property> - <type>hive-interactive-site</type> - <name>hive.server2.tez.sessions.per.default.queue</name> - </property> - </depends-on> - <on-ambari-upgrade add="true"/> - </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/4fcc4938/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml index b331736..9e588e9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml @@ -83,8 +83,11 @@ </property> <property> <name>tez.am.resource.memory.mb</name> - <value>0</value> + <value>1536</value> <description>The amount of memory to be used by the AppMaster</description> + <value-attributes> + <type>int</type> + </value-attributes> <depends-on> <property> <type>hive-interactive-env</type> @@ -92,71 +95,7 @@ </property> <property> <type>hive-interactive-env</type> - <name>num_llap_nodes</name> - </property> - <property> - <type>hive-interactive-site</type> - <name>hive.llap.daemon.queue.name</name> - </property> - <property> - <type>capacity-scheduler</type> - <name>yarn.scheduler.capacity.root.queues</name> - </property> - <property> - <type>hive-interactive-site</type> - <name>hive.server2.tez.sessions.per.default.queue</name> - </property> - </depends-on> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>tez.runtime.io.sort.mb</name> - <value>512</value> - <description>The size of the sort buffer when output needs to be sorted</description> - <depends-on> - <property> - <type>tez-site</type> - <name>tez.task.resource.memory.mb</name> - </property> - <property> - <type>hive-interactive-env</type> - <name>enable_hive_interactive</name> - </property> - <property> - <type>hive-interactive-env</type> - <name>num_llap_nodes</name> - </property> - <property> - <type>hive-interactive-site</type> - <name>hive.llap.daemon.queue.name</name> - </property> - <property> - <type>capacity-scheduler</type> - <name>yarn.scheduler.capacity.root.queues</name> - </property> - <property> - <type>hive-interactive-site</type> - <name>hive.server2.tez.sessions.per.default.queue</name> - </property> - </depends-on> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>tez.runtime.unordered.output.buffer.size-mb</name> - <value>100</value> - <description>The size of the buffer when output does not require to be sorted</description> - <depends-on> - <property> - <type>tez-site</type> - <name>tez.task.resource.memory.mb</name> - </property> - <property> - <type>hive-interactive-env</type> - <name>enable_hive_interactive</name> - </property> - <property> - <type>hive-interactive-env</type> - <name>num_llap_nodes</name> + <name>llap_queue_capacity</name> </property> <property> <type>hive-interactive-site</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/4fcc4938/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json index 1d50b6a..452537d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json @@ -63,7 +63,11 @@ ] }, { - "config": "hive-interactive-env/num_llap_nodes", + "config": "hive-interactive-env/llap_queue_capacity", + "subsection-name": "interactive-query-row1-col1" + }, + { + "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue", "subsection-name": "interactive-query-row1-col1", "depends-on": [ { @@ -85,8 +89,12 @@ ] }, { - "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue", + "config": "hive-interactive-env/copy_num_llap_nodes", "subsection-name": "interactive-query-row1-col1", + "property_value_attributes": { + "ui_only_property": true, + "copy": "hive-interactive-env/num_llap_nodes" + }, "depends-on": [ { "configs":[ @@ -218,9 +226,9 @@ } }, { - "config": "hive-interactive-env/num_llap_nodes", + "config": "hive-interactive-env/copy_num_llap_nodes", "widget": { - "type": "slider", + "type": "label", "units": [ { "unit-name": "int" @@ -229,6 +237,17 @@ } }, { + "config": "hive-interactive-env/llap_queue_capacity", + "widget": { + "type": "slider", + "units": [ + { + "unit-name": "percent" + } + ] + } + }, + { "config": "hive-interactive-site/ambari.copy.hive.llap.daemon.yarn.container.mb", "widget": { "type": "label", http://git-wip-us.apache.org/repos/asf/ambari/blob/4fcc4938/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml index bc0ecc4..9ff8484 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml @@ -26,7 +26,7 @@ </property> <property> <type>hive-interactive-env</type> - <name>num_llap_nodes</name> + <name>llap_queue_capacity</name> </property> </depends-on> <on-ambari-upgrade add="false"/>
