AMBARI-20568. Update hash aggregation settings for llap in hdp stack (Siddharth Seth via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/82d81ea9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/82d81ea9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/82d81ea9 Branch: refs/heads/branch-3.0-perf Commit: 82d81ea9d55a9e777091100bf84d1b162dbf528e Parents: fc6bab1 Author: Sumit Mohanty <[email protected]> Authored: Fri Mar 24 16:05:04 2017 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Fri Mar 24 16:05:12 2017 -0700 ---------------------------------------------------------------------- .../HIVE/configuration/hive-interactive-site.xml | 19 +++++++++++++++++++ .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 6 ++++++ .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 7 +++++++ .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 1 + 4 files changed, 33 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/82d81ea9/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml index e11bb18..52d5db4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml @@ -703,4 +703,23 @@ limitations under the License. </depends-on> <on-ambari-upgrade add="true"/> </property> + <property> + <name>hive.map.aggr.hash.min.reduction</name> + <value>0.99</value> + <description> + Hash aggregation will be turned off if the ratio between hash table size and input rows is bigger than this number. + Set to 1 to make sure hash aggregation is never turned off. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.vectorized.groupby.maxentries</name> + <value>1000000</value> + <description> + Max number of entries in the vector group by aggregation hashtables. + Exceeding this will trigger a flush irrelevant of memory pressure condition. + </description> + <on-ambari-upgrade add="false"/> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/82d81ea9/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 941e28a..f908654 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -470,6 +470,12 @@ <set key="llap_headroom_space" value="12288"/> </definition> + <definition xsi:type="configure" id="llap_update_hashaggregation" summary="Update Hash Aggregation settings for LLAP"> + <type>hive-interactive-site</type> + <set key="hive.map.aggr.hash.min.reduction" value="0.99"/> + <set key="hive.vectorized.groupby.maxentries" value="1000000"/> + </definition> + <definition xsi:type="configure" id="hdp_2_6_0_0_copy_hive_tez_container_size_to_hiveInteractive"> <type>hive-interactive-site</type> <transfer operation="copy" from-type="hive-site" from-key="hive.tez.container.size" to-key="hive.tez.container.size" default-value="682" if-type="hive-interactive-site" if-key="hive.tez.container.size" if-key-state="absent"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/82d81ea9/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 5ee6ba4..c2ec6fd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -683,6 +683,13 @@ <summary>Updating the Hive Log4J2 properties to include parameterizations</summary> </task> </execute-stage> + + <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Update Hash Aggregation settings for LLAP"> + <task xsi:type="configure" id="llap_update_hashaggregation"> + <summary>Updating Hash Aggregation settings for LLAP</summary> + </task> + </execute-stage> + </group> <group xsi:type="restart" name="KAFKA" title="Kafka"> http://git-wip-us.apache.org/repos/asf/ambari/blob/82d81ea9/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index 7c31e57..381d6cc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -818,6 +818,7 @@ <task xsi:type="configure" id="hive_log4j2_parameterize"/> <task xsi:type="configure" id="llap_cli_log4j2_parameterize"/> <task xsi:type="configure" id="llap_update_headroom"/> + <task xsi:type="configure" id="llap_update_hashaggregation"/> </pre-upgrade> <pre-downgrade />
