Repository: ambari Updated Branches: refs/heads/trunk d0290c095 -> 132cf31bc
AMBARI-10810. Make Hive theme changes to reduce number of configs (srimanth) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/132cf31b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/132cf31b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/132cf31b Branch: refs/heads/trunk Commit: 132cf31bc4eb797f708b09c388ce06860d598dd6 Parents: d0290c0 Author: Srimanth Gunturi <[email protected]> Authored: Tue Apr 28 16:35:14 2015 -0700 Committer: Srimanth Gunturi <[email protected]> Committed: Tue Apr 28 17:00:37 2015 -0700 ---------------------------------------------------------------------- .../HIVE/0.12.0.2.0/configuration/hive-site.xml | 10 +- .../services/HIVE/configuration/hive-site.xml | 16 +- .../HDP/2.2/services/HIVE/themes/theme.json | 370 ++++--------------- .../stacks/HDP/2.2/services/stack_advisor.py | 2 +- .../stacks/2.2/common/test_stack_advisor.py | 2 +- 5 files changed, 82 insertions(+), 318 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/132cf31b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml index 6cfb4ec..8715be2 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml @@ -356,21 +356,13 @@ limitations under the License. <type>value-list</type> <entries> <entry> - <value>ORCFile</value> + <value>ORC</value> <description>The Optimized Row Columnar (ORC) file format provides a highly efficient way to store Hive data. It was designed to overcome limitations of the other Hive file formats. Using ORC files improves performance when Hive is reading, writing, and processing data.</description> </entry> <entry> <value>TextFile</value> <description>Text file format saves Hive data as normal text.</description> </entry> - <entry> - <value>SequenceFile</value> - <description>Compressed text file format saves Hive data in CompressedStorage.</description> - </entry> - <entry> - <value>RCfile</value> - <description>Short of Record Columnar File, are flat files consisting of binary key/value pairs.</description> - </entry> </entries> </value-attributes> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/132cf31b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml index 95fd405..4525345 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml @@ -175,7 +175,7 @@ limitations under the License. <name>hive.exec.reducers.bytes.per.reducer</name> <value>67108864</value> <description>size per reducer.The default is 256Mb, i.e if the input size is 1G, it will use 4 reducers.</description> - <display-name>Memory</display-name> + <display-name>Data per Reducer</display-name> <value-attributes> <type>int</type> <minimum>64</minimum> @@ -1096,7 +1096,7 @@ limitations under the License. </entry> <entry> <value>LDAP</value> - <label>Lightweight Directory Access Protocol</label> + <label>LDAP</label> </entry> <entry> <value>KERBEROS</value> @@ -1104,7 +1104,11 @@ limitations under the License. </entry> <entry> <value>PAM</value> - <label>Pluggable Authentication Modules</label> + <label>PAM</label> + </entry> + <entry> + <value>CUSTOM</value> + <label>Custom</label> </entry> </entries> <selection-cardinality>1</selection-cardinality> @@ -1399,6 +1403,12 @@ limitations under the License. <unit>B</unit> <step-increment></step-increment> </value-attributes> + <depends-on> + <property> + <type>hive-site</type> + <name>hive.tez.container.size</name> + </property> + </depends-on> </property> <property> http://git-wip-us.apache.org/repos/asf/ambari/blob/132cf31b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json index efbe83b..c810f6d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json @@ -13,121 +13,83 @@ "tab-rows": 6, "tab-columns": 3, "sections": [ - { - "name": "optimization", - "display-name": "Optimization", + { + "name": "acid-transactions", + "display-name": "ACID Transactions", "row-index": "0", "column-index": "0", - "row-span": "3", - "column-span": "2", - "section-columns": "2", - "section-rows": "4", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", "subsections": [ { - "name": "optimization-row1-col1", - "display-name": "Storage", - "row-index": "0", - "column-index": "0", - "row-span": "2", - "column-span": "1" - }, - { - "name": "optimization-row1-col2-1", - "display-name": "", + "name": "acid-transactions-row1-col1-1", "row-index": "0", - "column-index": "1", - "row-span": "1", - "column-span": "1" - }, - { - "name": "optimization-row1-col2-2", - "display-name": "", - "row-index": "1", - "column-index": "1", - "row-span": "1", - "column-span": "1", - "border": true - }, - { - "name": "optimization-row2-col1", - "display-name": "Optimization Vectorization", - "row-index": "2", "column-index": "0", "row-span": "1", "column-span": "1" - }, - { - "name": "optimization-row2-col2", - "display-name": "", - "row-index": "2", - "column-index": "1", - "row-span": "1", - "column-span": "1" - }, + } + ] + }, + { + "name": "interactive-query", + "display-name": "Interactive Query", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ { - "name": "optimization-row3-col1", - "display-name": "Memory", - "row-index": "3", + "name": "interactive-query-row1-col1", + "row-index": "0", "column-index": "0", "row-span": "1", "column-span": "1" - }, - { - "name": "optimization-row3-col2", - "display-name": "", - "row-index": "3", - "column-index": "1", - "row-span": "1", - "column-span": "1" } ] }, { - "name": "acid-transactions", - "display-name": "Acid Transactions", + "name": "security", + "display-name": "Security", "row-index": "0", "column-index": "2", - "row-span": "2", + "row-span": "1", "column-span": "1", "section-columns": "1", - "section-rows": "2", + "section-rows": "1", "subsections": [ { - "name": "acid-transactions-row1-col1-1", + "name": "security-row1-col1", "row-index": "0", "column-index": "0", "row-span": "1", "column-span": "1" - }, - { - "name": "acid-transactions-row1-col1-2", - "row-index": "1", - "column-index": "0", - "row-span": "1", - "column-span": "1" } ] }, { - "name": "performance", - "display-name": "Performance", - "row-index": "3", + "name": "optimization", + "display-name": "Optimization", + "row-index": "1", "column-index": "0", - "row-span": "3", - "column-span": "2", - "section-columns": "2", + "row-span": "1", + "column-span": "3", + "section-columns": "3", "section-rows": "2", "subsections": [ { - "name": "performance-row1-col1", - "display-name": "TEZ Engine", + "name": "optimization-row1-col1", + "display-name": "Tez", "row-index": "0", "column-index": "0", "row-span": "1", "column-span": "1" }, { - "name": "performance-row1-col2", + "name": "optimization-row1-col2", "display-name": "", "row-index": "0", "column-index": "1", @@ -135,55 +97,35 @@ "column-span": "1" }, { - "name": "performance-row2-col1", + "name": "optimization-row1-col3", + "display-name": "CBO", + "row-index": "0", + "column-index": "2", + "row-span": "1", + "column-span": "1" + }, + { + "name": "optimization-row2-col1", + "display-name": "Storage", "row-index": "1", "column-index": "0", "row-span": "1", "column-span": "1" }, { - "name": "performance-row2-col2", + "name": "optimization-row2-col2", + "display-name": "", "row-index": "1", "column-index": "1", "row-span": "1", "column-span": "1" - } - ] - }, - { - "name": "interactive-query", - "display-name": "Interactive Query", - "row-index": "2", - "column-index": "2", - "row-span": "2", - "column-span": "1", - "section-columns": "1", - "section-rows": "2", - "subsections": [ - { - "name": "interactive-query-row1-col1", - "row-index": "0", - "column-index": "0", - "row-span": "2", - "column-span": "1" - } - ] - }, - { - "name": "security", - "display-name": "Security", - "row-index": "4", - "column-index": "2", - "row-span": "2", - "column-span": "1", - "section-columns": "1", - "section-rows": "2", - "subsections": [ + }, { - "name": "security-row1-col1", - "row-index": "0", - "column-index": "0", - "row-span": "2", + "name": "optimization-row2-col3", + "display-name": "Memory", + "row-index": "1", + "column-index": "2", + "row-span": "1", "column-span": "1" } ] @@ -198,66 +140,30 @@ "configuration-layout": "default", "configs": [ { - "config": "hive-site/hive.default.fileformat", - "subsection-name": "optimization-row1-col1" - }, - { "config": "hive-site/hive.exec.orc.default.stripe.size", - "subsection-name": "optimization-row1-col1" + "subsection-name": "optimization-row2-col1" }, { "config": "hive-site/hive.exec.orc.default.compress", - "subsection-name": "optimization-row1-col2-1" - }, - { - "config": "hive-env/hive_exec_orc_storage_strategy", - "subsection-name": "optimization-row1-col2-2" - }, - { - "config": "hive-site/hive.optimize.index.filter", - "subsection-name": "optimization-row1-col1" - }, - { - "config": "hive-site/hive.optimize.sort.dynamic.partition", - "subsection-name": "optimization-row1-col1" - }, - { - "config": "hive-site/hive.exec.orc.encoding.strategy", - "subsection-name": "optimization-row1-col2-2" - }, - { - "config": "hive-site/hive.exec.orc.compression.strategy", - "subsection-name": "optimization-row1-col2-2" - }, - { - "config": "hive-site/hive.vectorized.execution.enabled", "subsection-name": "optimization-row2-col1" }, { - "config": "hive-site/hive.vectorized.execution.reduce.enabled", + "config": "hive-env/hive_exec_orc_storage_strategy", "subsection-name": "optimization-row2-col2" }, { "config": "hive-site/hive.auto.convert.join.noconditionaltask.size", - "subsection-name": "optimization-row3-col1" + "subsection-name": "optimization-row2-col3" }, { "config": "hive-site/hive.exec.reducers.bytes.per.reducer", - "subsection-name": "optimization-row3-col2" + "subsection-name": "optimization-row2-col3" }, { "config": "hive-env/hive_txn_acid", "subsection-name": "acid-transactions-row1-col1-1" }, { - "config": "hive-site/hive.txn.manager", - "subsection-name": "acid-transactions-row1-col1-1" - }, - { - "config": "hive-site/hive.support.concurrency", - "subsection-name": "acid-transactions-row1-col1-1" - }, - { "config": "hive-site/hive.compactor.initiator.on", "subsection-name": "acid-transactions-row1-col1-1" }, @@ -266,56 +172,28 @@ "subsection-name": "acid-transactions-row1-col1-1" }, { - "config": "hive-site/hive.enforce.bucketing", - "subsection-name": "acid-transactions-row1-col1-2" - }, - { - "config": "hive-site/hive.exec.dynamic.partition.mode", - "subsection-name": "acid-transactions-row1-col1-2" - }, - { "config": "hive-site/hive.execution.engine", - "subsection-name": "performance-row1-col1" - }, - { - "config": "hive-env/hive_timeline_logging_enabled", - "subsection-name": "performance-row1-col1" + "subsection-name": "optimization-row1-col1" }, { "config": "hive-site/hive.tez.container.size", - "subsection-name": "performance-row1-col1" + "subsection-name": "optimization-row1-col1" }, { "config": "hive-site/hive.prewarm.enabled", - "subsection-name": "performance-row1-col1" + "subsection-name": "optimization-row1-col2" }, { "config": "hive-site/hive.prewarm.numcontainers", - "subsection-name": "performance-row1-col2" - }, - { - "config": "hive-site/hive.tez.auto.reducer.parallelism", - "subsection-name": "performance-row1-col2" - }, - { - "config": "hive-site/hive.tez.dynamic.partition.pruning", - "subsection-name": "performance-row1-col2" + "subsection-name": "optimization-row1-col2" }, { "config": "hive-site/hive.cbo.enable", - "subsection-name": "performance-row2-col1" - }, - { - "config": "hive-site/hive.stats.fetch.partition.stats", - "subsection-name": "performance-row2-col1" + "subsection-name": "optimization-row1-col3" }, { "config": "hive-site/hive.stats.fetch.column.stats", - "subsection-name": "performance-row2-col1" - }, - { - "config": "hive-site/hive.compute.query.using.stats", - "subsection-name": "performance-row2-col2" + "subsection-name": "optimization-row1-col3" }, { "config": "hiveserver2-site/hive.server2.tez.default.queues", @@ -330,10 +208,6 @@ "subsection-name": "interactive-query-row1-col1" }, { - "config": "hiveserver2-site/hive.server2.enable.doAs", - "subsection-name": "interactive-query-row1-col1" - }, - { "config": "hiveserver2-site/tez.session.am.dag.submit.timeout.secs", "subsection-name": "interactive-query-row1-col1" }, @@ -342,11 +216,7 @@ "subsection-name": "security-row1-col1" }, { - "config": "hive-site/hive.security.authorization.enabled", - "subsection-name": "security-row1-col1" - }, - { - "config": "hive-site/hive.security.metastore.authorization.manager", + "config": "hiveserver2-site/hive.server2.enable.doAs", "subsection-name": "security-row1-col1" }, { @@ -361,12 +231,6 @@ }, "widgets": [ { - "config": "hive-site/hive.default.fileformat", - "widget": { - "type": "combo" - } - }, - { "config": "hive-site/hive.exec.orc.default.stripe.size", "widget": { "type": "slider", @@ -384,48 +248,12 @@ } }, { - "config": "hive-site/hive.optimize.index.filter", - "widget": { - "type": "toggle" - } - }, - { - "config": "hive-site/hive.optimize.sort.dynamic.partition", - "widget": { - "type": "toggle" - } - }, - { - "config": "hive-site/hive.exec.orc.encoding.strategy", - "widget": { - "type": "toggle" - } - }, - { "config": "hive-env/hive_exec_orc_storage_strategy", "widget": { "type": "toggle" } }, { - "config": "hive-site/hive.exec.orc.compression.strategy", - "widget": { - "type": "toggle" - } - }, - { - "config": "hive-site/hive.vectorized.execution.enabled", - "widget": { - "type": "toggle" - } - }, - { - "config": "hive-site/hive.vectorized.execution.reduce.enabled", - "widget": { - "type": "toggle" - } - }, - { "config": "hive-site/hive.auto.convert.join.noconditionaltask.size", "widget": { "type": "slider", @@ -454,24 +282,6 @@ } }, { - "config": "hive-site/hive.txn.manager", - "widget": { - "type": "combo" - } - }, - { - "config": "hive-site/hive.security.metastore.authorization.manager", - "widget": { - "type": "string" - } - }, - { - "config": "hive-site/hive.support.concurrency", - "widget": { - "type": "toggle" - } - }, - { "config": "hive-site/hive.compactor.initiator.on", "widget": { "type": "toggle" @@ -489,18 +299,6 @@ } }, { - "config": "hive-site/hive.enforce.bucketing", - "widget": { - "type": "toggle" - } - }, - { - "config": "hive-site/hive.exec.dynamic.partition.mode", - "widget": { - "type": "toggle" - } - }, - { "config": "hive-site/hive.execution.engine", "widget": { "type": "combo" @@ -535,42 +333,18 @@ } }, { - "config": "hive-site/hive.tez.auto.reducer.parallelism", - "widget": { - "type": "toggle" - } - }, - { - "config": "hive-site/hive.tez.dynamic.partition.pruning", - "widget": { - "type": "toggle" - } - }, - { "config": "hive-site/hive.cbo.enable", "widget": { "type": "toggle" } }, { - "config": "hive-site/hive.stats.fetch.partition.stats", - "widget": { - "type": "toggle" - } - }, - { "config": "hive-site/hive.stats.fetch.column.stats", "widget": { "type": "toggle" } }, { - "config": "hive-site/hive.compute.query.using.stats", - "widget": { - "type": "toggle" - } - }, - { "config": "hiveserver2-site/hive.server2.tez.initialize.default.sessions", "widget": { "type": "toggle" @@ -611,12 +385,6 @@ } }, { - "config": "hive-site/hive.security.authorization.enabled", - "widget": { - "type": "toggle" - } - }, - { "config": "hive-site/hive.server2.authentication", "widget": { "type": "combo" @@ -633,12 +401,6 @@ "widget": { "type": "list" } - }, - { - "config": "hive-env/hive_timeline_logging_enabled", - "widget": { - "type": "toggle" - } } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/132cf31b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 3700a2d..d5e1534 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -233,7 +233,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putHiveSiteProperty("hive.vectorized.execution.reduce.enabled", "false") # Memory - putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", "2147483648") putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864") # Transactions @@ -297,6 +296,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): "yarn.scheduler.minimum-allocation-mb" in configurations["yarn-site"]["properties"]: container_size = configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"] putHiveSiteProperty("hive.tez.container.size", container_size) + putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", str(int(int(container_size)/3)*1024*1024)) putHiveSiteProperty("hive.prewarm.enabled", "false") putHiveSiteProperty("hive.prewarm.numcontainers", "3") putHiveSiteProperty("hive.tez.auto.reducer.parallelism", "true") http://git-wip-us.apache.org/repos/asf/ambari/blob/132cf31b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 1253781..e0a5006 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -874,7 +874,7 @@ class TestHDP22StackAdvisor(TestCase): }, 'hive-site': { 'properties': { - 'hive.auto.convert.join.noconditionaltask.size': '2147483648', + 'hive.auto.convert.join.noconditionaltask.size': '89128960', 'hive.cbo.enable': 'true', 'hive.compactor.initiator.on': 'false', 'hive.compactor.worker.threads': '0',
