AMBARI-14989. Inconsistent HIVE CBO setting presentation. (Jaimin)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/984d9e90 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/984d9e90 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/984d9e90 Branch: refs/heads/2.2.1-maint Commit: 984d9e90b28da343d4fb80abfdceca3ce6b4dc65 Parents: 9df0339 Author: Jaimin Jetly <jai...@hortonworks.com> Authored: Thu Feb 11 15:02:09 2016 +0530 Committer: Yusaku Sako <yus...@hortonworks.com> Committed: Thu Feb 11 07:57:10 2016 -0800 ---------------------------------------------------------------------- .../services/HIVE/configuration/hive-env.xml | 20 --------- .../services/HIVE/configuration/hive-site.xml | 14 ------ .../stacks/HDP/2.2/services/stack_advisor.py | 13 +++--- .../services/HIVE/configuration/hive-env.xml | 20 --------- .../services/HIVE/configuration/hive-site.xml | 8 ---- .../stacks/HDPWIN/2.2/services/stack_advisor.py | 13 +++--- .../2.1/configs/hive-metastore-upgrade.json | 3 +- .../stacks/2.2/common/test_stack_advisor.py | 12 +++-- .../2.3/common/services-sparkts-hive.json | 47 -------------------- .../stacks/2.3/common/test_stack_advisor.py | 7 ++- 10 files changed, 19 insertions(+), 138 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/984d9e90/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml index f84bc01..77eace4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml @@ -127,26 +127,6 @@ export METASTORE_PORT={{hive_metastore_port}} </property> <property> - <name>cost_based_optimizer</name> - <display-name>Cost Based Optimizer</display-name> - <value>Off</value> - <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>On</value> - <label>On</label> - </entry> - <entry> - <value>Off</value> - <label>Off</label> - </entry> - </entries> - <selection-cardinality>1</selection-cardinality> - </value-attributes> - </property> - - <property> <name>hive_security_authorization</name> <display-name>Choose Authorization</display-name> <value>None</value> http://git-wip-us.apache.org/repos/asf/ambari/blob/984d9e90/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml index cc3e75f..7d6c490 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml @@ -450,12 +450,6 @@ limitations under the License. <name>hive.cbo.enable</name> <value>true</value> <description>Flag to control enabling Cost Based Optimizations using Calcite framework.</description> - <depends-on> - <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - </depends-on> </property> <property> @@ -799,10 +793,6 @@ limitations under the License. </value-attributes> <depends-on> <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - <property> <type>hive-site</type> <name>hive.cbo.enable</name> </property> @@ -834,10 +824,6 @@ limitations under the License. </value-attributes> <depends-on> <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - <property> <type>hive-site</type> <name>hive.cbo.enable</name> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/984d9e90/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 9be6410..3ebb25f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -347,14 +347,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864") # CBO - putHiveEnvProperty("cost_based_optimizer", "On") - if str(configurations["hive-env"]["properties"]["cost_based_optimizer"]).lower() == "on": - putHiveSiteProperty("hive.cbo.enable", "true") - else: - putHiveSiteProperty("hive.cbo.enable", "false") - hive_cbo_enable = configurations["hive-site"]["properties"]["hive.cbo.enable"] - putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable) - putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable) + if "hive-site" in services["configurations"] and "hive.cbo.enable" in services["configurations"]["hive-site"]["properties"]: + hive_cbo_enable = services["configurations"]["hive-site"]["properties"]["hive.cbo.enable"] + putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable) + putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable) + putHiveSiteProperty("hive.compute.query.using.stats", "true") # Interactive Query http://git-wip-us.apache.org/repos/asf/ambari/blob/984d9e90/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml index 3d1b549..b5a9124 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml @@ -74,26 +74,6 @@ </property> <property> - <name>cost_based_optimizer</name> - <display-name>Cost Based Optimizer</display-name> - <value>Off</value> - <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>On</value> - <label>On</label> - </entry> - <entry> - <value>Off</value> - <label>Off</label> - </entry> - </entries> - <selection-cardinality>1</selection-cardinality> - </value-attributes> - </property> - - <property> <name>hive_security_authorization</name> <display-name>Choose Authorization</display-name> <value>None</value> http://git-wip-us.apache.org/repos/asf/ambari/blob/984d9e90/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml index 7e13436..8acc4dc 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml @@ -772,10 +772,6 @@ limitations under the License. </value-attributes> <depends-on> <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - <property> <type>hive-site</type> <name>hive.cbo.enable</name> </property> @@ -807,10 +803,6 @@ limitations under the License. </value-attributes> <depends-on> <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - <property> <type>hive-site</type> <name>hive.cbo.enable</name> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/984d9e90/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py index c6941f0..bf50aac 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py @@ -360,14 +360,11 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor): putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864") # CBO - putHiveEnvProperty("cost_based_optimizer", "On") - if str(configurations["hive-env"]["properties"]["cost_based_optimizer"]).lower() == "on": - putHiveSiteProperty("hive.cbo.enable", "true") - else: - putHiveSiteProperty("hive.cbo.enable", "false") - hive_cbo_enable = configurations["hive-site"]["properties"]["hive.cbo.enable"] - putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable) - putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable) + if "hive-site" in services["configurations"] and "hive.cbo.enable" in services["configurations"]["hive-site"]["properties"]: + hive_cbo_enable = services["configurations"]["hive-site"]["properties"]["hive.cbo.enable"] + putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable) + putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable) + putHiveSiteProperty("hive.compute.query.using.stats", "true") # Interactive Query http://git-wip-us.apache.org/repos/asf/ambari/blob/984d9e90/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json index 7829f05..0fbc282 100644 --- a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json +++ b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json @@ -77,8 +77,7 @@ "hive_user": "hive", "hive_database": "New MySQL Database", "hive.heapsize": "703", - "hcat_pid_dir": "/var/run/webhcat", - "cost_based_optimizer": "On", + "hcat_pid_dir": "/var/run/webhcat", "webhcat_user": "hcat", "hive_security_authorization": "None", "hive_exec_orc_storage_strategy": "SPEED", http://git-wip-us.apache.org/repos/asf/ambari/blob/984d9e90/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 904688a..86d560a 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -983,7 +983,6 @@ class TestHDP22StackAdvisor(TestCase): }, 'hive-env': { 'properties': { - 'cost_based_optimizer': 'On', 'hive_exec_orc_storage_strategy': 'SPEED', 'hive_security_authorization': 'None', 'hive_timeline_logging_enabled': 'true', @@ -997,7 +996,6 @@ class TestHDP22StackAdvisor(TestCase): 'hive.server2.tez.initialize.default.sessions': 'false', 'hive.server2.tez.sessions.per.default.queue': '1', 'hive.auto.convert.join.noconditionaltask.size': '268435456', - 'hive.cbo.enable': 'true', 'hive.compactor.initiator.on': 'false', 'hive.compactor.worker.threads': '0', 'hive.compute.query.using.stats': 'true', @@ -1140,7 +1138,8 @@ class TestHDP22StackAdvisor(TestCase): "hive.server2.authentication.kerberos.keytab": "", "hive.server2.authentication.kerberos.principal": "", "hive.server2.authentication.pam.services": "", - "hive.server2.custom.authentication.class": "" + "hive.server2.custom.authentication.class": "", + "hive.cbo.enable": "true" } }, "hiveserver2-site": { @@ -1215,7 +1214,8 @@ class TestHDP22StackAdvisor(TestCase): "hive.server2.authentication.kerberos.keytab": "", "hive.server2.authentication.kerberos.principal": "", "hive.server2.authentication.pam.services": "", - "hive.server2.custom.authentication.class": "" + "hive.server2.custom.authentication.class": "", + "hive.cbo.enable": "true" } }, "hiveserver2-site": { @@ -1279,10 +1279,8 @@ class TestHDP22StackAdvisor(TestCase): #test recommendations services["configurations"]["hive-site"]["properties"]["hive.cbo.enable"] = "false" services["configurations"]["hive-env"]["properties"]["hive_security_authorization"] = "sqlstdauth" - services["changed-configurations"] = [{"type": "hive-site", "name": "hive.cbo.enable"}, - {"type": "hive-env", "name": "hive_security_authorization"}] + services["changed-configurations"] = [{"type": "hive-env", "name": "hive_security_authorization"}] expected["hive-env"]["properties"]["hive_security_authorization"] = "sqlstdauth" - expected["hive-site"]["properties"]["hive.cbo.enable"] = "false" expected["hive-site"]["properties"]["hive.stats.fetch.partition.stats"]="false" expected["hive-site"]["properties"]["hive.stats.fetch.column.stats"]="false" expected["hive-site"]["properties"]["hive.security.authorization.enabled"]="true" http://git-wip-us.apache.org/repos/asf/ambari/blob/984d9e90/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json b/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json index 51ae36a..20a672e 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json +++ b/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json @@ -2402,47 +2402,6 @@ }, "dependencies" : [ ] }, { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer", - "StackConfigurations" : { - "property_depends_on" : [ ], - "property_name" : "cost_based_optimizer", - "service_name" : "HIVE", - "stack_name" : "HDP", - "stack_version" : "2.3", - "type" : "hive-env.xml" - }, - "dependencies" : [ { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer/dependencies/hive.cbo.enable", - "StackConfigurationDependency" : { - "dependency_name" : "hive.cbo.enable", - "dependency_type" : "hive-site", - "property_name" : "cost_based_optimizer", - "service_name" : "HIVE", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - }, { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer/dependencies/hive.stats.fetch.column.stats", - "StackConfigurationDependency" : { - "dependency_name" : "hive.stats.fetch.column.stats", - "dependency_type" : "hive-site", - "property_name" : "cost_based_optimizer", - "service_name" : "HIVE", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - }, { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer/dependencies/hive.stats.fetch.partition.stats", - "StackConfigurationDependency" : { - "dependency_name" : "hive.stats.fetch.partition.stats", - "dependency_type" : "hive-site", - "property_name" : "cost_based_optimizer", - "service_name" : "HIVE", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - } ] - }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/datanucleus.autoCreateSchema", "StackConfigurations" : { "property_depends_on" : [ ], @@ -4199,9 +4158,6 @@ "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/hive.stats.fetch.column.stats", "StackConfigurations" : { "property_depends_on" : [ { - "type" : "hive-env", - "name" : "cost_based_optimizer" - }, { "type" : "hive-site", "name" : "hive.cbo.enable" } ], @@ -4216,9 +4172,6 @@ "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/hive.stats.fetch.partition.stats", "StackConfigurations" : { "property_depends_on" : [ { - "type" : "hive-env", - "name" : "cost_based_optimizer" - }, { "type" : "hive-site", "name" : "hive.cbo.enable" } ], http://git-wip-us.apache.org/repos/asf/ambari/blob/984d9e90/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 1e6a1eb..d14e23e 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -870,7 +870,6 @@ class TestHDP23StackAdvisor(TestCase): }, 'hive-env': { 'properties': { - 'cost_based_optimizer': 'On', 'hive_exec_orc_storage_strategy': 'SPEED', 'hive_security_authorization': 'None', 'hive_timeline_logging_enabled': 'true', @@ -884,7 +883,6 @@ class TestHDP23StackAdvisor(TestCase): 'hive.server2.tez.initialize.default.sessions': 'false', 'hive.server2.tez.sessions.per.default.queue': '1', 'hive.auto.convert.join.noconditionaltask.size': '268435456', - 'hive.cbo.enable': 'true', 'hive.compactor.initiator.on': 'false', 'hive.compactor.worker.threads': '0', 'hive.compute.query.using.stats': 'true', @@ -1028,7 +1026,8 @@ class TestHDP23StackAdvisor(TestCase): "hive.server2.authentication.kerberos.keytab": "", "hive.server2.authentication.kerberos.principal": "", "hive.server2.authentication.pam.services": "", - "hive.server2.custom.authentication.class": "" + "hive.server2.custom.authentication.class": "", + "hive.cbo.enable": "true" } }, "hiveserver2-site": { @@ -1508,4 +1507,4 @@ class TestHDP23StackAdvisor(TestCase): recommendedConfigurations = {} self.stackAdvisor.recommendRangerConfigurations(recommendedConfigurations, clusterData, services, None) - self.assertEquals(recommendedConfigurations['ranger-admin-site']['properties']['ranger.audit.solr.zookeepers'], 'NONE') \ No newline at end of file + self.assertEquals(recommendedConfigurations['ranger-admin-site']['properties']['ranger.audit.solr.zookeepers'], 'NONE')