ODPI-193. Add Hive 1.2 to ODPi ambari reference implementation (cherry picked from commit d03f5b8b9796da11b2a0ade72704f7f3739af9ec)
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/ba8d7f50 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/ba8d7f50 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/ba8d7f50 Branch: refs/heads/master Commit: ba8d7f50d581c74c17b0799a813ef753d50c767d Parents: 490bcb6 Author: Roman Shaposhnik <[email protected]> Authored: Sat Oct 22 20:33:46 2016 -0700 Committer: Roman Shaposhnik <[email protected]> Committed: Tue Mar 21 23:08:08 2017 -0700 ---------------------------------------------------------------------- .../ambari/ODPi/1.0/services/HIVE/alerts.json | 232 ++ .../HIVE/configuration/beeline-log4j2.xml | 62 + .../services/HIVE/configuration/hcat-env.xml | 41 + .../hive-atlas-application.properties.xml | 67 + .../services/HIVE/configuration/hive-env.xml | 540 ++++ .../HIVE/configuration/hive-exec-log4j.xml | 96 + .../HIVE/configuration/hive-exec-log4j2.xml | 83 + .../HIVE/configuration/hive-interactive-env.xml | 373 +++ .../configuration/hive-interactive-site.xml | 909 ++++++ .../services/HIVE/configuration/hive-log4j.xml | 106 + .../services/HIVE/configuration/hive-log4j2.xml | 90 + .../services/HIVE/configuration/hive-site.xml | 2796 ++++++++++++++++++ .../HIVE/configuration/hivemetastore-site.xml | 43 + .../hiveserver2-interactive-site.xml | 56 + .../HIVE/configuration/hiveserver2-site.xml | 122 + .../HIVE/configuration/llap-cli-log4j2.xml | 91 + .../HIVE/configuration/llap-daemon-log4j.xml | 158 + .../HIVE/configuration/ranger-hive-audit.xml | 136 + .../ranger-hive-plugin-properties.xml | 63 + .../configuration/ranger-hive-policymgr-ssl.xml | 71 + .../HIVE/configuration/ranger-hive-security.xml | 81 + .../HIVE/configuration/tez-interactive-site.xml | 144 + .../services/HIVE/configuration/webhcat-env.xml | 38 + .../HIVE/configuration/webhcat-log4j.xml | 63 + .../HIVE/configuration/webhcat-site.xml | 287 ++ .../HIVE/etc/hive-schema-0.12.0.mysql.sql | 777 +++++ .../HIVE/etc/hive-schema-0.12.0.oracle.sql | 718 +++++ .../HIVE/etc/hive-schema-0.12.0.postgres.sql | 1406 +++++++++ .../HIVE/etc/hive-schema-0.13.0.mysql.sql | 889 ++++++ .../HIVE/etc/hive-schema-0.13.0.oracle.sql | 835 ++++++ .../HIVE/etc/hive-schema-0.13.0.postgres.sql | 1538 ++++++++++ .../etc/upgrade-0.12.0-to-0.13.0.oracle.sql | 165 ++ .../services/HIVE/etc/upgrade-0.13.0.oracle.sql | 38 + .../ambari/ODPi/1.0/services/HIVE/kerberos.json | 149 + .../ambari/ODPi/1.0/services/HIVE/metainfo.xml | 518 ++++ .../alert_hive_interactive_thrift_port.py | 216 ++ .../HIVE/package/alerts/alert_hive_metastore.py | 270 ++ .../package/alerts/alert_hive_thrift_port.py | 274 ++ .../package/alerts/alert_llap_app_status.py | 299 ++ .../HIVE/package/alerts/alert_webhcat_server.py | 228 ++ .../package/etc/hive-schema-0.12.0.mysql.sql | 777 +++++ .../package/etc/hive-schema-0.12.0.oracle.sql | 718 +++++ .../package/etc/hive-schema-0.12.0.postgres.sql | 1406 +++++++++ .../services/HIVE/package/files/addMysqlUser.sh | 39 + .../services/HIVE/package/files/hcatSmoke.sh | 41 + .../services/HIVE/package/files/hiveSmoke.sh | 24 + .../HIVE/package/files/hiveTezSetup.cmd | 58 + .../services/HIVE/package/files/hiveserver2.sql | 23 + .../HIVE/package/files/hiveserver2Smoke.sh | 32 + .../1.0/services/HIVE/package/files/pigSmoke.sh | 18 + .../HIVE/package/files/removeMysqlUser.sh | 33 + .../HIVE/package/files/startMetastore.sh | 25 + .../HIVE/package/files/templetonSmoke.sh | 95 + .../services/HIVE/package/scripts/__init__.py | 19 + .../1.0/services/HIVE/package/scripts/hcat.py | 81 + .../HIVE/package/scripts/hcat_client.py | 85 + .../HIVE/package/scripts/hcat_service_check.py | 86 + .../1.0/services/HIVE/package/scripts/hive.py | 481 +++ .../HIVE/package/scripts/hive_client.py | 68 + .../HIVE/package/scripts/hive_interactive.py | 302 ++ .../HIVE/package/scripts/hive_metastore.py | 259 ++ .../HIVE/package/scripts/hive_server.py | 211 ++ .../package/scripts/hive_server_interactive.py | 535 ++++ .../HIVE/package/scripts/hive_server_upgrade.py | 141 + .../HIVE/package/scripts/hive_service.py | 187 ++ .../package/scripts/hive_service_interactive.py | 109 + .../HIVE/package/scripts/mysql_server.py | 64 + .../HIVE/package/scripts/mysql_service.py | 49 + .../HIVE/package/scripts/mysql_users.py | 70 + .../HIVE/package/scripts/mysql_utils.py | 35 + .../1.0/services/HIVE/package/scripts/params.py | 29 + .../HIVE/package/scripts/params_linux.py | 736 +++++ .../HIVE/package/scripts/params_windows.py | 74 + .../HIVE/package/scripts/service_check.py | 190 ++ .../HIVE/package/scripts/setup_ranger_hive.py | 98 + .../scripts/setup_ranger_hive_interactive.py | 78 + .../HIVE/package/scripts/status_params.py | 118 + .../services/HIVE/package/scripts/webhcat.py | 145 + .../HIVE/package/scripts/webhcat_server.py | 164 + .../HIVE/package/scripts/webhcat_service.py | 96 + .../package/scripts/webhcat_service_check.py | 128 + .../hadoop-metrics2-hivemetastore.properties.j2 | 54 + .../hadoop-metrics2-hiveserver2.properties.j2 | 54 + .../templates/hadoop-metrics2-llapdaemon.j2 | 52 + .../hadoop-metrics2-llaptaskscheduler.j2 | 52 + .../HIVE/package/templates/hive.conf.j2 | 35 + .../package/templates/startHiveserver2.sh.j2 | 24 + .../templates/startHiveserver2Interactive.sh.j2 | 24 + .../package/templates/templeton_smoke.pig.j2 | 24 + 89 files changed, 23314 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/ba8d7f50/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/alerts.json ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/alerts.json b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/alerts.json new file mode 100755 index 0000000..e2431c3 --- /dev/null +++ b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/alerts.json @@ -0,0 +1,232 @@ +{ + "HIVE": { + "service": [], + "HIVE_METASTORE": [ + { + "name": "hive_metastore_process", + "label": "Hive Metastore Process", + "description": "This host-level alert is triggered if the Hive Metastore process cannot be determined to be up and listening on the network.", + "interval": 3, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_hive_metastore.py", + "parameters": [ + { + "name": "check.command.timeout", + "display_name": "Command Timeout", + "value": 60.0, + "type": "NUMERIC", + "description": "The maximum time before check command will be killed by timeout", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "default.smoke.user", + "display_name": "Default Smoke User", + "value": "ambari-qa", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env/smokeuser", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.principal", + "display_name": "Default Smoke Principal", + "value": "[email protected]", + "type": "STRING", + "description": "The principal to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_principal_name", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.keytab", + "display_name": "Default Smoke Keytab", + "value": "/etc/security/keytabs/smokeuser.headless.keytab", + "type": "STRING", + "description": "The keytab to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_keytab", + "visibility": "HIDDEN" + } + ] + } + } + ], + "HIVE_SERVER": [ + { + "name": "hive_server_process", + "label": "HiveServer2 Process", + "description": "This host-level alert is triggered if the HiveServer cannot be determined to be up and responding to client requests.", + "interval": 3, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py", + "parameters": [ + { + "name": "check.command.timeout", + "display_name": "Command Timeout", + "value": 60.0, + "type": "NUMERIC", + "description": "The maximum time before check command will be killed by timeout", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "default.smoke.user", + "display_name": "Default Smoke User", + "value": "ambari-qa", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env/smokeuser", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.principal", + "display_name": "Default Smoke Principal", + "value": "[email protected]", + "type": "STRING", + "description": "The principal to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_principal_name", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.keytab", + "display_name": "Default Smoke Keytab", + "value": "/etc/security/keytabs/smokeuser.headless.keytab", + "type": "STRING", + "description": "The keytab to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_keytab", + "visibility": "HIDDEN" + } + ] + } + } + ], + "HIVE_SERVER_INTERACTIVE": [ + { + "name": "hive_server_interactive_process", + "label": "HiveServer2 Interactive Process", + "description": "This host-level alert is triggered if the HiveServerInteractive cannot be determined to be up and responding to client requests.", + "interval": 3, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_hive_interactive_thrift_port.py", + "parameters": [ + { + "name": "check.command.timeout", + "display_name": "Command Timeout", + "value": 60.0, + "type": "NUMERIC", + "description": "The maximum time before check command will be killed by timeout", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "default.smoke.user", + "display_name": "Default Smoke User", + "value": "ambari-qa", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env/smokeuser", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.principal", + "display_name": "Default Smoke Principal", + "value": "[email protected]", + "type": "STRING", + "description": "The principal to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_principal_name", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.keytab", + "display_name": "Default Smoke Keytab", + "value": "/etc/security/keytabs/smokeuser.headless.keytab", + "type": "STRING", + "description": "The keytab to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_keytab", + "visibility": "HIDDEN" + } + ] + } + }, + { + "name": "llap_application", + "label": "LLAP Application", + "description": "This alert is triggered if the LLAP Application cannot be determined to be up and responding to requests.", + "interval": 3, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_llap_app_status.py", + "parameters": [ + { + "name": "check.command.timeout", + "display_name": "Command Timeout", + "value": 120.0, + "type": "NUMERIC", + "description": "The maximum time before check command will be killed by timeout", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "default.hive.user", + "display_name": "Default HIVE User", + "value": "hive", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env", + "visibility": "HIDDEN" + }, + { + "name": "default.hive.principal", + "display_name": "Default HIVE Principal", + "value": "hive/[email protected]", + "type": "STRING", + "description": "The principal to use when retrieving the kerberos ticket if not specified in cluster-env", + "visibility": "HIDDEN" + }, + { + "name": "default.hive.keytab", + "display_name": "Default HIVE Keytab", + "value": "/etc/security/keytabs/hive.llap.zk.sm.keytab", + "type": "STRING", + "description": "The keytab to use when retrieving the kerberos ticket if not specified in cluster-env.", + "visibility": "HIDDEN" + } + ] + } + } + ], + "WEBHCAT_SERVER": [ + { + "name": "hive_webhcat_server_status", + "label": "WebHCat Server Status", + "description": "This host-level alert is triggered if the templeton server status is not healthy.", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_webhcat_server.py", + "parameters": [ + { + "name": "default.smoke.user", + "display_name": "Default Smoke User", + "value": "ambari-qa", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env/smokeuser", + "visibility": "HIDDEN" + }, + { + "name": "connection.timeout", + "display_name": "Connection Timeout", + "value": 5.0, + "type": "NUMERIC", + "description": "The maximum time before this alert is considered to be CRITICAL", + "units": "seconds", + "threshold": "CRITICAL" + } + ] + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ba8d7f50/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/beeline-log4j2.xml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/beeline-log4j2.xml b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/beeline-log4j2.xml new file mode 100755 index 0000000..03de64e --- /dev/null +++ b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/beeline-log4j2.xml @@ -0,0 +1,62 @@ +<configuration><property require-input="false"> + <name>content</name> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = BeelineLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = WARN +property.hive.root.logger = console + +# list of all appenders +appenders = console + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# list of all loggers +loggers = HiveConnection + +# HiveConnection logs useful info for dynamic service discovery +logger.HiveConnection.name = org.apache.hive.jdbc.HiveConnection +logger.HiveConnection.level = INFO + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} + </value> + <description>Custom beeline-log4j2.properties</description> + <display-name>beeline-log4j template</display-name> + <filename>beeline-log4j2.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property></configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/ba8d7f50/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hcat-env.xml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hcat-env.xml b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hcat-env.xml new file mode 100755 index 0000000..3908d61 --- /dev/null +++ b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hcat-env.xml @@ -0,0 +1,41 @@ +<configuration><property require-input="false"> + <name>content</name> + <value> + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + JAVA_HOME={{java64_home}} + HCAT_PID_DIR={{hcat_pid_dir}}/ + HCAT_LOG_DIR={{hcat_log_dir}}/ + HCAT_CONF_DIR={{hcat_conf_dir}} + HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + #DBROOT is the path where the connector jars are downloaded + DBROOT={{hcat_dbroot}} + USER={{hcat_user}} + METASTORE_PORT={{hive_metastore_port}} + </value> + <description>This is the jinja template for hcat-env.sh file</description> + <display-name>hcat-env template</display-name> + <filename>hcat-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>content</type> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property></configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/ba8d7f50/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-atlas-application.properties.xml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-atlas-application.properties.xml b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-atlas-application.properties.xml new file mode 100755 index 0000000..7eb72ef --- /dev/null +++ b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-atlas-application.properties.xml @@ -0,0 +1,67 @@ +<configuration><property require-input="false"> + <name>atlas.hook.hive.synchronous</name> + <value>false</value> + <description></description> + <filename>hive-atlas-application.properties.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes/> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>atlas.hook.hive.numRetries</name> + <value>3</value> + <description></description> + <filename>hive-atlas-application.properties.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes/> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>atlas.hook.hive.minThreads</name> + <value>5</value> + <description></description> + <filename>hive-atlas-application.properties.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes/> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>atlas.hook.hive.maxThreads</name> + <value>5</value> + <description></description> + <filename>hive-atlas-application.properties.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes/> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>atlas.hook.hive.keepAliveTime</name> + <value>10</value> + <description></description> + <filename>hive-atlas-application.properties.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes/> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>atlas.hook.hive.queueSize</name> + <value>1000</value> + <description></description> + <filename>hive-atlas-application.properties.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes/> + <depends-on/> + <property_depended_by/> +</property></configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/ba8d7f50/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-env.xml b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-env.xml new file mode 100755 index 0000000..e5ed319 --- /dev/null +++ b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-env.xml @@ -0,0 +1,540 @@ +<configuration><property require-input="false"> + <name>content</name> + <value> + export HADOOP_USER_CLASSPATH_FIRST=true #this prevents old metrics libs from mapreduce lib from bringing in old jar deps overriding HIVE_LIB + if [ "$SERVICE" = "cli" ]; then + if [ -z "$DEBUG" ]; then + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseNUMA -XX:+UseParallelGC -XX:-UseGCOverheadLimit" + else + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" + fi + fi + + # The heap size of the jvm stared by hive shell script can be controlled via: + + if [ "$SERVICE" = "metastore" ]; then + export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore + else + export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client + fi + + export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m" + + # Larger heap size may be required when running queries over large number of files or partitions. + # By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be + # appropriate for hive server (hwi etc). + + + # Set HADOOP_HOME to point to a specific hadoop install directory + HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + + export HIVE_HOME=${HIVE_HOME:-{{hive_home_dir}}} + + # Hive Configuration Directory can be controlled by: + export HIVE_CONF_DIR=${HIVE_CONF_DIR:-{{hive_config_dir}}} + + # Folder containing extra libraries required for hive compilation/execution can be controlled by: + if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then + if [ -f "${HIVE_AUX_JARS_PATH}" ]; then + export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} + elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar + fi + elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar + fi + + export METASTORE_PORT={{hive_metastore_port}} + + {% if sqla_db_used or lib_dir_available %} + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}" + export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}" + {% endif %} + </value> + <description>This is the jinja template for hive-env.sh file</description> + <display-name>hive-env template</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>content</type> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>hive.heapsize</name> + <value>512</value> + <description>Hive Java heap size</description> + <display-name>HiveServer2 Heap Size</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>int</type> + <maximum>2048</maximum> + <minimum>512</minimum> + <unit>MB</unit> + <overridable>false</overridable> + <increment-step>512</increment-step> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>hive_exec_orc_storage_strategy</name> + <value>SPEED</value> + <description> + Define the encoding strategy to use while writing data. Changing this will only affect the light weight encoding for integers. + This flag will not change the compression level of higher level compression codec (like ZLIB). Possible options are SPEED and COMPRESSION. + </description> + <display-name>ORC Storage Strategy</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>SPEED</value> + <label>Speed</label> + </entry> + <entry> + <value>COMPRESSION</value> + <label>Compression</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <depends-on/> + <property_depended_by> + <dependedByProperties> + <name>hive.exec.orc.encoding.strategy</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.exec.orc.compression.strategy</name> + <type>hive-site</type> + </dependedByProperties> + </property_depended_by> +</property><property require-input="false"> + <name>hive_txn_acid</name> + <value>off</value> + <display-name>ACID Transactions</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>on</value> + <label>On</label> + </entry> + <entry> + <value>off</value> + <label>Off</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <depends-on/> + <property_depended_by> + <dependedByProperties> + <name>hive.enforce.bucketing</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.exec.dynamic.partition.mode</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.support.concurrency</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.txn.manager</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.compactor.initiator.on</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.compactor.worker.threads</name> + <type>hive-site</type> + </dependedByProperties> + </property_depended_by> +</property><property require-input="false"> + <name>hive_security_authorization</name> + <value>None</value> + <description> + Authorization mode, default NONE. Options are NONE, Ranger, SQLStdAuth. + SQL standard authorization provides grant/revoke functionality at database, table level. + Ranger provides a centralized authorization interface for Hive and provides more granular + access control at column level through the Hive plugin. + </description> + <display-name>Choose Authorization</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>None</value> + <label>None</label> + </entry> + <entry> + <value>SQLStdAuth</value> + <label>SQLStdAuth</label> + </entry> + <entry> + <value>Ranger</value> + <label>Ranger</label> + </entry> + </entries> + </value-attributes> + <depends-on> + <property> + <name>ranger-hive-plugin-enabled</name> + <type>ranger-env</type> + </property> + </depends-on> + <property_depended_by> + <dependedByProperties> + <name>hive.server2.enable.doAs</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.security.authenticator.manager</name> + <type>hiveserver2-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.server2.enable.doAs</name> + <type>hive-interactive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.security.authorization.enabled</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.conf.restricted.list</name> + <type>hiveserver2-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.security.authenticator.manager</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.security.authorization.manager</name> + <type>hiveserver2-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.security.authorization.manager</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.security.authorization.enabled</name> + <type>hiveserver2-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.security.metastore.authorization.manager</name> + <type>hive-site</type> + </dependedByProperties> + </property_depended_by> +</property><property require-input="false"> + <name>hive_timeline_logging_enabled</name> + <value>true</value> + <display-name>Use ATS Logging</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>true</value> + <label>True</label> + </entry> + <entry> + <value>false</value> + <label>False</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <depends-on/> + <property_depended_by> + <dependedByProperties> + <name>hive.exec.pre.hooks</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.exec.post.hooks</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.exec.failure.hooks</name> + <type>hive-site</type> + </dependedByProperties> + </property_depended_by> +</property><property require-input="false"> + <name>hive.client.heapsize</name> + <value>512</value> + <description>Hive Client Java heap size</description> + <display-name>Client Heap Size</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>int</type> + <maximum>2048</maximum> + <minimum>512</minimum> + <unit>MB</unit> + <overridable>false</overridable> + <increment-step>512</increment-step> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>hive.metastore.heapsize</name> + <value>1024</value> + <description>Hive Metastore Java heap size</description> + <display-name>Metastore Heap Size</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>int</type> + <maximum>2048</maximum> + <minimum>512</minimum> + <unit>MB</unit> + <increment-step>512</increment-step> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>hive_database_type</name> + <value>mysql</value> + <description>Default HIVE DB type.</description> + <display-name>Hive Database Type</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <depends-on> + <property> + <name>hive_database</name> + <type>hive-env</type> + </property> + </depends-on> + <property_depended_by/> +</property><property require-input="false"> + <name>hive_database</name> + <value>New MySQL Database</value> + <description> + Property that determines whether the HIVE DB is managed by Ambari. + </description> + <display-name>Hive Database</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <depends-on/> + <property_depended_by> + <dependedByProperties> + <name>hive_database_type</name> + <type>hive-env</type> + </dependedByProperties> + <dependedByProperties> + <name>javax.jdo.option.ConnectionURL</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>javax.jdo.option.ConnectionDriverName</name> + <type>hive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>datanucleus.rdbms.datastoreAdapterClassName</name> + <type>hive-site</type> + </dependedByProperties> + </property_depended_by> +</property><property require-input="false"> + <name>hive_ambari_database</name> + <value>MySQL</value> + <description>Database type.</description> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes/> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>hive_database_name</name> + <value>hive</value> + <description>Database name.</description> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>database</type> + <visible>false</visible> + <overridable>false</overridable> + <editable-only-at-install>true</editable-only-at-install> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>hive_log_dir</name> + <value>/var/log/hive</value> + <description>Directory for Hive Log files.</description> + <display-name>Hive Log Dir</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>hive_pid_dir</name> + <value>/var/run/hive</value> + <description>Hive PID Dir.</description> + <display-name>Hive PID Dir</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + <editable-only-at-install>true</editable-only-at-install> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>hive_user</name> + <value>hive</value> + <description>Hive User.</description> + <display-name>Hive User</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type>USER</property-type> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + <depends-on/> + <property_depended_by> + <dependedByProperties> + <name>ranger.plugins.hive.serviceuser</name> + <type>ranger-admin-site</type> + </dependedByProperties> + <dependedByProperties> + <name>ranger.kms.service.user.hive</name> + <type>ranger-admin-site</type> + </dependedByProperties> + </property_depended_by> +</property><property require-input="false"> + <name>hcat_log_dir</name> + <value>/var/log/webhcat</value> + <description>WebHCat Log Dir.</description> + <display-name>WebHCat Log Dir</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>hcat_pid_dir</name> + <value>/var/run/webhcat</value> + <description>WebHCat Pid Dir.</description> + <display-name>WebHCat Pid Dir</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + <editable-only-at-install>true</editable-only-at-install> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>hcat_user</name> + <value>hcat</value> + <description>HCat User.</description> + <display-name>HCat User</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type>USER</property-type> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>webhcat_user</name> + <value>hcat</value> + <description>WebHCat User.</description> + <display-name>WebHCat User</display-name> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type>USER</property-type> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>hive_user_nofile_limit</name> + <value>32000</value> + <description>Max open files limit setting for HIVE user.</description> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes/> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>hive_user_nproc_limit</name> + <value>16000</value> + <description>Max number of processes limit setting for HIVE user.</description> + <filename>hive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes/> + <depends-on/> + <property_depended_by/> +</property></configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/ba8d7f50/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-exec-log4j.xml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-exec-log4j.xml b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-exec-log4j.xml new file mode 100755 index 0000000..538334c --- /dev/null +++ b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-exec-log4j.xml @@ -0,0 +1,96 @@ +<configuration><property require-input="false"> + <name>content</name> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties + +hive.log.threshold=ALL +hive.root.logger=INFO,FA +hive.log.dir=${java.io.tmpdir}/${user.name} +hive.query.id=hadoop +hive.log.file=${hive.query.id}.log + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hive.root.logger}, EventCounter + +# Logging Threshold +log4j.threshhold=${hive.log.threshold} + +# +# File Appender +# + +log4j.appender.FA=org.apache.log4j.FileAppender +log4j.appender.FA.File=${hive.log.dir}/${hive.log.file} +log4j.appender.FA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +#custom logging levels +#log4j.logger.xxx=DEBUG + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter + + +log4j.category.DataNucleus=ERROR,FA +log4j.category.Datastore=ERROR,FA +log4j.category.Datastore.Schema=ERROR,FA +log4j.category.JPOX.Datastore=ERROR,FA +log4j.category.JPOX.Plugin=ERROR,FA +log4j.category.JPOX.MetaData=ERROR,FA +log4j.category.JPOX.Query=ERROR,FA +log4j.category.JPOX.General=ERROR,FA +log4j.category.JPOX.Enhancer=ERROR,FA + + +# Silence useless ZK logs +log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA +log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA + + </value> + <description>Custom hive-exec-log4j</description> + <display-name>hive-exec-log4j template</display-name> + <filename>hive-exec-log4j.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property></configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/ba8d7f50/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-exec-log4j2.xml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-exec-log4j2.xml b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-exec-log4j2.xml new file mode 100755 index 0000000..c818d43 --- /dev/null +++ b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-exec-log4j2.xml @@ -0,0 +1,83 @@ +<configuration><property require-input="false"> + <name>content</name> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = HiveExecLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = INFO +property.hive.root.logger = FA +property.hive.query.id = hadoop +property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} +property.hive.log.file = ${sys:hive.query.id}.log + +# list of all appenders +appenders = console, FA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# simple file appender +appender.FA.type = File +appender.FA.name = FA +appender.FA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file} +appender.FA.layout.type = PatternLayout +appender.FA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n + +# list of all loggers +loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} + </value> + <description>Custom hive-exec-log4j2.properties</description> + <display-name>hive-exec-log4j2 template</display-name> + <filename>hive-exec-log4j2.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property></configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/ba8d7f50/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-interactive-env.xml b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-interactive-env.xml new file mode 100755 index 0000000..7035283 --- /dev/null +++ b/bigtop-packages/src/common/ambari/ODPi/1.0/services/HIVE/configuration/hive-interactive-env.xml @@ -0,0 +1,373 @@ +<configuration><property require-input="false"> + <name>enable_hive_interactive</name> + <value>false</value> + <description>Enable or disable interactive query in this cluster.</description> + <display-name>Enable Interactive Query (Tech Preview)</display-name> + <filename>hive-interactive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>value-list</type> + <overridable>false</overridable> + <entries> + <entry> + <value>true</value> + <label>Yes</label> + </entry> + <entry> + <value>false</value> + <label>No</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <depends-on/> + <property_depended_by> + <dependedByProperties> + <name>hive.llap.daemon.queue.name</name> + <type>hive-interactive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.llap.daemon.num.executors</name> + <type>hive-interactive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>llap_queue_capacity</name> + <type>hive-interactive-env</type> + </dependedByProperties> + <dependedByProperties> + <name>llap_heap_size</name> + <type>hive-interactive-env</type> + </dependedByProperties> + <dependedByProperties> + <name>tez.am.resource.memory.mb</name> + <type>tez-interactive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.llap.daemon.yarn.container.mb</name> + <type>hive-interactive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>capacity-scheduler</name> + <type>capacity-scheduler</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.server2.tez.sessions.per.default.queue</name> + <type>hive-interactive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.llap.io.memory.size</name> + <type>hive-interactive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>num_llap_nodes</name> + <type>hive-interactive-env</type> + </dependedByProperties> + <dependedByProperties> + <name>slider_am_container_mb</name> + <type>hive-interactive-env</type> + </dependedByProperties> + </property_depended_by> +</property><property require-input="false"> + <name>hive_server_interactive_host</name> + <value>localhost</value> + <description>HiveServer2 Interactive Host</description> + <display-name>HiveServer2 Interactive Host</display-name> + <filename>hive-interactive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>llap_queue_capacity</name> + <value>0</value> + <description>Percentage of the cluster dedicated to interactive query.</description> + <display-name>% of Cluster Capacity</display-name> + <filename>hive-interactive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>int</type> + <maximum>100</maximum> + <minimum>20</minimum> + <increment-step>1</increment-step> + </value-attributes> + <depends-on> + <property> + <name>yarn.scheduler.capacity.root.queues</name> + <type>capacity-scheduler</type> + </property> + <property> + <name>hive.llap.daemon.queue.name</name> + <type>hive-interactive-site</type> + </property> + <property> + <name>enable_hive_interactive</name> + <type>hive-interactive-env</type> + </property> + </depends-on> + <property_depended_by> + <dependedByProperties> + <name>hive.llap.daemon.num.executors</name> + <type>hive-interactive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>llap_heap_size</name> + <type>hive-interactive-env</type> + </dependedByProperties> + <dependedByProperties> + <name>tez.am.resource.memory.mb</name> + <type>tez-interactive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.llap.daemon.yarn.container.mb</name> + <type>hive-interactive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>capacity-scheduler</name> + <type>capacity-scheduler</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.server2.tez.sessions.per.default.queue</name> + <type>hive-interactive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>hive.llap.io.memory.size</name> + <type>hive-interactive-site</type> + </dependedByProperties> + <dependedByProperties> + <name>num_llap_nodes</name> + <type>hive-interactive-env</type> + </dependedByProperties> + <dependedByProperties> + <name>slider_am_container_mb</name> + <type>hive-interactive-env</type> + </dependedByProperties> + </property_depended_by> +</property><property require-input="false"> + <name>num_llap_nodes</name> + <value>1</value> + <description>The number of Hive LLAP daemons to run.</description> + <display-name>Number of LLAP Daemons</display-name> + <filename>hive-interactive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>int</type> + <minimum>1</minimum> + </value-attributes> + <depends-on> + <property> + <name>yarn.scheduler.capacity.root.queues</name> + <type>capacity-scheduler</type> + </property> + <property> + <name>hive.llap.daemon.queue.name</name> + <type>hive-interactive-site</type> + </property> + <property> + <name>llap_queue_capacity</name> + <type>hive-interactive-env</type> + </property> + <property> + <name>enable_hive_interactive</name> + <type>hive-interactive-env</type> + </property> + <property> + <name>hive.server2.tez.sessions.per.default.queue</name> + <type>hive-interactive-site</type> + </property> + </depends-on> + <property_depended_by/> +</property><property require-input="false"> + <name>num_retries_for_checking_llap_status</name> + <value>10</value> + <description>After starting LLAP app, retry count to check LLAP status before starting HiveServer2.</description> + <display-name>Number of retries while checking LLAP app status</display-name> + <filename>hive-interactive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>int</type> + <maximum>20</maximum> + <minimum>0</minimum> + <increment-step>1</increment-step> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>llap_heap_size</name> + <value>0</value> + <description>Heap Size used by LLAP app.</description> + <display-name>LLAP heap size</display-name> + <filename>hive-interactive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>int</type> + </value-attributes> + <depends-on> + <property> + <name>tez.am.resource.memory.mb</name> + <type>tez-site</type> + </property> + <property> + <name>yarn.scheduler.capacity.root.queues</name> + <type>capacity-scheduler</type> + </property> + <property> + <name>hive.llap.daemon.queue.name</name> + <type>hive-interactive-site</type> + </property> + <property> + <name>llap_queue_capacity</name> + <type>hive-interactive-env</type> + </property> + <property> + <name>enable_hive_interactive</name> + <type>hive-interactive-env</type> + </property> + <property> + <name>hive.server2.tez.sessions.per.default.queue</name> + <type>hive-interactive-site</type> + </property> + </depends-on> + <property_depended_by/> +</property><property require-input="false"> + <name>slider_am_container_mb</name> + <value>341</value> + <description>Slider's app master container size in MB.</description> + <display-name>Slider AM container size</display-name> + <filename>hive-interactive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>int</type> + <unit>MB</unit> + </value-attributes> + <depends-on> + <property> + <name>yarn.scheduler.capacity.root.queues</name> + <type>capacity-scheduler</type> + </property> + <property> + <name>hive.llap.daemon.queue.name</name> + <type>hive-interactive-site</type> + </property> + <property> + <name>llap_queue_capacity</name> + <type>hive-interactive-env</type> + </property> + <property> + <name>enable_hive_interactive</name> + <type>hive-interactive-env</type> + </property> + <property> + <name>hive.server2.tez.sessions.per.default.queue</name> + <type>hive-interactive-site</type> + </property> + </depends-on> + <property_depended_by/> +</property><property require-input="false"> + <name>llap_log_level</name> + <value>INFO</value> + <description>LLAP app logging level (WARN/INFO/DEBUG/TRACE)</description> + <display-name>LLAP app logging level (WARN/INFO/DEBUG/TRACE)</display-name> + <filename>hive-interactive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes/> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>llap_app_name</name> + <value>llap0</value> + <description>LLAP app name</description> + <display-name>LLAP app name</display-name> + <filename>hive-interactive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes/> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>llap_java_opts</name> + <value>-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}</value> + <description>Java opts for llap application</description> + <display-name>LLAP app java opts</display-name> + <filename>hive-interactive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes/> + <depends-on/> + <property_depended_by/> +</property><property require-input="false"> + <name>content</name> + <value> + if [ "$SERVICE" = "cli" ]; then + if [ -z "$DEBUG" ]; then + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit" + else + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" + fi + fi + + # The heap size of the jvm stared by hive shell script can be controlled via: + + if [ "$SERVICE" = "metastore" ]; then + export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore + else + export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client + fi + + export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m" + + # Larger heap size may be required when running queries over large number of files or partitions. + # By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be + # appropriate for hive server (hwi etc). + + + # Set HADOOP_HOME to point to a specific hadoop install directory + HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + + # Hive Configuration Directory can be controlled by: + export HIVE_CONF_DIR={{hive_server_interactive_conf_dir}} + + # Add additional hcatalog jars + if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then + export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} + else + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-server2-hive2/lib/hive-hcatalog-core.jar + fi + + export METASTORE_PORT={{hive_metastore_port}} + + # Spark assembly contains a conflicting copy of HiveConf from hive-1.2 + export HIVE_SKIP_SPARK_ASSEMBLY=true + + </value> + <description>This is the jinja template for hive-env.sh file</description> + <display-name>hive-interactive-env template</display-name> + <filename>hive-interactive-env.xml</filename> + <deleted>false</deleted> + <on-ambari-upgrade add="true" delete="false" update="false"/> + <property-type></property-type> + <value-attributes> + <type>content</type> + </value-attributes> + <depends-on/> + <property_depended_by/> +</property></configuration> \ No newline at end of file
