AMBARI-19832. HDP 3.0 support for Hive with configs, kerberos, widgets, metrics, quicklinks, and themes.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/80b87400 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/80b87400 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/80b87400 Branch: refs/heads/trunk Commit: 80b874004c745c11fed682583a27136cb5f154c0 Parents: a00e7eb Author: Vitaly Brodetskyi <[email protected]> Authored: Fri Mar 24 12:25:42 2017 +0200 Committer: Vitaly Brodetskyi <[email protected]> Committed: Fri Mar 24 12:25:42 2017 +0200 ---------------------------------------------------------------------- .../common-services/HIVE/2.1.0.3.0/alerts.json | 232 ++ .../2.1.0.3.0/configuration/beeline-log4j2.xml | 80 + .../HIVE/2.1.0.3.0/configuration/hcat-env.xml | 60 + .../hive-atlas-application.properties.xml | 61 + .../HIVE/2.1.0.3.0/configuration/hive-env.xml | 495 +++++ .../2.1.0.3.0/configuration/hive-exec-log4j.xml | 114 + .../configuration/hive-exec-log4j2.xml | 101 + .../configuration/hive-interactive-env.xml | 332 +++ .../configuration/hive-interactive-site.xml | 762 +++++++ .../HIVE/2.1.0.3.0/configuration/hive-log4j.xml | 147 ++ .../2.1.0.3.0/configuration/hive-log4j2.xml | 131 ++ .../configuration/hive-logsearch-conf.xml | 117 + .../HIVE/2.1.0.3.0/configuration/hive-site.xml | 2000 ++++++++++++++++++ .../configuration/hivemetastore-site.xml | 46 + .../hiveserver2-interactive-site.xml | 55 + .../configuration/hiveserver2-site.xml | 110 + .../2.1.0.3.0/configuration/llap-cli-log4j2.xml | 148 ++ .../configuration/llap-daemon-log4j.xml | 210 ++ .../configuration/ranger-hive-audit.xml | 121 ++ .../ranger-hive-plugin-properties.xml | 127 ++ .../configuration/ranger-hive-policymgr-ssl.xml | 66 + .../configuration/ranger-hive-security.xml | 74 + .../configuration/tez-interactive-site.xml | 257 +++ .../2.1.0.3.0/configuration/webhcat-env.xml | 57 + .../2.1.0.3.0/configuration/webhcat-log4j.xml | 104 + .../2.1.0.3.0/configuration/webhcat-site.xml | 193 ++ .../HIVE/2.1.0.3.0/kerberos.json | 151 ++ .../common-services/HIVE/2.1.0.3.0/metainfo.xml | 517 +++++ .../alert_hive_interactive_thrift_port.py | 230 ++ .../package/alerts/alert_hive_metastore.py | 273 +++ .../package/alerts/alert_hive_thrift_port.py | 285 +++ .../package/alerts/alert_llap_app_status.py | 299 +++ .../package/alerts/alert_webhcat_server.py | 228 ++ .../package/etc/hive-schema-0.12.0.mysql.sql | 777 +++++++ .../package/etc/hive-schema-0.12.0.oracle.sql | 718 +++++++ .../package/etc/hive-schema-0.12.0.postgres.sql | 1406 ++++++++++++ .../package/etc/hive-schema-0.13.0.mysql.sql | 889 ++++++++ .../package/etc/hive-schema-0.13.0.oracle.sql | 835 ++++++++ .../package/etc/hive-schema-0.13.0.postgres.sql | 1538 ++++++++++++++ .../etc/upgrade-0.12.0-to-0.13.0.oracle.sql | 165 ++ .../package/etc/upgrade-0.13.0.oracle.sql | 38 + .../2.1.0.3.0/package/files/addMysqlUser.sh | 39 + .../HIVE/2.1.0.3.0/package/files/hcatSmoke.sh | 41 + .../HIVE/2.1.0.3.0/package/files/hiveSmoke.sh | 24 + .../2.1.0.3.0/package/files/hiveTezSetup.cmd | 58 + .../2.1.0.3.0/package/files/hiveserver2.sql | 23 + .../2.1.0.3.0/package/files/hiveserver2Smoke.sh | 32 + .../HIVE/2.1.0.3.0/package/files/pigSmoke.sh | 18 + .../2.1.0.3.0/package/files/removeMysqlUser.sh | 33 + .../2.1.0.3.0/package/files/startMetastore.sh | 25 + .../2.1.0.3.0/package/files/templetonSmoke.sh | 101 + .../HIVE/2.1.0.3.0/package/scripts/__init__.py | 19 + .../HIVE/2.1.0.3.0/package/scripts/hcat.py | 87 + .../2.1.0.3.0/package/scripts/hcat_client.py | 85 + .../package/scripts/hcat_service_check.py | 89 + .../HIVE/2.1.0.3.0/package/scripts/hive.py | 520 +++++ .../2.1.0.3.0/package/scripts/hive_client.py | 68 + .../package/scripts/hive_interactive.py | 346 +++ .../2.1.0.3.0/package/scripts/hive_metastore.py | 263 +++ .../2.1.0.3.0/package/scripts/hive_server.py | 230 ++ .../package/scripts/hive_server_interactive.py | 679 ++++++ .../package/scripts/hive_server_upgrade.py | 145 ++ .../2.1.0.3.0/package/scripts/hive_service.py | 187 ++ .../package/scripts/hive_service_interactive.py | 110 + .../2.1.0.3.0/package/scripts/mysql_server.py | 63 + .../2.1.0.3.0/package/scripts/mysql_service.py | 51 + .../2.1.0.3.0/package/scripts/mysql_users.py | 72 + .../2.1.0.3.0/package/scripts/mysql_utils.py | 35 + .../HIVE/2.1.0.3.0/package/scripts/params.py | 30 + .../2.1.0.3.0/package/scripts/params_linux.py | 822 +++++++ .../2.1.0.3.0/package/scripts/params_windows.py | 75 + .../2.1.0.3.0/package/scripts/service_check.py | 194 ++ .../package/scripts/setup_ranger_hive.py | 98 + .../scripts/setup_ranger_hive_interactive.py | 77 + .../2.1.0.3.0/package/scripts/status_params.py | 118 ++ .../HIVE/2.1.0.3.0/package/scripts/webhcat.py | 139 ++ .../2.1.0.3.0/package/scripts/webhcat_server.py | 167 ++ .../package/scripts/webhcat_service.py | 111 + .../package/scripts/webhcat_service_check.py | 133 ++ .../hadoop-metrics2-hivemetastore.properties.j2 | 56 + .../hadoop-metrics2-hiveserver2.properties.j2 | 55 + .../templates/hadoop-metrics2-llapdaemon.j2 | 54 + .../hadoop-metrics2-llaptaskscheduler.j2 | 54 + .../2.1.0.3.0/package/templates/hive.conf.j2 | 35 + .../package/templates/startHiveserver2.sh.j2 | 24 + .../templates/startHiveserver2Interactive.sh.j2 | 24 + .../package/templates/templeton_smoke.pig.j2 | 24 + .../package/templates/zkmigrator_jaas.conf.j2 | 26 + .../HIVE/2.1.0.3.0/quicklinks/quicklinks.json | 62 + .../HIVE/2.1.0.3.0/themes/theme.json | 753 +++++++ .../stacks/HDP/3.0/services/HIVE/metainfo.xml | 27 + 91 files changed, 21150 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/alerts.json b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/alerts.json new file mode 100644 index 0000000..e2431c3 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/alerts.json @@ -0,0 +1,232 @@ +{ + "HIVE": { + "service": [], + "HIVE_METASTORE": [ + { + "name": "hive_metastore_process", + "label": "Hive Metastore Process", + "description": "This host-level alert is triggered if the Hive Metastore process cannot be determined to be up and listening on the network.", + "interval": 3, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_hive_metastore.py", + "parameters": [ + { + "name": "check.command.timeout", + "display_name": "Command Timeout", + "value": 60.0, + "type": "NUMERIC", + "description": "The maximum time before check command will be killed by timeout", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "default.smoke.user", + "display_name": "Default Smoke User", + "value": "ambari-qa", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env/smokeuser", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.principal", + "display_name": "Default Smoke Principal", + "value": "[email protected]", + "type": "STRING", + "description": "The principal to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_principal_name", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.keytab", + "display_name": "Default Smoke Keytab", + "value": "/etc/security/keytabs/smokeuser.headless.keytab", + "type": "STRING", + "description": "The keytab to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_keytab", + "visibility": "HIDDEN" + } + ] + } + } + ], + "HIVE_SERVER": [ + { + "name": "hive_server_process", + "label": "HiveServer2 Process", + "description": "This host-level alert is triggered if the HiveServer cannot be determined to be up and responding to client requests.", + "interval": 3, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py", + "parameters": [ + { + "name": "check.command.timeout", + "display_name": "Command Timeout", + "value": 60.0, + "type": "NUMERIC", + "description": "The maximum time before check command will be killed by timeout", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "default.smoke.user", + "display_name": "Default Smoke User", + "value": "ambari-qa", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env/smokeuser", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.principal", + "display_name": "Default Smoke Principal", + "value": "[email protected]", + "type": "STRING", + "description": "The principal to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_principal_name", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.keytab", + "display_name": "Default Smoke Keytab", + "value": "/etc/security/keytabs/smokeuser.headless.keytab", + "type": "STRING", + "description": "The keytab to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_keytab", + "visibility": "HIDDEN" + } + ] + } + } + ], + "HIVE_SERVER_INTERACTIVE": [ + { + "name": "hive_server_interactive_process", + "label": "HiveServer2 Interactive Process", + "description": "This host-level alert is triggered if the HiveServerInteractive cannot be determined to be up and responding to client requests.", + "interval": 3, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_hive_interactive_thrift_port.py", + "parameters": [ + { + "name": "check.command.timeout", + "display_name": "Command Timeout", + "value": 60.0, + "type": "NUMERIC", + "description": "The maximum time before check command will be killed by timeout", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "default.smoke.user", + "display_name": "Default Smoke User", + "value": "ambari-qa", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env/smokeuser", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.principal", + "display_name": "Default Smoke Principal", + "value": "[email protected]", + "type": "STRING", + "description": "The principal to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_principal_name", + "visibility": "HIDDEN" + }, + { + "name": "default.smoke.keytab", + "display_name": "Default Smoke Keytab", + "value": "/etc/security/keytabs/smokeuser.headless.keytab", + "type": "STRING", + "description": "The keytab to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_keytab", + "visibility": "HIDDEN" + } + ] + } + }, + { + "name": "llap_application", + "label": "LLAP Application", + "description": "This alert is triggered if the LLAP Application cannot be determined to be up and responding to requests.", + "interval": 3, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_llap_app_status.py", + "parameters": [ + { + "name": "check.command.timeout", + "display_name": "Command Timeout", + "value": 120.0, + "type": "NUMERIC", + "description": "The maximum time before check command will be killed by timeout", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "default.hive.user", + "display_name": "Default HIVE User", + "value": "hive", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env", + "visibility": "HIDDEN" + }, + { + "name": "default.hive.principal", + "display_name": "Default HIVE Principal", + "value": "hive/[email protected]", + "type": "STRING", + "description": "The principal to use when retrieving the kerberos ticket if not specified in cluster-env", + "visibility": "HIDDEN" + }, + { + "name": "default.hive.keytab", + "display_name": "Default HIVE Keytab", + "value": "/etc/security/keytabs/hive.llap.zk.sm.keytab", + "type": "STRING", + "description": "The keytab to use when retrieving the kerberos ticket if not specified in cluster-env.", + "visibility": "HIDDEN" + } + ] + } + } + ], + "WEBHCAT_SERVER": [ + { + "name": "hive_webhcat_server_status", + "label": "WebHCat Server Status", + "description": "This host-level alert is triggered if the templeton server status is not healthy.", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "HIVE/0.12.0.2.0/package/alerts/alert_webhcat_server.py", + "parameters": [ + { + "name": "default.smoke.user", + "display_name": "Default Smoke User", + "value": "ambari-qa", + "type": "STRING", + "description": "The user that will run the Hive commands if not specified in cluster-env/smokeuser", + "visibility": "HIDDEN" + }, + { + "name": "connection.timeout", + "display_name": "Connection Timeout", + "value": 5.0, + "type": "NUMERIC", + "description": "The maximum time before this alert is considered to be CRITICAL", + "units": "seconds", + "threshold": "CRITICAL" + } + ] + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/beeline-log4j2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/beeline-log4j2.xml b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/beeline-log4j2.xml new file mode 100644 index 0000000..5f27ff2 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/beeline-log4j2.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>content</name> + <display-name>beeline-log4j template</display-name> + <description>Custom beeline-log4j2.properties</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = BeelineLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = {{hive_log_level}} +property.hive.root.logger = console + +# list of all appenders +appenders = console + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# list of all loggers +loggers = HiveConnection + +# HiveConnection logs useful info for dynamic service discovery +logger.HiveConnection.name = org.apache.hive.jdbc.HiveConnection +logger.HiveConnection.level = INFO + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hcat-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hcat-env.xml b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hcat-env.xml new file mode 100644 index 0000000..964abdb --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hcat-env.xml @@ -0,0 +1,60 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <!-- hcat-env.sh --> + <property> + <name>content</name> + <display-name>hcat-env template</display-name> + <description>This is the jinja template for hcat-env.sh file</description> + <value> + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + JAVA_HOME={{java64_home}} + HCAT_PID_DIR={{hcat_pid_dir}}/ + HCAT_LOG_DIR={{hcat_log_dir}}/ + HCAT_CONF_DIR={{hcat_conf_dir}} + HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + #DBROOT is the path where the connector jars are downloaded + DBROOT={{hcat_dbroot}} + USER={{webhcat_user}} + METASTORE_PORT={{hive_metastore_port}} + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-atlas-application.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-atlas-application.properties.xml b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-atlas-application.properties.xml new file mode 100644 index 0000000..9022d0a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-atlas-application.properties.xml @@ -0,0 +1,61 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false"> + <!-- These are the Atlas Hooks properties specific to this service. This file is then merged with common properties + that apply to all services. --> + <property> + <name>atlas.hook.hive.synchronous</name> + <value>false</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.hive.numRetries</name> + <value>3</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.hive.minThreads</name> + <value>5</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.hive.maxThreads</name> + <value>5</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.hive.keepAliveTime</name> + <value>10</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.hive.queueSize</name> + <value>1000</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-env.xml b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-env.xml new file mode 100644 index 0000000..4ed26f7 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-env.xml @@ -0,0 +1,495 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <property> + <name>hive.atlas.hook</name> + <value>false</value> + <display-name>Enable Atlas Hook</display-name> + <description>Enable Atlas Hook</description> + <value-attributes> + <type>boolean</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + <depends-on> + <property> + <type>application-properties</type> + <name>atlas.rest.address</name> + </property> + </depends-on> + </property> + <property> + <name>hive.client.heapsize</name> + <value>512</value> + <description>Hive Client Java heap size</description> + <display-name>Client Heap Size</display-name> + <value-attributes> + <type>int</type> + <minimum>512</minimum> + <maximum>2048</maximum> + <unit>MB</unit> + <increment-step>512</increment-step> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.metastore.heapsize</name> + <value>1024</value> + <description>Hive Metastore Java heap size</description> + <display-name>Metastore Heap Size</display-name> + <value-attributes> + <type>int</type> + <minimum>512</minimum> + <maximum>2048</maximum> + <unit>MB</unit> + <increment-step>512</increment-step> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_database_type</name> + <value>mysql</value> + <display-name>Hive Database Type</display-name> + <description>Default HIVE DB type.</description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <depends-on> + <property> + <type>hive-env</type> + <name>hive_database</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_ambari_database</name> + <value>MySQL</value> + <description>Database type.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_database_name</name> + <value>hive</value> + <description>Database name.</description> + <value-attributes> + <type>database</type> + <visible>false</visible> + <editable-only-at-install>true</editable-only-at-install> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_log_dir</name> + <value>/var/log/hive</value> + <display-name>Hive Log Dir</display-name> + <description>Directory for Hive Log files.</description> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_pid_dir</name> + <value>/var/run/hive</value> + <display-name>Hive PID Dir</display-name> + <description>Hive PID Dir.</description> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + <editable-only-at-install>true</editable-only-at-install> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_user</name> + <display-name>Hive User</display-name> + <value>hive</value> + <property-type>USER</property-type> + <description>Hive User.</description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <!--HCAT--> + <property> + <name>hcat_log_dir</name> + <value>/var/log/webhcat</value> + <display-name>WebHCat Log Dir</display-name> + <description>WebHCat Log Dir.</description> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hcat_pid_dir</name> + <value>/var/run/webhcat</value> + <display-name>WebHCat Pid Dir</display-name> + <description>WebHCat Pid Dir.</description> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + <editable-only-at-install>true</editable-only-at-install> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>webhcat_user</name> + <display-name>WebHCat User</display-name> + <value>hcat</value> + <property-type>USER</property-type> + <description>WebHCat User.</description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_user_nofile_limit</name> + <value>32000</value> + <description>Max open files limit setting for HIVE user.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_user_nproc_limit</name> + <value>16000</value> + <description>Max number of processes limit setting for HIVE user.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.heapsize</name> + <value>512</value> + <description>Hive Java heap size</description> + <display-name>HiveServer2 Heap Size</display-name> + <value-attributes> + <type>int</type> + <minimum>512</minimum> + <maximum>2048</maximum> + <unit>MB</unit> + <increment-step>512</increment-step> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_exec_orc_storage_strategy</name> + <display-name>ORC Storage Strategy</display-name> + <description> + Define the encoding strategy to use while writing data. Changing this will only affect the light weight encoding for integers. + This flag will not change the compression level of higher level compression codec (like ZLIB). Possible options are SPEED and COMPRESSION. + </description> + <value>SPEED</value> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>SPEED</value> + <label>Speed</label> + </entry> + <entry> + <value>COMPRESSION</value> + <label>Compression</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_txn_acid</name> + <display-name>ACID Transactions</display-name> + <description>Enable/Disable ACID transactions for HIVE</description> + <value>off</value> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>on</value> + <label>On</label> + </entry> + <entry> + <value>off</value> + <label>Off</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_security_authorization</name> + <display-name>Choose Authorization</display-name> + <description> + Authorization mode, default NONE. Options are NONE, Ranger, SQLStdAuth. + SQL standard authorization provides grant/revoke functionality at database, table level. + Ranger provides a centralized authorization interface for Hive and provides more granular + access control at column level through the Hive plugin. + </description> + <value>None</value> + <depends-on> + <property> + <type>ranger-env</type> + <name>ranger-hive-plugin-enabled</name> + </property> + </depends-on> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>None</value> + <label>None</label> + </entry> + <entry> + <value>SQLStdAuth</value> + <label>SQLStdAuth</label> + </entry> + <entry> + <value>Ranger</value> + <label>Ranger</label> + </entry> + </entries> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_timeline_logging_enabled</name> + <display-name>Use ATS Logging</display-name> + <value>true</value> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>true</value> + <label>True</label> + </entry> + <entry> + <value>false</value> + <label>False</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_database</name> + <value>New MySQL Database</value> + <display-name>Hive Database</display-name> + <description> + Property that determines whether the HIVE DB is managed by Ambari. + </description> + <value-attributes> + <overridable>false</overridable> + <type>value-list</type> + <entries> + <entry> + <value>New MySQL Database</value> + <label>New MySQL</label> + </entry> + <entry> + <value>Existing MySQL / MariaDB Database</value> + <label>Existing MySQL / MariaDB</label> + </entry> + <entry> + <value>Existing PostgreSQL Database</value> + <label>Existing PostgreSQL</label> + </entry> + <entry> + <value>Existing Oracle Database</value> + <label>Existing Oracle</label> + </entry> + <entry> + <value>Existing SQL Anywhere Database</value> + <label>Existing SQL Anywhere</label> + </entry> + </entries> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive.log.level</name> + <description>Hive Log level to control log4j - Options are INFO, DEBUG, WARN, ERROR</description> + <value>INFO</value> + <display-name>Hive Log Level</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>INFO</value> + <label>INFO (Recommended)</label> + </entry> + <entry> + <value>DEBUG</value> + <label>DEBUG (Most Verbose)</label> + </entry> + <entry> + <value>WARN</value> + <label>WARN</label> + </entry> + <entry> + <value>ERROR</value> + <label>ERROR (Least Verbose)</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>alert_ldap_username</name> + <value></value> + <description>LDAP username to be used for alerts</description> + <display-name>LDAP user for Alerts</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <depends-on> + <property> + <type>hive-site</type> + <name>hive.server2.authentication</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>alert_ldap_password</name> + <value></value> + <property-type>PASSWORD</property-type> + <description>Password to be used for LDAP user used in alerts</description> + <display-name>LDAP password for Alerts</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + <type>password</type> + </value-attributes> + <depends-on> + <property> + <type>hive-site</type> + <name>hive.server2.authentication</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>enable_heap_dump</name> + <value>false</value> + <description>Enable or disable taking Heap Dump. (true/false)</description> + <display-name>Enable heap dump</display-name> + <value-attributes> + <type>value-list</type> + <overridable>false</overridable> + <entries> + <entry> + <value>true</value> + <label>Enabled</label> + </entry> + <entry> + <value>false</value> + <label>Disabled</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>heap_dump_location</name> + <value>/tmp</value> + <description>Location for heap dump file</description> + <display-name>Heap dump location</display-name> + <on-ambari-upgrade add="false"/> + <value-attributes> + <empty-value-valid>false</empty-value-valid> + </value-attributes> + </property> + + + <!-- hive-env.sh --> + <property> + <name>content</name> + <display-name>hive-env template</display-name> + <description>This is the jinja template for hive-env.sh file</description> + <value> + export HADOOP_USER_CLASSPATH_FIRST=true #this prevents old metrics libs from mapreduce lib from bringing in old jar deps overriding HIVE_LIB + if [ "$SERVICE" = "cli" ]; then + if [ -z "$DEBUG" ]; then + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseNUMA -XX:+UseParallelGC -XX:-UseGCOverheadLimit" + else + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" + fi + fi + + # The heap size of the jvm stared by hive shell script can be controlled via: + + if [ "$SERVICE" = "metastore" ]; then + export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore + else + export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client + fi + + export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m" + export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS{{heap_dump_opts}}" + + # Larger heap size may be required when running queries over large number of files or partitions. + # By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be + # appropriate for hive server (hwi etc). + + + # Set HADOOP_HOME to point to a specific hadoop install directory + HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + + export HIVE_HOME=${HIVE_HOME:-{{hive_home_dir}}} + + # Hive Configuration Directory can be controlled by: + export HIVE_CONF_DIR=${HIVE_CONF_DIR:-{{hive_config_dir}}} + + # Folder containing extra libraries required for hive compilation/execution can be controlled by: + if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then + if [ -f "${HIVE_AUX_JARS_PATH}" ]; then + export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} + elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar + fi + elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar + fi + + export METASTORE_PORT={{hive_metastore_port}} + + {% if sqla_db_used or lib_dir_available %} + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}" + export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}" + {% endif %} + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-exec-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-exec-log4j.xml b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-exec-log4j.xml new file mode 100644 index 0000000..4d09756 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-exec-log4j.xml @@ -0,0 +1,114 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="false"> + <property> + <name>content</name> + <display-name>hive-exec-log4j template</display-name> + <description>Custom hive-exec-log4j</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties + +hive.log.threshold=ALL +hive.root.logger={{hive_log_level}},FA +hive.log.dir=${java.io.tmpdir}/${user.name} +hive.query.id=hadoop +hive.log.file=${hive.query.id}.log + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hive.root.logger}, EventCounter + +# Logging Threshold +log4j.threshhold=${hive.log.threshold} + +# +# File Appender +# + +log4j.appender.FA=org.apache.log4j.FileAppender +log4j.appender.FA.File=${hive.log.dir}/${hive.log.file} +log4j.appender.FA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +#custom logging levels +#log4j.logger.xxx=DEBUG + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter + + +log4j.category.DataNucleus=ERROR,FA +log4j.category.Datastore=ERROR,FA +log4j.category.Datastore.Schema=ERROR,FA +log4j.category.JPOX.Datastore=ERROR,FA +log4j.category.JPOX.Plugin=ERROR,FA +log4j.category.JPOX.MetaData=ERROR,FA +log4j.category.JPOX.Query=ERROR,FA +log4j.category.JPOX.General=ERROR,FA +log4j.category.JPOX.Enhancer=ERROR,FA + + +# Silence useless ZK logs +log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA +log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA + + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-exec-log4j2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-exec-log4j2.xml b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-exec-log4j2.xml new file mode 100644 index 0000000..c0b1790 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-exec-log4j2.xml @@ -0,0 +1,101 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="false"> + <property> + <name>content</name> + <display-name>hive-exec-log4j2 template</display-name> + <description>Custom hive-exec-log4j2.properties</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = HiveExecLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = {{hive_log_level}} +property.hive.root.logger = FA +property.hive.query.id = hadoop +property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} +property.hive.log.file = ${sys:hive.query.id}.log + +# list of all appenders +appenders = console, FA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# simple file appender +appender.FA.type = File +appender.FA.name = FA +appender.FA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file} +appender.FA.layout.type = PatternLayout +appender.FA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n + +# list of all loggers +loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/80b87400/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-interactive-env.xml new file mode 100644 index 0000000..e2048a2 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-interactive-env.xml @@ -0,0 +1,332 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + + <property> + <name>num_llap_nodes</name> + <value>1</value> + <description>Number of nodes used by Hive's LLAP, which includes nodes running : LLAP daemon, Slider and Tez App Master(s).</description> + <display-name>Number of nodes used by Hive's LLAP</display-name> + <value-attributes> + <type>int</type> + <minimum>1</minimum> + <increment-step>1</increment-step> + </value-attributes> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>llap_extra_slider_opts</name> + <value></value> + <description>Additional slider specific opts for llap such as port range</description> + <display-name>Additional slider specific opts for llap</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>llap_heap_size</name> + <value>0</value> + <description>LLAP Daemon Heap Size in MB.</description> + <display-name>LLAP Daemon Heap Size (MB)</display-name> + <value-attributes> + <type>int</type> + </value-attributes> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> + <property> + <type>tez-site</type> + <name>tez.am.resource.memory.mb</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>slider_am_container_mb</name> + <value>341</value> + <description>Slider's app master container size in MB.</description> + <display-name>Slider AM container size</display-name> + <value-attributes> + <type>int</type> + <unit>MB</unit> + </value-attributes> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>llap_headroom_space</name> + <value>12288</value> + <description> + Maximum headroom reserved from the YARN container running LLAP daemons. + This is an upper limit used during automatic size calculations, and the actual + value may be lower. + </description> + <display-name>LLAP Daemon Container Max Headroom</display-name> + <value-attributes> + <type>int</type> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>llap_log_level</name> + <value>INFO</value> + <description>LLAP daemon log level (WARN/INFO/DEBUG/TRACE)</description> + <display-name>LLAP daemon log level (WARN/INFO/DEBUG/TRACE)</display-name> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hive_aux_jars</name> + <value/> + <description>List of jars to be made available to LLAP daemons</description> + <display-name>Auxillary JAR list</display-name> + <on-ambari-upgrade add="false"/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>llap_app_name</name> + <value>llap0</value> + <description>LLAP app name</description> + <display-name>LLAP app name</display-name> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hive_heapsize</name> + <value>512</value> + <description>Hive Java heap size</description> + <display-name>HiveServer Interactive Heap Size</display-name> + <value-attributes> + <type>int</type> + <minimum>512</minimum> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.server2.tez.sessions.per.default.queue</name> + </property> + </depends-on> + </property> + <property> + <name>enable_hive_interactive</name> + <value>false</value> + <description>Enable or disable interactive query in this cluster. <br> Note: Hive LLAP may not start up on busy clusters if YARN pre-emption is not enabled. Enable via the YARN configs page.</description> + <display-name>Enable Interactive Query (requires YARN pre-emption)</display-name> + <value-attributes> + <type>value-list</type> + <overridable>false</overridable> + <entries> + <entry> + <value>true</value> + <label>Yes</label> + </entry> + <entry> + <value>false</value> + <label>No</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>llap_java_opts</name> + <value>-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:InitiatingHeapOccupancyPercent=40 -XX:G1ReservePercent=20 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}{{heap_dump_opts}}</value> + <description>Java opts for llap application</description> + <display-name>LLAP app java opts</display-name> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>num_retries_for_checking_llap_status</name> + <value>20</value> + <description>After starting LLAP app, retry count to check LLAP status before starting HiveServer2.</description> + <display-name>Number of retries while checking LLAP app status</display-name> + <value-attributes> + <type>int</type> + <minimum>1</minimum> + <maximum>30</maximum> + <increment-step>1</increment-step> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>num_llap_nodes_for_llap_daemons</name> + <value>1</value> + <description>Number of Node(s) on which Hive LLAP daemon runs.</description> + <display-name>Number of Node(s) for running Hive LLAP daemon</display-name> + <value-attributes> + <type>int</type> + <minimum>1</minimum> + <increment-step>1</increment-step> + </value-attributes> + <depends-on> + <property> + <type>hive-interactive-env</type> + <name>num_llap_nodes</name> + </property> + <property> + <type>hive-interactive-env</type> + <name>enable_hive_interactive</name> + </property> + <property> + <type>hive-interactive-site</type> + <name>hive.llap.daemon.queue.name</name> + </property> + <property> + <type>capacity-scheduler</type> + <name>yarn.scheduler.capacity.root.queues</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + + + <!-- hive-env.sh --> + <property> + <name>content</name> + <display-name>hive-interactive-env template</display-name> + <description>This is the jinja template for hive-env.sh file</description> + <value> + if [ "$SERVICE" = "cli" ]; then + if [ -z "$DEBUG" ]; then + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit" + else + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" + fi + fi + + # The heap size of the jvm stared by hive shell script can be controlled via: + + if [ "$SERVICE" = "metastore" ]; then + export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore + else + export HADOOP_HEAPSIZE={{hive_interactive_heapsize}} # Setting for HiveServer2 and Client + fi + + export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m" + export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS{{heap_dump_opts}}" + + # Larger heap size may be required when running queries over large number of files or partitions. + # By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be + # appropriate for hive server (hwi etc). + + + # Set HADOOP_HOME to point to a specific hadoop install directory + HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + + # Hive Configuration Directory can be controlled by: + export HIVE_CONF_DIR={{hive_server_interactive_conf_dir}} + + # Add additional hcatalog jars + if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then + export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} + else + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-server2-hive2/lib/hive-hcatalog-core.jar + fi + + export METASTORE_PORT={{hive_metastore_port}} + + # Spark assembly contains a conflicting copy of HiveConf from hive-1.2 + export HIVE_SKIP_SPARK_ASSEMBLY=true + + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration>
