AMBARI-21445. Fixes the following bugs : (1). Make Hive Kerberos keytab files group non-readable (2). HiveServer2 Authentication via LDAP to work correctly (3). Remove leading while spaces for the hive-env and hive-interactive-env template.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/eb3d3ea6 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/eb3d3ea6 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/eb3d3ea6 Branch: refs/heads/branch-feature-AMBARI-14714 Commit: eb3d3ea6e5eb9464a135f851658d4aa5b3988efa Parents: 9f788c3 Author: Swapan Shridhar <sshrid...@hortonworks.com> Authored: Tue Jul 11 15:37:08 2017 -0700 Committer: Swapan Shridhar <sshrid...@hortonworks.com> Committed: Wed Jul 12 11:55:44 2017 -0700 ---------------------------------------------------------------------- .../0.12.0.2.0/package/scripts/params_linux.py | 4 + .../0.12.0.2.0/package/scripts/service_check.py | 3 +- .../services/HIVE/configuration/hive-env.xml | 78 +++++----- .../HIVE/configuration/hive-interactive-env.xml | 62 ++++---- .../stacks/HDP/2.6/services/HIVE/kerberos.json | 151 +++++++++++++++++++ .../stacks/HDP/2.6/services/YARN/kerberos.json | 2 +- 6 files changed, 228 insertions(+), 72 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/eb3d3ea6/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 21b3d8b..9939536 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -849,3 +849,7 @@ ranger_hive_metastore_lookup = default('/configurations/ranger-hive-plugin-prope if security_enabled: hive_metastore_principal_with_host = hive_metastore_principal.replace('_HOST', hostname.lower()) + +# For ldap - hive_check +hive_ldap_user= config['configurations']['hive-env'].get('alert_ldap_username','') +hive_ldap_passwd=config['configurations']['hive-env'].get('alert_ldap_password','') \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/eb3d3ea6/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py index d144c34..271fff9 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py @@ -123,7 +123,8 @@ class HiveServiceCheckDefault(HiveServiceCheck): params.hive_server_principal, kinit_cmd, params.smokeuser, transport_mode=params.hive_transport_mode, http_endpoint=params.hive_http_endpoint, ssl=params.hive_ssl, ssl_keystore=ssl_keystore, - ssl_password=ssl_password) + ssl_password=ssl_password, ldap_username=params.hive_ldap_user, + ldap_password=params.hive_ldap_passwd) Logger.info("Successfully connected to {0} on port {1}".format(address, server_port)) workable_server_available = True except: http://git-wip-us.apache.org/repos/asf/ambari/blob/eb3d3ea6/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-env.xml index a6cf1bc..929c10d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-env.xml @@ -60,56 +60,56 @@ <display-name>hive-env template</display-name> <description>This is the jinja template for hive-env.sh file</description> <value> - export HADOOP_USER_CLASSPATH_FIRST=true #this prevents old metrics libs from mapreduce lib from bringing in old jar deps overriding HIVE_LIB - if [ "$SERVICE" = "cli" ]; then - if [ -z "$DEBUG" ]; then - export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseNUMA -XX:+UseParallelGC -XX:-UseGCOverheadLimit" - else - export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" - fi - fi +export HADOOP_USER_CLASSPATH_FIRST=true #this prevents old metrics libs from mapreduce lib from bringing in old jar deps overriding HIVE_LIB +if [ "$SERVICE" = "cli" ]; then + if [ -z "$DEBUG" ]; then + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseNUMA -XX:+UseParallelGC -XX:-UseGCOverheadLimit" + else + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" + fi +fi - # The heap size of the jvm stared by hive shell script can be controlled via: +# The heap size of the jvm stared by hive shell script can be controlled via: - if [ "$SERVICE" = "metastore" ]; then - export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore - else - export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client - fi +if [ "$SERVICE" = "metastore" ]; then + export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore +else + export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client +fi - export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m" - export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS{{heap_dump_opts}}" +export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m" +export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS{{heap_dump_opts}}" - # Larger heap size may be required when running queries over large number of files or partitions. - # By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be - # appropriate for hive server (hwi etc). +# Larger heap size may be required when running queries over large number of files or partitions. +# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be +# appropriate for hive server (hwi etc). - # Set HADOOP_HOME to point to a specific hadoop install directory - HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} +# Set HADOOP_HOME to point to a specific hadoop install directory +HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} - export HIVE_HOME=${HIVE_HOME:-{{hive_home_dir}}} +export HIVE_HOME=${HIVE_HOME:-{{hive_home_dir}}} - # Hive Configuration Directory can be controlled by: - export HIVE_CONF_DIR=${HIVE_CONF_DIR:-{{hive_config_dir}}} +# Hive Configuration Directory can be controlled by: +export HIVE_CONF_DIR=${HIVE_CONF_DIR:-{{hive_config_dir}}} - # Folder containing extra libraries required for hive compilation/execution can be controlled by: - if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then - if [ -f "${HIVE_AUX_JARS_PATH}" ]; then - export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} - elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then - export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar - fi - elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then - export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar - fi +# Folder containing extra libraries required for hive compilation/execution can be controlled by: +if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then + if [ -f "${HIVE_AUX_JARS_PATH}" ]; then + export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} + elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar + fi +elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar +fi - export METASTORE_PORT={{hive_metastore_port}} +export METASTORE_PORT={{hive_metastore_port}} - {% if sqla_db_used or lib_dir_available %} - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}" - export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}" - {% endif %} +{% if sqla_db_used or lib_dir_available %} +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}" +export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}" +{% endif %} </value> <value-attributes> <type>content</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/eb3d3ea6/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml index ada4859..86720f4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml @@ -100,47 +100,47 @@ <display-name>hive-interactive-env template</display-name> <description>This is the jinja template for hive-env.sh file</description> <value> - if [ "$SERVICE" = "cli" ]; then - if [ -z "$DEBUG" ]; then - export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit" - else - export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" - fi - fi +if [ "$SERVICE" = "cli" ]; then + if [ -z "$DEBUG" ]; then + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit" + else + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" + fi +fi - # The heap size of the jvm stared by hive shell script can be controlled via: +# The heap size of the jvm stared by hive shell script can be controlled via: - if [ "$SERVICE" = "metastore" ]; then - export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore - else - export HADOOP_HEAPSIZE={{hive_interactive_heapsize}} # Setting for HiveServer2 and Client - fi +if [ "$SERVICE" = "metastore" ]; then + export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore +else + export HADOOP_HEAPSIZE={{hive_interactive_heapsize}} # Setting for HiveServer2 and Client +fi - export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m" - export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS{{heap_dump_opts}}" +export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m" +export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS{{heap_dump_opts}}" - # Larger heap size may be required when running queries over large number of files or partitions. - # By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be - # appropriate for hive server (hwi etc). +# Larger heap size may be required when running queries over large number of files or partitions. +# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be +# appropriate for hive server (hwi etc). - # Set HADOOP_HOME to point to a specific hadoop install directory - HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} +# Set HADOOP_HOME to point to a specific hadoop install directory +HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} - # Hive Configuration Directory can be controlled by: - export HIVE_CONF_DIR={{hive_server_interactive_conf_dir}} +# Hive Configuration Directory can be controlled by: +export HIVE_CONF_DIR={{hive_server_interactive_conf_dir}} - # Add additional hcatalog jars - if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then - export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} - else - export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-server2-hive2/lib/hive-hcatalog-core.jar - fi +# Add additional hcatalog jars +if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then + export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} +else + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-server2-hive2/lib/hive-hcatalog-core.jar +fi - export METASTORE_PORT={{hive_metastore_port}} +export METASTORE_PORT={{hive_metastore_port}} - # Spark assembly contains a conflicting copy of HiveConf from hive-1.2 - export HIVE_SKIP_SPARK_ASSEMBLY=true +# Spark assembly contains a conflicting copy of HiveConf from hive-1.2 +export HIVE_SKIP_SPARK_ASSEMBLY=true </value> <value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/eb3d3ea6/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/kerberos.json new file mode 100644 index 0000000..b6e57e1 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/kerberos.json @@ -0,0 +1,151 @@ +{ + "services": [ + { + "name": "HIVE", + "identities": [ + { + "name": "/spnego" + }, + { + "name": "/smokeuser" + } + ], + "configurations": [ + { + "hive-site": { + "hive.metastore.sasl.enabled": "true", + "hive.server2.authentication": "KERBEROS" + } + }, + { + "ranger-hive-audit": { + "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule", + "xasecure.audit.jaas.Client.loginModuleControlFlag": "required", + "xasecure.audit.jaas.Client.option.useKeyTab": "true", + "xasecure.audit.jaas.Client.option.storeKey": "false", + "xasecure.audit.jaas.Client.option.serviceName": "solr", + "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true" + } + } + ], + "components": [ + { + "name": "HIVE_METASTORE", + "identities": [ + { + "name": "/HIVE/HIVE_SERVER/hive_server_hive", + "principal": { + "configuration": "hive-site/hive.metastore.kerberos.principal" + }, + "keytab": { + "configuration": "hive-site/hive.metastore.kerberos.keytab.file" + } + } + ] + }, + { + "name": "HIVE_SERVER", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "hive_server_hive", + "principal": { + "value": "hive/_HOST@${realm}", + "type": "service", + "configuration": "hive-site/hive.server2.authentication.kerberos.principal", + "local_username": "${hive-env/hive_user}" + }, + "keytab": { + "file": "${keytab_dir}/hive.service.keytab", + "owner": { + "name": "${hive-env/hive_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hive-site/hive.server2.authentication.kerberos.keytab" + } + }, + { + "name": "atlas_kafka", + "reference": "/HIVE/HIVE_SERVER/hive_server_hive", + "principal": { + "configuration": "hive-atlas-application.properties/atlas.jaas.KafkaClient.option.principal" + }, + "keytab": { + "configuration": "hive-atlas-application.properties/atlas.jaas.KafkaClient.option.keyTab" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "hive-site/hive.server2.authentication.spnego.principal" + }, + "keytab": { + "configuration": "hive-site/hive.server2.authentication.spnego.keytab" + } + }, + { + "name": "ranger_audit", + "reference": "/HIVE/HIVE_SERVER/hive_server_hive", + "principal": { + "configuration": "ranger-hive-audit/xasecure.audit.jaas.Client.option.principal" + }, + "keytab": { + "configuration": "ranger-hive-audit/xasecure.audit.jaas.Client.option.keyTab" + } + } + ] + }, + { + "name": "HIVE_SERVER_INTERACTIVE", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "/HIVE/HIVE_SERVER/hive_server_hive" + }, + { + "name": "/HIVE/HIVE_SERVER/spnego" + }, + { + "name": "/YARN/NODEMANAGER/llap_zk_hive" + } + ] + }, + { + "name": "WEBHCAT_SERVER", + "identities": [ + { + "name": "/spnego", + "principal": { + "configuration": "webhcat-site/templeton.kerberos.principal" + }, + "keytab": { + "configuration": "webhcat-site/templeton.kerberos.keytab" + } + } + ], + "configurations": [ + { + "core-site": { + "hadoop.proxyuser.HTTP.hosts": "${clusterHostInfo/webhcat_server_host|append(core-site/hadoop.proxyuser.HTTP.hosts, \\\\,, true)}" + } + }, + { + "webhcat-site": { + "templeton.kerberos.secret": "secret", + "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}" + } + } + ] + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/eb3d3ea6/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json index b1501b8..60d50eb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json @@ -117,7 +117,7 @@ }, "group": { "name": "${cluster-env/user_group}", - "access": "r" + "access": "" }, "configuration": "hive-interactive-site/hive.llap.zk.sm.keytab.file" },
