Repository: ambari Updated Branches: refs/heads/branch-1.7.0 0e0e51fba -> 1d974624b
AMBARI-7739. hive command line fails to start after setting hive.execution.engine to tez in HDP 2.2.1 (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1d974624 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1d974624 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1d974624 Branch: refs/heads/branch-1.7.0 Commit: 1d974624b7d282073e0d14f5d13bc9d0e90de04f Parents: 0e0e51f Author: Alejandro Fernandez <[email protected]> Authored: Mon Oct 13 11:13:08 2014 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Mon Oct 13 11:13:08 2014 -0700 ---------------------------------------------------------------------- .../HDP/2.1/services/TEZ/package/scripts/tez.py | 1 + .../services/HDFS/configuration/hadoop-env.xml | 13 ++-- .../services/HIVE/configuration/hive-env.xml | 64 ++++++++++++++++++++ .../services/HIVE/configuration/hive-site.xml | 6 -- .../2.2/services/TEZ/configuration/tez-site.xml | 7 ++- 5 files changed, 77 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1d974624/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/tez.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/tez.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/tez.py index 62e3cd6..fe4324c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/tez.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/tez.py @@ -30,6 +30,7 @@ def tez(): recursive = True ) + # TODO, tez-site.xml needs to use versioned tez.tar.gz file to allow currently running jobs to finish during a rolling upgrade. XmlConfig( "tez-site.xml", conf_dir = params.config_dir, configurations = params.config['configurations']['tez-site'], http://git-wip-us.apache.org/repos/asf/ambari/blob/1d974624/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml index 4d0e68f..984897f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml @@ -165,24 +165,25 @@ for jarFile in `ls /usr/share/java/*mysql* 2>/dev/null` do JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile done -#Add libraries required by oracle connector +# Add libraries required by oracle connector for jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null` do JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile done -#Add libraries required by nodemanager +# Add libraries required by nodemanager MAPREDUCE_LIBS={{mapreduce_libs_path}} export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} -if [ -d "/usr/lib/tez" ]; then - export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/tez/*:/usr/lib/tez/lib/*:/etc/tez/conf +if [[ -d "/usr/hdp/current/tez-client" && -d "/etc/tez/conf/" ]]; then + # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP. + export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf/ fi # Setting path to hdfs command line export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} -#Mostly required for hadoop 2.0 -export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop/lib/native/Linux-amd64-64 +# Mostly required for hadoop 2.0 +export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/hdp/current/hadoop-client/lib/native/Linux-amd64-64 #Hadoop logging options export HADOOP_ROOT_LOGGER={{hadoop_root_logger}} http://git-wip-us.apache.org/repos/asf/ambari/blob/1d974624/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml new file mode 100644 index 0000000..0eb4ef2 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml @@ -0,0 +1,64 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <!-- hive-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for hive-env.sh file</description> + <value> + if [ "$SERVICE" = "cli" ]; then + if [ -z "$DEBUG" ]; then + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit" + else + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" + fi + fi + +# The heap size of the jvm stared by hive shell script can be controlled via: + +export HADOOP_HEAPSIZE="{{hive_heapsize}}" +export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS" + +# Larger heap size may be required when running queries over large number of files or partitions. +# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be +# appropriate for hive server (hwi etc). + + +# Set HADOOP_HOME to point to a specific hadoop install directory +HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + +# Hive Configuration Directory can be controlled by: +export HIVE_CONF_DIR={{hive_config_dir}} + +# Folder containing extra libraries required for hive compilation/execution can be controlled by: +if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then + export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} +elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog +fi + +export METASTORE_PORT={{hive_metastore_port}} + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/1d974624/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml index 8fd22b0..43a470b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml @@ -27,12 +27,6 @@ limitations under the License. </property> <property> - <name>hive.aux.jars.path</name> - <value>file:///usr/hdp/current/hive-hcatalog/share/hcatalog</value> - <description>Comma-separated list of the location of the plugin jars that contain implementations of user defined functions and SerDes.</description> - </property> - - <property> <name>hive.cluster.delegation.token.store.class</name> <value>org.apache.hadoop.hive.thrift.ZooKeeperTokenStore</value> <description>The delegation token store implementation class.</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/1d974624/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml index dfedb6b..f587346 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml @@ -21,8 +21,11 @@ <property> <name>tez.lib.uris</name> - <value>hdfs:///apps/tez/,hdfs:///apps/tez/lib/,hdfs:///apps/tez/tez.tar.gz</value> - <description>The location of the Tez libraries which will be localized for DAGs</description> + <value>hdfs:///apps/tez/tez.tar.gz</value> <!-- TODO, this needs to be a versioned file to allow currently running jobs to finish during a rolling upgrade. --> + <description>Comma-delimited list of the location of the Tez libraries which will be localized for DAGs. + Specifying a single .tar.gz or .tgz assumes that a compressed version of the tez libs is being used. This is uncompressed into a tezlibs directory when running containers, and tezlibs/;tezlibs/lib/ are added to the classpath (after . and .*). + If multiple files are specified - files are localized as regular files, contents of directories are localized as regular files (non-recursive). + </description> </property> </configuration>
