This is an automated email from the ASF dual-hosted git repository. rmani pushed a commit to branch RANGER-5413-Patch in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 635f76eb5706b3de9ff18b804d5569a5a3eedb90 Author: Ramesh Mani <[email protected]> AuthorDate: Sun Nov 30 18:01:52 2025 -0800 RANGER-5413:Beeline failing in Hive Docker setup due to hive failure to connect to YARN resource manager --- dev-support/ranger-docker/Dockerfile.ranger-hive | 2 + .../scripts/hive/hive-log4j2.properties | 97 ++++++++++++++++++++++ .../scripts/hive/hive-metastore-log4j2.properties | 97 ++++++++++++++++++++++ .../scripts/hive/ranger-hive-setup.sh | 6 ++ .../ranger-docker/scripts/hive/ranger-hive.sh | 4 +- 5 files changed, 204 insertions(+), 2 deletions(-) diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hive b/dev-support/ranger-docker/Dockerfile.ranger-hive index e0bcdea3a..b4798cec3 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hive +++ b/dev-support/ranger-docker/Dockerfile.ranger-hive @@ -40,6 +40,8 @@ COPY ./scripts/hive/ranger-hive-setup.sh ${RANGER_SCRIPTS}/ COPY ./scripts/hive/ranger-hive.sh ${RANGER_SCRIPTS}/ COPY ./scripts/hive/ranger-hive-plugin-install.properties ${RANGER_SCRIPTS}/ COPY ./scripts/hive/hive-site-${RANGER_DB_TYPE}.xml ${RANGER_SCRIPTS}/hive-site.xml +COPY ./scripts/hive/hive-log4j2.properties ${RANGER_SCRIPTS}/ +COPY ./scripts/hive/hive-metastore-log4j2.properties ${RANGER_SCRIPTS}/ COPY ./scripts/hadoop/core-site.xml ${RANGER_SCRIPTS}/ COPY ./scripts/wait_for_keytab.sh ${RANGER_SCRIPTS}/ COPY ./scripts/kdc/krb5.conf /etc/krb5.conf diff --git a/dev-support/ranger-docker/scripts/hive/hive-log4j2.properties b/dev-support/ranger-docker/scripts/hive/hive-log4j2.properties new file mode 100644 index 000000000..09b0d1b9c --- /dev/null +++ b/dev-support/ranger-docker/scripts/hive/hive-log4j2.properties @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = HiveLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +# Change to DEBUG for detailed troubleshooting +property.hive.log.level = DEBUG +property.hive.root.logger = DRFA +property.hive.log.dir = /opt/hive/logs +property.hive.log.file = hiveserver2.log +property.hive.perflogger.log.level = INFO + +# list of all appenders +appenders = console, DRFA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n + +# daily rolling file appender +appender.DRFA.type = RollingRandomAccessFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${hive.log.dir}/${hive.log.file} +# Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session +appender.DRFA.filePattern = ${hive.log.dir}/${hive.log.file}.%d{yyyy-MM-dd} +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = 30 + +# list of all loggers +loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, PerfLogger, AmazonAws, ApacheHttp, RangerAuth + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +logger.AmazonAws.name=com.amazonaws +logger.AmazonAws.level = INFO + +logger.ApacheHttp.name=org.apache.http +logger.ApacheHttp.level = INFO + +logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger +logger.PerfLogger.level = ${hive.perflogger.log.level} + +# Ranger authorization logger +logger.RangerAuth.name = org.apache.ranger +logger.RangerAuth.level = DEBUG + +# +# Hive authentication +# logger.HiveAuth.name = org.apache.hadoop.hive.ql.security +# logger.HiveAuth.level = DEBUG +# +# Security and Kerberos +# logger.SecurityAuth.name = org.apache.hadoop.security +# logger.SecurityAuth.level = DEBUG + +# root logger (controls all packages unless overridden above) +rootLogger.level = ${hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${hive.root.logger} diff --git a/dev-support/ranger-docker/scripts/hive/hive-metastore-log4j2.properties b/dev-support/ranger-docker/scripts/hive/hive-metastore-log4j2.properties new file mode 100644 index 000000000..88a8a4925 --- /dev/null +++ b/dev-support/ranger-docker/scripts/hive/hive-metastore-log4j2.properties @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = HiveMetastoreLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +# Change to DEBUG for detailed troubleshooting +property.hive.log.level = INFO +property.hive.root.logger = DRFA +property.hive.log.dir = /opt/hive/logs +property.hive.log.file = metastore.log +property.hive.perflogger.log.level = INFO + +# list of all appenders +appenders = console, DRFA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n + +# daily rolling file appender +appender.DRFA.type = RollingRandomAccessFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${hive.log.dir}/${hive.log.file} +# Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session +appender.DRFA.filePattern = ${hive.log.dir}/${hive.log.file}.%d{yyyy-MM-dd} +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = 30 + +# list of all loggers +loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, PerfLogger, AmazonAws, ApacheHttp, RangerAuth + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +logger.AmazonAws.name=com.amazonaws +logger.AmazonAws.level = INFO + +logger.ApacheHttp.name=org.apache.http +logger.ApacheHttp.level = INFO + +logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger +logger.PerfLogger.level = ${hive.perflogger.log.level} + +# Ranger authorization logger +logger.RangerAuth.name = org.apache.ranger +logger.RangerAuth.level = INFO +# +# Hive authentication +# logger.HiveAuth.name = org.apache.hadoop.hive.ql.security +# logger.HiveAuth.level = INFO +# +# Security and Kerberos +# logger.SecurityAuth.name = org.apache.hadoop.security +# logger.SecurityAuth.level = INFO + +# root logger (controls all packages unless overridden above) +rootLogger.level = ${hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${hive.root.logger} + diff --git a/dev-support/ranger-docker/scripts/hive/ranger-hive-setup.sh b/dev-support/ranger-docker/scripts/hive/ranger-hive-setup.sh index 591e2182b..bacf00400 100755 --- a/dev-support/ranger-docker/scripts/hive/ranger-hive-setup.sh +++ b/dev-support/ranger-docker/scripts/hive/ranger-hive-setup.sh @@ -31,6 +31,8 @@ fi cp ${RANGER_SCRIPTS}/hive-site.xml ${HIVE_HOME}/conf/hive-site.xml cp ${RANGER_SCRIPTS}/hive-site.xml ${HIVE_HOME}/conf/hiveserver2-site.xml +cp ${RANGER_SCRIPTS}/hive-log4j2.properties ${HIVE_HOME}/conf/hive-log4j2.properties +cp ${RANGER_SCRIPTS}/hive-metastore-log4j2.properties ${HIVE_HOME}/conf/hive-metastore-log4j2.properties # Configure Tez mkdir -p ${TEZ_HOME}/conf @@ -77,6 +79,10 @@ cat <<EOF > ${HADOOP_HOME}/etc/hadoop/yarn-site.xml <name>yarn.resourcemanager.address</name> <value>ranger-hadoop:8032</value> </property> + <property> + <name>yarn.resourcemanager.principal</name> + <value>rm/[email protected]</value> + </property> </configuration> EOF diff --git a/dev-support/ranger-docker/scripts/hive/ranger-hive.sh b/dev-support/ranger-docker/scripts/hive/ranger-hive.sh index ee380b13f..3606f1bb2 100755 --- a/dev-support/ranger-docker/scripts/hive/ranger-hive.sh +++ b/dev-support/ranger-docker/scripts/hive/ranger-hive.sh @@ -91,11 +91,11 @@ cd "${HIVE_HOME}" || exit # Start Hive MetaStore echo "Starting Hive MetaStore..." -su -c "nohup ${HIVE_HOME}/bin/hive --service metastore > metastore.log 2>&1 &" hive +su -c "export HADOOP_CLIENT_OPTS='${HADOOP_CLIENT_OPTS} -Dlog4j2.configurationFile=file:${HIVE_HOME}/conf/hive-metastore-log4j2.properties' && nohup ${HIVE_HOME}/bin/hive --service metastore &" hive # Start HiveServer2 echo "Starting HiveServer2..." -su -c "nohup ${HIVE_HOME}/bin/hiveserver2 > hive-server2.log 2>&1 &" hive +su -c "export HADOOP_CLIENT_OPTS='${HADOOP_CLIENT_OPTS} -Dlog4j2.configurationFile=file:${HIVE_HOME}/conf/hive-log4j2.properties' && nohup ${HIVE_HOME}/bin/hiveserver2 &" hive # Wait for services to initialize echo "Waiting for Hive services to initialize..."
