This is an automated email from the ASF dual-hosted git repository. nixonrodrigues pushed a commit to branch ATLAS-5409 in repository https://gitbox.apache.org/repos/asf/atlas.git
commit d5e3aff203b9a72f53841018358c7331834a62ee Author: nixonrodrigues <[email protected]> AuthorDate: Thu Sep 17 16:44:22 2026 +0530 ATLAS-5408: Fix hook import classpath and logging in docker/minimal envs HBase: ship commons-configuration 1.x in atlas-hbase-plugin-impl because import-hbase.sh excludes it from HBase lib while Hadoop still needs it when HADOOP_HOME/classpath is absent. Logging: create /var/log/atlas in atlas-base and mkdir -p ATLAS_LOG_DIR in import-hbase, import-hive, and import-kafka so logback can open import logs. Kafka: stop putting broker config and log4j jars on the import classpath to avoid broker log4j.properties writing to /server.log; default KAFKA_CONF to $KAFKA_HOME/config. Co-authored-by: Cursor <[email protected]> --- addons/hbase-bridge/pom.xml | 6 ++++++ addons/hbase-bridge/src/bin/import-hbase.sh | 2 ++ addons/hive-bridge/src/bin/import-hive.sh | 1 + addons/kafka-bridge/src/bin/import-kafka.sh | 9 ++++++--- dev-support/atlas-docker/Dockerfile.atlas-base | 3 +++ 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/addons/hbase-bridge/pom.xml b/addons/hbase-bridge/pom.xml index 76ea8dbc1..0387bc991 100644 --- a/addons/hbase-bridge/pom.xml +++ b/addons/hbase-bridge/pom.xml @@ -582,6 +582,12 @@ <artifactId>commons-configuration2</artifactId> <version>${commons-conf2.version}</version> </artifactItem> + <!-- Hadoop metrics use commons-configuration 1.x; excluded from HBASE_CP in import-hbase.sh --> + <artifactItem> + <groupId>commons-configuration</groupId> + <artifactId>commons-configuration</artifactId> + <version>1.10</version> + </artifactItem> <artifactItem> <groupId>org.apache.commons</groupId> <artifactId>commons-text</artifactId> diff --git a/addons/hbase-bridge/src/bin/import-hbase.sh b/addons/hbase-bridge/src/bin/import-hbase.sh index c92ff1907..f05bc8a8a 100644 --- a/addons/hbase-bridge/src/bin/import-hbase.sh +++ b/addons/hbase-bridge/src/bin/import-hbase.sh @@ -68,6 +68,7 @@ fi # log dir for applications ATLAS_LOG_DIR="${ATLAS_LOG_DIR:-/var/log/atlas}" export ATLAS_LOG_DIR +mkdir -p "${ATLAS_LOG_DIR}" LOGFILE="$ATLAS_LOG_DIR/import-hbase.log" TIME=`date +%Y%m%d%H%M%s` @@ -104,6 +105,7 @@ HBASE_CP="${HBASE_CONF}" # Multiple jars in HBASE_CP_EXCLUDE_LIST can be added using "\|" separator # Ex: HBASE_CP_EXCLUDE_LIST="commons-configuration-1." +# commons-configuration 1.x is in atlas-hbase-plugin-impl (Hadoop needs it; Atlas uses configuration2). HBASE_CP_EXCLUDE_LIST="commons-configuration-1\|jersey-client" for i in "${HBASE_HOME}/lib/"*.jar "${HBASE_HOME}/lib/client-facing-thirdparty/"*.jar; do diff --git a/addons/hive-bridge/src/bin/import-hive.sh b/addons/hive-bridge/src/bin/import-hive.sh index eca58fab8..f230d65dd 100755 --- a/addons/hive-bridge/src/bin/import-hive.sh +++ b/addons/hive-bridge/src/bin/import-hive.sh @@ -63,6 +63,7 @@ fi # log dir for applications ATLAS_LOG_DIR="${ATLAS_LOG_DIR:-/var/log/atlas}" export ATLAS_LOG_DIR +mkdir -p "${ATLAS_LOG_DIR}" LOGFILE="$ATLAS_LOG_DIR/import-hive.log" TIME=`date +%Y%m%d%H%M%s` diff --git a/addons/kafka-bridge/src/bin/import-kafka.sh b/addons/kafka-bridge/src/bin/import-kafka.sh index bcf22ebc1..a4f6389e8 100644 --- a/addons/kafka-bridge/src/bin/import-kafka.sh +++ b/addons/kafka-bridge/src/bin/import-kafka.sh @@ -70,6 +70,7 @@ fi # log dir for applications ATLAS_LOG_DIR="${ATLAS_LOG_DIR:-/var/log/atlas}" export ATLAS_LOG_DIR +mkdir -p "${ATLAS_LOG_DIR}" LOGFILE="$ATLAS_LOG_DIR/import-kafka.log" TIME=`date +%Y%m%d%H%M%s` @@ -78,7 +79,7 @@ TIME=`date +%Y%m%d%H%M%s` if [ ! -z "$KAFKA_CONF_DIR" ]; then KAFKA_CONF=$KAFKA_CONF_DIR elif [ ! -z "$KAFKA_HOME" ]; then - KAFKA_CONF="$KAFKA_HOME/conf" + KAFKA_CONF="$KAFKA_HOME/config" elif [ -e /etc/kafka/conf ]; then KAFKA_CONF="/etc/kafka/conf" else @@ -102,9 +103,11 @@ if [ -z "$KAFKA_HOME" ]; then fi fi -KAFKA_CP="${KAFKA_CONF}" +# Do not add ${KAFKA_CONF} to the classpath: broker log4j.properties is picked up +# automatically and writes to ${kafka.logs.dir}/server.log (i.e. /server.log when unset). +KAFKA_CP="" -for i in "${KAFKA_HOME}/libs/kafka-clients"*.jar "${KAFKA_HOME}/libs/slf4j-"*.jar "${KAFKA_HOME}/libs/log4j-"*.jar "${KAFKA_HOME}/libs/commons-"*.jar "${KAFKA_HOME}/libs/jackson-module-jaxb-annotations"*.jar; do +for i in "${KAFKA_HOME}/libs/kafka-clients"*.jar "${KAFKA_HOME}/libs/slf4j-"*.jar "${KAFKA_HOME}/libs/commons-"*.jar "${KAFKA_HOME}/libs/jackson-module-jaxb-annotations"*.jar; do KAFKA_CP="${KAFKA_CP}:$i" done diff --git a/dev-support/atlas-docker/Dockerfile.atlas-base b/dev-support/atlas-docker/Dockerfile.atlas-base index a7af42844..1c5aece9a 100644 --- a/dev-support/atlas-docker/Dockerfile.atlas-base +++ b/dev-support/atlas-docker/Dockerfile.atlas-base @@ -45,6 +45,9 @@ RUN groupadd atlas && \ useradd -g hadoop -ms /bin/bash kafka && \ mkdir -p /home/atlas/dist && \ mkdir -p /home/atlas/scripts && \ + mkdir -p /var/log/atlas && \ + chown root:hadoop /var/log/atlas && \ + chmod 775 /var/log/atlas && \ chown -R atlas:atlas /home/atlas ENTRYPOINT [ "/bin/bash" ]
