This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new a71c511 [KYUUBI #2081] YARN_CONF_DIR shall be added to kyuubi server
classpath as HADOOP_CONF_DIR
a71c511 is described below
commit a71c511ba36ba496659cdb6983937ac2c2a681bf
Author: SteNicholas <[email protected]>
AuthorDate: Thu Mar 10 15:03:11 2022 +0800
[KYUUBI #2081] YARN_CONF_DIR shall be added to kyuubi server classpath as
HADOOP_CONF_DIR
### _Why are the changes needed?_
YARN_CONF_DIR shall be added to kyuubi server classpath as HADOOP_CONF_DIR.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #2082 from SteNicholas/KYUUBI-2081.
Closes #2081
c6f76c5c [SteNicholas] [KYUUBI #2081] YARN_CONF_DIR shall be added to
kyuubi server classpath as HADOOP_CONF_DIR
fee5c21f [SteNicholas] [KYUUBI #2081] YARN_CONF_DIR shall be added to
kyuubi server classpath as HADOOP_CONF_DIR
Authored-by: SteNicholas <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
bin/beeline | 6 +++++-
bin/kyuubi | 7 ++++++-
bin/kyuubi-ctl | 6 +++++-
bin/kyuubi-zk-cli | 6 +++++-
bin/load-kyuubi-env.sh | 2 ++
conf/kyuubi-env.sh.template | 2 ++
docs/deployment/settings.md | 2 ++
docs/integrations/delta_lake_with_azure_blob.md | 1 +
docs/monitor/logging.md | 1 +
docs/quick_start/quick_start.md | 1 +
10 files changed, 30 insertions(+), 4 deletions(-)
diff --git a/bin/beeline b/bin/beeline
index 2dd7259..91e6654 100755
--- a/bin/beeline
+++ b/bin/beeline
@@ -39,6 +39,10 @@ if [[ -z "$KYUUBI_BEELINE_JARS" ]]; then
fi
fi
-KYUUBI_BEELINE_CLASSPATH="${KYUUBI_BEELINE_JARS}/*:${HADOOP_CONF_DIR}"
+if [[ -z ${YARN_CONF_DIR} ]]; then
+ KYUUBI_BEELINE_CLASSPATH="${KYUUBI_BEELINE_JARS}/*:${HADOOP_CONF_DIR}"
+else
+
KYUUBI_BEELINE_CLASSPATH="${KYUUBI_BEELINE_JARS}/*:${HADOOP_CONF_DIR}:${YARN_CONF_DIR}"
+fi
exec ${RUNNER} ${KYUUBI_BEELINE_OPTS} -cp ${KYUUBI_BEELINE_CLASSPATH} $CLASS
"$@"
diff --git a/bin/kyuubi b/bin/kyuubi
index edd5465..0f1789e 100755
--- a/bin/kyuubi
+++ b/bin/kyuubi
@@ -85,7 +85,12 @@ if [[ -z "$KYUUBI_JAR_DIR" ]]; then
fi
fi
-KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}"
+if [[ -z ${YARN_CONF_DIR} ]]; then
+ KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}"
+else
+
KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}:${YARN_CONF_DIR}"
+fi
+
cmd="${RUNNER} ${KYUUBI_JAVA_OPTS} -cp ${KYUUBI_CLASSPATH} $CLASS"
pid="${KYUUBI_PID_DIR}/kyuubi-$USER-$CLASS.pid"
diff --git a/bin/kyuubi-ctl b/bin/kyuubi-ctl
index f6c3be6..3ff03dd 100755
--- a/bin/kyuubi-ctl
+++ b/bin/kyuubi-ctl
@@ -39,6 +39,10 @@ if [[ -z "$KYUUBI_JAR_DIR" ]]; then
fi
fi
-KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}"
+if [[ -z ${YARN_CONF_DIR} ]]; then
+ KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}"
+else
+
KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}:${YARN_CONF_DIR}"
+fi
exec ${RUNNER} ${KYUUBI_CTL_JAVA_OPTS} -cp ${KYUUBI_CLASSPATH} $CLASS "$@"
diff --git a/bin/kyuubi-zk-cli b/bin/kyuubi-zk-cli
index 369a45d..089b7ad 100755
--- a/bin/kyuubi-zk-cli
+++ b/bin/kyuubi-zk-cli
@@ -39,6 +39,10 @@ if [[ -z "$KYUUBI_JAR_DIR" ]]; then
fi
fi
-KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}"
+if [[ -z ${YARN_CONF_DIR} ]]; then
+ KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}"
+else
+
KYUUBI_CLASSPATH="${KYUUBI_JAR_DIR}/*:${KYUUBI_CONF_DIR}:${HADOOP_CONF_DIR}:${YARN_CONF_DIR}"
+fi
exec ${RUNNER} ${KYUUBI_JAVA_OPTS} -cp ${KYUUBI_CLASSPATH} $CLASS "$@"
\ No newline at end of file
diff --git a/bin/load-kyuubi-env.sh b/bin/load-kyuubi-env.sh
index 7d65cca..087eb21 100755
--- a/bin/load-kyuubi-env.sh
+++ b/bin/load-kyuubi-env.sh
@@ -102,4 +102,6 @@ if [ $silent -eq 0 ]; then
echo "TRINO_ENGINE_HOME: ${TRINO_ENGINE_HOME}"
echo "HADOOP_CONF_DIR: ${HADOOP_CONF_DIR}"
+
+ echo "YARN_CONF_DIR: ${YARN_CONF_DIR}"
fi
diff --git a/conf/kyuubi-env.sh.template b/conf/kyuubi-env.sh.template
index 969d47f..4afa70d 100755
--- a/conf/kyuubi-env.sh.template
+++ b/conf/kyuubi-env.sh.template
@@ -38,6 +38,7 @@
# - KYUUBI_WORK_DIR_ROOT Root directory for launching sql engine
applications.
# (Default: $KYUUBI_HOME/work)
# - HADOOP_CONF_DIR Directory containing the Hadoop / YARN
configuration to use.
+# - YARN_CONF_DIR Directory containing the YARN configuration to use.
#
# - SPARK_HOME Spark distribution which you would like to use in
Kyuubi.
# - SPARK_CONF_DIR Optional directory where the Spark configuration
lives.
@@ -50,5 +51,6 @@
# export JAVA_HOME=/usr/jdk64/jdk1.8.0_152
# export SPARK_HOME=/opt/spark
# export HADOOP_CONF_DIR=/usr/ndp/current/mapreduce_client/conf
+# export YARN_CONF_DIR=/usr/ndp/current/yarn/conf
# export KYUUBI_JAVA_OPTS="-Xmx10g -XX:+UnlockDiagnosticVMOptions
-XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC
-XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70
-XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled
-XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark
-XX:MaxDirectMemorySize=1024m -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=./logs -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps
-XX:+PrintTenuringDistribut [...]
# export KYUUBI_BEELINE_OPTS="-Xmx2g -XX:+UnlockDiagnosticVMOptions
-XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC
-XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70
-XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled
-XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark"
diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index fc07eb8..38a176d 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -73,6 +73,7 @@ You can configure the environment variables in
`$KYUUBI_HOME/conf/kyuubi-env.sh`
# - KYUUBI_WORK_DIR_ROOT Root directory for launching sql engine
applications.
# (Default: $KYUUBI_HOME/work)
# - HADOOP_CONF_DIR Directory containing the Hadoop / YARN
configuration to use.
+# - YARN_CONF_DIR Directory containing the YARN configuration to use.
#
# - SPARK_HOME Spark distribution which you would like to use in
Kyuubi.
# - SPARK_CONF_DIR Optional directory where the Spark configuration
lives.
@@ -85,6 +86,7 @@ You can configure the environment variables in
`$KYUUBI_HOME/conf/kyuubi-env.sh`
# export JAVA_HOME=/usr/jdk64/jdk1.8.0_152
# export SPARK_HOME=/opt/spark
# export HADOOP_CONF_DIR=/usr/ndp/current/mapreduce_client/conf
+# export YARN_CONF_DIR=/usr/ndp/current/yarn/conf
# export KYUUBI_JAVA_OPTS="-Xmx10g -XX:+UnlockDiagnosticVMOptions
-XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC
-XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70
-XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled
-XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark
-XX:MaxDirectMemorySize=1024m -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=./logs -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps
-XX:+PrintTenuringDistribut [...]
# export KYUUBI_BEELINE_OPTS="-Xmx2g -XX:+UnlockDiagnosticVMOptions
-XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC
-XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70
-XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled
-XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark"
```
diff --git a/docs/integrations/delta_lake_with_azure_blob.md
b/docs/integrations/delta_lake_with_azure_blob.md
index 03a9818..dfd60b8 100644
--- a/docs/integrations/delta_lake_with_azure_blob.md
+++ b/docs/integrations/delta_lake_with_azure_blob.md
@@ -195,6 +195,7 @@ KYUUBI_WORK_DIR_ROOT: /usr/apache/current/kyuubi/work
SPARK_HOME: /usr/apache/current/spark
SPARK_CONF_DIR: /usr/apache/current/spark/conf
HADOOP_CONF_DIR:
+YARN_CONF_DIR:
Starting org.apache.kyuubi.server.KyuubiServer, logging to
/usr/apache/current/kyuubi/logs/kyuubi-hadoop-org.apache.kyuubi.server.KyuubiServer-host.out
Welcome to
__ __ __
diff --git a/docs/monitor/logging.md b/docs/monitor/logging.md
index f1655e8..9081446 100644
--- a/docs/monitor/logging.md
+++ b/docs/monitor/logging.md
@@ -75,6 +75,7 @@ KYUUBI_WORK_DIR_ROOT:
/Users/kentyao/svn-kyuubi/v1.3.1-incubating-rc0/apache-kyu
SPARK_HOME: /Users/kentyao/Downloads/spark/spark-3.2.0-bin-hadoop3.2
SPARK_CONF_DIR: /Users/kentyao/Downloads/spark/spark-3.2.0-bin-hadoop3.2/conf
HADOOP_CONF_DIR:
+YARN_CONF_DIR:
Starting org.apache.kyuubi.server.KyuubiServer, logging to
/Users/kentyao/svn-kyuubi/v1.3.1-incubating-rc0/apache-kyuubi-1.3.1-incubating-bin/logs/kyuubi-kentyao-org.apache.kyuubi.server.KyuubiServer-hulk.local.out
Welcome to
__ __ __
diff --git a/docs/quick_start/quick_start.md b/docs/quick_start/quick_start.md
index f3d72dd..b03c034 100644
--- a/docs/quick_start/quick_start.md
+++ b/docs/quick_start/quick_start.md
@@ -162,6 +162,7 @@ KYUUBI_WORK_DIR_ROOT:
/Users/kentyao/svn-kyuubi/v1.3.1-incubating-rc0/apache-kyu
SPARK_HOME: /Users/kentyao/Downloads/spark/spark-3.2.0-bin-hadoop3.2
SPARK_CONF_DIR: /Users/kentyao/Downloads/spark/spark-3.2.0-bin-hadoop3.2/conf
HADOOP_CONF_DIR:
+YARN_CONF_DIR:
Starting org.apache.kyuubi.server.KyuubiServer, logging to
/Users/kentyao/svn-kyuubi/v1.3.1-incubating-rc0/apache-kyuubi-1.3.1-incubating-bin/logs/kyuubi-kentyao-org.apache.kyuubi.server.KyuubiServer-hulk.local.out
Welcome to
__ __ __
