This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 74abe85 [KYUUBI #1950] Remove ambiguous SPARK_HADOOP_VERSION
74abe85 is described below
commit 74abe8510c3a01283e52d8ba59a7fe350e42434e
Author: Kent Yao <[email protected]>
AuthorDate: Tue Feb 22 11:26:38 2022 +0800
[KYUUBI #1950] Remove ambiguous SPARK_HADOOP_VERSION
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines:
https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in
https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your
PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g.,
'[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
The original idea of SPARK_HADOOP_VERSION is used to concat spark release
names only, now we need to remove it as
- SPARK_HADOOP_VERSION is misunderstood by developers and misused somewhere
like the one of kyuubi compiled
- multi-engine support now
- the release names of spark(or something else) are very easy to get
through code with different environments, prod/test/dev
- A `mvn` job is bundled with `bin/load-kyuubi-env.sh` which is truly
worrisome
- SPARK_HADOOP_VERSION on spark side hass broken already for spark 3.2
which actually bundled with hadoop 3.3, see
https://github.com/apache/spark-website/pull/361#discussion_r730716668
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #1950 from yaooqinn/hadoop.
Closes #1950
b47be7c6 [Kent Yao] Remove ambiguous SPARK_HADOOP_VERSION
3b33ee56 [Kent Yao] Remove ambiguous SPARK_HADOOP_VERSION
Authored-by: Kent Yao <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
bin/load-kyuubi-env.sh | 30 ++++--------------------------
build/dist | 16 +++++-----------
pom.xml | 17 +----------------
3 files changed, 10 insertions(+), 53 deletions(-)
diff --git a/bin/load-kyuubi-env.sh b/bin/load-kyuubi-env.sh
index 050fac3..7d65cca 100755
--- a/bin/load-kyuubi-env.sh
+++ b/bin/load-kyuubi-env.sh
@@ -71,32 +71,11 @@ fi
export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.12"}"
if [[ -f ${KYUUBI_HOME}/RELEASE ]]; then
- KYUUBI_VERSION="$(grep "Kyuubi " "$KYUUBI_HOME/RELEASE" | awk -F ' ' '{print
$2}')"
- FLINK_VERSION_BUILD="$(grep "Flink " "$KYUUBI_HOME/RELEASE" | awk -F ' '
'{print $2}')"
- SPARK_VERSION_BUILD="$(grep "Spark " "$KYUUBI_HOME/RELEASE" | awk -F ' '
'{print $2}' | grep -v 'Hadoop')"
- HADOOP_VERSION_BUILD="$(grep "Spark Hadoop " "$KYUUBI_HOME/RELEASE" | awk -F
' ' '{print $3}')"
- FLINK_BUILTIN="${KYUUBI_HOME}/externals/flink-$FLINK_VERSION_BUILD"
-
SPARK_BUILTIN="${KYUUBI_HOME}/externals/spark-$SPARK_VERSION_BUILD-bin-hadoop${HADOOP_VERSION_BUILD:0:3}"
+ FLINK_BUILTIN="$(find "$KYUUBI_HOME/externals" -name 'flink-*' -type d |
head -n 1)"
+ SPARK_BUILTIN="$(find "$KYUUBI_HOME/externals" -name 'spark-*' -type d |
head -n 1)"
else
- MVN="${MVN:-"${KYUUBI_HOME}/build/mvn"}"
- KYUUBI_VERSION=$("$MVN" help:evaluate -Dexpression=project.version
2>/dev/null\
- | grep -v "INFO"\
- | grep -v "WARNING"\
- | tail -n 1)
- FLINK_VERSION_BUILD=$("$MVN" help:evaluate -Dexpression=flink.version
2>/dev/null\
- | grep -v "INFO"\
- | grep -v "WARNING"\
- | tail -n 1)
- SPARK_VERSION_BUILD=$("$MVN" help:evaluate -Dexpression=spark.version
2>/dev/null\
- | grep -v "INFO"\
- | grep -v "WARNING"\
- | tail -n 1)
- HADOOP_VERSION_BUILD=$("$MVN" help:evaluate
-Dexpression=hadoop.binary.version 2>/dev/null\
- | grep -v "INFO"\
- | grep -v "WARNING"\
- | tail -n 1)
-
FLINK_BUILTIN="${KYUUBI_HOME}/externals/kyuubi-download/target/flink-$FLINK_VERSION_BUILD"
-
SPARK_BUILTIN="${KYUUBI_HOME}/externals/kyuubi-download/target/spark-$SPARK_VERSION_BUILD-bin-hadoop${HADOOP_VERSION_BUILD}"
+ FLINK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name
'flink-*' -type d | head -n 1)"
+ SPARK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name
'spark-*' -type d | head -n 1)"
fi
export FLINK_HOME="${FLINK_HOME:-"${FLINK_BUILTIN}"}"
@@ -109,7 +88,6 @@ if [ $silent -eq 0 ]; then
echo "JAVA_HOME: ${JAVA_HOME}"
echo "KYUUBI_HOME: ${KYUUBI_HOME}"
- echo "KYUUBI_VERSION: ${KYUUBI_VERSION}"
echo "KYUUBI_CONF_DIR: ${KYUUBI_CONF_DIR}"
echo "KYUUBI_LOG_DIR: ${KYUUBI_LOG_DIR}"
echo "KYUUBI_PID_DIR: ${KYUUBI_PID_DIR}"
diff --git a/build/dist b/build/dist
index f4d0d42..6874ecc 100755
--- a/build/dist
+++ b/build/dist
@@ -153,11 +153,6 @@ SPARK_VERSION=$("$MVN" help:evaluate
-Dexpression=spark.version $@ 2>/dev/null\
| grep -v "WARNING"\
| tail -n 1)
-SPARK_HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.binary.version
$@ 2>/dev/null\
- | grep -v "INFO"\
- | grep -v "WARNING"\
- | tail -n 1)
-
HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.version $@
2>/dev/null\
| grep -v "INFO"\
| grep -v "WARNING"\
@@ -175,7 +170,7 @@ if [[ "$NAME" == "none" ]]; then
if [[ "$SPARK_PROVIDED" == "true" ]]; then
SUFFIX=""
else
- SUFFIX="-spark-${SPARK_VERSION:0:3}-hadoop${SPARK_HADOOP_VERSION}"
+ SUFFIX="-spark-${SPARK_VERSION:0:3}"
fi
fi
@@ -219,7 +214,6 @@ echo "Java $JAVA_VERSION" >> "$DISTDIR/RELEASE"
echo "Scala $SCALA_VERSION" >> "$DISTDIR/RELEASE"
echo "Flink $FLINK_VERSION" >> "$DISTDIR/RELEASE"
echo "Spark $SPARK_VERSION" >> "$DISTDIR/RELEASE"
-echo "Spark Hadoop $SPARK_HADOOP_VERSION" >> "$DISTDIR/RELEASE"
echo "Kyuubi Hadoop $HADOOP_VERSION" >> "$DISTDIR/RELEASE"
echo "Hive $HIVE_VERSION" >> "$DISTDIR/RELEASE"
echo "Build flags: $@" >> "$DISTDIR/RELEASE"
@@ -275,14 +269,14 @@ done
if [[ "$FLINK_PROVIDED" != "true" ]]; then
# Copy flink binary dist
- cp -r "$KYUUBI_HOME/externals/kyuubi-download/target/flink-$FLINK_VERSION/" \
- "$DISTDIR/externals/flink-$FLINK_VERSION/"
+ FLINK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name
'flink-*' -type d)"
+ cp -r "$FLINK_BUILTIN" "$DISTDIR/externals/"
fi
if [[ "$SPARK_PROVIDED" != "true" ]]; then
# Copy spark binary dist
- cp -r
"$KYUUBI_HOME/externals/kyuubi-download/target/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}$HIVE_VERSION_SUFFIX/"
\
-
"$DISTDIR/externals/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}$HIVE_VERSION_SUFFIX/"
+ SPARK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name
'spark-*' -type d)"
+ cp -r "$SPARK_BUILTIN" "$DISTDIR/externals/"
fi
# Copy license files
diff --git a/pom.xml b/pom.xml
index adae5e3..ecbe31b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -110,7 +110,6 @@
<guava.version>30.1-jre</guava.version>
<guava.failureaccess.version>1.0.1</guava.failureaccess.version>
<hadoop.version>3.3.1</hadoop.version>
- <hadoop.binary.version>3.2</hadoop.binary.version>
<hive.version>2.3.9</hive.version>
<hudi.version>0.10.0</hudi.version>
<iceberg.name>iceberg-spark-runtime-3.2_${scala.binary.version}</iceberg.name>
@@ -138,7 +137,7 @@
`delta.version`, `iceberg.name`
-->
<spark.version>3.2.1</spark.version>
-
<spark.archive.name>spark-${spark.version}-bin-hadoop${hadoop.binary.version}.tgz</spark.archive.name>
+
<spark.archive.name>spark-${spark.version}-bin-hadoop3.2.tgz</spark.archive.name>
<spark.archive.mirror>https://archive.apache.org/dist/spark/spark-${spark.version}</spark.archive.mirror>
<spark.archive.download.skip>false</spark.archive.download.skip>
<swagger.version>2.1.11</swagger.version>
@@ -1902,20 +1901,6 @@
</profile>
<profile>
- <id>spark-hadoop-2.7</id>
- <properties>
- <hadoop.binary.version>2.7</hadoop.binary.version>
- </properties>
- </profile>
-
- <profile>
- <id>spark-hadoop-3.2</id>
- <properties>
- <hadoop.binary.version>3.2</hadoop.binary.version>
- </properties>
- </profile>
-
- <profile>
<id>spark-provided</id>
<properties>
<spark.archive.download.skip>true</spark.archive.download.skip>
