This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new bfd089ecc [CORE] Support JDK 11 (#6112)
bfd089ecc is described below
commit bfd089ecca22365d50592de5326f662a55c3d463
Author: Suraj Naik <[email protected]>
AuthorDate: Thu Jun 20 07:46:56 2024 +0530
[CORE] Support JDK 11 (#6112)
---
.github/workflows/velox_docker.yml | 32 ++++++++++++++++++++++++++++----
docs/developers/NewToGluten.md | 2 +-
pom.xml | 15 +++++++++++++--
tools/gluten-it/pom.xml | 9 +++++++++
4 files changed, 51 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/velox_docker.yml
b/.github/workflows/velox_docker.yml
index f690ac7ae..5f64c9f7e 100644
--- a/.github/workflows/velox_docker.yml
+++ b/.github/workflows/velox_docker.yml
@@ -87,7 +87,7 @@ jobs:
matrix:
os: [ "ubuntu:20.04", "ubuntu:22.04" ]
spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5" ]
- java: [ "java-8", "java-17" ]
+ java: [ "java-8", "java-11", "java-17" ]
# Spark supports JDK17 since 3.3 and later, see
https://issues.apache.org/jira/browse/SPARK-33772
exclude:
- spark: spark-3.2
@@ -96,8 +96,16 @@ jobs:
java: java-17
- spark: spark-3.5
java: java-17
+ - spark: spark-3.2
+ java: java-11
+ - spark: spark-3.3
+ java: java-11
+ - spark: spark-3.4
+ java: java-11
- os: ubuntu:22.04
java: java-17
+ - os: ubuntu:22.04
+ java: java-11
runs-on: ubuntu-20.04
container: ${{ matrix.os }}
steps:
@@ -116,10 +124,13 @@ jobs:
run: |
if [ "${{ matrix.java }}" = "java-17" ]; then
apt-get update && apt-get install -y openjdk-17-jdk maven
+ apt remove openjdk-11* -y
+ elif [ "${{ matrix.java }}" = "java-11" ]; then
+ apt-get update && apt-get install -y openjdk-11-jdk maven
else
apt-get update && apt-get install -y openjdk-8-jdk maven
+ apt remove openjdk-11* -y
fi
- apt remove openjdk-11* -y
ls -l
/root/.m2/repository/org/apache/arrow/arrow-dataset/15.0.0-gluten/
- name: Build and run TPCH/DS
run: |
@@ -141,7 +152,7 @@ jobs:
matrix:
os: [ "centos:7", "centos:8" ]
spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5" ]
- java: [ "java-8", "java-17" ]
+ java: [ "java-8", "java-11", "java-17" ]
# Spark supports JDK17 since 3.3 and later, see
https://issues.apache.org/jira/browse/SPARK-33772
exclude:
- spark: spark-3.2
@@ -150,8 +161,16 @@ jobs:
java: java-17
- spark: spark-3.5
java: java-17
+ - spark: spark-3.2
+ java: java-11
+ - spark: spark-3.3
+ java: java-11
+ - spark: spark-3.4
+ java: java-11
- os: centos:7
java: java-17
+ - os: centos:7
+ java: java-11
runs-on: ubuntu-20.04
container: ${{ matrix.os }}
steps:
@@ -175,6 +194,8 @@ jobs:
run: |
if [ "${{ matrix.java }}" = "java-17" ]; then
yum update -y && yum install -y java-17-openjdk-devel wget
+ elif [ "${{ matrix.java }}" = "java-11" ]; then
+ yum update -y && yum install -y java-11-openjdk-devel wget
else
yum update -y && yum install -y java-1.8.0-openjdk-devel wget
fi
@@ -186,6 +207,8 @@ jobs:
echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
if [ "${{ matrix.java }}" = "java-17" ]; then
echo "JAVA_HOME=/usr/lib/jvm/java-17-openjdk" >> $GITHUB_ENV
+ elif [ "${{ matrix.java }}" = "java-11" ]; then
+ echo "JAVA_HOME=/usr/lib/jvm/java-11-openjdk" >> $GITHUB_ENV
else
echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
fi
@@ -510,13 +533,14 @@ jobs:
- name: Setup java and maven
run: |
apt-get update && apt-get install -y openjdk-8-jdk maven wget
+ apt remove openjdk-11* -y
+ echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV
- name: Build for Spark ${{ matrix.spark }}
run: |
cd $GITHUB_WORKSPACE/
$MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox
-Pceleborn -DskipTests
- name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with ${{
matrix.celeborn }}
run: |
- export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
EXTRA_PROFILE=""
if [ "${{ matrix.celeborn }}" = "celeborn-0.4.0" ]; then
EXTRA_PROFILE="-Pceleborn-0.4"
diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md
index a8862f7a5..fa2ca520f 100644
--- a/docs/developers/NewToGluten.md
+++ b/docs/developers/NewToGluten.md
@@ -43,7 +43,7 @@ export PATH="$PATH:$JAVA_HOME/bin"
## OpenJDK 17
-By default, Gluten compiles package using JDK8. Enable maven profile by
`-Pjava-17` to use JDK17, and please make sure your JAVA_HOME points to jdk17.
+By default, Gluten compiles package using JDK8. Enable maven profile by
`-Pjava-17` to use JDK17 or `-Pjava-11` to use JDK 11, and please make sure
your JAVA_HOME points to jdk17 or jdk11 respectively.
Apache Spark and Arrow requires setting java args
`-Dio.netty.tryReflectionSetAccessible=true`, see
[SPARK-29924](https://issues.apache.org/jira/browse/SPARK-29924) and
[ARROW-6206](https://issues.apache.org/jira/browse/ARROW-6206).
So please add following configs in `spark-defaults.conf`:
diff --git a/pom.xml b/pom.xml
index 88cbb724e..87b60e40e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -41,7 +41,7 @@
<java.version>1.8</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
- <caffeine.version.java8>2.9.3</caffeine.version.java8>
+ <caffeine.version>2.9.3</caffeine.version>
<scala.binary.version>2.12</scala.binary.version>
<scala.version>2.12.15</scala.version>
<spark.major.version>3</spark.major.version>
@@ -122,6 +122,16 @@
<java.version>1.8</java.version>
</properties>
</profile>
+ <profile>
+ <id>java-11</id>
+ <activation>
+ <jdk>11</jdk>
+ </activation>
+ <properties>
+ <java.version>11</java.version>
+ <caffeine.version>3.1.8</caffeine.version>
+ </properties>
+ </profile>
<profile>
<id>java-17</id>
<activation>
@@ -129,6 +139,7 @@
</activation>
<properties>
<java.version>17</java.version>
+ <caffeine.version>3.1.8</caffeine.version>
</properties>
</profile>
<profile>
@@ -301,7 +312,7 @@
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
- <version>${caffeine.version.java8}</version>
+ <version>${caffeine.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml
index ccb59ade8..3f1760069 100644
--- a/tools/gluten-it/pom.xml
+++ b/tools/gluten-it/pom.xml
@@ -119,6 +119,15 @@
<java.version>1.8</java.version>
</properties>
</profile>
+ <profile>
+ <id>java-11</id>
+ <activation>
+ <jdk>11</jdk>
+ </activation>
+ <properties>
+ <java.version>11</java.version>
+ </properties>
+ </profile>
<profile>
<id>java-17</id>
<activation>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
