This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 7208e0379b [GLUTEN-11316][VL] Enable JDK-21 for Spark-4.0 (#11304)
7208e0379b is described below
commit 7208e0379b1ad85bee6aa1e007d19bd016be7a8d
Author: Yuan <[email protected]>
AuthorDate: Mon Dec 22 11:25:45 2025 +0800
[GLUTEN-11316][VL] Enable JDK-21 for Spark-4.0 (#11304)
JDK-21 is supported since Spark-4.0. This patch adds:
the new java-21 profile
tpc tests
necessary dependencies update to support java-21
---------
Signed-off-by: Yuan <[email protected]>
---
.github/workflows/velox_backend_x86.yml | 18 ++++++++++++---
.../spark/source/TestIcebergSourceTablesBase.java | 2 ++
.../apache/gluten/memory/MemoryUsageRecorder.java | 2 ++
pom.xml | 26 +++++++++++++++++-----
tools/gluten-it/3rd/pom.xml | 2 +-
tools/gluten-it/pom.xml | 16 +++++++++++--
6 files changed, 54 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index 1bcf1f97ef..a8b2c6aa8d 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -108,9 +108,17 @@ jobs:
matrix:
os: [ "ubuntu:20.04", "ubuntu:22.04" ]
spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5",
"spark-4.0" ]
- java: [ "java-8", "java-11", "java-17" ]
+ java: [ "java-8", "java-11", "java-17", "java-21" ]
# Spark supports JDK17 since 3.3.
exclude:
+ - spark: spark-3.2
+ java: java-21
+ - spark: spark-3.3
+ java: java-21
+ - spark: spark-3.4
+ java: java-21
+ - spark: spark-3.5
+ java: java-21
- spark: spark-3.2
java: java-17
- spark: spark-3.4
@@ -123,10 +131,12 @@ jobs:
java: java-11
- spark: spark-3.4
java: java-11
- - os: ubuntu:22.04
+ - os: ubuntu:20.04
java: java-17
- - os: ubuntu:22.04
+ - os: ubuntu:20.04
java: java-11
+ - os: ubuntu:20.04
+ java: java-21
- spark: spark-4.0
java: java-8
- spark: spark-4.0
@@ -158,6 +168,8 @@ jobs:
if [ "${{ matrix.java }}" = "java-17" ]; then
apt-get update && apt-get install -y openjdk-17-jdk maven
apt remove openjdk-11* -y
+ elif [ "${{ matrix.java }}" = "java-21" ]; then
+ apt-get update && apt-get install -y openjdk-21-jdk maven
elif [ "${{ matrix.java }}" = "java-11" ]; then
apt-get update && apt-get install -y openjdk-11-jdk maven
else
diff --git
a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java
b/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java
index b05fd3b502..98d1925c6c 100644
---
a/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java
+++
b/backends-velox/src-iceberg-spark34/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java
@@ -1994,11 +1994,13 @@ public abstract class TestIcebergSourceTablesBase
extends SparkTestBase {
.mode("append")
.save(loadLocation(tableIdentifier));
+ // spotless:off
List<Integer> actual =
spark.read().format("iceberg").load(loadLocation(tableIdentifier,
"files"))
.sort(DataFile.SPEC_ID.name()).collectAsList().stream()
.map(r -> (Integer) r.getAs(DataFile.SPEC_ID.name()))
.collect(Collectors.toList());
+ // spotless:on
Assert.assertEquals("Should have two partition specs",
ImmutableList.of(spec0, spec1), actual);
}
diff --git
a/gluten-core/src/main/java/org/apache/gluten/memory/MemoryUsageRecorder.java
b/gluten-core/src/main/java/org/apache/gluten/memory/MemoryUsageRecorder.java
index 9013494a46..234e06dd23 100644
---
a/gluten-core/src/main/java/org/apache/gluten/memory/MemoryUsageRecorder.java
+++
b/gluten-core/src/main/java/org/apache/gluten/memory/MemoryUsageRecorder.java
@@ -17,9 +17,11 @@
package org.apache.gluten.memory;
public interface MemoryUsageRecorder extends MemoryUsageStatsBuilder {
+ // spotless:off
void inc(long bytes);
// peak used bytes
long peak();
// current used bytes
long current();
+ // spotless:on
}
diff --git a/pom.xml b/pom.xml
index 89d98a036b..f4e30677a1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -110,6 +110,7 @@
<guava.version>32.0.1-jre</guava.version>
<!--spotless-->
<spotless.version>2.27.2</spotless.version>
+ <googlejavaformat.version>1.7</googlejavaformat.version>
<spotless.scalafmt.version>3.8.3</spotless.scalafmt.version>
<spotless.delimiter>package</spotless.delimiter>
<!-- spotless:off -->
@@ -136,7 +137,7 @@
<!-- plugin version-->
<build-helper-maven-plugin.version>3.6.0</build-helper-maven-plugin.version>
<scala.compiler.version>4.8.0</scala.compiler.version>
- <maven.compiler.plugin>3.12.1</maven.compiler.plugin>
+ <maven.compiler.plugin>3.14.1</maven.compiler.plugin>
<maven.jar.plugin>3.2.2</maven.jar.plugin>
<scalastyle.version>1.0.0</scalastyle.version>
<scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
@@ -678,7 +679,7 @@
</plugin>
<plugin>
<artifactId>maven-shade-plugin</artifactId>
- <version>3.4.1</version>
+ <version>3.6.1</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
</configuration>
@@ -696,7 +697,7 @@
<java>
<toggleOffOn/>
<googleJavaFormat>
- <version>1.7</version>
+ <version>${googlejavaformat.version}</version>
</googleJavaFormat>
<!-- \# refers to the static imports -->
@@ -1055,6 +1056,18 @@
<caffeine.version>3.1.8</caffeine.version>
</properties>
</profile>
+ <profile>
+ <id>java-21</id>
+ <activation>
+ <jdk>21</jdk>
+ </activation>
+ <properties>
+ <java.version>21</java.version>
+ <caffeine.version>3.1.8</caffeine.version>
+ <spotless.version>2.44.2</spotless.version>
+ <googlejavaformat.version>1.17.0</googlejavaformat.version>
+ </properties>
+ </profile>
<profile>
<id>spark-3.2</id>
<properties>
@@ -1214,7 +1227,7 @@
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
- <id>enforce-java-17</id>
+ <id>enforce-java-17+</id>
<goals>
<goal>enforce</goal>
</goals>
@@ -1222,8 +1235,9 @@
<rules>
<requireActiveProfile>
<!-- Spark 4.0 requires Java 17+ -->
- <profiles>java-17</profiles>
- <message>"-P spark-4.0" requires JDK 17</message>
+ <profiles>java-17,java-21</profiles>
+ <all>false</all>
+ <message>"-P spark-4.0" requires JDK 17+</message>
</requireActiveProfile>
</rules>
</configuration>
diff --git a/tools/gluten-it/3rd/pom.xml b/tools/gluten-it/3rd/pom.xml
index cb7ba0c7ed..2f5d2e5fc3 100644
--- a/tools/gluten-it/3rd/pom.xml
+++ b/tools/gluten-it/3rd/pom.xml
@@ -49,7 +49,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>3.4.1</version>
+ <version>3.6.1</version>
<configuration>
<shadedArtifactAttached>false</shadedArtifactAttached>
<relocations>
diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml
index 692a74960b..27be4a1b5b 100644
--- a/tools/gluten-it/pom.xml
+++ b/tools/gluten-it/pom.xml
@@ -45,6 +45,7 @@
<tpcds.version>1.4</tpcds.version>
<spotless.version>2.27.2</spotless.version>
+ <googlejavaformat.version>1.7</googlejavaformat.version>
<spotless.scalafmt.version>3.8.3</spotless.scalafmt.version>
<spotless.delimiter>package</spotless.delimiter>
<!-- spotless:off -->
@@ -170,7 +171,7 @@
<java>
<toggleOffOn/>
<googleJavaFormat>
- <version>1.7</version>
+ <version>${googlejavaformat.version}</version>
</googleJavaFormat>
<!-- \# refers to the static imports -->
@@ -262,6 +263,17 @@
<java.version>17</java.version>
</properties>
</profile>
+ <profile>
+ <id>java-21</id>
+ <activation>
+ <jdk>21</jdk>
+ </activation>
+ <properties>
+ <java.version>21</java.version>
+ <spotless.version>2.44.2</spotless.version>
+ <googlejavaformat.version>1.17.0</googlejavaformat.version>
+ </properties>
+ </profile>
<profile>
<id>spark-3.2</id>
<activation>
@@ -309,7 +321,7 @@
<id>spark-4.0</id>
<properties>
<spark.version>4.0.1</spark.version>
- <scala.version>2.13.8</scala.version>
+ <scala.version>2.13.16</scala.version>
<scala.binary.version>2.13</scala.binary.version>
<delta.package.name>delta-spark</delta.package.name>
<delta.version>4.0.0</delta.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
