This is an automated email from the ASF dual-hosted git repository.
xuba pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new b24ad3ced [AMORO-3851] Separate spark-3.3 and spark-3.5 modules (#3848)
b24ad3ced is described below
commit b24ad3ced325ce9942dd087d8c7cafdcf1971828
Author: Fei Wang <[email protected]>
AuthorDate: Fri Oct 31 01:40:00 2025 -0700
[AMORO-3851] Separate spark-3.3 and spark-3.5 modules (#3848)
* separate
* java17
* build
* GA
* jdk17 tests
* fix conflicts
* <java.source.version>17</java.source.version>
* JDK8 + Spark-3.5
* revert JDK17
---
.github/workflows/core-hadoop3-ci.yml | 5 +++--
amoro-ams/pom.xml | 24 +++++++++++++++++----
.../amoro-mixed-spark-3-common/pom.xml | 1 -
.../amoro/spark/test/utils/TestTableUtil.java | 25 +++++++++++++++++++++-
amoro-format-mixed/amoro-mixed-spark/pom.xml | 6 ++----
.../v3.3/amoro-mixed-spark-3.3/pom.xml | 1 -
.../v3.5/amoro-mixed-spark-3.5/pom.xml | 1 -
pom.xml | 22 +++++++++++++++----
8 files changed, 67 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/core-hadoop3-ci.yml
b/.github/workflows/core-hadoop3-ci.yml
index 5dc84772f..5440afcbc 100644
--- a/.github/workflows/core-hadoop3-ci.yml
+++ b/.github/workflows/core-hadoop3-ci.yml
@@ -38,7 +38,8 @@ jobs:
strategy:
matrix:
jdk: [ '8', '11' ]
- name: Build Amoro with JDK ${{ matrix.jdk }}
+ spark: [ '3.3', '3.5' ]
+ name: Build Amoro with JDK ${{ matrix.jdk }} Spark-${{ matrix.spark }}
steps:
- uses: actions/checkout@v3
@@ -53,7 +54,7 @@ jobs:
run: mvn validate
- name: Build all modules with Maven
- run: mvn clean install -Djacoco.flink.skip=true -B -ntp -Pjava${{
matrix.jdk }}
+ run: mvn clean install -Djacoco.flink.skip=true -B -ntp -Pjava${{
matrix.jdk }} -Pspark-${{ matrix.spark }}
- name: Code coverage
uses: codecov/codecov-action@v3
diff --git a/amoro-ams/pom.xml b/amoro-ams/pom.xml
index c073c602c..58546041b 100644
--- a/amoro-ams/pom.xml
+++ b/amoro-ams/pom.xml
@@ -73,7 +73,7 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.12</artifactId>
- <version>${terminal.spark.version}</version>
+ <version>${spark.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.parquet</groupId>
@@ -83,6 +83,14 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j2-impl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>*</artifactId>
@@ -113,7 +121,7 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.12</artifactId>
- <version>${terminal.spark.version}</version>
+ <version>${spark.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.parquet</groupId>
@@ -139,13 +147,17 @@
<groupId>org.apache.arrow</groupId>
<artifactId>*</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.12</artifactId>
- <version>${terminal.spark.version}</version>
+ <version>${spark.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hive</groupId>
@@ -187,6 +199,10 @@
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -322,7 +338,7 @@
<!-- runtime dependencies -->
<dependency>
<groupId>org.apache.amoro</groupId>
-
<artifactId>amoro-format-mixed-spark-${terminal.spark.major.version}</artifactId>
+
<artifactId>amoro-format-mixed-spark-${spark.major.version}</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
</dependency>
diff --git
a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml
b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml
index 3a170ad9c..d9ad9cc68 100644
--- a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml
@@ -33,7 +33,6 @@
<url>https://amoro.apache.org</url>
<properties>
- <spark.version>3.3.4</spark.version>
<scala.version>2.12.15</scala.version>
</properties>
<dependencies>
diff --git
a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/test/java/org/apache/amoro/spark/test/utils/TestTableUtil.java
b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/test/java/org/apache/amoro/spark/test/utils/TestTableUtil.java
index 342bbf8d4..6946bc53f 100644
---
a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/test/java/org/apache/amoro/spark/test/utils/TestTableUtil.java
+++
b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/test/java/org/apache/amoro/spark/test/utils/TestTableUtil.java
@@ -56,11 +56,13 @@ import org.apache.iceberg.types.Types;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder;
import org.apache.spark.sql.catalyst.encoders.RowEncoder;
import org.apache.spark.sql.types.StructType;
import java.io.IOException;
import java.io.UncheckedIOException;
+import java.lang.reflect.Method;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
@@ -103,7 +105,28 @@ public class TestTableUtil {
public static InternalRow recordToInternalRow(Schema schema, Record record) {
StructType structType = SparkSchemaUtil.convert(schema);
Row row = recordToRow(record);
- return RowEncoder.apply(structType).createSerializer().apply(row);
+ try {
+ // for spark-3.5
+ Method rowEncoderEncoderForMethod =
+ RowEncoder.class.getDeclaredMethod("encoderFor", StructType.class);
+ Method expressionEncoderApply =
+ ExpressionEncoder.class.getDeclaredMethod(
+ "apply",
Class.forName("org.apache.spark.sql.catalyst.encoders.AgnosticEncoder"));
+ ExpressionEncoder encoder =
+ (ExpressionEncoder)
+ expressionEncoderApply.invoke(
+ null, rowEncoderEncoderForMethod.invoke(null, structType));
+ return encoder.createSerializer().apply(row);
+ } catch (Exception ignore) {
+ try {
+ // for spark-3.3
+ Method rowApplyMethod = RowEncoder.class.getDeclaredMethod("apply",
StructType.class);
+ ExpressionEncoder encoder = (ExpressionEncoder)
rowApplyMethod.invoke(null, structType);
+ return encoder.createSerializer().apply(row);
+ } catch (Throwable e) {
+ throw new RuntimeException(e);
+ }
+ }
}
public static Record rowToRecord(Row row, Types.StructType type) {
diff --git a/amoro-format-mixed/amoro-mixed-spark/pom.xml
b/amoro-format-mixed/amoro-mixed-spark/pom.xml
index a7463dafc..99138e7aa 100644
--- a/amoro-format-mixed/amoro-mixed-spark/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-spark/pom.xml
@@ -33,9 +33,7 @@
<modules>
<module>amoro-mixed-spark-3-common</module>
- <module>v3.3/amoro-mixed-spark-3.3</module>
- <module>v3.3/amoro-mixed-spark-runtime-3.3</module>
- <module>v3.5/amoro-mixed-spark-3.5</module>
- <module>v3.5/amoro-mixed-spark-runtime-3.5</module>
+
<module>v${spark.major.version}/amoro-mixed-spark-${spark.major.version}</module>
+
<module>v${spark.major.version}/amoro-mixed-spark-runtime-${spark.major.version}</module>
</modules>
</project>
diff --git
a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
index 949c3494c..8a508f89b 100644
--- a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
@@ -35,7 +35,6 @@
<properties>
<hive.version>2.3.9</hive.version>
- <spark.version>3.3.4</spark.version>
<scala.version>2.12.15</scala.version>
<scala.collection.compat>2.11.0</scala.collection.compat>
</properties>
diff --git
a/amoro-format-mixed/amoro-mixed-spark/v3.5/amoro-mixed-spark-3.5/pom.xml
b/amoro-format-mixed/amoro-mixed-spark/v3.5/amoro-mixed-spark-3.5/pom.xml
index cd696e32f..c482a9c47 100644
--- a/amoro-format-mixed/amoro-mixed-spark/v3.5/amoro-mixed-spark-3.5/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-spark/v3.5/amoro-mixed-spark-3.5/pom.xml
@@ -35,7 +35,6 @@
<properties>
<hive.version>2.3.9</hive.version>
- <spark.version>3.5.3</spark.version>
<scala.version>2.12.15</scala.version>
<scala.collection.compat>2.11.0</scala.collection.compat>
</properties>
diff --git a/pom.xml b/pom.xml
index a95b0a07c..8189ece91 100644
--- a/pom.xml
+++ b/pom.xml
@@ -125,8 +125,8 @@
<orc-core.version>1.8.3</orc-core.version>
<awssdk.version>2.24.12</awssdk.version>
<aliyun-sdk-oss.version>3.10.2</aliyun-sdk-oss.version>
- <terminal.spark.version>3.3.4</terminal.spark.version>
- <terminal.spark.major.version>3.3</terminal.spark.major.version>
+ <spark.version>3.3.4</spark.version>
+ <spark.major.version>3.3</spark.major.version>
<dropwizard.metrics.version>4.2.19</dropwizard.metrics.version>
<caffeine.version>2.9.3</caffeine.version>
<snakeyaml.version>2.2</snakeyaml.version>
@@ -1418,8 +1418,8 @@
<!-- Do not use Hive 2.3.9 for Hive 4 Metastore support due to
incompatible API changes. -->
<hive.version>2.3.8</hive.version>
<hadoop.version>2.10.2</hadoop.version>
- <terminal.spark.version>3.3.4</terminal.spark.version>
-
<terminal.spark.major.version>3.3</terminal.spark.major.version>
+ <spark.version>3.3.4</spark.version>
+ <spark.major.version>3.3</spark.major.version>
</properties>
</profile>
<profile>
@@ -1452,5 +1452,19 @@
<java.target.version>11</java.target.version>
</properties>
</profile>
+ <profile>
+ <id>spark-3.3</id>
+ <properties>
+ <spark.version>3.3.4</spark.version>
+ <spark.major.version>3.3</spark.major.version>
+ </properties>
+ </profile>
+ <profile>
+ <id>spark-3.5</id>
+ <properties>
+ <spark.version>3.5.3</spark.version>
+ <spark.major.version>3.5</spark.major.version>
+ </properties>
+ </profile>
</profiles>
</project>
