This is an automated email from the ASF dual-hosted git repository.
xuba pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new b3f11aaf0 [AMORO-3531] Support build project with JDK 8, 11 (#3568)
b3f11aaf0 is described below
commit b3f11aaf0fa83ce0a368fb128af6c3fdab1b19d2
Author: Xu Bai <[email protected]>
AuthorDate: Thu May 29 16:24:29 2025 +0800
[AMORO-3531] Support build project with JDK 8, 11 (#3568)
* Update CI configuration to support JDK 8, 11, and 17
* Update CI configuration to use actions/setup-java@v4 and
actions/checkout@master
* Update CI configuration to use JDK setup and Maven versions for improved
compatibility
* Revert CI configuration with comments
* Update hive.version to 3.1.3 in pom.xml for compatibility
* add extra java args
* Replace './mvnw' with 'mvn'
* revert
* Refactor assertions to compare task ID only
* Update pom.xml to enhance Java arguments for testing
* remove jdk 17
* exclude slf4j-log4j12 confilct
* Refactor CI configuration to simplify JDK setup and remove unnecessary
parameters
* Add Java version profile
* remove
* fix trino compile
---
.github/workflows/core-hadoop2-ci.yml | 12 ++++---
.github/workflows/core-hadoop3-ci.yml | 12 ++++---
.github/workflows/docker-images.yml | 12 +++----
README.md | 2 +-
.../amoro/server/TestDefaultOptimizingService.java | 42 +++++++++++++++++++---
.../v3.2/amoro-mixed-spark-3.2/pom.xml | 4 +++
docker/amoro/Dockerfile | 4 +--
pom.xml | 28 ++++++++++++---
8 files changed, 90 insertions(+), 26 deletions(-)
diff --git a/.github/workflows/core-hadoop2-ci.yml
b/.github/workflows/core-hadoop2-ci.yml
index 15eb710d7..bf03c7612 100644
--- a/.github/workflows/core-hadoop2-ci.yml
+++ b/.github/workflows/core-hadoop2-ci.yml
@@ -35,21 +35,25 @@ on:
jobs:
build:
runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ jdk: [ '8', '11' ]
+ name: Build Amoro with JDK ${{ matrix.jdk }}
steps:
- uses: actions/checkout@v3
- - name: Set up JDK 8
+ - name: Set up JDK ${{ matrix.jdk }}
uses: actions/setup-java@v3
with:
- java-version: '8'
+ java-version: ${{ matrix.jdk }}
distribution: 'temurin'
cache: maven
- name: Validate checkstyle first
run: mvn validate
- - name: Build all module with Maven
- run: mvn clean install -Djacoco.flink.skip=true -B -ntp -Phadoop2
+ - name: Build all modules with Maven
+ run: mvn clean install -Djacoco.flink.skip=true -B -ntp -Phadoop2
-Pjava${{ matrix.jdk }}
- name: Code coverage
uses: codecov/codecov-action@v3
diff --git a/.github/workflows/core-hadoop3-ci.yml
b/.github/workflows/core-hadoop3-ci.yml
index 742ae0092..5dc84772f 100644
--- a/.github/workflows/core-hadoop3-ci.yml
+++ b/.github/workflows/core-hadoop3-ci.yml
@@ -35,21 +35,25 @@ on:
jobs:
build:
runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ jdk: [ '8', '11' ]
+ name: Build Amoro with JDK ${{ matrix.jdk }}
steps:
- uses: actions/checkout@v3
- - name: Set up JDK 8
+ - name: Set up JDK ${{ matrix.jdk }}
uses: actions/setup-java@v3
with:
- java-version: '8'
+ java-version: ${{ matrix.jdk }}
distribution: 'temurin'
cache: maven
- name: Validate checkstyle first
run: mvn validate
- - name: Build all module with Maven
- run: mvn clean install -Djacoco.flink.skip=true -B -ntp
+ - name: Build all modules with Maven
+ run: mvn clean install -Djacoco.flink.skip=true -B -ntp -Pjava${{
matrix.jdk }}
- name: Code coverage
uses: codecov/codecov-action@v3
diff --git a/.github/workflows/docker-images.yml
b/.github/workflows/docker-images.yml
index fa0bd9f5a..343960ed0 100644
--- a/.github/workflows/docker-images.yml
+++ b/.github/workflows/docker-images.yml
@@ -43,10 +43,10 @@ jobs:
hadoop: [ "v2", "v3" ]
steps:
- uses: actions/checkout@v3
- - name: Set up JDK 8
+ - name: Set up JDK 11
uses: actions/setup-java@v3
with:
- java-version: '8'
+ java-version: '11'
distribution: 'temurin'
cache: maven
- name: Set up QEMU
@@ -106,10 +106,10 @@ jobs:
flink: [ "1.14.6", "1.20.0" ]
steps:
- uses: actions/checkout@v3
- - name: Set up JDK 8
+ - name: Set up JDK 11
uses: actions/setup-java@v3
with:
- java-version: '8'
+ java-version: '11'
distribution: 'temurin'
cache: maven
- name: Set up QEMU
@@ -187,10 +187,10 @@ jobs:
spark: [ "3.3.3" ]
steps:
- uses: actions/checkout@v3
- - name: Set up JDK 8
+ - name: Set up JDK 11
uses: actions/setup-java@v3
with:
- java-version: '8'
+ java-version: '11'
distribution: 'temurin'
cache: maven
- name: Set up QEMU
diff --git a/README.md b/README.md
index e1b3eae5c..53c71850b 100644
--- a/README.md
+++ b/README.md
@@ -116,7 +116,7 @@ Amoro contains modules as below:
## Building
-Amoro is built using Maven with JDK 8 and JDK 17(only for
`amoro-format-mixed/amoro-mixed-trino` module).
+Amoro is built using Maven with JDK 8, 11 and 17(required for
`amoro-format-mixed/amoro-mixed-trino` module).
* Build all modules without `amoro-mixed-trino`: `mvn clean package`
* Build and skip tests: `mvn clean package -DskipTests`
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java
b/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java
index cd69f3002..10ed01f8e 100644
---
a/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java
+++
b/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java
@@ -181,7 +181,13 @@ public class TestDefaultOptimizingService extends
AMSTableTestBase {
// 4.retry poll task
OptimizingTask task2 = optimizingService().pollTask(token, THREAD_ID);
- Assertions.assertEquals(task2, task);
+ Assertions.assertEquals(task2.getTaskId(), task.getTaskId());
+ Assertions.assertNotEquals(task2.getTaskInput(), task.getTaskInput());
+ TableOptimizing.OptimizingInput input =
+ SerializationUtil.simpleDeserialize(task.getTaskInput());
+ TableOptimizing.OptimizingInput input2 =
+ SerializationUtil.simpleDeserialize(task2.getTaskInput());
+ Assertions.assertEquals(input2.toString(), input.toString());
assertTaskStatus(TaskRuntime.Status.SCHEDULED);
optimizingService().ackTask(token, THREAD_ID, task.getTaskId());
assertTaskStatus(TaskRuntime.Status.ACKED);
@@ -205,7 +211,13 @@ public class TestDefaultOptimizingService extends
AMSTableTestBase {
// 4.retry poll task
OptimizingTask task2 = optimizingService().pollTask(token, THREAD_ID);
- Assertions.assertEquals(task2, task);
+ Assertions.assertEquals(task2.getTaskId(), task.getTaskId());
+ Assertions.assertNotEquals(task2.getTaskInput(), task.getTaskInput());
+ TableOptimizing.OptimizingInput input =
+ SerializationUtil.simpleDeserialize(task.getTaskInput());
+ TableOptimizing.OptimizingInput input2 =
+ SerializationUtil.simpleDeserialize(task2.getTaskInput());
+ Assertions.assertEquals(input2.toString(), input.toString());
optimizingService().ackTask(token, THREAD_ID, task.getTaskId());
optimizingService()
@@ -213,7 +225,11 @@ public class TestDefaultOptimizingService extends
AMSTableTestBase {
// retry again
OptimizingTask task3 = optimizingService().pollTask(token, THREAD_ID);
- Assertions.assertEquals(task3, task);
+ Assertions.assertEquals(task3.getTaskId(), task.getTaskId());
+ Assertions.assertNotEquals(task2.getTaskInput(), task.getTaskInput());
+ TableOptimizing.OptimizingInput input3 =
+ SerializationUtil.simpleDeserialize(task2.getTaskInput());
+ Assertions.assertEquals(input3.toString(), input.toString());
assertTaskStatus(TaskRuntime.Status.SCHEDULED);
// third time would be null
Assertions.assertNull(optimizingService().pollTask(token, THREAD_ID));
@@ -252,7 +268,12 @@ public class TestDefaultOptimizingService extends
AMSTableTestBase {
Thread.sleep(1000);
assertTaskStatus(TaskRuntime.Status.PLANNED);
OptimizingTask task2 = optimizingService().pollTask(token, THREAD_ID);
- Assertions.assertEquals(task2, task);
+ Assertions.assertEquals(task2.getTaskId(), task.getTaskId());
+ TableOptimizing.OptimizingInput input =
+ SerializationUtil.simpleDeserialize(task.getTaskInput());
+ TableOptimizing.OptimizingInput input2 =
+ SerializationUtil.simpleDeserialize(task2.getTaskInput());
+ Assertions.assertEquals(input2.toString(), input.toString());
}
@Test
@@ -267,6 +288,11 @@ public class TestDefaultOptimizingService extends
AMSTableTestBase {
OptimizingTask task2 = optimizingService().pollTask(token, THREAD_ID);
Assertions.assertNotNull(task2);
Assertions.assertEquals(task2.getTaskId(), task.getTaskId());
+ TableOptimizing.OptimizingInput input =
+ SerializationUtil.simpleDeserialize(task.getTaskInput());
+ TableOptimizing.OptimizingInput input2 =
+ SerializationUtil.simpleDeserialize(task2.getTaskInput());
+ Assertions.assertEquals(input2.toString(), input.toString());
}
@Test
@@ -354,6 +380,11 @@ public class TestDefaultOptimizingService extends
AMSTableTestBase {
OptimizingTask task2 = optimizingService().pollTask(token, THREAD_ID);
Assertions.assertEquals(task2.getTaskId(), task.getTaskId());
+ TableOptimizing.OptimizingInput input =
+ SerializationUtil.simpleDeserialize(task.getTaskInput());
+ TableOptimizing.OptimizingInput input2 =
+ SerializationUtil.simpleDeserialize(task2.getTaskInput());
+ Assertions.assertEquals(input2.toString(), input.toString());
optimizingService().ackTask(token, THREAD_ID, task.getTaskId());
optimizingService()
.completeTask(token, buildOptimizingTaskFailResult(task.getTaskId(),
"error"));
@@ -363,6 +394,9 @@ public class TestDefaultOptimizingService extends
AMSTableTestBase {
OptimizingTask task3 = optimizingService().pollTask(token, THREAD_ID);
Assertions.assertEquals(task3.getTaskId(), task.getTaskId());
+ TableOptimizing.OptimizingInput input3 =
+ SerializationUtil.simpleDeserialize(task2.getTaskInput());
+ Assertions.assertEquals(input3.toString(), input.toString());
optimizingService().ackTask(token, THREAD_ID, task.getTaskId());
optimizingService()
.completeTask(token, buildOptimizingTaskFailResult(task.getTaskId(),
"error"));
diff --git
a/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml
b/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml
index aed3f284b..f7bde90c3 100644
--- a/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml
@@ -147,6 +147,10 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
</exclusions>
</dependency>
diff --git a/docker/amoro/Dockerfile b/docker/amoro/Dockerfile
index 71de9fdd3..8315411c7 100644
--- a/docker/amoro/Dockerfile
+++ b/docker/amoro/Dockerfile
@@ -23,7 +23,7 @@
# --tag apache/amoro:tagname
# .
-FROM eclipse-temurin:8-jdk-jammy as builder
+FROM eclipse-temurin:11-jdk-jammy AS builder
# Add the entire project to the build container, unzip it,
# and remove flink-optimizer to reduce the container size.
@@ -40,7 +40,7 @@ RUN AMORO_VERSION=`cat pom.xml | grep 'amoro-parent' -C 3 |
grep -Eo '<version>.
&& rm -rf /workspace/amoro
-FROM eclipse-temurin:8-jdk-jammy
+FROM eclipse-temurin:11-jdk-jammy
ARG MAVEN_MIRROR=https://repo.maven.apache.org/maven2
diff --git a/pom.xml b/pom.xml
index 91e62c842..bde0125df 100644
--- a/pom.xml
+++ b/pom.xml
@@ -71,12 +71,13 @@
</issueManagement>
<properties>
- <maven.compiler.source>8</maven.compiler.source>
- <maven.compiler.target>8</maven.compiler.target>
+ <java.source.version>11</java.source.version>
+ <java.target.version>11</java.target.version>
+ <maven.compiler.source>${java.source.version}</maven.compiler.source>
+ <maven.compiler.target>${java.target.version}</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.build.outputTimestamp>2024-06-27T08:00:00Z</project.build.outputTimestamp>
- <java.version>1.8</java.version>
<root.dir>${project.basedir}</root.dir>
<jacoco.flink.skip>false</jacoco.flink.skip>
@@ -90,7 +91,7 @@
<maven-scala-plugin.version>4.8.1</maven-scala-plugin.version>
<maven-antlr4-plugin.version>4.3</maven-antlr4-plugin.version>
<maven-surefire-plugin.version>3.0.0-M7</maven-surefire-plugin.version>
- <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
+ <maven-compiler-plugin.version>3.13.0</maven-compiler-plugin.version>
<maven-dependency-plugin.version>3.3.0</maven-dependency-plugin.version>
<maven-antrun-plugin.version>3.0.0</maven-antrun-plugin.version>
<maven-jacoco-plugin.version>0.8.7</maven-jacoco-plugin.version>
@@ -103,7 +104,7 @@
<iceberg.version>1.6.1</iceberg.version>
<paimon.version>0.9.0</paimon.version>
- <hive.version>3.1.1</hive.version>
+ <hive.version>3.1.3</hive.version>
<hadoop.version>3.4.0</hadoop.version>
<kerby.version>2.0.3</kerby.version>
<scala.binary.version>2.12</scala.binary.version>
@@ -1388,5 +1389,22 @@
<aliyun-sdk-dependency-scope>compile</aliyun-sdk-dependency-scope>
</properties>
</profile>
+
+ <profile>
+ <id>java8</id>
+ <properties>
+ <java.target.version>8</java.target.version>
+ <java.source.version>8</java.source.version>
+ </properties>
+ </profile>
+ <profile>
+ <id>java11</id>
+ <activation>
+ <jdk>[11,)</jdk>
+ </activation>
+ <properties>
+ <java.target.version>11</java.target.version>
+ </properties>
+ </profile>
</profiles>
</project>
