This is an automated email from the ASF dual-hosted git repository.
JNSimba pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-flink-connector.git
The following commit(s) were added to refs/heads/master by this push:
new cedf5fef [Enhancement] Add JDK 21 support for Flink 2.x (#678)
cedf5fef is described below
commit cedf5fef659b5e5be28bf6e5b443c3b00ba289f2
Author: Roberto Serafin <[email protected]>
AuthorDate: Tue Aug 18 07:59:35 2026 +0200
[Enhancement] Add JDK 21 support for Flink 2.x (#678)
Keep Flink 2 artifacts compatible with Java 17 while adding JDK 21
package, integration, and end-to-end coverage. Allow JDK 17 or 21 in
the interactive build and document Java 21 as experimental.
---
.github/workflows/build-connector.yml | 51 +++++++++++++++-
.github/workflows/run-e2ecase-flink2.yml | 30 ++++++---
.github/workflows/run-itcase-flink2.yml | 58 ++++++++++++++++++
README.md | 28 +++++----
flink-doris-connector/build.sh | 41 +++++++------
flink-doris-connector/pom.xml | 102 ++++++++++++++++---------------
6 files changed, 222 insertions(+), 88 deletions(-)
diff --git a/.github/workflows/build-connector.yml
b/.github/workflows/build-connector.yml
index 783b4459..adc92788 100644
--- a/.github/workflows/build-connector.yml
+++ b/.github/workflows/build-connector.yml
@@ -21,6 +21,9 @@ on:
pull_request:
push:
+permissions:
+ contents: read
+
jobs:
build-extension:
name: "Build Connector"
@@ -115,4 +118,50 @@ jobs:
-pl flink-doris-connector-flink2 -am \
-Pflink2 \
-Dflink.version=2.2.0 \
- -Dflink.major.version=2.2
\ No newline at end of file
+ -Dflink.major.version=2.2
+
+ build-flink2-jdk21:
+ name: "Build Flink ${{ matrix.flink-major }} on JDK 21"
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - flink-version: 2.0.0
+ flink-major: '2.0'
+ - flink-version: 2.1.0
+ flink-major: '2.1'
+ - flink-version: 2.2.0
+ flink-major: '2.2'
+ defaults:
+ run:
+ shell: bash
+ working-directory: flink-doris-connector
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Setup Java 21
+ uses: actions/setup-java@v4
+ with:
+ distribution: temurin
+ java-version: '21'
+ cache: maven
+
+ - name: Build Flink connector
+ run: |
+ mvn clean package \
+ -pl flink-doris-connector-flink2 -am \
+ -Pflink2 \
+ -Dflink.version=${{ matrix.flink-version }} \
+ -Dflink.major.version=${{ matrix.flink-major }}
+
+ - name: Verify Java 17 bytecode
+ run: |
+ javap -verbose \
+
flink-doris-connector-base/target/classes/org/apache/doris/flink/serialization/RowBatch.class
\
+ | grep -q 'major version: 61'
+ javap -verbose \
+
flink-doris-connector-flink2/target/classes/org/apache/doris/flink/table/DorisDynamicTableFactory.class
\
+ | grep -q 'major version: 61'
diff --git a/.github/workflows/run-e2ecase-flink2.yml
b/.github/workflows/run-e2ecase-flink2.yml
index 8883e324..b695936f 100644
--- a/.github/workflows/run-e2ecase-flink2.yml
+++ b/.github/workflows/run-e2ecase-flink2.yml
@@ -21,22 +21,31 @@ on:
pull_request:
push:
+permissions:
+ contents: read
+
jobs:
build-extension:
- name: "Run E2ECases"
+ name: "Run E2ECases (Flink 2.2, JDK ${{ matrix.java }})"
runs-on: ubuntu-latest
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ java: ['17', '21']
defaults:
run:
shell: bash
steps:
- name: Checkout
- uses: actions/checkout@master
+ uses: actions/checkout@v4
- - name: Setup java
- uses: actions/setup-java@v2
+ - name: Setup Java ${{ matrix.java }}
+ uses: actions/setup-java@v4
with:
- distribution: adopt
- java-version: '17'
+ distribution: temurin
+ java-version: ${{ matrix.java }}
+ cache: maven
- name: Run E2ECases (Flink2 module)
run: |
@@ -44,4 +53,11 @@ jobs:
-Pflink2 -pl flink-doris-connector-it -am \
-DfailIfNoTests=false \
-Dtest="*E2ECase" \
- -Dimage="apache/doris:doris-all-in-one-2.1.0"
\ No newline at end of file
+ -Dflink.version=2.2.0 \
+ -Dflink.major.version=2.2 \
+ -Dimage="apache/doris:doris-all-in-one-2.1.0"
+
+ - name: Verify Doris-to-Doris E2E report
+ run: |
+ test -s \
+
flink-doris-connector/flink-doris-connector-it/target/surefire-reports/TEST-org.apache.doris.flink.container.e2e.Doris2DorisE2ECase.xml
diff --git a/.github/workflows/run-itcase-flink2.yml
b/.github/workflows/run-itcase-flink2.yml
index 4c55f9e7..2c43b17d 100644
--- a/.github/workflows/run-itcase-flink2.yml
+++ b/.github/workflows/run-itcase-flink2.yml
@@ -21,6 +21,9 @@ on:
pull_request:
push:
+permissions:
+ contents: read
+
jobs:
build-extension:
name: "Run ITCases"
@@ -44,5 +47,60 @@ jobs:
-Pflink2 -pl flink-doris-connector-it -am \
-DfailIfNoTests=false \
-Dtest="*ITCase" \
+ -Dflink.version=2.2.0 \
+ -Dflink.major.version=2.2 \
-Dimage="apache/doris:doris-all-in-one-2.1.0"
+ - name: Verify sensitive ITCase reports
+ run: |
+ for report in \
+ TEST-org.apache.doris.flink.source.DorisSourceITCase.xml \
+ TEST-org.apache.doris.flink.sink.DorisSinkITCase.xml \
+ TEST-org.apache.doris.flink.sink.DorisSinkFailoverITCase.xml
+ do
+ test -s
"flink-doris-connector/flink-doris-connector-it/target/surefire-reports/${report}"
+ done
+
+ jdk21-sensitive-itcases:
+ name: "Run sensitive ITCases (Flink ${{ matrix.flink-major }}, JDK 21)"
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - flink-version: 2.2.0
+ flink-major: '2.2'
+ defaults:
+ run:
+ shell: bash
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Setup Java 21
+ uses: actions/setup-java@v4
+ with:
+ distribution: temurin
+ java-version: '21'
+ cache: maven
+
+ - name: Run sensitive ITCases (Flink2 module)
+ run: |
+ cd flink-doris-connector && mvn test \
+ -Pflink2 -pl flink-doris-connector-it -am \
+ -DfailIfNoTests=false \
+ -Dtest="DorisSourceITCase,DorisSinkITCase,DorisSinkFailoverITCase" \
+ -Dflink.version=${{ matrix.flink-version }} \
+ -Dflink.major.version=${{ matrix.flink-major }} \
+ -Dimage="apache/doris:doris-all-in-one-2.1.0"
+
+ - name: Verify selected ITCase reports
+ run: |
+ for report in \
+ TEST-org.apache.doris.flink.source.DorisSourceITCase.xml \
+ TEST-org.apache.doris.flink.sink.DorisSinkITCase.xml \
+ TEST-org.apache.doris.flink.sink.DorisSinkFailoverITCase.xml
+ do
+ test -s
"flink-doris-connector/flink-doris-connector-it/target/surefire-reports/${report}"
+ done
diff --git a/README.md b/README.md
index 71c2a439..e9e2159c 100644
--- a/README.md
+++ b/README.md
@@ -26,21 +26,23 @@ under the License.
Flink Doris Connector supports the following Flink versions:
-| Flink Version | JDK Requirement |
-|:--------------|:----------------|
-| 1.15.x | JDK 8 |
-| 1.16.x | JDK 8 |
-| 1.17.x | JDK 8 |
-| 1.18.x | JDK 8 |
-| 1.19.x | JDK 8 |
-| 1.20.x | JDK 8 |
-| 2.0.x | JDK 17 |
-| 2.1.x | JDK 17 |
-| 2.2.x | JDK 17 |
+| Flink Version | Runtime JDK |
+|:--------------|:-------------------------------|
+| 1.15.x | JDK 8 |
+| 1.16.x | JDK 8 |
+| 1.17.x | JDK 8 |
+| 1.18.x | JDK 8 |
+| 1.19.x | JDK 8 |
+| 1.20.x | JDK 8 |
+| 2.0.x | JDK 17 |
+| 2.1.x | JDK 17; JDK 21 (experimental) |
+| 2.2.x | JDK 17; JDK 21 (experimental) |
+
+Flink 2.x artifacts are compiled with Java 17 (`--release 17`, classfile major
version 61) and can be built with JDK 17 or JDK 21. JDK 21 runtime
compatibility is experimental for the Flink versions listed above; JDK 17
remains the recommended runtime.
If you wish to contribute or use a connector from flink 1.13 (and earlier),
please use the
[branch-for-flink-before-1.13](https://github.com/apache/doris-flink-connector/tree/branch-for-flink-before-1.13)
-More information about compilation and usage, please visit [Flink Doris
Connector](https://doris.apache.org/docs/dev/ecosystem/flink-doris-connector)
+More information about compilation and usage is available in the [Flink Doris
Connector
documentation](https://doris.apache.org/docs/dev/ecosystem/flink-doris-connector).
The website documentation is maintained outside this repository and requires a
separate change to keep its compatibility guidance in sync.
### TLS Configuration
@@ -69,7 +71,7 @@ The CA path is a local file path, not an HDFS or HTTP URI.
For standalone deploy
You need to copy customer_env.sh.tpl to customer_env.sh before build and you
need to configure it before build.
-> **JDK requirement:** Use JDK 8 for Flink 1.x, and JDK 17 for Flink 2.x.
+> **Build JDK requirement:** Use JDK 8 for Flink 1.x. Use JDK 17 or JDK 21 for
Flink 2.x; the resulting Flink 2.x artifacts still target Java 17.
```shell
git clone [email protected]:apache/doris-flink-connector.git
diff --git a/flink-doris-connector/build.sh b/flink-doris-connector/build.sh
index 3b4be40d..ade88b9d 100755
--- a/flink-doris-connector/build.sh
+++ b/flink-doris-connector/build.sh
@@ -37,6 +37,7 @@ if [[ ${have_tty} -eq 1 ]]; then
GREEN=$(printf '\033[32m')
YELLOW=$(printf '\033[33m')
BLUE=$(printf '\033[34m')
+ WHITE=$(printf '\033[37m')
BOLD=$(printf '\033[1m')
RESET=$(printf '\033[0m')
else
@@ -52,29 +53,25 @@ fi
echo_r () {
# Color red: Error, Failed
[[ $# -ne 1 ]] && return 1
- # shellcheck disable=SC2059
- printf "[%sDoris%s] %s$1%s\n" $BLUE $RESET $RED $RESET
+ printf "[%sDoris%s] %s%s%s\n" "$BLUE" "$RESET" "$RED" "$1" "$RESET"
}
echo_g () {
# Color green: Success
[[ $# -ne 1 ]] && return 1
- # shellcheck disable=SC2059
- printf "[%sDoris%s] %s$1%s\n" $BLUE $RESET $GREEN $RESET
+ printf "[%sDoris%s] %s%s%s\n" "$BLUE" "$RESET" "$GREEN" "$1" "$RESET"
}
echo_y () {
# Color yellow: Warning
[[ $# -ne 1 ]] && return 1
- # shellcheck disable=SC2059
- printf "[%sDoris%s] %s$1%s\n" $BLUE $RESET $YELLOW $RESET
+ printf "[%sDoris%s] %s%s%s\n" "$BLUE" "$RESET" "$YELLOW" "$1" "$RESET"
}
echo_w () {
# Color yellow: White
[[ $# -ne 1 ]] && return 1
- # shellcheck disable=SC2059
- printf "[%sDoris%s] %s$1%s\n" $BLUE $RESET $WHITE $RESET
+ printf "[%sDoris%s] %s%s%s\n" "$BLUE" "$RESET" "$WHITE" "$1" "$RESET"
}
# OS specific support. $var _must_ be set to either true or false.
@@ -117,7 +114,7 @@ fi
selectFlink() {
echo 'Flink-Doris-Connector supports multiple versions of flink. Which
version do you need ?'
echo ' [Flink 1.x requires JDK 8]'
- echo ' [Flink 2.x requires JDK 17]'
+ echo ' [Flink 2.x builds require JDK 17 or 21; artifacts target Java 17]'
select flink in "1.15.x" "1.16.x" "1.17.x" "1.18.x" "1.19.x" "1.20.x"
"2.0.x" "2.1.x" "2.2.x"
do
case $flink in
@@ -194,22 +191,32 @@ fi
FLINK_MAJOR_VERSION=0
[ ${FLINK_VERSION} != 0 ] && FLINK_MAJOR_VERSION=${FLINK_VERSION%.*}
-# Sanity check: Flink 1.x requires JDK 1.8, Flink 2.x requires JDK 17
+# Sanity check: Flink 1.x requires JDK 8; Flink 2.x builds require JDK 17 or
21.
JAVA_VERSION=$(java -version 2>&1 | awk -F'"' '/version/ {print $2}')
+if [[ "${JAVA_VERSION}" =~ ^1\.([0-9]+)([-._+].*)?$ ]]; then
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[1]}
+elif [[ "${JAVA_VERSION}" =~ ^([0-9]+)([-._+].*)?$ ]]; then
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[1]}
+else
+ echo_r "Error: Unable to determine the build JDK major version from
'${JAVA_VERSION}'."
+ exit 1
+fi
+
if [ "${FLINK_PROFILE}" = "flink1" ]; then
- if [[ "${JAVA_VERSION}" != 1.8* ]]; then
- echo_r "Error: Flink ${FLINK_VERSION} requires JDK 1.8, but found
version '${JAVA_VERSION}'."
- echo_r "Please switch to JDK 1.8 before building Flink 1.x."
+ if [[ "${JAVA_MAJOR_VERSION}" != "8" ]]; then
+ echo_r "Error: Flink ${FLINK_VERSION} builds require JDK 8, but found
build JDK '${JAVA_VERSION}'."
+ echo_r "Please switch the build JDK to 8 before building Flink 1.x."
exit 1
fi
elif [ "${FLINK_PROFILE}" = "flink2" ]; then
- if [[ "${JAVA_VERSION}" != 17* ]]; then
- echo_r "Error: Flink ${FLINK_VERSION} requires JDK 17, but found
version '${JAVA_VERSION}'."
- echo_r "Please switch to JDK 17 before building Flink 2.x."
+ if [[ "${JAVA_MAJOR_VERSION}" != "17" && "${JAVA_MAJOR_VERSION}" != "21"
]]; then
+ echo_r "Error: Flink ${FLINK_VERSION} builds require JDK 17 or 21, but
found build JDK '${JAVA_VERSION}'."
+ echo_r "Please switch the build JDK to 17 or 21; Flink 2.x artifacts
target Java 17."
exit 1
fi
fi
-echo_g " JDK version check passed: ${JAVA_VERSION}"
+echo_g " Build JDK version check passed: ${JAVA_VERSION}"
+[ "${FLINK_PROFILE}" = "flink2" ] && echo_g " Flink 2.x artifact target: Java
17"
echo_g " flink version: ${FLINK_VERSION}, major version:
${FLINK_MAJOR_VERSION}"
echo_g " build starting..."
diff --git a/flink-doris-connector/pom.xml b/flink-doris-connector/pom.xml
index 6789e3c3..9547fb1b 100644
--- a/flink-doris-connector/pom.xml
+++ b/flink-doris-connector/pom.xml
@@ -82,13 +82,14 @@ under the License.
<maven.compiler.target>8</maven.compiler.target>
<checkstyle.version>8.14</checkstyle.version>
<maven-checkstyle-plugin.version>2.17</maven-checkstyle-plugin.version>
- <spotless.version>2.27.1</spotless.version>
+ <spotless.version>2.44.0</spotless.version>
<httpcomponents.version>4.5.13</httpcomponents.version>
<commons-codec.version>1.15</commons-codec.version>
<fasterxml.version>2.13.3</fasterxml.version>
<guava.version>31.1-jre</guava.version>
<slf4j.version>1.7.25</slf4j.version>
<mockito.version>4.2.0</mockito.version>
+ <byte-buddy.version>1.12.4</byte-buddy.version>
<testcontainers.version>1.17.6</testcontainers.version>
<junit.version>4.12</junit.version>
<hamcrest.version>1.3</hamcrest.version>
@@ -259,6 +260,18 @@ under the License.
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>net.bytebuddy</groupId>
+ <artifactId>byte-buddy</artifactId>
+ <version>${byte-buddy.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>net.bytebuddy</groupId>
+ <artifactId>byte-buddy-agent</artifactId>
+ <version>${byte-buddy.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime-web</artifactId>
@@ -418,36 +431,6 @@ under the License.
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
</plugin>
- <plugin>
- <groupId>com.diffplug.spotless</groupId>
- <artifactId>spotless-maven-plugin</artifactId>
- <version>${spotless.version}</version>
- <configuration>
- <java>
- <googleJavaFormat>
- <version>1.24.0</version>
- <style>AOSP</style>
- </googleJavaFormat>
-
- <!-- \# refers to the static imports -->
- <importOrder>
-
<order>org.apache.flink,org.apache.flink.shaded,,javax,java,scala,\#
- </order>
- </importOrder>
-
- <removeUnusedImports/>
- </java>
- </configuration>
- <executions>
- <execution>
- <id>spotless-check</id>
- <phase>validate</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
@@ -508,37 +491,21 @@ under the License.
<flink.major.version>2.2</flink.major.version>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
+ <maven.compiler.release>17</maven.compiler.release>
</properties>
<modules>
<module>flink-doris-connector-flink2</module>
</modules>
</profile>
- <profile>
- <id>java-8-target</id>
- <activation>
- <jdk>[1.8,11)</jdk>
- </activation>
- <build>
- <plugins>
- <!-- There is a conflict between JDK 8 and 17; JDK 8 skips
format checks.-->
- <plugin>
- <groupId>com.diffplug.spotless</groupId>
- <artifactId>spotless-maven-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
-
<profile>
<id>java-17-target</id>
<activation>
<jdk>[17,)</jdk>
</activation>
<properties>
+ <!-- Mockito 4 retains JDK 8 support; newer Byte Buddy enables
inline mocks on JDK 21. -->
+ <byte-buddy.version>1.14.12</byte-buddy.version>
<test.jvm.args>
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
@@ -551,6 +518,41 @@ under the License.
--add-exports=java.base/sun.nio.ch=ALL-UNNAMED
</test.jvm.args>
</properties>
+ <build>
+ <plugins>
+ <!-- Spotless 2.44 requires Java 11; Flink 1/JDK 8 already
skips format checks. -->
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <version>${spotless.version}</version>
+ <configuration>
+ <java>
+ <googleJavaFormat>
+ <version>1.24.0</version>
+ <style>AOSP</style>
+ </googleJavaFormat>
+
+ <!-- \# refers to the static imports -->
+ <importOrder>
+
<order>org.apache.flink,org.apache.flink.shaded,,javax,java,scala,\#
+ </order>
+ </importOrder>
+
+ <removeUnusedImports/>
+ </java>
+ </configuration>
+ <executions>
+ <execution>
+ <id>spotless-check</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</profile>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
