This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 704dc3c955 [Fix][CI] Fix upgrade compatibility workflow setup (#11611)
704dc3c955 is described below
commit 704dc3c955ab718dd4ffbdd50eb84c90e009148c
Author: Goutam Adwant <[email protected]>
AuthorDate: Sat Aug 22 18:18:50 2026 -0700
[Fix][CI] Fix upgrade compatibility workflow setup (#11611)
Signed-off-by: goutamadwant <[email protected]>
---
.github/workflows/upgrade_compatibility.yml | 3 +
tools/upgrade_compatibility/README.md | 6 +
.../run_upgrade_compatibility.sh | 54 +++++---
.../run_upgrade_compatibility_test.sh | 144 +++++++++++++++++++++
.../mysql-cdc-multitable-localfile/setup.sh | 10 +-
5 files changed, 195 insertions(+), 22 deletions(-)
diff --git a/.github/workflows/upgrade_compatibility.yml
b/.github/workflows/upgrade_compatibility.yml
index e76a33e81d..1c3819c08c 100644
--- a/.github/workflows/upgrade_compatibility.yml
+++ b/.github/workflows/upgrade_compatibility.yml
@@ -54,6 +54,9 @@ jobs:
distribution: temurin
java-version: "8"
+ - name: Test upgrade compatibility tooling
+ run: bash tools/upgrade_compatibility/run_upgrade_compatibility_test.sh
+
- name: Free disk space
run: bash tools/github/free_disk_space.sh
diff --git a/tools/upgrade_compatibility/README.md
b/tools/upgrade_compatibility/README.md
index 4c3f52a29a..4283ade996 100644
--- a/tools/upgrade_compatibility/README.md
+++ b/tools/upgrade_compatibility/README.md
@@ -65,6 +65,12 @@ OLD_SEATUNNEL_VERSION=2.3.13
SCENARIO=mysql-cdc-multitable-localfile \
bash tools/upgrade_compatibility/run_upgrade_compatibility.sh
```
+Run the focused runner tests with:
+
+```shell
+bash tools/upgrade_compatibility/run_upgrade_compatibility_test.sh
+```
+
## Adding Scenarios
Add a new directory under `scenarios/` with these files:
diff --git a/tools/upgrade_compatibility/run_upgrade_compatibility.sh
b/tools/upgrade_compatibility/run_upgrade_compatibility.sh
index 7f7e459bc3..66fa2bd8b6 100755
--- a/tools/upgrade_compatibility/run_upgrade_compatibility.sh
+++ b/tools/upgrade_compatibility/run_upgrade_compatibility.sh
@@ -18,7 +18,7 @@
set -euo pipefail
-ROOT_DIR="$(cd "$(dirname "$0")/../.." >/dev/null; pwd)"
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null; pwd)"
OLD_SEATUNNEL_VERSION="${OLD_SEATUNNEL_VERSION:-2.3.13}"
SCENARIO="${SCENARIO:-generic-fake-localfile}"
WORK_DIR="${WORK_DIR:-${ROOT_DIR}/target/upgrade-compatibility}"
@@ -75,7 +75,15 @@ cleanup() {
stop_process() {
local pid="$1"
- if [ -n "${pid}" ] && kill -0 "${pid}" >/dev/null 2>&1; then
+ local child_pid
+
+ [ -n "${pid}" ] || return 0
+
+ while IFS= read -r child_pid; do
+ stop_process "${child_pid}"
+ done < <(pgrep -P "${pid}" 2>/dev/null || true)
+
+ if kill -0 "${pid}" >/dev/null 2>&1; then
kill "${pid}" >/dev/null 2>&1 || true
wait "${pid}" >/dev/null 2>&1 || true
fi
@@ -124,13 +132,23 @@ find_current_distribution() {
return
fi
+ local -a archives=()
local archive
- archive="$(find "${ROOT_DIR}/seatunnel-dist/target" -maxdepth 1 \
- -name "apache-seatunnel-*-bin.tar.gz" \
- ! -name "*-src.tar.gz" \
- | sort | tail -n 1)"
- [ -n "${archive}" ] || fail "No current dev distribution found under
seatunnel-dist/target"
- echo "${archive}"
+ while IFS= read -r archive; do
+ archives+=("${archive}")
+ done < <(
+ find "${ROOT_DIR}/seatunnel-dist/target" -maxdepth 1 \
+ -name "apache-seatunnel-*-bin.tar.gz" \
+ ! -name "apache-seatunnel-edge-agent-*-bin.tar.gz" \
+ ! -name "*-src.tar.gz" \
+ | sort
+ )
+
+ [ "${#archives[@]}" -gt 0 ] \
+ || fail "No current dev distribution found under seatunnel-dist/target"
+ [ "${#archives[@]}" -eq 1 ] \
+ || fail "Multiple current dev distributions found: ${archives[*]}"
+ echo "${archives[0]}"
}
extract_distribution() {
@@ -207,12 +225,10 @@ install_old_release_connectors() {
continue
fi
echo "Installing ${connector}:${OLD_SEATUNNEL_VERSION}"
- "${ROOT_DIR}/mvnw" --batch-mode dependency:get \
- -Dtransitive=false \
- -DgroupId=org.apache.seatunnel \
- -DartifactId="${connector}" \
- -Dversion="${OLD_SEATUNNEL_VERSION}" \
- -Ddest="${OLD_DIST_DIR}/connectors"
+ "${ROOT_DIR}/mvnw" --batch-mode dependency:copy \
+
-Dartifact="org.apache.seatunnel:${connector}:${OLD_SEATUNNEL_VERSION}" \
+ -DoutputDirectory="${OLD_DIST_DIR}/connectors" \
+ -Dmdep.stripVersion=false
done < "${SCENARIO_DIR}/plugin_config"
end_stage
}
@@ -333,12 +349,16 @@ wait_for_process_to_stop() {
start_cluster() {
local dist_dir="$1"
local log_file="$2"
+ local engine_log="${dist_dir}/logs/seatunnel-engine-server.log"
mkdir -p "$(dirname "${log_file}")"
+ rm -f "${engine_log}"
SEATUNNEL_CONFIG="${dist_dir}/config/seatunnel.yaml" \
"${dist_dir}/bin/seatunnel-cluster.sh" > "${log_file}" 2>&1 &
local pid="$!"
- if ! wait_for_log "${log_file}" "received new worker register"
"${STARTUP_TIMEOUT_SECONDS}"; then
+ if ! wait_for_log \
+ "${engine_log}" "received new worker register"
"${STARTUP_TIMEOUT_SECONDS}"; then
+ tail -n 200 "${log_file}" >&2 || true
stop_process "${pid}"
return 1
fi
@@ -502,4 +522,6 @@ main() {
echo "Upgrade compatibility scenario passed: ${OLD_SEATUNNEL_VERSION} ->
current dev (${SCENARIO})"
}
-main "$@"
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+ main "$@"
+fi
diff --git a/tools/upgrade_compatibility/run_upgrade_compatibility_test.sh
b/tools/upgrade_compatibility/run_upgrade_compatibility_test.sh
new file mode 100755
index 0000000000..50a72cc018
--- /dev/null
+++ b/tools/upgrade_compatibility/run_upgrade_compatibility_test.sh
@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" >/dev/null; pwd)"
+source "${SCRIPT_DIR}/run_upgrade_compatibility.sh"
+
+TEST_ROOT="$(mktemp -d)"
+trap 'cleanup; rm -rf "${TEST_ROOT}"' EXIT
+
+ROOT_DIR="${TEST_ROOT}"
+DIST_DIR="${ROOT_DIR}/seatunnel-dist/target"
+
+reset_distributions() {
+ rm -rf "${DIST_DIR}"
+ mkdir -p "${DIST_DIR}"
+ CURRENT_DIST_ARCHIVE=""
+}
+
+assert_equals() {
+ local expected="$1"
+ local actual="$2"
+ [ "${actual}" = "${expected}" ] || {
+ echo "Expected '${expected}', got '${actual}'" >&2
+ exit 1
+ }
+}
+
+assert_contains() {
+ local expected="$1"
+ local actual="$2"
+ [[ "${actual}" == *"${expected}"* ]] || {
+ echo "Expected '${actual}' to contain '${expected}'" >&2
+ exit 1
+ }
+}
+
+test_selects_regular_distribution() {
+ reset_distributions
+ local
regular_archive="${DIST_DIR}/apache-seatunnel-3.0.0-SNAPSHOT-bin.tar.gz"
+ touch "${regular_archive}"
+ touch "${DIST_DIR}/apache-seatunnel-edge-agent-3.0.0-SNAPSHOT-bin.tar.gz"
+
+ assert_equals "${regular_archive}" "$(find_current_distribution)"
+}
+
+test_uses_explicit_distribution() {
+ reset_distributions
+ local explicit_archive="${TEST_ROOT}/current-seatunnel.tar.gz"
+ touch "${explicit_archive}"
+ CURRENT_DIST_ARCHIVE="${explicit_archive}"
+
+ assert_equals "${explicit_archive}" "$(find_current_distribution)"
+}
+
+test_rejects_missing_regular_distribution() {
+ reset_distributions
+ touch "${DIST_DIR}/apache-seatunnel-edge-agent-3.0.0-SNAPSHOT-bin.tar.gz"
+
+ local output
+ if output="$(find_current_distribution 2>&1)"; then
+ echo "Expected missing regular distribution to fail" >&2
+ exit 1
+ fi
+ assert_contains "No current dev distribution found" "${output}"
+}
+
+test_rejects_multiple_regular_distributions() {
+ reset_distributions
+ touch "${DIST_DIR}/apache-seatunnel-3.0.0-SNAPSHOT-bin.tar.gz"
+ touch "${DIST_DIR}/apache-seatunnel-3.1.0-SNAPSHOT-bin.tar.gz"
+
+ local output
+ if output="$(find_current_distribution 2>&1)"; then
+ echo "Expected multiple regular distributions to fail" >&2
+ exit 1
+ fi
+ assert_contains "Multiple current dev distributions found" "${output}"
+}
+
+test_waits_for_engine_log() {
+ local dist_dir="${TEST_ROOT}/distribution"
+ local launch_log="${TEST_ROOT}/cluster-launch.log"
+ mkdir -p "${dist_dir}/bin" "${dist_dir}/config" "${dist_dir}/logs"
+ touch "${dist_dir}/config/seatunnel.yaml"
+ cat > "${dist_dir}/bin/seatunnel-cluster.sh" <<'EOF'
+#!/usr/bin/env bash
+echo "start master_and_worker node"
+echo "received new worker register" > "$(dirname
"$0")/../logs/seatunnel-engine-server.log"
+sleep 5
+EOF
+ chmod +x "${dist_dir}/bin/seatunnel-cluster.sh"
+
+ STARTUP_TIMEOUT_SECONDS=2
+ local pid
+ pid="$(start_cluster "${dist_dir}" "${launch_log}")"
+ [[ "${pid}" =~ ^[0-9]+$ ]] || {
+ echo "Expected cluster process id, got '${pid}'" >&2
+ exit 1
+ }
+
+ local child_pid
+ child_pid="$(pgrep -P "${pid}" | head -n 1)"
+ [ -n "${child_pid}" ] || {
+ echo "Expected cluster process ${pid} to have a child" >&2
+ exit 1
+ }
+ stop_process "${pid}"
+ local attempt
+ for attempt in {1..20}; do
+ if ! kill -0 "${child_pid}" >/dev/null 2>&1; then
+ break
+ fi
+ sleep 0.1
+ done
+ if kill -0 "${child_pid}" >/dev/null 2>&1; then
+ echo "Expected child process ${child_pid} to stop with the cluster" >&2
+ exit 1
+ fi
+}
+
+test_selects_regular_distribution
+test_uses_explicit_distribution
+test_rejects_missing_regular_distribution
+test_rejects_multiple_regular_distributions
+test_waits_for_engine_log
+
+echo "Upgrade compatibility runner tests passed"
diff --git
a/tools/upgrade_compatibility/scenarios/mysql-cdc-multitable-localfile/setup.sh
b/tools/upgrade_compatibility/scenarios/mysql-cdc-multitable-localfile/setup.sh
index 531cbb62c6..6150a8c20f 100755
---
a/tools/upgrade_compatibility/scenarios/mysql-cdc-multitable-localfile/setup.sh
+++
b/tools/upgrade_compatibility/scenarios/mysql-cdc-multitable-localfile/setup.sh
@@ -46,12 +46,10 @@ docker exec "${MYSQL_CONTAINER_NAME}" \
docker exec -i "${MYSQL_CONTAINER_NAME}" \
mysql -uroot -p"${MYSQL_ROOT_PASSWORD}" < "${SCENARIO_DIR}/mysql.sql"
-"${ROOT_DIR}/mvnw" --batch-mode dependency:get \
- -Dtransitive=false \
- -DgroupId=com.mysql \
- -DartifactId=mysql-connector-j \
- -Dversion="${MYSQL_DRIVER_VERSION}" \
- -Ddest="${MYSQL_DRIVER_JAR}"
+"${ROOT_DIR}/mvnw" --batch-mode dependency:copy \
+ -Dartifact="com.mysql:mysql-connector-j:${MYSQL_DRIVER_VERSION}" \
+ -DoutputDirectory="${RUN_DIR}" \
+ -Dmdep.stripVersion=false
install_driver() {
local dist_dir="$1"
