This is an automated email from the ASF dual-hosted git repository.
dombizita pushed a commit to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-14496-zdu by this push:
new cb71a0a3fb6 HDDS-14821. Version specific data validation in
rolling-upgrade suite (#10033)
cb71a0a3fb6 is described below
commit cb71a0a3fb66324ad0c6e9f109e54f8d2d46519d
Author: Zita Dombi <[email protected]>
AuthorDate: Wed May 13 12:54:36 2026 +0200
HDDS-14821. Version specific data validation in rolling-upgrade suite
(#10033)
---
.../compose/upgrade/compose/ha/docker-compose.yaml | 2 +
.../dist/src/main/compose/upgrade/testlib.sh | 6 +-
.../rolling-upgrade/callbacks/common/callback.sh | 56 +++++++++-
.../upgrade/upgrades/rolling-upgrade/driver.sh | 122 +++++++++++++--------
4 files changed, 133 insertions(+), 53 deletions(-)
diff --git
a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml
index c9db073f825..1ac5a0ccca9 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml
@@ -325,6 +325,8 @@ services:
client:
command: ["sleep","1000000"]
<<: *common-config
+ environment:
+ <<: *environment
hostname: client
networks:
net:
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh
b/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh
index c83fc285014..805c96ef4ba 100755
--- a/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh
@@ -49,8 +49,10 @@ prepare_for_image() {
## the upgrade being tested if one exists. If neither exists, print a
## warning that nothing was tested.
## @param The name of the function to run.
+## @param All remaining parameters are passed as parameters to the called
function
callback() {
local func="$1"
+ shift
set -u
: "${OZONE_UPGRADE_CALLBACK}"
@@ -61,7 +63,7 @@ callback() {
# Common callback always exists.
source "$OZONE_COMMON_CALLBACK"
if [[ "$(type -t "$func")" = function ]]; then
- "$func"
+ "$func" "$@"
fi
)
@@ -70,7 +72,7 @@ callback() {
if [[ -f "$OZONE_UPGRADE_CALLBACK" ]]; then
source "$OZONE_UPGRADE_CALLBACK"
if [[ "$(type -t "$func")" = function ]]; then
- "$func"
+ "$func" "$@"
fi
fi
)
diff --git
a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh
b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh
index e5e693e41b5..00ffce1a671 100644
---
a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh
+++
b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh
@@ -19,10 +19,58 @@ source "$TEST_DIR"/testlib.sh
### CALLBACKS ###
-before_service_restart() {
- generate "generate-${SERVICE}" "$CLIENT"
+## @description Run while a service is stopped, it'll generate data on the
cluster (upgrade/generate.robot)
+## @param The prefix to use for data generated.
+with_service_stopped() {
+ generate "${1}" "$CLIENT"
}
-after_service_restart() {
- validate "generate-${SERVICE}" "$CLIENT"
+## @description Run while a service is started again (with new image), it'll
validate data on the cluster (upgrade/validate.robot)
+## @param The prefix to use for data generated.
+with_service_restarted() {
+ validate "${1}" "$CLIENT"
+}
+
+with_old_version() {
+ execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization"
--include finalized upgrade/check-finalization.robot
+ generate old1 "$CLIENT"
+ validate old1 "$CLIENT"
+
+ generate overwrite "$CLIENT"
+}
+
+with_this_version_pre_finalized() {
+ # No check for pre-finalized status here, because the release may not have
+ # added layout features to OM or HDDS.
+ validate old1 "$CLIENT"
+
+ generate new1 "$CLIENT"
+ validate new1 "$CLIENT"
+
+ validate overwrite "$CLIENT"
+ generate overwrite "$CLIENT" --exclude create-volume-and-bucket
+}
+
+with_old_version_downgraded() {
+ execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization"
--include finalized upgrade/check-finalization.robot
+ validate old1 "$CLIENT"
+ validate new1 "$CLIENT"
+
+ generate old2 "$CLIENT"
+ validate old2 "$CLIENT"
+
+ validate overwrite "$CLIENT"
+ generate overwrite "$CLIENT" --exclude create-volume-and-bucket
+}
+
+with_this_version_finalized() {
+ execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization"
--include finalized upgrade/check-finalization.robot
+ validate old1 "$CLIENT"
+ validate new1 "$CLIENT"
+ validate old2 "$CLIENT"
+
+ generate new2 "$CLIENT"
+ validate new2 "$CLIENT"
+
+ validate overwrite "$CLIENT"
}
diff --git
a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh
b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh
index a9bf7030021..1a87156de18 100755
---
a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh
+++
b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh
@@ -26,7 +26,6 @@ set -u
: "${OZONE_UPGRADE_FROM}"
: "${OZONE_UPGRADE_TO}"
: "${TEST_DIR}"
-: "${SCM}"
: "${CLIENT}"
: "${OZONE_CURRENT_VERSION}"
set +u
@@ -35,95 +34,124 @@ echo "--- RUNNING ROLLING UPGRADE TEST FROM
$OZONE_UPGRADE_FROM TO $OZONE_UPGRAD
source "$TEST_DIR"/testlib.sh
-# Restart one service with the target image.
+## @description Restart one service with the target image.
+## @param name of the service
+## @param stage prefix, used for the data prefix
rolling_restart_service() {
- SERVICE="$1"
+ local service="$1"
+ local stage_prefix="$2"
- echo "--- RESTARTING ${SERVICE} WITH IMAGE ${OZONE_UPGRADE_TO} ---"
+ local callback_data_prefix="generate-${service}-${stage_prefix}"
# Stop service
- stop_containers "${SERVICE}"
+ stop_containers "${service}"
- callback before_service_restart
+ callback with_service_stopped "$callback_data_prefix"
- # Restart service with new image.
- prepare_for_image "${OZONE_UPGRADE_TO}"
- create_containers "${SERVICE}"
+ # Start service
+ create_containers "${service}"
- callback after_service_restart
+ callback with_service_restarted "$callback_data_prefix"
# Service-specific readiness checks.
- case "${SERVICE}" in
+ case "${service}" in
om*)
- wait_for_port "${SERVICE}" 9862 120
+ wait_for_port "${service}" 9862 120
;;
scm*)
# SCM hostnames in this compose are scmX.org
- wait_for_port "${SERVICE}.org" 9876 120
+ wait_for_port "${service}.org" 9876 120
;;
dn*)
- wait_for_port "${SERVICE}" 9882 120
+ wait_for_port "${service}" 9882 120
;;
s3g*)
- wait_for_port "${SERVICE}" 9878 120
+ wait_for_port "${service}" 9878 120
;;
esac
}
+## @description Restart all services with the target image.
+## @param stage prefix, used for the generated data as prefix, also for the
OUTPUT_NAME
+## @param target image
+rolling_restart_all_services() {
+ local stage_prefix="$1"
+ local target_image="$2"
+ local s
+
+ # Prepare the requested image
+ prepare_for_image "${target_image}"
+ echo "--- PREPARED ${target_image} IMAGE ---"
+
+ # SCMs first
+ for s in scm2 scm1 scm3; do
+
OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-${stage_prefix}-${s}"
+ echo "--- RESTARTING ${s} WITH IMAGE ${target_image} ---"
+ rolling_restart_service "$s" "$stage_prefix"
+ done
+
+ # Recon
+ OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-${stage_prefix}-recon"
+ echo "--- RESTARTING recon WITH IMAGE ${target_image} ---"
+ rolling_restart_service "recon" "$stage_prefix"
+
+ # DNs
+ for s in dn1 dn2 dn3 dn4 dn5; do
+
OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-${stage_prefix}-${s}"
+ echo "--- RESTARTING ${s} WITH IMAGE ${target_image} ---"
+ rolling_restart_service "$s" "$stage_prefix"
+ done
+
+ # OMs
+ for s in om1 om2 om3; do
+
OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-${stage_prefix}-${s}"
+ echo "--- RESTARTING ${s} WITH IMAGE ${target_image} ---"
+ rolling_restart_service "$s" "$stage_prefix"
+ done
+
+ # S3 Gateways (s3g is HAProxy and does not need to be upgraded)
+ for s in s3g1 s3g2 s3g3; do
+
OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-${stage_prefix}-${s}"
+ echo "--- RESTARTING ${s} WITH IMAGE ${target_image} ---"
+ rolling_restart_service "$s" "$stage_prefix"
+ done
+}
+
echo "--- SETTING UP OLD VERSION $OZONE_UPGRADE_FROM ---"
OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-1-original"
-export OM_HA_ARGS='--'
prepare_for_image "$OZONE_UPGRADE_FROM"
echo "--- RUNNING WITH OLD VERSION $OZONE_UPGRADE_FROM ---"
start_docker_env
-# TODO Add old data generation
+callback with_old_version
echo "--- ROLLING UPGRADE TO $OZONE_UPGRADE_TO PRE-FINALIZED ---"
+rolling_restart_all_services "2-upgrade" "$OZONE_UPGRADE_TO"
-# SCMs first
-for s in scm2 scm1 scm3; do
- OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
- rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
-done
-
-# Recon
-OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-recon"
-rolling_restart_service "recon" "$OZONE_UPGRADE_TO"
+OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-pre-finalized"
+callback with_this_version_pre_finalized
-# DNs
-for s in dn1 dn2 dn3 dn4 dn5; do
- OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
- rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
-done
+echo "--- ROLLING DOWNGRADE TO $OZONE_UPGRADE_FROM ---"
+rolling_restart_all_services "3-downgrade" "$OZONE_UPGRADE_FROM"
-# OMs
-for s in om1 om2 om3; do
- OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
- rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
-done
+OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-3-downgraded"
+callback with_old_version_downgraded
-# S3 Gateways (s3g is HAProxy and does not need to be upgraded)
-for s in s3g1 s3g2 s3g3; do
- OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
- rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
-done
-
-# TODO Add downgrade scenario
+echo "--- ROLLING UPGRADE TO $OZONE_UPGRADE_TO ---"
+rolling_restart_all_services "4-upgrade" "$OZONE_UPGRADE_TO"
# Upgrade client after all server components are upgraded but before
finalization,
# so new client APIs can be exercised against pre-finalized servers.
echo "--- UPGRADING CLIENT TO $OZONE_UPGRADE_TO ---"
-OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-client"
+OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-4-upgrade-client"
stop_containers "$CLIENT"
prepare_for_image "${OZONE_UPGRADE_TO}"
create_containers "$CLIENT"
echo "--- RUNNING WITH NEW VERSION $OZONE_UPGRADE_TO FINALIZED ---"
-OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-3-finalized"
-
-# TODO Add validation for pre-finalized state
+OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-5-finalized"
# Sends commands to finalize OM and SCM.
execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-finalize"
upgrade/finalize.robot
+callback with_this_version_finalized
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
