On Mon, Jan 19, 2026 at 2:16 PM Xavier Simonart via dev < [email protected]> wrote:
> When ovn is upgraded, ovn-controller is updated first on > the compute nodes. Then ovn-northd and DB are upgraded. > This patch tests whether the intermediate state (i.e. with > ovn-controller being upgraded) works properly, running system > tests from the base line (i.e. before the upgrade). > > Flow tables might change between releases. > Hence this patch must take that into account by updating the (old) > system tests with any updated table numbers. > In some cases, (new) ovn-controller might change flows in existing > tables, causing some 'upgrade' tests to fail. > Such tests can be skipped using the TAG_TEST_NOT_UPGRADABLE tag. > > This patch upgrades the ci to run automatically some upgrade tests > weekly. It also provides a shell script to run those tests locally. > > Upgrade-tests are run on push/pull only for LTS (24.03) and latest > release (25.09) to avoid too long tests. Upgrades from other > branches are run on schedule. > > This patch depends on patch [1] on branch-25.09. > > [1] "tests: Add new TAG_TEST_NOT_UPGRADABLE to some tests." > > Reported-at: https://issues.redhat.com/browse/FDP-1240 > Signed-off-by: Xavier Simonart <[email protected]> > --- > Hi Xaiver, thank you for working on this. I have some comments down below. > .ci/ci.sh | 1 + > .ci/linux-build.sh | 95 +++++++++- > .ci/linux-util.sh | 367 +++++++++++++++++++++++++++++++++++++ > .ci/test-upgrade-local.sh | 202 ++++++++++++++++++++ > .github/workflows/test.yml | 41 ++++- > Makefile.am | 1 + > tests/ovn-macros.at | 6 + > 7 files changed, 697 insertions(+), 16 deletions(-) > create mode 100755 .ci/test-upgrade-local.sh > > diff --git a/.ci/ci.sh b/.ci/ci.sh > index 3640d3243..6798fbd78 100755 > --- a/.ci/ci.sh > +++ b/.ci/ci.sh > @@ -102,6 +102,7 @@ function run_tests() { > ARCH=$ARCH CC=$CC LIBS=$LIBS OPTS=$OPTS TESTSUITE=$TESTSUITE \ > TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS DPDK=$DPDK \ > RECHECK=$RECHECK UNSTABLE=$UNSTABLE TIMEOUT=$TIMEOUT \ > + BASE_VERSION=$BASE_VERSION \ > ./.ci/linux-build.sh > " > } > diff --git a/.ci/linux-build.sh b/.ci/linux-build.sh > index 183833a16..890f4c59d 100755 > --- a/.ci/linux-build.sh > +++ b/.ci/linux-build.sh > @@ -1,7 +1,10 @@ > #!/bin/bash > > set -o errexit > -set -x > +# Enable debug output for CI, optional for local > +if [ "${NO_DEBUG:-0}" = "0" ]; then > + set -x > +fi > > ARCH=${ARCH:-"x86_64"} > USE_SPARSE=${USE_SPARSE:-"yes"} > @@ -181,7 +184,7 @@ function run_system_tests() > > if ! sudo timeout -k 5m -v $TIMEOUT make $JOBS $type \ > TESTSUITEFLAGS="$TEST_RANGE" RECHECK=$RECHECK \ > - SKIP_UNSTABLE=$SKIP_UNSTABLE; then > + SKIP_UNSTABLE=$SKIP_UNSTABLE UPGRADE_TEST=$UPGRADE_TEST; then > # $log_file is necessary for debugging. > cat tests/$log_file > return 1 > @@ -190,19 +193,28 @@ function run_system_tests() > > function execute_system_tests() > { > - configure_ovn $OPTS > - make $JOBS || { cat config.log; exit 1; } > + local test_type=$1 > + local log_file=$2 > + local skip_build=$3 > + > + # Only build if not already built (upgrade tests build separately) > + if [ "$skip_build" != "yes" ]; then > + configure_ovn $OPTS > + make $JOBS || { cat config.log; exit 1; } > + fi > > local stable_rc=0 > local unstable_rc=0 > > - if ! SKIP_UNSTABLE=yes run_system_tests $@; then > + if ! SKIP_UNSTABLE=yes UPGRADE_TEST=$UPGRADE_TEST \ > + run_system_tests $test_type $log_file; then > stable_rc=1 > fi > > if [ "$UNSTABLE" ]; then > if ! SKIP_UNSTABLE=no TEST_RANGE="-k unstable" RECHECK=yes \ > - run_system_tests $@; then > + UPGRADE_TEST=$UPGRADE_TEST run_system_tests $test_type \ > + $log_file; > then > unstable_rc=1 > fi > fi > @@ -212,6 +224,72 @@ function execute_system_tests() > fi > } > > +function execute_upgrade_tests() > +{ > + . .ci/linux-util.sh > + > + # Save current CI scripts (will be replaced by base version after > checkout) > + cp -rf .ci /tmp/ovn-upgrade-ci > + > + # Build current version > + log "Building current version..." > + mkdir -p logs > + configure_ovn $OPTS >> logs/build-current.log 2>&1 || { > + log "configure ovn failed - see config.log and > logs/build-current.log" > + exit 1 > + } > + make $JOBS >> logs/build-current.log 2>&1 || { > + log "building ovn failed - see logs/build-current.log" > + exit 1 > + } > + > + ovn_upgrade_save_current_binaries > + > + # Checkout base version > + ovn_upgrade_checkout_base "$BASE_VERSION" logs/git.log > + > + # Clean from current version > + log "Cleaning build artifacts..." > + make distclean >> logs/build-base.log 2>&1 || true > + (cd ovs && make distclean >> ../logs/build-base.log 2>&1) || true > + > + # Apply test patches > + ovn_upgrade_apply_tests_patches > + > + # Build base with patches > + ovn_upgrade_patch_for_ovn_debug > + > + # Build (modified) base version > + log "Building base version (with patched lflow.h)..." > + configure_ovn $OPTS >> logs/build-base.log 2>&1 || { > + log "configure ovn failed - see config.log and > logs/build-base.log" > + exit 1 > + } > + make $JOBS >> logs/build-base.log 2>&1 || { > + log "building ovn failed - see logs/build-base.log" > + exit 1 > + } > + ovn_upgrade_save_ovn_debug > + > + # Build (clean) base version > + log "Rebuilding base version (clean lflow.h)..." > + git checkout controller/lflow.h >> logs/git.log 2>&1 > + make $JOBS >> logs/build-base.log 2>&1 || { > + log "building ovn failed - see logs/build-base.log" > + exit 1 > + } > + > + # Restore binaries > + ovn_upgrade_restore_binaries > + > + # Restore current CI scripts for test execution > + cp -f /tmp/ovn-upgrade-ci/linux-build.sh .ci/linux-build.sh > + cp -f /tmp/ovn-upgrade-ci/linux-util.sh .ci/linux-util.sh > + > + UPGRADE_TEST=yes execute_system_tests "check-kernel" \ > + "system-kmod-testsuite.log" > "yes" > +} > + > configure_$CC > > if [ "$TESTSUITE" ]; then > @@ -238,6 +316,11 @@ if [ "$TESTSUITE" ]; then > sudo bash -c "echo 2048 > > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" > execute_system_tests "check-system-dpdk" > "system-dpdk-testsuite.log" > ;; > + > + "upgrade-test") > + execute_upgrade_tests > + ;; > + > esac > else > configure_ovn $OPTS > diff --git a/.ci/linux-util.sh b/.ci/linux-util.sh > index b5bd1f8c9..a62ff2ea0 100755 > --- a/.ci/linux-util.sh > +++ b/.ci/linux-util.sh > @@ -49,3 +49,370 @@ function disable_apparmor() > sudo aa-teardown || true > sudo systemctl disable --now apparmor.service > } > + > +function log() { > + echo "[$(date '+%H:%M:%S')] $*" > +} > + > +# ovn_upgrade_save_current_binaries > +# Saves current version's binaries and schemas to > /tmp/ovn-upgrade-binaries/ > +function ovn_upgrade_save_current_binaries() > +{ > + mkdir -p /tmp/ovn-upgrade-binaries > + > + # New ovn-controller may generate OpenFlow flows with new actions > that old > + # OVS doesn't understand, so we need also to use new OVS. > + # New ovs-vswitchd binary expects columns/tables defined in the > current > + # schema, so we also need to use new schemas. > + files="controller/ovn-controller ovs/vswitchd/ovs-vswitchd > + ovs/ovsdb/ovsdb-server ovs/utilities/ovs-vsctl > + ovs/utilities/ovs-ofctl ovs/utilities/ovs-appctl > + ovs/utilities/ovs-dpctl ovs/vswitchd/vswitch.ovsschema" > + for file in $files; do > + if [ ! -f "$file" ]; then > + log "ERROR: $file not found" > + return 1 > + fi > + cp "$file" /tmp/ovn-upgrade-binaries/ > + done > + > + # In the upgrade scenario we use old ovn-northd and new > ovn-controller. > + # OFTABLES are defined through a combination of northd/northd.h and > + # controller/lflow.h. Tests uses either (old) table numbers, table > names > + # (defined in ovn-macros) or ovn-debug. > + # > + # Extract OFCTL_* table defines from current lflow.h > + if ! grep '^#define OFTABLE_' controller/lflow.h > \ > + /tmp/ovn-upgrade-ofctl-defines.h; then > + log "No #define OFTABLE_ found in lflow.h" > + return 1 > + fi > + > + # Extract OFTABLE m4 defines from current tests/ovn-macros.at > + # These are used by tests to reference table numbers > + # In old tests, there might be no OFTABLE_ in ovn-macros, so grep can > fail. > + grep '^m4_define(\[OFTABLE_' tests/ovn-macros.at > \ > + /tmp/ovn-upgrade-oftable-m4-defines.txt || true > + > + # Extract key table numbers for calculating shifts in hardcoded table > + # references. OFTABLE_SAVE_INPORT is where normal (unshifted) tables > + # resume. > + LINE=$(grep "define OFTABLE_LOG_EGRESS_PIPELINE" controller/lflow.h) > + NEW_LOG_EGRESS=$(echo "$LINE" | grep -oE '[0-9]+') > + LINE=$(grep "define OFTABLE_SAVE_INPORT" controller/lflow.h) > + NEW_SAVE_INPORT=$(echo "$LINE" | grep -oE '[0-9]+') > + if [ -z "$NEW_LOG_EGRESS" ]; then > + log "ERROR: Could not extract OFTABLE_LOG_EGRESS_PIPELINE value" > + return 1 > + fi > + if [ -z "$NEW_SAVE_INPORT" ]; then > + log "ERROR: Could not extract OFTABLE_SAVE_INPORT value" > + return 1 > + fi > + > + echo "$NEW_LOG_EGRESS" > /tmp/ovn-upgrade-new-log-egress.txt > + echo "$NEW_SAVE_INPORT" > /tmp/ovn-upgrade-new-save-inport.txt > + > + echo "" > + log "Saved current versions:" > + log " ovn-controller:$(/tmp/ovn-upgrade-binaries/ovn-controller > --version | > + grep ovn-controller)" > + log " SB DB schema:$(/tmp/ovn-upgrade-binaries/ovn-controller > --version | > + grep "SB DB Schema")" > + log " ovs-vswitchd:$(/tmp/ovn-upgrade-binaries/ovs-vswitchd --version > | > + grep vSwitch)" > +} > + > +# ovn_upgrade_checkout_base BASE_VERSION LOG_FILE > +# Checks out base version from git > +function ovn_upgrade_checkout_base() > +{ > + local base_version=$1 > + local log_file=$2 > + > + log "Checking out base version: $base_version" > + > + # Try to checkout directly first (might already exist locally) > + if git checkout "$base_version" >> "$log_file" 2>&1; then > + log "Using locally available $base_version" > + else > + # Not available locally, try to fetch it > + log "Fetching $base_version from origin..." > + > + # Try as a tag first > + if git fetch --depth=1 origin tag "$base_version" \ > + >> "$log_file" 2>&1; then > + log "Fetched tag $base_version" > + > + # Try as a branch > + elif git fetch --depth=1 origin "$base_version" \ > + >> "$log_file" 2>&1; then > + log "Fetched branch $base_version" > + > + else > + git fetch origin >> "$log_file" 2>&1 || true > + log "Fetched all refs from origin" > + fi > + > + # Try checkout > + if git checkout "$base_version" >> "$log_file" 2>&1; then > + log "Using $base_version from origin" > + else > + # origin might be a private repo w/o all branches. > + # Try ovn-org as fallback. > + log "Not in origin, fetching from ovn-org..." > + git fetch https://github.com/ovn-org/ovn.git \ > + "$base_version:$base_version" >> "$log_file" 2>&1 || > return 1 > + log "Fetched $base_version from ovn-org" > + git checkout "$base_version" >> "$log_file" 2>&1 || return 1 > + fi > + fi > + > + git submodule update --init >> "$log_file" 2>&1 || return 1 > +} > + > +# Patch base version's lflow.h with current OFTABLE table defines > +# This ensures ovn-debug uses correct table numbers > +function ovn_upgrade_patch_for_ovn_debug() > +{ > + if [ -f /tmp/ovn-upgrade-ofctl-defines.h ] && \ > + [ -f controller/lflow.h ]; then > + # Replace old OFCTL defines with current ones in one pass > + awk ' > + !inserted && /^#define OFTABLE_/ { > + system("cat /tmp/ovn-upgrade-ofctl-defines.h") > + inserted = 1 > + } > + /^#define OFTABLE_/ { next } > + { print } > + ' controller/lflow.h > controller/lflow.h.tmp > + > + mv controller/lflow.h.tmp controller/lflow.h > + fi > +} > + > +# ovn_upgrade_save_ovn_debug > +# Saves ovn-debug binary built with current OFTABLE defines > +# This creates a hybrid ovn-debug: current table numbers + base logical > flow > +# stages > +function ovn_upgrade_save_ovn_debug() > +{ > + log "Saving hybrid ovn-debug..." > + cp utilities/ovn-debug /tmp/ovn-upgrade-binaries/ovn-debug > +} > + > +# update_test old_first_table old_last_table shift test_file > +# Update test tables in test_file, for old_first <= tables < > old_last_table > +function update_test() > +{ > + test_file=$4 > + awk -v old_start=$1 \ > + -v old_end=$2 \ > + -v shift=$3 ' > + { > + result = "" > + rest = $0 > + # Process all table=NUMBER matches in the line > + while (match(rest, /table *= *[0-9]+/)) { > + # Save match position before calling match() again > + pos = RSTART > + len = RLENGTH > + > + # Add everything before the match > + result = result substr(rest, 1, pos-1) > + > + # Extract the matched text and the number > + matched = substr(rest, pos, len) > + if (match(matched, /[0-9]+/)) { > + num = substr(matched, RSTART, RLENGTH) > + } else { > + num = 0 > + } > + > + # Check if this table number needs updating > + if (num >= old_start && num < old_end) { > + result = result "table=" (num + shift) > + } else { > + result = result matched > + } > + > + # Continue with the rest of the line (use saved pos/len) > + rest = substr(rest, pos + len) > + } > + # Add any remaining text > + print result rest > + }' "$test_file" > "$test_file.tmp" && mv "$test_file.tmp" "$test_file" > +} > + > +# ovn_upgrade_table_numbers_in_tests_patch: fix hardcoded table numbers in > +# test files > +function ovn_upgrade_table_numbers_in_tests_patch() > +{ > + # Old tests (e.g., branch-24.03) have hardcoded numbers like > "table=45" > + # which refer to specific logical tables. When OFTABLE defines shift, > + # these numbers must be updated. > + # Example: v24.03.0 has OFTABLE_LOG_EGRESS_PIPELINE=42, so "table=45" > + # means egress+3. > + # In main, OFTABLE_LOG_EGRESS_PIPELINE=47, so it should become > "table=50". > + if [ ! -f /tmp/ovn-upgrade-new-log-egress.txt ] || > + [ ! -f /tmp/ovn-upgrade-new-save-inport.txt ]; then > + log "WARNING: Table shift data not found, skipping hardcoded > table \ > + number updates" > + return > + fi > + > + if [ ! -f controller/lflow.h ]; then > + log "WARNING: controller/lflow.h not found, skipping hardcoded > table \ > + number updates" > + return > + fi > + > + NEW_LOG_EGRESS=$(cat /tmp/ovn-upgrade-new-log-egress.txt) > + NEW_SAVE_INPORT=$(cat /tmp/ovn-upgrade-new-save-inport.txt) > + > + # Get old values from base version's lflow.h (before we patched it) > + LINE=$(grep "#define OFTABLE_LOG_EGRESS_PIPELINE" controller/lflow.h) > + OLD_LOG_EGRESS=$(echo "$LINE" | grep -oE '[0-9]+') > + LINE=$(grep "#define OFTABLE_SAVE_INPORT" controller/lflow.h) > + OLD_SAVE_INPORT=$(echo "$LINE" | grep -oE '[0-9]+') > + > + if [ -z "$OLD_LOG_EGRESS" ] || [ -z "$OLD_SAVE_INPORT" ] || \ > + [ "$OLD_LOG_EGRESS" == "$NEW_LOG_EGRESS" ]; then > + log "No change in tests files as old_log_egress=$OLD_LOG_EGRESS, > + old_save_inport=$OLD_SAVE_INPORT and > + new_log_egress=$NEW_LOG_EGRESS" > + return > + fi > + > + # Calculate the shift > + SHIFT=$((NEW_LOG_EGRESS - OLD_LOG_EGRESS)) > + > + log "Updating hardcoded table numbers in tests (shift: +$SHIFT for > tables \ > + $OLD_LOG_EGRESS-$((OLD_SAVE_INPORT-1)))" > + > + # Update hardcoded table numbers in test files > + for test_file in tests/system-ovn.at tests/system-ovn-kmod.at; do > + if [ -f "$test_file" ]; then > + log "Updating $test_file" > + update_test "$OLD_LOG_EGRESS" "$OLD_SAVE_INPORT" "$SHIFT" \ > + "$test_file" > + fi > + done > +} > + > +# ovn_upgrade_cleanup_sbox_patch: filter out expected schema warnings. > +function ovn_upgrade_cleanup_sbox_patch() > +{ > + cat << 'EOF' > /tmp/upgrade-schema-filter.patch > +diff --git a/tests/ovn-macros.at b/tests/ovn-macros.at > +index a08252d50..a31dc87b4 100644 > +--- a/tests/ovn-macros.at > ++++ b/tests/ovn-macros.at > +@@ -98,6 +98,7 @@ m4_define([OVN_CLEANUP_SBOX],[ > + $error > + /connection failed (No such file or directory)/d > + /has no network name*/d > ++ /OVN_Southbound database lacks/d > + /receive tunnel port not found*/d > + /Failed to locate tunnel to reach main chassis/d > + /Transaction causes multiple rows.*MAC_Binding/d > +diff --git a/tests/system-kmod-macros.at b/tests/system-kmod-macros.at > +index 6f6670199..4bd1a2c90 100644 > +--- a/tests/system-kmod-macros.at > ++++ b/tests/system-kmod-macros.at > +@@ -45,7 +45,8 @@ m4_define([OVS_TRAFFIC_VSWITCHD_START], > + # invoked. They can be used to perform additional cleanups such as name > space > + # removal. > + m4_define([OVS_TRAFFIC_VSWITCHD_STOP], > +- [OVS_VSWITCHD_STOP([$1]) > ++ [OVS_VSWITCHD_STOP([dnl > ++$1";/OVN_Southbound database lacks/d"]) > + AT_CHECK([:; $2]) > + ]) > + > +EOF > + > + # Try to apply schema filter patch. May fail on old OVN versions where > + # OVN_CLEANUP_SBOX doesn't check errors - this is expected and okay. > + # If patch fails for more recent OVN, then the test will fail due to > the > + # "OVN_Southbound database lacks". > + patch -p1 < /tmp/upgrade-schema-filter.patch > /dev/null 2>&1 || true > + rm -f /tmp/upgrade-schema-filter.patch > +} > + > +# ovn_upgrade_oftable_ovn_macro_patch: update table numbers in ovn-macro > +function ovn_upgrade_oftable_ovn_macro_patch() > +{ > + # Patch base version's tests/ovn-macros.at with current OFTABLE m4 > defines > + # This ensures tests use correct table numbers when checking flows > + if [ -f /tmp/ovn-upgrade-oftable-m4-defines.txt ] && > + [ -f tests/ovn-macros.at ]; then > + # Check if the base version has OFTABLE m4 defines > + if grep -q '^m4_define(\[OFTABLE_' tests/ovn-macros.at; then > + # Replace old m4_define OFTABLE statements with current ones > + awk ' > + !inserted && /^m4_define\(\[OFTABLE_/ { > + system("cat /tmp/ovn-upgrade-oftable-m4-defines.txt") > + inserted = 1 > + } > + /^m4_define\(\[OFTABLE_/ { next } > + { print } > + ' tests/ovn-macros.at > tests/ovn-macros.at.tmp > + > + mv tests/ovn-macros.at.tmp tests/ovn-macros.at > + fi > + fi > +} > + > +# Applies patches to base version after second build: > +# 1. Schema error patch (filters "OVN_Southbound database lacks" warnings) > +# 2. OFTABLE m4 defines patch in tests/ovn-macros.at (for test table > numbers) > +# 3. Hardcoded table numbers patch in test files > +function ovn_upgrade_apply_tests_patches() > +{ > + log "Applying schema filter and table number patches..." > + ovn_upgrade_table_numbers_in_tests_patch > + ovn_upgrade_cleanup_sbox_patch > + ovn_upgrade_oftable_ovn_macro_patch > +} > All of this patching is very fragile, if we are not careful it might easily break. I'm not sure what would be the best way to make it more robust. > + > +# ovn_upgrade_restore_binaries > +# > +# Replaces base version binaries with saved current versions: > +# - ovn-controller (from current) > +# - OVS binaries and schema (from current) > +# - ovn-debug (hybrid: current OFTABLE + base logical stages) > +function ovn_upgrade_restore_binaries() > +{ > + log "Replacing binaries with current versions" > + > + # Replace OVN controller > + cp /tmp/ovn-upgrade-binaries/ovn-controller controller/ovn-controller > + > + # Replace ovn-debug with hybrid version (built with current OFTABLE + > base > + # northd.h) > + cp /tmp/ovn-upgrade-binaries/ovn-debug utilities/ovn-debug > + > + # Replace OVS binaries > + cp /tmp/ovn-upgrade-binaries/ovs-vswitchd ovs/vswitchd/ovs-vswitchd > + cp /tmp/ovn-upgrade-binaries/ovsdb-server ovs/ovsdb/ovsdb-server > + cp /tmp/ovn-upgrade-binaries/ovs-vsctl ovs/utilities/ovs-vsctl > + cp /tmp/ovn-upgrade-binaries/ovs-ofctl ovs/utilities/ovs-ofctl > + cp /tmp/ovn-upgrade-binaries/ovs-appctl ovs/utilities/ovs-appctl > + cp /tmp/ovn-upgrade-binaries/ovs-dpctl ovs/utilities/ovs-dpctl > + > + # Replace OVS schema (current binaries expect current schema) > + cp /tmp/ovn-upgrade-binaries/vswitch.ovsschema \ > + ovs/vswitchd/vswitch.ovsschema > + > + echo "" > + log "Verification - Current versions (from current patch):" > + log " ovn-controller: $(controller/ovn-controller --version | > + grep ovn-controller)" > + log " SB DB Schema: $(controller/ovn-controller --version | > + grep "SB DB Schema")" > + log " ovs-vswitchd: $(ovs/vswitchd/ovs-vswitchd --version | grep > vSwitch)" > + log "Verification - Base versions (for compatibility testing):" > + log " ovn-northd: $(northd/ovn-northd --version | grep ovn-northd)" > + log " ovn-nbctl: $(utilities/ovn-nbctl --version | grep ovn-nbctl)" > +} > diff --git a/.ci/test-upgrade-local.sh b/.ci/test-upgrade-local.sh > new file mode 100755 > index 000000000..3818ac465 > --- /dev/null > +++ b/.ci/test-upgrade-local.sh > @@ -0,0 +1,202 @@ > +#!/bin/bash > + > +set -e > + > +. "$(dirname $0)/linux-util.sh" > + > +BASE_VERSION="${BASE_VERSION:-branch-24.03}" > +TEST_RANGE="${TEST_RANGE:-1-}" > +KEEPALIVE_INT=50 > + > +CLEANUP_DONE=0 > +TEST_STATUS=1 > + > +usage() { > + cat << EOF > +Usage: $0 [options] > + > +Test OVN upgrade compatibility without GitHub. > + > +Options: > + -b, --base-version VERSION Base version to test (default: > branch-24.03) > + -t, --test-range RANGE Test range to run (default: 1-) > + Examples: -100, 101-, 55 > + -h, --help Show this help message > + > +Environment Variables: > + BASE_VERSION Same as --base-version > + TEST_RANGE Same as --test-range > + > +Examples: > + # Test against branch-24.03 with all tests > + $0 > + > + # Test against specific version > + $0 --base-version v24.03.0 > + > + # Test specific test range > + $0 --test-range 101-200 > +EOF > +} > + > +# Parse command line arguments > +while [[ $# -gt 0 ]]; do > + case $1 in > + -b|--base-version) > + BASE_VERSION="$2" > + shift 2 > + ;; > + -t|--test-range) > + TEST_RANGE="$2" > + shift 2 > + ;; > + -h|--help) > + usage > + exit 0 > + ;; > + *) > + echo "Unknown option: $1" > + usage > + exit 1 > + ;; > + esac > +done > + > +log "========================================" > +log "OVN Upgrade Test" > +log "Base version: $BASE_VERSION" > +log "Test range: $TEST_RANGE" > + > +# Check if we're in the OVN repository root > +if [ ! -f "configure.ac" ] || ! grep -q "ovn" configure.ac; then > + log "Error: This script must be run from the OVN repository root" > + exit 1 > +fi > + > +start_sudo_keepalive() { > + (while true; do sudo -n true; sleep "$KEEPALIVE_INT"; done) > 2>/dev/null & > + SUDO_KEEPALIVE_PID=$! > + if ! kill -0 $SUDO_KEEPALIVE_PID 2>/dev/null; then > + log "ERROR: sudo keepalive failed to start" > + exit 1 > + fi > +} > + > +stop_sudo_keepalive() { > + if [ -n "$SUDO_KEEPALIVE_PID" ]; then > + kill $SUDO_KEEPALIVE_PID 2>/dev/null || true > + fi > +} > + > +# Cleanup function - always runs on exit > +cleanup() { > + if [ $CLEANUP_DONE -eq 1 ]; then > + return > + fi > + CLEANUP_DONE=1 > + > + stop_sudo_keepalive > + > + echo > + log "Cleaning up..." > + log "Restoring modified files..." > + # Restore OVN test files > + git checkout tests/ovn-macros.at tests/system-kmod-macros.at \ > + tests/system-ovn.at tests/system-ovn-kmod.at \ > + >> logs/git.log 2>&1 || true > + # Restore OVS submodule files > + (cd ovs && git checkout vswitchd/vswitch.ovsschema \ > + >> ../logs/git.log 2>&1 || true) > + # Restore CI scripts (may have been replaced during upgrade test) > + git checkout .ci/linux-build.sh .ci/linux-util.sh \ > + >> logs/git.log 2>&1 || true > + > + log "Restoring original branch/commit..." > + # If we were on a branch, restore to it; otherwise restore to commit > + if [ "$CURRENT_BRANCH" != "HEAD" ]; then > + if ! git checkout "$CURRENT_BRANCH" >> logs/git.log 2>&1; then > + log "WARNING: Failed to restore branch $CURRENT_BRANCH" >&2 > + fi > + else > + # We were in detached HEAD state, restore to the commit > + if ! git checkout "$CURRENT_COMMIT" >> logs/git.log 2>&1; then > + log "WARNING: Failed to restore commit $CURRENT_COMMIT" >&2 > + fi > + fi > + > + log "Updating submodules..." > + git submodule update --init >> logs/git.log 2>&1 || true > + log "Restored to: $CURRENT_BRANCH ($CURRENT_COMMIT)" > + > + # Cleanup temporary files > + rm -rf /tmp/ovn-upgrade-binaries /tmp/ovn-upgrade-ci > + rm -f /tmp/ovn-upgrade-ofctl-defines.h > + rm -f /tmp/ovn-upgrade-oftable-m4-defines.txt > + rm -f /tmp/ovn-upgrade-new-log-egress.txt > + rm -f /tmp/ovn-upgrade-new-save-inport.txt > +} > + > +trap cleanup EXIT INT TERM QUIT HUP > + > +# Request sudo credentials early > +if ! sudo -nv 2>/dev/null; then > + log "This script requires sudo for running system tests." > + log "Please enter your password now:" > + sudo -v || { > + log "Error: sudo authentication failed" > + exit 1 > + } > +fi > + > +start_sudo_keepalive > + > +# Save current branch/commit > +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) > +CURRENT_COMMIT=$(git rev-parse HEAD) > +log "Current branch: $CURRENT_BRANCH" > +log "Current commit: $CURRENT_COMMIT" > +echo > + > +# Check if working directory is clean > +if ! git diff-index --quiet HEAD --; then > + log "Warning: Working directory has uncommitted changes" > + read -p "Continue anyway? (y/n) " -n 1 -r > + echo > + if [[ ! $REPLY =~ ^[Yy]$ ]]; then > + exit 1 > + fi > +fi > + > +# Create logs directory > +mkdir -p logs > + > +# Export environment variables for linux-build.sh > +export TESTSUITE="upgrade-test" > +export BASE_VERSION="$BASE_VERSION" > +export TEST_RANGE="$TEST_RANGE" > +export JOBS="${JOBS:--j$(nproc 2>/dev/null || echo 4)}" > +export CC="${CC:-gcc}" > +export NO_DEBUG="${NO_DEBUG:-1}" # Disable verbose set -x output by > default > +export USE_SPARSE="${USE_SPARSE:-no}" # Disable sparse for local tests > + > +log "Running upgrade tests via linux-build.sh..." > +echo > + > +# Run linux-build.sh which will call execute_upgrade_tests() > +if ./.ci/linux-build.sh; then > + echo > + log "Upgrade test completed successfully" > + TEST_STATUS=0 > +else > + echo > + log "Upgrade test failed - check logs" > + TEST_STATUS=1 > +fi > + > +# Print summary > +echo > +log "Logs saved to:" > +log " - logs/git.log" > +log " - tests/system-kmod-testsuite.log" > + > +exit $TEST_STATUS > diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml > index b6e461129..06b677f55 100644 > --- a/.github/workflows/test.yml > +++ b/.github/workflows/test.yml > @@ -94,7 +94,6 @@ jobs: > > name: linux ${{ join(matrix.cfg.*, ' ') }} > runs-on: ubuntu-24.04 > - > nit: Unrelated change. > strategy: > fail-fast: false > matrix: > @@ -123,30 +122,42 @@ jobs: > - { compiler: clang, testsuite: system-test, sanitizers: > sanitizers, test_range: "-100" } > - { compiler: clang, testsuite: system-test, sanitizers: > sanitizers, test_range: "101-200" } > - { compiler: clang, testsuite: system-test, sanitizers: > sanitizers, test_range: "201-", unstable: unstable } > + - { compiler: gcc, testsuite: upgrade-test, base_version: > branch-24.03, test_range: "-100" } > + - { compiler: gcc, testsuite: upgrade-test, base_version: > branch-24.03, test_range: "101-", unstable: unstable } > + - { compiler: gcc, testsuite: upgrade-test, base_version: > branch-24.09, test_range: "-100", run_on: schedule } > + - { compiler: gcc, testsuite: upgrade-test, base_version: > branch-24.09, test_range: "101-200", run_on: schedule } > + - { compiler: gcc, testsuite: upgrade-test, base_version: > branch-24.09, test_range: "201-", unstable: unstable, run_on: schedule } > + - { compiler: gcc, testsuite: upgrade-test, base_version: > branch-25.03, test_range: "-100", run_on: schedule } > + - { compiler: gcc, testsuite: upgrade-test, base_version: > branch-25.03, test_range: "101-200", run_on: schedule } > + - { compiler: gcc, testsuite: upgrade-test, base_version: > branch-25.03, test_range: "201-", unstable: unstable, run_on: schedule } > + - { compiler: gcc, testsuite: upgrade-test, base_version: > branch-25.09, test_range: "-100" } > + - { compiler: gcc, testsuite: upgrade-test, base_version: > branch-25.09, test_range: "101-200" } > + - { compiler: gcc, testsuite: upgrade-test, base_version: > branch-25.09, test_range: "201-", unstable: unstable } > I know it will result in a certain amount of duplication but we should add this into separate step IMO. It is very hard to track what runs when. I also wonder if it wouldn't be enough to run all those on schedule weekly like we do with multinode tests, WDYT? > - { arch: x86, compiler: gcc, opts: --disable-ssl } > > steps: > - name: system-level-dependencies > - if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }} > + if: ${{ (startsWith(matrix.cfg.testsuite, 'system-test') || > (matrix.cfg.testsuite == 'upgrade-test')) && (matrix.cfg.run_on != > 'schedule' || github.event_name == 'schedule') }} > run: | > sudo apt update > sudo apt -y install linux-modules-extra-$(uname -r) > > - name: checkout > - if: github.event_name == 'push' || github.event_name == > 'pull_request' > + if: (github.event_name == 'push' || github.event_name == > 'pull_request' || matrix.cfg.testsuite == 'upgrade-test') && > (matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule') > uses: actions/checkout@v4 > with: > submodules: recursive > + fetch-depth: ${{ matrix.cfg.testsuite == 'upgrade-test' && 0 || 1 > }} > > - # For weekly runs, don't update submodules > + # For weekly runs (no upgrade-tests), don't update submodules > - name: checkout without submodule > - if: github.event_name == 'schedule' > + if: github.event_name == 'schedule' && matrix.cfg.testsuite != > 'upgrade-test' > uses: actions/checkout@v4 > > - # Weekly runs test using the tip of the most recent stable OVS branch > + # Weekly runs (no upgrade-tests) test using the tip of the most > recent stable OVS branch > # instead of the submodule. > - name: checkout OVS > - if: github.event_name == 'schedule' > + if: github.event_name == 'schedule' && matrix.cfg.testsuite != > 'upgrade-test' > uses: actions/checkout@v4 > with: > repository: 'openvswitch/ovs' > @@ -154,7 +165,7 @@ jobs: > path: 'ovs' > > - name: checkout OVS most recent stable branch. > - if: github.event_name == 'schedule' > + if: github.event_name == 'schedule' && matrix.cfg.testsuite != > 'upgrade-test' > run: | > git checkout \ > $(git branch -a -l '*branch-*' | sed 's/remotes\/origin\///' | \ > @@ -162,16 +173,19 @@ jobs: > working-directory: ovs > > - name: Fix /etc/hosts file > + if: matrix.cfg.run_on != 'schedule' || github.event_name == > 'schedule' > run: | > . .ci/linux-util.sh > fix_etc_hosts > > - name: Disable apparmor > + if: matrix.cfg.run_on != 'schedule' || github.event_name == > 'schedule' > run: | > . .ci/linux-util.sh > disable_apparmor > > - name: image cache > + if: matrix.cfg.run_on != 'schedule' || github.event_name == > 'schedule' > id: image_cache > uses: actions/cache@v4 > with: > @@ -179,17 +193,24 @@ jobs: > key: ${{ github.sha }}/${{ github.event_name }} > > - name: load image > + if: matrix.cfg.run_on != 'schedule' || github.event_name == > 'schedule' > run: | > sudo podman load -i /tmp/image.tar > podman load -i /tmp/image.tar > rm -rf /tmp/image.tar > > + # Set BASE_VERSION env var for upgrade tests > + - name: Set upgrade test env > + if: matrix.cfg.testsuite == 'upgrade-test' && (matrix.cfg.run_on != > 'schedule' || github.event_name == 'schedule') > + run: echo "BASE_VERSION=${{ matrix.cfg.base_version }}" >> > $GITHUB_ENV > + > + # Regular build steps > - name: build > - if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }} > + if: ${{ (startsWith(matrix.cfg.testsuite, 'system-test') || > (matrix.cfg.testsuite == 'upgrade-test')) && (matrix.cfg.run_on != > 'schedule' || github.event_name == 'schedule') }} > run: sudo -E ./.ci/ci.sh --archive-logs --timeout=2h > > - name: build > - if: ${{ !startsWith(matrix.cfg.testsuite, 'system-test') }} > + if: ${{ !startsWith(matrix.cfg.testsuite, 'system-test') && > matrix.cfg.testsuite != 'upgrade-test' && (matrix.cfg.run_on != 'schedule' > || github.event_name == 'schedule') }} > run: ./.ci/ci.sh --archive-logs --timeout=2h > > - name: upload logs on failure > diff --git a/Makefile.am b/Makefile.am > index 3ad2077b3..e57bfb297 100644 > --- a/Makefile.am > +++ b/Makefile.am > @@ -91,6 +91,7 @@ EXTRA_DIST = \ > .ci/linux-util.sh \ > .ci/osx-build.sh \ > .ci/osx-prepare.sh \ > + .ci/test-upgrade-local.sh \ > .ci/ovn-kubernetes/Dockerfile \ > .ci/ovn-kubernetes/prepare.sh \ > .ci/ovn-kubernetes/custom.patch \ > diff --git a/tests/ovn-macros.at b/tests/ovn-macros.at > index 1c981e5a8..834ed60be 100644 > --- a/tests/ovn-macros.at > +++ b/tests/ovn-macros.at > @@ -1520,6 +1520,12 @@ m4_define([TAG_UNSTABLE], [ > AT_SKIP_IF([test X"$SKIP_UNSTABLE" = Xyes]) > ]) > > +# TAG_TEST_NOT_UPGRADABLE tag indicates that the test would fail > +# "upgrade" test (i.e. running old ovn-northd and new ovn-controller) > +m4_define([TAG_TEST_NOT_UPGRADABLE], [ > + AT_SKIP_IF([test X"$UPGRADE_TEST" = Xyes]) > +]) > + > m4_define([OVN_CHECK_SCAPY_EDNS_CLIENT_SUBNET_SUPPORT], > [ > AT_SKIP_IF([test $HAVE_SCAPY = no]) > -- > 2.47.1 > > _______________________________________________ > dev mailing list > [email protected] > https://mail.openvswitch.org/mailman/listinfo/ovs-dev > > Regards, Ales _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
