On Wed, Jan 21, 2026 at 9:41 AM Ales Musil <[email protected]> wrote:
> > > On Mon, Jan 19, 2026 at 2:16 PM Xavier Simonart via dev < > [email protected]> wrote: > >> When ovn is upgraded, ovn-controller is updated first on >> the compute nodes. Then ovn-northd and DB are upgraded. >> This patch tests whether the intermediate state (i.e. with >> ovn-controller being upgraded) works properly, running system >> tests from the base line (i.e. before the upgrade). >> >> Flow tables might change between releases. >> Hence this patch must take that into account by updating the (old) >> system tests with any updated table numbers. >> In some cases, (new) ovn-controller might change flows in existing >> tables, causing some 'upgrade' tests to fail. >> Such tests can be skipped using the TAG_TEST_NOT_UPGRADABLE tag. >> >> This patch upgrades the ci to run automatically some upgrade tests >> weekly. It also provides a shell script to run those tests locally. >> >> Upgrade-tests are run on push/pull only for LTS (24.03) and latest >> release (25.09) to avoid too long tests. Upgrades from other >> branches are run on schedule. >> >> This patch depends on patch [1] on branch-25.09. >> >> [1] "tests: Add new TAG_TEST_NOT_UPGRADABLE to some tests." >> >> Reported-at: https://issues.redhat.com/browse/FDP-1240 >> Signed-off-by: Xavier Simonart <[email protected]> >> --- >> > > Hi Xaiver, > > thank you for working on this. I have some comments down below. > > > >> .ci/ci.sh | 1 + >> .ci/linux-build.sh | 95 +++++++++- >> .ci/linux-util.sh | 367 +++++++++++++++++++++++++++++++++++++ >> .ci/test-upgrade-local.sh | 202 ++++++++++++++++++++ >> .github/workflows/test.yml | 41 ++++- >> Makefile.am | 1 + >> tests/ovn-macros.at | 6 + >> 7 files changed, 697 insertions(+), 16 deletions(-) >> create mode 100755 .ci/test-upgrade-local.sh >> >> diff --git a/.ci/ci.sh b/.ci/ci.sh >> index 3640d3243..6798fbd78 100755 >> --- a/.ci/ci.sh >> +++ b/.ci/ci.sh >> @@ -102,6 +102,7 @@ function run_tests() { >> ARCH=$ARCH CC=$CC LIBS=$LIBS OPTS=$OPTS TESTSUITE=$TESTSUITE \ >> TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS DPDK=$DPDK \ >> RECHECK=$RECHECK UNSTABLE=$UNSTABLE TIMEOUT=$TIMEOUT \ >> + BASE_VERSION=$BASE_VERSION \ >> ./.ci/linux-build.sh >> " >> } >> diff --git a/.ci/linux-build.sh b/.ci/linux-build.sh >> index 183833a16..890f4c59d 100755 >> --- a/.ci/linux-build.sh >> +++ b/.ci/linux-build.sh >> @@ -1,7 +1,10 @@ >> #!/bin/bash >> >> set -o errexit >> -set -x >> +# Enable debug output for CI, optional for local >> +if [ "${NO_DEBUG:-0}" = "0" ]; then >> + set -x >> +fi >> >> ARCH=${ARCH:-"x86_64"} >> USE_SPARSE=${USE_SPARSE:-"yes"} >> @@ -181,7 +184,7 @@ function run_system_tests() >> >> if ! sudo timeout -k 5m -v $TIMEOUT make $JOBS $type \ >> TESTSUITEFLAGS="$TEST_RANGE" RECHECK=$RECHECK \ >> - SKIP_UNSTABLE=$SKIP_UNSTABLE; then >> + SKIP_UNSTABLE=$SKIP_UNSTABLE UPGRADE_TEST=$UPGRADE_TEST; then >> # $log_file is necessary for debugging. >> cat tests/$log_file >> return 1 >> @@ -190,19 +193,28 @@ function run_system_tests() >> >> function execute_system_tests() >> { >> - configure_ovn $OPTS >> - make $JOBS || { cat config.log; exit 1; } >> + local test_type=$1 >> + local log_file=$2 >> + local skip_build=$3 >> + >> + # Only build if not already built (upgrade tests build separately) >> + if [ "$skip_build" != "yes" ]; then >> + configure_ovn $OPTS >> + make $JOBS || { cat config.log; exit 1; } >> + fi >> >> local stable_rc=0 >> local unstable_rc=0 >> >> - if ! SKIP_UNSTABLE=yes run_system_tests $@; then >> + if ! SKIP_UNSTABLE=yes UPGRADE_TEST=$UPGRADE_TEST \ >> + run_system_tests $test_type $log_file; then >> stable_rc=1 >> fi >> >> if [ "$UNSTABLE" ]; then >> if ! SKIP_UNSTABLE=no TEST_RANGE="-k unstable" RECHECK=yes \ >> - run_system_tests $@; then >> + UPGRADE_TEST=$UPGRADE_TEST run_system_tests $test_type \ >> + $log_file; >> then >> unstable_rc=1 >> fi >> fi >> @@ -212,6 +224,72 @@ function execute_system_tests() >> fi >> } >> >> +function execute_upgrade_tests() >> +{ >> + . .ci/linux-util.sh >> + >> + # Save current CI scripts (will be replaced by base version after >> checkout) >> + cp -rf .ci /tmp/ovn-upgrade-ci >> + >> + # Build current version >> + log "Building current version..." >> + mkdir -p logs >> + configure_ovn $OPTS >> logs/build-current.log 2>&1 || { >> + log "configure ovn failed - see config.log and >> logs/build-current.log" >> + exit 1 >> + } >> + make $JOBS >> logs/build-current.log 2>&1 || { >> + log "building ovn failed - see logs/build-current.log" >> + exit 1 >> + } >> + >> + ovn_upgrade_save_current_binaries >> + >> + # Checkout base version >> + ovn_upgrade_checkout_base "$BASE_VERSION" logs/git.log >> + >> + # Clean from current version >> + log "Cleaning build artifacts..." >> + make distclean >> logs/build-base.log 2>&1 || true >> + (cd ovs && make distclean >> ../logs/build-base.log 2>&1) || true >> + >> + # Apply test patches >> + ovn_upgrade_apply_tests_patches >> + >> + # Build base with patches >> + ovn_upgrade_patch_for_ovn_debug >> + >> + # Build (modified) base version >> + log "Building base version (with patched lflow.h)..." >> + configure_ovn $OPTS >> logs/build-base.log 2>&1 || { >> + log "configure ovn failed - see config.log and >> logs/build-base.log" >> + exit 1 >> + } >> + make $JOBS >> logs/build-base.log 2>&1 || { >> + log "building ovn failed - see logs/build-base.log" >> + exit 1 >> + } >> + ovn_upgrade_save_ovn_debug >> + >> + # Build (clean) base version >> + log "Rebuilding base version (clean lflow.h)..." >> + git checkout controller/lflow.h >> logs/git.log 2>&1 >> + make $JOBS >> logs/build-base.log 2>&1 || { >> + log "building ovn failed - see logs/build-base.log" >> + exit 1 >> + } >> + >> + # Restore binaries >> + ovn_upgrade_restore_binaries >> + >> + # Restore current CI scripts for test execution >> + cp -f /tmp/ovn-upgrade-ci/linux-build.sh .ci/linux-build.sh >> + cp -f /tmp/ovn-upgrade-ci/linux-util.sh .ci/linux-util.sh >> + >> + UPGRADE_TEST=yes execute_system_tests "check-kernel" \ >> + "system-kmod-testsuite.log" >> "yes" >> +} >> + >> configure_$CC >> >> if [ "$TESTSUITE" ]; then >> @@ -238,6 +316,11 @@ if [ "$TESTSUITE" ]; then >> sudo bash -c "echo 2048 > >> /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" >> execute_system_tests "check-system-dpdk" >> "system-dpdk-testsuite.log" >> ;; >> + >> + "upgrade-test") >> + execute_upgrade_tests >> + ;; >> + >> esac >> else >> configure_ovn $OPTS >> diff --git a/.ci/linux-util.sh b/.ci/linux-util.sh >> index b5bd1f8c9..a62ff2ea0 100755 >> --- a/.ci/linux-util.sh >> +++ b/.ci/linux-util.sh >> @@ -49,3 +49,370 @@ function disable_apparmor() >> sudo aa-teardown || true >> sudo systemctl disable --now apparmor.service >> } >> + >> +function log() { >> + echo "[$(date '+%H:%M:%S')] $*" >> +} >> + >> +# ovn_upgrade_save_current_binaries >> +# Saves current version's binaries and schemas to >> /tmp/ovn-upgrade-binaries/ >> +function ovn_upgrade_save_current_binaries() >> +{ >> + mkdir -p /tmp/ovn-upgrade-binaries >> + >> + # New ovn-controller may generate OpenFlow flows with new actions >> that old >> + # OVS doesn't understand, so we need also to use new OVS. >> + # New ovs-vswitchd binary expects columns/tables defined in the >> current >> + # schema, so we also need to use new schemas. >> + files="controller/ovn-controller ovs/vswitchd/ovs-vswitchd >> + ovs/ovsdb/ovsdb-server ovs/utilities/ovs-vsctl >> + ovs/utilities/ovs-ofctl ovs/utilities/ovs-appctl >> + ovs/utilities/ovs-dpctl ovs/vswitchd/vswitch.ovsschema" >> + for file in $files; do >> + if [ ! -f "$file" ]; then >> + log "ERROR: $file not found" >> + return 1 >> + fi >> + cp "$file" /tmp/ovn-upgrade-binaries/ >> + done >> + >> + # In the upgrade scenario we use old ovn-northd and new >> ovn-controller. >> + # OFTABLES are defined through a combination of northd/northd.h and >> + # controller/lflow.h. Tests uses either (old) table numbers, table >> names >> + # (defined in ovn-macros) or ovn-debug. >> + # >> + # Extract OFCTL_* table defines from current lflow.h >> + if ! grep '^#define OFTABLE_' controller/lflow.h > \ >> + /tmp/ovn-upgrade-ofctl-defines.h; then >> + log "No #define OFTABLE_ found in lflow.h" >> + return 1 >> + fi >> + >> + # Extract OFTABLE m4 defines from current tests/ovn-macros.at >> + # These are used by tests to reference table numbers >> + # In old tests, there might be no OFTABLE_ in ovn-macros, so grep >> can fail. >> + grep '^m4_define(\[OFTABLE_' tests/ovn-macros.at > \ >> + /tmp/ovn-upgrade-oftable-m4-defines.txt || true >> + >> + # Extract key table numbers for calculating shifts in hardcoded table >> + # references. OFTABLE_SAVE_INPORT is where normal (unshifted) tables >> + # resume. >> + LINE=$(grep "define OFTABLE_LOG_EGRESS_PIPELINE" controller/lflow.h) >> + NEW_LOG_EGRESS=$(echo "$LINE" | grep -oE '[0-9]+') >> + LINE=$(grep "define OFTABLE_SAVE_INPORT" controller/lflow.h) >> + NEW_SAVE_INPORT=$(echo "$LINE" | grep -oE '[0-9]+') >> + if [ -z "$NEW_LOG_EGRESS" ]; then >> + log "ERROR: Could not extract OFTABLE_LOG_EGRESS_PIPELINE value" >> + return 1 >> + fi >> + if [ -z "$NEW_SAVE_INPORT" ]; then >> + log "ERROR: Could not extract OFTABLE_SAVE_INPORT value" >> + return 1 >> + fi >> + >> + echo "$NEW_LOG_EGRESS" > /tmp/ovn-upgrade-new-log-egress.txt >> + echo "$NEW_SAVE_INPORT" > /tmp/ovn-upgrade-new-save-inport.txt >> + >> + echo "" >> + log "Saved current versions:" >> + log " ovn-controller:$(/tmp/ovn-upgrade-binaries/ovn-controller >> --version | >> + grep ovn-controller)" >> + log " SB DB schema:$(/tmp/ovn-upgrade-binaries/ovn-controller >> --version | >> + grep "SB DB Schema")" >> + log " ovs-vswitchd:$(/tmp/ovn-upgrade-binaries/ovs-vswitchd >> --version | >> + grep vSwitch)" >> +} >> + >> +# ovn_upgrade_checkout_base BASE_VERSION LOG_FILE >> +# Checks out base version from git >> +function ovn_upgrade_checkout_base() >> +{ >> + local base_version=$1 >> + local log_file=$2 >> + >> + log "Checking out base version: $base_version" >> + >> + # Try to checkout directly first (might already exist locally) >> + if git checkout "$base_version" >> "$log_file" 2>&1; then >> + log "Using locally available $base_version" >> + else >> + # Not available locally, try to fetch it >> + log "Fetching $base_version from origin..." >> + >> + # Try as a tag first >> + if git fetch --depth=1 origin tag "$base_version" \ >> + >> "$log_file" 2>&1; then >> + log "Fetched tag $base_version" >> + >> + # Try as a branch >> + elif git fetch --depth=1 origin "$base_version" \ >> + >> "$log_file" 2>&1; then >> + log "Fetched branch $base_version" >> + >> + else >> + git fetch origin >> "$log_file" 2>&1 || true >> + log "Fetched all refs from origin" >> + fi >> + >> + # Try checkout >> + if git checkout "$base_version" >> "$log_file" 2>&1; then >> + log "Using $base_version from origin" >> + else >> + # origin might be a private repo w/o all branches. >> + # Try ovn-org as fallback. >> + log "Not in origin, fetching from ovn-org..." >> + git fetch https://github.com/ovn-org/ovn.git \ >> + "$base_version:$base_version" >> "$log_file" 2>&1 || >> return 1 >> + log "Fetched $base_version from ovn-org" >> + git checkout "$base_version" >> "$log_file" 2>&1 || return 1 >> + fi >> + fi >> + >> + git submodule update --init >> "$log_file" 2>&1 || return 1 >> +} >> + >> +# Patch base version's lflow.h with current OFTABLE table defines >> +# This ensures ovn-debug uses correct table numbers >> +function ovn_upgrade_patch_for_ovn_debug() >> +{ >> + if [ -f /tmp/ovn-upgrade-ofctl-defines.h ] && \ >> + [ -f controller/lflow.h ]; then >> + # Replace old OFCTL defines with current ones in one pass >> + awk ' >> + !inserted && /^#define OFTABLE_/ { >> + system("cat /tmp/ovn-upgrade-ofctl-defines.h") >> + inserted = 1 >> + } >> + /^#define OFTABLE_/ { next } >> + { print } >> + ' controller/lflow.h > controller/lflow.h.tmp >> + >> + mv controller/lflow.h.tmp controller/lflow.h >> + fi >> +} >> + >> +# ovn_upgrade_save_ovn_debug >> +# Saves ovn-debug binary built with current OFTABLE defines >> +# This creates a hybrid ovn-debug: current table numbers + base logical >> flow >> +# stages >> +function ovn_upgrade_save_ovn_debug() >> +{ >> + log "Saving hybrid ovn-debug..." >> + cp utilities/ovn-debug /tmp/ovn-upgrade-binaries/ovn-debug >> +} >> + >> +# update_test old_first_table old_last_table shift test_file >> +# Update test tables in test_file, for old_first <= tables < >> old_last_table >> +function update_test() >> +{ >> + test_file=$4 >> + awk -v old_start=$1 \ >> + -v old_end=$2 \ >> + -v shift=$3 ' >> + { >> + result = "" >> + rest = $0 >> + # Process all table=NUMBER matches in the line >> + while (match(rest, /table *= *[0-9]+/)) { >> + # Save match position before calling match() again >> + pos = RSTART >> + len = RLENGTH >> + >> + # Add everything before the match >> + result = result substr(rest, 1, pos-1) >> + >> + # Extract the matched text and the number >> + matched = substr(rest, pos, len) >> + if (match(matched, /[0-9]+/)) { >> + num = substr(matched, RSTART, RLENGTH) >> + } else { >> + num = 0 >> + } >> + >> + # Check if this table number needs updating >> + if (num >= old_start && num < old_end) { >> + result = result "table=" (num + shift) >> + } else { >> + result = result matched >> + } >> + >> + # Continue with the rest of the line (use saved pos/len) >> + rest = substr(rest, pos + len) >> + } >> + # Add any remaining text >> + print result rest >> + }' "$test_file" > "$test_file.tmp" && mv "$test_file.tmp" >> "$test_file" >> +} >> + >> +# ovn_upgrade_table_numbers_in_tests_patch: fix hardcoded table numbers >> in >> +# test files >> +function ovn_upgrade_table_numbers_in_tests_patch() >> +{ >> + # Old tests (e.g., branch-24.03) have hardcoded numbers like >> "table=45" >> + # which refer to specific logical tables. When OFTABLE defines shift, >> + # these numbers must be updated. >> + # Example: v24.03.0 has OFTABLE_LOG_EGRESS_PIPELINE=42, so "table=45" >> + # means egress+3. >> + # In main, OFTABLE_LOG_EGRESS_PIPELINE=47, so it should become >> "table=50". >> + if [ ! -f /tmp/ovn-upgrade-new-log-egress.txt ] || >> + [ ! -f /tmp/ovn-upgrade-new-save-inport.txt ]; then >> + log "WARNING: Table shift data not found, skipping hardcoded >> table \ >> + number updates" >> + return >> + fi >> + >> + if [ ! -f controller/lflow.h ]; then >> + log "WARNING: controller/lflow.h not found, skipping hardcoded >> table \ >> + number updates" >> + return >> + fi >> + >> + NEW_LOG_EGRESS=$(cat /tmp/ovn-upgrade-new-log-egress.txt) >> + NEW_SAVE_INPORT=$(cat /tmp/ovn-upgrade-new-save-inport.txt) >> + >> + # Get old values from base version's lflow.h (before we patched it) >> + LINE=$(grep "#define OFTABLE_LOG_EGRESS_PIPELINE" controller/lflow.h) >> + OLD_LOG_EGRESS=$(echo "$LINE" | grep -oE '[0-9]+') >> + LINE=$(grep "#define OFTABLE_SAVE_INPORT" controller/lflow.h) >> + OLD_SAVE_INPORT=$(echo "$LINE" | grep -oE '[0-9]+') >> + >> + if [ -z "$OLD_LOG_EGRESS" ] || [ -z "$OLD_SAVE_INPORT" ] || \ >> + [ "$OLD_LOG_EGRESS" == "$NEW_LOG_EGRESS" ]; then >> + log "No change in tests files as old_log_egress=$OLD_LOG_EGRESS, >> + old_save_inport=$OLD_SAVE_INPORT and >> + new_log_egress=$NEW_LOG_EGRESS" >> + return >> + fi >> + >> + # Calculate the shift >> + SHIFT=$((NEW_LOG_EGRESS - OLD_LOG_EGRESS)) >> + >> + log "Updating hardcoded table numbers in tests (shift: +$SHIFT for >> tables \ >> + $OLD_LOG_EGRESS-$((OLD_SAVE_INPORT-1)))" >> + >> + # Update hardcoded table numbers in test files >> + for test_file in tests/system-ovn.at tests/system-ovn-kmod.at; do >> + if [ -f "$test_file" ]; then >> + log "Updating $test_file" >> + update_test "$OLD_LOG_EGRESS" "$OLD_SAVE_INPORT" "$SHIFT" \ >> + "$test_file" >> + fi >> + done >> +} >> + >> +# ovn_upgrade_cleanup_sbox_patch: filter out expected schema warnings. >> +function ovn_upgrade_cleanup_sbox_patch() >> +{ >> + cat << 'EOF' > /tmp/upgrade-schema-filter.patch >> +diff --git a/tests/ovn-macros.at b/tests/ovn-macros.at >> +index a08252d50..a31dc87b4 100644 >> +--- a/tests/ovn-macros.at >> ++++ b/tests/ovn-macros.at >> +@@ -98,6 +98,7 @@ m4_define([OVN_CLEANUP_SBOX],[ >> + $error >> + /connection failed (No such file or directory)/d >> + /has no network name*/d >> ++ /OVN_Southbound database lacks/d >> + /receive tunnel port not found*/d >> + /Failed to locate tunnel to reach main chassis/d >> + /Transaction causes multiple rows.*MAC_Binding/d >> +diff --git a/tests/system-kmod-macros.at b/tests/system-kmod-macros.at >> +index 6f6670199..4bd1a2c90 100644 >> +--- a/tests/system-kmod-macros.at >> ++++ b/tests/system-kmod-macros.at >> +@@ -45,7 +45,8 @@ m4_define([OVS_TRAFFIC_VSWITCHD_START], >> + # invoked. They can be used to perform additional cleanups such as name >> space >> + # removal. >> + m4_define([OVS_TRAFFIC_VSWITCHD_STOP], >> +- [OVS_VSWITCHD_STOP([$1]) >> ++ [OVS_VSWITCHD_STOP([dnl >> ++$1";/OVN_Southbound database lacks/d"]) >> + AT_CHECK([:; $2]) >> + ]) >> + >> +EOF >> + >> + # Try to apply schema filter patch. May fail on old OVN versions >> where >> + # OVN_CLEANUP_SBOX doesn't check errors - this is expected and okay. >> + # If patch fails for more recent OVN, then the test will fail due to >> the >> + # "OVN_Southbound database lacks". >> + patch -p1 < /tmp/upgrade-schema-filter.patch > /dev/null 2>&1 || true >> + rm -f /tmp/upgrade-schema-filter.patch >> +} >> + >> +# ovn_upgrade_oftable_ovn_macro_patch: update table numbers in ovn-macro >> +function ovn_upgrade_oftable_ovn_macro_patch() >> +{ >> + # Patch base version's tests/ovn-macros.at with current OFTABLE m4 >> defines >> + # This ensures tests use correct table numbers when checking flows >> + if [ -f /tmp/ovn-upgrade-oftable-m4-defines.txt ] && >> + [ -f tests/ovn-macros.at ]; then >> + # Check if the base version has OFTABLE m4 defines >> + if grep -q '^m4_define(\[OFTABLE_' tests/ovn-macros.at; then >> + # Replace old m4_define OFTABLE statements with current ones >> + awk ' >> + !inserted && /^m4_define\(\[OFTABLE_/ { >> + system("cat /tmp/ovn-upgrade-oftable-m4-defines.txt") >> + inserted = 1 >> + } >> + /^m4_define\(\[OFTABLE_/ { next } >> + { print } >> + ' tests/ovn-macros.at > tests/ovn-macros.at.tmp >> + >> + mv tests/ovn-macros.at.tmp tests/ovn-macros.at >> + fi >> + fi >> +} >> + >> +# Applies patches to base version after second build: >> +# 1. Schema error patch (filters "OVN_Southbound database lacks" >> warnings) >> +# 2. OFTABLE m4 defines patch in tests/ovn-macros.at (for test table >> numbers) >> +# 3. Hardcoded table numbers patch in test files >> +function ovn_upgrade_apply_tests_patches() >> +{ >> + log "Applying schema filter and table number patches..." >> + ovn_upgrade_table_numbers_in_tests_patch >> + ovn_upgrade_cleanup_sbox_patch >> + ovn_upgrade_oftable_ovn_macro_patch >> +} >> > > All of this patching is very fragile, if we are not careful it might > easily break. I'm not sure what would be the best way to make > it more robust. > After some offline discussion we have agreed that some of the new additions might be better written in python rather than bash. Especially the OFTABLE_ definition replacements. > >> + >> +# ovn_upgrade_restore_binaries >> +# >> +# Replaces base version binaries with saved current versions: >> +# - ovn-controller (from current) >> +# - OVS binaries and schema (from current) >> +# - ovn-debug (hybrid: current OFTABLE + base logical stages) >> +function ovn_upgrade_restore_binaries() >> +{ >> + log "Replacing binaries with current versions" >> + >> + # Replace OVN controller >> + cp /tmp/ovn-upgrade-binaries/ovn-controller controller/ovn-controller >> + >> + # Replace ovn-debug with hybrid version (built with current OFTABLE >> + base >> + # northd.h) >> + cp /tmp/ovn-upgrade-binaries/ovn-debug utilities/ovn-debug >> + >> + # Replace OVS binaries >> + cp /tmp/ovn-upgrade-binaries/ovs-vswitchd ovs/vswitchd/ovs-vswitchd >> + cp /tmp/ovn-upgrade-binaries/ovsdb-server ovs/ovsdb/ovsdb-server >> + cp /tmp/ovn-upgrade-binaries/ovs-vsctl ovs/utilities/ovs-vsctl >> + cp /tmp/ovn-upgrade-binaries/ovs-ofctl ovs/utilities/ovs-ofctl >> + cp /tmp/ovn-upgrade-binaries/ovs-appctl ovs/utilities/ovs-appctl >> + cp /tmp/ovn-upgrade-binaries/ovs-dpctl ovs/utilities/ovs-dpctl >> + >> + # Replace OVS schema (current binaries expect current schema) >> + cp /tmp/ovn-upgrade-binaries/vswitch.ovsschema \ >> + ovs/vswitchd/vswitch.ovsschema >> + >> + echo "" >> + log "Verification - Current versions (from current patch):" >> + log " ovn-controller: $(controller/ovn-controller --version | >> + grep ovn-controller)" >> + log " SB DB Schema: $(controller/ovn-controller --version | >> + grep "SB DB Schema")" >> + log " ovs-vswitchd: $(ovs/vswitchd/ovs-vswitchd --version | grep >> vSwitch)" >> + log "Verification - Base versions (for compatibility testing):" >> + log " ovn-northd: $(northd/ovn-northd --version | grep ovn-northd)" >> + log " ovn-nbctl: $(utilities/ovn-nbctl --version | grep ovn-nbctl)" >> +} >> diff --git a/.ci/test-upgrade-local.sh b/.ci/test-upgrade-local.sh >> new file mode 100755 >> index 000000000..3818ac465 >> --- /dev/null >> +++ b/.ci/test-upgrade-local.sh >> @@ -0,0 +1,202 @@ >> +#!/bin/bash >> + >> +set -e >> + >> +. "$(dirname $0)/linux-util.sh" >> + >> +BASE_VERSION="${BASE_VERSION:-branch-24.03}" >> +TEST_RANGE="${TEST_RANGE:-1-}" >> +KEEPALIVE_INT=50 >> + >> +CLEANUP_DONE=0 >> +TEST_STATUS=1 >> + >> +usage() { >> + cat << EOF >> +Usage: $0 [options] >> + >> +Test OVN upgrade compatibility without GitHub. >> + >> +Options: >> + -b, --base-version VERSION Base version to test (default: >> branch-24.03) >> + -t, --test-range RANGE Test range to run (default: 1-) >> + Examples: -100, 101-, 55 >> + -h, --help Show this help message >> + >> +Environment Variables: >> + BASE_VERSION Same as --base-version >> + TEST_RANGE Same as --test-range >> + >> +Examples: >> + # Test against branch-24.03 with all tests >> + $0 >> + >> + # Test against specific version >> + $0 --base-version v24.03.0 >> + >> + # Test specific test range >> + $0 --test-range 101-200 >> +EOF >> +} >> + >> +# Parse command line arguments >> +while [[ $# -gt 0 ]]; do >> + case $1 in >> + -b|--base-version) >> + BASE_VERSION="$2" >> + shift 2 >> + ;; >> + -t|--test-range) >> + TEST_RANGE="$2" >> + shift 2 >> + ;; >> + -h|--help) >> + usage >> + exit 0 >> + ;; >> + *) >> + echo "Unknown option: $1" >> + usage >> + exit 1 >> + ;; >> + esac >> +done >> + >> +log "========================================" >> +log "OVN Upgrade Test" >> +log "Base version: $BASE_VERSION" >> +log "Test range: $TEST_RANGE" >> + >> +# Check if we're in the OVN repository root >> +if [ ! -f "configure.ac" ] || ! grep -q "ovn" configure.ac; then >> + log "Error: This script must be run from the OVN repository root" >> + exit 1 >> +fi >> + >> +start_sudo_keepalive() { >> + (while true; do sudo -n true; sleep "$KEEPALIVE_INT"; done) >> 2>/dev/null & >> + SUDO_KEEPALIVE_PID=$! >> + if ! kill -0 $SUDO_KEEPALIVE_PID 2>/dev/null; then >> + log "ERROR: sudo keepalive failed to start" >> + exit 1 >> + fi >> +} >> + >> +stop_sudo_keepalive() { >> + if [ -n "$SUDO_KEEPALIVE_PID" ]; then >> + kill $SUDO_KEEPALIVE_PID 2>/dev/null || true >> + fi >> +} >> + >> +# Cleanup function - always runs on exit >> +cleanup() { >> + if [ $CLEANUP_DONE -eq 1 ]; then >> + return >> + fi >> + CLEANUP_DONE=1 >> + >> + stop_sudo_keepalive >> + >> + echo >> + log "Cleaning up..." >> + log "Restoring modified files..." >> + # Restore OVN test files >> + git checkout tests/ovn-macros.at tests/system-kmod-macros.at \ >> + tests/system-ovn.at tests/system-ovn-kmod.at \ >> + >> logs/git.log 2>&1 || true >> + # Restore OVS submodule files >> + (cd ovs && git checkout vswitchd/vswitch.ovsschema \ >> + >> ../logs/git.log 2>&1 || true) >> + # Restore CI scripts (may have been replaced during upgrade test) >> + git checkout .ci/linux-build.sh .ci/linux-util.sh \ >> + >> logs/git.log 2>&1 || true >> + >> + log "Restoring original branch/commit..." >> + # If we were on a branch, restore to it; otherwise restore to commit >> + if [ "$CURRENT_BRANCH" != "HEAD" ]; then >> + if ! git checkout "$CURRENT_BRANCH" >> logs/git.log 2>&1; then >> + log "WARNING: Failed to restore branch $CURRENT_BRANCH" >&2 >> + fi >> + else >> + # We were in detached HEAD state, restore to the commit >> + if ! git checkout "$CURRENT_COMMIT" >> logs/git.log 2>&1; then >> + log "WARNING: Failed to restore commit $CURRENT_COMMIT" >&2 >> + fi >> + fi >> + >> + log "Updating submodules..." >> + git submodule update --init >> logs/git.log 2>&1 || true >> + log "Restored to: $CURRENT_BRANCH ($CURRENT_COMMIT)" >> + >> + # Cleanup temporary files >> + rm -rf /tmp/ovn-upgrade-binaries /tmp/ovn-upgrade-ci >> + rm -f /tmp/ovn-upgrade-ofctl-defines.h >> + rm -f /tmp/ovn-upgrade-oftable-m4-defines.txt >> + rm -f /tmp/ovn-upgrade-new-log-egress.txt >> + rm -f /tmp/ovn-upgrade-new-save-inport.txt >> +} >> + >> +trap cleanup EXIT INT TERM QUIT HUP >> + >> +# Request sudo credentials early >> +if ! sudo -nv 2>/dev/null; then >> + log "This script requires sudo for running system tests." >> + log "Please enter your password now:" >> + sudo -v || { >> + log "Error: sudo authentication failed" >> + exit 1 >> + } >> +fi >> + >> +start_sudo_keepalive >> + >> +# Save current branch/commit >> +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) >> +CURRENT_COMMIT=$(git rev-parse HEAD) >> +log "Current branch: $CURRENT_BRANCH" >> +log "Current commit: $CURRENT_COMMIT" >> +echo >> + >> +# Check if working directory is clean >> +if ! git diff-index --quiet HEAD --; then >> + log "Warning: Working directory has uncommitted changes" >> + read -p "Continue anyway? (y/n) " -n 1 -r >> + echo >> + if [[ ! $REPLY =~ ^[Yy]$ ]]; then >> + exit 1 >> + fi >> +fi >> + >> +# Create logs directory >> +mkdir -p logs >> + >> +# Export environment variables for linux-build.sh >> +export TESTSUITE="upgrade-test" >> +export BASE_VERSION="$BASE_VERSION" >> +export TEST_RANGE="$TEST_RANGE" >> +export JOBS="${JOBS:--j$(nproc 2>/dev/null || echo 4)}" >> +export CC="${CC:-gcc}" >> +export NO_DEBUG="${NO_DEBUG:-1}" # Disable verbose set -x output by >> default >> +export USE_SPARSE="${USE_SPARSE:-no}" # Disable sparse for local tests >> + >> +log "Running upgrade tests via linux-build.sh..." >> +echo >> + >> +# Run linux-build.sh which will call execute_upgrade_tests() >> +if ./.ci/linux-build.sh; then >> + echo >> + log "Upgrade test completed successfully" >> + TEST_STATUS=0 >> +else >> + echo >> + log "Upgrade test failed - check logs" >> + TEST_STATUS=1 >> +fi >> + >> +# Print summary >> +echo >> +log "Logs saved to:" >> +log " - logs/git.log" >> +log " - tests/system-kmod-testsuite.log" >> + >> +exit $TEST_STATUS >> diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml >> index b6e461129..06b677f55 100644 >> --- a/.github/workflows/test.yml >> +++ b/.github/workflows/test.yml >> @@ -94,7 +94,6 @@ jobs: >> >> name: linux ${{ join(matrix.cfg.*, ' ') }} >> runs-on: ubuntu-24.04 >> - >> > > nit: Unrelated change. > > >> strategy: >> fail-fast: false >> matrix: >> @@ -123,30 +122,42 @@ jobs: >> - { compiler: clang, testsuite: system-test, sanitizers: >> sanitizers, test_range: "-100" } >> - { compiler: clang, testsuite: system-test, sanitizers: >> sanitizers, test_range: "101-200" } >> - { compiler: clang, testsuite: system-test, sanitizers: >> sanitizers, test_range: "201-", unstable: unstable } >> + - { compiler: gcc, testsuite: upgrade-test, base_version: >> branch-24.03, test_range: "-100" } >> + - { compiler: gcc, testsuite: upgrade-test, base_version: >> branch-24.03, test_range: "101-", unstable: unstable } >> + - { compiler: gcc, testsuite: upgrade-test, base_version: >> branch-24.09, test_range: "-100", run_on: schedule } >> + - { compiler: gcc, testsuite: upgrade-test, base_version: >> branch-24.09, test_range: "101-200", run_on: schedule } >> + - { compiler: gcc, testsuite: upgrade-test, base_version: >> branch-24.09, test_range: "201-", unstable: unstable, run_on: schedule } >> + - { compiler: gcc, testsuite: upgrade-test, base_version: >> branch-25.03, test_range: "-100", run_on: schedule } >> + - { compiler: gcc, testsuite: upgrade-test, base_version: >> branch-25.03, test_range: "101-200", run_on: schedule } >> + - { compiler: gcc, testsuite: upgrade-test, base_version: >> branch-25.03, test_range: "201-", unstable: unstable, run_on: schedule } >> + - { compiler: gcc, testsuite: upgrade-test, base_version: >> branch-25.09, test_range: "-100" } >> + - { compiler: gcc, testsuite: upgrade-test, base_version: >> branch-25.09, test_range: "101-200" } >> + - { compiler: gcc, testsuite: upgrade-test, base_version: >> branch-25.09, test_range: "201-", unstable: unstable } >> > > I know it will result in a certain amount of duplication > but we should add this into separate step IMO. > It is very hard to track what runs when. I also wonder > if it wouldn't be enough to run all those on schedule weekly > like we do with multinode tests, WDYT? > > >> - { arch: x86, compiler: gcc, opts: --disable-ssl } >> >> steps: >> - name: system-level-dependencies >> - if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }} >> + if: ${{ (startsWith(matrix.cfg.testsuite, 'system-test') || >> (matrix.cfg.testsuite == 'upgrade-test')) && (matrix.cfg.run_on != >> 'schedule' || github.event_name == 'schedule') }} >> run: | >> sudo apt update >> sudo apt -y install linux-modules-extra-$(uname -r) >> >> - name: checkout >> - if: github.event_name == 'push' || github.event_name == >> 'pull_request' >> + if: (github.event_name == 'push' || github.event_name == >> 'pull_request' || matrix.cfg.testsuite == 'upgrade-test') && >> (matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule') >> uses: actions/checkout@v4 >> with: >> submodules: recursive >> + fetch-depth: ${{ matrix.cfg.testsuite == 'upgrade-test' && 0 || >> 1 }} >> >> - # For weekly runs, don't update submodules >> + # For weekly runs (no upgrade-tests), don't update submodules >> - name: checkout without submodule >> - if: github.event_name == 'schedule' >> + if: github.event_name == 'schedule' && matrix.cfg.testsuite != >> 'upgrade-test' >> uses: actions/checkout@v4 >> >> - # Weekly runs test using the tip of the most recent stable OVS branch >> + # Weekly runs (no upgrade-tests) test using the tip of the most >> recent stable OVS branch >> # instead of the submodule. >> - name: checkout OVS >> - if: github.event_name == 'schedule' >> + if: github.event_name == 'schedule' && matrix.cfg.testsuite != >> 'upgrade-test' >> uses: actions/checkout@v4 >> with: >> repository: 'openvswitch/ovs' >> @@ -154,7 +165,7 @@ jobs: >> path: 'ovs' >> >> - name: checkout OVS most recent stable branch. >> - if: github.event_name == 'schedule' >> + if: github.event_name == 'schedule' && matrix.cfg.testsuite != >> 'upgrade-test' >> run: | >> git checkout \ >> $(git branch -a -l '*branch-*' | sed 's/remotes\/origin\///' | >> \ >> @@ -162,16 +173,19 @@ jobs: >> working-directory: ovs >> >> - name: Fix /etc/hosts file >> + if: matrix.cfg.run_on != 'schedule' || github.event_name == >> 'schedule' >> run: | >> . .ci/linux-util.sh >> fix_etc_hosts >> >> - name: Disable apparmor >> + if: matrix.cfg.run_on != 'schedule' || github.event_name == >> 'schedule' >> run: | >> . .ci/linux-util.sh >> disable_apparmor >> >> - name: image cache >> + if: matrix.cfg.run_on != 'schedule' || github.event_name == >> 'schedule' >> id: image_cache >> uses: actions/cache@v4 >> with: >> @@ -179,17 +193,24 @@ jobs: >> key: ${{ github.sha }}/${{ github.event_name }} >> >> - name: load image >> + if: matrix.cfg.run_on != 'schedule' || github.event_name == >> 'schedule' >> run: | >> sudo podman load -i /tmp/image.tar >> podman load -i /tmp/image.tar >> rm -rf /tmp/image.tar >> >> + # Set BASE_VERSION env var for upgrade tests >> + - name: Set upgrade test env >> + if: matrix.cfg.testsuite == 'upgrade-test' && (matrix.cfg.run_on >> != 'schedule' || github.event_name == 'schedule') >> + run: echo "BASE_VERSION=${{ matrix.cfg.base_version }}" >> >> $GITHUB_ENV >> + >> + # Regular build steps >> - name: build >> - if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }} >> + if: ${{ (startsWith(matrix.cfg.testsuite, 'system-test') || >> (matrix.cfg.testsuite == 'upgrade-test')) && (matrix.cfg.run_on != >> 'schedule' || github.event_name == 'schedule') }} >> run: sudo -E ./.ci/ci.sh --archive-logs --timeout=2h >> >> - name: build >> - if: ${{ !startsWith(matrix.cfg.testsuite, 'system-test') }} >> + if: ${{ !startsWith(matrix.cfg.testsuite, 'system-test') && >> matrix.cfg.testsuite != 'upgrade-test' && (matrix.cfg.run_on != 'schedule' >> || github.event_name == 'schedule') }} >> run: ./.ci/ci.sh --archive-logs --timeout=2h >> >> - name: upload logs on failure >> diff --git a/Makefile.am b/Makefile.am >> index 3ad2077b3..e57bfb297 100644 >> --- a/Makefile.am >> +++ b/Makefile.am >> @@ -91,6 +91,7 @@ EXTRA_DIST = \ >> .ci/linux-util.sh \ >> .ci/osx-build.sh \ >> .ci/osx-prepare.sh \ >> + .ci/test-upgrade-local.sh \ >> .ci/ovn-kubernetes/Dockerfile \ >> .ci/ovn-kubernetes/prepare.sh \ >> .ci/ovn-kubernetes/custom.patch \ >> diff --git a/tests/ovn-macros.at b/tests/ovn-macros.at >> index 1c981e5a8..834ed60be 100644 >> --- a/tests/ovn-macros.at >> +++ b/tests/ovn-macros.at >> @@ -1520,6 +1520,12 @@ m4_define([TAG_UNSTABLE], [ >> AT_SKIP_IF([test X"$SKIP_UNSTABLE" = Xyes]) >> ]) >> >> +# TAG_TEST_NOT_UPGRADABLE tag indicates that the test would fail >> +# "upgrade" test (i.e. running old ovn-northd and new ovn-controller) >> +m4_define([TAG_TEST_NOT_UPGRADABLE], [ >> + AT_SKIP_IF([test X"$UPGRADE_TEST" = Xyes]) >> +]) >> + >> m4_define([OVN_CHECK_SCAPY_EDNS_CLIENT_SUBNET_SUPPORT], >> [ >> AT_SKIP_IF([test $HAVE_SCAPY = no]) >> -- >> 2.47.1 >> >> _______________________________________________ >> dev mailing list >> [email protected] >> https://mail.openvswitch.org/mailman/listinfo/ovs-dev >> >> > Regards, > Ales > _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
