When ovn is upgraded, ovn-controller is updated first on the compute nodes. Then ovn-northd and DB are upgraded. This patch tests whether the intermediate state (i.e. with ovn-controller being upgraded) works properly, running system tests from the base line (i.e. before the upgrade).
Flow tables might change between releases. Hence this patch must take that into account by updating the (old) system tests with any updated table numbers. In some cases, (new) ovn-controller might change flows in existing tables, causing some 'upgrade' tests to fail. Such tests can be skipped using the TAG_TEST_NOT_UPGRADABLE tag. This patch upgrades the ci to run automatically some upgrade tests weekly. It also provides a shell script to run those tests locally. Upgrade-tests are run on push/pull only for LTS (24.03) and latest release (25.09) to avoid too long tests. Upgrades from other branches are run on schedule. This patch depends on patch [1] on branch-25.09. [1] "tests: Add new TAG_TEST_NOT_UPGRADABLE to some tests." Reported-at: https://issues.redhat.com/browse/FDP-1240 Signed-off-by: Xavier Simonart <[email protected]> --- .ci/ci.sh | 1 + .ci/linux-build.sh | 95 +++++++++- .ci/linux-util.sh | 367 +++++++++++++++++++++++++++++++++++++ .ci/test-upgrade-local.sh | 202 ++++++++++++++++++++ .github/workflows/test.yml | 41 ++++- Makefile.am | 1 + tests/ovn-macros.at | 6 + 7 files changed, 697 insertions(+), 16 deletions(-) create mode 100755 .ci/test-upgrade-local.sh diff --git a/.ci/ci.sh b/.ci/ci.sh index 3640d3243..6798fbd78 100755 --- a/.ci/ci.sh +++ b/.ci/ci.sh @@ -102,6 +102,7 @@ function run_tests() { ARCH=$ARCH CC=$CC LIBS=$LIBS OPTS=$OPTS TESTSUITE=$TESTSUITE \ TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS DPDK=$DPDK \ RECHECK=$RECHECK UNSTABLE=$UNSTABLE TIMEOUT=$TIMEOUT \ + BASE_VERSION=$BASE_VERSION \ ./.ci/linux-build.sh " } diff --git a/.ci/linux-build.sh b/.ci/linux-build.sh index 183833a16..890f4c59d 100755 --- a/.ci/linux-build.sh +++ b/.ci/linux-build.sh @@ -1,7 +1,10 @@ #!/bin/bash set -o errexit -set -x +# Enable debug output for CI, optional for local +if [ "${NO_DEBUG:-0}" = "0" ]; then + set -x +fi ARCH=${ARCH:-"x86_64"} USE_SPARSE=${USE_SPARSE:-"yes"} @@ -181,7 +184,7 @@ function run_system_tests() if ! sudo timeout -k 5m -v $TIMEOUT make $JOBS $type \ TESTSUITEFLAGS="$TEST_RANGE" RECHECK=$RECHECK \ - SKIP_UNSTABLE=$SKIP_UNSTABLE; then + SKIP_UNSTABLE=$SKIP_UNSTABLE UPGRADE_TEST=$UPGRADE_TEST; then # $log_file is necessary for debugging. cat tests/$log_file return 1 @@ -190,19 +193,28 @@ function run_system_tests() function execute_system_tests() { - configure_ovn $OPTS - make $JOBS || { cat config.log; exit 1; } + local test_type=$1 + local log_file=$2 + local skip_build=$3 + + # Only build if not already built (upgrade tests build separately) + if [ "$skip_build" != "yes" ]; then + configure_ovn $OPTS + make $JOBS || { cat config.log; exit 1; } + fi local stable_rc=0 local unstable_rc=0 - if ! SKIP_UNSTABLE=yes run_system_tests $@; then + if ! SKIP_UNSTABLE=yes UPGRADE_TEST=$UPGRADE_TEST \ + run_system_tests $test_type $log_file; then stable_rc=1 fi if [ "$UNSTABLE" ]; then if ! SKIP_UNSTABLE=no TEST_RANGE="-k unstable" RECHECK=yes \ - run_system_tests $@; then + UPGRADE_TEST=$UPGRADE_TEST run_system_tests $test_type \ + $log_file; then unstable_rc=1 fi fi @@ -212,6 +224,72 @@ function execute_system_tests() fi } +function execute_upgrade_tests() +{ + . .ci/linux-util.sh + + # Save current CI scripts (will be replaced by base version after checkout) + cp -rf .ci /tmp/ovn-upgrade-ci + + # Build current version + log "Building current version..." + mkdir -p logs + configure_ovn $OPTS >> logs/build-current.log 2>&1 || { + log "configure ovn failed - see config.log and logs/build-current.log" + exit 1 + } + make $JOBS >> logs/build-current.log 2>&1 || { + log "building ovn failed - see logs/build-current.log" + exit 1 + } + + ovn_upgrade_save_current_binaries + + # Checkout base version + ovn_upgrade_checkout_base "$BASE_VERSION" logs/git.log + + # Clean from current version + log "Cleaning build artifacts..." + make distclean >> logs/build-base.log 2>&1 || true + (cd ovs && make distclean >> ../logs/build-base.log 2>&1) || true + + # Apply test patches + ovn_upgrade_apply_tests_patches + + # Build base with patches + ovn_upgrade_patch_for_ovn_debug + + # Build (modified) base version + log "Building base version (with patched lflow.h)..." + configure_ovn $OPTS >> logs/build-base.log 2>&1 || { + log "configure ovn failed - see config.log and logs/build-base.log" + exit 1 + } + make $JOBS >> logs/build-base.log 2>&1 || { + log "building ovn failed - see logs/build-base.log" + exit 1 + } + ovn_upgrade_save_ovn_debug + + # Build (clean) base version + log "Rebuilding base version (clean lflow.h)..." + git checkout controller/lflow.h >> logs/git.log 2>&1 + make $JOBS >> logs/build-base.log 2>&1 || { + log "building ovn failed - see logs/build-base.log" + exit 1 + } + + # Restore binaries + ovn_upgrade_restore_binaries + + # Restore current CI scripts for test execution + cp -f /tmp/ovn-upgrade-ci/linux-build.sh .ci/linux-build.sh + cp -f /tmp/ovn-upgrade-ci/linux-util.sh .ci/linux-util.sh + + UPGRADE_TEST=yes execute_system_tests "check-kernel" \ + "system-kmod-testsuite.log" "yes" +} + configure_$CC if [ "$TESTSUITE" ]; then @@ -238,6 +316,11 @@ if [ "$TESTSUITE" ]; then sudo bash -c "echo 2048 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" execute_system_tests "check-system-dpdk" "system-dpdk-testsuite.log" ;; + + "upgrade-test") + execute_upgrade_tests + ;; + esac else configure_ovn $OPTS diff --git a/.ci/linux-util.sh b/.ci/linux-util.sh index b5bd1f8c9..a62ff2ea0 100755 --- a/.ci/linux-util.sh +++ b/.ci/linux-util.sh @@ -49,3 +49,370 @@ function disable_apparmor() sudo aa-teardown || true sudo systemctl disable --now apparmor.service } + +function log() { + echo "[$(date '+%H:%M:%S')] $*" +} + +# ovn_upgrade_save_current_binaries +# Saves current version's binaries and schemas to /tmp/ovn-upgrade-binaries/ +function ovn_upgrade_save_current_binaries() +{ + mkdir -p /tmp/ovn-upgrade-binaries + + # New ovn-controller may generate OpenFlow flows with new actions that old + # OVS doesn't understand, so we need also to use new OVS. + # New ovs-vswitchd binary expects columns/tables defined in the current + # schema, so we also need to use new schemas. + files="controller/ovn-controller ovs/vswitchd/ovs-vswitchd + ovs/ovsdb/ovsdb-server ovs/utilities/ovs-vsctl + ovs/utilities/ovs-ofctl ovs/utilities/ovs-appctl + ovs/utilities/ovs-dpctl ovs/vswitchd/vswitch.ovsschema" + for file in $files; do + if [ ! -f "$file" ]; then + log "ERROR: $file not found" + return 1 + fi + cp "$file" /tmp/ovn-upgrade-binaries/ + done + + # In the upgrade scenario we use old ovn-northd and new ovn-controller. + # OFTABLES are defined through a combination of northd/northd.h and + # controller/lflow.h. Tests uses either (old) table numbers, table names + # (defined in ovn-macros) or ovn-debug. + # + # Extract OFCTL_* table defines from current lflow.h + if ! grep '^#define OFTABLE_' controller/lflow.h > \ + /tmp/ovn-upgrade-ofctl-defines.h; then + log "No #define OFTABLE_ found in lflow.h" + return 1 + fi + + # Extract OFTABLE m4 defines from current tests/ovn-macros.at + # These are used by tests to reference table numbers + # In old tests, there might be no OFTABLE_ in ovn-macros, so grep can fail. + grep '^m4_define(\[OFTABLE_' tests/ovn-macros.at > \ + /tmp/ovn-upgrade-oftable-m4-defines.txt || true + + # Extract key table numbers for calculating shifts in hardcoded table + # references. OFTABLE_SAVE_INPORT is where normal (unshifted) tables + # resume. + LINE=$(grep "define OFTABLE_LOG_EGRESS_PIPELINE" controller/lflow.h) + NEW_LOG_EGRESS=$(echo "$LINE" | grep -oE '[0-9]+') + LINE=$(grep "define OFTABLE_SAVE_INPORT" controller/lflow.h) + NEW_SAVE_INPORT=$(echo "$LINE" | grep -oE '[0-9]+') + if [ -z "$NEW_LOG_EGRESS" ]; then + log "ERROR: Could not extract OFTABLE_LOG_EGRESS_PIPELINE value" + return 1 + fi + if [ -z "$NEW_SAVE_INPORT" ]; then + log "ERROR: Could not extract OFTABLE_SAVE_INPORT value" + return 1 + fi + + echo "$NEW_LOG_EGRESS" > /tmp/ovn-upgrade-new-log-egress.txt + echo "$NEW_SAVE_INPORT" > /tmp/ovn-upgrade-new-save-inport.txt + + echo "" + log "Saved current versions:" + log " ovn-controller:$(/tmp/ovn-upgrade-binaries/ovn-controller --version | + grep ovn-controller)" + log " SB DB schema:$(/tmp/ovn-upgrade-binaries/ovn-controller --version | + grep "SB DB Schema")" + log " ovs-vswitchd:$(/tmp/ovn-upgrade-binaries/ovs-vswitchd --version | + grep vSwitch)" +} + +# ovn_upgrade_checkout_base BASE_VERSION LOG_FILE +# Checks out base version from git +function ovn_upgrade_checkout_base() +{ + local base_version=$1 + local log_file=$2 + + log "Checking out base version: $base_version" + + # Try to checkout directly first (might already exist locally) + if git checkout "$base_version" >> "$log_file" 2>&1; then + log "Using locally available $base_version" + else + # Not available locally, try to fetch it + log "Fetching $base_version from origin..." + + # Try as a tag first + if git fetch --depth=1 origin tag "$base_version" \ + >> "$log_file" 2>&1; then + log "Fetched tag $base_version" + + # Try as a branch + elif git fetch --depth=1 origin "$base_version" \ + >> "$log_file" 2>&1; then + log "Fetched branch $base_version" + + else + git fetch origin >> "$log_file" 2>&1 || true + log "Fetched all refs from origin" + fi + + # Try checkout + if git checkout "$base_version" >> "$log_file" 2>&1; then + log "Using $base_version from origin" + else + # origin might be a private repo w/o all branches. + # Try ovn-org as fallback. + log "Not in origin, fetching from ovn-org..." + git fetch https://github.com/ovn-org/ovn.git \ + "$base_version:$base_version" >> "$log_file" 2>&1 || return 1 + log "Fetched $base_version from ovn-org" + git checkout "$base_version" >> "$log_file" 2>&1 || return 1 + fi + fi + + git submodule update --init >> "$log_file" 2>&1 || return 1 +} + +# Patch base version's lflow.h with current OFTABLE table defines +# This ensures ovn-debug uses correct table numbers +function ovn_upgrade_patch_for_ovn_debug() +{ + if [ -f /tmp/ovn-upgrade-ofctl-defines.h ] && \ + [ -f controller/lflow.h ]; then + # Replace old OFCTL defines with current ones in one pass + awk ' + !inserted && /^#define OFTABLE_/ { + system("cat /tmp/ovn-upgrade-ofctl-defines.h") + inserted = 1 + } + /^#define OFTABLE_/ { next } + { print } + ' controller/lflow.h > controller/lflow.h.tmp + + mv controller/lflow.h.tmp controller/lflow.h + fi +} + +# ovn_upgrade_save_ovn_debug +# Saves ovn-debug binary built with current OFTABLE defines +# This creates a hybrid ovn-debug: current table numbers + base logical flow +# stages +function ovn_upgrade_save_ovn_debug() +{ + log "Saving hybrid ovn-debug..." + cp utilities/ovn-debug /tmp/ovn-upgrade-binaries/ovn-debug +} + +# update_test old_first_table old_last_table shift test_file +# Update test tables in test_file, for old_first <= tables < old_last_table +function update_test() +{ + test_file=$4 + awk -v old_start=$1 \ + -v old_end=$2 \ + -v shift=$3 ' + { + result = "" + rest = $0 + # Process all table=NUMBER matches in the line + while (match(rest, /table *= *[0-9]+/)) { + # Save match position before calling match() again + pos = RSTART + len = RLENGTH + + # Add everything before the match + result = result substr(rest, 1, pos-1) + + # Extract the matched text and the number + matched = substr(rest, pos, len) + if (match(matched, /[0-9]+/)) { + num = substr(matched, RSTART, RLENGTH) + } else { + num = 0 + } + + # Check if this table number needs updating + if (num >= old_start && num < old_end) { + result = result "table=" (num + shift) + } else { + result = result matched + } + + # Continue with the rest of the line (use saved pos/len) + rest = substr(rest, pos + len) + } + # Add any remaining text + print result rest + }' "$test_file" > "$test_file.tmp" && mv "$test_file.tmp" "$test_file" +} + +# ovn_upgrade_table_numbers_in_tests_patch: fix hardcoded table numbers in +# test files +function ovn_upgrade_table_numbers_in_tests_patch() +{ + # Old tests (e.g., branch-24.03) have hardcoded numbers like "table=45" + # which refer to specific logical tables. When OFTABLE defines shift, + # these numbers must be updated. + # Example: v24.03.0 has OFTABLE_LOG_EGRESS_PIPELINE=42, so "table=45" + # means egress+3. + # In main, OFTABLE_LOG_EGRESS_PIPELINE=47, so it should become "table=50". + if [ ! -f /tmp/ovn-upgrade-new-log-egress.txt ] || + [ ! -f /tmp/ovn-upgrade-new-save-inport.txt ]; then + log "WARNING: Table shift data not found, skipping hardcoded table \ + number updates" + return + fi + + if [ ! -f controller/lflow.h ]; then + log "WARNING: controller/lflow.h not found, skipping hardcoded table \ + number updates" + return + fi + + NEW_LOG_EGRESS=$(cat /tmp/ovn-upgrade-new-log-egress.txt) + NEW_SAVE_INPORT=$(cat /tmp/ovn-upgrade-new-save-inport.txt) + + # Get old values from base version's lflow.h (before we patched it) + LINE=$(grep "#define OFTABLE_LOG_EGRESS_PIPELINE" controller/lflow.h) + OLD_LOG_EGRESS=$(echo "$LINE" | grep -oE '[0-9]+') + LINE=$(grep "#define OFTABLE_SAVE_INPORT" controller/lflow.h) + OLD_SAVE_INPORT=$(echo "$LINE" | grep -oE '[0-9]+') + + if [ -z "$OLD_LOG_EGRESS" ] || [ -z "$OLD_SAVE_INPORT" ] || \ + [ "$OLD_LOG_EGRESS" == "$NEW_LOG_EGRESS" ]; then + log "No change in tests files as old_log_egress=$OLD_LOG_EGRESS, + old_save_inport=$OLD_SAVE_INPORT and + new_log_egress=$NEW_LOG_EGRESS" + return + fi + + # Calculate the shift + SHIFT=$((NEW_LOG_EGRESS - OLD_LOG_EGRESS)) + + log "Updating hardcoded table numbers in tests (shift: +$SHIFT for tables \ + $OLD_LOG_EGRESS-$((OLD_SAVE_INPORT-1)))" + + # Update hardcoded table numbers in test files + for test_file in tests/system-ovn.at tests/system-ovn-kmod.at; do + if [ -f "$test_file" ]; then + log "Updating $test_file" + update_test "$OLD_LOG_EGRESS" "$OLD_SAVE_INPORT" "$SHIFT" \ + "$test_file" + fi + done +} + +# ovn_upgrade_cleanup_sbox_patch: filter out expected schema warnings. +function ovn_upgrade_cleanup_sbox_patch() +{ + cat << 'EOF' > /tmp/upgrade-schema-filter.patch +diff --git a/tests/ovn-macros.at b/tests/ovn-macros.at +index a08252d50..a31dc87b4 100644 +--- a/tests/ovn-macros.at ++++ b/tests/ovn-macros.at +@@ -98,6 +98,7 @@ m4_define([OVN_CLEANUP_SBOX],[ + $error + /connection failed (No such file or directory)/d + /has no network name*/d ++ /OVN_Southbound database lacks/d + /receive tunnel port not found*/d + /Failed to locate tunnel to reach main chassis/d + /Transaction causes multiple rows.*MAC_Binding/d +diff --git a/tests/system-kmod-macros.at b/tests/system-kmod-macros.at +index 6f6670199..4bd1a2c90 100644 +--- a/tests/system-kmod-macros.at ++++ b/tests/system-kmod-macros.at +@@ -45,7 +45,8 @@ m4_define([OVS_TRAFFIC_VSWITCHD_START], + # invoked. They can be used to perform additional cleanups such as name space + # removal. + m4_define([OVS_TRAFFIC_VSWITCHD_STOP], +- [OVS_VSWITCHD_STOP([$1]) ++ [OVS_VSWITCHD_STOP([dnl ++$1";/OVN_Southbound database lacks/d"]) + AT_CHECK([:; $2]) + ]) + +EOF + + # Try to apply schema filter patch. May fail on old OVN versions where + # OVN_CLEANUP_SBOX doesn't check errors - this is expected and okay. + # If patch fails for more recent OVN, then the test will fail due to the + # "OVN_Southbound database lacks". + patch -p1 < /tmp/upgrade-schema-filter.patch > /dev/null 2>&1 || true + rm -f /tmp/upgrade-schema-filter.patch +} + +# ovn_upgrade_oftable_ovn_macro_patch: update table numbers in ovn-macro +function ovn_upgrade_oftable_ovn_macro_patch() +{ + # Patch base version's tests/ovn-macros.at with current OFTABLE m4 defines + # This ensures tests use correct table numbers when checking flows + if [ -f /tmp/ovn-upgrade-oftable-m4-defines.txt ] && + [ -f tests/ovn-macros.at ]; then + # Check if the base version has OFTABLE m4 defines + if grep -q '^m4_define(\[OFTABLE_' tests/ovn-macros.at; then + # Replace old m4_define OFTABLE statements with current ones + awk ' + !inserted && /^m4_define\(\[OFTABLE_/ { + system("cat /tmp/ovn-upgrade-oftable-m4-defines.txt") + inserted = 1 + } + /^m4_define\(\[OFTABLE_/ { next } + { print } + ' tests/ovn-macros.at > tests/ovn-macros.at.tmp + + mv tests/ovn-macros.at.tmp tests/ovn-macros.at + fi + fi +} + +# Applies patches to base version after second build: +# 1. Schema error patch (filters "OVN_Southbound database lacks" warnings) +# 2. OFTABLE m4 defines patch in tests/ovn-macros.at (for test table numbers) +# 3. Hardcoded table numbers patch in test files +function ovn_upgrade_apply_tests_patches() +{ + log "Applying schema filter and table number patches..." + ovn_upgrade_table_numbers_in_tests_patch + ovn_upgrade_cleanup_sbox_patch + ovn_upgrade_oftable_ovn_macro_patch +} + +# ovn_upgrade_restore_binaries +# +# Replaces base version binaries with saved current versions: +# - ovn-controller (from current) +# - OVS binaries and schema (from current) +# - ovn-debug (hybrid: current OFTABLE + base logical stages) +function ovn_upgrade_restore_binaries() +{ + log "Replacing binaries with current versions" + + # Replace OVN controller + cp /tmp/ovn-upgrade-binaries/ovn-controller controller/ovn-controller + + # Replace ovn-debug with hybrid version (built with current OFTABLE + base + # northd.h) + cp /tmp/ovn-upgrade-binaries/ovn-debug utilities/ovn-debug + + # Replace OVS binaries + cp /tmp/ovn-upgrade-binaries/ovs-vswitchd ovs/vswitchd/ovs-vswitchd + cp /tmp/ovn-upgrade-binaries/ovsdb-server ovs/ovsdb/ovsdb-server + cp /tmp/ovn-upgrade-binaries/ovs-vsctl ovs/utilities/ovs-vsctl + cp /tmp/ovn-upgrade-binaries/ovs-ofctl ovs/utilities/ovs-ofctl + cp /tmp/ovn-upgrade-binaries/ovs-appctl ovs/utilities/ovs-appctl + cp /tmp/ovn-upgrade-binaries/ovs-dpctl ovs/utilities/ovs-dpctl + + # Replace OVS schema (current binaries expect current schema) + cp /tmp/ovn-upgrade-binaries/vswitch.ovsschema \ + ovs/vswitchd/vswitch.ovsschema + + echo "" + log "Verification - Current versions (from current patch):" + log " ovn-controller: $(controller/ovn-controller --version | + grep ovn-controller)" + log " SB DB Schema: $(controller/ovn-controller --version | + grep "SB DB Schema")" + log " ovs-vswitchd: $(ovs/vswitchd/ovs-vswitchd --version | grep vSwitch)" + log "Verification - Base versions (for compatibility testing):" + log " ovn-northd: $(northd/ovn-northd --version | grep ovn-northd)" + log " ovn-nbctl: $(utilities/ovn-nbctl --version | grep ovn-nbctl)" +} diff --git a/.ci/test-upgrade-local.sh b/.ci/test-upgrade-local.sh new file mode 100755 index 000000000..3818ac465 --- /dev/null +++ b/.ci/test-upgrade-local.sh @@ -0,0 +1,202 @@ +#!/bin/bash + +set -e + +. "$(dirname $0)/linux-util.sh" + +BASE_VERSION="${BASE_VERSION:-branch-24.03}" +TEST_RANGE="${TEST_RANGE:-1-}" +KEEPALIVE_INT=50 + +CLEANUP_DONE=0 +TEST_STATUS=1 + +usage() { + cat << EOF +Usage: $0 [options] + +Test OVN upgrade compatibility without GitHub. + +Options: + -b, --base-version VERSION Base version to test (default: branch-24.03) + -t, --test-range RANGE Test range to run (default: 1-) + Examples: -100, 101-, 55 + -h, --help Show this help message + +Environment Variables: + BASE_VERSION Same as --base-version + TEST_RANGE Same as --test-range + +Examples: + # Test against branch-24.03 with all tests + $0 + + # Test against specific version + $0 --base-version v24.03.0 + + # Test specific test range + $0 --test-range 101-200 +EOF +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -b|--base-version) + BASE_VERSION="$2" + shift 2 + ;; + -t|--test-range) + TEST_RANGE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +log "========================================" +log "OVN Upgrade Test" +log "Base version: $BASE_VERSION" +log "Test range: $TEST_RANGE" + +# Check if we're in the OVN repository root +if [ ! -f "configure.ac" ] || ! grep -q "ovn" configure.ac; then + log "Error: This script must be run from the OVN repository root" + exit 1 +fi + +start_sudo_keepalive() { + (while true; do sudo -n true; sleep "$KEEPALIVE_INT"; done) 2>/dev/null & + SUDO_KEEPALIVE_PID=$! + if ! kill -0 $SUDO_KEEPALIVE_PID 2>/dev/null; then + log "ERROR: sudo keepalive failed to start" + exit 1 + fi +} + +stop_sudo_keepalive() { + if [ -n "$SUDO_KEEPALIVE_PID" ]; then + kill $SUDO_KEEPALIVE_PID 2>/dev/null || true + fi +} + +# Cleanup function - always runs on exit +cleanup() { + if [ $CLEANUP_DONE -eq 1 ]; then + return + fi + CLEANUP_DONE=1 + + stop_sudo_keepalive + + echo + log "Cleaning up..." + log "Restoring modified files..." + # Restore OVN test files + git checkout tests/ovn-macros.at tests/system-kmod-macros.at \ + tests/system-ovn.at tests/system-ovn-kmod.at \ + >> logs/git.log 2>&1 || true + # Restore OVS submodule files + (cd ovs && git checkout vswitchd/vswitch.ovsschema \ + >> ../logs/git.log 2>&1 || true) + # Restore CI scripts (may have been replaced during upgrade test) + git checkout .ci/linux-build.sh .ci/linux-util.sh \ + >> logs/git.log 2>&1 || true + + log "Restoring original branch/commit..." + # If we were on a branch, restore to it; otherwise restore to commit + if [ "$CURRENT_BRANCH" != "HEAD" ]; then + if ! git checkout "$CURRENT_BRANCH" >> logs/git.log 2>&1; then + log "WARNING: Failed to restore branch $CURRENT_BRANCH" >&2 + fi + else + # We were in detached HEAD state, restore to the commit + if ! git checkout "$CURRENT_COMMIT" >> logs/git.log 2>&1; then + log "WARNING: Failed to restore commit $CURRENT_COMMIT" >&2 + fi + fi + + log "Updating submodules..." + git submodule update --init >> logs/git.log 2>&1 || true + log "Restored to: $CURRENT_BRANCH ($CURRENT_COMMIT)" + + # Cleanup temporary files + rm -rf /tmp/ovn-upgrade-binaries /tmp/ovn-upgrade-ci + rm -f /tmp/ovn-upgrade-ofctl-defines.h + rm -f /tmp/ovn-upgrade-oftable-m4-defines.txt + rm -f /tmp/ovn-upgrade-new-log-egress.txt + rm -f /tmp/ovn-upgrade-new-save-inport.txt +} + +trap cleanup EXIT INT TERM QUIT HUP + +# Request sudo credentials early +if ! sudo -nv 2>/dev/null; then + log "This script requires sudo for running system tests." + log "Please enter your password now:" + sudo -v || { + log "Error: sudo authentication failed" + exit 1 + } +fi + +start_sudo_keepalive + +# Save current branch/commit +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) +CURRENT_COMMIT=$(git rev-parse HEAD) +log "Current branch: $CURRENT_BRANCH" +log "Current commit: $CURRENT_COMMIT" +echo + +# Check if working directory is clean +if ! git diff-index --quiet HEAD --; then + log "Warning: Working directory has uncommitted changes" + read -p "Continue anyway? (y/n) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi +fi + +# Create logs directory +mkdir -p logs + +# Export environment variables for linux-build.sh +export TESTSUITE="upgrade-test" +export BASE_VERSION="$BASE_VERSION" +export TEST_RANGE="$TEST_RANGE" +export JOBS="${JOBS:--j$(nproc 2>/dev/null || echo 4)}" +export CC="${CC:-gcc}" +export NO_DEBUG="${NO_DEBUG:-1}" # Disable verbose set -x output by default +export USE_SPARSE="${USE_SPARSE:-no}" # Disable sparse for local tests + +log "Running upgrade tests via linux-build.sh..." +echo + +# Run linux-build.sh which will call execute_upgrade_tests() +if ./.ci/linux-build.sh; then + echo + log "Upgrade test completed successfully" + TEST_STATUS=0 +else + echo + log "Upgrade test failed - check logs" + TEST_STATUS=1 +fi + +# Print summary +echo +log "Logs saved to:" +log " - logs/git.log" +log " - tests/system-kmod-testsuite.log" + +exit $TEST_STATUS diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b6e461129..06b677f55 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -94,7 +94,6 @@ jobs: name: linux ${{ join(matrix.cfg.*, ' ') }} runs-on: ubuntu-24.04 - strategy: fail-fast: false matrix: @@ -123,30 +122,42 @@ jobs: - { compiler: clang, testsuite: system-test, sanitizers: sanitizers, test_range: "-100" } - { compiler: clang, testsuite: system-test, sanitizers: sanitizers, test_range: "101-200" } - { compiler: clang, testsuite: system-test, sanitizers: sanitizers, test_range: "201-", unstable: unstable } + - { compiler: gcc, testsuite: upgrade-test, base_version: branch-24.03, test_range: "-100" } + - { compiler: gcc, testsuite: upgrade-test, base_version: branch-24.03, test_range: "101-", unstable: unstable } + - { compiler: gcc, testsuite: upgrade-test, base_version: branch-24.09, test_range: "-100", run_on: schedule } + - { compiler: gcc, testsuite: upgrade-test, base_version: branch-24.09, test_range: "101-200", run_on: schedule } + - { compiler: gcc, testsuite: upgrade-test, base_version: branch-24.09, test_range: "201-", unstable: unstable, run_on: schedule } + - { compiler: gcc, testsuite: upgrade-test, base_version: branch-25.03, test_range: "-100", run_on: schedule } + - { compiler: gcc, testsuite: upgrade-test, base_version: branch-25.03, test_range: "101-200", run_on: schedule } + - { compiler: gcc, testsuite: upgrade-test, base_version: branch-25.03, test_range: "201-", unstable: unstable, run_on: schedule } + - { compiler: gcc, testsuite: upgrade-test, base_version: branch-25.09, test_range: "-100" } + - { compiler: gcc, testsuite: upgrade-test, base_version: branch-25.09, test_range: "101-200" } + - { compiler: gcc, testsuite: upgrade-test, base_version: branch-25.09, test_range: "201-", unstable: unstable } - { arch: x86, compiler: gcc, opts: --disable-ssl } steps: - name: system-level-dependencies - if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }} + if: ${{ (startsWith(matrix.cfg.testsuite, 'system-test') || (matrix.cfg.testsuite == 'upgrade-test')) && (matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule') }} run: | sudo apt update sudo apt -y install linux-modules-extra-$(uname -r) - name: checkout - if: github.event_name == 'push' || github.event_name == 'pull_request' + if: (github.event_name == 'push' || github.event_name == 'pull_request' || matrix.cfg.testsuite == 'upgrade-test') && (matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule') uses: actions/checkout@v4 with: submodules: recursive + fetch-depth: ${{ matrix.cfg.testsuite == 'upgrade-test' && 0 || 1 }} - # For weekly runs, don't update submodules + # For weekly runs (no upgrade-tests), don't update submodules - name: checkout without submodule - if: github.event_name == 'schedule' + if: github.event_name == 'schedule' && matrix.cfg.testsuite != 'upgrade-test' uses: actions/checkout@v4 - # Weekly runs test using the tip of the most recent stable OVS branch + # Weekly runs (no upgrade-tests) test using the tip of the most recent stable OVS branch # instead of the submodule. - name: checkout OVS - if: github.event_name == 'schedule' + if: github.event_name == 'schedule' && matrix.cfg.testsuite != 'upgrade-test' uses: actions/checkout@v4 with: repository: 'openvswitch/ovs' @@ -154,7 +165,7 @@ jobs: path: 'ovs' - name: checkout OVS most recent stable branch. - if: github.event_name == 'schedule' + if: github.event_name == 'schedule' && matrix.cfg.testsuite != 'upgrade-test' run: | git checkout \ $(git branch -a -l '*branch-*' | sed 's/remotes\/origin\///' | \ @@ -162,16 +173,19 @@ jobs: working-directory: ovs - name: Fix /etc/hosts file + if: matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule' run: | . .ci/linux-util.sh fix_etc_hosts - name: Disable apparmor + if: matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule' run: | . .ci/linux-util.sh disable_apparmor - name: image cache + if: matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule' id: image_cache uses: actions/cache@v4 with: @@ -179,17 +193,24 @@ jobs: key: ${{ github.sha }}/${{ github.event_name }} - name: load image + if: matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule' run: | sudo podman load -i /tmp/image.tar podman load -i /tmp/image.tar rm -rf /tmp/image.tar + # Set BASE_VERSION env var for upgrade tests + - name: Set upgrade test env + if: matrix.cfg.testsuite == 'upgrade-test' && (matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule') + run: echo "BASE_VERSION=${{ matrix.cfg.base_version }}" >> $GITHUB_ENV + + # Regular build steps - name: build - if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }} + if: ${{ (startsWith(matrix.cfg.testsuite, 'system-test') || (matrix.cfg.testsuite == 'upgrade-test')) && (matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule') }} run: sudo -E ./.ci/ci.sh --archive-logs --timeout=2h - name: build - if: ${{ !startsWith(matrix.cfg.testsuite, 'system-test') }} + if: ${{ !startsWith(matrix.cfg.testsuite, 'system-test') && matrix.cfg.testsuite != 'upgrade-test' && (matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule') }} run: ./.ci/ci.sh --archive-logs --timeout=2h - name: upload logs on failure diff --git a/Makefile.am b/Makefile.am index 3ad2077b3..e57bfb297 100644 --- a/Makefile.am +++ b/Makefile.am @@ -91,6 +91,7 @@ EXTRA_DIST = \ .ci/linux-util.sh \ .ci/osx-build.sh \ .ci/osx-prepare.sh \ + .ci/test-upgrade-local.sh \ .ci/ovn-kubernetes/Dockerfile \ .ci/ovn-kubernetes/prepare.sh \ .ci/ovn-kubernetes/custom.patch \ diff --git a/tests/ovn-macros.at b/tests/ovn-macros.at index 1c981e5a8..834ed60be 100644 --- a/tests/ovn-macros.at +++ b/tests/ovn-macros.at @@ -1520,6 +1520,12 @@ m4_define([TAG_UNSTABLE], [ AT_SKIP_IF([test X"$SKIP_UNSTABLE" = Xyes]) ]) +# TAG_TEST_NOT_UPGRADABLE tag indicates that the test would fail +# "upgrade" test (i.e. running old ovn-northd and new ovn-controller) +m4_define([TAG_TEST_NOT_UPGRADABLE], [ + AT_SKIP_IF([test X"$UPGRADE_TEST" = Xyes]) +]) + m4_define([OVN_CHECK_SCAPY_EDNS_CLIENT_SUBNET_SUPPORT], [ AT_SKIP_IF([test $HAVE_SCAPY = no]) -- 2.47.1 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
