On Fri, Aug 7, 2026 at 7:22 PM Xavier Simonart via dev < [email protected]> wrote:
> This will make further addition to the test easier. > In addition, a few fixes/improvements: > - Reset log flag to info on exit and re-enable rate limit. > - Add third ovn-chassis - just checking it does not prevent packets to be > sent properly. > - Modify logging (replace a few AS_BOX by echo). > - Read ping request and reply related statistics in one command. > - Few typos such as s/interfac/interface/ > > Signed-off-by: Xavier Simonart <[email protected]> > --- > Hi Xavier, I tried to run this change on my fork and it failed, could you please check it out? https://github.com/almusil/ovn/actions/runs/31782367292 Regards, Ales > tests/multinode.at | 439 ++++++++++++++++++++++++++------------------- > 1 file changed, 257 insertions(+), 182 deletions(-) > > diff --git a/tests/multinode.at b/tests/multinode.at > index d7fde2e26..c69ab3cb4 100644 > --- a/tests/multinode.at > +++ b/tests/multinode.at > @@ -3036,42 +3036,42 @@ AT_CLEANUP > > AT_SETUP([HA: Check for missing garp on leader when BFD goes back up]) > # Network topology > -# > ┌────────────────────────────────────────────────────────────────────────────────────────────────────────┐ > -# │ > │ > -# │ ┌───────────────────┐ ┌───────────────────┐ > ┌───────────────────┐ ┌───────────────────┐ │ > -# │ │ ovn-chassis-1 │ │ ovn-gw-1 │ │ ovn-gw-2 > │ │ ovn-chassis-2 │ │ > -# │ └─────────┬─────────┘ └───────────────────┘ > └───────────────────┘ └───────────────────┘ │ > -# │ ┌─────────┴─────────┐ > │ > -# │ │ inside1 │ > │ > -# │ │ 192.168.1.1/24 │ > │ > -# │ └─────────┬─────────┘ > │ > -# │ ┌─────────┴─────────┐ > │ > -# │ │ inside │ > │ > -# │ └─────────┬─────────┘ > │ > -# │ ┌─────────┴─────────┐ > │ > -# │ │ 192.168.1.254 │ > │ > -# │ │ R1 │ > │ > -# │ │ 192.168.0.254 │ > │ > -# │ └─────────┬─────────┘ > │ > -# │ └------eth1---------------┬--------eth1-----------┐ > │ > -# │ ┌──────────┴────────┐ > ┌─────────┴─────────┐ │ > -# │ │ 192.168.1.254 │ │ > 192.168.1.254 │ │ > -# │ │ R1 │ │ R1 > │ │ > -# │ │ 192.168.0.254 │ │ > 192.168.0.254 │ │ > -# │ └─────────┬─────────┘ > └─────────┬─────────┘ │ > -# │ │ │ > ┌───────────────────┐ │ > -# │ ┌─────────┴─────────┐ > ┌─────────┴─────────┐ │ 192.168.0.1 │ │ > -# │ │ outside │ │ > outside │ │ ext1 │ │ > -# │ └─────────┬─────────┘ > └─────────┬─────────┘ └─────────┬─────────┘ │ > -# │ ┌─────────┴─────────┐ > ┌─────────┴─────────┐ ┌─────────┴─────────┐ │ > -# │ │ ln-outside │ │ > ln-outside │ │ ln-ext1 │ │ > -# │ └─────────┬─────────┘ > └─────────┬─────────┘ └─────────┬─────────┘ │ > -# │ ┌─────────┴─────────┐ > ┌─────────┴─────────┐ ┌─────────┴─────────┐ │ > -# │ │ br-ex │ │ > br-ex │ │ br-ex │ │ > -# │ └─────────┬─────────┘ > └─────────┬─────────┘ └─────────┬─────────┘ │ > -# │ > └---------eth2-----------┴-------eth2-------------┘ │ > -# │ > │ > -# > └────────────────────────────────────────────────────────────────────────────────────────────────────────┘ > +# > ┌────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ > +# │ > │ > +# │ ┌───────────────────┐ ┌───────────────────┐ > ┌───────────────────┐ ┌───────────────────┐ ┌───────────────────┐ │ > +# │ │ ovn-chassis-1 │ │ ovn-chassis-2 │ │ ovn-gw-1 > │ │ ovn-gw-2 │ │ ovn-chassis-3 │ │ > +# │ └─────────┬─────────┘ └─────────┬─────────┘ > └───────────────────┘ └───────────────────┘ └───────────────────┘ │ > +# │ ┌─────────┴─────────┐ ┌─────────┴─────────┐ > │ > +# │ │ inside1 │ │ inside2 │ > │ > +# │ │ 192.168.1.1/24 │ │ 192.168.1.2/24 │ > │ > +# │ └─────────┬─────────┘ └─────────┬─────────┘ > │ > +# │ ┌─┴────────────────────────┴─┐ > │ > +# │ │ inside │ > │ > +# │ └──────────────┬─────────────┘ > │ > +# │ ┌─────────┴─────────┐ > │ > +# │ │ 192.168.1.254 │ > │ > +# │ │ R1 │ > │ > +# │ │ 192.168.0.254 │ > │ > +# │ └─────────┬─────────┘ > │ > +# │ > └------eth1---------------------------┬--------eth1-----------┐ > │ > +# │ > ┌──────────┴────────┐ ┌─────────┴─────────┐ > │ > +# │ │ > 192.168.1.254 │ │ 192.168.1.254 │ │ > +# │ │ R1 > │ │ R1 │ │ > +# │ │ > 192.168.0.254 │ │ 192.168.0.254 │ │ > +# │ > └─────────┬─────────┘ └─────────┬─────────┘ > │ > +# │ │ > │ ┌───────────────────┐ │ > +# │ > ┌─────────┴─────────┐ ┌─────────┴─────────┐ │ 192.168.0.1 │ > │ > +# │ │ > outside │ │ outside │ │ ext1 │ │ > +# │ > └─────────┬─────────┘ └─────────┬─────────┘ └─────────┬─────────┘ > │ > +# │ > ┌─────────┴─────────┐ ┌─────────┴─────────┐ ┌─────────┴─────────┐ > │ > +# │ │ > ln-outside │ │ ln-outside │ │ ln-ext1 │ │ > +# │ > └─────────┬─────────┘ └─────────┬─────────┘ └─────────┬─────────┘ > │ > +# │ > ┌─────────┴─────────┐ ┌─────────┴─────────┐ ┌─────────┴─────────┐ > │ > +# │ │ br-ex > │ │ br-ex │ │ br-ex │ │ > +# │ > └─────────┬─────────┘ └─────────┬─────────┘ └─────────┬─────────┘ > │ > +# │ > └---------eth2-----------┴-------eth2-------------┘ │ > +# │ > │ > +# > └────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ > > # The goal of this test is the check that GARP are properly generated by > higest priority traffic when > # BFD goes down, and back up, and this whether the BFD event is due > either to some bfd packet lost > @@ -3080,6 +3080,12 @@ AT_SETUP([HA: Check for missing garp on leader when > BFD goes back up]) > # So gw3 should in this test neither send garp or receive packets. > # > # Enable vconn so we can check the GARP from a log perspective. > +on_exit "podman exec ovn-gw-1 ovn-appctl vlog/set info" > +on_exit "podman exec ovn-gw-1 ovn-appctl vlog/enable-rate-limit" > +on_exit "podman exec ovn-gw-2 ovn-appctl vlog/set info" > +on_exit "podman exec ovn-gw-2 ovn-appctl vlog/enable-rate-limit" > +on_exit "podman exec ovn-gw-3 ovn-appctl vlog/set info" > +on_exit "podman exec ovn-gw-3 ovn-appctl vlog/enable-rate-limit" > nit: The cleanup should also use m_as. > m_as ovn-gw-1 ovn-appctl vlog/set vconn:dbg > m_as ovn-gw-2 ovn-appctl vlog/set vconn:dbg > m_as ovn-gw-3 ovn-appctl vlog/set vconn:dbg > @@ -3093,6 +3099,7 @@ check_fake_multinode_setup > cleanup_multinode_resources > > ip_ch1=$(m_as ovn-chassis-1 ip a show dev eth1 | grep "inet " | awk > '{print $2}'| cut -d '/' -f1) > +ip_ch2=$(m_as ovn-chassis-2 ip a show dev eth1 | grep "inet " | awk > '{print $2}'| cut -d '/' -f1) > ip_gw1=$(m_as ovn-gw-1 ip a show dev eth1 | grep "inet " | awk '{print > $2}'| cut -d '/' -f1) > ip_gw2=$(m_as ovn-gw-2 ip a show dev eth1 | grep "inet " | awk '{print > $2}'| cut -d '/' -f1) > ip_gw3=$(m_as ovn-gw-3 ip a show dev eth1 | grep "inet " | awk '{print > $2}'| cut -d '/' -f1) > @@ -3100,17 +3107,23 @@ ip_gw3=$(m_as ovn-gw-3 ip a show dev eth1 | grep > "inet " | awk '{print $2}'| cut > from_gw1_to_gw2=$(m_as ovn-gw-1 ovs-vsctl --bare --columns=name find > interface options:remote_ip=$ip_gw2) > from_gw1_to_gw3=$(m_as ovn-gw-1 ovs-vsctl --bare --columns=name find > interface options:remote_ip=$ip_gw3) > from_gw1_to_ch1=$(m_as ovn-gw-1 ovs-vsctl --bare --columns=name find > interface options:remote_ip=$ip_ch1) > +from_gw1_to_ch2=$(m_as ovn-gw-1 ovs-vsctl --bare --columns=name find > interface options:remote_ip=$ip_ch2) > from_gw2_to_gw1=$(m_as ovn-gw-2 ovs-vsctl --bare --columns=name find > interface options:remote_ip=$ip_gw1) > from_gw2_to_gw3=$(m_as ovn-gw-2 ovs-vsctl --bare --columns=name find > interface options:remote_ip=$ip_gw3) > from_gw2_to_ch1=$(m_as ovn-gw-2 ovs-vsctl --bare --columns=name find > interface options:remote_ip=$ip_ch1) > +from_gw2_to_ch2=$(m_as ovn-gw-2 ovs-vsctl --bare --columns=name find > interface options:remote_ip=$ip_ch2) > from_ch1_to_gw1=$(m_as ovn-chassis-1 ovs-vsctl --bare --columns=name find > interface options:remote_ip=$ip_gw1) > from_ch1_to_gw2=$(m_as ovn-chassis-1 ovs-vsctl --bare --columns=name find > interface options:remote_ip=$ip_gw2) > +from_ch2_to_gw1=$(m_as ovn-chassis-2 ovs-vsctl --bare --columns=name find > interface options:remote_ip=$ip_gw1) > +from_ch2_to_gw2=$(m_as ovn-chassis-2 ovs-vsctl --bare --columns=name find > interface options:remote_ip=$ip_gw2) > > m_as ovn-chassis-1 ip link del hv1-vif1-p > -m_as ovn-chassis-2 ip link del ext1-p > +m_as ovn-chassis-2 ip link del hv2-vif1-p > +m_as ovn-chassis-3 ip link del ext1-p > > OVS_WAIT_UNTIL([m_as ovn-chassis-1 ip link show | grep -q genev_sys]) > OVS_WAIT_UNTIL([m_as ovn-chassis-2 ip link show | grep -q genev_sys]) > +OVS_WAIT_UNTIL([m_as ovn-chassis-3 ip link show | grep -q genev_sys]) > OVS_WAIT_UNTIL([m_as ovn-gw-1 ip link show | grep -q genev_sys]) > OVS_WAIT_UNTIL([m_as ovn-gw-2 ip link show | grep -q genev_sys]) > OVS_WAIT_UNTIL([m_as ovn-gw-3 ip link show | grep -q genev_sys]) > @@ -3119,6 +3132,7 @@ check multinode_nbctl ls-add inside > check multinode_nbctl ls-add outside > check multinode_nbctl ls-add ext > check multinode_nbctl lsp-add inside inside1 -- lsp-set-addresses inside1 > "f0:00:c0:a8:01:01 192.168.1.1" > +check multinode_nbctl lsp-add inside inside2 -- lsp-set-addresses inside2 > "f0:00:c0:a8:01:02 192.168.1.2" > check multinode_nbctl lsp-add ext ext1 -- lsp-set-addresses ext1 > "00:00:c0:a8:00:01 192.168.0.1" > > multinode_nbctl create Logical_Router name=R1 > @@ -3150,12 +3164,14 @@ m_as ovn-gw-3 ovs-vsctl remove open . external_ids > garp-max-timeout-sec > > m_as ovn-chassis-1 ovs-vsctl set open . > external-ids:ovn-bridge-mappings=public:br-ex > m_as ovn-chassis-2 ovs-vsctl set open . > external-ids:ovn-bridge-mappings=public:br-ex > +m_as ovn-chassis-3 ovs-vsctl set open . > external-ids:ovn-bridge-mappings=public:br-ex > m_as ovn-gw-1 ovs-vsctl set open . > external-ids:ovn-bridge-mappings=public:br-ex > m_as ovn-gw-2 ovs-vsctl set open . > external-ids:ovn-bridge-mappings=public:br-ex > m_as ovn-gw-3 ovs-vsctl set open . > external-ids:ovn-bridge-mappings=public:br-ex > > m_as ovn-chassis-1 /data/create_fake_vm.sh inside1 hv1-vif1 > f0:00:c0:a8:01:01 1500 192.168.1.1 24 192.168.1.254 2000::1/64 2000::a > -m_as ovn-chassis-2 /data/create_fake_vm.sh ext1 ext1 00:00:c0:a8:00:01 > 1500 192.168.0.1 24 192.168.0.254 1000::3/64 1000::a > +m_as ovn-chassis-2 /data/create_fake_vm.sh inside2 hv2-vif1 > f0:00:c0:a8:01:02 1500 192.168.1.2 24 192.168.1.254 2000::2/64 2000::a > +m_as ovn-chassis-3 /data/create_fake_vm.sh ext1 ext1 00:00:c0:a8:00:01 > 1500 192.168.0.1 24 192.168.0.254 1000::3/64 1000::a > > # There should be one ha_chassis_group with the name "R1_outside" > m_check_row_count HA_Chassis_Group 1 name=R1_outside > @@ -3210,20 +3226,29 @@ for chassis in $from_ch1_to_gw1 $from_ch1_to_gw2; > do > wait_bfd_enabled ovn-chassis-1 $chassis > done > > +# check BFD enablement on tunnel ports from ovn-chassis-2 ########### > +for chassis in $from_ch2_to_gw1 $from_ch2_to_gw2; do > + echo "checking ovn-chassis-2 -> $chassis" > + wait_bfd_enabled ovn-chassis-2 $chassis > +done > + > # Make sure there is no nft table left. Do not use nft directly as might > not be installed in container. > gw1_pid=$(podman inspect -f '{{.State.Pid}}' ovn-gw-1) > nsenter --net=/proc/$gw1_pid/ns/net nft list tables | grep ovn-test && > nsenter --net=/proc/$gw1_pid/ns/net nft delete table ip ovn-test > -on_exit "nsenter --net=/proc/$gw1_pid/ns/net nft list tables | grep > ovn-test && nsenter --net=/proc/$gw1_pid/ns/net nft delete table ip > ovn-test" > +on_exit "if [[ -d "/proc/$gw1_pid" ]]; then nsenter > --net=/proc/$gw1_pid/ns/net nft list tables | grep ovn-test && nsenter > --net=/proc/$gw1_pid/ns/net nft delete table ip ovn-test; fi" > > -for chassis in $from_gw1_to_gw2 $from_gw1_to_gw3 $from_gw1_to_ch1; do > +for chassis in $from_gw1_to_gw2 $from_gw1_to_gw3 $from_gw1_to_ch1 > $from_gw1_to_ch2; do > wait_bfd_up ovn-gw-1 $chassis > done > -for chassis in $from_gw2_to_gw1 $from_gw2_to_gw3 $from_gw2_to_ch1; do > +for chassis in $from_gw2_to_gw1 $from_gw2_to_gw3 $from_gw2_to_ch1 > $from_gw2_to_ch2; do > wait_bfd_up ovn-gw-2 $chassis > done > for chassis in $from_ch1_to_gw1 $from_ch1_to_gw2; do > wait_bfd_up ovn-chassis-1 $chassis > done > +for chassis in $from_ch2_to_gw1 $from_ch2_to_gw2; do > + wait_bfd_up ovn-chassis-2 $chassis > +done > > m_wait_row_count Port_Binding 1 logical_port=cr-R1_outside > chassis=$gw1_chassis > check multinode_nbctl --wait=hv sync > @@ -3231,7 +3256,8 @@ check multinode_nbctl --wait=hv sync > start_tcpdump() { > echo "$(date +%H:%M:%S.%03N) Starting tcpdump" > M_START_TCPDUMPS([ovn-chassis-1], [-neei hv1-vif1-p], [ch1], > - [ovn-chassis-2], [-neei eth2], [ch2], > + [ovn-chassis-2], [-neei hv2-vif1-p], [ch2], > + [ovn-chassis-3], [-neei eth2], [ch3], > [ovn-gw-1], [-neei eth2], [gw1], > [ovn-gw-1], [-neei eth2 -Q out], [gw1_out], > [ovn-gw-2], [-neei eth2], [gw2], > @@ -3242,22 +3268,22 @@ start_tcpdump() { > > stop_tcpdump() { > echo "$(date +%H:%M:%S.%03N) Stopping tcpdump" > - m_kill 'ovn-gw-1 ovn-gw-2 ovn-gw-3 ovn-chassis-1 ovn-chassis-2' > tcpdump > + m_kill 'ovn-gw-1 ovn-gw-2 ovn-gw-3 ovn-chassis-1 ovn-chassis-2 > ovn-chassis-3' tcpdump > } > > -# Send packets from chassis2 (ext1) to chassis1 > +# Send packets from ovn-chassis-3 (ext1) to ovn-chassis-1 > send_background_packets() { > echo "$(date +%H:%M:%S.%03N) Sending packets in Background" > start_tcpdump > - M_NS_DAEMONIZE([ovn-chassis-2], [ext1], [ping -f -i 0.1 192.168.1.1], > [ping.pid]) > + M_NS_DAEMONIZE([ovn-chassis-3], [ext1], [ping -qf -i 0.1 > 192.168.1.1], [ping.pid]) > } > > stop_sending_background_packets() { > echo "$(date +%H:%M:%S.%03N) Stopping Background process" > m_as ovn-chassis-1 ps -ef | grep -v grep | grep -q ping && \ > m_as ovn-chassis-1 echo "Stopping ping on ovn-chassis-1" && > killall ping > - m_as ovn-chassis-2 ps -ef | grep -v grep | grep -q ping && \ > - m_as ovn-chassis-2 echo "Stopping ping on ovn-chassis-2" && > killall ping > + m_as ovn-chassis-3 ps -ef | grep -v grep | grep -q ping && \ > + m_as ovn-chassis-3 echo "Stopping ping on ovn-chassis-3" && > killall ping > stop_tcpdump > } > > @@ -3266,8 +3292,8 @@ check_for_new_garps() { > expecting_garp=$2 > n_new_garps=$(cat ${hv}_out.tcpdump | grep -c "f0:00:c0:a8:00:fe > > Broadcast, ethertype ARP (0x0806), length 42: Request who-has 192.168.0.254 > tell 192.168.0.254, length 28") > > - if [ "$expecting_garp" == "true" ]; then > - AS_BOX([$(date +%H:%M:%S.%03N) Waiting/checking for garp from $hv > - Starting with $n_new_garps]) > + if [[ "$expecting_garp" == "true" ]]; then > + echo "$(date +%H:%M:%S.%03N) Waiting/checking for garp from $hv - > Starting with $n_new_garps" > OVS_WAIT_UNTIL([ > n_garps=$n_new_garps > n_new_garps=$(cat ${hv}_out.tcpdump | grep -c > "f0:00:c0:a8:00:fe > Broadcast, ethertype ARP (0x0806), length 42: Request > who-has 192.168.0.254 tell 192.168.0.254, length 28") > @@ -3275,7 +3301,7 @@ check_for_new_garps() { > test "$n_garps" -ne "$n_new_garps" > ]) > else > - AS_BOX([$(date +%H:%M:%S.%03N) Checking no garp from ${hv}]) > + echo "$(date +%H:%M:%S.%03N) Checking no garp from ${hv}" > # Waiting a few seconds to get a chance to see unexpected garps. > sleep 3 > n_garps=$(cat ${hv}_out.tcpdump | grep -c "f0:00:c0:a8:00:fe > > Broadcast, ethertype ARP (0x0806), length 42: Request who-has 192.168.0.254 > tell 192.168.0.254, length 28") > @@ -3291,8 +3317,8 @@ check_for_new_echo_pkts() { > n_new_echo_req=$(cat ${hv}.tcpdump | grep -c "$mac_src > $mac_dst, > ethertype IPv4 (0x0800), length 98: 192.168.0.1 > 192.168.1.1: ICMP echo > request") > n_new_echo_rep=$(cat ${hv}.tcpdump | grep -c "$mac_dst > $mac_src, > ethertype IPv4 (0x0800), length 98: 192.168.1.1 > 192.168.0.1: ICMP echo > reply") > > - if [ "$expecting_pkts" == "true" ]; then > - AS_BOX([$(date +%H:%M:%S.%03N) Waiting/checking for echo pkts > through ${hv}]) > + if [[ "$expecting_pkts" == "true" ]]; then > + echo "$(date +%H:%M:%S.%03N) Waiting/checking for echo pkts > through ${hv}" > echo "Starting with $n_new_echo_req requests and $n_new_echo_rep > replies so far on ${hv}." > OVS_WAIT_UNTIL([ > n_echo_req=$n_new_echo_req > @@ -3302,8 +3328,8 @@ check_for_new_echo_pkts() { > echo "We saw $n_new_echo_req requests and $n_new_echo_rep > replies so far on ${hv}." > test "$n_echo_req" -ne "$n_new_echo_req" && test > "$n_echo_rep" -ne "$n_new_echo_rep" > ]) > - else > - AS_BOX([$(date +%H:%M:%S.%03N) Checking no pkts from ${hv}]) > + elif [[ "$expecting_pkts" == "false" ]]; then > + echo "$(date +%H:%M:%S.%03N) Checking no pkts from ${hv}" > # Waiting a few seconds to get a chance to see unexpected pkts. > sleep 3 > n_echo_req=$(cat ${hv}.tcpdump | grep -c "$mac_src > $mac_dst, > ethertype IPv4 (0x0800), length 98: 192.168.0.1 > 192.168.1.1: ICMP echo > request") > @@ -3317,26 +3343,24 @@ dump_statistics() { > n1=$(m_as ovn-gw-1 grep -c Changing /var/log/ovn/ovn-controller.log) > n2=$(m_as ovn-gw-2 grep -c Changing /var/log/ovn/ovn-controller.log) > n3=$(m_as ovn-gw-3 grep -c Changing /var/log/ovn/ovn-controller.log) > - ch1_req=$(grep -c "ICMP echo request" ch1.tcpdump) > - ch1_rep=$(grep -c "ICMP echo reply" ch1.tcpdump) > - ch2_req=$(grep -c "ICMP echo request" ch2.tcpdump) > - ch2_rep=$(grep -c "ICMP echo reply" ch2.tcpdump) > - gw1_req=$(grep -c "ICMP echo request" gw1.tcpdump) > - gw1_rep=$(grep -c "ICMP echo reply" gw1.tcpdump) > - gw2_req=$(grep -c "ICMP echo request" gw2.tcpdump) > - gw2_rep=$(grep -c "ICMP echo reply" gw2.tcpdump) > - gw3_req=$(grep -c "ICMP echo request" gw3.tcpdump) > - gw3_rep=$(grep -c "ICMP echo reply" gw3.tcpdump) > - echo "$n1 claims in gw1, $n2 in gw2 and $n3 on gw3" > - echo "ch2_request=$ch2_req gw1_request=$gw1_req gw2_request=$gw2_req > gw3_request=$gw3_req ch1_request=$ch1_req ch1_reply=$ch1_rep > gw1_reply=$gw1_rep gw2_reply=$gw2_rep gw3_reply=$gw3_rep ch2_reply=$ch2_rep" > + > + # Read request and reply in one command to avoid having e.g. more > replies than requests. > + read -r ch1_req ch1_rep <<< $(awk '/ICMP echo request/{req++} /ICMP > echo reply/{rep++} END {print req+0, rep+0}' ch1.tcpdump) > + read -r ch2_req ch2_rep <<< $(awk '/ICMP echo request/{req++} /ICMP > echo reply/{rep++} END {print req+0, rep+0}' ch2.tcpdump) > + read -r ch3_req ch3_rep <<< $(awk '/ICMP echo request/{req++} /ICMP > echo reply/{rep++} END {print req+0, rep+0}' ch3.tcpdump) > + read -r gw1_req gw1_rep <<< $(awk '/ICMP echo request/{req++} /ICMP > echo reply/{rep++} END {print req+0, rep+0}' gw1.tcpdump) > + read -r gw2_req gw2_rep <<< $(awk '/ICMP echo request/{req++} /ICMP > echo reply/{rep++} END {print req+0, rep+0}' gw2.tcpdump) > + read -r gw3_req gw3_rep <<< $(awk '/ICMP echo request/{req++} /ICMP > echo reply/{rep++} END {print req+0, rep+0}' gw3.tcpdump) > + > + echo "$n1 claims in gw1, $n2 in gw2 and $n3 on gw3" >&2 > + echo "ch3_req=$ch3_req gw_req=($gw1_req + $gw2_req +$gw3_req) > ch1_req=$ch1_req ch1_rep=$ch1_rep gw_rep=($gw1_rep + $gw2_rep + $gw3_rep) > ch3_rep=$ch3_rep ch2=($ch2_req+$ch2_rep)" >&2 > + echo "$((ch3_req - ch3_rep))" > } > > -check_migration_between_gw1_and_gw2() { > - action=$1 > +prepare() { > send_background_packets > - > # We make sure gw1 is leader since enough time that it generated all > its garps. > - AS_BOX([$(date +%H:%M:%S.%03N) Waiting all garps sent by gw1]) > + echo $(date +%H:%M:%S.%03N) Waiting all garps sent by gw1 > n_new_garps=$(cat gw1_out.tcpdump | grep -c "f0:00:c0:a8:00:fe > > Broadcast, ethertype ARP (0x0806), length 42: Request who-has 192.168.0.254 > tell 192.168.0.254, length 28") > OVS_WAIT_UNTIL([ > n_garps=$n_new_garps > @@ -3347,135 +3371,186 @@ check_migration_between_gw1_and_gw2() { > test "$n_garps" -eq "$n_new_garps" > ]) > > - # All packets should go through gw1, and none through gw2 or gw3. > - check_for_new_echo_pkts gw1 "00:00:c0:a8:00:01" "f0:00:c0:a8:00:fe" > "true" > - check_for_new_echo_pkts gw2 "00:00:c0:a8:00:01" "f0:00:c0:a8:00:fe" > "false" > - check_for_new_echo_pkts gw3 "00:00:c0:a8:00:01" "f0:00:c0:a8:00:fe" > "false" > - > + # All packets should go through gw1, none through gw2 or gw3, and all > through ch1. > + check_packets "true" "false" "false" "true" > flap_count_gw_1=$(m_as ovn-gw-1 ovs-vsctl get interface > $from_gw1_to_gw2 bfd_status | sed 's/.*flap_count=\"\([[0-9]]*\).*/\1/g') > flap_count_gw_2=$(m_as ovn-gw-2 ovs-vsctl get interface > $from_gw2_to_gw1 bfd_status | sed 's/.*flap_count=\"\([[0-9]]*\).*/\1/g') > +} > > - if [ test "$action" == "stop_bfd" ]; then > - AS_BOX([$(date +%H:%M:%S.%03N) Blocking bfd on gw1 (from $ip_gw1 > to $ip_gw2)]) > - nsenter --net=/proc/$gw1_pid/ns/net nft add table ip ovn-test > - nsenter --net=/proc/$gw1_pid/ns/net nft 'add chain ip ovn-test > INPUT { type filter hook input priority 0; policy accept; }' > - # Drop BFD from gw-1 to gw-2: geneve port (6081), inner port 3784 > (0xec8), Session state Up, Init, Down. > - nsenter --net=/proc/$gw1_pid/ns/net nft add rule ip ovn-test > INPUT ip daddr $ip_gw1 ip saddr $ip_gw2 udp dport 6081 '@th,416,16 == > 0x0ec8 @th,472,8 == 0xc0 counter drop' > - nsenter --net=/proc/$gw1_pid/ns/net nft add rule ip ovn-test > INPUT ip daddr $ip_gw1 ip saddr $ip_gw2 udp dport 6081 '@th,416,16 == > 0x0ec8 @th,472,8 == 0x80 counter drop' > - nsenter --net=/proc/$gw1_pid/ns/net nft add rule ip ovn-test > INPUT ip daddr $ip_gw1 ip saddr $ip_gw2 udp dport 6081 '@th,416,16 == > 0x0ec8 @th,472,8 == 0x40 counter drop' > - > - # We do not check that packets go through gw2 as BFD between > chassis-2 and gw1 is still up > - fi > - > - if [ test "$action" == "kill_gw2" ]; then > - AS_BOX([$(date +%H:%M:%S.%03N) Killing gw2 ovn-controller]) > - on_exit 'm_as ovn-gw-2 /usr/share/openvswitch/scripts/ovs-ctl > status || > - m_as ovn-gw-2 /usr/share/openvswitch/scripts/ovs-ctl > start --system-id=ovn-gw-2' > - on_exit 'm_as ovn-gw-2 /usr/share/ovn/scripts/ovn-ctl > status_controller || > - m_as ovn-gw-2 /usr/share/ovn/scripts/ovn-ctl > start_controller ${CONTROLLER_SSL_ARGS}' > - > - m_as ovn-gw-2 kill -9 $(m_as ovn-gw-2 cat > /run/ovn/ovn-controller.pid) > - m_as ovn-gw-2 kill -9 $(m_as ovn-gw-2 cat > /run/openvswitch/ovs-vswitchd.pid) > - m_as ovn-gw-2 kill -9 $(m_as ovn-gw-2 cat > /run/openvswitch/ovsdb-server.pid) > - # Also delete datapath (flows) > - m_as ovn-gw-2 ovs-dpctl del-dp system@ovs-system > - fi > - > - if [ test "$action" == "kill_gw1" ]; then > - AS_BOX([$(date +%H:%M:%S.%03N) Killing gw1 ovn-controller]) > - on_exit 'm_as ovn-gw-1 /usr/share/openvswitch/scripts/ovs-ctl > status || > - m_as ovn-gw-1 /usr/share/openvswitch/scripts/ovs-ctl > start --system-id=ovn-gw-1' > - on_exit 'm_as ovn-gw-1 /usr/share/ovn/scripts/ovn-ctl > status_controller || > - m_as ovn-gw-1 /usr/share/ovn/scripts/ovn-ctl > start_controller ${CONTROLLER_SSL_ARGS}' > - > - m_as ovn-gw-1 kill -9 $(m_as ovn-gw-1 cat > /run/ovn/ovn-controller.pid) > - m_as ovn-gw-1 kill -9 $(m_as ovn-gw-1 cat > /run/openvswitch/ovs-vswitchd.pid) > - m_as ovn-gw-1 kill -9 $(m_as ovn-gw-1 cat > /run/openvswitch/ovsdb-server.pid) > - # Also delete datapath (flows) > - m_as ovn-gw-1 ovs-dpctl del-dp system@ovs-system > - fi > +check_loss_after_flap() > +{ > + dead=$1 > + max_expected_loss=$2 > > - if [ test "$action" == "kill_gw2" ]; then > - AS_BOX([$(date +%H:%M:%S.%03N) Waiting for flap count between gw1 > and gw2 to increase]) > + if [[ "$dead" == "gw2" ]]; then > + echo "$(date +%H:%M:%S.%03N) Waiting for flap count between gw1 > and gw2 to increase" > OVS_WAIT_UNTIL([ > - new_flap_count=$(m_as ovn-gw-1 ovs-vsctl get interfac > $from_gw1_to_gw2 bfd_status | sed 's/.*flap_count=\"\([[0-9]]*\).*/\1/g') > + new_flap_count=$(m_as ovn-gw-1 ovs-vsctl get interface > $from_gw1_to_gw2 bfd_status | sed 's/.*flap_count=\"\([[0-9]]*\).*/\1/g') > echo "Comparing $new_flap_count versus $flap_count_gw_1" > test "$new_flap_count" -gt "$((flap_count_gw_1))" > ]) > else > - AS_BOX([$(date +%H:%M:%S.%03N) Waiting for flap count between gw2 > and gw1 to increase]) > + echo "$(date +%H:%M:%S.%03N) Waiting for flap count between gw2 > and gw1 to increase" > OVS_WAIT_UNTIL([ > - new_flap_count=$(m_as ovn-gw-2 ovs-vsctl get interfac > $from_gw2_to_gw1 bfd_status | sed 's/.*flap_count=\"\([[0-9]]*\).*/\1/g') > + new_flap_count=$(m_as ovn-gw-2 ovs-vsctl get interface > $from_gw2_to_gw1 bfd_status | sed 's/.*flap_count=\"\([[0-9]]*\).*/\1/g') > echo "Comparing $new_flap_count versus $flap_count_gw_2" > test "$new_flap_count" -gt "$((flap_count_gw_2))" > ]) > - > fi > - AS_BOX([$(date +%H:%M:%S.%03N) Flapped!]) > > + echo "$(date +%H:%M:%S.%03N) Flapped!" > # Wait a few more second for the fight. > + sleep 4 > + > + echo "$(date +%H:%M:%S.%03N) Statistics after flapping" > + lost=$(dump_statistics) > + echo "===> $lost packet lost while handling migration" > + AT_CHECK([test "$lost" -le "$max_expected_loss"]) > +} > + > +final_check() > +{ > + action=$1 > + max_expected_loss_after_restoration=$2 > + > + # Wait a little more to get packets while network is restored > sleep 2 > - AS_BOX([$(date +%H:%M:%S.%03N) Statistics after flapping]) > - dump_statistics > - > - if [ test "$action" == "stop_bfd" ]; then > - # gw1 still alive and gw2 tried to claim => gw1 should restart > generating garps. > - check_for_new_garps gw1 "true" > - check_for_new_garps gw2 "false" > - check_for_new_garps gw3 "false" > - check_for_new_echo_pkts gw1 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe > "true" > - check_for_new_echo_pkts gw2 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe > "false" > - check_for_new_echo_pkts gw3 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe > "false" > - check_for_new_echo_pkts ch1 f0:00:c0:a8:01:fe f0:00:c0:a8:01:01 > "true" > - AS_BOX([$(date +%H:%M:%S.%03N) Unblocking bfd on gw1]) > - nsenter --net=/proc/$gw1_pid/ns/net nft -a list ruleset > - nsenter --net=/proc/$gw1_pid/ns/net nft delete table ip ovn-test > - fi > + echo "$(date +%H:%M:%S.%03N) Statistics after network restored (after > $action)" > + new_lost=$(dump_statistics) > + echo "===> $((new_lost - lost)) packets lost during network > restoration" > + AT_CHECK([test "$((new_lost - lost))" -le > "$max_expected_loss_after_restoration"]) > + stop_sending_background_packets > +} > > - if [ test "$action" == "kill_gw2" ]; then > - # gw1 still alive, but gw2 did not try to claim => gw1 should not > generate new garps. > - check_for_new_garps gw1 "false" > - check_for_new_garps gw2 "false" > - check_for_new_garps gw3 "false" > - check_for_new_echo_pkts gw1 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe > "true" > - check_for_new_echo_pkts gw2 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe > "false" > - check_for_new_echo_pkts gw3 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe > "false" > - check_for_new_echo_pkts ch1 f0:00:c0:a8:01:fe f0:00:c0:a8:01:01 > "true" > - AS_BOX([$(date +%H:%M:%S.%03N) Restarting gw2 ovn-vswitchd]) > - m_as ovn-gw-2 /usr/share/openvswitch/scripts/ovs-ctl start > --system-id=ovn-gw-2 > - > - AS_BOX([$(date +%H:%M:%S.%03N) Restarting gw2 ovn-controller]) > - m_as ovn-gw-2 /usr/share/ovn/scripts/ovn-ctl start_controller > ${CONTROLLER_SSL_ARGS} > - fi > +check_garps() > +{ > + check_for_new_garps gw1 "$1" > + check_for_new_garps gw2 "$2" > + check_for_new_garps gw3 "$3" > +} > > - if [ test "$action" == "kill_gw1" ]; then > - # gw1 died => gw2 should generate garps. > - check_for_new_garps gw1 "false" > - check_for_new_garps gw2 "true" > - check_for_new_garps gw3 "false" > - check_for_new_echo_pkts gw1 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe > "false" > - check_for_new_echo_pkts gw2 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe > "true" > - check_for_new_echo_pkts gw3 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe > "false" > - check_for_new_echo_pkts ch1 f0:00:c0:a8:01:fe f0:00:c0:a8:01:01 > "true" > - AS_BOX([$(date +%H:%M:%S.%03N) Restarting gw1 ovn-vswitchd]) > - m_as ovn-gw-1 /usr/share/openvswitch/scripts/ovs-ctl start > --system-id=ovn-gw-1 > - > - AS_BOX([$(date +%H:%M:%S.%03N) Restarting gw1 ovn-controller]) > - m_as ovn-gw-1 /usr/share/ovn/scripts/ovn-ctl start_controller > ${CONTROLLER_SSL_ARGS} > - fi > +check_packets() > +{ > + check_for_new_echo_pkts gw1 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe "$1" > + check_for_new_echo_pkts gw2 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe "$2" > + check_for_new_echo_pkts gw3 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe "$3" > + check_for_new_echo_pkts ch1 f0:00:c0:a8:01:fe f0:00:c0:a8:01:01 "$4" > +} > + > +start_openvswitch() > +{ > + chassis=$1 > + echo "$(date +%H:%M:%S.%03N) Simulate openvswitch start: start ovsdb, > delete transient ports, start ovs-vswitchd" > + m_as $chassis /usr/share/openvswitch/scripts/ovs-ctl start > --system-id=$chassis --no-ovs-vswitchd > + m_as $chassis /usr/share/openvswitch/scripts/ovs-ctl > delete-transient-ports > + m_as $chassis /usr/share/openvswitch/scripts/ovs-ctl start > --system-id=$chassis --no-ovsdb-server > +} > + > +check_migration_between_gw1_and_gw2_bfd_stop() > +{ > + AS_BOX([$(date +%H:%M:%S.%03N) Testing migration after bfd_stop]) > + max_expected_loss1=$1 > + max_expected_loss2=$2 > + prepare > + > + echo "$(date +%H:%M:%S.%03N) Blocking bfd on gw1 (from $ip_gw1 to > $ip_gw2)" > + nsenter --net=/proc/$gw1_pid/ns/net nft add table ip ovn-test > + nsenter --net=/proc/$gw1_pid/ns/net nft 'add chain ip ovn-test INPUT > { type filter hook input priority 0; policy accept; }' > + # Drop BFD from gw-1 to gw-2: geneve port (6081), inner port 3784 > (0xec8), Session state Up, Init, Down. > + nsenter --net=/proc/$gw1_pid/ns/net nft add rule ip ovn-test INPUT ip > daddr $ip_gw1 ip saddr $ip_gw2 udp dport 6081 '@th,416,16 == 0x0ec8 > @th,472,8 == 0xc0 counter drop' > + nsenter --net=/proc/$gw1_pid/ns/net nft add rule ip ovn-test INPUT ip > daddr $ip_gw1 ip saddr $ip_gw2 udp dport 6081 '@th,416,16 == 0x0ec8 > @th,472,8 == 0x80 counter drop' > + nsenter --net=/proc/$gw1_pid/ns/net nft add rule ip ovn-test INPUT ip > daddr $ip_gw1 ip saddr $ip_gw2 udp dport 6081 '@th,416,16 == 0x0ec8 > @th,472,8 == 0x40 counter drop' > + > + check_loss_after_flap "gw1" $max_expected_loss1 > + > + # gw1 still alive and gw2 tried to claim => gw1 should restart > generating garps. > + check_garps "true" "false" "false" > + check_packets "true" "false" "false" "true" > + > + echo "$(date +%H:%M:%S.%03N) Unblocking bfd on gw1" > + nsenter --net=/proc/$gw1_pid/ns/net nft -a list ruleset > + nsenter --net=/proc/$gw1_pid/ns/net nft delete table ip ovn-test > > # The network is now restored => packets should go through gw1 and > reach chassis-1. > - check_for_new_echo_pkts gw1 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe "true" > - check_for_new_echo_pkts gw2 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe > "false" > - check_for_new_echo_pkts gw3 00:00:c0:a8:00:01 f0:00:c0:a8:00:fe > "false" > - check_for_new_echo_pkts ch1 f0:00:c0:a8:01:fe f0:00:c0:a8:01:01 "true" > - AS_BOX([$(date +%H:%M:%S.%03N) Statistics after network restored]) > - dump_statistics > - stop_sending_background_packets > + check_packets "true" "false" "false" "true" > + final_check "bfd_stop" $max_expected_loss2 > +} > + > +check_migration_between_gw1_and_gw2_kill_gw2() { > + AS_BOX([$(date +%H:%M:%S.%03N) Check migration after killing gw2 > ovn-controller & vswitchd]) > + max_expected_loss1=$1 > + max_expected_loss2=$2 > + prepare > + > + gw2_ip_mask=$(m_as ovn-gw-2 ip -4 -o addr show eth1 | awk '{print > $4}') > + on_exit 'm_as ovn-gw-2 /usr/share/openvswitch/scripts/ovs-ctl status > || > + m_as ovn-gw-2 /usr/share/openvswitch/scripts/ovs-ctl start > --system-id=ovn-gw-2' > + on_exit 'm_as ovn-gw-2 /usr/share/ovn/scripts/ovn-ctl > status_controller || > + m_as ovn-gw-2 /usr/share/ovn/scripts/ovn-ctl > start_controller ${CONTROLLER_SSL_ARGS}' > + on_exit 'm_as ovn-gw-2 ip addr add $gw2_ip_mask dev eth1' > + on_exit 'm_as ovn-gw-2 ip link set eth1 up' > + > + m_as ovn-gw-2 kill -9 $(m_as ovn-gw-2 cat /run/ovn/ovn-controller.pid) > + m_as ovn-gw-2 kill -9 $(m_as ovn-gw-2 cat > /run/openvswitch/ovs-vswitchd.pid) > + m_as ovn-gw-2 kill -9 $(m_as ovn-gw-2 cat > /run/openvswitch/ovsdb-server.pid) > + m_as ovn-gw-2 ovs-dpctl del-dp system@ovs-system > + > + check_loss_after_flap "gw2" $max_expected_loss1 > + > + # gw1 still alive, but gw2 did not try to claim => gw1 should not > generate new garps. > + check_garps "false" "false" "false" > + check_packets "true" "false" "false" "true" > + > + start_openvswitch ovn-gw-2 > + > + echo "$(date +%H:%M:%S.%03N) Restarting gw2 ovn-controller" > + m_as ovn-gw-2 /usr/share/ovn/scripts/ovn-ctl start_controller > ${CONTROLLER_SSL_ARGS} > + > + # The network is now restored => packets should go through gw1 and > reach chassis-1. > + check_packets "true" "false" "false" "true" > + final_check "kill_gw2" $max_expected_loss2 > +} > + > +check_migration_between_gw1_and_gw2_kill_gw1() { > + AS_BOX([$(date +%H:%M:%S.%03N) Killing gw1 ovn-controller and > ovs-vswitchd]) > + max_expected_loss1=$1 > + max_expected_loss2=$2 > + prepare > + > + on_exit 'm_as ovn-gw-1 /usr/share/openvswitch/scripts/ovs-ctl status > || > + m_as ovn-gw-1 /usr/share/openvswitch/scripts/ovs-ctl start > --system-id=ovn-gw-1' > + on_exit 'm_as ovn-gw-1 /usr/share/ovn/scripts/ovn-ctl > status_controller || > + m_as ovn-gw-1 /usr/share/ovn/scripts/ovn-ctl > start_controller ${CONTROLLER_SSL_ARGS}' > + > + m_as ovn-gw-1 kill -9 $(m_as ovn-gw-1 cat /run/ovn/ovn-controller.pid) > + m_as ovn-gw-1 kill -9 $(m_as ovn-gw-1 cat > /run/openvswitch/ovs-vswitchd.pid) > + m_as ovn-gw-1 kill -9 $(m_as ovn-gw-1 cat > /run/openvswitch/ovsdb-server.pid) > + # Also delete datapath (flows) > + m_as ovn-gw-1 ovs-dpctl del-dp system@ovs-system > + > + check_loss_after_flap "gw1" $max_expected_loss1 > + > + # gw1 died => gw2 should generate garps. > + check_garps "false" "true" "false" > + check_packets "false" "true" "false" "true" > + start_openvswitch ovn-gw-1 > + > + # Wait some long time before restarting ovn-controller > + sleep 10 > + > + # gw2 should still be handling packets as OVN not restarted on gw1 > + check_packets "false" "true" "false" "true" > + > + echo "$(date +%H:%M:%S.%03N) Restarting gw1 ovn-controller after > killing gw1" > + m_as ovn-gw-1 /usr/share/ovn/scripts/ovn-ctl start_controller > ${CONTROLLER_SSL_ARGS} > + > + # The network is now restored => packets should go through gw1 and > reach chassis-1. > + check_packets "true" "false" "false" "true" > + final_check "kill_gw1" $max_expected_loss2 > } > > start_tcpdump > -AS_BOX([$(date +%H:%M:%S.%03N) Sending packet from hv1-vif1(inside1) to > ext1]) > +echo "$(date +%H:%M:%S.%03N) Sending packet from hv1-vif1(inside1) to > ext1" > M_NS_CHECK_EXEC([ovn-chassis-1], [hv1-vif1], [ping -c3 -q -i 0.1 > 192.168.0.1 | FORMAT_PING], > [0], [dnl > 3 packets transmitted, 3 received, 0% packet loss, time 0ms > @@ -3483,7 +3558,7 @@ M_NS_CHECK_EXEC([ovn-chassis-1], [hv1-vif1], [ping > -c3 -q -i 0.1 192.168.0.1 | F > stop_tcpdump > > # It should have gone through gw1 and not gw2 > -AS_BOX([$(date +%H:%M:%S.%03N) Checking it went through gw1 and not gw2]) > +echo "$(date +%H:%M:%S.%03N) Checking it went through gw1 and not gw2" > AT_CHECK([cat gw2.tcpdump | grep "ICMP echo"], [1], [dnl > ]) > > @@ -3496,17 +3571,17 @@ f0:00:c0:a8:00:fe > 00:00:c0:a8:00:01, ethertype > IPv4 (0x0800), length 98: 192.1 > 00:00:c0:a8:00:01 > f0:00:c0:a8:00:fe, ethertype IPv4 (0x0800), length > 98: 192.168.0.1 > 192.168.1.1: ICMP echo reply, > ]) > > -# We stop bfd between gw1 & gw2, but keep gw1 & gw2 running. > -check_migration_between_gw1_and_gw2 "stop_bfd" > +lost=0 > +# We stop bfd between gw1 & gw2, but keep gw1 & gw2 running. We should > not lose packets. > +check_migration_between_gw1_and_gw2_bfd_stop 1 1 > > # We simulate death of gw2. It should not have any effect. > -check_migration_between_gw1_and_gw2 "kill_gw2" > +check_migration_between_gw1_and_gw2_kill_gw2 1 1 > > -# We simulate death of gw1. gw2 should take over. > -check_migration_between_gw1_and_gw2 "kill_gw1" > +# We simulate restart of both OVS & OVN gw1. gw2 should take over. > +check_migration_between_gw1_and_gw2_kill_gw1 400 200 > > AT_CLEANUP > -]) > > AT_SETUP([ovn multinode bgp L2 EVPN]) > check_fake_multinode_setup > -- > 2.47.1 > > _______________________________________________ > dev mailing list > [email protected] > https://mail.openvswitch.org/mailman/listinfo/ovs-dev > > _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
