Add HA tests, simulating bfd going down between two ha gw.
Signed-off-by: Xavier Simonart <[email protected]>
---
tests/multinode.at | 324 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 324 insertions(+)
diff --git a/tests/multinode.at b/tests/multinode.at
index 49e128387..3840ebd50 100644
--- a/tests/multinode.at
+++ b/tests/multinode.at
@@ -3078,4 +3078,328 @@ m_as ovn-chassis-3 killall tcpdump
AT_CLEANUP
+AT_SETUP([HA: Check for missing garp on leader when BFD goes back up])
+# Network topology
+#
┌────────────────────────────────────────────────────────────────────────────────────────────────────────┐
+# │
│
+# │ ┌───────────────────┐ ┌───────────────────┐
┌───────────────────┐ ┌───────────────────┐ │
+# │ │ ovn-chassis-1 │ │ ovn-gw-1 │ │ ovn-gw-2
│ │ ovn-chassis-2 │ │
+# │ └─────────┬─────────┘ └───────────────────┘
└───────────────────┘ └───────────────────┘ │
+# │ ┌─────────┴─────────┐
│
+# │ │ inside1 │
│
+# │ │ 192.168.1.1/24 │
│
+# │ └─────────┬─────────┘
│
+# │ ┌─────────┴─────────┐
│
+# │ │ inside │
│
+# │ └─────────┬─────────┘
│
+# │ ┌─────────┴─────────┐
│
+# │ │ 192.168.1.254 │
│
+# │ │ R1 │
│
+# │ │ 192.168.0.254 │
│
+# │ └─────────┬─────────┘
│
+# │ └------eth1---------------┬--------eth1-----------┐
│
+# │ ┌──────────┴────────┐
┌─────────┴─────────┐ │
+# │ │ 192.168.1.254 │ │ 192.168.1.254
│ │
+# │ │ R1 │ │ R1
│ │
+# │ │ 192.168.0.254 │ │ 192.168.0.254
│ │
+# │ └─────────┬─────────┘
└─────────┬─────────┘ │
+# │ │ │
┌───────────────────┐ │
+# │ ┌─────────┴─────────┐
┌─────────┴─────────┐ │ 192.168.0.1 │ │
+# │ │ outside │ │ outside
│ │ ext1 │ │
+# │ └─────────┬─────────┘
└─────────┬─────────┘ └─────────┬─────────┘ │
+# │ ┌─────────┴─────────┐
┌─────────┴─────────┐ ┌─────────┴─────────┐ │
+# │ │ ln-outside │ │ ln-outside
│ │ ln-ext1 │ │
+# │ └─────────┬─────────┘
└─────────┬─────────┘ └─────────┬─────────┘ │
+# │ ┌─────────┴─────────┐
┌─────────┴─────────┐ ┌─────────┴─────────┐ │
+# │ │ br-ex │ │ br-ex
│ │ br-ex │ │
+# │ └─────────┬─────────┘
└─────────┬─────────┘ └─────────┬─────────┘ │
+# │
└---------eth2-----------┴-------eth2-------------┘ │
+# │
│
+#
└────────────────────────────────────────────────────────────────────────────────────────────────────────┘
+
+# The goal of this test is the check that GARP are properly generated by
higest priority traffic when
+# BFD comes back up.
+#
+# Enable vconn so we can check the GARP from a log perspective.
+m_as ovn-gw-1 ovn-appctl vlog/set vconn:dbg
+m_as ovn-gw-2 ovn-appctl vlog/set vconn:dbg
+m_as ovn-gw-1 ovn-appctl vlog/disable-rate-limit
+m_as ovn-gw-2 ovn-appctl vlog/disable-rate-limit
+
+check_fake_multinode_setup_by_nodes 'ovn-chassis-1 ovn-chassis-2 ovn-gw-1
ovn-gw-2'
+
+# Delete the multinode NB and OVS resources before starting the test.
+cleanup_multinode_resources_by_nodes 'ovn-chassis-1 ovn-chassis-2 ovn-gw-1
ovn-gw-2'
+
+ip_ch1=$(m_as ovn-chassis-1 ip a show dev eth1 | grep "inet " | awk '{print
$2}'| cut -d '/' -f1)
+ip_gw1=$(m_as ovn-gw-1 ip a show dev eth1 | grep "inet " | awk '{print $2}'|
cut -d '/' -f1)
+ip_gw2=$(m_as ovn-gw-2 ip a show dev eth1 | grep "inet " | awk '{print $2}'|
cut -d '/' -f1)
+
+from_gw1_to_ch1=$(m_as ovn-gw-1 ovs-vsctl --bare --columns=name find interface
options:remote_ip=$ip_ch1)
+from_gw2_to_ch1=$(m_as ovn-gw-2 ovs-vsctl --bare --columns=name find interface
options:remote_ip=$ip_ch1)
+m_as ovn-chassis-1 ip link del hv1-vif1-p
+m_as ovn-chassis-2 ip link del ext1-p
+
+OVS_WAIT_UNTIL([m_as ovn-chassis-1 ip link show | grep -q genev_sys])
+OVS_WAIT_UNTIL([m_as ovn-chassis-2 ip link show | grep -q genev_sys])
+OVS_WAIT_UNTIL([m_as ovn-gw-1 ip link show | grep -q genev_sys])
+OVS_WAIT_UNTIL([m_as ovn-gw-2 ip link show | grep -q genev_sys])
+
+check multinode_nbctl ls-add inside
+check multinode_nbctl ls-add outside
+check multinode_nbctl ls-add ext
+check multinode_nbctl lsp-add inside inside1 -- lsp-set-addresses inside1
"f0:00:c0:a8:01:01 192.168.1.1"
+check multinode_nbctl lsp-add ext ext1 -- lsp-set-addresses ext1
"00:00:c0:a8:00:01 192.168.0.1"
+
+multinode_nbctl create Logical_Router name=R1
+check multinode_nbctl lrp-add R1 R1_inside f0:00:c0:a8:01:fe 192.168.1.254/24
+check multinode_nbctl -- lsp-add inside inside_R1 \
+ -- set Logical_Switch_Port inside_R1 type=router
options:router-port=R1_inside \
+ -- lsp-set-addresses inside_R1 router
+
+check multinode_nbctl lrp-add R1 R1_outside f0:00:c0:a8:00:fe 192.168.0.254/24
+check multinode_nbctl -- lsp-add outside outside_R1 \
+ -- set Logical_Switch_Port outside_R1 type=router
options:router-port=R1_outside \
+ -- lsp-set-addresses outside_R1 router
+
+multinode_nbctl -- --id=@gc0 create Gateway_Chassis name=outside_gw1
chassis_name=ovn-gw-1 priority=20 \
+ -- --id=@gc1 create Gateway_Chassis name=outside_gw2
chassis_name=ovn-gw-2 priority=10 \
+ -- set Logical_Router_Port R1_outside
'gateway_chassis=[@gc0,@gc1]'
+
+# Create localnet port in outside
+check multinode_nbctl lsp-add outside ln-outside
+check multinode_nbctl lsp-set-addresses ln-outside unknown
+check multinode_nbctl lsp-set-type ln-outside localnet
+check multinode_nbctl lsp-set-options ln-outside network_name=public
+
+# Create localnet port in ext1
+check multinode_nbctl lsp-add ext ln-ext1
+check multinode_nbctl lsp-set-addresses ln-ext1 unknown
+check multinode_nbctl lsp-set-type ln-ext1 localnet
+check multinode_nbctl lsp-set-options ln-ext1 network_name=public
+
+# Make sure garp-max-timeout-sec is not set
+m_as ovn-gw-1 ovs-vsctl remove open . external_ids garp-max-timeout-sec
+m_as ovn-gw-2 ovs-vsctl remove open . external_ids garp-max-timeout-sec
+
+m_as ovn-chassis-1 ovs-vsctl set open .
external-ids:ovn-bridge-mappings=public:br-ex
+m_as ovn-chassis-2 ovs-vsctl set open .
external-ids:ovn-bridge-mappings=public:br-ex
+m_as ovn-gw-1 ovs-vsctl set open .
external-ids:ovn-bridge-mappings=public:br-ex
+m_as ovn-gw-2 ovs-vsctl set open .
external-ids:ovn-bridge-mappings=public:br-ex
+
+m_as ovn-chassis-1 /data/create_fake_vm.sh inside1 hv1-vif1 f0:00:c0:a8:01:01
1500 192.168.1.1 24 192.168.1.254 2000::1/64 2000::a
+m_as ovn-chassis-2 /data/create_fake_vm.sh ext1 ext1 00:00:c0:a8:00:01 1500
192.168.0.1 24 192.168.0.254 1000::3/64 1000::a
+
+# There should be one ha_chassis_group with the name "R1_outside"
+m_check_row_count HA_Chassis_Group 1 name=R1_outside
+
+# There should be 2 ha_chassis rows in SB DB.
+m_check_row_count HA_Chassis 2 'chassis!=[[]]'
+
+ha_ch=$(m_fetch_column HA_Chassis_Group ha_chassis)
+m_check_column "$ha_ch" HA_Chassis _uuid
+
+gw1_chassis=$(m_fetch_column Chassis _uuid name=ovn-gw-1)
+gw2_chassis=$(m_fetch_column Chassis _uuid name=ovn-gw-2)
+
+wait_bfd_enabled() {
+ chassis=$1
+ interface=$2
+ echo "Using $chassis $interface"
+ OVS_WAIT_UNTIL([test 1 = $(m_as $chassis ovs-vsctl --bare --columns bfd
find Interface name=$interface | \
+grep "enable=true" | wc -l)
+])
+}
+
+wait_bfd_up() {
+ hv1=$1
+ hv2=$2
+
+ echo "checking bfd_status for $hv1 => $hv2"
+ OVS_WAIT_UNTIL([
+ state=$(m_as $hv1 ovs-vsctl get interface $hv2 bfd_status:state)
+ remote_state=$(m_as $hv1 ovs-vsctl get interface $hv2
bfd_status:remote_state)
+ echo "$(date +%H:%M:%S.%03N) bfd state = $state, remote_state =
$remote_state"
+ test "$state" = "up" -a "$remote_state" = "up"
+ ])
+}
+
+# check BFD enablement on tunnel ports from ovn-gw-1 ##########
+for chassis in ovn-ovn-gw-0 $from_gw1_to_ch1; do
+ echo "checking ovn-gw-1 -> $chassis"
+ wait_bfd_enabled ovn-gw-1 $chassis
+done
+
+# check BFD enablement on tunnel ports from ovn-gw-2 ##########
+for chassis in ovn-ovn-gw-0 $from_gw2_to_ch1; do
+ echo "checking ovn-gw-2 -> $chassis"
+ wait_bfd_enabled ovn-gw-2 $chassis
+done
+
+# check BFD enablement on tunnel ports from ovn-chassis-1 ###########
+for chassis in ovn-ovn-gw-0 ovn-ovn-gw-1; do
+ echo "checking ovn-chassis-1 -> $chassis"
+ wait_bfd_enabled ovn-chassis-1 $chassis
+done
+
+# Make sure there is no nft table left. Do not use nft directly as might not
be installed in container.
+gw1_pid=$(podman inspect -f '{{.State.Pid}}' ovn-gw-1)
+nsenter --net=/proc/$gw1_pid/ns/net nft list tables | grep ovn-test && nsenter
--net=/proc/$gw1_pid/ns/net nft delete table ip ovn-test
+on_exit "nsenter --net=/proc/$gw1_pid/ns/net nft list tables | grep ovn-test
&& nsenter --net=/proc/$gw1_pid/ns/net nft delete table ip ovn-test"
+
+wait_bfd_up ovn-gw-1 ovn-ovn-gw-0
+wait_bfd_up ovn-gw-1 $from_gw1_to_ch1
+wait_bfd_up ovn-gw-2 ovn-ovn-gw-0
+wait_bfd_up ovn-gw-2 $from_gw1_to_ch1
+wait_bfd_up ovn-chassis-1 ovn-ovn-gw-0
+wait_bfd_up ovn-chassis-1 ovn-ovn-gw-1
+
+m_wait_row_count Port_Binding 1 logical_port=cr-R1_outside chassis=$gw1_chassis
+check multinode_nbctl --wait=hv sync
+
+start_tcpdump() {
+ echo "$(date +%H:%M:%S.%03N) Starting tcpdump"
+ M_START_TCPDUMP([ovn-chassis-1], [-neei hv1-vif1-p], [ch1])
+ M_START_TCPDUMP([ovn-chassis-2], [-neei eth2], [ch2])
+ M_START_TCPDUMP([ovn-gw-1], [-neei eth2], [gw1])
+ M_START_TCPDUMP([ovn-gw-1], [-neei eth2 -Q out], [gw1_out])
+ M_START_TCPDUMP([ovn-gw-2], [-neei eth2], [gw2])
+ M_START_TCPDUMP([ovn-gw-2], [-neei eth2 -Q out], [gw2_out])
+}
+stop_tcpdump() {
+ echo "$(date +%H:%M:%S.%03N) Stopping tcpdump"
+ m_kill 'ovn-gw-1 ovn-gw-2 ovn-chassis-1 ovn-chassis-2' tcpdump
+}
+
+# Send packets from chassis2 (ext1) to chassis1
+send_background_packets() {
+ echo "$(date +%H:%M:%S.%03N) Sending packets in Background"
+ start_tcpdump
+ M_NS_DAEMONIZE([ovn-chassis-2], [ext1], [ping -f -i 0.1 192.168.1.1],
[ping.pid])
+}
+
+stop_sending_background_packets() {
+ echo "$(date +%H:%M:%S.%03N) Stopping Background process"
+ m_as ovn-chassis-1 ps -ef | grep -v grep | grep -q ping && \
+ m_as ovn-chassis-1 echo "Stopping ping on ovn-chassis-1" && killall
ping
+ m_as ovn-chassis-2 ps -ef | grep -v grep | grep -q ping && \
+ m_as ovn-chassis-2 echo "Stopping ping on ovn-chassis-2" && killall
ping
+ stop_tcpdump
+}
+
+check_migration_between_gw1_and_gw2() {
+ send_background_packets
+
+ # We do not check whether GARP have been generated on gw1: the test might
have been start already quite some time ago.
+ # We make sure gw1 is leader since enough time that it generated all its
garps.
+ AS_BOX([$(date +%H:%M:%S.%03N) Waiting all garps sent by gw1])
+ new_n_garps=$(cat gw1_out.tcpdump | grep -c "f0:00:c0:a8:00:fe >
Broadcast, ethertype ARP (0x0806), length 42: Request who-has 192.168.0.254
tell 192.168.0.254, length 28")
+ OVS_WAIT_UNTIL([
+ n_garps=$new_n_garps
+ echo "We saw $n_garps so far."
+ # Garp delay might be up to 8 seconds.
+ sleep 10
+ new_n_garps=$(cat gw1_out.tcpdump | grep -c "f0:00:c0:a8:00:fe >
Broadcast, ethertype ARP (0x0806), length 42: Request who-has 192.168.0.254
tell 192.168.0.254, length 28")
+ test "$n_garps" == "$new_n_garps"
+])
+
+ # It should go through gw1
+ AS_BOX([$(date +%H:%M:%S.%03N) Waiting/checking it went through gw1])
+ OVS_WAIT_FOR_OUTPUT([cat gw1.tcpdump | grep "ICMP echo" | cut -d ' '
-f2-15 | sort | uniq], [0], [dnl
+00:00:c0:a8:00:01 > f0:00:c0:a8:00:fe, ethertype IPv4 (0x0800), length 98:
192.168.0.1 > 192.168.1.1: ICMP echo request,
+f0:00:c0:a8:00:fe > 00:00:c0:a8:00:01, ethertype IPv4 (0x0800), length 98:
192.168.1.1 > 192.168.0.1: ICMP echo reply,
+])
+
+ flap_count=$(m_as ovn-gw-1 ovs-vsctl get interfac ovn-ovn-gw-0 bfd_status
| sed 's/.*flap_count=\"\([[0-9]]*\).*/\1/g')
+
+ AS_BOX([$(date +%H:%M:%S.%03N) Blocking bfd on gw1 (from $ip_gw1 to
$ip_gw2)])
+ nsenter --net=/proc/$gw1_pid/ns/net nft add table ip ovn-test
+ nsenter --net=/proc/$gw1_pid/ns/net nft 'add chain ip ovn-test INPUT {
type filter hook input priority 0; policy accept; }'
+ # Drop BFD from gw-1 to gw-2: geneve port (6081), inner port 3784 (0xec8),
Session state Up, Init, Down.
+ nsenter --net=/proc/$gw1_pid/ns/net nft add rule ip ovn-test INPUT ip
daddr $ip_gw1 ip saddr $ip_gw2 udp dport 6081 '@th,416,16 == 0x0ec8 @th,472,8
== 0xc0 counter drop'
+ nsenter --net=/proc/$gw1_pid/ns/net nft add rule ip ovn-test INPUT ip
daddr $ip_gw1 ip saddr $ip_gw2 udp dport 6081 '@th,416,16 == 0x0ec8 @th,472,8
== 0x80 counter drop'
+ nsenter --net=/proc/$gw1_pid/ns/net nft add rule ip ovn-test INPUT ip
daddr $ip_gw1 ip saddr $ip_gw2 udp dport 6081 '@th,416,16 == 0x0ec8 @th,472,8
== 0x40 counter drop'
+
+ # We do not check that packets go through gw2 as BFD between chassis-2 and
gw1 is still up
+ AS_BOX([$(date +%H:%M:%S.%03N) Waiting for flap count between gw1 and gw2
to increase])
+ OVS_WAIT_UNTIL([
+ new_flap_count=$(m_as ovn-gw-1 ovs-vsctl get interfac ovn-ovn-gw-0
bfd_status | sed 's/.*flap_count=\"\([[0-9]]*\).*/\1/g')
+ echo "Comparing $new_flap_count versus $flap_count"
+ test "$new_flap_count" -gt "$((flap_count))"
+ ])
+
+ AS_BOX([$(date +%H:%M:%S.%03N) Flapped!])
+ stop_sending_background_packets
+ cp gw1.tcpdump gw1.1.tcpdump
+ cp gw2.tcpdump gw2.1.tcpdump
+ cp ch1.tcpdump ch1.1.tcpdump
+ cp ch2.tcpdump ch2.1.tcpdump
+
+ send_background_packets
+ # Give some time for gw1 and gw2 to fight.
+ sleep 5
+ nsenter --net=/proc/$gw1_pid/ns/net nft -a list ruleset
+
+ AS_BOX([$(date +%H:%M:%S.%03N) Unblocking bfd on gw1])
+ nsenter --net=/proc/$gw1_pid/ns/net nft delete table ip ovn-test
+
+ AS_BOX([$(date +%H:%M:%S.%03N) Waiting/checking for garp from gw1])
+ OVS_WAIT_FOR_OUTPUT([cat gw1_out.tcpdump | grep "Broadcast" | cut -d ' '
-f2-16 | sort | uniq], [0], [dnl
+f0:00:c0:a8:00:fe > Broadcast, ethertype ARP (0x0806), length 42: Request
who-has 192.168.0.254 tell 192.168.0.254, length 28
+])
+
+ AS_BOX([$(date +%H:%M:%S.%03N) Waiting traffic went through gw1])
+ OVS_WAIT_FOR_OUTPUT([cat gw1.tcpdump| grep "ICMP echo" | cut -d ' '
-f2-15 | sort | uniq], [0], [dnl
+00:00:c0:a8:00:01 > f0:00:c0:a8:00:fe, ethertype IPv4 (0x0800), length 98:
192.168.0.1 > 192.168.1.1: ICMP echo request,
+f0:00:c0:a8:00:fe > 00:00:c0:a8:00:01, ethertype IPv4 (0x0800), length 98:
192.168.1.1 > 192.168.0.1: ICMP echo reply,
+])
+
+ AS_BOX([$(date +%H:%M:%S.%03N) Waiting it reaches ch1])
+ OVS_WAIT_FOR_OUTPUT([cat ch1.tcpdump | grep "ICMP echo" | cut -d ' '
-f2-15 | sort | uniq], [0], [dnl
+f0:00:c0:a8:01:01 > f0:00:c0:a8:01:fe, ethertype IPv4 (0x0800), length 98:
192.168.1.1 > 192.168.0.1: ICMP echo reply,
+f0:00:c0:a8:01:fe > f0:00:c0:a8:01:01, ethertype IPv4 (0x0800), length 98:
192.168.0.1 > 192.168.1.1: ICMP echo request,
+])
+
+ stop_sending_background_packets
+}
+
+start_tcpdump
+AS_BOX([$(date +%H:%M:%S.%03N) Sending packet from hv1-vif1(inside1) to ext1])
+M_NS_CHECK_EXEC([ovn-chassis-1], [hv1-vif1], [ping -c3 -q -i 0.1 192.168.0.1 |
FORMAT_PING],
+[0], [dnl
+3 packets transmitted, 3 received, 0% packet loss, time 0ms
+])
+stop_tcpdump
+
+# It should have gone through gw1 and not gw2
+AS_BOX([$(date +%H:%M:%S.%03N) Checking it went through gw1 and not gw2])
+AT_CHECK([cat gw2.tcpdump | grep "ICMP echo"], [1], [dnl
+])
+
+AT_CHECK([cat gw1.tcpdump | grep "ICMP echo" | cut -d ' ' -f2-15], [0], [dnl
+f0:00:c0:a8:00:fe > 00:00:c0:a8:00:01, ethertype IPv4 (0x0800), length 98:
192.168.1.1 > 192.168.0.1: ICMP echo request,
+00:00:c0:a8:00:01 > f0:00:c0:a8:00:fe, ethertype IPv4 (0x0800), length 98:
192.168.0.1 > 192.168.1.1: ICMP echo reply,
+f0:00:c0:a8:00:fe > 00:00:c0:a8:00:01, ethertype IPv4 (0x0800), length 98:
192.168.1.1 > 192.168.0.1: ICMP echo request,
+00:00:c0:a8:00:01 > f0:00:c0:a8:00:fe, ethertype IPv4 (0x0800), length 98:
192.168.0.1 > 192.168.1.1: ICMP echo reply,
+f0:00:c0:a8:00:fe > 00:00:c0:a8:00:01, ethertype IPv4 (0x0800), length 98:
192.168.1.1 > 192.168.0.1: ICMP echo request,
+00:00:c0:a8:00:01 > f0:00:c0:a8:00:fe, ethertype IPv4 (0x0800), length 98:
192.168.0.1 > 192.168.1.1: ICMP echo reply,
+])
+
+check_migration_between_gw1_and_gw2
+
+n1=$(m_as ovn-gw-1 grep -c Changing /var/log/ovn//ovn-controller.log)
+n2=$(m_as ovn-gw-2 grep -c Changing /var/log/ovn//ovn-controller.log)
+ch1_req=$(grep -c "ICMP echo request" ch1.tcpdump)
+ch1_rep=$(grep -c "ICMP echo reply" ch1.tcpdump)
+ch2_req=$(grep -c "ICMP echo request" ch2.tcpdump)
+ch2_rep=$(grep -c "ICMP echo reply" ch2.tcpdump)
+gw1_req=$(grep -c "ICMP echo request" gw1.tcpdump)
+gw1_rep=$(grep -c "ICMP echo reply" gw1.tcpdump)
+gw2_req=$(grep -c "ICMP echo request" gw2.tcpdump)
+gw2_rep=$(grep -c "ICMP echo reply" gw2.tcpdump)
+echo "$n1 claims in gw1 and $n2 in gw2"
+echo "ch2_request=$ch2_req gw1_request=$gw1_req gw2_request=$gw2_req
ch1_request=$ch1_req ch1_reply=$ch1_rep gw1_reply=$gw1_rep gw2_reply=$gw2_rep
ch2_reply=$ch2_rep"
+
+AT_CLEANUP
+])
--
2.47.1
_______________________________________________
dev mailing list
[email protected]
https://mail.openvswitch.org/mailman/listinfo/ovs-dev
