A backend is only probed by the ovn-controller of the chassis its port
is bound to (pinctrl's sync_svc_monitors() skips ports bound elsewhere).
When Port_Binding.chassis is cleared while "up" stays true, nobody probes
the backend anymore and its last "online" status is trusted forever, so
traffic keeps being sent to a backend that may be gone. This happens
when the Chassis row is deleted (weak reference), after "ovn-appctl exit"
without --restart, and when ovn-controller restarts on a hypervisor whose
VMs did not come back and releases ports it was not tracking.
Commit 23e203a3f ("northd: set svc_mon status to offline if port_binding
released") only checks "up". Also treat an unbound port as offline, for
both load balancer and LSP health checks, and clear chassis_name so it
never names a chassis that stopped probing. ovn-controller sets the
status back to "online" once the port is bound again and a probe
succeeds; northd never writes "online" itself.
The ovn-ic service monitor test emulated a probed backend by setting
Port_Binding "up" without a chassis; bind the ports to a chassis instead,
which is what happens in practice.
CC: Vladislav Odintsov <[email protected]>
Fixes: 23e203a3f30b ("northd: set svc_mon status to offline if port_binding
released")
Assisted-by: Claude Opus 5, Claude Code
Signed-off-by: Jaygue Lee <[email protected]>
---
northd/northd.c | 25 +++++++++------
tests/ovn-ic.at | 18 ++++++-----
tests/ovn-northd.at | 75 +++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 100 insertions(+), 18 deletions(-)
diff --git a/northd/northd.c b/northd/northd.c
index 1c9e5d0703..3f5f7bda48 100644
--- a/northd/northd.c
+++ b/northd/northd.c
@@ -3522,19 +3522,22 @@ ovn_lb_svc_create(struct ovsdb_idl_txn *ovnsb_txn,
backend_nb->svc_mon_src_ip);
}
+ /* Unbound ports are not probed by any chassis: fail closed. */
if (!backend_nb->remote_backend &&
- (!op->sb->n_up || !op->sb->up[0])
+ (!op->sb->chassis || !op->sb->n_up || !op->sb->up[0])
&& mon_info->sbrec_mon->status
&& !strcmp(mon_info->sbrec_mon->status, "online")) {
sbrec_service_monitor_set_status(mon_info->sbrec_mon,
"offline");
}
- if (!backend_nb->remote_backend && op->sb->chassis &&
- strcmp(mon_info->sbrec_mon->chassis_name,
- op->sb->chassis->name)) {
- sbrec_service_monitor_set_chassis_name(mon_info->sbrec_mon,
- op->sb->chassis->name);
+ if (!backend_nb->remote_backend) {
+ const char *chassis_name = op->sb->chassis
+ ? op->sb->chassis->name : "";
+ if (strcmp(mon_info->sbrec_mon->chassis_name, chassis_name)) {
+ sbrec_service_monitor_set_chassis_name(mon_info->sbrec_mon,
+ chassis_name);
+ }
}
}
}
@@ -3853,16 +3856,18 @@ ovn_lsp_svc_monitors_process_port(
lsp_hc->src_ip);
}
- if ((!op->sb->n_up || !op->sb->up[0]) &&
+ /* Unbound ports are not probed by any chassis: fail closed. */
+ if ((!op->sb->chassis || !op->sb->n_up || !op->sb->up[0]) &&
mon_info->sbrec_mon->status &&
!strcmp(mon_info->sbrec_mon->status, "online")) {
sbrec_service_monitor_set_status(mon_info->sbrec_mon, "offline");
}
- if (op->sb->chassis && strcmp(mon_info->sbrec_mon->chassis_name,
- op->sb->chassis->name)) {
+ const char *chassis_name = op->sb->chassis ? op->sb->chassis->name
+ : "";
+ if (strcmp(mon_info->sbrec_mon->chassis_name, chassis_name)) {
sbrec_service_monitor_set_chassis_name(mon_info->sbrec_mon,
- op->sb->chassis->name);
+ chassis_name);
}
}
}
diff --git a/tests/ovn-ic.at b/tests/ovn-ic.at
index c9e07250bb..812ff22803 100644
--- a/tests/ovn-ic.at
+++ b/tests/ovn-ic.at
@@ -4962,16 +4962,20 @@ AT_CHECK([ovn-sbctl lflow-list az3_ls1 | grep
'ls_in_arp_rsp' | ovn_strip_lflows
# Service monitors are created in the "offline" state by default, so local
# backends are not used until their first health check succeeds. This is a
-# mock test, so bring the local backends' ports up and mark their monitors
-# online to emulate a successful health check before validating load balancing.
+# mock test, so bind the local backends' ports to a chassis (which also brings
+# them up) and mark their monitors online to emulate a successful health check
+# before validating load balancing. northd keeps monitors of unbound ports
+# offline because no ovn-controller could have probed them.
ovn_as az1
-check ovn-sbctl set Port_Binding lport1_az1 up=true
+check ovn-sbctl chassis-add hv1 geneve 127.0.0.1
+check ovn-sbctl lsp-bind lport1_az1 hv1
sm_lport1=$(fetch_column Service_Monitor _uuid logical_port="lport1_az1")
check ovn-sbctl set Service_Monitor $sm_lport1 status=online
check ovn-nbctl --wait=sb sync
ovn_as az2
-check ovn-sbctl set Port_Binding lport3_az2 up=true
+check ovn-sbctl chassis-add hv2 geneve 127.0.0.2
+check ovn-sbctl lsp-bind lport3_az2 hv2
sm_lport3=$(fetch_column Service_Monitor _uuid logical_port="lport3_az2")
check ovn-sbctl set Service_Monitor $sm_lport3 status=online
check ovn-nbctl --wait=sb sync
@@ -5105,16 +5109,14 @@ AT_CHECK([ovn-sbctl lflow-list az2_ls1 | grep ls_in_lb
| grep backends | ovn_str
ovn_as az1
# set other statuses to online state.
svc_lport1_uuid=$(ovn-sbctl -d bare --no-headings --columns _uuid find
Service_Monitor logical_port=lport1_az1)
-lport1_uuid=$(ovn-sbctl -d bare --no-headings --columns _uuid find
Port_Binding logical_port=lport1_az1)
-check ovn-sbctl set Port_Binding $lport1_uuid up=true
+check ovn-sbctl --may-exist lsp-bind lport1_az1 hv1
check ovn-sbctl set Service_Monitor $svc_lport1_uuid status="online"
# Sync IC DB (AZ1->ISB)
check ovn-ic-nbctl --wait=sb sync
check_column online ic-sb:Service_Monitor status logical_port=lport1_az1
svc_lport2_uuid=$(ovn-sbctl -d bare --no-headings --columns _uuid find
Service_Monitor logical_port=lport2_az1)
-lport2_uuid=$(ovn-sbctl -d bare --no-headings --columns _uuid find
Port_Binding logical_port=lport2_az1)
-check ovn-sbctl set Port_Binding $lport2_uuid up=true
+check ovn-sbctl --may-exist lsp-bind lport2_az1 hv1
check ovn-sbctl set Service_Monitor $svc_lport2_uuid status="online"
# Sync IC DB (AZ1->ISB)
check ovn-ic-nbctl --wait=sb sync
diff --git a/tests/ovn-northd.at b/tests/ovn-northd.at
index af231fb871..7da84e0f93 100644
--- a/tests/ovn-northd.at
+++ b/tests/ovn-northd.at
@@ -1806,6 +1806,81 @@ OVN_CLEANUP_NORTHD
AT_CLEANUP
])
+OVN_FOR_EACH_NORTHD_NO_HV([
+AT_SETUP([Load balancer health check unbound backends go offline])
+ovn_start
+
+check ovn-nbctl lb-add lb1 10.0.0.10:80 10.0.0.3:80,10.0.0.4:80
+check ovn-nbctl --wait=sb set load_balancer .
ip_port_mappings:10.0.0.3=sw0-p1:10.0.0.2
+check ovn-nbctl --wait=sb set load_balancer .
ip_port_mappings:10.0.0.4=sw0-p2:10.0.0.2
+
+check ovn-nbctl ls-add sw0
+check ovn-nbctl --wait=sb lsp-add sw0 sw0-p1 -- lsp-set-addresses sw0-p1 \
+"00:00:00:00:00:03 10.0.0.3"
+check ovn-nbctl --wait=sb lsp-add sw0 sw0-p2 -- lsp-set-addresses sw0-p2 \
+"00:00:00:00:00:04 10.0.0.4"
+
+check ovn-sbctl chassis-add hv1 geneve 127.0.0.1
+check ovn-sbctl lsp-bind sw0-p1 hv1
+check ovn-sbctl lsp-bind sw0-p2 hv1
+wait_row_count nb:Logical_Switch_Port 1 name=sw0-p1 'up=true'
+wait_row_count nb:Logical_Switch_Port 1 name=sw0-p2 'up=true'
+
+check ovn-nbctl --wait=sb ls-lb-add sw0 lb1
+check_uuid ovn-nbctl --wait=sb -- --id=@hc create \
+Load_Balancer_Health_Check vip="10.0.0.10\:80" -- add Load_Balancer . \
+health_check @hc
+wait_row_count Service_Monitor 2
+wait_row_count Service_Monitor 2 chassis_name=hv1
+
+# Simulate successful probes from hv1.
+check ovn-sbctl set service_monitor sw0-p1 status=online
+check ovn-sbctl set service_monitor sw0-p2 status=online
+AT_CAPTURE_FILE([sbflows_online])
+OVS_WAIT_FOR_OUTPUT(
+ [ovn-sbctl dump-flows sw0 | tee sbflows_online | grep
'priority=120.*backends' | ovn_strip_lflows], 0, [dnl
+ table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst ==
10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 =
10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80,10.0.0.4:80);)
+])
+
+AS_BOX([Port released without clearing "up": nobody probes it anymore.])
+# This is what happens when the chassis row is deleted (weak reference) or
+# when ovn-controller releases a port it was not tracking, e.g. after the
+# hypervisor rebooted with the VM not restarted.
+check ovn-sbctl clear Port_Binding sw0-p1 chassis
+wait_row_count Port_Binding 1 logical_port=sw0-p1 'up=true'
+wait_row_count Service_Monitor 1 logical_port=sw0-p1 status=offline
+wait_row_count Service_Monitor 1 logical_port=sw0-p1 'chassis_name=""'
+wait_row_count Service_Monitor 1 logical_port=sw0-p2 status=online
+AT_CAPTURE_FILE([sbflows_unbound])
+OVS_WAIT_FOR_OUTPUT(
+ [ovn-sbctl dump-flows sw0 | tee sbflows_unbound | grep
'priority=120.*backends' | ovn_strip_lflows], 0, [dnl
+ table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst ==
10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 =
10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.4:80);)
+])
+
+AS_BOX([Port bound again: stays offline until a probe succeeds.])
+check ovn-sbctl lsp-bind sw0-p1 hv1
+wait_row_count nb:Logical_Switch_Port 1 name=sw0-p1 'up=true'
+wait_row_count Service_Monitor 1 logical_port=sw0-p1 chassis_name=hv1
+check ovn-nbctl --wait=sb sync
+check_row_count Service_Monitor 1 logical_port=sw0-p1 status=offline
+AT_CAPTURE_FILE([sbflows_rebound])
+OVS_WAIT_FOR_OUTPUT(
+ [ovn-sbctl dump-flows sw0 | tee sbflows_rebound | grep
'priority=120.*backends' | ovn_strip_lflows], 0, [dnl
+ table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst ==
10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 =
10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.4:80);)
+])
+
+# Only a successful probe (reported by ovn-controller) brings it back.
+check ovn-sbctl set service_monitor sw0-p1 status=online
+AT_CAPTURE_FILE([sbflows_recovered])
+OVS_WAIT_FOR_OUTPUT(
+ [ovn-sbctl dump-flows sw0 | tee sbflows_recovered | grep
'priority=120.*backends' | ovn_strip_lflows], 0, [dnl
+ table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst ==
10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 =
10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80,10.0.0.4:80);)
+])
+
+OVN_CLEANUP_NORTHD
+AT_CLEANUP
+])
+
OVN_FOR_EACH_NORTHD_NO_HV([
AT_SETUP([Load balancer VIP in NAT entries])
AT_SKIP_IF([test $HAVE_PYTHON = no])
--
2.49.0
_______________________________________________
dev mailing list
[email protected]
https://mail.openvswitch.org/mailman/listinfo/ovs-dev
