If all the health-checked backends of a VIP are down, traffic to the VIP is dropped (or rejected).
Add the "fail_open" option to the NB Load_Balancer_Health_Check table. If it is set to true and all the backends of the VIP are down, traffic is load balanced across all the backends that have health checks configured. As soon as one backend is back online, only the online backends are used. Since newly created Service_Monitor rows start in the "offline" state, a VIP whose backends are all new is load balanced across all of them until the first health check succeeds. Signed-off-by: Alexandra Rukomoinikova <[email protected]> --- v1 --> v2: Rename option from fail-open to fail_open in the same pattern as all other option for LB Changed the northd check as discussed with Rosemarie. Removed unused code in northd. --- NEWS | 5 ++ northd/lb.c | 4 ++ northd/lb.h | 1 + northd/northd.c | 86 +++++++++++++++++------------- ovn-nb.xml | 27 +++++++++- tests/ovn-northd.at | 124 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 211 insertions(+), 36 deletions(-) diff --git a/NEWS b/NEWS index 4176f8d7f..808b92796 100644 --- a/NEWS +++ b/NEWS @@ -44,6 +44,11 @@ Post v26.09.0 table at startup; it defers its kernel route updates until the initial dump of the advertised routes of the local route-exchange datapaths is complete. + - Added a new "options:fail_open" key to the Load_Balancer_Health_Check + table. If set to true and all the backends of the VIP are detected as + down, traffic is load balanced across all of them instead of being + dropped or rejected. As soon as at least one backend is online again, + only the online backends are used. OVN v26.09.0 - xxx xx xxxx -------------------------- diff --git a/northd/lb.c b/northd/lb.c index 5ff9d1fad..0f6d93091 100644 --- a/northd/lb.c +++ b/northd/lb.c @@ -104,6 +104,10 @@ ovn_northd_lb_vip_init(struct ovn_northd_lb_vip *lb_vip_nb, sizeof *lb_vip_nb->backends_nb); lb_vip_nb->lb_health_check = ovn_lb_get_health_check(nbrec_lb, vip_port_str, template); + lb_vip_nb->fail_open = + lb_vip_nb->lb_health_check + && smap_get_bool(&lb_vip_nb->lb_health_check->options, + "fail_open", false); } /* diff --git a/northd/lb.h b/northd/lb.h index 7a98c2f55..6ed3c1ed8 100644 --- a/northd/lb.h +++ b/northd/lb.h @@ -91,6 +91,7 @@ struct ovn_northd_lb_vip { size_t n_backends; struct nbrec_load_balancer_health_check *lb_health_check; + bool fail_open; }; struct ovn_northd_lb_backend { diff --git a/northd/northd.c b/northd/northd.c index 19a62ace8..1db73ccdd 100644 --- a/northd/northd.c +++ b/northd/northd.c @@ -3568,6 +3568,45 @@ backend_is_available(const struct ovn_northd_lb *lb, false : true; } +static size_t +build_lb_vip_backends(const struct ovn_northd_lb *lb, + const struct ovn_lb_vip *lb_vip, + const struct ovn_northd_lb_vip *lb_vip_nb, + struct ds *action, + const struct svc_monitors_map_data *svc_mons_data, + bool ignore_health_status) +{ + const struct ovn_lb_backend *backend; + size_t n_backends = 0; + size_t i = 0; + + VECTOR_FOR_EACH_PTR (&lb_vip->backends, backend) { + struct ovn_northd_lb_backend *backend_nb = + &lb_vip_nb->backends_nb[i++]; + bool ipv6_backend = !IN6_IS_ADDR_V4MAPPED(&backend->ip); + + if (!backend_nb->logical_port) { + continue; + } + + if (!ignore_health_status && backend_nb->health_check + && !backend_is_available(lb, backend, backend_nb, + svc_mons_data)) { + continue; + } + + if (backend_nb->distributed_backend) { + ds_put_format(action, "\"%s\":", backend_nb->logical_port); + } + ds_put_format(action, + ipv6_backend ? "[%s]:%"PRIu16"," : "%s:%"PRIu16",", + backend->ip_str, backend->port); + n_backends++; + } + + return n_backends; +} + static bool build_lb_vip_actions(const struct ovn_northd_lb *lb, const struct ovn_lb_vip *lb_vip, @@ -3598,41 +3637,18 @@ build_lb_vip_actions(const struct ovn_northd_lb *lb, : "ct_lb_mark(backends="); if (lb_vip_nb->lb_health_check || lb->is_distributed) { - size_t i = 0; - size_t n_active_backends = 0; - const struct ovn_lb_backend *backend; - VECTOR_FOR_EACH_PTR (&lb_vip->backends, backend) { - struct ovn_northd_lb_backend *backend_nb = - &lb_vip_nb->backends_nb[i++]; - bool ipv6_backend = !IN6_IS_ADDR_V4MAPPED(&backend->ip); - - /* XXX: Remove these checks: by changing the iteration - * only for selected backends. */ - if (lb_vip_nb->lb_health_check && - !backend_nb->health_check) { - continue; - } - - if (lb->is_distributed && - !backend_nb->distributed_backend) { - continue; - } - - if (backend_nb->health_check && - !backend_is_available(lb, - backend, - backend_nb, - svc_mons_data)) { - continue; - } - - if (backend_nb->distributed_backend) { - ds_put_format(action, "\"%s\":", backend_nb->logical_port); - } - ds_put_format(action, - ipv6_backend ? "[%s]:%"PRIu16"," : "%s:%"PRIu16",", - backend->ip_str, backend->port); - n_active_backends++; + size_t n_active_backends = + build_lb_vip_backends(lb, lb_vip, lb_vip_nb, action, + svc_mons_data, false); + + /* If all the health-checked backends are offline and "fail_open" is + * enabled, use all of them instead of dropping (or rejecting) the + * traffic. As soon as at least one backend is reported online + * again, only the online ones are used. */ + if (!n_active_backends && lb_vip_nb->fail_open) { + n_active_backends = + build_lb_vip_backends(lb, lb_vip, lb_vip_nb, action, + svc_mons_data, true); } ds_chomp(action, ','); diff --git a/ovn-nb.xml b/ovn-nb.xml index d8c63567a..3edab5055 100644 --- a/ovn-nb.xml +++ b/ovn-nb.xml @@ -2587,7 +2587,9 @@ <group title="Health Checks"> <p> OVN supports health checks for load balancer endpoints. When health - checks are enabled, the load balancer uses only healthy endpoints. + checks are enabled, the load balancer uses only healthy endpoints, + unless <ref table="Load_Balancer_Health_Check" column="options" + key="fail_open"/> is enabled and no endpoint is healthy. </p> <p> @@ -2912,6 +2914,29 @@ or The number of failure checks after which the endpoint is considered offline. </column> + + <column name="options" key="fail_open" type='{"type": "boolean"}'> + <p> + If set to <code>true</code> and none of the VIP's endpoints is + healthy, traffic is load balanced across all the endpoints that + have health checks configured, instead of being dropped or + rejected. In that case this option takes precedence over the + <ref table="Load_Balancer" column="options" key="reject"/> option. + As soon as at least one endpoint is healthy again, only the + healthy endpoints are used. + </p> + + <p> + A newly added endpoint is considered offline until its first + health check succeeds. Therefore, when all the endpoints of a VIP + are new, traffic is load balanced across all of them until the + first health check succeeds. + </p> + + <p> + The default is <code>false</code>. + </p> + </column> </group> <group title="Common Columns"> diff --git a/tests/ovn-northd.at b/tests/ovn-northd.at index 6c37337ad..f65fd16bf 100644 --- a/tests/ovn-northd.at +++ b/tests/ovn-northd.at @@ -25340,3 +25340,127 @@ OVN_CLEANUP_NORTHD AT_CLEANUP ]) +OVN_FOR_EACH_NORTHD_NO_HV([ +AT_SETUP([Load balancer health check fail_open]) +ovn_start +# 30.0.0.3 has no ip_port_mappings entry, so it is never used, not even +# when fail_open is in effect. +check ovn-nbctl lb-add lb1 10.0.0.10:80 10.0.0.3:80,20.0.0.3:80,30.0.0.3:80 + +check ovn-nbctl ls-add sw0 +check ovn-nbctl lsp-add sw0 sw0-p1 -- lsp-set-addresses sw0-p1 \ +"00:00:00:00:00:03 10.0.0.3" +check ovn-nbctl ls-add sw1 +check ovn-nbctl --wait=sb lsp-add sw1 sw1-p1 -- lsp-set-addresses sw1-p1 \ +"02:00:00:00:00:03 20.0.0.3" + +check ovn-sbctl chassis-add hv1 geneve 127.0.0.1 +check ovn-sbctl lsp-bind sw0-p1 hv1 +check ovn-sbctl lsp-bind sw1-p1 hv1 +wait_row_count nb:Logical_Switch_Port 1 name=sw0-p1 'up=true' +wait_row_count nb:Logical_Switch_Port 1 name=sw1-p1 'up=true' + +check ovn-nbctl set load_balancer . ip_port_mappings:10.0.0.3=sw0-p1:10.0.0.2 +check ovn-nbctl --wait=sb set load_balancer . \ +ip_port_mappings:20.0.0.3=sw1-p1:20.0.0.2 + +check_uuid ovn-nbctl --wait=sb -- --id=@hc create \ +Load_Balancer_Health_Check vip="10.0.0.10\:80" -- add Load_Balancer . \ +health_check @hc +wait_row_count Service_Monitor 2 + +check ovn-nbctl lr-add lr0 +check ovn-nbctl lr-lb-add lr0 lb1 +check ovn-nbctl --wait=sb ls-lb-add sw0 lb1 + +sm_sw0_p1=$(fetch_column Service_Monitor _uuid logical_port=sw0-p1) +sm_sw1_p1=$(fetch_column Service_Monitor _uuid logical_port=sw1-p1) + +AS_BOX([Both backends online]) +check ovn-sbctl set service_monitor $sm_sw0_p1 status=online +check ovn-sbctl set service_monitor $sm_sw1_p1 status=online + +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | grep 'priority=120.*backends' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80,20.0.0.3:80);) +]) + +AT_CHECK([ovn-sbctl dump-flows lr0 | grep 'lr_in_dnat.*priority=120' | ovn_strip_lflows], [0], [dnl + table=??(lr_in_dnat ), priority=120 , match=(ct.new && !ct.rel && ip4 && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(ct_lb_mark(backends=10.0.0.3:80,20.0.0.3:80);) +]) + +AS_BOX([All backends offline without fail_open - traffic is dropped]) +check ovn-sbctl set service_monitor $sm_sw0_p1 status=offline +check ovn-sbctl set service_monitor $sm_sw1_p1 status=offline + +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | grep 'ls_in_lb.*priority=120' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(drop;) +]) + +AT_CHECK([ovn-sbctl dump-flows lr0 | grep 'lr_in_dnat.*priority=120' | ovn_strip_lflows], [0], [dnl + table=??(lr_in_dnat ), priority=120 , match=(ct.new && !ct.rel && ip4 && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(drop;) +]) + +AS_BOX([All backends offline with fail_open - all mapped backends are used]) +check ovn-nbctl --wait=sb set load_balancer_health_check . \ +options:fail_open=true + +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | grep 'priority=120.*backends' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80,20.0.0.3:80);) +]) + +AT_CHECK([ovn-sbctl dump-flows lr0 | grep 'lr_in_dnat.*priority=120' | ovn_strip_lflows], [0], [dnl + table=??(lr_in_dnat ), priority=120 , match=(ct.new && !ct.rel && ip4 && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(ct_lb_mark(backends=10.0.0.3:80,20.0.0.3:80);) +]) + +AS_BOX([fail_open takes precedence over the reject option]) +check ovn-nbctl --wait=sb set load_balancer . options:reject=true + +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | grep 'ls_in_lb.*priority=120' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80,20.0.0.3:80);) +]) + +AT_CHECK([ovn-sbctl dump-flows lr0 | grep 'lr_in_dnat.*priority=120' | ovn_strip_lflows], [0], [dnl + table=??(lr_in_dnat ), priority=120 , match=(ct.new && !ct.rel && ip4 && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(ct_lb_mark(backends=10.0.0.3:80,20.0.0.3:80);) +]) +check ovn-nbctl --wait=sb remove load_balancer . options reject + +AS_BOX([One backend back online - only the online one is used]) +check ovn-sbctl set service_monitor $sm_sw1_p1 status=online + +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | grep 'priority=120.*backends' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=20.0.0.3:80);) +]) + +AT_CHECK([ovn-sbctl dump-flows lr0 | grep 'lr_in_dnat.*priority=120' | ovn_strip_lflows], [0], [dnl + table=??(lr_in_dnat ), priority=120 , match=(ct.new && !ct.rel && ip4 && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(ct_lb_mark(backends=20.0.0.3:80);) +]) + +AS_BOX([Last backend goes down again - all mapped backends are used]) +check ovn-sbctl set service_monitor $sm_sw1_p1 status=offline + +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | grep 'priority=120.*backends' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80,20.0.0.3:80);) +]) + +AS_BOX([Disabling fail_open restores the drop behavior]) +check ovn-nbctl --wait=sb set load_balancer_health_check . \ +options:fail_open=false + +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | grep 'ls_in_lb.*priority=120' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(drop;) +]) + +AT_CHECK([ovn-sbctl dump-flows lr0 | grep 'lr_in_dnat.*priority=120' | ovn_strip_lflows], [0], [dnl + table=??(lr_in_dnat ), priority=120 , match=(ct.new && !ct.rel && ip4 && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(drop;) +]) + +OVN_CLEANUP_NORTHD +AT_CLEANUP +]) -- 2.48.1 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
