Load balancer health checks always probe the backend's port using the load balancer's protocol. Some services expose their health endpoint on a different port, or can only be probed with a different protocol, e.g. a TCP service with a UDP health endpoint. Extend the Load_Balancer ip_port_mappings value format to: "backed_ip=port_name:source_ip[:protocol:port][:az_name]".
Signed-off-by: Alexandra Rukomoinikova <[email protected]> --- NEWS | 4 + northd/en-advertised-route-sync.c | 6 +- northd/lb.c | 118 ++++++++++++++++++++++++------ northd/lb.h | 6 ++ northd/northd.c | 40 +++------- ovn-nb.xml | 24 ++++++ tests/ovn-northd.at | 110 ++++++++++++++++++++++++++++ 7 files changed, 255 insertions(+), 53 deletions(-) diff --git a/NEWS b/NEWS index b2416f98b..f9edcf59f 100644 --- a/NEWS +++ b/NEWS @@ -77,6 +77,10 @@ Post v26.09.0 the replies from the backends of its load balancers. The CMS must make sure that the replies re-enter the router on the chassis that load balanced the request. See ovn-nb(5). + - Load balancer health checks can now use a port and a protocol + different from the backend's port and the load balancer's protocol. + They are set in the ip_port_mappings column in Load_Balancer table. + See the ovn-nb(5) man page for more information. OVN v26.09.0 - xxx xx xxxx -------------------------- diff --git a/northd/en-advertised-route-sync.c b/northd/en-advertised-route-sync.c index eb8a24a39..1c646b855 100644 --- a/northd/en-advertised-route-sync.c +++ b/northd/en-advertised-route-sync.c @@ -548,8 +548,6 @@ build_lb_routes_for_lb(const struct ovn_port *advertising_op, } const struct ovn_northd_lb *lb = lb_dps->lb; - const char *protocol = lb->nlb->protocol && lb->nlb->protocol[0] - ? lb->nlb->protocol : "tcp"; for (size_t v = 0; v < lb->n_vips; v++) { const struct ovn_lb_vip *vip = &lb->vips[v]; const struct ovn_northd_lb_vip *vip_nb = &lb->vips_nb[v]; @@ -603,8 +601,8 @@ build_lb_routes_for_lb(const struct ovn_port *advertising_op, if (backend_nb->health_check) { sset_add_and_free( &route_e->health_checks, - xasprintf("%s,%s,%"PRIu16, protocol, backend->ip_str, - backend->port)); + xasprintf("%s,%s,%"PRIu16, backend_nb->svc_mon_protocol, + backend->ip_str, backend_nb->svc_mon_port)); } else { route_e->has_ungated_lb = true; } diff --git a/northd/lb.c b/northd/lb.c index 5ff9d1fad..d2ca376f8 100644 --- a/northd/lb.c +++ b/northd/lb.c @@ -106,11 +106,82 @@ ovn_northd_lb_vip_init(struct ovn_northd_lb_vip *lb_vip_nb, ovn_lb_get_health_check(nbrec_lb, vip_port_str, template); } +static bool +parse_ip_port_mapping_optional_params(char *fields, + const char *mapping, + const char **hc_protocol, + uint16_t *hc_port, + bool *is_remote, char **az_name) +{ + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1); + char *field[3]; + size_t n_fields = 0; + + for (char *f = fields; f; n_fields++) { + if (n_fields == ARRAY_SIZE(field)) { + VLOG_WARN_RL(&rl, "Too many fields in ip_port_mappings %s", + mapping); + return false; + } + field[n_fields] = f; + f = strchr(f, ':'); + if (f) { + *f++ = '\0'; + } + } + + const char *protocol = NULL; + const char *port = NULL; + char *az = NULL; + if (n_fields == 1) { + az = field[0]; + } else { + protocol = field[0]; + port = field[1]; + az = n_fields == 3 ? field[2] : NULL; + } + + if (az && !*az) { + VLOG_WARN_RL(&rl, "Empty AZ name specified in ip_port_mappings %s", + mapping); + return false; + } + + if (protocol && *protocol) { + if (!strcmp(protocol, "tcp")) { + *hc_protocol = "tcp"; + } else if (!strcmp(protocol, "udp")) { + *hc_protocol = "udp"; + } else { + VLOG_WARN_RL(&rl, "Invalid health check protocol \"%s\" in " + "ip_port_mappings %s, expected tcp or udp", + protocol, mapping); + return false; + } + } + + if (port && *port) { + unsigned int value; + if (!str_to_uint(port, 10, &value) || !value || value > 65535) { + VLOG_WARN_RL(&rl, "Invalid health check port \"%s\" in " + "ip_port_mappings %s", port, mapping); + return false; + } + *hc_port = value; + } + + *az_name = az; + *is_remote = az != NULL; + + return true; +} + /* * Parses ip_port_mappings in the format : - * "ip:logical_port[:src_ip][:az_name]". + * "ip:logical_port[:src_ip][:protocol:port][:az_name]". * src_ip parameter is optional when distributed mode is enabled, - * without health checks configured. + * without health checks configured. Protocol and port override + * the protocol and the port used by health checks. * If az_name is present and non-empty, it indicates this is a * remote service monitor (backend is in another availability zone), * it should be propogated to another AZ by interconnection processing. @@ -144,6 +215,8 @@ ovn_lb_vip_backends_ip_port_mappings_init(const struct ovn_northd_lb *lb, char *port_name = NULL; char *first_colon = NULL; bool is_remote = false; + uint16_t hc_port = 0; + const char *hc_protocol = NULL; port_name = xstrdup(s); first_colon = strchr(port_name, ':'); @@ -162,8 +235,9 @@ ovn_lb_vip_backends_ip_port_mappings_init(const struct ovn_northd_lb *lb, } *first_colon = '\0'; + char *fields = NULL; if (first_colon[1] == '[') { - /* IPv6 case - format: port:[ipv6]:az or port:[ipv6] */ + /* IPv6 case - format: port:[ipv6][:fields] */ char *ip_end = strchr(first_colon + 2, ']'); if (!ip_end) { VLOG_WARN_RL(&rl, "Malformed IPv6 address in backend %s", s); @@ -174,27 +248,22 @@ ovn_lb_vip_backends_ip_port_mappings_init(const struct ovn_northd_lb *lb, *ip_end = '\0'; if (ip_end[1] == ':') { - az_name = ip_end + 2; - if (!*az_name) { - VLOG_WARN_RL(&rl, "Empty AZ name specified for backend %s", - port_name); - goto cleanup; - } - is_remote = true; + fields = ip_end + 2; } } else { - /* IPv4 case - format: port:ip:az or port:ip */ + /* IPv4 case - format: port:ip[:fields] */ src_ip = first_colon + 1; - char *az_colon = strchr(src_ip, ':'); - if (az_colon) { - *az_colon = '\0'; - az_name = az_colon + 1; - if (!*az_name) { - VLOG_WARN_RL(&rl, "Empty AZ name specified for backend %s", - port_name); - goto cleanup; - } - is_remote = true; + fields = strchr(src_ip, ':'); + if (fields) { + *fields++ = '\0'; + } + } + + if (fields) { + if (!parse_ip_port_mapping_optional_params(fields, s, + &hc_protocol, &hc_port, + &is_remote, &az_name)) { + goto cleanup; } } @@ -218,6 +287,12 @@ init_backend: backend_nb->remote_backend = is_remote; backend_nb->svc_mon_lrp = NULL; backend_nb->distributed_backend = lb->is_distributed; + backend_nb->svc_mon_port = hc_port ? hc_port : backend->port; + if (!hc_protocol) { + hc_protocol = lb->nlb->protocol && lb->nlb->protocol[0] + ? lb->nlb->protocol : "tcp"; + } + backend_nb->svc_mon_protocol = xstrdup(hc_protocol); cleanup: free(port_name); } @@ -233,6 +308,7 @@ void ovn_northd_lb_vip_destroy(struct ovn_northd_lb_vip *vip) free(vip->backends_nb[i].logical_port); free(vip->backends_nb[i].svc_mon_src_ip); free(vip->backends_nb[i].az_name); + free(vip->backends_nb[i].svc_mon_protocol); } free(vip->backends_nb); } diff --git a/northd/lb.h b/northd/lb.h index 7a98c2f55..72ec8b072 100644 --- a/northd/lb.h +++ b/northd/lb.h @@ -104,6 +104,12 @@ struct ovn_northd_lb_backend { char *svc_mon_src_ip; /* Target Availability Zone name for service monitoring. */ char *az_name; + /* Port and protocol used by the health check. They default to the + * backend's port and the load balancer's protocol, but can be + * overridden by the "hc_port" and "hc_protocol" options of the + * backend's ip_port_mappings entry. */ + uint16_t svc_mon_port; + char *svc_mon_protocol; /* LRP used as the source of health_check traffic. This may be NULL if * health_check uses "reserved" IP address that does not belong to any * LRP in the backend's datapath. */ diff --git a/northd/northd.c b/northd/northd.c index e4acb11ab..40b9ac25c 100644 --- a/northd/northd.c +++ b/northd/northd.c @@ -3483,11 +3483,6 @@ ovn_lb_svc_create(struct ovsdb_idl_txn *ovnsb_txn, source_mac = svc_global_addresses->mac_src; } - const char *protocol = lb->nlb->protocol; - if (!protocol || !protocol[0]) { - protocol = "tcp"; - } - struct service_monitor_info *mon_info = create_or_get_service_mon(ovnsb_txn, local_svc_monitors_map, @@ -3496,8 +3491,8 @@ ovn_lb_svc_create(struct ovsdb_idl_txn *ovnsb_txn, backend->ip_str, backend_nb->logical_port, NULL, - backend->port, - protocol, + backend_nb->svc_mon_port, + backend_nb->svc_mon_protocol, backend_nb->remote_backend); set_service_mon_options(mon_info->sbrec_mon, @@ -3541,23 +3536,17 @@ ovn_lb_svc_create(struct ovsdb_idl_txn *ovnsb_txn, } static bool -backend_is_available(const struct ovn_northd_lb *lb, - const struct ovn_lb_backend *backend, +backend_is_available(const struct ovn_lb_backend *backend, const struct ovn_northd_lb_backend *backend_nb, const struct svc_monitors_map_data *svc_mons_data) { - char *protocol = lb->nlb->protocol; - if (!protocol || !protocol[0]) { - protocol = "tcp"; - } - struct service_monitor_info *mon_info = get_service_mon(svc_mons_data->local_svc_monitors_map, svc_mons_data->ic_learned_svc_monitors_map, backend->ip_str, backend_nb->logical_port, - backend->port, - protocol); + backend_nb->svc_mon_port, + backend_nb->svc_mon_protocol); if (!mon_info) { return false; @@ -3621,8 +3610,7 @@ build_lb_vip_actions(const struct ovn_northd_lb *lb, } if (backend_nb->health_check && - !backend_is_available(lb, - backend, + !backend_is_available(backend, backend_nb, svc_mons_data)) { continue; @@ -9401,7 +9389,6 @@ build_lb_rules_for_stateless_acl(struct lflow_table *lflows, static void build_lb_health_check_response_lflows( struct lflow_table *lflows, - const struct ovn_northd_lb *lb, const struct ovn_lb_vip *lb_vip, const struct ovn_northd_lb_vip *lb_vip_nb, const struct ovn_lb_datapaths *lb_dps, @@ -9426,10 +9413,7 @@ build_lb_health_check_response_lflows( continue; } - const char *protocol = lb->nlb->protocol; - if (!protocol || !protocol[0]) { - protocol = "tcp"; - } + const char *protocol = backend_nb->svc_mon_protocol; /* A type=external backend has no VIF of its own. Its replies reach * br-int through one of the switch's localnet ports, so they need @@ -9480,8 +9464,8 @@ build_lb_health_check_response_lflows( backend->ip_str, backend_nb->svc_mon_lrp->lrp_networks.ea_s); if (!strcmp(protocol, "tcp")) { - ds_put_format(&probe_match, "tcp.src == %s", - backend->port_str); + ds_put_format(&probe_match, "tcp.src == %"PRIu16, + backend_nb->svc_mon_port); } else { ds_put_cstr(&probe_match, "icmp6.type == 1"); } @@ -9492,8 +9476,8 @@ build_lb_health_check_response_lflows( backend->ip_str, backend_nb->svc_mon_lrp->lrp_networks.ea_s); if (!strcmp(protocol, "tcp")) { - ds_put_format(&probe_match, "tcp.src == %s", - backend->port_str); + ds_put_format(&probe_match, "tcp.src == %"PRIu16, + backend_nb->svc_mon_port); } else { ds_put_cstr(&probe_match, "icmp4.type == 3"); } @@ -14253,7 +14237,7 @@ build_lrouter_flows_for_lb(struct ovn_lb_datapaths *lb_dps, lr_datapaths); build_lb_health_check_response_lflows( - lflows, lb, lb_vip, &lb->vips_nb[i], lb_dps, lr_datapaths, + lflows, lb_vip, &lb->vips_nb[i], lb_dps, lr_datapaths, ls_ports, meter_groups, match, action); if (!build_empty_lb_event_flow(lb_vip, lb, match, action)) { diff --git a/ovn-nb.xml b/ovn-nb.xml index 0d18010aa..5eb76f87c 100644 --- a/ovn-nb.xml +++ b/ovn-nb.xml @@ -2708,6 +2708,30 @@ <code>[2001::1]</code>=<code>sw0-p1:[2002::1]</code>. Remote endpoint: same as for IP. </p> + + <p> + By default, health checks are sent to the endpoint's port using the + load balancer's <ref column="protocol"/>. Both can be overridden + per endpoint by adding them after the source IP: + <code>port_name:source_ip:protocol:port</code>, optionally + followed by <code>:<var>target_zone_name</var></code> + for a remote endpoint. <var>protocol</var> is either + <code>tcp</code> or <code>udp</code> and <var>port</var> is in + the range 1 to 65535. Either of them may be left empty to keep + its default. An endpoint whose health check protocol or port is + invalid is not monitored. + </p> + + <p> + For example, <code>10.0.0.4</code>= + <code>sw0-p1:10.0.0.2:udp:8081</code> probes the endpoint + <code>10.0.0.4</code> of a TCP load balancer with UDP health checks + to port <code>8081</code>, + <code>sw0-p1:10.0.0.2::8081:az1</code> probes the remote endpoint + with the load balancer's protocol on port <code>8081</code>, and + <code>sw0-p1:[2002::1]:udp:8081</code> is the IPv6 equivalent of + the first example. + </p> </column> </group> diff --git a/tests/ovn-northd.at b/tests/ovn-northd.at index f1e88b4bd..ab051c688 100644 --- a/tests/ovn-northd.at +++ b/tests/ovn-northd.at @@ -2043,6 +2043,116 @@ OVN_CLEANUP_NORTHD AT_CLEANUP ]) +OVN_FOR_EACH_NORTHD_NO_HV([ +AT_SETUP([Load balancer health check custom port and protocol]) +ovn_start + +check ovn-nbctl ls-add sw0 +for i in 3 4 5 6; do + check ovn-nbctl lsp-add sw0 sw0-p$i \ + -- lsp-set-addresses sw0-p$i "00:00:00:00:00:0$i 10.0.0.$i" +done +check ovn-nbctl lsp-add sw0 sw0-p7 \ + -- lsp-set-addresses sw0-p7 "00:00:00:00:00:06 2001::7" + +check ovn-sbctl chassis-add hv1 geneve 127.0.0.1 +for i in 3 4 5 6; do + check ovn-sbctl lsp-bind sw0-p$i hv1 + wait_row_count nb:Logical_Switch_Port 1 name=sw0-p$i 'up=true' +done + +check ovn-nbctl lb-add lb1 10.0.0.10:80 10.0.0.3:80,10.0.0.4:80,10.0.0.5:80,10.0.0.6:80 tcp +check ovn-nbctl --wait=sb ls-lb-add sw0 lb1 +check_uuid ovn-nbctl --wait=sb -- --id=@hc create \ +Load_Balancer_Health_Check vip="10.0.0.10\:80" -- add Load_Balancer lb1 \ +health_check @hc + +AS_BOX([Without options the backend port and the LB protocol are used.]) +check ovn-nbctl --wait=sb set load_balancer lb1 \ + ip_port_mappings:10.0.0.3=sw0-p3:10.0.0.2 +wait_row_count Service_Monitor 1 logical_port=sw0-p3 port=80 protocol=tcp + +AS_BOX([Custom health check port.]) +check ovn-nbctl --wait=sb set load_balancer lb1 \ + 'ip_port_mappings:10.0.0.4="sw0-p4:10.0.0.2::8080"' +wait_row_count Service_Monitor 1 logical_port=sw0-p4 port=8080 protocol=tcp + +AS_BOX([Custom health check protocol.]) +check ovn-nbctl --wait=sb set load_balancer lb1 \ + 'ip_port_mappings:10.0.0.5="sw0-p5:10.0.0.2:udp:"' +wait_row_count Service_Monitor 1 logical_port=sw0-p5 port=80 protocol=udp +check_row_count Service_Monitor 3 + +AS_BOX([Backends keep their own port in the LB action.]) +for i in 3 4 5; do + check ovn-sbctl set service_monitor sw0-p$i status=online +done + +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | grep 'priority=120.*backends' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80,10.0.0.4:80,10.0.0.5:80);) +]) + +AS_BOX([The status of the custom monitors gates the backends.]) +check ovn-sbctl set service_monitor sw0-p4 status=offline +check ovn-sbctl set service_monitor sw0-p5 status=offline +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | grep 'priority=120.*backends' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80);) +]) + +AS_BOX([Custom port and protocol together.]) +check ovn-nbctl --wait=sb set load_balancer lb1 \ + 'ip_port_mappings:10.0.0.4="sw0-p4:10.0.0.2:udp:5353"' +wait_row_count Service_Monitor 1 logical_port=sw0-p4 port=5353 protocol=udp +check_row_count Service_Monitor 0 logical_port=sw0-p4 port=8080 +check_row_count Service_Monitor 3 + +AS_BOX([Empty protocol and port keep the defaults.]) +check ovn-nbctl --wait=sb set load_balancer lb1 \ + 'ip_port_mappings:10.0.0.4="sw0-p4:10.0.0.2::"' +wait_row_count Service_Monitor 1 logical_port=sw0-p4 port=80 protocol=tcp +check_row_count Service_Monitor 3 + +AS_BOX([Removing protocol and port restores the defaults.]) +check ovn-nbctl --wait=sb set load_balancer lb1 \ + 'ip_port_mappings:10.0.0.4="sw0-p4:10.0.0.2:udp:5353"' +wait_row_count Service_Monitor 1 logical_port=sw0-p4 port=5353 protocol=udp +check ovn-nbctl --wait=sb set load_balancer lb1 \ + ip_port_mappings:10.0.0.4=sw0-p4:10.0.0.2 +wait_row_count Service_Monitor 1 logical_port=sw0-p4 port=80 protocol=tcp +check_row_count Service_Monitor 3 + + +AS_BOX([Remote health check with AZ name.]) +check ovn-nbctl --wait=sb set load_balancer lb1 \ + 'ip_port_mappings:10.0.0.6="sw0-p6:10.0.0.2:az2"' +wait_row_count Service_Monitor 1 logical_port=sw0-p6 port=80 protocol=tcp \ + remote=true +check_row_count Service_Monitor 4 + +AS_BOX([Protocol and port can be combined with a remote endpoint.]) +check ovn-nbctl --wait=sb set load_balancer lb1 \ + 'ip_port_mappings:10.0.0.6="sw0-p6:10.0.0.2::8080:az2"' +wait_row_count Service_Monitor 1 logical_port=sw0-p6 port=8080 protocol=tcp \ + remote=true +check_row_count Service_Monitor 4 + +AS_BOX([IPv6 backend.]) +check ovn-nbctl lb-add lb2 [[2001::10]]:80 [[2001::7]]:80 tcp +check ovn-nbctl --wait=sb ls-lb-add sw0 lb2 +check_uuid ovn-nbctl --wait=sb -- --id=@hc create \ +Load_Balancer_Health_Check vip="\[\[2001\:\:10\]\]\:80" -- add Load_Balancer lb2 \ +health_check @hc +check ovn-nbctl --wait=sb set load_balancer lb2 \ + 'ip_port_mappings:"[[2001::7]]"="sw0-p7:[[2001::2]]:udp:8080"' +wait_row_count Service_Monitor 1 logical_port=sw0-p7 ip=\"2001::7\" \ + port=8080 protocol=udp + +OVN_CLEANUP_NORTHD +AT_CLEANUP +]) + OVN_FOR_EACH_NORTHD_NO_HV([ AT_SETUP([Load balancer VIP in NAT entries]) AT_SKIP_IF([test $HAVE_PYTHON = no]) -- 2.48.1 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
