Load balancer health checks always probe the backend's port using the
load balancer's protocol.  Some services expose their health endpoint
on a different port, or can only be probed with a different protocol,
e.g. a TCP service with a UDP health endpoint. Extend the Load_Balancer
ip_port_mappings value format to:
"backed_ip=port_name:source_ip[:protocol:port][:az_name]".

Signed-off-by: Alexandra Rukomoinikova <[email protected]>
---
 NEWS                              |   4 +
 northd/en-advertised-route-sync.c |   6 +-
 northd/lb.c                       | 118 ++++++++++++++++++++++++------
 northd/lb.h                       |   6 ++
 northd/northd.c                   |  40 +++-------
 ovn-nb.xml                        |  24 ++++++
 tests/ovn-northd.at               | 110 ++++++++++++++++++++++++++++
 7 files changed, 255 insertions(+), 53 deletions(-)

diff --git a/NEWS b/NEWS
index b2416f98b..f9edcf59f 100644
--- a/NEWS
+++ b/NEWS
@@ -77,6 +77,10 @@ Post v26.09.0
      the replies from the backends of its load balancers.  The CMS must make
      sure that the replies re-enter the router on the chassis that load
      balanced the request.  See ovn-nb(5).
+   - Load balancer health checks can now use a port and a protocol
+     different from the backend's port and the load balancer's protocol.
+     They are set in the ip_port_mappings column in Load_Balancer table.
+     See the ovn-nb(5) man page for more information.
 
 OVN v26.09.0 - xxx xx xxxx
 --------------------------
diff --git a/northd/en-advertised-route-sync.c 
b/northd/en-advertised-route-sync.c
index eb8a24a39..1c646b855 100644
--- a/northd/en-advertised-route-sync.c
+++ b/northd/en-advertised-route-sync.c
@@ -548,8 +548,6 @@ build_lb_routes_for_lb(const struct ovn_port 
*advertising_op,
     }
 
     const struct ovn_northd_lb *lb = lb_dps->lb;
-    const char *protocol = lb->nlb->protocol && lb->nlb->protocol[0]
-                           ? lb->nlb->protocol : "tcp";
     for (size_t v = 0; v < lb->n_vips; v++) {
         const struct ovn_lb_vip *vip = &lb->vips[v];
         const struct ovn_northd_lb_vip *vip_nb = &lb->vips_nb[v];
@@ -603,8 +601,8 @@ build_lb_routes_for_lb(const struct ovn_port 
*advertising_op,
             if (backend_nb->health_check) {
                 sset_add_and_free(
                     &route_e->health_checks,
-                    xasprintf("%s,%s,%"PRIu16, protocol, backend->ip_str,
-                              backend->port));
+                    xasprintf("%s,%s,%"PRIu16, backend_nb->svc_mon_protocol,
+                              backend->ip_str, backend_nb->svc_mon_port));
             } else {
                 route_e->has_ungated_lb = true;
             }
diff --git a/northd/lb.c b/northd/lb.c
index 5ff9d1fad..d2ca376f8 100644
--- a/northd/lb.c
+++ b/northd/lb.c
@@ -106,11 +106,82 @@ ovn_northd_lb_vip_init(struct ovn_northd_lb_vip 
*lb_vip_nb,
         ovn_lb_get_health_check(nbrec_lb, vip_port_str, template);
 }
 
+static bool
+parse_ip_port_mapping_optional_params(char *fields,
+                                      const char *mapping,
+                                      const char **hc_protocol,
+                                      uint16_t *hc_port,
+                                      bool *is_remote, char **az_name)
+{
+    static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1);
+    char *field[3];
+    size_t n_fields = 0;
+
+    for (char *f = fields; f; n_fields++) {
+        if (n_fields == ARRAY_SIZE(field)) {
+            VLOG_WARN_RL(&rl, "Too many fields in ip_port_mappings %s",
+                         mapping);
+            return false;
+        }
+        field[n_fields] = f;
+        f = strchr(f, ':');
+        if (f) {
+            *f++ = '\0';
+        }
+    }
+
+    const char *protocol = NULL;
+    const char *port = NULL;
+    char *az = NULL;
+    if (n_fields == 1) {
+        az = field[0];
+    } else {
+        protocol = field[0];
+        port = field[1];
+        az = n_fields == 3 ? field[2] : NULL;
+    }
+
+    if (az && !*az) {
+        VLOG_WARN_RL(&rl, "Empty AZ name specified in ip_port_mappings %s",
+                     mapping);
+        return false;
+    }
+
+    if (protocol && *protocol) {
+        if (!strcmp(protocol, "tcp")) {
+            *hc_protocol = "tcp";
+        } else if (!strcmp(protocol, "udp")) {
+            *hc_protocol = "udp";
+        } else {
+            VLOG_WARN_RL(&rl, "Invalid health check protocol \"%s\" in "
+                         "ip_port_mappings %s, expected tcp or udp",
+                         protocol, mapping);
+            return false;
+        }
+    }
+
+    if (port && *port) {
+        unsigned int value;
+        if (!str_to_uint(port, 10, &value) || !value || value > 65535) {
+            VLOG_WARN_RL(&rl, "Invalid health check port \"%s\" in "
+                         "ip_port_mappings %s", port, mapping);
+            return false;
+        }
+        *hc_port = value;
+    }
+
+    *az_name = az;
+    *is_remote = az != NULL;
+
+    return true;
+}
+
 /*
  * Parses ip_port_mappings in the format :
- * "ip:logical_port[:src_ip][:az_name]".
+ * "ip:logical_port[:src_ip][:protocol:port][:az_name]".
  * src_ip parameter is optional when distributed mode is enabled,
- * without health checks configured.
+ * without health checks configured. Protocol and port override
+ * the protocol and the port used by health checks.
  * If az_name is present and non-empty, it indicates this is a
  * remote service monitor (backend is in another availability zone),
  * it should be propogated to another AZ by interconnection processing.
@@ -144,6 +215,8 @@ ovn_lb_vip_backends_ip_port_mappings_init(const struct 
ovn_northd_lb *lb,
         char *port_name = NULL;
         char *first_colon = NULL;
         bool is_remote = false;
+        uint16_t hc_port = 0;
+        const char *hc_protocol = NULL;
 
         port_name = xstrdup(s);
         first_colon = strchr(port_name, ':');
@@ -162,8 +235,9 @@ ovn_lb_vip_backends_ip_port_mappings_init(const struct 
ovn_northd_lb *lb,
         }
 
         *first_colon = '\0';
+        char *fields = NULL;
         if (first_colon[1] == '[') {
-            /* IPv6 case - format: port:[ipv6]:az or port:[ipv6] */
+            /* IPv6 case - format: port:[ipv6][:fields] */
             char *ip_end = strchr(first_colon + 2, ']');
             if (!ip_end) {
                 VLOG_WARN_RL(&rl, "Malformed IPv6 address in backend %s", s);
@@ -174,27 +248,22 @@ ovn_lb_vip_backends_ip_port_mappings_init(const struct 
ovn_northd_lb *lb,
             *ip_end = '\0';
 
             if (ip_end[1] == ':') {
-                az_name = ip_end + 2;
-                if (!*az_name) {
-                    VLOG_WARN_RL(&rl, "Empty AZ name specified for backend %s",
-                                 port_name);
-                    goto cleanup;
-                }
-                is_remote = true;
+                fields = ip_end + 2;
             }
         } else {
-            /* IPv4 case - format: port:ip:az or port:ip */
+            /* IPv4 case - format: port:ip[:fields] */
             src_ip = first_colon + 1;
-            char *az_colon = strchr(src_ip, ':');
-            if (az_colon) {
-                *az_colon = '\0';
-                az_name = az_colon + 1;
-                if (!*az_name) {
-                    VLOG_WARN_RL(&rl, "Empty AZ name specified for backend %s",
-                                 port_name);
-                    goto cleanup;
-                }
-                is_remote = true;
+            fields = strchr(src_ip, ':');
+            if (fields) {
+                *fields++ = '\0';
+            }
+        }
+
+        if (fields) {
+            if (!parse_ip_port_mapping_optional_params(fields, s,
+                                                       &hc_protocol, &hc_port,
+                                                       &is_remote, &az_name)) {
+                goto cleanup;
             }
         }
 
@@ -218,6 +287,12 @@ init_backend:
         backend_nb->remote_backend = is_remote;
         backend_nb->svc_mon_lrp = NULL;
         backend_nb->distributed_backend = lb->is_distributed;
+        backend_nb->svc_mon_port = hc_port ? hc_port : backend->port;
+        if (!hc_protocol) {
+            hc_protocol = lb->nlb->protocol && lb->nlb->protocol[0]
+                          ? lb->nlb->protocol : "tcp";
+        }
+        backend_nb->svc_mon_protocol = xstrdup(hc_protocol);
 cleanup:
         free(port_name);
     }
@@ -233,6 +308,7 @@ void ovn_northd_lb_vip_destroy(struct ovn_northd_lb_vip 
*vip)
         free(vip->backends_nb[i].logical_port);
         free(vip->backends_nb[i].svc_mon_src_ip);
         free(vip->backends_nb[i].az_name);
+        free(vip->backends_nb[i].svc_mon_protocol);
     }
     free(vip->backends_nb);
 }
diff --git a/northd/lb.h b/northd/lb.h
index 7a98c2f55..72ec8b072 100644
--- a/northd/lb.h
+++ b/northd/lb.h
@@ -104,6 +104,12 @@ struct ovn_northd_lb_backend {
     char *svc_mon_src_ip;
     /* Target Availability Zone name for service monitoring. */
     char *az_name;
+    /* Port and protocol used by the health check. They default to the
+     * backend's port and the load balancer's protocol, but can be
+     * overridden by the "hc_port" and "hc_protocol" options of the
+     * backend's ip_port_mappings entry. */
+    uint16_t svc_mon_port;
+    char *svc_mon_protocol;
     /* LRP used as the source of health_check traffic.  This may be NULL if
      * health_check uses "reserved" IP address that does not belong to any
      * LRP in the backend's datapath. */
diff --git a/northd/northd.c b/northd/northd.c
index e4acb11ab..40b9ac25c 100644
--- a/northd/northd.c
+++ b/northd/northd.c
@@ -3483,11 +3483,6 @@ ovn_lb_svc_create(struct ovsdb_idl_txn *ovnsb_txn,
                 source_mac = svc_global_addresses->mac_src;
             }
 
-            const char *protocol = lb->nlb->protocol;
-            if (!protocol || !protocol[0]) {
-                protocol = "tcp";
-            }
-
             struct service_monitor_info *mon_info =
                 create_or_get_service_mon(ovnsb_txn,
                                           local_svc_monitors_map,
@@ -3496,8 +3491,8 @@ ovn_lb_svc_create(struct ovsdb_idl_txn *ovnsb_txn,
                                           backend->ip_str,
                                           backend_nb->logical_port,
                                           NULL,
-                                          backend->port,
-                                          protocol,
+                                          backend_nb->svc_mon_port,
+                                          backend_nb->svc_mon_protocol,
                                           backend_nb->remote_backend);
 
             set_service_mon_options(mon_info->sbrec_mon,
@@ -3541,23 +3536,17 @@ ovn_lb_svc_create(struct ovsdb_idl_txn *ovnsb_txn,
 }
 
 static bool
-backend_is_available(const struct ovn_northd_lb *lb,
-                     const struct ovn_lb_backend *backend,
+backend_is_available(const struct ovn_lb_backend *backend,
                      const struct ovn_northd_lb_backend *backend_nb,
                      const struct svc_monitors_map_data *svc_mons_data)
 {
-    char *protocol = lb->nlb->protocol;
-    if (!protocol || !protocol[0]) {
-        protocol = "tcp";
-    }
-
     struct service_monitor_info *mon_info =
         get_service_mon(svc_mons_data->local_svc_monitors_map,
                         svc_mons_data->ic_learned_svc_monitors_map,
                         backend->ip_str,
                         backend_nb->logical_port,
-                        backend->port,
-                        protocol);
+                        backend_nb->svc_mon_port,
+                        backend_nb->svc_mon_protocol);
 
     if (!mon_info) {
         return false;
@@ -3621,8 +3610,7 @@ build_lb_vip_actions(const struct ovn_northd_lb *lb,
             }
 
             if (backend_nb->health_check &&
-                !backend_is_available(lb,
-                                      backend,
+                !backend_is_available(backend,
                                       backend_nb,
                                       svc_mons_data)) {
                 continue;
@@ -9401,7 +9389,6 @@ build_lb_rules_for_stateless_acl(struct lflow_table 
*lflows,
 static void
 build_lb_health_check_response_lflows(
     struct lflow_table *lflows,
-    const struct ovn_northd_lb *lb,
     const struct ovn_lb_vip *lb_vip,
     const struct ovn_northd_lb_vip *lb_vip_nb,
     const struct ovn_lb_datapaths *lb_dps,
@@ -9426,10 +9413,7 @@ build_lb_health_check_response_lflows(
             continue;
         }
 
-        const char *protocol = lb->nlb->protocol;
-        if (!protocol || !protocol[0]) {
-            protocol = "tcp";
-        }
+        const char *protocol = backend_nb->svc_mon_protocol;
 
         /* A type=external backend has no VIF of its own.  Its replies reach
          * br-int through one of the switch's localnet ports, so they need
@@ -9480,8 +9464,8 @@ build_lb_health_check_response_lflows(
                               backend->ip_str,
                               backend_nb->svc_mon_lrp->lrp_networks.ea_s);
                 if (!strcmp(protocol, "tcp")) {
-                    ds_put_format(&probe_match, "tcp.src == %s",
-                                  backend->port_str);
+                    ds_put_format(&probe_match, "tcp.src == %"PRIu16,
+                                  backend_nb->svc_mon_port);
                 } else {
                     ds_put_cstr(&probe_match, "icmp6.type == 1");
                 }
@@ -9492,8 +9476,8 @@ build_lb_health_check_response_lflows(
                               backend->ip_str,
                               backend_nb->svc_mon_lrp->lrp_networks.ea_s);
                 if (!strcmp(protocol, "tcp")) {
-                    ds_put_format(&probe_match, "tcp.src == %s",
-                                  backend->port_str);
+                    ds_put_format(&probe_match, "tcp.src == %"PRIu16,
+                                  backend_nb->svc_mon_port);
                 } else {
                     ds_put_cstr(&probe_match, "icmp4.type == 3");
                 }
@@ -14253,7 +14237,7 @@ build_lrouter_flows_for_lb(struct ovn_lb_datapaths 
*lb_dps,
                                                  lr_datapaths);
 
         build_lb_health_check_response_lflows(
-            lflows, lb, lb_vip, &lb->vips_nb[i], lb_dps, lr_datapaths,
+            lflows, lb_vip, &lb->vips_nb[i], lb_dps, lr_datapaths,
             ls_ports, meter_groups, match, action);
 
         if (!build_empty_lb_event_flow(lb_vip, lb, match, action)) {
diff --git a/ovn-nb.xml b/ovn-nb.xml
index 0d18010aa..5eb76f87c 100644
--- a/ovn-nb.xml
+++ b/ovn-nb.xml
@@ -2708,6 +2708,30 @@
           <code>[2001::1]</code>=<code>sw0-p1:[2002::1]</code>.
           Remote endpoint: same as for IP.
         </p>
+
+        <p>
+          By default, health checks are sent to the endpoint's port using the
+          load balancer's <ref column="protocol"/>.  Both can be overridden
+          per endpoint by adding them after the source IP:
+          <code>port_name:source_ip:protocol:port</code>, optionally
+          followed by <code>:<var>target_zone_name</var></code>
+          for a remote endpoint.  <var>protocol</var> is either
+          <code>tcp</code> or <code>udp</code> and <var>port</var> is in
+          the range 1 to 65535.  Either of them may be left empty to keep
+          its default.  An endpoint whose health check protocol or port is
+          invalid is not monitored.
+        </p>
+
+        <p>
+          For example, <code>10.0.0.4</code>=
+          <code>sw0-p1:10.0.0.2:udp:8081</code> probes the endpoint
+          <code>10.0.0.4</code> of a TCP load balancer with UDP health checks
+          to port <code>8081</code>,
+          <code>sw0-p1:10.0.0.2::8081:az1</code> probes the remote endpoint
+          with the load balancer's protocol on port <code>8081</code>, and
+          <code>sw0-p1:[2002::1]:udp:8081</code> is the IPv6 equivalent of
+          the first example.
+        </p>
       </column>
     </group>
 
diff --git a/tests/ovn-northd.at b/tests/ovn-northd.at
index f1e88b4bd..ab051c688 100644
--- a/tests/ovn-northd.at
+++ b/tests/ovn-northd.at
@@ -2043,6 +2043,116 @@ OVN_CLEANUP_NORTHD
 AT_CLEANUP
 ])
 
+OVN_FOR_EACH_NORTHD_NO_HV([
+AT_SETUP([Load balancer health check custom port and protocol])
+ovn_start
+
+check ovn-nbctl ls-add sw0
+for i in 3 4 5 6; do
+    check ovn-nbctl lsp-add sw0 sw0-p$i \
+        -- lsp-set-addresses sw0-p$i "00:00:00:00:00:0$i 10.0.0.$i"
+done
+check ovn-nbctl lsp-add sw0 sw0-p7 \
+    -- lsp-set-addresses sw0-p7 "00:00:00:00:00:06 2001::7"
+
+check ovn-sbctl chassis-add hv1 geneve 127.0.0.1
+for i in 3 4 5 6; do
+    check ovn-sbctl lsp-bind sw0-p$i hv1
+    wait_row_count nb:Logical_Switch_Port 1 name=sw0-p$i 'up=true'
+done
+
+check ovn-nbctl lb-add lb1 10.0.0.10:80 
10.0.0.3:80,10.0.0.4:80,10.0.0.5:80,10.0.0.6:80 tcp
+check ovn-nbctl --wait=sb ls-lb-add sw0 lb1
+check_uuid ovn-nbctl --wait=sb -- --id=@hc create \
+Load_Balancer_Health_Check vip="10.0.0.10\:80" -- add Load_Balancer lb1 \
+health_check @hc
+
+AS_BOX([Without options the backend port and the LB protocol are used.])
+check ovn-nbctl --wait=sb set load_balancer lb1 \
+    ip_port_mappings:10.0.0.3=sw0-p3:10.0.0.2
+wait_row_count Service_Monitor 1 logical_port=sw0-p3 port=80 protocol=tcp
+
+AS_BOX([Custom health check port.])
+check ovn-nbctl --wait=sb set load_balancer lb1 \
+    'ip_port_mappings:10.0.0.4="sw0-p4:10.0.0.2::8080"'
+wait_row_count Service_Monitor 1 logical_port=sw0-p4 port=8080 protocol=tcp
+
+AS_BOX([Custom health check protocol.])
+check ovn-nbctl --wait=sb set load_balancer lb1 \
+    'ip_port_mappings:10.0.0.5="sw0-p5:10.0.0.2:udp:"'
+wait_row_count Service_Monitor 1 logical_port=sw0-p5 port=80 protocol=udp
+check_row_count Service_Monitor 3
+
+AS_BOX([Backends keep their own port in the LB action.])
+for i in 3 4 5; do
+    check ovn-sbctl set service_monitor sw0-p$i status=online
+done
+
+OVS_WAIT_FOR_OUTPUT(
+  [ovn-sbctl dump-flows sw0 | grep 'priority=120.*backends' | 
ovn_strip_lflows], 0, [dnl
+  table=??(ls_in_lb           ), priority=120  , match=(ct.new && ip4.dst == 
10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 
10.0.0.10; reg2[[0..15]] = 80; 
ct_lb_mark(backends=10.0.0.3:80,10.0.0.4:80,10.0.0.5:80);)
+])
+
+AS_BOX([The status of the custom monitors gates the backends.])
+check ovn-sbctl set service_monitor sw0-p4 status=offline
+check ovn-sbctl set service_monitor sw0-p5 status=offline
+OVS_WAIT_FOR_OUTPUT(
+  [ovn-sbctl dump-flows sw0 | grep 'priority=120.*backends' | 
ovn_strip_lflows], 0, [dnl
+  table=??(ls_in_lb           ), priority=120  , match=(ct.new && ip4.dst == 
10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 
10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80);)
+])
+
+AS_BOX([Custom port and protocol together.])
+check ovn-nbctl --wait=sb set load_balancer lb1 \
+    'ip_port_mappings:10.0.0.4="sw0-p4:10.0.0.2:udp:5353"'
+wait_row_count Service_Monitor 1 logical_port=sw0-p4 port=5353 protocol=udp
+check_row_count Service_Monitor 0 logical_port=sw0-p4 port=8080
+check_row_count Service_Monitor 3
+
+AS_BOX([Empty protocol and port keep the defaults.])
+check ovn-nbctl --wait=sb set load_balancer lb1 \
+    'ip_port_mappings:10.0.0.4="sw0-p4:10.0.0.2::"'
+wait_row_count Service_Monitor 1 logical_port=sw0-p4 port=80 protocol=tcp
+check_row_count Service_Monitor 3
+
+AS_BOX([Removing protocol and port restores the defaults.])
+check ovn-nbctl --wait=sb set load_balancer lb1 \
+    'ip_port_mappings:10.0.0.4="sw0-p4:10.0.0.2:udp:5353"'
+wait_row_count Service_Monitor 1 logical_port=sw0-p4 port=5353 protocol=udp
+check ovn-nbctl --wait=sb set load_balancer lb1 \
+    ip_port_mappings:10.0.0.4=sw0-p4:10.0.0.2
+wait_row_count Service_Monitor 1 logical_port=sw0-p4 port=80 protocol=tcp
+check_row_count Service_Monitor 3
+
+
+AS_BOX([Remote health check with AZ name.])
+check ovn-nbctl --wait=sb set load_balancer lb1 \
+    'ip_port_mappings:10.0.0.6="sw0-p6:10.0.0.2:az2"'
+wait_row_count Service_Monitor 1 logical_port=sw0-p6 port=80 protocol=tcp \
+    remote=true
+check_row_count Service_Monitor 4
+
+AS_BOX([Protocol and port can be combined with a remote endpoint.])
+check ovn-nbctl --wait=sb set load_balancer lb1 \
+    'ip_port_mappings:10.0.0.6="sw0-p6:10.0.0.2::8080:az2"'
+wait_row_count Service_Monitor 1 logical_port=sw0-p6 port=8080 protocol=tcp \
+    remote=true
+check_row_count Service_Monitor 4
+
+AS_BOX([IPv6 backend.])
+check ovn-nbctl lb-add lb2 [[2001::10]]:80 [[2001::7]]:80 tcp
+check ovn-nbctl --wait=sb ls-lb-add sw0 lb2
+check_uuid ovn-nbctl --wait=sb -- --id=@hc create \
+Load_Balancer_Health_Check vip="\[\[2001\:\:10\]\]\:80" -- add Load_Balancer 
lb2 \
+health_check @hc
+check ovn-nbctl --wait=sb set load_balancer lb2 \
+    'ip_port_mappings:"[[2001::7]]"="sw0-p7:[[2001::2]]:udp:8080"'
+wait_row_count Service_Monitor 1 logical_port=sw0-p7 ip=\"2001::7\" \
+    port=8080 protocol=udp
+
+OVN_CLEANUP_NORTHD
+AT_CLEANUP
+])
+
 OVN_FOR_EACH_NORTHD_NO_HV([
 AT_SETUP([Load balancer VIP in NAT entries])
 AT_SKIP_IF([test $HAVE_PYTHON = no])
-- 
2.48.1

_______________________________________________
dev mailing list
[email protected]
https://mail.openvswitch.org/mailman/listinfo/ovs-dev

Reply via email to