For each distributed-LB Advertised_Route, northd writes its monitored backends to external_ids as protocol, IP and port selectors. The controller looks up matching Service_Monitor rows for the tracked logical port. It withdraws the route when monitors exist and none are online. A route with no matching monitor is installed.
The selectors are encoded in a single external_ids:health-checks key as semicolon-delimited protocol,IP,port tuples. IP addresses are compared in binary form so equivalent IPv6 spellings continue to match. Use an IDL index on Service_Monitor logical port, type, protocol and port to narrow each selector lookup. Record the UUIDs of rows that matched during the route run. A tracked change to one of those UUIDs forces recomputation, which covers deletion and movement away from a selector. A new or updated load-balancer monitor on a referenced logical port also forces recomputation, which covers insertion and movement into a selector. Other monitor changes leave the route node unchanged. Route evaluation reads the live IDL rows. Apply advertising-port locality and dynamic-routing-redistribute-local-only eligibility before registering a route health gate. A remote local-only backend does not make unrelated controllers parse its selectors or recompute routes when its monitor changes. If an LB route and a non-LB route share the same (datapath, logical_port, ip_prefix, tracked_port) key - for example, when the same IP is used as both a distributed-LB VIP and a NAT external IP pointing at the same backend LSP - omit source=lb, distributed-lb and health-checks metadata so the controller does not health-gate the shared route. Track the non-LB route independently of route-builder order. The shared kernel route remains unconditional. This is an edge case a CMS would typically prevent, but both paths must survive when it occurs. external_ids:enabled is intentionally not written by northd. It is an operator- or CMS-owned administrative override for marked routes. Setting it to false withdraws the route. Centralized load balancers and non-LB routes do not use either gate. Signed-off-by: Dmitrii Shcherbakov <[email protected]> --- controller/ovn-controller.c | 81 ++- controller/route.c | 273 +++++++- controller/route.h | 16 + lib/ovn-util.h | 4 + northd/en-advertised-route-sync.c | 71 +- ovn-sb.xml | 45 +- tests/ovn-inc-proc-graph-dump.at | 2 + tests/ovn-northd.at | 80 ++- tests/system-ovn.at | 1011 ++++++++++++++++++++++++++++- 9 files changed, 1533 insertions(+), 50 deletions(-) diff --git a/controller/ovn-controller.c b/controller/ovn-controller.c index 141485237..4dcb4b895 100644 --- a/controller/ovn-controller.c +++ b/controller/ovn-controller.c @@ -62,6 +62,7 @@ #include "lib/ovn-dirs.h" #include "lib/ovn-sb-idl.h" #include "lib/ovn-util.h" +#include "lib/uuidset.h" #include "ovsport.h" #include "patch.h" #include "vif-plug.h" @@ -364,7 +365,6 @@ update_sb_monitors(struct ovsdb_idl *ovnsb_idl, * ones. */ ovsdb_idl_condition_add_clause_true(&ar); } - if (local_ifaces) { const char *name; @@ -5263,6 +5263,13 @@ struct ed_type_route { * locally. */ struct sset tracked_ports_remote; + /* Contains logical ports referenced by health-gated advertised routes. */ + struct sset health_check_ports; + + /* Service_Monitor rows matched during the last route run. Their UUIDs + * identify relevant deletions and selector changes. */ + struct uuidset relevant_service_monitors; + /* Contains all the currently configured dynamic-routing-port-name values * on all datapaths. */ @@ -5270,7 +5277,6 @@ struct ed_type_route { /* Contains struct advertise_datapath_entry */ struct hmap announce_routes; - struct ovsdb_idl *ovnsb_idl; }; @@ -5301,7 +5307,9 @@ en_route_run(struct engine_node *node, void *data) const struct sbrec_advertised_route_table *advertised_route_table = EN_OVSDB_GET(engine_get_input("SB_advertised_route", node)); - + struct ovsdb_idl_index *service_monitor_by_selector = + engine_ovsdb_node_get_index( + engine_get_input("SB_service_monitor", node), "selector"); const struct ovsrec_open_vswitch *cfg = ovsrec_open_vswitch_table_first(ovs_table); const char *dynamic_routing_port_mapping = @@ -5309,6 +5317,7 @@ en_route_run(struct engine_node *node, void *data) struct route_ctx_in r_ctx_in = { .advertised_route_table = advertised_route_table, + .service_monitor_by_selector = service_monitor_by_selector, .sbrec_port_binding_by_name = sbrec_port_binding_by_name, .chassis = chassis, .dynamic_routing_port_mapping = dynamic_routing_port_mapping, @@ -5319,6 +5328,9 @@ en_route_run(struct engine_node *node, void *data) struct route_ctx_out r_ctx_out = { .tracked_re_datapaths = &re_data->tracked_route_datapaths, .tracked_ports_local = &re_data->tracked_ports_local, + .health_check_ports = &re_data->health_check_ports, + .relevant_service_monitors = + &re_data->relevant_service_monitors, .filtered_ports = &re_data->filtered_ports, .tracked_ports_remote = &re_data->tracked_ports_remote, .announce_routes = &re_data->announce_routes, @@ -5328,9 +5340,12 @@ en_route_run(struct engine_node *node, void *data) tracked_datapaths_clear(r_ctx_out.tracked_re_datapaths); sset_clear(r_ctx_out.tracked_ports_local); sset_clear(r_ctx_out.tracked_ports_remote); + sset_clear(r_ctx_out.health_check_ports); + uuidset_clear(r_ctx_out.relevant_service_monitors); sset_clear(r_ctx_out.filtered_ports); route_run(&r_ctx_in, &r_ctx_out); + return EN_UPDATED; } @@ -5344,6 +5359,8 @@ en_route_init(struct engine_node *node OVS_UNUSED, hmap_init(&data->tracked_route_datapaths); sset_init(&data->tracked_ports_local); sset_init(&data->tracked_ports_remote); + sset_init(&data->health_check_ports); + uuidset_init(&data->relevant_service_monitors); sset_init(&data->filtered_ports); hmap_init(&data->announce_routes); data->ovnsb_idl = arg->sb_idl; @@ -5359,6 +5376,8 @@ en_route_cleanup(void *data) tracked_datapaths_destroy(&re_data->tracked_route_datapaths); sset_destroy(&re_data->tracked_ports_local); sset_destroy(&re_data->tracked_ports_remote); + sset_destroy(&re_data->health_check_ports); + uuidset_destroy(&re_data->relevant_service_monitors); sset_destroy(&re_data->filtered_ports); route_cleanup(&re_data->announce_routes); hmap_destroy(&re_data->announce_routes); @@ -5578,6 +5597,11 @@ route_sb_advertised_route_data_handler(struct engine_node *node, void *data) return EN_UNHANDLED; } + if (sbrec_advertised_route_is_updated( + sbrec_route, SBREC_ADVERTISED_ROUTE_COL_EXTERNAL_IDS)) { + return EN_UNHANDLED; + } + if (sbrec_route->tracked_port) { const char *name = sbrec_route->tracked_port->logical_port; if (!(sset_contains(&re_data->tracked_ports_local, name) || @@ -5629,6 +5653,31 @@ route_sb_datapath_binding_handler(struct engine_node *node, return EN_HANDLED_UNCHANGED; } +static enum engine_input_handler_result +route_sb_service_monitor_handler(struct engine_node *node, + void *data) +{ + struct ed_type_route *re_data = data; + const struct sbrec_service_monitor_table *sm_table = + EN_OVSDB_GET(engine_get_input("SB_service_monitor", node)); + + const struct sbrec_service_monitor *sm; + SBREC_SERVICE_MONITOR_TABLE_FOR_EACH_TRACKED (sm, sm_table) { + if (uuidset_find(&re_data->relevant_service_monitors, + &sm->header_.uuid)) { + return EN_UNHANDLED; + } + if (!sbrec_service_monitor_is_deleted(sm) && + sm->type && !strcmp(sm->type, "load-balancer") && + sset_contains(&re_data->health_check_ports, + sm->logical_port)) { + return EN_UNHANDLED; + } + } + + return EN_HANDLED_UNCHANGED; +} + static int table_id_cmp(const void *a_, const void *b_) { @@ -5665,7 +5714,6 @@ static enum engine_node_state en_route_exchange_run(struct engine_node *node, void *data) { struct ed_type_route_exchange *re = data; - struct ovsdb_idl_index *sbrec_learned_route_by_datapath = engine_ovsdb_node_get_index( engine_get_input("SB_learned_route", node), @@ -5675,7 +5723,6 @@ en_route_exchange_run(struct engine_node *node, void *data) engine_ovsdb_node_get_index( engine_get_input("SB_port_binding", node), "name"); - struct ed_type_route *route_data = engine_get_input_data("route", node); struct ed_type_route_table_notify *rt_notify = @@ -6936,7 +6983,8 @@ evpn_arp_vtep_binding_handler(struct engine_node *node, void *data OVS_UNUSED) SB_NODE(acl_id) \ SB_NODE(advertised_route) \ SB_NODE(learned_route) \ - SB_NODE(advertised_mac_binding) + SB_NODE(advertised_mac_binding) \ + SB_NODE(service_monitor) enum sb_engine_node { #define SB_NODE(NAME) SB_##NAME, @@ -7069,6 +7117,8 @@ inc_proc_ovn_controller_init( route_sb_advertised_route_data_handler); engine_add_input(&en_route, &en_sb_datapath_binding, route_sb_datapath_binding_handler); + engine_add_input(&en_route, &en_sb_service_monitor, + route_sb_service_monitor_handler); engine_add_input(&en_route_exchange, &en_ovs_open_vswitch, NULL); engine_add_input(&en_route_exchange, &en_sb_chassis, NULL); @@ -7380,6 +7430,19 @@ inc_proc_ovn_controller_init( engine_ovsdb_node_add_index(&en_sb_datapath_binding, "key", sbrec_datapath_binding_by_key); + const struct ovsdb_idl_index_column service_monitor_selector_columns[] = { + { .column = &sbrec_service_monitor_col_logical_port }, + { .column = &sbrec_service_monitor_col_type }, + { .column = &sbrec_service_monitor_col_protocol }, + { .column = &sbrec_service_monitor_col_port }, + }; + struct ovsdb_idl_index *service_monitor_by_selector = + ovsdb_idl_index_create( + sb_idl_loop->idl, service_monitor_selector_columns, + ARRAY_SIZE(service_monitor_selector_columns)); + engine_ovsdb_node_add_index(&en_sb_service_monitor, "selector", + service_monitor_by_selector); + struct ovsdb_idl_index *sbrec_fdb_by_dp_key = ovsdb_idl_index_create1(sb_idl_loop->idl, &sbrec_fdb_col_dp_key); @@ -7408,7 +7471,6 @@ inc_proc_ovn_controller_init( &sbrec_learned_route_col_datapath); engine_ovsdb_node_add_index(&en_sb_learned_route, "datapath", sbrec_learned_route_index_by_datapath); - struct ovsdb_idl_index *sbrec_advertised_mac_binding_index_by_dp = ovsdb_idl_index_create1(sb_idl_loop->idl, &sbrec_advertised_mac_binding_col_datapath); @@ -7655,8 +7717,6 @@ main(int argc, char *argv[]) ovsdb_idl_omit(ovnsb_idl_loop.idl, &sbrec_ha_chassis_col_external_ids); ovsdb_idl_omit(ovnsb_idl_loop.idl, &sbrec_ha_chassis_group_col_external_ids); - ovsdb_idl_omit(ovnsb_idl_loop.idl, - &sbrec_advertised_route_col_external_ids); ovsdb_idl_omit(ovnsb_idl_loop.idl, &sbrec_learned_route_col_external_ids); ovsdb_idl_omit(ovnsb_idl_loop.idl, @@ -7679,7 +7739,7 @@ main(int argc, char *argv[]) * other_config column so we no longer need to monitor it */ ovsdb_idl_omit_alert(ovnsb_idl_loop.idl, &sbrec_chassis_col_external_ids); - /* Do not monitor Chassis_Private external_ids */ + /* Do not monitor Chassis_Private external_ids. */ ovsdb_idl_omit(ovnsb_idl_loop.idl, &sbrec_chassis_private_col_external_ids); @@ -8500,7 +8560,6 @@ loop_done: ? chassis_private_lookup_by_name( sbrec_chassis_private_by_name, chassis_id) : NULL); - /* Run all of the cleanup functions, even if one of them returns * false. We're done if all of them return true. */ done = binding_cleanup(ovnsb_idl_txn, port_binding_table, chassis); diff --git a/controller/route.c b/controller/route.c index 553ed4f78..0230e372a 100644 --- a/controller/route.c +++ b/controller/route.c @@ -18,6 +18,8 @@ #include <config.h> #include <net/if.h> +#include <netinet/in.h> +#include <arpa/inet.h> #include "vswitch-idl.h" #include "openvswitch/hmap.h" @@ -25,6 +27,8 @@ #include "openvswitch/ofp-parse.h" #include "lib/ovn-sb-idl.h" +#include "lib/ovn-util.h" +#include "lib/uuidset.h" #include "binding.h" #include "ha-chassis.h" @@ -74,6 +78,214 @@ find_veth_peer(const struct ovsrec_interface *iface) return xstrdup(peer_ifname); } +/* Per-candidate state for an LB-derived Advertised_Route that is eligible for + * Service_Monitor gating. */ +struct lb_route_gate { + struct hmap_node node; + const struct sbrec_advertised_route *route; + bool seen_monitor; + bool any_online; +}; + +static bool +route_has_health_checks(const struct sbrec_advertised_route *route) +{ + return smap_get(&route->external_ids, OVN_AR_HEALTH_CHECKS_KEY) != NULL; +} + +static bool +route_is_distributed_lb(const struct sbrec_advertised_route *route) +{ + const char *source = smap_get(&route->external_ids, OVN_AR_SOURCE_ID); + return source && !strcmp(source, "lb") && + smap_get_bool(&route->external_ids, + OVN_AR_DISTRIBUTED_LB_ID, false); +} + +static bool +route_advertising_port_is_local( + const struct sbrec_advertised_route *route, + struct ovsdb_idl_index *sbrec_port_binding_by_name, + const struct sbrec_chassis *chassis) +{ + return lport_is_local(sbrec_port_binding_by_name, chassis, + route->logical_port->logical_port); +} + +static bool +route_is_local_only_eligible( + const struct sbrec_advertised_route *route, + struct ovsdb_idl_index *sbrec_port_binding_by_name, + const struct sbrec_chassis *chassis, bool *tracked_port_local) +{ + if (!route->tracked_port) { + *tracked_port_local = false; + return true; + } + + *tracked_port_local = lport_is_local( + sbrec_port_binding_by_name, chassis, + route->tracked_port->logical_port); + return *tracked_port_local || + !smap_get_bool(&route->logical_port->options, + "dynamic-routing-redistribute-local-only", false); +} + +/* Build gates only for routes that northd annotated with health selectors. */ +static void +build_lb_route_gates(struct hmap *gates, + const struct sbrec_advertised_route_table *ar_table, + struct hmap *announce_routes, + struct ovsdb_idl_index *sbrec_port_binding_by_name, + const struct sbrec_chassis *chassis, + struct sset *health_check_ports) +{ + const struct sbrec_advertised_route *route; + SBREC_ADVERTISED_ROUTE_TABLE_FOR_EACH (route, ar_table) { + struct advertise_datapath_entry *ad = + advertise_datapath_find(announce_routes, route->datapath); + if (!ad) { + continue; + } + if (!route_is_distributed_lb(route)) { + continue; + } + if (!route->tracked_port || !route_has_health_checks(route)) { + continue; + } + if (!route_advertising_port_is_local( + route, sbrec_port_binding_by_name, chassis)) { + continue; + } + + bool tracked_port_local; + if (!route_is_local_only_eligible( + route, sbrec_port_binding_by_name, chassis, + &tracked_port_local)) { + continue; + } + + struct lb_route_gate *g = xmalloc(sizeof *g); + *g = (struct lb_route_gate) { + .route = route, + .seen_monitor = false, + .any_online = false, + }; + hmap_insert(gates, &g->node, hash_pointer(route, 0)); + sset_add(health_check_ports, route->tracked_port->logical_port); + } +} + +static void +destroy_lb_route_gates(struct hmap *gates) +{ + struct lb_route_gate *g; + HMAP_FOR_EACH_POP (g, node, gates) { + free(g); + } + hmap_destroy(gates); +} + +/* Resolve northd-provided protocol,backend-IP,backend-port selectors against + * Service_Monitor rows with the same logical port, port and protocol. IP is + * compared separately in binary form so equivalent IPv6 spellings match. */ +static void +evaluate_lb_route_gates(struct hmap *gates, + struct ovsdb_idl_index *service_monitor_by_selector, + struct uuidset *relevant_service_monitors) +{ + struct sbrec_service_monitor *filter = + sbrec_service_monitor_index_init_row(service_monitor_by_selector); + + struct lb_route_gate *g; + HMAP_FOR_EACH (g, node, gates) { + const char *checks = smap_get(&g->route->external_ids, + OVN_AR_HEALTH_CHECKS_KEY); + if (!checks) { + continue; + } + + char *buf = xstrdup(checks); + char *save_ptr = NULL; + char *selector; + for (selector = strtok_r(buf, ";", &save_ptr); + selector; selector = strtok_r(NULL, ";", &save_ptr)) { + + char *protocol = selector; + char *backend_ip = strchr(protocol, ','); + if (!backend_ip) { + continue; + } + *backend_ip++ = '\0'; + char *port_s = strchr(backend_ip, ','); + if (!port_s) { + continue; + } + *port_s++ = '\0'; + + unsigned int backend_port; + if (!str_to_uint(port_s, 10, &backend_port) || + backend_port > UINT16_MAX) { + continue; + } + + struct in6_addr backend_ip_addr; + if (!ip46_parse(backend_ip, &backend_ip_addr)) { + continue; + } + + const char *tracked_lp = g->route->tracked_port->logical_port; + sbrec_service_monitor_index_set_logical_port(filter, tracked_lp); + sbrec_service_monitor_index_set_type(filter, "load-balancer"); + sbrec_service_monitor_index_set_protocol(filter, protocol); + sbrec_service_monitor_index_set_port(filter, backend_port); + + const struct sbrec_service_monitor *monitor; + SBREC_SERVICE_MONITOR_FOR_EACH_EQUAL ( + monitor, filter, service_monitor_by_selector) { + struct in6_addr sm_ip; + if (!ip46_parse(monitor->ip, &sm_ip) || + !ipv6_addr_equals(&sm_ip, &backend_ip_addr)) { + continue; + } + + uuidset_insert(relevant_service_monitors, + &monitor->header_.uuid); + g->seen_monitor = true; + g->any_online |= monitor->status && + !strcmp(monitor->status, "online"); + } + + if (g->any_online) { + break; + } + } + free(buf); + } + + sbrec_service_monitor_index_destroy_row(filter); +} + +/* Look up the gate decision for a specific route. Returns: + * -1 if no gate exists (route is not LB-derived) + * 0 if gate says withdraw (seen_monitor && !any_online) + * 1 if gate says install */ +static int +lb_route_gate_decision(const struct hmap *gates, + const struct sbrec_advertised_route *route) +{ + const struct lb_route_gate *g; + HMAP_FOR_EACH_WITH_HASH (g, node, hash_pointer(route, 0), gates) { + if (g->route == route) { + if (g->seen_monitor && !g->any_online) { + return 0; + } + return 1; + } + } + return -1; +} + static bool route_exchange_relevant_port(const struct sbrec_port_binding *pb) { @@ -450,6 +662,22 @@ route_run(struct route_ctx_in *r_ctx_in, } } + struct hmap lb_route_gates = HMAP_INITIALIZER(&lb_route_gates); + if (!hmap_is_empty(r_ctx_out->announce_routes)) { + build_lb_route_gates(&lb_route_gates, + r_ctx_in->advertised_route_table, + r_ctx_out->announce_routes, + r_ctx_in->sbrec_port_binding_by_name, + r_ctx_in->chassis, + r_ctx_out->health_check_ports); + } + + if (!hmap_is_empty(&lb_route_gates)) { + evaluate_lb_route_gates(&lb_route_gates, + r_ctx_in->service_monitor_by_selector, + r_ctx_out->relevant_service_monitors); + } + const struct sbrec_advertised_route *route; SBREC_ADVERTISED_ROUTE_TABLE_FOR_EACH (route, r_ctx_in->advertised_route_table) { @@ -470,9 +698,9 @@ route_run(struct route_ctx_in *r_ctx_in, continue; } - if (!lport_is_local(r_ctx_in->sbrec_port_binding_by_name, - r_ctx_in->chassis, - route->logical_port->logical_port)) { + if (!route_advertising_port_is_local( + route, r_ctx_in->sbrec_port_binding_by_name, + r_ctx_in->chassis)) { sset_add(r_ctx_out->tracked_ports_remote, route->logical_port->logical_port); continue; @@ -480,27 +708,40 @@ route_run(struct route_ctx_in *r_ctx_in, sset_add(r_ctx_out->tracked_ports_local, route->logical_port->logical_port); + bool distributed_lb = route_is_distributed_lb(route); + unsigned int priority = PRIORITY_DEFAULT; if (route->tracked_port) { - bool redistribute_local_bound_only = - smap_get_bool(&route->logical_port->options, - "dynamic-routing-redistribute-local-only", - false); - if (lport_is_local(r_ctx_in->sbrec_port_binding_by_name, - r_ctx_in->chassis, - route->tracked_port->logical_port)) { + bool tracked_port_local; + bool local_only_eligible = route_is_local_only_eligible( + route, r_ctx_in->sbrec_port_binding_by_name, + r_ctx_in->chassis, &tracked_port_local); + if (tracked_port_local) { priority = PRIORITY_LOCAL_BOUND; sset_add(r_ctx_out->tracked_ports_local, route->tracked_port->logical_port); } else { sset_add(r_ctx_out->tracked_ports_remote, route->tracked_port->logical_port); - if (redistribute_local_bound_only) { - /* We're not advertising routes whose 'tracked_port' is - * not local, skip this route. */ - continue; - } } + + /* A route not selected for this chassis must not get a + * status row: evaluate local-only eligibility before any + * administrative or health withdrawal so the documented + * "selected for its chassis" meaning holds. */ + if (!local_only_eligible) { + continue; + } + } + + if (distributed_lb && + !smap_get_bool(&route->external_ids, "enabled", true)) { + continue; + } + + int gate = lb_route_gate_decision(&lb_route_gates, route); + if (gate == 0) { + continue; } struct in6_addr nexthop = IN6_IS_ADDR_V4MAPPED(&prefix) @@ -521,6 +762,8 @@ route_run(struct route_ctx_in *r_ctx_in, advertise_route_hash(&ar->addr, &ar->nexthop, plen)); } + destroy_lb_route_gates(&lb_route_gates); + smap_destroy(&port_mapping); } diff --git a/controller/route.h b/controller/route.h index f1d03a9e5..5ddc94456 100644 --- a/controller/route.h +++ b/controller/route.h @@ -22,18 +22,22 @@ #include <netinet/in.h> #include <net/if.h> #include "openvswitch/hmap.h" +#include "vec.h" #include "sset.h" #include "smap.h" struct hmap; struct ovsdb_idl_index; +struct uuidset; struct route_data; struct sbrec_chassis; struct sbrec_port_binding; struct sbrec_datapath_binding; +struct sbrec_advertised_route; struct route_ctx_in { const struct sbrec_advertised_route_table *advertised_route_table; + struct ovsdb_idl_index *service_monitor_by_selector; struct ovsdb_idl_index *sbrec_port_binding_by_name; const struct sbrec_chassis *chassis; const char *dynamic_routing_port_mapping; @@ -51,6 +55,12 @@ struct route_ctx_out { * locally. */ struct sset *tracked_ports_remote; + /* Contains logical ports referenced by health-gated advertised routes. */ + struct sset *health_check_ports; + + /* Contains Service_Monitor UUIDs matched during the last route run. */ + struct uuidset *relevant_service_monitors; + /* Contains all the currently configured dynamic-routing-port-name values * on all datapaths. */ @@ -71,6 +81,12 @@ struct advertise_datapath_entry { struct hmap routes; + /* Result of the most recent attempt to reconcile 'routes' with the + * chassis routing table. */ + bool routes_synced; + int route_error; + const char *route_error_description; + /* The name of the port bindings locally bound for this datapath and * running route exchange logic. * The key is the port name and the value is the ifname if set. This may diff --git a/lib/ovn-util.h b/lib/ovn-util.h index 4209ab07f..3ffcf42c3 100644 --- a/lib/ovn-util.h +++ b/lib/ovn-util.h @@ -38,6 +38,10 @@ #define OVN_AR_SOURCE_ID "source" #define OVN_AR_DISTRIBUTED_LB_ID "distributed-lb" +/* Semicolon-delimited "protocol,IP,port" tuples stored in + * Advertised_Route.external_ids so the controller can look up + * matching Service_Monitor rows. */ +#define OVN_AR_HEALTH_CHECKS_KEY "health-checks" #define ETH_CRC_LENGTH 4 #define ETHERNET_OVERHEAD (ETH_HEADER_LEN + ETH_CRC_LENGTH) diff --git a/northd/en-advertised-route-sync.c b/northd/en-advertised-route-sync.c index ed8f61e3c..eb8a24a39 100644 --- a/northd/en-advertised-route-sync.c +++ b/northd/en-advertised-route-sync.c @@ -41,7 +41,15 @@ struct ar_entry { * advertises this route with a * higher priority. */ enum route_source source; + /* A shared route with a non-LB contributor must remain unconditional. */ + bool has_non_lb_contributor; bool distributed_lb; + /* True when at least one contributor to this (VIP IP, backend LSP) + * route is a backend without a health check. If set, the controller + * must NOT gate the route on Service_Monitor health, because the + * unmonitored listener must remain reachable regardless. */ + bool has_ungated_lb; + struct sset health_checks; }; /* Add a new entries to the to-be-advertised routes. @@ -59,6 +67,8 @@ ar_entry_add_nocopy(struct hmap *routes, const struct ovn_datapath *od, route_e->ip_prefix = ip_prefix; route_e->tracked_port = tracked_port; route_e->source = source; + route_e->has_non_lb_contributor = source != ROUTE_SOURCE_LB; + sset_init(&route_e->health_checks); uint32_t hash = uuid_hash(&od->sdp->sb_dp->header_.uuid); hash = hash_string(op->sb->logical_port, hash); hash = hash_string(ip_prefix, hash); @@ -131,13 +141,21 @@ static void ar_entry_free(struct ar_entry *route_e) { free(route_e->ip_prefix); + sset_destroy(&route_e->health_checks); free(route_e); } static void ar_entry_merge_metadata(struct ar_entry *dst, const struct ar_entry *src) { + dst->has_non_lb_contributor |= src->has_non_lb_contributor; dst->distributed_lb |= src->distributed_lb; + dst->has_ungated_lb |= src->has_ungated_lb; + + const char *check; + SSET_FOR_EACH (check, &src->health_checks) { + sset_add(&dst->health_checks, check); + } } static void @@ -147,18 +165,36 @@ ar_entry_sync_external_ids(const struct sbrec_advertised_route *sb_route, struct smap ids; smap_clone(&ids, &sb_route->external_ids); - if (route_e->source == ROUTE_SOURCE_LB) { + bool lb_owned = route_e->source == ROUTE_SOURCE_LB && + !route_e->has_non_lb_contributor; + if (lb_owned) { smap_replace(&ids, OVN_AR_SOURCE_ID, "lb"); } else { smap_remove(&ids, OVN_AR_SOURCE_ID); } - if (route_e->distributed_lb) { + /* A NAT or another non-LB contributor can share the complete SB route + * key with an LB. In that case the shared kernel route must remain + * unconditional, so do not publish LB-only metadata on the row. */ + if (lb_owned && route_e->distributed_lb) { smap_replace(&ids, OVN_AR_DISTRIBUTED_LB_ID, "true"); } else { smap_remove(&ids, OVN_AR_DISTRIBUTED_LB_ID); } + /* Selectors are packed into one semicolon-delimited value to avoid + * a schema change for data consumed only by the controller. + * Suppress health-checks when an ungated listener exists: the + * kernel route is per-prefix, so an unmonitored listener sharing + * the (VIP IP, backend LSP) must remain reachable regardless of + * other listeners' Service_Monitor state. */ + smap_remove(&ids, OVN_AR_HEALTH_CHECKS_KEY); + if (lb_owned && !route_e->has_ungated_lb && + !sset_is_empty(&route_e->health_checks)) { + smap_add_nocopy(&ids, xstrdup(OVN_AR_HEALTH_CHECKS_KEY), + sset_join(&route_e->health_checks, ";", "")); + } + if (!smap_equal(&ids, &sb_route->external_ids)) { sbrec_advertised_route_set_external_ids(sb_route, &ids); } @@ -359,13 +395,16 @@ build_nat_route_for_port(const struct ovn_port *advertising_op, ? ovn_port_find(ls_ports, nat->nb->logical_port) : nat->l3dgw_port; - if (!ar_entry_find(routes, advertising_od->sdp->sb_dp, - advertising_op->sb, - nat->nb->external_ip, - tracked_port ? tracked_port->sb : NULL)) { + struct ar_entry *route_e = ar_entry_find( + routes, advertising_od->sdp->sb_dp, advertising_op->sb, + nat->nb->external_ip, + tracked_port ? tracked_port->sb : NULL); + if (!route_e) { ar_entry_add(routes, advertising_od, advertising_op, nat->nb->external_ip, tracked_port, ROUTE_SOURCE_NAT); + } else { + route_e->has_non_lb_contributor = true; } if (forwarding_port && parsed_routes_out) { @@ -509,6 +548,8 @@ build_lb_routes_for_lb(const struct ovn_port *advertising_op, } const struct ovn_northd_lb *lb = lb_dps->lb; + const char *protocol = lb->nlb->protocol && lb->nlb->protocol[0] + ? lb->nlb->protocol : "tcp"; for (size_t v = 0; v < lb->n_vips; v++) { const struct ovn_lb_vip *vip = &lb->vips[v]; const struct ovn_northd_lb_vip *vip_nb = &lb->vips_nb[v]; @@ -534,7 +575,11 @@ build_lb_routes_for_lb(const struct ovn_port *advertising_op, bool emitted_any = false; for (size_t b = 0; b < vip_nb->n_backends; b++) { - const char *lsp_name = vip_nb->backends_nb[b].logical_port; + const struct ovn_northd_lb_backend *backend_nb = + &vip_nb->backends_nb[b]; + const struct ovn_lb_backend *backend = + vector_get_ptr(&vip->backends, b); + const char *lsp_name = backend_nb->logical_port; if (!lsp_name) { continue; } @@ -555,6 +600,14 @@ build_lb_routes_for_lb(const struct ovn_port *advertising_op, backend_op, ROUTE_SOURCE_LB); } route_e->distributed_lb = true; + if (backend_nb->health_check) { + sset_add_and_free( + &route_e->health_checks, + xasprintf("%s,%s,%"PRIu16, protocol, backend->ip_str, + backend->port)); + } else { + route_e->has_ungated_lb = true; + } emitted_any = true; } if (!emitted_any) { @@ -667,8 +720,8 @@ build_lb_connected_routes(const struct ovn_datapath *od, /* Generate routes for LB VIPs owned by the advertising LR itself. * Uses build_lb_lr_routes() so that distributed LBs emit per-backend - * Advertised_Route rows with backend LSPs as tracked ports, the same as - * for neighbor-owned LBs. Forwarding parsed_routes are not needed here + * Advertised_Route rows with health selectors, the same as for + * neighbor-owned LBs. Forwarding parsed_routes are not needed here * because the advertising LR's own pipeline handles ingress. */ static void build_lb_routes(const struct ovn_datapath *od, diff --git a/ovn-sb.xml b/ovn-sb.xml index 4ad2c0b89..037954800 100644 --- a/ovn-sb.xml +++ b/ovn-sb.xml @@ -5491,22 +5491,53 @@ tcp.flags = RST; See <em>External IDs</em> at the beginning of this document. <p> - <code>ovn-northd</code> sets the following key on routes - derived from Load_Balancer VIPs: + The following keys control distributed Load_Balancer routes: </p> <dl> <dt><code>source</code></dt> <dd> - Set to <code>lb</code> by <code>ovn-northd</code> on routes - derived from Load_Balancer VIPs. Absent on all other routes. - The key identifies the route's origin for consumers that - distinguish Load_Balancer-derived routes. + <code>ovn-northd</code> sets this to <code>lb</code> on routes + derived from Load_Balancer VIPs. It is absent on other routes. + The controller applies LB-specific handling when this key and + <code>distributed-lb=true</code> are both present. </dd> + + <dt><code>distributed-lb</code></dt> + <dd> + Set to <code>true</code> by <code>ovn-northd</code> when the route + belongs to a Load Balancer whose + <ref table="Load_Balancer" column="options" + key="distributed" db="OVN_Northbound"/> option is enabled. + The health gate, administrative enable flag and status table apply + only to routes carrying this key. + </dd> + + <dt><code>health-checks</code></dt> + <dd> + For a health-checked distributed Load_Balancer route, + <code>ovn-northd</code> records each relevant backend as + <code>protocol,IP,port</code>. When more than one backend + is relevant the tuples are separated by semicolons. + <code>ovn-controller</code> looks up matching + <ref table="Service_Monitor"/> rows. When a shared + route (same VIP IP and backend LSP) has at least one + contributor without a health check, the key is absent so + the controller does not gate the route. + </dd> + + <dt><code>enabled</code></dt> + <dd> + Administrative enable flag for routes carrying + <code>distributed-lb=true</code>. A value of <code>false</code> + withdraws the route regardless of Service_Monitor health. An + absent key is treated as enabled. A privileged administrative + writer can use this key to withdraw a route. + </dd> + </dl> </column> </table> - <table name="Learned_Route"> <p> Each record represents a route that learned by ovn using some dynamic diff --git a/tests/ovn-inc-proc-graph-dump.at b/tests/ovn-inc-proc-graph-dump.at index 8efb994e1..81f8c151f 100644 --- a/tests/ovn-inc-proc-graph-dump.at +++ b/tests/ovn-inc-proc-graph-dump.at @@ -453,6 +453,7 @@ digraph "Incremental-Processing-Engine" { SB_chassis -> bfd_chassis [[label=""]]; SB_ha_chassis_group -> bfd_chassis [[label=""]]; SB_advertised_route [[style=filled, shape=box, fillcolor=white, label="SB_advertised_route"]]; + SB_service_monitor [[style=filled, shape=box, fillcolor=white, label="SB_service_monitor"]]; route [[style=filled, shape=box, fillcolor=white, label="route"]]; OVS_open_vswitch -> route [[label=""]]; SB_chassis -> route [[label=""]]; @@ -460,6 +461,7 @@ digraph "Incremental-Processing-Engine" { runtime_data -> route [[label="route_runtime_data_handler"]]; SB_advertised_route -> route [[label="route_sb_advertised_route_data_handler"]]; SB_datapath_binding -> route [[label="route_sb_datapath_binding_handler"]]; + SB_service_monitor -> route [[label="route_sb_service_monitor_handler"]]; SB_learned_route [[style=filled, shape=box, fillcolor=white, label="SB_learned_route"]]; route_table_notify [[style=filled, shape=box, fillcolor=white, label="route_table_notify"]]; route_exchange_status [[style=filled, shape=box, fillcolor=white, label="route_exchange_status"]]; diff --git a/tests/ovn-northd.at b/tests/ovn-northd.at index f5c52f8b8..8a7707362 100644 --- a/tests/ovn-northd.at +++ b/tests/ovn-northd.at @@ -18648,7 +18648,8 @@ ovn_start # row using the peer LR's gateway LRP. That moves the chassis-locality # decision from a chassis-unbound patch port (the peer LRP) to the # actual backend port - the controller can then per-chassis install -# the kernel route via dynamic-routing-redistribute-local-only=true. +# the kernel route via dynamic-routing-redistribute-local-only=true, +# and gate it on Service_Monitor.status. check ovn-nbctl lr-add lr0 check ovn-nbctl set Logical_Router lr0 \ @@ -18768,7 +18769,7 @@ ovn_start # A distributed LB attached directly to the advertising LR must use # the per-backend path (build_lb_routes_for_lb), not the aggregated # lb_ips path, so that Advertised_Route rows carry distributed-lb=true, -# and the backend LSP as tracked_port. +# health selectors, and the backend LSP as tracked_port. check ovn-nbctl lr-add lr0 check ovn-nbctl set Logical_Router lr0 \ @@ -18819,6 +18820,78 @@ OVN_CLEANUP_NORTHD AT_CLEANUP ]) +OVN_FOR_EACH_NORTHD_NO_HV([ +AT_SETUP([dynamic-routing - ungated listener suppresses health-checks]) +AT_KEYWORDS([dynamic-routing]) +ovn_start + +# When two LB listeners share the same VIP IP and backend LSP, and one +# has a health check while the other does not, the shared +# Advertised_Route must NOT carry health-checks. Otherwise the +# controller would withdraw the route when the monitored listener's +# Service_Monitor is offline, making the unmonitored listener +# unreachable too. + +check ovn-nbctl lr-add lr0 +check ovn-nbctl set Logical_Router lr0 \ + options:dynamic-routing=true \ + options:chassis=hv1 +check ovn-nbctl lrp-add lr0 lr0-up 00:00:00:00:00:01 +check ovn-nbctl lrp-set-options lr0-up dynamic-routing-redistribute=lb +check ovn-nbctl ls-add up +check ovn-nbctl lsp-add-router-port up up-lr0 lr0-up + +check ovn-nbctl lr-add lr1 +check ovn-nbctl lrp-add lr1 lr1-up 00:00:00:00:00:02 10.0.0.1/24 +check ovn-nbctl lsp-add-router-port up up-lr1 lr1-up + +check ovn-nbctl lrp-add lr1 lr1-be 00:00:00:00:00:03 192.168.1.1/24 +check ovn-nbctl ls-add be +check ovn-nbctl lsp-add-router-port be be-lr1 lr1-be +check ovn-nbctl lsp-add be be-vm1 +check ovn-nbctl lsp-set-addresses be-vm1 "00:00:00:00:01:01 192.168.1.10" + +# lb0: VIP:80 with health check. +check ovn-nbctl \ + -- lb-add lb0 172.16.1.10:80 192.168.1.10:80 \ + -- set Load_Balancer lb0 options:distributed=true \ + ip_port_mappings:192.168.1.10="be-vm1" +check ovn-nbctl --wait=sb lr-lb-add lr1 lb0 +check_uuid ovn-nbctl --wait=sb \ + -- --id=@hc create Load_Balancer_Health_Check vip="172.16.1.10\:80" \ + -- add Load_Balancer lb0 health_check @hc + +# lb1: same VIP IP, port :443, no health check. +check ovn-nbctl \ + -- lb-add lb1 172.16.1.10:443 192.168.1.10:443 \ + -- set Load_Balancer lb1 options:distributed=true \ + ip_port_mappings:192.168.1.10="be-vm1" +check ovn-nbctl --wait=sb lr-lb-add lr1 lb1 + +datapath_lr0=$(fetch_column Datapath_Binding _uuid external_ids:name=lr0) +pb_lr0_up=$(fetch_column Port_Binding _uuid logical_port=lr0-up) +pb_be_vm1=$(fetch_column Port_Binding _uuid logical_port=be-vm1) + +# Both listeners map to one row (same VIP IP, same backend LSP). +check_row_count Advertised_Route 1 \ + ip_prefix="172.16.1.10" \ + datapath=$datapath_lr0 \ + logical_port=$pb_lr0_up \ + tracked_port=$pb_be_vm1 + +# The row has distributed-lb=true but NO health-checks, because the +# unmonitored :443 listener is an ungated listener. +AT_CHECK([ovn-sbctl --bare --columns=external_ids \ + find Advertised_Route ip_prefix=172.16.1.10 | + grep -q 'distributed-lb.*true']) +AT_CHECK([! ovn-sbctl --bare --columns=external_ids \ + find Advertised_Route ip_prefix=172.16.1.10 | + grep -q 'health-checks']) + +OVN_CLEANUP_NORTHD +AT_CLEANUP +]) + OVN_FOR_EACH_NORTHD_NO_HV([ AT_SETUP([dynamic-routing - LB redistribute SCTP per-backend route emission]) AT_KEYWORDS([dynamic-routing]) @@ -19152,7 +19225,8 @@ AT_KEYWORDS([dynamic-routing]) ovn_start # Regression test: routes from NAT redistribution must not carry -# external_ids:source=lb. +# external_ids:source=lb, so the controller never applies Service_Monitor +# gating to non-LB routes. check ovn-nbctl lr-add lr0 check ovn-nbctl set Logical_Router lr0 \ diff --git a/tests/system-ovn.at b/tests/system-ovn.at index 06372dc9b..b28f4c991 100644 --- a/tests/system-ovn.at +++ b/tests/system-ovn.at @@ -21179,7 +21179,6 @@ OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d AT_CLEANUP ]) -]) OVN_FOR_EACH_NORTHD([ AT_SETUP([dynamic-routing - Advertised route nexthop]) @@ -22518,6 +22517,8 @@ ADD_BR([br-ex]) check ovs-ofctl add-flow br-ex action=normal +# Set external-ids in br-int needed for ovn-controller + # Set external-ids in br-int needed for ovn-controller check ovs-vsctl \ -- set Open_vSwitch . external-ids:system-id=hv1 \ @@ -22620,10 +22621,925 @@ as OVS_TRAFFIC_VSWITCHD_STOP(["/.*error receiving.*/d /failed to query port patch-.*/d /.*terminating with signal 15.*/d"]) +AT_CLEANUP +]) + +AT_SETUP([dynamic-routing - LB redistribute gated by Service_Monitor.status]) +AT_KEYWORDS([dynamic-routing]) + +VRF_RESERVE([1339]) + +# Distributed-LB routes carry backend selectors joined to Service_Monitor by +# logical port, type, IP, port and protocol. The selector omits chassis_name, +# so it also gates remote backends. A route without a matching monitor is +# installed. +# +# Topology: two chassis-bound LRs (lr-origin, lr-target) share a +# common LS (ls-share). lr-origin has redistribute=lb on its LRP into +# ls-share, so northd emits one Advertised_Route row per backend LSP +# on lr-origin's datapath with tracked_port = the backend LSP. +# lr-target owns the LB. The backend LSP (be0) is on ls-share, bound +# to hv1. + +ovn_start +OVS_TRAFFIC_VSWITCHD_START() + +ADD_BR([br-int]) +ovn_remote=unix:$ovs_base/ovn-sb/ovn-sb.sock +if test -n "$SSL_OVN_SB_DB"; then + check ovs-vsctl set-ssl \ + $PKIDIR/testpki-hv1-privkey.pem \ + $PKIDIR/testpki-hv1-cert.pem \ + $PKIDIR/testpki-cacert.pem + ovn_remote=$SSL_OVN_SB_DB +fi +check ovs-vsctl \ + -- set Open_vSwitch . external-ids:system-id=hv1 \ + -- set Open_vSwitch . external-ids:ovn-remote=$ovn_remote \ + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true + +start_daemon ovn-controller + +# Shared LS holding both LR LRPs and the backend LSP. +check ovn-nbctl ls-add ls-share + +# lr-origin: GW, vrf 1339, has the redistribute LRP into ls-share. +check ovn-nbctl lr-add lr-origin \ + -- set Logical_Router lr-origin options:chassis=hv1 \ + options:dynamic-routing=true \ + options:dynamic-routing-vrf-id=1339 +check ovn-nbctl lrp-add lr-origin lr-origin-share 00:de:ad:00:00:01 \ + 192.168.0.1/24 fd00::1/64 \ + -- set Logical_Router_Port lr-origin-share \ + options:dynamic-routing-redistribute="lb" \ + options:dynamic-routing-maintain-vrf=true +check ovn-nbctl lsp-add ls-share share-lr-origin \ + -- set Logical_Switch_Port share-lr-origin type=router \ + options:router-port=lr-origin-share \ + -- lsp-set-addresses share-lr-origin router + +# lr-target: GW, owns the LB. Also attached to ls-share so lr-origin +# can walk LR-LS-LR to discover the LB. +check ovn-nbctl lr-add lr-target \ + -- set Logical_Router lr-target options:chassis=hv1 +check ovn-nbctl lrp-add lr-target lr-target-share 00:de:ad:00:00:02 \ + 192.168.0.2/24 fd00::2/64 +check ovn-nbctl lsp-add ls-share share-lr-target \ + -- set Logical_Switch_Port share-lr-target type=router \ + options:router-port=lr-target-share \ + -- lsp-set-addresses share-lr-target router + +# Backend LSP on the same LS. Veth-backed so it's claimed locally. +check ovn-nbctl lsp-add ls-share be0 +check ovn-nbctl lsp-set-addresses be0 "00:de:ad:00:00:10 192.168.0.10 fd00::10" +ADD_NAMESPACES(be0_ns) +ADD_VETH(be0, be0_ns, br-int, "192.168.0.10/24", "00:de:ad:00:00:10") +check ip netns exec be0_ns ip addr add fd00::10/64 dev be0 + +# ip_port_mappings associates the backend with be0. The health check makes +# northd create a Service_Monitor row. ovn-controller updates its status. +check ovn-nbctl \ + -- lb-add lb0 172.16.1.10:80 192.168.0.10:80 \ + -- set Load_Balancer lb0 options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb0 +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.10\:80" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb0 health_check @hc + +check ovn-nbctl --wait=hv sync +wait_for_ports_up +OVS_CTL_TIMEOUT=30 + +# With no listener, the monitor is offline and the route is absent. +OVS_WAIT_UNTIL([test -n "`ip vrf show ovnvrf1339 2>/dev/null`"]) +wait_row_count Service_Monitor 1 logical_port=be0 +wait_row_count Service_Monitor 1 logical_port=be0 status=offline +wait_row_count sb:Advertised_Route 1 ip_prefix='"172.16.1.10"' + +# Record the route UUID for the administrative override checks below. +lb_ar=$(ovn-sbctl --bare --columns=_uuid find Advertised_Route \ + ip_prefix=172.16.1.10) +test -n "$lb_ar" + +# The offline monitor withdraws the route. +AT_CHECK([ + ip route list vrf ovnvrf1339 | grep -c "blackhole 172.16.1.10" || true +], [0], [0 +]) + +# A centralized LB has one VIP-level route. Its offline monitor does not +# gate the route. +check ovn-nbctl --wait=hv set Load_Balancer lb0 options:distributed=false +central_ar=$(ovn-sbctl --bare --columns=_uuid find Advertised_Route \ + ip_prefix=172.16.1.10) +test -n "$central_ar" +AT_CHECK([! ovn-sbctl --bare get Advertised_Route $central_ar external_ids | + grep -q 'distributed-lb']) +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) + +# The administrative key does not affect a centralized route. +check ovn-sbctl set Advertised_Route $central_ar external_ids:enabled=false +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) +check ovn-sbctl remove Advertised_Route $central_ar external_ids enabled + +# Restore the distributed route. +check ovn-nbctl --wait=hv set Load_Balancer lb0 options:distributed=true +lb_ar=$(ovn-sbctl --bare --columns=_uuid find Advertised_Route \ + ip_prefix=172.16.1.10) +test -n "$lb_ar" +AT_CHECK([ovn-sbctl --bare get Advertised_Route $lb_ar external_ids | + grep -q 'distributed-lb.*true']) +OVS_WAIT_UNTIL([ + ! ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) + +# Bind the offline backend to hv2. The advertising router stays on hv1 and +# local-only remains false, so the monitor still withdraws the route. +check ovn-appctl inc-engine/clear-stats +check ovs-vsctl del-port br-int ovs-be0 +AT_CHECK([ip netns exec be0_ns ip link del be0], [ignore], [ignore], [ignore]) +wait_column "" Port_Binding chassis logical_port=be0 +OVS_WAIT_UNTIL([ + route_recompute_ct=$(ovn-appctl inc-engine/show-stats route recompute) + test "$route_recompute_ct" -ne 0 +]) +check ovn-sbctl chassis-add hv2 geneve 169.0.0.2 +hv2=$(fetch_column Chassis _uuid name=hv2) +check ovn-sbctl set Port_Binding be0 chassis=$hv2 +wait_row_count Service_Monitor 1 logical_port=be0 status=offline +AT_CHECK([ + ip route list vrf ovnvrf1339 | grep -c "blackhole 172.16.1.10" || true +], [0], [0 +]) + +# Exclude remote backends from local-only advertisement before registering +# their health selectors. A health update for the remote backend must not +# recompute hv1's route node. +check ovn-nbctl --wait=hv set Logical_Router_Port lr-origin-share \ + options:dynamic-routing-redistribute-local-only=true + +sm_v4=$(ovn-sbctl --bare --columns=_uuid find Service_Monitor \ + logical_port=be0 ip='"192.168.0.10"' port=80 protocol=tcp) +test -n "$sm_v4" +check as northd ovn-appctl -t ovn-northd pause +check ovn-appctl inc-engine/clear-stats +check ovn-sbctl set Service_Monitor $sm_v4 status=online +OVS_WAIT_UNTIL([ + route_compute_ct=$(ovn-appctl inc-engine/show-stats route compute) + test "$route_compute_ct" -ne 0 +]) +route_recompute_ct=$(ovn-appctl inc-engine/show-stats route recompute) +check test "$route_recompute_ct" -eq 0 +check as northd ovn-appctl -t ovn-northd resume + +# Return the backend to hv1. +check ovn-sbctl clear Port_Binding be0 chassis +check ovn-sbctl destroy Chassis $hv2 +ADD_VETH(be0, be0_ns, br-int, "192.168.0.10/24", "00:de:ad:00:00:10") +check ip netns exec be0_ns ip addr add fd00::10/64 dev be0 +hv1_uuid=$(fetch_column Chassis _uuid name=hv1) +wait_row_count Port_Binding 1 logical_port=be0 chassis=$hv1_uuid +wait_row_count Service_Monitor 1 logical_port=be0 chassis_name=hv1 \ + status=offline + +# Start the backend directly and use Service_Monitor.status for readiness. +be0_pid_file=$(mktemp be0_http.XXX.pid) +check ovn-appctl inc-engine/clear-stats +NETNS_DAEMONIZE([be0_ns], + [[$PYTHON $srcdir/test-l7.py http]], [$be0_pid_file]) +wait_row_count Service_Monitor 1 logical_port=be0 status=online +OVS_WAIT_UNTIL([ + route_recompute_ct=$(ovn-appctl inc-engine/show-stats route recompute) + test "$route_recompute_ct" -ne 0 +]) +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) + +# Stop the backend and wait for route withdrawal. +kill `cat $be0_pid_file` +wait_row_count Service_Monitor 1 logical_port=be0 status=offline +OVS_WAIT_UNTIL([ + ! ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) + +# A monitor that moves away from a selector must stop gating the route. A +# monitor that moves into the selector must start gating it. Deleting and +# recreating the row exercises both sides with different UUIDs. +sm_v4=$(ovn-sbctl --bare --columns=_uuid find Service_Monitor \ + logical_port=be0 ip='"192.168.0.10"' port=80 protocol=tcp) +test -n "$sm_v4" +check as northd ovn-appctl -t ovn-northd pause + +check ovn-appctl inc-engine/clear-stats +check ovn-sbctl set Service_Monitor $sm_v4 logical_port=moved-be0 +OVS_WAIT_UNTIL([ + route_recompute_ct=$(ovn-appctl inc-engine/show-stats route recompute) + test "$route_recompute_ct" -ne 0 +]) +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) + +check ovn-appctl inc-engine/clear-stats +check ovn-sbctl set Service_Monitor $sm_v4 logical_port=be0 +OVS_WAIT_UNTIL([ + route_recompute_ct=$(ovn-appctl inc-engine/show-stats route recompute) + test "$route_recompute_ct" -ne 0 +]) +OVS_WAIT_UNTIL([ + ! ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) + +check ovn-appctl inc-engine/clear-stats +check ovn-sbctl destroy Service_Monitor $sm_v4 +OVS_WAIT_UNTIL([ + route_recompute_ct=$(ovn-appctl inc-engine/show-stats route recompute) + test "$route_recompute_ct" -ne 0 +]) +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) + +check ovn-appctl inc-engine/clear-stats +sm_v4=$(ovn-sbctl create Service_Monitor type=load-balancer \ + logical_port=be0 ip='"192.168.0.10"' port=80 protocol=tcp \ + status=offline chassis_name=hv1) +test -n "$sm_v4" +OVS_WAIT_UNTIL([ + route_recompute_ct=$(ovn-appctl inc-engine/show-stats route recompute) + test "$route_recompute_ct" -ne 0 +]) +OVS_WAIT_UNTIL([ + ! ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) +check as northd ovn-appctl -t ovn-northd resume + +# An online monitor in the same logical-port, type, port and protocol index +# bucket must not affect the route when its IP does not match the selector. +check as northd ovn-appctl -t ovn-northd pause +check ovn-appctl inc-engine/clear-stats +unrelated_sm=$(ovn-sbctl create Service_Monitor type=load-balancer \ + logical_port=be0 ip='"192.168.0.99"' port=80 protocol=tcp status=online \ + chassis_name=other) +test -n "$unrelated_sm" +OVS_WAIT_UNTIL([ + route_recompute_ct=$(ovn-appctl inc-engine/show-stats route recompute) + test "$route_recompute_ct" -ne 0 +]) +OVS_WAIT_UNTIL([ + ! ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) +check ovn-sbctl destroy Service_Monitor $unrelated_sm +check as northd ovn-appctl -t ovn-northd resume + +# IPv6 selector normalization: add an IPv6 VIP and verify the controller +# matches selector and Service_Monitor IPs using binary in6_addr +# comparison rather than string comparison. +check ovn-nbctl lb-add lb0 [[fd00::200]]:80 [[fd00::10]]:80 +check ovn-nbctl set load_balancer lb0 \ + ip_port_mappings:\"[[fd00::10]]\"=\"be0:[[fd00::2]]\" +check_uuid ovn-nbctl --id=@hc6 create Load_Balancer_Health_Check \ + vip="\[\[fd00\:\:200\]\]\:80" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb0 health_check @hc6 +check ovn-nbctl --wait=hv sync + +# An offline Service_Monitor is created for the IPv6 backend. +wait_row_count Service_Monitor 1 logical_port=be0 ip='"fd00::10"' \ + status=offline +sm_v6=$(ovn-sbctl --bare --columns=_uuid find Service_Monitor \ + logical_port=be0 ip='"fd00::10"') +test -n "$sm_v6" + +# The IPv6 route is withdrawn because the monitor is offline. +lb_ar_v6=$(ovn-sbctl --bare --columns=_uuid find Advertised_Route \ + ip_prefix='"fd00::200"') +test -n "$lb_ar_v6" +AT_CHECK([ + ip -6 route list vrf ovnvrf1339 | grep -c "blackhole fd00::200" || true +], [0], [0 +]) + +# Pause northd so it cannot overwrite the monitor ip, then rewrite the +# ip to its expanded textual form. A forced recompute makes the +# controller re-read the monitor and re-evaluate the selector. +# ip46_parse normalizes both "fd00::10" and +# "fd00:0:0:0:0:0:0:10" to the same in6_addr, so the controller must +# still match the selector and keep the route withdrawn. +check as northd ovn-appctl -t ovn-northd pause +check ovn-appctl inc-engine/clear-stats +check ovn-sbctl set Service_Monitor $sm_v6 ip='"fd00:0:0:0:0:0:0:10"' +check ovn-appctl inc-engine/recompute +OVS_WAIT_UNTIL([ + route_recompute_ct=$(ovn-appctl inc-engine/show-stats route recompute) + test "$route_recompute_ct" -ne 0 +]) +# With northd paused the monitor still holds the expanded form. +wait_row_count Service_Monitor 1 logical_port=be0 \ + ip='"fd00:0:0:0:0:0:0:10"' status=offline +# The controller matched the normalized in6_addr against the selector +# and kept the route withdrawn. +AT_CHECK([ + ip -6 route list vrf ovnvrf1339 | grep -c "blackhole fd00::200" || true +], [0], [0 +]) +# Resume northd. It repairs the monitor ip back to compressed form. +check as northd ovn-appctl -t ovn-northd resume +wait_row_count Service_Monitor 1 logical_port=be0 ip='"fd00::10"' \ + status=offline + +# Removing the health check removes the monitor and makes the route +# unconditional. +check ovn-nbctl clear Load_Balancer lb0 health_check +wait_row_count Service_Monitor 0 logical_port=be0 +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) + +# A monitor on a tracked port without a health-gated route does not require a +# route recompute. +check as northd ovn-appctl -t ovn-northd pause +check ovn-appctl inc-engine/clear-stats +ungated_sm=$(ovn-sbctl create Service_Monitor type=load-balancer \ + logical_port=be0 ip='"192.168.0.10"' port=81 protocol=tcp status=offline) +test -n "$ungated_sm" +OVS_WAIT_UNTIL([ + route_compute_ct=$(ovn-appctl inc-engine/show-stats route compute) + test "$route_compute_ct" -ne 0 +]) +route_recompute_ct=$(ovn-appctl inc-engine/show-stats route recompute) +check test "$route_recompute_ct" -eq 0 +check ovn-sbctl destroy Service_Monitor $ungated_sm +check as northd ovn-appctl -t ovn-northd resume + +# Remove the routes. +check ovn-nbctl --wait=hv lr-lb-del lr-target lb0 +wait_row_count sb:Advertised_Route 0 ip_prefix='"172.16.1.10"' +wait_row_count sb:Advertised_Route 0 ip_prefix='"fd00::200"' + +OVS_APP_EXIT_AND_WAIT([ovn-controller]) + +as ovn-sb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as ovn-nb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as northd +OVS_APP_EXIT_AND_WAIT([ovn-northd]) + +as +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d +/connection dropped.*/d"]) + +AT_CLEANUP +]) + +OVN_FOR_EACH_NORTHD([ +AT_SETUP([dynamic-routing - shared backend LSP gates per-VIP]) +AT_KEYWORDS([dynamic-routing]) + +VRF_RESERVE([1340]) + +# lb-a and lb-b use different ports on the same backend LSP. Each route must +# follow the monitor for its own backend port. A gate keyed only by +# (tracked_port, chassis) would couple the two routes. + +ovn_start +OVS_TRAFFIC_VSWITCHD_START() + +ADD_BR([br-int]) +check ovs-vsctl \ + -- set Open_vSwitch . external-ids:system-id=hv1 \ + -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \ + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true + +start_daemon ovn-controller + +check ovn-nbctl ls-add ls-share + +check ovn-nbctl lr-add lr-origin \ + -- set Logical_Router lr-origin options:chassis=hv1 \ + options:dynamic-routing=true \ + options:dynamic-routing-vrf-id=1340 +check ovn-nbctl lrp-add lr-origin lr-origin-share 00:de:ad:00:00:01 \ + 192.168.0.1/24 \ + -- set Logical_Router_Port lr-origin-share \ + options:dynamic-routing-redistribute="lb" \ + options:dynamic-routing-maintain-vrf=true +check ovn-nbctl lsp-add ls-share share-lr-origin \ + -- set Logical_Switch_Port share-lr-origin type=router \ + options:router-port=lr-origin-share \ + -- lsp-set-addresses share-lr-origin router + +check ovn-nbctl lr-add lr-target \ + -- set Logical_Router lr-target options:chassis=hv1 +check ovn-nbctl lrp-add lr-target lr-target-share 00:de:ad:00:00:02 \ + 192.168.0.2/24 +check ovn-nbctl lsp-add ls-share share-lr-target \ + -- set Logical_Switch_Port share-lr-target type=router \ + options:router-port=lr-target-share \ + -- lsp-set-addresses share-lr-target router + +# One backend LSP shared by both LBs, with two listeners on different +# ports (:80 and :443) inside the same netns. +check ovn-nbctl lsp-add ls-share be0 +check ovn-nbctl lsp-set-addresses be0 "00:de:ad:00:00:10 192.168.0.10" +ADD_NAMESPACES(be0_ns) +ADD_VETH(be0, be0_ns, br-int, "192.168.0.10/24", "00:de:ad:00:00:10") + +# lb-a: VIP 172.16.1.10:80 -> 192.168.0.10:80 +check ovn-nbctl \ + -- lb-add lb-a 172.16.1.10:80 192.168.0.10:80 \ + -- set Load_Balancer lb-a options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb-a +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.10\:80" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb-a health_check @hc + +# lb-b: VIP 172.16.1.11:443 -> 192.168.0.10:443 (same backend LSP). +check ovn-nbctl \ + -- lb-add lb-b 172.16.1.11:443 192.168.0.10:443 \ + -- set Load_Balancer lb-b options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb-b +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.11\:443" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb-b health_check @hc + +check ovn-nbctl --wait=hv sync +wait_for_ports_up +OVS_CTL_TIMEOUT=30 + +# Both monitors start offline. +OVS_WAIT_UNTIL([test -n "`ip vrf show ovnvrf1340 2>/dev/null`"]) +wait_row_count Service_Monitor 1 logical_port=be0 port=80 +wait_row_count Service_Monitor 1 logical_port=be0 port=80 status=offline +wait_row_count Service_Monitor 1 logical_port=be0 port=443 status=offline +AT_CHECK([ + ip route list vrf ovnvrf1340 | grep -c "blackhole 172.16.1." || true +], [0], [0 +]) + +# Start only the :80 listener. +be80_pid_file=$(mktemp be0_http80.XXX.pid) +NETNS_DAEMONIZE([be0_ns], + [[$PYTHON $srcdir/test-l7.py http]], [$be80_pid_file]) +wait_row_count Service_Monitor 1 logical_port=be0 port=80 status=online +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1340 | grep -q "blackhole 172.16.1.10"]) +# The :443 route stays withdrawn. +AT_CHECK([ + ip route list vrf ovnvrf1340 | grep -c "blackhole 172.16.1.11" || true +], [0], [0 +]) + +# Start a separate listener for :443. +be443_pid_file=$(mktemp be0_tcp443.XXX.pid) +NETNS_DAEMONIZE([be0_ns], + [python3 -c "import socket; s=socket.socket(); s.bind(('0.0.0.0',443)); s.listen(1) +while True: + c,_=s.accept(); c.close()"], + [$be443_pid_file]) +wait_row_count Service_Monitor 1 logical_port=be0 port=443 status=online +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1340 | grep -q "blackhole 172.16.1.11"]) +# The :80 route remains installed. +AT_CHECK([ + ip route list vrf ovnvrf1340 | grep -c "blackhole 172.16.1.10" +], [0], [1 +]) + +# Stop :80. The :443 route remains installed. +kill `cat $be80_pid_file` +wait_row_count Service_Monitor 1 logical_port=be0 port=80 status=offline +OVS_WAIT_UNTIL([ + ! ip route list vrf ovnvrf1340 | grep -q "blackhole 172.16.1.10"]) +AT_CHECK([ + ip route list vrf ovnvrf1340 | grep -c "blackhole 172.16.1.11" +], [0], [1 +]) + +OVS_APP_EXIT_AND_WAIT([ovn-controller]) + +as ovn-sb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as ovn-nb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as northd +OVS_APP_EXIT_AND_WAIT([ovn-northd]) + +as +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d +/connection dropped.*/d"]) + +AT_CLEANUP +]) + +OVN_FOR_EACH_NORTHD([ +AT_SETUP([dynamic-routing - shared VIP IP and cross-VIP isolation]) +AT_KEYWORDS([dynamic-routing]) + +VRF_RESERVE([1341]) + +# VIP A has two listeners on be0 and therefore two selectors on one route. +# VIP B uses another port on be0. VIP A is installed while either of its +# monitors is online. VIP B's monitor does not affect it. + +ovn_start +OVS_TRAFFIC_VSWITCHD_START() + +ADD_BR([br-int]) +check ovs-vsctl \ + -- set Open_vSwitch . external-ids:system-id=hv1 \ + -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \ + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true + +start_daemon ovn-controller + +check ovn-nbctl ls-add ls-share + +check ovn-nbctl lr-add lr-origin \ + -- set Logical_Router lr-origin options:chassis=hv1 \ + options:dynamic-routing=true \ + options:dynamic-routing-vrf-id=1341 +check ovn-nbctl lrp-add lr-origin lr-origin-share 00:de:ad:00:00:01 \ + 192.168.0.1/24 \ + -- set Logical_Router_Port lr-origin-share \ + options:dynamic-routing-redistribute="lb" \ + options:dynamic-routing-maintain-vrf=true +check ovn-nbctl lsp-add ls-share share-lr-origin \ + -- set Logical_Switch_Port share-lr-origin type=router \ + options:router-port=lr-origin-share \ + -- lsp-set-addresses share-lr-origin router + +check ovn-nbctl lr-add lr-target \ + -- set Logical_Router lr-target options:chassis=hv1 +check ovn-nbctl lrp-add lr-target lr-target-share 00:de:ad:00:00:02 \ + 192.168.0.2/24 +check ovn-nbctl lsp-add ls-share share-lr-target \ + -- set Logical_Switch_Port share-lr-target type=router \ + options:router-port=lr-target-share \ + -- lsp-set-addresses share-lr-target router + +check ovn-nbctl lsp-add ls-share be0 +check ovn-nbctl lsp-set-addresses be0 "00:de:ad:00:00:10 192.168.0.10" +ADD_NAMESPACES(be0_ns) +ADD_VETH(be0, be0_ns, br-int, "192.168.0.10/24", "00:de:ad:00:00:10") + +# VIP A, listener 1: lb-c carries 172.16.1.20:80 -> 192.168.0.10:80. +check ovn-nbctl \ + -- lb-add lb-c 172.16.1.20:80 192.168.0.10:80 \ + -- set Load_Balancer lb-c options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb-c +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.20\:80" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb-c health_check @hc + +# VIP A, listener 2: lb-d carries 172.16.1.20:443 -> 192.168.0.10:443. +check ovn-nbctl \ + -- lb-add lb-d 172.16.1.20:443 192.168.0.10:443 \ + -- set Load_Balancer lb-d options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb-d +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.20\:443" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb-d health_check @hc + +# VIP B: lb-e carries 172.16.1.30:8080 -> 192.168.0.10:8080. +check ovn-nbctl \ + -- lb-add lb-e 172.16.1.30:8080 192.168.0.10:8080 \ + -- set Load_Balancer lb-e options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb-e +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.30\:8080" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb-e health_check @hc + +check ovn-nbctl --wait=hv sync +wait_for_ports_up +OVS_CTL_TIMEOUT=30 + +# Northd emits one route per (VIP IP, backend LSP). +OVS_WAIT_UNTIL([test -n "`ip vrf show ovnvrf1341 2>/dev/null`"]) +wait_row_count Service_Monitor 1 logical_port=be0 port=80 +wait_row_count Service_Monitor 1 logical_port=be0 port=80 status=offline +wait_row_count Service_Monitor 1 logical_port=be0 port=443 status=offline +wait_row_count Service_Monitor 1 logical_port=be0 port=8080 status=offline +wait_row_count sb:Advertised_Route 1 ip_prefix='"172.16.1.20"' +wait_row_count sb:Advertised_Route 1 ip_prefix='"172.16.1.30"' +ar_uuid=$(ovn-sbctl --bare --columns=_uuid find Advertised_Route \ + ip_prefix='"172.16.1.20"') +OVS_WAIT_UNTIL([ + hc=$(ovn-sbctl get Advertised_Route $ar_uuid \ + external_ids:health-checks 2>/dev/null) + test "$hc" = '"tcp,192.168.0.10,443;tcp,192.168.0.10,80"' +]) + +# All monitors start offline. +AT_CHECK([ + ip route list vrf ovnvrf1341 | grep -c "blackhole 172.16.1." || true +], [0], [0 +]) + +# Start VIP B. VIP A stays withdrawn because its :80 and :443 monitors are +# still offline. +be8080_pid_file=$(mktemp be0_tcp8080.XXX.pid) +NETNS_DAEMONIZE([be0_ns], + [python3 -c "import socket; s=socket.socket(); s.bind(('0.0.0.0',8080)); s.listen(1) +while True: + c,_=s.accept(); c.close()"], + [$be8080_pid_file]) +wait_row_count Service_Monitor 1 logical_port=be0 port=8080 status=online +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1341 | grep -q "blackhole 172.16.1.30"]) +AT_CHECK([ + ip route list vrf ovnvrf1341 | grep -c "blackhole 172.16.1.20" || true +], [0], [0 +]) + +# An unrelated SB Load_Balancer has VIP A's IP and VIP B's healthy backend. +# It is absent from VIP A's route selectors and must not open the gate. +check as northd ovn-appctl -t ovn-northd pause +unrelated_lb=$(ovn-sbctl create Load_Balancer name=unrelated protocol=tcp \ + vips='"172.16.1.20:9999"="192.168.0.10:8080"') +check ovn-appctl inc-engine/recompute +AT_CHECK([ + ip route list vrf ovnvrf1341 | grep -c "blackhole 172.16.1.20" || true +], [0], [0 +]) +check ovn-sbctl destroy Load_Balancer $unrelated_lb + +# An unrelated Service_Monitor update is handled without a route recompute. +check ovn-appctl inc-engine/clear-stats +unrelated_sm=$(ovn-sbctl create Service_Monitor type=load-balancer \ + ip=192.0.2.10 port=8080 protocol=tcp logical_port=unrelated \ + status=offline) +OVS_WAIT_UNTIL([ + route_compute_ct=$(ovn-appctl inc-engine/show-stats route compute) + test "$route_compute_ct" -ne 0 +]) +route_recompute_ct=$(ovn-appctl inc-engine/show-stats route recompute) +check test "$route_recompute_ct" -eq 0 +check ovn-sbctl destroy Service_Monitor $unrelated_sm +check as northd ovn-appctl -t ovn-northd resume + +# Start VIP A's :80 listener. Its route is installed while :443 remains +# offline. +be80_pid_file=$(mktemp be0_http80.XXX.pid) +NETNS_DAEMONIZE([be0_ns], + [[$PYTHON $srcdir/test-l7.py http]], [$be80_pid_file]) +wait_row_count Service_Monitor 1 logical_port=be0 port=80 status=online +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1341 | grep -q "blackhole 172.16.1.20"]) + +# Stop VIP A's :80 listener. VIP A withdraws and VIP B stays installed. +kill `cat $be80_pid_file` +wait_row_count Service_Monitor 1 logical_port=be0 port=80 status=offline +OVS_WAIT_UNTIL([ + ! ip route list vrf ovnvrf1341 | grep -q "blackhole 172.16.1.20"]) +AT_CHECK([ + ip route list vrf ovnvrf1341 | grep -c "blackhole 172.16.1.30" +], [0], [1 +]) + +OVS_APP_EXIT_AND_WAIT([ovn-controller]) + +as ovn-sb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as ovn-nb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as northd +OVS_APP_EXIT_AND_WAIT([ovn-northd]) + +as +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d +/connection dropped.*/d"]) AT_CLEANUP ]) +OVN_FOR_EACH_NORTHD([ +AT_SETUP([dynamic-routing - non-LB route overlap and LB health gating]) +AT_KEYWORDS([dynamic-routing]) + +VRF_RESERVE([1342]) + +# A NAT external IP overlaps an LB VIP. The monitor gates only the LB route. +# the NAT route stays installed. Check the LB status transitions as well. + +ovn_start +OVS_TRAFFIC_VSWITCHD_START() + +ADD_BR([br-int]) +check ovs-vsctl \ + -- set Open_vSwitch . external-ids:system-id=hv1 \ + -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \ + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true + +start_daemon ovn-controller + +check ovn-nbctl ls-add ls-share + +# lr-origin: GW with redistribute="lb,nat". The NAT rule creates a +# non-LB Advertised_Route for 172.16.1.10 (no source=lb key). +check ovn-nbctl lr-add lr-origin \ + -- set Logical_Router lr-origin options:chassis=hv1 \ + options:dynamic-routing=true \ + options:dynamic-routing-vrf-id=1342 +check ovn-nbctl lrp-add lr-origin lr-origin-share 00:de:ad:00:00:01 \ + 192.168.0.1/24 \ + -- set Logical_Router_Port lr-origin-share \ + options:dynamic-routing-redistribute="lb,nat" \ + options:dynamic-routing-maintain-vrf=true +check ovn-nbctl lsp-add ls-share share-lr-origin \ + -- set Logical_Switch_Port share-lr-origin type=router \ + options:router-port=lr-origin-share \ + -- lsp-set-addresses share-lr-origin router + +# NAT rule whose external IP matches the LB VIP. Northd emits a NAT +# Advertised_Route for 172.16.1.10 with no source=lb key. +check ovn-nbctl lr-nat-add lr-origin dnat_and_snat 172.16.1.10 192.168.0.10 + +# lr-target: owns the LB. +check ovn-nbctl lr-add lr-target \ + -- set Logical_Router lr-target options:chassis=hv1 +check ovn-nbctl lrp-add lr-target lr-target-share 00:de:ad:00:00:02 \ + 192.168.0.2/24 +check ovn-nbctl lsp-add ls-share share-lr-target \ + -- set Logical_Switch_Port share-lr-target type=router \ + options:router-port=lr-target-share \ + -- lsp-set-addresses share-lr-target router + +# Backend LSP. +check ovn-nbctl lsp-add ls-share be0 +check ovn-nbctl lsp-set-addresses be0 "00:de:ad:00:00:10 192.168.0.10" +ADD_NAMESPACES(be0_ns) +ADD_VETH(be0, be0_ns, br-int, "192.168.0.10/24", "00:de:ad:00:00:10") + +# LB with VIP 172.16.1.10:80 (same IP as NAT external IP). +check ovn-nbctl \ + -- lb-add lb0 172.16.1.10:80 192.168.0.10:80 \ + -- set Load_Balancer lb0 options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb0 +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.10\:80" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb0 health_check @hc + +check ovn-nbctl --wait=hv sync +wait_for_ports_up +OVS_CTL_TIMEOUT=30 + +# Two AR rows for 172.16.1.10: NAT-derived (no source) and LB-derived +# (source=lb). +OVS_WAIT_UNTIL([test -n "`ip vrf show ovnvrf1342 2>/dev/null`"]) +wait_row_count Service_Monitor 1 logical_port=be0 status=offline +wait_row_count sb:Advertised_Route 2 ip_prefix='"172.16.1.10"' + +# The offline monitor withdraws the LB route. The NAT route stays installed. +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1342 | grep -q "blackhole 172.16.1.10"]) + +# Find the LB-derived route for the administrative override below. +lb_ar=$(ovn-sbctl --columns=_uuid,external_ids find Advertised_Route \ + ip_prefix=172.16.1.10 | awk ' + /_uuid/ { uuid = $3 } + /source.*lb/ { print uuid; exit } + ') +test -n "$lb_ar" + +# Start the listener and install the LB route. +be_pid_file=$(mktemp be0_http.XXX.pid) +NETNS_DAEMONIZE([be0_ns], + [[$PYTHON $srcdir/test-l7.py http]], [$be_pid_file]) +wait_row_count Service_Monitor 1 logical_port=be0 status=online +# Both routes are installed at different priorities. +AT_CHECK([ + ip route list vrf ovnvrf1342 | grep -c "blackhole 172.16.1.10" +], [0], [2 +]) + +# Disable the LB route. The NAT route remains. +check ovn-sbctl set Advertised_Route $lb_ar external_ids:enabled=false +AT_CHECK([ + ip route list vrf ovnvrf1342 | grep -c "blackhole 172.16.1.10" +], [0], [1 +]) + +# Re-enable the LB route. +check ovn-sbctl remove Advertised_Route $lb_ar external_ids enabled + +# Stop the listener and withdraw the LB route. +kill `cat $be_pid_file` +wait_row_count Service_Monitor 1 logical_port=be0 status=offline +# The NAT route remains installed. +AT_CHECK([ + ip route list vrf ovnvrf1342 | grep -c "blackhole 172.16.1.10" +], [0], [1 +]) + +# A distributed NAT and distributed LB can also map to the exact same +# Advertised_Route key. The NAT contributor owns that shared row, so LB-only +# metadata must be absent and an offline monitor must not withdraw it. +check ovn-nbctl lr-add lr-exact +check ovn-nbctl lrp-add lr-exact lr-exact-share 00:de:ad:00:00:03 \ + 192.168.0.3/24 +check ovn-nbctl lsp-add ls-share share-lr-exact \ + -- set Logical_Switch_Port share-lr-exact type=router \ + options:router-port=lr-exact-share \ + -- lsp-set-addresses share-lr-exact router +check ovn-nbctl ls-add ls-exact-ext +check ovn-nbctl lrp-add lr-exact lr-exact-dgp 00:de:ad:00:00:05 \ + 203.0.113.1/24 +check ovn-nbctl lrp-set-gateway-chassis lr-exact-dgp hv1 +check ovn-nbctl lsp-add ls-exact-ext exact-ext-lr \ + -- set Logical_Switch_Port exact-ext-lr type=router \ + options:router-port=lr-exact-dgp \ + -- lsp-set-addresses exact-ext-lr router +check ovn-nbctl \ + -- lsp-add ls-exact-ext exact-ext-localnet \ + -- lsp-set-type exact-ext-localnet localnet \ + -- lsp-set-options exact-ext-localnet network_name=physnet-exact +check ovn-nbctl ls-add ls-exact-be +check ovn-nbctl lrp-add lr-exact lr-exact-be 00:de:ad:00:00:04 \ + 10.0.0.1/24 +check ovn-nbctl lsp-add ls-exact-be exact-be-lr \ + -- set Logical_Switch_Port exact-be-lr type=router \ + options:router-port=lr-exact-be \ + -- lsp-set-addresses exact-be-lr router +check ovn-nbctl lsp-add ls-exact-be be-exact +check ovn-nbctl lsp-set-addresses be-exact \ + "00:de:ad:00:00:20 10.0.0.10" +check ovn-nbctl lr-nat-add lr-exact dnat_and_snat \ + 172.16.1.20 10.0.0.10 be-exact 00:de:ad:00:00:20 +check ovn-nbctl \ + -- lb-add lb-exact 172.16.1.20:80 10.0.0.10:80 \ + -- set Load_Balancer lb-exact options:distributed=true \ + ip_port_mappings:10.0.0.10="be-exact:10.0.0.1" \ + -- lr-lb-add lr-exact lb-exact +check_uuid ovn-nbctl --id=@hc2 create Load_Balancer_Health_Check \ + vip="172.16.1.20\:80" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb-exact health_check @hc2 +check ovn-nbctl --wait=hv sync + +wait_row_count sb:Advertised_Route 1 ip_prefix='"172.16.1.20"' +exact_ar=$(ovn-sbctl --bare --columns=_uuid find Advertised_Route \ + ip_prefix=172.16.1.20) +test -n "$exact_ar" +AT_CHECK([! ovn-sbctl --bare get Advertised_Route $exact_ar external_ids | \ + grep -Eq 'source|distributed-lb|health-checks']) +AT_CHECK([ + ip route list vrf ovnvrf1342 | grep -c "blackhole 172.16.1.20" +], [0], [1 +]) + +OVS_APP_EXIT_AND_WAIT([ovn-controller]) + +as ovn-sb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as ovn-nb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as northd +OVS_APP_EXIT_AND_WAIT([ovn-northd]) + +as +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d +/connection dropped.*/d +/Unable to sync routes.*invalid table id: 252/d"]) + +AT_CLEANUP +]) + +OVN_FOR_EACH_NORTHD([ AT_SETUP([dynamic-routing - LB redistribute local-only restricts to backend chassis]) AT_KEYWORDS([dynamic-routing]) @@ -22682,8 +23598,8 @@ check ovn-nbctl lsp-set-addresses be0 "00:de:ad:00:00:10 192.168.0.10" ADD_NAMESPACES(be0_ns) ADD_VETH(be0, be0_ns, br-int, "192.168.0.10/24", "00:de:ad:00:00:10") -# Distributed LB with one VIP and one backend. This test isolates the -# local-only route-selection mechanism. +# Distributed LB with one VIP and one backend. No health check: this +# test isolates the local-only mechanism, not Service_Monitor gating. check ovn-nbctl \ -- lb-add lb0 172.16.1.10:80 192.168.0.10:80 \ -- set Load_Balancer lb0 options:distributed=true \ @@ -22701,7 +23617,7 @@ wait_row_count sb:Advertised_Route 1 ip_prefix='"172.16.1.10"' OVS_WAIT_UNTIL([ ip route list vrf ovnvrf1342 | grep -q "blackhole 172.16.1.10"]) -# Move the backend to a simulated hv2 by deleting the veth so hv1 +# Move the backend to a simulated hv2 by removing the OVS port so hv1 # drops the claim, then bind be0 to hv2. check ovs-vsctl del-port br-int ovs-be0 wait_column "" Port_Binding chassis logical_port=be0 @@ -22745,6 +23661,7 @@ OVS_WAIT_UNTIL([ # Cleanup. check ovn-sbctl destroy Chassis $hv2 + OVS_APP_EXIT_AND_WAIT([ovn-controller]) as ovn-sb @@ -22758,7 +23675,91 @@ OVS_APP_EXIT_AND_WAIT([ovn-northd]) as OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d -/connection dropped.*/d"]) +/connection dropped.*/d +/modprobe.*vrf/d"]) + +AT_CLEANUP +]) + +OVN_FOR_EACH_NORTHD([ +AT_SETUP([dynamic-routing - admin override on LB route without tracked_port]) +AT_KEYWORDS([dynamic-routing]) + +VRF_RESERVE([1343]) + +# A distributed-LB route without ip_port_mappings has tracked_port=NULL. +# external_ids:enabled=false must still withdraw it even though the +# tracked_port block is skipped. The LB is attached directly to the +# gateway router (lr-origin) so northd emits the route on lr-origin's +# datapath with no peer LRP to use as fallback tracked_port. + +ovn_start +OVS_TRAFFIC_VSWITCHD_START() + +ADD_BR([br-int]) +check ovs-vsctl \ + -- set Open_vSwitch . external-ids:system-id=hv1 \ + -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \ + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true + +start_daemon ovn-controller + +check ovn-nbctl ls-add ls-pub + +check ovn-nbctl lr-add lr-origin \ + -- set Logical_Router lr-origin options:chassis=hv1 \ + options:dynamic-routing=true \ + options:dynamic-routing-vrf-id=1343 +check ovn-nbctl lrp-add lr-origin lr-origin-pub 00:de:ad:00:00:01 \ + 192.168.0.1/24 \ + -- set Logical_Router_Port lr-origin-pub \ + options:dynamic-routing-redistribute="lb" \ + options:dynamic-routing-maintain-vrf=true +check ovn-nbctl lsp-add ls-pub pub-lr-origin \ + -- set Logical_Switch_Port pub-lr-origin type=router \ + options:router-port=lr-origin-pub \ + addresses=router + +# Distributed LB attached directly to the gateway router, without +# ip_port_mappings. The resulting Advertised_Route has no tracked_port. +check ovn-nbctl lb-add lb0 172.16.1.10:80 192.168.0.10:80 +check ovn-nbctl set Load_Balancer lb0 options:distributed=true +check ovn-nbctl --wait=hv lr-lb-add lr-origin lb0 + +# Verify the route has no tracked_port. +wait_row_count sb:Advertised_Route 1 \ + ip_prefix='"172.16.1.10"' tracked_port='[[]]' + +# Route is installed. +OVN_ROUTE_EQUAL([ovnvrf1343], [dnl +blackhole 172.16.1.10 proto ovn metric 1000]) + +# Administrative override withdraws the route even without tracked_port. +ar_uuid=$(fetch_column sb:Advertised_Route _uuid \ + ip_prefix='"172.16.1.10"' tracked_port='[[]]') +check ovn-sbctl set Advertised_Route $ar_uuid \ + external_ids:enabled=false + +OVS_WAIT_UNTIL([ + ! ip route list vrf ovnvrf1343 | grep -q "172.16.1.10"]) + +OVS_APP_EXIT_AND_WAIT([ovn-controller]) + +as ovn-sb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as ovn-nb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as northd +OVS_APP_EXIT_AND_WAIT([ovn-northd]) + +as +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d +/connection dropped.*/d +/modprobe.*vrf/d"]) AT_CLEANUP ]) -- 2.53.0 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
