northd_handle_lb_data_changes() gave up on any change that involved a load balancer with a health check, and the northd engine node had no handler for the SB Service_Monitor table, so a backend added to such a load balancer and every status change reported by ovn-controller rebuilt all logical flows. A distributed load balancer needs a health check for its per-backend route advertisement to be gated, so with many of them every backend change or health transition in the deployment costs a full recompute.
Keep, for every service monitor northd owns, the set of load balancers whose backends it probes, and for every load balancer the set of its monitors. On a load balancer change, drop the monitors the load balancer held and build its current set, deleting a monitor that no load balancer, network function or logical switch port health check uses any more. Handle the SB Service_Monitor changes in the northd node: - A status change of a load balancer monitor marks the load balancers that use it for the logical flow rebuild the lflow node already does per load balancer. - A monitor northd does not know, or one deleted behind its back, falls back to a recompute. - A status change of a monitor that a network function or a logical switch port health check uses also falls back to a recompute. The interconnect learned monitors node only recomputes for rows that carry or lose the ic_learned flag. Test backend addition and removal, status changes, a monitor shared by two load balancers, and recovery from a monitor deleted directly in the Southbound database. Signed-off-by: Dmitrii Shcherbakov <[email protected]> --- northd/en-northd.c | 64 ++++++- northd/en-northd.h | 5 + northd/inc-proc-northd.c | 6 +- northd/lb.c | 2 + northd/lb.h | 6 + northd/northd.c | 301 ++++++++++++++++++++++++++++--- northd/northd.h | 12 +- tests/ovn-inc-proc-graph-dump.at | 4 +- tests/ovn-northd.at | 241 +++++++++++++++++++++++-- 9 files changed, 595 insertions(+), 46 deletions(-) diff --git a/northd/en-northd.c b/northd/en-northd.c index 96859e4ae..a40679270 100644 --- a/northd/en-northd.c +++ b/northd/en-northd.c @@ -224,14 +224,26 @@ northd_lb_data_handler(struct engine_node *node, void *data) return EN_UNHANDLED; } + const struct engine_context *eng_ctx = engine_get_context(); struct northd_data *nd = data; + struct northd_input input_data; + + northd_get_input_data(node, &input_data); + + struct hmap *ic_learned_map = input_data.ic_learned_svc_monitors_map; if (!northd_handle_lb_data_changes(&lb_data->tracked_lb_data, &nd->ls_datapaths, &nd->lr_datapaths, &nd->lb_datapaths_map, &nd->lb_group_datapaths_map, &lb_data->lr_lb_map, - &nd->trk_data)) { + &nd->trk_data, + eng_ctx->ovnsb_idl_txn, + input_data.svc_global_addresses, + &nd->ls_ports, + &nd->svc_monitor_lsps, + &nd->local_svc_monitors_map, + ic_learned_map)) { return EN_UNHANDLED; } @@ -687,6 +699,56 @@ en_northd_clear_tracked_data(void *data_) destroy_northd_data_tracked_changes(data); } +enum engine_input_handler_result +northd_sb_service_monitor_handler(struct engine_node *node, void *data) +{ + struct northd_data *nd = data; + const struct sbrec_service_monitor_table *sbrec_service_monitor_table = + EN_OVSDB_GET(engine_get_input("SB_service_monitor", node)); + + if (!northd_handle_sb_service_monitor_changes(sbrec_service_monitor_table, + &nd->local_svc_monitors_map, + &nd->trk_data)) { + return EN_UNHANDLED; + } + + if (northd_has_lbs_in_tracked_data(&nd->trk_data)) { + return EN_HANDLED_UPDATED; + } + + return EN_HANDLED_UNCHANGED; +} + +/* The interconnect learned monitors map only depends on the rows with + * ic_learned set, and on rows that lose the flag when northd takes them + * over. Any other change, such as a status update by ovn-controller, + * leaves the map as it is. */ +enum engine_input_handler_result +ic_learned_svc_monitors_sb_service_monitor_handler(struct engine_node *node, + void *data OVS_UNUSED) +{ + const struct sbrec_service_monitor_table *sb_mon_table = + EN_OVSDB_GET(engine_get_input("SB_service_monitor", node)); + + const struct sbrec_service_monitor *sbrec_mon; + SBREC_SERVICE_MONITOR_TABLE_FOR_EACH_TRACKED (sbrec_mon, sb_mon_table) { + if (sbrec_mon->ic_learned) { + return EN_UNHANDLED; + } + /* A row that lost the flag was taken over by northd. An inserted + * row reports every column as updated, and a deleted one is gone + * with its flag, so only look at modified rows. */ + if (!sbrec_service_monitor_is_new(sbrec_mon) + && !sbrec_service_monitor_is_deleted(sbrec_mon) + && sbrec_service_monitor_is_updated( + sbrec_mon, SBREC_SERVICE_MONITOR_COL_IC_LEARNED)) { + return EN_UNHANDLED; + } + } + + return EN_HANDLED_UNCHANGED; +} + enum engine_input_handler_result northd_sb_fdb_change_handler(struct engine_node *node, void *data) { diff --git a/northd/en-northd.h b/northd/en-northd.h index c62631008..c9f71b0aa 100644 --- a/northd/en-northd.h +++ b/northd/en-northd.h @@ -29,6 +29,11 @@ enum engine_input_handler_result northd_nb_port_group_handler(struct engine_node *node, void *data); enum engine_input_handler_result northd_sb_fdb_change_handler(struct engine_node *node, void *data); +enum engine_input_handler_result +northd_sb_service_monitor_handler(struct engine_node *node, void *data); +enum engine_input_handler_result +ic_learned_svc_monitors_sb_service_monitor_handler(struct engine_node *node, + void *data); void *en_routes_init(struct engine_node *node OVS_UNUSED, struct engine_arg *arg OVS_UNUSED); void en_route_policies_cleanup(void *data); diff --git a/northd/inc-proc-northd.c b/northd/inc-proc-northd.c index 53604a20a..73cc70bb4 100644 --- a/northd/inc-proc-northd.c +++ b/northd/inc-proc-northd.c @@ -251,7 +251,8 @@ void inc_proc_northd_init(struct ovsdb_idl_loop *nb, lb_data_synced_logical_router_handler); engine_add_input(&en_ic_learned_svc_monitors, - &en_sb_service_monitor, NULL); + &en_sb_service_monitor, + ic_learned_svc_monitors_sb_service_monitor_handler); engine_add_input(&en_northd, &en_nb_mirror, NULL); engine_add_input(&en_northd, &en_nb_mirror_rule, NULL); @@ -272,7 +273,8 @@ void inc_proc_northd_init(struct ovsdb_idl_loop *nb, engine_add_input(&en_northd, &en_sb_meter, NULL); engine_add_input(&en_northd, &en_sb_dns, NULL); engine_add_input(&en_northd, &en_sb_ha_chassis_group, NULL); - engine_add_input(&en_northd, &en_sb_service_monitor, NULL); + engine_add_input(&en_northd, &en_sb_service_monitor, + northd_sb_service_monitor_handler); engine_add_input(&en_northd, &en_sb_static_mac_binding, NULL); engine_add_input(&en_northd, &en_sb_chassis_template_var, NULL); engine_add_input(&en_northd, &en_ic_learned_svc_monitors, NULL); diff --git a/northd/lb.c b/northd/lb.c index 5ff9d1fad..fcd6e56f8 100644 --- a/northd/lb.c +++ b/northd/lb.c @@ -717,6 +717,7 @@ ovn_lb_datapaths_create(const struct ovn_northd_lb *lb, size_t n_ls_datapaths, dynamic_bitmap_alloc(&lb_dps->nb_lr_map, n_lr_datapaths); lb_dps->lflow_ref = lflow_ref_create(); hmapx_init(&lb_dps->ls_lb_with_stateless_mode); + hmapx_init(&lb_dps->svc_mons); return lb_dps; } @@ -727,6 +728,7 @@ ovn_lb_datapaths_destroy(struct ovn_lb_datapaths *lb_dps) dynamic_bitmap_free(&lb_dps->nb_ls_map); lflow_ref_destroy(lb_dps->lflow_ref); hmapx_destroy(&lb_dps->ls_lb_with_stateless_mode); + hmapx_destroy(&lb_dps->svc_mons); free(lb_dps); } diff --git a/northd/lb.h b/northd/lb.h index 7a98c2f55..8d799b974 100644 --- a/northd/lb.h +++ b/northd/lb.h @@ -162,6 +162,12 @@ struct ovn_lb_datapaths { struct hmapx ls_lb_with_stateless_mode; + /* The service monitors (struct service_monitor_info, owned by the + * en_northd node) that probe the backends of this load balancer. Kept + * so that a change to the load balancer can drop the monitors it no + * longer needs without a full recompute. */ + struct hmapx svc_mons; + /* Reference of lflows generated for this load balancer. * * This data is initialized and destroyed by the en_northd node, but diff --git a/northd/northd.c b/northd/northd.c index 3f59894bd..6020a3305 100644 --- a/northd/northd.c +++ b/northd/northd.c @@ -3172,20 +3172,62 @@ delete_fdb_entries(struct ovsdb_idl_index *sbrec_fdb_by_dp_and_port, struct service_monitor_info { struct hmap_node hmap_node; + /* The SB row. A row northd inserted is freed by the IDL once the + * transaction completes and comes back as a new row with the server's + * update, so the pointer is re-pointed in + * northd_handle_sb_service_monitor_changes() and the key of the monitor + * is kept here rather than read through it. */ const struct sbrec_service_monitor *sbrec_mon; + char *ip; + char *logical_port; + char *protocol; + uint16_t port; bool required; + /* The load balancers (struct ovn_lb_datapaths) whose backends this + * monitor probes, so that a change of the monitor rebuilds only their + * flows and a change of a load balancer drops only the monitors it + * no longer needs. */ + struct hmapx lbs; + /* True when a network function or a logical switch port health check + * uses this monitor too. Such a monitor is never deleted on a load + * balancer change, and a status change of it needs a recompute. */ + bool non_lb_user; }; +static struct service_monitor_info * +service_monitor_info_alloc(const struct sbrec_service_monitor *sbrec_mon) +{ + struct service_monitor_info *mon_info = xzalloc(sizeof *mon_info); + mon_info->sbrec_mon = sbrec_mon; + mon_info->ip = xstrdup(sbrec_mon->ip); + mon_info->logical_port = xstrdup(sbrec_mon->logical_port); + mon_info->protocol = xstrdup(sbrec_mon->protocol ? sbrec_mon->protocol + : ""); + mon_info->port = sbrec_mon->port; + hmapx_init(&mon_info->lbs); + return mon_info; +} + +static void +service_monitor_info_destroy(struct service_monitor_info *mon_info) +{ + hmapx_destroy(&mon_info->lbs); + free(mon_info->ip); + free(mon_info->logical_port); + free(mon_info->protocol); + free(mon_info); +} + static inline bool monitor_info_matches(const struct service_monitor_info *mon_info, const char *ip, const char *logical_port, uint16_t service_port, const char *protocol) { - return (mon_info->sbrec_mon->port == service_port && - !strcmp(mon_info->sbrec_mon->ip, ip) && - !strcmp(mon_info->sbrec_mon->protocol, protocol) && - !strcmp(mon_info->sbrec_mon->logical_port, logical_port)); + return (mon_info->port == service_port && + !strcmp(mon_info->ip, ip) && + !strcmp(mon_info->protocol, protocol) && + !strcmp(mon_info->logical_port, logical_port)); } static struct service_monitor_info * @@ -3287,8 +3329,7 @@ create_or_get_service_mon(struct ovsdb_idl_txn *ovnsb_txn, * the first health check probe succeeds. */ sbrec_service_monitor_set_status(sbrec_mon, "offline"); - mon_info = xzalloc(sizeof *mon_info); - mon_info->sbrec_mon = sbrec_mon; + mon_info = service_monitor_info_alloc(sbrec_mon); mon_info->required = true; hmap_insert(local_svc_monitors_map, &mon_info->hmap_node, hash); @@ -3390,6 +3431,7 @@ ovn_nf_svc_create(struct ovsdb_idl_txn *ovnsb_txn, nbrec_nf->inport->name, 0, "icmp", false); + mon_info->non_lb_user = true; set_service_mon_options(mon_info->sbrec_mon, &nbrec_nf->health_check->options, NULL); @@ -3429,9 +3471,13 @@ ovn_nf_svc_create(struct ovsdb_idl_txn *ovnsb_txn, } } -static void +/* Creates or updates the service monitors of the load balancer's backends + * and records them in lb_dps->svc_mons. Returns false when a monitor is + * owned by the interconnect learned map, which incremental processing + * cannot track. */ +static bool ovn_lb_svc_create(struct ovsdb_idl_txn *ovnsb_txn, - const struct ovn_lb_datapaths *lb_dps, + struct ovn_lb_datapaths *lb_dps, const struct svc_monitor_addresses *svc_global_addresses, struct hmap *ls_ports, struct sset *svc_monitor_lsps, @@ -3439,9 +3485,10 @@ ovn_lb_svc_create(struct ovsdb_idl_txn *ovnsb_txn, struct hmap *ic_learned_svc_monitors_map) { const struct ovn_northd_lb *lb = lb_dps->lb; + bool all_local = true; if (lb->template) { - return; + return true; } for (size_t i = 0; i < lb->n_vips; i++) { @@ -3498,6 +3545,13 @@ ovn_lb_svc_create(struct ovsdb_idl_txn *ovnsb_txn, protocol, backend_nb->remote_backend); + if (hmap_contains(local_svc_monitors_map, &mon_info->hmap_node)) { + hmapx_add(&mon_info->lbs, lb_dps); + hmapx_add(&lb_dps->svc_mons, mon_info); + } else { + all_local = false; + } + set_service_mon_options(mon_info->sbrec_mon, &lb_vip_nb->lb_health_check->options, backend_nb->az_name); @@ -3536,6 +3590,29 @@ ovn_lb_svc_create(struct ovsdb_idl_txn *ovnsb_txn, } } } + return all_local; +} + +/* Drops the references lb_dps held on the monitors in old_mons that are no + * longer in lb_dps->svc_mons, and deletes the monitors nothing else uses. + * A monitor the load balancer still needs keeps its row and its status. */ +static void +ovn_lb_svc_prune(struct ovn_lb_datapaths *lb_dps, const struct hmapx *old_mons, + struct hmap *local_svc_monitors_map) +{ + struct hmapx_node *node; + HMAPX_FOR_EACH (node, old_mons) { + struct service_monitor_info *mon_info = node->data; + if (hmapx_contains(&lb_dps->svc_mons, mon_info)) { + continue; + } + hmapx_find_and_delete(&mon_info->lbs, lb_dps); + if (hmapx_is_empty(&mon_info->lbs) && !mon_info->non_lb_user) { + sbrec_service_monitor_delete(mon_info->sbrec_mon); + hmap_remove(local_svc_monitors_map, &mon_info->hmap_node); + service_monitor_info_destroy(mon_info); + } + } } static bool @@ -3814,16 +3891,17 @@ ovn_lsp_svc_monitors_process_port( lport_addresses_contains_ip(op->lsp_addrs, op->n_lsp_addrs, lsp_hc->address); - /* Remove outdated records */ + /* Remove outdated records, unless a load balancer still probes the + * same backend through the monitor. */ if (!lsp_contain_ip) { mon_info = get_service_mon(local_svc_monitors_map, NULL, lsp_hc->address, op->key, lsp_hc->port, lsp_hc->protocol); - if (mon_info) { + if (mon_info && hmapx_is_empty(&mon_info->lbs)) { sbrec_service_monitor_delete(mon_info->sbrec_mon); hmap_remove(local_svc_monitors_map, &mon_info->hmap_node); - free(mon_info); + service_monitor_info_destroy(mon_info); } continue; } @@ -3835,6 +3913,7 @@ ovn_lsp_svc_monitors_process_port( lsp_hc->port, lsp_hc->protocol, false); + mon_info->non_lb_user = true; set_service_mon_options(mon_info->sbrec_mon, &lsp_hc->options, NULL); struct eth_addr ea; @@ -3891,8 +3970,8 @@ build_svc_monitors_data( uint32_t hash = sbrec_mon->port; hash = hash_string(sbrec_mon->ip, hash); hash = hash_string(sbrec_mon->logical_port, hash); - struct service_monitor_info *mon_info = xzalloc(sizeof *mon_info); - mon_info->sbrec_mon = sbrec_mon; + struct service_monitor_info *mon_info = + service_monitor_info_alloc(sbrec_mon); mon_info->required = false; hmap_insert(local_svc_monitors_map, &mon_info->hmap_node, hash); @@ -3948,7 +4027,7 @@ build_svc_monitors_data( if (!mon_info->required) { sbrec_service_monitor_delete(mon_info->sbrec_mon); hmap_remove(local_svc_monitors_map, &mon_info->hmap_node); - free(mon_info); + service_monitor_info_destroy(mon_info); } } } @@ -5935,12 +6014,19 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, struct hmap *lb_datapaths_map, struct hmap *lbgrp_datapaths_map, const struct hmap *lr_lb_map, - struct northd_tracked_data *nd_changes) -{ - if (trk_lb_data->has_health_checks) { - /* Fall back to recompute since a tracked load balancer - * has health checks configured and I-P is not yet supported - * for such load balancers. */ + struct northd_tracked_data *nd_changes, + struct ovsdb_idl_txn *ovnsb_txn, + const struct svc_monitor_addresses + *svc_global_addresses, + struct hmap *ls_ports, + struct sset *svc_monitor_lsps, + struct hmap *local_svc_monitors_map, + struct hmap *ic_learned_svc_monitors_map) +{ + if (trk_lb_data->has_health_checks && !ovnsb_txn) { + /* The service monitors of a load balancer with health checks live + * in the Southbound database, so their update needs a transaction. + * Without one, fall back to recompute. */ return false; } @@ -6022,7 +6108,10 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, hmap_remove(lb_datapaths_map, &lb_dps->hmap_node); - /* Add the deleted lb to the northd tracked data. */ + /* Add the deleted lb to the northd tracked data. A service monitor + * change handled earlier in this run may have marked it as updated, + * and a deleted load balancer gets no flows rebuilt. */ + hmapx_find_and_delete(&nd_changes->trk_lbs.crupdated, lb_dps); hmapx_add(&nd_changes->trk_lbs.deleted, lb_dps); } @@ -6212,6 +6301,166 @@ northd_handle_lb_data_changes(struct tracked_lb_data *trk_lb_data, } } + /* Service monitors. Every load balancer that changed, was deleted, or + * was attached to a router, which decides the source address of its + * monitors, drops the monitors it held and builds its current set. + * The datapath associations above have to be in place for that. */ + struct hmapx svc_lbs = HMAPX_INITIALIZER(&svc_lbs); + HMAPX_FOR_EACH (hmapx_node, &nd_changes->trk_lbs.deleted) { + hmapx_add(&svc_lbs, hmapx_node->data); + } + HMAP_FOR_EACH (clb, hmap_node, &trk_lb_data->crupdated_lbs) { + lb_dps = ovn_lb_datapaths_find(lb_datapaths_map, + &clb->lb->nlb->header_.uuid); + hmapx_add(&svc_lbs, lb_dps); + } + HMAP_FOR_EACH (crupdated_lbgrp, hmap_node, + &trk_lb_data->crupdated_lbgrps) { + struct hmapx_node *hnode; + HMAPX_FOR_EACH (hnode, &crupdated_lbgrp->assoc_lbs) { + lb = hnode->data; + lb_dps = ovn_lb_datapaths_find(lb_datapaths_map, + &lb->nlb->header_.uuid); + hmapx_add(&svc_lbs, lb_dps); + } + } + LIST_FOR_EACH (codlb, list_node, &trk_lb_data->crupdated_lr_lbs) { + struct uuidset_node *uuidnode; + UUIDSET_FOR_EACH (uuidnode, &codlb->assoc_lbs) { + lb_dps = ovn_lb_datapaths_find(lb_datapaths_map, &uuidnode->uuid); + hmapx_add(&svc_lbs, lb_dps); + } + UUIDSET_FOR_EACH (uuidnode, &codlb->assoc_lbgrps) { + lbgrp_dps = ovn_lb_group_datapaths_find(lbgrp_datapaths_map, + &uuidnode->uuid); + for (size_t j = 0; j < lbgrp_dps->lb_group->n_lbs; j++) { + lb_dps = ovn_lb_datapaths_find( + lb_datapaths_map, + &lbgrp_dps->lb_group->lbs[j]->nlb->header_.uuid); + hmapx_add(&svc_lbs, lb_dps); + } + } + } + /* Build the current set of every load balancer first, so that a monitor + * a load balancer keeps, or that moves from one load balancer to another + * in the same change, is updated in place and keeps its status. Only + * then drop the monitors no load balancer needs any more. A deleted + * load balancer only gives its monitors up. */ + size_t n_svc_lbs = 0; + struct ovn_lb_datapaths **svc_lb_dps = + xmalloc(hmapx_count(&svc_lbs) * sizeof *svc_lb_dps); + struct hmapx *old_mons = xmalloc(hmapx_count(&svc_lbs) * sizeof *old_mons); + bool ok = true; + HMAPX_FOR_EACH (hmapx_node, &svc_lbs) { + lb_dps = hmapx_node->data; + bool deleted = hmapx_contains(&nd_changes->trk_lbs.deleted, lb_dps); + if (hmapx_is_empty(&lb_dps->svc_mons) + && (deleted || !lb_dps->lb->health_checks)) { + continue; + } + if (!ovnsb_txn) { + ok = false; + break; + } + hmapx_init(&old_mons[n_svc_lbs]); + hmapx_swap(&old_mons[n_svc_lbs], &lb_dps->svc_mons); + svc_lb_dps[n_svc_lbs++] = lb_dps; + } + for (size_t i = 0; ok && i < n_svc_lbs; i++) { + lb_dps = svc_lb_dps[i]; + if (!hmapx_contains(&nd_changes->trk_lbs.deleted, lb_dps) + && lb_dps->lb->health_checks + && !ovn_lb_svc_create(ovnsb_txn, lb_dps, svc_global_addresses, + ls_ports, svc_monitor_lsps, + local_svc_monitors_map, + ic_learned_svc_monitors_map)) { + ok = false; + } + } + for (size_t i = 0; i < n_svc_lbs; i++) { + ovn_lb_svc_prune(svc_lb_dps[i], &old_mons[i], local_svc_monitors_map); + hmapx_destroy(&old_mons[i]); + } + free(old_mons); + free(svc_lb_dps); + hmapx_destroy(&svc_lbs); + + return ok; +} + +/* Handles the tracked changes of the SB Service_Monitor table for the + * monitors northd owns. A status change rebuilds the flows of the load + * balancers the monitor probes. Returns false when the change needs a + * recompute: a monitor northd does not know, one that was deleted behind + * its back, or a status change of a monitor that a network function or a + * logical switch port health check uses. */ +bool +northd_handle_sb_service_monitor_changes( + const struct sbrec_service_monitor_table *sb_mon_table, + struct hmap *local_svc_monitors_map, + struct northd_tracked_data *nd_changes) +{ + const struct sbrec_service_monitor *sbrec_mon; + SBREC_SERVICE_MONITOR_TABLE_FOR_EACH_TRACKED (sbrec_mon, sb_mon_table) { + if (sbrec_mon->ic_learned) { + /* Owned by the interconnect learned monitors node. */ + continue; + } + + struct service_monitor_info *mon_info = + get_service_mon(local_svc_monitors_map, NULL, sbrec_mon->ip, + sbrec_mon->logical_port, sbrec_mon->port, + sbrec_mon->protocol); + + if (sbrec_service_monitor_is_deleted(sbrec_mon)) { + if (mon_info && mon_info->sbrec_mon == sbrec_mon) { + /* Deleted by someone else, recreate it. */ + return false; + } + continue; + } + + bool status_changed; + if (sbrec_service_monitor_is_new(sbrec_mon)) { + if (!mon_info) { + /* A row northd did not create. */ + return false; + } + /* The row northd inserted was freed with its transaction and + * this is the server's copy of it. northd inserted it as + * "offline" and built the flows that way, so a status that + * ovn-controller set before this copy arrived is a change. */ + mon_info->sbrec_mon = sbrec_mon; + status_changed = !sbrec_mon->status + || strcmp(sbrec_mon->status, "offline"); + } else { + if (!mon_info || mon_info->sbrec_mon != sbrec_mon) { + /* A row northd did not create. */ + return false; + } + status_changed = sbrec_service_monitor_is_updated( + sbrec_mon, SBREC_SERVICE_MONITOR_COL_STATUS); + } + + if (!status_changed) { + continue; + } + + if (mon_info->non_lb_user) { + return false; + } + + struct hmapx_node *node; + HMAPX_FOR_EACH (node, &mon_info->lbs) { + if (hmapx_contains(&nd_changes->trk_lbs.deleted, node->data)) { + /* Deleted earlier in this run. */ + continue; + } + hmapx_add(&nd_changes->trk_lbs.crupdated, node->data); + nd_changes->type |= NORTHD_TRACKED_LBS; + } + } + return true; } @@ -12333,8 +12582,8 @@ build_ic_learned_svc_monitors_map( uint32_t hash = sbrec_mon->port; hash = hash_string(sbrec_mon->ip, hash); hash = hash_string(sbrec_mon->logical_port, hash); - struct service_monitor_info *mon_info = xzalloc(sizeof *mon_info); - mon_info->sbrec_mon = sbrec_mon; + struct service_monitor_info *mon_info = + service_monitor_info_alloc(sbrec_mon); mon_info->required = true; hmap_insert(ic_learned_svc_monitors_map, &mon_info->hmap_node, hash); @@ -22079,7 +22328,7 @@ northd_destroy(struct northd_data *data) struct service_monitor_info *mon_info; HMAP_FOR_EACH_POP (mon_info, hmap_node, &data->local_svc_monitors_map) { - free(mon_info); + service_monitor_info_destroy(mon_info); } hmap_destroy(&data->local_svc_monitors_map); @@ -22122,7 +22371,7 @@ __ic_learned_svcs_cleanup(struct hmap *ic_learned_svc_monitors_map) { struct service_monitor_info *mon_info; HMAP_FOR_EACH_POP (mon_info, hmap_node, ic_learned_svc_monitors_map) { - free(mon_info); + service_monitor_info_destroy(mon_info); } hmap_destroy(ic_learned_svc_monitors_map); } diff --git a/northd/northd.h b/northd/northd.h index 3d1fd2d31..5c4c7fbe9 100644 --- a/northd/northd.h +++ b/northd/northd.h @@ -1015,7 +1015,17 @@ bool northd_handle_lb_data_changes(struct tracked_lb_data *, struct hmap *lb_datapaths_map, struct hmap *lbgrp_datapaths_map, const struct hmap *lr_lb_map, - struct northd_tracked_data *); + struct northd_tracked_data *, + struct ovsdb_idl_txn *ovnsb_txn, + const struct svc_monitor_addresses *, + struct hmap *ls_ports, + struct sset *svc_monitor_lsps, + struct hmap *local_svc_monitors_map, + struct hmap *ic_learned_svc_monitors_map); +bool northd_handle_sb_service_monitor_changes( + const struct sbrec_service_monitor_table *, + struct hmap *local_svc_monitors_map, + struct northd_tracked_data *); void build_route_policies(struct ovn_datapath *, const struct hmap *, const struct hmap *, struct hmap *, struct hmap *, diff --git a/tests/ovn-inc-proc-graph-dump.at b/tests/ovn-inc-proc-graph-dump.at index b3df37a01..1f9c5cc62 100644 --- a/tests/ovn-inc-proc-graph-dump.at +++ b/tests/ovn-inc-proc-graph-dump.at @@ -29,7 +29,7 @@ digraph "Incremental-Processing-Engine" { SB_static_mac_binding [[style=filled, shape=box, fillcolor=white, label="SB_static_mac_binding"]]; SB_chassis_template_var [[style=filled, shape=box, fillcolor=white, label="SB_chassis_template_var"]]; ic_learned_svc_monitors [[style=filled, shape=box, fillcolor=white, label="ic_learned_svc_monitors"]]; - SB_service_monitor -> ic_learned_svc_monitors [[label=""]]; + SB_service_monitor -> ic_learned_svc_monitors [[label="ic_learned_svc_monitors_sb_service_monitor_handler"]]; SB_fdb [[style=filled, shape=box, fillcolor=white, label="SB_fdb"]]; NB_nb_global [[style=filled, shape=box, fillcolor=white, label="NB_nb_global"]]; NB_logical_switch [[style=filled, shape=box, fillcolor=white, label="NB_logical_switch"]]; @@ -84,7 +84,7 @@ digraph "Incremental-Processing-Engine" { SB_meter -> northd [[label=""]]; SB_dns -> northd [[label=""]]; SB_ha_chassis_group -> northd [[label=""]]; - SB_service_monitor -> northd [[label=""]]; + SB_service_monitor -> northd [[label="northd_sb_service_monitor_handler"]]; SB_static_mac_binding -> northd [[label=""]]; SB_chassis_template_var -> northd [[label=""]]; ic_learned_svc_monitors -> northd [[label=""]]; diff --git a/tests/ovn-northd.at b/tests/ovn-northd.at index 93af177ab..39ec38c6c 100644 --- a/tests/ovn-northd.at +++ b/tests/ovn-northd.at @@ -2043,6 +2043,218 @@ OVN_CLEANUP_NORTHD AT_CLEANUP ]) +OVN_FOR_EACH_NORTHD_NO_HV([ +AT_SETUP([Load balancer health check incremental processing]) +ovn_start + +check ovn-nbctl ls-add sw0 +check ovn-nbctl lsp-add sw0 sw0-p1 -- lsp-set-addresses sw0-p1 \ + "00:00:00:00:00:03 10.0.0.3" +check ovn-nbctl lsp-add sw0 sw0-p2 -- lsp-set-addresses sw0-p2 \ + "00:00:00:00:00:04 10.0.0.4" +check ovn-sbctl chassis-add hv1 geneve 127.0.0.1 +check ovn-sbctl lsp-bind sw0-p1 hv1 +check ovn-sbctl lsp-bind sw0-p2 hv1 +wait_row_count nb:Logical_Switch_Port 1 name=sw0-p1 'up=true' +wait_row_count nb:Logical_Switch_Port 1 name=sw0-p2 'up=true' + +check ovn-nbctl lb-add lb1 10.0.0.10:80 10.0.0.3:80 \ + -- set load_balancer lb1 ip_port_mappings:10.0.0.3=sw0-p1:10.0.0.2 +check_uuid ovn-nbctl -- --id=@hc create Load_Balancer_Health_Check \ + vip="10.0.0.10\:80" -- add Load_Balancer lb1 health_check @hc +check ovn-nbctl --wait=sb ls-lb-add sw0 lb1 +wait_row_count Service_Monitor 1 logical_port=sw0-p1 +check ovn-sbctl set service_monitor sw0-p1 status=online + +AT_CAPTURE_FILE([sbflows]) +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | tee sbflows | grep 'priority=120.*backends' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80);) +]) + +AS_BOX([A backend added to the load balancer gets its monitor without a recompute]) +check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats +check ovn-nbctl --wait=sb set load_balancer lb1 \ + vips:\"10.0.0.10:80\"=\"10.0.0.3:80,10.0.0.4:80\" \ + ip_port_mappings:10.0.0.4=sw0-p2:10.0.0.2 +wait_row_count Service_Monitor 1 logical_port=sw0-p2 +wait_row_count Service_Monitor 2 +check_engine_stats northd norecompute compute +check_engine_stats lflow norecompute compute + +# The new monitor starts offline, so its backend is not programmed yet. +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | tee sbflows | grep 'priority=120.*backends' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80);) +]) + +AS_BOX([A status change rebuilds the load balancer flows without a recompute]) +check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats +check ovn-sbctl set service_monitor sw0-p2 status=online +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | tee sbflows | grep 'priority=120.*backends' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80,10.0.0.4:80);) +]) +check_engine_stats northd norecompute compute +check_engine_stats lflow norecompute compute + +check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats +check ovn-sbctl set service_monitor sw0-p2 status=offline +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | tee sbflows | grep 'priority=120.*backends' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80);) +]) +check_engine_stats northd norecompute compute +check_engine_stats lflow norecompute compute + +AS_BOX([A health check option change updates the monitors without a recompute]) +check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats +check ovn-nbctl --wait=sb set load_balancer_health_check . options:interval=5 +wait_row_count Service_Monitor 2 options:interval=5 +check_engine_stats northd norecompute compute + +AS_BOX([A monitor that goes online before northd sees its row programs the backend]) +check ovn-nbctl lsp-add sw0 sw0-p3 -- lsp-set-addresses sw0-p3 \ + "00:00:00:00:00:05 10.0.0.5" +check ovn-sbctl lsp-bind sw0-p3 hv1 +wait_row_count nb:Logical_Switch_Port 1 name=sw0-p3 'up=true' +check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats +# Hold the Southbound server so that northd's transaction with the new +# monitor stays in flight, then hold northd once it has sent it. The +# server then commits the insert while northd cannot read the echo, the +# status changes in the meantime, and northd sees the row for the first +# time with the status already online. +on_exit "test -e ovn-sb/ovsdb-server.pid && kill -CONT $(cat ovn-sb/ovsdb-server.pid)" +on_exit "test -e northd/ovn-northd.pid && kill -CONT $(cat northd/ovn-northd.pid)" +AT_CHECK([kill -STOP $(cat ovn-sb/ovsdb-server.pid)]) +check ovn-nbctl set load_balancer lb1 \ + vips:\"10.0.0.10:80\"=\"10.0.0.3:80,10.0.0.4:80,10.0.0.5:80\" \ + ip_port_mappings:10.0.0.5=sw0-p3:10.0.0.2 +OVS_WAIT_UNTIL([test "$(as northd ovn-appctl -t ovn-northd inc-engine/show-stats northd | cut -d '-' -f3 | cut -d ':' -f2 | tr -d ' ')" != "0"]) +AT_CHECK([kill -STOP $(cat northd/ovn-northd.pid)]) +AT_CHECK([kill -CONT $(cat ovn-sb/ovsdb-server.pid)]) +wait_row_count Service_Monitor 1 logical_port=sw0-p3 +check ovn-sbctl set service_monitor sw0-p3 status=online +AT_CHECK([kill -CONT $(cat northd/ovn-northd.pid)]) +check ovn-nbctl --wait=sb sync +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | tee sbflows | grep 'priority=120.*backends' | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.3:80,10.0.0.5:80);) +]) +check_engine_stats northd norecompute compute + +AS_BOX([Backends swapped between two load balancers keep their monitors]) +check ovn-nbctl lb-add lb3 10.0.0.30:80 10.0.0.4:80 \ + -- set load_balancer lb3 ip_port_mappings:10.0.0.4=sw0-p2:10.0.0.2 +check_uuid ovn-nbctl --wait=sb -- --id=@hc create Load_Balancer_Health_Check \ + vip="10.0.0.30\:80" -- add Load_Balancer lb3 health_check @hc +check ovn-nbctl --wait=sb set load_balancer lb1 \ + vips:\"10.0.0.10:80\"=\"10.0.0.3:80\" +wait_row_count Service_Monitor 2 +check ovn-sbctl set service_monitor sw0-p1 status=online +check ovn-sbctl set service_monitor sw0-p2 status=online +check ovn-nbctl --wait=sb sync +sm_p1=$(fetch_column Service_Monitor _uuid logical_port=sw0-p1) +sm_p2=$(fetch_column Service_Monitor _uuid logical_port=sw0-p2) +check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats +check ovn-nbctl --wait=sb \ + -- set load_balancer lb1 vips:\"10.0.0.10:80\"=\"10.0.0.4:80\" \ + ip_port_mappings:10.0.0.4=sw0-p2:10.0.0.2 \ + -- set load_balancer lb3 vips:\"10.0.0.30:80\"=\"10.0.0.3:80\" \ + ip_port_mappings:10.0.0.3=sw0-p1:10.0.0.2 +wait_row_count Service_Monitor 2 +check_engine_stats northd norecompute compute +check test "$sm_p1" = "$(fetch_column Service_Monitor _uuid logical_port=sw0-p1)" +check test "$sm_p2" = "$(fetch_column Service_Monitor _uuid logical_port=sw0-p2)" +check_column online Service_Monitor status logical_port=sw0-p1 +check_column online Service_Monitor status logical_port=sw0-p2 +OVS_WAIT_FOR_OUTPUT( + [ovn-sbctl dump-flows sw0 | tee sbflows | grep 'priority=120.*backends' | grep 10.0.0.10 | ovn_strip_lflows], 0, [dnl + table=??(ls_in_lb ), priority=120 , match=(ct.new && ip4.dst == 10.0.0.10 && reg1[[16..23]] == 6 && reg1[[0..15]] == 80), action=(reg4 = 10.0.0.10; reg2[[0..15]] = 80; ct_lb_mark(backends=10.0.0.4:80);) +]) +check ovn-nbctl --wait=sb \ + -- set load_balancer lb1 vips:\"10.0.0.10:80\"=\"10.0.0.3:80,10.0.0.4:80\" \ + ip_port_mappings:10.0.0.3=sw0-p1:10.0.0.2 \ + -- lb-del lb3 +wait_row_count Service_Monitor 2 +check test "$sm_p1" = "$(fetch_column Service_Monitor _uuid logical_port=sw0-p1)" +check ovn-sbctl set service_monitor sw0-p2 status=offline +check ovn-nbctl --wait=sb sync + +AS_BOX([A backend removed from the load balancer drops its monitor without a recompute]) +check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats +check ovn-nbctl --wait=sb set load_balancer lb1 \ + vips:\"10.0.0.10:80\"=\"10.0.0.3:80\" +wait_row_count Service_Monitor 0 logical_port=sw0-p2 +wait_row_count Service_Monitor 1 +check_engine_stats northd norecompute compute + +AS_BOX([A monitor shared by two load balancers survives the deletion of one]) +check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats +check ovn-nbctl lb-add lb2 10.0.0.20:80 10.0.0.3:80 \ + -- set load_balancer lb2 ip_port_mappings:10.0.0.3=sw0-p1:10.0.0.2 +check_uuid ovn-nbctl --wait=sb -- --id=@hc create Load_Balancer_Health_Check \ + vip="10.0.0.20\:80" -- add Load_Balancer lb2 health_check @hc +wait_row_count Service_Monitor 1 +check_engine_stats northd norecompute compute +check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats +check ovn-nbctl --wait=sb lb-del lb2 +wait_row_count Service_Monitor 1 logical_port=sw0-p1 +check_engine_stats northd norecompute compute + +AS_BOX([A port health check that shares a monitor with a load balancer]) +check ovn-nbctl --wait=sb lsp-hc-add sw0-p1 tcp 10.0.0.2 80 10.0.0.3 +wait_row_count Service_Monitor 1 +# The port loses the address. The port health check is outdated, but the +# load balancer still probes its backend through the same monitor, which +# has to survive the recompute and the next load balancer change. +check ovn-nbctl --wait=sb lsp-set-addresses sw0-p1 \ + "00:00:00:00:00:03 10.0.0.6" +wait_row_count Service_Monitor 1 logical_port=sw0-p1 +check ovn-nbctl --wait=sb set load_balancer lb1 \ + vips:\"10.0.0.10:80\"=\"10.0.0.3:80,10.0.0.4:80\" +wait_row_count Service_Monitor 2 +check ovn-nbctl --wait=sb set load_balancer lb1 \ + vips:\"10.0.0.10:80\"=\"10.0.0.3:80\" +wait_row_count Service_Monitor 1 +check ovn-nbctl --wait=sb lsp-hc-del sw0-p1 +check ovn-nbctl --wait=sb lsp-set-addresses sw0-p1 \ + "00:00:00:00:00:03 10.0.0.3" +wait_row_count Service_Monitor 1 logical_port=sw0-p1 + +AS_BOX([A status change and the deletion of the load balancer in one run]) +check ovn-nbctl lb-add lb4 10.0.0.40:80 10.0.0.4:80 \ + -- set load_balancer lb4 ip_port_mappings:10.0.0.4=sw0-p2:10.0.0.2 +check_uuid ovn-nbctl --wait=sb -- --id=@hc create Load_Balancer_Health_Check \ + vip="10.0.0.40\:80" -- add Load_Balancer lb4 health_check @hc +wait_row_count Service_Monitor 1 logical_port=sw0-p2 +check ovn-sbctl set service_monitor sw0-p2 status=online +check ovn-nbctl --wait=sb sync +check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats +# Hold northd so that it sees the status change and the deletion together. +kill -STOP $(cat northd/ovn-northd.pid) +check ovn-sbctl set service_monitor sw0-p2 status=offline +check ovn-nbctl lb-del lb4 +kill -CONT $(cat northd/ovn-northd.pid) +check ovn-nbctl --wait=sb sync +wait_row_count Service_Monitor 0 logical_port=sw0-p2 +check_engine_stats northd norecompute compute +CHECK_NO_CHANGE_AFTER_RECOMPUTE + +AS_BOX([A monitor deleted behind northd's back is recreated by a recompute]) +check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats +check ovn-sbctl destroy service_monitor sw0-p1 +wait_row_count Service_Monitor 1 logical_port=sw0-p1 +check_engine_compute northd recompute + +AS_BOX([Deleting the load balancer drops its monitor]) +check ovn-nbctl --wait=sb lb-del lb1 +wait_row_count Service_Monitor 0 + +OVN_CLEANUP_NORTHD +AT_CLEANUP +]) + OVN_FOR_EACH_NORTHD_NO_HV([ AT_SETUP([Load balancer VIP in NAT entries]) AT_SKIP_IF([test $HAVE_PYTHON = no]) @@ -13336,28 +13548,29 @@ AT_CHECK([ovn-nbctl --wait=sb \ -- add Load_Balancer . health_check @hc | uuidfilt], [0], [<0> ]) check_engine_stats lb_data norecompute compute -check_engine_stats northd recompute nocompute -check_engine_stats lr_stateful recompute nocompute -check_engine_stats lflow recompute nocompute -check_engine_stats sync_to_sb_lb recompute nocompute +check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute +check_engine_stats lflow norecompute compute +check_engine_stats sync_to_sb_lb norecompute compute -# Any change to load balancer health check should also result in full recompute -# of northd node (but not northd_lb_data node) +# A change to a load balancer health check updates the service monitors of +# the load balancer without a full recompute. check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats check ovn-nbctl --wait=sb set load_balancer_health_check . options:foo=bar1 check_engine_stats lb_data norecompute compute -check_engine_stats northd recompute nocompute -check_engine_stats lflow recompute nocompute -check_engine_stats sync_to_sb_lb recompute nocompute +check_engine_stats northd norecompute compute +check_engine_stats lflow norecompute compute +check_engine_stats sync_to_sb_lb norecompute compute -# Delete the health check from the load balancer. northd engine node should do a full recompute. +# Delete the health check from the load balancer. The northd engine node +# drops the service monitors without a full recompute. check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats check ovn-nbctl --wait=sb clear Load_Balancer . health_check check_engine_stats lb_data norecompute compute -check_engine_stats northd recompute nocompute -check_engine_stats lr_stateful recompute nocompute -check_engine_stats lflow recompute nocompute -check_engine_stats sync_to_sb_lb recompute nocompute +check_engine_stats northd norecompute compute +check_engine_stats lr_stateful norecompute compute +check_engine_stats lflow norecompute compute +check_engine_stats sync_to_sb_lb norecompute compute check as northd ovn-appctl -t ovn-northd inc-engine/clear-stats check ovn-nbctl ls-add sw0 -- 2.53.0 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
