For each vrf we use we open a netlink watcher. This allows us to reconcile on changed route entries from outside routing agents.
Signed-off-by: Felix Huettner <felix.huettner@stackit.cloud> --- controller/automake.mk | 7 +- controller/ovn-controller.c | 48 ++++++- controller/route-exchange.c | 6 +- controller/route-exchange.h | 3 + controller/route-table-notify-stub.c | 44 +++++++ controller/route-table-notify.c | 179 +++++++++++++++++++++++++++ controller/route-table-notify.h | 37 ++++++ tests/system-ovn.at | 4 - 8 files changed, 319 insertions(+), 9 deletions(-) create mode 100644 controller/route-table-notify-stub.c create mode 100644 controller/route-table-notify.c create mode 100644 controller/route-table-notify.h diff --git a/controller/automake.mk b/controller/automake.mk index 66aff8643..df24a674f 100644 --- a/controller/automake.mk +++ b/controller/automake.mk @@ -53,6 +53,7 @@ controller_ovn_controller_SOURCES = \ controller/ovn-dns.c \ controller/ovn-dns.h \ controller/route-exchange.h \ + controller/route-table-notify.h \ controller/route.h \ controller/route.c @@ -60,10 +61,12 @@ if HAVE_NETLINK controller_ovn_controller_SOURCES += \ controller/route-exchange-netlink.h \ controller/route-exchange-netlink.c \ - controller/route-exchange.c + controller/route-exchange.c \ + controller/route-table-notify.c else controller_ovn_controller_SOURCES += \ - controller/route-exchange-stub.c + controller/route-exchange-stub.c \ + controller/route-table-notify-stub.c endif controller_ovn_controller_LDADD = lib/libovn.la $(OVS_LIBDIR)/libopenvswitch.la diff --git a/controller/ovn-controller.c b/controller/ovn-controller.c index 612fccb42..70c3a5d1a 100644 --- a/controller/ovn-controller.c +++ b/controller/ovn-controller.c @@ -90,6 +90,7 @@ #include "ovn-dns.h" #include "route.h" #include "route-exchange.h" +#include "route-table-notify.h" VLOG_DEFINE_THIS_MODULE(main); @@ -5104,10 +5105,14 @@ en_route_exchange_run(struct engine_node *node, void *data) .announce_routes = &route_data->announce_routes, }; - struct route_exchange_ctx_out r_ctx_out = { - }; + struct route_exchange_ctx_out r_ctx_out = {}; + + hmap_init(&r_ctx_out.route_table_watches); route_exchange_run(&r_ctx_in, &r_ctx_out); + route_table_notify_update_watches(&r_ctx_out.route_table_watches); + + hmap_destroy(&r_ctx_out.route_table_watches); engine_set_node_state(node, EN_UPDATED); } @@ -5126,6 +5131,38 @@ static void en_route_exchange_cleanup(void *data OVS_UNUSED) {} +struct ed_type_route_table_notify { + /* For incremental processing this could be tracked per datapath in + * the future. */ + bool changed; +}; + +static void +en_route_table_notify_run(struct engine_node *node, void *data) +{ + struct ed_type_route_table_notify *rtn = data; + if (rtn->changed) { + engine_set_node_state(node, EN_UPDATED); + } else { + engine_set_node_state(node, EN_UNCHANGED); + } + rtn->changed = false; +} + + +static void * +en_route_table_notify_init(struct engine_node *node OVS_UNUSED, + struct engine_arg *arg OVS_UNUSED) +{ + struct ed_type_route_table_notify *rtn = xzalloc(sizeof *rtn); + rtn->changed = true; + return rtn; +} + +static void +en_route_table_notify_cleanup(void *data OVS_UNUSED) +{} + /* Returns false if the northd internal version stored in SB_Global * and ovn-controller internal version don't match. */ @@ -5425,6 +5462,7 @@ main(int argc, char *argv[]) ENGINE_NODE(bfd_chassis, "bfd_chassis"); ENGINE_NODE(dns_cache, "dns_cache"); ENGINE_NODE(route, "route"); + ENGINE_NODE(route_table_notify, "route_table_notify"); ENGINE_NODE(route_exchange, "route_exchange"); #define SB_NODE(NAME, NAME_STR) ENGINE_NODE_SB(NAME, NAME_STR); @@ -5462,6 +5500,7 @@ main(int argc, char *argv[]) engine_noop_handler); engine_add_input(&en_route_exchange, &en_sb_port_binding, engine_noop_handler); + engine_add_input(&en_route_exchange, &en_route_table_notify, NULL); engine_add_input(&en_addr_sets, &en_sb_address_set, addr_sets_sb_address_set_handler); @@ -5979,6 +6018,10 @@ main(int argc, char *argv[]) &transport_zones, bridge_table); + struct ed_type_route_table_notify *rtn = + engine_get_internal_data(&en_route_table_notify); + route_table_notify_run(&rtn->changed); + stopwatch_start(CONTROLLER_LOOP_STOPWATCH_NAME, time_msec()); @@ -6254,6 +6297,7 @@ main(int argc, char *argv[]) } binding_wait(); + route_table_notify_wait(); } unixctl_server_run(unixctl); diff --git a/controller/route-exchange.c b/controller/route-exchange.c index dc3e5657c..e7a85eecf 100644 --- a/controller/route-exchange.c +++ b/controller/route-exchange.c @@ -29,6 +29,7 @@ #include "ha-chassis.h" #include "local_data.h" #include "route.h" +#include "route-table-notify.h" #include "route-exchange.h" #include "route-exchange-netlink.h" @@ -169,7 +170,7 @@ sb_sync_learned_routes(const struct ovs_list *learned_routes, void route_exchange_run(struct route_exchange_ctx_in *r_ctx_in, - struct route_exchange_ctx_out *r_ctx_out OVS_UNUSED) + struct route_exchange_ctx_out *r_ctx_out) { struct sset old_maintained_vrfs = SSET_INITIALIZER(&old_maintained_vrfs); sset_swap(&_maintained_vrfs, &old_maintained_vrfs); @@ -211,6 +212,9 @@ route_exchange_run(struct route_exchange_ctx_in *r_ctx_in, r_ctx_in->sbrec_port_binding_by_name, r_ctx_in->sbrec_learned_route_by_datapath); + route_table_add_watch_request( + &r_ctx_out->route_table_watches, table_id); + re_nl_learned_routes_destroy(&received_routes); } diff --git a/controller/route-exchange.h b/controller/route-exchange.h index d23bb37a2..0eb0477ee 100644 --- a/controller/route-exchange.h +++ b/controller/route-exchange.h @@ -18,6 +18,8 @@ #ifndef ROUTE_EXCHANGE_H #define ROUTE_EXCHANGE_H 1 +#include "openvswitch/hmap.h" + struct route_exchange_ctx_in { struct ovsdb_idl_txn *ovnsb_idl_txn; struct ovsdb_idl_index *sbrec_port_binding_by_name; @@ -27,6 +29,7 @@ struct route_exchange_ctx_in { }; struct route_exchange_ctx_out { + struct hmap route_table_watches; }; void route_exchange_run(struct route_exchange_ctx_in *, diff --git a/controller/route-table-notify-stub.c b/controller/route-table-notify-stub.c new file mode 100644 index 000000000..0a83a3e93 --- /dev/null +++ b/controller/route-table-notify-stub.c @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2024, STACKIT GmbH & Co. KG + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <config.h> + +#include <stdbool.h> + +#include "openvswitch/compiler.h" +#include "route-table-notify.h" + +void +route_table_notify_run(bool *changed) +{ + *changed = false; +} + +void +route_table_notify_wait(void) +{ +} + +void +route_table_add_watch_request(struct hmap *route_table_watches OVS_UNUSED, + uint32_t table_id OVS_UNUSED) { +} + +void +route_table_notify_update_watches(struct hmap *route_table_watches OVS_UNUSED) +{ +} + diff --git a/controller/route-table-notify.c b/controller/route-table-notify.c new file mode 100644 index 000000000..0d931e133 --- /dev/null +++ b/controller/route-table-notify.c @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2024, STACKIT GmbH & Co. KG + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <config.h> + +#include <net/if.h> +#include <linux/rtnetlink.h> + +#include "netlink-notifier.h" +#include "openvswitch/vlog.h" + +#include "binding.h" +#include "hash.h" +#include "hmapx.h" +#include "route-table.h" +#include "route.h" +#include "route-table-notify.h" +#include "route-exchange-netlink.h" + +VLOG_DEFINE_THIS_MODULE(route_table_notify); + +struct route_table_watch_request { + struct hmap_node node; + uint32_t table_id; +}; + +struct route_table_watch_entry { + struct hmap_node node; + uint32_t table_id; + struct nln *nln; + struct nln_notifier *route_notifier; + struct nln_notifier *route6_notifier; +}; + +static struct hmap watches = HMAP_INITIALIZER(&watches); +static bool any_route_table_changed; +static struct route_table_msg rtmsg; + +static uint32_t +route_table_notify_hash_watch(uint32_t table_id) +{ + return hash_add(0, table_id); +} + +void +route_table_add_watch_request(struct hmap *route_table_watches, + uint32_t table_id) +{ + struct route_table_watch_request *wr = xzalloc(sizeof *wr); + wr->table_id = table_id; + hmap_insert(route_table_watches, &wr->node, + route_table_notify_hash_watch(wr->table_id)); +} + +static void +route_table_change(const void *change_, void *aux OVS_UNUSED) +{ + const struct route_table_msg *change = change_; + if (change && change->rd.rtm_protocol != RTPROT_OVN) { + any_route_table_changed = true; + } +} + +static void +add_watch_entry(uint32_t table_id) +{ + struct route_table_watch_entry *we; + uint32_t hash = route_table_notify_hash_watch(table_id); + we = xzalloc(sizeof *we); + we->table_id = table_id; + VLOG_INFO("Registering new route table watcher for table %d.", + table_id); + we->nln = nln_create(NETLINK_ROUTE, route_table_parse, &rtmsg); + + we->route_notifier = + nln_notifier_create(we->nln, RTNLGRP_IPV4_ROUTE, + route_table_change, NULL); + if (!we->route_notifier) { + VLOG_WARN("Failed to create ipv4 route table watcher for " + "table %d. We will miss routing table updates.", table_id); + } + + we->route6_notifier = + nln_notifier_create(we->nln, RTNLGRP_IPV6_ROUTE, + route_table_change, NULL); + if (!we->route6_notifier) { + VLOG_WARN("Failed to create ipv6 route table watcher for " + "table %d. We will miss routing table updates.", table_id); + } + + hmap_insert(&watches, &we->node, hash); +} + +static void +remove_watch_entry(struct route_table_watch_entry *we) +{ + hmap_remove(&watches, &we->node); + VLOG_INFO("Removing route table watcher for table %d.", + we->table_id); + nln_notifier_destroy(we->route_notifier); + nln_notifier_destroy(we->route6_notifier); + nln_destroy(we->nln); + free(we); +} + +void +route_table_notify_run(bool *changed) +{ + any_route_table_changed = false; + + struct route_table_watch_entry *we; + HMAP_FOR_EACH (we, node, &watches) { + nln_run(we->nln); + } + + *changed = any_route_table_changed; +} + +void +route_table_notify_wait(void) +{ + struct route_table_watch_entry *we; + HMAP_FOR_EACH (we, node, &watches) { + nln_wait(we->nln); + } +} + +static struct route_table_watch_entry* +find_watch_entry(uint32_t table_id) +{ + struct route_table_watch_entry *we; + uint32_t hash = route_table_notify_hash_watch(table_id); + HMAP_FOR_EACH_WITH_HASH (we, node, hash, &watches) { + if (table_id == we->table_id) { + return we; + } + } + return NULL; +} + +void +route_table_notify_update_watches(struct hmap *route_table_watches) +{ + struct hmapx sync_watches = HMAPX_INITIALIZER(&sync_watches); + struct route_table_watch_entry *we; + HMAP_FOR_EACH (we, node, &watches) { + hmapx_add(&sync_watches, we); + } + + struct route_table_watch_request *wr; + HMAP_FOR_EACH_SAFE (wr, node, route_table_watches) { + we = find_watch_entry(wr->table_id); + if (we) { + hmapx_find_and_delete(&sync_watches, we); + } else { + add_watch_entry(wr->table_id); + } + hmap_remove(route_table_watches, &wr->node); + free(wr); + } + + struct hmapx_node *node; + HMAPX_FOR_EACH (node, &sync_watches) { + remove_watch_entry(node->data); + } +} diff --git a/controller/route-table-notify.h b/controller/route-table-notify.h new file mode 100644 index 000000000..b7211f6e7 --- /dev/null +++ b/controller/route-table-notify.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2024, STACKIT GmbH & Co. KG + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ROUTE_TABLE_NOTIFY_H +#define ROUTE_TABLE_NOTIFY_H 1 + +#include <stdbool.h> +#include "openvswitch/hmap.h" + +/* Sets "changed" to true if any route table has changed enough that we need + * to learn new routes. */ +void route_table_notify_run(bool *changed); +void route_table_notify_wait(void); + +/* Add a watch request to the hmap. The hmap should later be passed to + * route_table_notify_update_watches*/ +void route_table_add_watch_request(struct hmap *route_table_watches, + uint32_t table_id); + +/* Updates the list of route table watches that are currently active. + * hmap should contain struct route_table_watch_request */ +void route_table_notify_update_watches(struct hmap *route_table_watches); + +#endif /* ROUTE_TABLE_NOTIFY_H */ diff --git a/tests/system-ovn.at b/tests/system-ovn.at index ef43e242e..3f5672dad 100644 --- a/tests/system-ovn.at +++ b/tests/system-ovn.at @@ -15075,8 +15075,6 @@ blackhole 198.51.100.0/24 proto 84 metric 1000]) # Now we test route learning. check_row_count Learned_Route 0 check ip route add 233.252.0.0/24 via 192.168.10.10 dev lo onlink vrf ovnvrf1337 -# For now we trigger a recompute as route watching is not yet implemented. -check ovn-appctl -t ovn-controller inc-engine/recompute check ovn-nbctl --wait=hv sync check_row_count Learned_Route 1 lp=$(fetch_column port_binding _uuid logical_port=internet-phys) @@ -15324,8 +15322,6 @@ blackhole 198.51.100.0/24 proto 84 metric 1000]) # Now we test route learning. check_row_count Learned_Route 0 check ip route add 233.252.0.0/24 via 192.168.10.10 dev lo onlink vrf ovnvrf1337 -# For now we trigger a recompute as route watching is not yet implemented. -check ovn-appctl -t ovn-controller inc-engine/recompute check ovn-nbctl --wait=hv sync check_row_count Learned_Route 1 lp=$(fetch_column port_binding _uuid logical_port=internet-phys) -- 2.47.1 _______________________________________________ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev