If logical router has more than one LRP as gateway router port and dynamic routing configured, dynamic-routing-port-name could be used to specify the LRP that will be used to dynamic routing. However, if all LRPs learning routes, routes from LRP without dynamic-routing-port-name must be flushed. This happens when LRPs are scheduled in the same chassis.
Signed-off-by: Lucas Vargas Dias <[email protected]> --- controller/ovn-controller.c | 15 ++++ controller/route-exchange.c | 48 ++++++++++- controller/route-exchange.h | 1 + tests/multinode.at | 142 +++++++++++++++++++++++++++++++ tests/ovn-inc-proc-graph-dump.at | 2 + tests/system-ovn.at | 133 +++++++++++++++++++++++++++++ 6 files changed, 337 insertions(+), 4 deletions(-) diff --git a/controller/ovn-controller.c b/controller/ovn-controller.c index fd848c54c..e65962d9f 100644 --- a/controller/ovn-controller.c +++ b/controller/ovn-controller.c @@ -5681,11 +5681,24 @@ en_route_exchange_run(struct engine_node *node, void *data) } vector_clear(&rt_notify->watches); + const struct ovsrec_open_vswitch_table *ovs_table = + EN_OVSDB_GET(engine_get_input("OVS_open_vswitch", node)); + const char *chassis_id = get_ovs_chassis_id(ovs_table); + ovs_assert(chassis_id); + + struct ovsdb_idl_index *sbrec_chassis_by_name = + engine_ovsdb_node_get_index( + engine_get_input("SB_chassis", node), + "name"); + const struct sbrec_chassis *chassis + = chassis_lookup_by_name(sbrec_chassis_by_name, chassis_id); + ovs_assert(chassis); struct route_exchange_ctx_in r_ctx_in = { .ovnsb_idl_txn = engine_get_context()->ovnsb_idl_txn, .sbrec_learned_route_by_datapath = sbrec_learned_route_by_datapath, .sbrec_port_binding_by_name = sbrec_port_binding_by_name, + .chassis = chassis, .announce_routes = &route_data->announce_routes, }; struct route_exchange_ctx_out r_ctx_out = { @@ -7003,6 +7016,8 @@ inc_proc_ovn_controller_init( engine_add_input(&en_route, &en_sb_datapath_binding, route_sb_datapath_binding_handler); + engine_add_input(&en_route_exchange, &en_ovs_open_vswitch, NULL); + engine_add_input(&en_route_exchange, &en_sb_chassis, NULL); engine_add_input(&en_route_exchange, &en_route, NULL); engine_add_input(&en_route_exchange, &en_sb_learned_route, engine_noop_handler); diff --git a/controller/route-exchange.c b/controller/route-exchange.c index b86eb43bf..a3c0968c6 100644 --- a/controller/route-exchange.c +++ b/controller/route-exchange.c @@ -27,6 +27,7 @@ #include "openvswitch/list.h" #include "lib/ovn-sb-idl.h" +#include "lib/uuidset.h" #include "binding.h" #include "ha-chassis.h" @@ -86,7 +87,7 @@ maintained_route_table_add(uint32_t table_id) hmap_insert(&_maintained_route_tables, &mrt->node, hash); } -static void +static struct route_entry * route_add_entry(struct hmap *routes, const struct sbrec_learned_route *sb_route, bool stale) @@ -102,6 +103,7 @@ route_add_entry(struct hmap *routes, hash = hash_string(sb_route->ip_prefix, hash); hmap_insert(routes, &route_e->hmap_node, hash); + return route_e; } static struct route_entry * @@ -144,28 +146,65 @@ sb_sync_learned_routes(const struct vector *learned_routes, struct ovsdb_idl_txn *ovnsb_idl_txn, struct ovsdb_idl_index *sbrec_port_binding_by_name, struct ovsdb_idl_index *sbrec_learned_route_by_datapath, - bool *sb_changes_pending) + bool *sb_changes_pending, + const struct sbrec_chassis *chassis) { struct hmap sync_routes = HMAP_INITIALIZER(&sync_routes); const struct sbrec_learned_route *sb_route; - struct route_entry *route_e; + struct route_entry *route_e = NULL; + struct uuidset lrp_with_dr_port_name = + UUIDSET_INITIALIZER(&lrp_with_dr_port_name); struct sbrec_learned_route *filter = sbrec_learned_route_index_init_row(sbrec_learned_route_by_datapath); sbrec_learned_route_index_set_datapath(filter, datapath); SBREC_LEARNED_ROUTE_FOR_EACH_EQUAL (sb_route, filter, sbrec_learned_route_by_datapath) { + const struct sbrec_port_binding *cr_pb = + lport_get_cr_port(sbrec_port_binding_by_name, + sb_route->logical_port, NULL); + const char *dynamic_routing_port_name = + smap_get(&sb_route->logical_port->options, + "dynamic-routing-port-name"); + if (!dynamic_routing_port_name && cr_pb) { + dynamic_routing_port_name = + smap_get(&cr_pb->options, "dynamic-routing-port-name"); + } + + if (sb_route->logical_port->chassis == chassis || + (cr_pb && cr_pb->chassis == chassis)) { + route_e = route_add_entry(&sync_routes, sb_route, false); + if (dynamic_routing_port_name) { + uuidset_insert(&lrp_with_dr_port_name, + &sb_route->logical_port->header_.uuid); + } + } + /* If the port is not local we don't care about it. * Some other ovn-controller will handle it. * We may not use smap_get since the value might be validly NULL. */ if (!smap_get_node(bound_ports, sb_route->logical_port->logical_port)) { + route_e = NULL; + continue; + } + if (route_e) { + route_e->stale = true; continue; } route_add_entry(&sync_routes, sb_route, true); } sbrec_learned_route_index_destroy_row(filter); + if (!uuidset_is_empty(&lrp_with_dr_port_name)) { + HMAP_FOR_EACH_SAFE (route_e, hmap_node, &sync_routes) { + if (!uuidset_find(&lrp_with_dr_port_name, + &route_e->sb_route->logical_port->header_.uuid)) { + route_e->stale = true; + } + } + } + uuidset_destroy(&lrp_with_dr_port_name); struct re_nl_received_route_node *learned_route; VECTOR_FOR_EACH_PTR (learned_routes, learned_route) { char *ip_prefix = normalize_v46_prefix(&learned_route->prefix, @@ -363,7 +402,8 @@ route_exchange_run(const struct route_exchange_ctx_in *r_ctx_in, r_ctx_in->ovnsb_idl_txn, r_ctx_in->sbrec_port_binding_by_name, sbrec_learned_route_by_datapath, - &r_ctx_out->sb_changes_pending); + &r_ctx_out->sb_changes_pending, + r_ctx_in->chassis); } vector_push(r_ctx_out->route_table_watches, &arte->table_id); vector_destroy(&received_routes); diff --git a/controller/route-exchange.h b/controller/route-exchange.h index 25db35568..a1ef4a359 100644 --- a/controller/route-exchange.h +++ b/controller/route-exchange.h @@ -24,6 +24,7 @@ struct route_exchange_ctx_in { struct ovsdb_idl_txn *ovnsb_idl_txn; struct ovsdb_idl_index *sbrec_port_binding_by_name; struct ovsdb_idl_index *sbrec_learned_route_by_datapath; + const struct sbrec_chassis *chassis; /* Contains struct advertise_datapath_entry */ const struct hmap *announce_routes; diff --git a/tests/multinode.at b/tests/multinode.at index 069f2a677..e5be00be7 100644 --- a/tests/multinode.at +++ b/tests/multinode.at @@ -2984,6 +2984,148 @@ OVS_WAIT_UNTIL([m_as ovn-gw-2 ip netns exec frr-ns ping -W 1 -c 1 172.16.10.2]) AT_CLEANUP +AT_SETUP([ovn multinode dynamic-routing - BGP learned routes with router filter name and multiple DGPs]) + +# This is the multinode counterpart of the system test +# "dynamic-routing - BGP learned routes with router filter name and multiple +# DGPs": a single logical router with dynamic routing enabled has two +# distributed gateway ports scheduled on different chassis. Both LRPs are used +# for dynamic routing and each has its own dynamic-routing-port-name filter +# resolving to the interface local to the chassis hosting the port. +# +# Each chassis learns a route for the same prefix from its own VRF, so both +# gateway ports learn it. Because every LRP has a dynamic-routing-port-name +# that resolves to a local interface, neither route is flushed: the route +# learned on ovn-gw-1 (lrp-local-bgp-port) and the route learned on ovn-gw-2 +# (lrp-dgp-dummy) both remain. +# +# Topology: +# +# public ls-dummy +# (local-bgp-port, ovn-gw-1) (ovn-gw-2) +# | | +# lrp-local-bgp-port lrp-dgp-dummy +# (DGP @ ovn-gw-1, (DGP @ ovn-gw-2, +# port-name local-bgp-port) port-name dummy-bgp-port) +# \ / +# +------------- lr-frr -----------+ +# (dynamic-routing, vrf-id $vrf) + +# Check that ovn-fake-multinode setup is up and running. +check_fake_multinode_setup + +CHECK_VRF() + +# Delete the multinode NB and OVS resources before starting the test. +cleanup_multinode_resources + +CHECK_VRF() + +vrf=1000 + +# Create the VRF manually on both gateway chassis (dynamic-routing-maintain-vrf +# is false, so OVN does not create it) and learn/read routes from table $vrf. +for gw in ovn-gw-1 ovn-gw-2; do + check m_as $gw ip link add vrf-bgp type vrf table $vrf + on_exit "m_as $gw ip link del vrf-bgp" + check m_as $gw ip link set vrf-bgp up +done + +# Create the logical router with dynamic routing in VRF $vrf. +check multinode_nbctl lr-add lr-frr +check multinode_nbctl set Logical_Router lr-frr \ + options:dynamic-routing=true \ + options:dynamic-routing-vrf-id=$vrf \ + options:dynamic-routing-redistribute=static + +# The DGP that is used for dynamic routing, scheduled on ovn-gw-1. +check multinode_nbctl lrp-add lr-frr lrp-local-bgp-port 00:00:00:00:00:03 20.0.0.3/24 \ + -- set Logical_Router_Port lrp-local-bgp-port \ + options:dynamic-routing-maintain-vrf=false \ + -- set Logical_Router_Port lrp-local-bgp-port \ + options:routing-protocol-redirect=local-bgp-port +check multinode_nbctl lrp-set-gateway-chassis lrp-local-bgp-port ovn-gw-1 + +check multinode_nbctl ls-add public +check multinode_nbctl lsp-add-router-port public public-lr-frr lrp-local-bgp-port +check multinode_nbctl lsp-add public local-bgp-port \ + -- lsp-set-addresses local-bgp-port unknown + +# The second DGP used for dynamic routing, scheduled on ovn-gw-2. bgp-dummy is +# not an OVN logical port, so the dynamic-routing-port-name filter is resolved +# through the dynamic-routing-port-mapping configured on ovn-gw-2 below. +check multinode_nbctl lrp-add lr-frr lrp-dgp-dummy 00:00:00:00:00:04 20.0.1.3/24 \ + -- set Logical_Router_Port lrp-dgp-dummy \ + options:dynamic-routing-maintain-vrf=false +check multinode_nbctl lrp-set-gateway-chassis lrp-dgp-dummy ovn-gw-2 + +check multinode_nbctl ls-add ls-dummy +check multinode_nbctl lsp-add-router-port ls-dummy lsp-dummy lrp-dgp-dummy + +check multinode_nbctl --wait=hv sync + +# Bind local-bgp-port on ovn-gw-1 and move its interface into the VRF. The +# dynamic-routing-port-name filter will be resolved to this interface name. +check m_as ovn-gw-1 ovs-vsctl add-port br-int local-bgp-port \ + -- set Interface local-bgp-port type=internal \ + -- set Interface local-bgp-port external_ids:iface-id=local-bgp-port +on_exit "m_as ovn-gw-1 ovs-vsctl del-port br-int local-bgp-port" +check m_as ovn-gw-1 ip link set local-bgp-port master vrf-bgp +check m_as ovn-gw-1 ip link set local-bgp-port address 00:00:00:00:00:03 +check m_as ovn-gw-1 ip addr add dev local-bgp-port 20.0.0.3/24 +check m_as ovn-gw-1 ip link set local-bgp-port up + +# ovn-gw-2 hosts lrp-dgp-dummy. Add an interface to its VRF so a route can be +# present in the VRF table and get learned on this gateway port, and map the +# dynamic-routing-port-name (dummy-bgp-port) to this interface so the filter +# resolves locally on ovn-gw-2. +check m_as ovn-gw-2 ip link add bgp-dummy type dummy +on_exit "m_as ovn-gw-2 ip link del bgp-dummy" +check m_as ovn-gw-2 ip link set bgp-dummy master vrf-bgp +check m_as ovn-gw-2 ip addr add dev bgp-dummy 20.0.1.3/24 +check m_as ovn-gw-2 ip link set bgp-dummy up + +check m_as ovn-gw-2 ovs-vsctl set open . \ + external-ids:dynamic-routing-port-mapping="dummy-bgp-port=bgp-dummy" +on_exit "m_as ovn-gw-2 ovs-vsctl remove open . external-ids dynamic-routing-port-mapping" + +m_wait_for_ports_up +check multinode_nbctl --wait=hv sync + +# Both gateway ports are flagged for dynamic routing. +m_check_row_count Port_Binding 1 logical_port=cr-lrp-local-bgp-port 'options:dynamic-routing=true' +m_check_row_count Port_Binding 1 logical_port=cr-lrp-dgp-dummy 'options:dynamic-routing=true' + +# Simulate a route learned via a dynamic routing protocol in each chassis' VRF +# for the same prefix. +check m_as ovn-gw-1 ip route add 10.10.3.0/24 via 20.0.0.25 vrf vrf-bgp proto bgp +check m_as ovn-gw-2 ip route add 10.10.3.0/24 via 20.0.1.25 vrf vrf-bgp proto bgp + +# Each chassis learns the route on its own gateway port, so there are two +# Learned_Route rows for the prefix. +m_wait_row_count Learned_Route 2 ip_prefix=10.10.3.0/24 + +# Configure a dynamic-routing-port-name filter on both LRPs, each resolving to +# the interface local to the chassis hosting the port: lrp-local-bgp-port -> +# local-bgp-port (ovn-gw-1) and lrp-dgp-dummy -> dummy-bgp-port (ovn-gw-2, via +# the port-mapping configured above). +check multinode_nbctl set Logical_Router_Port lrp-local-bgp-port \ + options:dynamic-routing-port-name=local-bgp-port +check multinode_nbctl --wait=hv set Logical_Router_Port lrp-dgp-dummy \ + options:dynamic-routing-port-name=dummy-bgp-port + +local_lp=$(m_fetch_column port_binding _uuid logical_port=lrp-local-bgp-port) +dummy_lp=$(m_fetch_column port_binding _uuid logical_port=lrp-dgp-dummy) + +# Both filters resolve to a local interface, so no route is flushed: the route +# learned on ovn-gw-1 (lrp-local-bgp-port) and the route learned on ovn-gw-2 +# (lrp-dgp-dummy) both remain. +m_wait_row_count Learned_Route 2 ip_prefix=10.10.3.0/24 +m_wait_row_count Learned_Route 1 ip_prefix=10.10.3.0/24 logical_port=$local_lp +m_wait_row_count Learned_Route 1 ip_prefix=10.10.3.0/24 logical_port=$dummy_lp + +AT_CLEANUP + AT_SETUP([HA: Check for missing garp on leader when BFD goes back up]) # Network topology # ┌────────────────────────────────────────────────────────────────────────────────────────────────────────┐ diff --git a/tests/ovn-inc-proc-graph-dump.at b/tests/ovn-inc-proc-graph-dump.at index 3750339d0..4fe39cf69 100644 --- a/tests/ovn-inc-proc-graph-dump.at +++ b/tests/ovn-inc-proc-graph-dump.at @@ -461,6 +461,8 @@ digraph "Incremental-Processing-Engine" { route_table_notify [[style=filled, shape=box, fillcolor=white, label="route_table_notify"]]; route_exchange_status [[style=filled, shape=box, fillcolor=white, label="route_exchange_status"]]; route_exchange [[style=filled, shape=box, fillcolor=white, label="route_exchange"]]; + OVS_open_vswitch -> route_exchange [[label=""]]; + SB_chassis -> route_exchange [[label=""]]; route -> route_exchange [[label=""]]; SB_learned_route -> route_exchange [[label="engine_noop_handler"]]; SB_port_binding -> route_exchange [[label="engine_noop_handler"]]; diff --git a/tests/system-ovn.at b/tests/system-ovn.at index 35df0ec2f..af2dc8a83 100644 --- a/tests/system-ovn.at +++ b/tests/system-ovn.at @@ -21865,3 +21865,136 @@ OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d AT_CLEANUP ]) + +OVN_FOR_EACH_NORTHD([ +AT_SETUP([dynamic-routing - BGP learned routes with router filter name and multiple DGPs]) + +# This test validates that BGP learned routes work correctly: +# 1. Routes added to the VRF appear in Learned_Route table +# 2. Remove routes learned in LRPs in same hypervisor when configure dynamic-routing-port-name +# 3. Keep learned route just in LRP with dynamic-routing-port-name configured. +# +# Topology: +# +---------+ +# | public | +# +----+----+ +# | +# +----+---+ +---------------+ +# | lr-frr |-----| lrp-dgp-dummy | (dynamic-routing-port-name NOT configured (hv1)) +# +----+---+ +---------------+ +# | +# +------+-----------+ +# |lrp-local-bgp-port| (dynamic-routing-port-name configured (hv1)) +# +------------------+ + +ovn_start +OVS_TRAFFIC_VSWITCHD_START() +ADD_BR([br-int]) +ADD_BR([br-ex]) + +check ovs-ofctl add-flow br-ex action=normal + +# Set external-ids in br-int needed for ovn-controller +check ovs-vsctl \ + -- set Open_vSwitch . external-ids:system-id=hv1 \ + -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \ + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true + +# Configure bridge mappings for localnet. +check ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=phys:br-ex + +vrf=10 +VRF_RESERVE([$vrf]) + +# Start ovn-controller. +start_daemon ovn-controller + +check ip link add vrf-$vrf type vrf table $vrf +on_exit "ip link del vrf-$vrf" +check ip link set vrf-$vrf up + +# Create public logical switch with localnet port. +check ovn-nbctl ls-add public +check ovn-nbctl lsp-add-localnet-port public ln_port phys + +# Create lr-frr with dynamic routing in VRF $vrf. +check ovn-nbctl lr-add lr-frr \ + -- set Logical_Router lr-frr \ + options:dynamic-routing=true \ + options:dynamic-routing-vrf-id=$vrf \ + options:dynamic-routing-redistribute=static + +check ovn-nbctl lrp-add lr-frr lrp-local-bgp-port 00:00:00:00:00:03 20.0.0.3/24 \ + -- set Logical_Router_Port lrp-local-bgp-port options:dynamic-routing-maintain-vrf=false \ + -- set Logical_Router_Port lrp-local-bgp-port options:routing-protocol-redirect=local-bgp-port + +check ovn-nbctl lrp-set-gateway-chassis lrp-local-bgp-port hv1 +check ovn-nbctl lsp-add-router-port public public-lr-frr lrp-local-bgp-port + +check ovn-nbctl lrp-add lr-frr lrp-dgp-dummy 00:00:00:00:00:04 20.0.1.3/24 +check ovn-nbctl lrp-set-gateway-chassis lrp-dgp-dummy hv1 +check ovn-nbctl ls-add ls-dummy +check ovn-nbctl lsp-add-router-port ls-dummy lsp-dummy lrp-dgp-dummy + +# Create local-bgp-port in VRF 10. +check ovs-vsctl add-port br-int local-bgp-port \ + -- set Interface local-bgp-port type=internal \ + -- set Interface local-bgp-port external_ids:iface-id=local-bgp-port + +check ovn-nbctl lsp-add public local-bgp-port \ + -- lsp-set-addresses local-bgp-port unknown + +# Configure local-bgp-port interface and add to VRF. +check ip link set local-bgp-port master vrf-$vrf +check ip link set local-bgp-port address 00:00:00:00:00:03 +check ip addr add dev local-bgp-port 20.0.0.3/24 +check ip link set local-bgp-port up + +# Wait for everything to be ready. +wait_for_ports_up +check ovn-nbctl --wait=hv sync + +# Check lrp-local-bgp-port has dynamic-routing option set. +check_row_count Port_Binding 1 logical_port=cr-lrp-local-bgp-port 'options:dynamic-routing=true' +check_row_count Port_Binding 1 logical_port=cr-lrp-dgp-dummy 'options:dynamic-routing=true' + +# Add static routes. +check ovn-nbctl lr-route-add lr-frr 10.10.2.1 20.0.0.42 lrp-local-bgp-port + +# Verify advertised routes exist. +AS_BOX([Advertised_Route]) +wait_row_count Advertised_Route 1 ip_prefix=10.10.2.1 + +# Add a route to the VRF (simulating BGP learning a route). +check ip route add 10.10.3.1 via 20.0.0.25 vrf vrf-$vrf proto zebra + +# Verify learned route appears in SB database. +check_row_count Learned_Route 2 ip_prefix=10.10.3.1 + +check ovn-nbctl --wait=hv set Logical_Router_Port lrp-local-bgp-port options:dynamic-routing-port-name=local-bgp-port +lp=$(fetch_column port_binding _uuid logical_port=lrp-local-bgp-port) + +check_row_count Learned_Route 1 ip_prefix=10.10.3.1 +check_row_count Learned_Route 1 ip_prefix=10.10.3.1 logical_port=$lp + +check ovn-nbctl --wait=hv ls-del ls-dummy +check ovn-nbctl --wait=hv ls-del public +check ovn-nbctl --wait=hv lr-del lr-frr + +# Stop ovn-controller +OVN_CLEANUP_CONTROLLER([hv1]) +check ovn-nbctl --wait=sb sync + +# Verify routes are removed in SB database. +wait_row_count Learned_Route 0 + +OVN_CLEANUP_NORTHD + +as +OVS_TRAFFIC_VSWITCHD_STOP(["/.*error receiving.*/d +/failed to query port patch-.*/d +/.*terminating with signal 15.*/d"]) +AT_CLEANUP +]) -- 2.43.0 -- _‘Esta mensagem é direcionada apenas para os endereços constantes no cabeçalho inicial. Se você não está listado nos endereços constantes no cabeçalho, pedimos-lhe que desconsidere completamente o conteúdo dessa mensagem e cuja cópia, encaminhamento e/ou execução das ações citadas estão imediatamente anuladas e proibidas’._ * **‘Apesar do Magazine Luiza tomar todas as precauções razoáveis para assegurar que nenhum vírus esteja presente nesse e-mail, a empresa não poderá aceitar a responsabilidade por quaisquer perdas ou danos causados por esse e-mail ou por seus anexos’.* _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
