Fix ovn-ic mode when vxlan is used as encapsulation mode reducing the maximum local dp key to ((2<<10)-1) in order to make some room for OVN_MAX_DP_VXLAN_KEY_GLOBAL (vxlan tunnels export just 12 bit for metadata key).
Reported-at: https://issues.redhat.com/browse/FDP-1023 Signed-off-by: Lorenzo Bianconi <[email protected]> --- - Changes in v5: - Fix comments - Fix code indentation - Do not set en_nb_glocal_config node state to EN_UPDATED in global_config_nb_logical_switch_handler() if max_tunid is not updated - Changes in v4: - Introduce vxlan_mode in option column of IC_NB_Global table to enable VXLAN protocol for cross-AZ traffic. Default value is false. - Changes in v3: - Reduce the max local dp key to 1023 just if the cluster is running in ovn-ic mode - Changes in v2: - Document local datapath limitation --- NEWS | 4 +++ ic/ovn-ic.c | 48 +++++++++++++++++++++++----- lib/ovn-util.h | 4 ++- northd/en-global-config.c | 66 ++++++++++++++++++++++++++++++++++++--- northd/en-global-config.h | 2 ++ northd/inc-proc-northd.c | 2 ++ northd/northd.c | 17 +++++++--- northd/northd.h | 2 +- ovn-ic-nb.xml | 7 +++++ ovn-nb.xml | 14 +++++++++ tests/ovn-ic.at | 32 +++++++++++++++++++ tests/ovn-northd.at | 22 +++++++++++++ 12 files changed, 203 insertions(+), 17 deletions(-) diff --git a/NEWS b/NEWS index efe7a0e8a..14143aeb5 100644 --- a/NEWS +++ b/NEWS @@ -47,6 +47,10 @@ Post v24.09.0 - Logical router policies can now be arranged in chains. Using the new "jump" action, combined with new "chain" and "jump_chain" columns, allows for policies to be chained together. + - Reduce the max number of local datapath to 1024 when OVN is using VXLAN + encapsulation type in OVN-interconnect mode. + - Added vxlan_mode parameter in IC_NB_GLOBAL option column to enable or + disable VXLAN encapsulation type in OVN-interconnect mode. OVN v24.09.0 - 13 Sep 2024 -------------------------- diff --git a/ic/ovn-ic.c b/ic/ovn-ic.c index 75b5d1787..a2e5cb8f3 100644 --- a/ic/ovn-ic.c +++ b/ic/ovn-ic.c @@ -181,18 +181,20 @@ az_run(struct ic_context *ctx) } static uint32_t -allocate_ts_dp_key(struct hmap *dp_tnlids) +allocate_ts_dp_key(struct hmap *dp_tnlids, bool vxlan_mode) { - static uint32_t hint = OVN_MIN_DP_KEY_GLOBAL; - return ovn_allocate_tnlid(dp_tnlids, "transit switch datapath", - OVN_MIN_DP_KEY_GLOBAL, OVN_MAX_DP_KEY_GLOBAL, - &hint); + uint32_t hint = vxlan_mode ? OVN_MIN_DP_VXLAN_KEY_GLOBAL + : OVN_MIN_DP_KEY_GLOBAL; + return ovn_allocate_tnlid(dp_tnlids, "transit switch datapath", hint, + vxlan_mode ? OVN_MAX_DP_VXLAN_KEY_GLOBAL : OVN_MAX_DP_KEY_GLOBAL, + &hint); } static void ts_run(struct ic_context *ctx) { const struct icnbrec_transit_switch *ts; + bool dp_key_refresh = false; struct hmap dp_tnlids = HMAP_INITIALIZER(&dp_tnlids); struct shash isb_dps = SHASH_INITIALIZER(&isb_dps); @@ -202,6 +204,20 @@ ts_run(struct ic_context *ctx) ovn_add_tnlid(&dp_tnlids, isb_dp->tunnel_key); } + bool vxlan_mode = false; + const struct icnbrec_ic_nb_global *ic_nb = + icnbrec_ic_nb_global_first(ctx->ovninb_idl); + + if (ic_nb && smap_get_bool(&ic_nb->options, "vxlan_mode", false)) { + const struct icsbrec_encap *encap; + ICSBREC_ENCAP_FOR_EACH (encap, ctx->ovnisb_idl) { + if (!strcmp(encap->type, "vxlan")) { + vxlan_mode = true; + break; + } + } + } + /* Sync INB TS to AZ NB */ if (ctx->ovnnb_txn) { struct shash nb_tses = SHASH_INITIALIZER(&nb_tses); @@ -224,7 +240,19 @@ ts_run(struct ic_context *ctx) nbrec_logical_switch_update_other_config_setkey(ls, "interconn-ts", ts->name); + nbrec_logical_switch_update_other_config_setkey( + ls, "ic-vxlan_mode", vxlan_mode ? "true" : "false"); + } else { + bool _vxlan_mode = smap_get_bool(&ls->other_config, + "ic-vxlan_mode", false); + if (_vxlan_mode != vxlan_mode) { + dp_key_refresh = true; + nbrec_logical_switch_update_other_config_setkey( + ls, "ic-vxlan_mode", + vxlan_mode ? "true" : "false"); + } } + isb_dp = shash_find_data(&isb_dps, ts->name); if (isb_dp) { int64_t nb_tnl_key = smap_get_int(&ls->other_config, @@ -260,7 +288,7 @@ ts_run(struct ic_context *ctx) isb_dp = shash_find_and_delete(&isb_dps, ts->name); if (!isb_dp) { /* Allocate tunnel key */ - int64_t dp_key = allocate_ts_dp_key(&dp_tnlids); + int64_t dp_key = allocate_ts_dp_key(&dp_tnlids, vxlan_mode); if (!dp_key) { continue; } @@ -268,6 +296,12 @@ ts_run(struct ic_context *ctx) isb_dp = icsbrec_datapath_binding_insert(ctx->ovnisb_txn); icsbrec_datapath_binding_set_transit_switch(isb_dp, ts->name); icsbrec_datapath_binding_set_tunnel_key(isb_dp, dp_key); + } else if (dp_key_refresh) { + /* Refresh tunnel key since encap mode has changhed. */ + int64_t dp_key = allocate_ts_dp_key(&dp_tnlids, vxlan_mode); + if (dp_key) { + icsbrec_datapath_binding_set_tunnel_key(isb_dp, dp_key); + } } } @@ -1930,8 +1964,8 @@ static void ovn_db_run(struct ic_context *ctx, const struct icsbrec_availability_zone *az) { - ts_run(ctx); gateway_run(ctx, az); + ts_run(ctx); port_binding_run(ctx, az); route_run(ctx, az); } diff --git a/lib/ovn-util.h b/lib/ovn-util.h index f2f70dd72..dabe72254 100644 --- a/lib/ovn-util.h +++ b/lib/ovn-util.h @@ -163,7 +163,9 @@ void set_idl_probe_interval(struct ovsdb_idl *idl, const char *remote, #define OVN_MAX_DP_KEY_GLOBAL OVN_MAX_DP_KEY #define OVN_MAX_DP_VXLAN_KEY ((1u << 12) - 1) -#define OVN_MAX_DP_VXLAN_KEY_LOCAL (OVN_MAX_DP_KEY - OVN_MAX_DP_GLOBAL_NUM) +#define OVN_MAX_DP_VXLAN_KEY_LOCAL ((1u << 10) - 1) +#define OVN_MIN_DP_VXLAN_KEY_GLOBAL (OVN_MAX_DP_VXLAN_KEY_LOCAL + 1) +#define OVN_MAX_DP_VXLAN_KEY_GLOBAL ((1u << 12) - 1) struct hmap; void ovn_destroy_tnlids(struct hmap *tnlids); diff --git a/northd/en-global-config.c b/northd/en-global-config.c index ed171449a..c103b137f 100644 --- a/northd/en-global-config.c +++ b/northd/en-global-config.c @@ -71,6 +71,8 @@ en_global_config_run(struct engine_node *node , void *data) const struct nbrec_nb_global_table *nb_global_table = EN_OVSDB_GET(engine_get_input("NB_nb_global", node)); + const struct nbrec_logical_switch_table *nbrec_ls_table = + EN_OVSDB_GET(engine_get_input("NB_logical_switch", node)); const struct sbrec_sb_global_table *sb_global_table = EN_OVSDB_GET(engine_get_input("SB_sb_global", node)); const struct sbrec_chassis_table *sbrec_chassis_table = @@ -121,10 +123,19 @@ en_global_config_run(struct engine_node *node , void *data) config_data->svc_monitor_mac); } - char *max_tunid = xasprintf("%d", - get_ovn_max_dp_key_local( - is_vxlan_mode(&nb->options, - sbrec_chassis_table))); + bool ic_vxlan_mode = false; + const struct nbrec_logical_switch *nbs; + NBREC_LOGICAL_SWITCH_TABLE_FOR_EACH (nbs, nbrec_ls_table) { + if (smap_get(&nbs->other_config, "ic-vxlan_mode")) { + ic_vxlan_mode = true; + break; + } + } + uint32_t max_dp_key = + get_ovn_max_dp_key_local(is_vxlan_mode(&nb->options, + sbrec_chassis_table), + ic_vxlan_mode); + char *max_tunid = xasprintf("%d", max_dp_key); smap_replace(options, "max_tunid", max_tunid); free(max_tunid); @@ -371,6 +382,53 @@ node_global_config_handler(struct engine_node *node, void *data OVS_UNUSED) return true; } +bool +global_config_nb_logical_switch_handler(struct engine_node *node, + void *data) +{ + struct ed_type_global_config *config_data = data; + const struct nbrec_logical_switch_table *nbrec_ls_table = + EN_OVSDB_GET(engine_get_input("NB_logical_switch", node)); + const struct nbrec_nb_global *nb = nbrec_nb_global_table_first( + EN_OVSDB_GET(engine_get_input("NB_nb_global", node))); + const struct sbrec_chassis_table *sbrec_chassis_table = + EN_OVSDB_GET(engine_get_input("SB_chassis", node)); + + bool ic_vxlan_mode = false; + const struct nbrec_logical_switch *nbs; + NBREC_LOGICAL_SWITCH_TABLE_FOR_EACH (nbs, nbrec_ls_table) { + if (smap_get(&nbs->other_config, "ic-vxlan_mode")) { + ic_vxlan_mode = true; + break; + } + } + uint32_t max_dp_key = + get_ovn_max_dp_key_local(is_vxlan_mode(&nb->options, + sbrec_chassis_table), + ic_vxlan_mode); + char *max_tunid = xasprintf("%d", max_dp_key); + struct smap *options = &config_data->nb_options; + const char *cur_max_tunid = smap_get(options, "max_tunid"); + + if (!cur_max_tunid || strcmp(max_tunid, cur_max_tunid)) { + engine_set_node_state(node, EN_UPDATED); + } else { + engine_set_node_state(node, EN_UNCHANGED); + } + + smap_replace(options, "max_tunid", max_tunid); + free(max_tunid); + + if (!smap_equal(&nb->options, options)) { + nbrec_nb_global_verify_options(nb); + nbrec_nb_global_set_options(nb, options); + } + + config_data->tracked = true; + + return true; +} + /* static functions. */ static void northd_enable_all_features(struct ed_type_global_config *data) diff --git a/northd/en-global-config.h b/northd/en-global-config.h index a95ba2a6c..de88db18b 100644 --- a/northd/en-global-config.h +++ b/northd/en-global-config.h @@ -60,6 +60,8 @@ void en_global_config_clear_tracked_data(void *data); bool global_config_nb_global_handler(struct engine_node *, void *data); bool global_config_sb_global_handler(struct engine_node *, void *data); bool global_config_sb_chassis_handler(struct engine_node *, void *data); +bool global_config_nb_logical_switch_handler(struct engine_node *node, + void *data); /* generic global config handler for any engine node which has global_config * has an input node . */ diff --git a/northd/inc-proc-northd.c b/northd/inc-proc-northd.c index fd3fa04da..9232eb6c6 100644 --- a/northd/inc-proc-northd.c +++ b/northd/inc-proc-northd.c @@ -188,6 +188,8 @@ void inc_proc_northd_init(struct ovsdb_idl_loop *nb, engine_add_input(&en_global_config, &en_nb_nb_global, global_config_nb_global_handler); + engine_add_input(&en_global_config, &en_nb_logical_switch, + global_config_nb_logical_switch_handler); engine_add_input(&en_global_config, &en_sb_sb_global, global_config_sb_global_handler); engine_add_input(&en_global_config, &en_sb_chassis, diff --git a/northd/northd.c b/northd/northd.c index cc8fb4cd9..ca67221b3 100644 --- a/northd/northd.c +++ b/northd/northd.c @@ -95,6 +95,8 @@ static bool default_acl_drop; * and ports tunnel key allocation (12 bits for each instead of default 16). */ static bool vxlan_mode; +static bool vxlan_ic_mode; + #define MAX_OVN_TAGS 4096 @@ -853,6 +855,7 @@ join_datapaths(const struct nbrec_logical_switch_table *nbrec_ls_table, ovs_list_push_back(sb_only, &od->list); } + vxlan_ic_mode = false; const struct nbrec_logical_switch *nbs; NBREC_LOGICAL_SWITCH_TABLE_FOR_EACH (nbs, nbrec_ls_table) { struct ovn_datapath *od = ovn_datapath_find_(datapaths, @@ -870,6 +873,10 @@ join_datapaths(const struct nbrec_logical_switch_table *nbrec_ls_table, init_ipam_info_for_datapath(od); init_mcast_info_for_datapath(od); + + if (smap_get_bool(&nbs->other_config, "ic-vxlan_mode", false)) { + vxlan_ic_mode = true; + } } const struct nbrec_logical_router *nbr; @@ -927,13 +934,14 @@ is_vxlan_mode(const struct smap *nb_options, } uint32_t -get_ovn_max_dp_key_local(bool _vxlan_mode) +get_ovn_max_dp_key_local(bool _vxlan_mode, bool _vxlan_ic_mode) { if (_vxlan_mode) { /* OVN_MAX_DP_GLOBAL_NUM doesn't apply for VXLAN mode. */ - return OVN_MAX_DP_VXLAN_KEY; + return _vxlan_ic_mode ? OVN_MAX_DP_VXLAN_KEY_LOCAL + : OVN_MAX_DP_VXLAN_KEY; } - return OVN_MAX_DP_KEY - OVN_MAX_DP_GLOBAL_NUM; + return _vxlan_ic_mode ? OVN_MAX_DP_VXLAN_KEY_LOCAL : OVN_MAX_DP_KEY_LOCAL; } static void @@ -942,7 +950,8 @@ ovn_datapath_allocate_key(struct hmap *datapaths, struct hmap *dp_tnlids, { if (!od->tunnel_key) { od->tunnel_key = ovn_allocate_tnlid(dp_tnlids, "datapath", - OVN_MIN_DP_KEY_LOCAL, get_ovn_max_dp_key_local(vxlan_mode), hint); + OVN_MIN_DP_KEY_LOCAL, + get_ovn_max_dp_key_local(vxlan_mode, vxlan_ic_mode), hint); if (!od->tunnel_key) { if (od->sb) { sbrec_datapath_binding_delete(od->sb); diff --git a/northd/northd.h b/northd/northd.h index 07dba5b27..1a6e7a5d4 100644 --- a/northd/northd.h +++ b/northd/northd.h @@ -877,7 +877,7 @@ bool is_vxlan_mode(const struct smap *nb_options, const struct sbrec_chassis_table *sbrec_chassis_table); -uint32_t get_ovn_max_dp_key_local(bool _vxlan_mode); +uint32_t get_ovn_max_dp_key_local(bool _vxlan_mode, bool ic_mode); /* Returns true if the logical router port 'enabled' column is empty or * set to true. Otherwise, returns false. */ diff --git a/ovn-ic-nb.xml b/ovn-ic-nb.xml index 80138400d..abd8114fc 100644 --- a/ovn-ic-nb.xml +++ b/ovn-ic-nb.xml @@ -78,6 +78,13 @@ at least 1000 ms. </p> </column> + + <column name="options" key="vxlan_mode"> + <p> + This field allows the client to enable VXLAN as encapsulation + protocol for for cross-AZ traffic. Default value is false. + </p> + </column> </group> <group title="Connection Options"> diff --git a/ovn-nb.xml b/ovn-nb.xml index 0d5777d30..034a5d644 100644 --- a/ovn-nb.xml +++ b/ovn-nb.xml @@ -394,6 +394,14 @@ support HW VTEP functionality and main encap type is GENEVE or STT, set this option to <code>false</code> to use default non-<code>VXLAN mode</code> tunnel IDs allocation logic. + Please consider when OVN is running in <code>OVN-interconnect</code> + mode and it is using <code>VXLAN</code> encapsulation type, the max + number of non-transit logical switches and logical routers is reduced + to 1024. + Plese note, in order to enable <code>VXLAN</code> encapsulation type + for for cross-AZ traffic, <code>vxlan_mode</code> parameter in + <ref table="IC_NB_Global TABLE" db="OVN_IC_Northbound"/> must be set + to true. </column> <column name="options" key="always_tunnel" @@ -888,6 +896,12 @@ database. This kind of logical switch is created and controlled by <code>ovn-ic</code>. </column> + <column name="other_config" key="ic-vxlan_mode" + type='{"type": "boolean"}'> + <code>ic-vxlan_mode</code> is set to true by <code>ovn-ic</code> when + it runs <code>VXLAN</code> as encapsulation protocol for cross-AZ + traffic. Default value is false. + </column> </group> <group title="Tunnel Key"> diff --git a/tests/ovn-ic.at b/tests/ovn-ic.at index fbcfca2e4..a1eccb165 100644 --- a/tests/ovn-ic.at +++ b/tests/ovn-ic.at @@ -94,6 +94,38 @@ OVN_CLEANUP_IC([az1]) AT_CLEANUP ]) +OVN_FOR_EACH_NORTHD([ +AT_SETUP([ovn-ic -- VXLAN tunnel key]) +ovn_init_ic_db +net_add n1 + +ovn_start az1 +sim_add gw-az1 +as gw-az1 + +check ovs-vsctl add-br br-phys +ovn_az_attach az1 n1 br-phys 192.168.1.1 +check ovs-vsctl set open . external-ids:ovn-is-interconn=true + +AT_CHECK([ovn-ic-nbctl --wait=sb ts-add ts1]) + +# Check ISB +check_row_count ic-sb:Datapath_Binding 1 transit_switch=ts1 +check_column "ts1" ic-sb:Datapath_Binding transit_switch +check_column "ts1" nb:Logical_Switch name + +wait_column "ic-vxlan_mode=false interconn-ts=ts1 requested-tnl-key=16711682" nb:Logical_Switch other_config name="ts1" +# Check tunnel key fits in VXLAN space +check ovn-ic-nbctl --wait=sb set IC_NB_Global . options:vxlan_mode=true +wait_column "ic-vxlan_mode=true interconn-ts=ts1 requested-tnl-key=1025" nb:Logical_Switch other_config name="ts1" + +check ovn-ic-nbctl --wait=sb set IC_NB_Global . options:vxlan_mode=false +wait_column "ic-vxlan_mode=false interconn-ts=ts1 requested-tnl-key=16711682" nb:Logical_Switch other_config name="ts1" + +OVN_CLEANUP_IC([az1]) +AT_CLEANUP +]) + OVN_FOR_EACH_NORTHD([ AT_SETUP([ovn-ic -- port-bindings deletion upon TS deletion]) diff --git a/tests/ovn-northd.at b/tests/ovn-northd.at index b584cafbc..69bdc5c2c 100644 --- a/tests/ovn-northd.at +++ b/tests/ovn-northd.at @@ -2983,7 +2983,29 @@ OVS_WAIT_UNTIL([grep "all port tunnel ids exhausted" northd/ovn-northd.log]) AT_CLEANUP ]) +OVN_FOR_EACH_NORTHD_NO_HV([ +AT_SETUP([check VXLAN encap in IC-mode]) +ovn_start + +get_max_tunid() { + echo $(ovn-nbctl get NB_Global . options:max_tunid | sed s/":"//g | sed s/\"//g) +} + +check_uuid ovn-sbctl \ + --id=@e create encap chassis_name=hv1 ip="192.168.0.1" type="vxlan" \ + -- --id=@c create chassis name=hv1 encaps=@e +check ovn-nbctl --wait=sb ls-add LS +AT_CHECK([test "$(get_max_tunid)" -eq 4095]) + +check ovn-nbctl --wait=sb set logical-switch LS other-config:interconn-ts=LS +check ovn-nbctl --wait=sb set logical-switch LS other-config:ic-vxlan_mode=true +AT_CHECK([test "$(get_max_tunid)" -eq 1023]) +check ovn-nbctl --wait=sb clear logical-switch LS other-config +AT_CHECK([test "$(get_max_tunid)" -eq 4095]) + +AT_CLEANUP +]) OVN_FOR_EACH_NORTHD_NO_HV([ AT_SETUP([Logical Flow Datapath Groups]) -- 2.48.1 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
