A multicast flood carries one set of actions per destination, so a large
enough group outgrows the FLOW_MOD message that has to carry them.
ofctrl then discards the flow with "The FLOW_MOD message is too big" and
the flood reaches nobody. A logical switch with more than about 800
ports on remote chassis is enough.
Install such a group as a chain of flows instead, each holding a slice
of the outputs and resubmitting to the next. The next slice is found by
an id published in reg0, with MLF_MC_CHUNK marking that lookup. A slice
only ends on a completed delivery, so no destination is cut in half.
That size is more than a test wants to build, so debug/set-mc-chunk-max
lowers the budget a flood is split at.
Fixes: f4d24c09d848 ("ofctrl: Do not try to program long flows")
Co-authored-by: Ram Vepa <[email protected]>
Signed-off-by: Ram Vepa <[email protected]>
Assisted-by: Claude Opus 5, Cursor
Signed-off-by: Han Zhou <[email protected]>
---
On why the test lowers the chunk budget instead of building a group big
enough to really overrun a FLOW_MOD:
That takes more than 800 remote ports, and at that size the test would
no longer be testing this patch. Past about 710 destinations a flood
does not fit in one datapath action list either, and translation drops
whatever is left over without reporting anything, so "every destination
is reached" would fail for a reason that has nothing to do with the
split. That truncation is an OVS bug in its own right, fixed separately
by:
https://patchwork.ozlabs.org/project/openvswitch/patch/[email protected]/
Lowering the budget keeps the test cheap, and lets it cover a chain of
several slices rather than the two that 800 ports would produce.
controller/ovn-controller.8.xml | 7 ++
controller/ovn-controller.c | 22 ++++
controller/physical.c | 185 +++++++++++++++++++++++++++++++-
controller/physical.h | 1 +
include/ovn/logical-fields.h | 5 +
ovn-architecture.7.xml | 11 ++
tests/ovn.at | 74 +++++++++++++
7 files changed, 300 insertions(+), 5 deletions(-)
diff --git a/controller/ovn-controller.8.xml b/controller/ovn-controller.8.xml
index 2b90f25d9ff1..ca04f28395da 100644
--- a/controller/ovn-controller.8.xml
+++ b/controller/ovn-controller.8.xml
@@ -1078,6 +1078,13 @@
debugging.
</dd>
+ <dt><code>debug/set-mc-chunk-max</code> <var>bytes</var></dt>
+ <dd>
+ Sets how many encoded OpenFlow action bytes a multicast flood may
+ put in one flow before it is split across several, for testing.
+ Zero restores the default.
+ </dd>
+
<dt><code>debug/resume</code></dt>
<dd>
Resumes a processing loop paused by <code>debug/pause</code>.
diff --git a/controller/ovn-controller.c b/controller/ovn-controller.c
index 7bc3139ab435..8311cafb578e 100644
--- a/controller/ovn-controller.c
+++ b/controller/ovn-controller.c
@@ -127,6 +127,7 @@ static unixctl_cb_func debug_dump_lflow_conj_ids;
static unixctl_cb_func lflow_cache_flush_cmd;
static unixctl_cb_func lflow_cache_show_stats_cmd;
static unixctl_cb_func debug_delay_nb_cfg_report;
+static unixctl_cb_func debug_set_mc_chunk_max;
#define DEFAULT_BRIDGE_NAME "br-int"
#define DEFAULT_DATAPATH "system"
@@ -8455,6 +8456,8 @@ main(int argc, char *argv[])
unsigned int delay_nb_cfg_report = 0;
unixctl_command_register("debug/delay-nb-cfg-report", "SECONDS", 1, 1,
debug_delay_nb_cfg_report, &delay_nb_cfg_report);
+ unixctl_command_register("debug/set-mc-chunk-max", "BYTES", 1, 1,
+ debug_set_mc_chunk_max, NULL);
unixctl_command_register("debug/dump-local-bindings", "", 0, 0,
debug_dump_local_bindings,
@@ -9581,6 +9584,25 @@ debug_status_execution(struct unixctl_conn *conn, int
argc OVS_UNUSED,
}
}
+/* Lowers the number of encoded OpenFlow action bytes a multicast flood is
+ * allowed to put in one flow, so that a test can split a small group.
+ * Zero restores the default. */
+static void
+debug_set_mc_chunk_max(struct unixctl_conn *conn, int argc OVS_UNUSED,
+ const char *argv[], void *arg OVS_UNUSED)
+{
+ unsigned int bytes;
+
+ if (!str_to_uint(argv[1], 10, &bytes)) {
+ unixctl_command_reply_error(conn, "unsigned integer required");
+ return;
+ }
+
+ physical_set_mc_chunk_max(bytes);
+ engine_set_force_recompute();
+ unixctl_command_reply(conn, NULL);
+}
+
static void
debug_delay_nb_cfg_report(struct unixctl_conn *conn, int argc OVS_UNUSED,
const char *argv[], void *delay_)
diff --git a/controller/physical.c b/controller/physical.c
index 01013be48c73..dbd0cc9970c5 100644
--- a/controller/physical.c
+++ b/controller/physical.c
@@ -2938,6 +2938,181 @@ flood_querier_port_add(struct mcast_querier_port
*flood_ports,
(*n_flood_ports)++;
}
+/* A multicast flood carries one set of actions per destination, and a
+ * large enough group outgrows the FLOW_MOD that has to carry them: past
+ * that, ofctrl discards the flow and the flood reaches nobody. Such a
+ * group is installed as a chain of flows instead, each holding a slice of
+ * the outputs and resubmitting to the next.
+ *
+ * The next slice is found by an id published in reg0, with MLF_MC_CHUNK
+ * marking that lookup. reg0 is stacked around it because the egress
+ * pipeline reads it. */
+
+/* Encoded OpenFlow action bytes per flow. add_flow_mod() has a little
+ * over 65,000 to spend on actions, once the bundle and flow_mod headers,
+ * the match and the instruction header are taken out. */
+#define MC_CHUNK_OF_DEFAULT (60 * 1024)
+
+static size_t mc_chunk_of_max = MC_CHUNK_OF_DEFAULT;
+
+void
+physical_set_mc_chunk_max(size_t bytes)
+{
+ mc_chunk_of_max = bytes ? bytes : MC_CHUNK_OF_DEFAULT;
+}
+
+static size_t
+mc_chunk_cost(const struct ofpact *ofpacts, size_t len,
+ struct ofpbuf *scratch)
+{
+ ofpbuf_clear(scratch);
+ return ofpacts_put_openflow_actions(ofpacts, len, scratch,
+ OFP15_VERSION);
+}
+
+static bool
+mc_fits_one_chunk(const struct ofpbuf *ofpacts)
+{
+ struct ofpbuf scratch;
+ size_t size;
+
+ ofpbuf_init(&scratch, ofpacts->size);
+ size = mc_chunk_cost(ofpacts->data, ofpacts->size, &scratch);
+ ofpbuf_uninit(&scratch);
+
+ return size <= mc_chunk_of_max;
+}
+
+static void
+mc_chunk_match(const struct sbrec_multicast_group *mc,
+ const struct mc_flow_ctx *ctx, uint32_t chunk_id,
+ struct match *match)
+{
+ uint32_t flags = ctx->flags;
+ uint32_t mask = ctx->flags_mask | MLF_MC_CHUNK;
+
+ match_init_catchall(match);
+ match_outport_dp_and_port_keys(match, mc->datapath->tunnel_key,
+ mc->tunnel_key);
+ if (chunk_id > 0) {
+ flags |= MLF_MC_CHUNK;
+ match_set_reg(match, MFF_REG0 - MFF_REG0, chunk_id);
+ }
+ match_set_reg_masked(match, MFF_LOG_FLAGS - MFF_REG0, flags, mask);
+}
+
+static void
+mc_compose_chunk(struct ofpbuf *dst, const struct ofpbuf *body,
+ bool first, bool last, uint32_t next_chunk,
+ int64_t mc_key, uint8_t same_table, bool resume_next,
+ uint8_t next_table)
+{
+ ofpbuf_clear(dst);
+ if (!first) {
+ put_stack(MFF_REG0, ofpact_put_STACK_POP(dst));
+ put_load(0, MFF_LOG_FLAGS, MLF_MC_CHUNK_BIT, 1, dst);
+ }
+ ofpbuf_put(dst, body->data, body->size);
+ if (!last) {
+ /* The body may have overwritten the outport this table matches,
+ * and the egress pipeline reads reg0. */
+ put_stack(MFF_REG0, ofpact_put_STACK_PUSH(dst));
+ put_load(mc_key, MFF_LOG_OUTPORT, 0, 32, dst);
+ put_load(next_chunk, MFF_REG0, 0, 32, dst);
+ put_load(1, MFF_LOG_FLAGS, MLF_MC_CHUNK_BIT, 1, dst);
+ put_resubmit(same_table, dst);
+ } else if (resume_next) {
+ put_load(mc_key, MFF_LOG_OUTPORT, 0, 32, dst);
+ put_resubmit(next_table, dst);
+ }
+}
+
+static void
+mc_add_chunk_flow(const struct sbrec_multicast_group *mc,
+ const struct mc_flow_ctx *ctx, uint32_t chunk_id,
+ const struct ofpbuf *actions,
+ struct ovn_desired_flow_table *flow_table)
+{
+ struct match match;
+
+ mc_chunk_match(mc, ctx, chunk_id, &match);
+ ofctrl_add_flow(flow_table, ctx->stage, ctx->prio,
+ mc->header_.uuid.parts[0], &match, actions,
+ &mc->header_.uuid);
+}
+
+static void
+mc_split_flows(const struct sbrec_multicast_group *mc,
+ struct mc_flow_ctx *ctx, bool resume_next, uint8_t next_table,
+ struct ovn_desired_flow_table *flow_table)
+{
+ struct ofpbuf body, actions, scratch;
+ const struct ofpact *a;
+ const struct ofpact *base = ctx->ofpacts.data;
+ size_t total = ctx->ofpacts.size;
+ uint32_t chunk_id = 0;
+ size_t sum = 0;
+
+ ofpbuf_init(&body, 0);
+ ofpbuf_init(&actions, 0);
+ ofpbuf_init(&scratch, 64);
+
+ /* A slice has to end on a completed delivery: cutting between a port's
+ * register loads and the resubmit that consumes them would deliver that
+ * port to the multicast key the continuation restores. */
+ const struct ofpact *last = NULL;
+ OFPACT_FOR_EACH (a, base, total) {
+ size_t cost = mc_chunk_cost(a, OFPACT_ALIGN(a->len), &scratch);
+ bool boundary = last && (last->type == OFPACT_RESUBMIT
+ || last->type == OFPACT_OUTPUT);
+
+ if (body.size > 0 && sum + cost > mc_chunk_of_max && boundary) {
+ mc_compose_chunk(&actions, &body, chunk_id == 0, false,
+ chunk_id + 1, mc->tunnel_key, ctx->stage,
+ resume_next, next_table);
+ mc_add_chunk_flow(mc, ctx, chunk_id, &actions, flow_table);
+ chunk_id++;
+ ofpbuf_clear(&body);
+ last = NULL;
+ sum = 0;
+ }
+ ofpbuf_put(&body, a, OFPACT_ALIGN(a->len));
+ sum += cost;
+ last = a;
+ }
+
+ mc_compose_chunk(&actions, &body, chunk_id == 0, true, 0,
+ mc->tunnel_key, ctx->stage, resume_next, next_table);
+ mc_add_chunk_flow(mc, ctx, chunk_id, &actions, flow_table);
+
+ ofpbuf_uninit(&body);
+ ofpbuf_uninit(&actions);
+ ofpbuf_uninit(&scratch);
+}
+
+/* Install the multicast output in 'ctx', splitting it across several flows
+ * if it is too large for one. When 'resume_next' is true the last slice
+ * resubmits to 'next_table'. */
+static void
+mc_install_flows(const struct sbrec_multicast_group *mc,
+ struct mc_flow_ctx *ctx, bool resume_next, uint8_t next_table,
+ struct ovn_desired_flow_table *flow_table)
+{
+ size_t plain = ctx->ofpacts.size;
+
+ if (resume_next) {
+ put_resubmit(next_table, &ctx->ofpacts);
+ }
+ if (mc_fits_one_chunk(&ctx->ofpacts)) {
+ mc_ofctrl_add_flow(mc, ctx, flow_table);
+ return;
+ }
+
+ ofpbuf_truncate(&ctx->ofpacts, plain);
+ mc_split_flows(mc, ctx, resume_next, next_table, flow_table);
+ ofpbuf_clear(&ctx->ofpacts);
+}
+
static void
consider_mc_group(const struct physical_ctx *ctx,
const struct sbrec_multicast_group *mc,
@@ -3116,7 +3291,7 @@ consider_mc_group(const struct physical_ctx *ctx,
if (local_lports) {
put_load(mc->tunnel_key, MFF_LOG_OUTPORT, 0, 32, &local_ctx->ofpacts);
- mc_ofctrl_add_flow(mc, local_ctx, flow_table);
+ mc_install_flows(mc, local_ctx, false, 0, flow_table);
}
if (remote_ports) {
@@ -3148,14 +3323,14 @@ consider_mc_group(const struct physical_ctx *ctx,
remote_ports = remote_ctx->ofpacts.size > 0;
if (remote_ports) {
- put_resubmit(OFTABLE_REMOTE_VTEP_OUTPUT, &remote_ctx->ofpacts);
- mc_ofctrl_add_flow(mc, remote_ctx, flow_table);
+ mc_install_flows(mc, remote_ctx, true, OFTABLE_REMOTE_VTEP_OUTPUT,
+ flow_table);
}
if (ramp_ports && has_vtep) {
put_load(mc->tunnel_key, MFF_LOG_OUTPORT, 0, 32, &ramp_ctx->ofpacts);
- put_resubmit(OFTABLE_REMOTE_VTEP_OUTPUT, &ramp_ctx->ofpacts);
- mc_ofctrl_add_flow(mc, ramp_ctx, flow_table);
+ mc_install_flows(mc, ramp_ctx, true, OFTABLE_REMOTE_VTEP_OUTPUT,
+ flow_table);
}
for (size_t i = 0; i < MC_FLOWS_MAX; i++) {
diff --git a/controller/physical.h b/controller/physical.h
index 8218096d5e49..6828342a82a4 100644
--- a/controller/physical.h
+++ b/controller/physical.h
@@ -85,6 +85,7 @@ struct physical_ctx {
};
void physical_register_ovs_idl(struct ovsdb_idl *);
+void physical_set_mc_chunk_max(size_t bytes);
void physical_run(struct physical_ctx *,
struct ovn_desired_flow_table *);
void physical_handle_mc_group_changes(struct physical_ctx *,
diff --git a/include/ovn/logical-fields.h b/include/ovn/logical-fields.h
index de81a7fe255a..433ba35d168f 100644
--- a/include/ovn/logical-fields.h
+++ b/include/ovn/logical-fields.h
@@ -144,6 +144,7 @@ enum mff_log_flags_bits {
MLF_RECIRC_BIT = 24,
MLF_EVPN_LOOKUP_BIT = 25,
MLF_POST_NF_LOOP_BACK_BIT = 26,
+ MLF_MC_CHUNK_BIT = 27,
MLF_NETWORK_ID_START_BIT = 28,
MLF_NETWORK_ID_END_BIT = 31,
};
@@ -224,6 +225,10 @@ enum mff_log_flags {
/* Set on a post-NF packet going back to the original ingress port. */
MLF_POST_NF_LOOP_BACK = (1 << MLF_POST_NF_LOOP_BACK_BIT),
+ /* Set on a multicast flood that was split across several OpenFlow
+ * flows, for the table lookup of the next chunk only. */
+ MLF_MC_CHUNK = (1 << MLF_MC_CHUNK_BIT),
+
/* Assign network ID to packet to choose correct network for snat when
* lb_force_snat_ip=router_ip. */
MLF_NETWORK_ID = (OVN_MAX_NETWORK_ID << MLF_NETWORK_ID_START_BIT),
diff --git a/ovn-architecture.7.xml b/ovn-architecture.7.xml
index 00728ee9b7e6..c3ecdca15d36 100644
--- a/ovn-architecture.7.xml
+++ b/ovn-architecture.7.xml
@@ -1553,6 +1553,17 @@
multicast group includes a logical port or ports on the local
hypervisor, then its actions also resubmit to
<code>OFTABLE_LOCAL_OUTPUT</code>.
+ A multicast group carries one set of actions per destination, and
+ a large enough group outgrows the <code>FLOW_MOD</code> message
+ that has to carry them. Such a group is installed as several
+ flows instead, in this table or in
+ <code>OFTABLE_LOCAL_OUTPUT</code>, each holding one slice of the
+ outputs and, except the last, ending in a <code>resubmit</code>
+ back to the same table. The next slice is found by an id carried
+ in <code>reg0</code>, with <code>MLF_MC_CHUNK</code> marking that
+ lookup. <code>reg0</code> is saved on the stack across it, and
+ <code>MLF_MC_CHUNK</code> cleared, before a slice runs the egress
+ pipeline.
<code>OFTABLE_REMOTE_OUTPUT</code> also includes:
</p>
diff --git a/tests/ovn.at b/tests/ovn.at
index 6582760fcef0..7512d825f924 100644
--- a/tests/ovn.at
+++ b/tests/ovn.at
@@ -47355,6 +47355,80 @@ OVN_CLEANUP([hv1])
AT_CLEANUP
])
+OVN_FOR_EACH_NORTHD([
+AT_SETUP([broadcast to many remote ports reaches every port])
+ovn_start
+net_add n1
+sim_add hv1
+as hv1
+ovs-vsctl add-br br-phys
+ovn_attach n1 br-phys 192.168.0.11 16
+
+dnl Tunnels to a chassis in another availability zone are only built when
+dnl this chassis is an interconnection gateway.
+check ovs-vsctl set open . external_ids:ovn-is-interconn=true
+
+dnl A type=remote port models a port in another availability zone. Unlike
+dnl a port on a peer chassis in this zone, it cannot share an output with
+dnl its neighbours: the remote zone has no copy of this multicast group, so
+dnl the destination's own tunnel key has to go into the Geneve option. Each
+dnl one therefore costs three field writes plus an output, some 80 bytes of
+dnl OpenFlow, so a flood outgrows its FLOW_MOD somewhere past 800 ports and
+dnl is discarded whole. Rather than create that many, shrink the budget so
+dnl that a few dozen ports are enough to split, and enough to chain.
+as hv1 ovs-appctl -t ovn-controller debug/set-mc-chunk-max 4096
+N=120
+
+check ovn-sbctl chassis-add rch geneve 192.168.0.12 \
+ -- set chassis rch other_config:is-remote=true
+
+check ovn-nbctl ls-add ls1
+check ovn-nbctl lsp-add ls1 vif-local \
+ -- lsp-set-addresses vif-local "f0:00:00:00:00:01"
+check ovs-vsctl add-port br-int vif-local \
+ -- set Interface vif-local external-ids:iface-id=vif-local
+
+dnl Every remote port is encapsulated to individually, so they can all sit
+dnl behind one remote chassis.
+nbctl_cmd=""
+for i in $(seq 1 $N); do
+ hi=$(printf "%02x" $(($i / 256))); lo=$(printf "%02x" $(($i % 256)))
+ nbctl_cmd="$nbctl_cmd -- lsp-add ls1 rp-$i"
+ nbctl_cmd="$nbctl_cmd -- lsp-set-type rp-$i remote"
+ nbctl_cmd="$nbctl_cmd -- lsp-set-options rp-$i requested-chassis=rch"
+ nbctl_cmd="$nbctl_cmd -- lsp-set-addresses rp-$i \"f0:00:01:00:$hi:$lo\""
+done
+eval check ovn-nbctl $nbctl_cmd
+check ovn-nbctl --wait=sb sync
+
+dnl Resolve the tunnel next hop so the outputs actually encapsulate.
+as hv1 ovs-appctl tnl/neigh/set br-phys 192.168.0.12 f0:00:02:00:00:01
+OVS_WAIT_UNTIL([as hv1 ovs-vsctl list-ports br-int | grep -q '^ovn-rch'])
+check ovn-nbctl --wait=hv sync
+
+dnl The flood did not fit one flow, so it was split across a chain of them.
+AT_CHECK([test $(as hv1 ovs-ofctl dump-flows br-int
table=OFTABLE_REMOTE_OUTPUT |
+ grep -c 'reg15=0x8000') -ge 3])
+
+dnl Nothing was dropped for being too big to install.
+AT_CHECK([grep "FLOW_MOD message is too big" hv1/ovn-controller.log], [1])
+
+in_ofport=$(as hv1 ovs-vsctl get Interface vif-local ofport)
+flow="in_port=$in_ofport,dl_src=f0:00:00:00:00:01,dl_dst=ff:ff:ff:ff:ff:ff,arp,arp_op=1,arp_spa=10.0.0.1,arp_tpa=10.0.0.250,arp_sha=f0:00:00:00:00:01,arp_tha=ff:ff:ff:ff:ff:ff"
+AT_CHECK([as hv1 ovs-appctl ofproto/trace br-int "$flow"], [0], [stdout])
+
+AT_CHECK([grep "Translation failed" stdout], [1])
+AT_CHECK([grep "over 4096 resubmit" stdout], [1])
+dnl Every remote port is encapsulated to across the slices. They share one
+dnl tunnel, so the destinations are told apart by the Geneve option that
+dnl carries the destination's own tunnel key.
+AT_CHECK([test $(grep -o 'len=4,0x[[0-9a-f]]*' stdout |
+ sort -u | wc -l) -eq $N])
+
+OVN_CLEANUP([hv1])
+AT_CLEANUP
+])
+
OVN_FOR_EACH_NORTHD([
AT_SETUP([Port security - VRRPv3 ARP/ND])
CHECK_SCAPY
--
2.38.1
_______________________________________________
dev mailing list
[email protected]
https://mail.openvswitch.org/mailman/listinfo/ovs-dev