A multicast flood carries one set of actions per destination, so a large
enough group outgrows the FLOW_MOD message that has to carry them.
ofctrl then discards the flow with "The FLOW_MOD message is too big" and
the flood reaches nobody.  A logical switch with more than about 800
ports on remote chassis is enough.

Install such a group as a chain of flows instead, each holding a slice
of the outputs and resubmitting to the next.  The next slice is found by
an id published in reg0, with MLF_MC_CHUNK marking that lookup.  A slice
only ends on a completed delivery, so no destination is cut in half.

That size is more than a test wants to build, so debug/set-mc-chunk-max
lowers the budget a flood is split at.

Fixes: f4d24c09d848 ("ofctrl: Do not try to program long flows")
Co-authored-by: Ram Vepa <[email protected]>
Signed-off-by: Ram Vepa <[email protected]>
Assisted-by: Claude Opus 5, Cursor
Signed-off-by: Han Zhou <[email protected]>
---
On why the test lowers the chunk budget instead of building a group big
enough to really overrun a FLOW_MOD:

That takes more than 800 remote ports, and at that size the test would
no longer be testing this patch.  Past about 710 destinations a flood
does not fit in one datapath action list either, and translation drops
whatever is left over without reporting anything, so "every destination
is reached" would fail for a reason that has nothing to do with the
split.  That truncation is an OVS bug in its own right, fixed separately
by:
https://patchwork.ozlabs.org/project/openvswitch/patch/[email protected]/

Lowering the budget keeps the test cheap, and lets it cover a chain of
several slices rather than the two that 800 ports would produce.

 controller/ovn-controller.8.xml |   7 ++
 controller/ovn-controller.c     |  22 ++++
 controller/physical.c           | 185 +++++++++++++++++++++++++++++++-
 controller/physical.h           |   1 +
 include/ovn/logical-fields.h    |   5 +
 ovn-architecture.7.xml          |  11 ++
 tests/ovn.at                    |  74 +++++++++++++
 7 files changed, 300 insertions(+), 5 deletions(-)

diff --git a/controller/ovn-controller.8.xml b/controller/ovn-controller.8.xml
index 2b90f25d9ff1..ca04f28395da 100644
--- a/controller/ovn-controller.8.xml
+++ b/controller/ovn-controller.8.xml
@@ -1078,6 +1078,13 @@
         debugging.
       </dd>
 
+      <dt><code>debug/set-mc-chunk-max</code> <var>bytes</var></dt>
+      <dd>
+        Sets how many encoded OpenFlow action bytes a multicast flood may
+        put in one flow before it is split across several, for testing.
+        Zero restores the default.
+      </dd>
+
       <dt><code>debug/resume</code></dt>
       <dd>
         Resumes a processing loop paused by <code>debug/pause</code>.
diff --git a/controller/ovn-controller.c b/controller/ovn-controller.c
index 7bc3139ab435..8311cafb578e 100644
--- a/controller/ovn-controller.c
+++ b/controller/ovn-controller.c
@@ -127,6 +127,7 @@ static unixctl_cb_func debug_dump_lflow_conj_ids;
 static unixctl_cb_func lflow_cache_flush_cmd;
 static unixctl_cb_func lflow_cache_show_stats_cmd;
 static unixctl_cb_func debug_delay_nb_cfg_report;
+static unixctl_cb_func debug_set_mc_chunk_max;
 
 #define DEFAULT_BRIDGE_NAME "br-int"
 #define DEFAULT_DATAPATH "system"
@@ -8455,6 +8456,8 @@ main(int argc, char *argv[])
     unsigned int delay_nb_cfg_report = 0;
     unixctl_command_register("debug/delay-nb-cfg-report", "SECONDS", 1, 1,
                              debug_delay_nb_cfg_report, &delay_nb_cfg_report);
+    unixctl_command_register("debug/set-mc-chunk-max", "BYTES", 1, 1,
+                             debug_set_mc_chunk_max, NULL);
 
     unixctl_command_register("debug/dump-local-bindings", "", 0, 0,
                              debug_dump_local_bindings,
@@ -9581,6 +9584,25 @@ debug_status_execution(struct unixctl_conn *conn, int 
argc OVS_UNUSED,
     }
 }
 
+/* Lowers the number of encoded OpenFlow action bytes a multicast flood is
+ * allowed to put in one flow, so that a test can split a small group.
+ * Zero restores the default. */
+static void
+debug_set_mc_chunk_max(struct unixctl_conn *conn, int argc OVS_UNUSED,
+                       const char *argv[], void *arg OVS_UNUSED)
+{
+    unsigned int bytes;
+
+    if (!str_to_uint(argv[1], 10, &bytes)) {
+        unixctl_command_reply_error(conn, "unsigned integer required");
+        return;
+    }
+
+    physical_set_mc_chunk_max(bytes);
+    engine_set_force_recompute();
+    unixctl_command_reply(conn, NULL);
+}
+
 static void
 debug_delay_nb_cfg_report(struct unixctl_conn *conn, int argc OVS_UNUSED,
                           const char *argv[], void *delay_)
diff --git a/controller/physical.c b/controller/physical.c
index 01013be48c73..dbd0cc9970c5 100644
--- a/controller/physical.c
+++ b/controller/physical.c
@@ -2938,6 +2938,181 @@ flood_querier_port_add(struct mcast_querier_port 
*flood_ports,
     (*n_flood_ports)++;
 }
 
+/* A multicast flood carries one set of actions per destination, and a
+ * large enough group outgrows the FLOW_MOD that has to carry them: past
+ * that, ofctrl discards the flow and the flood reaches nobody.  Such a
+ * group is installed as a chain of flows instead, each holding a slice of
+ * the outputs and resubmitting to the next.
+ *
+ * The next slice is found by an id published in reg0, with MLF_MC_CHUNK
+ * marking that lookup.  reg0 is stacked around it because the egress
+ * pipeline reads it. */
+
+/* Encoded OpenFlow action bytes per flow.  add_flow_mod() has a little
+ * over 65,000 to spend on actions, once the bundle and flow_mod headers,
+ * the match and the instruction header are taken out. */
+#define MC_CHUNK_OF_DEFAULT (60 * 1024)
+
+static size_t mc_chunk_of_max = MC_CHUNK_OF_DEFAULT;
+
+void
+physical_set_mc_chunk_max(size_t bytes)
+{
+    mc_chunk_of_max = bytes ? bytes : MC_CHUNK_OF_DEFAULT;
+}
+
+static size_t
+mc_chunk_cost(const struct ofpact *ofpacts, size_t len,
+              struct ofpbuf *scratch)
+{
+    ofpbuf_clear(scratch);
+    return ofpacts_put_openflow_actions(ofpacts, len, scratch,
+                                        OFP15_VERSION);
+}
+
+static bool
+mc_fits_one_chunk(const struct ofpbuf *ofpacts)
+{
+    struct ofpbuf scratch;
+    size_t size;
+
+    ofpbuf_init(&scratch, ofpacts->size);
+    size = mc_chunk_cost(ofpacts->data, ofpacts->size, &scratch);
+    ofpbuf_uninit(&scratch);
+
+    return size <= mc_chunk_of_max;
+}
+
+static void
+mc_chunk_match(const struct sbrec_multicast_group *mc,
+               const struct mc_flow_ctx *ctx, uint32_t chunk_id,
+               struct match *match)
+{
+    uint32_t flags = ctx->flags;
+    uint32_t mask = ctx->flags_mask | MLF_MC_CHUNK;
+
+    match_init_catchall(match);
+    match_outport_dp_and_port_keys(match, mc->datapath->tunnel_key,
+                                   mc->tunnel_key);
+    if (chunk_id > 0) {
+        flags |= MLF_MC_CHUNK;
+        match_set_reg(match, MFF_REG0 - MFF_REG0, chunk_id);
+    }
+    match_set_reg_masked(match, MFF_LOG_FLAGS - MFF_REG0, flags, mask);
+}
+
+static void
+mc_compose_chunk(struct ofpbuf *dst, const struct ofpbuf *body,
+                 bool first, bool last, uint32_t next_chunk,
+                 int64_t mc_key, uint8_t same_table, bool resume_next,
+                 uint8_t next_table)
+{
+    ofpbuf_clear(dst);
+    if (!first) {
+        put_stack(MFF_REG0, ofpact_put_STACK_POP(dst));
+        put_load(0, MFF_LOG_FLAGS, MLF_MC_CHUNK_BIT, 1, dst);
+    }
+    ofpbuf_put(dst, body->data, body->size);
+    if (!last) {
+        /* The body may have overwritten the outport this table matches,
+         * and the egress pipeline reads reg0. */
+        put_stack(MFF_REG0, ofpact_put_STACK_PUSH(dst));
+        put_load(mc_key, MFF_LOG_OUTPORT, 0, 32, dst);
+        put_load(next_chunk, MFF_REG0, 0, 32, dst);
+        put_load(1, MFF_LOG_FLAGS, MLF_MC_CHUNK_BIT, 1, dst);
+        put_resubmit(same_table, dst);
+    } else if (resume_next) {
+        put_load(mc_key, MFF_LOG_OUTPORT, 0, 32, dst);
+        put_resubmit(next_table, dst);
+    }
+}
+
+static void
+mc_add_chunk_flow(const struct sbrec_multicast_group *mc,
+                  const struct mc_flow_ctx *ctx, uint32_t chunk_id,
+                  const struct ofpbuf *actions,
+                  struct ovn_desired_flow_table *flow_table)
+{
+    struct match match;
+
+    mc_chunk_match(mc, ctx, chunk_id, &match);
+    ofctrl_add_flow(flow_table, ctx->stage, ctx->prio,
+                    mc->header_.uuid.parts[0], &match, actions,
+                    &mc->header_.uuid);
+}
+
+static void
+mc_split_flows(const struct sbrec_multicast_group *mc,
+               struct mc_flow_ctx *ctx, bool resume_next, uint8_t next_table,
+               struct ovn_desired_flow_table *flow_table)
+{
+    struct ofpbuf body, actions, scratch;
+    const struct ofpact *a;
+    const struct ofpact *base = ctx->ofpacts.data;
+    size_t total = ctx->ofpacts.size;
+    uint32_t chunk_id = 0;
+    size_t sum = 0;
+
+    ofpbuf_init(&body, 0);
+    ofpbuf_init(&actions, 0);
+    ofpbuf_init(&scratch, 64);
+
+    /* A slice has to end on a completed delivery: cutting between a port's
+     * register loads and the resubmit that consumes them would deliver that
+     * port to the multicast key the continuation restores. */
+    const struct ofpact *last = NULL;
+    OFPACT_FOR_EACH (a, base, total) {
+        size_t cost = mc_chunk_cost(a, OFPACT_ALIGN(a->len), &scratch);
+        bool boundary = last && (last->type == OFPACT_RESUBMIT
+                                 || last->type == OFPACT_OUTPUT);
+
+        if (body.size > 0 && sum + cost > mc_chunk_of_max && boundary) {
+            mc_compose_chunk(&actions, &body, chunk_id == 0, false,
+                             chunk_id + 1, mc->tunnel_key, ctx->stage,
+                             resume_next, next_table);
+            mc_add_chunk_flow(mc, ctx, chunk_id, &actions, flow_table);
+            chunk_id++;
+            ofpbuf_clear(&body);
+            last = NULL;
+            sum = 0;
+        }
+        ofpbuf_put(&body, a, OFPACT_ALIGN(a->len));
+        sum += cost;
+        last = a;
+    }
+
+    mc_compose_chunk(&actions, &body, chunk_id == 0, true, 0,
+                     mc->tunnel_key, ctx->stage, resume_next, next_table);
+    mc_add_chunk_flow(mc, ctx, chunk_id, &actions, flow_table);
+
+    ofpbuf_uninit(&body);
+    ofpbuf_uninit(&actions);
+    ofpbuf_uninit(&scratch);
+}
+
+/* Install the multicast output in 'ctx', splitting it across several flows
+ * if it is too large for one.  When 'resume_next' is true the last slice
+ * resubmits to 'next_table'. */
+static void
+mc_install_flows(const struct sbrec_multicast_group *mc,
+                 struct mc_flow_ctx *ctx, bool resume_next, uint8_t next_table,
+                 struct ovn_desired_flow_table *flow_table)
+{
+    size_t plain = ctx->ofpacts.size;
+
+    if (resume_next) {
+        put_resubmit(next_table, &ctx->ofpacts);
+    }
+    if (mc_fits_one_chunk(&ctx->ofpacts)) {
+        mc_ofctrl_add_flow(mc, ctx, flow_table);
+        return;
+    }
+
+    ofpbuf_truncate(&ctx->ofpacts, plain);
+    mc_split_flows(mc, ctx, resume_next, next_table, flow_table);
+    ofpbuf_clear(&ctx->ofpacts);
+}
+
 static void
 consider_mc_group(const struct physical_ctx *ctx,
                   const struct sbrec_multicast_group *mc,
@@ -3116,7 +3291,7 @@ consider_mc_group(const struct physical_ctx *ctx,
 
     if (local_lports) {
         put_load(mc->tunnel_key, MFF_LOG_OUTPORT, 0, 32, &local_ctx->ofpacts);
-        mc_ofctrl_add_flow(mc, local_ctx, flow_table);
+        mc_install_flows(mc, local_ctx, false, 0, flow_table);
     }
 
     if (remote_ports) {
@@ -3148,14 +3323,14 @@ consider_mc_group(const struct physical_ctx *ctx,
 
     remote_ports = remote_ctx->ofpacts.size > 0;
     if (remote_ports) {
-        put_resubmit(OFTABLE_REMOTE_VTEP_OUTPUT, &remote_ctx->ofpacts);
-        mc_ofctrl_add_flow(mc, remote_ctx, flow_table);
+        mc_install_flows(mc, remote_ctx, true, OFTABLE_REMOTE_VTEP_OUTPUT,
+                         flow_table);
     }
 
     if (ramp_ports && has_vtep) {
         put_load(mc->tunnel_key, MFF_LOG_OUTPORT, 0, 32, &ramp_ctx->ofpacts);
-        put_resubmit(OFTABLE_REMOTE_VTEP_OUTPUT, &ramp_ctx->ofpacts);
-        mc_ofctrl_add_flow(mc, ramp_ctx, flow_table);
+        mc_install_flows(mc, ramp_ctx, true, OFTABLE_REMOTE_VTEP_OUTPUT,
+                         flow_table);
     }
 
     for (size_t i = 0; i < MC_FLOWS_MAX; i++) {
diff --git a/controller/physical.h b/controller/physical.h
index 8218096d5e49..6828342a82a4 100644
--- a/controller/physical.h
+++ b/controller/physical.h
@@ -85,6 +85,7 @@ struct physical_ctx {
 };
 
 void physical_register_ovs_idl(struct ovsdb_idl *);
+void physical_set_mc_chunk_max(size_t bytes);
 void physical_run(struct physical_ctx *,
                   struct ovn_desired_flow_table *);
 void physical_handle_mc_group_changes(struct physical_ctx *,
diff --git a/include/ovn/logical-fields.h b/include/ovn/logical-fields.h
index de81a7fe255a..433ba35d168f 100644
--- a/include/ovn/logical-fields.h
+++ b/include/ovn/logical-fields.h
@@ -144,6 +144,7 @@ enum mff_log_flags_bits {
     MLF_RECIRC_BIT = 24,
     MLF_EVPN_LOOKUP_BIT = 25,
     MLF_POST_NF_LOOP_BACK_BIT = 26,
+    MLF_MC_CHUNK_BIT = 27,
     MLF_NETWORK_ID_START_BIT = 28,
     MLF_NETWORK_ID_END_BIT = 31,
 };
@@ -224,6 +225,10 @@ enum mff_log_flags {
     /* Set on a post-NF packet going back to the original ingress port. */
     MLF_POST_NF_LOOP_BACK = (1 << MLF_POST_NF_LOOP_BACK_BIT),
 
+    /* Set on a multicast flood that was split across several OpenFlow
+     * flows, for the table lookup of the next chunk only. */
+    MLF_MC_CHUNK = (1 << MLF_MC_CHUNK_BIT),
+
     /* Assign network ID to packet to choose correct network for snat when
      * lb_force_snat_ip=router_ip. */
     MLF_NETWORK_ID = (OVN_MAX_NETWORK_ID << MLF_NETWORK_ID_START_BIT),
diff --git a/ovn-architecture.7.xml b/ovn-architecture.7.xml
index 00728ee9b7e6..c3ecdca15d36 100644
--- a/ovn-architecture.7.xml
+++ b/ovn-architecture.7.xml
@@ -1553,6 +1553,17 @@
         multicast group includes a logical port or ports on the local
         hypervisor, then its actions also resubmit to
         <code>OFTABLE_LOCAL_OUTPUT</code>.
+        A multicast group carries one set of actions per destination, and
+        a large enough group outgrows the <code>FLOW_MOD</code> message
+        that has to carry them.  Such a group is installed as several
+        flows instead, in this table or in
+        <code>OFTABLE_LOCAL_OUTPUT</code>, each holding one slice of the
+        outputs and, except the last, ending in a <code>resubmit</code>
+        back to the same table.  The next slice is found by an id carried
+        in <code>reg0</code>, with <code>MLF_MC_CHUNK</code> marking that
+        lookup.  <code>reg0</code> is saved on the stack across it, and
+        <code>MLF_MC_CHUNK</code> cleared, before a slice runs the egress
+        pipeline.
         <code>OFTABLE_REMOTE_OUTPUT</code> also includes:
       </p>
 
diff --git a/tests/ovn.at b/tests/ovn.at
index 6582760fcef0..7512d825f924 100644
--- a/tests/ovn.at
+++ b/tests/ovn.at
@@ -47355,6 +47355,80 @@ OVN_CLEANUP([hv1])
 AT_CLEANUP
 ])
 
+OVN_FOR_EACH_NORTHD([
+AT_SETUP([broadcast to many remote ports reaches every port])
+ovn_start
+net_add n1
+sim_add hv1
+as hv1
+ovs-vsctl add-br br-phys
+ovn_attach n1 br-phys 192.168.0.11 16
+
+dnl Tunnels to a chassis in another availability zone are only built when
+dnl this chassis is an interconnection gateway.
+check ovs-vsctl set open . external_ids:ovn-is-interconn=true
+
+dnl A type=remote port models a port in another availability zone.  Unlike
+dnl a port on a peer chassis in this zone, it cannot share an output with
+dnl its neighbours: the remote zone has no copy of this multicast group, so
+dnl the destination's own tunnel key has to go into the Geneve option.  Each
+dnl one therefore costs three field writes plus an output, some 80 bytes of
+dnl OpenFlow, so a flood outgrows its FLOW_MOD somewhere past 800 ports and
+dnl is discarded whole.  Rather than create that many, shrink the budget so
+dnl that a few dozen ports are enough to split, and enough to chain.
+as hv1 ovs-appctl -t ovn-controller debug/set-mc-chunk-max 4096
+N=120
+
+check ovn-sbctl chassis-add rch geneve 192.168.0.12 \
+    -- set chassis rch other_config:is-remote=true
+
+check ovn-nbctl ls-add ls1
+check ovn-nbctl lsp-add ls1 vif-local \
+    -- lsp-set-addresses vif-local "f0:00:00:00:00:01"
+check ovs-vsctl add-port br-int vif-local \
+    -- set Interface vif-local external-ids:iface-id=vif-local
+
+dnl Every remote port is encapsulated to individually, so they can all sit
+dnl behind one remote chassis.
+nbctl_cmd=""
+for i in $(seq 1 $N); do
+    hi=$(printf "%02x" $(($i / 256))); lo=$(printf "%02x" $(($i % 256)))
+    nbctl_cmd="$nbctl_cmd -- lsp-add ls1 rp-$i"
+    nbctl_cmd="$nbctl_cmd -- lsp-set-type rp-$i remote"
+    nbctl_cmd="$nbctl_cmd -- lsp-set-options rp-$i requested-chassis=rch"
+    nbctl_cmd="$nbctl_cmd -- lsp-set-addresses rp-$i \"f0:00:01:00:$hi:$lo\""
+done
+eval check ovn-nbctl $nbctl_cmd
+check ovn-nbctl --wait=sb sync
+
+dnl Resolve the tunnel next hop so the outputs actually encapsulate.
+as hv1 ovs-appctl tnl/neigh/set br-phys 192.168.0.12 f0:00:02:00:00:01
+OVS_WAIT_UNTIL([as hv1 ovs-vsctl list-ports br-int | grep -q '^ovn-rch'])
+check ovn-nbctl --wait=hv sync
+
+dnl The flood did not fit one flow, so it was split across a chain of them.
+AT_CHECK([test $(as hv1 ovs-ofctl dump-flows br-int 
table=OFTABLE_REMOTE_OUTPUT |
+                 grep -c 'reg15=0x8000') -ge 3])
+
+dnl Nothing was dropped for being too big to install.
+AT_CHECK([grep "FLOW_MOD message is too big" hv1/ovn-controller.log], [1])
+
+in_ofport=$(as hv1 ovs-vsctl get Interface vif-local ofport)
+flow="in_port=$in_ofport,dl_src=f0:00:00:00:00:01,dl_dst=ff:ff:ff:ff:ff:ff,arp,arp_op=1,arp_spa=10.0.0.1,arp_tpa=10.0.0.250,arp_sha=f0:00:00:00:00:01,arp_tha=ff:ff:ff:ff:ff:ff"
+AT_CHECK([as hv1 ovs-appctl ofproto/trace br-int "$flow"], [0], [stdout])
+
+AT_CHECK([grep "Translation failed" stdout], [1])
+AT_CHECK([grep "over 4096 resubmit" stdout], [1])
+dnl Every remote port is encapsulated to across the slices.  They share one
+dnl tunnel, so the destinations are told apart by the Geneve option that
+dnl carries the destination's own tunnel key.
+AT_CHECK([test $(grep -o 'len=4,0x[[0-9a-f]]*' stdout |
+                 sort -u | wc -l) -eq $N])
+
+OVN_CLEANUP([hv1])
+AT_CLEANUP
+])
+
 OVN_FOR_EACH_NORTHD([
 AT_SETUP([Port security - VRRPv3 ARP/ND])
 CHECK_SCAPY
-- 
2.38.1

_______________________________________________
dev mailing list
[email protected]
https://mail.openvswitch.org/mailman/listinfo/ovs-dev

Reply via email to