From: Loke Berne <[email protected]> The tc flow dump translates each tc flower rule into odp actions inside a fixed-size OFPBUF_STACK buffer of ODPUTIL_FLOW_KEY_BYTES. Unlike the key and the mask, the actions have no such bound: a single flower rule can carry up to TCA_ACT_MAX_PRIO actions, and a tunnel_push plus output pair alone takes about 100 bytes. When a rule does not fit, ofpbuf_resize__() hits OVS_NOT_REACHED() and ovs-vswitchd aborts. tc_parse_flow_get() has the same problem with its 1024-byte buffer.
This was hit in production with flood rules that output to many remote tunnel endpoints. Every non-terse "ovs-appctl dpctl/dump-flows" (every type= except "ovs") walks this path, so a monitoring agent polling the flows crash-looped ovs-vswitchd on each poll. tc_parse_flow_get() also returns a dpif_flow whose key and mask point into buffers on its own stack frame, so callers such as "ovs-appctl dpctl/get-flow" read a stale stack frame. Translate the actions, key and mask of a flow into a single buffer: a stub-backed buffer per batch slot for the dump, and the caller's buffer for flow get. The flow's pointers are resolved after the last write, as the buffer may be reallocated. The added test sends a flow to four Geneve tunnels with a 124-byte option each, which dumps back as about 750 bytes of odp actions, and dumps it with type=tc. Without the fix ovs-vswitchd aborts on the dump. CC: Paul Blakey <[email protected]> Fixes: f2280b419860 ("dpif-netlink: Dump netdevs flows on flow dump") Fixes: 6c343984809c ("dpif-netlink: Use netdev flow get api to query a flow") Assisted-by: Claude Fable 5.1, Claude Code Signed-off-by: Loke Berne <[email protected]> --- lib/dpif-offload-tc.c | 90 ++++++++++++++------------------ tests/system-offloads-traffic.at | 47 +++++++++++++++++ 2 files changed, 87 insertions(+), 50 deletions(-) diff --git a/lib/dpif-offload-tc.c b/lib/dpif-offload-tc.c index 3b84d9f20..c7d0a7996 100644 --- a/lib/dpif-offload-tc.c +++ b/lib/dpif-offload-tc.c @@ -58,11 +58,10 @@ struct tc_flow_dump_thread { bool netdev_dump_done; size_t netdev_dump_index; - /* (Flows/Key/Mask/Actions) Buffers for netdev dumping. */ + /* Netlink flows, and per batch slot a flow's actions, key and mask. */ struct ofpbuf nl_flows; - struct odputil_keybuf keybuf[FLOW_DUMP_MAX_BATCH]; - struct odputil_keybuf maskbuf[FLOW_DUMP_MAX_BATCH]; - struct odputil_keybuf actbuf[FLOW_DUMP_MAX_BATCH]; + struct odputil_keybuf buf_stub[FLOW_DUMP_MAX_BATCH][3]; + struct ofpbuf buf[FLOW_DUMP_MAX_BATCH]; }; static struct tc_offload * @@ -317,8 +316,8 @@ tc_flow_dump_create(const struct dpif_offload *offload, bool terse) } static int -tc_match_to_dpif_flow(struct match *match, struct ofpbuf *key_buf, - struct ofpbuf *mask_buf, struct nlattr *actions, +tc_match_to_dpif_flow(struct match *match, struct ofpbuf *buf, + struct nlattr *actions, struct dpif_flow_stats *stats, struct dpif_flow_attrs *attrs, ovs_u128 *ufid, struct dpif_flow *flow, bool terse) @@ -338,24 +337,28 @@ tc_match_to_dpif_flow(struct match *match, struct ofpbuf *key_buf, .ct_label = true, }, }; - size_t offset; - - /* Key */ - offset = key_buf->size; - flow->key = ofpbuf_tail(key_buf); - odp_flow_key_from_flow(&odp_parms, key_buf); - flow->key_len = key_buf->size - offset; - - /* Mask */ - offset = mask_buf->size; - flow->mask = ofpbuf_tail(mask_buf); - odp_parms.key_buf = key_buf; - odp_flow_key_from_mask(&odp_parms, mask_buf); - flow->mask_len = mask_buf->size - offset; - - /* Actions */ - flow->actions = nl_attr_get(actions); + size_t act_ofs, key_ofs, mask_ofs; + struct ofpbuf key; + + act_ofs = (char *) nl_attr_get(actions) - (char *) buf->data; flow->actions_len = nl_attr_get_size(actions); + + key_ofs = buf->size; + odp_flow_key_from_flow(&odp_parms, buf); + flow->key_len = buf->size - key_ofs; + + /* The mask is built while reading 'key', so 'buf' must not move. */ + ofpbuf_prealloc_tailroom(buf, ODPUTIL_FLOW_KEY_BYTES); + ofpbuf_use_const(&key, ofpbuf_at_assert(buf, key_ofs, flow->key_len), + flow->key_len); + odp_parms.key_buf = &key; + mask_ofs = buf->size; + odp_flow_key_from_mask(&odp_parms, buf); + flow->mask_len = buf->size - mask_ofs; + + flow->actions = ofpbuf_at_assert(buf, act_ofs, flow->actions_len); + flow->key = ofpbuf_at_assert(buf, key_ofs, flow->key_len); + flow->mask = ofpbuf_at_assert(buf, mask_ofs, flow->mask_len); } /* Stats */ @@ -411,13 +414,10 @@ tc_flow_dump_next(struct dpif_offload_flow_dump_thread *thread_, max_flows = MIN(max_flows, FLOW_DUMP_MAX_BATCH); while (!thread->netdev_dump_done && n_flows < max_flows) { - struct odputil_keybuf *maskbuf = &thread->maskbuf[n_flows]; - struct odputil_keybuf *keybuf = &thread->keybuf[n_flows]; - struct odputil_keybuf *actbuf = &thread->actbuf[n_flows]; + struct ofpbuf *buf = &thread->buf[n_flows]; struct netdev_tc_flow_dump *netdev_dump; struct dpif_flow *f = &flows[n_flows]; int cur = thread->netdev_dump_index; - struct ofpbuf key, mask, act; struct dpif_flow_stats stats; struct dpif_flow_attrs attrs; struct nlattr *actions; @@ -426,15 +426,13 @@ tc_flow_dump_next(struct dpif_offload_flow_dump_thread *thread_, bool has_next; netdev_dump = thread->dump->netdev_dumps[cur]; - ofpbuf_use_stack(&key, keybuf, sizeof *keybuf); - ofpbuf_use_stack(&act, actbuf, sizeof *actbuf); - ofpbuf_use_stack(&mask, maskbuf, sizeof *maskbuf); + ofpbuf_clear(buf); has_next = tc_netdev_flow_dump_next(netdev_dump, &match, &actions, &stats, &attrs, &ufid, - &thread->nl_flows, &act); + &thread->nl_flows, buf); if (has_next) { - tc_match_to_dpif_flow(&match, &key, &mask, actions, &stats, - &attrs, &ufid, f, thread->dump->dump.terse); + tc_match_to_dpif_flow(&match, buf, actions, &stats, &attrs, &ufid, + f, thread->dump->dump.terse); n_flows++; } else { tc_advance_provider_dump(thread); @@ -473,6 +471,10 @@ tc_flow_dump_thread_create(struct dpif_offload_flow_dump *dump) thread->netdev_dump_index = 0; thread->netdev_dump_done = !thread->dump->netdev_dump_count; ofpbuf_init(&thread->nl_flows, NL_DUMP_BUFSIZE); + for (int i = 0; i < FLOW_DUMP_MAX_BATCH; i++) { + ofpbuf_use_stub(&thread->buf[i], thread->buf_stub[i], + sizeof thread->buf_stub[i]); + } return &thread->thread; } @@ -481,6 +483,9 @@ tc_flow_dump_thread_destroy(struct dpif_offload_flow_dump_thread *thread_) { struct tc_flow_dump_thread *thread = tc_flow_dump_thread_cast(thread_); + for (int i = 0; i < FLOW_DUMP_MAX_BATCH; i++) { + ofpbuf_uninit(&thread->buf[i]); + } ofpbuf_uninit(&thread->nl_flows); free(thread); } @@ -616,23 +621,15 @@ tc_parse_flow_get(struct tc_offload *offload_tc, struct dpif_flow_get *get) { struct dpif_flow *dpif_flow = get->flow; struct dpif_offload_port *port; - struct odputil_keybuf maskbuf; - struct odputil_keybuf keybuf; - struct odputil_keybuf actbuf; - struct ofpbuf key, mask, act; struct dpif_flow_stats stats; struct dpif_flow_attrs attrs; - uint64_t act_buf[1024 / 8]; struct nlattr *actions; struct match match; - struct ofpbuf buf; int err = ENOENT; - ofpbuf_use_stack(&buf, &act_buf, sizeof act_buf); - DPIF_OFFLOAD_PORT_FOR_EACH (port, &offload_tc->offload) { if (!tc_netdev_flow_get(port->netdev, &match, &actions, get->ufid, - &stats, &attrs, &buf)) { + &stats, &attrs, get->buffer)) { err = 0; break; } @@ -644,15 +641,8 @@ tc_parse_flow_get(struct tc_offload *offload_tc, struct dpif_flow_get *get) VLOG_DBG("found flow from netdev, translating to dpif flow"); - ofpbuf_use_stack(&key, &keybuf, sizeof keybuf); - ofpbuf_use_stack(&act, &actbuf, sizeof actbuf); - ofpbuf_use_stack(&mask, &maskbuf, sizeof maskbuf); - tc_match_to_dpif_flow(&match, &key, &mask, actions, &stats, &attrs, + tc_match_to_dpif_flow(&match, get->buffer, actions, &stats, &attrs, (ovs_u128 *) get->ufid, dpif_flow, false); - ofpbuf_put(get->buffer, nl_attr_get(actions), nl_attr_get_size(actions)); - dpif_flow->actions = ofpbuf_at(get->buffer, 0, 0); - dpif_flow->actions_len = nl_attr_get_size(actions); - return 0; } diff --git a/tests/system-offloads-traffic.at b/tests/system-offloads-traffic.at index cfcc5bd6c..5e7cf9ae0 100644 --- a/tests/system-offloads-traffic.at +++ b/tests/system-offloads-traffic.at @@ -1285,3 +1285,50 @@ m4_for([id], [2], [17], [1], [ OVS_TRAFFIC_VSWITCHD_STOP AT_CLEANUP + +AT_SETUP([offloads - dump flow with large actions]) +OVS_CHECK_GENEVE() +OVS_TRAFFIC_VSWITCHD_START([], [], + [-- set Open_vSwitch . other_config:hw-offload=true]) + +AT_CHECK([ovs-appctl vlog/set tc:file:dbg dpif_offload_tc:file:dbg]) + +dnl Four Geneve outputs with a 124-byte option exceed ODPUTIL_FLOW_KEY_BYTES. +ADD_BR([br-underlay]) +AT_CHECK([ovs-ofctl add-flow br-underlay "actions=normal"]) + +ADD_NAMESPACES(at_ns0, at_ns1) + +ADD_VETH(p0, at_ns0, br-underlay, "172.31.1.1/24") +m4_for([id], [2], [4], [1], [ + NS_CHECK_EXEC([at_ns0], [ip addr add 172.31.1.id/24 dev p0]) +]) +AT_CHECK([ip addr add dev br-underlay "172.31.1.100/24"]) +AT_CHECK([ip link set dev br-underlay up]) + +m4_for([id], [1], [4], [1], [ + AT_CHECK([ovs-vsctl add-port br0 at_gnv[]id -- \ + set int at_gnv[]id type=geneve options:remote_ip=172.31.1.id \ + options:key=id]) +]) + +ADD_VETH(p1, at_ns1, br0, "10.1.1.1/24") +NS_CHECK_EXEC([at_ns1], + [ip neigh replace 10.1.1.2 lladdr 02:00:00:00:00:02 dev p1]) + +AT_CHECK([ovs-ofctl add-tlv-map br0 "{class=0xffff,type=0,len=124}->tun_metadata0"]) +AT_CHECK([ovs-ofctl add-flow br0 "priority=10,in_port=ovs-p1,ip,actions=set_field:0x1234->tun_metadata0,output:at_gnv1,output:at_gnv2,output:at_gnv3,output:at_gnv4"]) + +NS_CHECK_EXEC([at_ns1], [ping -q -c 3 -i 0.3 -W 1 10.1.1.2], [ignore], [ignore]) +AT_CHECK([ovs-appctl revalidator/wait]) + +AT_CHECK([ovs-appctl dpctl/dump-flows --names type=tc], [0], [stdout]) +AT_CHECK([grep "in_port(ovs-p1)" stdout | grep "eth_type(0x0800)" | head -1 | \ + grep -o "set[[(]]tunnel[[(]]" | wc -l], [0], [dnl +4 +]) +AT_CHECK([grep "in_port(ovs-p1)" stdout | \ + grep -q "geneve[[(]][[{]]class=0xffff,type=0,len=124,"]) + +OVS_TRAFFIC_VSWITCHD_STOP +AT_CLEANUP -- 2.56.1 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
