Allow the netdev_hw_post_process() callback to return an optional set of alternative actions via a new 'alt_actions' parameter. When set, these actions are executed instead of the flow's own actions, enabling offload providers to indicate that a subset of the actions was already performed in hardware.
Note that batching for packets with alternative actions is not optimal, as it requires a linear scan over the current batch list to find a matching (flow, alt_actions) pair. Acked-by: Eli Britstein <[email protected]> Signed-off-by: Eelco Chaudron <[email protected]> --- lib/dpif-netdev-perf.c | 3 + lib/dpif-netdev-perf.h | 2 + lib/dpif-netdev.c | 113 +++++++++++++++++++++++++++--------- lib/dpif-offload-dpdk.c | 5 +- lib/dpif-offload-dummy.c | 7 ++- lib/dpif-offload-provider.h | 7 ++- lib/dpif-offload.c | 4 +- lib/dpif-offload.h | 7 +++ tests/pmd.at | 3 +- 9 files changed, 118 insertions(+), 33 deletions(-) diff --git a/lib/dpif-netdev-perf.c b/lib/dpif-netdev-perf.c index 4b3b35852..7bd288216 100644 --- a/lib/dpif-netdev-perf.c +++ b/lib/dpif-netdev-perf.c @@ -267,6 +267,7 @@ pmd_perf_format_overall_stats(struct ds *str, struct pmd_perf_stats *s, ds_put_format(str, " Datapath passes: %12"PRIu64" (%.2f passes/pkt)\n" " - PHWOL hits: %12"PRIu64" (%5.1f %%)\n" + " - PHWOL + ACT hits: %12"PRIu64" (%5.1f %%)\n" " - Simple Match hits:%12"PRIu64" (%5.1f %%)\n" " - EMC hits: %12"PRIu64" (%5.1f %%)\n" " - SMC hits: %12"PRIu64" (%5.1f %%)\n" @@ -275,6 +276,8 @@ pmd_perf_format_overall_stats(struct ds *str, struct pmd_perf_stats *s, passes, 1.0 * passes / rx_packets, stats[PMD_STAT_PHWOL_HIT], 100.0 * stats[PMD_STAT_PHWOL_HIT] / passes, + stats[PMD_STAT_PHWOL_ACT_HIT], + 100.0 * stats[PMD_STAT_PHWOL_ACT_HIT] / passes, stats[PMD_STAT_SIMPLE_HIT], 100.0 * stats[PMD_STAT_SIMPLE_HIT] / passes, stats[PMD_STAT_EXACT_HIT], diff --git a/lib/dpif-netdev-perf.h b/lib/dpif-netdev-perf.h index cf4b6e103..3d17e978f 100644 --- a/lib/dpif-netdev-perf.h +++ b/lib/dpif-netdev-perf.h @@ -57,6 +57,8 @@ extern "C" { enum pmd_stat_type { PMD_STAT_PHWOL_HIT, /* Packets that had a partial HWOL hit (phwol). */ + PMD_STAT_PHWOL_ACT_HIT, /* Packets that had a partial HWOL hit with + alternative actions. */ PMD_STAT_SIMPLE_HIT, /* Packets that had a simple match hit. */ PMD_STAT_EXACT_HIT, /* Packets that had an exact match (emc). */ PMD_STAT_SMC_HIT, /* Packets that had a sig match hit (SMC). */ diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c index 4151ea056..e15561ba8 100644 --- a/lib/dpif-netdev.c +++ b/lib/dpif-netdev.c @@ -197,6 +197,7 @@ struct dpcls { struct dp_packet_flow_map { struct dp_packet *packet; struct dp_netdev_flow *flow; + struct dp_netdev_actions *alt_actions; /* NULL = use flow's own actions. */ uint16_t tcp_flags; }; @@ -468,6 +469,17 @@ struct dpif_netdev { uint64_t last_port_seq; }; +/* Verify that struct offload_actions (defined in dpif-offload.h) is + * layout-identical to struct dp_netdev_actions. The two structs are kept + * separate to avoid exposing dp_netdev_actions as a public API, but must + * remain binary-compatible so that ALIGNED_CAST() between them is safe. */ +BUILD_ASSERT_DECL(sizeof(struct offload_actions) == + sizeof(struct dp_netdev_actions)); +BUILD_ASSERT_DECL(offsetof(struct offload_actions, size) == + offsetof(struct dp_netdev_actions, size)); +BUILD_ASSERT_DECL(offsetof(struct offload_actions, actions) == + offsetof(struct dp_netdev_actions, actions)); + static int get_port_by_number(struct dp_netdev *dp, odp_port_t port_no, struct dp_netdev_port **portp) OVS_REQ_RDLOCK(dp->port_rwlock); @@ -7059,6 +7071,7 @@ struct packet_batch_per_flow { unsigned int byte_count; uint16_t tcp_flags; struct dp_netdev_flow *flow; + struct dp_netdev_actions *alt_actions; /* NULL = use flow's own actions. */ struct dp_packet_batch array; }; @@ -7075,11 +7088,15 @@ packet_batch_per_flow_update(struct packet_batch_per_flow *batch, static inline void packet_batch_per_flow_init(struct packet_batch_per_flow *batch, - struct dp_netdev_flow *flow) + struct dp_netdev_flow *flow, + struct dp_netdev_actions *alt_actions) { - flow->batch = batch; + if (!alt_actions) { + flow->batch = batch; + } batch->flow = flow; + batch->alt_actions = alt_actions; dp_packet_batch_init(&batch->array); batch->byte_count = 0; batch->tcp_flags = 0; @@ -7096,37 +7113,67 @@ packet_batch_per_flow_execute(struct packet_batch_per_flow *batch, batch->byte_count, batch->tcp_flags, pmd->ctx.now / 1000); - actions = dp_netdev_flow_get_actions(flow); + actions = batch->alt_actions ? batch->alt_actions + : dp_netdev_flow_get_actions(flow); dp_netdev_execute_actions(pmd, &batch->array, true, &flow->flow, actions->actions, actions->size); } -static inline void -dp_netdev_queue_batches(struct dp_packet *pkt, - struct dp_netdev_flow *flow, uint16_t tcp_flags, - struct packet_batch_per_flow *batches, - size_t *n_batches) +static struct packet_batch_per_flow * +dp_netdev_get_batch(struct dp_netdev_flow *flow, + struct dp_netdev_actions *alt_actions, + struct packet_batch_per_flow *batches, + size_t *n_batches) { - struct packet_batch_per_flow *batch = flow->batch; + struct packet_batch_per_flow *batch = NULL; + + if (!alt_actions) { + batch = flow->batch; + } else { + /* Scan for an existing batch with matching (flow, alt_actions). */ + for (size_t i = 0; i < *n_batches; i++) { + if (batches[i].flow == flow && + batches[i].alt_actions == alt_actions) { + batch = &batches[i]; + break; + } + } + } if (OVS_UNLIKELY(!batch)) { batch = &batches[(*n_batches)++]; - packet_batch_per_flow_init(batch, flow); + packet_batch_per_flow_init(batch, flow, alt_actions); } + return batch; +} + +static inline void +dp_netdev_queue_batches(struct dp_packet *pkt, + struct dp_netdev_flow *flow, + struct dp_netdev_actions *alt_actions, + uint16_t tcp_flags, + struct packet_batch_per_flow *batches, + size_t *n_batches) +{ + struct packet_batch_per_flow *batch; + + batch = dp_netdev_get_batch(flow, alt_actions, batches, n_batches); packet_batch_per_flow_update(batch, pkt, tcp_flags); } static inline void packet_enqueue_to_flow_map(struct dp_packet *packet, struct dp_netdev_flow *flow, + struct dp_netdev_actions *alt_actions, uint16_t tcp_flags, struct dp_packet_flow_map *flow_map, size_t index) { struct dp_packet_flow_map *map = &flow_map[index]; map->flow = flow; + map->alt_actions = alt_actions; map->packet = packet; map->tcp_flags = tcp_flags; } @@ -7180,7 +7227,7 @@ smc_lookup_batch(struct dp_netdev_pmd_thread *pmd, /* Add these packets into the flow map in the same order * as received. */ - packet_enqueue_to_flow_map(packet, flow, tcp_flags, + packet_enqueue_to_flow_map(packet, flow, NULL, tcp_flags, flow_map, recv_idx); n_smc_hit++; hit = true; @@ -7233,8 +7280,10 @@ smc_lookup_single(struct dp_netdev_pmd_thread *pmd, static inline int dp_netdev_hw_flow(const struct dp_netdev_pmd_thread *pmd, struct dp_packet *packet, + struct dp_netdev_actions **alt_actions, struct dp_netdev_flow **flow) { + struct offload_actions *offload_actions = NULL; struct dp_netdev_rxq *rxq = pmd->ctx.last_rxq; bool post_process_api_supported; void *flow_reference = NULL; @@ -7245,11 +7294,13 @@ dp_netdev_hw_flow(const struct dp_netdev_pmd_thread *pmd, if (!post_process_api_supported) { *flow = NULL; + *alt_actions = NULL; return 0; } err = dpif_offload_netdev_hw_post_process(rxq->port->netdev, pmd->core_id, - packet, &flow_reference); + packet, &offload_actions, + &flow_reference); if (err && err != EOPNOTSUPP) { if (err != ECANCELED) { COVERAGE_INC(datapath_drop_hw_post_process); @@ -7260,6 +7311,7 @@ dp_netdev_hw_flow(const struct dp_netdev_pmd_thread *pmd, } *flow = flow_reference; + *alt_actions = ALIGNED_CAST(struct dp_netdev_actions *, offload_actions); return 0; } @@ -7268,6 +7320,7 @@ dp_netdev_hw_flow(const struct dp_netdev_pmd_thread *pmd, static inline void dfc_processing_enqueue_classified_packet(struct dp_packet *packet, struct dp_netdev_flow *flow, + struct dp_netdev_actions *alt_actions, uint16_t tcp_flags, bool batch_enable, struct packet_batch_per_flow *batches, @@ -7277,17 +7330,16 @@ dfc_processing_enqueue_classified_packet(struct dp_packet *packet, { if (OVS_LIKELY(batch_enable)) { - dp_netdev_queue_batches(packet, flow, tcp_flags, batches, + dp_netdev_queue_batches(packet, flow, alt_actions, tcp_flags, batches, n_batches); } else { /* Flow batching should be performed only after fast-path * processing is also completed for packets with emc miss * or else it will result in reordering of packets with * same datapath flows. */ - packet_enqueue_to_flow_map(packet, flow, tcp_flags, + packet_enqueue_to_flow_map(packet, flow, alt_actions, tcp_flags, flow_map, (*map_cnt)++); } - } /* Try to process all ('cnt') the 'packets' using only the datapath flow cache @@ -7315,10 +7367,11 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd, size_t *n_flows, uint8_t *index_map, bool md_is_valid, odp_port_t port_no) { - size_t n_missed = 0, n_emc_hit = 0, n_phwol_hit = 0, n_simple_hit = 0; + size_t n_missed = 0, n_emc_hit = 0, n_simple_hit = 0; const bool offload_enabled = dpif_offload_enabled(); const uint32_t recirc_depth = *recirc_depth_get(); const size_t cnt = dp_packet_batch_size(packets_); + size_t n_phwol_hit = 0, n_phwol_act_hit = 0; struct dfc_cache *cache = &pmd->flow_cache; struct netdev_flow_key *key = &keys[0]; struct dp_packet *packet; @@ -7338,6 +7391,7 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd, cnt); int i; DP_PACKET_BATCH_REFILL_FOR_EACH (i, cnt, packet, packets_) { + struct dp_netdev_actions *alt_actions = NULL; struct dp_netdev_flow *flow = NULL; uint16_t tcp_flags; @@ -7359,17 +7413,22 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd, } if (offload_enabled && recirc_depth == 0) { - if (OVS_UNLIKELY(dp_netdev_hw_flow(pmd, packet, &flow))) { + if (OVS_UNLIKELY(dp_netdev_hw_flow(pmd, packet, &alt_actions, + &flow))) { /* Packet restoration failed and it was dropped, do not - * continue processing. - */ + * continue processing. */ continue; } if (OVS_LIKELY(flow)) { + if (alt_actions) { + n_phwol_act_hit++; + } else { + n_phwol_hit++; + } + tcp_flags = parse_tcp_flags(packet, NULL, NULL, NULL); - n_phwol_hit++; dfc_processing_enqueue_classified_packet( - packet, flow, tcp_flags, batch_enable, + packet, flow, alt_actions, tcp_flags, batch_enable, batches, n_batches, flow_map, &map_cnt); continue; } @@ -7385,7 +7444,7 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd, if (OVS_LIKELY(flow)) { n_simple_hit++; dfc_processing_enqueue_classified_packet( - packet, flow, tcp_flags, batch_enable, + packet, flow, NULL, tcp_flags, batch_enable, batches, n_batches, flow_map, &map_cnt); continue; } @@ -7404,7 +7463,7 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd, tcp_flags = miniflow_get_tcp_flags(&key->mf); n_emc_hit++; dfc_processing_enqueue_classified_packet( - packet, flow, tcp_flags, batch_enable, + packet, flow, NULL, tcp_flags, batch_enable, batches, n_batches, flow_map, &map_cnt); } else { /* Exact match cache missed. Group missed packets together at @@ -7431,6 +7490,8 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd, *n_flows = map_cnt; pmd_perf_update_counter(&pmd->perf_stats, PMD_STAT_PHWOL_HIT, n_phwol_hit); + pmd_perf_update_counter(&pmd->perf_stats, PMD_STAT_PHWOL_ACT_HIT, + n_phwol_act_hit); pmd_perf_update_counter(&pmd->perf_stats, PMD_STAT_SIMPLE_HIT, n_simple_hit); pmd_perf_update_counter(&pmd->perf_stats, PMD_STAT_EXACT_HIT, n_emc_hit); @@ -7624,7 +7685,7 @@ fast_path_processing(struct dp_netdev_pmd_thread *pmd, * as received. */ tcp_flags = miniflow_get_tcp_flags(&keys[i]->mf); - packet_enqueue_to_flow_map(packet, flow, tcp_flags, + packet_enqueue_to_flow_map(packet, flow, NULL, tcp_flags, flow_map, recv_idx); } @@ -7676,8 +7737,8 @@ dp_netdev_input__(struct dp_netdev_pmd_thread *pmd, if (OVS_UNLIKELY(!map->flow)) { continue; } - dp_netdev_queue_batches(map->packet, map->flow, map->tcp_flags, - batches, &n_batches); + dp_netdev_queue_batches(map->packet, map->flow, map->alt_actions, + map->tcp_flags, batches, &n_batches); } /* All the flow batches need to be reset before any call to diff --git a/lib/dpif-offload-dpdk.c b/lib/dpif-offload-dpdk.c index 293b3b8e9..0ea16ec8b 100644 --- a/lib/dpif-offload-dpdk.c +++ b/lib/dpif-offload-dpdk.c @@ -832,10 +832,13 @@ dpdk_flow_count_by_thread(struct dpdk_offload *offload, unsigned int tid) static int dpdk_offload_hw_post_process(const struct dpif_offload *offload_, struct netdev *netdev, unsigned pmd_id, - struct dp_packet *packet, void **flow_reference) + struct dp_packet *packet, + struct offload_actions **alt_actions, + void **flow_reference) { struct dpdk_offload *offload = dpdk_offload_cast(offload_); + *alt_actions = NULL; return dpdk_netdev_hw_miss_packet_recover(offload, netdev, pmd_id, packet, flow_reference); } diff --git a/lib/dpif-offload-dummy.c b/lib/dpif-offload-dummy.c index 878276a94..1c70c0bae 100644 --- a/lib/dpif-offload-dummy.c +++ b/lib/dpif-offload-dummy.c @@ -580,13 +580,16 @@ dummy_offload_get_port_by_odp_port(const struct dpif_offload *offload_, static int dummy_offload_hw_post_process(const struct dpif_offload *offload_, struct netdev *netdev, unsigned pmd_id, - struct dp_packet *packet, void **flow_reference_) + struct dp_packet *packet, + struct offload_actions **alt_actions, + void **flow_reference_) { struct dummy_offloaded_flow *off_flow; struct dummy_offload_port *port; void *flow_reference = NULL; uint32_t flow_mark; + *alt_actions = NULL; port = dummy_offload_get_port_by_netdev(offload_, netdev); if (!port || !dp_packet_has_flow_mark(packet, &flow_mark)) { *flow_reference_ = NULL; @@ -607,7 +610,7 @@ dummy_offload_hw_post_process(const struct dpif_offload *offload_, } ovs_mutex_unlock(&port->port_mutex); - *flow_reference_ = flow_reference; + *flow_reference_ = flow_reference; return 0; } diff --git a/lib/dpif-offload-provider.h b/lib/dpif-offload-provider.h index 444b13138..aed03b094 100644 --- a/lib/dpif-offload-provider.h +++ b/lib/dpif-offload-provider.h @@ -269,10 +269,13 @@ struct dpif_offload_class { * * When zero (0) is returned, the 'flow_reference' pointer may reference * the flow_reference passed to the matching flow. This can be used to - * support partial offloads. The returned pointer must remain valid until - * the end of the next RCU grace period. */ + * support partial offloads. The 'alt_actions' pointer may also be set to + * override the flow's actions, for example because a subset of the actions + * was already executed in hardware. Any pointers set must remain valid + * until the end of the next RCU grace period. */ int (*netdev_hw_post_process)(const struct dpif_offload *, struct netdev *, unsigned pmd_id, struct dp_packet *, + struct offload_actions **alt_actions, void **flow_reference); /* Allows the offload provider to override the default UDP tunnel source diff --git a/lib/dpif-offload.c b/lib/dpif-offload.c index 04dabc42c..b381eeaa6 100644 --- a/lib/dpif-offload.c +++ b/lib/dpif-offload.c @@ -1438,6 +1438,7 @@ dpif_offload_datapath_flow_stats(const char *dpif_name, odp_port_t in_port, int dpif_offload_netdev_hw_post_process(struct netdev *netdev, unsigned pmd_id, struct dp_packet *packet, + struct offload_actions **alt_actions, void **flow_reference) { const struct dpif_offload *offload; @@ -1463,7 +1464,8 @@ dpif_offload_netdev_hw_post_process(struct netdev *netdev, unsigned pmd_id, } rc = offload->class->netdev_hw_post_process(offload, netdev, pmd_id, - packet, flow_reference); + packet, alt_actions, + flow_reference); if (rc == EOPNOTSUPP) { /* API unsupported by the port; avoid subsequent calls. */ atomic_store_relaxed(&netdev->hw_info.post_process_api_supported, diff --git a/lib/dpif-offload.h b/lib/dpif-offload.h index bf7643320..d48fcb1f0 100644 --- a/lib/dpif-offload.h +++ b/lib/dpif-offload.h @@ -40,6 +40,12 @@ enum dpif_offload_impl_type { DPIF_OFFLOAD_IMPL_FLOWS_PROVIDER_ONLY, }; +/* Immutable structure that holds a set of actions. */ +struct offload_actions { + unsigned int size; /* Size of 'actions', in bytes. */ + struct nlattr actions[]; /* Sequence of OVS_ACTION_ATTR_* attributes. */ +}; + /* Global functions. */ void dpif_offload_set_global_cfg(const struct ovsrec_open_vswitch *); @@ -112,6 +118,7 @@ bool dpif_offload_netdev_same_offload(const struct netdev *, const struct netdev *); int dpif_offload_netdev_hw_post_process(struct netdev *, unsigned pmd_id, struct dp_packet *, + struct offload_actions **alt_actions, void **flow_reference); bool dpif_offload_netdev_udp_tnl_get_src_port(const struct netdev *, struct dp_packet *, diff --git a/tests/pmd.at b/tests/pmd.at index 66fccc1d5..d6e79bde8 100644 --- a/tests/pmd.at +++ b/tests/pmd.at @@ -469,11 +469,12 @@ recirc_id(0),in_port(1),packet_type(ns=0,id=0),eth(src=50:54:00:00:00:77,dst=50: AT_CHECK([ovs-appctl dpif-netdev/pmd-perf-show | sed SED_NUMA_CORE_PATTERN \ | sed '/cycles/d' | sed '/[[Ii]]teration/d' \ | sed 's/, .* us/, <cleared> us/' \ - | sed 's/[[0-9]]* Kpps/<cleared> Kpps/' | grep pmd -A 11], [0], [dnl + | sed 's/[[0-9]]* Kpps/<cleared> Kpps/' | grep pmd -A 12], [0], [dnl pmd thread numa_id <cleared> core_id <cleared>: Datapath passes: 20 (1.00 passes/pkt) - PHWOL hits: 0 ( 0.0 %) + - PHWOL + ACT hits: 0 ( 0.0 %) - Simple Match hits: 0 ( 0.0 %) - EMC hits: 19 ( 95.0 %) - SMC hits: 0 ( 0.0 %) -- 2.56.0 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
