Allow the netdev_hw_post_process() callback to return an optional set of
alternative actions via a new 'alt_actions' parameter.  When set, these
actions are executed instead of the flow's own actions, enabling offload
providers to indicate that a subset of the actions was already performed
in hardware.

Note that batching for packets with alternative actions is not optimal,
as it requires a linear scan over the current batch list to find a
matching (flow, alt_actions) pair.

Acked-by: Eli Britstein <[email protected]>
Signed-off-by: Eelco Chaudron <[email protected]>
---
 lib/dpif-netdev-perf.c      |   3 +
 lib/dpif-netdev-perf.h      |   2 +
 lib/dpif-netdev.c           | 113 +++++++++++++++++++++++++++---------
 lib/dpif-offload-dpdk.c     |   5 +-
 lib/dpif-offload-dummy.c    |   7 ++-
 lib/dpif-offload-provider.h |   7 ++-
 lib/dpif-offload.c          |   4 +-
 lib/dpif-offload.h          |   7 +++
 tests/pmd.at                |   3 +-
 9 files changed, 118 insertions(+), 33 deletions(-)

diff --git a/lib/dpif-netdev-perf.c b/lib/dpif-netdev-perf.c
index 4b3b35852..7bd288216 100644
--- a/lib/dpif-netdev-perf.c
+++ b/lib/dpif-netdev-perf.c
@@ -267,6 +267,7 @@ pmd_perf_format_overall_stats(struct ds *str, struct 
pmd_perf_stats *s,
         ds_put_format(str,
             "  Datapath passes:    %12"PRIu64"  (%.2f passes/pkt)\n"
             "  - PHWOL hits:       %12"PRIu64"  (%5.1f %%)\n"
+            "  - PHWOL + ACT hits: %12"PRIu64"  (%5.1f %%)\n"
             "  - Simple Match hits:%12"PRIu64"  (%5.1f %%)\n"
             "  - EMC hits:         %12"PRIu64"  (%5.1f %%)\n"
             "  - SMC hits:         %12"PRIu64"  (%5.1f %%)\n"
@@ -275,6 +276,8 @@ pmd_perf_format_overall_stats(struct ds *str, struct 
pmd_perf_stats *s,
             passes, 1.0 * passes / rx_packets,
             stats[PMD_STAT_PHWOL_HIT],
             100.0 * stats[PMD_STAT_PHWOL_HIT] / passes,
+            stats[PMD_STAT_PHWOL_ACT_HIT],
+            100.0 * stats[PMD_STAT_PHWOL_ACT_HIT] / passes,
             stats[PMD_STAT_SIMPLE_HIT],
             100.0 * stats[PMD_STAT_SIMPLE_HIT] / passes,
             stats[PMD_STAT_EXACT_HIT],
diff --git a/lib/dpif-netdev-perf.h b/lib/dpif-netdev-perf.h
index cf4b6e103..3d17e978f 100644
--- a/lib/dpif-netdev-perf.h
+++ b/lib/dpif-netdev-perf.h
@@ -57,6 +57,8 @@ extern "C" {
 
 enum pmd_stat_type {
     PMD_STAT_PHWOL_HIT,     /* Packets that had a partial HWOL hit (phwol). */
+    PMD_STAT_PHWOL_ACT_HIT, /* Packets that had a partial HWOL hit with
+                               alternative actions. */
     PMD_STAT_SIMPLE_HIT,    /* Packets that had a simple match hit. */
     PMD_STAT_EXACT_HIT,     /* Packets that had an exact match (emc). */
     PMD_STAT_SMC_HIT,       /* Packets that had a sig match hit (SMC). */
diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
index 4151ea056..e15561ba8 100644
--- a/lib/dpif-netdev.c
+++ b/lib/dpif-netdev.c
@@ -197,6 +197,7 @@ struct dpcls {
 struct dp_packet_flow_map {
     struct dp_packet *packet;
     struct dp_netdev_flow *flow;
+    struct dp_netdev_actions *alt_actions; /* NULL = use flow's own actions. */
     uint16_t tcp_flags;
 };
 
@@ -468,6 +469,17 @@ struct dpif_netdev {
     uint64_t last_port_seq;
 };
 
+/* Verify that struct offload_actions (defined in dpif-offload.h) is
+ * layout-identical to struct dp_netdev_actions.  The two structs are kept
+ * separate to avoid exposing dp_netdev_actions as a public API, but must
+ * remain binary-compatible so that ALIGNED_CAST() between them is safe. */
+BUILD_ASSERT_DECL(sizeof(struct offload_actions) ==
+                  sizeof(struct dp_netdev_actions));
+BUILD_ASSERT_DECL(offsetof(struct offload_actions, size) ==
+                  offsetof(struct dp_netdev_actions, size));
+BUILD_ASSERT_DECL(offsetof(struct offload_actions, actions) ==
+                  offsetof(struct dp_netdev_actions, actions));
+
 static int get_port_by_number(struct dp_netdev *dp, odp_port_t port_no,
                               struct dp_netdev_port **portp)
     OVS_REQ_RDLOCK(dp->port_rwlock);
@@ -7059,6 +7071,7 @@ struct packet_batch_per_flow {
     unsigned int byte_count;
     uint16_t tcp_flags;
     struct dp_netdev_flow *flow;
+    struct dp_netdev_actions *alt_actions; /* NULL = use flow's own actions. */
 
     struct dp_packet_batch array;
 };
@@ -7075,11 +7088,15 @@ packet_batch_per_flow_update(struct 
packet_batch_per_flow *batch,
 
 static inline void
 packet_batch_per_flow_init(struct packet_batch_per_flow *batch,
-                           struct dp_netdev_flow *flow)
+                           struct dp_netdev_flow *flow,
+                           struct dp_netdev_actions *alt_actions)
 {
-    flow->batch = batch;
+    if (!alt_actions) {
+        flow->batch = batch;
+    }
 
     batch->flow = flow;
+    batch->alt_actions = alt_actions;
     dp_packet_batch_init(&batch->array);
     batch->byte_count = 0;
     batch->tcp_flags = 0;
@@ -7096,37 +7113,67 @@ packet_batch_per_flow_execute(struct 
packet_batch_per_flow *batch,
                         batch->byte_count,
                         batch->tcp_flags, pmd->ctx.now / 1000);
 
-    actions = dp_netdev_flow_get_actions(flow);
+    actions = batch->alt_actions ? batch->alt_actions
+                                 : dp_netdev_flow_get_actions(flow);
 
     dp_netdev_execute_actions(pmd, &batch->array, true, &flow->flow,
                               actions->actions, actions->size);
 }
 
-static inline void
-dp_netdev_queue_batches(struct dp_packet *pkt,
-                        struct dp_netdev_flow *flow, uint16_t tcp_flags,
-                        struct packet_batch_per_flow *batches,
-                        size_t *n_batches)
+static struct packet_batch_per_flow *
+dp_netdev_get_batch(struct dp_netdev_flow *flow,
+                    struct dp_netdev_actions *alt_actions,
+                    struct packet_batch_per_flow *batches,
+                    size_t *n_batches)
 {
-    struct packet_batch_per_flow *batch = flow->batch;
+    struct packet_batch_per_flow *batch = NULL;
+
+    if (!alt_actions) {
+        batch = flow->batch;
+    } else {
+        /* Scan for an existing batch with matching (flow, alt_actions). */
+        for (size_t i = 0; i < *n_batches; i++) {
+            if (batches[i].flow == flow &&
+                batches[i].alt_actions == alt_actions) {
+                batch = &batches[i];
+                break;
+            }
+        }
+    }
 
     if (OVS_UNLIKELY(!batch)) {
         batch = &batches[(*n_batches)++];
-        packet_batch_per_flow_init(batch, flow);
+        packet_batch_per_flow_init(batch, flow, alt_actions);
     }
 
+    return batch;
+}
+
+static inline void
+dp_netdev_queue_batches(struct dp_packet *pkt,
+                        struct dp_netdev_flow *flow,
+                        struct dp_netdev_actions *alt_actions,
+                        uint16_t tcp_flags,
+                        struct packet_batch_per_flow *batches,
+                        size_t *n_batches)
+{
+    struct packet_batch_per_flow *batch;
+
+    batch = dp_netdev_get_batch(flow, alt_actions, batches, n_batches);
     packet_batch_per_flow_update(batch, pkt, tcp_flags);
 }
 
 static inline void
 packet_enqueue_to_flow_map(struct dp_packet *packet,
                            struct dp_netdev_flow *flow,
+                           struct dp_netdev_actions *alt_actions,
                            uint16_t tcp_flags,
                            struct dp_packet_flow_map *flow_map,
                            size_t index)
 {
     struct dp_packet_flow_map *map = &flow_map[index];
     map->flow = flow;
+    map->alt_actions = alt_actions;
     map->packet = packet;
     map->tcp_flags = tcp_flags;
 }
@@ -7180,7 +7227,7 @@ smc_lookup_batch(struct dp_netdev_pmd_thread *pmd,
                     /* Add these packets into the flow map in the same order
                      * as received.
                      */
-                    packet_enqueue_to_flow_map(packet, flow, tcp_flags,
+                    packet_enqueue_to_flow_map(packet, flow, NULL, tcp_flags,
                                                flow_map, recv_idx);
                     n_smc_hit++;
                     hit = true;
@@ -7233,8 +7280,10 @@ smc_lookup_single(struct dp_netdev_pmd_thread *pmd,
 static inline int
 dp_netdev_hw_flow(const struct dp_netdev_pmd_thread *pmd,
                   struct dp_packet *packet,
+                  struct dp_netdev_actions **alt_actions,
                   struct dp_netdev_flow **flow)
 {
+    struct offload_actions *offload_actions = NULL;
     struct dp_netdev_rxq *rxq = pmd->ctx.last_rxq;
     bool post_process_api_supported;
     void *flow_reference = NULL;
@@ -7245,11 +7294,13 @@ dp_netdev_hw_flow(const struct dp_netdev_pmd_thread 
*pmd,
 
     if (!post_process_api_supported) {
         *flow = NULL;
+        *alt_actions = NULL;
         return 0;
     }
 
     err = dpif_offload_netdev_hw_post_process(rxq->port->netdev, pmd->core_id,
-                                              packet, &flow_reference);
+                                              packet, &offload_actions,
+                                              &flow_reference);
     if (err && err != EOPNOTSUPP) {
         if (err != ECANCELED) {
             COVERAGE_INC(datapath_drop_hw_post_process);
@@ -7260,6 +7311,7 @@ dp_netdev_hw_flow(const struct dp_netdev_pmd_thread *pmd,
     }
 
     *flow = flow_reference;
+    *alt_actions = ALIGNED_CAST(struct dp_netdev_actions *, offload_actions);
     return 0;
 }
 
@@ -7268,6 +7320,7 @@ dp_netdev_hw_flow(const struct dp_netdev_pmd_thread *pmd,
 static inline void
 dfc_processing_enqueue_classified_packet(struct dp_packet *packet,
                                          struct dp_netdev_flow *flow,
+                                         struct dp_netdev_actions *alt_actions,
                                          uint16_t tcp_flags,
                                          bool batch_enable,
                                          struct packet_batch_per_flow *batches,
@@ -7277,17 +7330,16 @@ dfc_processing_enqueue_classified_packet(struct 
dp_packet *packet,
 
 {
     if (OVS_LIKELY(batch_enable)) {
-        dp_netdev_queue_batches(packet, flow, tcp_flags, batches,
+        dp_netdev_queue_batches(packet, flow, alt_actions, tcp_flags, batches,
                                 n_batches);
     } else {
         /* Flow batching should be performed only after fast-path
          * processing is also completed for packets with emc miss
          * or else it will result in reordering of packets with
          * same datapath flows. */
-        packet_enqueue_to_flow_map(packet, flow, tcp_flags,
+        packet_enqueue_to_flow_map(packet, flow, alt_actions, tcp_flags,
                                    flow_map, (*map_cnt)++);
     }
-
 }
 
 /* Try to process all ('cnt') the 'packets' using only the datapath flow cache
@@ -7315,10 +7367,11 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd,
                size_t *n_flows, uint8_t *index_map,
                bool md_is_valid, odp_port_t port_no)
 {
-    size_t n_missed = 0, n_emc_hit = 0, n_phwol_hit = 0, n_simple_hit = 0;
+    size_t n_missed = 0, n_emc_hit = 0, n_simple_hit = 0;
     const bool offload_enabled = dpif_offload_enabled();
     const uint32_t recirc_depth = *recirc_depth_get();
     const size_t cnt = dp_packet_batch_size(packets_);
+    size_t n_phwol_hit = 0, n_phwol_act_hit = 0;
     struct dfc_cache *cache = &pmd->flow_cache;
     struct netdev_flow_key *key = &keys[0];
     struct dp_packet *packet;
@@ -7338,6 +7391,7 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd,
                             cnt);
     int i;
     DP_PACKET_BATCH_REFILL_FOR_EACH (i, cnt, packet, packets_) {
+        struct dp_netdev_actions *alt_actions = NULL;
         struct dp_netdev_flow *flow = NULL;
         uint16_t tcp_flags;
 
@@ -7359,17 +7413,22 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd,
         }
 
         if (offload_enabled && recirc_depth == 0) {
-            if (OVS_UNLIKELY(dp_netdev_hw_flow(pmd, packet, &flow))) {
+            if (OVS_UNLIKELY(dp_netdev_hw_flow(pmd, packet, &alt_actions,
+                                               &flow))) {
                 /* Packet restoration failed and it was dropped, do not
-                 * continue processing.
-                 */
+                 * continue processing. */
                 continue;
             }
             if (OVS_LIKELY(flow)) {
+                if (alt_actions) {
+                    n_phwol_act_hit++;
+                } else {
+                    n_phwol_hit++;
+                }
+
                 tcp_flags = parse_tcp_flags(packet, NULL, NULL, NULL);
-                n_phwol_hit++;
                 dfc_processing_enqueue_classified_packet(
-                        packet, flow, tcp_flags, batch_enable,
+                        packet, flow, alt_actions, tcp_flags, batch_enable,
                         batches, n_batches, flow_map, &map_cnt);
                 continue;
             }
@@ -7385,7 +7444,7 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd,
             if (OVS_LIKELY(flow)) {
                 n_simple_hit++;
                 dfc_processing_enqueue_classified_packet(
-                        packet, flow, tcp_flags, batch_enable,
+                        packet, flow, NULL, tcp_flags, batch_enable,
                         batches, n_batches, flow_map, &map_cnt);
                 continue;
             }
@@ -7404,7 +7463,7 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd,
             tcp_flags = miniflow_get_tcp_flags(&key->mf);
             n_emc_hit++;
             dfc_processing_enqueue_classified_packet(
-                    packet, flow, tcp_flags, batch_enable,
+                    packet, flow, NULL, tcp_flags, batch_enable,
                     batches, n_batches, flow_map, &map_cnt);
         } else {
             /* Exact match cache missed. Group missed packets together at
@@ -7431,6 +7490,8 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd,
     *n_flows = map_cnt;
 
     pmd_perf_update_counter(&pmd->perf_stats, PMD_STAT_PHWOL_HIT, n_phwol_hit);
+    pmd_perf_update_counter(&pmd->perf_stats, PMD_STAT_PHWOL_ACT_HIT,
+                            n_phwol_act_hit);
     pmd_perf_update_counter(&pmd->perf_stats, PMD_STAT_SIMPLE_HIT,
                             n_simple_hit);
     pmd_perf_update_counter(&pmd->perf_stats, PMD_STAT_EXACT_HIT, n_emc_hit);
@@ -7624,7 +7685,7 @@ fast_path_processing(struct dp_netdev_pmd_thread *pmd,
          * as received.
          */
         tcp_flags = miniflow_get_tcp_flags(&keys[i]->mf);
-        packet_enqueue_to_flow_map(packet, flow, tcp_flags,
+        packet_enqueue_to_flow_map(packet, flow, NULL, tcp_flags,
                                    flow_map, recv_idx);
     }
 
@@ -7676,8 +7737,8 @@ dp_netdev_input__(struct dp_netdev_pmd_thread *pmd,
         if (OVS_UNLIKELY(!map->flow)) {
             continue;
         }
-        dp_netdev_queue_batches(map->packet, map->flow, map->tcp_flags,
-                                batches, &n_batches);
+        dp_netdev_queue_batches(map->packet, map->flow, map->alt_actions,
+                                map->tcp_flags, batches, &n_batches);
      }
 
     /* All the flow batches need to be reset before any call to
diff --git a/lib/dpif-offload-dpdk.c b/lib/dpif-offload-dpdk.c
index 293b3b8e9..0ea16ec8b 100644
--- a/lib/dpif-offload-dpdk.c
+++ b/lib/dpif-offload-dpdk.c
@@ -832,10 +832,13 @@ dpdk_flow_count_by_thread(struct dpdk_offload *offload, 
unsigned int tid)
 static int
 dpdk_offload_hw_post_process(const struct dpif_offload *offload_,
                              struct netdev *netdev, unsigned pmd_id,
-                             struct dp_packet *packet, void **flow_reference)
+                             struct dp_packet *packet,
+                             struct offload_actions **alt_actions,
+                             void **flow_reference)
 {
     struct dpdk_offload *offload = dpdk_offload_cast(offload_);
 
+    *alt_actions = NULL;
     return dpdk_netdev_hw_miss_packet_recover(offload, netdev, pmd_id, packet,
                                               flow_reference);
 }
diff --git a/lib/dpif-offload-dummy.c b/lib/dpif-offload-dummy.c
index 878276a94..1c70c0bae 100644
--- a/lib/dpif-offload-dummy.c
+++ b/lib/dpif-offload-dummy.c
@@ -580,13 +580,16 @@ dummy_offload_get_port_by_odp_port(const struct 
dpif_offload *offload_,
 static int
 dummy_offload_hw_post_process(const struct dpif_offload *offload_,
                               struct netdev *netdev, unsigned pmd_id,
-                              struct dp_packet *packet, void **flow_reference_)
+                              struct dp_packet *packet,
+                              struct offload_actions **alt_actions,
+                              void **flow_reference_)
 {
     struct dummy_offloaded_flow *off_flow;
     struct dummy_offload_port *port;
     void *flow_reference = NULL;
     uint32_t flow_mark;
 
+    *alt_actions = NULL;
     port = dummy_offload_get_port_by_netdev(offload_, netdev);
     if (!port || !dp_packet_has_flow_mark(packet, &flow_mark)) {
         *flow_reference_ = NULL;
@@ -607,7 +610,7 @@ dummy_offload_hw_post_process(const struct dpif_offload 
*offload_,
     }
     ovs_mutex_unlock(&port->port_mutex);
 
-     *flow_reference_ = flow_reference;
+    *flow_reference_ = flow_reference;
     return 0;
 }
 
diff --git a/lib/dpif-offload-provider.h b/lib/dpif-offload-provider.h
index 444b13138..aed03b094 100644
--- a/lib/dpif-offload-provider.h
+++ b/lib/dpif-offload-provider.h
@@ -269,10 +269,13 @@ struct dpif_offload_class {
      *
      * When zero (0) is returned, the 'flow_reference' pointer may reference
      * the flow_reference passed to the matching flow.  This can be used to
-     * support partial offloads.  The returned pointer must remain valid until
-     * the end of the next RCU grace period. */
+     * support partial offloads.  The 'alt_actions' pointer may also be set to
+     * override the flow's actions, for example because a subset of the actions
+     * was already executed in hardware.  Any pointers set must remain valid
+     * until the end of the next RCU grace period. */
     int (*netdev_hw_post_process)(const struct dpif_offload *, struct netdev *,
                                   unsigned pmd_id, struct dp_packet *,
+                                  struct offload_actions **alt_actions,
                                   void **flow_reference);
 
     /* Allows the offload provider to override the default UDP tunnel source
diff --git a/lib/dpif-offload.c b/lib/dpif-offload.c
index 04dabc42c..b381eeaa6 100644
--- a/lib/dpif-offload.c
+++ b/lib/dpif-offload.c
@@ -1438,6 +1438,7 @@ dpif_offload_datapath_flow_stats(const char *dpif_name, 
odp_port_t in_port,
 int
 dpif_offload_netdev_hw_post_process(struct netdev *netdev, unsigned pmd_id,
                                     struct dp_packet *packet,
+                                    struct offload_actions **alt_actions,
                                     void **flow_reference)
 {
     const struct dpif_offload *offload;
@@ -1463,7 +1464,8 @@ dpif_offload_netdev_hw_post_process(struct netdev 
*netdev, unsigned pmd_id,
     }
 
     rc = offload->class->netdev_hw_post_process(offload, netdev, pmd_id,
-                                                packet, flow_reference);
+                                                packet, alt_actions,
+                                                flow_reference);
     if (rc == EOPNOTSUPP) {
         /* API unsupported by the port; avoid subsequent calls. */
         atomic_store_relaxed(&netdev->hw_info.post_process_api_supported,
diff --git a/lib/dpif-offload.h b/lib/dpif-offload.h
index bf7643320..d48fcb1f0 100644
--- a/lib/dpif-offload.h
+++ b/lib/dpif-offload.h
@@ -40,6 +40,12 @@ enum dpif_offload_impl_type {
     DPIF_OFFLOAD_IMPL_FLOWS_PROVIDER_ONLY,
 };
 
+/* Immutable structure that holds a set of actions. */
+struct offload_actions {
+    unsigned int size;       /* Size of 'actions', in bytes. */
+    struct nlattr actions[]; /* Sequence of OVS_ACTION_ATTR_* attributes. */
+};
+
 
 /* Global functions. */
 void dpif_offload_set_global_cfg(const struct ovsrec_open_vswitch *);
@@ -112,6 +118,7 @@ bool dpif_offload_netdev_same_offload(const struct netdev *,
                                       const struct netdev *);
 int dpif_offload_netdev_hw_post_process(struct netdev *, unsigned pmd_id,
                                         struct dp_packet *,
+                                        struct offload_actions **alt_actions,
                                         void **flow_reference);
 bool dpif_offload_netdev_udp_tnl_get_src_port(const struct netdev *,
                                               struct dp_packet *,
diff --git a/tests/pmd.at b/tests/pmd.at
index 66fccc1d5..d6e79bde8 100644
--- a/tests/pmd.at
+++ b/tests/pmd.at
@@ -469,11 +469,12 @@ 
recirc_id(0),in_port(1),packet_type(ns=0,id=0),eth(src=50:54:00:00:00:77,dst=50:
 AT_CHECK([ovs-appctl dpif-netdev/pmd-perf-show | sed SED_NUMA_CORE_PATTERN \
             | sed '/cycles/d' | sed '/[[Ii]]teration/d' \
             | sed 's/, .* us/, <cleared> us/' \
-            | sed 's/[[0-9]]* Kpps/<cleared> Kpps/' | grep pmd -A 11], [0], 
[dnl
+            | sed 's/[[0-9]]* Kpps/<cleared> Kpps/' | grep pmd -A 12], [0], 
[dnl
 pmd thread numa_id <cleared> core_id <cleared>:
 
   Datapath passes:              20  (1.00 passes/pkt)
   - PHWOL hits:                  0  (  0.0 %)
+  - PHWOL + ACT hits:            0  (  0.0 %)
   - Simple Match hits:           0  (  0.0 %)
   - EMC hits:                   19  ( 95.0 %)
   - SMC hits:                    0  (  0.0 %)
-- 
2.56.0

_______________________________________________
dev mailing list
[email protected]
https://mail.openvswitch.org/mailman/listinfo/ovs-dev

Reply via email to