On Wed, Jul 7, 2021, at 17:05, Eli Britstein wrote:
> Port numbers are usually small. Maintain an array of netdev handles indexed
> by port numbers. It accelerates looking up for them for
> netdev_hw_miss_packet_recover().
>
> Reported-by: Cian Ferriter <cian.ferri...@intel.com>
> Signed-off-by: Eli Britstein <el...@nvidia.com>
> Reviewed-by: Gaetan Rivet <gaet...@nvidia.com>
> ---
> lib/dpif-netdev.c | 41 +++++++++++++++++++++++++++++++++++++----
> 1 file changed, 37 insertions(+), 4 deletions(-)
>
> diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
> index 2e654426e..accb23a1a 100644
> --- a/lib/dpif-netdev.c
> +++ b/lib/dpif-netdev.c
> @@ -650,6 +650,9 @@ struct dp_netdev_pmd_thread_ctx {
> uint32_t emc_insert_min;
> };
>
> +/* Size of netdev's cache. */
> +#define DP_PMD_NETDEV_CACHE_SIZE 1024
> +
> /* PMD: Poll modes drivers. PMD accesses devices via polling to eliminate
> * the performance overhead of interrupt processing. Therefore netdev can
> * not implement rx-wait for these devices. dpif-netdev needs to poll
> @@ -786,6 +789,7 @@ struct dp_netdev_pmd_thread {
> * other instance will only be accessed by its own pmd thread. */
> struct hmap tnl_port_cache;
> struct hmap send_port_cache;
> + struct netdev *send_netdev_cache[DP_PMD_NETDEV_CACHE_SIZE];
>
> /* Keep track of detailed PMD performance statistics. */
> struct pmd_perf_stats perf_stats;
> @@ -5910,6 +5914,10 @@ pmd_free_cached_ports(struct
> dp_netdev_pmd_thread *pmd)
> free(tx_port_cached);
> }
> HMAP_FOR_EACH_POP (tx_port_cached, node, &pmd->send_port_cache) {
> + if (tx_port_cached->port->port_no <
> + ARRAY_SIZE(pmd->send_netdev_cache)) {
> + pmd->send_netdev_cache[tx_port_cached->port->port_no] =
> NULL;
> + }
> free(tx_port_cached);
> }
> }
> @@ -5939,6 +5947,11 @@ pmd_load_cached_ports(struct
> dp_netdev_pmd_thread *pmd)
> tx_port_cached = xmemdup(tx_port, sizeof *tx_port_cached);
> hmap_insert(&pmd->send_port_cache, &tx_port_cached->node,
> hash_port_no(tx_port_cached->port->port_no));
> + if (tx_port_cached->port->port_no <
> + ARRAY_SIZE(pmd->send_netdev_cache)) {
> + pmd->send_netdev_cache[tx_port_cached->port->port_no] =
> + tx_port_cached->port->netdev;
> + }
> }
> }
> }
> @@ -6585,6 +6598,7 @@ dp_netdev_configure_pmd(struct
> dp_netdev_pmd_thread *pmd, struct dp_netdev *dp,
> hmap_init(&pmd->tx_ports);
> hmap_init(&pmd->tnl_port_cache);
> hmap_init(&pmd->send_port_cache);
> + memset(pmd->send_netdev_cache, 0, sizeof pmd->send_netdev_cache);
> cmap_init(&pmd->tx_bonds);
> /* init the 'flow_cache' since there is no
> * actual thread created for NON_PMD_CORE_ID. */
> @@ -6603,6 +6617,7 @@ dp_netdev_destroy_pmd(struct dp_netdev_pmd_thread
> *pmd)
> struct dpcls *cls;
>
> dp_netdev_pmd_flow_flush(pmd);
> + memset(pmd->send_netdev_cache, 0, sizeof pmd->send_netdev_cache);
> hmap_destroy(&pmd->send_port_cache);
> hmap_destroy(&pmd->tnl_port_cache);
> hmap_destroy(&pmd->tx_ports);
> @@ -7090,20 +7105,38 @@ smc_lookup_batch(struct dp_netdev_pmd_thread *pmd,
> static struct tx_port * pmd_send_port_cache_lookup(
> const struct dp_netdev_pmd_thread *pmd, odp_port_t port_no);
>
> +OVS_UNUSED
> +static inline struct netdev *
> +pmd_netdev_cache_lookup(const struct dp_netdev_pmd_thread *pmd,
> + odp_port_t port_no)
> +{
> + struct tx_port *p;
> +
> + if (port_no < ARRAY_SIZE(pmd->send_netdev_cache)) {
> + return pmd->send_netdev_cache[port_no];
> + }
> +
> + p = pmd_send_port_cache_lookup(pmd, port_no);
> + if (p) {
> + return p->port->netdev;
> + }
> + return NULL;
> +}
> +
> static inline int
> dp_netdev_hw_flow(const struct dp_netdev_pmd_thread *pmd,
> odp_port_t port_no OVS_UNUSED,
> struct dp_packet *packet,
> struct dp_netdev_flow **flow)
> {
> - struct tx_port *p OVS_UNUSED;
> + struct netdev *netdev OVS_UNUSED;
> uint32_t mark;
>
> #ifdef ALLOW_EXPERIMENTAL_API /* Packet restoration API required. */
> /* Restore the packet if HW processing was terminated before completion.
> */
> - p = pmd_send_port_cache_lookup(pmd, port_no);
> - if (OVS_LIKELY(p)) {
> - int err = netdev_hw_miss_packet_recover(p->port->netdev, packet);
> + netdev = pmd_netdev_cache_lookup(pmd, port_no);
> + if (OVS_LIKELY(netdev)) {
> + int err = netdev_hw_miss_packet_recover(netdev, packet);
>
> if (err && err != EOPNOTSUPP) {
> COVERAGE_INC(datapath_drop_hw_miss_recover);FI-86194-0059
> --
> 2.28.0.2311.g225365fb51
>
> _______________________________________________
> dev mailing list
> d...@openvswitch.org
> https://mail.openvswitch.org/mailman/listinfo/ovs-dev
>
Hello,
I tested the performance impact of this patch with a partial offload setup.
As reported by pmd-stats-show, in average cycles per packet:
Before vxlan-decap: 525 c/p
After vxlan-decap: 542 c/p
After this fix: 530 c/p
Without those fixes, vxlan-decap has a 3.2% negative impact on cycles,
with the fixes, the impact is reduced to 0.95%.
As I had to force partial offloads for our hardware, it would be better
with an outside confirmation on a proper setup.
Kind regards,
--
Gaetan Rivet
_______________________________________________
dev mailing list
d...@openvswitch.org
https://mail.openvswitch.org/mailman/listinfo/ovs-dev
