On 27.06.2019 14:12, Kevin Traynor wrote:
> vhost tx retries can provide some mitigation against
> dropped packets due to a temporarily slow guest/limited queue
> size for an interface, but on the other hand when a system
> is fully loaded those extra cycles retrying could mean
> packets are dropped elsewhere.
>
> Up to now max vhost tx retries have been hardcoded, which meant
> no tuning and no way to disable for debugging to see if extra
> cycles spent retrying resulted in rx drops on some other
> interface.
>
> Add an option to change the max retries, with a value of
> 0 effectively disabling vhost tx retries.
>
> Signed-off-by: Kevin Traynor <[email protected]>
> ---
> Documentation/topics/dpdk/vhost-user.rst | 25 ++++++++++++++++
> lib/netdev-dpdk.c | 36 +++++++++++++++++++++---
> vswitchd/vswitch.xml | 10 +++++++
> 3 files changed, 67 insertions(+), 4 deletions(-)
>
> diff --git a/Documentation/topics/dpdk/vhost-user.rst
> b/Documentation/topics/dpdk/vhost-user.rst
> index 3caa88231..d8508d8f8 100644
> --- a/Documentation/topics/dpdk/vhost-user.rst
> +++ b/Documentation/topics/dpdk/vhost-user.rst
> @@ -392,4 +392,29 @@ The default value is ``false``.
> .. _dpdk-testpmd:
>
> +vhost-user-client tx retries config
> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> +
> +For vhost-user-client interfaces, the max amount of retries can be changed
> from
> +the default 8 by setting ``vhost-tx-retries``.
> +
> +The minimum is 0 which means there will be no retries and if any packets in
> +each batch cannot be sent immediately they will be dropped. The maximum is
> 32,
> +which would mean that after the first packet(s) in the batch was sent there
> +could be a maximum of 32 more retries.
> +
> +Retries can help with avoiding packet loss when temporarily unable to send
> to a
> +vhost interface because the virtqueue is full. However, spending more time
> +retrying to send to one interface, will reduce the time available for rx/tx
> and
> +processing packets on other interfaces, so some tuning may be required for
> best
> +performance.
> +
> +Tx retries can be set for vhost-user-client ports::
> +
> + $ ovs-vsctl set Interface vhost-client-1 options:vhost-tx-retries=0
> +
> +.. note::
> +
> + Configurable vhost tx retries are not supported with vhost-user ports.
Please, use at least 2 spaces for 'note' section indentation.
> +
> DPDK in the Guest
> -----------------
> diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
> index 65161deaf..2befbf9a7 100644
> --- a/lib/netdev-dpdk.c
> +++ b/lib/netdev-dpdk.c
> @@ -159,5 +159,10 @@ typedef uint16_t dpdk_port_t;
> #define DPDK_PORT_ID_FMT "%"PRIu16
>
> -#define VHOST_ENQ_RETRY_NUM 8
> +/* Minimum amount of vhost tx retries, effectively a disable. */
> +#define VHOST_ENQ_RETRY_MIN 0
> +/* Maximum amount of vhost tx retries. */
> +#define VHOST_ENQ_RETRY_MAX 32
> +/* Legacy default value for vhost tx retries. */
> +#define VHOST_ENQ_RETRY_DEF 8
> #define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ)
>
> @@ -409,5 +414,6 @@ struct netdev_dpdk {
> /* True if vHost device is 'up' and has been reconfigured at least
> once */
> bool vhost_reconfigured;
> - /* 3 pad bytes here. */
> + atomic_uint8_t vhost_tx_retries;
> + /* 2 pad bytes here. */
> );
>
> @@ -1240,4 +1246,6 @@ vhost_common_construct(struct netdev *netdev)
> }
>
> + atomic_store_relaxed(&dev->vhost_tx_retries, VHOST_ENQ_RETRY_DEF);
> +
> return common_construct(netdev, DPDK_ETH_PORT_ID_INVALID,
> DPDK_DEV_VHOST, socket_id);
> @@ -1899,4 +1907,5 @@ netdev_dpdk_vhost_client_set_config(struct netdev
> *netdev,
> struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
> const char *path;
> + int max_tx_retries, cur_max_tx_retries;
>
> ovs_mutex_lock(&dev->mutex);
> @@ -1915,4 +1924,17 @@ netdev_dpdk_vhost_client_set_config(struct netdev
> *netdev,
> }
> }
> +
> + max_tx_retries = smap_get_int(args, "vhost-tx-retries",
> + VHOST_ENQ_RETRY_DEF);
> + if (max_tx_retries < VHOST_ENQ_RETRY_MIN
> + || max_tx_retries > VHOST_ENQ_RETRY_MAX) {
> + max_tx_retries = VHOST_ENQ_RETRY_DEF;
> + }
> + atomic_read_relaxed(&dev->vhost_tx_retries, &cur_max_tx_retries);
> + if (max_tx_retries != cur_max_tx_retries) {
> + atomic_store_relaxed(&dev->vhost_tx_retries, max_tx_retries);
> + VLOG_INFO("Max Tx retries for vhost device '%s' set to %d",
> + dev->up.name, max_tx_retries);
> + }
> ovs_mutex_unlock(&dev->mutex);
>
> @@ -2322,4 +2344,5 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
> unsigned int dropped = 0;
> int i, retries = 0;
> + int max_retries = VHOST_ENQ_RETRY_MIN;
> int vid = netdev_dpdk_get_vid(dev);
>
> @@ -2351,9 +2374,13 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, int
> qid,
> /* Prepare for possible retry.*/
> cur_pkts = &cur_pkts[tx_pkts];
> + if (OVS_UNLIKELY(cnt && !retries)) {
> + /* Read max_retries before there may be some. */
> + atomic_read_relaxed(&dev->vhost_tx_retries, &max_retries);
> + }
> } else {
> /* No packets sent - do not retry.*/
> break;
> }
> - } while (cnt && (retries++ < VHOST_ENQ_RETRY_NUM));
> + } while (cnt && (retries++ < max_retries));
>
> rte_spinlock_unlock(&dev->tx_q[qid].tx_lock);
> @@ -2361,5 +2388,6 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
> rte_spinlock_lock(&dev->stats_lock);
> netdev_dpdk_vhost_update_tx_counters(&dev->stats, pkts, total_pkts,
> - cnt + dropped, retries);
> + cnt + dropped,
> + max_retries ? retries : 0);
> rte_spinlock_unlock(&dev->stats_lock);
>
> diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml
> index 69fce7ffb..8353607ba 100644
> --- a/vswitchd/vswitch.xml
> +++ b/vswitchd/vswitch.xml
> @@ -3120,4 +3120,14 @@ ovs-vsctl add-port br0 p0 -- set Interface p0
> type=patch options:peer=p1 \
> </column>
>
> + <column name="options" key="vhost-tx-retries"
> + type='{"type": "integer", "minInteger": 0, "maxInteger": 32}'>
> + <p>
> + The value specifies the maximum amount of tx vhost retries that can
> + be made while trying to send a batch of packets to a
> dpdkvhostclient
s/dpdkvhostclient/dpdkvhostuserclient/ or /vhost-user/
> + interface.
> + Only supported by dpdkvhostuserclient interfaces.
Default value should be specified here.
> + </p>
> + </column>
> +
> <column name="options" key="n_rxq_desc"
> type='{"type": "integer", "minInteger": 1, "maxInteger":
> 4096}'>
>
_______________________________________________
dev mailing list
[email protected]
https://mail.openvswitch.org/mailman/listinfo/ovs-dev
