Hi,
> -----Original Message-----
> From: Shiri Kuzin <shi...@mellanox.com>
> Sent: Tuesday, March 24, 2020 1:40 PM
> To: dev@dpdk.org
> Cc: Matan Azrad <ma...@mellanox.com>; Raslan Darawsheh
> <rasl...@mellanox.com>; Slava Ovsiienko <viachesl...@mellanox.com>
> Subject: [PATCH] net/mlx5: creating relaxed ordering memory regions
>
> In the current state, when preforming read/write
> transactions we must wait for a completion in order
> to run the next transaction, and all transactions are
> performed by order.
>
> Relaxed Ordering is a PCI optimization which by enabling it
> we allow the system to perform read/writes in a different
> order without having to wait for completion and improve
> the performance in that matter.
>
> This commit introduces the creation of relaxed ordering
> memory regions in mlx5.
> As relaxed ordering is an optimization, drivers that
> do not support it can simply ignore it and therefore
> it is enabled by default.
>
> Signed-off-by: Shiri Kuzin <shi...@mellanox.com>
> Acked-by: Matan Azrad <ma...@mellanox.com>
> ---
> doc/guides/rel_notes/release_20_05.rst | 2 +-
> drivers/common/mlx5/mlx5_devx_cmds.c | 4 ++++
> drivers/common/mlx5/mlx5_devx_cmds.h | 1 +
> drivers/common/mlx5/mlx5_glue.h | 4 ++++
> drivers/common/mlx5/mlx5_prm.h | 4 +++-
> drivers/net/mlx5/mlx5_flow_dv.c | 1 +
> drivers/net/mlx5/mlx5_mr.c | 6 ++++--
> drivers/vdpa/mlx5/mlx5_vdpa_lm.c | 1 +
> drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 1 +
> 9 files changed, 20 insertions(+), 4 deletions(-)
>
> diff --git a/doc/guides/rel_notes/release_20_05.rst
> b/doc/guides/rel_notes/release_20_05.rst
> index 000bbf5..c960fd2 100644
> --- a/doc/guides/rel_notes/release_20_05.rst
> +++ b/doc/guides/rel_notes/release_20_05.rst
> @@ -61,7 +61,7 @@ New Features
> Updated Mellanox mlx5 driver with new features and improvements,
> including:
>
> * Added support for matching on IPv4 Time To Live and IPv6 Hop Limit.
> -
> + * Added support for creating Relaxed Ordering Memory Regions.
>
> Removed Items
> -------------
> diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c
> b/drivers/common/mlx5/mlx5_devx_cmds.c
> index d960bc9..1157a44 100644
> --- a/drivers/common/mlx5/mlx5_devx_cmds.c
> +++ b/drivers/common/mlx5/mlx5_devx_cmds.c
> @@ -196,6 +196,10 @@ struct mlx5_devx_obj *
> MLX5_SET(mkc, mkc, pd, attr->pd);
> MLX5_SET(mkc, mkc, mkey_7_0, attr->umem_id & 0xFF);
> MLX5_SET(mkc, mkc, translations_octword_size, translation_size);
> + if (attr->relaxed_ordering == 1) {
> + MLX5_SET(mkc, mkc, relaxed_ordering_write, 0x1);
> + MLX5_SET(mkc, mkc, relaxed_ordering_read, 0x1);
> + }
> MLX5_SET64(mkc, mkc, start_addr, attr->addr);
> MLX5_SET64(mkc, mkc, len, attr->size);
> mkey->obj = mlx5_glue->devx_obj_create(ctx, in, in_size_dw * 4,
> out,
> diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h
> b/drivers/common/mlx5/mlx5_devx_cmds.h
> index 6912dc6..20bb294 100644
> --- a/drivers/common/mlx5/mlx5_devx_cmds.h
> +++ b/drivers/common/mlx5/mlx5_devx_cmds.h
> @@ -33,6 +33,7 @@ struct mlx5_devx_mkey_attr {
> uint32_t pd;
> uint32_t log_entity_size;
> uint32_t pg_access:1;
> + uint32_t relaxed_ordering:1;
> struct mlx5_klm *klm_array;
> int klm_num;
> };
> diff --git a/drivers/common/mlx5/mlx5_glue.h
> b/drivers/common/mlx5/mlx5_glue.h
> index 6238b43..cd1136f 100644
> --- a/drivers/common/mlx5/mlx5_glue.h
> +++ b/drivers/common/mlx5/mlx5_glue.h
> @@ -98,6 +98,10 @@
> uint64_t comp_mask; };
> #endif
>
> +#ifndef IBV_ACCESS_RELAXED_ORDERING
> +#define IBV_ACCESS_RELAXED_ORDERING 0
> +#endif
> +
> /* LIB_GLUE_VERSION must be updated every time this structure is
> modified. */
> struct mlx5_glue {
> const char *version;
> diff --git a/drivers/common/mlx5/mlx5_prm.h
> b/drivers/common/mlx5/mlx5_prm.h
> index eee3a4b..00fd7c1 100644
> --- a/drivers/common/mlx5/mlx5_prm.h
> +++ b/drivers/common/mlx5/mlx5_prm.h
> @@ -882,7 +882,9 @@ struct mlx5_ifc_mkc_bits {
>
> u8 translations_octword_size[0x20];
>
> - u8 reserved_at_1c0[0x1b];
> + u8 reserved_at_1c0[0x19];
> + u8 relaxed_ordering_read[0x1];
> + u8 reserved_at_1da[0x1];
> u8 log_page_size[0x5];
>
> u8 reserved_at_1e0[0x20];
> diff --git a/drivers/net/mlx5/mlx5_flow_dv.c
> b/drivers/net/mlx5/mlx5_flow_dv.c
> index 2090631..809833b 100644
> --- a/drivers/net/mlx5/mlx5_flow_dv.c
> +++ b/drivers/net/mlx5/mlx5_flow_dv.c
> @@ -3962,6 +3962,7 @@ struct field_modify_info modify_tcp[] = {
> mkey_attr.pg_access = 0;
> mkey_attr.klm_array = NULL;
> mkey_attr.klm_num = 0;
> + mkey_attr.relaxed_ordering = 1;
> mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx,
> &mkey_attr);
> if (!mem_mng->dm) {
> mlx5_glue->devx_umem_dereg(mem_mng->umem);
> diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
> index cb97c87..fd1245d 100644
> --- a/drivers/net/mlx5/mlx5_mr.c
> +++ b/drivers/net/mlx5/mlx5_mr.c
> @@ -768,7 +768,8 @@ struct mr_update_mp_data {
> * through mlx5_alloc_verbs_buf().
> */
> mr->ibv_mr = mlx5_glue->reg_mr(sh->pd, (void *)data.start, len,
> - IBV_ACCESS_LOCAL_WRITE);
> + IBV_ACCESS_LOCAL_WRITE |
> + IBV_ACCESS_RELAXED_ORDERING);
> if (mr->ibv_mr == NULL) {
> DEBUG("port %u fail to create a verbs MR for address (%p)",
> dev->data->port_id, (void *)addr);
> @@ -1217,7 +1218,8 @@ struct mr_update_mp_data {
> if (mr == NULL)
> return NULL;
> mr->ibv_mr = mlx5_glue->reg_mr(priv->sh->pd, (void *)addr, len,
> - IBV_ACCESS_LOCAL_WRITE);
> + IBV_ACCESS_LOCAL_WRITE |
> + IBV_ACCESS_RELAXED_ORDERING);
> if (mr->ibv_mr == NULL) {
> DRV_LOG(WARNING,
> "port %u fail to create a verbs MR for address (%p)",
> diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
> b/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
> index 3358704..4457760 100644
> --- a/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
> +++ b/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
> @@ -39,6 +39,7 @@
> .pg_access = 1,
> .klm_array = NULL,
> .klm_num = 0,
> + .relaxed_ordering = 0,
> };
> struct mlx5_devx_virtq_attr attr = {
> .type =
> MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS,
> diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
> b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
> index 398ca35..da31b47 100644
> --- a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
> +++ b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
> @@ -263,6 +263,7 @@
> mkey_attr.pg_access = 1;
> mkey_attr.klm_array = NULL;
> mkey_attr.klm_num = 0;
> + mkey_attr.relaxed_ordering = 0;
> entry->mkey = mlx5_devx_cmd_mkey_create(priv->ctx,
> &mkey_attr);
> if (!entry->mkey) {
> DRV_LOG(ERR, "Failed to create direct Mkey.");
> --
> 1.8.3.1
Patch applied to next-net-mlx,
Kindest regards,
Raslan Darawsheh
