From: Vamsi Attunuru <[email protected]> Adds DMA driver support for O20 hardware.
Signed-off-by: Vamsi Attunuru <[email protected]> --- doc/guides/dmadevs/cnxk.rst | 65 +++- drivers/dma/cnxk/cnxk_dmadev.c | 530 ++++++++++++++++++++++++++++-- drivers/dma/cnxk/cnxk_dmadev.h | 72 +++- drivers/dma/cnxk/cnxk_dmadev_fp.c | 232 ++++++++++++- 4 files changed, 863 insertions(+), 36 deletions(-) diff --git a/doc/guides/dmadevs/cnxk.rst b/doc/guides/dmadevs/cnxk.rst index 1a7185deb9..2794f393a7 100644 --- a/doc/guides/dmadevs/cnxk.rst +++ b/doc/guides/dmadevs/cnxk.rst @@ -7,8 +7,37 @@ CNXK DMA Device Driver ====================== The ``cnxk`` dmadev driver provides a poll-mode driver (PMD) for Marvell DPI DMA -Hardware Accelerator block found in OCTEON 9 and OCTEON 10 family of SoCs. -Each DMA queue is exposed as a VF function when SRIOV is enabled. +Hardware Accelerator block found in OCTEON CN9K, CN10K and CN20K family of SoCs. + +Supported OCTEON cnxk SoCs +-------------------------- + +- CN9XX +- CN10XX +- CN20XX + +Supported PCI devices +--------------------- + +.. list-table:: + :widths: 30 20 50 + :header-rows: 1 + + * - SoC family + - PCI device ID + - Description + * - CN9K/CN10K + - 0xA081 + - DPI VF + * - CN20K + - 0xA0E8 + - DPI PF + * - CN20K + - 0xA0E9 + - DPI VF + +On CN9K/CN10K, each DMA queue is exposed as a VF function when SRIOV is enabled. +On CN20K, both DPI PF and VF devices can be used directly by the PMD. The block supports following modes of DMA transfers: @@ -38,15 +67,39 @@ For example:: $ dpdk-devbind.py -b vfio-pci 0000:05:00.1 +On CN20K, DPI PF and VF devices can be bound directly to the ``vfio-pci`` driver +and used by the DPDK PMD. The ``octeontx2_dpi.ko`` kernel driver is not required +on CN20K. + Device Probing and Initialization ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To use the devices from an application, the dmadev API can be used. CNXK DMA device configuration requirements: -* Only one ``vchan`` is supported per device. +* CN9K/CN10K: only one ``vchan`` is supported per device. +* CN20K: multiple ``vchans`` are supported per device. * CNXK DMA devices do not support silent mode. +CN20K runtime config options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following ``devargs`` parameters can be used to configure CN20K DPI devices. +For example:: + + -a 0002:02:00.0,num_vchans=16,num_lfs=8 + +``num_vchans`` + + Number of virtual channels to configure per device (default ``8``). + The value must be a power of 2 and must not exceed ``512``. + +``num_lfs`` + + Number of local functions to configure per device (default ``num_vchans / 2``). + Each LF has two hardware rings. The value must be a power of 2 + and must not exceed ``256``. + Once configured, the device can then be made ready for use by calling the ``rte_dma_start()`` API. @@ -60,7 +113,7 @@ for details on operation enqueue and submission API usage. Performance Tuning Parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To achieve higher performance, DMA device needs to be tuned +On CN9K/CN10K, to achieve higher performance, DMA device needs to be tuned using PF kernel driver module parameters. The PF kernel driver is part of the OCTEON SDK. Module parameters shall be configured during module insert as in below example:: @@ -95,3 +148,7 @@ Module parameters shall be configured during module insert as in below example:: MPS and MRRS performance tuning parameters help achieve higher performance only for inbound and outbound DMA transfers. The parameter has no effect for internal only DMA transfer. + + Performance tuning via ``octeontx2_dpi.ko`` applies to CN9K/CN10K only. + CN20K DPI devices do not require the kernel driver and can be used directly + after binding to ``vfio-pci``. diff --git a/drivers/dma/cnxk/cnxk_dmadev.c b/drivers/dma/cnxk/cnxk_dmadev.c index 6ae7fdca3b..602241d755 100644 --- a/drivers/dma/cnxk/cnxk_dmadev.c +++ b/drivers/dma/cnxk/cnxk_dmadev.c @@ -2,6 +2,9 @@ * Copyright (C) 2021 Marvell International Ltd. */ +#include <errno.h> +#include <stdlib.h> + #include <rte_event_dma_adapter.h> #include <cnxk_dmadev.h> @@ -9,6 +12,68 @@ static int cnxk_stats_reset(struct rte_dma_dev *dev, uint16_t vchan); static void cnxk_set_fp_ops(struct rte_dma_dev *dev, uint8_t enable_enq_deq); +static int +parse_val_u16(const char *key, const char *value, void *extra_args) +{ + unsigned long val; + char *end; + + RTE_SET_USED(key); + + if (value == NULL || extra_args == NULL) + return -EINVAL; + + errno = 0; + val = strtoul(value, &end, 0); + if (errno != 0 || end == value || *end != '\0' || val > UINT16_MAX) + return -EINVAL; + + *(uint16_t *)extra_args = (uint16_t)val; + + return 0; +} + +static int +cn20k_dmadev_parse_devargs(struct rte_devargs *devargs, struct cnxk_dpi_vf_s *dpivf) +{ + uint16_t num_vchans = CN20K_DPI_DEF_VCHANS; + uint16_t num_lfs = num_vchans >> 1; /* Each LF has 2 rings */ + struct rte_kvargs *kvlist = NULL; + + if (devargs == NULL) { + dpivf->max_vchans = num_vchans; + dpivf->max_lfs = num_lfs; + return 0; + } + + kvlist = rte_kvargs_parse(devargs->args, NULL); + if (kvlist == NULL) + goto exit; + + if (rte_kvargs_process(kvlist, CN20K_DPI_NUM_VCHANS, &parse_val_u16, &num_vchans) < 0) + goto exit; + if (rte_kvargs_process(kvlist, CN20K_DPI_NUM_LFS, &parse_val_u16, &num_lfs) < 0) + goto exit; + + if (!num_vchans || !num_lfs || num_vchans > CN20K_DPI_MAX_VCHANS || + num_lfs > CN20K_DPI_MAX_LFS) + goto exit; + + if (!rte_is_power_of_2(num_vchans) || !rte_is_power_of_2(num_lfs)) { + plt_err("num_vchans or num_lfs is not a power of 2"); + goto exit; + } + + dpivf->max_vchans = num_vchans; + dpivf->max_lfs = num_lfs; + + rte_kvargs_free(kvlist); + return 0; +exit: + rte_kvargs_free(kvlist); + return -EINVAL; +} + static int cnxk_dmadev_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_info, uint32_t size) { @@ -25,9 +90,21 @@ cnxk_dmadev_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_inf dev_info->dev_capa |= RTE_DMA_CAPA_PRI_POLICY_SP; dev_info->nb_priorities = CN10K_DPI_MAX_PRI; } - dev_info->max_desc = CNXK_DPI_MAX_DESC; - dev_info->min_desc = CNXK_DPI_MIN_DESC; - dev_info->max_sges = CNXK_DPI_MAX_POINTER; + + if (roc_model_is_cn20k()) { + const uint16_t vpr = dpivf->vchans_per_ring ? dpivf->vchans_per_ring : 1; + + dev_info->max_desc = CN20K_DPI_MAX_DESC / vpr; + dev_info->min_desc = CN20K_DPI_MIN_DESC; + dev_info->max_sges = CN20K_DPI_MAX_POINTER; + dev_info->max_vchans = dpivf->max_vchans; + dev_info->dev_capa |= RTE_DMA_CAPA_OPS_FILL; + } else { + dev_info->max_desc = CNXK_DPI_MAX_DESC; + dev_info->min_desc = CNXK_DPI_MIN_DESC; + dev_info->max_sges = CNXK_DPI_MAX_POINTER; + dev_info->max_vchans = CNXK_DPI_MAX_VCHANS_PER_QUEUE; + } return 0; } @@ -52,8 +129,10 @@ cnxk_dmadev_vchan_free(struct cnxk_dpi_vf_s *dpivf, uint16_t vchan) for (; i < num_vchans; i++) { dpi_conf = &dpivf->conf[i]; - rte_free(dpi_conf->c_desc.compl_ptr); - dpi_conf->c_desc.compl_ptr = NULL; + if (dpi_conf->c_desc.compl_ptr) { + rte_free(dpi_conf->c_desc.compl_ptr); + dpi_conf->c_desc.compl_ptr = NULL; + } } return 0; @@ -99,10 +178,39 @@ cnxk_dmadev_chunk_pool_create(struct rte_dma_dev *dev, uint32_t nb_chunks, uint3 return rc; } +static int +cnxk_dmadev_vchan_rsrc_free(struct cnxk_dpi_vf_s *dpivf) +{ + struct roc_dpi *rdpi = &dpivf->rdpi; + int rc; + + if (dpivf->ring_conf) { + rte_free(dpivf->ring_conf); + dpivf->ring_conf = NULL; + } + + if (rdpi->lfs == NULL) + return 0; + + rc = roc_dpi_lf_chan_tbl_free(&(rdpi->lfs[0])); + if (rc < 0) + plt_err("Failed to free dpi lf channel table"); + + rc = roc_dpi_rsrc_fini(rdpi); + if (rc < 0) + plt_err("Failed to free dpi lfs"); + + dpivf->is_ring_conf_done = false; + return rc; +} + static int cnxk_dmadev_configure(struct rte_dma_dev *dev, const struct rte_dma_conf *conf, uint32_t conf_sz) { struct cnxk_dpi_vf_s *dpivf = NULL; + struct roc_dpi_lf *lf; + uint16_t num_rings; + int rc = 0; RTE_SET_USED(conf_sz); dpivf = dev->fp_obj->dev_private; @@ -111,13 +219,78 @@ cnxk_dmadev_configure(struct rte_dma_dev *dev, const struct rte_dma_conf *conf, * Free up vchan memory if any, before configuring num_vchans. */ cnxk_dmadev_vchan_free(dpivf, RTE_DMA_ALL_VCHAN); + dpivf->num_vchans = conf->nb_vchans; + if (roc_model_is_cn20k()) { + if (!rte_is_power_of_2(dpivf->num_vchans)) + dpivf->num_vchans = rte_align32pow2(dpivf->num_vchans); + + if (dpivf->num_vchans > dpivf->max_vchans) { + plt_info("Limiting vchans from %u to max_vchans %u", + dpivf->num_vchans, dpivf->max_vchans); + dpivf->num_vchans = dpivf->max_vchans; + } + } if (roc_feature_dpi_has_priority()) dpivf->rdpi.priority = conf->priority; cnxk_set_fp_ops(dev, conf->flags & RTE_DMA_CFG_FLAG_ENQ_DEQ); - return 0; + if (roc_model_is_cn20k()) { + rc = cnxk_dmadev_vchan_rsrc_free(dpivf); + if (rc < 0) + goto error; + + dpivf->rdpi.nr_lfs = dpivf->max_lfs; + num_rings = dpivf->rdpi.nr_lfs << 1; + + if (dpivf->num_vchans > num_rings) { + dpivf->vchans_per_ring = dpivf->num_vchans / num_rings; + } else { + /* Each vchan has got it's own hardware ring */ + dpivf->vchans_per_ring = 1; + dpivf->rdpi.nr_lfs = dpivf->num_vchans >> 1; + if (!dpivf->rdpi.nr_lfs) + dpivf->rdpi.nr_lfs = 1; + } + + if ((CN20K_DPI_MAX_DESC / dpivf->vchans_per_ring) < CN20K_DPI_MIN_DESC) { + rc = -EINVAL; + plt_err("Very few LFs are requested than required"); + goto error; + } + + num_rings = dpivf->rdpi.nr_lfs << 1; + dpivf->ring_conf = rte_zmalloc("dpi_ring_conf", sizeof(struct cn20k_ring_conf) * + num_rings, 0); + if (dpivf->ring_conf == NULL) { + plt_err("Failed to allocate memory for ring conf"); + rc = -ENOMEM; + goto error; + } + + rc = roc_dpi_rsrc_init(&dpivf->rdpi); + if (rc < 0) { + plt_err("rsrc alloc failed"); + goto error; + } + + lf = &(dpivf->rdpi.lfs[0]); + rc = roc_dpi_lf_chan_tbl_alloc(lf, dpivf->num_vchans); + if (rc < 0) { + plt_err("Failed to allocate chan tbl"); + goto error; + } + + dpivf->chan_tbl = lf->chan_tbl; + dpivf->is_ring_conf_done = false; + } + +error: + if (roc_model_is_cn20k() && rc < 0) + cnxk_dmadev_vchan_rsrc_free(dpivf); + + return rc; } static int @@ -232,6 +405,87 @@ cn10k_dmadev_setup_hdr(union cnxk_dpi_instr_cmd *header, const struct rte_dma_vc return 0; } +static int +cn20k_dmadev_setup(struct cnxk_dpi_vf_s *dpivf, uint16_t vchan, + const struct rte_dma_vchan_conf *conf) +{ + struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + struct cn20k_ring_conf *rconf = dpivf->ring_conf; + uint16_t ridx, max_desc, num_rings; + union cnxk_dpi_instr_cmd *header; + int rc = 0; + int aura; + + header = (union cnxk_dpi_instr_cmd *)&dpi_conf->cmd.u; + header->cn20k.ct = DPI_HDR_PT_ZBW_CA; + header->cn20k.xt = 0; + + switch (conf->direction) { + case RTE_DMA_DIR_DEV_TO_MEM: + dpi_conf->cfg.xtype = DPI_XTYPE_INBOUND; + dpi_conf->cfg.rport = conf->src_port.pcie.coreid; + dpi_conf->cfg.wport = 0; + dpi_conf->chan_cfg.pf_func = conf->src_port.pcie.pfid; + dpi_conf->chan_cfg.vf_func = conf->src_port.pcie.vfid; + dpi_conf->chan_cfg.valid = 1; + break; + case RTE_DMA_DIR_MEM_TO_DEV: + dpi_conf->cfg.xtype = DPI_XTYPE_OUTBOUND; + dpi_conf->cfg.rport = 0; + dpi_conf->cfg.wport = conf->dst_port.pcie.coreid; + dpi_conf->chan_cfg.pf_func = conf->dst_port.pcie.pfid; + dpi_conf->chan_cfg.vf_func = conf->dst_port.pcie.vfid; + dpi_conf->chan_cfg.valid = 1; + aura = dmadev_src_buf_aura_get(conf->auto_free.m2d.pool, "cn20k_mempool_ops"); + if (aura < 0) + return aura; + header->cn20k.aura = aura; + break; + case RTE_DMA_DIR_MEM_TO_MEM: + dpi_conf->cfg.xtype = DPI_XTYPE_INTERNAL_ONLY; + dpi_conf->cfg.rport = 0; + dpi_conf->cfg.wport = 0; + break; + case RTE_DMA_DIR_DEV_TO_DEV: + dpi_conf->cfg.xtype = DPI_XTYPE_EXTERNAL_ONLY; + dpi_conf->cfg.rport = conf->src_port.pcie.coreid; + dpi_conf->cfg.wport = conf->dst_port.pcie.coreid; + }; + + max_desc = conf->nb_desc; + if (!rte_is_power_of_2(max_desc)) + max_desc = rte_align32pow2(max_desc); + + num_rings = dpivf->rdpi.nr_lfs << 1; + + for (ridx = 0; ridx < num_rings; ridx++) { + if (!rconf[ridx].used) { + if (!rconf[ridx].num_vchans) { + rconf[ridx].direction = conf->direction; + rconf[ridx].num_desc = max_desc; + } else if ((rconf[ridx].direction == conf->direction) && + (rconf[ridx].num_vchans < dpivf->vchans_per_ring)) { + rconf[ridx].num_desc += max_desc; + } else { + continue; + } + + rconf[ridx].num_vchans++; + dpi_conf->ridx = ridx; + + if (rconf[ridx].num_vchans == dpivf->vchans_per_ring) + rconf[ridx].used = true; + + break; + } + } + + if (ridx == num_rings) + rc = -ENODEV; + + return rc; +} + static int cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan, const struct rte_dma_vchan_conf *conf, uint32_t conf_sz) @@ -245,9 +499,14 @@ cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan, RTE_SET_USED(conf_sz); + if (conf->auto_free.m2d.pool != NULL && conf->direction != RTE_DMA_DIR_MEM_TO_DEV) + return -EINVAL; + header = (union cnxk_dpi_instr_cmd *)&dpi_conf->cmd.u; - if (dpivf->is_cn10k) + if (roc_model_is_cn20k()) + ret = cn20k_dmadev_setup(dpivf, vchan, conf); + else if (roc_model_is_cn10k()) ret = cn10k_dmadev_setup_hdr(header, conf); else ret = cn9k_dmadev_setup_hdr(header, conf); @@ -262,7 +521,7 @@ cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan, if (!rte_is_power_of_2(max_desc)) max_desc = rte_align32pow2(max_desc); - if (max_desc > CNXK_DPI_MAX_DESC) + if (!roc_model_is_cn20k() && (max_desc > CNXK_DPI_MAX_DESC)) max_desc = CNXK_DPI_MAX_DESC; size = (max_desc * sizeof(uint8_t) * CNXK_DPI_COMPL_OFFSET); @@ -285,10 +544,121 @@ cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan, dpi_conf->c_desc.compl_ptr[i * CNXK_DPI_COMPL_OFFSET] = CNXK_DPI_REQ_CDATA; dpi_conf->c_desc.max_cnt = (max_desc - 1); + dpi_conf->cfg_done = true; return 0; } +static int +cn20k_dmadev_queue_setup(struct cnxk_dpi_vf_s *dpivf) +{ + struct cn20k_ring_conf *rconf = dpivf->ring_conf; + struct roc_dpi *rdpi = &dpivf->rdpi; + struct cnxk_dpi_conf *dpi_conf; + struct roc_dpi_lf_que *que; + struct roc_dpi_lf_ring_cfg cfg; + struct roc_dpi_lf *lf; + uint16_t idx, qidx, ridx, vchan; + int rc = 0; + + for (idx = 0; idx < rdpi->nr_lfs; idx++) { + lf = &rdpi->lfs[idx]; + lf->chan_tbl = dpivf->chan_tbl; + + for (qidx = 0; qidx < 2; qidx++) { + ridx = (idx << 1) + qidx; + + if (!rconf[ridx].used) + continue; + + que = &lf->queue[qidx]; + + que->qsize = rconf[ridx].num_desc; + que->cmd_len = DPI_CMD_SIZE_128B; + que->first_skip = 0; + que->later_skip = 0; + memset(&cfg, 0, sizeof(cfg)); + cfg.ring_idx = qidx; + cfg.isize = que->cmd_len / DPI_CMD_SIZE_128B; + cfg.xtype = DPI_XTYPE_INTERNAL_ONLY; + cfg.rport = 0; /* Default is PEM:0 */ + cfg.wport = 0; /* Default is PEM:0 */ + cfg.pri = 0; /* 0 - High Priority */ + for (vchan = 0; vchan < dpivf->num_vchans; vchan++) { + dpi_conf = &dpivf->conf[vchan]; + + if (dpi_conf->ridx == ridx) + cfg.xtype = dpi_conf->cfg.xtype; + } + + rc = roc_dpi_lf_ring_init(que, &cfg); + if (rc) + return rc; + + for (vchan = 0; vchan < dpivf->num_vchans; vchan++) { + dpi_conf = &dpivf->conf[vchan]; + + if (dpi_conf->ridx == ridx) { + dpi_conf->que = que; + dpi_conf->dbell = lf->rbase + DPI_LF_RINGX_WIDX(qidx); + } + } + } + + rc = roc_dpi_lf_chan_tbl_select(lf); + if (rc) + return rc; + + /* FIX ME */ + rc = roc_dpi_lf_pffunc_cfg(lf); + if (rc) + return rc; + } + + return rc; +} + +static int +cn20k_dmadev_chan_tbl_setup(struct cnxk_dpi_vf_s *dpivf) +{ + uint16_t vchan, idx = 0, offset = 0; + struct cnxk_dpi_conf *dpi_conf; + uint64_t config[64] = {0}; + struct roc_dpi_lf *lf; + int rc = 0; + + lf = &(dpivf->rdpi.lfs[0]); + + for (vchan = 0; vchan < dpivf->num_vchans; vchan++) { + dpi_conf = &dpivf->conf[vchan]; + if (!dpi_conf->cfg_done || (dpi_conf->cfg.xtype == DPI_XTYPE_INTERNAL_ONLY)) + continue; + + dpi_conf->cmd.cn20k.chan = offset + idx; + config[idx++] = dpi_conf->chan_cfg.u; + + if (idx == 64) { + rc = roc_dpi_lf_chan_tbl_update(lf, config, offset, idx); + if (rc < 0) { + plt_err("Failed to update chan tbl = %d", rc); + return rc; + } + offset += idx; + idx = 0; + } + } + + if (idx) { + rc = roc_dpi_lf_chan_tbl_update(lf, config, offset, idx); + if (rc < 0) { + plt_err("Failed to update chan tbl = %d", rc); + return rc; + } + } + + return rc; +} + static int cnxk_dmadev_start(struct rte_dma_dev *dev) { @@ -306,6 +676,10 @@ cnxk_dmadev_start(struct rte_dma_dev *dev) dpi_conf->c_desc.head = 0; dpi_conf->c_desc.tail = 0; dpi_conf->desc_idx = 0; + + if (dpi_conf->c_desc.compl_ptr == NULL) + continue; + for (j = 0; j < dpi_conf->c_desc.max_cnt + 1; j++) dpi_conf->c_desc.compl_ptr[j * CNXK_DPI_COMPL_OFFSET] = CNXK_DPI_REQ_CDATA; nb_desc += dpi_conf->c_desc.max_cnt + 1; @@ -313,6 +687,31 @@ cnxk_dmadev_start(struct rte_dma_dev *dev) dpi_conf->completed_offset = 0; } + if (roc_model_is_cn20k()) { + for (i = 0; i < (dpivf->rdpi.nr_lfs << 1); i++) + dpivf->ring_conf[i].pending = 0; + rc = roc_dpi_reset(&dpivf->rdpi); + if (rc < 0) { + plt_err("Failed to reset DPI LFs"); + goto error; + } + + if (dpivf->is_ring_conf_done) + goto enable_dpi; + + rc = cn20k_dmadev_queue_setup(dpivf); + if (rc) + goto error; + + rc = cn20k_dmadev_chan_tbl_setup(dpivf); + if (rc) + goto error; + + dpivf->is_ring_conf_done = true; + + goto enable_dpi; + } + queue_buf_sz = CNXK_DPI_QUEUE_BUF_SIZE_V2; /* Max block size allowed by cnxk mempool driver is (128 * 1024). * Block size = elt_size + mp->header + mp->trailer. @@ -349,6 +748,7 @@ cnxk_dmadev_start(struct rte_dma_dev *dev) dpivf->chunk_head = 0; dpivf->chunk_size_m1 = (queue_buf_sz >> 3) - 2; +enable_dpi: roc_dpi_enable(&dpivf->rdpi); error: return rc; @@ -359,14 +759,18 @@ cnxk_dmadev_stop(struct rte_dma_dev *dev) { struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private; - if (roc_dpi_wait_queue_idle(&dpivf->rdpi)) - return -EAGAIN; + if (!roc_model_is_cn20k()) { + if (roc_dpi_wait_queue_idle(&dpivf->rdpi)) + return -EAGAIN; + } roc_dpi_disable(&dpivf->rdpi); - rte_mempool_free(dpivf->chunk_pool); - dpivf->chunk_pool = NULL; - dpivf->chunk_base = NULL; - dpivf->chunk_size_m1 = 0; + if (!roc_model_is_cn20k()) { + rte_mempool_free(dpivf->chunk_pool); + dpivf->chunk_pool = NULL; + dpivf->chunk_base = NULL; + dpivf->chunk_size_m1 = 0; + } return 0; } @@ -378,6 +782,10 @@ cnxk_dmadev_close(struct rte_dma_dev *dev) roc_dpi_disable(&dpivf->rdpi); cnxk_dmadev_vchan_free(dpivf, RTE_DMA_ALL_VCHAN); + + if (roc_model_is_cn20k()) + cnxk_dmadev_vchan_rsrc_free(dpivf); + roc_dpi_dev_fini(&dpivf->rdpi); /* Clear all flags as we close the device. */ @@ -454,11 +862,17 @@ cnxk_damdev_burst_capacity(const void *dev_private, uint16_t vchan) { const struct cnxk_dpi_vf_s *dpivf = (const struct cnxk_dpi_vf_s *)dev_private; const struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + struct cn20k_ring_conf *ring_conf; uint16_t burst_cap; burst_cap = dpi_conf->c_desc.max_cnt - (dpi_conf->stats.submitted - dpi_conf->stats.completed) + 1; + if (roc_model_is_cn20k()) { + ring_conf = &dpivf->ring_conf[dpi_conf->ridx]; + burst_cap -= ring_conf->pending; + } + return burst_cap; } @@ -472,7 +886,7 @@ cnxk_dmadev_submit(void *dev_private, uint16_t vchan) if (!num_words) return 0; - rte_wmb(); + rte_io_wmb(); plt_write64(num_words, dpivf->rdpi.rbase + DPI_VDMA_DBELL); dpivf->total_pnum_words = 0; @@ -480,6 +894,26 @@ cnxk_dmadev_submit(void *dev_private, uint16_t vchan) return 0; } +static int +cn20k_dmadev_submit(void *dev_private, uint16_t vchan) +{ + struct cnxk_dpi_vf_s *dpivf = dev_private; + struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + struct cn20k_ring_conf *ring_conf = &dpivf->ring_conf[dpi_conf->ridx]; + uint16_t num_words = ring_conf->pending; + + if (!num_words) + return 0; + + rte_io_wmb(); + plt_write64(num_words, dpi_conf->dbell); + dpi_conf->stats.submitted += num_words; + + ring_conf->pending = 0; + + return 0; +} + static int cnxk_stats_get(const struct rte_dma_dev *dev, uint16_t vchan, struct rte_dma_stats *rte_stats, uint32_t size) @@ -552,11 +986,16 @@ cnxk_set_fp_ops(struct rte_dma_dev *dev, uint8_t ena_enq_deq) dev->fp_obj->submit = cnxk_dmadev_submit; dev->fp_obj->completed = cnxk_dmadev_completed; dev->fp_obj->completed_status = cnxk_dmadev_completed_status; - dev->fp_obj->burst_capacity = cnxk_damdev_burst_capacity; + dev->fp_obj->burst_capacity = cnxk_damdev_burst_capacity; if (roc_model_is_cn10k()) { dev->fp_obj->copy = cn10k_dmadev_copy; dev->fp_obj->copy_sg = cn10k_dmadev_copy_sg; + } else if (roc_model_is_cn20k()) { + dev->fp_obj->submit = cn20k_dmadev_submit; + dev->fp_obj->copy = cn20k_dmadev_copy; + dev->fp_obj->copy_sg = cn20k_dmadev_copy_sg; + dev->fp_obj->fill = cn20k_dmadev_fill; } if (ena_enq_deq) { @@ -571,6 +1010,9 @@ cnxk_set_fp_ops(struct rte_dma_dev *dev, uint8_t ena_enq_deq) if (roc_model_is_cn10k()) dev->fp_obj->enqueue = cn10k_dma_ops_enqueue; + + if (roc_model_is_cn20k()) + dev->fp_obj->enqueue = cn20k_dma_ops_enqueue; } } @@ -594,8 +1036,10 @@ cnxk_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_de struct roc_dpi *rdpi = NULL; int rc; - if (!pci_dev->mem_resource[0].addr) - return -ENODEV; + if (!roc_model_is_cn20k()) { + if (!pci_dev->mem_resource[0].addr) + return -ENODEV; + } rc = roc_plt_init(); if (rc) { @@ -612,19 +1056,65 @@ cnxk_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_de } dpivf = dmadev->data->dev_private; + dmadev->device = &pci_dev->device; + + if (roc_model_is_cn20k()) { + /* Parse devargs string */ + rc = cn20k_dmadev_parse_devargs(dmadev->device->devargs, dpivf); + if (rc) { + plt_err("Failed to parse devargs rc=%d", rc); + goto err_out_free; + } + dpivf->num_vchans = dpivf->max_vchans; + } dmadev->device = &pci_dev->device; dmadev->fp_obj->dev_private = dpivf; dmadev->dev_ops = &cnxk_dmadev_ops; + /* + * dev_private (and the roc_dpi it holds) lives in shared memory. Only + * the primary owns the HW and the per-process pci_dev pointer; a + * secondary must not run roc_dpi_dev_init nor overwrite rdpi->pci_dev, + * otherwise the primary later dereferences the secondary's stale + * pointer during rte_eal_cleanup() and crashes. + */ if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; - dpivf->is_cn10k = roc_model_is_cn10k(); + dmadev->fp_obj->submit = cnxk_dmadev_submit; + dmadev->fp_obj->completed = cnxk_dmadev_completed; + dmadev->fp_obj->completed_status = cnxk_dmadev_completed_status; + dmadev->fp_obj->burst_capacity = cnxk_damdev_burst_capacity; + + if (roc_model_is_cn10k()) { + dmadev->fp_obj->copy = cn10k_dmadev_copy; + dmadev->fp_obj->copy_sg = cn10k_dmadev_copy_sg; + dpivf->num_vchans = CNXK_DPI_MAX_VCHANS_PER_QUEUE; + } else if (roc_model_is_cn20k()) { + dmadev->fp_obj->submit = cn20k_dmadev_submit; + dmadev->fp_obj->copy = cn20k_dmadev_copy; + dmadev->fp_obj->copy_sg = cn20k_dmadev_copy_sg; + dmadev->fp_obj->fill = cn20k_dmadev_fill; + } else { + dmadev->fp_obj->copy = cnxk_dmadev_copy; + dmadev->fp_obj->copy_sg = cnxk_dmadev_copy_sg; + dpivf->num_vchans = CNXK_DPI_MAX_VCHANS_PER_QUEUE; + } + + dpivf->conf = rte_zmalloc("dpi_vchan_conf", dpivf->num_vchans * + sizeof(struct cnxk_dpi_conf), 0); + if (dpivf->conf == NULL) { + plt_err("Failed to allocate memory for vchan conf"); + rc = -ENOMEM; + goto err_out_free; + } + dpivf->mcs_lock = NULL; rdpi = &dpivf->rdpi; rdpi->pci_dev = pci_dev; + rc = roc_dpi_dev_init(rdpi, offsetof(struct rte_dma_op, impl_opaque)); if (rc < 0) goto err_out_free; @@ -653,6 +1143,8 @@ cnxk_dmadev_remove(struct rte_pci_device *pci_dev) static const struct rte_pci_id cnxk_dma_pci_map[] = { {RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CNXK_DPI_VF)}, + {RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN20K_DPI_PF)}, + {RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN20K_DPI_VF)}, { .vendor_id = 0, }, diff --git a/drivers/dma/cnxk/cnxk_dmadev.h b/drivers/dma/cnxk/cnxk_dmadev.h index 18039e43fb..3ba404dda8 100644 --- a/drivers/dma/cnxk/cnxk_dmadev.h +++ b/drivers/dma/cnxk/cnxk_dmadev.h @@ -9,9 +9,11 @@ #include <bus_pci_driver.h> #include <rte_common.h> +#include <rte_devargs.h> #include <rte_dmadev.h> #include <rte_dmadev_pmd.h> #include <rte_eal.h> +#include <rte_kvargs.h> #include <rte_lcore.h> #include <rte_mbuf_pool_ops.h> #include <rte_mcslock.h> @@ -31,15 +33,31 @@ #define CN10K_DPI_MAX_PRI 2 #define CNXK_DPI_MAX_VCHANS_PER_QUEUE 128 #define CNXK_DPI_QUEUE_BUF_SIZE 16256 -#define CNXK_DPI_QUEUE_BUF_SIZE_V2 130944 +/* Maximum pool size supported by device is 128 * 1024. When RTE_LIBRTE_MEMPOOL_DEBUG is enabled + * mempool->trailer size will be increased by 8B. Additionally if the pool is not created with + * RTE_MEMPOOL_F_NO_CACHE_ALIGN, trailer will be expanded to cache line size. + * To allow future needs, limit the max size to 127KB + */ +#define CNXK_DPI_QUEUE_BUF_SIZE_V2 130048 #define CNXK_DPI_POOL_MAX_CACHE_SZ (16) #define CNXK_DPI_DW_PER_SINGLE_CMD 8 #define CNXK_DPI_HDR_LEN 4 #define CNXK_DPI_CMD_LEN(src, dst) (CNXK_DPI_HDR_LEN + ((src) << 1) + ((dst) << 1)) -#define CNXK_DPI_MAX_CMD_SZ CNXK_DPI_CMD_LEN(CNXK_DPI_MAX_POINTER, \ +#define CNXK_DPI_MAX_CMD_SZ CNXK_DPI_CMD_LEN(CNXK_DPI_MAX_POINTER, \ CNXK_DPI_MAX_POINTER) #define CNXK_DPI_CHUNKS_FROM_DESC(cz, desc) (((desc) / (((cz) / 8) / CNXK_DPI_MAX_CMD_SZ)) + 1) #define CNXK_DPI_COMPL_OFFSET ROC_CACHE_LINE_SZ + +#define CN20K_DPI_MAX_POINTER 4 +#define CN20K_DPI_MAX_DESC 2048 +#define CN20K_DPI_MIN_DESC 128 +#define CN20K_DPI_MAX_VCHANS 512 +#define CN20K_DPI_DEF_VCHANS 8 +#define CN20K_DPI_MAX_LFS 256 + +#define CN20K_DPI_NUM_VCHANS "num_vchans" +#define CN20K_DPI_NUM_LFS "num_lfs" + /* Set Completion data to 0xFF when request submitted, * upon successful request completion engine reset to completion status */ @@ -86,6 +104,32 @@ union cnxk_dpi_instr_cmd { uint64_t reserved_62_63 : 2; /* Word 0 - End */ } cn10k; + + struct cn20k_dpi_instr_cmd { + uint64_t nfst : 3; + uint64_t reserved_3 : 1; + uint64_t nlst : 3; + uint64_t reserved_7 : 1; + uint64_t msix_int : 1; + uint64_t ct : 3; + uint64_t chan : 14; + uint64_t reserved_26_29 : 4; + uint64_t aura : 20; + uint64_t xt : 2; + uint64_t ivec : 9; + uint64_t fe : 1; + uint64_t reserved_62 : 1; + uint64_t vld : 1; + /* Word 0 - End */ + } cn20k; +}; + +struct cn20k_ring_conf { + enum rte_dma_direction direction; + uint16_t pending; + uint16_t num_desc; + uint8_t num_vchans; + bool used; }; struct cnxk_dpi_cdesc_data_s { @@ -100,9 +144,15 @@ struct cnxk_dpi_conf { union cnxk_dpi_instr_cmd cmd; struct cnxk_dpi_cdesc_data_s c_desc; uint16_t desc_idx; + uintptr_t dbell; struct rte_dma_stats stats; uint64_t completed_offset; + struct roc_dpi_lf_que *que; + union roc_dpi_lf_ccfg chan_cfg; + struct roc_dpi_lf_ring_cfg cfg; + uint16_t ridx; bool adapter_enabled; + bool cfg_done; }; struct cnxk_dpi_vf_s { @@ -111,15 +161,21 @@ struct cnxk_dpi_vf_s { uint16_t chunk_head; uint16_t chunk_size_m1; uint16_t total_pnum_words; + uint16_t vchans_per_ring; struct rte_mempool *chunk_pool; - struct cnxk_dpi_conf conf[CNXK_DPI_MAX_VCHANS_PER_QUEUE]; + struct cnxk_dpi_conf *conf; + struct cn20k_ring_conf *ring_conf; RTE_ATOMIC(rte_mcslock_t *) mcs_lock; /* Slow path */ struct roc_dpi rdpi; uint32_t aura; + uint16_t max_lfs; + uint16_t max_vchans; uint16_t num_vchans; + uint16_t chan_tbl; uint16_t flag; uint8_t is_cn10k; + uint8_t is_ring_conf_done; } __plt_cache_aligned; int cnxk_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst, @@ -138,5 +194,13 @@ uint16_t cn10k_dma_ops_enqueue(void *dev_private, uint16_t vchan, struct rte_dma uint16_t nb_ops); uint16_t cnxk_dma_ops_dequeue(void *dev_private, uint16_t vchan, struct rte_dma_op **ops, uint16_t nb_ops); - +int cn20k_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst, + uint32_t length, uint64_t flags); +int cn20k_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge *src, + const struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst, + uint64_t flags); +int cn20k_dmadev_fill(void *dev_private, uint16_t vchan, uint64_t pattern, rte_iova_t dst, + uint32_t length, uint64_t flags); +uint16_t cn20k_dma_ops_enqueue(void *dev_private, uint16_t vchan, struct rte_dma_op **ops, + uint16_t nb_ops); #endif diff --git a/drivers/dma/cnxk/cnxk_dmadev_fp.c b/drivers/dma/cnxk/cnxk_dmadev_fp.c index 4435adc38b..26ac6e0804 100644 --- a/drivers/dma/cnxk/cnxk_dmadev_fp.c +++ b/drivers/dma/cnxk/cnxk_dmadev_fp.c @@ -282,7 +282,7 @@ cnxk_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t d } if (flags & RTE_DMA_OP_FLAG_SUBMIT) { - rte_wmb(); + rte_io_wmb(); plt_write64(dpivf->total_pnum_words + CNXK_DPI_DW_PER_SINGLE_CMD, dpivf->rdpi.rbase + DPI_VDMA_DBELL); dpivf->total_pnum_words = 0; @@ -338,7 +338,7 @@ cnxk_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge } if (flags & RTE_DMA_OP_FLAG_SUBMIT) { - rte_wmb(); + rte_io_wmb(); plt_write64(dpivf->total_pnum_words + CNXK_DPI_CMD_LEN(nb_src, nb_dst), dpivf->rdpi.rbase + DPI_VDMA_DBELL); dpivf->total_pnum_words = 0; @@ -384,7 +384,7 @@ cn10k_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t } if (flags & RTE_DMA_OP_FLAG_SUBMIT) { - rte_wmb(); + rte_io_wmb(); plt_write64(dpivf->total_pnum_words + CNXK_DPI_DW_PER_SINGLE_CMD, dpivf->rdpi.rbase + DPI_VDMA_DBELL); dpivf->total_pnum_words = 0; @@ -427,7 +427,7 @@ cn10k_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge } if (flags & RTE_DMA_OP_FLAG_SUBMIT) { - rte_wmb(); + rte_io_wmb(); plt_write64(dpivf->total_pnum_words + CNXK_DPI_CMD_LEN(nb_src, nb_dst), dpivf->rdpi.rbase + DPI_VDMA_DBELL); dpivf->total_pnum_words = 0; @@ -440,6 +440,163 @@ cn10k_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge return dpi_conf->desc_idx++; } +int +cn20k_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst, + uint32_t length, uint64_t flags) +{ + struct cnxk_dpi_vf_s *dpivf = dev_private; + struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + const uint16_t max_cnt = dpi_conf->c_desc.max_cnt; + struct roc_dpi_lf_que *queue = dpi_conf->que; + struct cn20k_ring_conf *ring_conf; + uint8_t *comp_ptr; + uint64_t *cmd; + + if (unlikely(((dpi_conf->c_desc.tail + 1) & max_cnt) == (dpi_conf->c_desc.head & max_cnt))) + return -ENOSPC; + + if (dpivf->vchans_per_ring == 1) { + cmd = queue->cmd_base + ((dpi_conf->c_desc.tail & max_cnt) << 4); + } else { + cmd = queue->cmd_base + (queue->widx << 4); + queue->widx = (queue->widx + 1) & (queue->qsize - 1); + } + + ring_conf = &(dpivf->ring_conf[dpi_conf->ridx]); + comp_ptr = &dpi_conf->c_desc + .compl_ptr[(dpi_conf->c_desc.tail & max_cnt) * CNXK_DPI_COMPL_OFFSET]; + dpi_conf->c_desc.tail++; + + cmd[1] = (uint64_t)comp_ptr; + cmd[4] = ((uint64_t)length << 32) | length | ((flags & RTE_DMA_OP_FLAG_AUTO_FREE) << 28); + cmd[5] = src; + cmd[6] = dst; + cmd[0] = DPI_CMD_VLD_BIT | dpi_conf->cmd.u | 0x11U; + + if (flags & RTE_DMA_OP_FLAG_SUBMIT) { + rte_io_wmb(); + plt_write64(ring_conf->pending + 1, dpi_conf->dbell); + dpi_conf->stats.submitted += (ring_conf->pending + 1); + ring_conf->pending = 0; + } else { + ring_conf->pending++; + } + + return dpi_conf->desc_idx++; +} + +/* Helper macro to write length and address */ +#define DPI_WRITE_SEGMENT(ptr, seg, i, idx, eidx, tmp) \ + do { \ + if ((tmp) % 2 == 0) { \ + ptr[eidx] = seg[i].length; \ + idx++; \ + } else { \ + ptr[eidx] |= ((uint64_t)seg[i].length << 32); \ + eidx += 3; \ + } \ + ptr[idx++] = (uint64_t)seg[i].addr; \ + tmp++; \ + } while (0) + +int +cn20k_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge *src, + const struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst, + uint64_t flags) +{ + struct cnxk_dpi_vf_s *dpivf = dev_private; + struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + const uint16_t max_cnt = dpi_conf->c_desc.max_cnt; + struct roc_dpi_lf_que *queue = dpi_conf->que; + struct cn20k_ring_conf *ring_conf = &dpivf->ring_conf[dpi_conf->ridx]; + uint16_t idx = 4, eidx = 4, tmp = 0; + uint8_t *comp_ptr, i; + uint64_t *cmd; + + if (unlikely(((dpi_conf->c_desc.tail + 1) & max_cnt) == (dpi_conf->c_desc.head & max_cnt))) + return -ENOSPC; + + if (dpivf->vchans_per_ring == 1) { + cmd = queue->cmd_base + ((dpi_conf->c_desc.tail & max_cnt) << 4); + } else { + cmd = queue->cmd_base + (queue->widx << 4); + queue->widx = (queue->widx + 1) & (queue->qsize - 1); + } + comp_ptr = &dpi_conf->c_desc + .compl_ptr[(dpi_conf->c_desc.tail & max_cnt) * CNXK_DPI_COMPL_OFFSET]; + dpi_conf->c_desc.tail++; + + cmd[1] = (uint64_t)comp_ptr; + + /* Fill source segments */ + for (i = 0; i < nb_src; i++) + DPI_WRITE_SEGMENT(cmd, src, i, idx, eidx, tmp); + + /* Fill destination segments */ + for (i = 0; i < nb_dst; i++) + DPI_WRITE_SEGMENT(cmd, dst, i, idx, eidx, tmp); + + cmd[0] = DPI_CMD_VLD_BIT | dpi_conf->cmd.u | (nb_dst << 4) | nb_src; + + if (flags & RTE_DMA_OP_FLAG_SUBMIT) { + rte_io_wmb(); + plt_write64(ring_conf->pending + 1, dpi_conf->dbell); + dpi_conf->stats.submitted += ring_conf->pending + 1; + ring_conf->pending = 0; + } else { + ring_conf->pending++; + } + + return dpi_conf->desc_idx++; +} + +int +cn20k_dmadev_fill(void *dev_private, uint16_t vchan, uint64_t pattern, rte_iova_t dst, + uint32_t length, uint64_t flags) +{ + struct cnxk_dpi_vf_s *dpivf = dev_private; + struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + const uint16_t max_cnt = dpi_conf->c_desc.max_cnt; + struct roc_dpi_lf_que *queue = dpi_conf->que; + struct cn20k_ring_conf *ring_conf; + uint8_t *comp_ptr; + uint64_t *cmd; + +#define DPI_XT_TYPE_FILL BIT_ULL(50) + + if (unlikely(((dpi_conf->c_desc.tail + 1) & max_cnt) == (dpi_conf->c_desc.head & max_cnt))) + return -ENOSPC; + + if (dpivf->vchans_per_ring == 1) { + cmd = queue->cmd_base + ((dpi_conf->c_desc.tail & max_cnt) << 4); + } else { + cmd = queue->cmd_base + (queue->widx << 4); + queue->widx = (queue->widx + 1) & (queue->qsize - 1); + } + + ring_conf = &(dpivf->ring_conf[dpi_conf->ridx]); + comp_ptr = &dpi_conf->c_desc + .compl_ptr[(dpi_conf->c_desc.tail & max_cnt) * CNXK_DPI_COMPL_OFFSET]; + dpi_conf->c_desc.tail++; + + cmd[1] = (uint64_t)comp_ptr; + cmd[4] = (uint64_t)length << 32; + cmd[5] = pattern; + cmd[6] = dst; + cmd[0] = DPI_CMD_VLD_BIT | DPI_XT_TYPE_FILL | dpi_conf->cmd.u | 0x10U; + + if (flags & RTE_DMA_OP_FLAG_SUBMIT) { + rte_io_wmb(); + plt_write64(ring_conf->pending + 1, dpi_conf->dbell); + dpi_conf->stats.submitted += (ring_conf->pending + 1); + ring_conf->pending = 0; + } else { + ring_conf->pending++; + } + + return dpi_conf->desc_idx++; +} + static inline uint64_t cnxk_dma_adapter_format_event(uint64_t event) { @@ -450,6 +607,63 @@ cnxk_dma_adapter_format_event(uint64_t event) return w0; } +uint16_t +cn20k_dma_ops_enqueue(void *dev_private, uint16_t vchan, struct rte_dma_op **ops, uint16_t nb_ops) +{ + struct cnxk_dpi_vf_s *dpivf = dev_private; + struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + const uint16_t max_cnt = dpi_conf->c_desc.max_cnt; + struct roc_dpi_lf_que *queue = dpi_conf->que; + uint16_t idx, eidx, tmp; + struct rte_dma_op *op; + uint16_t space, i, j; + uint16_t src, dst; + uint8_t *comp_ptr; + uint64_t *cmd; + + space = (max_cnt + dpi_conf->c_desc.head - dpi_conf->c_desc.tail) & max_cnt; + space = RTE_MIN(space, nb_ops); + + for (j = 0; j < space; j++) { + op = ops[j]; + src = op->nb_src; + dst = op->nb_dst; + + idx = 4; eidx = 4; tmp = 0; + + if (dpivf->vchans_per_ring == 1) { + cmd = queue->cmd_base + ((dpi_conf->c_desc.tail & max_cnt) << 4); + } else { + cmd = queue->cmd_base + (queue->widx << 4); + queue->widx = (queue->widx + 1) & (queue->qsize - 1); + } + comp_ptr = &dpi_conf->c_desc.compl_ptr[(dpi_conf->c_desc.tail & max_cnt) * + CNXK_DPI_COMPL_OFFSET]; + dpi_conf->c_desc.ops[dpi_conf->c_desc.tail & max_cnt] = op; + dpi_conf->c_desc.tail++; + + cmd[1] = (uint64_t)comp_ptr; + + /* Fill source segments */ + for (i = 0; i < src; i++) + DPI_WRITE_SEGMENT(cmd, op->src_dst_seg, i, idx, eidx, tmp); + + /* Fill destination segments */ + for (i = 0; i < dst; i++) + DPI_WRITE_SEGMENT(cmd, (op->src_dst_seg + src), i, idx, eidx, tmp); + + cmd[0] = DPI_CMD_VLD_BIT | dpi_conf->cmd.u | (dst << 4) | src; + } + + if (space) { + rte_io_wmb(); + plt_write64(space, dpi_conf->dbell); + dpi_conf->stats.submitted += space; + } + + return j; +} + RTE_EXPORT_INTERNAL_SYMBOL(cn10k_dma_adapter_enqueue) uint16_t cn10k_dma_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events) @@ -496,7 +710,7 @@ cn10k_dma_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events) } if (op->flags & RTE_DMA_OP_FLAG_SUBMIT) { - rte_wmb(); + rte_io_wmb(); plt_write64(dpivf->total_pnum_words + CNXK_DPI_CMD_LEN(nb_src, nb_dst), dpivf->rdpi.rbase + DPI_VDMA_DBELL); dpivf->total_pnum_words = 0; @@ -567,7 +781,7 @@ cn9k_dma_adapter_dual_enqueue(void *ws, struct rte_event ev[], uint16_t nb_event } if (op->flags & RTE_DMA_OP_FLAG_SUBMIT) { - rte_wmb(); + rte_io_wmb(); plt_write64(dpivf->total_pnum_words + CNXK_DPI_CMD_LEN(nb_src, nb_dst), dpivf->rdpi.rbase + DPI_VDMA_DBELL); dpivf->total_pnum_words = 0; @@ -635,7 +849,7 @@ cn9k_dma_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events) } if (op->flags & RTE_DMA_OP_FLAG_SUBMIT) { - rte_wmb(); + rte_io_wmb(); plt_write64(dpivf->total_pnum_words + CNXK_DPI_CMD_LEN(nb_src, nb_dst), dpivf->rdpi.rbase + DPI_VDMA_DBELL); dpivf->total_pnum_words = 0; @@ -729,7 +943,7 @@ cnxk_dma_ops_enqueue(void *dev_private, uint16_t vchan, struct rte_dma_op **ops, done: if (nwords) { - rte_wmb(); + rte_io_wmb(); plt_write64(nwords, dpivf->rdpi.rbase + DPI_VDMA_DBELL); dpi_conf->stats.submitted += i; } @@ -778,7 +992,7 @@ cn10k_dma_ops_enqueue(void *dev_private, uint16_t vchan, struct rte_dma_op **ops done: if (nwords) { - rte_wmb(); + rte_io_wmb(); plt_write64(nwords, dpivf->rdpi.rbase + DPI_VDMA_DBELL); dpi_conf->stats.submitted += i; } -- 2.34.1
