From: Vamsi Attunuru <[email protected]> Adds DMA driver support for O20 hardware.
Signed-off-by: Vamsi Attunuru <[email protected]> --- drivers/dma/cnxk/cnxk_dmadev.c | 527 +++++++++++++++++++++++++++++- drivers/dma/cnxk/cnxk_dmadev.h | 72 +++- drivers/dma/cnxk/cnxk_dmadev_fp.c | 214 ++++++++++++ 3 files changed, 791 insertions(+), 22 deletions(-) diff --git a/drivers/dma/cnxk/cnxk_dmadev.c b/drivers/dma/cnxk/cnxk_dmadev.c index 6ae7fdca3b..3970abe1c3 100644 --- a/drivers/dma/cnxk/cnxk_dmadev.c +++ b/drivers/dma/cnxk/cnxk_dmadev.c @@ -2,6 +2,9 @@ * Copyright (C) 2021 Marvell International Ltd. */ +#include <errno.h> +#include <stdlib.h> + #include <rte_event_dma_adapter.h> #include <cnxk_dmadev.h> @@ -9,6 +12,68 @@ static int cnxk_stats_reset(struct rte_dma_dev *dev, uint16_t vchan); static void cnxk_set_fp_ops(struct rte_dma_dev *dev, uint8_t enable_enq_deq); +static int +parse_val_u16(const char *key, const char *value, void *extra_args) +{ + unsigned long val; + char *end; + + RTE_SET_USED(key); + + if (value == NULL || extra_args == NULL) + return -EINVAL; + + errno = 0; + val = strtoul(value, &end, 0); + if (errno != 0 || end == value || *end != '\0' || val > UINT16_MAX) + return -EINVAL; + + *(uint16_t *)extra_args = (uint16_t)val; + + return 0; +} + +static int +cn20k_dmadev_parse_devargs(struct rte_devargs *devargs, struct cnxk_dpi_vf_s *dpivf) +{ + uint16_t num_vchans = CN20K_DPI_DEF_VCHANS; + uint16_t num_lfs = num_vchans >> 1; /* Each LF has 2 rings */ + struct rte_kvargs *kvlist = NULL; + + if (devargs == NULL) { + dpivf->max_vchans = num_vchans; + dpivf->max_lfs = num_lfs; + return 0; + } + + kvlist = rte_kvargs_parse(devargs->args, NULL); + if (kvlist == NULL) + goto exit; + + if (rte_kvargs_process(kvlist, CN20K_DPI_NUM_VCHANS, &parse_val_u16, &num_vchans) < 0) + goto exit; + if (rte_kvargs_process(kvlist, CN20K_DPI_NUM_LFS, &parse_val_u16, &num_lfs) < 0) + goto exit; + + if (!num_vchans || !num_lfs || num_vchans > CN20K_DPI_MAX_VCHANS || + num_lfs > CN20K_DPI_MAX_LFS) + goto exit; + + if (!rte_is_power_of_2(num_vchans) || !rte_is_power_of_2(num_lfs)) { + plt_err("num_vchans or num_lfs is not a power of 2"); + goto exit; + } + + dpivf->max_vchans = num_vchans; + dpivf->max_lfs = num_lfs; + + rte_kvargs_free(kvlist); + return 0; +exit: + rte_kvargs_free(kvlist); + return -EINVAL; +} + static int cnxk_dmadev_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_info, uint32_t size) { @@ -25,9 +90,21 @@ cnxk_dmadev_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_inf dev_info->dev_capa |= RTE_DMA_CAPA_PRI_POLICY_SP; dev_info->nb_priorities = CN10K_DPI_MAX_PRI; } - dev_info->max_desc = CNXK_DPI_MAX_DESC; - dev_info->min_desc = CNXK_DPI_MIN_DESC; - dev_info->max_sges = CNXK_DPI_MAX_POINTER; + + if (roc_model_is_cn20k()) { + const uint16_t vpr = dpivf->vchans_per_ring ? dpivf->vchans_per_ring : 1; + + dev_info->max_desc = CN20K_DPI_MAX_DESC / vpr; + dev_info->min_desc = CN20K_DPI_MIN_DESC; + dev_info->max_sges = CN20K_DPI_MAX_POINTER; + dev_info->max_vchans = dpivf->max_vchans; + dev_info->dev_capa |= RTE_DMA_CAPA_OPS_FILL; + } else { + dev_info->max_desc = CNXK_DPI_MAX_DESC; + dev_info->min_desc = CNXK_DPI_MIN_DESC; + dev_info->max_sges = CNXK_DPI_MAX_POINTER; + dev_info->max_vchans = CNXK_DPI_MAX_VCHANS_PER_QUEUE; + } return 0; } @@ -52,8 +129,10 @@ cnxk_dmadev_vchan_free(struct cnxk_dpi_vf_s *dpivf, uint16_t vchan) for (; i < num_vchans; i++) { dpi_conf = &dpivf->conf[i]; - rte_free(dpi_conf->c_desc.compl_ptr); - dpi_conf->c_desc.compl_ptr = NULL; + if (dpi_conf->c_desc.compl_ptr) { + rte_free(dpi_conf->c_desc.compl_ptr); + dpi_conf->c_desc.compl_ptr = NULL; + } } return 0; @@ -99,10 +178,42 @@ cnxk_dmadev_chunk_pool_create(struct rte_dma_dev *dev, uint32_t nb_chunks, uint3 return rc; } +static int +cnxk_dmadev_vchan_rsrc_free(struct cnxk_dpi_vf_s *dpivf) +{ + struct roc_dpi *rdpi = &dpivf->rdpi; + int rc; + + if (dpivf->ring_conf) { + rte_free(dpivf->ring_conf); + dpivf->ring_conf = NULL; + } + + if (rdpi->lfs == NULL) + return 0; + + rc = roc_dpi_lf_chan_tbl_free(&(rdpi->lfs[0])); + if (rc < 0) { + plt_err("Failed to free dpi lf channel table"); + goto error; + } + + rc = roc_dpi_rsrc_fini(rdpi); + if (rc < 0) + plt_err("Failed to free dpi lfs"); + + dpivf->is_ring_conf_done = false; +error: + return rc; +} + static int cnxk_dmadev_configure(struct rte_dma_dev *dev, const struct rte_dma_conf *conf, uint32_t conf_sz) { struct cnxk_dpi_vf_s *dpivf = NULL; + struct roc_dpi_lf *lf; + uint16_t num_rings; + int rc = 0; RTE_SET_USED(conf_sz); dpivf = dev->fp_obj->dev_private; @@ -111,13 +222,78 @@ cnxk_dmadev_configure(struct rte_dma_dev *dev, const struct rte_dma_conf *conf, * Free up vchan memory if any, before configuring num_vchans. */ cnxk_dmadev_vchan_free(dpivf, RTE_DMA_ALL_VCHAN); + dpivf->num_vchans = conf->nb_vchans; + if (roc_model_is_cn20k()) { + if (!rte_is_power_of_2(dpivf->num_vchans)) + dpivf->num_vchans = rte_align32pow2(dpivf->num_vchans); + + if (dpivf->num_vchans > dpivf->max_vchans) { + plt_info("Limiting vchans from %u to max_vchans %u", + dpivf->num_vchans, dpivf->max_vchans); + dpivf->num_vchans = dpivf->max_vchans; + } + } if (roc_feature_dpi_has_priority()) dpivf->rdpi.priority = conf->priority; cnxk_set_fp_ops(dev, conf->flags & RTE_DMA_CFG_FLAG_ENQ_DEQ); - return 0; + if (roc_model_is_cn20k()) { + rc = cnxk_dmadev_vchan_rsrc_free(dpivf); + if (rc < 0) + goto error; + + dpivf->rdpi.nr_lfs = dpivf->max_lfs; + num_rings = dpivf->rdpi.nr_lfs << 1; + + if (dpivf->num_vchans > num_rings) { + dpivf->vchans_per_ring = dpivf->num_vchans / num_rings; + } else { + /* Each vchan has got it's own hardware ring */ + dpivf->vchans_per_ring = 1; + dpivf->rdpi.nr_lfs = dpivf->num_vchans >> 1; + if (!dpivf->rdpi.nr_lfs) + dpivf->rdpi.nr_lfs = 1; + } + + if ((CN20K_DPI_MAX_DESC / dpivf->vchans_per_ring) < CN20K_DPI_MIN_DESC) { + rc = -EINVAL; + plt_err("Very few LFs are requested than required"); + goto error; + } + + num_rings = dpivf->rdpi.nr_lfs << 1; + dpivf->ring_conf = rte_zmalloc("dpi_ring_conf", sizeof(struct cn20k_ring_conf) * + num_rings, 0); + if (dpivf->ring_conf == NULL) { + plt_err("Failed to allocate memory for ring conf"); + rc = -ENOMEM; + goto error; + } + + rc = roc_dpi_rsrc_init(&dpivf->rdpi); + if (rc < 0) { + plt_err("rsrc alloc failed"); + goto error; + } + + lf = &(dpivf->rdpi.lfs[0]); + rc = roc_dpi_lf_chan_tbl_alloc(lf, dpivf->num_vchans); + if (rc < 0) { + plt_err("Failed to allocate chan tbl"); + goto error; + } + + dpivf->chan_tbl = lf->chan_tbl; + dpivf->is_ring_conf_done = false; + } + +error: + if (roc_model_is_cn20k() && rc < 0) + cnxk_dmadev_vchan_rsrc_free(dpivf); + + return rc; } static int @@ -232,6 +408,87 @@ cn10k_dmadev_setup_hdr(union cnxk_dpi_instr_cmd *header, const struct rte_dma_vc return 0; } +static int +cn20k_dmadev_setup(struct cnxk_dpi_vf_s *dpivf, uint16_t vchan, + const struct rte_dma_vchan_conf *conf) +{ + struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + struct cn20k_ring_conf *rconf = dpivf->ring_conf; + uint16_t ridx, max_desc, num_rings; + union cnxk_dpi_instr_cmd *header; + int rc = 0; + int aura; + + header = (union cnxk_dpi_instr_cmd *)&dpi_conf->cmd.u; + header->cn20k.ct = DPI_HDR_PT_ZBW_CA; + header->cn20k.xt = 0; + + switch (conf->direction) { + case RTE_DMA_DIR_DEV_TO_MEM: + dpi_conf->cfg.xtype = DPI_XTYPE_INBOUND; + dpi_conf->cfg.rport = conf->src_port.pcie.coreid; + dpi_conf->cfg.wport = 0; + dpi_conf->chan_cfg.pf_func = conf->src_port.pcie.pfid; + dpi_conf->chan_cfg.vf_func = conf->src_port.pcie.vfid; + dpi_conf->chan_cfg.valid = 1; + break; + case RTE_DMA_DIR_MEM_TO_DEV: + dpi_conf->cfg.xtype = DPI_XTYPE_OUTBOUND; + dpi_conf->cfg.rport = 0; + dpi_conf->cfg.wport = conf->dst_port.pcie.coreid; + dpi_conf->chan_cfg.pf_func = conf->dst_port.pcie.pfid; + dpi_conf->chan_cfg.vf_func = conf->dst_port.pcie.vfid; + dpi_conf->chan_cfg.valid = 1; + aura = dmadev_src_buf_aura_get(conf->auto_free.m2d.pool, "cn20k_mempool_ops"); + if (aura < 0) + return aura; + header->cn20k.aura = aura; + break; + case RTE_DMA_DIR_MEM_TO_MEM: + dpi_conf->cfg.xtype = DPI_XTYPE_INTERNAL_ONLY; + dpi_conf->cfg.rport = 0; + dpi_conf->cfg.wport = 0; + break; + case RTE_DMA_DIR_DEV_TO_DEV: + dpi_conf->cfg.xtype = DPI_XTYPE_EXTERNAL_ONLY; + dpi_conf->cfg.rport = conf->src_port.pcie.coreid; + dpi_conf->cfg.wport = conf->dst_port.pcie.coreid; + }; + + max_desc = conf->nb_desc; + if (!rte_is_power_of_2(max_desc)) + max_desc = rte_align32pow2(max_desc); + + num_rings = dpivf->rdpi.nr_lfs << 1; + + for (ridx = 0; ridx < num_rings; ridx++) { + if (!rconf[ridx].used) { + if (!rconf[ridx].num_vchans) { + rconf[ridx].direction = conf->direction; + rconf[ridx].num_desc = max_desc; + } else if ((rconf[ridx].direction == conf->direction) && + (rconf[ridx].num_vchans < dpivf->vchans_per_ring)) { + rconf[ridx].num_desc += max_desc; + } else { + continue; + } + + rconf[ridx].num_vchans++; + dpi_conf->ridx = ridx; + + if (rconf[ridx].num_vchans == dpivf->vchans_per_ring) + rconf[ridx].used = true; + + break; + } + } + + if (ridx == num_rings) + rc = -ENODEV; + + return rc; +} + static int cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan, const struct rte_dma_vchan_conf *conf, uint32_t conf_sz) @@ -245,9 +502,14 @@ cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan, RTE_SET_USED(conf_sz); + if (conf->auto_free.m2d.pool != NULL && conf->direction != RTE_DMA_DIR_MEM_TO_DEV) + return -EINVAL; + header = (union cnxk_dpi_instr_cmd *)&dpi_conf->cmd.u; - if (dpivf->is_cn10k) + if (roc_model_is_cn20k()) + ret = cn20k_dmadev_setup(dpivf, vchan, conf); + else if (roc_model_is_cn10k()) ret = cn10k_dmadev_setup_hdr(header, conf); else ret = cn9k_dmadev_setup_hdr(header, conf); @@ -262,7 +524,7 @@ cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan, if (!rte_is_power_of_2(max_desc)) max_desc = rte_align32pow2(max_desc); - if (max_desc > CNXK_DPI_MAX_DESC) + if (!roc_model_is_cn20k() && (max_desc > CNXK_DPI_MAX_DESC)) max_desc = CNXK_DPI_MAX_DESC; size = (max_desc * sizeof(uint8_t) * CNXK_DPI_COMPL_OFFSET); @@ -285,10 +547,121 @@ cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan, dpi_conf->c_desc.compl_ptr[i * CNXK_DPI_COMPL_OFFSET] = CNXK_DPI_REQ_CDATA; dpi_conf->c_desc.max_cnt = (max_desc - 1); + dpi_conf->cfg_done = true; return 0; } +static int +cn20k_dmadev_queue_setup(struct cnxk_dpi_vf_s *dpivf) +{ + struct cn20k_ring_conf *rconf = dpivf->ring_conf; + struct roc_dpi *rdpi = &dpivf->rdpi; + struct cnxk_dpi_conf *dpi_conf; + struct roc_dpi_lf_que *que; + struct roc_dpi_lf_ring_cfg cfg; + struct roc_dpi_lf *lf; + uint16_t idx, qidx, ridx, vchan; + int rc = 0; + + for (idx = 0; idx < rdpi->nr_lfs; idx++) { + lf = &rdpi->lfs[idx]; + lf->chan_tbl = dpivf->chan_tbl; + + for (qidx = 0; qidx < 2; qidx++) { + ridx = (idx << 1) + qidx; + + if (!rconf[ridx].used) + continue; + + que = &lf->queue[qidx]; + + que->qsize = rconf[ridx].num_desc; + que->cmd_len = DPI_CMD_SIZE_128B; + que->first_skip = 0; + que->later_skip = 0; + memset(&cfg, 0, sizeof(cfg)); + cfg.ring_idx = qidx; + cfg.isize = que->cmd_len / DPI_CMD_SIZE_128B; + cfg.xtype = DPI_XTYPE_INTERNAL_ONLY; + cfg.rport = 0; /* Default is PEM:0 */ + cfg.wport = 0; /* Default is PEM:0 */ + cfg.pri = 0; /* 0 - High Priority */ + for (vchan = 0; vchan < dpivf->num_vchans; vchan++) { + dpi_conf = &dpivf->conf[vchan]; + + if (dpi_conf->ridx == ridx) + cfg.xtype = dpi_conf->cfg.xtype; + } + + rc = roc_dpi_lf_ring_init(que, &cfg); + if (rc) + return rc; + + for (vchan = 0; vchan < dpivf->num_vchans; vchan++) { + dpi_conf = &dpivf->conf[vchan]; + + if (dpi_conf->ridx == ridx) { + dpi_conf->que = que; + dpi_conf->dbell = lf->rbase + DPI_LF_RINGX_WIDX(qidx); + } + } + } + + rc = roc_dpi_lf_chan_tbl_select(lf); + if (rc) + return rc; + + /* FIX ME */ + rc = roc_dpi_lf_pffunc_cfg(lf); + if (rc) + return rc; + } + + return rc; +} + +static int +cn20k_dmadev_chan_tbl_setup(struct cnxk_dpi_vf_s *dpivf) +{ + uint16_t vchan, idx = 0, offset = 0; + struct cnxk_dpi_conf *dpi_conf; + uint64_t config[64] = {0}; + struct roc_dpi_lf *lf; + int rc = 0; + + lf = &(dpivf->rdpi.lfs[0]); + + for (vchan = 0; vchan < dpivf->num_vchans; vchan++) { + dpi_conf = &dpivf->conf[vchan]; + if (!dpi_conf->cfg_done || (dpi_conf->cfg.xtype == DPI_XTYPE_INTERNAL_ONLY)) + continue; + + dpi_conf->cmd.cn20k.chan = idx; + config[idx++] = dpi_conf->chan_cfg.u; + + if (idx == 64) { + rc = roc_dpi_lf_chan_tbl_update(lf, config, offset, idx); + if (rc < 0) { + plt_err("Failed to update chan tbl = %d", rc); + return rc; + } + offset += idx; + idx = 0; + } + } + + if (idx) { + rc = roc_dpi_lf_chan_tbl_update(lf, config, offset, idx); + if (rc < 0) { + plt_err("Failed to update chan tbl = %d", rc); + return rc; + } + } + + return rc; +} + static int cnxk_dmadev_start(struct rte_dma_dev *dev) { @@ -306,6 +679,10 @@ cnxk_dmadev_start(struct rte_dma_dev *dev) dpi_conf->c_desc.head = 0; dpi_conf->c_desc.tail = 0; dpi_conf->desc_idx = 0; + + if (dpi_conf->c_desc.compl_ptr == NULL) + continue; + for (j = 0; j < dpi_conf->c_desc.max_cnt + 1; j++) dpi_conf->c_desc.compl_ptr[j * CNXK_DPI_COMPL_OFFSET] = CNXK_DPI_REQ_CDATA; nb_desc += dpi_conf->c_desc.max_cnt + 1; @@ -313,6 +690,27 @@ cnxk_dmadev_start(struct rte_dma_dev *dev) dpi_conf->completed_offset = 0; } + if (roc_model_is_cn20k()) { + for (i = 0; i < (dpivf->rdpi.nr_lfs << 1); i++) + dpivf->ring_conf[i].pending = 0; + roc_dpi_reset(&dpivf->rdpi); + + if (dpivf->is_ring_conf_done) + goto enable_dpi; + + rc = cn20k_dmadev_queue_setup(dpivf); + if (rc) + goto error; + + rc = cn20k_dmadev_chan_tbl_setup(dpivf); + if (rc) + goto error; + + dpivf->is_ring_conf_done = true; + + goto enable_dpi; + } + queue_buf_sz = CNXK_DPI_QUEUE_BUF_SIZE_V2; /* Max block size allowed by cnxk mempool driver is (128 * 1024). * Block size = elt_size + mp->header + mp->trailer. @@ -349,6 +747,7 @@ cnxk_dmadev_start(struct rte_dma_dev *dev) dpivf->chunk_head = 0; dpivf->chunk_size_m1 = (queue_buf_sz >> 3) - 2; +enable_dpi: roc_dpi_enable(&dpivf->rdpi); error: return rc; @@ -359,14 +758,18 @@ cnxk_dmadev_stop(struct rte_dma_dev *dev) { struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private; - if (roc_dpi_wait_queue_idle(&dpivf->rdpi)) - return -EAGAIN; + if (!roc_model_is_cn20k()) { + if (roc_dpi_wait_queue_idle(&dpivf->rdpi)) + return -EAGAIN; + } roc_dpi_disable(&dpivf->rdpi); - rte_mempool_free(dpivf->chunk_pool); - dpivf->chunk_pool = NULL; - dpivf->chunk_base = NULL; - dpivf->chunk_size_m1 = 0; + if (!roc_model_is_cn20k()) { + rte_mempool_free(dpivf->chunk_pool); + dpivf->chunk_pool = NULL; + dpivf->chunk_base = NULL; + dpivf->chunk_size_m1 = 0; + } return 0; } @@ -378,6 +781,10 @@ cnxk_dmadev_close(struct rte_dma_dev *dev) roc_dpi_disable(&dpivf->rdpi); cnxk_dmadev_vchan_free(dpivf, RTE_DMA_ALL_VCHAN); + + if (roc_model_is_cn20k()) + cnxk_dmadev_vchan_rsrc_free(dpivf); + roc_dpi_dev_fini(&dpivf->rdpi); /* Clear all flags as we close the device. */ @@ -454,11 +861,17 @@ cnxk_damdev_burst_capacity(const void *dev_private, uint16_t vchan) { const struct cnxk_dpi_vf_s *dpivf = (const struct cnxk_dpi_vf_s *)dev_private; const struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + struct cn20k_ring_conf *ring_conf; uint16_t burst_cap; burst_cap = dpi_conf->c_desc.max_cnt - (dpi_conf->stats.submitted - dpi_conf->stats.completed) + 1; + if (roc_model_is_cn20k()) { + ring_conf = &dpivf->ring_conf[dpi_conf->ridx]; + burst_cap -= ring_conf->pending; + } + return burst_cap; } @@ -480,6 +893,26 @@ cnxk_dmadev_submit(void *dev_private, uint16_t vchan) return 0; } +static int +cn20k_dmadev_submit(void *dev_private, uint16_t vchan) +{ + struct cnxk_dpi_vf_s *dpivf = dev_private; + struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + struct cn20k_ring_conf *ring_conf = &dpivf->ring_conf[dpi_conf->ridx]; + uint16_t num_words = ring_conf->pending; + + if (!num_words) + return 0; + + rte_wmb(); + plt_write64(num_words, dpi_conf->dbell); + dpi_conf->stats.submitted += num_words; + + ring_conf->pending = 0; + + return 0; +} + static int cnxk_stats_get(const struct rte_dma_dev *dev, uint16_t vchan, struct rte_dma_stats *rte_stats, uint32_t size) @@ -552,11 +985,16 @@ cnxk_set_fp_ops(struct rte_dma_dev *dev, uint8_t ena_enq_deq) dev->fp_obj->submit = cnxk_dmadev_submit; dev->fp_obj->completed = cnxk_dmadev_completed; dev->fp_obj->completed_status = cnxk_dmadev_completed_status; - dev->fp_obj->burst_capacity = cnxk_damdev_burst_capacity; + dev->fp_obj->burst_capacity = cnxk_damdev_burst_capacity; if (roc_model_is_cn10k()) { dev->fp_obj->copy = cn10k_dmadev_copy; dev->fp_obj->copy_sg = cn10k_dmadev_copy_sg; + } else if (roc_model_is_cn20k()) { + dev->fp_obj->submit = cn20k_dmadev_submit; + dev->fp_obj->copy = cn20k_dmadev_copy; + dev->fp_obj->copy_sg = cn20k_dmadev_copy_sg; + dev->fp_obj->fill = cn20k_dmadev_fill; } if (ena_enq_deq) { @@ -571,6 +1009,9 @@ cnxk_set_fp_ops(struct rte_dma_dev *dev, uint8_t ena_enq_deq) if (roc_model_is_cn10k()) dev->fp_obj->enqueue = cn10k_dma_ops_enqueue; + + if (roc_model_is_cn20k()) + dev->fp_obj->enqueue = cn20k_dma_ops_enqueue; } } @@ -594,8 +1035,10 @@ cnxk_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_de struct roc_dpi *rdpi = NULL; int rc; - if (!pci_dev->mem_resource[0].addr) - return -ENODEV; + if (!roc_model_is_cn20k()) { + if (!pci_dev->mem_resource[0].addr) + return -ENODEV; + } rc = roc_plt_init(); if (rc) { @@ -612,19 +1055,65 @@ cnxk_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_de } dpivf = dmadev->data->dev_private; + dmadev->device = &pci_dev->device; + + if (roc_model_is_cn20k()) { + /* Parse devargs string */ + rc = cn20k_dmadev_parse_devargs(dmadev->device->devargs, dpivf); + if (rc) { + plt_err("Failed to parse devargs rc=%d", rc); + goto err_out_free; + } + dpivf->num_vchans = dpivf->max_vchans; + } dmadev->device = &pci_dev->device; dmadev->fp_obj->dev_private = dpivf; dmadev->dev_ops = &cnxk_dmadev_ops; + /* + * dev_private (and the roc_dpi it holds) lives in shared memory. Only + * the primary owns the HW and the per-process pci_dev pointer; a + * secondary must not run roc_dpi_dev_init nor overwrite rdpi->pci_dev, + * otherwise the primary later dereferences the secondary's stale + * pointer during rte_eal_cleanup() and crashes. + */ if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; - dpivf->is_cn10k = roc_model_is_cn10k(); + dmadev->fp_obj->submit = cnxk_dmadev_submit; + dmadev->fp_obj->completed = cnxk_dmadev_completed; + dmadev->fp_obj->completed_status = cnxk_dmadev_completed_status; + dmadev->fp_obj->burst_capacity = cnxk_damdev_burst_capacity; + + if (roc_model_is_cn10k()) { + dmadev->fp_obj->copy = cn10k_dmadev_copy; + dmadev->fp_obj->copy_sg = cn10k_dmadev_copy_sg; + dpivf->num_vchans = CNXK_DPI_MAX_VCHANS_PER_QUEUE; + } else if (roc_model_is_cn20k()) { + dmadev->fp_obj->submit = cn20k_dmadev_submit; + dmadev->fp_obj->copy = cn20k_dmadev_copy; + dmadev->fp_obj->copy_sg = cn20k_dmadev_copy_sg; + dmadev->fp_obj->fill = cn20k_dmadev_fill; + } else { + dmadev->fp_obj->copy = cnxk_dmadev_copy; + dmadev->fp_obj->copy_sg = cnxk_dmadev_copy_sg; + dpivf->num_vchans = CNXK_DPI_MAX_VCHANS_PER_QUEUE; + } + + dpivf->conf = rte_zmalloc("dpi_vchan_conf", dpivf->num_vchans * + sizeof(struct cnxk_dpi_conf), 0); + if (dpivf->conf == NULL) { + plt_err("Failed to allocate memory for vchan conf"); + rc = -ENOMEM; + goto err_out_free; + } + dpivf->mcs_lock = NULL; rdpi = &dpivf->rdpi; rdpi->pci_dev = pci_dev; + rc = roc_dpi_dev_init(rdpi, offsetof(struct rte_dma_op, impl_opaque)); if (rc < 0) goto err_out_free; @@ -653,6 +1142,8 @@ cnxk_dmadev_remove(struct rte_pci_device *pci_dev) static const struct rte_pci_id cnxk_dma_pci_map[] = { {RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CNXK_DPI_VF)}, + {RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN20K_DPI_PF)}, + {RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN20K_DPI_VF)}, { .vendor_id = 0, }, diff --git a/drivers/dma/cnxk/cnxk_dmadev.h b/drivers/dma/cnxk/cnxk_dmadev.h index 18039e43fb..3ba404dda8 100644 --- a/drivers/dma/cnxk/cnxk_dmadev.h +++ b/drivers/dma/cnxk/cnxk_dmadev.h @@ -9,9 +9,11 @@ #include <bus_pci_driver.h> #include <rte_common.h> +#include <rte_devargs.h> #include <rte_dmadev.h> #include <rte_dmadev_pmd.h> #include <rte_eal.h> +#include <rte_kvargs.h> #include <rte_lcore.h> #include <rte_mbuf_pool_ops.h> #include <rte_mcslock.h> @@ -31,15 +33,31 @@ #define CN10K_DPI_MAX_PRI 2 #define CNXK_DPI_MAX_VCHANS_PER_QUEUE 128 #define CNXK_DPI_QUEUE_BUF_SIZE 16256 -#define CNXK_DPI_QUEUE_BUF_SIZE_V2 130944 +/* Maximum pool size supported by device is 128 * 1024. When RTE_LIBRTE_MEMPOOL_DEBUG is enabled + * mempool->trailer size will be increased by 8B. Additionally if the pool is not created with + * RTE_MEMPOOL_F_NO_CACHE_ALIGN, trailer will be expanded to cache line size. + * To allow future needs, limit the max size to 127KB + */ +#define CNXK_DPI_QUEUE_BUF_SIZE_V2 130048 #define CNXK_DPI_POOL_MAX_CACHE_SZ (16) #define CNXK_DPI_DW_PER_SINGLE_CMD 8 #define CNXK_DPI_HDR_LEN 4 #define CNXK_DPI_CMD_LEN(src, dst) (CNXK_DPI_HDR_LEN + ((src) << 1) + ((dst) << 1)) -#define CNXK_DPI_MAX_CMD_SZ CNXK_DPI_CMD_LEN(CNXK_DPI_MAX_POINTER, \ +#define CNXK_DPI_MAX_CMD_SZ CNXK_DPI_CMD_LEN(CNXK_DPI_MAX_POINTER, \ CNXK_DPI_MAX_POINTER) #define CNXK_DPI_CHUNKS_FROM_DESC(cz, desc) (((desc) / (((cz) / 8) / CNXK_DPI_MAX_CMD_SZ)) + 1) #define CNXK_DPI_COMPL_OFFSET ROC_CACHE_LINE_SZ + +#define CN20K_DPI_MAX_POINTER 4 +#define CN20K_DPI_MAX_DESC 2048 +#define CN20K_DPI_MIN_DESC 128 +#define CN20K_DPI_MAX_VCHANS 512 +#define CN20K_DPI_DEF_VCHANS 8 +#define CN20K_DPI_MAX_LFS 256 + +#define CN20K_DPI_NUM_VCHANS "num_vchans" +#define CN20K_DPI_NUM_LFS "num_lfs" + /* Set Completion data to 0xFF when request submitted, * upon successful request completion engine reset to completion status */ @@ -86,6 +104,32 @@ union cnxk_dpi_instr_cmd { uint64_t reserved_62_63 : 2; /* Word 0 - End */ } cn10k; + + struct cn20k_dpi_instr_cmd { + uint64_t nfst : 3; + uint64_t reserved_3 : 1; + uint64_t nlst : 3; + uint64_t reserved_7 : 1; + uint64_t msix_int : 1; + uint64_t ct : 3; + uint64_t chan : 14; + uint64_t reserved_26_29 : 4; + uint64_t aura : 20; + uint64_t xt : 2; + uint64_t ivec : 9; + uint64_t fe : 1; + uint64_t reserved_62 : 1; + uint64_t vld : 1; + /* Word 0 - End */ + } cn20k; +}; + +struct cn20k_ring_conf { + enum rte_dma_direction direction; + uint16_t pending; + uint16_t num_desc; + uint8_t num_vchans; + bool used; }; struct cnxk_dpi_cdesc_data_s { @@ -100,9 +144,15 @@ struct cnxk_dpi_conf { union cnxk_dpi_instr_cmd cmd; struct cnxk_dpi_cdesc_data_s c_desc; uint16_t desc_idx; + uintptr_t dbell; struct rte_dma_stats stats; uint64_t completed_offset; + struct roc_dpi_lf_que *que; + union roc_dpi_lf_ccfg chan_cfg; + struct roc_dpi_lf_ring_cfg cfg; + uint16_t ridx; bool adapter_enabled; + bool cfg_done; }; struct cnxk_dpi_vf_s { @@ -111,15 +161,21 @@ struct cnxk_dpi_vf_s { uint16_t chunk_head; uint16_t chunk_size_m1; uint16_t total_pnum_words; + uint16_t vchans_per_ring; struct rte_mempool *chunk_pool; - struct cnxk_dpi_conf conf[CNXK_DPI_MAX_VCHANS_PER_QUEUE]; + struct cnxk_dpi_conf *conf; + struct cn20k_ring_conf *ring_conf; RTE_ATOMIC(rte_mcslock_t *) mcs_lock; /* Slow path */ struct roc_dpi rdpi; uint32_t aura; + uint16_t max_lfs; + uint16_t max_vchans; uint16_t num_vchans; + uint16_t chan_tbl; uint16_t flag; uint8_t is_cn10k; + uint8_t is_ring_conf_done; } __plt_cache_aligned; int cnxk_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst, @@ -138,5 +194,13 @@ uint16_t cn10k_dma_ops_enqueue(void *dev_private, uint16_t vchan, struct rte_dma uint16_t nb_ops); uint16_t cnxk_dma_ops_dequeue(void *dev_private, uint16_t vchan, struct rte_dma_op **ops, uint16_t nb_ops); - +int cn20k_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst, + uint32_t length, uint64_t flags); +int cn20k_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge *src, + const struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst, + uint64_t flags); +int cn20k_dmadev_fill(void *dev_private, uint16_t vchan, uint64_t pattern, rte_iova_t dst, + uint32_t length, uint64_t flags); +uint16_t cn20k_dma_ops_enqueue(void *dev_private, uint16_t vchan, struct rte_dma_op **ops, + uint16_t nb_ops); #endif diff --git a/drivers/dma/cnxk/cnxk_dmadev_fp.c b/drivers/dma/cnxk/cnxk_dmadev_fp.c index 4435adc38b..29ce13c8e4 100644 --- a/drivers/dma/cnxk/cnxk_dmadev_fp.c +++ b/drivers/dma/cnxk/cnxk_dmadev_fp.c @@ -440,6 +440,163 @@ cn10k_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge return dpi_conf->desc_idx++; } +int +cn20k_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst, + uint32_t length, uint64_t flags) +{ + struct cnxk_dpi_vf_s *dpivf = dev_private; + struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + const uint16_t max_cnt = dpi_conf->c_desc.max_cnt; + struct roc_dpi_lf_que *queue = dpi_conf->que; + struct cn20k_ring_conf *ring_conf; + uint8_t *comp_ptr; + uint64_t *cmd; + + if (unlikely(((dpi_conf->c_desc.tail + 1) & max_cnt) == (dpi_conf->c_desc.head & max_cnt))) + return -ENOSPC; + + if (dpivf->vchans_per_ring == 1) { + cmd = queue->cmd_base + ((dpi_conf->c_desc.tail & max_cnt) << 4); + } else { + cmd = queue->cmd_base + (queue->widx << 4); + queue->widx = (queue->widx + 1) & (queue->qsize - 1); + } + + ring_conf = &(dpivf->ring_conf[dpi_conf->ridx]); + comp_ptr = &dpi_conf->c_desc + .compl_ptr[(dpi_conf->c_desc.tail & max_cnt) * CNXK_DPI_COMPL_OFFSET]; + dpi_conf->c_desc.tail++; + + cmd[1] = (uint64_t)comp_ptr; + cmd[4] = ((uint64_t)length << 32) | length | ((flags & RTE_DMA_OP_FLAG_AUTO_FREE) << 28); + cmd[5] = src; + cmd[6] = dst; + cmd[0] = DPI_CMD_VLD_BIT | dpi_conf->cmd.u | 0x11U; + + if (flags & RTE_DMA_OP_FLAG_SUBMIT) { + rte_wmb(); + plt_write64(ring_conf->pending + 1, dpi_conf->dbell); + dpi_conf->stats.submitted += (ring_conf->pending + 1); + ring_conf->pending = 0; + } else { + ring_conf->pending++; + } + + return dpi_conf->desc_idx++; +} + +/* Helper macro to write length and address */ +#define DPI_WRITE_SEGMENT(ptr, seg, i, idx, eidx, tmp) \ + do { \ + if ((tmp) % 2 == 0) { \ + ptr[eidx] = seg[i].length; \ + idx++; \ + } else { \ + ptr[eidx] |= ((uint64_t)seg[i].length << 32); \ + eidx += 3; \ + } \ + ptr[idx++] = (uint64_t)seg[i].addr; \ + tmp++; \ + } while (0) + +int +cn20k_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge *src, + const struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst, + uint64_t flags) +{ + struct cnxk_dpi_vf_s *dpivf = dev_private; + struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + const uint16_t max_cnt = dpi_conf->c_desc.max_cnt; + struct roc_dpi_lf_que *queue = dpi_conf->que; + struct cn20k_ring_conf *ring_conf = &dpivf->ring_conf[dpi_conf->ridx]; + uint16_t idx = 4, eidx = 4, tmp = 0; + uint8_t *comp_ptr, i; + uint64_t *cmd; + + if (unlikely(((dpi_conf->c_desc.tail + 1) & max_cnt) == (dpi_conf->c_desc.head & max_cnt))) + return -ENOSPC; + + if (dpivf->vchans_per_ring == 1) { + cmd = queue->cmd_base + ((dpi_conf->c_desc.tail & max_cnt) << 4); + } else { + cmd = queue->cmd_base + (queue->widx << 4); + queue->widx = (queue->widx + 1) & (queue->qsize - 1); + } + comp_ptr = &dpi_conf->c_desc + .compl_ptr[(dpi_conf->c_desc.tail & max_cnt) * CNXK_DPI_COMPL_OFFSET]; + dpi_conf->c_desc.tail++; + + cmd[1] = (uint64_t)comp_ptr; + + /* Fill source segments */ + for (i = 0; i < nb_src; i++) + DPI_WRITE_SEGMENT(cmd, src, i, idx, eidx, tmp); + + /* Fill destination segments */ + for (i = 0; i < nb_dst; i++) + DPI_WRITE_SEGMENT(cmd, dst, i, idx, eidx, tmp); + + cmd[0] = DPI_CMD_VLD_BIT | dpi_conf->cmd.u | (nb_dst << 4) | nb_src; + + if (flags & RTE_DMA_OP_FLAG_SUBMIT) { + rte_wmb(); + plt_write64(ring_conf->pending + 1, dpi_conf->dbell); + dpi_conf->stats.submitted += ring_conf->pending + 1; + ring_conf->pending = 0; + } else { + ring_conf->pending++; + } + + return dpi_conf->desc_idx++; +} + +int +cn20k_dmadev_fill(void *dev_private, uint16_t vchan, uint64_t pattern, rte_iova_t dst, + uint32_t length, uint64_t flags) +{ + struct cnxk_dpi_vf_s *dpivf = dev_private; + struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + const uint16_t max_cnt = dpi_conf->c_desc.max_cnt; + struct roc_dpi_lf_que *queue = dpi_conf->que; + struct cn20k_ring_conf *ring_conf; + uint8_t *comp_ptr; + uint64_t *cmd; + +#define DPI_XT_TYPE_FILL BIT_ULL(50) + + if (unlikely(((dpi_conf->c_desc.tail + 1) & max_cnt) == (dpi_conf->c_desc.head & max_cnt))) + return -ENOSPC; + + if (dpivf->vchans_per_ring == 1) { + cmd = queue->cmd_base + ((dpi_conf->c_desc.tail & max_cnt) << 4); + } else { + cmd = queue->cmd_base + (queue->widx << 4); + queue->widx = (queue->widx + 1) & (queue->qsize - 1); + } + + ring_conf = &(dpivf->ring_conf[dpi_conf->ridx]); + comp_ptr = &dpi_conf->c_desc + .compl_ptr[(dpi_conf->c_desc.tail & max_cnt) * CNXK_DPI_COMPL_OFFSET]; + dpi_conf->c_desc.tail++; + + cmd[1] = (uint64_t)comp_ptr; + cmd[4] = (uint64_t)length << 32; + cmd[5] = pattern; + cmd[6] = dst; + cmd[0] = DPI_CMD_VLD_BIT | DPI_XT_TYPE_FILL | dpi_conf->cmd.u | 0x10U; + + if (flags & RTE_DMA_OP_FLAG_SUBMIT) { + rte_wmb(); + plt_write64(ring_conf->pending + 1, dpi_conf->dbell); + dpi_conf->stats.submitted += (ring_conf->pending + 1); + ring_conf->pending = 0; + } else { + ring_conf->pending++; + } + + return dpi_conf->desc_idx++; +} + static inline uint64_t cnxk_dma_adapter_format_event(uint64_t event) { @@ -450,6 +607,63 @@ cnxk_dma_adapter_format_event(uint64_t event) return w0; } +uint16_t +cn20k_dma_ops_enqueue(void *dev_private, uint16_t vchan, struct rte_dma_op **ops, uint16_t nb_ops) +{ + struct cnxk_dpi_vf_s *dpivf = dev_private; + struct cnxk_dpi_conf *dpi_conf = &dpivf->conf[vchan]; + const uint16_t max_cnt = dpi_conf->c_desc.max_cnt; + struct roc_dpi_lf_que *queue = dpi_conf->que; + uint16_t idx, eidx, tmp; + struct rte_dma_op *op; + uint16_t space, i, j; + uint16_t src, dst; + uint8_t *comp_ptr; + uint64_t *cmd; + + space = (max_cnt + dpi_conf->c_desc.head - dpi_conf->c_desc.tail) & max_cnt; + space = RTE_MIN(space, nb_ops); + + for (j = 0; j < space; j++) { + op = ops[j]; + src = op->nb_src; + dst = op->nb_dst; + + idx = 4; eidx = 4; tmp = 0; + + if (dpivf->vchans_per_ring == 1) { + cmd = queue->cmd_base + ((dpi_conf->c_desc.tail & max_cnt) << 4); + } else { + cmd = queue->cmd_base + (queue->widx << 4); + queue->widx = (queue->widx + 1) & (queue->qsize - 1); + } + comp_ptr = &dpi_conf->c_desc.compl_ptr[(dpi_conf->c_desc.tail & max_cnt) * + CNXK_DPI_COMPL_OFFSET]; + dpi_conf->c_desc.ops[dpi_conf->c_desc.tail & max_cnt] = op; + dpi_conf->c_desc.tail++; + + cmd[1] = (uint64_t)comp_ptr; + + /* Fill source segments */ + for (i = 0; i < src; i++) + DPI_WRITE_SEGMENT(cmd, op->src_dst_seg, i, idx, eidx, tmp); + + /* Fill destination segments */ + for (i = 0; i < dst; i++) + DPI_WRITE_SEGMENT(cmd, (op->src_dst_seg + src), i, idx, eidx, tmp); + + cmd[0] = DPI_CMD_VLD_BIT | dpi_conf->cmd.u | (dst << 4) | src; + } + + if (space) { + rte_wmb(); + plt_write64(space, dpi_conf->dbell); + dpi_conf->stats.submitted += space; + } + + return j; +} + RTE_EXPORT_INTERNAL_SYMBOL(cn10k_dma_adapter_enqueue) uint16_t cn10k_dma_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events) -- 2.34.1
