On 8/12/2026 5:29 PM, Gagandeep Singh wrote:
> Add the memory-to-memory data path for the eDMA5 dmadev.
>
> The eDMA5 exposes a single transfer control descriptor and a single
> completion flag per hardware channel, so jobs are serialised in software.
> Each enqueued job is played out synchronously as one or more single-block
> transfers: the per-channel TCD is programmed, the transfer is
> software-started and the driver busy-waits for the DONE flag with a
> wall-clock bounded timeout. Scatter-gather copies walk the source and
> destination segment lists as two cursors, emitting one single-block
> sub-transfer per iteration that fits both current segments.
>
> As the eDMA5 is a non-coherent bus master, source and destination buffers
> are cleaned from the CPU cache before a transfer and the destination is
> invalidated after completion so the application observes the DMA result.
>
> This adds copy, copy_sg, submit, completed, completed_status and
> burst_capacity, wired through the device fast-path object.
>
> Signed-off-by: Gagandeep Singh <[email protected]>
> Signed-off-by: Prashant Gupta <[email protected]>
> ---
> doc/guides/dmadevs/imx_edma5.rst | 9 +
> drivers/dma/imx_edma5/imx_edma5_dmadev.c | 599 +++++++++++++++++++++++
> drivers/dma/imx_edma5/imx_edma5_dmadev.h | 14 +
> 3 files changed, 622 insertions(+)
>
> diff --git a/doc/guides/dmadevs/imx_edma5.rst
> b/doc/guides/dmadevs/imx_edma5.rst
> index f57881816e..de7c49101e 100644
> --- a/doc/guides/dmadevs/imx_edma5.rst
> +++ b/doc/guides/dmadevs/imx_edma5.rst
> @@ -58,3 +58,12 @@ Limitations
> independently and programs one hardware transfer per consumed segment;
> full TCD scatter-gather linking is not yet implemented.
> - The driver operates in poll mode only; completion interrupts are not used.
> +- Each operation executes synchronously: the driver programs the TCD,
> + starts the channel and busy-waits for completion inside the enqueue or
> + submit call. No transfer-offload benefit over memcpy is provided by this
> + first revision; the synchronous model is documented rather than implied.
> +- Cache maintenance (source clean, destination clean+invalidate) is skipped
> + for any address that cannot be resolved to a CPU virtual address via
> + ``rte_mem_iova2virt()``. This affects externally-allocated memory not
> + registered with DPDK. Applications using such memory must ensure cache
> + coherency independently or register the memory with DPDK.
> diff --git a/drivers/dma/imx_edma5/imx_edma5_dmadev.c
> b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
> index fe5539b612..82eb1c87cb 100644
> --- a/drivers/dma/imx_edma5/imx_edma5_dmadev.c
> +++ b/drivers/dma/imx_edma5/imx_edma5_dmadev.c
> @@ -337,6 +337,598 @@ imx_edma5_close(struct rte_dma_dev *dev)
> return 0;
> }
>
> +/*
> + * Encode the largest natural transfer size (SSIZE/DSIZE) usable for a given
> + * source, destination and length. The address must be aligned to the
> transfer
> + * size and the byte count must be a multiple of it.
> + */
> +static uint16_t
> +imx_edma5_calc_attr(uint64_t src, uint64_t dst, uint32_t len)
> +{
> + uint32_t sz = IMX_EDMA5_TCD_SIZE_1B;
> +
> + if (((src | dst | len) & 0x1F) == 0)
> + sz = IMX_EDMA5_TCD_SIZE_32B;
> + else if (((src | dst | len) & 0xF) == 0)
> + sz = IMX_EDMA5_TCD_SIZE_16B;
> + else if (((src | dst | len) & 0x7) == 0)
> + sz = IMX_EDMA5_TCD_SIZE_8B;
> + else if (((src | dst | len) & 0x3) == 0)
> + sz = IMX_EDMA5_TCD_SIZE_4B;
> + else if (((src | dst | len) & 0x1) == 0)
> + sz = IMX_EDMA5_TCD_SIZE_2B;
> +
> + return IMX_EDMA5_TCD_ATTR_SSIZE(sz) | IMX_EDMA5_TCD_ATTR_DSIZE(sz);
> +}
> +
> +/*
> + * Program the channel TCD for a single-block copy: one minor loop of "len"
> + * bytes with a major count of 1. Completion is polled via CH_CSR.DONE.
> + */
> +static inline void
> +imx_edma5_program_copy(struct imx_edma5_vchan *vc, uint64_t src, uint64_t
> dst,
> + uint32_t len)
> +{
> + uint8_t *tcd = vc->tcd_regs;
> + uint16_t attr = imx_edma5_calc_attr(src, dst, len);
> +
> + imx_edma5_write64(tcd, IMX_EDMA5_TCD_SADDR, src);
> + imx_edma5_write64(tcd, IMX_EDMA5_TCD_DADDR, dst);
> + imx_edma5_write16(tcd, IMX_EDMA5_TCD_ATTR, attr);
> + imx_edma5_write16(tcd, IMX_EDMA5_TCD_SOFF,
> + (uint16_t)(1u << IMX_EDMA5_TCD_ATTR_GET_SSIZE(attr)));
> + imx_edma5_write16(tcd, IMX_EDMA5_TCD_DOFF,
> + (uint16_t)(1u << IMX_EDMA5_TCD_ATTR_GET_DSIZE(attr)));
> + imx_edma5_write32(tcd, IMX_EDMA5_TCD_NBYTES, len);
> + imx_edma5_write64(tcd, IMX_EDMA5_TCD_SLAST, 0);
> + imx_edma5_write64(tcd, IMX_EDMA5_TCD_DLAST_SGA, 0);
> + imx_edma5_write16(tcd, IMX_EDMA5_TCD_CITER, 1);
> + imx_edma5_write16(tcd, IMX_EDMA5_TCD_BITER, 1);
> + /* Clear CSR so no scatter-gather link is followed. */
> + imx_edma5_write16(tcd, IMX_EDMA5_TCD_CSR, 0);
> +}
> +
> +/* Fill an in-memory hardware TCD64 descriptor for one copy segment. */
> +static inline void
> +imx_edma5_fill_tcd(struct imx_edma5_hw_tcd64 *t, uint64_t src, uint64_t dst,
> + uint32_t len)
> +{
> + uint16_t attr = imx_edma5_calc_attr(src, dst, len);
> + uint16_t soff = (uint16_t)(1u << IMX_EDMA5_TCD_ATTR_GET_SSIZE(attr));
> + uint16_t doff = (uint16_t)(1u << IMX_EDMA5_TCD_ATTR_GET_DSIZE(attr));
> +
> + t->saddr = rte_cpu_to_le_64(src);
> + t->soff = rte_cpu_to_le_16(soff);
> + t->attr = rte_cpu_to_le_16(attr);
> + t->nbytes = rte_cpu_to_le_32(len);
> + t->slast = 0;
> + t->daddr = rte_cpu_to_le_64(dst);
> + t->dlast_sga = 0;
> + t->doff = rte_cpu_to_le_16(doff);
> + t->citer = rte_cpu_to_le_16(1);
> + t->csr = 0;
> + t->biter = rte_cpu_to_le_16(1);
> +}
> +
> +static inline void
> +imx_edma5_hw_start(struct imx_edma5_vchan *vc)
> +{
> + uint16_t csr = imx_edma5_read16(vc->tcd_regs, IMX_EDMA5_TCD_CSR);
> +
> + csr |= IMX_EDMA5_TCD_CSR_START;
> + imx_edma5_write16(vc->tcd_regs, IMX_EDMA5_TCD_CSR, csr);
> +}
> +
> +/*
> + * Invalidate the CPU cache lines covering a completed job's destination(s)
> so
> + * the application reads the DMA result rather than stale cache. The lines
> were
> + * cleaned at enqueue time, so this clean+invalidate behaves as a pure
> + * invalidate.
> + */
> +static inline void
> +imx_edma5_job_invalidate_dst(struct imx_edma5_job *job)
> +{
> + if (job->nb_sg > 0) {
> + uint16_t s;
> +
> + for (s = 0; s < job->nb_sg; s++) {
> + rte_iova_t da = rte_le_to_cpu_64(job->sg_tcd[s].daddr);
> + uint32_t len = rte_le_to_cpu_32(job->sg_tcd[s].nbytes);
> + void *va = imx_edma5_iova_to_virt(da);
> +
> + if (va != NULL)
> + imx_edma5_cache_inval(va, len);
> + }
> + } else if (job->dst_va != NULL) {
> + imx_edma5_cache_inval(job->dst_va, job->len);
> + }
> +}
> +
> +/*
> + * Upper bound on how long to poll for an entire job to complete.
> + * A wall-clock deadline is computed once per job in imx_edma5_run_job()
> + * and passed to every sub-transfer wait, so the bound covers the whole job
> + * rather than each sub-transfer individually.
> + * Each single-block transfer is capped at IMX_EDMA5_MAX_NBYTES (1 GiB - 1);
> + * at an AXI bus rate of 2 GB/s that is 512 ms in the worst case, so
> + * 1000 ms is a safe ceiling while still bounding a wedged channel.
> + */
> +#define IMX_EDMA5_WAIT_TIMEOUT_MS 1000
> +
> +/*
> + * Wait for the single register TCD transfer to finish and clear its latched
> + * status. Returns true on success, false on a logged channel error
> (CH_ES.ERR)
> + * or timeout. The per-transfer completion flag CH_CSR.DONE and the CH_ES
> error
> + * bit are both write-1-to-clear.
> + */
> +static inline bool
> +imx_edma5_wait_done(struct imx_edma5_vchan *vc, uint64_t deadline)
> +{
> + do {
> + uint32_t ch_es = imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_ES);
> + uint32_t ch_csr;
> +
> + if (ch_es & IMX_EDMA5_CH_ES_ERR) {
> + imx_edma5_write32(vc->ch_regs, IMX_EDMA5_CH_ES,
> + IMX_EDMA5_CH_ES_ERR);
> + /* Reset the errored channel before the next job reuses
> it. */
> + imx_edma5_reset_hw_chan(vc);
> + return false;
> + }
> +
> + ch_csr = imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_CSR);
> + if (ch_csr & IMX_EDMA5_CH_CSR_DONE) {
> + imx_edma5_write32(vc->ch_regs, IMX_EDMA5_CH_CSR,
> + IMX_EDMA5_CH_CSR_DONE);
> + return true;
> + }
> + } while (rte_get_timer_cycles() < deadline);
> +
> + IMX_EDMA5_LOG(ERR,
> + "channel %u timed out waiting for DONE (CH_CSR=0x%08x "
> + "CH_ES=0x%08x)",
> + vc->hw_chan,
> + imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_CSR),
> + imx_edma5_read32(vc->ch_regs, IMX_EDMA5_CH_ES));
> +
> + /*
> + * Reset the channel registers to a known idle state. Note that this
> does
> + * not cancel an in-flight DMA transfer: the eDMA5 has no software-
> + * accessible abort/cancel bit (MP_CSR.CX does not exist on this
> + * variant). A channel that is genuinely stuck continues its bus
> + * transaction; reprogramming it here races with the active transfer.
> + */
> + imx_edma5_reset_hw_chan(vc);
> + return false;
> +}
> +
> +/*
> + * Execute one job to completion on the channel's single register TCD.
> + *
> + * The eDMA5 exposes a single TCD and a single completion flag per channel,
> so
> + * jobs are serialised in software: a job is run synchronously here (program
> the
> + * TCD, software-start, busy-wait for DONE) and its completion recorded in
> + * job->done for the completion API to reap. Scatter-gather segments are
> played
> + * out one at a time as single-block transfers, since the eDMA5 does not
> + * auto-advance a hardware TCD chain for software-started mem-to-mem
> transfers.
> + * On success the destination cache lines are invalidated (non-coherent
> master).
> + */
> +static inline void
> +imx_edma5_run_job(struct imx_edma5_vchan *vc, struct imx_edma5_job *job)
> +{
> + bool ok = true;
> + /*
> + * Compute a single deadline for the entire job before entering the
> + * sub-transfer loop. Passing it to every imx_edma5_wait_done() call
> + * means the timeout budget is shared across all sub-transfers rather
> + * than reset to a fresh 1000 ms per sub-transfer.
> + */
> + uint64_t deadline = rte_get_timer_cycles() +
> + (rte_get_timer_hz() * IMX_EDMA5_WAIT_TIMEOUT_MS) / 1000;
> +
> + if (job->nb_sg > 0) {
> + uint16_t s;
> +
> + for (s = 0; s < job->nb_sg; s++) {
> + uint64_t src = rte_le_to_cpu_64(job->sg_tcd[s].saddr);
> + uint64_t dst = rte_le_to_cpu_64(job->sg_tcd[s].daddr);
> + uint32_t len = rte_le_to_cpu_32(job->sg_tcd[s].nbytes);
> +
> + imx_edma5_program_copy(vc, src, dst, len);
> + imx_edma5_hw_start(vc);
> + if (!imx_edma5_wait_done(vc, deadline)) {
> + ok = false;
> + break;
> + }
> + }
> + } else {
> + imx_edma5_program_copy(vc, job->src_iova, job->dst_iova,
> + job->len);
> + imx_edma5_hw_start(vc);
> + ok = imx_edma5_wait_done(vc, deadline);
> + }
> +
> + if (ok)
> + imx_edma5_job_invalidate_dst(job);
> +
> + job->error = ok ? 0 : 1;
> + job->done = 1;
> + vc->submitted_count++;
> +}
> +
> +static int
> +imx_edma5_copy(void *dev_private, uint16_t vchan, rte_iova_t src,
> + rte_iova_t dst, uint32_t length, uint64_t flags)
> +{
> + struct imx_edma5_dev *ed = dev_private;
> + struct imx_edma5_vchan *vc = &ed->vchans[vchan];
> + struct imx_edma5_job *job;
> + uint16_t slot;
> +
> + /* NBYTES = 0 is undefined on the eDMA5 and can wedge the channel. */
> + if (length == 0)
> + return -EINVAL;
> + /*
> + * TCD_NBYTES bits 31:30 are SMLOE/DMLOE when minor-loop offsets are
> + * enabled; writing a count larger than the 30-bit max would corrupt
> + * those control bits. Reject oversized requests.
> + */
> + if (length > IMX_EDMA5_MAX_NBYTES)
> + return -EINVAL;
> +
> + /* Ring full? one slot is kept free to distinguish full from empty. */
> + if (vc->nb_enqueued >= (uint16_t)(vc->nb_desc - 1))
> + return -ENOSPC;
> +
> + slot = vc->head;
> + job = &vc->jobs[slot];
> + job->ridx = vc->ridx;
> + job->submitted = 0;
> + job->done = 0;
> + job->error = 0;
> + job->nb_sg = 0;
> + job->len = length;
> + job->src_iova = src;
> + job->dst_iova = dst;
> + /*
> + * rte_mem_iova2virt() returns NULL when the IOVA is not in the
> + * memzone table (e.g. externally-allocated IOVA-contiguous memory not
> + * registered with DPDK). Cache maintenance is silently skipped for such
> + * addresses; callers are responsible for ensuring coherency in that
> case
> + * or for registering the memory so a VA mapping is available.
> + */
> + job->dst_va = imx_edma5_iova_to_virt(dst);
> +
> + /*
> + * Non-cache-coherent master: clean the source so the device reads the
> + * CPU's latest writes, and clean the destination so a prior dirty line
> + * cannot be written back over the DMA result (the destination is
> + * invalidated after completion).
> + */
> + {
> + void *src_va = imx_edma5_iova_to_virt(src);
> +
> + if (src_va != NULL)
> + imx_edma5_cache_clean(src_va, length);
> + if (job->dst_va != NULL)
> + imx_edma5_cache_clean(job->dst_va, length);
> + }
> +
> + /*
> + * RTE_DMA_OP_FLAG_SUBMIT is equivalent to calling rte_dma_submit()
> + * after this enqueue: advance the head to include the new job and then
> + * run all pending (unsubmitted) jobs from tail to the new head in FIFO
> + * order, matching the behaviour of imx_edma5_submit().
> + */
> + if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
> + uint16_t idx;
> +
> + vc->head = (vc->head + 1) & vc->desc_mask;
> + vc->nb_enqueued++;
> +
> + idx = vc->tail;
> + while (idx != vc->head) {
> + struct imx_edma5_job *j = &vc->jobs[idx];
> +
> + if (!j->submitted) {
> + imx_edma5_run_job(vc, j);
> + j->submitted = 1;
> + }
> + idx = (idx + 1) & vc->desc_mask;
> + }
> +
> + return vc->ridx++;
> + }
> +
> + vc->head = (vc->head + 1) & vc->desc_mask;
> + vc->nb_enqueued++;
> +
> + return vc->ridx++;
> +}
> +
> +static int
> +imx_edma5_copy_sg(void *dev_private, uint16_t vchan,
> + const struct rte_dma_sge *src, const struct rte_dma_sge *dst,
> + uint16_t nb_src, uint16_t nb_dst, uint64_t flags)
> +{
> + struct imx_edma5_dev *ed = dev_private;
> + struct imx_edma5_vchan *vc = &ed->vchans[vchan];
> + struct imx_edma5_hw_tcd64 *tcd;
> + struct imx_edma5_job *job;
> + uint16_t slot;
> + uint16_t s;
> + uint16_t si = 0, di = 0; /* current source/destination seg index
> */
> + uint32_t s_off = 0, d_off = 0; /* byte offset within current segment */
> + uint16_t nsg = 0; /* sub-transfers produced so far */
> + uint64_t src_total = 0, dst_total = 0;
> +
> + /*
> + * The scatter-gather contract is a byte stream: the source and
> + * destination lists may be segmented independently but their
> + * concatenations are equal. The lists are walked as two cursors,
> + * emitting one single-block sub-transfer per run that fits in both the
> + * current source and destination segments. This yields at most
> + * nb_src + nb_dst - 1 sub-transfers, which fit in
> IMX_EDMA5_SG_TCD_PER_JOB.
> + */
> + if (nb_src == 0 || nb_dst == 0 ||
> + nb_src > IMX_EDMA5_MAX_SGES || nb_dst > IMX_EDMA5_MAX_SGES) {
> + IMX_EDMA5_LOG(ERR, "Unsupported SG shape src=%u dst=%u",
> + nb_src, nb_dst);
Suggest use datapath logtype, e.g. add one new type: RTE_LOGTYPE_IMX_EDMA5_DP
> + return -EINVAL;
> + }
> +
> + /* Ring full? one slot is kept free to distinguish full from empty. */
> + if (vc->nb_enqueued >= (uint16_t)(vc->nb_desc - 1))
> + return -ENOSPC;
> +
> + slot = vc->head;
> + job = &vc->jobs[slot];
> + job->ridx = vc->ridx;
> + job->submitted = 0;
> + job->done = 0;
> + job->error = 0;
> +
> + /* This job's dedicated slice of the in-memory TCD pool. */
> + job->sg_tcd = &vc->sg_tcd_pool[(size_t)slot * IMX_EDMA5_SG_TCD_PER_JOB];
> + tcd = job->sg_tcd;
> +
> + /*
> + * Clean every source and destination segment up front (non-coherent
> + * master): the device must read current source data, and dirty
> + * destination lines must be flushed before the transfer.
> + */
> + for (s = 0; s < nb_src; s++) {
> + void *va = imx_edma5_iova_to_virt(src[s].addr);
> +
> + src_total += src[s].length;
> + if (va != NULL)
> + imx_edma5_cache_clean(va, src[s].length);
> + }
> + for (s = 0; s < nb_dst; s++) {
> + void *va = imx_edma5_iova_to_virt(dst[s].addr);
> +
> + dst_total += dst[s].length;
> + if (va != NULL)
> + imx_edma5_cache_clean(va, dst[s].length);
> + }
> +
> + /* copy_sg requires equal total bytes on both lists; reject misuse. */
> + if (src_total != dst_total) {
> + IMX_EDMA5_LOG(ERR,
> + "SG byte count mismatch src=%" PRIu64
> + " dst=%" PRIu64, src_total, dst_total);
> + return -EINVAL;
> + }
> +
> + while (si < nb_src && di < nb_dst) {
> + uint32_t s_rem = src[si].length - s_off;
> + uint32_t d_rem = dst[di].length - d_off;
> + uint32_t len = RTE_MIN(s_rem, d_rem);
> +
> + /*
> + * TCD_NBYTES bits 31:30 are SMLOE/DMLOE; a sub-transfer that
> + * spans more than IMX_EDMA5_MAX_NBYTES would corrupt those
> bits.
> + * Individual segment lengths up to UINT32_MAX are permitted by
> + * the dmadev API, so validate here rather than assume the
> caller
> + * has split them.
> + */
> + if (len > IMX_EDMA5_MAX_NBYTES) {
> + IMX_EDMA5_LOG(ERR,
> + "SG sub-transfer length %u exceeds "
> + "TCD_NBYTES max %u",
> + len, IMX_EDMA5_MAX_NBYTES);
> + job->nb_sg = 0;
> + return -EINVAL;
> + }
> +
> + /* Skip zero-length segments without emitting a descriptor. */
> + if (len == 0) {
> + if (s_rem == 0) {
> + si++;
> + s_off = 0;
> + }
> + if (d_rem == 0) {
> + di++;
> + d_off = 0;
> + }
> + continue;
> + }
> +
> + if (nsg >= IMX_EDMA5_SG_TCD_PER_JOB) {
> + IMX_EDMA5_LOG(ERR,
> + "SG produced too many sub-transfers "
> + "(src=%u dst=%u)", nb_src, nb_dst);
> + /* Defensive: clear nb_sg so the abandoned slot is not
> reused. */
> + job->nb_sg = 0;
> + return -EINVAL;
> + }
> +
> + imx_edma5_fill_tcd(&tcd[nsg], src[si].addr + s_off,
> + dst[di].addr + d_off, len);
> + nsg++;
> +
> + s_off += len;
> + d_off += len;
> + if (s_off == src[si].length) {
> + si++;
> + s_off = 0;
> + }
> + if (d_off == dst[di].length) {
> + di++;
> + d_off = 0;
> + }
> + }
> +
> + /* No sub-transfer (all segments zero-length): NBYTES = 0 wedges eDMA5.
> */
> + if (nsg == 0) {
> + IMX_EDMA5_LOG(ERR, "SG produced zero sub-transfers");
> + return -EINVAL;
> + }
> +
> + job->nb_sg = nsg;
> +
> + /*
> + * RTE_DMA_OP_FLAG_SUBMIT is equivalent to calling rte_dma_submit()
> + * after this enqueue: advance the head to include the new job and then
> + * run all pending (unsubmitted) jobs from tail to the new head in FIFO
> + * order, matching the behaviour of imx_edma5_submit().
> + */
> + if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
> + uint16_t idx;
> +
> + vc->head = (vc->head + 1) & vc->desc_mask;
> + vc->nb_enqueued++;
> +
> + idx = vc->tail;
> + while (idx != vc->head) {
> + struct imx_edma5_job *j = &vc->jobs[idx];
> +
> + if (!j->submitted) {
> + imx_edma5_run_job(vc, j);
> + j->submitted = 1;
> + }
> + idx = (idx + 1) & vc->desc_mask;
> + }
> +
> + return vc->ridx++;
> + }
> +
> + vc->head = (vc->head + 1) & vc->desc_mask;
> + vc->nb_enqueued++;
> +
> + return vc->ridx++;
> +}
> +
> +static int
> +imx_edma5_submit(void *dev_private, uint16_t vchan)
> +{
> + struct imx_edma5_dev *ed = dev_private;
> + struct imx_edma5_vchan *vc = &ed->vchans[vchan];
> + uint16_t idx = vc->tail;
> +
> + /* Run every enqueued-but-not-yet-submitted job to completion (FIFO). */
> + while (idx != vc->head) {
> + struct imx_edma5_job *job = &vc->jobs[idx];
> +
> + if (!job->submitted) {
> + imx_edma5_run_job(vc, job);
> + job->submitted = 1;
> + }
> + idx = (idx + 1) & vc->desc_mask;
> + }
> +
> + return 0;
> +}
> +
> +/*
> + * Reap completed jobs from the software ring in FIFO order. Jobs run
> + * synchronously, so a submitted job's result is already in job->done/error.
> + * An unsubmitted job stops the walk.
> + */
> +static uint16_t
> +imx_edma5_completed(void *dev_private, uint16_t vchan, const uint16_t
> nb_cpls,
> + uint16_t *last_idx, bool *has_error)
> +{
> + struct imx_edma5_dev *ed = dev_private;
> + struct imx_edma5_vchan *vc = &ed->vchans[vchan];
> + uint16_t count = 0;
> +
> + *has_error = false;
> +
> + while (count < nb_cpls && vc->tail != vc->head) {
> + struct imx_edma5_job *job = &vc->jobs[vc->tail];
> +
> + if (!job->submitted || !job->done)
> + break;
> +
> + /*
> + * Stop before an errored job: it is left in the ring for
> + * rte_dma_completed_status(), and last_idx stays at the last
> + * successful transfer.
> + */
> + if (job->error) {
> + *has_error = true;
> + break;
> + }
> +
> + vc->last_idx = job->ridx;
> + vc->completed_count++;
> +
> + vc->tail = (vc->tail + 1) & vc->desc_mask;
> + vc->nb_enqueued--;
> + count++;
> + }
> +
> + *last_idx = vc->last_idx;
> +
> + return count;
> +}
> +
> +static uint16_t
> +imx_edma5_completed_status(void *dev_private, uint16_t vchan,
> + const uint16_t nb_cpls, uint16_t *last_idx,
> + enum rte_dma_status_code *status)
> +{
> + struct imx_edma5_dev *ed = dev_private;
> + struct imx_edma5_vchan *vc = &ed->vchans[vchan];
> + uint16_t count = 0;
> +
> + while (count < nb_cpls && vc->tail != vc->head) {
> + struct imx_edma5_job *job = &vc->jobs[vc->tail];
> +
> + if (!job->submitted || !job->done)
> + break;
> +
> + if (job->error) {
> + status[count] = RTE_DMA_STATUS_BUS_ERROR;
> + vc->errors_count++;
> + } else {
> + status[count] = RTE_DMA_STATUS_SUCCESSFUL;
> + }
> +
> + vc->last_idx = job->ridx;
> + vc->completed_count++;
> +
> + vc->tail = (vc->tail + 1) & vc->desc_mask;
> + vc->nb_enqueued--;
> + count++;
> + }
> +
> + *last_idx = vc->last_idx;
> +
> + return count;
> +}
> +
> +static uint16_t
> +imx_edma5_burst_capacity(const void *dev_private, uint16_t vchan)
> +{
> + const struct imx_edma5_dev *ed = dev_private;
> + const struct imx_edma5_vchan *vc = &ed->vchans[vchan];
> +
> + /* One slot is reserved to distinguish full from empty. */
> + return vc->nb_desc - 1 - vc->nb_enqueued;
> +}
> +
> static const struct rte_dma_dev_ops imx_edma5_ops = {
> .dev_info_get = imx_edma5_info_get,
> .dev_configure = imx_edma5_configure,
> @@ -382,6 +974,13 @@ imx_edma5_probe(struct rte_platform_device *pdev)
>
> dev->device = &pdev->device;
> dev->dev_ops = &imx_edma5_ops;
> + dev->fp_obj->dev_private = dev->data->dev_private;
> + dev->fp_obj->copy = imx_edma5_copy;
> + dev->fp_obj->copy_sg = imx_edma5_copy_sg;
> + dev->fp_obj->submit = imx_edma5_submit;
> + dev->fp_obj->completed = imx_edma5_completed;
> + dev->fp_obj->completed_status = imx_edma5_completed_status;
> + dev->fp_obj->burst_capacity = imx_edma5_burst_capacity;
>
> ed = dev->data->dev_private;
> ed->reg_base = res->mem.addr;
> diff --git a/drivers/dma/imx_edma5/imx_edma5_dmadev.h
> b/drivers/dma/imx_edma5/imx_edma5_dmadev.h
> index 3da7957457..c85ea4a726 100644
> --- a/drivers/dma/imx_edma5/imx_edma5_dmadev.h
> +++ b/drivers/dma/imx_edma5/imx_edma5_dmadev.h
> @@ -12,6 +12,7 @@
> #include <rte_byteorder.h>
> #include <rte_common.h>
> #include <rte_dmadev.h>
> +#include <rte_eal.h>
> #include <rte_io.h>
> #include <rte_memory.h>
>
> @@ -217,4 +218,17 @@ imx_edma5_write64(uint8_t *base, uint32_t off, uint64_t
> val)
> imx_edma5_write32(base, off + 4, (uint32_t)(val >> 32));
> }
>
> +/*
> + * Resolve an IOVA to a CPU virtual address for cache maintenance.
> + * In IOVA=VA mode the IOVA is the VA directly; otherwise walk the memseg
> + * table. Avoids the memseg list walk on the common IOVA=VA fast path.
> + */
> +static inline void *
> +imx_edma5_iova_to_virt(rte_iova_t iova)
> +{
> + if (rte_eal_iova_mode() == RTE_IOVA_VA)
> + return (void *)(uintptr_t)iova;
The rte_eal_iova_mode() is not inline, how about save the info in vc struct at
vchan-setup()?
> + return rte_mem_iova2virt(iova);
> +}
> +
> #endif /* IMX_EDMA5_DMADEV_H */
The above comments are only suggestions, regardless of whether they are
modified or not,
Acked-by: Chengwen Feng <[email protected]>
