From: Gagandeep Singh <[email protected]>
Scatter-gather batching, the post-transfer data validation helper and the
ERR050757 read-transaction workaround already exist in the driver. This
patch does not add them; it makes them configurable at runtime and gates
the previously always-on data-validation and ERR050757 programming.
Add three new device arguments:
dpaa_dma_sg_disable
Submit each pending descriptor as an individual single-entry transfer
instead of grouping them into one SG command. SG batching stays on by
default.
dpaa_dma_data_validation
Enable the existing read-back-and-compare data validation helper, which
is otherwise skipped. For debugging only; significant performance cost.
dpaa_dma_pci_read_disable
Skip the ERR050757 source-frame-descriptor programming (read
transaction type and stride limiting). Only available when built with
RTE_DMA_DPAA_ERRATA_ERR050757. May improve throughput on unaffected
platforms.
Also let dma_pool_alloc() accept a NULL phy_addr and update the dmadev
documentation for the new arguments.
Signed-off-by: Gagandeep Singh <[email protected]>
---
doc/guides/dmadevs/dpaa.rst | 30 +++++++++-
drivers/dma/dpaa/dpaa_qdma.c | 107 +++++++++++++++++++++++++++--------
2 files changed, 110 insertions(+), 27 deletions(-)
diff --git a/doc/guides/dmadevs/dpaa.rst b/doc/guides/dmadevs/dpaa.rst
index 01104d7f1e..17aac5208c 100644
--- a/doc/guides/dmadevs/dpaa.rst
+++ b/doc/guides/dmadevs/dpaa.rst
@@ -76,5 +76,31 @@ Device Arguments
----------------
``dpaa_dma_err_check=1``
- Check DMA errors at driver level.
- Usage example: ``dpaa_bus:dpaa_qdma-1,dpaa_dma_err_check=1``
+ Enable DMA error checking at driver level. When set, the driver reads
+ the hardware error detect register after each dequeue and increments
+ the error statistics counter on any reported error.
+ Usage example: ``-a dpaa_bus:dpaa_qdma-1,dpaa_dma_err_check=1``
+
+``dpaa_dma_sg_disable=1``
+ Disable scatter-gather (SG) batching. By default the driver groups
+ multiple pending descriptors into a single SG command to reduce
+ doorbell overhead. Setting this devarg forces each descriptor to be
+ submitted as an individual single-entry transfer. Useful for latency
+ benchmarking or debugging.
+ Usage example: ``-a dpaa_bus:dpaa_qdma-1,dpaa_dma_sg_disable=1``
+
+``dpaa_dma_data_validation=1``
+ Enable post-DMA data validation. After each completed transfer the
+ driver reads back source and destination buffers and compares them
+ byte by byte, logging any mismatch. This has significant performance
+ impact and is intended for debugging only.
+ Usage example: ``-a dpaa_bus:dpaa_qdma-1,dpaa_dma_data_validation=1``
+
+``dpaa_dma_pci_read_disable=1``
+ Disable the ERR050757 workaround programming. The workaround sets the
+ read transaction type on the source frame descriptor and, for transfers
+ longer than the errata safe length, programs the source stride so that
+ read transactions stay within that limit. Only available when the driver
+ is built with ``RTE_DMA_DPAA_ERRATA_ERR050757`` enabled. Disabling it may
+ improve throughput on platforms not affected by the erratum.
+ Usage example: ``-a dpaa_bus:dpaa_qdma-1,dpaa_dma_pci_read_disable=1``
diff --git a/drivers/dma/dpaa/dpaa_qdma.c b/drivers/dma/dpaa/dpaa_qdma.c
index 3981bf277e..0dfb835d5b 100644
--- a/drivers/dma/dpaa/dpaa_qdma.c
+++ b/drivers/dma/dpaa/dpaa_qdma.c
@@ -11,8 +11,18 @@
static uint32_t s_sg_max_entry_sz = 2000;
static bool s_hw_err_check;
+static bool s_data_validation;
+static bool s_sg_enable = true;
+#ifdef RTE_DMA_DPAA_ERRATA_ERR050757
+static bool s_pci_read = true;
+#endif
-#define DPAA_DMA_ERROR_CHECK "dpaa_dma_err_check"
+#define DPAA_DMA_ERROR_CHECK "dpaa_dma_err_check"
+#define DPAA_DMA_SG_DISABLE "dpaa_dma_sg_disable"
+#define DPAA_DMA_DATA_VALID "dpaa_dma_data_validation"
+#ifdef RTE_DMA_DPAA_ERRATA_ERR050757
+#define DPAA_DMA_PCI_READ_DISABLE "dpaa_dma_pci_read_disable"
+#endif
static inline void
qdma_desc_addr_set64(struct fsl_qdma_comp_cmd_desc *ccdf, u64 addr)
@@ -112,7 +122,8 @@ dma_pool_alloc(char *nm, int size, int aligned, dma_addr_t
*phy_addr)
if (!virt_addr)
return NULL;
- *phy_addr = rte_mem_virt2iova(virt_addr);
+ if (phy_addr)
+ *phy_addr = rte_mem_virt2iova(virt_addr);
return virt_addr;
}
@@ -392,6 +403,8 @@ fsl_qdma_data_validation(struct fsl_qdma_desc *desc[],
char err_msg[512];
int offset;
+ if (likely(!s_data_validation))
+ return;
offset = sprintf(err_msg, "Fatal TC%d/queue%d: ",
fsl_queue->block_id,
@@ -716,19 +729,21 @@ fsl_qdma_enqueue_desc_single(struct fsl_qdma_queue
*fsl_queue,
ft = fsl_queue->ft[fsl_queue->ci];
#ifdef RTE_DMA_DPAA_ERRATA_ERR050757
- sdf = &ft->df.sdf;
- sdf->srttype = FSL_QDMA_CMD_RWTTYPE;
+ if (s_pci_read) {
+ sdf = &ft->df.sdf;
+ sdf->srttype = FSL_QDMA_CMD_RWTTYPE;
#ifdef RTE_DMA_DPAA_ERRATA_ERR050265
- sdf->prefetch = 1;
+ sdf->prefetch = 1;
#endif
- if (len > FSL_QDMA_CMD_SS_ERR050757_LEN) {
- sdf->ssen = 1;
- sdf->sss = FSL_QDMA_CMD_SS_ERR050757_LEN;
- sdf->ssd = FSL_QDMA_CMD_SS_ERR050757_LEN;
- } else {
- sdf->ssen = 0;
- sdf->sss = 0;
- sdf->ssd = 0;
+ if (len > FSL_QDMA_CMD_SS_ERR050757_LEN) {
+ sdf->ssen = 1;
+ sdf->sss = FSL_QDMA_CMD_SS_ERR050757_LEN;
+ sdf->ssd = FSL_QDMA_CMD_SS_ERR050757_LEN;
+ } else {
+ sdf->ssen = 0;
+ sdf->sss = 0;
+ sdf->ssd = 0;
+ }
}
#endif
csgf_src = &ft->desc_sbuf;
@@ -837,19 +852,21 @@ fsl_qdma_enqueue_desc_sg(struct fsl_qdma_queue *fsl_queue)
csgf_src->length = total_len;
csgf_dest->length = total_len;
#ifdef RTE_DMA_DPAA_ERRATA_ERR050757
- sdf = &ft->df.sdf;
- sdf->srttype = FSL_QDMA_CMD_RWTTYPE;
+ if (s_pci_read) {
+ sdf = &ft->df.sdf;
+ sdf->srttype = FSL_QDMA_CMD_RWTTYPE;
#ifdef RTE_DMA_DPAA_ERRATA_ERR050265
- sdf->prefetch = 1;
+ sdf->prefetch = 1;
#endif
- if (total_len > FSL_QDMA_CMD_SS_ERR050757_LEN) {
- sdf->ssen = 1;
- sdf->sss = FSL_QDMA_CMD_SS_ERR050757_LEN;
- sdf->ssd = FSL_QDMA_CMD_SS_ERR050757_LEN;
- } else {
- sdf->ssen = 0;
- sdf->sss = 0;
- sdf->ssd = 0;
+ if (total_len > FSL_QDMA_CMD_SS_ERR050757_LEN) {
+ sdf->ssen = 1;
+ sdf->sss = FSL_QDMA_CMD_SS_ERR050757_LEN;
+ sdf->ssd = FSL_QDMA_CMD_SS_ERR050757_LEN;
+ } else {
+ sdf->ssen = 0;
+ sdf->sss = 0;
+ sdf->ssd = 0;
+ }
}
#endif
ret = fsl_qdma_enqueue_desc_to_ring(fsl_queue, num);
@@ -888,6 +905,25 @@ fsl_qdma_enqueue_desc(struct fsl_qdma_queue *fsl_queue)
fsl_queue->pending_num = 0;
}
return ret;
+ } else if (!s_sg_enable) {
+ while (fsl_queue->pending_num > 0) {
+ ret = fsl_qdma_enqueue_desc_single(fsl_queue,
+ fsl_queue->pending_desc[start].dst,
+ fsl_queue->pending_desc[start].src,
+ fsl_queue->pending_desc[start].len);
+ if (!ret) {
+ start = (start + 1) &
+ (fsl_queue->pending_max - 1);
+ fsl_queue->pending_start = start;
+ fsl_queue->pending_num--;
+ } else {
+ DPAA_QDMA_ERR("Eq pending desc failed(%d)",
+ ret);
+ return -EIO;
+ }
+ }
+
+ return 0;
}
return fsl_qdma_enqueue_desc_sg(fsl_queue);
@@ -1358,6 +1394,20 @@ dpaa_qdma_init(struct rte_dma_dev *dmadev)
s_hw_err_check = true;
DPAA_QDMA_INFO("Enable DMA error checks");
}
+ if (dpaa_get_devargs(dmadev->device->devargs, DPAA_DMA_SG_DISABLE)) {
+ s_sg_enable = false;
+ DPAA_QDMA_INFO("SG mode disabled via devarg");
+ }
+ if (dpaa_get_devargs(dmadev->device->devargs, DPAA_DMA_DATA_VALID)) {
+ s_data_validation = true;
+ DPAA_QDMA_INFO("Data validation enabled via devarg");
+ }
+#ifdef RTE_DMA_DPAA_ERRATA_ERR050757
+ if (dpaa_get_devargs(dmadev->device->devargs,
DPAA_DMA_PCI_READ_DISABLE)) {
+ s_pci_read = false;
+ DPAA_QDMA_INFO("ERR050757 PCI read workaround disabled via
devarg");
+ }
+#endif
fsl_qdma->n_queues = QDMA_QUEUES * QDMA_BLOCKS;
fsl_qdma->num_blocks = QDMA_BLOCKS;
@@ -1483,5 +1533,12 @@ static struct rte_dpaa_driver rte_dpaa_qdma_pmd = {
};
RTE_PMD_REGISTER_DPAA(dpaa_qdma, rte_dpaa_qdma_pmd);
-RTE_PMD_REGISTER_PARAM_STRING(dpaa_qdma, DPAA_DMA_ERROR_CHECK "=<int>");
+RTE_PMD_REGISTER_PARAM_STRING(dpaa_qdma,
+ DPAA_DMA_ERROR_CHECK "=<int> "
+ DPAA_DMA_SG_DISABLE "=<int> "
+ DPAA_DMA_DATA_VALID "=<int> "
+#ifdef RTE_DMA_DPAA_ERRATA_ERR050757
+ DPAA_DMA_PCI_READ_DISABLE "=<int>"
+#endif
+ );
RTE_LOG_REGISTER_DEFAULT(dpaa_qdma_logtype, INFO);
--
2.25.1
