Currently, finished irq work is signaled by a completion. This is lightweight but doesn't integrate with other kernel buffer synchronization.
Using dma-fences instead of completions allows for the fences to be added to their respective DMA-BUFs which enables polling on the buffer instead of relying on the ~_WAIT_BO IOCTL for synchronization. This is also useful in preparation for the addition of drm_sched support. Fences can be linked together to create more complex synchronization structures. Co-developed-by: Pranjal Ramajor Asha Kanojiya <[email protected]> Signed-off-by: Pranjal Ramajor Asha Kanojiya <[email protected]> Signed-off-by: Carl Vanderlip <[email protected]> --- drivers/accel/qaic/Makefile | 1 + drivers/accel/qaic/qaic.h | 26 +++++- drivers/accel/qaic/qaic_data.c | 84 +++++++++++-------- drivers/accel/qaic/qaic_drv.c | 2 +- drivers/accel/qaic/qaic_fence.c | 144 ++++++++++++++++++++++++++++++++ 5 files changed, 220 insertions(+), 37 deletions(-) create mode 100644 drivers/accel/qaic/qaic_fence.c diff --git a/drivers/accel/qaic/Makefile b/drivers/accel/qaic/Makefile index 71f727b74da3..90582b0693da 100644 --- a/drivers/accel/qaic/Makefile +++ b/drivers/accel/qaic/Makefile @@ -10,6 +10,7 @@ qaic-y := \ qaic_control.o \ qaic_data.o \ qaic_drv.o \ + qaic_fence.o \ qaic_ras.o \ qaic_ssr.o \ qaic_sysfs.o \ diff --git a/drivers/accel/qaic/qaic.h b/drivers/accel/qaic/qaic.h index 02ca99000e2a..4cdd6579aa89 100644 --- a/drivers/accel/qaic/qaic.h +++ b/drivers/accel/qaic/qaic.h @@ -6,6 +6,7 @@ #ifndef _QAIC_H_ #define _QAIC_H_ +#include <linux/dma-fence.h> #include <linux/interrupt.h> #include <linux/kref.h> #include <linux/mhi.h> @@ -18,9 +19,11 @@ #include <drm/drm_device.h> #include <drm/drm_gem.h> +#define QAIC_NAME "qaic" #define QAIC_DBC_BASE SZ_128K #define QAIC_DBC_SIZE SZ_4K #define QAIC_SSR_DBC_SENTINEL U32_MAX /* No ongoing SSR sentinel */ +#define QAIC_TIMELINE_LEN 32 /* Should fit max sized DBC id/BO handle */ #define QAIC_NO_PARTITION -1 @@ -227,6 +230,16 @@ struct qaic_drm_device { void *sysfs_attrs; }; +struct qaic_fence { + struct dma_fence base; + /* Spinlock used for initializing this fence */ + spinlock_t lock; + /* Name of fence context */ + char timeline_name[QAIC_TIMELINE_LEN]; + /* Cached value of dbc_id for use in populating timeline_name */ + u32 dbc_id; +}; + struct qaic_bo { struct drm_gem_object base; /* Scatter/gather table for allocate/imported BO */ @@ -253,8 +266,10 @@ struct qaic_bo { bool sliced; /* Request ID of this BO if it is queued for execution */ u16 req_id; - /* Wait on this for completion of DMA transfer of this BO */ - struct completion xfer_done; + /* Wait on this fence for DMA transfer of this BO */ + struct qaic_fence *fence; + /* Current fence context for this BO */ + u64 fence_context; /* * Node in linked list where head is dbc->xfer_list. * This link list contain BO's that are queued for DMA transfer. @@ -290,6 +305,8 @@ struct qaic_bo { } perf_stats; /* Synchronizes BO operations */ struct mutex lock; + /* Callback to sync BO with CPU */ + struct dma_fence_cb cb; }; struct bo_slice { @@ -351,6 +368,11 @@ void qaic_irq_polling_work(struct work_struct *work); void qaic_dbc_enter_ssr(struct qaic_device *qdev, u32 dbc_id); void qaic_dbc_exit_ssr(struct qaic_device *qdev); +/* qaic_fence.c */ +int qaic_fence_wait(struct dma_fence *fence, signed long timeout); +void qaic_remove_bo_fence(struct qaic_bo *bo); +int qaic_acquire_bo_fence(struct qaic_bo *bo); + /* qaic_sysfs.c */ int qaic_sysfs_init(struct qaic_drm_device *qddev); void qaic_sysfs_remove(struct qaic_drm_device *qddev); diff --git a/drivers/accel/qaic/qaic_data.c b/drivers/accel/qaic/qaic_data.c index 64bc3cc50d1f..3326d9da0e1e 100644 --- a/drivers/accel/qaic/qaic_data.c +++ b/drivers/accel/qaic/qaic_data.c @@ -184,9 +184,12 @@ struct qaic_exec_data_req { int idx; }; +/* Caller should be holding bo->lock */ static inline bool bo_queued(struct qaic_bo *bo) { - return !list_empty(&bo->xfer_list); + if (bo->fence) + return !dma_fence_is_signaled(&bo->fence->base); + return false; } inline int get_dbc_req_elem_size(void) @@ -745,16 +748,8 @@ static const struct drm_gem_object_funcs qaic_gem_funcs = { .vm_ops = &drm_vm_ops, }; -static void qaic_init_bo(struct qaic_bo *bo, bool reinit) +static void qaic_init_bo_lists(struct qaic_bo *bo) { - if (reinit) { - bo->sliced = false; - reinit_completion(&bo->xfer_done); - } else { - mutex_init(&bo->lock); - init_completion(&bo->xfer_done); - } - complete_all(&bo->xfer_done); INIT_LIST_HEAD(&bo->slices); INIT_LIST_HEAD(&bo->xfer_list); } @@ -767,7 +762,9 @@ static struct qaic_bo *qaic_alloc_init_bo(void) if (!bo) return ERR_PTR(-ENOMEM); - qaic_init_bo(bo, false); + mutex_init(&bo->lock); + bo->fence_context = dma_fence_context_alloc(1); + qaic_init_bo_lists(bo); return bo; } @@ -996,6 +993,7 @@ static void qaic_unprepare_bo(struct qaic_device *qdev, struct qaic_bo *bo) bo->dir = 0; bo->dbc = NULL; bo->nr_slice = 0; + bo->sliced = false; } static void qaic_free_slices_bo(struct qaic_bo *bo) @@ -1304,9 +1302,7 @@ static int send_bo_list_to_device(struct qaic_device *qdev, struct drm_file *fil goto unlock_bo; } - spin_lock_irqsave(&dbc->xfer_lock, flags); if (bo_queued(bo)) { - spin_unlock_irqrestore(&dbc->xfer_lock, flags); ret = -EINVAL; goto unlock_bo; } @@ -1327,12 +1323,15 @@ static int send_bo_list_to_device(struct qaic_device *qdev, struct drm_file *fil head, tail); else ret = copy_exec_reqs(qdev, slice, dbc->id, head, tail); - if (ret) { - spin_unlock_irqrestore(&dbc->xfer_lock, flags); + if (ret) goto unlock_bo; - } } - reinit_completion(&bo->xfer_done); + + ret = qaic_acquire_bo_fence(bo); + if (ret) + goto unlock_bo; + + spin_lock_irqsave(&dbc->xfer_lock, flags); list_add_tail(&bo->xfer_list, &dbc->xfer_list); spin_unlock_irqrestore(&dbc->xfer_lock, flags); dma_sync_sgtable_for_device(&qdev->pdev->dev, bo->sgt, bo->dir); @@ -1787,11 +1786,13 @@ irqreturn_t dbc_irq_threaded_fn(int irq, void *data) * At this point we have received all the interrupts for * BO, which means BO execution is complete. */ - dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir); bo->nr_slice_xfer_done = 0; list_del_init(&bo->xfer_list); - bo->perf_stats.req_processed_ts = ktime_get_ns(); - complete_all(&bo->xfer_done); + /* + * fence is put when it is executed next, during xfer_list + * cleanup, or when slice is detached + */ + dma_fence_signal(&bo->fence->base); drm_gem_object_put(&bo->base); break; } @@ -1840,6 +1841,7 @@ int qaic_wait_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file struct dma_bridge_chan *dbc; struct drm_gem_object *obj; struct qaic_device *qdev; + struct dma_fence *fence; unsigned long timeout; struct qaic_user *usr; struct qaic_bo *bo; @@ -1881,6 +1883,9 @@ int qaic_wait_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file goto unlock_ch_srcu; } + timeout = args->timeout ? args->timeout : wait_exec_default_timeout_ms; + timeout = msecs_to_jiffies(timeout); + obj = drm_gem_object_lookup(file_priv, args->handle); if (!obj) { ret = -ENOENT; @@ -1888,15 +1893,24 @@ int qaic_wait_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file } bo = to_qaic_bo(obj); - timeout = args->timeout ? args->timeout : wait_exec_default_timeout_ms; - timeout = msecs_to_jiffies(timeout); - ret = wait_for_completion_interruptible_timeout(&bo->xfer_done, timeout); - if (!ret) { - ret = -ETIMEDOUT; + ret = mutex_lock_interruptible(&bo->lock); + if (ret) + goto put_obj; + + if (!bo->fence) { + ret = -ENOENT; + mutex_unlock(&bo->lock); goto put_obj; } - if (ret > 0) - ret = 0; + + fence = &bo->fence->base; + dma_fence_get(fence); + mutex_unlock(&bo->lock); + + ret = qaic_fence_wait(fence, timeout); + dma_fence_put(fence); + if (ret) + goto put_obj; if (!dbc->usr) ret = -EPERM; @@ -2000,9 +2014,10 @@ int qaic_perf_stats_bo_ioctl(struct drm_device *dev, void *data, struct drm_file static void detach_slice_bo(struct qaic_device *qdev, struct qaic_bo *bo) { + qaic_remove_bo_fence(bo); qaic_free_slices_bo(bo); qaic_unprepare_bo(qdev, bo); - qaic_init_bo(bo, true); + qaic_init_bo_lists(bo); list_del(&bo->bo_list); drm_gem_object_put(&bo->base); } @@ -2015,7 +2030,6 @@ int qaic_detach_slice_bo_ioctl(struct drm_device *dev, void *data, struct drm_fi struct drm_gem_object *obj; struct qaic_device *qdev; struct qaic_user *usr; - unsigned long flags; struct qaic_bo *bo; int ret; @@ -2060,13 +2074,10 @@ int qaic_detach_slice_bo_ioctl(struct drm_device *dev, void *data, struct drm_fi } /* Check if BO is committed to H/W for DMA */ - spin_lock_irqsave(&dbc->xfer_lock, flags); if (bo_queued(bo)) { - spin_unlock_irqrestore(&dbc->xfer_lock, flags); ret = -EBUSY; goto unlock_ch_srcu; } - spin_unlock_irqrestore(&dbc->xfer_lock, flags); detach_slice_bo(qdev, bo); @@ -2093,14 +2104,19 @@ static void empty_xfer_list(struct qaic_device *qdev, struct dma_bridge_chan *db bo = list_first_entry(&dbc->xfer_list, typeof(*bo), xfer_list); list_del_init(&bo->xfer_list); spin_unlock_irqrestore(&dbc->xfer_lock, flags); + mutex_lock(&bo->lock); bo->nr_slice_xfer_done = 0; bo->req_id = 0; bo->perf_stats.req_received_ts = 0; bo->perf_stats.req_submit_ts = 0; bo->perf_stats.req_processed_ts = 0; bo->perf_stats.queue_level_before = 0; - dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir); - complete_all(&bo->xfer_done); + dma_fence_get(&bo->fence->base); + dma_fence_set_error(&bo->fence->base, -ECANCELED); + dma_fence_signal(&bo->fence->base); + dma_fence_put(&bo->fence->base); + qaic_remove_bo_fence(bo); + mutex_unlock(&bo->lock); drm_gem_object_put(&bo->base); spin_lock_irqsave(&dbc->xfer_lock, flags); } diff --git a/drivers/accel/qaic/qaic_drv.c b/drivers/accel/qaic/qaic_drv.c index c1bb6db78d12..219f58fdfde8 100644 --- a/drivers/accel/qaic/qaic_drv.c +++ b/drivers/accel/qaic/qaic_drv.c @@ -40,7 +40,7 @@ MODULE_IMPORT_NS("DMA_BUF"); #define PCI_DEVICE_ID_QCOM_AIC080 0xa080 #define PCI_DEVICE_ID_QCOM_AIC100 0xa100 #define PCI_DEVICE_ID_QCOM_AIC200 0xa110 -#define QAIC_NAME "qaic" +#define PCI_DEVICE_ID_QCOM_AIC200VF 0xa111 #define QAIC_DESC "Qualcomm Cloud AI Accelerators" #define CNTL_MAJOR 5 #define CNTL_MINOR 0 diff --git a/drivers/accel/qaic/qaic_fence.c b/drivers/accel/qaic/qaic_fence.c new file mode 100644 index 000000000000..ec7e555c454b --- /dev/null +++ b/drivers/accel/qaic/qaic_fence.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* Copyright (c) 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved. */ + +#include <linux/dma-fence.h> +#include <linux/dma-resv.h> +#include <linux/ktime.h> +#include <linux/spinlock.h> +#include <linux/sprintf.h> + +#include "qaic.h" + +static const char *qaic_fence_get_driver_name(struct dma_fence *f) +{ + return QAIC_NAME; +} + +static const char *qaic_fence_get_timeline_name(struct dma_fence *f) +{ + struct qaic_fence *qfence = container_of(f, struct qaic_fence, base); + + if (!qfence->timeline_name[0]) + scnprintf(qfence->timeline_name, ARRAY_SIZE(qfence->timeline_name), "D%02u", + qfence->dbc_id); + return qfence->timeline_name; +} + +static const struct dma_fence_ops qaic_fence_ops = { + .get_driver_name = qaic_fence_get_driver_name, + .get_timeline_name = qaic_fence_get_timeline_name, +}; + +static void qaic_fence_sync_cb_func(struct dma_fence *f, struct dma_fence_cb *cb) +{ + struct qaic_bo *bo = container_of(cb, struct qaic_bo, cb); + struct qaic_device *qdev; + + if (f->error) + return; + + qdev = bo->dbc->qdev; + dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir); + bo->perf_stats.req_processed_ts = ktime_to_ns(f->timestamp); +} + +int qaic_fence_wait(struct dma_fence *fence, signed long timeout) +{ + int ret; + + timeout = dma_fence_wait_timeout(fence, true, timeout); + if (!timeout) { /* Wait timed out */ + ret = -ETIMEDOUT; + } else if (timeout < 0) { /* Wait interrupted */ + ret = -EINTR; + } else { /* Wait completed */ + ret = dma_fence_get_status(fence); + WARN_ON_ONCE(ret == 0); + if (!ret) /* Fence not signaled yet */ + ret = -EBUSY; + else if (ret > 0) /* Fence is signaled successfully */ + ret = 0; + /* otherwise ret already contains the error code of the fence */ + } + return ret; +} + +static struct qaic_fence *qaic_create_fence(struct qaic_bo *bo) +{ + struct qaic_fence *fence; + + fence = kzalloc_obj(*fence); + if (!fence) + return ERR_PTR(-ENOMEM); + + spin_lock_init(&fence->lock); + fence->dbc_id = bo->dbc->id; + dma_fence_init(&fence->base, &qaic_fence_ops, &fence->lock, bo->fence_context, 1); + + return fence; +} + +/* Caller should be holding bo->lock */ +static int qaic_set_bo_resv_fence(struct qaic_bo *bo, struct qaic_fence *fence) +{ + int ret; + + ret = dma_resv_reserve_fences(bo->base.resv, 1); + if (ret) + goto err_out_fence; + + /* + * Add a fence in dma-buf reservation object for this BO, + * enables buffer sharing across devices. + */ + dma_resv_add_fence(bo->base.resv, &fence->base, + dma_resv_usage_rw(bo->dir == DMA_TO_DEVICE)); + + ret = dma_fence_add_callback(&fence->base, &bo->cb, qaic_fence_sync_cb_func); + if (ret) + goto err_out_fence; + + return ret; + +err_out_fence: + dma_fence_put(&fence->base); + bo->fence = NULL; + return ret; +} + +/* Caller should be holding bo->lock */ +void qaic_remove_bo_fence(struct qaic_bo *bo) +{ + if (bo->fence) { + if (!dma_fence_is_signaled(&bo->fence->base)) { + dma_fence_set_error(&bo->fence->base, -ECANCELED); + dma_fence_signal(&bo->fence->base); + } + dma_fence_put(&bo->fence->base); + } + bo->fence = NULL; +} + +/* Caller should be holding bo->lock */ +static void qaic_replace_bo_fence(struct qaic_bo *bo, struct qaic_fence *fence) +{ + qaic_remove_bo_fence(bo); + bo->fence = fence; +} + +/* Caller should be holding bo->lock */ +int qaic_acquire_bo_fence(struct qaic_bo *bo) +{ + struct qaic_fence *fence; + int ret = 0; + + fence = qaic_create_fence(bo); + if (IS_ERR(fence)) + return PTR_ERR(fence); + + qaic_replace_bo_fence(bo, fence); + ret = qaic_set_bo_resv_fence(bo, fence); + + return ret; +} -- 2.43.0
