When a BO's requests would exceed FIFO size and credit balance, divide the requests into groups to be handled by multiple jobs. This addresses limitations where large or fragmented buffers could not be submitted due to FIFO capacity.
Co-developed-by: Pranjal Ramajor Asha Kanojiya <[email protected]> Signed-off-by: Pranjal Ramajor Asha Kanojiya <[email protected]> Signed-off-by: Carl Vanderlip <[email protected]> --- drivers/accel/qaic/qaic.h | 9 ++++--- drivers/accel/qaic/qaic_data.c | 45 +++++++++++++++++++++------------ drivers/accel/qaic/qaic_sched.c | 25 ++++++++++++++---- 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/drivers/accel/qaic/qaic.h b/drivers/accel/qaic/qaic.h index 40c7492ea121..53c71584edf9 100644 --- a/drivers/accel/qaic/qaic.h +++ b/drivers/accel/qaic/qaic.h @@ -88,6 +88,8 @@ struct qaic_job { /* Node of list of jobs associated with a DBC */ struct list_head queue; struct kref ref_count; + /* Offset of requests this job needs to skip within the slice */ + unsigned int req_offset; /* Number of requests this job executes */ unsigned int num_req; /* True if this job is partially executed */ @@ -461,8 +463,8 @@ void enable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr); void wakeup_dbc(struct qaic_device *qdev, u32 dbc_id); void release_dbc(struct qaic_device *qdev, u32 dbc_id); void qaic_data_get_fifo_info(struct dma_bridge_chan *dbc, u32 *head, u32 *tail); -int qaic_submit_reqs_to_hw(struct bo_slice *slice, unsigned int num_req, - bool is_partial, u32 partial_size, u16 job_id); +int qaic_submit_reqs_to_hw(struct bo_slice *slice, struct dbc_req *reqs, + unsigned int num_req, bool is_partial, u32 partial_size, u16 job_id); void wake_all_cntl(struct qaic_device *qdev); void qaic_dev_reset_clean_local_state(struct qaic_device *qdev); @@ -491,7 +493,8 @@ void qaic_remove_bo_fence(struct qaic_bo *bo); int qaicm_sched_init(struct drm_device *drm, struct dma_bridge_chan *dbc); void qaic_sched_entity_init(struct dma_bridge_chan *dbc); struct qaic_job *qaic_create_job(struct drm_file *file_priv, struct bo_slice *slice, - unsigned int num_req, u64 seq_no, struct list_head *tmp_list); + unsigned int req_offset, unsigned int num_full_reqs, u64 seq_no, + struct list_head *tmp_list); void qaic_submit_job(struct dma_bridge_chan *dbc, struct qaic_job *job); void qaic_free_job(struct kref *ref); void qaic_put_job(struct qaic_job *job); diff --git a/drivers/accel/qaic/qaic_data.c b/drivers/accel/qaic/qaic_data.c index 63ca4a489a9a..7287370e4fab 100644 --- a/drivers/accel/qaic/qaic_data.c +++ b/drivers/accel/qaic/qaic_data.c @@ -948,11 +948,6 @@ static int qaic_attach_slicing_bo(struct qaic_device *qdev, struct qaic_bo *bo, } } - if (bo->total_slice_nents > bo->dbc->nelem) { - qaic_free_slices_bo(bo); - return -ENOSPC; - } - return 0; } @@ -1108,11 +1103,11 @@ static void submit_reqs_to_fifo(struct dma_bridge_chan *dbc, struct dbc_req *req } /* Caller should be holding dbc->req_lock */ -int qaic_submit_reqs_to_hw(struct bo_slice *slice, unsigned int num_req, - bool is_partial, u32 partial_size, u16 job_id) +int qaic_submit_reqs_to_hw(struct bo_slice *slice, struct dbc_req *reqs, + unsigned int num_req, bool is_partial, + u32 partial_size, u16 job_id) { struct dma_bridge_chan *dbc = slice->bo->dbc; - struct dbc_req *reqs = slice->reqs; struct qaic_bo *bo = slice->bo; struct dbc_req *last_req; u32 head, tail, avail; @@ -1129,12 +1124,12 @@ int qaic_submit_reqs_to_hw(struct bo_slice *slice, unsigned int num_req, submit_reqs_to_fifo(dbc, reqs, num_req, head, tail); + last_req = fifo_at(dbc->req_q_base, (tail + num_req - 1) % dbc->nelem); if (is_partial) { /* * Copy over the last entry. Here we need to adjust len to the left over * size, and set src and dst to the entry it is copied to. */ - last_req = fifo_at(dbc->req_q_base, (tail + num_req - 1) % dbc->nelem); memcpy(last_req, &slice->reqs[slice->nents - 1], sizeof(*last_req)); last_req->len = cpu_to_le32(partial_size); last_req->src_addr = reqs[num_req - 1].src_addr; @@ -1144,6 +1139,8 @@ int qaic_submit_reqs_to_hw(struct bo_slice *slice, unsigned int num_req, if (!last_req->len) last_req->cmd &= ~GENMASK(1, 0); } + /* When a BO has been split into multiple jobs, each job needs to signal its completion */ + last_req->cmd |= GEN_COMPLETION; tail = (tail + num_req) % dbc->nelem; @@ -1169,11 +1166,11 @@ static inline void qaic_dbc_put_job(struct dma_bridge_chan *dbc, struct qaic_job static int create_slice_jobs(struct drm_file *file_priv, struct bo_slice *slice, bool partial, u64 resize, struct list_head *tmp_list) { + unsigned int nents_offset, nents_xfer; struct dbc_req *reqs = slice->reqs; - struct qaic_job *job = NULL; + struct qaic_job *job, *job_i; unsigned int job_count = 0; - unsigned int nents_xfer; - u64 last_bytes; + u64 last_bytes, seq_no; u32 first_n; /* @@ -1194,16 +1191,32 @@ static int create_slice_jobs(struct drm_file *file_priv, struct bo_slice *slice, else nents_xfer = slice->nents; - job = qaic_create_job(file_priv, slice, nents_xfer, 0, tmp_list); - if (IS_ERR(job)) - return PTR_ERR(job); - job_count++; + nents_offset = 0; + seq_no = 0; + job = NULL; + while (nents_xfer > nents_offset) { + job = qaic_create_job(file_priv, slice, nents_offset, nents_xfer, + seq_no++, tmp_list); + if (IS_ERR(job)) + goto free_jobs; + + job_count++; + nents_offset += job->num_req; + } /* job points to the last job for this slice, only valid for partial execute ioctl */ job->partial_size = last_bytes; job->partial = partial; return job_count; + +free_jobs: + list_for_each_entry_safe_reverse(job, job_i, tmp_list, queue) + if (job_count--) + qaic_cleanup_job(job); + else + break; + return job_count; } static int send_bo_list_to_sched(struct qaic_device *qdev, struct drm_file *file_priv, diff --git a/drivers/accel/qaic/qaic_sched.c b/drivers/accel/qaic/qaic_sched.c index b0992d63c2b6..ded0bdccdfdb 100644 --- a/drivers/accel/qaic/qaic_sched.c +++ b/drivers/accel/qaic/qaic_sched.c @@ -41,8 +41,8 @@ static struct dma_fence *qaic_job_run(struct drm_sched_job *sched_job) if (ret) goto exit; - ret = qaic_submit_reqs_to_hw(slice, job->num_req, job->partial, - job->partial_size, job->id); + ret = qaic_submit_reqs_to_hw(slice, &slice->reqs[job->req_offset], job->num_req, + job->partial, job->partial_size, job->id); if (ret) goto unlock_dbc; mutex_unlock(&dbc->req_lock); @@ -146,15 +146,19 @@ void qaic_sched_entity_init(struct dma_bridge_chan *dbc) } struct qaic_job *qaic_create_job(struct drm_file *file_priv, struct bo_slice *slice, - unsigned int num_req, u64 seq_no, struct list_head *tmp_list) + unsigned int req_offset, unsigned int num_full_reqs, u64 seq_no, + struct list_head *tmp_list) { struct dma_bridge_chan *dbc = slice->bo->dbc; struct qaic_bo *bo = slice->bo; - struct qaic_job *job; + struct qaic_job *job, *dep_job; + unsigned int num_req; u32 credits; u16 req_id; int ret, i; + num_req = min(slice->bo->dbc->nelem - 1, num_full_reqs - req_offset); + if (check_mul_overflow((u32)num_req, dbc->credit_ratio, &credits)) return ERR_PTR(-EINVAL); @@ -185,13 +189,24 @@ struct qaic_job *qaic_create_job(struct drm_file *file_priv, struct bo_slice *sl goto job_cleanup; job->slice = slice; + job->req_offset = req_offset; job->num_req = num_req; job->dbc = dbc; INIT_LIST_HEAD(&job->queue); job->id = req_id; kref_init(&job->ref_count); for (i = 0; i < job->num_req; i++) - slice->reqs[i].req_id = cpu_to_le16(job->id); + slice->reqs[i + job->req_offset].req_id = cpu_to_le16(job->id); + + /* Handle BOs that have been split into many jobs */ + if (seq_no) { + dep_job = list_last_entry(tmp_list, struct qaic_job, queue); + ret = drm_sched_job_add_dependency(&job->base, + dma_fence_get(&dep_job->irq_fence->base)); + /* add_dependency() will put() the fence on error */ + if (ret) + goto job_cleanup; + } list_add_tail(&job->queue, tmp_list); drm_gem_object_get(&bo->base); -- 2.43.0
