The timeout worker and threaded IRQ handler can both clear
in_flight_job. If a new job is scheduled in between those,
in_flight_job could be overwritten when the IRQ thread resumes.
Serialize these updates so a late completion cannot clear the next job's
tracking pointer.
Fixes: 5a5e9c0228e6 ("accel: Add Arm Ethos-U NPU driver")
Cc: [email protected]
Signed-off-by: Rob Herring (Arm) <[email protected]>
---
v4:
- New patch
---
drivers/accel/ethosu/ethosu_device.h | 1 +
drivers/accel/ethosu/ethosu_job.c | 43 +++++++++++++++++++++++++++++-------
2 files changed, 36 insertions(+), 8 deletions(-)
diff --git a/drivers/accel/ethosu/ethosu_device.h
b/drivers/accel/ethosu/ethosu_device.h
index 1eca8590e68d..51d5bc5d6137 100644
--- a/drivers/accel/ethosu/ethosu_device.h
+++ b/drivers/accel/ethosu/ethosu_device.h
@@ -7,6 +7,7 @@
#include <linux/bitfield.h>
#include <linux/bits.h>
#include <linux/mutex.h>
+#include <linux/spinlock.h>
#include <linux/types.h>
#include <drm/drm_device.h>
diff --git a/drivers/accel/ethosu/ethosu_job.c
b/drivers/accel/ethosu/ethosu_job.c
index d418ae7961cf..f22cacb19e3d 100644
--- a/drivers/accel/ethosu/ethosu_job.c
+++ b/drivers/accel/ethosu/ethosu_job.c
@@ -42,6 +42,25 @@ static const struct dma_fence_ops ethosu_fence_ops = {
.get_timeline_name = ethosu_fence_get_timeline_name,
};
+static void ethosu_job_set_in_flight(struct ethosu_device *dev,
+ struct ethosu_job *job)
+{
+ guard(spinlock)(&dev->job_lock);
+ dev->in_flight_job = job;
+}
+
+static struct ethosu_job *
+ethosu_job_clear_in_flight(struct ethosu_device *dev)
+{
+ struct ethosu_job *job;
+
+ guard(spinlock)(&dev->job_lock);
+ job = dev->in_flight_job;
+ dev->in_flight_job = NULL;
+
+ return job;
+}
+
static void ethosu_job_hw_submit(struct ethosu_device *dev, struct ethosu_job
*job)
{
struct drm_gem_dma_object *cmd_bo = to_drm_gem_dma_obj(job->cmd_bo);
@@ -235,7 +254,7 @@ static struct dma_fence *ethosu_job_run(struct
drm_sched_job *sched_job)
ethosu_switch_perfmon(dev, job);
- WRITE_ONCE(dev->in_flight_job, job);
+ ethosu_job_set_in_flight(dev, job);
ethosu_job_hw_submit(dev, job);
return fence;
@@ -252,11 +271,9 @@ static void ethosu_job_handle_irq(struct ethosu_device
*dev)
return;
}
- job = READ_ONCE(dev->in_flight_job);
- if (job) {
- WRITE_ONCE(dev->in_flight_job, NULL);
+ job = ethosu_job_clear_in_flight(dev);
+ if (job)
dma_fence_signal(job->done_fence);
- }
}
static irqreturn_t ethosu_job_irq_handler_thread(int irq, void *data)
@@ -285,6 +302,7 @@ static enum drm_gpu_sched_stat ethosu_job_timedout(struct
drm_sched_job *bad)
struct ethosu_job *job = to_ethosu_job(bad);
struct ethosu_device *dev = job->dev;
bool running;
+ int ret;
u32 *bocmds = to_drm_gem_dma_obj(job->cmd_bo)->vaddr;
u32 cmdaddr;
@@ -311,11 +329,19 @@ static enum drm_gpu_sched_stat ethosu_job_timedout(struct
drm_sched_job *bad)
drm_sched_stop(&dev->sched, bad);
- WRITE_ONCE(dev->in_flight_job, NULL);
+ ethosu_job_clear_in_flight(dev);
/* Proceed with reset now. */
- pm_runtime_force_suspend(dev->base.dev);
- pm_runtime_force_resume(dev->base.dev);
+ ret = pm_runtime_force_suspend(dev->base.dev);
+ if (!ret)
+ ret = pm_runtime_force_resume(dev->base.dev);
+ if (ret) {
+ dev_err(dev->base.dev, "Failed to recover NPU: %d\n", ret);
+ return DRM_GPU_SCHED_STAT_ENODEV;
+ }
+
+ /* Drain a pending completion IRQ before a replacement job can run. */
+ synchronize_irq(dev->irq);
/* Restart the scheduler */
drm_sched_start(&dev->sched, 0);
@@ -344,6 +370,7 @@ int ethosu_job_init(struct ethosu_device *edev)
int ret;
spin_lock_init(&edev->fence_lock);
+ spin_lock_init(&edev->job_lock);
ret = devm_mutex_init(dev, &edev->sched_lock);
if (ret)
return ret;
--
2.53.0
