A secondary weight or scale stream must be explicitly disabled or
configured when the hardware uses it. Otherwise, stale stream state from
a previous command stream can bypass buffer bounds validation.
Derive the active stream count from the NPU configuration on U85 and
from PARALLEL_MODE on U65. Require every active stream's length command,
while allowing a zero length to disable it.
Fixes: 5a5e9c0228e6 ("accel: Add Arm Ethos-U NPU driver")
Cc: [email protected]
Signed-off-by: Rob Herring (Arm) <[email protected]>
---
v4:
- New patch
---
drivers/accel/ethosu/ethosu_device.h | 4 +++
drivers/accel/ethosu/ethosu_gem.c | 60 +++++++++++++++++++++++++++++++-----
2 files changed, 57 insertions(+), 7 deletions(-)
diff --git a/drivers/accel/ethosu/ethosu_device.h
b/drivers/accel/ethosu/ethosu_device.h
index 135c33f57045..f41a8e0e1e1b 100644
--- a/drivers/accel/ethosu/ethosu_device.h
+++ b/drivers/accel/ethosu/ethosu_device.h
@@ -63,6 +63,7 @@ struct gen_pool;
#define CONFIG_MACS_PER_CC_MASK GENMASK(3, 0)
#define CONFIG_CMD_STREAM_VER_MASK GENMASK(7, 4)
+#define CONFIG_NUM_WD_MASK GENMASK(13, 12)
#define STATUS_STATE_RUNNING BIT(0)
#define STATUS_IRQ_RAISED BIT(1)
@@ -130,6 +131,7 @@ enum ethosu_cmds {
NPU_SET_KERNEL_WIDTH_M1 = 0x120,
NPU_SET_KERNEL_HEIGHT_M1 = 0x121,
NPU_SET_KERNEL_STRIDE = 0x122,
+ NPU_SET_PARALLEL_MODE = 0x123, // U65 only
NPU_SET_ACC_FORMAT = 0x124,
NPU_SET_WEIGHT_REGION = 0x128,
NPU_SET_SCALE_REGION = 0x129,
@@ -193,6 +195,8 @@ enum ethosu_cmds {
#define NPU_ACC_FORMAT_INPUT_MASK GENMASK(5, 4)
#define NPU_ACC_INPUT_IFM2 2
+#define NPU_MACS_PER_CC_256 8
+#define NPU_PARALLEL_MODE_DUAL_CORE_DEPTH 1
#define ETHOSU_SRAM_REGION 2 /* Matching Vela compiler */
diff --git a/drivers/accel/ethosu/ethosu_gem.c
b/drivers/accel/ethosu/ethosu_gem.c
index 1935440fb815..21458aacfcad 100644
--- a/drivers/accel/ethosu/ethosu_gem.c
+++ b/drivers/accel/ethosu/ethosu_gem.c
@@ -165,6 +165,7 @@ struct cmd_state {
DECLARE_BITMAP(cmd0, NPU_CMD0_REGS);
DECLARE_BITMAP(cmd1, NPU_CMD1_REGS);
bool acc_input_ifm2;
+ u8 parallel_mode;
struct dma_state dma;
struct buffer scale[2];
struct buffer weight[4];
@@ -767,19 +768,45 @@ static int calc_sizes_resize(struct drm_device *ddev,
return 0;
}
+static int weight_streams(struct ethosu_device *edev, struct cmd_state *st,
+ u8 *streams)
+{
+ if (ethosu_is_u65(edev)) {
+ if (FIELD_GET(CONFIG_MACS_PER_CC_MASK, edev->npu_info.config) ==
+ NPU_MACS_PER_CC_256) {
+ *streams = 1;
+ return 0;
+ }
+
+ if (!cmd_state_reg_is_set(st, NPU_SET_PARALLEL_MODE))
+ return -EINVAL;
+
+ *streams = st->parallel_mode + 1;
+ } else {
+ *streams = 1 << FIELD_GET(CONFIG_NUM_WD_MASK,
+ edev->npu_info.config);
+ }
+
+ return 0;
+}
+
static int buffer_size(struct ethosu_validated_cmdstream_info *info,
struct cmd_state *st, struct buffer *buf, s8 region,
- u16 region_cmd, u16 base_cmd, u16 length_cmd, bool
optional)
+ u16 region_cmd, u16 base_cmd, u16 length_cmd, bool
can_omit)
{
u64 end;
bool base_set = cmd_state_reg_is_set(st, base_cmd);
bool length_set = cmd_state_reg_is_set(st, length_cmd);
- if (optional && !base_set && !length_set)
+ if (can_omit && !length_set)
return 0;
if (region < 0 || !cmd_state_reg_is_set(st, region_cmd) ||
- !base_set || !length_set)
+ !length_set)
+ return -EINVAL;
+ if (!buf->length)
+ return 0;
+ if (!base_set)
return -EINVAL;
if (check_add_overflow(buf->base, (u64)buf->length, &end))
@@ -797,6 +824,7 @@ static int calc_sizes(struct drm_device *ddev,
{
struct ethosu_device *edev = to_ethosu_device(ddev);
u64 len;
+ u8 streams;
int ret;
if (ifm) {
@@ -848,6 +876,10 @@ static int calc_sizes(struct drm_device *ddev,
}
if (weight) {
+ ret = weight_streams(edev, st, &streams);
+ if (ret)
+ return ret;
+
dev_dbg(ddev->dev, "op %d: W:%d:0x%llx-0x%llx\n",
op, st->weight[0].region, st->weight[0].base,
st->weight[0].base + st->weight[0].length - 1);
@@ -858,19 +890,23 @@ static int calc_sizes(struct drm_device *ddev,
if (buffer_size(info, st, &st->weight[1], st->weight[0].region,
NPU_SET_WEIGHT_REGION, NPU_SET_WEIGHT1_BASE,
- NPU_SET_WEIGHT1_LENGTH, true) ||
+ NPU_SET_WEIGHT1_LENGTH, streams < 2) ||
buffer_size(info, st, &st->weight[3], st->weight[0].region,
NPU_SET_WEIGHT_REGION, NPU_SET_WEIGHT3_BASE,
- NPU_SET_WEIGHT3_LENGTH, true))
+ NPU_SET_WEIGHT3_LENGTH, streams < 4))
return -EINVAL;
if (!ethosu_is_u65(edev) &&
buffer_size(info, st, &st->weight[2], st->weight[0].region,
NPU_SET_WEIGHT_REGION, NPU_SET_WEIGHT2_BASE,
- NPU_SET_WEIGHT2_LENGTH, true))
+ NPU_SET_WEIGHT2_LENGTH, streams < 3))
return -EINVAL;
}
if (scale) {
+ ret = weight_streams(edev, st, &streams);
+ if (ret)
+ return ret;
+
dev_dbg(ddev->dev, "op %d: S:%d:0x%llx-0x%llx\n",
op, st->scale[0].region, st->scale[0].base,
st->scale[0].base + st->scale[0].length - 1);
@@ -882,7 +918,7 @@ static int calc_sizes(struct drm_device *ddev,
if (ethosu_is_u65(edev) &&
buffer_size(info, st, &st->scale[1], st->scale[0].region,
NPU_SET_SCALE_REGION, NPU_SET_SCALE1_BASE,
- NPU_SET_SCALE1_LENGTH, true))
+ NPU_SET_SCALE1_LENGTH, streams < 2))
return -EINVAL;
}
@@ -1093,6 +1129,16 @@ static int ethosu_gem_cmdstream_copy_and_validate(struct
drm_device *ddev,
case NPU_SET_KERNEL_STRIDE:
st.ifm.stride_kernel = param;
break;
+ case NPU_SET_PARALLEL_MODE:
+ if (!ethosu_is_u65(edev) ||
+ param > NPU_PARALLEL_MODE_DUAL_CORE_DEPTH)
+ return -EINVAL;
+ if (FIELD_GET(CONFIG_MACS_PER_CC_MASK,
edev->npu_info.config) ==
+ NPU_MACS_PER_CC_256 &&
+ param)
+ return -EINVAL;
+ st.parallel_mode = param;
+ break;
case NPU_SET_ACC_FORMAT:
if (!ethosu_is_u65(edev)) {
u32 acc_input =
FIELD_GET(NPU_ACC_FORMAT_INPUT_MASK, param);
--
2.53.0
