Although VCE has HW limits much smaller than this and should fail without causing damage, prevent the overflow just in case.
Signed-off-by: Benjamin Cheng <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index efdebd9c0a1f..d73be4d7eff2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -877,9 +877,15 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, goto out; } - *size = amdgpu_ib_get_value(ib, idx + 8) * - amdgpu_ib_get_value(ib, idx + 10) * - 8 * 3 / 2; + if (check_mul_overflow(amdgpu_ib_get_value(ib, idx + 8), + amdgpu_ib_get_value(ib, idx + 10), + &tmp) || + check_mul_overflow(tmp, 8 * 3 / 2, size)) { + DRM_ERROR("enc size overflow!\n"); + r = -EINVAL; + goto out; + } + break; case 0x04000001: /* config extension */ -- 2.54.0
