[git:media_stage/master] media: vivid: fix wrong sizeimage value for mplane

[Hans Verkuil]

This is an automatic generated email to let you know that the following patch 
were queued:

Subject: media: vivid: fix wrong sizeimage value for mplane
Author:  Hans Verkuil <hverkuil-ci...@xs4all.nl>
Date:    Wed Jun 26 12:59:13 2024 +0200

In several places a division by fmt->vdownsampling[p] was
missing in the sizeimage[p] calculation, causing incorrect
behavior for multiplanar formats were some planes are smaller
than the first plane.

Found by new v4l2-compliance tests.

Signed-off-by: Hans Verkuil <hverkuil-ci...@xs4all.nl>

 drivers/media/test-drivers/vivid/vivid-vid-cap.c |  5 +++--
 drivers/media/test-drivers/vivid/vivid-vid-out.c | 16 +++++++++-------
 2 files changed, 12 insertions(+), 9 deletions(-)

---

diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.c 
b/drivers/media/test-drivers/vivid/vivid-vid-cap.c
index 2804975fe278..3a3041a0378f 100644
--- a/drivers/media/test-drivers/vivid/vivid-vid-cap.c
+++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.c
@@ -106,8 +106,9 @@ static int vid_cap_queue_setup(struct vb2_queue *vq,
                if (*nplanes != buffers)
                        return -EINVAL;
                for (p = 0; p < buffers; p++) {
-                       if (sizes[p] < tpg_g_line_width(&dev->tpg, p) * h +
-                                               dev->fmt_cap->data_offset[p])
+                       if (sizes[p] < tpg_g_line_width(&dev->tpg, p) * h /
+                                       dev->fmt_cap->vdownsampling[p] +
+                                       dev->fmt_cap->data_offset[p])
                                return -EINVAL;
                }
        } else {
diff --git a/drivers/media/test-drivers/vivid/vivid-vid-out.c 
b/drivers/media/test-drivers/vivid/vivid-vid-out.c
index 1653b2988f7e..7a0f4c61ac80 100644
--- a/drivers/media/test-drivers/vivid/vivid-vid-out.c
+++ b/drivers/media/test-drivers/vivid/vivid-vid-out.c
@@ -63,14 +63,16 @@ static int vid_out_queue_setup(struct vb2_queue *vq,
                if (sizes[0] < size)
                        return -EINVAL;
                for (p = 1; p < planes; p++) {
-                       if (sizes[p] < dev->bytesperline_out[p] * h +
-                                      vfmt->data_offset[p])
+                       if (sizes[p] < dev->bytesperline_out[p] * h /
+                                       vfmt->vdownsampling[p] +
+                                       vfmt->data_offset[p])
                                return -EINVAL;
                }
        } else {
                for (p = 0; p < planes; p++)
-                       sizes[p] = p ? dev->bytesperline_out[p] * h +
-                                      vfmt->data_offset[p] : size;
+                       sizes[p] = p ? dev->bytesperline_out[p] * h /
+                                       vfmt->vdownsampling[p] +
+                                       vfmt->data_offset[p] : size;
        }
 
        *nplanes = planes;
@@ -124,7 +126,7 @@ static int vid_out_buf_prepare(struct vb2_buffer *vb)
 
        for (p = 0; p < planes; p++) {
                if (p)
-                       size = dev->bytesperline_out[p] * h;
+                       size = dev->bytesperline_out[p] * h / 
vfmt->vdownsampling[p];
                size += vb->planes[p].data_offset;
 
                if (vb2_get_plane_payload(vb, p) < size) {
@@ -331,8 +333,8 @@ int vivid_g_fmt_vid_out(struct file *file, void *priv,
        for (p = 0; p < mp->num_planes; p++) {
                mp->plane_fmt[p].bytesperline = dev->bytesperline_out[p];
                mp->plane_fmt[p].sizeimage =
-                       mp->plane_fmt[p].bytesperline * mp->height +
-                       fmt->data_offset[p];
+                       mp->plane_fmt[p].bytesperline * mp->height /
+                       fmt->vdownsampling[p] + fmt->data_offset[p];
        }
        for (p = fmt->buffers; p < fmt->planes; p++) {
                unsigned stride = dev->bytesperline_out[p];