[PATCH v2 6/6] drm/virtio: fix DRM_FORMAT_* handling
Use DRM_FORMAT_HOST_XRGB, so we are using the correct format code
on bigendian machines. Also set the quirk_addfb_prefer_host_byte_order
mode_config bit so drm_mode_addfb() asks for the correct format code.
Both DRM_FORMAT_* and VIRTIO_GPU_FORMAT_* are defined to be little
endian, so using a different mapping on bigendian machines is wrong.
It's there because of broken drm_mode_addfb() behavior. So with
drm_mode_addfb() being fixed we can fix this too.
While wading through the code I've noticed we have a little issue in
virtio: We attach a format to the bo when it is created
(DRM_IOCTL_MODE_CREATE_DUMB), not when we map it as framebuffer
(DRM_IOCTL_MODE_ADDFB). Easy way out: Support a single format only.
Pick DRM_FORMAT_HOST_XRGB, it is the only one actually used in
practice. Drop unused mappings in virtio_gpu_translate_format().
With this patch applied both ADDFB and ADDFB2 ioctls work correctly in
the virtio-gpu.ko driver on big endian machines. Without the patch only
ADDFB (which still seems to be used by the majority of userspace) works
correctly.
Signed-off-by: Gerd Hoffmann
---
drivers/gpu/drm/virtio/virtgpu_display.c | 5 +++
drivers/gpu/drm/virtio/virtgpu_fb.c | 2 +-
drivers/gpu/drm/virtio/virtgpu_gem.c | 7 +++--
drivers/gpu/drm/virtio/virtgpu_plane.c | 54 ++--
4 files changed, 13 insertions(+), 55 deletions(-)
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c
b/drivers/gpu/drm/virtio/virtgpu_display.c
index 25503b9335..14a13edc02 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -306,6 +306,10 @@ virtio_gpu_user_framebuffer_create(struct drm_device *dev,
struct virtio_gpu_framebuffer *virtio_gpu_fb;
int ret;
+ if (mode_cmd->pixel_format != DRM_FORMAT_HOST_XRGB &&
+ mode_cmd->pixel_format != DRM_FORMAT_HOST_ARGB)
+ return ERR_PTR(-ENOENT);
+
/* lookup object associated with res handle */
obj = drm_gem_object_lookup(file_priv, mode_cmd->handles[0]);
if (!obj)
@@ -354,6 +358,7 @@ int virtio_gpu_modeset_init(struct virtio_gpu_device *vgdev)
int i;
drm_mode_config_init(vgdev->ddev);
+ vgdev->ddev->mode_config.quirk_addfb_prefer_host_byte_order = true;
vgdev->ddev->mode_config.funcs = _gpu_mode_funcs;
vgdev->ddev->mode_config.helper_private = _mode_config_helpers;
diff --git a/drivers/gpu/drm/virtio/virtgpu_fb.c
b/drivers/gpu/drm/virtio/virtgpu_fb.c
index a121b1c795..9d87ebd645 100644
--- a/drivers/gpu/drm/virtio/virtgpu_fb.c
+++ b/drivers/gpu/drm/virtio/virtgpu_fb.c
@@ -233,7 +233,7 @@ static int virtio_gpufb_create(struct drm_fb_helper *helper,
mode_cmd.width = sizes->surface_width;
mode_cmd.height = sizes->surface_height;
mode_cmd.pitches[0] = mode_cmd.width * 4;
- mode_cmd.pixel_format = drm_mode_legacy_fb_format(32, 24);
+ mode_cmd.pixel_format = DRM_FORMAT_HOST_XRGB;
format = virtio_gpu_translate_format(mode_cmd.pixel_format);
if (format == 0)
diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c
b/drivers/gpu/drm/virtio/virtgpu_gem.c
index 0f2768eaca..82c817f37c 100644
--- a/drivers/gpu/drm/virtio/virtgpu_gem.c
+++ b/drivers/gpu/drm/virtio/virtgpu_gem.c
@@ -90,7 +90,10 @@ int virtio_gpu_mode_dumb_create(struct drm_file *file_priv,
uint32_t resid;
uint32_t format;
- pitch = args->width * ((args->bpp + 1) / 8);
+ if (args->bpp != 32)
+ return -EINVAL;
+
+ pitch = args->width * 4;
args->size = pitch * args->height;
args->size = ALIGN(args->size, PAGE_SIZE);
@@ -99,7 +102,7 @@ int virtio_gpu_mode_dumb_create(struct drm_file *file_priv,
if (ret)
goto fail;
- format = virtio_gpu_translate_format(DRM_FORMAT_XRGB);
+ format = virtio_gpu_translate_format(DRM_FORMAT_HOST_XRGB);
virtio_gpu_resource_id_get(vgdev, );
virtio_gpu_cmd_create_resource(vgdev, resid, format,
args->width, args->height);
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c
b/drivers/gpu/drm/virtio/virtgpu_plane.c
index dc5b5b2b7a..3221d50b9a 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -28,22 +28,11 @@
#include
static const uint32_t virtio_gpu_formats[] = {
- DRM_FORMAT_XRGB,
- DRM_FORMAT_ARGB,
- DRM_FORMAT_BGRX,
- DRM_FORMAT_BGRA,
- DRM_FORMAT_RGBX,
- DRM_FORMAT_RGBA,
- DRM_FORMAT_XBGR,
- DRM_FORMAT_ABGR,
+ DRM_FORMAT_HOST_XRGB,
};
static const uint32_t virtio_gpu_cursor_formats[] = {
-#ifdef __BIG_ENDIAN
- DRM_FORMAT_BGRA,
-#else
- DRM_FORMAT_ARGB,
-#endif
+ DRM_FORMAT_HOST_ARGB,
};
uint32_t virtio_gpu_translate_format(uint32_t drm_fourcc)
@@ -51,32 +40,6 @@ uint32_t
[PATCH v2 6/6] drm/virtio: fix DRM_FORMAT_* handling
Use DRM_FORMAT_HOST_XRGB, so we are using the correct format code
on bigendian machines. Also set the quirk_addfb_prefer_host_byte_order
mode_config bit so drm_mode_addfb() asks for the correct format code.
Both DRM_FORMAT_* and VIRTIO_GPU_FORMAT_* are defined to be little
endian, so using a different mapping on bigendian machines is wrong.
It's there because of broken drm_mode_addfb() behavior. So with
drm_mode_addfb() being fixed we can fix this too.
While wading through the code I've noticed we have a little issue in
virtio: We attach a format to the bo when it is created
(DRM_IOCTL_MODE_CREATE_DUMB), not when we map it as framebuffer
(DRM_IOCTL_MODE_ADDFB). Easy way out: Support a single format only.
Pick DRM_FORMAT_HOST_XRGB, it is the only one actually used in
practice. Drop unused mappings in virtio_gpu_translate_format().
With this patch applied both ADDFB and ADDFB2 ioctls work correctly in
the virtio-gpu.ko driver on big endian machines. Without the patch only
ADDFB (which still seems to be used by the majority of userspace) works
correctly.
Signed-off-by: Gerd Hoffmann
---
drivers/gpu/drm/virtio/virtgpu_display.c | 5 +++
drivers/gpu/drm/virtio/virtgpu_fb.c | 2 +-
drivers/gpu/drm/virtio/virtgpu_gem.c | 7 +++--
drivers/gpu/drm/virtio/virtgpu_plane.c | 54 ++--
4 files changed, 13 insertions(+), 55 deletions(-)
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c
b/drivers/gpu/drm/virtio/virtgpu_display.c
index 25503b9335..14a13edc02 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -306,6 +306,10 @@ virtio_gpu_user_framebuffer_create(struct drm_device *dev,
struct virtio_gpu_framebuffer *virtio_gpu_fb;
int ret;
+ if (mode_cmd->pixel_format != DRM_FORMAT_HOST_XRGB &&
+ mode_cmd->pixel_format != DRM_FORMAT_HOST_ARGB)
+ return ERR_PTR(-ENOENT);
+
/* lookup object associated with res handle */
obj = drm_gem_object_lookup(file_priv, mode_cmd->handles[0]);
if (!obj)
@@ -354,6 +358,7 @@ int virtio_gpu_modeset_init(struct virtio_gpu_device *vgdev)
int i;
drm_mode_config_init(vgdev->ddev);
+ vgdev->ddev->mode_config.quirk_addfb_prefer_host_byte_order = true;
vgdev->ddev->mode_config.funcs = _gpu_mode_funcs;
vgdev->ddev->mode_config.helper_private = _mode_config_helpers;
diff --git a/drivers/gpu/drm/virtio/virtgpu_fb.c
b/drivers/gpu/drm/virtio/virtgpu_fb.c
index a121b1c795..9d87ebd645 100644
--- a/drivers/gpu/drm/virtio/virtgpu_fb.c
+++ b/drivers/gpu/drm/virtio/virtgpu_fb.c
@@ -233,7 +233,7 @@ static int virtio_gpufb_create(struct drm_fb_helper *helper,
mode_cmd.width = sizes->surface_width;
mode_cmd.height = sizes->surface_height;
mode_cmd.pitches[0] = mode_cmd.width * 4;
- mode_cmd.pixel_format = drm_mode_legacy_fb_format(32, 24);
+ mode_cmd.pixel_format = DRM_FORMAT_HOST_XRGB;
format = virtio_gpu_translate_format(mode_cmd.pixel_format);
if (format == 0)
diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c
b/drivers/gpu/drm/virtio/virtgpu_gem.c
index 0f2768eaca..82c817f37c 100644
--- a/drivers/gpu/drm/virtio/virtgpu_gem.c
+++ b/drivers/gpu/drm/virtio/virtgpu_gem.c
@@ -90,7 +90,10 @@ int virtio_gpu_mode_dumb_create(struct drm_file *file_priv,
uint32_t resid;
uint32_t format;
- pitch = args->width * ((args->bpp + 1) / 8);
+ if (args->bpp != 32)
+ return -EINVAL;
+
+ pitch = args->width * 4;
args->size = pitch * args->height;
args->size = ALIGN(args->size, PAGE_SIZE);
@@ -99,7 +102,7 @@ int virtio_gpu_mode_dumb_create(struct drm_file *file_priv,
if (ret)
goto fail;
- format = virtio_gpu_translate_format(DRM_FORMAT_XRGB);
+ format = virtio_gpu_translate_format(DRM_FORMAT_HOST_XRGB);
virtio_gpu_resource_id_get(vgdev, );
virtio_gpu_cmd_create_resource(vgdev, resid, format,
args->width, args->height);
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c
b/drivers/gpu/drm/virtio/virtgpu_plane.c
index dc5b5b2b7a..3221d50b9a 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -28,22 +28,11 @@
#include
static const uint32_t virtio_gpu_formats[] = {
- DRM_FORMAT_XRGB,
- DRM_FORMAT_ARGB,
- DRM_FORMAT_BGRX,
- DRM_FORMAT_BGRA,
- DRM_FORMAT_RGBX,
- DRM_FORMAT_RGBA,
- DRM_FORMAT_XBGR,
- DRM_FORMAT_ABGR,
+ DRM_FORMAT_HOST_XRGB,
};
static const uint32_t virtio_gpu_cursor_formats[] = {
-#ifdef __BIG_ENDIAN
- DRM_FORMAT_BGRA,
-#else
- DRM_FORMAT_ARGB,
-#endif
+ DRM_FORMAT_HOST_ARGB,
};
uint32_t virtio_gpu_translate_format(uint32_t drm_fourcc)
@@ -51,32 +40,6 @@ uint32_t
