[PATCH] vmwgfx: integer overflow in vmw_kms_update_layout_ioctl()

2011-11-28 Thread Thomas Hellstrom
From: Xi Wang 

There are two issues in vmw_kms_update_layout_ioctl().  First, the
for loop forgets to index rects and only checks the first element.
Second, there is a potential integer overflow if userspace passes
in a large arg->num_outputs.  The call to kzalloc() would allocate
a small buffer, leading to out-of-bounds read.

Reported-by: Haogang Chen 
Signed-off-by: Xi Wang 
Signed-off-by: Thomas Hellstrom 
---
 drivers/gpu/drm/vmwgfx/vmwgfx_kms.c |   11 ++-
 1 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c 
b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 880e285..37d4054 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -1809,7 +1809,8 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, 
void *data,
}

rects_size = arg->num_outputs * sizeof(struct drm_vmw_rect);
-   rects = kzalloc(rects_size, GFP_KERNEL);
+   rects = kcalloc(arg->num_outputs, sizeof(struct drm_vmw_rect),
+   GFP_KERNEL);
if (unlikely(!rects)) {
ret = -ENOMEM;
goto out_unlock;
@@ -1824,10 +1825,10 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, 
void *data,
}

for (i = 0; i < arg->num_outputs; ++i) {
-   if (rects->x < 0 ||
-   rects->y < 0 ||
-   rects->x + rects->w > mode_config->max_width ||
-   rects->y + rects->h > mode_config->max_height) {
+   if (rects[i].x < 0 ||
+   rects[i].y < 0 ||
+   rects[i].x + rects[i].w > mode_config->max_width ||
+   rects[i].y + rects[i].h > mode_config->max_height) {
DRM_ERROR("Invalid GUI layout.\n");
ret = -EINVAL;
goto out_free;
-- 
1.7.4.4



[PATCH] vmwgfx: integer overflow in vmw_kms_update_layout_ioctl()

2011-11-28 Thread Thomas Hellstrom
From: Xi Wang 

There are two issues in vmw_kms_update_layout_ioctl().  First, the
for loop forgets to index rects and only checks the first element.
Second, there is a potential integer overflow if userspace passes
in a large arg->num_outputs.  The call to kzalloc() would allocate
a small buffer, leading to out-of-bounds read.

Reported-by: Haogang Chen 
Signed-off-by: Xi Wang 
Signed-off-by: Thomas Hellstrom 
---
 drivers/gpu/drm/vmwgfx/vmwgfx_kms.c |   11 ++-
 1 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c 
b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 880e285..37d4054 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -1809,7 +1809,8 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, 
void *data,
}
 
rects_size = arg->num_outputs * sizeof(struct drm_vmw_rect);
-   rects = kzalloc(rects_size, GFP_KERNEL);
+   rects = kcalloc(arg->num_outputs, sizeof(struct drm_vmw_rect),
+   GFP_KERNEL);
if (unlikely(!rects)) {
ret = -ENOMEM;
goto out_unlock;
@@ -1824,10 +1825,10 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, 
void *data,
}
 
for (i = 0; i < arg->num_outputs; ++i) {
-   if (rects->x < 0 ||
-   rects->y < 0 ||
-   rects->x + rects->w > mode_config->max_width ||
-   rects->y + rects->h > mode_config->max_height) {
+   if (rects[i].x < 0 ||
+   rects[i].y < 0 ||
+   rects[i].x + rects[i].w > mode_config->max_width ||
+   rects[i].y + rects[i].h > mode_config->max_height) {
DRM_ERROR("Invalid GUI layout.\n");
ret = -EINVAL;
goto out_free;
-- 
1.7.4.4

___
dri-devel mailing list
dri-devel@lists.freedesktop.org
http://lists.freedesktop.org/mailman/listinfo/dri-devel