[Mesa-dev] [PATCH] swrast: Fix unsigned promotion in pointer arithmetic

2012-01-23 Thread Chad Versace 
When rowstride was negatie, unsigned promotion caused a segfault here:

299│if (rb-Format == MESA_FORMAT_S8) {
300│   const GLuint rowStride = rb-RowStride;
301│   for (i = 0; i  count; i++) {
302│  if (x[i] = 0  y[i] = 0  x[i]  w  y[i]  h) {
303├stencil[i] = *(map + y[i] * rowStride + x[i]);
304│  }
305│   }
306│}

Fixes segfault in oglconform
separatestencil-neu(NonPolygon.BothFacesBitmapCoreAPI),
though test still fails.

Note: This is a candidate for the stable branches.
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=43327
Signed-off-by: Chad Versace chad.vers...@linux.intel.com
---
 src/mesa/swrast/s_stencil.c |6 +++---
 1 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/mesa/swrast/s_stencil.c b/src/mesa/swrast/s_stencil.c
index dbcbd2b..fb95ef1 100644
--- a/src/mesa/swrast/s_stencil.c
+++ b/src/mesa/swrast/s_stencil.c
@@ -297,7 +297,7 @@ get_s8_values(struct gl_context *ctx, struct 
gl_renderbuffer *rb,
GLuint i;
 
if (rb-Format == MESA_FORMAT_S8) {
-  const GLuint rowStride = rb-RowStride;
+  const GLint rowStride = rb-RowStride;
   for (i = 0; i  count; i++) {
  if (x[i] = 0  y[i] = 0  x[i]  w  y[i]  h) {
 stencil[i] = *(map + y[i] * rowStride + x[i]);
@@ -305,8 +305,8 @@ get_s8_values(struct gl_context *ctx, struct 
gl_renderbuffer *rb,
   }
}
else {
-  const GLuint bpp = _mesa_get_format_bytes(rb-Format);
-  const GLuint rowStride = rb-RowStride * bpp;
+  const GLint bpp = _mesa_get_format_bytes(rb-Format);
+  const GLint rowStride = rb-RowStride * bpp;
   for (i = 0; i  count; i++) {
  if (x[i] = 0  y[i] = 0  x[i]  w  y[i]  h) {
 const GLubyte *src = map + y[i] * rowStride + x[i] * bpp;
-- 
1.7.7.5

___
mesa-dev mailing list
mesa-dev@lists.freedesktop.org
http://lists.freedesktop.org/mailman/listinfo/mesa-dev

Re: [Mesa-dev] [PATCH] swrast: Fix unsigned promotion in pointer arithmetic

2012-01-23 Thread Brian Paul 
On Mon, Jan 23, 2012 at 10:15 AM, Chad Versace
chad.vers...@linux.intel.com wrote:
 When rowstride was negatie, unsigned promotion caused a segfault here:

 299│    if (rb-Format == MESA_FORMAT_S8) {
 300│       const GLuint rowStride = rb-RowStride;
 301│       for (i = 0; i  count; i++) {
 302│          if (x[i] = 0  y[i] = 0  x[i]  w  y[i]  h) {
 303├            stencil[i] = *(map + y[i] * rowStride + x[i]);
 304│          }
 305│       }
 306│    }

 Fixes segfault in oglconform
 separatestencil-neu(NonPolygon.BothFacesBitmapCoreAPI),
 though test still fails.

 Note: This is a candidate for the stable branches.
 Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=43327
 Signed-off-by: Chad Versace chad.vers...@linux.intel.com
 ---
  src/mesa/swrast/s_stencil.c |    6 +++---
  1 files changed, 3 insertions(+), 3 deletions(-)

 diff --git a/src/mesa/swrast/s_stencil.c b/src/mesa/swrast/s_stencil.c
 index dbcbd2b..fb95ef1 100644
 --- a/src/mesa/swrast/s_stencil.c
 +++ b/src/mesa/swrast/s_stencil.c
 @@ -297,7 +297,7 @@ get_s8_values(struct gl_context *ctx, struct 
 gl_renderbuffer *rb,
    GLuint i;

    if (rb-Format == MESA_FORMAT_S8) {
 -      const GLuint rowStride = rb-RowStride;
 +      const GLint rowStride = rb-RowStride;
       for (i = 0; i  count; i++) {
          if (x[i] = 0  y[i] = 0  x[i]  w  y[i]  h) {
             stencil[i] = *(map + y[i] * rowStride + x[i]);
 @@ -305,8 +305,8 @@ get_s8_values(struct gl_context *ctx, struct 
 gl_renderbuffer *rb,
       }
    }
    else {
 -      const GLuint bpp = _mesa_get_format_bytes(rb-Format);
 -      const GLuint rowStride = rb-RowStride * bpp;
 +      const GLint bpp = _mesa_get_format_bytes(rb-Format);
 +      const GLint rowStride = rb-RowStride * bpp;
       for (i = 0; i  count; i++) {
          if (x[i] = 0  y[i] = 0  x[i]  w  y[i]  h) {
             const GLubyte *src = map + y[i] * rowStride + x[i] * bpp;
 --
 1.7.7.5

Looks good.

Reviewed-by: Brian Paul bri...@vmware.com
___
mesa-dev mailing list
mesa-dev@lists.freedesktop.org
http://lists.freedesktop.org/mailman/listinfo/mesa-dev