[libav-devel] [PATCH] swscale: fix clipping of 9/10bit YUV420P.

[Ronald S. Bultje]

---
 libswscale/swscale.c |   36 +++++++++++++++++++-----------------
 1 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index b63a386..8131927 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -209,6 +209,7 @@ DECLARE_ALIGNED(8, const uint8_t, dither_8x8_220)[8][8]={
 };
 #endif
 
+#define POW2CLIP(x,max) (((x) & ~max) ? (-(x))>>31 & max : (x))
 static av_always_inline void yuv2yuvX16inC_template(const int16_t *lumFilter, 
const int16_t **lumSrc, int lumFilterSize,
                                                     const int16_t *chrFilter, 
const int16_t **chrSrc, int chrFilterSize,
                                                     const int16_t **alpSrc, 
uint16_t *dest, uint16_t *uDest, uint16_t *vDest, uint16_t *aDest,
@@ -218,6 +219,20 @@ static av_always_inline void yuv2yuvX16inC_template(const 
int16_t *lumFilter, co
     int i;
     int shift = 11 + 16 - output_bits;
 
+#define output_pixel(pos, val) \
+    if (big_endian) { \
+        if (output_bits == 16) { \
+            AV_WB16(pos, av_clip_uint16(val >> shift)); \
+        } else { \
+            AV_WB16(pos, POW2CLIP(val >> shift, (1 << output_bits) - 1)); \
+        } \
+    } else { \
+        if (output_bits == 16) { \
+            AV_WL16(pos, av_clip_uint16(val >> shift)); \
+        } else { \
+            AV_WL16(pos, POW2CLIP(val >> shift, (1 << output_bits) - 1)); \
+        } \
+    }
     for (i = 0; i < dstW; i++) {
         int val = 1 << 10;
         int j;
@@ -225,11 +240,7 @@ static av_always_inline void yuv2yuvX16inC_template(const 
int16_t *lumFilter, co
         for (j = 0; j < lumFilterSize; j++)
             val += lumSrc[j][i] * lumFilter[j];
 
-        if (big_endian) {
-            AV_WB16(&dest[i], av_clip_uint16(val >> shift));
-        } else {
-            AV_WL16(&dest[i], av_clip_uint16(val >> shift));
-        }
+        output_pixel(&dest[i], val);
     }
 
     if (uDest) {
@@ -243,13 +254,8 @@ static av_always_inline void yuv2yuvX16inC_template(const 
int16_t *lumFilter, co
                 v += chrSrc[j][i + VOFW] * chrFilter[j];
             }
 
-            if (big_endian) {
-                AV_WB16(&uDest[i], av_clip_uint16(u >> shift));
-                AV_WB16(&vDest[i], av_clip_uint16(v >> shift));
-            } else {
-                AV_WL16(&uDest[i], av_clip_uint16(u >> shift));
-                AV_WL16(&vDest[i], av_clip_uint16(v >> shift));
-            }
+            output_pixel(&uDest[i], u);
+            output_pixel(&vDest[i], v);
         }
     }
 
@@ -261,11 +267,7 @@ static av_always_inline void yuv2yuvX16inC_template(const 
int16_t *lumFilter, co
             for (j = 0; j < lumFilterSize; j++)
                 val += alpSrc[j][i] * lumFilter[j];
 
-            if (big_endian) {
-                AV_WB16(&aDest[i], av_clip_uint16(val >> shift));
-            } else {
-                AV_WL16(&aDest[i], av_clip_uint16(val >> shift));
-            }
+            output_pixel(&aDest[i], val);
         }
     }
 }
-- 
1.7.4.4

_______________________________________________
libav-devel mailing list
libav-devel@libav.org
https://lists.libav.org/mailman/listinfo/libav-devel