[libav-devel] [PATCH] swscale: fix isGray() macro madness.

[Ronald S. Bultje]

---
 libswscale/swscale.c          |    3 +--
 libswscale/swscale_internal.h |   13 ++++---------
 libswscale/swscale_unscaled.c |    6 +++---
 libswscale/utils.c            |    4 +++-
 4 files changed, 11 insertions(+), 15 deletions(-)

diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 98daf22..514a291 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -2862,8 +2862,7 @@ static av_cold void sws_init_swScale_c(SwsContext *c)
         }
     }
 
-    if (!(isGray(srcFormat) || isGray(c->dstFormat) ||
-          srcFormat == PIX_FMT_MONOBLACK || srcFormat == PIX_FMT_MONOWHITE))
+    if (!(isGray(srcFormat) || isGray(c->dstFormat)))
         c->needs_hcscale = 1;
 }
 
diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h
index 7bfdc7e..d952859 100644
--- a/libswscale/swscale_internal.h
+++ b/libswscale/swscale_internal.h
@@ -552,18 +552,13 @@ const char *sws_format_name(enum PixelFormat format);
 #define isRGB(x) \
     (av_pix_fmt_descriptors[x].flags & PIX_FMT_RGB)
 
-#if 0 // FIXME
 #define isGray(x) \
     (!(av_pix_fmt_descriptors[x].flags & PIX_FMT_PAL) && \
      av_pix_fmt_descriptors[x].nb_components <= 2)
-#else
-#define isGray(x)       (           \
-           (x)==PIX_FMT_GRAY8       \
-        || (x)==PIX_FMT_Y400A      \
-        || (x)==PIX_FMT_GRAY16BE    \
-        || (x)==PIX_FMT_GRAY16LE    \
-    )
-#endif
+
+#define isGrayMod8(x) \
+    (isGray(x) && \
+     (av_pix_fmt_descriptors[x].comp[0].depth_minus1 & 7) == 7)
 
 #define isRGBinInt(x)   (           \
            (x)==PIX_FMT_RGB48BE     \
diff --git a/libswscale/swscale_unscaled.c b/libswscale/swscale_unscaled.c
index 86bc5ac..f9017dc 100644
--- a/libswscale/swscale_unscaled.c
+++ b/libswscale/swscale_unscaled.c
@@ -754,9 +754,9 @@ void ff_get_unscaled_swscale(SwsContext *c)
     if (  srcFormat == dstFormat
         || (srcFormat == PIX_FMT_YUVA420P && dstFormat == PIX_FMT_YUV420P)
         || (srcFormat == PIX_FMT_YUV420P && dstFormat == PIX_FMT_YUVA420P)
-        || (isPlanarYUV(srcFormat) && isGray(dstFormat))
-        || (isPlanarYUV(dstFormat) && isGray(srcFormat))
-        || (isGray(dstFormat) && isGray(srcFormat))
+        || (isPlanarYUV(srcFormat) && isGrayMod8(dstFormat))
+        || (isPlanarYUV(dstFormat) && isGrayMod8(srcFormat))
+        || (isGrayMod8(dstFormat) && isGrayMod8(srcFormat))
         || (isPlanarYUV(srcFormat) && isPlanarYUV(dstFormat)
             && c->chrDstHSubSample == c->chrSrcHSubSample
             && c->chrDstVSubSample == c->chrSrcVSubSample
diff --git a/libswscale/utils.c b/libswscale/utils.c
index 7a96986..385b558 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -698,7 +698,9 @@ int sws_setColorspaceDetails(struct SwsContext *c, const 
int inv_table[4],
     c->saturation= saturation;
     c->srcRange  = srcRange;
     c->dstRange  = dstRange;
-    if (isYUV(c->dstFormat) || isGray(c->dstFormat)) return -1;
+    // FIXME the isGrayMod8 here is silly, we're creating (and using)
+    // rgb tables for monowhite/monoblack output
+    if (isYUV(c->dstFormat) || isGrayMod8(c->dstFormat)) return -1;
 
     c->dstFormatBpp = 
av_get_bits_per_pixel(&av_pix_fmt_descriptors[c->dstFormat]);
     c->srcFormatBpp = 
av_get_bits_per_pixel(&av_pix_fmt_descriptors[c->srcFormat]);
-- 
1.7.2.1

_______________________________________________
libav-devel mailing list
libav-devel@libav.org
https://lists.libav.org/mailman/listinfo/libav-devel