[libav-devel] [PATCH] ff_get_format: call ->get_format again if hwaccel init fails

[Rémi Denis-Courmont]

This allows the application to fallback to another hwaccel, or more
likely, to software decoding.
---
 libavcodec/utils.c | 46 ++++++++++++++++++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 10 deletions(-)

diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index c5fa50d..317e90f 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -902,11 +902,26 @@ static AVHWAccel *find_hwaccel(enum AVCodecID codec_id,
 int ff_get_format(AVCodecContext *avctx, const enum AVPixelFormat *fmt)
 {
     const AVPixFmtDescriptor *desc;
-    enum AVPixelFormat ret = avctx->get_format(avctx, fmt);
+    enum AVPixelFormat *choices;
+    enum AVPixelFormat ret;
+    unsigned n = 0;
+
+    while (fmt[n] != AV_PIX_FMT_NONE)
+        ++n;
+
+    choices = av_malloc(++n * sizeof(*choices));
+    if (choices == NULL)
+        return AV_PIX_FMT_NONE;
+
+    memcpy(choices, fmt, n * sizeof(*choices));
+retry:
+    ret = avctx->get_format(avctx, choices);
 
     desc = av_pix_fmt_desc_get(ret);
-    if (!desc)
+    if (!desc) {
+        av_freep(&choices);
         return AV_PIX_FMT_NONE;
+    }
 
     if (avctx->hwaccel && avctx->hwaccel->uninit)
         avctx->hwaccel->uninit(avctx);
@@ -915,32 +930,43 @@ int ff_get_format(AVCodecContext *avctx, const enum 
AVPixelFormat *fmt)
 
     if (desc->flags & AV_PIX_FMT_FLAG_HWACCEL) {
         AVHWAccel *hwaccel;
-        int err;
+        int err = 0;
 
         hwaccel = find_hwaccel(avctx->codec_id, ret);
         if (!hwaccel) {
             av_log(avctx, AV_LOG_ERROR,
                    "Could not find an AVHWAccel for the pixel format: %s",
                    desc->name);
-            return AV_PIX_FMT_NONE;
+            err = AVERROR(ENOENT);
         }
 
-        if (hwaccel->priv_data_size) {
+        if (err == 0 && hwaccel->priv_data_size) {
             avctx->internal->hwaccel_priv_data = 
av_mallocz(hwaccel->priv_data_size);
             if (!avctx->internal->hwaccel_priv_data)
-                return AV_PIX_FMT_NONE;
+                err = AVERROR(ENOMEM);
         }
 
-        if (hwaccel->init) {
+        if (err == 0 && hwaccel->init) {
             err = hwaccel->init(avctx);
-            if (err < 0) {
+            if (err < 0)
                 av_freep(&avctx->internal->hwaccel_priv_data);
-                return AV_PIX_FMT_NONE;
-            }
+        }
+
+        if (err < 0) {
+            /* Remove failed hwaccel from choices and reiterate */
+            for (n = 0; fmt[n] != ret; n++)
+                av_assert0(fmt[n] != AV_PIX_FMT_NONE);
+
+            do
+                choices[n] = fmt[n + 1];
+            while (choices[n] != AV_PIX_FMT_NONE);
+
+            goto retry;
         }
         avctx->hwaccel = hwaccel;
     }
 
+    av_freep(&choices);
     return ret;
 }
 
-- 
1.8.1.5

_______________________________________________
libav-devel mailing list
libav-devel@libav.org
https://lists.libav.org/mailman/listinfo/libav-devel