Jean-Baptiste Kempf pushed to branch master at VideoLAN / VLC
Commits:
1d43024f by Pierre Lamot at 2022-06-04T10:57:27+00:00
avcodec: moves the cleanup of the AVCodecContext to the va module
If the va module succeed to open but is identified as unusable before the end of
FFmpeg getFormat callback, the module may have to do some cleanups in the
AVCodecContext.
the close callback will be called with a NULL AVCodecContext outside the
getFormat callback as the AVCodecContext ownership is transferred to FFmpeg
- - - - -
0654e138 by Pierre Lamot at 2022-06-04T10:57:27+00:00
avcodec: fix crash in FFMPEG when VAAPI is initialized but can't be used
The typical case is an application that use vgl callback with a GLX context,
vaapi decoder will initiliaze properly and provide a hw_frames_ctx context, but
due to the lack of interop, the decoder won't be usable and fallback to the
sw
decoder, but hw_frames_ctx will still be set and will cause a crash later on.
- - - - -
7 changed files:
- modules/codec/avcodec/d3d11va.c
- modules/codec/avcodec/dxva2.c
- modules/codec/avcodec/va.c
- modules/codec/avcodec/va.h
- modules/codec/avcodec/vaapi.c
- modules/codec/avcodec/video.c
- modules/hw/vdpau/avcodec.c
Changes:
=====================================
modules/codec/avcodec/d3d11va.c
=====================================
@@ -209,7 +209,7 @@ static int Get(vlc_va_t *va, picture_t *pic, AVCodecContext
*ctx, AVFrame *frame
return VLC_SUCCESS;
}
-static void Close(vlc_va_t *va)
+static void Close(vlc_va_t *va, AVCodecContext* ctx)
{
vlc_va_sys_t *sys = va->sys;
@@ -218,6 +218,9 @@ static void Close(vlc_va_t *va)
if (sys->va_pool)
va_pool_Close(sys->va_pool);
+
+ if (ctx)
+ ctx->hwaccel_context = NULL;
}
static const struct vlc_va_operations ops = { Get, Close, };
@@ -297,7 +300,7 @@ static int Open(vlc_va_t *va, AVCodecContext *ctx, enum
AVPixelFormat hwfmt, con
return VLC_SUCCESS;
error:
- Close(va);
+ Close(va, ctx);
return err;
}
=====================================
modules/codec/avcodec/dxva2.c
=====================================
@@ -219,10 +219,13 @@ static int Get(vlc_va_t *va, picture_t *pic,
AVCodecContext *ctx, AVFrame *frame
return VLC_SUCCESS;
}
-static void Close(vlc_va_t *va)
+static void Close(vlc_va_t *va, AVCodecContext *ctx)
{
vlc_va_sys_t *sys = va->sys;
+ if (ctx)
+ ctx->hwaccel_context = NULL;
+
if (sys->vctx)
vlc_video_context_Release(sys->vctx);
@@ -322,7 +325,7 @@ static int Open(vlc_va_t *va, AVCodecContext *ctx, enum
AVPixelFormat hwfmt, con
return VLC_SUCCESS;
error:
- Close(va);
+ Close(va, ctx);
return VLC_EGENERIC;
}
/* */
=====================================
modules/codec/avcodec/va.c
=====================================
@@ -126,9 +126,9 @@ vlc_va_t *vlc_va_New(vlc_object_t *obj, AVCodecContext
*avctx,
return NULL;
}
-void vlc_va_Delete(vlc_va_t *va)
+void vlc_va_Delete(vlc_va_t *va, AVCodecContext *avctx)
{
if (va->ops->close != NULL)
- va->ops->close(va);
+ va->ops->close(va, avctx);
vlc_object_delete(va);
}
=====================================
modules/codec/avcodec/va.h
=====================================
@@ -32,7 +32,7 @@ typedef struct vlc_video_context vlc_video_context;
struct vlc_va_operations {
int (*get)(vlc_va_t *, picture_t *pic, AVCodecContext *ctx, AVFrame
*frame);
- void (*close)(vlc_va_t *);
+ void (*close)(vlc_va_t *, AVCodecContext* ctx);
};
struct vlc_va_t {
@@ -103,7 +103,9 @@ static inline int vlc_va_Get(vlc_va_t *va, picture_t *pic,
AVCodecContext *ctx,
/**
* Destroys a libavcodec hardware acceleration back-end.
* All allocated surfaces shall have been released beforehand.
+ *
+ * @param avctx pointer to the current AVCodecContext if it needs to be
cleaned, NULL otherwise [IN]
*/
-void vlc_va_Delete(vlc_va_t *);
+void vlc_va_Delete(vlc_va_t *, AVCodecContext * avctx);
#endif
=====================================
modules/codec/avcodec/vaapi.c
=====================================
@@ -135,8 +135,10 @@ static int Get(vlc_va_t *va, picture_t *pic,
AVCodecContext *ctx, AVFrame *frame
return VLC_SUCCESS;
}
-static void Delete(vlc_va_t *va)
+static void Delete(vlc_va_t *va, AVCodecContext* ctx)
{
+ if (ctx)
+ av_buffer_unref(&ctx->hw_frames_ctx);
vlc_video_context_Release(va->sys);
}
=====================================
modules/codec/avcodec/video.c
=====================================
@@ -1345,7 +1345,7 @@ void EndVideoDec( vlc_object_t *obj )
if( p_sys->p_va )
{
- vlc_va_Delete( p_sys->p_va );
+ vlc_va_Delete( p_sys->p_va, NULL );
vlc_video_context_Release( p_sys->vctx_out );
p_sys->vctx_out = NULL;
}
@@ -1667,7 +1667,7 @@ no_reuse:
if (p_sys->p_va != NULL)
{
msg_Err(p_dec, "existing hardware acceleration cannot be reused");
- vlc_va_Delete(p_sys->p_va);
+ vlc_va_Delete(p_sys->p_va, NULL);
p_sys->p_va = NULL;
vlc_video_context_Release( p_sys->vctx_out );
p_sys->vctx_out = NULL;
@@ -1744,8 +1744,7 @@ no_reuse:
if (decoder_UpdateVideoOutput(p_dec, vctx_out))
{
- vlc_va_Delete(va);
- p_context->hwaccel_context = NULL;
+ vlc_va_Delete(va, p_context);
continue; /* Unsupported codec profile or such */
}
=====================================
modules/hw/vdpau/avcodec.c
=====================================
@@ -116,7 +116,7 @@ static int Lock(vlc_va_t *va, picture_t *pic,
AVCodecContext *ctx, AVFrame *fram
return VLC_SUCCESS;
}
-static void Close(vlc_va_t *va)
+static void Close(vlc_va_t *va, AVCodecContext* ctx)
{
vlc_va_sys_t *sys = va->sys;
@@ -126,6 +126,8 @@ static void Close(vlc_va_t *va)
vlc_video_context_Release(sys->vctx);
if (sys->hwaccel_context)
av_free(sys->hwaccel_context);
+ if (ctx)
+ ctx->hwaccel_context = NULL;
free(sys);
}
@@ -220,7 +222,7 @@ static int Open(vlc_va_t *va, AVCodecContext *avctx, enum
AVPixelFormat hwfmt, c
if (i < refs)
{
msg_Err(va, "not enough video RAM");
- Close(va);
+ Close(va, avctx);
return VLC_ENOMEM;
}
View it on GitLab:
https://code.videolan.org/videolan/vlc/-/compare/6144634a178f6e876b7dd301e357b6b0bcaa1862...0654e1386b87947b5ff6adf90d2d41f6ae49694e
--
View it on GitLab:
https://code.videolan.org/videolan/vlc/-/compare/6144634a178f6e876b7dd301e357b6b0bcaa1862...0654e1386b87947b5ff6adf90d2d41f6ae49694e
You're receiving this email because of your account on code.videolan.org.
VideoLAN code repository instance_______________________________________________
vlc-commits mailing list
vlc-commits@videolan.org
https://mailman.videolan.org/listinfo/vlc-commits
