On 02/08/2017 09:52, Huang, Zhengxu wrote:
> From fc3c6ff4c738d5cf6de84fb6d210efbe3f1a491f Mon Sep 17 00:00:00 2001
> From: "Huang, Zhengxu" <[email protected]>
> Date: Tue, 25 Jul 2017 21:55:50 +0800
> Subject: [PATCH 1/2] libavfilter/overlay_qsv: Add QSV overlay vpp filter
>
> Add intel QSV overlay filter. Now it supports two input. And it also supports
> the second input scale(implicit) during composition compared to the sw
> overlay.
>
> Code has been separated into common interface part and qsv overlay implement
> part.
> The common part mainly creates the qsv session and manages the surface which
> are
> nearly the same for all qsv filters. So the qsvvpp.c/qsvvpp.h API can be used
> by
> other QSV vpp filters to reduce code redundancy.
>
> Usage:
> -hwaccel qsv -c:v mpeg2_qsv -r 25 -i in.m2v -hwaccel qsv -c:v h264_qsv -i
> in.h264 -filter_complex
> "overlay_qsv=eof_action=repeat:x=(W-w)/2:y=(H-h)/2" -b 2M -maxrate 3M -c:v
> h264_qsv -y out.h264
>
> two input should have different size otherwise one will be completely covered
> or you need scale the
> second input as follow:
> -hwaccel qsv -c:v mpeg2_qsv -r 25 -i in.m2v -hwaccel qsv -c:v h264_qsv -i
> in.h264 -filter_complex
> "overlay_qsv=w=720:h=576:x=(W-w)/2:y=(H-h)/2" -b 2M -maxrate 3M -c:v h264_qsv
> -y out.h264
>
> Signed-off-by: ChaoX A Liu <[email protected]>
> Signed-off-by: Zhengxu Huang <[email protected]>
> Signed-off-by: Andrew Zhang <[email protected]>
> Change-Id: I5c381febb0af6e2f9622c54ba00490ab99d48297
> ---
> Changelog | 1 +
> configure | 3 +
> libavfilter/Makefile | 1 +
> libavfilter/allfilters.c | 1 +
> libavfilter/qsvvpp.c | 697
> +++++++++++++++++++++++++++++++++++++++++++
> libavfilter/qsvvpp.h | 66 ++++
> libavfilter/vf_overlay_qsv.c | 482 ++++++++++++++++++++++++++++++
> 7 files changed, 1251 insertions(+)
> create mode 100644 libavfilter/qsvvpp.c
> create mode 100644 libavfilter/qsvvpp.h
> create mode 100644 libavfilter/vf_overlay_qsv.c
>
> diff --git a/Changelog b/Changelog
> index 92a7249..51c3f85 100644
> --- a/Changelog
> +++ b/Changelog
> @@ -19,6 +19,7 @@ version <next>:
> - Cinepak encoder
> - Intel QSV-accelerated MJPEG encoding
> - NVIDIA CUVID-accelerated H.264 and HEVC decoding
> +- Intel QSV-accelerated overlay filter
>
>
> version 12:
> diff --git a/configure b/configure
> index 4510100..a80f24f 100755
> --- a/configure
> +++ b/configure
> @@ -1786,6 +1786,7 @@ CONFIG_EXTRA="
> qsv
> qsvdec
> qsvenc
> + qsvvpp
> rangecoder
> riffdec
> riffenc
> @@ -2528,6 +2529,8 @@ hqdn3d_filter_deps="gpl"
> interlace_filter_deps="gpl"
> movie_filter_deps="avcodec avformat"
> ocv_filter_deps="libopencv"
> +overlay_qsv_filter_deps="libmfx qsv"
> +overlay_qsv_filter_select="qsvvpp"
> resample_filter_deps="avresample"
> scale_filter_deps="swscale"
> scale_qsv_filter_deps="libmfx"
> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> index 348ad92..2edb6c7 100644
> --- a/libavfilter/Makefile
> +++ b/libavfilter/Makefile
> @@ -75,6 +75,7 @@ OBJS-$(CONFIG_NOFORMAT_FILTER) += vf_format.o
> OBJS-$(CONFIG_NULL_FILTER) += vf_null.o
> OBJS-$(CONFIG_OCV_FILTER) += vf_libopencv.o
> OBJS-$(CONFIG_OVERLAY_FILTER) += vf_overlay.o
> +OBJS-$(CONFIG_OVERLAY_QSV_FILTER) += qsvvpp.o vf_overlay_qsv.o
> OBJS-$(CONFIG_PAD_FILTER) += vf_pad.o
> OBJS-$(CONFIG_PIXDESCTEST_FILTER) += vf_pixdesctest.o
> OBJS-$(CONFIG_SCALE_FILTER) += vf_scale.o
The .h should be added to the SKIPHEADERS otherwise make check would
fail on qsv-less systems, something along:
SKIPHEADERS-$(CONFIG_LIBMFX) += qsvvpp.h
Should do.
> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
> index 3f09f46..dc59ccd 100644
> --- a/libavfilter/allfilters.c
> +++ b/libavfilter/allfilters.c
> @@ -98,6 +98,7 @@ void avfilter_register_all(void)
> REGISTER_FILTER(NULL, null, vf);
> REGISTER_FILTER(OCV, ocv, vf);
> REGISTER_FILTER(OVERLAY, overlay, vf);
> + REGISTER_FILTER(OVERLAY_QSV, overlay_qsv, vf);
> REGISTER_FILTER(PAD, pad, vf);
> REGISTER_FILTER(PIXDESCTEST, pixdesctest, vf);
> REGISTER_FILTER(SCALE, scale, vf);
> diff --git a/libavfilter/qsvvpp.c b/libavfilter/qsvvpp.c
> new file mode 100644
> index 0000000..464a030
> --- /dev/null
> +++ b/libavfilter/qsvvpp.c
> @@ -0,0 +1,697 @@
> +/*
> + * This file is part of Libav.
> + *
> + * Libav is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * Libav is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with Libav; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
> USA
> + */
> +/**
> + * @file
> + * Intel Quick Sync Video VPP base function
> + */
This description could be put in the .h as well.
> +
> +#include "libavutil/common.h"
> +#include "libavutil/mathematics.h"
> +#include "libavutil/hwcontext.h"
> +#include "libavutil/hwcontext_qsv.h"
> +#include "libavutil/time.h"
> +#include "libavutil/pixdesc.h"
> +#include "internal.h"
> +#include "qsvvpp.h"
> +#include "video.h"
> +
> +typedef struct QSVFrame {
> + AVFrame *frame;
> + mfxFrameSurface1 *surface;
> + mfxFrameSurface1 surface_internal; ///< for system memory
> + struct QSVFrame *next;
> +} QSVFrame;
> +
> +/* abstract struct for all QSV filters */
> +struct FFQSVVPPContext {
> + mfxSession session;
> + int (*filter_frame) (AVFilterLink *outlink, AVFrame *frame);///<
> callback function
> + enum AVPixelFormat out_sw_format; ///< Real output format
> + mfxVideoParam vpp_param;
> + mfxFrameInfo *frame_infos; ///< frame info for each input
> +
> + /* members releated to the input/output surface */
> + int in_mem_mode;
> + int out_mem_mode;
> + QSVFrame *in_frame_list;
> + QSVFrame *out_frame_list;
> + int nb_surface_ptrs_in;
> + int nb_surface_ptrs_out;
> + mfxFrameSurface1 **surface_ptrs_in;
> + mfxFrameSurface1 **surface_ptrs_out;
> +
> + /* MFXVPP extern parameters*/
> + mfxExtOpaqueSurfaceAlloc opaque_alloc;
> + mfxExtBuffer **ext_buffers;
> + int nb_ext_buffers;
> +};
> +
> +static const mfxHandleType handle_types[] = {
> + MFX_HANDLE_VA_DISPLAY,
> + MFX_HANDLE_D3D9_DEVICE_MANAGER,
> + MFX_HANDLE_D3D11_DEVICE,
> +};
> +
> +static const AVRational default_tb = {1, 90000};
> +
> +/* functions for frameAlloc */
> +static mfxStatus frame_alloc(mfxHDL pthis, mfxFrameAllocRequest *req,
> + mfxFrameAllocResponse *resp)
> +{
> + FFQSVVPPContext *s = pthis;
> + int i;
> +
> + if (!(req->Type & MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET) ||
> + !(req->Type & (MFX_MEMTYPE_FROM_VPPIN | MFX_MEMTYPE_FROM_VPPOUT)) ||
> + !(req->Type & MFX_MEMTYPE_EXTERNAL_FRAME))
> + return MFX_ERR_UNSUPPORTED;
> +
> + if (req->Type & MFX_MEMTYPE_FROM_VPPIN) {
> + resp->mids = av_mallocz(s->nb_surface_ptrs_in * sizeof(*resp->mids));
> + if (!resp->mids)
> + return AVERROR(ENOMEM);
> +
> + for (i = 0; i < s->nb_surface_ptrs_in; i++)
> + resp->mids[i] = s->surface_ptrs_in[i]->Data.MemId;
> +
> + resp->NumFrameActual = s->nb_surface_ptrs_in;
> + } else {
> + resp->mids = av_mallocz(s->nb_surface_ptrs_in * sizeof(*resp->mids));
> + if (!resp->mids)
> + return AVERROR(ENOMEM);
> +
> + for (i = 0; i < s->nb_surface_ptrs_out; i++)
> + resp->mids[i] = s->surface_ptrs_out[i]->Data.MemId;
> +
> + resp->NumFrameActual = s->nb_surface_ptrs_out;
> + }
> +
> + return MFX_ERR_NONE;
> +}
> +
> +static mfxStatus frame_free(mfxHDL pthis, mfxFrameAllocResponse *resp)
> +{
> + av_freep(&resp->mids);
> + return MFX_ERR_NONE;
> +}
> +
> +static mfxStatus frame_lock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
> +{
> + return MFX_ERR_UNSUPPORTED;
> +}
> +
> +static mfxStatus frame_unlock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
> +{
> + return MFX_ERR_UNSUPPORTED;
> +}
> +
> +static mfxStatus frame_get_hdl(mfxHDL pthis, mfxMemId mid, mfxHDL *hdl)
> +{
> + *hdl = mid;
> + return MFX_ERR_NONE;
> +}
> +
> +static int pix_fmt_to_mfx_fourcc(int format)
> +{
> + switch(format) {
> + case AV_PIX_FMT_YUV420P:
> + return MFX_FOURCC_YV12;
> + case AV_PIX_FMT_NV12:
> + return MFX_FOURCC_NV12;
> + case AV_PIX_FMT_YUYV422:
> + return MFX_FOURCC_YUY2;
> + case AV_PIX_FMT_RGB32:
> + return MFX_FOURCC_RGB4;
> + }
> +
> + return MFX_FOURCC_NV12;
> +}
> +
> +static int map_frame_to_surface(AVFrame *frame, mfxFrameSurface1 *surface)
> +{
> + switch (frame->format) {
> + case AV_PIX_FMT_NV12:
> + surface->Data.Y = frame->data[0];
> + surface->Data.UV = frame->data[1];
> + break;
> + case AV_PIX_FMT_YUV420P:
> + surface->Data.Y = frame->data[0];
> + surface->Data.U = frame->data[1];
> + surface->Data.V = frame->data[2];
> + break;
> + case AV_PIX_FMT_YUYV422:
> + surface->Data.Y = frame->data[0];
> + surface->Data.U = frame->data[0] + 1;
> + surface->Data.V = frame->data[0] + 3;
> + break;
> + case AV_PIX_FMT_RGB32:
> + surface->Data.B = frame->data[0];
> + surface->Data.G = frame->data[0] + 1;
> + surface->Data.R = frame->data[0] + 2;
> + surface->Data.A = frame->data[0] + 3;
> + break;
> + default:
> + return MFX_ERR_UNSUPPORTED;
> + }
> + surface->Data.Pitch = frame->linesize[0];
> +
> + return 0;
> +}
> +
> +/* fill the surface info */
> +static int fill_frameinfo_by_link(mfxFrameInfo *frameinfo, AVFilterLink
> *link)
> +{
> + enum AVPixelFormat pix_fmt;
> + AVHWFramesContext *frames_ctx;
> + const AVPixFmtDescriptor *desc;
> +
> + if (link->format == AV_PIX_FMT_QSV) {
> + if (!link->hw_frames_ctx)
> + return AVERROR(EINVAL);
> +
> + frames_ctx = (AVHWFramesContext *)link->hw_frames_ctx->data;
> + pix_fmt = frames_ctx->sw_format;
> + } else
> + pix_fmt = link->format;
> +
> + desc = av_pix_fmt_desc_get(pix_fmt);
> + if (!desc)
> + return AVERROR_BUG;
> +
> + frameinfo->CropX = 0;
> + frameinfo->CropY = 0;
> + frameinfo->CropW = link->w;
> + frameinfo->CropH = link->h;
> + frameinfo->Width = FFALIGN(link->w, 32);
> + frameinfo->Height = FFALIGN(link->h, 32);
> + frameinfo->PicStruct = MFX_PICSTRUCT_PROGRESSIVE;
> + frameinfo->FrameRateExtN = link->frame_rate.num;
> + frameinfo->FrameRateExtD = link->frame_rate.den;
> + frameinfo->FourCC = pix_fmt_to_mfx_fourcc(pix_fmt);
> + frameinfo->BitDepthLuma = desc->comp[0].depth;
> + frameinfo->BitDepthChroma = desc->comp[0].depth;
> + frameinfo->Shift = desc->comp[0].depth > 8;
> + if (desc->log2_chroma_w && desc->log2_chroma_h)
> + frameinfo->ChromaFormat = MFX_CHROMAFORMAT_YUV420;
> + else if (desc->log2_chroma_w)
> + frameinfo->ChromaFormat = MFX_CHROMAFORMAT_YUV422;
> + else
> + frameinfo->ChromaFormat = MFX_CHROMAFORMAT_YUV444;
> + frameinfo->AspectRatioW = link->sample_aspect_ratio.num ?
> link->sample_aspect_ratio.num : 1;
> + frameinfo->AspectRatioH = link->sample_aspect_ratio.den ?
> link->sample_aspect_ratio.den : 1;
> +
> + return 0;
> +}
> +
> +static void clear_unused_frames(QSVFrame *list)
> +{
> + while (list) {
> + if (list->surface && !list->surface->Data.Locked) {
> + list->surface = NULL;
> + av_frame_free(&list->frame);
> + }
> + list = list->next;
> + }
> +}
> +
> +static void clear_frame_list(QSVFrame **list)
> +{
> + QSVFrame *frame;
> +
> + while (*list) {
> + frame = *list;
> + *list = (*list)->next;
> + av_frame_free(&frame->frame);
> + av_freep(&frame);
> + }
> +}
> +
> +static QSVFrame *get_free_frame(QSVFrame **list)
> +{
> + QSVFrame *out = *list;
> +
> + for (; out; out = out->next) {
> + if (!out->surface)
> + break;
> + }
> +
> + if (!out) {
> + out = av_mallocz(sizeof(*out));
> + if (!out)
> + return NULL;
> + out->next = *list;
> + *list = out;
> + }
> +
> + return out;
> +}
> +
> +/* get the input surface */
> +static QSVFrame *submit_frame(FFQSVVPPContext *s, AVFilterLink *inlink,
> AVFrame *picref)
> +{
> + QSVFrame *qsv_frame;
> + AVFilterContext *ctx = inlink->dst;
> +
> + clear_unused_frames(s->in_frame_list);
> +
> + qsv_frame = get_free_frame(&s->in_frame_list);
> + if (!qsv_frame) {
> + av_log(ctx, AV_LOG_ERROR, "Can't alloc new frame.\n");
> + return NULL;
> + }
> +
> + qsv_frame->frame = picref;
> +
> + /**
> + * Turn AVFrame into mfxFrameSurface1.
> + * For video/opaque memory mode, pix_fmt is AV_PIX_FMT_QSV, and
> + * mfxFrameSurface1 is stored in AVFrame->data[3];
> + * for system memory mode, raw video data is stored in
> + * AVFrame, we should map it into mfxFrameSurface1.
> + */
> + if (s->in_mem_mode != MFX_MEMTYPE_SYSTEM_MEMORY) {
> + if (qsv_frame->frame->format != AV_PIX_FMT_QSV) {
> + av_log(ctx, AV_LOG_ERROR, "QSVVPP gets a wrong frame.\n");
> + return NULL;
> + }
> + qsv_frame->surface = (mfxFrameSurface1*)qsv_frame->frame->data[3];
> + } else {
> + if (map_frame_to_surface(qsv_frame->frame,
> + &qsv_frame->surface_internal) < 0) {
> + av_log(ctx, AV_LOG_ERROR, "Unsupported frame.\n");
> + return NULL;
> + }
> + qsv_frame->surface = &qsv_frame->surface_internal;
> + }
> +
> + qsv_frame->surface->Info = s->frame_infos[FF_INLINK_IDX(inlink)];
> + qsv_frame->surface->Data.TimeStamp = av_rescale_q(qsv_frame->frame->pts,
> + inlink->time_base, default_tb);
> +
> + return qsv_frame;
> +}
> +
> +/* get the output surface */
> +static QSVFrame *query_frame(FFQSVVPPContext *s, AVFilterLink *outlink)
> +{
> + AVFilterContext *ctx = outlink->src;
> + QSVFrame *out_frame;
> + int ret;
> +
> + clear_unused_frames(s->out_frame_list);
> +
> + out_frame = get_free_frame(&s->out_frame_list);
> + if (!out_frame) {
> + av_log(ctx, AV_LOG_ERROR, "Can't alloc new output frame.\n");
> + return NULL;
> + }
> +
> + /**
> + * For video memory, get a hw frame;
> + * for system memory, get a sw frame and map it into a mfx_surface.
> + */
> + if (s->out_mem_mode != MFX_MEMTYPE_SYSTEM_MEMORY) {
> + out_frame->frame = av_frame_alloc();
> + if (!out_frame->frame)
> + return NULL;
> +
> + ret = av_hwframe_get_buffer(outlink->hw_frames_ctx,
> out_frame->frame, 0);
> + if (ret < 0) {
> + av_log(ctx, AV_LOG_ERROR, "Can't allocate a surface.\n");
> + return NULL;
> + }
> + out_frame->surface = (mfxFrameSurface1*)out_frame->frame->data[3];
> + } else {
> + /**
> + * Get a frame with aligned dimensions.
> + * Libmfx need system memory being 128x64 aligned
> + */
> + out_frame->frame = ff_get_video_buffer(outlink,
> + FFALIGN(outlink->w, 128),
> + FFALIGN(outlink->h, 64));
> + if (!out_frame->frame)
> + return NULL;
> +
> + out_frame->frame->width = outlink->w;
> + out_frame->frame->height = outlink->h;
> +
> + ret = map_frame_to_surface(out_frame->frame,
> + &out_frame->surface_internal);
> + if (ret < 0)
> + return NULL;
> +
> + out_frame->surface = &out_frame->surface_internal;
> + }
> +
> + out_frame->surface->Info = s->vpp_param.vpp.Out;
> +
> + return out_frame;
> +}
> +
> +/* create the QSV session */
> +static int init_vpp_session(AVFilterContext *avctx, FFQSVVPPContext *s)
> +{
> + AVFilterLink *inlink = avctx->inputs[0];
> + AVFilterLink *outlink = avctx->outputs[0];
> + AVQSVFramesContext *in_frames_hwctx = NULL;
> + AVQSVFramesContext *out_frames_hwctx = NULL;
> +
> + AVBufferRef *device_ref;
> + AVHWDeviceContext *device_ctx;
> + AVQSVDeviceContext *device_hwctx;
> + mfxHDL handle;
> + mfxHandleType handle_type;
> + mfxVersion ver;
> + mfxIMPL impl;
> + int ret, i;
> +
> + /**
> + * @mem-mode
> + * 1.Input has a HW context, Video or Opaque mode
> + * 2.Otherwise, System mode.
> + */
> + if (inlink->hw_frames_ctx) {
> + AVHWFramesContext *frames_ctx = (AVHWFramesContext
> *)inlink->hw_frames_ctx->data;
> +
> + device_ref = frames_ctx->device_ref;
> + in_frames_hwctx = frames_ctx->hwctx;
> +
> + if (in_frames_hwctx->frame_type & MFX_MEMTYPE_OPAQUE_FRAME)
> + s->in_mem_mode = MFX_MEMTYPE_OPAQUE_FRAME;
> + else
> + s->in_mem_mode = MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
> +
> + s->surface_ptrs_in = av_mallocz_array(in_frames_hwctx->nb_surfaces,
> + sizeof(*s->surface_ptrs_in));
> + if (!s->surface_ptrs_in)
> + return AVERROR(ENOMEM);
> + for (i = 0; i < in_frames_hwctx->nb_surfaces; i++)
> + s->surface_ptrs_in[i] = in_frames_hwctx->surfaces + i;
> + s->nb_surface_ptrs_in = in_frames_hwctx->nb_surfaces;
> + } else if (avctx->hw_device_ctx) {
> + device_ref = avctx->hw_device_ctx;
> + s->in_mem_mode = MFX_MEMTYPE_SYSTEM_MEMORY;
> + } else {
> + av_log(avctx, AV_LOG_ERROR, "No hw context provided.\n");
> + return AVERROR(EINVAL);
> + }
> +
> + device_ctx = (AVHWDeviceContext *)device_ref->data;
> + device_hwctx = device_ctx->hwctx;
> +
> + if (outlink->format == AV_PIX_FMT_QSV) {
> + AVHWFramesContext *out_frames_ctx;
> + AVBufferRef *out_frames_ref = av_hwframe_ctx_alloc(device_ref);
> + if (!out_frames_ref)
> + return AVERROR(ENOMEM);
> +
> + out_frames_ctx = (AVHWFramesContext*)out_frames_ref->data;
> + out_frames_hwctx = out_frames_ctx->hwctx;
> +
> + out_frames_ctx->format = AV_PIX_FMT_QSV;
> + out_frames_ctx->width = FFALIGN(outlink->w, 32);
> + out_frames_ctx->height = FFALIGN(outlink->h, 32);
> + out_frames_ctx->sw_format = s->out_sw_format;
> + out_frames_ctx->initial_pool_size = 64;
> + out_frames_hwctx->frame_type = in_frames_hwctx->frame_type;
> + s->out_mem_mode = s->in_mem_mode;
> +
> + ret = av_hwframe_ctx_init(out_frames_ref);
> + if (ret < 0) {
> + av_buffer_unref(&out_frames_ref);
> + av_log(avctx, AV_LOG_ERROR, "Error creating frames_ctx for
> output pad.\n");
> + return ret;
> + }
> +
> + s->surface_ptrs_out = av_mallocz_array(out_frames_hwctx->nb_surfaces,
> + sizeof(*s->surface_ptrs_out));
> + if (!s->surface_ptrs_out)
> + return AVERROR(ENOMEM);
> + for (i = 0; i < out_frames_hwctx->nb_surfaces; i++)
> + s->surface_ptrs_out[i] = out_frames_hwctx->surfaces + i;
> + s->nb_surface_ptrs_out = out_frames_hwctx->nb_surfaces;
> +
> + av_buffer_unref(&outlink->hw_frames_ctx);
> + outlink->hw_frames_ctx = out_frames_ref;
> + } else
> + s->out_mem_mode = MFX_MEMTYPE_SYSTEM_MEMORY;
> +
> + /* extract the properties of the "master" session given to us */
> + ret = MFXQueryIMPL(device_hwctx->session, &impl);
> + if (ret == MFX_ERR_NONE)
> + ret = MFXQueryVersion(device_hwctx->session, &ver);
> + if (ret != MFX_ERR_NONE) {
> + av_log(avctx, AV_LOG_ERROR, "Error querying the session
> attributes\n");
> + return AVERROR_UNKNOWN;
> + }
> +
> + for (i = 0; i < FF_ARRAY_ELEMS(handle_types); i++) {
> + ret = MFXVideoCORE_GetHandle(device_hwctx->session, handle_types[i],
> &handle);
> + if (ret == MFX_ERR_NONE) {
> + handle_type = handle_types[i];
> + break;
> + }
> + }
> +
> + /* create a "slave" session with those same properties, to be used for
> vpp*/
> + ret = MFXInit(impl, &ver, &s->session);
> + if (ret != MFX_ERR_NONE) {
> + av_log(avctx, AV_LOG_ERROR, "Error initializing a session for
> scaling\n");
> + return AVERROR_UNKNOWN;
> + }
> +
> + if (handle) {
> + ret = MFXVideoCORE_SetHandle(s->session, handle_type, handle);
> + if (ret != MFX_ERR_NONE)
> + return AVERROR_UNKNOWN;
> + }
> +
> + if (s->in_mem_mode == MFX_MEMTYPE_OPAQUE_FRAME || s->out_mem_mode ==
> MFX_MEMTYPE_OPAQUE_FRAME) {
> + s->opaque_alloc.In.Surfaces = s->surface_ptrs_in;
> + s->opaque_alloc.In.NumSurface = s->nb_surface_ptrs_in;
> + s->opaque_alloc.In.Type = in_frames_hwctx ?
> in_frames_hwctx->frame_type :
> + MFX_MEMTYPE_SYSTEM_MEMORY;
> +
> + s->opaque_alloc.Out.Surfaces = s->surface_ptrs_out;
> + s->opaque_alloc.Out.NumSurface = s->nb_surface_ptrs_out;
> + s->opaque_alloc.Out.Type = out_frames_hwctx ?
> out_frames_hwctx->frame_type :
> + MFX_MEMTYPE_SYSTEM_MEMORY;
> +
> + s->opaque_alloc.Header.BufferId =
> MFX_EXTBUFF_OPAQUE_SURFACE_ALLOCATION;
> + s->opaque_alloc.Header.BufferSz = sizeof(s->opaque_alloc);
> + } else if (s->in_mem_mode == MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET ||
> + s->out_mem_mode == MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET) {
> + mfxFrameAllocator frame_allocator = {
> + .pthis = s,
> + .Alloc = frame_alloc,
> + .Lock = frame_lock,
> + .Unlock = frame_unlock,
> + .GetHDL = frame_get_hdl,
> + .Free = frame_free,
> + };
> +
> + ret = MFXVideoCORE_SetFrameAllocator(s->session, &frame_allocator);
> + if (ret != MFX_ERR_NONE)
> + return AVERROR_UNKNOWN;
> + }
> +
> + return 0;
> +}
> +
> +int ff_qsvvpp_create(AVFilterContext *avctx, FFQSVVPPContext **vpp,
> FFQSVVPPParam *param)
> +{
> + int ret = 0, i;
> + FFQSVVPPContext *s;
> +
> + if (!avctx || !vpp)
> + return AVERROR(EINVAL);
> +
> + s = av_mallocz(sizeof(FFQSVVPPContext));
> + if (!s) {
> + ret = AVERROR(ENOMEM);
> + goto failed;
> + }
> +
> + s->filter_frame = param->filter_frame;
> + if (!s->filter_frame)
> + s->filter_frame = ff_filter_frame;
> + s->out_sw_format = param->out_sw_format;
> +
> + /* create the vpp session */
> + ret = init_vpp_session(avctx, s);
> + if (ret < 0)
> + goto failed;
> +
> + s->frame_infos = av_mallocz_array(sizeof(*s->frame_infos),
> avctx->nb_inputs);
> + if (!s->frame_infos) {
> + ret = AVERROR(ENOMEM);
> + goto failed;
> + }
> +
> + /* Init each input's information*/
> + for (i = 0; i < avctx->nb_inputs; i++) {
> + ret = fill_frameinfo_by_link(&s->frame_infos[i], avctx->inputs[i]);
> + if (ret < 0)
> + goto failed;
> + }
> +
> + /* Update input's frame info according to crop */
> + for (i = 0; i < param->num_crop; i++) {
> + FFQSVVPPCrop *crop = param->crop + i;
> + if (crop->in_idx > avctx->nb_inputs) {
> + ret = AVERROR(EINVAL);
> + goto failed;
> + }
> + s->frame_infos[crop->in_idx].CropX = crop->x;
> + s->frame_infos[crop->in_idx].CropY = crop->y;
> + s->frame_infos[crop->in_idx].CropW = crop->w;
> + s->frame_infos[crop->in_idx].CropH = crop->h;
> + }
> +
> + s->vpp_param.vpp.In = s->frame_infos[0];
> +
> + ret = fill_frameinfo_by_link(&s->vpp_param.vpp.Out, avctx->outputs[0]);
> + if (ret < 0) {
> + av_log(avctx, AV_LOG_ERROR, "Fail to get frame info from link.\n");
> + goto failed;
> + }
> +
> + if (s->in_mem_mode == MFX_MEMTYPE_OPAQUE_FRAME || s->out_mem_mode ==
> MFX_MEMTYPE_OPAQUE_FRAME) {
> + s->nb_ext_buffers = param->num_ext_buf + 1;
> + s->ext_buffers = av_mallocz(sizeof(*s->ext_buffers) *
> s->nb_ext_buffers);
> + if (!s->ext_buffers)
> + return AVERROR(ENOMEM);
> +
> + s->ext_buffers[0] = (mfxExtBuffer *)&s->opaque_alloc;
> + for (i = 1; i < param->num_ext_buf; i++)
> + s->ext_buffers[i] = param->ext_buf[i - 1];
> + s->vpp_param.ExtParam = s->ext_buffers;
> + s->vpp_param.NumExtParam = s->nb_ext_buffers;
> + } else {
> + s->vpp_param.NumExtParam = param->num_ext_buf;
> + s->vpp_param.ExtParam = param->ext_buf;
> + }
> +
> + s->vpp_param.AsyncDepth = 1;
> +
> + if (s->in_mem_mode == MFX_MEMTYPE_SYSTEM_MEMORY)
> + s->vpp_param.IOPattern |= MFX_IOPATTERN_IN_SYSTEM_MEMORY;
> + else if (s->in_mem_mode == MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET)
> + s->vpp_param.IOPattern |= MFX_IOPATTERN_IN_VIDEO_MEMORY;
> + else if (s->in_mem_mode == MFX_MEMTYPE_OPAQUE_FRAME)
> + s->vpp_param.IOPattern |= MFX_IOPATTERN_IN_OPAQUE_MEMORY;
> +
> +
> + if (s->out_mem_mode == MFX_MEMTYPE_SYSTEM_MEMORY)
> + s->vpp_param.IOPattern |= MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
> + else if (s->out_mem_mode == MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET)
> + s->vpp_param.IOPattern |= MFX_IOPATTERN_OUT_VIDEO_MEMORY;
> + else if (s->out_mem_mode == MFX_MEMTYPE_OPAQUE_FRAME)
> + s->vpp_param.IOPattern |= MFX_IOPATTERN_OUT_OPAQUE_MEMORY;
> +
> + ret = MFXVideoVPP_Init(s->session, &s->vpp_param);
> + if (ret < 0) {
> + av_log(avctx, AV_LOG_ERROR, "Failed to create a qsvvpp, ret =
> %d.\n", ret);
> + goto failed;
> + }
> +
> + *vpp = s;
> + return 0;
> +
> +failed:
> + if (s)
> + ff_qsvvpp_free(&s);
> +
> + return ret;
> +}
> +
> +int ff_qsvvpp_free(FFQSVVPPContext **vpp)
> +{
> + FFQSVVPPContext *s = *vpp;
> +
> + if (!s)
> + return 0;
> +
> + MFXVideoVPP_Close(s->session);
> + MFXClose(s->session);
> +
> + /* release all the resources */
> + clear_frame_list(&s->in_frame_list);
> + clear_frame_list(&s->out_frame_list);
> + av_freep(&s->surface_ptrs_in);
> + av_freep(&s->surface_ptrs_out);
> + av_freep(&s->ext_buffers);
> + av_freep(&s->frame_infos);
> + av_freep(vpp);
> +
> + return 0;
> +}
> +
> +int ff_qsvvpp_filter_frame(FFQSVVPPContext *s, AVFilterLink *inlink, AVFrame
> *picref)
> +{
> + AVFilterContext *ctx = inlink->dst;
> + AVFilterLink *outlink = ctx->outputs[0];
> + mfxSyncPoint sync;
> + QSVFrame *in_frame, *out_frame;
> + int ret, filter_ret;
> +
> + in_frame = submit_frame(s, inlink, picref);
> + if (!in_frame) {
> + av_log(ctx, AV_LOG_ERROR, "Fail to submit frame on input[%d]\n",
> + FF_INLINK_IDX(inlink));
> + return AVERROR(EINVAL);
> + }
> +
> + do {
> + out_frame = query_frame(s, outlink);
> + if(!out_frame) {
> + av_log(ctx, AV_LOG_ERROR, "Fail to query an output frame.\n");
> + return AVERROR(ENOMEM);
> + }
> +
> + do {
> + ret = MFXVideoVPP_RunFrameVPPAsync(s->session, in_frame->surface,
> + out_frame->surface, NULL,
> &sync);
> + if (ret == MFX_WRN_DEVICE_BUSY)
> + av_usleep(500);
> + } while (ret == MFX_WRN_DEVICE_BUSY);
> +
> + if (ret < 0 && ret != MFX_ERR_MORE_SURFACE) {
> + /*Ignore more_data error*/
> + if (ret == MFX_ERR_MORE_DATA)
> + ret = AVERROR(EAGAIN);
> + break;
> + }
> +
> + if (MFXVideoCORE_SyncOperation(s->session, sync, 1000) < 0)
> + av_log(ctx, AV_LOG_WARNING, "Sync failed.\n");
> +
> + out_frame->frame->pts =
> av_rescale_q(out_frame->surface->Data.TimeStamp,
> + default_tb, outlink->time_base);
> +
> + filter_ret = s->filter_frame(outlink, out_frame->frame);
> + if (filter_ret < 0) {
> + av_frame_free(&out_frame->frame);
> + ret = filter_ret;
> + break;
> + }
> + out_frame->frame = NULL;
> + } while(ret == MFX_ERR_MORE_SURFACE);
> +
> + return ret;
> +}
> diff --git a/libavfilter/qsvvpp.h b/libavfilter/qsvvpp.h
> new file mode 100644
> index 0000000..b8ada20
> --- /dev/null
> +++ b/libavfilter/qsvvpp.h
> @@ -0,0 +1,66 @@
> +/*
> + * This file is part of Libav.
> + *
> + * Libav is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * Libav is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with Libav; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
> USA
> + */
> +
> +#ifndef AVFILTER_QSVVPP_H
> +#define AVFILTER_QSVVPP_H
> +
> +#include "avfilter.h"
> +#include <mfx/mfxvideo.h>
> +
> +#define FF_INLINK_IDX(link) ((int)((link)->dstpad -
> (link)->dst->input_pads))
> +#define FF_OUTLINK_IDX(link) ((int)((link)->srcpad -
> (link)->src->output_pads))
> +
> +typedef struct FFQSVVPPContext FFQSVVPPContext;
> +
> +typedef struct FFQSVVPPCrop {
> + int in_idx; ///< Input index
> + int x, y, w, h; ///< Crop rectangle
> +} FFQSVVPPCrop;
> +
> +typedef struct FFQSVVPPParam {
> + /* default is ff_filter_frame */
> + int (*filter_frame) (AVFilterLink *outlink, AVFrame *frame);
> +
> + /* To fill with MFX enhanced filter configurations */
> + int num_ext_buf;
> + mfxExtBuffer **ext_buf;
> +
> + /* Real output format */
> + enum AVPixelFormat out_sw_format;
> +
> + /* Crop information for each input, if needed */
> + int num_crop;
> + FFQSVVPPCrop *crop;
> +} FFQSVVPPParam;
> +
> +/**
> + * Blow APIs are abstracted to be called by different QSV VPP filters,> + *
> most codes of different QSV filters
(scale/deinterlace/framerate/overlay) are nearly
> + * the same so use the common APIs to reduce code redundancy
> + */
Probably the comment above could be dropped or simplified as "common
code for VPP-based filters".
> +
> +/* create and initialize the QSV session*/
> +int ff_qsvvpp_create(AVFilterContext *avctx, FFQSVVPPContext **vpp,
> FFQSVVPPParam *param);
> +
> +/* release the resources eg.surfaces */
> +int ff_qsvvpp_free(FFQSVVPPContext **vpp);
> +
> +/* vpp filter frame and call the cb if needed */
> +int ff_qsvvpp_filter_frame(FFQSVVPPContext *vpp, AVFilterLink *inlink,
> AVFrame *frame);
> +
> +#endif /* QSVVPP_H */
> diff --git a/libavfilter/vf_overlay_qsv.c b/libavfilter/vf_overlay_qsv.c
> new file mode 100644
> index 0000000..f4c78fa
> --- /dev/null
> +++ b/libavfilter/vf_overlay_qsv.c
> @@ -0,0 +1,482 @@
> +/*
> + * This file is part of Libav.
> + *
> + * Libav is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * Libav is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with Libav; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
> USA
> + */
> +
> +/**
> + * @vf_overlay_qsv
> + * A hardware accelerated overlay filter based on Intel Quick Sync Video VPP
> + */
> +
> +#include "internal.h"
> +#include "avfilter.h"
> +#include "formats.h"
> +#include "libavutil/opt.h"
> +#include "libavutil/common.h"
> +#include "libavutil/pixdesc.h"
> +#include "libavutil/eval.h"
> +#include "libavutil/hwcontext.h"
> +#include "libavutil/avstring.h"
> +#include "libavutil/avassert.h"
> +#include "libavutil/imgutils.h"
> +#include "libavutil/mathematics.h"
> +#include "video.h"
> +
> +#include "qsvvpp.h"
> +
> +#define MAIN 0
> +#define OVERLAY 1
> +
> +#define OFFSET(x) offsetof(QSVOverlayContext, x)
> +#define FLAGS AV_OPT_FLAG_VIDEO_PARAM
> +
> +enum var_name {
> + VAR_MAIN_iW, VAR_MW,
> + VAR_MAIN_iH, VAR_MH,
> + VAR_OVERLAY_iW,
> + VAR_OVERLAY_iH,
> + VAR_OVERLAY_X, VAR_OX,
> + VAR_OVERLAY_Y, VAR_OY,
> + VAR_OVERLAY_W, VAR_OW,
> + VAR_OVERLAY_H, VAR_OH,
> + VAR_VARS_NB
> +};
> +
> +enum EOFAction {
> + EOF_ACTION_REPEAT,
> + EOF_ACTION_ENDALL
> +};
While at it, a newline here.
> +typedef struct QSVOverlayContext {
> + AVClass *class;
> +
> + FFQSVVPPContext *qsv;
> + FFQSVVPPParam qsv_param;
> + mfxExtVPPComposite comp_conf;
> + double var_values[VAR_VARS_NB];
> +
> + char *overlay_ox, *overlay_oy, *overlay_ow, *overlay_oh;
> + uint16_t overlay_alpha, overlay_pixel_alpha;
> +
> + enum EOFAction eof_action; ///< action to take on EOF from source
> +
> + AVFrame *main;
> + AVFrame *over_prev, *over_next;
> +} QSVOverlayContext;
> +
> +/**
> + * qsv composite support overlay implicitly scale so the params much
> + * more complicated than sw overlay
> + */
> +static const char *const var_names[] = {
> + "main_w", "W", ///< input width of the main layer
> + "main_h", "H", ///< input height of the main layer
> + "overlay_iw", ///< input width of the overlay layer
> + "overlay_ih", ///< input height of the overlay layer
> + "overlay_x", "x", ///< x position of the overlay layer inside of main
> + "overlay_y", "y", ///< y position of the overlay layer inside of main
> + "overlay_w", "w", ///< output width of overlay layer
> + "overlay_h", "h", ///< output height of overlay layer
> + NULL
> +};
> +
> +static const AVOption options[] = {
> + { "x", "Overlay x position", OFFSET(overlay_ox), AV_OPT_TYPE_STRING,
> {.str="0"}, 0, 255, .flags = FLAGS},
> + { "y", "Overlay y position", OFFSET(overlay_oy), AV_OPT_TYPE_STRING,
> {.str="0"}, 0, 255, .flags = FLAGS},
> + { "w", "Overlay width", OFFSET(overlay_ow), AV_OPT_TYPE_STRING,
> {.str="overlay_iw"}, 0, 255, .flags = FLAGS},
> + { "h", "Overlay height", OFFSET(overlay_oh), AV_OPT_TYPE_STRING,
> {.str="overlay_ih*w/overlay_iw"}, 0, 255, .flags = FLAGS},
> + { "alpha", "Overlay global alpha", OFFSET(overlay_alpha),
> AV_OPT_TYPE_INT, {.i64 = 255}, 0, 255, .flags = FLAGS},
> + { "eof_action", "Action to take when encountering EOF from secondary
> input ",
> + OFFSET(eof_action), AV_OPT_TYPE_INT, { .i64 = EOF_ACTION_REPEAT },
> + EOF_ACTION_REPEAT, EOF_ACTION_ENDALL, .flags = FLAGS, "eof_action" },
> + { "repeat", "Repeat the previous frame.", 0, AV_OPT_TYPE_CONST, {
> .i64 = EOF_ACTION_REPEAT }, .flags = FLAGS, "eof_action" },
> + { "endall", "End both streams.", 0, AV_OPT_TYPE_CONST, {
> .i64 = EOF_ACTION_ENDALL }, .flags = FLAGS, "eof_action" },
> + { NULL }
> +};
> +
> +static int eval_expr(AVFilterContext *ctx)
> +{
> + QSVOverlayContext *vpp = ctx->priv;
> + double *var_values = vpp->var_values;
> + AVExpr *ox_expr, *oy_expr, *ow_expr, *oh_expr;
> + int ret;
> +
> +#define PASS_EXPR(e, s) {\
> + ret = av_expr_parse(&e, s, var_names, NULL, NULL, NULL, NULL, 0, ctx); \
> + if (ret < 0) {\
> + av_log(ctx, AV_LOG_ERROR, "Error when passing '%s'.\n", s);\
> + return ret;\
> + }\
> +}
> + PASS_EXPR(ox_expr, vpp->overlay_ox);
> + PASS_EXPR(oy_expr, vpp->overlay_oy);
> + PASS_EXPR(ow_expr, vpp->overlay_ow);
> + PASS_EXPR(oh_expr, vpp->overlay_oh);
> +#undef PASS_EXPR
> +
> + var_values[VAR_OVERLAY_W] =
> + var_values[VAR_OW] = av_expr_eval(ow_expr, var_values, NULL);
> + var_values[VAR_OVERLAY_H] =
> + var_values[VAR_OH] = av_expr_eval(oh_expr, var_values, NULL);
> +
> + /* calc again in case ow is relative to oh */
> + var_values[VAR_OVERLAY_W] =
> + var_values[VAR_OW] = av_expr_eval(ow_expr, var_values, NULL);
> +
> + var_values[VAR_OVERLAY_X] =
> + var_values[VAR_OX] = av_expr_eval(ox_expr, var_values, NULL);
> + var_values[VAR_OVERLAY_Y] =
> + var_values[VAR_OY] = av_expr_eval(oy_expr, var_values, NULL);
> +
> + /* calc again in case ox is relative to oy */
> + var_values[VAR_OVERLAY_X] =
> + var_values[VAR_OX] = av_expr_eval(ox_expr, var_values, NULL);
> +
> + /* calc overlay_w and overlay_h again incase relative to ox,oy */
> + var_values[VAR_OVERLAY_W] =
> + var_values[VAR_OW] = av_expr_eval(ow_expr, var_values, NULL);
> + var_values[VAR_OVERLAY_H] =
> + var_values[VAR_OH] = av_expr_eval(oh_expr, var_values, NULL);
> + var_values[VAR_OVERLAY_W] =
> + var_values[VAR_OW] = av_expr_eval(ow_expr, var_values, NULL);
> +
> + av_expr_free(ox_expr);
> + av_expr_free(oy_expr);
> + av_expr_free(ow_expr);
> + av_expr_free(oh_expr);
> +
> + return 0;
> +}
> +
> +static int have_alpha_planar(AVFilterLink *link)
> +{
> + enum AVPixelFormat pix_fmt;
> + const AVPixFmtDescriptor *desc;
> + AVHWFramesContext *fctx;
> +
> + if (link->format == AV_PIX_FMT_QSV) {
> + fctx = (AVHWFramesContext *)link->hw_frames_ctx->data;
> + pix_fmt = fctx->sw_format;
> + }
> +
> + desc = av_pix_fmt_desc_get(pix_fmt);
> + if (!desc)
> + return 0;
> +
> + return !!(desc->flags & AV_PIX_FMT_FLAG_ALPHA);
> +}
> +
> +static int config_main_input(AVFilterLink *inlink)
> +{
> + AVFilterContext *ctx = inlink->dst;
> + QSVOverlayContext *vpp = ctx->priv;
> + mfxVPPCompInputStream *st = &vpp->comp_conf.InputStream[0];
> +
> + av_log(ctx, AV_LOG_DEBUG, "Input[%d] is of %s.\n", FF_INLINK_IDX(inlink),
> + av_get_pix_fmt_name(inlink->format));
> +
> + vpp->var_values[VAR_MAIN_iW] =
> + vpp->var_values[VAR_MW] = inlink->w;
> + vpp->var_values[VAR_MAIN_iH] =
> + vpp->var_values[VAR_MH] = inlink->h;
> +
> + st->DstX = 0;
> + st->DstY = 0;
> + st->DstW = inlink->w;
> + st->DstH = inlink->h;
> + st->GlobalAlphaEnable = 0;
> + st->PixelAlphaEnable = 0;
> +
> + return 0;
> +}
> +
> +static int config_overlay_input(AVFilterLink *inlink)
> +{
> + AVFilterContext *ctx = inlink->dst;
> + AVFilterLink *in0 = ctx->inputs[0];
> + QSVOverlayContext *vpp = ctx->priv;
> + mfxVPPCompInputStream *st = &vpp->comp_conf.InputStream[1];
> + int ret;
> +
> + av_log(ctx, AV_LOG_DEBUG, "Input[%d] is of %s.\n", FF_INLINK_IDX(inlink),
> + av_get_pix_fmt_name(inlink->format));
> + if ((in0->format == AV_PIX_FMT_QSV && inlink->format != AV_PIX_FMT_QSV)
> ||
> + (in0->format != AV_PIX_FMT_QSV && inlink->format == AV_PIX_FMT_QSV))
> {
> + av_log(ctx, AV_LOG_ERROR, "One of the inputs is of AV_PIX_FMT_QSV,"
> + "but the other is of soft pixel format.\n");
> + av_log(ctx, AV_LOG_ERROR, "HW/SW mixed format is not supported
> now.\n");
> + return AVERROR(EINVAL);
> + }
> +
> + vpp->var_values[VAR_OVERLAY_iW] = inlink->w;
> + vpp->var_values[VAR_OVERLAY_iH] = inlink->h;
> + ret = eval_expr(ctx);
> + if (ret < 0)
> + return ret;
> +
> + st->DstX = vpp->var_values[VAR_OX];
> + st->DstY = vpp->var_values[VAR_OY];
> + st->DstW = vpp->var_values[VAR_OW];
> + st->DstH = vpp->var_values[VAR_OH];
> + st->GlobalAlpha = vpp->overlay_alpha;
> + st->GlobalAlphaEnable = (st->GlobalAlpha < 255);
> + st->PixelAlphaEnable = have_alpha_planar(inlink);
> +
> + return 0;
> +}
> +
> +static int config_output(AVFilterLink *outlink)
> +{
> + AVFilterContext *ctx = outlink->src;
> + QSVOverlayContext *vpp = ctx->priv;
> + AVFilterLink *inlink = ctx->inputs[0];
> +
> + av_log(ctx, AV_LOG_DEBUG, "Output is of %s.\n",
> av_get_pix_fmt_name(outlink->format));
> +
> + outlink->w = vpp->var_values[VAR_MW];
> + outlink->h = vpp->var_values[VAR_MH];
> + outlink->frame_rate = inlink->frame_rate;
> + outlink->time_base = av_inv_q(outlink->frame_rate);
> +
> + return ff_qsvvpp_create(ctx, &vpp->qsv, &vpp->qsv_param);
> +}
> +
> +static int blend_frame(AVFilterContext *ctx,
> + AVFrame *mpic, AVFrame *opic)
> +{
> + int ret = 0;
> + QSVOverlayContext *vpp = ctx->priv;
> + AVFrame *opic_copy = NULL;
> +
> + ret = ff_qsvvpp_filter_frame(vpp->qsv, ctx->inputs[0], mpic);
> + if (ret == 0 || ret == AVERROR(EAGAIN)) {
> + /**
> + * Make a copy of the overlay frame. Because:
> + * 1. ff_qsvvpp_filter_frame will take control of the given frame
> + * 2. We need to repeat the overlay frame when 2nd input goes into
> EOF
> + */
> + opic_copy = av_frame_clone(opic);
> + if (!opic_copy)
> + return AVERROR(ENOMEM);
> +
> + ret = ff_qsvvpp_filter_frame(vpp->qsv, ctx->inputs[1], opic_copy);
> + }
> +
> + return ret;
> +}
> +
> +static int handle_overlay_eof(AVFilterContext *ctx)
> +{
> + int ret = 0;
> + QSVOverlayContext *s = ctx->priv;
> + /* Repeat previous frame on secondary input */
> + if (s->over_prev && s->eof_action == EOF_ACTION_REPEAT)
> + ret = blend_frame(ctx, s->main, s->over_prev);
> + /* End both streams */
> + else if (s->eof_action == EOF_ACTION_ENDALL)
> + return AVERROR_EOF;
> +
> + s->main = NULL;
> +
> + return ret;
> +}
> +
> +static int request_frame(AVFilterLink *outlink)
> +{
> + AVFilterContext *ctx = outlink->src;
> + QSVOverlayContext *s = ctx->priv;
> + AVRational tb_main = ctx->inputs[MAIN]->time_base;
> + AVRational tb_over = ctx->inputs[OVERLAY]->time_base;
> + int ret = 0;
> +
> + /* get a frame on the main input */
> + if (!s->main) {
> + ret = ff_request_frame(ctx->inputs[MAIN]);
> + if (ret < 0)
> + return ret;
> + }
> +
> + /* get a new frame on the overlay input, on EOF check setting
> 'eof_action' */
> + if (!s->over_next) {
> + ret = ff_request_frame(ctx->inputs[OVERLAY]);
> + if (ret == AVERROR_EOF)
> + return handle_overlay_eof(ctx);
> + else if (ret < 0)
> + return ret;
> + }
> +
> + while (s->main->pts != AV_NOPTS_VALUE &&
> + s->over_next->pts != AV_NOPTS_VALUE &&
> + av_compare_ts(s->over_next->pts, tb_over, s->main->pts, tb_main)
> < 0) {
> + av_frame_free(&s->over_prev);
> + FFSWAP(AVFrame*, s->over_prev, s->over_next);
> +
> + ret = ff_request_frame(ctx->inputs[OVERLAY]);
> + if (ret == AVERROR_EOF)
> + return handle_overlay_eof(ctx);
> + else if (ret < 0)
> + return ret;
> + }
> +
> + if (s->main->pts == AV_NOPTS_VALUE ||
> + s->over_next->pts == AV_NOPTS_VALUE ||
> + !av_compare_ts(s->over_next->pts, tb_over, s->main->pts, tb_main)) {
> + ret = blend_frame(ctx, s->main, s->over_next);
> + av_frame_free(&s->over_prev);
> + FFSWAP(AVFrame*, s->over_prev, s->over_next);
> + } else if (s->over_prev) {
> + ret = blend_frame(ctx, s->main, s->over_prev);
> + } else {
> + av_frame_free(&s->main);
> + ret = AVERROR(EAGAIN);
> + }
> +
> + s->main = NULL;
> +
> + return ret ;
> +}
> +
> +static int filter_frame_main(AVFilterLink *inlink, AVFrame *frame)
> +{
> + QSVOverlayContext *s = inlink->dst->priv;
> +
> + av_assert0(!s->main);
> + s->main = frame;
> +
> + return 0;
> +}
> +
> +static int filter_frame_overlay(AVFilterLink *inlink, AVFrame *frame)
> +{
> + QSVOverlayContext *s = inlink->dst->priv;
> +
> + av_assert0(!s->over_next);
> + s->over_next = frame;
> +
> + return 0;
> +}
> +
> +static int overlay_qsv_init(AVFilterContext *ctx)
> +{
> + QSVOverlayContext *vpp = ctx->priv;
> +
> + /* fill composite config */
> + vpp->comp_conf.Header.BufferId = MFX_EXTBUFF_VPP_COMPOSITE;
> + vpp->comp_conf.Header.BufferSz = sizeof(vpp->comp_conf);
> + vpp->comp_conf.NumInputStream = ctx->nb_inputs;
> + vpp->comp_conf.InputStream =
> + av_mallocz(sizeof(*vpp->comp_conf.InputStream) * ctx->nb_inputs);
> + if (!vpp->comp_conf.InputStream)
> + return AVERROR(ENOMEM);
> +
> + /* initiallize QSVVPP params */
typo here ^^^
> + vpp->qsv_param.filter_frame = NULL;
> + vpp->qsv_param.ext_buf =
> av_mallocz(sizeof(*vpp->qsv_param.ext_buf));
> + if (!vpp->qsv_param.ext_buf)
> + return AVERROR(ENOMEM);
> +
> + vpp->qsv_param.ext_buf[0] = (mfxExtBuffer *)&vpp->comp_conf;
> + vpp->qsv_param.num_ext_buf = 1;
> + vpp->qsv_param.out_sw_format = AV_PIX_FMT_NV12;
> + vpp->qsv_param.num_crop = 0;
> +
> + return 0;
> +}
> +
> +static void overlay_qsv_uninit(AVFilterContext *ctx)
> +{
> + QSVOverlayContext *vpp = ctx->priv;
> +
> + av_frame_free(&vpp->main);
> + av_frame_free(&vpp->over_prev);
> + av_frame_free(&vpp->over_next);
> + ff_qsvvpp_free(&vpp->qsv);
> + av_freep(&vpp->comp_conf.InputStream);
> + av_freep(&vpp->qsv_param.ext_buf);
> +}
> +
> +static int overlay_qsv_query_formats(AVFilterContext *ctx)
> +{
> + int i;
> +
> + static const enum AVPixelFormat main_in_fmts[] = {
> + AV_PIX_FMT_YUV420P,
> + AV_PIX_FMT_NV12,
> + AV_PIX_FMT_YUYV422,
> + AV_PIX_FMT_RGB32,
> + AV_PIX_FMT_QSV,
> + AV_PIX_FMT_NONE
> + };
> + static const enum AVPixelFormat out_pix_fmts[] = {
> + AV_PIX_FMT_NV12,
> + AV_PIX_FMT_QSV,
> + AV_PIX_FMT_NONE
> + };
> +
> + for(i = 0; i < ctx->nb_inputs; i++)
> + ff_formats_ref(ff_make_format_list(main_in_fmts),
> &ctx->inputs[i]->out_formats);
> +
> + ff_formats_ref(ff_make_format_list(out_pix_fmts),
> &ctx->outputs[0]->in_formats);
> +
> + return 0;
> +}
> +
> +static const AVClass overlay_qsv_class = {
> + .class_name = "overlay_qsv",
> + .item_name = av_default_item_name,
> + .option = options,
> + .version = LIBAVUTIL_VERSION_INT,
> +};
> +
> +static const AVFilterPad overlay_qsv_inputs[] = {
> + {
> + .name = "main",
> + .type = AVMEDIA_TYPE_VIDEO,
> + .filter_frame = filter_frame_main,
> + .config_props = config_main_input,
> + .needs_fifo = 1,
> + },
> + {
> + .name = "overlay",
> + .type = AVMEDIA_TYPE_VIDEO,
> + .filter_frame = filter_frame_overlay,
> + .config_props = config_overlay_input,
> + .needs_fifo = 1,
> + },
> + { NULL }
> +};
> +
> +static const AVFilterPad overlay_qsv_outputs[] = {
> + {
> + .name = "default",
> + .type = AVMEDIA_TYPE_VIDEO,
> + .config_props = config_output,
> + .request_frame = request_frame,
> + },
> + { NULL }
> +};
> +
> +AVFilter ff_vf_overlay_qsv = {
> + .name = "overlay_qsv",
> + .description = NULL_IF_CONFIG_SMALL("Quick Sync Video overlay."),
> + .priv_size = sizeof(QSVOverlayContext),
> + .query_formats = overlay_qsv_query_formats,
> + .init = overlay_qsv_init,
> + .uninit = overlay_qsv_uninit,
> + .inputs = overlay_qsv_inputs,
> + .outputs = overlay_qsv_outputs,
> + .priv_class = &overlay_qsv_class,
> + .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
> +};
Beside those small issues I guess it is good to go.
lu
_______________________________________________
libav-devel mailing list
[email protected]
https://lists.libav.org/mailman/listinfo/libav-devel
