Re: [FFmpeg-devel] [PATCH v1 3/3] libavutil: Add thumbnail output to vaapi_h264 decoder
> -Original Message-
> From: Zhou, Zachary
> Sent: Monday, April 8, 2019 3:10 PM
> To: ffmpeg-devel@ffmpeg.org
> Cc: Zhou, Zachary
> Subject: [PATCH v1 3/3] libavutil: Add thumbnail output to vaapi_h264
> decoder
>
> This is sample code for reference
>
> HW support for decode+scaling in a single HW command (VDBOX+SFC).
> The primary target usage is video analytics, but can be used playback,
> transcoding, etc.
>
> For VAAPI -
> https://github.com/intel/libva
> basically, it allows multiple outputs (in different resolutions) using the
> decode
> context in a single call (you can search for “additional_outputs” in va.h).
>
> VAAPI sample code -
> https://github.com/intel/libva-
> utils/commit/957a269f02b00760b7e807643c821ee26abc529b
> ---
> libavutil/frame.c | 1 +
> libavutil/frame.h | 13 +++
> libavutil/hwcontext_vaapi.c | 73 -
> libavutil/hwcontext_vaapi.h | 6 +++
> 4 files changed, 92 insertions(+), 1 deletion(-)
>
> diff --git a/libavutil/frame.c b/libavutil/frame.c index
> dcf1fc3d17..aa9cd744ae
> 100644
> --- a/libavutil/frame.c
> +++ b/libavutil/frame.c
> @@ -842,6 +842,7 @@ const char *av_frame_side_data_name(enum
> AVFrameSideDataType type) #endif
> case AV_FRAME_DATA_DYNAMIC_HDR_PLUS: return "HDR Dynamic
> Metadata SMPTE2094-40 (HDR10+)";
> case AV_FRAME_DATA_REGIONS_OF_INTEREST: return "Regions Of
> Interest";
> +case AV_FRAME_DATA_SFC_INFO: return "SFC Information";
> }
> return NULL;
> }
> diff --git a/libavutil/frame.h b/libavutil/frame.h index
> 8aa3e88367..616e7d9d4e 100644
> --- a/libavutil/frame.h
> +++ b/libavutil/frame.h
> @@ -179,6 +179,11 @@ enum AVFrameSideDataType {
> * array element is implied by AVFrameSideData.size /
> AVRegionOfInterest.self_size.
> */
> AV_FRAME_DATA_REGIONS_OF_INTEREST,
> +
> +/**
> + * VDENC SFC information.
> + */
> +AV_FRAME_DATA_SFC_INFO,
> };
>
> enum AVActiveFormatDescription {
> @@ -235,6 +240,14 @@ typedef struct AVRegionOfInterest {
> AVRational qoffset;
> } AVRegionOfInterest;
>
> +/**
> + * Structure to hold Information of SFC.
> + *
> + */
> +typedef struct AVSFCInfo {
> +uint8_t *data;
> +} AVSFCInfo;
> +
> /**
> * This structure describes decoded (raw) audio or video data.
> *
> diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c index
> 8624369bb9..caa6d02b15 100644
> --- a/libavutil/hwcontext_vaapi.c
> +++ b/libavutil/hwcontext_vaapi.c
> @@ -642,8 +642,74 @@ static void
> vaapi_frames_uninit(AVHWFramesContext *hwfc)
> av_freep(>attributes);
> }
>
> +static int vaapi_get_sfc_buffer(AVHWFramesContext *hwfc, AVFrame
> +*frame) {
> +AVVAAPIFramesContext *avfc = hwfc->hwctx;
> +AVFrameSideData *frame_sfc_sd = av_frame_get_side_data(frame,
> +
> AV_FRAME_DATA_SFC_INFO);
> +if (frame_sfc_sd) {
> +AVVAAPIDeviceContext *hwctx = hwfc->device_ctx->hwctx;
> +VASurfaceID sfc_surface_id;
> +VASurfaceAttrib attrib;
> +unsigned int rt_format;
> +AVBufferRef *ref;
> +VAStatus vas;
> +
> +memset(, 0, sizeof(attrib));
> +attrib.type = VASurfaceAttribPixelFormat;
> +attrib.flags = VA_SURFACE_ATTRIB_SETTABLE;
> +attrib.value.type = VAGenericValueTypeInteger;
> +
> +switch (avfc->sfc_format) {
> +case AV_PIX_FMT_NV12:
> +attrib.value.value.i = VA_FOURCC_NV12;
> +rt_format = VA_RT_FORMAT_YUV420;
> +break;
> +case AV_PIX_FMT_ARGB:
> +attrib.value.value.i = VA_FOURCC_ARGB;
> +rt_format = VA_RT_FORMAT_RGB32;
> +break;
> +default:
> +av_log(hwfc, AV_LOG_ERROR, "Unknown sfc format: %d\n", avfc-
> >sfc_format);
> +return AVERROR(EINVAL);
> +}
> +
> +vas = vaCreateSurfaces(hwctx->display, rt_format,
> + avfc->sfc_width, avfc->sfc_height,
> + _surface_id, 1,
> + , 1);
> +if (vas != VA_STATUS_SUCCESS) {
> +av_log(hwfc, AV_LOG_ERROR, "Failed to create sfc surface: "
> + "%d (%s).\n", vas, vaErrorStr(vas));
> +return AVERROR(EIO);
> +}
> +
> +av_log(hwfc, AV_LOG_DEBUG, "Created sfc surface %#x. sfc_format(%d),
> w(%d), h(%d)\n",
> + sfc_surface_id, avfc->sfc_format, avfc->sfc_width,
> + avfc->sfc_height);
> +
> +ref = av_buffer_create((uint8_t*)(uintptr_t)sfc_surface_id,
> + sizeof(sfc_surface_id), _buffer_free,
> + hwfc, AV_BUFFER_FLAG_READONLY);
> +if (!ref) {
> +vaDestroySurfaces(hwctx->display, _surface_id, 1);
> +return AVERROR(EIO);
> +}
> +
> +frame_sfc_sd->buf = ref;
> +
> +av_log(hwfc, AV_LOG_DEBUG,
[FFmpeg-devel] [PATCH v1 3/3] libavutil: Add thumbnail output to vaapi_h264 decoder
This is sample code for reference
HW support for decode+scaling in a single HW command (VDBOX+SFC).
The primary target usage is video analytics, but can be used playback,
transcoding, etc.
For VAAPI -
https://github.com/intel/libva
basically, it allows multiple outputs (in different resolutions) using the
decode context in a single call (you can search for “additional_outputs” in
va.h).
VAAPI sample code -
https://github.com/intel/libva-utils/commit/957a269f02b00760b7e807643c821ee26abc529b
---
libavutil/frame.c | 1 +
libavutil/frame.h | 13 +++
libavutil/hwcontext_vaapi.c | 73 -
libavutil/hwcontext_vaapi.h | 6 +++
4 files changed, 92 insertions(+), 1 deletion(-)
diff --git a/libavutil/frame.c b/libavutil/frame.c
index dcf1fc3d17..aa9cd744ae 100644
--- a/libavutil/frame.c
+++ b/libavutil/frame.c
@@ -842,6 +842,7 @@ const char *av_frame_side_data_name(enum
AVFrameSideDataType type)
#endif
case AV_FRAME_DATA_DYNAMIC_HDR_PLUS: return "HDR Dynamic Metadata
SMPTE2094-40 (HDR10+)";
case AV_FRAME_DATA_REGIONS_OF_INTEREST: return "Regions Of Interest";
+case AV_FRAME_DATA_SFC_INFO: return "SFC Information";
}
return NULL;
}
diff --git a/libavutil/frame.h b/libavutil/frame.h
index 8aa3e88367..616e7d9d4e 100644
--- a/libavutil/frame.h
+++ b/libavutil/frame.h
@@ -179,6 +179,11 @@ enum AVFrameSideDataType {
* array element is implied by AVFrameSideData.size /
AVRegionOfInterest.self_size.
*/
AV_FRAME_DATA_REGIONS_OF_INTEREST,
+
+/**
+ * VDENC SFC information.
+ */
+AV_FRAME_DATA_SFC_INFO,
};
enum AVActiveFormatDescription {
@@ -235,6 +240,14 @@ typedef struct AVRegionOfInterest {
AVRational qoffset;
} AVRegionOfInterest;
+/**
+ * Structure to hold Information of SFC.
+ *
+ */
+typedef struct AVSFCInfo {
+uint8_t *data;
+} AVSFCInfo;
+
/**
* This structure describes decoded (raw) audio or video data.
*
diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 8624369bb9..caa6d02b15 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -642,8 +642,74 @@ static void vaapi_frames_uninit(AVHWFramesContext *hwfc)
av_freep(>attributes);
}
+static int vaapi_get_sfc_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
+{
+AVVAAPIFramesContext *avfc = hwfc->hwctx;
+AVFrameSideData *frame_sfc_sd = av_frame_get_side_data(frame,
+
AV_FRAME_DATA_SFC_INFO);
+if (frame_sfc_sd) {
+AVVAAPIDeviceContext *hwctx = hwfc->device_ctx->hwctx;
+VASurfaceID sfc_surface_id;
+VASurfaceAttrib attrib;
+unsigned int rt_format;
+AVBufferRef *ref;
+VAStatus vas;
+
+memset(, 0, sizeof(attrib));
+attrib.type = VASurfaceAttribPixelFormat;
+attrib.flags = VA_SURFACE_ATTRIB_SETTABLE;
+attrib.value.type = VAGenericValueTypeInteger;
+
+switch (avfc->sfc_format) {
+case AV_PIX_FMT_NV12:
+attrib.value.value.i = VA_FOURCC_NV12;
+rt_format = VA_RT_FORMAT_YUV420;
+break;
+case AV_PIX_FMT_ARGB:
+attrib.value.value.i = VA_FOURCC_ARGB;
+rt_format = VA_RT_FORMAT_RGB32;
+break;
+default:
+av_log(hwfc, AV_LOG_ERROR, "Unknown sfc format: %d\n",
avfc->sfc_format);
+return AVERROR(EINVAL);
+}
+
+vas = vaCreateSurfaces(hwctx->display, rt_format,
+ avfc->sfc_width, avfc->sfc_height,
+ _surface_id, 1,
+ , 1);
+if (vas != VA_STATUS_SUCCESS) {
+av_log(hwfc, AV_LOG_ERROR, "Failed to create sfc surface: "
+ "%d (%s).\n", vas, vaErrorStr(vas));
+return AVERROR(EIO);
+}
+
+av_log(hwfc, AV_LOG_DEBUG, "Created sfc surface %#x. sfc_format(%d),
w(%d), h(%d)\n",
+ sfc_surface_id, avfc->sfc_format, avfc->sfc_width,
avfc->sfc_height);
+
+ref = av_buffer_create((uint8_t*)(uintptr_t)sfc_surface_id,
+ sizeof(sfc_surface_id), _buffer_free,
+ hwfc, AV_BUFFER_FLAG_READONLY);
+if (!ref) {
+vaDestroySurfaces(hwctx->display, _surface_id, 1);
+return AVERROR(EIO);
+}
+
+frame_sfc_sd->buf = ref;
+
+av_log(hwfc, AV_LOG_DEBUG, "Created sfc av buffer ref: %p\n", ref);
+
+} else {
+av_log(hwfc, AV_LOG_DEBUG, "VDSFC frame_sfc_sd is null\n");
+}
+
+return 0;
+}
+
static int vaapi_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
{
+int err;
+
frame->buf[0] = av_buffer_pool_get(hwfc->pool);
if (!frame->buf[0])
return AVERROR(ENOMEM);
@@ -653,6 +719,11 @@ static int vaapi_get_buffer(AVHWFramesContext *hwfc,
AVFrame *frame)
frame->width = hwfc->width;
frame->height =
