[FFmpeg-devel] [PATCH 2/3] avcodec/mediacodecenc: workaround the alignment requirement only for H.264
From: Zhao Zhili
There is no bsf for other codecs to modify crop info except H.265.
For H.265, the assumption that FFALIGN(width, 16)xFFALIGN(height, 16)
is the video resolution can be wrong, since the encoder can use CTU
larger than 16x16. In that case, use FFALIGN(width, 16) - width
as crop_right is incorrect. So disable the workaround for H.265 now.
---
libavcodec/mediacodecenc.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c
index bbf570e7be..bfff149039 100644
--- a/libavcodec/mediacodecenc.c
+++ b/libavcodec/mediacodecenc.c
@@ -234,7 +234,8 @@ static av_cold int mediacodec_init(AVCodecContext *avctx)
ff_AMediaFormat_setString(format, "mime", codec_mime);
// Workaround the alignment requirement of mediacodec. We can't do it
// silently for AV_PIX_FMT_MEDIACODEC.
-if (avctx->pix_fmt != AV_PIX_FMT_MEDIACODEC) {
+if (avctx->pix_fmt != AV_PIX_FMT_MEDIACODEC &&
+avctx->codec_id == AV_CODEC_ID_H264) {
s->width = FFALIGN(avctx->width, 16);
s->height = FFALIGN(avctx->height, 16);
} else {
--
2.42.0
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH 1/3] avcodec/h265_metadata: Add options to set width/height after crop
From: Zhao Zhili
It's a common usecase to request a video size after crop. Before
this patch, user must know the video size before crop, then set
crop_right/crop_bottom accordingly. Since HEVC can have different
CTU size, it's not easy to get/deduce the video size before crop.
With the new width/height options, there is no such requirement.
Signed-off-by: Zhao Zhili
---
doc/bitstream_filters.texi | 4 ++
libavcodec/bsf/h265_metadata.c | 101 -
libavcodec/version.h | 2 +-
3 files changed, 103 insertions(+), 4 deletions(-)
diff --git a/doc/bitstream_filters.texi b/doc/bitstream_filters.texi
index 3d4dda04fc..c03f04f858 100644
--- a/doc/bitstream_filters.texi
+++ b/doc/bitstream_filters.texi
@@ -456,6 +456,10 @@ will replace the current ones if the stream is already
cropped.
These fields are set in pixels. Note that some sizes may not be
representable if the chroma is subsampled (H.265 section 7.4.3.2.1).
+@item width
+@item height
+Set width and height after crop.
+
@item level
Set the level in the VPS and SPS. See H.265 section A.4 and tables
A.6 and A.7.
diff --git a/libavcodec/bsf/h265_metadata.c b/libavcodec/bsf/h265_metadata.c
index c9e1cc3eed..eba00c20d5 100644
--- a/libavcodec/bsf/h265_metadata.c
+++ b/libavcodec/bsf/h265_metadata.c
@@ -58,6 +58,8 @@ typedef struct H265MetadataContext {
int crop_right;
int crop_top;
int crop_bottom;
+int width;
+int height;
int level;
int level_guess;
@@ -187,12 +189,94 @@ static int h265_metadata_update_vps(AVBSFContext *bsf,
return 0;
}
+static int h265_metadata_deduce_crop(AVBSFContext *bsf, const H265RawSPS *sps,
+ int *crop_left, int *crop_right,
+ int *crop_top, int *crop_bottom)
+{
+const H265MetadataContext *ctx = bsf->priv_data;
+int left = ctx->crop_left;
+int right = ctx->crop_right;
+int top = ctx->crop_top;
+int bottom = ctx->crop_bottom;
+
+if (ctx->width > 0) {
+if (ctx->width > sps->pic_width_in_luma_samples) {
+av_log(bsf, AV_LOG_ERROR,
+ "The width option value %d is larger than picture width
%d\n",
+ ctx->width, sps->pic_width_in_luma_samples);
+return AVERROR(EINVAL);
+}
+
+if (left < 0) {
+if (right > 0)
+left = sps->pic_width_in_luma_samples - ctx->width - right;
+else
+left = 0;
+}
+
+if (right < 0)
+right = sps->pic_width_in_luma_samples - ctx->width - left;
+
+if (left < 0 || right < 0 || (left + right + ctx->width) !=
+sps->pic_width_in_luma_samples) {
+av_log(bsf, AV_LOG_ERROR,
+ "Invalid value for crop_left %d, crop_right %d, width after
"
+ "crop %d, with picture width %d\n",
+ ctx->crop_left, ctx->crop_right, ctx->width,
+ sps->pic_width_in_luma_samples);
+return AVERROR(EINVAL);
+}
+}
+
+if (ctx->height > 0) {
+if (ctx->height > sps->pic_height_in_luma_samples) {
+av_log(bsf, AV_LOG_ERROR,
+ "The height option value %d is larger than picture height
%d\n",
+ ctx->height, sps->pic_height_in_luma_samples);
+return AVERROR(EINVAL);
+}
+
+if (top < 0) {
+if (bottom > 0)
+top = sps->pic_height_in_luma_samples - ctx->height - bottom;
+else
+top = 0;
+}
+
+if (bottom < 0)
+bottom = sps->pic_height_in_luma_samples - ctx->height - top;
+
+if (top < 0 || bottom < 0 || (top + bottom + ctx->height) !=
+sps->pic_height_in_luma_samples) {
+av_log(bsf, AV_LOG_ERROR,
+ "Invalid value for crop_top %d, crop_bottom %d, height
after "
+ "crop %d, with picture height %d\n",
+ ctx->crop_top, ctx->crop_bottom, ctx->height,
+ sps->pic_height_in_luma_samples);
+return AVERROR(EINVAL);
+}
+}
+
+*crop_left = left;
+*crop_right = right;
+*crop_top = top;
+*crop_bottom = bottom;
+
+return 0;
+}
+
static int h265_metadata_update_sps(AVBSFContext *bsf,
H265RawSPS *sps)
{
H265MetadataContext *ctx = bsf->priv_data;
int need_vui = 0;
int crop_unit_x, crop_unit_y;
+/* Use local variables to avoid modifying context fields in case of video
+ * resolution changed. Crop doesn't work well with resolution change, this
+ * is the best we can do.
+ */
+int crop_left, crop_right, cr
[FFmpeg-devel] [PATCH 3/3] avcodec/mediacodecenc: workaround the alignment requirement for H.265
From: Zhao Zhili
Signed-off-by: Zhao Zhili
---
libavcodec/mediacodecenc.c | 11 ---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c
index bfff149039..a42270551b 100644
--- a/libavcodec/mediacodecenc.c
+++ b/libavcodec/mediacodecenc.c
@@ -154,8 +154,12 @@ static int mediacodec_init_bsf(AVCodecContext *avctx)
ret = snprintf(str, sizeof(str),
"h264_metadata=crop_right=%d:crop_bottom=%d",
crop_right, crop_bottom);
else if (avctx->codec_id == AV_CODEC_ID_HEVC)
-ret = snprintf(str, sizeof(str),
"hevc_metadata=crop_right=%d:crop_bottom=%d",
- crop_right, crop_bottom);
+/* Encoder can use CTU size larger than 16x16, so the real crop
+ * margin can be larger than crop_right/crop_bottom. Let bsf figure
+ * out the real crop margin.
+ */
+ret = snprintf(str, sizeof(str),
"hevc_metadata=width=%d:height=%d",
+ avctx->width, avctx->height);
if (ret >= sizeof(str))
return AVERROR_BUFFER_TOO_SMALL;
}
@@ -235,7 +239,8 @@ static av_cold int mediacodec_init(AVCodecContext *avctx)
// Workaround the alignment requirement of mediacodec. We can't do it
// silently for AV_PIX_FMT_MEDIACODEC.
if (avctx->pix_fmt != AV_PIX_FMT_MEDIACODEC &&
-avctx->codec_id == AV_CODEC_ID_H264) {
+(avctx->codec_id == AV_CODEC_ID_H264 ||
+ avctx->codec_id == AV_CODEC_ID_HEVC)) {
s->width = FFALIGN(avctx->width, 16);
s->height = FFALIGN(avctx->height, 16);
} else {
--
2.42.0
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH] avcodec/h265_metadata: Add option to set width/height after crop
From: Zhao Zhili
It's a common usecase to request a video size after crop. Before
this patch, user must know the video size before crop, then set
crop_right/crop_bottom accordingly. Since HEVC can have different
CTU size, it's not easy to get/deduce the video size before crop.
With the new width/height options, there is no such requirement.
Signed-off-by: Zhao Zhili
---
doc/bitstream_filters.texi | 4 ++
libavcodec/bsf/h265_metadata.c | 101 -
libavcodec/version.h | 2 +-
3 files changed, 103 insertions(+), 4 deletions(-)
diff --git a/doc/bitstream_filters.texi b/doc/bitstream_filters.texi
index 3d4dda04fc..c03f04f858 100644
--- a/doc/bitstream_filters.texi
+++ b/doc/bitstream_filters.texi
@@ -456,6 +456,10 @@ will replace the current ones if the stream is already
cropped.
These fields are set in pixels. Note that some sizes may not be
representable if the chroma is subsampled (H.265 section 7.4.3.2.1).
+@item width
+@item height
+Set width and height after crop.
+
@item level
Set the level in the VPS and SPS. See H.265 section A.4 and tables
A.6 and A.7.
diff --git a/libavcodec/bsf/h265_metadata.c b/libavcodec/bsf/h265_metadata.c
index c9e1cc3eed..eba00c20d5 100644
--- a/libavcodec/bsf/h265_metadata.c
+++ b/libavcodec/bsf/h265_metadata.c
@@ -58,6 +58,8 @@ typedef struct H265MetadataContext {
int crop_right;
int crop_top;
int crop_bottom;
+int width;
+int height;
int level;
int level_guess;
@@ -187,12 +189,94 @@ static int h265_metadata_update_vps(AVBSFContext *bsf,
return 0;
}
+static int h265_metadata_deduce_crop(AVBSFContext *bsf, const H265RawSPS *sps,
+ int *crop_left, int *crop_right,
+ int *crop_top, int *crop_bottom)
+{
+const H265MetadataContext *ctx = bsf->priv_data;
+int left = ctx->crop_left;
+int right = ctx->crop_right;
+int top = ctx->crop_top;
+int bottom = ctx->crop_bottom;
+
+if (ctx->width > 0) {
+if (ctx->width > sps->pic_width_in_luma_samples) {
+av_log(bsf, AV_LOG_ERROR,
+ "The width option value %d is larger than picture width
%d\n",
+ ctx->width, sps->pic_width_in_luma_samples);
+return AVERROR(EINVAL);
+}
+
+if (left < 0) {
+if (right > 0)
+left = sps->pic_width_in_luma_samples - ctx->width - right;
+else
+left = 0;
+}
+
+if (right < 0)
+right = sps->pic_width_in_luma_samples - ctx->width - left;
+
+if (left < 0 || right < 0 || (left + right + ctx->width) !=
+sps->pic_width_in_luma_samples) {
+av_log(bsf, AV_LOG_ERROR,
+ "Invalid value for crop_left %d, crop_right %d, width after
"
+ "crop %d, with picture width %d\n",
+ ctx->crop_left, ctx->crop_right, ctx->width,
+ sps->pic_width_in_luma_samples);
+return AVERROR(EINVAL);
+}
+}
+
+if (ctx->height > 0) {
+if (ctx->height > sps->pic_height_in_luma_samples) {
+av_log(bsf, AV_LOG_ERROR,
+ "The height option value %d is larger than picture height
%d\n",
+ ctx->height, sps->pic_height_in_luma_samples);
+return AVERROR(EINVAL);
+}
+
+if (top < 0) {
+if (bottom > 0)
+top = sps->pic_height_in_luma_samples - ctx->height - bottom;
+else
+top = 0;
+}
+
+if (bottom < 0)
+bottom = sps->pic_height_in_luma_samples - ctx->height - top;
+
+if (top < 0 || bottom < 0 || (top + bottom + ctx->height) !=
+sps->pic_height_in_luma_samples) {
+av_log(bsf, AV_LOG_ERROR,
+ "Invalid value for crop_top %d, crop_bottom %d, height
after "
+ "crop %d, with picture height %d\n",
+ ctx->crop_top, ctx->crop_bottom, ctx->height,
+ sps->pic_height_in_luma_samples);
+return AVERROR(EINVAL);
+}
+}
+
+*crop_left = left;
+*crop_right = right;
+*crop_top = top;
+*crop_bottom = bottom;
+
+return 0;
+}
+
static int h265_metadata_update_sps(AVBSFContext *bsf,
H265RawSPS *sps)
{
H265MetadataContext *ctx = bsf->priv_data;
int need_vui = 0;
int crop_unit_x, crop_unit_y;
+/* Use local variables to avoid modifying context fields in case of video
+ * resolution changed. Crop doesn't work well with resolution change, this
+ * is the best we can do.
+ */
+int crop_left, crop_right, cr
Re: [FFmpeg-devel] [PATCH] avutil/hwcontext: Set proper CVBuffer colorspace
> On May 20, 2024, at 09:12, Marvin Scholz wrote:
>
> Fix #10884
LGTM except the commit subject should mention videotoolbox or vt in short.
> ---
> libavutil/hwcontext_videotoolbox.c | 54 +-
> 1 file changed, 38 insertions(+), 16 deletions(-)
>
> diff --git a/libavutil/hwcontext_videotoolbox.c
> b/libavutil/hwcontext_videotoolbox.c
> index 9f82b104c3..4a35bfc7ff 100644
> --- a/libavutil/hwcontext_videotoolbox.c
> +++ b/libavutil/hwcontext_videotoolbox.c
> @@ -530,6 +530,8 @@ CFStringRef av_map_videotoolbox_color_trc_from_av(enum
> AVColorTransferCharacteri
> static int vt_pixbuf_set_colorspace(void *log_ctx,
> CVPixelBufferRef pixbuf, const AVFrame
> *src)
> {
> +CGColorSpaceRef colorspace = NULL;
> +CFMutableDictionaryRef attachments = NULL;
> CFStringRef colormatrix = NULL, colorpri = NULL, colortrc = NULL;
> Float32 gamma = 0;
>
> @@ -550,37 +552,57 @@ static int vt_pixbuf_set_colorspace(void *log_ctx,
> else if (src->color_trc == AVCOL_TRC_GAMMA28)
> gamma = 2.8;
>
> +attachments = CFDictionaryCreateMutable(NULL, 0,
> +
> ,
> +
> );
> +if (!attachments)
> +return AVERROR(ENOMEM);
> +
> if (colormatrix) {
> -CVBufferSetAttachment(
> -pixbuf,
> +CFDictionarySetValue(
> +attachments,
> kCVImageBufferYCbCrMatrixKey,
> -colormatrix,
> -kCVAttachmentMode_ShouldPropagate);
> +colormatrix);
> }
> if (colorpri) {
> -CVBufferSetAttachment(
> -pixbuf,
> +CFDictionarySetValue(
> +attachments,
> kCVImageBufferColorPrimariesKey,
> -colorpri,
> -kCVAttachmentMode_ShouldPropagate);
> +colorpri);
> }
> if (colortrc) {
> -CVBufferSetAttachment(
> -pixbuf,
> +CFDictionarySetValue(
> +attachments,
> kCVImageBufferTransferFunctionKey,
> -colortrc,
> -kCVAttachmentMode_ShouldPropagate);
> +colortrc);
> }
> if (gamma != 0) {
> CFNumberRef gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type,
> );
> -CVBufferSetAttachment(
> -pixbuf,
> +CFDictionarySetValue(
> +attachments,
> kCVImageBufferGammaLevelKey,
> -gamma_level,
> -kCVAttachmentMode_ShouldPropagate);
> +gamma_level);
> CFRelease(gamma_level);
> }
>
> +if (__builtin_available(macOS 10.8, iOS 10, *))
> +colorspace =
> CVImageBufferCreateColorSpaceFromAttachments(attachments);
> +
> +if (colorspace) {
> +CFDictionarySetValue(
> +attachments,
> +kCVImageBufferCGColorSpaceKey,
> +colorspace);
> +CFRelease(colorspace);
> +} else
> +av_log(log_ctx, AV_LOG_WARNING, "Unable to set proper colorspace for
> the CVImageBuffer.\n");
> +
> +CVBufferSetAttachments(
> +pixbuf,
> +attachments,
> +kCVAttachmentMode_ShouldPropagate);
> +CFRelease(attachments);
> +
> return 0;
> }
>
>
> base-commit: 463c573e6b6489c588bee90124d5cf92db8ccaaa
> --
> 2.39.3 (Apple Git-145)
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [RFC PATCH] avfilter/video: Protect frame_pool from multi-threads access
> On May 10, 2024, at 16:57, Zhao Zhili wrote:
>
> From: Zhao Zhili
>
> Fix crash with
> ./ffplay -f lavfi -i movie=foo.mp4,drawtext=text=bar
This is an RFC because I’m not sure whether it’s a valid use case or not to call
ff_default_get_video_buffer2 from multiple threads. On one hand, it’s rare used
with multithreads. On the other hand, it can be hidden deep and dangerous.
> ---
> libavfilter/avfilter.c | 2 ++
> libavfilter/avfilter_internal.h | 2 ++
> libavfilter/video.c | 14 --
> 3 files changed, 12 insertions(+), 6 deletions(-)
>
> diff --git a/libavfilter/avfilter.c b/libavfilter/avfilter.c
> index 049e4f62ca..44da809c15 100644
> --- a/libavfilter/avfilter.c
> +++ b/libavfilter/avfilter.c
> @@ -176,6 +176,7 @@ int avfilter_link(AVFilterContext *src, unsigned srcpad,
> li = av_mallocz(sizeof(*li));
> if (!li)
> return AVERROR(ENOMEM);
> +ff_mutex_init(>frame_pool_mutex, NULL);
> link = >l;
>
> src->outputs[srcpad] = dst->inputs[dstpad] = link;
> @@ -203,6 +204,7 @@ static void link_free(AVFilterLink **link)
>
> ff_framequeue_free(>fifo);
> ff_frame_pool_uninit(>frame_pool);
> +ff_mutex_destroy(>frame_pool_mutex);
> av_channel_layout_uninit(&(*link)->ch_layout);
>
> av_freep(link);
> diff --git a/libavfilter/avfilter_internal.h b/libavfilter/avfilter_internal.h
> index 2c31c3e7de..eae3a699c3 100644
> --- a/libavfilter/avfilter_internal.h
> +++ b/libavfilter/avfilter_internal.h
> @@ -27,6 +27,7 @@
>
> #include
>
> +#include "libavutil/thread.h"
> #include "avfilter.h"
> #include "framequeue.h"
>
> @@ -34,6 +35,7 @@ typedef struct FilterLinkInternal {
> AVFilterLink l;
>
> struct FFFramePool *frame_pool;
> +AVMutex frame_pool_mutex;
>
> /**
> * Queue of frames waiting to be filtered.
> diff --git a/libavfilter/video.c b/libavfilter/video.c
> index 89d0797ab5..30eba76cbd 100644
> --- a/libavfilter/video.c
> +++ b/libavfilter/video.c
> @@ -70,17 +70,17 @@ AVFrame *ff_default_get_video_buffer2(AVFilterLink *link,
> int w, int h, int alig
> return frame;
> }
>
> +ff_mutex_lock(>frame_pool_mutex);
> if (!li->frame_pool) {
> li->frame_pool = ff_frame_pool_video_init(av_buffer_allocz, w, h,
> link->format, align);
> if (!li->frame_pool)
> -return NULL;
> +goto out;
> } else {
> if (ff_frame_pool_get_video_config(li->frame_pool,
>_width, _height,
> - _format, _align) < 0) {
> -return NULL;
> -}
> + _format, _align) < 0)
> +goto out;
>
> if (pool_width != w || pool_height != h ||
> pool_format != link->format || pool_align != align) {
> @@ -89,18 +89,20 @@ AVFrame *ff_default_get_video_buffer2(AVFilterLink *link,
> int w, int h, int alig
> li->frame_pool = ff_frame_pool_video_init(av_buffer_allocz, w, h,
> link->format, align);
> if (!li->frame_pool)
> -return NULL;
> +goto out;
> }
> }
>
> frame = ff_frame_pool_get(li->frame_pool);
> if (!frame)
> -return NULL;
> +goto out;
>
> frame->sample_aspect_ratio = link->sample_aspect_ratio;
> frame->colorspace = link->colorspace;
> frame->color_range = link->color_range;
>
> +out:
> +ff_mutex_unlock(>frame_pool_mutex);
> return frame;
> }
>
> --
> 2.42.0
>
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [RFC PATCH] avfilter/video: Protect frame_pool from multi-threads access
From: Zhao Zhili
Fix crash with
./ffplay -f lavfi -i movie=foo.mp4,drawtext=text=bar
---
libavfilter/avfilter.c | 2 ++
libavfilter/avfilter_internal.h | 2 ++
libavfilter/video.c | 14 --
3 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/libavfilter/avfilter.c b/libavfilter/avfilter.c
index 049e4f62ca..44da809c15 100644
--- a/libavfilter/avfilter.c
+++ b/libavfilter/avfilter.c
@@ -176,6 +176,7 @@ int avfilter_link(AVFilterContext *src, unsigned srcpad,
li = av_mallocz(sizeof(*li));
if (!li)
return AVERROR(ENOMEM);
+ff_mutex_init(>frame_pool_mutex, NULL);
link = >l;
src->outputs[srcpad] = dst->inputs[dstpad] = link;
@@ -203,6 +204,7 @@ static void link_free(AVFilterLink **link)
ff_framequeue_free(>fifo);
ff_frame_pool_uninit(>frame_pool);
+ff_mutex_destroy(>frame_pool_mutex);
av_channel_layout_uninit(&(*link)->ch_layout);
av_freep(link);
diff --git a/libavfilter/avfilter_internal.h b/libavfilter/avfilter_internal.h
index 2c31c3e7de..eae3a699c3 100644
--- a/libavfilter/avfilter_internal.h
+++ b/libavfilter/avfilter_internal.h
@@ -27,6 +27,7 @@
#include
+#include "libavutil/thread.h"
#include "avfilter.h"
#include "framequeue.h"
@@ -34,6 +35,7 @@ typedef struct FilterLinkInternal {
AVFilterLink l;
struct FFFramePool *frame_pool;
+AVMutex frame_pool_mutex;
/**
* Queue of frames waiting to be filtered.
diff --git a/libavfilter/video.c b/libavfilter/video.c
index 89d0797ab5..30eba76cbd 100644
--- a/libavfilter/video.c
+++ b/libavfilter/video.c
@@ -70,17 +70,17 @@ AVFrame *ff_default_get_video_buffer2(AVFilterLink *link,
int w, int h, int alig
return frame;
}
+ff_mutex_lock(>frame_pool_mutex);
if (!li->frame_pool) {
li->frame_pool = ff_frame_pool_video_init(av_buffer_allocz, w, h,
link->format, align);
if (!li->frame_pool)
-return NULL;
+goto out;
} else {
if (ff_frame_pool_get_video_config(li->frame_pool,
_width, _height,
- _format, _align) < 0) {
-return NULL;
-}
+ _format, _align) < 0)
+goto out;
if (pool_width != w || pool_height != h ||
pool_format != link->format || pool_align != align) {
@@ -89,18 +89,20 @@ AVFrame *ff_default_get_video_buffer2(AVFilterLink *link,
int w, int h, int alig
li->frame_pool = ff_frame_pool_video_init(av_buffer_allocz, w, h,
link->format, align);
if (!li->frame_pool)
-return NULL;
+goto out;
}
}
frame = ff_frame_pool_get(li->frame_pool);
if (!frame)
-return NULL;
+goto out;
frame->sample_aspect_ratio = link->sample_aspect_ratio;
frame->colorspace = link->colorspace;
frame->color_range = link->color_range;
+out:
+ff_mutex_unlock(>frame_pool_mutex);
return frame;
}
--
2.42.0
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH] avfilter/src_movie: Remove align dimension to fix crash
From: Zhao Zhili
The alignment is handled by ff_default_get_video_buffer2. We
shouldn't use the aligned width/height as FFFramePool width/height.
It cause recreate FFFramePool inside ff_default_get_video_buffer2.
The recreate of FFFramePool together with multi-threads decoding
leading to data race and crash, for example,
./ffplay -f lavfi -i movie=foo.mp4,drawtext=text=bar
---
libavfilter/src_movie.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/libavfilter/src_movie.c b/libavfilter/src_movie.c
index e2cdcf17db..5099012100 100644
--- a/libavfilter/src_movie.c
+++ b/libavfilter/src_movie.c
@@ -187,7 +187,6 @@ static int get_buffer(AVCodecContext *avctx, AVFrame
*frame, int flags)
switch (avctx->codec_type) {
case AVMEDIA_TYPE_VIDEO:
-avcodec_align_dimensions2(avctx, , , linesize_align);
new = ff_default_get_video_buffer(outlink, w, h);
break;
case AVMEDIA_TYPE_AUDIO:
--
2.42.0
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH v3 01/10] avfilter/dnn: Refactor DNN parameter configuration system
On 2024/5/6 21:42, Zhao Zhili wrote: On May 6, 2024, at 19:48, Guo, Yejun wrote: -Original Message- From: ffmpeg-devel On Behalf Of Zhao Zhili Sent: Tuesday, April 30, 2024 3:12 PM To: ffmpeg-devel@ffmpeg.org Cc: Zhao Zhili Subject: [FFmpeg-devel] [PATCH v3 01/10] avfilter/dnn: Refactor DNN parameter configuration system From: Zhao Zhili This patch trying to resolve mulitiple issues related to parameter configuration: Firstly, each DNN filters duplicate DNN_COMMON_OPTIONS, which should be the common options of backend. Secondly, backend options are hidden behind the scene. It's a AV_OPT_TYPE_STRING backend_configs for user, and parsed by each backend. We don't know each backend support what kind of options from the help message. Third, DNN backends duplicate DNN_BACKEND_COMMON_OPTIONS. Last but not the least, pass backend options via AV_OPT_TYPE_STRING makes it hard to pass AV_OPT_TYPE_BINARY to backend, if not impossible. This patch puts backend common options and each backend options inside DnnContext to reduce code duplication, make options user friendly, and easy to extend for future usecase. There is a known issue that, for a filter which only support one or two of the backends, the help message still show the option of all three backends. Each DNN filter should be able to run on any backend. Current issue is mostly due to incomplete implementation (e.g., libtorch only support DFT_PROCESS_FRAME), and lack of maintenance on the filters. This patch 01 basically looks good, two comments: - it is possible that we add one dnn filter with one backend support first, and then other backends one by one some-time later. So, please adjust the help message accordingly with only the supported backends. I can’t figure out how to do that properly without duplicating some code for each filter. Do you have any idea? Fixed in patch v4. - is it possible to split this patch into small patches for an easier detail review? The configuration methods were scattered across filter common code, filters, backend common code and backend implementation. I have written six version, patch 1/10 is hard to split to small patches like patch 2-10. For example, ./ffmpeg -h filter=dnn_processing dnn_processing AVOptions: dnn_backend ..FV... DNN backend (from INT_MIN to INT_MAX) (default tensorflow) tensorflow 1 ..FV... tensorflow backend flag openvino 2 ..FV... openvino backend flag torch 3 ..FV... torch backend flag dnn_base AVOptions: model ..F path to model file input ..F input name of the model output ..F output name of the model backend_configs ..F...P backend configs (deprecated) options ..F...P backend configs (deprecated) nireq ..F number of request (from 0 to INT_MAX) (default 0) async ..F use DNN async inference (default true) device ..F device to run model dnn_tensorflow AVOptions: sess_config ..F config for SessionOptions dnn_openvino AVOptions: batch_size ..F batch size per request (from 1 to 1000) (default 1) input_resizable ..F can input be resizable or not (default false) layout ..F input layout of model (from 0 to 2) (default none) none 0 ..F none nchw 1 ..F nchw nhwc 2 ..F nhwc scale ..F Add scale preprocess operation. Divide each element of input by specified value. (from INT_MIN to INT_MAX) (default 0) mean ..F Add mean preprocess operation. Subtract specified value from each element of input. (from INT_MIN to INT_MAX) (default 0) dnn_th AVOptions: optimize ..F turn on graph executor optimization (from 0 to 1) (default 0) --- libavfilter/dnn/dnn_backend_common.h | 13 ++- libavfilter/dnn/dnn_backend_openvino.c | 146 ++--- libavfilter/dnn/dnn_backend_tf.c | 82 +- libavfilter/dnn/dnn_backend_torch.cpp | 67 libavfilter/dnn/dnn_interface.c | 89 +++ libavfilter/dnn_filter_common.c | 38 ++- libavfilter/dnn_filter_common.h | 39 +++ libavfilter/dnn_interface.h | 67 +++- libavfilter/vf_derain.c | 6 +- libavfilter/vf_dnn_classify.c | 4 +- libavfilter/vf_dnn_detect.c | 4 +- libavfilter/vf_dnn_processing.c | 4 +- libavfilter/vf_sr.c | 6 +- 13 files changed, 336 insertions(+), 229 deletions(-) ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https
[FFmpeg-devel] [PATCH v4 10/11] avfilter/dnn: Remove a level of dereference
From: Zhao Zhili
For code such as 'model->model = ov_model' is confusing. We can
just drop the member variable and use cast to get the subclass.
Signed-off-by: Zhao Zhili
---
libavfilter/dnn/dnn_backend_openvino.c | 17 -
libavfilter/dnn/dnn_backend_tf.c | 19 +--
libavfilter/dnn/dnn_backend_torch.cpp | 15 +++
libavfilter/dnn_filter_common.c| 6 +++---
libavfilter/dnn_interface.h| 6 ++
5 files changed, 29 insertions(+), 34 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c
b/libavfilter/dnn/dnn_backend_openvino.c
index 1a5ed6383a..9c699cdc8c 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -517,7 +517,7 @@ static void dnn_free_model_ov(DNNModel **model)
if (!model || !*model)
return;
-ov_model = (*model)->model;
+ov_model = (OVModel *)(*model);
while (ff_safe_queue_size(ov_model->request_queue) != 0) {
OVRequestItem *item = ff_safe_queue_pop_front(ov_model->request_queue);
if (item && item->infer_request) {
@@ -1059,9 +1059,9 @@ err:
return ret;
}
-static int get_input_ov(void *model, DNNData *input, const char *input_name)
+static int get_input_ov(DNNModel *model, DNNData *input, const char
*input_name)
{
-OVModel *ov_model = model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
int input_resizable = ctx->ov_option.input_resizable;
@@ -1255,7 +1255,7 @@ static int extract_lltask_from_task(DNNFunctionType
func_type, TaskItem *task, Q
}
}
-static int get_output_ov(void *model, const char *input_name, int input_width,
int input_height,
+static int get_output_ov(DNNModel *model, const char *input_name, int
input_width, int input_height,
const char *output_name, int *output_width,
int *output_height)
{
#if HAVE_OPENVINO2
@@ -1268,7 +1268,7 @@ static int get_output_ov(void *model, const char
*input_name, int input_width, i
input_shapes_t input_shapes;
#endif
int ret;
-OVModel *ov_model = model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
TaskItem task;
OVRequestItem *request;
@@ -1383,7 +1383,6 @@ static DNNModel *dnn_load_model_ov(DnnContext *ctx,
DNNFunctionType func_type, A
return NULL;
ov_model->ctx = ctx;
model = _model->model;
-model->model = ov_model;
#if HAVE_OPENVINO2
status = ov_core_create();
@@ -1470,7 +1469,7 @@ err:
static int dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams
*exec_params)
{
-OVModel *ov_model = model->model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
OVRequestItem *request;
TaskItem *task;
@@ -1558,13 +1557,13 @@ static int dnn_execute_model_ov(const DNNModel *model,
DNNExecBaseParams *exec_p
static DNNAsyncStatusType dnn_get_result_ov(const DNNModel *model, AVFrame
**in, AVFrame **out)
{
-OVModel *ov_model = model->model;
+OVModel *ov_model = (OVModel *)model;
return ff_dnn_get_result_common(ov_model->task_queue, in, out);
}
static int dnn_flush_ov(const DNNModel *model)
{
-OVModel *ov_model = model->model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
OVRequestItem *request;
#if HAVE_OPENVINO2
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index 583ebd7c13..6afefe8115 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -262,9 +262,9 @@ static TF_Tensor *allocate_input_tensor(const DNNData
*input)
input_dims[1] * input_dims[2] * input_dims[3] *
size);
}
-static int get_input_tf(void *model, DNNData *input, const char *input_name)
+static int get_input_tf(DNNModel *model, DNNData *input, const char
*input_name)
{
-TFModel *tf_model = model;
+TFModel *tf_model = (TFModel *)model;
DnnContext *ctx = tf_model->ctx;
TF_Status *status;
TF_DataType dt;
@@ -310,11 +310,11 @@ static int get_input_tf(void *model, DNNData *input,
const char *input_name)
return 0;
}
-static int get_output_tf(void *model, const char *input_name, int input_width,
int input_height,
+static int get_output_tf(DNNModel *model, const char *input_name, int
input_width, int input_height,
const char *output_name, int *output_width,
int *output_height)
{
int ret;
-TFModel *tf_model = model;
+TFModel *tf_model = (TFModel *)model;
DnnContext *ctx = tf_model->ctx;
TaskItem task;
TFRequestItem *request;
@@ -486,7 +486,7 @@ static void dnn_free_model_tf(DNNModel **model)
if (!model || !*model)
return;
-tf_model = (*model)->model;
+tf_model = (TFModel *)(*model);
while (ff_safe_queu
[FFmpeg-devel] [PATCH v4 11/11] avfilter/vf_dnn_detect: Fix null pointer dereference
From: Zhao Zhili
Signed-off-by: Zhao Zhili
---
libavfilter/vf_dnn_detect.c | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/libavfilter/vf_dnn_detect.c b/libavfilter/vf_dnn_detect.c
index b4eee06fe7..2a277d4169 100644
--- a/libavfilter/vf_dnn_detect.c
+++ b/libavfilter/vf_dnn_detect.c
@@ -807,11 +807,13 @@ static av_cold void dnn_detect_uninit(AVFilterContext
*context)
DnnDetectContext *ctx = context->priv;
AVDetectionBBox *bbox;
ff_dnn_uninit(>dnnctx);
-while(av_fifo_can_read(ctx->bboxes_fifo)) {
-av_fifo_read(ctx->bboxes_fifo, , 1);
-av_freep();
+if (ctx->bboxes_fifo) {
+while (av_fifo_can_read(ctx->bboxes_fifo)) {
+av_fifo_read(ctx->bboxes_fifo, , 1);
+av_freep();
+}
+av_fifo_freep2(>bboxes_fifo);
}
-av_fifo_freep2(>bboxes_fifo);
av_freep(>anchors);
free_detect_labels(ctx);
}
--
2.34.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v4 05/11] avfilter/dnn_backend_openvino: simplify memory allocation
From: Zhao Zhili
Signed-off-by: Zhao Zhili
---
libavfilter/dnn/dnn_backend_openvino.c | 47 +++---
1 file changed, 20 insertions(+), 27 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c
b/libavfilter/dnn/dnn_backend_openvino.c
index ba79eba73f..1a5ed6383a 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -41,8 +41,8 @@
#include "dnn_backend_common.h"
typedef struct OVModel{
+DNNModel model;
DnnContext *ctx;
-DNNModel *model;
#if HAVE_OPENVINO2
ov_core_t *core;
ov_model_t *ov_model;
@@ -300,11 +300,11 @@ static int fill_model_input_ov(OVModel *ov_model,
OVRequestItem *request)
return ov2_map_error(status, NULL);
}
#endif
-switch (ov_model->model->func_type) {
+switch (ov_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
-if (ov_model->model->frame_pre_proc != NULL) {
-ov_model->model->frame_pre_proc(task->in_frame, ,
ov_model->model->filter_ctx);
+if (ov_model->model.frame_pre_proc != NULL) {
+ov_model->model.frame_pre_proc(task->in_frame, ,
ov_model->model.filter_ctx);
} else {
ff_proc_from_frame_to_dnn(task->in_frame, , ctx);
}
@@ -442,11 +442,11 @@ static void infer_completion_callback(void *args)
for (int i = 0; i < request->lltask_count; ++i) {
task = request->lltasks[i]->task;
-switch (ov_model->model->func_type) {
+switch (ov_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
-if (ov_model->model->frame_post_proc != NULL) {
-ov_model->model->frame_post_proc(task->out_frame, outputs,
ov_model->model->filter_ctx);
+if (ov_model->model.frame_post_proc != NULL) {
+ov_model->model.frame_post_proc(task->out_frame, outputs,
ov_model->model.filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);
}
@@ -458,23 +458,23 @@ static void infer_completion_callback(void *args)
}
break;
case DFT_ANALYTICS_DETECT:
-if (!ov_model->model->detect_post_proc) {
+if (!ov_model->model.detect_post_proc) {
av_log(ctx, AV_LOG_ERROR, "detect filter needs to provide post
proc\n");
goto end;
}
-ov_model->model->detect_post_proc(task->in_frame, outputs,
+ov_model->model.detect_post_proc(task->in_frame, outputs,
ov_model->nb_outputs,
- ov_model->model->filter_ctx);
+ ov_model->model.filter_ctx);
break;
case DFT_ANALYTICS_CLASSIFY:
-if (!ov_model->model->classify_post_proc) {
+if (!ov_model->model.classify_post_proc) {
av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide
post proc\n");
goto end;
}
for (int output_i = 0; output_i < ov_model->nb_outputs; output_i++)
-ov_model->model->classify_post_proc(task->in_frame, outputs,
+ov_model->model.classify_post_proc(task->in_frame, outputs,
request->lltasks[i]->bbox_index,
-
ov_model->model->filter_ctx);
+
ov_model->model.filter_ctx);
break;
default:
av_assert0(!"should not reach here");
@@ -571,7 +571,7 @@ static void dnn_free_model_ov(DNNModel **model)
av_free(ov_model->all_input_names);
#endif
av_freep(_model);
-av_freep(model);
+*model = NULL;
}
@@ -598,7 +598,7 @@ static int init_model_ov(OVModel *ov_model, const char
*input_name, const char *
#endif
// We scale pixel by default when do frame processing.
if (fabsf(ctx->ov_option.scale) < 1e-6f)
-ctx->ov_option.scale = ov_model->model->func_type == DFT_PROCESS_FRAME
? 255 : 1;
+ctx->ov_option.scale = ov_model->model.func_type == DFT_PROCESS_FRAME
? 255 : 1;
// batch size
if (ctx->ov_option.batch_size <= 0) {
ctx->ov_option.batch_size = 1;
@@ -702,7 +702,7 @@ static int init_model_ov(OVModel *ov_model, const char
*input_name, const char *
ret = ov2_map_error(status, NULL);
goto err;
}
-if (ov_model->model->func_t
[FFmpeg-devel] [PATCH v4 06/11] avfilter/dnn_backend_tf: Remove one level of indentation
From: Zhao Zhili
Signed-off-by: Zhao Zhili
---
libavfilter/dnn/dnn_backend_tf.c | 63
1 file changed, 32 insertions(+), 31 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index 0ab3bd13ed..eb4a2a5289 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -483,41 +483,42 @@ static void dnn_free_model_tf(DNNModel **model)
{
TFModel *tf_model;
-if (*model){
-tf_model = (*model)->model;
-while (ff_safe_queue_size(tf_model->request_queue) != 0) {
-TFRequestItem *item =
ff_safe_queue_pop_front(tf_model->request_queue);
-destroy_request_item();
-}
-ff_safe_queue_destroy(tf_model->request_queue);
+if (!model || !*model)
+return;
-while (ff_queue_size(tf_model->lltask_queue) != 0) {
-LastLevelTaskItem *item =
ff_queue_pop_front(tf_model->lltask_queue);
-av_freep();
-}
-ff_queue_destroy(tf_model->lltask_queue);
+tf_model = (*model)->model;
+while (ff_safe_queue_size(tf_model->request_queue) != 0) {
+TFRequestItem *item = ff_safe_queue_pop_front(tf_model->request_queue);
+destroy_request_item();
+}
+ff_safe_queue_destroy(tf_model->request_queue);
-while (ff_queue_size(tf_model->task_queue) != 0) {
-TaskItem *item = ff_queue_pop_front(tf_model->task_queue);
-av_frame_free(>in_frame);
-av_frame_free(>out_frame);
-av_freep();
-}
-ff_queue_destroy(tf_model->task_queue);
+while (ff_queue_size(tf_model->lltask_queue) != 0) {
+LastLevelTaskItem *item = ff_queue_pop_front(tf_model->lltask_queue);
+av_freep();
+}
+ff_queue_destroy(tf_model->lltask_queue);
-if (tf_model->graph){
-TF_DeleteGraph(tf_model->graph);
-}
-if (tf_model->session){
-TF_CloseSession(tf_model->session, tf_model->status);
-TF_DeleteSession(tf_model->session, tf_model->status);
-}
-if (tf_model->status){
-TF_DeleteStatus(tf_model->status);
-}
-av_freep(_model);
-av_freep();
+while (ff_queue_size(tf_model->task_queue) != 0) {
+TaskItem *item = ff_queue_pop_front(tf_model->task_queue);
+av_frame_free(>in_frame);
+av_frame_free(>out_frame);
+av_freep();
+}
+ff_queue_destroy(tf_model->task_queue);
+
+if (tf_model->graph){
+TF_DeleteGraph(tf_model->graph);
+}
+if (tf_model->session){
+TF_CloseSession(tf_model->session, tf_model->status);
+TF_DeleteSession(tf_model->session, tf_model->status);
+}
+if (tf_model->status){
+TF_DeleteStatus(tf_model->status);
}
+av_freep(_model);
+av_freep();
}
static DNNModel *dnn_load_model_tf(DnnContext *ctx, DNNFunctionType func_type,
AVFilterContext *filter_ctx)
--
2.34.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v4 09/11] avfilter/dnn_backend_torch: Simplify memory allocation
From: Zhao Zhili
Signed-off-by: Zhao Zhili
---
libavfilter/dnn/dnn_backend_torch.cpp | 31 +++
1 file changed, 12 insertions(+), 19 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_torch.cpp
b/libavfilter/dnn/dnn_backend_torch.cpp
index b70e4db600..0aa0b95269 100644
--- a/libavfilter/dnn/dnn_backend_torch.cpp
+++ b/libavfilter/dnn/dnn_backend_torch.cpp
@@ -37,8 +37,8 @@ extern "C" {
}
typedef struct THModel {
+DNNModel model;
DnnContext *ctx;
-DNNModel *model;
torch::jit::Module *jit_model;
SafeQueue *request_queue;
Queue *task_queue;
@@ -141,7 +141,7 @@ static void dnn_free_model_th(DNNModel **model)
ff_queue_destroy(th_model->task_queue);
delete th_model->jit_model;
av_freep(_model);
-av_freep(model);
+*model = NULL;
}
static int get_input_th(void *model, DNNData *input, const char *input_name)
@@ -195,19 +195,19 @@ static int fill_model_input_th(THModel *th_model,
THRequestItem *request)
infer_request->input_tensor = new torch::Tensor();
infer_request->output = new torch::Tensor();
-switch (th_model->model->func_type) {
+switch (th_model->model.func_type) {
case DFT_PROCESS_FRAME:
input.scale = 255;
if (task->do_ioproc) {
-if (th_model->model->frame_pre_proc != NULL) {
-th_model->model->frame_pre_proc(task->in_frame, ,
th_model->model->filter_ctx);
+if (th_model->model.frame_pre_proc != NULL) {
+th_model->model.frame_pre_proc(task->in_frame, ,
th_model->model.filter_ctx);
} else {
ff_proc_from_frame_to_dnn(task->in_frame, , ctx);
}
}
break;
default:
-avpriv_report_missing_feature(NULL, "model function type %d",
th_model->model->func_type);
+avpriv_report_missing_feature(NULL, "model function type %d",
th_model->model.func_type);
break;
}
*infer_request->input_tensor = torch::from_blob(input.data,
@@ -282,13 +282,13 @@ static void infer_completion_callback(void *args) {
goto err;
}
-switch (th_model->model->func_type) {
+switch (th_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
outputs.scale = 255;
outputs.data = output->data_ptr();
-if (th_model->model->frame_post_proc != NULL) {
-th_model->model->frame_post_proc(task->out_frame, ,
th_model->model->filter_ctx);
+if (th_model->model.frame_post_proc != NULL) {
+th_model->model.frame_post_proc(task->out_frame, ,
th_model->model.filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, ,
th_model->ctx);
}
@@ -298,7 +298,7 @@ static void infer_completion_callback(void *args) {
}
break;
default:
-avpriv_report_missing_feature(th_model->ctx, "model function type %d",
th_model->model->func_type);
+avpriv_report_missing_feature(th_model->ctx, "model function type %d",
th_model->model.func_type);
goto err;
}
task->inference_done++;
@@ -417,17 +417,10 @@ static DNNModel *dnn_load_model_th(DnnContext *ctx,
DNNFunctionType func_type, A
THRequestItem *item = NULL;
const char *device_name = ctx->device ? ctx->device : "cpu";
-model = (DNNModel *)av_mallocz(sizeof(DNNModel));
-if (!model) {
-return NULL;
-}
-
th_model = (THModel *)av_mallocz(sizeof(THModel));
-if (!th_model) {
-av_freep();
+if (!th_model)
return NULL;
-}
-th_model->model = model;
+model = _model->model;
model->model = th_model;
th_model->ctx = ctx;
--
2.34.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v4 08/11] avfilter/dnn_backend_tf: Simplify memory allocation
From: Zhao Zhili
Signed-off-by: Zhao Zhili
---
libavfilter/dnn/dnn_backend_tf.c | 33 +---
1 file changed, 13 insertions(+), 20 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index 8b53a3b87a..583ebd7c13 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -37,8 +37,8 @@
#include
typedef struct TFModel {
+DNNModel model;
DnnContext *ctx;
-DNNModel *model;
TF_Graph *graph;
TF_Session *session;
TF_Status *status;
@@ -518,7 +518,7 @@ static void dnn_free_model_tf(DNNModel **model)
TF_DeleteStatus(tf_model->status);
}
av_freep(_model);
-av_freep();
+*model = NULL;
}
static DNNModel *dnn_load_model_tf(DnnContext *ctx, DNNFunctionType func_type,
AVFilterContext *filter_ctx)
@@ -526,18 +526,11 @@ static DNNModel *dnn_load_model_tf(DnnContext *ctx,
DNNFunctionType func_type, A
DNNModel *model = NULL;
TFModel *tf_model = NULL;
-model = av_mallocz(sizeof(DNNModel));
-if (!model){
-return NULL;
-}
-
tf_model = av_mallocz(sizeof(TFModel));
-if (!tf_model){
-av_freep();
+if (!tf_model)
return NULL;
-}
+model = _model->model;
model->model = tf_model;
-tf_model->model = model;
tf_model->ctx = ctx;
if (load_tf_model(tf_model, ctx->model_filename) != 0){
@@ -650,11 +643,11 @@ static int fill_model_input_tf(TFModel *tf_model,
TFRequestItem *request) {
}
input.data = (float *)TF_TensorData(infer_request->input_tensor);
-switch (tf_model->model->func_type) {
+switch (tf_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
-if (tf_model->model->frame_pre_proc != NULL) {
-tf_model->model->frame_pre_proc(task->in_frame, ,
tf_model->model->filter_ctx);
+if (tf_model->model.frame_pre_proc != NULL) {
+tf_model->model.frame_pre_proc(task->in_frame, ,
tf_model->model.filter_ctx);
} else {
ff_proc_from_frame_to_dnn(task->in_frame, , ctx);
}
@@ -664,7 +657,7 @@ static int fill_model_input_tf(TFModel *tf_model,
TFRequestItem *request) {
ff_frame_to_dnn_detect(task->in_frame, , ctx);
break;
default:
-avpriv_report_missing_feature(ctx, "model function type %d",
tf_model->model->func_type);
+avpriv_report_missing_feature(ctx, "model function type %d",
tf_model->model.func_type);
break;
}
@@ -724,12 +717,12 @@ static void infer_completion_callback(void *args) {
outputs[i].data = TF_TensorData(infer_request->output_tensors[i]);
outputs[i].dt =
(DNNDataType)TF_TensorType(infer_request->output_tensors[i]);
}
-switch (tf_model->model->func_type) {
+switch (tf_model->model.func_type) {
case DFT_PROCESS_FRAME:
//it only support 1 output if it's frame in & frame out
if (task->do_ioproc) {
-if (tf_model->model->frame_post_proc != NULL) {
-tf_model->model->frame_post_proc(task->out_frame, outputs,
tf_model->model->filter_ctx);
+if (tf_model->model.frame_post_proc != NULL) {
+tf_model->model.frame_post_proc(task->out_frame, outputs,
tf_model->model.filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);
}
@@ -741,11 +734,11 @@ static void infer_completion_callback(void *args) {
}
break;
case DFT_ANALYTICS_DETECT:
-if (!tf_model->model->detect_post_proc) {
+if (!tf_model->model.detect_post_proc) {
av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post
proc\n");
return;
}
-tf_model->model->detect_post_proc(task->in_frame, outputs,
task->nb_output, tf_model->model->filter_ctx);
+tf_model->model.detect_post_proc(task->in_frame, outputs,
task->nb_output, tf_model->model.filter_ctx);
break;
default:
av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this
kind of dnn filter now\n");
--
2.34.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v4 07/11] avfilter/dnn_backend_tf: Fix free context at random place
From: Zhao Zhili
It will be freed again by ff_dnn_uninit.
Signed-off-by: Zhao Zhili
---
libavfilter/dnn/dnn_backend_tf.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index eb4a2a5289..8b53a3b87a 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -804,7 +804,7 @@ err:
if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
destroy_request_item();
}
-dnn_free_model_tf(_model->model);
+
return ret;
}
--
2.34.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v4 01/11] avfilter/dnn: Refactor DNN parameter configuration system
From: Zhao Zhili
This patch trying to resolve mulitiple issues related to parameter
configuration:
Firstly, each DNN filters duplicate DNN_COMMON_OPTIONS, which should
be the common options of backend.
Secondly, backend options are hidden behind the scene. It's a
AV_OPT_TYPE_STRING backend_configs for user, and parsed by each
backend. We don't know each backend support what kind of options
from the help message.
Third, DNN backends duplicate DNN_BACKEND_COMMON_OPTIONS.
Last but not the least, pass backend options via AV_OPT_TYPE_STRING
makes it hard to pass AV_OPT_TYPE_BINARY to backend, if not impossible.
This patch puts backend common options and each backend options inside
DnnContext to reduce code duplication, make options user friendly, and
easy to extend for future usecase.
For example,
./ffmpeg -h filter=dnn_processing
dnn_processing AVOptions:
dnn_backend ..FV... DNN backend (from INT_MIN to
INT_MAX) (default tensorflow)
tensorflow 1..FV... tensorflow backend flag
openvino2..FV... openvino backend flag
torch 3..FV... torch backend flag
dnn_base AVOptions:
model ..F path to model file
input ..F input name of the model
output ..F output name of the model
backend_configs..F...P backend configs (deprecated)
options..F...P backend configs (deprecated)
nireq ..F number of request (from 0 to
INT_MAX) (default 0)
async ..F use DNN async inference (default
true)
device ..F device to run model
dnn_tensorflow AVOptions:
sess_config..F config for SessionOptions
dnn_openvino AVOptions:
batch_size..F batch size per request (from 1 to
1000) (default 1)
input_resizable ..F can input be resizable or not
(default false)
layout..F input layout of model (from 0 to
2) (default none)
none0..F none
nchw1..F nchw
nhwc2..F nhwc
scale ..F Add scale preprocess operation.
Divide each element of input by specified value. (from INT_MIN to INT_MAX)
(default 0)
mean..F Add mean preprocess operation.
Subtract specified value from each element of input. (from INT_MIN to INT_MAX)
(default 0)
dnn_th AVOptions:
optimize ..F turn on graph executor
optimization (from 0 to 1) (default 0)
Signed-off-by: Zhao Zhili
---
libavfilter/dnn/dnn_backend_common.h | 13 ++-
libavfilter/dnn/dnn_backend_openvino.c | 146 ++---
libavfilter/dnn/dnn_backend_tf.c | 82 +-
libavfilter/dnn/dnn_backend_torch.cpp | 67
libavfilter/dnn/dnn_interface.c| 89 +++
libavfilter/dnn_filter_common.c| 38 ++-
libavfilter/dnn_filter_common.h| 39 +++
libavfilter/dnn_interface.h| 67 +++-
libavfilter/vf_derain.c| 6 +-
libavfilter/vf_dnn_classify.c | 4 +-
libavfilter/vf_dnn_detect.c| 4 +-
libavfilter/vf_dnn_processing.c| 4 +-
libavfilter/vf_sr.c| 6 +-
13 files changed, 336 insertions(+), 229 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_common.h
b/libavfilter/dnn/dnn_backend_common.h
index 42c67c7040..9f5d37b3e0 100644
--- a/libavfilter/dnn/dnn_backend_common.h
+++ b/libavfilter/dnn/dnn_backend_common.h
@@ -28,9 +28,16 @@
#include "../dnn_interface.h"
#include "libavutil/thread.h"
-#define DNN_BACKEND_COMMON_OPTIONS \
-{ "nireq", "number of request",
OFFSET(options.nireq), AV_OPT_TYPE_INT,{ .i64 = 0 }, 0,
INT_MAX, FLAGS }, \
-{ "async", "use DNN async inference",
OFFSET(options.async), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0,
1, FLAGS },
+#define DNN_DEFINE_CLASS_EXT(name, desc, options) \
+{ \
+.class_name = desc, \
+.item_name = av_default_item_name, \
+.option = options, \
+.version= LIBAVUTIL_VERSION_INT,\
+.category = AV_CLASS_CATEGORY_FILTER, \
+}
+#define DNN_DEFINE_CLASS(fname) \
+DNN_DEFINE_CLASS_EXT(fname, #fname, fname##_options)
// one task for one function call from dnn interface
typedef struct TaskItem {
diff --git a/libavfilter/dnn/dnn_backend_openvino.c
b/libavfilter/dnn/dnn_backend_openvino.c
index 374f21b7a1..c4b0682f11 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.
[FFmpeg-devel] [PATCH v4 04/11] avfilter/dnn_backend_openvino: Fix free context at random place
From: Zhao Zhili It will be freed again by ff_dnn_uninit. Signed-off-by: Zhao Zhili --- libavfilter/dnn/dnn_backend_openvino.c | 1 - 1 file changed, 1 deletion(-) diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c index 5ea8fb8d1f..ba79eba73f 100644 --- a/libavfilter/dnn/dnn_backend_openvino.c +++ b/libavfilter/dnn/dnn_backend_openvino.c @@ -959,7 +959,6 @@ err: if (input_model_info) ov_preprocess_input_model_info_free(input_model_info); #endif -dnn_free_model_ov(_model->model); return ret; } -- 2.34.1 ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v4 03/11] avfilter/dnn: Don't show backends which are not supported by a filter
From: Zhao Zhili
---
libavfilter/dnn/dnn_interface.c | 11 ---
libavfilter/dnn_filter_common.h | 11 +--
libavfilter/dnn_interface.h | 8 ++--
libavfilter/vf_derain.c | 2 +-
libavfilter/vf_dnn_classify.c | 2 +-
libavfilter/vf_dnn_detect.c | 2 +-
libavfilter/vf_dnn_processing.c | 2 +-
libavfilter/vf_sr.c | 2 +-
8 files changed, 28 insertions(+), 12 deletions(-)
diff --git a/libavfilter/dnn/dnn_interface.c b/libavfilter/dnn/dnn_interface.c
index b56c22e4c7..dc1593821d 100644
--- a/libavfilter/dnn/dnn_interface.c
+++ b/libavfilter/dnn/dnn_interface.c
@@ -120,11 +120,16 @@ void *ff_dnn_child_next(DnnContext *obj, void *prev) {
return NULL;
}
-const AVClass *ff_dnn_child_class_iterate(void **iter)
+const AVClass *ff_dnn_child_class_iterate_with_mask(void **iter, uint32_t
backend_mask)
{
-uintptr_t i = (uintptr_t) *iter;
+for (uintptr_t i = (uintptr_t)*iter; i <
FF_ARRAY_ELEMS(dnn_backend_info_list); i++) {
+if (i > 0) {
+const DNNModule *module = dnn_backend_info_list[i].module;
+
+if (!(module->type & backend_mask))
+continue;
+}
-if (i < FF_ARRAY_ELEMS(dnn_backend_info_list)) {
*iter = (void *)(i + 1);
return dnn_backend_info_list[i].class;
}
diff --git a/libavfilter/dnn_filter_common.h b/libavfilter/dnn_filter_common.h
index b52b55a90d..42a4719997 100644
--- a/libavfilter/dnn_filter_common.h
+++ b/libavfilter/dnn_filter_common.h
@@ -26,6 +26,12 @@
#include "dnn_interface.h"
+#define DNN_FILTER_CHILD_CLASS_ITERATE(name, backend_mask) \
+static const AVClass *name##_child_class_iterate(void **iter) \
+{ \
+return ff_dnn_child_class_iterate_with_mask(iter, (backend_mask)); \
+}
+
#define AVFILTER_DNN_DEFINE_CLASS_EXT(name, desc, options) \
static const AVClass name##_class = { \
.class_name = desc, \
@@ -34,10 +40,11 @@
.version= LIBAVUTIL_VERSION_INT,\
.category = AV_CLASS_CATEGORY_FILTER,\
.child_next = ff_dnn_filter_child_next,\
-.child_class_iterate = ff_dnn_child_class_iterate, \
+.child_class_iterate = name##_child_class_iterate, \
}
-#define AVFILTER_DNN_DEFINE_CLASS(fname) \
+#define AVFILTER_DNN_DEFINE_CLASS(fname, backend_mask) \
+DNN_FILTER_CHILD_CLASS_ITERATE(fname, backend_mask) \
AVFILTER_DNN_DEFINE_CLASS_EXT(fname, #fname, fname##_options)
void *ff_dnn_filter_child_next(void *obj, void *prev);
diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h
index dd603534b2..697b9f3318 100644
--- a/libavfilter/dnn_interface.h
+++ b/libavfilter/dnn_interface.h
@@ -32,7 +32,11 @@
#define DNN_GENERIC_ERROR FFERRTAG('D','N','N','!')
-typedef enum {DNN_TF = 1, DNN_OV, DNN_TH} DNNBackendType;
+typedef enum {
+DNN_TF = 1,
+DNN_OV = 1 << 1,
+DNN_TH = 1 << 2
+} DNNBackendType;
typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
@@ -190,7 +194,7 @@ const DNNModule *ff_get_dnn_module(DNNBackendType
backend_type, void *log_ctx);
void ff_dnn_init_child_class(DnnContext *ctx);
void *ff_dnn_child_next(DnnContext *obj, void *prev);
-const AVClass *ff_dnn_child_class_iterate(void **iter);
+const AVClass *ff_dnn_child_class_iterate_with_mask(void **iter, uint32_t
backend_mask);
static inline int dnn_get_width_idx_by_layout(DNNLayout layout)
{
diff --git a/libavfilter/vf_derain.c b/libavfilter/vf_derain.c
index 7f665b73ab..5cefca6b55 100644
--- a/libavfilter/vf_derain.c
+++ b/libavfilter/vf_derain.c
@@ -49,7 +49,7 @@ static const AVOption derain_options[] = {
{ NULL }
};
-AVFILTER_DNN_DEFINE_CLASS(derain);
+AVFILTER_DNN_DEFINE_CLASS(derain, DNN_TF);
static int filter_frame(AVFilterLink *inlink, AVFrame *in)
{
diff --git a/libavfilter/vf_dnn_classify.c b/libavfilter/vf_dnn_classify.c
index 965779a8ab..f6d3678796 100644
--- a/libavfilter/vf_dnn_classify.c
+++ b/libavfilter/vf_dnn_classify.c
@@ -56,7 +56,7 @@ static const AVOption dnn_classify_options[] = {
{ NULL }
};
-AVFILTER_DNN_DEFINE_CLASS(dnn_classify);
+AVFILTER_DNN_DEFINE_CLASS(dnn_classify, DNN_OV);
static int dnn_classify_post_proc(AVFrame *frame, DNNData *output, uint32_t
bbox_index, AVFilterContext *filter_ctx)
{
diff --git a/libavfilter/vf_dnn_detect.c b/libavfilter/vf_dnn_detect.c
index 926966368a..b4eee06fe7 100644
--- a/libavfilter/vf_dnn_detect.c
+++ b/libavfilter/vf_dnn_detect.c
@@ -84,7 +84,7 @@ static const AVOption dnn_detect_options[] = {
{ NULL }
};
-AVFILTER_DNN_DEFINE_CLASS(dnn_detect);
+AVFILTER_DNN_DEFINE_CLASS(dnn_detect, DNN_TF | DNN_OV);
static inline float sigmoid(float x) {
return 1.f / (1.f + exp(-x));
diff --git a/libavfilter/vf_dnn_processing.c b/libavfilter/vf_dnn_processing
[FFmpeg-devel] [PATCH v4 00/11] Refactor DNN
From: Zhao Zhili v4: 1. Move [patch v3 9/10] to [patch v4 2/11], since patch 3 depends on it. 2. Don't show backends which are not supported by a filter (patch 3). v3: 1. Init class with ff_dnn_filter_init_child_class and called by AVFilter preinit. 2. Fix crash in vf_dnn_detect (patch 10/10). Zhao Zhili (11): avfilter/dnn: Refactor DNN parameter configuration system avfilter/dnn: Use dnn_backend_info_list to search for dnn module avfilter/dnn: Don't show backends which are not supported by a filter avfilter/dnn_backend_openvino: Fix free context at random place avfilter/dnn_backend_openvino: simplify memory allocation avfilter/dnn_backend_tf: Remove one level of indentation avfilter/dnn_backend_tf: Fix free context at random place avfilter/dnn_backend_tf: Simplify memory allocation avfilter/dnn_backend_torch: Simplify memory allocation avfilter/dnn: Remove a level of dereference avfilter/vf_dnn_detect: Fix null pointer dereference libavfilter/dnn/dnn_backend_common.h | 13 +- libavfilter/dnn/dnn_backend_openvino.c | 210 ++--- libavfilter/dnn/dnn_backend_tf.c | 194 ++- libavfilter/dnn/dnn_backend_torch.cpp | 112 + libavfilter/dnn/dnn_interface.c| 120 +++--- libavfilter/dnn_filter_common.c| 44 +- libavfilter/dnn_filter_common.h| 40 ++--- libavfilter/dnn_interface.h| 80 -- libavfilter/vf_derain.c| 6 +- libavfilter/vf_dnn_classify.c | 4 +- libavfilter/vf_dnn_detect.c| 14 +- libavfilter/vf_dnn_processing.c| 4 +- libavfilter/vf_sr.c| 6 +- 13 files changed, 470 insertions(+), 377 deletions(-) -- 2.34.1 ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v4 02/11] avfilter/dnn: Use dnn_backend_info_list to search for dnn module
From: Zhao Zhili
Signed-off-by: Zhao Zhili
---
libavfilter/dnn/dnn_backend_openvino.c | 1 +
libavfilter/dnn/dnn_backend_tf.c | 1 +
libavfilter/dnn/dnn_backend_torch.cpp | 1 +
libavfilter/dnn/dnn_interface.c| 26 --
libavfilter/dnn_interface.h| 1 +
5 files changed, 12 insertions(+), 18 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c
b/libavfilter/dnn/dnn_backend_openvino.c
index c4b0682f11..5ea8fb8d1f 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -1622,6 +1622,7 @@ static int dnn_flush_ov(const DNNModel *model)
const DNNModule ff_dnn_backend_openvino = {
.clazz = DNN_DEFINE_CLASS(dnn_openvino),
+.type = DNN_OV,
.load_model = dnn_load_model_ov,
.execute_model = dnn_execute_model_ov,
.get_result = dnn_get_result_ov,
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index d24591b90b..0ab3bd13ed 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -893,6 +893,7 @@ static int dnn_flush_tf(const DNNModel *model)
const DNNModule ff_dnn_backend_tf = {
.clazz = DNN_DEFINE_CLASS(dnn_tensorflow),
+.type = DNN_TF,
.load_model = dnn_load_model_tf,
.execute_model = dnn_execute_model_tf,
.get_result = dnn_get_result_tf,
diff --git a/libavfilter/dnn/dnn_backend_torch.cpp
b/libavfilter/dnn/dnn_backend_torch.cpp
index abdef1f178..b70e4db600 100644
--- a/libavfilter/dnn/dnn_backend_torch.cpp
+++ b/libavfilter/dnn/dnn_backend_torch.cpp
@@ -569,6 +569,7 @@ static int dnn_flush_th(const DNNModel *model)
extern const DNNModule ff_dnn_backend_torch = {
.clazz = DNN_DEFINE_CLASS(dnn_th),
+.type = DNN_TH,
.load_model = dnn_load_model_th,
.execute_model = dnn_execute_model_th,
.get_result = dnn_get_result_th,
diff --git a/libavfilter/dnn/dnn_interface.c b/libavfilter/dnn/dnn_interface.c
index 2d3bd229a4..b56c22e4c7 100644
--- a/libavfilter/dnn/dnn_interface.c
+++ b/libavfilter/dnn/dnn_interface.c
@@ -81,25 +81,15 @@ static const DnnBackendInfo dnn_backend_info_list[] = {
const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx)
{
-switch(backend_type){
-#if (CONFIG_LIBTENSORFLOW == 1)
-case DNN_TF:
-return _dnn_backend_tf;
-#endif
-#if (CONFIG_LIBOPENVINO == 1)
-case DNN_OV:
-return _dnn_backend_openvino;
-#endif
-#if (CONFIG_LIBTORCH == 1)
-case DNN_TH:
-return _dnn_backend_torch;
-#endif
-default:
-av_log(log_ctx, AV_LOG_ERROR,
-"Module backend_type %d is not supported or enabled.\n",
-backend_type);
-return NULL;
+for (int i = 1; i < FF_ARRAY_ELEMS(dnn_backend_info_list); i++) {
+if (dnn_backend_info_list[i].module->type == backend_type)
+return dnn_backend_info_list[i].module;
}
+
+av_log(log_ctx, AV_LOG_ERROR,
+"Module backend_type %d is not supported or enabled.\n",
+backend_type);
+return NULL;
}
void ff_dnn_init_child_class(DnnContext *ctx)
diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h
index 4e544486cc..dd603534b2 100644
--- a/libavfilter/dnn_interface.h
+++ b/libavfilter/dnn_interface.h
@@ -172,6 +172,7 @@ typedef struct DnnContext {
// Stores pointers to functions for loading, executing, freeing DNN models for
one of the backends.
struct DNNModule {
const AVClass clazz;
+DNNBackendType type;
// Loads model and parameters from given file. Returns NULL if it is not
possible.
DNNModel *(*load_model)(DnnContext *ctx, DNNFunctionType func_type,
AVFilterContext *filter_ctx);
// Executes model with specified input and output. Returns the error code
otherwise.
--
2.34.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH v3 01/10] avfilter/dnn: Refactor DNN parameter configuration system
> On May 6, 2024, at 19:48, Guo, Yejun > wrote: > >> -Original Message- >> From: ffmpeg-devel > <mailto:ffmpeg-devel-boun...@ffmpeg.org>> On Behalf Of Zhao >> Zhili >> Sent: Tuesday, April 30, 2024 3:12 PM >> To: ffmpeg-devel@ffmpeg.org <mailto:ffmpeg-devel@ffmpeg.org> >> Cc: Zhao Zhili mailto:zhiliz...@tencent.com>> >> Subject: [FFmpeg-devel] [PATCH v3 01/10] avfilter/dnn: Refactor DNN >> parameter configuration system >> >> From: Zhao Zhili >> >> This patch trying to resolve mulitiple issues related to parameter >> configuration: >> >> Firstly, each DNN filters duplicate DNN_COMMON_OPTIONS, which should >> be the common options of backend. >> >> Secondly, backend options are hidden behind the scene. It's a >> AV_OPT_TYPE_STRING backend_configs for user, and parsed by each >> backend. We don't know each backend support what kind of options >> from the help message. >> >> Third, DNN backends duplicate DNN_BACKEND_COMMON_OPTIONS. >> >> Last but not the least, pass backend options via AV_OPT_TYPE_STRING >> makes it hard to pass AV_OPT_TYPE_BINARY to backend, if not impossible. >> >> This patch puts backend common options and each backend options inside >> DnnContext to reduce code duplication, make options user friendly, and >> easy to extend for future usecase. >> >> There is a known issue that, for a filter which only support one or two >> of the backends, the help message still show the option of all three >> backends. Each DNN filter should be able to run on any backend. Current >> issue is mostly due to incomplete implementation (e.g., libtorch only >> support DFT_PROCESS_FRAME), and lack of maintenance on the filters. > > This patch 01 basically looks good, two comments: > - it is possible that we add one dnn filter with one backend support first, > and then > other backends one by one some-time later. So, please adjust the help message > accordingly with only the supported backends. I can’t figure out how to do that properly without duplicating some code for each filter. Do you have any idea? > > - is it possible to split this patch into small patches for an easier detail > review? The configuration methods were scattered across filter common code, filters, backend common code and backend implementation. I have written six version, patch 1/10 is hard to split to small patches like patch 2-10. > >> >> For example, >> >> ./ffmpeg -h filter=dnn_processing >> >> dnn_processing AVOptions: >> dnn_backend ..FV... DNN backend (from INT_MIN to >> INT_MAX) (default tensorflow) >> tensorflow 1..FV... tensorflow backend flag >> openvino2..FV... openvino backend flag >> torch 3..FV... torch backend flag >> >> dnn_base AVOptions: >> model ..F path to model file >> input ..F input name of the model >> output ..F output name of the model >> backend_configs..F...P backend configs (deprecated) >> options..F...P backend configs (deprecated) >> nireq ..F number of request (from 0 to >> INT_MAX) >> (default 0) >> async ..F use DNN async inference >> (default true) >> device ..F device to run model >> >> dnn_tensorflow AVOptions: >> sess_config..F config for SessionOptions >> >> dnn_openvino AVOptions: >> batch_size..F batch size per request (from 1 >> to 1000) >> (default 1) >> input_resizable ..F can input be resizable or not >> (default >> false) >> layout..F input layout of model (from 0 >> to 2) (default >> none) >> none0..F none >> nchw1..F nchw >> nhwc2..F nhwc >> scale ..F Add scale preprocess operation. >> Divide each >> element of input by specified value. (from INT_MIN to INT_MAX) (default 0) >> mean..F Add mean preprocess operation. >> Subtract >> specified value from each element of input. (from INT_MIN to INT_MAX) >> (default 0) >> >> dnn_th AVOptions: >> optimize ..F turn on graph executor >> optimizati
Re: [FFmpeg-devel] [RFC] 5 year plan & Inovation
> 在 2024年5月5日，上午5:51，epira...@gmail.com 写道： > > > >> On 4 May 2024, at 23:25, Andrew Sayers wrote: >> >>> On Sat, May 04, 2024 at 09:28:03PM +0200, Michael Niedermayer wrote: >>> Hi >>> On Fri, May 03, 2024 at 03:45:20PM +, Cosmin Stejerean via ffmpeg-devel wrote: >>> [...] What doesn't exist (yet) is a way to keep people on the exact email based workflow we currently have, and have bi-directional sync with something like github or gitlab. Such a thing could probably be built, but it might be worth first trying to see if those that insist on sticking with the CLI can use one of the existing CLI based workflows. >>> >>> Such a thing could be quite useful to many more projects than just ffmpeg. >>> There are many older projects that use ML based workflows. >>> >>> I imagine STF might be willing to fund such a thing if it is technically >>> feasable. As the goal of STF is about maintainance. And bridging the gap >>> between old ML and new browser based workflows allowing developers who >>> prefer to work through their web browser to do so. >>> >>> also, we need to find maintaince related projects worth minimum 150k € >>> for 2025 for STF. >>> We cant do many of the things we do in 2024 for STF again as they where >>> one time things and STF doesnt like sponsoring adding new features. >>> >>> thx >> >> It seems like the strongest argument for sticking with the ML is from >> experienced maintainers who don't want to jeopardise their existing >> workflow; while the strongest argument for switching is from people >> itching to try out new workflows. It’s not “try out new workflows”, but current workflow is inefficient and unbearable for some of us. >> So how about this for a plan... >> >> Make a repo on SourceHut, not necessarily for FFmpeg itself but for >> automated review tools (running fate tests, checking C11 compliance >> etc.). Their CI/CD system automatically runs those tests on every >> patch, then we manually forward genuine issues to the ML. That would >> let experimenters show off new things, and would let maintainers >> think through what their workflow would look like in a mixed >> environment. Then when we've got enough evidence to make a long-term >> plan, we can wind the repo down without too much fuss. > > I hardly see how SourceHut would improve much of any of the actual > struggles we talked about in this thread tbh… > > FWIW what most people are desiring is better review workflow/tooling > than a mail client can not offer (easily) and making it easier for > people to contribute by simply pushing a branch to their fork > which is for better or worse what a lot of people are familiar with > from GitHub. > > Both of which is nothing SourceHut offers, to my knowledge. > > So rather than spend efforts on something that only marginally improves > upon what is currently used it would IMHO be way more useful to evaluate > something like GitLab or Gitea/Forgejo. +1 > >> ___ >> ffmpeg-devel mailing list >> ffmpeg-devel@ffmpeg.org >> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel >> >> To unsubscribe, visit link above, or email >> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe". > ___ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe". ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [RFC] 5 year plan & Inovation
> -Original Message- > From: ffmpeg-devel On Behalf Of Ondřej Fiala > Sent: 2024年5月2日 22:25 > To: FFmpeg development discussions and patches > Subject: Re: [FFmpeg-devel] [RFC] 5 year plan & Inovation > > On Wed May 1, 2024 at 7:27 AM CEST, Rémi Denis-Courmont wrote: > > Le 30 avril 2024 22:15:10 GMT+03:00, "Ondřej Fiala" a > > écrit : > > >On Tue Apr 30, 2024 at 9:06 PM CEST, Hendrik Leppkes wrote: > > >> I will take the replacement instead, thanks. Email is archaic. The > > >> entire point is to get away from email, not dress it up. > > >> SourceHut usage would likely make me even less interested then today. > > >> > > >> - Hendrik > > >I guess that depends on how (and with what) you use it. Using it with > > >Gmail UI for example is obviously not a great idea. No idea whether you > > >do, but if you do, you should be upset at Gmail, not email. > > > > I don't use Gmail, and using email for review still sucks. No matter how you > > slice it, email was not meant for threaded code reviews. > Email was not meant for a lot of what it's used for today. Many email clients > have support for threading, and unlike GitHub allow threads of arbitrary > depth. Using such a client with commands for moving between messages in a > a thread etc. makes threaded code review over email quite usably in my > opinion. > > > Also while I can use git-send-email, not everyone can. And patches as > > attachments are simply awful. Unfortunately I can't dictate that people > > don't > > send patches that way. > How can anyone use git, but not git send-email? Any decent email provider > has support for external clients over SMTP. And I believe you *can* actually > dictate that people don't attach patches -- if you have control over the > mailing list software, you can set up a filter that rejects such emails > and auto-replies with instructions on how to send them properly. I have tested a few email providers trying to find one which works with git send-email. Some email providers blocked my emails because they don't know git send-email and treat patches in email as spam. And there are email client issues. Michael's email is blank in a very popular email client. I reported the bug to the developer of that email client, they don't understand the use case and have no interest to fix it. I know a developer which have contributed to FFmpeg and stop doing so after losing his git-send-email environment. > > > >But you did not answer my question: which specific code review features > > >are you missing? > > > > Proper threaded reviews with state tracking, ability to collapse and expand > > context and files, and proper listing of open MR (*not* like patchwork). > I can sort of understand everything except the last one. What is "a proper > listing of open MR" supposed to mean...? (I know what a merge request is, > of course, but I don't get how the way GitLab lists them is supposedly > superior to SourceHut's list of patches.) > ___ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe". ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH v4 3/4] all: Link to "context" from all contexts with documentation
> On May 2, 2024, at 21:27, Andrew Sayers wrote:
>
> On Thu, May 02, 2024 at 09:00:42PM +0800, Zhao Zhili wrote:
>>
>>
>>> On May 2, 2024, at 19:01, Lynne wrote:
>>>
>>> Apr 29, 2024, 11:24 by ffmpeg-de...@pileofstuff.org:
>>>
>>>> Some headings needed to be rewritten to accomodate the text,
>>>> (hopefully) without changing the meaning.
>>>> ---
>>>> libavcodec/aac/aacdec.h| 2 +-
>>>> libavcodec/aacenc.h| 2 +-
>>>> libavcodec/ac3enc.h| 2 +-
>>>> libavcodec/amfenc.h| 2 +-
>>>> libavcodec/atrac.h | 2 +-
>>>> libavcodec/avcodec.h | 3 ++-
>>>> libavcodec/bsf.h | 2 +-
>>>> libavcodec/cbs.h | 2 +-
>>>> libavcodec/d3d11va.h | 3 +--
>>>> libavcodec/h264dsp.h | 2 +-
>>>> libavcodec/h264pred.h | 2 +-
>>>> libavcodec/mediacodec.h| 2 +-
>>>> libavcodec/mpegaudiodec_template.c | 2 +-
>>>> libavcodec/pthread_frame.c | 4 ++--
>>>> libavcodec/qsv.h | 6 --
>>>> libavcodec/sbr.h | 2 +-
>>>> libavcodec/smacker.c | 2 +-
>>>> libavcodec/vdpau.h | 3 ++-
>>>> libavcodec/videotoolbox.h | 5 +++--
>>>> libavfilter/avfilter.h | 2 +-
>>>> libavformat/avformat.h | 3 ++-
>>>> libavformat/avio.h | 3 ++-
>>>> libavutil/audio_fifo.h | 2 +-
>>>> libavutil/hwcontext.h | 21 -
>>>> libavutil/hwcontext_cuda.h | 2 +-
>>>> libavutil/hwcontext_d3d11va.h | 4 ++--
>>>> libavutil/hwcontext_d3d12va.h | 6 +++---
>>>> libavutil/hwcontext_drm.h | 2 +-
>>>> libavutil/hwcontext_dxva2.h| 4 ++--
>>>> libavutil/hwcontext_mediacodec.h | 2 +-
>>>> libavutil/hwcontext_opencl.h | 4 ++--
>>>> libavutil/hwcontext_qsv.h | 4 ++--
>>>> libavutil/hwcontext_vaapi.h| 6 +++---
>>>> libavutil/hwcontext_vdpau.h| 2 +-
>>>> libavutil/hwcontext_vulkan.h | 4 ++--
>>>> libavutil/lfg.h| 2 +-
>>>> 36 files changed, 66 insertions(+), 57 deletions(-)
>>>>
>>>> diff --git a/libavcodec/aac/aacdec.h b/libavcodec/aac/aacdec.h
>>>> index 4cf764e2e9..71d61813f4 100644
>>>> --- a/libavcodec/aac/aacdec.h
>>>> +++ b/libavcodec/aac/aacdec.h
>>>> @@ -248,7 +248,7 @@ typedef struct AACDecDSP {
>>>> } AACDecDSP;
>>>>
>>>> /**
>>>> - * main AAC decoding context
>>>> + * main AAC decoding @ref md_doc_2context "context"
>>>> */
>>>> struct AACDecContext {
>>>> const struct AVClass *class;
>>>> diff --git a/libavcodec/aacenc.h b/libavcodec/aacenc.h
>>>> index d07960620e..1a645f4719 100644
>>>> --- a/libavcodec/aacenc.h
>>>> +++ b/libavcodec/aacenc.h
>>>> @@ -207,7 +207,7 @@ typedef struct AACPCEInfo {
>>>> } AACPCEInfo;
>>>>
>>>> /**
>>>> - * AAC encoder context
>>>> + * AAC encoder @ref md_doc_2context "context"
>>>> */
>>>> typedef struct AACEncContext {
>>>> AVClass *av_class;
>>>> diff --git a/libavcodec/ac3enc.h b/libavcodec/ac3enc.h
>>>> index 30812617cc..c725007077 100644
>>>> --- a/libavcodec/ac3enc.h
>>>> +++ b/libavcodec/ac3enc.h
>>>> @@ -152,7 +152,7 @@ typedef struct AC3Block {
>>>> } AC3Block;
>>>>
>>>> /**
>>>> - * AC-3 encoder private context.
>>>> + * AC-3 encoder private @ref md_doc_2context "context"
>>>> */
>>>> typedef struct AC3EncodeContext {
>>>> AVClass *av_class; ///< AVClass used for AVOption
>>>> diff --git a/libavcodec/amfenc.h b/libavcodec/amfenc.h
>>>> index 2dbd378ef8..f142ede63a 100644
>>>> --- a/libavcodec/amfenc.h
>>>> +++ b/libavcodec/amfenc.h
>>>> @@ -43,7 +43,7 @@ typedef struct AmfTraceWriter {
>>>> } AmfTraceWriter;
>>>>
>>>> /**
>>>> -* AMF encoder context
>>>> +* AMF encoder @ref md_doc_2context "c
Re: [FFmpeg-devel] [PATCH v4 3/4] all: Link to "context" from all contexts with documentation
> On May 2, 2024, at 19:01, Lynne wrote:
>
> Apr 29, 2024, 11:24 by ffmpeg-de...@pileofstuff.org:
>
>> Some headings needed to be rewritten to accomodate the text,
>> (hopefully) without changing the meaning.
>> ---
>> libavcodec/aac/aacdec.h| 2 +-
>> libavcodec/aacenc.h| 2 +-
>> libavcodec/ac3enc.h| 2 +-
>> libavcodec/amfenc.h| 2 +-
>> libavcodec/atrac.h | 2 +-
>> libavcodec/avcodec.h | 3 ++-
>> libavcodec/bsf.h | 2 +-
>> libavcodec/cbs.h | 2 +-
>> libavcodec/d3d11va.h | 3 +--
>> libavcodec/h264dsp.h | 2 +-
>> libavcodec/h264pred.h | 2 +-
>> libavcodec/mediacodec.h| 2 +-
>> libavcodec/mpegaudiodec_template.c | 2 +-
>> libavcodec/pthread_frame.c | 4 ++--
>> libavcodec/qsv.h | 6 --
>> libavcodec/sbr.h | 2 +-
>> libavcodec/smacker.c | 2 +-
>> libavcodec/vdpau.h | 3 ++-
>> libavcodec/videotoolbox.h | 5 +++--
>> libavfilter/avfilter.h | 2 +-
>> libavformat/avformat.h | 3 ++-
>> libavformat/avio.h | 3 ++-
>> libavutil/audio_fifo.h | 2 +-
>> libavutil/hwcontext.h | 21 -
>> libavutil/hwcontext_cuda.h | 2 +-
>> libavutil/hwcontext_d3d11va.h | 4 ++--
>> libavutil/hwcontext_d3d12va.h | 6 +++---
>> libavutil/hwcontext_drm.h | 2 +-
>> libavutil/hwcontext_dxva2.h| 4 ++--
>> libavutil/hwcontext_mediacodec.h | 2 +-
>> libavutil/hwcontext_opencl.h | 4 ++--
>> libavutil/hwcontext_qsv.h | 4 ++--
>> libavutil/hwcontext_vaapi.h| 6 +++---
>> libavutil/hwcontext_vdpau.h| 2 +-
>> libavutil/hwcontext_vulkan.h | 4 ++--
>> libavutil/lfg.h| 2 +-
>> 36 files changed, 66 insertions(+), 57 deletions(-)
>>
>> diff --git a/libavcodec/aac/aacdec.h b/libavcodec/aac/aacdec.h
>> index 4cf764e2e9..71d61813f4 100644
>> --- a/libavcodec/aac/aacdec.h
>> +++ b/libavcodec/aac/aacdec.h
>> @@ -248,7 +248,7 @@ typedef struct AACDecDSP {
>> } AACDecDSP;
>>
>> /**
>> - * main AAC decoding context
>> + * main AAC decoding @ref md_doc_2context "context"
>> */
>> struct AACDecContext {
>> const struct AVClass *class;
>> diff --git a/libavcodec/aacenc.h b/libavcodec/aacenc.h
>> index d07960620e..1a645f4719 100644
>> --- a/libavcodec/aacenc.h
>> +++ b/libavcodec/aacenc.h
>> @@ -207,7 +207,7 @@ typedef struct AACPCEInfo {
>> } AACPCEInfo;
>>
>> /**
>> - * AAC encoder context
>> + * AAC encoder @ref md_doc_2context "context"
>> */
>> typedef struct AACEncContext {
>> AVClass *av_class;
>> diff --git a/libavcodec/ac3enc.h b/libavcodec/ac3enc.h
>> index 30812617cc..c725007077 100644
>> --- a/libavcodec/ac3enc.h
>> +++ b/libavcodec/ac3enc.h
>> @@ -152,7 +152,7 @@ typedef struct AC3Block {
>> } AC3Block;
>>
>> /**
>> - * AC-3 encoder private context.
>> + * AC-3 encoder private @ref md_doc_2context "context"
>> */
>> typedef struct AC3EncodeContext {
>> AVClass *av_class; ///< AVClass used for AVOption
>> diff --git a/libavcodec/amfenc.h b/libavcodec/amfenc.h
>> index 2dbd378ef8..f142ede63a 100644
>> --- a/libavcodec/amfenc.h
>> +++ b/libavcodec/amfenc.h
>> @@ -43,7 +43,7 @@ typedef struct AmfTraceWriter {
>> } AmfTraceWriter;
>>
>> /**
>> -* AMF encoder context
>> +* AMF encoder @ref md_doc_2context "context"
>> */
>>
>> typedef struct AmfContext {
>> diff --git a/libavcodec/atrac.h b/libavcodec/atrac.h
>> index 05208bbee6..1527e376a9 100644
>> --- a/libavcodec/atrac.h
>> +++ b/libavcodec/atrac.h
>> @@ -39,7 +39,7 @@ typedef struct AtracGainInfo {
>> } AtracGainInfo;
>>
>> /**
>> - * Gain compensation context structure.
>> + * Gain compensation @ref md_doc_2context "context"
>> */
>> typedef struct AtracGCContext {
>> float gain_tab1[16]; ///< gain compensation level table
>> diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
>> index 968009a192..9180fedca7 100644
>> --- a/libavcodec/avcodec.h
>> +++ b/libavcodec/avcodec.h
>> @@ -430,7 +430,8 @@ typedef struct RcOverride{
>> #define AV_GET_ENCODE_BUFFER_FLAG_REF (1 << 0)
>>
>> /**
>> - * main external API structure.
>> + * @ref md_doc_2context "Context" for an encode or decode session
>> + *
>> * New fields can be added to the end with minor version bumps.
>> * Removal, reordering and changes to existing fields require a major
>> * version bump.
>> diff --git a/libavcodec/bsf.h b/libavcodec/bsf.h
>> index a09c69f242..bf79afa7cc 100644
>> --- a/libavcodec/bsf.h
>> +++ b/libavcodec/bsf.h
>> @@ -56,7 +56,7 @@
>> */
>>
>> /**
>> - * The bitstream filter state.
>> + * Bitstream filter @ref md_doc_2context "context"
>> *
>> * This struct must be allocated with av_bsf_alloc() and freed with
>> * av_bsf_free().
>> diff --git a/libavcodec/cbs.h b/libavcodec/cbs.h
>> index d479b1ac2d..0ff64d2fef 100644
Re: [FFmpeg-devel] [PATCH WIP v2 1/9] avfilter/dnn: Refactor DNN parameter configuration system
> On Apr 30, 2024, at 13:54, Chen, Wenbin
> wrote:
>
>> -Original Message-
>> From: ffmpeg-devel On Behalf Of Zhao
>> Zhili
>> Sent: Sunday, April 28, 2024 2:47 PM
>> To: ffmpeg-devel@ffmpeg.org
>> Cc: Zhao Zhili
>> Subject: [FFmpeg-devel] [PATCH WIP v2 1/9] avfilter/dnn: Refactor DNN
>> parameter configuration system
>>
>> From: Zhao Zhili
>> +
>> +void *ff_dnn_child_next(DnnContext *obj, void *prev) {
>> +size_t pre_offset;
>> +char *ptr;
>> +
>> +if (!prev) {
>> +obj->clazz = _base_class;
>> +return obj;
>> +}
>> +
>> +pre_offset = (char *)prev - (char *)obj;
>> +for (int i = 0; i < FF_ARRAY_ELEMS(dnn_backend_info_list) - 1; i++) {
>> +if (dnn_backend_info_list[i].offset == pre_offset) {
>> +ptr = (char *)obj + dnn_backend_info_list[i + 1].offset;
>> +*(const AVClass **) ptr = dnn_backend_info_list[i + 1].class;
>> +return ptr;
>> +}
>> +}
>> +
>> +return NULL;
>> +}
>
> Can this function be simplified by implementing AVFilter.preinit interface in
> each dnn filter
> and assign class to DNNContext and TF/OV/THOption in preinit function? I
> don't think setting AVClass
> in child_next() function is the proper way.
Sure, here comes v3:
https://ffmpeg.org/pipermail/ffmpeg-devel/2024-April/326495.html
>
> wenbin
>
>> +
>> +const AVClass *ff_dnn_child_class_iterate(void **iter)
>> +{
>> +uintptr_t i = (uintptr_t) *iter;
>> +
>> +if (i < FF_ARRAY_ELEMS(dnn_backend_info_list)) {
>> +*iter = (void *)(i + 1);
>> +return dnn_backend_info_list[i].class;
>> +}
>> +
>> +return NULL;
>> +}
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v3 10/10] avfilter/vf_dnn_detect: Fix null pointer dereference
From: Zhao Zhili
---
libavfilter/vf_dnn_detect.c | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/libavfilter/vf_dnn_detect.c b/libavfilter/vf_dnn_detect.c
index 926966368a..1830bae181 100644
--- a/libavfilter/vf_dnn_detect.c
+++ b/libavfilter/vf_dnn_detect.c
@@ -807,11 +807,13 @@ static av_cold void dnn_detect_uninit(AVFilterContext
*context)
DnnDetectContext *ctx = context->priv;
AVDetectionBBox *bbox;
ff_dnn_uninit(>dnnctx);
-while(av_fifo_can_read(ctx->bboxes_fifo)) {
-av_fifo_read(ctx->bboxes_fifo, , 1);
-av_freep();
+if (ctx->bboxes_fifo) {
+while (av_fifo_can_read(ctx->bboxes_fifo)) {
+av_fifo_read(ctx->bboxes_fifo, , 1);
+av_freep();
+}
+av_fifo_freep2(>bboxes_fifo);
}
-av_fifo_freep2(>bboxes_fifo);
av_freep(>anchors);
free_detect_labels(ctx);
}
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v3 09/10] avfilter/dnn: Use dnn_backend_info_list to search for dnn module
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_openvino.c | 1 +
libavfilter/dnn/dnn_backend_tf.c | 1 +
libavfilter/dnn/dnn_backend_torch.cpp | 1 +
libavfilter/dnn/dnn_interface.c| 26 --
libavfilter/dnn_interface.h| 1 +
5 files changed, 12 insertions(+), 18 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c
b/libavfilter/dnn/dnn_backend_openvino.c
index d8a6820dc2..9c699cdc8c 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -1613,6 +1613,7 @@ static int dnn_flush_ov(const DNNModel *model)
const DNNModule ff_dnn_backend_openvino = {
.clazz = DNN_DEFINE_CLASS(dnn_openvino),
+.type = DNN_OV,
.load_model = dnn_load_model_ov,
.execute_model = dnn_execute_model_ov,
.get_result = dnn_get_result_ov,
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index 06ea6cbb8c..6afefe8115 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -886,6 +886,7 @@ static int dnn_flush_tf(const DNNModel *model)
const DNNModule ff_dnn_backend_tf = {
.clazz = DNN_DEFINE_CLASS(dnn_tensorflow),
+.type = DNN_TF,
.load_model = dnn_load_model_tf,
.execute_model = dnn_execute_model_tf,
.get_result = dnn_get_result_tf,
diff --git a/libavfilter/dnn/dnn_backend_torch.cpp
b/libavfilter/dnn/dnn_backend_torch.cpp
index 24e9f2c8e2..2557264713 100644
--- a/libavfilter/dnn/dnn_backend_torch.cpp
+++ b/libavfilter/dnn/dnn_backend_torch.cpp
@@ -561,6 +561,7 @@ static int dnn_flush_th(const DNNModel *model)
extern const DNNModule ff_dnn_backend_torch = {
.clazz = DNN_DEFINE_CLASS(dnn_th),
+.type = DNN_TH,
.load_model = dnn_load_model_th,
.execute_model = dnn_execute_model_th,
.get_result = dnn_get_result_th,
diff --git a/libavfilter/dnn/dnn_interface.c b/libavfilter/dnn/dnn_interface.c
index 2d3bd229a4..b56c22e4c7 100644
--- a/libavfilter/dnn/dnn_interface.c
+++ b/libavfilter/dnn/dnn_interface.c
@@ -81,25 +81,15 @@ static const DnnBackendInfo dnn_backend_info_list[] = {
const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx)
{
-switch(backend_type){
-#if (CONFIG_LIBTENSORFLOW == 1)
-case DNN_TF:
-return _dnn_backend_tf;
-#endif
-#if (CONFIG_LIBOPENVINO == 1)
-case DNN_OV:
-return _dnn_backend_openvino;
-#endif
-#if (CONFIG_LIBTORCH == 1)
-case DNN_TH:
-return _dnn_backend_torch;
-#endif
-default:
-av_log(log_ctx, AV_LOG_ERROR,
-"Module backend_type %d is not supported or enabled.\n",
-backend_type);
-return NULL;
+for (int i = 1; i < FF_ARRAY_ELEMS(dnn_backend_info_list); i++) {
+if (dnn_backend_info_list[i].module->type == backend_type)
+return dnn_backend_info_list[i].module;
}
+
+av_log(log_ctx, AV_LOG_ERROR,
+"Module backend_type %d is not supported or enabled.\n",
+backend_type);
+return NULL;
}
void ff_dnn_init_child_class(DnnContext *ctx)
diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h
index 1154d50629..e2582c808e 100644
--- a/libavfilter/dnn_interface.h
+++ b/libavfilter/dnn_interface.h
@@ -170,6 +170,7 @@ typedef struct DnnContext {
// Stores pointers to functions for loading, executing, freeing DNN models for
one of the backends.
struct DNNModule {
const AVClass clazz;
+DNNBackendType type;
// Loads model and parameters from given file. Returns NULL if it is not
possible.
DNNModel *(*load_model)(DnnContext *ctx, DNNFunctionType func_type,
AVFilterContext *filter_ctx);
// Executes model with specified input and output. Returns the error code
otherwise.
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v3 07/10] avfilter/dnn_backend_torch: Simplify memory allocation
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_torch.cpp | 31 +++
1 file changed, 12 insertions(+), 19 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_torch.cpp
b/libavfilter/dnn/dnn_backend_torch.cpp
index abdef1f178..818ec5b713 100644
--- a/libavfilter/dnn/dnn_backend_torch.cpp
+++ b/libavfilter/dnn/dnn_backend_torch.cpp
@@ -37,8 +37,8 @@ extern "C" {
}
typedef struct THModel {
+DNNModel model;
DnnContext *ctx;
-DNNModel *model;
torch::jit::Module *jit_model;
SafeQueue *request_queue;
Queue *task_queue;
@@ -141,7 +141,7 @@ static void dnn_free_model_th(DNNModel **model)
ff_queue_destroy(th_model->task_queue);
delete th_model->jit_model;
av_freep(_model);
-av_freep(model);
+*model = NULL;
}
static int get_input_th(void *model, DNNData *input, const char *input_name)
@@ -195,19 +195,19 @@ static int fill_model_input_th(THModel *th_model,
THRequestItem *request)
infer_request->input_tensor = new torch::Tensor();
infer_request->output = new torch::Tensor();
-switch (th_model->model->func_type) {
+switch (th_model->model.func_type) {
case DFT_PROCESS_FRAME:
input.scale = 255;
if (task->do_ioproc) {
-if (th_model->model->frame_pre_proc != NULL) {
-th_model->model->frame_pre_proc(task->in_frame, ,
th_model->model->filter_ctx);
+if (th_model->model.frame_pre_proc != NULL) {
+th_model->model.frame_pre_proc(task->in_frame, ,
th_model->model.filter_ctx);
} else {
ff_proc_from_frame_to_dnn(task->in_frame, , ctx);
}
}
break;
default:
-avpriv_report_missing_feature(NULL, "model function type %d",
th_model->model->func_type);
+avpriv_report_missing_feature(NULL, "model function type %d",
th_model->model.func_type);
break;
}
*infer_request->input_tensor = torch::from_blob(input.data,
@@ -282,13 +282,13 @@ static void infer_completion_callback(void *args) {
goto err;
}
-switch (th_model->model->func_type) {
+switch (th_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
outputs.scale = 255;
outputs.data = output->data_ptr();
-if (th_model->model->frame_post_proc != NULL) {
-th_model->model->frame_post_proc(task->out_frame, ,
th_model->model->filter_ctx);
+if (th_model->model.frame_post_proc != NULL) {
+th_model->model.frame_post_proc(task->out_frame, ,
th_model->model.filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, ,
th_model->ctx);
}
@@ -298,7 +298,7 @@ static void infer_completion_callback(void *args) {
}
break;
default:
-avpriv_report_missing_feature(th_model->ctx, "model function type %d",
th_model->model->func_type);
+avpriv_report_missing_feature(th_model->ctx, "model function type %d",
th_model->model.func_type);
goto err;
}
task->inference_done++;
@@ -417,17 +417,10 @@ static DNNModel *dnn_load_model_th(DnnContext *ctx,
DNNFunctionType func_type, A
THRequestItem *item = NULL;
const char *device_name = ctx->device ? ctx->device : "cpu";
-model = (DNNModel *)av_mallocz(sizeof(DNNModel));
-if (!model) {
-return NULL;
-}
-
th_model = (THModel *)av_mallocz(sizeof(THModel));
-if (!th_model) {
-av_freep();
+if (!th_model)
return NULL;
-}
-th_model->model = model;
+model = _model->model;
model->model = th_model;
th_model->ctx = ctx;
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v3 05/10] avfilter/dnn_backend_tf: Fix free context at random place
From: Zhao Zhili
It will be freed again by ff_dnn_uninit.
---
libavfilter/dnn/dnn_backend_tf.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index 60f9e57fb7..3b4de6d13f 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -804,7 +804,7 @@ err:
if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
destroy_request_item();
}
-dnn_free_model_tf(_model->model);
+
return ret;
}
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v3 08/10] avfilter/dnn: Remove a level of dereference
From: Zhao Zhili
For code such as 'model->model = ov_model' is confusing. We can
just drop the member variable and use cast to get the subclass.
---
libavfilter/dnn/dnn_backend_openvino.c | 17 -
libavfilter/dnn/dnn_backend_tf.c | 19 +--
libavfilter/dnn/dnn_backend_torch.cpp | 15 +++
libavfilter/dnn_filter_common.c| 6 +++---
libavfilter/dnn_interface.h| 6 ++
5 files changed, 29 insertions(+), 34 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c
b/libavfilter/dnn/dnn_backend_openvino.c
index 1acc54b791..d8a6820dc2 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -517,7 +517,7 @@ static void dnn_free_model_ov(DNNModel **model)
if (!model || !*model)
return;
-ov_model = (*model)->model;
+ov_model = (OVModel *)(*model);
while (ff_safe_queue_size(ov_model->request_queue) != 0) {
OVRequestItem *item = ff_safe_queue_pop_front(ov_model->request_queue);
if (item && item->infer_request) {
@@ -1059,9 +1059,9 @@ err:
return ret;
}
-static int get_input_ov(void *model, DNNData *input, const char *input_name)
+static int get_input_ov(DNNModel *model, DNNData *input, const char
*input_name)
{
-OVModel *ov_model = model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
int input_resizable = ctx->ov_option.input_resizable;
@@ -1255,7 +1255,7 @@ static int extract_lltask_from_task(DNNFunctionType
func_type, TaskItem *task, Q
}
}
-static int get_output_ov(void *model, const char *input_name, int input_width,
int input_height,
+static int get_output_ov(DNNModel *model, const char *input_name, int
input_width, int input_height,
const char *output_name, int *output_width,
int *output_height)
{
#if HAVE_OPENVINO2
@@ -1268,7 +1268,7 @@ static int get_output_ov(void *model, const char
*input_name, int input_width, i
input_shapes_t input_shapes;
#endif
int ret;
-OVModel *ov_model = model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
TaskItem task;
OVRequestItem *request;
@@ -1383,7 +1383,6 @@ static DNNModel *dnn_load_model_ov(DnnContext *ctx,
DNNFunctionType func_type, A
return NULL;
ov_model->ctx = ctx;
model = _model->model;
-model->model = ov_model;
#if HAVE_OPENVINO2
status = ov_core_create();
@@ -1470,7 +1469,7 @@ err:
static int dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams
*exec_params)
{
-OVModel *ov_model = model->model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
OVRequestItem *request;
TaskItem *task;
@@ -1558,13 +1557,13 @@ static int dnn_execute_model_ov(const DNNModel *model,
DNNExecBaseParams *exec_p
static DNNAsyncStatusType dnn_get_result_ov(const DNNModel *model, AVFrame
**in, AVFrame **out)
{
-OVModel *ov_model = model->model;
+OVModel *ov_model = (OVModel *)model;
return ff_dnn_get_result_common(ov_model->task_queue, in, out);
}
static int dnn_flush_ov(const DNNModel *model)
{
-OVModel *ov_model = model->model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
OVRequestItem *request;
#if HAVE_OPENVINO2
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index c7716e696d..06ea6cbb8c 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -262,9 +262,9 @@ static TF_Tensor *allocate_input_tensor(const DNNData
*input)
input_dims[1] * input_dims[2] * input_dims[3] *
size);
}
-static int get_input_tf(void *model, DNNData *input, const char *input_name)
+static int get_input_tf(DNNModel *model, DNNData *input, const char
*input_name)
{
-TFModel *tf_model = model;
+TFModel *tf_model = (TFModel *)model;
DnnContext *ctx = tf_model->ctx;
TF_Status *status;
TF_DataType dt;
@@ -310,11 +310,11 @@ static int get_input_tf(void *model, DNNData *input,
const char *input_name)
return 0;
}
-static int get_output_tf(void *model, const char *input_name, int input_width,
int input_height,
+static int get_output_tf(DNNModel *model, const char *input_name, int
input_width, int input_height,
const char *output_name, int *output_width,
int *output_height)
{
int ret;
-TFModel *tf_model = model;
+TFModel *tf_model = (TFModel *)model;
DnnContext *ctx = tf_model->ctx;
TaskItem task;
TFRequestItem *request;
@@ -486,7 +486,7 @@ static void dnn_free_model_tf(DNNModel **model)
if (!model || !*model)
return;
-tf_model = (*model)->model;
+tf_model = (TFModel *)(*model);
while (ff_safe_queue_size(tf_model->request_queue)
[FFmpeg-devel] [PATCH v3 01/10] avfilter/dnn: Refactor DNN parameter configuration system
From: Zhao Zhili
This patch trying to resolve mulitiple issues related to parameter
configuration:
Firstly, each DNN filters duplicate DNN_COMMON_OPTIONS, which should
be the common options of backend.
Secondly, backend options are hidden behind the scene. It's a
AV_OPT_TYPE_STRING backend_configs for user, and parsed by each
backend. We don't know each backend support what kind of options
from the help message.
Third, DNN backends duplicate DNN_BACKEND_COMMON_OPTIONS.
Last but not the least, pass backend options via AV_OPT_TYPE_STRING
makes it hard to pass AV_OPT_TYPE_BINARY to backend, if not impossible.
This patch puts backend common options and each backend options inside
DnnContext to reduce code duplication, make options user friendly, and
easy to extend for future usecase.
There is a known issue that, for a filter which only support one or two
of the backends, the help message still show the option of all three
backends. Each DNN filter should be able to run on any backend. Current
issue is mostly due to incomplete implementation (e.g., libtorch only
support DFT_PROCESS_FRAME), and lack of maintenance on the filters.
For example,
./ffmpeg -h filter=dnn_processing
dnn_processing AVOptions:
dnn_backend ..FV... DNN backend (from INT_MIN to
INT_MAX) (default tensorflow)
tensorflow 1..FV... tensorflow backend flag
openvino2..FV... openvino backend flag
torch 3..FV... torch backend flag
dnn_base AVOptions:
model ..F path to model file
input ..F input name of the model
output ..F output name of the model
backend_configs..F...P backend configs (deprecated)
options..F...P backend configs (deprecated)
nireq ..F number of request (from 0 to
INT_MAX) (default 0)
async ..F use DNN async inference (default
true)
device ..F device to run model
dnn_tensorflow AVOptions:
sess_config..F config for SessionOptions
dnn_openvino AVOptions:
batch_size..F batch size per request (from 1 to
1000) (default 1)
input_resizable ..F can input be resizable or not
(default false)
layout..F input layout of model (from 0 to
2) (default none)
none0..F none
nchw1..F nchw
nhwc2..F nhwc
scale ..F Add scale preprocess operation.
Divide each element of input by specified value. (from INT_MIN to INT_MAX)
(default 0)
mean..F Add mean preprocess operation.
Subtract specified value from each element of input. (from INT_MIN to INT_MAX)
(default 0)
dnn_th AVOptions:
optimize ..F turn on graph executor
optimization (from 0 to 1) (default 0)
---
libavfilter/dnn/dnn_backend_common.h | 13 ++-
libavfilter/dnn/dnn_backend_openvino.c | 146 ++---
libavfilter/dnn/dnn_backend_tf.c | 82 +-
libavfilter/dnn/dnn_backend_torch.cpp | 67
libavfilter/dnn/dnn_interface.c| 89 +++
libavfilter/dnn_filter_common.c| 38 ++-
libavfilter/dnn_filter_common.h| 39 +++
libavfilter/dnn_interface.h| 67 +++-
libavfilter/vf_derain.c| 6 +-
libavfilter/vf_dnn_classify.c | 4 +-
libavfilter/vf_dnn_detect.c| 4 +-
libavfilter/vf_dnn_processing.c| 4 +-
libavfilter/vf_sr.c| 6 +-
13 files changed, 336 insertions(+), 229 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_common.h
b/libavfilter/dnn/dnn_backend_common.h
index 42c67c7040..9f5d37b3e0 100644
--- a/libavfilter/dnn/dnn_backend_common.h
+++ b/libavfilter/dnn/dnn_backend_common.h
@@ -28,9 +28,16 @@
#include "../dnn_interface.h"
#include "libavutil/thread.h"
-#define DNN_BACKEND_COMMON_OPTIONS \
-{ "nireq", "number of request",
OFFSET(options.nireq), AV_OPT_TYPE_INT,{ .i64 = 0 }, 0,
INT_MAX, FLAGS }, \
-{ "async", "use DNN async inference",
OFFSET(options.async), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0,
1, FLAGS },
+#define DNN_DEFINE_CLASS_EXT(name, desc, options) \
+{ \
+.class_name = desc, \
+.item_name = av_default_item_name, \
+.option = options, \
+.version= LIBAVUTIL_VERSION_INT,\
+.category = AV_CLASS_CATEGORY_FILTER, \
+}
+#define DNN_DEFINE_CLASS(fname) \
+DNN_DEFINE_CLASS_EXT(fname, #fname
[FFmpeg-devel] [PATCH v3 06/10] avfilter/dnn_backend_tf: Simplify memory allocation
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_tf.c | 33 +---
1 file changed, 13 insertions(+), 20 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index 3b4de6d13f..c7716e696d 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -37,8 +37,8 @@
#include
typedef struct TFModel {
+DNNModel model;
DnnContext *ctx;
-DNNModel *model;
TF_Graph *graph;
TF_Session *session;
TF_Status *status;
@@ -518,7 +518,7 @@ static void dnn_free_model_tf(DNNModel **model)
TF_DeleteStatus(tf_model->status);
}
av_freep(_model);
-av_freep();
+*model = NULL;
}
static DNNModel *dnn_load_model_tf(DnnContext *ctx, DNNFunctionType func_type,
AVFilterContext *filter_ctx)
@@ -526,18 +526,11 @@ static DNNModel *dnn_load_model_tf(DnnContext *ctx,
DNNFunctionType func_type, A
DNNModel *model = NULL;
TFModel *tf_model = NULL;
-model = av_mallocz(sizeof(DNNModel));
-if (!model){
-return NULL;
-}
-
tf_model = av_mallocz(sizeof(TFModel));
-if (!tf_model){
-av_freep();
+if (!tf_model)
return NULL;
-}
+model = _model->model;
model->model = tf_model;
-tf_model->model = model;
tf_model->ctx = ctx;
if (load_tf_model(tf_model, ctx->model_filename) != 0){
@@ -650,11 +643,11 @@ static int fill_model_input_tf(TFModel *tf_model,
TFRequestItem *request) {
}
input.data = (float *)TF_TensorData(infer_request->input_tensor);
-switch (tf_model->model->func_type) {
+switch (tf_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
-if (tf_model->model->frame_pre_proc != NULL) {
-tf_model->model->frame_pre_proc(task->in_frame, ,
tf_model->model->filter_ctx);
+if (tf_model->model.frame_pre_proc != NULL) {
+tf_model->model.frame_pre_proc(task->in_frame, ,
tf_model->model.filter_ctx);
} else {
ff_proc_from_frame_to_dnn(task->in_frame, , ctx);
}
@@ -664,7 +657,7 @@ static int fill_model_input_tf(TFModel *tf_model,
TFRequestItem *request) {
ff_frame_to_dnn_detect(task->in_frame, , ctx);
break;
default:
-avpriv_report_missing_feature(ctx, "model function type %d",
tf_model->model->func_type);
+avpriv_report_missing_feature(ctx, "model function type %d",
tf_model->model.func_type);
break;
}
@@ -724,12 +717,12 @@ static void infer_completion_callback(void *args) {
outputs[i].data = TF_TensorData(infer_request->output_tensors[i]);
outputs[i].dt =
(DNNDataType)TF_TensorType(infer_request->output_tensors[i]);
}
-switch (tf_model->model->func_type) {
+switch (tf_model->model.func_type) {
case DFT_PROCESS_FRAME:
//it only support 1 output if it's frame in & frame out
if (task->do_ioproc) {
-if (tf_model->model->frame_post_proc != NULL) {
-tf_model->model->frame_post_proc(task->out_frame, outputs,
tf_model->model->filter_ctx);
+if (tf_model->model.frame_post_proc != NULL) {
+tf_model->model.frame_post_proc(task->out_frame, outputs,
tf_model->model.filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);
}
@@ -741,11 +734,11 @@ static void infer_completion_callback(void *args) {
}
break;
case DFT_ANALYTICS_DETECT:
-if (!tf_model->model->detect_post_proc) {
+if (!tf_model->model.detect_post_proc) {
av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post
proc\n");
return;
}
-tf_model->model->detect_post_proc(task->in_frame, outputs,
task->nb_output, tf_model->model->filter_ctx);
+tf_model->model.detect_post_proc(task->in_frame, outputs,
task->nb_output, tf_model->model.filter_ctx);
break;
default:
av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this
kind of dnn filter now\n");
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v3 02/10] avfilter/dnn_backend_openvino: Fix free context at random place
From: Zhao Zhili It will be freed again by ff_dnn_uninit. --- libavfilter/dnn/dnn_backend_openvino.c | 1 - 1 file changed, 1 deletion(-) diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c index c4b0682f11..769ba0a54b 100644 --- a/libavfilter/dnn/dnn_backend_openvino.c +++ b/libavfilter/dnn/dnn_backend_openvino.c @@ -959,7 +959,6 @@ err: if (input_model_info) ov_preprocess_input_model_info_free(input_model_info); #endif -dnn_free_model_ov(_model->model); return ret; } -- 2.25.1 ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v3 04/10] avfilter/dnn_backend_tf: Remove one level of indentation
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_tf.c | 63
1 file changed, 32 insertions(+), 31 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index d24591b90b..60f9e57fb7 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -483,41 +483,42 @@ static void dnn_free_model_tf(DNNModel **model)
{
TFModel *tf_model;
-if (*model){
-tf_model = (*model)->model;
-while (ff_safe_queue_size(tf_model->request_queue) != 0) {
-TFRequestItem *item =
ff_safe_queue_pop_front(tf_model->request_queue);
-destroy_request_item();
-}
-ff_safe_queue_destroy(tf_model->request_queue);
+if (!model || !*model)
+return;
-while (ff_queue_size(tf_model->lltask_queue) != 0) {
-LastLevelTaskItem *item =
ff_queue_pop_front(tf_model->lltask_queue);
-av_freep();
-}
-ff_queue_destroy(tf_model->lltask_queue);
+tf_model = (*model)->model;
+while (ff_safe_queue_size(tf_model->request_queue) != 0) {
+TFRequestItem *item = ff_safe_queue_pop_front(tf_model->request_queue);
+destroy_request_item();
+}
+ff_safe_queue_destroy(tf_model->request_queue);
-while (ff_queue_size(tf_model->task_queue) != 0) {
-TaskItem *item = ff_queue_pop_front(tf_model->task_queue);
-av_frame_free(>in_frame);
-av_frame_free(>out_frame);
-av_freep();
-}
-ff_queue_destroy(tf_model->task_queue);
+while (ff_queue_size(tf_model->lltask_queue) != 0) {
+LastLevelTaskItem *item = ff_queue_pop_front(tf_model->lltask_queue);
+av_freep();
+}
+ff_queue_destroy(tf_model->lltask_queue);
-if (tf_model->graph){
-TF_DeleteGraph(tf_model->graph);
-}
-if (tf_model->session){
-TF_CloseSession(tf_model->session, tf_model->status);
-TF_DeleteSession(tf_model->session, tf_model->status);
-}
-if (tf_model->status){
-TF_DeleteStatus(tf_model->status);
-}
-av_freep(_model);
-av_freep();
+while (ff_queue_size(tf_model->task_queue) != 0) {
+TaskItem *item = ff_queue_pop_front(tf_model->task_queue);
+av_frame_free(>in_frame);
+av_frame_free(>out_frame);
+av_freep();
+}
+ff_queue_destroy(tf_model->task_queue);
+
+if (tf_model->graph){
+TF_DeleteGraph(tf_model->graph);
+}
+if (tf_model->session){
+TF_CloseSession(tf_model->session, tf_model->status);
+TF_DeleteSession(tf_model->session, tf_model->status);
+}
+if (tf_model->status){
+TF_DeleteStatus(tf_model->status);
}
+av_freep(_model);
+av_freep();
}
static DNNModel *dnn_load_model_tf(DnnContext *ctx, DNNFunctionType func_type,
AVFilterContext *filter_ctx)
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH v3 03/10] avfilter/dnn_backend_openvino: simplify memory allocation
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_openvino.c | 47 +++---
1 file changed, 20 insertions(+), 27 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c
b/libavfilter/dnn/dnn_backend_openvino.c
index 769ba0a54b..1acc54b791 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -41,8 +41,8 @@
#include "dnn_backend_common.h"
typedef struct OVModel{
+DNNModel model;
DnnContext *ctx;
-DNNModel *model;
#if HAVE_OPENVINO2
ov_core_t *core;
ov_model_t *ov_model;
@@ -300,11 +300,11 @@ static int fill_model_input_ov(OVModel *ov_model,
OVRequestItem *request)
return ov2_map_error(status, NULL);
}
#endif
-switch (ov_model->model->func_type) {
+switch (ov_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
-if (ov_model->model->frame_pre_proc != NULL) {
-ov_model->model->frame_pre_proc(task->in_frame, ,
ov_model->model->filter_ctx);
+if (ov_model->model.frame_pre_proc != NULL) {
+ov_model->model.frame_pre_proc(task->in_frame, ,
ov_model->model.filter_ctx);
} else {
ff_proc_from_frame_to_dnn(task->in_frame, , ctx);
}
@@ -442,11 +442,11 @@ static void infer_completion_callback(void *args)
for (int i = 0; i < request->lltask_count; ++i) {
task = request->lltasks[i]->task;
-switch (ov_model->model->func_type) {
+switch (ov_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
-if (ov_model->model->frame_post_proc != NULL) {
-ov_model->model->frame_post_proc(task->out_frame, outputs,
ov_model->model->filter_ctx);
+if (ov_model->model.frame_post_proc != NULL) {
+ov_model->model.frame_post_proc(task->out_frame, outputs,
ov_model->model.filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);
}
@@ -458,23 +458,23 @@ static void infer_completion_callback(void *args)
}
break;
case DFT_ANALYTICS_DETECT:
-if (!ov_model->model->detect_post_proc) {
+if (!ov_model->model.detect_post_proc) {
av_log(ctx, AV_LOG_ERROR, "detect filter needs to provide post
proc\n");
goto end;
}
-ov_model->model->detect_post_proc(task->in_frame, outputs,
+ov_model->model.detect_post_proc(task->in_frame, outputs,
ov_model->nb_outputs,
- ov_model->model->filter_ctx);
+ ov_model->model.filter_ctx);
break;
case DFT_ANALYTICS_CLASSIFY:
-if (!ov_model->model->classify_post_proc) {
+if (!ov_model->model.classify_post_proc) {
av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide
post proc\n");
goto end;
}
for (int output_i = 0; output_i < ov_model->nb_outputs; output_i++)
-ov_model->model->classify_post_proc(task->in_frame, outputs,
+ov_model->model.classify_post_proc(task->in_frame, outputs,
request->lltasks[i]->bbox_index,
-
ov_model->model->filter_ctx);
+
ov_model->model.filter_ctx);
break;
default:
av_assert0(!"should not reach here");
@@ -571,7 +571,7 @@ static void dnn_free_model_ov(DNNModel **model)
av_free(ov_model->all_input_names);
#endif
av_freep(_model);
-av_freep(model);
+*model = NULL;
}
@@ -598,7 +598,7 @@ static int init_model_ov(OVModel *ov_model, const char
*input_name, const char *
#endif
// We scale pixel by default when do frame processing.
if (fabsf(ctx->ov_option.scale) < 1e-6f)
-ctx->ov_option.scale = ov_model->model->func_type == DFT_PROCESS_FRAME
? 255 : 1;
+ctx->ov_option.scale = ov_model->model.func_type == DFT_PROCESS_FRAME
? 255 : 1;
// batch size
if (ctx->ov_option.batch_size <= 0) {
ctx->ov_option.batch_size = 1;
@@ -702,7 +702,7 @@ static int init_model_ov(OVModel *ov_model, const char
*input_name, const char *
ret = ov2_map_error(status, NULL);
goto err;
}
-if (ov_model->model->func_type != DFT_PROCESS_FRAME
[FFmpeg-devel] [PATCH v3 00/10] Refactor DNN
From: Zhao Zhili v3: 1. Init class with ff_dnn_filter_init_child_class and called by AVFilter preinit. 2. Fix crash in vf_dnn_detect (patch 10/10). Zhao Zhili (10): avfilter/dnn: Refactor DNN parameter configuration system avfilter/dnn_backend_openvino: Fix free context at random place avfilter/dnn_backend_openvino: simplify memory allocation avfilter/dnn_backend_tf: Remove one level of indentation avfilter/dnn_backend_tf: Fix free context at random place avfilter/dnn_backend_tf: Simplify memory allocation avfilter/dnn_backend_torch: Simplify memory allocation avfilter/dnn: Remove a level of dereference avfilter/dnn: Use dnn_backend_info_list to search for dnn module avfilter/vf_dnn_detect: Fix null pointer dereference libavfilter/dnn/dnn_backend_common.h | 13 +- libavfilter/dnn/dnn_backend_openvino.c | 210 ++--- libavfilter/dnn/dnn_backend_tf.c | 194 ++- libavfilter/dnn/dnn_backend_torch.cpp | 112 + libavfilter/dnn/dnn_interface.c| 115 +++--- libavfilter/dnn_filter_common.c| 44 +- libavfilter/dnn_filter_common.h| 39 ++--- libavfilter/dnn_interface.h| 74 +++-- libavfilter/vf_derain.c| 6 +- libavfilter/vf_dnn_classify.c | 4 +- libavfilter/vf_dnn_detect.c| 14 +- libavfilter/vf_dnn_processing.c| 4 +- libavfilter/vf_sr.c| 6 +- 13 files changed, 456 insertions(+), 379 deletions(-) -- 2.25.1 ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH WIP 0/9] Refactor DNN
> On Apr 29, 2024, at 18:29, Guo, Yejun
> wrote:
>
>
>
>> -Original Message-
>> From: ffmpeg-devel On Behalf Of Zhao
>> Zhili
>> Sent: Sunday, April 28, 2024 6:55 PM
>> To: FFmpeg development discussions and patches > de...@ffmpeg.org>
>> Subject: Re: [FFmpeg-devel] [PATCH WIP 0/9] Refactor DNN
>>
>>
>>
>>> On Apr 28, 2024, at 18:34, Guo, Yejun > intel@ffmpeg.org> wrote:
>>>
>>>> -----Original Message-
>>>> From: ffmpeg-devel >>> <mailto:ffmpeg-devel-boun...@ffmpeg.org>> On Behalf Of Zhao Zhili
>>>> Sent: Sunday, April 28, 2024 12:42 AM
>>>> To: ffmpeg-devel@ffmpeg.org <mailto:ffmpeg-devel@ffmpeg.org>
>>>> Cc: Zhao Zhili mailto:zhiliz...@tencent.com>>
>>>> Subject: [FFmpeg-devel] [PATCH WIP 0/9] Refactor DNN
>>>>
>>>> From: Zhao Zhili
>>>>
>>>> During the refactor progress, I have found some serious issues, which
>>>> is not resolved by the patchset:
>>>>
>>>> 1. Tensorflow backend is broken.
>>>>
>>>> I think it doesn't work since 2021 at least. For example, it destroy
>>>> a thread and create a new thread for each frame, and it destroy an
>>>> invalid thread at the first
>>>> frame:
>>>
>>> It works from the day that code is merged, till today. It is by design
>>> to keep the code simplicity by using the feature that pthread_join
>>> accepts a parameter that is not a joinable thread.
>>>
>>> Please share more info if you experienced a real case that it does not work.
>>
>> It will abort if ASSERT_LEVEL > 1.
>>
>> #define ASSERT_PTHREAD_ABORT(func, ret) do {\
>>char errbuf[AV_ERROR_MAX_STRING_SIZE] = ""; \
>>av_log(NULL, AV_LOG_FATAL, AV_STRINGIFY(func) \
>> " failed with error: %s\n", \
>> av_make_error_string(errbuf, AV_ERROR_MAX_STRING_SIZE, \
>>AVERROR(ret))); \
>>abort();\
>> } while (0)
>>
>> I think the check is there just to prevent call pthread_join(0, ) by
>> accident,
>> so we shouldn’t do that on purpose.
>>
> Nice catch with configure assert level > 1, will fix, and patch also welcome,
> thanks.
If I read the code correctly, it destroy a thread and create a new thread for
each
frame. I think this “async” mode isn’t common in ffmpeg’s design. Create new
thread
for each frame can be heavy on some platforms. We use slice threading to improve
parallel, and thread with command queue to improve throughput. In this case with
tensorflow do the heavy lift, if it doesn’t support async operation, simple
synchronous
operation with tensorflow backend should be find. The “async” mode is
unnecessary
and use more resource over the benefit it provides.
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH WIP 0/9] Refactor DNN
> On Apr 28, 2024, at 18:58, Paul B Mahol wrote: > > Extremely low quality filters, both in source code quality and > performance/security and output quality should be queued for removal. I don’t think there is any replacement for the function provided by dnn_detect, which is very important for ROI encoding. > ___ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe". ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH WIP 0/9] Refactor DNN
> On Apr 28, 2024, at 18:34, Guo, Yejun
> wrote:
>
>> -Original Message-
>> From: ffmpeg-devel > <mailto:ffmpeg-devel-boun...@ffmpeg.org>> On Behalf Of Zhao
>> Zhili
>> Sent: Sunday, April 28, 2024 12:42 AM
>> To: ffmpeg-devel@ffmpeg.org <mailto:ffmpeg-devel@ffmpeg.org>
>> Cc: Zhao Zhili mailto:zhiliz...@tencent.com>>
>> Subject: [FFmpeg-devel] [PATCH WIP 0/9] Refactor DNN
>>
>> From: Zhao Zhili
>>
>> During the refactor progress, I have found some serious issues, which is not
>> resolved by the patchset:
>>
>> 1. Tensorflow backend is broken.
>>
>> I think it doesn't work since 2021 at least. For example, it destroy a
>> thread and
>> create a new thread for each frame, and it destroy an invalid thread at the
>> first
>> frame:
>
> It works from the day that code is merged, till today. It is by design to
> keep the
> code simplicity by using the feature that pthread_join accepts a parameter
> that
> is not a joinable thread.
>
> Please share more info if you experienced a real case that it does not work.
It will abort if ASSERT_LEVEL > 1.
#define ASSERT_PTHREAD_ABORT(func, ret) do {\
char errbuf[AV_ERROR_MAX_STRING_SIZE] = ""; \
av_log(NULL, AV_LOG_FATAL, AV_STRINGIFY(func) \
" failed with error: %s\n", \
av_make_error_string(errbuf, AV_ERROR_MAX_STRING_SIZE, \
AVERROR(ret))); \
abort();\
} while (0)
I think the check is there just to prevent call pthread_join(0, ) by
accident, so we
shouldn’t do that on purpose.
>>
>>
>>pthread_join(async_module->thread_id, );
>>if (status == DNN_ASYNC_FAIL) {
>>av_log(ctx, AV_LOG_ERROR, "Unable to start inference as previous
>> inference failed.\n");
>>return DNN_GENERIC_ERROR;
>>}
>>ret = pthread_create(_module->thread_id, NULL,
>> async_thread_routine, async_module);
>>
>>
>> 2. Openvino V1 doesn't compile. It doesn't compile and no one complains, I
>> think it's a hint to just keep the code for V2.
>
> In plan, and patch is welcome.
>
>>
>> 3. Error handling. It's easy to crash with incorrect command line arguments.
>
> Thanks, will review your patchset one by one, it may take some time.
>
>>
>> I don't have enough test case. Please share your test case and help on test.
>>
>> Zhao Zhili (9):
>> avfilter/dnn: Refactor DNN parameter configuration system
>> avfilter/dnn_backend_openvino: Fix free context at random place
>> avfilter/dnn_backend_openvino: simplify memory allocation
>> avfilter/dnn_backend_tf: Remove one level of indentation
>> avfilter/dnn_backend_tf: Fix free context at random place
>> avfilter/dnn_backend_tf: Simplify memory allocation
>> avfilter/dnn_backend_torch: Simplify memory allocation
>> avfilter/dnn: Remove a level of dereference
>> avfilter/dnn: Use dnn_backend_info_list to search for dnn module
>>
>> libavfilter/dnn/dnn_backend_common.h | 13 +-
>> libavfilter/dnn/dnn_backend_openvino.c | 210 ++---
>> libavfilter/dnn/dnn_backend_tf.c | 194 ++-
>> libavfilter/dnn/dnn_backend_torch.cpp | 112 +
>> libavfilter/dnn/dnn_interface.c| 107 ++---
>> libavfilter/dnn_filter_common.c| 38 -
>> libavfilter/dnn_filter_common.h| 37 ++---
>> libavfilter/dnn_interface.h| 73 +++--
>> libavfilter/vf_derain.c| 5 +-
>> libavfilter/vf_dnn_classify.c | 3 +-
>> libavfilter/vf_dnn_detect.c| 3 +-
>> libavfilter/vf_dnn_processing.c| 3 +-
>> libavfilter/vf_sr.c| 5 +-
>> 13 files changed, 428 insertions(+), 375 deletions(-)
>>
>> --
>> 2.34.1
>>
>> ___
>> ffmpeg-devel mailing list
>> ffmpeg-devel@ffmpeg.org <mailto:ffmpeg-devel@ffmpeg.org>
>> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>>
>> To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org
>> <mailto:ffmpeg-devel-requ...@ffmpeg.org>
>> with subject "unsubscribe".
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org <mailto:ffmpeg-devel@ffmpeg.org>
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org <mailto:ffmpeg-devel-requ...@ffmpeg.org> with
> subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH WIP v2 6/9] avfilter/dnn_backend_tf: Simplify memory allocation
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_tf.c | 33 +---
1 file changed, 13 insertions(+), 20 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index 3b4de6d13f..c7716e696d 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -37,8 +37,8 @@
#include
typedef struct TFModel {
+DNNModel model;
DnnContext *ctx;
-DNNModel *model;
TF_Graph *graph;
TF_Session *session;
TF_Status *status;
@@ -518,7 +518,7 @@ static void dnn_free_model_tf(DNNModel **model)
TF_DeleteStatus(tf_model->status);
}
av_freep(_model);
-av_freep();
+*model = NULL;
}
static DNNModel *dnn_load_model_tf(DnnContext *ctx, DNNFunctionType func_type,
AVFilterContext *filter_ctx)
@@ -526,18 +526,11 @@ static DNNModel *dnn_load_model_tf(DnnContext *ctx,
DNNFunctionType func_type, A
DNNModel *model = NULL;
TFModel *tf_model = NULL;
-model = av_mallocz(sizeof(DNNModel));
-if (!model){
-return NULL;
-}
-
tf_model = av_mallocz(sizeof(TFModel));
-if (!tf_model){
-av_freep();
+if (!tf_model)
return NULL;
-}
+model = _model->model;
model->model = tf_model;
-tf_model->model = model;
tf_model->ctx = ctx;
if (load_tf_model(tf_model, ctx->model_filename) != 0){
@@ -650,11 +643,11 @@ static int fill_model_input_tf(TFModel *tf_model,
TFRequestItem *request) {
}
input.data = (float *)TF_TensorData(infer_request->input_tensor);
-switch (tf_model->model->func_type) {
+switch (tf_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
-if (tf_model->model->frame_pre_proc != NULL) {
-tf_model->model->frame_pre_proc(task->in_frame, ,
tf_model->model->filter_ctx);
+if (tf_model->model.frame_pre_proc != NULL) {
+tf_model->model.frame_pre_proc(task->in_frame, ,
tf_model->model.filter_ctx);
} else {
ff_proc_from_frame_to_dnn(task->in_frame, , ctx);
}
@@ -664,7 +657,7 @@ static int fill_model_input_tf(TFModel *tf_model,
TFRequestItem *request) {
ff_frame_to_dnn_detect(task->in_frame, , ctx);
break;
default:
-avpriv_report_missing_feature(ctx, "model function type %d",
tf_model->model->func_type);
+avpriv_report_missing_feature(ctx, "model function type %d",
tf_model->model.func_type);
break;
}
@@ -724,12 +717,12 @@ static void infer_completion_callback(void *args) {
outputs[i].data = TF_TensorData(infer_request->output_tensors[i]);
outputs[i].dt =
(DNNDataType)TF_TensorType(infer_request->output_tensors[i]);
}
-switch (tf_model->model->func_type) {
+switch (tf_model->model.func_type) {
case DFT_PROCESS_FRAME:
//it only support 1 output if it's frame in & frame out
if (task->do_ioproc) {
-if (tf_model->model->frame_post_proc != NULL) {
-tf_model->model->frame_post_proc(task->out_frame, outputs,
tf_model->model->filter_ctx);
+if (tf_model->model.frame_post_proc != NULL) {
+tf_model->model.frame_post_proc(task->out_frame, outputs,
tf_model->model.filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);
}
@@ -741,11 +734,11 @@ static void infer_completion_callback(void *args) {
}
break;
case DFT_ANALYTICS_DETECT:
-if (!tf_model->model->detect_post_proc) {
+if (!tf_model->model.detect_post_proc) {
av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post
proc\n");
return;
}
-tf_model->model->detect_post_proc(task->in_frame, outputs,
task->nb_output, tf_model->model->filter_ctx);
+tf_model->model.detect_post_proc(task->in_frame, outputs,
task->nb_output, tf_model->model.filter_ctx);
break;
default:
av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this
kind of dnn filter now\n");
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH WIP v2 5/9] avfilter/dnn_backend_tf: Fix free context at random place
From: Zhao Zhili
It will be freed again by ff_dnn_uninit.
---
libavfilter/dnn/dnn_backend_tf.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index 60f9e57fb7..3b4de6d13f 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -804,7 +804,7 @@ err:
if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
destroy_request_item();
}
-dnn_free_model_tf(_model->model);
+
return ret;
}
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH WIP v2 1/9] avfilter/dnn: Refactor DNN parameter configuration system
From: Zhao Zhili
This patch trying to resolve mulitiple issues related to parameter
configuration:
Firstly, each DNN filters duplicate DNN_COMMON_OPTIONS, which should
be the common options of backend.
Secondly, backend options are hidden behind the scene. It's a
AV_OPT_TYPE_STRING backend_configs for user, and parsed by each
backend. We don't know each backend support what kind of options
from the help message.
Third, DNN backends duplicate DNN_BACKEND_COMMON_OPTIONS.
Last but not the least, pass backend options via AV_OPT_TYPE_STRING
makes it hard to pass AV_OPT_TYPE_BINARY to backend, if not impossible.
This patch puts backend common options and each backend options inside
DnnContext to reduce code duplication, make options user friendly, and
easy to extend for future usecase.
There is a known issue that, for a filter which only support one or two
of the backends, the help message still show the option of all three
backends. Each DNN filter should be able to run on any backend. Current
issue is mostly due to incomplete implementation (e.g., libtorch only
support DFT_PROCESS_FRAME), and lack of maintenance on the filters.
For example,
./ffmpeg -h filter=dnn_processing
dnn_processing AVOptions:
dnn_backend ..FV... DNN backend (from INT_MIN to
INT_MAX) (default tensorflow)
tensorflow 1..FV... tensorflow backend flag
openvino2..FV... openvino backend flag
torch 3..FV... torch backend flag
dnn_base AVOptions:
model ..F path to model file
input ..F input name of the model
output ..F output name of the model
backend_configs..F...P backend configs (deprecated)
options..F...P backend configs (deprecated)
nireq ..F number of request (from 0 to
INT_MAX) (default 0)
async ..F use DNN async inference (default
true)
device ..F device to run model
dnn_tensorflow AVOptions:
sess_config..F config for SessionOptions
dnn_openvino AVOptions:
batch_size..F batch size per request (from 1 to
1000) (default 1)
input_resizable ..F can input be resizable or not
(default false)
layout..F input layout of model (from 0 to
2) (default none)
none0..F none
nchw1..F nchw
nhwc2..F nhwc
scale ..F Add scale preprocess operation.
Divide each element of input by specified value. (from INT_MIN to INT_MAX)
(default 0)
mean..F Add mean preprocess operation.
Subtract specified value from each element of input. (from INT_MIN to INT_MAX)
(default 0)
dnn_th AVOptions:
optimize ..F turn on graph executor
optimization (from 0 to 1) (default 0)
---
libavfilter/dnn/dnn_backend_common.h | 13 ++-
libavfilter/dnn/dnn_backend_openvino.c | 146 ++---
libavfilter/dnn/dnn_backend_tf.c | 82 +-
libavfilter/dnn/dnn_backend_torch.cpp | 67
libavfilter/dnn/dnn_interface.c| 81 ++
libavfilter/dnn_filter_common.c| 32 +-
libavfilter/dnn_filter_common.h| 37 +++
libavfilter/dnn_interface.h| 66 ++-
libavfilter/vf_derain.c| 5 +-
libavfilter/vf_dnn_classify.c | 3 +-
libavfilter/vf_dnn_detect.c| 3 +-
libavfilter/vf_dnn_processing.c| 3 +-
libavfilter/vf_sr.c| 5 +-
13 files changed, 314 insertions(+), 229 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_common.h
b/libavfilter/dnn/dnn_backend_common.h
index 42c67c7040..9f5d37b3e0 100644
--- a/libavfilter/dnn/dnn_backend_common.h
+++ b/libavfilter/dnn/dnn_backend_common.h
@@ -28,9 +28,16 @@
#include "../dnn_interface.h"
#include "libavutil/thread.h"
-#define DNN_BACKEND_COMMON_OPTIONS \
-{ "nireq", "number of request",
OFFSET(options.nireq), AV_OPT_TYPE_INT,{ .i64 = 0 }, 0,
INT_MAX, FLAGS }, \
-{ "async", "use DNN async inference",
OFFSET(options.async), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0,
1, FLAGS },
+#define DNN_DEFINE_CLASS_EXT(name, desc, options) \
+{ \
+.class_name = desc, \
+.item_name = av_default_item_name, \
+.option = options, \
+.version= LIBAVUTIL_VERSION_INT,\
+.category = AV_CLASS_CATEGORY_FILTER, \
+}
+#define DNN_DEFINE_CLASS(fname) \
+DNN_DEFINE_CLASS_EXT(fname, #fname, fn
[FFmpeg-devel] [PATCH WIP v2 2/9] avfilter/dnn_backend_openvino: Fix free context at random place
From: Zhao Zhili It will be freed again by ff_dnn_uninit. --- libavfilter/dnn/dnn_backend_openvino.c | 1 - 1 file changed, 1 deletion(-) diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c index c4b0682f11..769ba0a54b 100644 --- a/libavfilter/dnn/dnn_backend_openvino.c +++ b/libavfilter/dnn/dnn_backend_openvino.c @@ -959,7 +959,6 @@ err: if (input_model_info) ov_preprocess_input_model_info_free(input_model_info); #endif -dnn_free_model_ov(_model->model); return ret; } -- 2.25.1 ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH WIP v2 9/9] avfilter/dnn: Use dnn_backend_info_list to search for dnn module
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_openvino.c | 1 +
libavfilter/dnn/dnn_backend_tf.c | 1 +
libavfilter/dnn/dnn_backend_torch.cpp | 1 +
libavfilter/dnn/dnn_interface.c| 26 --
libavfilter/dnn_interface.h| 1 +
5 files changed, 12 insertions(+), 18 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c
b/libavfilter/dnn/dnn_backend_openvino.c
index d8a6820dc2..9c699cdc8c 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -1613,6 +1613,7 @@ static int dnn_flush_ov(const DNNModel *model)
const DNNModule ff_dnn_backend_openvino = {
.clazz = DNN_DEFINE_CLASS(dnn_openvino),
+.type = DNN_OV,
.load_model = dnn_load_model_ov,
.execute_model = dnn_execute_model_ov,
.get_result = dnn_get_result_ov,
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index 06ea6cbb8c..6afefe8115 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -886,6 +886,7 @@ static int dnn_flush_tf(const DNNModel *model)
const DNNModule ff_dnn_backend_tf = {
.clazz = DNN_DEFINE_CLASS(dnn_tensorflow),
+.type = DNN_TF,
.load_model = dnn_load_model_tf,
.execute_model = dnn_execute_model_tf,
.get_result = dnn_get_result_tf,
diff --git a/libavfilter/dnn/dnn_backend_torch.cpp
b/libavfilter/dnn/dnn_backend_torch.cpp
index 24e9f2c8e2..2557264713 100644
--- a/libavfilter/dnn/dnn_backend_torch.cpp
+++ b/libavfilter/dnn/dnn_backend_torch.cpp
@@ -561,6 +561,7 @@ static int dnn_flush_th(const DNNModel *model)
extern const DNNModule ff_dnn_backend_torch = {
.clazz = DNN_DEFINE_CLASS(dnn_th),
+.type = DNN_TH,
.load_model = dnn_load_model_th,
.execute_model = dnn_execute_model_th,
.get_result = dnn_get_result_th,
diff --git a/libavfilter/dnn/dnn_interface.c b/libavfilter/dnn/dnn_interface.c
index ebd308cd84..cce3c45856 100644
--- a/libavfilter/dnn/dnn_interface.c
+++ b/libavfilter/dnn/dnn_interface.c
@@ -80,25 +80,15 @@ static const DnnBackendInfo dnn_backend_info_list[] = {
const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx)
{
-switch(backend_type){
-#if (CONFIG_LIBTENSORFLOW == 1)
-case DNN_TF:
-return _dnn_backend_tf;
-#endif
-#if (CONFIG_LIBOPENVINO == 1)
-case DNN_OV:
-return _dnn_backend_openvino;
-#endif
-#if (CONFIG_LIBTORCH == 1)
-case DNN_TH:
-return _dnn_backend_torch;
-#endif
-default:
-av_log(log_ctx, AV_LOG_ERROR,
-"Module backend_type %d is not supported or enabled.\n",
-backend_type);
-return NULL;
+for (int i = 1; i < FF_ARRAY_ELEMS(dnn_backend_info_list); i++) {
+if (dnn_backend_info_list[i].module->type == backend_type)
+return dnn_backend_info_list[i].module;
}
+
+av_log(log_ctx, AV_LOG_ERROR,
+"Module backend_type %d is not supported or enabled.\n",
+backend_type);
+return NULL;
}
void *ff_dnn_child_next(DnnContext *obj, void *prev) {
diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h
index 4e14a42d00..4b25ac2b84 100644
--- a/libavfilter/dnn_interface.h
+++ b/libavfilter/dnn_interface.h
@@ -170,6 +170,7 @@ typedef struct DnnContext {
// Stores pointers to functions for loading, executing, freeing DNN models for
one of the backends.
struct DNNModule {
const AVClass clazz;
+DNNBackendType type;
// Loads model and parameters from given file. Returns NULL if it is not
possible.
DNNModel *(*load_model)(DnnContext *ctx, DNNFunctionType func_type,
AVFilterContext *filter_ctx);
// Executes model with specified input and output. Returns the error code
otherwise.
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH WIP v2 4/9] avfilter/dnn_backend_tf: Remove one level of indentation
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_tf.c | 63
1 file changed, 32 insertions(+), 31 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index d24591b90b..60f9e57fb7 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -483,41 +483,42 @@ static void dnn_free_model_tf(DNNModel **model)
{
TFModel *tf_model;
-if (*model){
-tf_model = (*model)->model;
-while (ff_safe_queue_size(tf_model->request_queue) != 0) {
-TFRequestItem *item =
ff_safe_queue_pop_front(tf_model->request_queue);
-destroy_request_item();
-}
-ff_safe_queue_destroy(tf_model->request_queue);
+if (!model || !*model)
+return;
-while (ff_queue_size(tf_model->lltask_queue) != 0) {
-LastLevelTaskItem *item =
ff_queue_pop_front(tf_model->lltask_queue);
-av_freep();
-}
-ff_queue_destroy(tf_model->lltask_queue);
+tf_model = (*model)->model;
+while (ff_safe_queue_size(tf_model->request_queue) != 0) {
+TFRequestItem *item = ff_safe_queue_pop_front(tf_model->request_queue);
+destroy_request_item();
+}
+ff_safe_queue_destroy(tf_model->request_queue);
-while (ff_queue_size(tf_model->task_queue) != 0) {
-TaskItem *item = ff_queue_pop_front(tf_model->task_queue);
-av_frame_free(>in_frame);
-av_frame_free(>out_frame);
-av_freep();
-}
-ff_queue_destroy(tf_model->task_queue);
+while (ff_queue_size(tf_model->lltask_queue) != 0) {
+LastLevelTaskItem *item = ff_queue_pop_front(tf_model->lltask_queue);
+av_freep();
+}
+ff_queue_destroy(tf_model->lltask_queue);
-if (tf_model->graph){
-TF_DeleteGraph(tf_model->graph);
-}
-if (tf_model->session){
-TF_CloseSession(tf_model->session, tf_model->status);
-TF_DeleteSession(tf_model->session, tf_model->status);
-}
-if (tf_model->status){
-TF_DeleteStatus(tf_model->status);
-}
-av_freep(_model);
-av_freep();
+while (ff_queue_size(tf_model->task_queue) != 0) {
+TaskItem *item = ff_queue_pop_front(tf_model->task_queue);
+av_frame_free(>in_frame);
+av_frame_free(>out_frame);
+av_freep();
+}
+ff_queue_destroy(tf_model->task_queue);
+
+if (tf_model->graph){
+TF_DeleteGraph(tf_model->graph);
+}
+if (tf_model->session){
+TF_CloseSession(tf_model->session, tf_model->status);
+TF_DeleteSession(tf_model->session, tf_model->status);
+}
+if (tf_model->status){
+TF_DeleteStatus(tf_model->status);
}
+av_freep(_model);
+av_freep();
}
static DNNModel *dnn_load_model_tf(DnnContext *ctx, DNNFunctionType func_type,
AVFilterContext *filter_ctx)
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH WIP v2 3/9] avfilter/dnn_backend_openvino: simplify memory allocation
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_openvino.c | 47 +++---
1 file changed, 20 insertions(+), 27 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c
b/libavfilter/dnn/dnn_backend_openvino.c
index 769ba0a54b..1acc54b791 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -41,8 +41,8 @@
#include "dnn_backend_common.h"
typedef struct OVModel{
+DNNModel model;
DnnContext *ctx;
-DNNModel *model;
#if HAVE_OPENVINO2
ov_core_t *core;
ov_model_t *ov_model;
@@ -300,11 +300,11 @@ static int fill_model_input_ov(OVModel *ov_model,
OVRequestItem *request)
return ov2_map_error(status, NULL);
}
#endif
-switch (ov_model->model->func_type) {
+switch (ov_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
-if (ov_model->model->frame_pre_proc != NULL) {
-ov_model->model->frame_pre_proc(task->in_frame, ,
ov_model->model->filter_ctx);
+if (ov_model->model.frame_pre_proc != NULL) {
+ov_model->model.frame_pre_proc(task->in_frame, ,
ov_model->model.filter_ctx);
} else {
ff_proc_from_frame_to_dnn(task->in_frame, , ctx);
}
@@ -442,11 +442,11 @@ static void infer_completion_callback(void *args)
for (int i = 0; i < request->lltask_count; ++i) {
task = request->lltasks[i]->task;
-switch (ov_model->model->func_type) {
+switch (ov_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
-if (ov_model->model->frame_post_proc != NULL) {
-ov_model->model->frame_post_proc(task->out_frame, outputs,
ov_model->model->filter_ctx);
+if (ov_model->model.frame_post_proc != NULL) {
+ov_model->model.frame_post_proc(task->out_frame, outputs,
ov_model->model.filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);
}
@@ -458,23 +458,23 @@ static void infer_completion_callback(void *args)
}
break;
case DFT_ANALYTICS_DETECT:
-if (!ov_model->model->detect_post_proc) {
+if (!ov_model->model.detect_post_proc) {
av_log(ctx, AV_LOG_ERROR, "detect filter needs to provide post
proc\n");
goto end;
}
-ov_model->model->detect_post_proc(task->in_frame, outputs,
+ov_model->model.detect_post_proc(task->in_frame, outputs,
ov_model->nb_outputs,
- ov_model->model->filter_ctx);
+ ov_model->model.filter_ctx);
break;
case DFT_ANALYTICS_CLASSIFY:
-if (!ov_model->model->classify_post_proc) {
+if (!ov_model->model.classify_post_proc) {
av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide
post proc\n");
goto end;
}
for (int output_i = 0; output_i < ov_model->nb_outputs; output_i++)
-ov_model->model->classify_post_proc(task->in_frame, outputs,
+ov_model->model.classify_post_proc(task->in_frame, outputs,
request->lltasks[i]->bbox_index,
-
ov_model->model->filter_ctx);
+
ov_model->model.filter_ctx);
break;
default:
av_assert0(!"should not reach here");
@@ -571,7 +571,7 @@ static void dnn_free_model_ov(DNNModel **model)
av_free(ov_model->all_input_names);
#endif
av_freep(_model);
-av_freep(model);
+*model = NULL;
}
@@ -598,7 +598,7 @@ static int init_model_ov(OVModel *ov_model, const char
*input_name, const char *
#endif
// We scale pixel by default when do frame processing.
if (fabsf(ctx->ov_option.scale) < 1e-6f)
-ctx->ov_option.scale = ov_model->model->func_type == DFT_PROCESS_FRAME
? 255 : 1;
+ctx->ov_option.scale = ov_model->model.func_type == DFT_PROCESS_FRAME
? 255 : 1;
// batch size
if (ctx->ov_option.batch_size <= 0) {
ctx->ov_option.batch_size = 1;
@@ -702,7 +702,7 @@ static int init_model_ov(OVModel *ov_model, const char
*input_name, const char *
ret = ov2_map_error(status, NULL);
goto err;
}
-if (ov_model->model->func_type != DFT_PROCESS_FRAME
[FFmpeg-devel] [PATCH WIP v2 7/9] avfilter/dnn_backend_torch: Simplify memory allocation
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_torch.cpp | 31 +++
1 file changed, 12 insertions(+), 19 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_torch.cpp
b/libavfilter/dnn/dnn_backend_torch.cpp
index abdef1f178..818ec5b713 100644
--- a/libavfilter/dnn/dnn_backend_torch.cpp
+++ b/libavfilter/dnn/dnn_backend_torch.cpp
@@ -37,8 +37,8 @@ extern "C" {
}
typedef struct THModel {
+DNNModel model;
DnnContext *ctx;
-DNNModel *model;
torch::jit::Module *jit_model;
SafeQueue *request_queue;
Queue *task_queue;
@@ -141,7 +141,7 @@ static void dnn_free_model_th(DNNModel **model)
ff_queue_destroy(th_model->task_queue);
delete th_model->jit_model;
av_freep(_model);
-av_freep(model);
+*model = NULL;
}
static int get_input_th(void *model, DNNData *input, const char *input_name)
@@ -195,19 +195,19 @@ static int fill_model_input_th(THModel *th_model,
THRequestItem *request)
infer_request->input_tensor = new torch::Tensor();
infer_request->output = new torch::Tensor();
-switch (th_model->model->func_type) {
+switch (th_model->model.func_type) {
case DFT_PROCESS_FRAME:
input.scale = 255;
if (task->do_ioproc) {
-if (th_model->model->frame_pre_proc != NULL) {
-th_model->model->frame_pre_proc(task->in_frame, ,
th_model->model->filter_ctx);
+if (th_model->model.frame_pre_proc != NULL) {
+th_model->model.frame_pre_proc(task->in_frame, ,
th_model->model.filter_ctx);
} else {
ff_proc_from_frame_to_dnn(task->in_frame, , ctx);
}
}
break;
default:
-avpriv_report_missing_feature(NULL, "model function type %d",
th_model->model->func_type);
+avpriv_report_missing_feature(NULL, "model function type %d",
th_model->model.func_type);
break;
}
*infer_request->input_tensor = torch::from_blob(input.data,
@@ -282,13 +282,13 @@ static void infer_completion_callback(void *args) {
goto err;
}
-switch (th_model->model->func_type) {
+switch (th_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
outputs.scale = 255;
outputs.data = output->data_ptr();
-if (th_model->model->frame_post_proc != NULL) {
-th_model->model->frame_post_proc(task->out_frame, ,
th_model->model->filter_ctx);
+if (th_model->model.frame_post_proc != NULL) {
+th_model->model.frame_post_proc(task->out_frame, ,
th_model->model.filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, ,
th_model->ctx);
}
@@ -298,7 +298,7 @@ static void infer_completion_callback(void *args) {
}
break;
default:
-avpriv_report_missing_feature(th_model->ctx, "model function type %d",
th_model->model->func_type);
+avpriv_report_missing_feature(th_model->ctx, "model function type %d",
th_model->model.func_type);
goto err;
}
task->inference_done++;
@@ -417,17 +417,10 @@ static DNNModel *dnn_load_model_th(DnnContext *ctx,
DNNFunctionType func_type, A
THRequestItem *item = NULL;
const char *device_name = ctx->device ? ctx->device : "cpu";
-model = (DNNModel *)av_mallocz(sizeof(DNNModel));
-if (!model) {
-return NULL;
-}
-
th_model = (THModel *)av_mallocz(sizeof(THModel));
-if (!th_model) {
-av_freep();
+if (!th_model)
return NULL;
-}
-th_model->model = model;
+model = _model->model;
model->model = th_model;
th_model->ctx = ctx;
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH WIP v2 8/9] avfilter/dnn: Remove a level of dereference
From: Zhao Zhili
For code such as 'model->model = ov_model' is confusing. We can
just drop the member variable and use cast to get the subclass.
---
libavfilter/dnn/dnn_backend_openvino.c | 17 -
libavfilter/dnn/dnn_backend_tf.c | 19 +--
libavfilter/dnn/dnn_backend_torch.cpp | 15 +++
libavfilter/dnn_filter_common.c| 6 +++---
libavfilter/dnn_interface.h| 6 ++
5 files changed, 29 insertions(+), 34 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c
b/libavfilter/dnn/dnn_backend_openvino.c
index 1acc54b791..d8a6820dc2 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -517,7 +517,7 @@ static void dnn_free_model_ov(DNNModel **model)
if (!model || !*model)
return;
-ov_model = (*model)->model;
+ov_model = (OVModel *)(*model);
while (ff_safe_queue_size(ov_model->request_queue) != 0) {
OVRequestItem *item = ff_safe_queue_pop_front(ov_model->request_queue);
if (item && item->infer_request) {
@@ -1059,9 +1059,9 @@ err:
return ret;
}
-static int get_input_ov(void *model, DNNData *input, const char *input_name)
+static int get_input_ov(DNNModel *model, DNNData *input, const char
*input_name)
{
-OVModel *ov_model = model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
int input_resizable = ctx->ov_option.input_resizable;
@@ -1255,7 +1255,7 @@ static int extract_lltask_from_task(DNNFunctionType
func_type, TaskItem *task, Q
}
}
-static int get_output_ov(void *model, const char *input_name, int input_width,
int input_height,
+static int get_output_ov(DNNModel *model, const char *input_name, int
input_width, int input_height,
const char *output_name, int *output_width,
int *output_height)
{
#if HAVE_OPENVINO2
@@ -1268,7 +1268,7 @@ static int get_output_ov(void *model, const char
*input_name, int input_width, i
input_shapes_t input_shapes;
#endif
int ret;
-OVModel *ov_model = model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
TaskItem task;
OVRequestItem *request;
@@ -1383,7 +1383,6 @@ static DNNModel *dnn_load_model_ov(DnnContext *ctx,
DNNFunctionType func_type, A
return NULL;
ov_model->ctx = ctx;
model = _model->model;
-model->model = ov_model;
#if HAVE_OPENVINO2
status = ov_core_create();
@@ -1470,7 +1469,7 @@ err:
static int dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams
*exec_params)
{
-OVModel *ov_model = model->model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
OVRequestItem *request;
TaskItem *task;
@@ -1558,13 +1557,13 @@ static int dnn_execute_model_ov(const DNNModel *model,
DNNExecBaseParams *exec_p
static DNNAsyncStatusType dnn_get_result_ov(const DNNModel *model, AVFrame
**in, AVFrame **out)
{
-OVModel *ov_model = model->model;
+OVModel *ov_model = (OVModel *)model;
return ff_dnn_get_result_common(ov_model->task_queue, in, out);
}
static int dnn_flush_ov(const DNNModel *model)
{
-OVModel *ov_model = model->model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
OVRequestItem *request;
#if HAVE_OPENVINO2
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index c7716e696d..06ea6cbb8c 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -262,9 +262,9 @@ static TF_Tensor *allocate_input_tensor(const DNNData
*input)
input_dims[1] * input_dims[2] * input_dims[3] *
size);
}
-static int get_input_tf(void *model, DNNData *input, const char *input_name)
+static int get_input_tf(DNNModel *model, DNNData *input, const char
*input_name)
{
-TFModel *tf_model = model;
+TFModel *tf_model = (TFModel *)model;
DnnContext *ctx = tf_model->ctx;
TF_Status *status;
TF_DataType dt;
@@ -310,11 +310,11 @@ static int get_input_tf(void *model, DNNData *input,
const char *input_name)
return 0;
}
-static int get_output_tf(void *model, const char *input_name, int input_width,
int input_height,
+static int get_output_tf(DNNModel *model, const char *input_name, int
input_width, int input_height,
const char *output_name, int *output_width,
int *output_height)
{
int ret;
-TFModel *tf_model = model;
+TFModel *tf_model = (TFModel *)model;
DnnContext *ctx = tf_model->ctx;
TaskItem task;
TFRequestItem *request;
@@ -486,7 +486,7 @@ static void dnn_free_model_tf(DNNModel **model)
if (!model || !*model)
return;
-tf_model = (*model)->model;
+tf_model = (TFModel *)(*model);
while (ff_safe_queue_size(tf_model->request_queue)
[FFmpeg-devel] [PATCH WIP 9/9] avfilter/dnn: Use dnn_backend_info_list to search for dnn module
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_openvino.c | 1 +
libavfilter/dnn/dnn_backend_tf.c | 1 +
libavfilter/dnn/dnn_backend_torch.cpp | 1 +
libavfilter/dnn/dnn_interface.c| 26 --
libavfilter/dnn_interface.h| 1 +
5 files changed, 12 insertions(+), 18 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c
b/libavfilter/dnn/dnn_backend_openvino.c
index d8a6820dc2..9c699cdc8c 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -1613,6 +1613,7 @@ static int dnn_flush_ov(const DNNModel *model)
const DNNModule ff_dnn_backend_openvino = {
.clazz = DNN_DEFINE_CLASS(dnn_openvino),
+.type = DNN_OV,
.load_model = dnn_load_model_ov,
.execute_model = dnn_execute_model_ov,
.get_result = dnn_get_result_ov,
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index 06ea6cbb8c..6afefe8115 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -886,6 +886,7 @@ static int dnn_flush_tf(const DNNModel *model)
const DNNModule ff_dnn_backend_tf = {
.clazz = DNN_DEFINE_CLASS(dnn_tensorflow),
+.type = DNN_TF,
.load_model = dnn_load_model_tf,
.execute_model = dnn_execute_model_tf,
.get_result = dnn_get_result_tf,
diff --git a/libavfilter/dnn/dnn_backend_torch.cpp
b/libavfilter/dnn/dnn_backend_torch.cpp
index 24e9f2c8e2..2557264713 100644
--- a/libavfilter/dnn/dnn_backend_torch.cpp
+++ b/libavfilter/dnn/dnn_backend_torch.cpp
@@ -561,6 +561,7 @@ static int dnn_flush_th(const DNNModel *model)
extern const DNNModule ff_dnn_backend_torch = {
.clazz = DNN_DEFINE_CLASS(dnn_th),
+.type = DNN_TH,
.load_model = dnn_load_model_th,
.execute_model = dnn_execute_model_th,
.get_result = dnn_get_result_th,
diff --git a/libavfilter/dnn/dnn_interface.c b/libavfilter/dnn/dnn_interface.c
index 245fe5f759..76fe364a47 100644
--- a/libavfilter/dnn/dnn_interface.c
+++ b/libavfilter/dnn/dnn_interface.c
@@ -80,25 +80,15 @@ static const DnnBackendInfo dnn_backend_info_list[] = {
const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx)
{
-switch(backend_type){
-#if (CONFIG_LIBTENSORFLOW == 1)
-case DNN_TF:
-return _dnn_backend_tf;
-#endif
-#if (CONFIG_LIBOPENVINO == 1)
-case DNN_OV:
-return _dnn_backend_openvino;
-#endif
-#if (CONFIG_LIBTORCH == 1)
-case DNN_TH:
-return _dnn_backend_torch;
-#endif
-default:
-av_log(log_ctx, AV_LOG_ERROR,
-"Module backend_type %d is not supported or enabled.\n",
-backend_type);
-return NULL;
+for (int i = 1; i < FF_ARRAY_ELEMS(dnn_backend_info_list); i++) {
+if (dnn_backend_info_list[i].module->type == backend_type)
+return dnn_backend_info_list[i].module;
}
+
+av_log(log_ctx, AV_LOG_ERROR,
+"Module backend_type %d is not supported or enabled.\n",
+backend_type);
+return NULL;
}
void *ff_dnn_child_next(DnnContext *obj, void *prev) {
diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h
index 2b35c9c9fb..3fe88d4029 100644
--- a/libavfilter/dnn_interface.h
+++ b/libavfilter/dnn_interface.h
@@ -170,6 +170,7 @@ typedef struct DnnContext {
// Stores pointers to functions for loading, executing, freeing DNN models for
one of the backends.
struct DNNModule {
const AVClass clazz;
+DNNBackendType type;
// Loads model and parameters from given file. Returns NULL if it is not
possible.
DNNModel *(*load_model)(DnnContext *ctx, DNNFunctionType func_type,
AVFilterContext *filter_ctx);
// Executes model with specified input and output. Returns the error code
otherwise.
--
2.34.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH WIP 7/9] avfilter/dnn_backend_torch: Simplify memory allocation
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_torch.cpp | 31 +++
1 file changed, 12 insertions(+), 19 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_torch.cpp
b/libavfilter/dnn/dnn_backend_torch.cpp
index abdef1f178..818ec5b713 100644
--- a/libavfilter/dnn/dnn_backend_torch.cpp
+++ b/libavfilter/dnn/dnn_backend_torch.cpp
@@ -37,8 +37,8 @@ extern "C" {
}
typedef struct THModel {
+DNNModel model;
DnnContext *ctx;
-DNNModel *model;
torch::jit::Module *jit_model;
SafeQueue *request_queue;
Queue *task_queue;
@@ -141,7 +141,7 @@ static void dnn_free_model_th(DNNModel **model)
ff_queue_destroy(th_model->task_queue);
delete th_model->jit_model;
av_freep(_model);
-av_freep(model);
+*model = NULL;
}
static int get_input_th(void *model, DNNData *input, const char *input_name)
@@ -195,19 +195,19 @@ static int fill_model_input_th(THModel *th_model,
THRequestItem *request)
infer_request->input_tensor = new torch::Tensor();
infer_request->output = new torch::Tensor();
-switch (th_model->model->func_type) {
+switch (th_model->model.func_type) {
case DFT_PROCESS_FRAME:
input.scale = 255;
if (task->do_ioproc) {
-if (th_model->model->frame_pre_proc != NULL) {
-th_model->model->frame_pre_proc(task->in_frame, ,
th_model->model->filter_ctx);
+if (th_model->model.frame_pre_proc != NULL) {
+th_model->model.frame_pre_proc(task->in_frame, ,
th_model->model.filter_ctx);
} else {
ff_proc_from_frame_to_dnn(task->in_frame, , ctx);
}
}
break;
default:
-avpriv_report_missing_feature(NULL, "model function type %d",
th_model->model->func_type);
+avpriv_report_missing_feature(NULL, "model function type %d",
th_model->model.func_type);
break;
}
*infer_request->input_tensor = torch::from_blob(input.data,
@@ -282,13 +282,13 @@ static void infer_completion_callback(void *args) {
goto err;
}
-switch (th_model->model->func_type) {
+switch (th_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
outputs.scale = 255;
outputs.data = output->data_ptr();
-if (th_model->model->frame_post_proc != NULL) {
-th_model->model->frame_post_proc(task->out_frame, ,
th_model->model->filter_ctx);
+if (th_model->model.frame_post_proc != NULL) {
+th_model->model.frame_post_proc(task->out_frame, ,
th_model->model.filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, ,
th_model->ctx);
}
@@ -298,7 +298,7 @@ static void infer_completion_callback(void *args) {
}
break;
default:
-avpriv_report_missing_feature(th_model->ctx, "model function type %d",
th_model->model->func_type);
+avpriv_report_missing_feature(th_model->ctx, "model function type %d",
th_model->model.func_type);
goto err;
}
task->inference_done++;
@@ -417,17 +417,10 @@ static DNNModel *dnn_load_model_th(DnnContext *ctx,
DNNFunctionType func_type, A
THRequestItem *item = NULL;
const char *device_name = ctx->device ? ctx->device : "cpu";
-model = (DNNModel *)av_mallocz(sizeof(DNNModel));
-if (!model) {
-return NULL;
-}
-
th_model = (THModel *)av_mallocz(sizeof(THModel));
-if (!th_model) {
-av_freep();
+if (!th_model)
return NULL;
-}
-th_model->model = model;
+model = _model->model;
model->model = th_model;
th_model->ctx = ctx;
--
2.34.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH WIP 8/9] avfilter/dnn: Remove a level of dereference
From: Zhao Zhili
For code such as 'model->model = ov_model' is confusing. We can
just drop the member variable and use cast to get the subclass.
---
libavfilter/dnn/dnn_backend_openvino.c | 17 -
libavfilter/dnn/dnn_backend_tf.c | 19 +--
libavfilter/dnn/dnn_backend_torch.cpp | 15 +++
libavfilter/dnn_filter_common.c| 6 +++---
libavfilter/dnn_interface.h| 6 ++
5 files changed, 29 insertions(+), 34 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c
b/libavfilter/dnn/dnn_backend_openvino.c
index 1acc54b791..d8a6820dc2 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -517,7 +517,7 @@ static void dnn_free_model_ov(DNNModel **model)
if (!model || !*model)
return;
-ov_model = (*model)->model;
+ov_model = (OVModel *)(*model);
while (ff_safe_queue_size(ov_model->request_queue) != 0) {
OVRequestItem *item = ff_safe_queue_pop_front(ov_model->request_queue);
if (item && item->infer_request) {
@@ -1059,9 +1059,9 @@ err:
return ret;
}
-static int get_input_ov(void *model, DNNData *input, const char *input_name)
+static int get_input_ov(DNNModel *model, DNNData *input, const char
*input_name)
{
-OVModel *ov_model = model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
int input_resizable = ctx->ov_option.input_resizable;
@@ -1255,7 +1255,7 @@ static int extract_lltask_from_task(DNNFunctionType
func_type, TaskItem *task, Q
}
}
-static int get_output_ov(void *model, const char *input_name, int input_width,
int input_height,
+static int get_output_ov(DNNModel *model, const char *input_name, int
input_width, int input_height,
const char *output_name, int *output_width,
int *output_height)
{
#if HAVE_OPENVINO2
@@ -1268,7 +1268,7 @@ static int get_output_ov(void *model, const char
*input_name, int input_width, i
input_shapes_t input_shapes;
#endif
int ret;
-OVModel *ov_model = model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
TaskItem task;
OVRequestItem *request;
@@ -1383,7 +1383,6 @@ static DNNModel *dnn_load_model_ov(DnnContext *ctx,
DNNFunctionType func_type, A
return NULL;
ov_model->ctx = ctx;
model = _model->model;
-model->model = ov_model;
#if HAVE_OPENVINO2
status = ov_core_create();
@@ -1470,7 +1469,7 @@ err:
static int dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams
*exec_params)
{
-OVModel *ov_model = model->model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
OVRequestItem *request;
TaskItem *task;
@@ -1558,13 +1557,13 @@ static int dnn_execute_model_ov(const DNNModel *model,
DNNExecBaseParams *exec_p
static DNNAsyncStatusType dnn_get_result_ov(const DNNModel *model, AVFrame
**in, AVFrame **out)
{
-OVModel *ov_model = model->model;
+OVModel *ov_model = (OVModel *)model;
return ff_dnn_get_result_common(ov_model->task_queue, in, out);
}
static int dnn_flush_ov(const DNNModel *model)
{
-OVModel *ov_model = model->model;
+OVModel *ov_model = (OVModel *)model;
DnnContext *ctx = ov_model->ctx;
OVRequestItem *request;
#if HAVE_OPENVINO2
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index c7716e696d..06ea6cbb8c 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -262,9 +262,9 @@ static TF_Tensor *allocate_input_tensor(const DNNData
*input)
input_dims[1] * input_dims[2] * input_dims[3] *
size);
}
-static int get_input_tf(void *model, DNNData *input, const char *input_name)
+static int get_input_tf(DNNModel *model, DNNData *input, const char
*input_name)
{
-TFModel *tf_model = model;
+TFModel *tf_model = (TFModel *)model;
DnnContext *ctx = tf_model->ctx;
TF_Status *status;
TF_DataType dt;
@@ -310,11 +310,11 @@ static int get_input_tf(void *model, DNNData *input,
const char *input_name)
return 0;
}
-static int get_output_tf(void *model, const char *input_name, int input_width,
int input_height,
+static int get_output_tf(DNNModel *model, const char *input_name, int
input_width, int input_height,
const char *output_name, int *output_width,
int *output_height)
{
int ret;
-TFModel *tf_model = model;
+TFModel *tf_model = (TFModel *)model;
DnnContext *ctx = tf_model->ctx;
TaskItem task;
TFRequestItem *request;
@@ -486,7 +486,7 @@ static void dnn_free_model_tf(DNNModel **model)
if (!model || !*model)
return;
-tf_model = (*model)->model;
+tf_model = (TFModel *)(*model);
while (ff_safe_queue_size(tf_model->request_queue)
[FFmpeg-devel] [PATCH WIP 4/9] avfilter/dnn_backend_tf: Remove one level of indentation
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_tf.c | 63
1 file changed, 32 insertions(+), 31 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index d24591b90b..60f9e57fb7 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -483,41 +483,42 @@ static void dnn_free_model_tf(DNNModel **model)
{
TFModel *tf_model;
-if (*model){
-tf_model = (*model)->model;
-while (ff_safe_queue_size(tf_model->request_queue) != 0) {
-TFRequestItem *item =
ff_safe_queue_pop_front(tf_model->request_queue);
-destroy_request_item();
-}
-ff_safe_queue_destroy(tf_model->request_queue);
+if (!model || !*model)
+return;
-while (ff_queue_size(tf_model->lltask_queue) != 0) {
-LastLevelTaskItem *item =
ff_queue_pop_front(tf_model->lltask_queue);
-av_freep();
-}
-ff_queue_destroy(tf_model->lltask_queue);
+tf_model = (*model)->model;
+while (ff_safe_queue_size(tf_model->request_queue) != 0) {
+TFRequestItem *item = ff_safe_queue_pop_front(tf_model->request_queue);
+destroy_request_item();
+}
+ff_safe_queue_destroy(tf_model->request_queue);
-while (ff_queue_size(tf_model->task_queue) != 0) {
-TaskItem *item = ff_queue_pop_front(tf_model->task_queue);
-av_frame_free(>in_frame);
-av_frame_free(>out_frame);
-av_freep();
-}
-ff_queue_destroy(tf_model->task_queue);
+while (ff_queue_size(tf_model->lltask_queue) != 0) {
+LastLevelTaskItem *item = ff_queue_pop_front(tf_model->lltask_queue);
+av_freep();
+}
+ff_queue_destroy(tf_model->lltask_queue);
-if (tf_model->graph){
-TF_DeleteGraph(tf_model->graph);
-}
-if (tf_model->session){
-TF_CloseSession(tf_model->session, tf_model->status);
-TF_DeleteSession(tf_model->session, tf_model->status);
-}
-if (tf_model->status){
-TF_DeleteStatus(tf_model->status);
-}
-av_freep(_model);
-av_freep();
+while (ff_queue_size(tf_model->task_queue) != 0) {
+TaskItem *item = ff_queue_pop_front(tf_model->task_queue);
+av_frame_free(>in_frame);
+av_frame_free(>out_frame);
+av_freep();
+}
+ff_queue_destroy(tf_model->task_queue);
+
+if (tf_model->graph){
+TF_DeleteGraph(tf_model->graph);
+}
+if (tf_model->session){
+TF_CloseSession(tf_model->session, tf_model->status);
+TF_DeleteSession(tf_model->session, tf_model->status);
+}
+if (tf_model->status){
+TF_DeleteStatus(tf_model->status);
}
+av_freep(_model);
+av_freep();
}
static DNNModel *dnn_load_model_tf(DnnContext *ctx, DNNFunctionType func_type,
AVFilterContext *filter_ctx)
--
2.34.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH WIP 5/9] avfilter/dnn_backend_tf: Fix free context at random place
From: Zhao Zhili
It will be freed again by ff_dnn_uninit.
---
libavfilter/dnn/dnn_backend_tf.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index 60f9e57fb7..3b4de6d13f 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -804,7 +804,7 @@ err:
if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
destroy_request_item();
}
-dnn_free_model_tf(_model->model);
+
return ret;
}
--
2.34.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH WIP 3/9] avfilter/dnn_backend_openvino: simplify memory allocation
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_openvino.c | 47 +++---
1 file changed, 20 insertions(+), 27 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c
b/libavfilter/dnn/dnn_backend_openvino.c
index 769ba0a54b..1acc54b791 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -41,8 +41,8 @@
#include "dnn_backend_common.h"
typedef struct OVModel{
+DNNModel model;
DnnContext *ctx;
-DNNModel *model;
#if HAVE_OPENVINO2
ov_core_t *core;
ov_model_t *ov_model;
@@ -300,11 +300,11 @@ static int fill_model_input_ov(OVModel *ov_model,
OVRequestItem *request)
return ov2_map_error(status, NULL);
}
#endif
-switch (ov_model->model->func_type) {
+switch (ov_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
-if (ov_model->model->frame_pre_proc != NULL) {
-ov_model->model->frame_pre_proc(task->in_frame, ,
ov_model->model->filter_ctx);
+if (ov_model->model.frame_pre_proc != NULL) {
+ov_model->model.frame_pre_proc(task->in_frame, ,
ov_model->model.filter_ctx);
} else {
ff_proc_from_frame_to_dnn(task->in_frame, , ctx);
}
@@ -442,11 +442,11 @@ static void infer_completion_callback(void *args)
for (int i = 0; i < request->lltask_count; ++i) {
task = request->lltasks[i]->task;
-switch (ov_model->model->func_type) {
+switch (ov_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
-if (ov_model->model->frame_post_proc != NULL) {
-ov_model->model->frame_post_proc(task->out_frame, outputs,
ov_model->model->filter_ctx);
+if (ov_model->model.frame_post_proc != NULL) {
+ov_model->model.frame_post_proc(task->out_frame, outputs,
ov_model->model.filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);
}
@@ -458,23 +458,23 @@ static void infer_completion_callback(void *args)
}
break;
case DFT_ANALYTICS_DETECT:
-if (!ov_model->model->detect_post_proc) {
+if (!ov_model->model.detect_post_proc) {
av_log(ctx, AV_LOG_ERROR, "detect filter needs to provide post
proc\n");
goto end;
}
-ov_model->model->detect_post_proc(task->in_frame, outputs,
+ov_model->model.detect_post_proc(task->in_frame, outputs,
ov_model->nb_outputs,
- ov_model->model->filter_ctx);
+ ov_model->model.filter_ctx);
break;
case DFT_ANALYTICS_CLASSIFY:
-if (!ov_model->model->classify_post_proc) {
+if (!ov_model->model.classify_post_proc) {
av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide
post proc\n");
goto end;
}
for (int output_i = 0; output_i < ov_model->nb_outputs; output_i++)
-ov_model->model->classify_post_proc(task->in_frame, outputs,
+ov_model->model.classify_post_proc(task->in_frame, outputs,
request->lltasks[i]->bbox_index,
-
ov_model->model->filter_ctx);
+
ov_model->model.filter_ctx);
break;
default:
av_assert0(!"should not reach here");
@@ -571,7 +571,7 @@ static void dnn_free_model_ov(DNNModel **model)
av_free(ov_model->all_input_names);
#endif
av_freep(_model);
-av_freep(model);
+*model = NULL;
}
@@ -598,7 +598,7 @@ static int init_model_ov(OVModel *ov_model, const char
*input_name, const char *
#endif
// We scale pixel by default when do frame processing.
if (fabsf(ctx->ov_option.scale) < 1e-6f)
-ctx->ov_option.scale = ov_model->model->func_type == DFT_PROCESS_FRAME
? 255 : 1;
+ctx->ov_option.scale = ov_model->model.func_type == DFT_PROCESS_FRAME
? 255 : 1;
// batch size
if (ctx->ov_option.batch_size <= 0) {
ctx->ov_option.batch_size = 1;
@@ -702,7 +702,7 @@ static int init_model_ov(OVModel *ov_model, const char
*input_name, const char *
ret = ov2_map_error(status, NULL);
goto err;
}
-if (ov_model->model->func_type != DFT_PROCESS_FRAME
[FFmpeg-devel] [PATCH WIP 6/9] avfilter/dnn_backend_tf: Simplify memory allocation
From: Zhao Zhili
---
libavfilter/dnn/dnn_backend_tf.c | 33 +---
1 file changed, 13 insertions(+), 20 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index 3b4de6d13f..c7716e696d 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -37,8 +37,8 @@
#include
typedef struct TFModel {
+DNNModel model;
DnnContext *ctx;
-DNNModel *model;
TF_Graph *graph;
TF_Session *session;
TF_Status *status;
@@ -518,7 +518,7 @@ static void dnn_free_model_tf(DNNModel **model)
TF_DeleteStatus(tf_model->status);
}
av_freep(_model);
-av_freep();
+*model = NULL;
}
static DNNModel *dnn_load_model_tf(DnnContext *ctx, DNNFunctionType func_type,
AVFilterContext *filter_ctx)
@@ -526,18 +526,11 @@ static DNNModel *dnn_load_model_tf(DnnContext *ctx,
DNNFunctionType func_type, A
DNNModel *model = NULL;
TFModel *tf_model = NULL;
-model = av_mallocz(sizeof(DNNModel));
-if (!model){
-return NULL;
-}
-
tf_model = av_mallocz(sizeof(TFModel));
-if (!tf_model){
-av_freep();
+if (!tf_model)
return NULL;
-}
+model = _model->model;
model->model = tf_model;
-tf_model->model = model;
tf_model->ctx = ctx;
if (load_tf_model(tf_model, ctx->model_filename) != 0){
@@ -650,11 +643,11 @@ static int fill_model_input_tf(TFModel *tf_model,
TFRequestItem *request) {
}
input.data = (float *)TF_TensorData(infer_request->input_tensor);
-switch (tf_model->model->func_type) {
+switch (tf_model->model.func_type) {
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
-if (tf_model->model->frame_pre_proc != NULL) {
-tf_model->model->frame_pre_proc(task->in_frame, ,
tf_model->model->filter_ctx);
+if (tf_model->model.frame_pre_proc != NULL) {
+tf_model->model.frame_pre_proc(task->in_frame, ,
tf_model->model.filter_ctx);
} else {
ff_proc_from_frame_to_dnn(task->in_frame, , ctx);
}
@@ -664,7 +657,7 @@ static int fill_model_input_tf(TFModel *tf_model,
TFRequestItem *request) {
ff_frame_to_dnn_detect(task->in_frame, , ctx);
break;
default:
-avpriv_report_missing_feature(ctx, "model function type %d",
tf_model->model->func_type);
+avpriv_report_missing_feature(ctx, "model function type %d",
tf_model->model.func_type);
break;
}
@@ -724,12 +717,12 @@ static void infer_completion_callback(void *args) {
outputs[i].data = TF_TensorData(infer_request->output_tensors[i]);
outputs[i].dt =
(DNNDataType)TF_TensorType(infer_request->output_tensors[i]);
}
-switch (tf_model->model->func_type) {
+switch (tf_model->model.func_type) {
case DFT_PROCESS_FRAME:
//it only support 1 output if it's frame in & frame out
if (task->do_ioproc) {
-if (tf_model->model->frame_post_proc != NULL) {
-tf_model->model->frame_post_proc(task->out_frame, outputs,
tf_model->model->filter_ctx);
+if (tf_model->model.frame_post_proc != NULL) {
+tf_model->model.frame_post_proc(task->out_frame, outputs,
tf_model->model.filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);
}
@@ -741,11 +734,11 @@ static void infer_completion_callback(void *args) {
}
break;
case DFT_ANALYTICS_DETECT:
-if (!tf_model->model->detect_post_proc) {
+if (!tf_model->model.detect_post_proc) {
av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post
proc\n");
return;
}
-tf_model->model->detect_post_proc(task->in_frame, outputs,
task->nb_output, tf_model->model->filter_ctx);
+tf_model->model.detect_post_proc(task->in_frame, outputs,
task->nb_output, tf_model->model.filter_ctx);
break;
default:
av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this
kind of dnn filter now\n");
--
2.34.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH WIP 1/9] avfilter/dnn: Refactor DNN parameter configuration system
From: Zhao Zhili
This patch trying to resolve mulitiple issues related to parameter
configuration:
Firstly, each DNN filters duplicate DNN_COMMON_OPTIONS, which should
be the common options of backend.
Secondly, backend options are hidden behind the scene. It's a
AV_OPT_TYPE_STRING backend_configs for user, and parsed by each
backend. We don't know each backend support what kind of options
from the help message.
Third, DNN backends duplicate DNN_BACKEND_COMMON_OPTIONS.
Last but not the least, pass backend options via AV_OPT_TYPE_STRING
makes it hard to pass AV_OPT_TYPE_BINARY to backend, if not impossible.
This patch puts backend common options and each backend options inside
DnnContext to reduce code duplication, make options user friendly, and
easy to extend for future usecase.
There is a known issue that, for a filter which only support one or two
of the backends, the help message still show the option of all three
backend. Each DNN filter should be able to run on any backend. Current
issue is mostly due to incomplete implementation of backend (e.g., only
support DFT_PROCESS_FRAME with libtorch backedn), and lack of maintenance
on the filters.
For example,
./ffmpeg -h filter=dnn_processing
dnn_processing AVOptions:
dnn_backend ..FV... DNN backend (from INT_MIN to
INT_MAX) (default tensorflow)
tensorflow 1..FV... tensorflow backend flag
openvino2..FV... openvino backend flag
torch 3..FV... torch backend flag
dnn_base AVOptions:
model ..F path to model file
input ..F input name of the model
output ..F output name of the model
backend_configs..F...P backend configs (deprecated)
options..F...P backend configs (deprecated)
async ..F use DNN async inference (ignored,
use backend_configs='async=1') (default true)
nireq ..F number of request (from 0 to
INT_MAX) (default 0)
async ..F use DNN async inference (default
true)
device ..F device to run model
dnn_tensorflow AVOptions:
sess_config..F config for SessionOptions
dnn_openvino AVOptions:
batch_size..F batch size per request (from 1 to
1000) (default 1)
input_resizable ..F can input be resizable or not
(default false)
layout..F input layout of model (from 0 to
2) (default none)
none0..F none
nchw1..F nchw
nhwc2..F nhwc
scale ..F Add scale preprocess operation.
Divide each element of input by specified value. (from INT_MIN to INT_MAX)
(default 0)
mean..F Add mean preprocess operation.
Subtract specified value from each element of input. (from INT_MIN to INT_MAX)
(default 0)
dnn_th AVOptions:
optimize ..F turn on graph executor
optimization (from 0 to 1) (default 0)
---
libavfilter/dnn/dnn_backend_common.h | 13 ++-
libavfilter/dnn/dnn_backend_openvino.c | 146 ++---
libavfilter/dnn/dnn_backend_tf.c | 82 +-
libavfilter/dnn/dnn_backend_torch.cpp | 67
libavfilter/dnn/dnn_interface.c| 81 ++
libavfilter/dnn_filter_common.c| 32 +-
libavfilter/dnn_filter_common.h| 37 +++
libavfilter/dnn_interface.h| 66 ++-
libavfilter/vf_derain.c| 5 +-
libavfilter/vf_dnn_classify.c | 3 +-
libavfilter/vf_dnn_detect.c| 3 +-
libavfilter/vf_dnn_processing.c| 3 +-
libavfilter/vf_sr.c| 5 +-
13 files changed, 314 insertions(+), 229 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_common.h
b/libavfilter/dnn/dnn_backend_common.h
index 42c67c7040..9f5d37b3e0 100644
--- a/libavfilter/dnn/dnn_backend_common.h
+++ b/libavfilter/dnn/dnn_backend_common.h
@@ -28,9 +28,16 @@
#include "../dnn_interface.h"
#include "libavutil/thread.h"
-#define DNN_BACKEND_COMMON_OPTIONS \
-{ "nireq", "number of request",
OFFSET(options.nireq), AV_OPT_TYPE_INT,{ .i64 = 0 }, 0,
INT_MAX, FLAGS }, \
-{ "async", "use DNN async inference",
OFFSET(options.async), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0,
1, FLAGS },
+#define DNN_DEFINE_CLASS_EXT(name, desc, options) \
+{ \
+.class_name = desc, \
+.item_name = av_default_item_name, \
+.option = options, \
+.version= LIBAVUTIL_VERSI
[FFmpeg-devel] [PATCH WIP 2/9] avfilter/dnn_backend_openvino: Fix free context at random place
From: Zhao Zhili It will be freed again by ff_dnn_uninit. --- libavfilter/dnn/dnn_backend_openvino.c | 1 - 1 file changed, 1 deletion(-) diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c index c4b0682f11..769ba0a54b 100644 --- a/libavfilter/dnn/dnn_backend_openvino.c +++ b/libavfilter/dnn/dnn_backend_openvino.c @@ -959,7 +959,6 @@ err: if (input_model_info) ov_preprocess_input_model_info_free(input_model_info); #endif -dnn_free_model_ov(_model->model); return ret; } -- 2.34.1 ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH WIP 0/9] Refactor DNN
From: Zhao Zhili
During the refactor progress, I have found some serious issues, which
is not resolved by the patchset:
1. Tensorflow backend is broken.
I think it doesn't work since 2021 at least. For example, it destroy a
thread and create a new thread for each frame, and it destroy an invalid
thread at the first frame:
pthread_join(async_module->thread_id, );
if (status == DNN_ASYNC_FAIL) {
av_log(ctx, AV_LOG_ERROR, "Unable to start inference as previous
inference failed.\n");
return DNN_GENERIC_ERROR;
}
ret = pthread_create(_module->thread_id, NULL, async_thread_routine,
async_module);
2. Openvino V1 doesn't compile. It doesn't compile and no one complains,
I think it's a hint to just keep the code for V2.
3. Error handling. It's easy to crash with incorrect command line arguments.
I don't have enough test case. Please share your test case and help on test.
Zhao Zhili (9):
avfilter/dnn: Refactor DNN parameter configuration system
avfilter/dnn_backend_openvino: Fix free context at random place
avfilter/dnn_backend_openvino: simplify memory allocation
avfilter/dnn_backend_tf: Remove one level of indentation
avfilter/dnn_backend_tf: Fix free context at random place
avfilter/dnn_backend_tf: Simplify memory allocation
avfilter/dnn_backend_torch: Simplify memory allocation
avfilter/dnn: Remove a level of dereference
avfilter/dnn: Use dnn_backend_info_list to search for dnn module
libavfilter/dnn/dnn_backend_common.h | 13 +-
libavfilter/dnn/dnn_backend_openvino.c | 210 ++---
libavfilter/dnn/dnn_backend_tf.c | 194 ++-
libavfilter/dnn/dnn_backend_torch.cpp | 112 +
libavfilter/dnn/dnn_interface.c| 107 ++---
libavfilter/dnn_filter_common.c| 38 -
libavfilter/dnn_filter_common.h| 37 ++---
libavfilter/dnn_interface.h| 73 +++--
libavfilter/vf_derain.c| 5 +-
libavfilter/vf_dnn_classify.c | 3 +-
libavfilter/vf_dnn_detect.c| 3 +-
libavfilter/vf_dnn_processing.c| 3 +-
libavfilter/vf_sr.c| 5 +-
13 files changed, 428 insertions(+), 375 deletions(-)
--
2.34.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH] avcodec/mediacodecenc: Flush bsf after create extradata
From: Zhao Zhili Avoid leaving any data inside bsf while also avoid keep bsf in EOF state. --- libavcodec/mediacodecenc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c index c87a0d86c5..bbf570e7be 100644 --- a/libavcodec/mediacodecenc.c +++ b/libavcodec/mediacodecenc.c @@ -681,6 +681,7 @@ bailout: s->eof_sent = 0; ff_AMediaCodec_flush(s->codec); } +av_bsf_flush(s->bsf); av_packet_free(); return ret; } -- 2.25.1 ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [RFC] 5 year plan & Inovation
> -Original Message- > From: ffmpeg-devel On Behalf Of Niklas Haas > Sent: 2024年4月19日 22:50 > To: FFmpeg development discussions and patches > Subject: Re: [FFmpeg-devel] [RFC] 5 year plan & Inovation > > On Thu, 18 Apr 2024 22:53:51 +0200 Michael Niedermayer > wrote: > > A plugin system moves this patch-management to people who actually > > care, that is the authors of the codecs and (de)muxers. > > A plugin system will only solve this insomuch as plugin authors will > just host their plugin code on GitHub instead of bothering with the > mailing list. > > I think it runs a good risk of basically killing the project. VLC is plugin based, gstreamer is plugin based too (which went t far ), I don't think plugin is that dangerous. Firstly, we can enable plugin interface only with enable-gpl. Secondly, we can have a less stable plugin interface than public API, for our development convenience, and encourage plugin authors to contribute to upstream. > > > Our productivity as is, is not good, many patches are ignored. > > The people caring about these patches are their Authors and yet they > > are powerless as they must sometimes wait many months for reviews > > So, rather than all of the above, what I think we should do is contract > somebody to set up, manage, host and maintain a GitLab instance for us. > > This would probably be the single most cost effective boost to both > community growth and innovation I can think of, as it will remove > several of the major grievances and barriers to entry with the > ML+pingspam model. +1. I can't remember how many patches I have ping. It's really frustration. I ask for permission to commit mostly due to this. Now I can keep track of my own patches, but it's still not easy to filter out patches I'm interested to review (I can blame the email client, but blame it doesn't help). I'm sure I can review more patches with a new workflow. > > We can use a system like VLC's auto-merge bot, where any MR that has at > least one developer approval, no unresolved issues, and no activity for > N days gets *automatically* merged. > > I'm sure that if we try, we can find an interested party willing to fund > this. (Maybe SPI?) > > > Besides that, developers are leaving for various reasons and they > > are forced to setup full forks not being able to maintain their > > code in any other way. > > IMO A plugin system would improve productivity as everyone could work > > on their own terms. > > No week or month long delays and weekly pinging patches > > No risk about patches being rejected or ignored > > No need to read every other discussion on the ML. One can just > > simply work on their own plugin looking just at the API documentation > > ... > > > > > > > > > > > > > * ffchat > > > > (expand into realtime chat / zoom) this would > > > > bring in more users and developers, and we basically have almost > > > > all parts for it already but some people where against it > > > > > > This seems like a user application on top of FFmpeg, not something that > > > should be part of FFmpeg core. Can you explain what modifications in > > > FFmpeg would be necessary for something like this? > > > > ffmpeg, ffplay, ffprobe are also user applications. > > > > > > > > > > > * client side / in browser support > > > > (expand towards webapps, webpages using ffmpeg client side in the > > > > browser) > > > > bring in more users and developers, and it will be costly for us > > > > if we let others take this area as its important and significant > > > > > > I don't understand this point - don't all major browsers already vendor > > > FFmpeg for decoding? > > > > FFmpeg does more than decoding. > > > > > > > > > > > * AI / neural network filters and codecs > > > > The future seems to be AI based. Future Filters and Codecs will use > > > > neural networks. FFmpeg can be at the forefront, developing these > > > > > > We already have TensorFlow support, no? (vf_sr etc.) What more is > > > needed? > > > > more of that AND more convenience > > > > lets pick a comparission > > to run fate > > you write "make fate" > > if you want to do it with the samples its > > make fate-rsync ; make fate > > > > if you want to use vf_sr, its reading the docs, looking at some scripts > > reading their docs > > and i presume selecting a training set ? creating a model ? > > > > how many people do that ? > > > > thx > > > > [...] > > > > -- > > Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB > > > > I have often repented speaking, but never of holding my tongue. > > -- Xenocrates > > ___ > > ffmpeg-devel mailing list > > ffmpeg-devel@ffmpeg.org > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > To unsubscribe, visit link above, or email > > ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe". > ___ > ffmpeg-devel mailing list >
[FFmpeg-devel] [PATCH 8/8] fftools: Fix implicit-const-int-float-conversion warning
From: Zhao Zhili
---
fftools/cmdutils.c | 2 +-
fftools/ffmpeg_opt.c | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/fftools/cmdutils.c b/fftools/cmdutils.c
index 2120fc7935..a8f5c6d89b 100644
--- a/fftools/cmdutils.c
+++ b/fftools/cmdutils.c
@@ -311,7 +311,7 @@ static int write_option(void *optctx, const OptionDef *po,
const char *opt,
*(int *)dst = num;
} else if (po->type == OPT_TYPE_INT64) {
-ret = parse_number(opt, arg, OPT_TYPE_INT64, INT64_MIN, INT64_MAX,
);
+ret = parse_number(opt, arg, OPT_TYPE_INT64, INT64_MIN,
(double)INT64_MAX, );
if (ret < 0)
goto finish;
diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c
index 6526e8e3e8..910e4a336b 100644
--- a/fftools/ffmpeg_opt.c
+++ b/fftools/ffmpeg_opt.c
@@ -317,7 +317,7 @@ static int opt_filter_threads(void *optctx, const char
*opt, const char *arg)
static int opt_abort_on(void *optctx, const char *opt, const char *arg)
{
static const AVOption opts[] = {
-{ "abort_on" , NULL, 0, AV_OPT_TYPE_FLAGS, { .i64 = 0 },
INT64_MIN, INT64_MAX, .unit = "flags" },
+{ "abort_on" , NULL, 0, AV_OPT_TYPE_FLAGS, { .i64 = 0 },
INT64_MIN, (double)INT64_MAX, .unit = "flags" },
{ "empty_output" , NULL, 0, AV_OPT_TYPE_CONST, { .i64 =
ABORT_ON_FLAG_EMPTY_OUTPUT}, .unit = "flags" },
{ "empty_output_stream", NULL, 0, AV_OPT_TYPE_CONST, { .i64 =
ABORT_ON_FLAG_EMPTY_OUTPUT_STREAM }, .unit = "flags" },
{ NULL },
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH 7/8] avcodec/mediacodecenc: add AV_CODEC_CAP_ENCODER_FLUSH support
From: Zhao Zhili
Signed-off-by: Zhao Zhili
---
libavcodec/mediacodecenc.c | 15 +--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c
index 64816ccf0a..c87a0d86c5 100644
--- a/libavcodec/mediacodecenc.c
+++ b/libavcodec/mediacodecenc.c
@@ -705,6 +705,15 @@ static av_cold int mediacodec_close(AVCodecContext *avctx)
return 0;
}
+static av_cold void mediacodec_flush(AVCodecContext *avctx)
+{
+MediaCodecEncContext *s = avctx->priv_data;
+if (s->bsf)
+av_bsf_flush(s->bsf);
+av_frame_unref(s->frame);
+ff_AMediaCodec_flush(s->codec);
+}
+
static const AVCodecHWConfigInternal *const mediacodec_hw_configs[] = {
&(const AVCodecHWConfigInternal) {
.public = {
@@ -755,13 +764,15 @@ const FFCodec ff_ ## short_name ## _mediacodec_encoder =
{ \
CODEC_LONG_NAME(long_name " Android MediaCodec encoder"), \
.p.type = AVMEDIA_TYPE_VIDEO, \
.p.id = codec_id, \
-.p.capabilities = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY \
-| AV_CODEC_CAP_HARDWARE,\
+.p.capabilities = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY | \
+AV_CODEC_CAP_HARDWARE | \
+AV_CODEC_CAP_ENCODER_FLUSH, \
.priv_data_size = sizeof(MediaCodecEncContext), \
.p.pix_fmts = avc_pix_fmts, \
.init = mediacodec_init,\
FF_CODEC_RECEIVE_PACKET_CB(mediacodec_encode), \
.close= mediacodec_close, \
+.flush= mediacodec_flush, \
.p.priv_class = _name ## _mediacodec_class, \
.caps_internal= FF_CODEC_CAP_INIT_CLEANUP, \
.p.wrapper_name = "mediacodec", \
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH 6/8] avcodec/mediacodecenc: Add global header support
From: Zhao Zhili
The extradata is generated by encoding a dummy frame, then reset
the encoder state by mediacodec flush(). It only works for pixel
format other than AV_PIX_FMT_MEDIACODEC, since I'm not sure how
to create a dummy frame safely with AV_PIX_FMT_MEDIACODEC.
Signed-off-by: Zhao Zhili
---
configure | 6 +-
libavcodec/mediacodecenc.c | 166 +
2 files changed, 155 insertions(+), 17 deletions(-)
diff --git a/configure b/configure
index d0d0e8430b..4da1c0be35 100755
--- a/configure
+++ b/configure
@@ -3313,6 +3313,7 @@ ac3_mf_encoder_deps="mediafoundation"
av1_cuvid_decoder_deps="cuvid CUVIDAV1PICPARAMS"
av1_mediacodec_decoder_deps="mediacodec"
av1_mediacodec_encoder_deps="mediacodec"
+av1_mediacodec_encoder_select="extract_extradata_bsf"
av1_nvenc_encoder_deps="nvenc NV_ENC_PIC_PARAMS_AV1"
av1_nvenc_encoder_select="atsc_a53"
h263_v4l2m2m_decoder_deps="v4l2_m2m h263_v4l2_m2m"
@@ -3323,7 +3324,7 @@ h264_cuvid_decoder_select="h264_mp4toannexb_bsf"
h264_mediacodec_decoder_deps="mediacodec"
h264_mediacodec_decoder_select="h264_mp4toannexb_bsf h264_parser"
h264_mediacodec_encoder_deps="mediacodec"
-h264_mediacodec_encoder_select="h264_metadata"
+h264_mediacodec_encoder_select="extract_extradata_bsf h264_metadata"
h264_mf_encoder_deps="mediafoundation"
h264_mmal_decoder_deps="mmal"
h264_nvenc_encoder_deps="nvenc"
@@ -3343,7 +3344,7 @@ hevc_cuvid_decoder_select="hevc_mp4toannexb_bsf"
hevc_mediacodec_decoder_deps="mediacodec"
hevc_mediacodec_decoder_select="hevc_mp4toannexb_bsf hevc_parser"
hevc_mediacodec_encoder_deps="mediacodec"
-hevc_mediacodec_encoder_select="hevc_metadata"
+hevc_mediacodec_encoder_select="extract_extradata_bsf hevc_metadata"
hevc_mf_encoder_deps="mediafoundation"
hevc_nvenc_encoder_deps="nvenc"
hevc_nvenc_encoder_select="atsc_a53"
@@ -3375,6 +3376,7 @@ mpeg2_v4l2m2m_decoder_deps="v4l2_m2m mpeg2_v4l2_m2m"
mpeg4_cuvid_decoder_deps="cuvid"
mpeg4_mediacodec_decoder_deps="mediacodec"
mpeg4_mediacodec_encoder_deps="mediacodec"
+mpeg4_mediacodec_encoder_select="extract_extradata_bsf"
mpeg4_mmal_decoder_deps="mmal"
mpeg4_omx_encoder_deps="omx"
mpeg4_v4l2m2m_decoder_deps="v4l2_m2m mpeg4_v4l2_m2m"
diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c
index 8caaad729a..64816ccf0a 100644
--- a/libavcodec/mediacodecenc.c
+++ b/libavcodec/mediacodecenc.c
@@ -23,6 +23,7 @@
#include "config_components.h"
#include "libavutil/avassert.h"
+#include "libavutil/avstring.h"
#include "libavutil/hwcontext_mediacodec.h"
#include "libavutil/imgutils.h"
#include "libavutil/mem.h"
@@ -74,6 +75,7 @@ typedef struct MediaCodecEncContext {
int bitrate_mode;
int level;
int pts_as_dts;
+int extract_extradata;
} MediaCodecEncContext;
enum {
@@ -112,6 +114,23 @@ static void mediacodec_output_format(AVCodecContext *avctx)
ff_AMediaFormat_delete(out_format);
}
+static int extract_extradata_support(AVCodecContext *avctx)
+{
+const AVBitStreamFilter *bsf = av_bsf_get_by_name("extract_extradata");
+
+if (!bsf) {
+av_log(avctx, AV_LOG_WARNING, "extract_extradata bsf not found\n");
+return 0;
+}
+
+for (int i = 0; bsf->codec_ids[i] != AV_CODEC_ID_NONE; i++) {
+if (bsf->codec_ids[i] == avctx->codec_id)
+return 1;
+}
+
+return 0;
+}
+
static int mediacodec_init_bsf(AVCodecContext *avctx)
{
MediaCodecEncContext *s = avctx->priv_data;
@@ -120,20 +139,32 @@ static int mediacodec_init_bsf(AVCodecContext *avctx)
int crop_right = s->width - avctx->width;
int crop_bottom = s->height - avctx->height;
-if (!crop_right && !crop_bottom)
+/* Nothing can be done for this format now */
+if (avctx->pix_fmt == AV_PIX_FMT_MEDIACODEC)
return 0;
-if (avctx->codec_id == AV_CODEC_ID_H264)
-ret = snprintf(str, sizeof(str),
"h264_metadata=crop_right=%d:crop_bottom=%d",
- crop_right, crop_bottom);
-else if (avctx->codec_id == AV_CODEC_ID_HEVC)
-ret = snprintf(str, sizeof(str),
"hevc_metadata=crop_right=%d:crop_bottom=%d",
- crop_right, crop_bottom);
-else
+s->extract_extradata = (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) &&
+ extract_extradata_support(avctx);
+if (!crop_right && !crop_bottom && !s->extract_extradata)
return 0;
-if (ret >= sizeof(str))
-return AVERROR_
[FFmpeg-devel] [PATCH 2/8] avcodec/mediacodec_wrapper: link to NDK mediacodec API directly
From: Zhao Zhili
Drop support of Android version before 5.0.
---
configure | 2 +-
libavcodec/mediacodec_wrapper.c | 164 +++-
2 files changed, 36 insertions(+), 130 deletions(-)
diff --git a/configure b/configure
index 47ec215f8c..d0d0e8430b 100755
--- a/configure
+++ b/configure
@@ -3131,7 +3131,7 @@ d3d11va_deps="dxva_h ID3D11VideoDecoder
ID3D11VideoContext"
d3d12va_deps="dxva_h ID3D12Device ID3D12VideoDecoder"
dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32"
ffnvcodec_deps_any="libdl LoadLibrary"
-mediacodec_deps="android"
+mediacodec_deps="android mediandk"
nvdec_deps="ffnvcodec"
vaapi_x11_deps="xlib_x11"
videotoolbox_hwaccel_deps="videotoolbox pthreads"
diff --git a/libavcodec/mediacodec_wrapper.c b/libavcodec/mediacodec_wrapper.c
index 306359071e..a9c8b522e0 100644
--- a/libavcodec/mediacodec_wrapper.c
+++ b/libavcodec/mediacodec_wrapper.c
@@ -1828,25 +1828,8 @@ typedef struct FFAMediaFormatNdk {
void *libmedia;
AMediaFormat *impl;
-AMediaFormat *(*new)(void);
-media_status_t (*delete)(AMediaFormat*);
-
-const char* (*toString)(AMediaFormat*);
-
-bool (*getInt32)(AMediaFormat*, const char *name, int32_t *out);
-bool (*getInt64)(AMediaFormat*, const char *name, int64_t *out);
-bool (*getFloat)(AMediaFormat*, const char *name, float *out);
-bool (*getSize)(AMediaFormat*, const char *name, size_t *out);
-bool (*getBuffer)(AMediaFormat*, const char *name, void** data, size_t
*size);
-bool (*getString)(AMediaFormat*, const char *name, const char **out);
bool (*getRect)(AMediaFormat *, const char *name,
int32_t *left, int32_t *top, int32_t *right, int32_t
*bottom);
-
-void (*setInt32)(AMediaFormat*, const char* name, int32_t value);
-void (*setInt64)(AMediaFormat*, const char* name, int64_t value);
-void (*setFloat)(AMediaFormat*, const char* name, float value);
-void (*setString)(AMediaFormat*, const char* name, const char* value);
-void (*setBuffer)(AMediaFormat*, const char* name, const void* data,
size_t size);
void (*setRect)(AMediaFormat *, const char *name,
int32_t left, int32_t top, int32_t right, int32_t bottom);
} FFAMediaFormatNdk;
@@ -1858,34 +1841,6 @@ typedef struct FFAMediaCodecNdk {
AMediaCodec *impl;
ANativeWindow *window;
-AMediaCodec* (*createCodecByName)(const char *name);
-AMediaCodec* (*createDecoderByType)(const char *mime_type);
-AMediaCodec* (*createEncoderByType)(const char *mime_type);
-media_status_t (*delete)(AMediaCodec*);
-
-media_status_t (*configure)(AMediaCodec *,
-const AMediaFormat *format,
-ANativeWindow *surface,
-AMediaCrypto *crypto,
-uint32_t flags);
-media_status_t (*start)(AMediaCodec*);
-media_status_t (*stop)(AMediaCodec*);
-media_status_t (*flush)(AMediaCodec*);
-
-uint8_t* (*getInputBuffer)(AMediaCodec*, size_t idx, size_t *out_size);
-uint8_t* (*getOutputBuffer)(AMediaCodec*, size_t idx, size_t *out_size);
-
-ssize_t (*dequeueInputBuffer)(AMediaCodec*, int64_t timeoutUs);
-media_status_t (*queueInputBuffer)(AMediaCodec*, size_t idx,
- long offset, size_t size,
- uint64_t time, uint32_t flags);
-
-ssize_t (*dequeueOutputBuffer)(AMediaCodec*, AMediaCodecBufferInfo *info,
int64_t timeoutUs);
-AMediaFormat* (*getOutputFormat)(AMediaCodec*);
-
-media_status_t (*releaseOutputBuffer)(AMediaCodec*, size_t idx, bool
render);
-media_status_t (*releaseOutputBufferAtTime)(AMediaCodec *mData, size_t
idx, int64_t timestampNs);
-
// Available since API level 28.
media_status_t (*getName)(AMediaCodec*, char** out_name);
void (*releaseName)(AMediaCodec*, char* name);
@@ -1925,38 +1880,15 @@ static FFAMediaFormat
*mediaformat_ndk_create(AMediaFormat *impl)
#define GET_OPTIONAL_SYMBOL(sym) \
format->sym = dlsym(format->libmedia, "AMediaFormat_" #sym);
-#define GET_SYMBOL(sym) \
-GET_OPTIONAL_SYMBOL(sym)\
-if (!format->sym) \
-goto error;
-
-GET_SYMBOL(new)
-GET_SYMBOL(delete)
-
-GET_SYMBOL(toString)
-
-GET_SYMBOL(getInt32)
-GET_SYMBOL(getInt64)
-GET_SYMBOL(getFloat)
-GET_SYMBOL(getSize)
-GET_SYMBOL(getBuffer)
-GET_SYMBOL(getString)
GET_OPTIONAL_SYMBOL(getRect)
-
-GET_SYMBOL(setInt32)
-GET_SYMBOL(setInt64)
-GET_SYMBOL(setFloat)
-GET_SYMBOL(setString)
-GET_SYMBOL(setBuffer)
GET_OPTIONAL_SYMBOL(setRect)
-#undef GET_SYMBOL
#undef GET_OPTIONAL_SYMBOL
if (impl) {
format->impl = impl;
} else {
-format->impl = format->new();
+
[FFmpeg-devel] [PATCH 1/8] configure: Lower Android API level requirement of mediandk
From: Zhao Zhili AImage_delete is introduced in API level 24, while AMediaFormat_new is introduced in API level 21. We only need API level 21 to support NDK mediacodec. This change doesn't break or change of android_camera_indev since camera2ndk still needs API level 24. --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index 55f1fc354d..47ec215f8c 100755 --- a/configure +++ b/configure @@ -6640,7 +6640,7 @@ check_lib shell32 "windows.h shellapi.h" CommandLineToArgvW -lshell32 check_lib psapi"windows.h psapi.h"GetProcessMemoryInfo -lpsapi check_lib android android/native_window.h ANativeWindow_acquire -landroid -check_lib mediandk "stdint.h media/NdkImage.h" AImage_delete -lmediandk +check_lib mediandk "stdint.h media/NdkMediaFormat.h" AMediaFormat_new -lmediandk check_lib camera2ndk "stdbool.h stdint.h camera/NdkCameraManager.h" ACameraManager_create -lcamera2ndk enabled appkit && check_apple_framework AppKit -- 2.25.1 ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH 5/8] avcodec/mediacodecenc: Remove write only variable
From: Zhao Zhili
Signed-off-by: Zhao Zhili
---
libavcodec/mediacodecenc.c | 10 +++---
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c
index e562181120..8caaad729a 100644
--- a/libavcodec/mediacodecenc.c
+++ b/libavcodec/mediacodecenc.c
@@ -352,9 +352,7 @@ bailout:
return ret;
}
-static int mediacodec_receive(AVCodecContext *avctx,
- AVPacket *pkt,
- int *got_packet)
+static int mediacodec_receive(AVCodecContext *avctx, AVPacket *pkt)
{
MediaCodecEncContext *s = avctx->priv_data;
FFAMediaCodec *codec = s->codec;
@@ -400,7 +398,7 @@ static int mediacodec_receive(AVCodecContext *avctx,
memcpy(s->extradata, out_buf + out_info.offset, out_info.size);
ff_AMediaCodec_releaseOutputBuffer(codec, index, false);
// try immediately
-return mediacodec_receive(avctx, pkt, got_packet);
+return mediacodec_receive(avctx, pkt);
}
ret = ff_get_encode_buffer(avctx, pkt, out_info.size + s->extradata_size,
0);
@@ -419,7 +417,6 @@ static int mediacodec_receive(AVCodecContext *avctx,
if (out_info.flags & ff_AMediaCodec_getBufferFlagKeyFrame(codec))
pkt->flags |= AV_PKT_FLAG_KEY;
ret = 0;
-*got_packet = 1;
av_log(avctx, AV_LOG_TRACE, "receive packet pts %" PRId64 " dts %" PRId64
" flags %d extradata %d\n",
@@ -510,7 +507,6 @@ static int mediacodec_encode(AVCodecContext *avctx,
AVPacket *pkt)
{
MediaCodecEncContext *s = avctx->priv_data;
int ret;
-int got_packet = 0;
// Return on three case:
// 1. Serious error
@@ -525,7 +521,7 @@ static int mediacodec_encode(AVCodecContext *avctx,
AVPacket *pkt)
return ret;
}
-ret = mediacodec_receive(avctx, pkt, _packet);
+ret = mediacodec_receive(avctx, pkt);
if (s->bsf) {
if (!ret || ret == AVERROR_EOF)
ret = av_bsf_send_packet(s->bsf, pkt);
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH 4/8] avcodec/mediacodecenc: Fix return empty packet when bsf is used
From: Zhao Zhili
Signed-off-by: Zhao Zhili
---
libavcodec/mediacodecenc.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c
index b59de75b9b..e562181120 100644
--- a/libavcodec/mediacodecenc.c
+++ b/libavcodec/mediacodecenc.c
@@ -534,7 +534,7 @@ static int mediacodec_encode(AVCodecContext *avctx,
AVPacket *pkt)
return 0;
}
-if (ret != AVERROR(EAGAIN))
+if (ret < 0 && ret != AVERROR(EAGAIN))
return ret;
if (!s->frame->buf[0]) {
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH 3/8] avcodec/mediacodec_wrapper: Fix unused variable warning
From: Zhao Zhili --- libavcodec/mediacodec_wrapper.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavcodec/mediacodec_wrapper.c b/libavcodec/mediacodec_wrapper.c index a9c8b522e0..96c88a 100644 --- a/libavcodec/mediacodec_wrapper.c +++ b/libavcodec/mediacodec_wrapper.c @@ -365,6 +365,7 @@ int ff_AMediaCodecProfile_getProfileFromAVCodecContext(AVCodecContext *avctx) (void)VP9Profile3HDR; (void)VP9Profile2HDR10Plus; (void)VP9Profile3HDR10Plus; +(void)MPEG4ProfileSimpleFace; (void)AV1ProfileMain10; (void)AV1ProfileMain10HDR10; (void)AV1ProfileMain10HDR10Plus; -- 2.25.1 ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH v1] libavfilter/af_channelsplit.c:fix memory leak
> On Apr 12, 2024, at 17:49, Zhao Zhili wrote:
>
>
>> On Apr 12, 2024, at 17:19, LuMingYin wrote:
>>
>> Signed-off-by: LuMingYin
>> ---
>> libavfilter/af_channelsplit.c | 4 +++-
>> 1 file changed, 3 insertions(+), 1 deletion(-)
>>
>> diff --git a/libavfilter/af_channelsplit.c b/libavfilter/af_channelsplit.c
>> index d18d91dcb6..2cfac19cd3 100644
>> --- a/libavfilter/af_channelsplit.c
>> +++ b/libavfilter/af_channelsplit.c
>> @@ -163,8 +163,10 @@ static int filter_frame(AVFilterLink *outlink, AVFrame
>> *buf)
>>
>>buf_out->data[0] = buf_out->extended_data[0] =
>> buf_out->extended_data[s->map[i]];
>>ret = av_channel_layout_from_mask(_out->ch_layout, 1ULL << channel);
>> -if (ret < 0)
>> +if (ret < 0){
>> +av_frame_free(_out);
>>return ret;
>> +}
>
> LGTM. Please pay attention to the coding style. I can fix it manually this
> time.
Applied as 5e380bcdb13dd47ce9c358a4edb281f05fde3f24.
>
>>
>>return ff_filter_frame(ctx->outputs[i], buf_out);
>> }
>> --
>> 2.25.1
>>
>> ___
>> ffmpeg-devel mailing list
>> ffmpeg-devel@ffmpeg.org
>> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>>
>> To unsubscribe, visit link above, or email
>> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
>
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [RFC] Bump minimum required version of Android to 5.0
> On Apr 12, 2024, at 18:50, Tomas Härdin wrote: > > tor 2024-04-11 klockan 21:57 +0800 skrev Zhao Zhili: >> >> >>> On Apr 11, 2024, at 21:17, Tomas Härdin wrote: >>> >>> tor 2024-04-11 klockan 20:16 +0800 skrev Zhao Zhili: >>>> We don’t have a minimum required version of Android in FFmpeg. >>>> libavdevice/android_camera requires Android 7, Java MediaCodec >>>> requires Android 4.1, and NDK MediaCodec requires Android 5.0. >>>> >>>> Without an explicit version, it’s unclear for development and >>>> test. >>>> >>>> Android 5.0 is released in 2014, is it OK to bump the minimum >>>> required >>>> version to Android 5.0, or any other version you prefer? >>> >>> Don't we already have stuff that detects the Android version and >>> acts >>> accordingly? Dropping 4.1 might lessen the maintenance burden >>> though. >> >> Check Android API level is easy. There is [no] minimum API level >> requirement >> in configure script as far as I know. > > Would be a good idea to add one > > What would dropping 4.1 support actually look like? I presume it would > make the code a lot simpler. I can remove a lot of dlsym by link directly. And it will be more clear to know how to handle Android version dependent API when adding new code. > >>> I have an old phone (Samsung Galaxy S5) running the most recent >>> LineageOS possible to install on it (16.0), and that uses Android >>> 9. So >>> for me bumping to version 5 sounds fine. >>> >>> Do you have any statistics on Android versions actually in use? >> >> Search by "Android distribution chart" shows version >= 5.0 is about >> 99.3%. > > Makes me wonder what kind of devices people run that haven't been > updated in 10+ years.. Like TV and OTT box. I know those devices exist, but I highly doubt that anyone would attempt to run FFmpeg 7.1 on it. Android toolchains also drop support of Android 4.4 now. "KitKat (APIs 19 and 20) is no longer supported.” https://developer.android.com/ndk/downloads/revision_history > > Compare to the recent Debian and Ubuntu support discussion, where we're > considering dropping support for 5 year old releases. Dropping support > for 10 year old ones doesn't sound bad. You're also the one most > involved in Android support, so unless someone else wants to take on > the maintenance burden I say go for it. If downstream apps make a fuss, > they can either take on the work themselves or pay. > > /Tomas > ___ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe". ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH v1] libavfilter/af_channelsplit.c:fix memory leak
> On Apr 12, 2024, at 17:19, LuMingYin wrote:
>
> Signed-off-by: LuMingYin
> ---
> libavfilter/af_channelsplit.c | 4 +++-
> 1 file changed, 3 insertions(+), 1 deletion(-)
>
> diff --git a/libavfilter/af_channelsplit.c b/libavfilter/af_channelsplit.c
> index d18d91dcb6..2cfac19cd3 100644
> --- a/libavfilter/af_channelsplit.c
> +++ b/libavfilter/af_channelsplit.c
> @@ -163,8 +163,10 @@ static int filter_frame(AVFilterLink *outlink, AVFrame
> *buf)
>
> buf_out->data[0] = buf_out->extended_data[0] =
> buf_out->extended_data[s->map[i]];
> ret = av_channel_layout_from_mask(_out->ch_layout, 1ULL << channel);
> -if (ret < 0)
> +if (ret < 0){
> +av_frame_free(_out);
> return ret;
> +}
LGTM. Please pay attention to the coding style. I can fix it manually this time.
>
> return ff_filter_frame(ctx->outputs[i], buf_out);
> }
> --
> 2.25.1
>
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [RFC] Bump minimum required version of Android to 5.0
> 在 2024年4月11日，下午9:57，Zhao Zhili 写道： > > >> >> On Apr 11, 2024, at 21:17, Tomas Härdin wrote: >> >> tor 2024-04-11 klockan 20:16 +0800 skrev Zhao Zhili: >>> We don’t have a minimum required version of Android in FFmpeg. >>> libavdevice/android_camera requires Android 7, Java MediaCodec >>> requires Android 4.1, and NDK MediaCodec requires Android 5.0. >>> >>> Without an explicit version, it’s unclear for development and test. >>> >>> Android 5.0 is released in 2014, is it OK to bump the minimum >>> required >>> version to Android 5.0, or any other version you prefer? >> >> Don't we already have stuff that detects the Android version and acts >> accordingly? Dropping 4.1 might lessen the maintenance burden though. > > Check Android API level is easy. There is minimum API level requirement > in configure script as far as I know. I mean there is no minimum API level requirement in configure. >> >> I have an old phone (Samsung Galaxy S5) running the most recent >> LineageOS possible to install on it (16.0), and that uses Android 9. So >> for me bumping to version 5 sounds fine. >> >> Do you have any statistics on Android versions actually in use? > > Search by "Android distribution chart" shows version >= 5.0 is about 99.3%. > >> >> /Tomas >> ___ >> ffmpeg-devel mailing list >> ffmpeg-devel@ffmpeg.org >> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel >> >> To unsubscribe, visit link above, or email >> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe". > ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [RFC] Bump minimum required version of Android to 5.0
> On Apr 11, 2024, at 21:17, Tomas Härdin wrote: > > tor 2024-04-11 klockan 20:16 +0800 skrev Zhao Zhili: >> We don’t have a minimum required version of Android in FFmpeg. >> libavdevice/android_camera requires Android 7, Java MediaCodec >> requires Android 4.1, and NDK MediaCodec requires Android 5.0. >> >> Without an explicit version, it’s unclear for development and test. >> >> Android 5.0 is released in 2014, is it OK to bump the minimum >> required >> version to Android 5.0, or any other version you prefer? > > Don't we already have stuff that detects the Android version and acts > accordingly? Dropping 4.1 might lessen the maintenance burden though. Check Android API level is easy. There is minimum API level requirement in configure script as far as I know. > > I have an old phone (Samsung Galaxy S5) running the most recent > LineageOS possible to install on it (16.0), and that uses Android 9. So > for me bumping to version 5 sounds fine. > > Do you have any statistics on Android versions actually in use? Search by "Android distribution chart" shows version >= 5.0 is about 99.3%. > > /Tomas > ___ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe". ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH] avformat/tls_mbedtls: Pass FLAG_NONBLOCK to underlying transport
> On Apr 8, 2024, at 21:26, Zhao Zhili wrote:
>
> From: Zhao Zhili
>
> This fix rtmps failure since rtmps requires nonblocking read.
>
> Signed-off-by: Zhao Zhili
> ---
> libavformat/tls_mbedtls.c | 4
> 1 file changed, 4 insertions(+)
>
> diff --git a/libavformat/tls_mbedtls.c b/libavformat/tls_mbedtls.c
> index 8503523b6d..f51cf43b1c 100644
> --- a/libavformat/tls_mbedtls.c
> +++ b/libavformat/tls_mbedtls.c
> @@ -309,6 +309,8 @@ static int tls_read(URLContext *h, uint8_t *buf, int size)
> TLSContext *tls_ctx = h->priv_data;
> int ret;
>
> +tls_ctx->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
> +tls_ctx->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
> if ((ret = mbedtls_ssl_read(_ctx->ssl_context, buf, size)) > 0) {
> // return read length
> return ret;
> @@ -322,6 +324,8 @@ static int tls_write(URLContext *h, const uint8_t *buf,
> int size)
> TLSContext *tls_ctx = h->priv_data;
> int ret;
>
> +tls_ctx->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
> +tls_ctx->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
> if ((ret = mbedtls_ssl_write(_ctx->ssl_context, buf, size)) > 0) {
> // return written length
> return ret;
> --
> 2.25.1
Will apply tomorrow.
>
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [RFC] Bump minimum required version of Android to 5.0
We don’t have a minimum required version of Android in FFmpeg. libavdevice/android_camera requires Android 7, Java MediaCodec requires Android 4.1, and NDK MediaCodec requires Android 5.0. Without an explicit version, it’s unclear for development and test. Android 5.0 is released in 2014, is it OK to bump the minimum required version to Android 5.0, or any other version you prefer? ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH] avformat/tls_mbedtls: Pass FLAG_NONBLOCK to underlying transport
> On Apr 8, 2024, at 21:32, Dennis Mungai wrote:
>
> On Mon, 8 Apr 2024 at 16:26, Zhao Zhili wrote:
>
>> From: Zhao Zhili
>>
>> This fix rtmps failure since rtmps requires nonblocking read.
>>
>> Signed-off-by: Zhao Zhili
>> ---
>> libavformat/tls_mbedtls.c | 4
>> 1 file changed, 4 insertions(+)
>>
>> diff --git a/libavformat/tls_mbedtls.c b/libavformat/tls_mbedtls.c
>> index 8503523b6d..f51cf43b1c 100644
>> --- a/libavformat/tls_mbedtls.c
>> +++ b/libavformat/tls_mbedtls.c
>> @@ -309,6 +309,8 @@ static int tls_read(URLContext *h, uint8_t *buf, int
>> size)
>> TLSContext *tls_ctx = h->priv_data;
>> int ret;
>>
>> +tls_ctx->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
>> +tls_ctx->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
>> if ((ret = mbedtls_ssl_read(_ctx->ssl_context, buf, size)) > 0) {
>> // return read length
>> return ret;
>> @@ -322,6 +324,8 @@ static int tls_write(URLContext *h, const uint8_t
>> *buf, int size)
>> TLSContext *tls_ctx = h->priv_data;
>> int ret;
>>
>> +tls_ctx->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
>> +tls_ctx->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
>> if ((ret = mbedtls_ssl_write(_ctx->ssl_context, buf, size)) > 0) {
>> // return written length
>> return ret;
>> --
>> 2.25.1
>>
>
> Are other TLS layers affected, say OpenSSL, etc?
openssl and gnutls don’t get affected, since they have done the same operation.
I don’t know about schannel and securetransport.
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH] avformat/tls_mbedtls: Pass FLAG_NONBLOCK to underlying transport
From: Zhao Zhili
This fix rtmps failure since rtmps requires nonblocking read.
Signed-off-by: Zhao Zhili
---
libavformat/tls_mbedtls.c | 4
1 file changed, 4 insertions(+)
diff --git a/libavformat/tls_mbedtls.c b/libavformat/tls_mbedtls.c
index 8503523b6d..f51cf43b1c 100644
--- a/libavformat/tls_mbedtls.c
+++ b/libavformat/tls_mbedtls.c
@@ -309,6 +309,8 @@ static int tls_read(URLContext *h, uint8_t *buf, int size)
TLSContext *tls_ctx = h->priv_data;
int ret;
+tls_ctx->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
+tls_ctx->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
if ((ret = mbedtls_ssl_read(_ctx->ssl_context, buf, size)) > 0) {
// return read length
return ret;
@@ -322,6 +324,8 @@ static int tls_write(URLContext *h, const uint8_t *buf, int
size)
TLSContext *tls_ctx = h->priv_data;
int ret;
+tls_ctx->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
+tls_ctx->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
if ((ret = mbedtls_ssl_write(_ctx->ssl_context, buf, size)) > 0) {
// return written length
return ret;
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH] lavf/matroskaenc: sort options by name
> On Apr 7, 2024, at 14:16, Anton Khirnov wrote: > > Quoting Andreas Rheinhardt (2024-04-06 13:25:49) >> See https://ffmpeg.org/pipermail/ffmpeg-devel/2024-February/320849.html >> Additionally I do not agree that sorting options by name is the best >> way; it should be sorted by what are (believed to be) the most commonly >> used options. > > +1 https://patchwork.ffmpeg.org/project/ffmpeg/patch/20240106165246.274472-1-stefa...@gmail.com/ I have the same consideration in another patch. Maybe group related options together than sort whole options. > > -- > Anton Khirnov > ___ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe". ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH v2] libavformat/libsrt.c: Add statistics option to output SRT statistics with av_log()
> 在 2024年3月19日，下午7:51，Ryan McCartney 写道：
>
> ---
> libavformat/libsrt.c | 37 +
> 1 file changed, 37 insertions(+)
>
> diff --git a/libavformat/libsrt.c b/libavformat/libsrt.c
> index a7aafea536..80d52b1737 100644
> --- a/libavformat/libsrt.c
> +++ b/libavformat/libsrt.c
> @@ -56,6 +56,8 @@ typedef struct SRTContext {
> int eid;
> int64_t rw_timeout;
> int64_t listen_timeout;
> +int64_t lastStatsTime;
> +int64_t stats;
> int recv_buffer_size;
> int send_buffer_size;
>
> @@ -100,6 +102,7 @@ typedef struct SRTContext {
> static const AVOption libsrt_options[] = {
> { "timeout","Timeout of socket I/O operations (in microseconds)",
> OFFSET(rw_timeout), AV_OPT_TYPE_INT64, { .i64 = -1 },
> -1, INT64_MAX, .flags = D|E },
> { "listen_timeout", "Connection awaiting timeout (in microseconds)" ,
> OFFSET(listen_timeout), AV_OPT_TYPE_INT64, { .i64 = -1 },
> -1, INT64_MAX, .flags = D|E },
> +{ "stats", "Show SRT statistics in the log output",
>OFFSET(stats),AV_OPT_TYPE_INT64, { .i64 = -1
> }, -1, INT64_MAX, .flags = D|E },
> { "send_buffer_size", "Socket send buffer size (in bytes)",
> OFFSET(send_buffer_size), AV_OPT_TYPE_INT, { .i64 = -1
> }, -1, INT_MAX, .flags = D|E },
> { "recv_buffer_size", "Socket receive buffer size (in bytes)",
> OFFSET(recv_buffer_size), AV_OPT_TYPE_INT, { .i64 = -1
> }, -1, INT_MAX, .flags = D|E },
> { "pkt_size", "Maximum SRT packet size",
> OFFSET(payload_size), AV_OPT_TYPE_INT, { .i64 = -1
> }, -1, SRT_LIVE_MAX_PAYLOAD_SIZE, .flags = D|E, .unit = "payload_size" },
> @@ -158,6 +161,29 @@ static int libsrt_neterrno(URLContext *h)
> return os_errno ? AVERROR(os_errno) : AVERROR_UNKNOWN;
> }
>
> +static int libsrt_stats(URLContext *h,int read)
> +{
> +SRTContext *s = h->priv_data;
> +SRT_TRACEBSTATS trace;
> +
> +int ret = srt_bistats(s->fd, , 0,1);
> +int64_t timeNow = trace.msTimeStamp;
> +int64_t timeNext = s->lastStatsTime + s->stats;
> +
> +if((ret >= 0) && (timeNow > timeNext )){
> +s->lastStatsTime = timeNow;
> +
> +if(read > 0){
> +av_log(h, AV_LOG_INFO, "[srt-stats] rate=%.2fMbps bw=%.2fMbps
> rtt=%.2fms total=%jdpkts retrans=%jdpkts loss=%jdpkts \n",
> trace.mbpsRecvRate,trace.mbpsBandwidth,trace.msRTT,trace.pktRecvTotal,trace.pktRcvRetrans,trace.pktRcvLossTotal);
> +}
> +else{
> +av_log(h, AV_LOG_INFO, "[srt-stats] rate=%.2fMbps bw=%.2fMbps
> rtt=%.2fms total=%jdpkts retrans=%jdpkts loss=%jdpkts \n",
> trace.mbpsSendRate,trace.mbpsBandwidth,trace.msRTT,trace.pktSentTotal,trace.pktRetrans,trace.pktSndLossTotal);
> +}
> +}
> +
> +return 0;
> +}
The implementation is simple but not flexible enough. We need more than print.
I have this issue on my mind but don’t figure out a solution yet.
> +
> static int libsrt_getsockopt(URLContext *h, int fd, SRT_SOCKOPT optname,
> const char * optnamestr, void * optval, int * optlen)
> {
> if (srt_getsockopt(fd, 0, optname, optval, optlen) < 0) {
> @@ -557,6 +583,9 @@ static int libsrt_open(URLContext *h, const char *uri,
> int flags)
> goto err;
> }
> }
> +if (av_find_info_tag(buf, sizeof(buf), "stats", p)) {
> +s->stats = strtol(buf, NULL, 10);
> +}
> #if SRT_VERSION_VALUE >= 0x010302
> if (av_find_info_tag(buf, sizeof(buf), "enforced_encryption", p)) {
> s->enforced_encryption = strtol(buf, NULL, 10);
> @@ -686,6 +715,10 @@ static int libsrt_read(URLContext *h, uint8_t *buf, int
> size)
> SRTContext *s = h->priv_data;
> int ret;
>
> +if(s->stats > 0){
> +libsrt_stats(h,1);
> +}
> +
> if (!(h->flags & AVIO_FLAG_NONBLOCK)) {
> ret = libsrt_network_wait_fd_timeout(h, s->eid, 0, h->rw_timeout,
> >interrupt_callback);
> if (ret)
> @@ -705,6 +738,10 @@ static int libsrt_write(URLContext *h, const uint8_t
> *buf, int size)
> SRTContext *s = h->priv_data;
> int ret;
>
> +if(s->stats > 0){
> +libsrt_stats(h,0);
> +}
> +
> if (!(h->flags & AVIO_FLAG_NONBLOCK)) {
> ret = libsrt_network_wait_fd_timeout(h, s->eid, 1, h->rw_timeout,
> >interrupt_callback);
> if (ret)
> --
> 2.43.0
>
>
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject
[FFmpeg-devel] [PATCH] avcodec/h264_parse: Fix error code in decode_extradata
From: Zhao Zhili
---
libavcodec/h264_parse.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libavcodec/h264_parse.c b/libavcodec/h264_parse.c
index 3378650fd5..09d3b9dc05 100644
--- a/libavcodec/h264_parse.c
+++ b/libavcodec/h264_parse.c
@@ -468,7 +468,7 @@ int ff_h264_decode_extradata(const uint8_t *data, int size,
H264ParamSets *ps,
int ret;
if (!data || size <= 0)
-return -1;
+return AVERROR(EINVAL);
if (data[0] == 1) {
int i, cnt, nalsize;
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH] tests: Remove fate-libx265-hdr10
Ping. Is it OK to apply the patch as it is? > On Mar 23, 2024, at 06:36, Jan Ekström wrote: > > On Fri, Mar 22, 2024 at 10:18 PM Anton Khirnov wrote: >> >> Quoting Zhao Zhili (2024-03-22 13:43:43) >>> From: Zhao Zhili >>> >>> The test depends on the compile option of x265. It failed when >>> HIGH_BIT_DEPTH isn't enabled. It also failed when asan is enabled >>> because of memory issue inside of x265, which I don't think can >>> be fixed within FFmpeg. >> >> I suggested some time ago we should mark x265 as experimental. It didn't >> receive much enthusiasm, and some probably considered it a joke, but I >> was mostly serious. It has major memory safety issues that have been >> ignored for years. >> > > Yea, I recall when I was testing my patch set with valgrind about half > a year ago, some of the memory issues showed up there as well. The > problem being that most users are happily oblivious to these issues as > long as the piece of software "seems to work". Which is a line that > x265 does cross. > > Thus at least on my side the reason for not much enthusiasm for > marking it experimental at this point came from it being on the output > side (and thus not handling random input/being a parser) as well as it > most likely receiving much less understanding from users than just > complaints about command lines which had worked for ages suddenly > breaking. > > Jan > ___ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe". ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH] avcodec/h264_mp4toannexb: Fix heap buffer overflow
From: Zhao Zhili
Fixes: out of array write
Fixes:
64407/clusterfuzz-testcase-minimized-ffmpeg_BSF_H264_MP4TOANNEXB_fuzzer-4966763443650560
mp4toannexb_filter counts the number of bytes needed in the first
pass and allocate the memory, then do memcpy in the second pass.
Update sps/pps size in the loop makes the count invalid in the
case of SPS/PPS occur after IDR slice. This patch process in-band
SPS/PPS before the two pass loops.
---
libavcodec/bsf/h264_mp4toannexb.c | 59 ---
1 file changed, 46 insertions(+), 13 deletions(-)
diff --git a/libavcodec/bsf/h264_mp4toannexb.c
b/libavcodec/bsf/h264_mp4toannexb.c
index 120241c892..92af6a6881 100644
--- a/libavcodec/bsf/h264_mp4toannexb.c
+++ b/libavcodec/bsf/h264_mp4toannexb.c
@@ -208,6 +208,49 @@ static int h264_mp4toannexb_save_ps(uint8_t **dst, int
*dst_size,
return 0;
}
+static int h264_mp4toannexb_filter_ps(H264BSFContext *s,
+ const uint8_t *buf,
+ const uint8_t *buf_end)
+{
+int sps_count = 0;
+int pps_count = 0;
+uint8_t unit_type;
+
+do {
+uint32_t nal_size = 0;
+
+/* possible overread ok due to padding */
+for (int i = 0; i < s->length_size; i++)
+nal_size = (nal_size << 8) | buf[i];
+
+buf += s->length_size;
+
+/* This check requires the cast as the right side might
+ * otherwise be promoted to an unsigned value. */
+if ((int64_t)nal_size > buf_end - buf)
+return AVERROR_INVALIDDATA;
+
+if (!nal_size)
+continue;
+
+unit_type = *buf & 0x1f;
+
+if (unit_type == H264_NAL_SPS) {
+h264_mp4toannexb_save_ps(>sps, >sps_size, >sps_buf_size,
buf,
+ nal_size, !sps_count);
+sps_count++;
+} else if (unit_type == H264_NAL_PPS) {
+h264_mp4toannexb_save_ps(>pps, >pps_size, >pps_buf_size,
buf,
+ nal_size, !pps_count);
+pps_count++;
+}
+
+buf += nal_size;
+} while (buf < buf_end);
+
+return 0;
+}
+
static int h264_mp4toannexb_init(AVBSFContext *ctx)
{
int extra_size = ctx->par_in->extradata_size;
@@ -263,14 +306,14 @@ static int h264_mp4toannexb_filter(AVBSFContext *ctx,
AVPacket *opkt)
}
buf_end = in->data + in->size;
+ret = h264_mp4toannexb_filter_ps(s, in->data, buf_end);
+if (ret < 0)
+goto fail;
#define LOG_ONCE(...) \
if (j) \
av_log(__VA_ARGS__)
for (int j = 0; j < 2; j++) {
-int sps_count = 0;
-int pps_count = 0;
-
buf = in->data;
new_idr = s->new_idr;
sps_seen = s->idr_sps_seen;
@@ -301,18 +344,8 @@ static int h264_mp4toannexb_filter(AVBSFContext *ctx,
AVPacket *opkt)
if (unit_type == H264_NAL_SPS) {
sps_seen = new_idr = 1;
-if (!j) {
-h264_mp4toannexb_save_ps(>sps, >sps_size,
>sps_buf_size,
- buf, nal_size, !sps_count);
-sps_count++;
-}
} else if (unit_type == H264_NAL_PPS) {
pps_seen = new_idr = 1;
-if (!j) {
-h264_mp4toannexb_save_ps(>pps, >pps_size,
>pps_buf_size,
- buf, nal_size, !pps_count);
-pps_count++;
-}
/* if SPS has not been seen yet, prepend the AVCC one to PPS */
if (!sps_seen) {
if (!s->sps_size) {
--
2.25.1
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH] libavformat\file:Fix duplicate JNI symbols
On 2024/3/24 21:23, Andreas Rheinhardt wrote: ad...@shaoxia.xyz: From: 联盟少侠 The errors indicate that there are multiple definitions of several JNI (Java Native Interface) symbols in the FFmpeg library. Specifically, the linker ld.lld has found duplicate definitions for the following symbols: - ff_jni_get_env - ff_jni_jstring_to_utf_chars - ff_jni_utf_chars_to_jstring - ff_jni_exception_get_summary - ff_jni_exception_check - ff_jni_init_jfields - ff_jni_reset_jfields These symbols are defined in both libavcodec.a and libavformat.a archives, leading to conflicts during the linking process. The duplicates are found in the same source file ffjni.c, but they are being compiled into two different static libraries. To resolve these errors, ensure that each symbol is defined only once across all linked libraries, or if these symbols are required in both libraries, they should be marked as weak symbols to allow the linker to handle them appropriately. Additionally, check for any build script misconfigurations that may be causing the same file to be included in multiple libraries. --- libavformat/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/file.c b/libavformat/file.c index dd5819c..fa13ae9 100644 --- a/libavformat/file.c +++ b/libavformat/file.c @@ -43,7 +43,7 @@ #if CONFIG_ANDROID_CONTENT_PROTOCOL #include #include "libavcodec/jni.h" -#include "libavcodec/ffjni.c" +#include "libavcodec/ffjni.h" #endif How can this even happen? In case of static builds with the android content protocol enabled, libavformat/file.o provides all the symbols that libavcodec/ffjni.o also provides. Given that the former is part of libavformat.a and this is linked before libavcodec.a, the stuff from the former is used and the ffjni.o in libavcodec.a should not be pulled in at all. I guess the error is triggered by creating a shared library from combine all static libraries via -Wl,--whole-archive. It's a common practice to reduce the number of shared libraries in mobile app. - Andreas ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe". ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH] libavformat\file:Fix duplicate JNI symbols
On 2024/3/24 10:16, Zhao Zhili wrote: diff --git a/libavformat/file.c b/libavformat/file.c index dd5819c..fa13ae9 100644 --- a/libavformat/file.c +++ b/libavformat/file.c @@ -43,7 +43,7 @@ #if CONFIG_ANDROID_CONTENT_PROTOCOL #include #include "libavcodec/jni.h" -#include "libavcodec/ffjni.c" +#include "libavcodec/ffjni.h" #endif This just break --enable-shared, so NACK. The following patch should work for static and shared both. https://ffmpeg.org/pipermail/ffmpeg-devel/2024-March/324138.html ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH] avformat/file: Fix duplicate symbol ff_jni_*
From: Zhao Zhili --- libavformat/Makefile | 1 + libavformat/ffjni.c | 1 + libavformat/file.c | 3 +-- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 libavformat/ffjni.c diff --git a/libavformat/Makefile b/libavformat/Makefile index 44aa485029..e10ad436a8 100644 --- a/libavformat/Makefile +++ b/libavformat/Makefile @@ -724,6 +724,7 @@ OBJS-$(CONFIG_LIBZMQ_PROTOCOL) += libzmq.o # Objects duplicated from other libraries for shared builds SHLIBOBJS+= log2_tab.o to_upper4.o +SHLIBOBJS-$(CONFIG_ANDROID_CONTENT_PROTOCOL) += ffjni.o SHLIBOBJS-$(CONFIG_ISO_MEDIA)+= mpegaudiotabs.o SHLIBOBJS-$(CONFIG_FLV_MUXER)+= mpeg4audio_sample_rates.o SHLIBOBJS-$(CONFIG_HLS_DEMUXER) += ac3_channel_layout_tab.o diff --git a/libavformat/ffjni.c b/libavformat/ffjni.c new file mode 100644 index 00..a5285564b2 --- /dev/null +++ b/libavformat/ffjni.c @@ -0,0 +1 @@ +#include "libavcodec/ffjni.c" diff --git a/libavformat/file.c b/libavformat/file.c index dd5819c06f..f80cb4b37a 100644 --- a/libavformat/file.c +++ b/libavformat/file.c @@ -41,9 +41,8 @@ #include "os_support.h" #include "url.h" #if CONFIG_ANDROID_CONTENT_PROTOCOL -#include +#include "libavcodec/ffjni.h" #include "libavcodec/jni.h" -#include "libavcodec/ffjni.c" #endif -- 2.34.1 ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH] libavformat\file:Fix duplicate JNI symbols
On 2024/3/24 03:05, ad...@shaoxia.xyz wrote: From: 联盟少侠 Could you configure git to avoid Chinese character? The errors indicate that there are multiple definitions of several JNI (Java Native Interface) symbols in the FFmpeg library. Specifically, the linker ld.lld has found duplicate definitions for the following symbols: How to reproduce the error? - ff_jni_get_env - ff_jni_jstring_to_utf_chars - ff_jni_utf_chars_to_jstring - ff_jni_exception_get_summary - ff_jni_exception_check - ff_jni_init_jfields - ff_jni_reset_jfields These symbols are defined in both libavcodec.a and libavformat.a archives, leading to conflicts during the linking process. The duplicates are found in the same source file ffjni.c, but they are being compiled into two different static libraries. To resolve these errors, ensure that each symbol is defined only once across all linked libraries, or if these symbols are required in both libraries, they should be marked as weak symbols to allow the linker to handle them appropriately. Additionally, check for any build script misconfigurations that may be causing the same file to be included in multiple libraries. --- libavformat/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/file.c b/libavformat/file.c index dd5819c..fa13ae9 100644 --- a/libavformat/file.c +++ b/libavformat/file.c @@ -43,7 +43,7 @@ #if CONFIG_ANDROID_CONTENT_PROTOCOL #include #include "libavcodec/jni.h" -#include "libavcodec/ffjni.c" +#include "libavcodec/ffjni.h" #endif This just break --enable-shared, so NACK. ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH] avcodec/libx264: Remove unused variable
> -Original Message-
> From: ffmpeg-devel On Behalf Of Andreas
> Rheinhardt
> Sent: 2024年3月22日 21:03
> To: ffmpeg-devel@ffmpeg.org
> Cc: Andreas Rheinhardt
> Subject: [FFmpeg-devel] [PATCH] avcodec/libx264: Remove unused variable
>
> Signed-off-by: Andreas Rheinhardt
> ---
> libavcodec/libx264.c | 1 -
> 1 file changed, 1 deletion(-)
>
> diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c
> index 2653941eb9..3d195fa6b6 100644
> --- a/libavcodec/libx264.c
> +++ b/libavcodec/libx264.c
> @@ -867,7 +867,6 @@ static int save_sei(AVCodecContext *avctx, x264_nal_t
> *nal)
> #if CONFIG_LIBX264_ENCODER
> static int set_avcc_extradata(AVCodecContext *avctx, x264_nal_t *nal, int
> nnal)
> {
> -X264Context *x4 = avctx->priv_data;
> x264_nal_t *sps_nal = NULL;
> x264_nal_t *pps_nal = NULL;
> uint8_t *p, *sps;
LGTM.
> --
> 2.40.1
>
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH v6] avcodec/libx264: fix extradata when config annexb=0
> On Mar 22, 2024, at 18:36, Anton Khirnov wrote: > >> Subject: Re: [FFmpeg-devel] [PATCH v6] avcodec/libx264: fix extradata when >> config annexb=0 > > This could really use more explanation on what exactly is broken. Added detailed description and pushed, thanks! > > -- > Anton Khirnov > ___ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe". ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
[FFmpeg-devel] [PATCH] tests: Remove fate-libx265-hdr10
From: Zhao Zhili The test depends on the compile option of x265. It failed when HIGH_BIT_DEPTH isn't enabled. It also failed when asan is enabled because of memory issue inside of x265, which I don't think can be fixed within FFmpeg. --- tests/fate/enc_external.mak | 5 - tests/ref/fate/libx265-hdr10 | 16 2 files changed, 21 deletions(-) delete mode 100644 tests/ref/fate/libx265-hdr10 diff --git a/tests/fate/enc_external.mak b/tests/fate/enc_external.mak index 30021efbcd..4095a4b51a 100644 --- a/tests/fate/enc_external.mak +++ b/tests/fate/enc_external.mak @@ -12,10 +12,5 @@ FATE_ENC_EXTERNAL-$(call ENCDEC, LIBX264 HEVC, MOV, LIBX264_HDR10 HEVC_DEMUXER H fate-libx264-hdr10: CMD = enc_external $(TARGET_SAMPLES)/hevc/hdr10_plus_h265_sample.hevc \ mp4 "-c:v libx264" "-show_frames -show_entries frame=side_data_list -of flat" -# test for x265 MDCV and CLL passthrough during encoding -FATE_ENC_EXTERNAL-$(call ENCDEC, LIBX265 HEVC, MOV, HEVC_DEMUXER) += fate-libx265-hdr10 -fate-libx265-hdr10: CMD = enc_external $(TARGET_SAMPLES)/hevc/hdr10_plus_h265_sample.hevc \ -mp4 "-c:v libx265" "-show_frames -show_entries frame=side_data_list -of flat" - FATE_SAMPLES_FFMPEG_FFPROBE += $(FATE_ENC_EXTERNAL-yes) fate-enc-external: $(FATE_ENC_EXTERNAL-yes) diff --git a/tests/ref/fate/libx265-hdr10 b/tests/ref/fate/libx265-hdr10 deleted file mode 100644 index 571c837cac..00 --- a/tests/ref/fate/libx265-hdr10 +++ /dev/null @@ -1,16 +0,0 @@ -frames.frame.0.side_data_list.side_data.0.side_data_type="H.26[45] User Data Unregistered SEI message" -frames.frame.0.side_data_list.side_data.1.side_data_type="H.26[45] User Data Unregistered SEI message" -frames.frame.0.side_data_list.side_data.2.side_data_type="Mastering display metadata" -frames.frame.0.side_data_list.side_data.2.red_x="13250/5" -frames.frame.0.side_data_list.side_data.2.red_y="34500/5" -frames.frame.0.side_data_list.side_data.2.green_x="7500/5" -frames.frame.0.side_data_list.side_data.2.green_y="3000/5" -frames.frame.0.side_data_list.side_data.2.blue_x="34000/5" -frames.frame.0.side_data_list.side_data.2.blue_y="16000/5" -frames.frame.0.side_data_list.side_data.2.white_point_x="15635/5" -frames.frame.0.side_data_list.side_data.2.white_point_y="16450/5" -frames.frame.0.side_data_list.side_data.2.min_luminance="50/1" -frames.frame.0.side_data_list.side_data.2.max_luminance="1000/1" -frames.frame.0.side_data_list.side_data.3.side_data_type="Content light level metadata" -frames.frame.0.side_data_list.side_data.3.max_content=1000 -frames.frame.0.side_data_list.side_data.3.max_average=200 -- 2.25.1 ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH v10 14/14] avcodec/libx265: add support for writing out CLL and MDCV
> On Mar 19, 2024, at 05:31, Jan Ekström wrote:
>
> The newer of these two are the separate integers for content light
> level, introduced in 3952bf3e98c76c31594529a3fe34e056d3e3e2ea ,
> with X265_BUILD 75. As we already require X265_BUILD of at least
> 89, no further conditions are required.
The fate test not only depends on the version of x265, but also depends on
the compile option of x265. The test failed when HIGH_BIT_DEPTH isn’t enabled.
It also failed when asan is enabled because of memory issue inside of x265.
> ---
> libavcodec/libx265.c | 89
> tests/fate/enc_external.mak | 5 ++
> tests/ref/fate/libx265-hdr10 | 16 +++
> 3 files changed, 110 insertions(+)
> create mode 100644 tests/ref/fate/libx265-hdr10
>
> diff --git a/libavcodec/libx265.c b/libavcodec/libx265.c
> index 92183b9ca2..70ec6d3539 100644
> --- a/libavcodec/libx265.c
> +++ b/libavcodec/libx265.c
> @@ -28,9 +28,11 @@
> #include
>
> #include "libavutil/avassert.h"
> +#include "libavutil/bprint.h"
> #include "libavutil/buffer.h"
> #include "libavutil/internal.h"
> #include "libavutil/common.h"
> +#include "libavutil/mastering_display_metadata.h"
> #include "libavutil/opt.h"
> #include "libavutil/pixdesc.h"
> #include "avcodec.h"
> @@ -176,6 +178,86 @@ static av_cold int
> libx265_param_parse_int(AVCodecContext *avctx,
> return 0;
> }
>
> +static int handle_mdcv(const AVClass **avcl, const x265_api *api,
> + x265_param *params,
> + const AVMasteringDisplayMetadata *mdcv)
> +{
> +int ret = AVERROR_BUG;
> +AVBPrint buf;
> +av_bprint_init(, 0, AV_BPRINT_SIZE_AUTOMATIC);
> +
> +// G(%hu,%hu)B(%hu,%hu)R(%hu,%hu)WP(%hu,%hu)L(%u,%u)
> +av_bprintf(
> +,
> +"G(%"PRId64",%"PRId64")B(%"PRId64",%"PRId64")R(%"PRId64",%"PRId64")"
> +"WP(%"PRId64",%"PRId64")L(%"PRId64",%"PRId64")",
> +av_rescale_q(1, mdcv->display_primaries[1][0], (AVRational){ 1,
> 5 }),
> +av_rescale_q(1, mdcv->display_primaries[1][1], (AVRational){ 1,
> 5 }),
> +av_rescale_q(1, mdcv->display_primaries[2][0], (AVRational){ 1,
> 5 }),
> +av_rescale_q(1, mdcv->display_primaries[2][1], (AVRational){ 1,
> 5 }),
> +av_rescale_q(1, mdcv->display_primaries[0][0], (AVRational){ 1,
> 5 }),
> +av_rescale_q(1, mdcv->display_primaries[0][1], (AVRational){ 1,
> 5 }),
> +av_rescale_q(1, mdcv->white_point[0], (AVRational){ 1, 5 }),
> +av_rescale_q(1, mdcv->white_point[1], (AVRational){ 1, 5 }),
> +av_rescale_q(1, mdcv->max_luminance, (AVRational){ 1, 1 }),
> +av_rescale_q(1, mdcv->min_luminance, (AVRational){ 1, 1 }));
> +
> +if (!av_bprint_is_complete()) {
> +av_log(avcl, AV_LOG_ERROR,
> + "MDCV string too long for its available space!\n");
> +ret = AVERROR(ENOMEM);
> +goto end;
> +}
> +
> +if (api->param_parse(params, "master-display", buf.str) ==
> +X265_PARAM_BAD_VALUE) {
> +av_log(avcl, AV_LOG_ERROR,
> + "Invalid value \"%s\" for param \"master-display\".\n",
> + buf.str);
> +ret = AVERROR(EINVAL);
> +goto end;
> +}
> +
> +ret = 0;
> +
> +end:
> +av_bprint_finalize(, NULL);
> +
> +return ret;
> +}
> +
> +static int handle_side_data(AVCodecContext *avctx, const x265_api *api,
> +x265_param *params)
> +{
> +const AVFrameSideData *cll_sd =
> +av_frame_side_data_get(
> +(const AVFrameSideData **)avctx->decoded_side_data,
> +avctx->nb_decoded_side_data, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL);
> +const AVFrameSideData *mdcv_sd =
> +av_frame_side_data_get(
> +(const AVFrameSideData **)avctx->decoded_side_data,
> +avctx->nb_decoded_side_data,
> +AV_FRAME_DATA_MASTERING_DISPLAY_METADATA);
> +
> +if (cll_sd) {
> +const AVContentLightMetadata *cll =
> +(AVContentLightMetadata *)cll_sd->data;
> +
> +params->maxCLL = cll->MaxCLL;
> +params->maxFALL = cll->MaxFALL;
> +}
> +
> +if (mdcv_sd) {
> +int ret = handle_mdcv(
> +>av_class, api, params,
> +(AVMasteringDisplayMetadata *)mdcv_sd->data);
> +if (ret < 0)
> +return ret;
> +}
> +
> +return 0;
> +}
> +
> static av_cold int libx265_encode_init(AVCodecContext *avctx)
> {
> libx265Context *ctx = avctx->priv_data;
> @@ -336,6 +418,13 @@ FF_ENABLE_DEPRECATION_WARNINGS
> return AVERROR_BUG;
> }
>
> +ret = handle_side_data(avctx, ctx->api, ctx->params);
> +if (ret < 0) {
> +av_log(avctx, AV_LOG_ERROR, "Failed handling side data! (%s)\n",
> + av_err2str(ret));
> +return ret;
> +}
> +
> if (ctx->crf >= 0) {
> char crf[6];
>
> diff --git a/tests/fate/enc_external.mak
Re: [FFmpeg-devel] [PATCH v6] avcodec/libx264: fix extradata when config annexb=0
Ping. Will apply this week if no more comments.
> -Original Message-
> From: ffmpeg-devel On Behalf Of Zhao Zhili
> Sent: 2024年3月17日 11:35
> To: ffmpeg-devel@ffmpeg.org
> Cc: Zhao Zhili
> Subject: [FFmpeg-devel] [PATCH v6] avcodec/libx264: fix extradata when config
> annexb=0
>
> From: Zhao Zhili
>
> ---
> v6: use av_assert2
> select golomb in configure
> conditional compile in case of CONFIG_LIBX264_ENCODER=0
> v5: Decode chroma_format_idc directly instead of
> ff_h264_decode_seq_parameter_set
> v4: Fix missing SEI in set_avcc_extradata
> v3: Remove unnecessary inclusion
>
> configure| 2 +-
> libavcodec/libx264.c | 166 ---
> 2 files changed, 143 insertions(+), 25 deletions(-)
>
> diff --git a/configure b/configure
> index 2b4c4ec9a2..d9d2183a47 100755
> --- a/configure
> +++ b/configure
> @@ -3484,7 +3484,7 @@ libwebp_encoder_deps="libwebp"
> libwebp_anim_encoder_deps="libwebp"
> libx262_encoder_deps="libx262"
> libx264_encoder_deps="libx264"
> -libx264_encoder_select="atsc_a53"
> +libx264_encoder_select="atsc_a53 golomb"
> libx264rgb_encoder_deps="libx264"
> libx264rgb_encoder_select="libx264_encoder"
> libx265_encoder_deps="libx265"
> diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c
> index 0997c4e134..ddec06d960 100644
> --- a/libavcodec/libx264.c
> +++ b/libavcodec/libx264.c
> @@ -38,6 +38,7 @@
> #include "packet_internal.h"
> #include "atsc_a53.h"
> #include "sei.h"
> +#include "golomb.h"
>
> #include
> #include
> @@ -847,6 +848,144 @@ static int convert_pix_fmt(enum AVPixelFormat pix_fmt)
> return 0;
> }
>
> +static int save_sei(AVCodecContext *avctx, x264_nal_t *nal)
> +{
> +X264Context *x4 = avctx->priv_data;
> +
> +av_log(avctx, AV_LOG_INFO, "%s\n", nal->p_payload + 25);
> +x4->sei_size = nal->i_payload;
> +x4->sei = av_malloc(x4->sei_size);
> +if (!x4->sei)
> +return AVERROR(ENOMEM);
> +
> +memcpy(x4->sei, nal->p_payload, nal->i_payload);
> +
> +return 0;
> +}
> +
> +#if CONFIG_LIBX264_ENCODER
> +static int set_avcc_extradata(AVCodecContext *avctx, x264_nal_t *nal, int
> nnal)
> +{
> +X264Context *x4 = avctx->priv_data;
> +x264_nal_t *sps_nal = NULL;
> +x264_nal_t *pps_nal = NULL;
> +uint8_t *p, *sps;
> +int ret;
> +
> +/* We know it's in the order of SPS/PPS/SEI, but it's not documented in
> x264 API.
> + * The x264 param i_sps_id implies there is a single pair of SPS/PPS.
> + */
> +for (int i = 0; i < nnal; i++) {
> +switch (nal[i].i_type) {
> +case NAL_SPS:
> +sps_nal = [i];
> +break;
> +case NAL_PPS:
> +pps_nal = [i];
> +break;
> +case NAL_SEI:
> +ret = save_sei(avctx, [i]);
> +if (ret < 0)
> +return ret;
> +break;
> +}
> +}
> +if (!sps_nal || !pps_nal)
> +return AVERROR_EXTERNAL;
> +
> +avctx->extradata_size = sps_nal->i_payload + pps_nal->i_payload + 7;
> +avctx->extradata = av_mallocz(avctx->extradata_size +
> AV_INPUT_BUFFER_PADDING_SIZE);
> +if (!avctx->extradata)
> +return AVERROR(ENOMEM);
> +
> +// Now create AVCDecoderConfigurationRecord
> +p = avctx->extradata;
> +// Skip size part
> +sps = sps_nal->p_payload + 4;
> +*p++ = 1; // version
> +*p++ = sps[1]; // AVCProfileIndication
> +*p++ = sps[2]; // profile_compatibility
> +*p++ = sps[3]; // AVCLevelIndication
> +*p++ = 0xFF;
> +*p++ = 0xE0 | 0x01; // 3 bits reserved (111) + 5 bits number of sps
> +memcpy(p, sps_nal->p_payload + 2, sps_nal->i_payload - 2);
> +// Make sps has AV_INPUT_BUFFER_PADDING_SIZE padding, so it can be used
> +// with GetBitContext
> +sps = p + 2;
> +p += sps_nal->i_payload - 2;
> +*p++ = 1;
> +memcpy(p, pps_nal->p_payload + 2, pps_nal->i_payload - 2);
> +p += pps_nal->i_payload - 2;
> +
> +if (sps[3] != 66 && sps[3] != 77 && sps[3] != 88) {
> +GetBitContext gbc;
> +int chroma_format_idc;
> +int bit_depth_luma_minus8, bit_depth_chroma_minus8;
> +
> +/* It's not possible to have emulation prevention byte before
> + * bit_depth_chroma_minus8 due to the range of sps id,
> chroma_format_idc
> + * and so
Re: [FFmpeg-devel] [PATCH 3/3] Revert "avcodec/h264_mp4toannexb_bsf: fix missing PS before IDR frames"
> -Original Message- > From: ffmpeg-devel On Behalf Of Michael > Niedermayer > Sent: 2024年3月20日 21:02 > To: FFmpeg development discussions and patches > Subject: Re: [FFmpeg-devel] [PATCH 3/3] Revert "avcodec/h264_mp4toannexb_bsf: > fix missing PS before IDR frames" > > On Wed, Mar 20, 2024 at 02:41:05PM +0800, Zhao Zhili wrote: > > > > > > > On Mar 20, 2024, at 10:19, Michael Niedermayer > > > wrote: > > > > > > This reverts commit d3aa0cd16f5e952bc346b7c74b4dcba95151a63a. > > > > > > Fixes: out of array write > > > Fixes: > > > 64407/clusterfuzz-testcase-minimized-ffmpeg_BSF_H264_MP4TOANNEXB_fuzzer-4966763443650560 > > > > > > The bsf code performs 2 iterations, the first counts how much space is > > > needed > > > than allocates > > > and the 2nd pass copies into teh allocated space > > > > > > The reverted code reallocates sps/pps in the first pass in a data > > > dependant way that leaves > > > the 2nd pass in a different state then the first > > > > Sorry for the break. How to access the fuzz report details? > > ossfuzz made this one public already > https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=64407 I can't open the "Detailed Report" link. Does it provide asan details? I'm currently tied up with daily jobs, I need time to learn how to use oss-fuzz. Feel free to revert the commit. I would be very grateful if anyone has a quick fix for it. Broken file is less severity than heap-buffer-overflow, but still a serious problem. > > thx > > [...] > -- > Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB > > Modern terrorism, a quick summary: Need oil, start war with country that > has oil, kill hundread thousand in war. Let country fall into chaos, > be surprised about raise of fundamantalists. Drop more bombs, kill more > people, be surprised about them taking revenge and drop even more bombs > and strip your own citizens of their rights and freedoms. to be continued ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH 3/3] Revert "avcodec/h264_mp4toannexb_bsf: fix missing PS before IDR frames"
> On Mar 20, 2024, at 10:19, Michael Niedermayer wrote:
>
> This reverts commit d3aa0cd16f5e952bc346b7c74b4dcba95151a63a.
>
> Fixes: out of array write
> Fixes:
> 64407/clusterfuzz-testcase-minimized-ffmpeg_BSF_H264_MP4TOANNEXB_fuzzer-4966763443650560
>
> The bsf code performs 2 iterations, the first counts how much space is needed
> than allocates
> and the 2nd pass copies into teh allocated space
>
> The reverted code reallocates sps/pps in the first pass in a data dependant
> way that leaves
> the 2nd pass in a different state then the first
Sorry for the break. How to access the fuzz report details?
Without the patch, it generates broken files in those cases. I want to dig
further to fix it.
>
> Found-by: continuous fuzzing process
> https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
> Signed-off-by: Michael Niedermayer
> ---
> libavcodec/bsf/h264_mp4toannexb.c | 83 +++
> tests/fate/h264.mak | 5 --
> 2 files changed, 6 insertions(+), 82 deletions(-)
>
> diff --git a/libavcodec/bsf/h264_mp4toannexb.c
> b/libavcodec/bsf/h264_mp4toannexb.c
> index 120241c892..b99de39ce9 100644
> --- a/libavcodec/bsf/h264_mp4toannexb.c
> +++ b/libavcodec/bsf/h264_mp4toannexb.c
> @@ -36,8 +36,6 @@ typedef struct H264BSFContext {
> uint8_t *pps;
> int sps_size;
> int pps_size;
> -unsigned sps_buf_size;
> -unsigned pps_buf_size;
> uint8_t length_size;
> uint8_t new_idr;
> uint8_t idr_sps_seen;
> @@ -133,33 +131,16 @@ pps:
> memset(out + total_size, 0, padding);
>
> if (pps_offset) {
> -uint8_t *sps;
> -
> +s->sps = out;
> s->sps_size = pps_offset;
> -sps = av_fast_realloc(s->sps, >sps_buf_size, s->sps_size);
> -if (!sps) {
> -av_free(out);
> -return AVERROR(ENOMEM);
> -}
> -s->sps = sps;
> -memcpy(s->sps, out, s->sps_size);
> } else {
> av_log(ctx, AV_LOG_WARNING,
>"Warning: SPS NALU missing or invalid. "
>"The resulting stream may not play.\n");
> }
> if (pps_offset < total_size) {
> -uint8_t *pps;
> -
> +s->pps = out + pps_offset;
> s->pps_size = total_size - pps_offset;
> -pps = av_fast_realloc(s->pps, >pps_buf_size, s->pps_size);
> -if (!pps) {
> -av_freep(>sps);
> -av_free(out);
> -return AVERROR(ENOMEM);
> -}
> -s->pps = pps;
> -memcpy(s->pps, out + pps_offset, s->pps_size);
> } else {
> av_log(ctx, AV_LOG_WARNING,
>"Warning: PPS NALU missing or invalid. "
> @@ -179,35 +160,6 @@ pps:
> return 0;
> }
>
> -static int h264_mp4toannexb_save_ps(uint8_t **dst, int *dst_size,
> -unsigned *dst_buf_size,
> -const uint8_t *nal, uint32_t nal_size,
> -int first)
> -{
> -static const uint8_t nalu_header[4] = { 0, 0, 0, 1 };
> -const int start_code_size = sizeof(nalu_header);
> -uint8_t *ptr;
> -uint32_t size;
> -
> -if (first)
> -size = 0;
> -else
> -size = *dst_size;
> -
> -ptr = av_fast_realloc(*dst, dst_buf_size, size + nal_size +
> start_code_size);
> -if (!ptr)
> -return AVERROR(ENOMEM);
> -
> -memcpy(ptr + size, nalu_header, start_code_size);
> -size += start_code_size;
> -memcpy(ptr + size, nal, nal_size);
> -size += nal_size;
> -
> -*dst = ptr;
> -*dst_size = size;
> -return 0;
> -}
> -
> static int h264_mp4toannexb_init(AVBSFContext *ctx)
> {
> int extra_size = ctx->par_in->extradata_size;
> @@ -268,9 +220,6 @@ static int h264_mp4toannexb_filter(AVBSFContext *ctx,
> AVPacket *opkt)
> if (j) \
> av_log(__VA_ARGS__)
> for (int j = 0; j < 2; j++) {
> -int sps_count = 0;
> -int pps_count = 0;
> -
> buf = in->data;
> new_idr = s->new_idr;
> sps_seen = s->idr_sps_seen;
> @@ -301,18 +250,8 @@ static int h264_mp4toannexb_filter(AVBSFContext *ctx,
> AVPacket *opkt)
>
> if (unit_type == H264_NAL_SPS) {
> sps_seen = new_idr = 1;
> -if (!j) {
> -h264_mp4toannexb_save_ps(>sps, >sps_size,
> >sps_buf_size,
> - buf, nal_size, !sps_count);
> -sps_count++;
> -}
> } else if (unit_type == H264_NAL_PPS) {
> pps_seen = new_idr = 1;
> -if (!j) {
> -h264_mp4toannexb_save_ps(>pps, >pps_size,
> >pps_buf_size,
> - buf, nal_size, !pps_count);
> -pps_count++;
> -}
> /* if SPS has not been seen yet, prepend the AVCC one to PPS
> */
> if (!sps_seen) {
>
[FFmpeg-devel] [PATCH] Changelog: mention ffplay with hwaccel decoding support
From: Zhao Zhili --- Changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/Changelog b/Changelog index e3ca52430c..c6e8f6bcaf 100644 --- a/Changelog +++ b/Changelog @@ -35,6 +35,7 @@ version : - AEA muxer - ffmpeg CLI loopback decoders - Support PacketTypeMetadata of PacketType in enhanced flv format +- ffplay with hwaccel decoding support (depends on vulkan renderer via libplacebo) version 6.1: -- 2.34.1 ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH v5 1/3] avformat/flvdec: support enhanced flv PacketTypeMetadata
> On Mar 18, 2024, at 17:14, Dennis Mungai wrote: > > Steven and Zhu Pengfei, > > Thank you for the patchset. Can this be back-ported to release/6.1? Only bug fix and security fix can be back ported to stable release. This patch is for new feature. > ___ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe". ___ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
Re: [FFmpeg-devel] [PATCH v2 2/6] avformat: add Android content resolver protocol support
Sorry for the long delay of review.
> On Mar 5, 2024, at 03:37, Matthieu Bouron wrote:
>
> On Tue, Feb 27, 2024 at 03:50:38PM +0100, Matthieu Bouron wrote:
>> Handles Android content-uri starting with content://.
>> ---
>
> [...]
>
> Handles Android content-uri starting with content://.
> ---
> configure | 2 +
> doc/APIchanges | 3 +
> libavcodec/jni.c| 1 +
> libavformat/Makefile| 1 +
> libavformat/file.c | 157
> libavformat/protocols.c | 1 +
> 6 files changed, 165 insertions(+)
>
> diff --git a/configure b/configure
> index bb5e630bad..790a1df7c8 100755
> --- a/configure
> +++ b/configure
> @@ -3655,6 +3655,8 @@ xcbgrab_indev_suggest="libxcb_shm libxcb_shape
> libxcb_xfixes"
> xv_outdev_deps="xlib_xv xlib_x11 xlib_xext"
>
> # protocols
> +android_content_protocol_deps="jni"
> +android_content_protocol_select="file_protocol"
> async_protocol_deps="threads"
> bluray_protocol_deps="libbluray"
> ffrtmpcrypt_protocol_conflict="librtmp_protocol"
> diff --git a/doc/APIchanges b/doc/APIchanges
> index 10f6667e9e..258e953ca6 100644
> --- a/doc/APIchanges
> +++ b/doc/APIchanges
> @@ -2,6 +2,9 @@ The last version increases of all libraries were on 2023-02-09
>
> API changes, most recent first:
>
> +2024-02-xx - xx - lavc 60.41.100 - jni.h
> + Add av_jni_set_android_app_ctx() and av_jni_get_android_app_ctx().
Should be added together with patch 1/6.
> +
> 2024-02-26 - xx - lavf 60.22.101 - avformat.h
>AV_DISPOSITION_DEPENDENT may now also be used for video streams
>intended to be merged with other video streams for presentation.
> diff --git a/libavcodec/jni.c b/libavcodec/jni.c
> index cfe95bd1ec..1193c608c3 100644
> --- a/libavcodec/jni.c
> +++ b/libavcodec/jni.c
> @@ -35,6 +35,7 @@
> #include "ffjni.h"
>
> static void *java_vm;
> +static void *android_app_ctx;
> static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
>
> int av_jni_set_java_vm(void *vm, void *log_ctx)
> diff --git a/libavformat/Makefile b/libavformat/Makefile
> index 4a380668bd..08fe98a535 100644
> --- a/libavformat/Makefile
> +++ b/libavformat/Makefile
> @@ -657,6 +657,7 @@ OBJS-$(CONFIG_LIBOPENMPT_DEMUXER)+= libopenmpt.o
> OBJS-$(CONFIG_VAPOURSYNTH_DEMUXER) += vapoursynth.o
>
> # protocols I/O
> +OBJS-$(CONFIG_ANDROID_CONTENT_PROTOCOL) += file.o
> OBJS-$(CONFIG_ASYNC_PROTOCOL)+= async.o
> OBJS-$(CONFIG_APPLEHTTP_PROTOCOL)+= hlsproto.o
> OBJS-$(CONFIG_BLURAY_PROTOCOL) += bluray.o
> diff --git a/libavformat/file.c b/libavformat/file.c
> index 64df7ff6fb..1b2b69f090 100644
> --- a/libavformat/file.c
> +++ b/libavformat/file.c
> @@ -40,6 +40,12 @@
> #include
> #include "os_support.h"
> #include "url.h"
> +#if CONFIG_ANDROID_CONTENT_PROTOCOL
> +#include
> +#include "libavcodec/jni.h"
> +#include "libavcodec/ffjni.c"
> +#endif
> +
>
> /* Some systems may not have S_ISFIFO */
> #ifndef S_ISFIFO
> @@ -101,6 +107,21 @@ typedef struct FileContext {
> int64_t initial_pos;
> } FileContext;
>
> +
> +#if CONFIG_ANDROID_CONTENT_PROTOCOL
> +static const AVOption android_content_options[] = {
> +{ "blocksize", "set I/O operation maximum block size",
> offsetof(FileContext, blocksize), AV_OPT_TYPE_INT, { .i64 = INT_MAX }, 1,
> INT_MAX, AV_OPT_FLAG_ENCODING_PARAM },
> +{ NULL }
> +};
> +
> +static const AVClass android_content_class = {
> +.class_name = "android_content",
> +.item_name = av_default_item_name,
> +.option = android_content_options,
> +.version= LIBAVUTIL_VERSION_INT,
> +};
> +#endif
> +
> static const AVOption file_options[] = {
> { "truncate", "truncate existing files on write", offsetof(FileContext,
> trunc), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, AV_OPT_FLAG_ENCODING_PARAM },
> { "blocksize", "set I/O operation maximum block size",
> offsetof(FileContext, blocksize), AV_OPT_TYPE_INT, { .i64 = INT_MAX }, 1,
> INT_MAX, AV_OPT_FLAG_ENCODING_PARAM },
> @@ -524,3 +545,139 @@ const URLProtocol ff_fd_protocol = {
> };
>
> #endif /* CONFIG_FD_PROTOCOL */
> +
> +#if CONFIG_ANDROID_CONTENT_PROTOCOL
> +
> +struct JFields {
> +jclass uri_class;
> +jmethodID parse_id;
> +
> +jclass context_class;
> +jmethodID get_content_resolver_id;
> +
> +jclass content_resolver_class;
> +jmethodID open_file_descriptor_id;
> +
> +jclass parcel_file_descriptor_class;
> +jmethodID detach_fd_id;
> +};
typedef struct is the preferred coding style of FFmpeg, it’s mentioned in
developer.texi.
> +
> +#define OFFSET(x) offsetof(struct JFields, x)
> +static const struct FFJniField jfields_mapping[] = {
> +{ "android/net/Uri", NULL, NULL, FF_JNI_CLASS, OFFSET(uri_class), 1 },
> +{ "android/net/Uri", "parse",
> "(Ljava/lang/String;)Landroid/net/Uri;", FF_JNI_STATIC_METHOD,
> OFFSET(parse_id), 1 },
> +
> +{ "android/content/Context", NULL, NULL, FF_JNI_CLASS,
>
Re: [FFmpeg-devel] [PATCH v5] avcodec/libx264: fix extradata when config annexb=0
> On Mar 14, 2024, at 21:14, Andreas Rheinhardt
> wrote:
>
> Zhao Zhili:
>> From: Zhao Zhili
>>
>> ---
>> v5: Decode chroma_format_idc directly instead of
>> ff_h264_decode_seq_parameter_set
>> v4: Fix missing SEI in set_avcc_extradata
>> v3: Remove unnecessary inclusion
>>
>> libavcodec/libx264.c | 162 ---
>> 1 file changed, 138 insertions(+), 24 deletions(-)
>>
>> diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c
>> index 10d646bd76..64cefb5fb0 100644
>> --- a/libavcodec/libx264.c
>> +++ b/libavcodec/libx264.c
>> @@ -38,6 +38,7 @@
>> #include "packet_internal.h"
>> #include "atsc_a53.h"
>> #include "sei.h"
>> +#include "golomb.h"
>>
>> #include
>> #include
>> @@ -865,6 +866,140 @@ static int convert_pix_fmt(enum AVPixelFormat pix_fmt)
>> return 0;
>> }
>>
>> +static int save_sei(AVCodecContext *avctx, x264_nal_t *nal)
>> +{
>> +X264Context *x4 = avctx->priv_data;
>> +
>> +av_log(avctx, AV_LOG_INFO, "%s\n", nal->p_payload + 25);
>> +x4->sei_size = nal->i_payload;
>> +x4->sei = av_malloc(x4->sei_size);
>> +if (!x4->sei)
>> +return AVERROR(ENOMEM);
>> +
>> +memcpy(x4->sei, nal->p_payload, nal->i_payload);
>> +
>> +return 0;
>> +}
>> +
>> +static int set_avcc_extradata(AVCodecContext *avctx, x264_nal_t *nal, int
>> nnal)
>> +{
>> +X264Context *x4 = avctx->priv_data;
>> +x264_nal_t *sps_nal = NULL;
>> +x264_nal_t *pps_nal = NULL;
>> +uint8_t *p, *sps;
>> +int ret;
>> +
>> +/* We know it's in the order of SPS/PPS/SEI, but it's not documented in
>> x264 API.
>> + * The x264 param i_sps_id implies there is a single pair of SPS/PPS.
>> + */
>> +for (int i = 0; i < nnal; i++) {
>> +switch (nal[i].i_type) {
>> +case NAL_SPS:
>> +sps_nal = [i];
>> +break;
>> +case NAL_PPS:
>> +pps_nal = [i];
>> +break;
>> +case NAL_SEI:
>> +ret = save_sei(avctx, [i]);
>> +if (ret < 0)
>> +return ret;
>> +break;
>> +}
>> +}
>> +if (!sps_nal || !pps_nal)
>> +return AVERROR_EXTERNAL;
>> +
>> +avctx->extradata_size = sps_nal->i_payload + pps_nal->i_payload + 7;
>> +avctx->extradata = av_mallocz(avctx->extradata_size +
>> AV_INPUT_BUFFER_PADDING_SIZE);
>> +if (!avctx->extradata)
>> +return AVERROR(ENOMEM);
>> +
>> +// Now create AVCDecoderConfigurationRecord
>> +p = avctx->extradata;
>> +// Skip size part
>> +sps = sps_nal->p_payload + 4;
>> +*p++ = 1; // version
>> +*p++ = sps[1]; // AVCProfileIndication
>> +*p++ = sps[2]; // profile_compatibility
>> +*p++ = sps[3]; // AVCLevelIndication
>> +*p++ = 0xFF;
>> +*p++ = 0xE0 | 0x01; // 3 bits reserved (111) + 5 bits number of sps
>> +memcpy(p, sps_nal->p_payload + 2, sps_nal->i_payload - 2);
>> +// Make sps has AV_INPUT_BUFFER_PADDING_SIZE padding, so it can be used
>> +// with GetBitContext
>> +sps = p + 2;
>> +p += sps_nal->i_payload - 2;
>> +*p++ = 1;
>> +memcpy(p, pps_nal->p_payload + 2, pps_nal->i_payload - 2);
>> +p += pps_nal->i_payload - 2;
>> +
>> +if (sps[3] != 66 && sps[3] != 77 && sps[3] != 88) {
>> +GetBitContext gbc;
>> +int chroma_format_idc;
>> +int bit_depth_luma_minus8, bit_depth_chroma_minus8;
>> +
>> +/* It's not possible to have emulation prevention byte before
>> + * bit_depth_chroma_minus8 due to the range of sps id,
>> chroma_format_idc
>> + * and so on. So we can read directly without need to escape
>> emulation
>> + * prevention byte.
>
> There is a slight problem here: You ignore the possibility of
> profile_idc being zero. After all, all values not being used are merely
> reserved and can be used at any time (although I doubt that they will
> ever use this). Shall we ignore this?
> The above check (and presumably the ISOBMFF spec for this) uses an
> opt-out check for profile, whereas the H.264 spec actually uses an
> opt-in check. I just wanted to mention it, feel free to ignore i
[FFmpeg-devel] [PATCH v6] avcodec/libx264: fix extradata when config annexb=0
From: Zhao Zhili
---
v6: use av_assert2
select golomb in configure
conditional compile in case of CONFIG_LIBX264_ENCODER=0
v5: Decode chroma_format_idc directly instead of
ff_h264_decode_seq_parameter_set
v4: Fix missing SEI in set_avcc_extradata
v3: Remove unnecessary inclusion
configure| 2 +-
libavcodec/libx264.c | 166 ---
2 files changed, 143 insertions(+), 25 deletions(-)
diff --git a/configure b/configure
index 2b4c4ec9a2..d9d2183a47 100755
--- a/configure
+++ b/configure
@@ -3484,7 +3484,7 @@ libwebp_encoder_deps="libwebp"
libwebp_anim_encoder_deps="libwebp"
libx262_encoder_deps="libx262"
libx264_encoder_deps="libx264"
-libx264_encoder_select="atsc_a53"
+libx264_encoder_select="atsc_a53 golomb"
libx264rgb_encoder_deps="libx264"
libx264rgb_encoder_select="libx264_encoder"
libx265_encoder_deps="libx265"
diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c
index 0997c4e134..ddec06d960 100644
--- a/libavcodec/libx264.c
+++ b/libavcodec/libx264.c
@@ -38,6 +38,7 @@
#include "packet_internal.h"
#include "atsc_a53.h"
#include "sei.h"
+#include "golomb.h"
#include
#include
@@ -847,6 +848,144 @@ static int convert_pix_fmt(enum AVPixelFormat pix_fmt)
return 0;
}
+static int save_sei(AVCodecContext *avctx, x264_nal_t *nal)
+{
+X264Context *x4 = avctx->priv_data;
+
+av_log(avctx, AV_LOG_INFO, "%s\n", nal->p_payload + 25);
+x4->sei_size = nal->i_payload;
+x4->sei = av_malloc(x4->sei_size);
+if (!x4->sei)
+return AVERROR(ENOMEM);
+
+memcpy(x4->sei, nal->p_payload, nal->i_payload);
+
+return 0;
+}
+
+#if CONFIG_LIBX264_ENCODER
+static int set_avcc_extradata(AVCodecContext *avctx, x264_nal_t *nal, int nnal)
+{
+X264Context *x4 = avctx->priv_data;
+x264_nal_t *sps_nal = NULL;
+x264_nal_t *pps_nal = NULL;
+uint8_t *p, *sps;
+int ret;
+
+/* We know it's in the order of SPS/PPS/SEI, but it's not documented in
x264 API.
+ * The x264 param i_sps_id implies there is a single pair of SPS/PPS.
+ */
+for (int i = 0; i < nnal; i++) {
+switch (nal[i].i_type) {
+case NAL_SPS:
+sps_nal = [i];
+break;
+case NAL_PPS:
+pps_nal = [i];
+break;
+case NAL_SEI:
+ret = save_sei(avctx, [i]);
+if (ret < 0)
+return ret;
+break;
+}
+}
+if (!sps_nal || !pps_nal)
+return AVERROR_EXTERNAL;
+
+avctx->extradata_size = sps_nal->i_payload + pps_nal->i_payload + 7;
+avctx->extradata = av_mallocz(avctx->extradata_size +
AV_INPUT_BUFFER_PADDING_SIZE);
+if (!avctx->extradata)
+return AVERROR(ENOMEM);
+
+// Now create AVCDecoderConfigurationRecord
+p = avctx->extradata;
+// Skip size part
+sps = sps_nal->p_payload + 4;
+*p++ = 1; // version
+*p++ = sps[1]; // AVCProfileIndication
+*p++ = sps[2]; // profile_compatibility
+*p++ = sps[3]; // AVCLevelIndication
+*p++ = 0xFF;
+*p++ = 0xE0 | 0x01; // 3 bits reserved (111) + 5 bits number of sps
+memcpy(p, sps_nal->p_payload + 2, sps_nal->i_payload - 2);
+// Make sps has AV_INPUT_BUFFER_PADDING_SIZE padding, so it can be used
+// with GetBitContext
+sps = p + 2;
+p += sps_nal->i_payload - 2;
+*p++ = 1;
+memcpy(p, pps_nal->p_payload + 2, pps_nal->i_payload - 2);
+p += pps_nal->i_payload - 2;
+
+if (sps[3] != 66 && sps[3] != 77 && sps[3] != 88) {
+GetBitContext gbc;
+int chroma_format_idc;
+int bit_depth_luma_minus8, bit_depth_chroma_minus8;
+
+/* It's not possible to have emulation prevention byte before
+ * bit_depth_chroma_minus8 due to the range of sps id,
chroma_format_idc
+ * and so on. So we can read directly without need to escape emulation
+ * prevention byte.
+ *
+ * +4 to skip until sps id.
+ */
+init_get_bits8(, sps + 4, sps_nal->i_payload - 4 - 4);
+// Skip sps id
+get_ue_golomb_31();
+chroma_format_idc = get_ue_golomb_31();
+if (chroma_format_idc == 3)
+skip_bits1();
+bit_depth_luma_minus8 = get_ue_golomb_31();
+bit_depth_chroma_minus8 = get_ue_golomb_31();
+
+*p++ = 0xFC | chroma_format_idc;
+*p++ = 0xF8 | bit_depth_luma_minus8;
+*p++ = 0xF8 | bit_depth_chroma_minus8;
+*p++ = 0;
+}
+av_assert2(avctx->extradata + avctx->extradata_size >= p);
+avctx->extradata_size = p - avctx->extradata;
+
+return 0;
+}
+#endif
+
+static int set_extradata(AVCodecContext *avctx)
+{
+X264Context *x4 = avctx->priv_data;
+x264_nal_t *nal;
+uin
