Extends the DNN filter common code to support CLIP/CLAP classification and model loading with tokenizers. Adds new execution functions for both image and audio classification.
Try the new filters using my Github Repo https://github.com/MaximilianKaindl/DeepFFMPEGVideoClassification. Any Feedback is appreciated! Signed-off-by: MaximilianKaindl <m.kaindl0...@gmail.com> --- libavfilter/dnn_filter_common.c | 78 ++++++++++++++++++++++++++++++--- libavfilter/dnn_filter_common.h | 4 ++ 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/libavfilter/dnn_filter_common.c b/libavfilter/dnn_filter_common.c index 6a1e9ace2e..d9b2cc9bcd 100644 --- a/libavfilter/dnn_filter_common.c +++ b/libavfilter/dnn_filter_common.c @@ -75,7 +75,7 @@ void *ff_dnn_filter_child_next(void *obj, void *prev) return ff_dnn_child_next(&base->dnnctx, prev); } -int ff_dnn_init(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx) +int ff_dnn_init_priv(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx) { DNNBackendType backend = ctx->backend_type; @@ -87,10 +87,19 @@ int ff_dnn_init(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *fil if (backend == DNN_TH) { if (ctx->model_inputname) av_log(filter_ctx, AV_LOG_WARNING, "LibTorch backend do not require inputname, "\ - "inputname will be ignored.\n"); + "inputname will be ignored.\n"); if (ctx->model_outputnames) av_log(filter_ctx, AV_LOG_WARNING, "LibTorch backend do not require outputname(s), "\ "all outputname(s) will be ignored.\n"); + +#if (CONFIG_LIBTOKENIZERS == 0) + if ((func_type == DFT_ANALYTICS_CLIP || func_type == DFT_ANALYTICS_CLAP)) { + av_log(ctx, AV_LOG_ERROR, + "tokenizers-cpp is not included. CLIP/CLAP Classification requires tokenizers-cpp library. Include " + "it with configure.\n"); + return AVERROR(EINVAL); + } +#endif ctx->nb_outputs = 1; } else if (backend == DNN_TF) { if (!ctx->model_inputname) { @@ -118,26 +127,50 @@ int ff_dnn_init(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *fil void *child = NULL; av_log(filter_ctx, AV_LOG_WARNING, - "backend_configs is deprecated, please set backend options directly\n"); + "backend_configs is deprecated, please set backend options directly\n"); while (child = ff_dnn_child_next(ctx, child)) { if (*(const AVClass **)child == &ctx->dnn_module->clazz) { int ret = av_opt_set_from_string(child, ctx->backend_options, - NULL, "=", "&"); + NULL, "=", "&"); if (ret < 0) { av_log(filter_ctx, AV_LOG_ERROR, "failed to parse options \"%s\"\n", - ctx->backend_options); + ctx->backend_options); return ret; } } } } + return 0; +} +int ff_dnn_init(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx) +{ + int ret = ff_dnn_init_priv(ctx, func_type, filter_ctx); + if (ret < 0) { + return ret; + } ctx->model = (ctx->dnn_module->load_model)(ctx, func_type, filter_ctx); if (!ctx->model) { av_log(filter_ctx, AV_LOG_ERROR, "could not load DNN model\n"); return AVERROR(EINVAL); } + return 0; +} +int ff_dnn_init_with_tokenizer(DnnContext *ctx, DNNFunctionType func_type, char **labels, int label_count, + int *softmax_units, int softmax_units_count, char *tokenizer_path, + AVFilterContext *filter_ctx) +{ + int ret = ff_dnn_init_priv(ctx, func_type, filter_ctx); + if (ret < 0) { + return ret; + } + ctx->model = (ctx->dnn_module->load_model_with_tokenizer)(ctx, func_type, labels, label_count, softmax_units, + softmax_units_count, tokenizer_path, filter_ctx); + if (!ctx->model) { + av_log(filter_ctx, AV_LOG_ERROR, "could not load DNN model\n"); + return AVERROR(EINVAL); + } return 0; } @@ -200,6 +233,41 @@ int ff_dnn_execute_model_classification(DnnContext *ctx, AVFrame *in_frame, AVFr return (ctx->dnn_module->execute_model)(ctx->model, &class_params.base); } +int ff_dnn_execute_model_clip(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char **labels, int label_count, const char* tokenizer_path, char *target) +{ + DNNExecZeroShotClassificationParams class_params = { + { + .input_name = ctx->model_inputname, + .output_names = (const char **)ctx->model_outputnames, + .nb_output = ctx->nb_outputs, + .in_frame = in_frame, + .out_frame = out_frame, + }, + .labels = labels, + .label_count = label_count, + .tokenizer_path = tokenizer_path, + .target = target, + }; + return (ctx->dnn_module->execute_model)(ctx->model, &class_params.base); +} + +int ff_dnn_execute_model_clap(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char **labels, int label_count, const char* tokenizer_path) +{ + DNNExecZeroShotClassificationParams class_params = { + { + .input_name = ctx->model_inputname, + .output_names = (const char **)ctx->model_outputnames, + .nb_output = ctx->nb_outputs, + .in_frame = in_frame, + .out_frame = out_frame, + }, + .labels = labels, + .label_count = label_count, + .tokenizer_path = tokenizer_path, + }; + return (ctx->dnn_module->execute_model)(ctx->model, &class_params.base); +} + DNNAsyncStatusType ff_dnn_get_result(DnnContext *ctx, AVFrame **in_frame, AVFrame **out_frame) { return (ctx->dnn_module->get_result)(ctx->model, in_frame, out_frame); diff --git a/libavfilter/dnn_filter_common.h b/libavfilter/dnn_filter_common.h index fffa676a9e..b05acf5d55 100644 --- a/libavfilter/dnn_filter_common.h +++ b/libavfilter/dnn_filter_common.h @@ -54,7 +54,9 @@ void *ff_dnn_filter_child_next(void *obj, void *prev); int ff_dnn_filter_init_child_class(AVFilterContext *filter); +int ff_dnn_init_priv(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx); int ff_dnn_init(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx); +int ff_dnn_init_with_tokenizer(DnnContext *ctx, DNNFunctionType func_type, char** labels, int label_count, int* softmax_units, int softmax_units_count, char* tokenizer_path, AVFilterContext *filter_ctx); int ff_dnn_set_frame_proc(DnnContext *ctx, FramePrePostProc pre_proc, FramePrePostProc post_proc); int ff_dnn_set_detect_post_proc(DnnContext *ctx, DetectPostProc post_proc); int ff_dnn_set_classify_post_proc(DnnContext *ctx, ClassifyPostProc post_proc); @@ -62,6 +64,8 @@ int ff_dnn_get_input(DnnContext *ctx, DNNData *input); int ff_dnn_get_output(DnnContext *ctx, int input_width, int input_height, int *output_width, int *output_height); int ff_dnn_execute_model(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame); int ff_dnn_execute_model_classification(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char *target); +int ff_dnn_execute_model_clip(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char **labels, int label_count, const char* tokenizer_path, char *target); +int ff_dnn_execute_model_clap(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char **labels, int label_count, const char* tokenizer_path); DNNAsyncStatusType ff_dnn_get_result(DnnContext *ctx, AVFrame **in_frame, AVFrame **out_frame); int ff_dnn_flush(DnnContext *ctx); void ff_dnn_uninit(DnnContext *ctx); -- 2.34.1 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
