some models such as ssd, yolo have more than one output. the clean up code in this patch is a little complex, it is because that set_input_output_tf coulde be called for many times together with ff_dnn_execute_model_tf, we have to clean resources for the case that the two interfaces are called interleaved.
Signed-off-by: Guo, Yejun <yejun....@intel.com> --- libavfilter/dnn_backend_native.c | 15 +++++--- libavfilter/dnn_backend_native.h | 2 +- libavfilter/dnn_backend_tf.c | 80 ++++++++++++++++++++++++++++++++-------- libavfilter/dnn_backend_tf.h | 2 +- libavfilter/dnn_interface.h | 6 ++- libavfilter/vf_sr.c | 11 +++--- 6 files changed, 85 insertions(+), 31 deletions(-) diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c index 18735c0..8a83c63 100644 --- a/libavfilter/dnn_backend_native.c +++ b/libavfilter/dnn_backend_native.c @@ -25,7 +25,7 @@ #include "dnn_backend_native.h" -static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char *output_name) +static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output) { ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; InputParams *input_params; @@ -275,7 +275,7 @@ static void depth_to_space(const float *input, float *output, int block_size, in } } -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output) +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output) { ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model; int cur_width, cur_height, cur_channels; @@ -317,10 +317,13 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output } } - output->data = network->layers[network->layers_num - 1].output; - output->height = cur_height; - output->width = cur_width; - output->channels = cur_channels; + // native mode does not support multiple outputs yet + if (nb_output > 1) + return DNN_ERROR; + outputs[0].data = network->layers[network->layers_num - 1].output; + outputs[0].height = cur_height; + outputs[0].width = cur_width; + outputs[0].channels = cur_channels; return DNN_SUCCESS; } diff --git a/libavfilter/dnn_backend_native.h b/libavfilter/dnn_backend_native.h index adaf4a7..e13a68a 100644 --- a/libavfilter/dnn_backend_native.h +++ b/libavfilter/dnn_backend_native.h @@ -63,7 +63,7 @@ typedef struct ConvolutionalNetwork{ DNNModel *ff_dnn_load_model_native(const char *model_filename); -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output); +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output); void ff_dnn_free_model_native(DNNModel **model); diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c index be8401e..ca6472d 100644 --- a/libavfilter/dnn_backend_tf.c +++ b/libavfilter/dnn_backend_tf.c @@ -26,6 +26,7 @@ #include "dnn_backend_tf.h" #include "dnn_backend_native.h" #include "libavformat/avio.h" +#include "libavutil/avassert.h" #include <tensorflow/c/c_api.h> @@ -33,9 +34,11 @@ typedef struct TFModel{ TF_Graph *graph; TF_Session *session; TF_Status *status; - TF_Output input, output; + TF_Output input; TF_Tensor *input_tensor; - TF_Tensor *output_tensor; + TF_Output *outputs; + TF_Tensor **output_tensors; + uint32_t nb_output; } TFModel; static void free_buffer(void *data, size_t length) @@ -76,7 +79,7 @@ static TF_Buffer *read_graph(const char *model_filename) return graph_buf; } -static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char *input_name, const char *output_name) +static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output) { TFModel *tf_model = (TFModel *)model; int64_t input_dims[] = {1, input->height, input->width, input->channels}; @@ -100,11 +103,38 @@ static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char input->data = (float *)TF_TensorData(tf_model->input_tensor); // Output operation - tf_model->output.oper = TF_GraphOperationByName(tf_model->graph, output_name); - if (!tf_model->output.oper){ + if (nb_output == 0) + return DNN_ERROR; + + av_freep(&tf_model->outputs); + tf_model->outputs = av_malloc_array(nb_output, sizeof(*tf_model->outputs)); + if (!tf_model->outputs) + return DNN_ERROR; + for (int i = 0; i < nb_output; ++i) { + tf_model->outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]); + if (!tf_model->outputs[i].oper){ + av_freep(&tf_model->outputs); + return DNN_ERROR; + } + tf_model->outputs[i].index = 0; + } + + if (tf_model->output_tensors) { + for (uint32_t i = 0; i < tf_model->nb_output; ++i) { + if (tf_model->output_tensors[i]) { + TF_DeleteTensor(tf_model->output_tensors[i]); + tf_model->output_tensors[i] = NULL; + } + } + } + av_freep(&tf_model->output_tensors); + tf_model->output_tensors = av_mallocz_array(nb_output, sizeof(*tf_model->output_tensors)); + if (!tf_model->output_tensors) { + av_freep(&tf_model->outputs); return DNN_ERROR; } - tf_model->output.index = 0; + + tf_model->nb_output = nb_output; if (tf_model->session){ TF_CloseSession(tf_model->session, tf_model->status); @@ -484,25 +514,36 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename) -DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *output) +DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output) { TFModel *tf_model = (TFModel *)model->model; - if (tf_model->output_tensor) - TF_DeleteTensor(tf_model->output_tensor); + uint32_t nb = FFMIN(nb_output, tf_model->nb_output); + if (nb == 0) + return DNN_ERROR; + + av_assert0(tf_model->output_tensors); + for (uint32_t i = 0; i < tf_model->nb_output; ++i) { + if (tf_model->output_tensors[i]) { + TF_DeleteTensor(tf_model->output_tensors[i]); + tf_model->output_tensors[i] = NULL; + } + } TF_SessionRun(tf_model->session, NULL, &tf_model->input, &tf_model->input_tensor, 1, - &tf_model->output, &tf_model->output_tensor, 1, + tf_model->outputs, tf_model->output_tensors, nb, NULL, 0, NULL, tf_model->status); if (TF_GetCode(tf_model->status) != TF_OK){ return DNN_ERROR; } - output->height = TF_Dim(tf_model->output_tensor, 1); - output->width = TF_Dim(tf_model->output_tensor, 2); - output->channels = TF_Dim(tf_model->output_tensor, 3); - output->data = TF_TensorData(tf_model->output_tensor); + for (uint32_t i = 0; i < nb; ++i) { + outputs[i].height = TF_Dim(tf_model->output_tensors[i], 1); + outputs[i].width = TF_Dim(tf_model->output_tensors[i], 2); + outputs[i].channels = TF_Dim(tf_model->output_tensors[i], 3); + outputs[i].data = TF_TensorData(tf_model->output_tensors[i]); + } return DNN_SUCCESS; } @@ -526,9 +567,16 @@ void ff_dnn_free_model_tf(DNNModel **model) if (tf_model->input_tensor){ TF_DeleteTensor(tf_model->input_tensor); } - if (tf_model->output_tensor){ - TF_DeleteTensor(tf_model->output_tensor); + if (tf_model->output_tensors) { + for (uint32_t i = 0; i < tf_model->nb_output; ++i) { + if (tf_model->output_tensors[i]) { + TF_DeleteTensor(tf_model->output_tensors[i]); + tf_model->output_tensors[i] = NULL; + } + } } + av_freep(&tf_model->outputs); + av_freep(&tf_model->output_tensors); av_freep(&tf_model); av_freep(model); } diff --git a/libavfilter/dnn_backend_tf.h b/libavfilter/dnn_backend_tf.h index 47a24ec..07877b1 100644 --- a/libavfilter/dnn_backend_tf.h +++ b/libavfilter/dnn_backend_tf.h @@ -31,7 +31,7 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename); -DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *output); +DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output); void ff_dnn_free_model_tf(DNNModel **model); diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h index 822f6e5..73d226e 100644 --- a/libavfilter/dnn_interface.h +++ b/libavfilter/dnn_interface.h @@ -26,6 +26,8 @@ #ifndef AVFILTER_DNN_INTERFACE_H #define AVFILTER_DNN_INTERFACE_H +#include <stdint.h> + typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType; typedef enum {DNN_NATIVE, DNN_TF} DNNBackendType; @@ -40,7 +42,7 @@ typedef struct DNNModel{ void *model; // Sets model input and output. // Should be called at least once before model execution. - DNNReturnType (*set_input_output)(void *model, DNNData *input, const char *input_name, const char *output_name); + DNNReturnType (*set_input_output)(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output); } DNNModel; // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends. @@ -48,7 +50,7 @@ typedef struct DNNModule{ // Loads model and parameters from given file. Returns NULL if it is not possible. DNNModel *(*load_model)(const char *model_filename); // Executes model with specified input and output. Returns DNN_ERROR otherwise. - DNNReturnType (*execute_model)(const DNNModel *model, DNNData *output); + DNNReturnType (*execute_model)(const DNNModel *model, DNNData *outputs, uint32_t nb_output); // Frees memory allocated for model. void (*free_model)(DNNModel **model); } DNNModule; diff --git a/libavfilter/vf_sr.c b/libavfilter/vf_sr.c index 53bd8ea..b4d4165 100644 --- a/libavfilter/vf_sr.c +++ b/libavfilter/vf_sr.c @@ -117,18 +117,19 @@ static int config_props(AVFilterLink *inlink) AVFilterLink *outlink = context->outputs[0]; DNNReturnType result; int sws_src_h, sws_src_w, sws_dst_h, sws_dst_w; + const char *model_output_name = "y"; sr_context->input.width = inlink->w * sr_context->scale_factor; sr_context->input.height = inlink->h * sr_context->scale_factor; sr_context->input.channels = 1; - result = (sr_context->model->set_input_output)(sr_context->model->model, &sr_context->input, "x", "y"); + result = (sr_context->model->set_input_output)(sr_context->model->model, &sr_context->input, "x", &model_output_name, 1); if (result != DNN_SUCCESS){ av_log(context, AV_LOG_ERROR, "could not set input and output for the model\n"); return AVERROR(EIO); } - result = (sr_context->dnn_module->execute_model)(sr_context->model, &sr_context->output); + result = (sr_context->dnn_module->execute_model)(sr_context->model, &sr_context->output, 1); if (result != DNN_SUCCESS){ av_log(context, AV_LOG_ERROR, "failed to execute loaded model\n"); return AVERROR(EIO); @@ -137,12 +138,12 @@ static int config_props(AVFilterLink *inlink) if (sr_context->input.height != sr_context->output.height || sr_context->input.width != sr_context->output.width){ sr_context->input.width = inlink->w; sr_context->input.height = inlink->h; - result = (sr_context->model->set_input_output)(sr_context->model->model, &sr_context->input, "x", "y"); + result = (sr_context->model->set_input_output)(sr_context->model->model, &sr_context->input, "x", &model_output_name, 1); if (result != DNN_SUCCESS){ av_log(context, AV_LOG_ERROR, "could not set input and output for the model\n"); return AVERROR(EIO); } - result = (sr_context->dnn_module->execute_model)(sr_context->model, &sr_context->output); + result = (sr_context->dnn_module->execute_model)(sr_context->model, &sr_context->output, 1); if (result != DNN_SUCCESS){ av_log(context, AV_LOG_ERROR, "failed to execute loaded model\n"); return AVERROR(EIO); @@ -259,7 +260,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) } av_frame_free(&in); - dnn_result = (sr_context->dnn_module->execute_model)(sr_context->model, &sr_context->output); + dnn_result = (sr_context->dnn_module->execute_model)(sr_context->model, &sr_context->output, 1); if (dnn_result != DNN_SUCCESS){ av_log(context, AV_LOG_ERROR, "failed to execute loaded model\n"); return AVERROR(EIO); -- 2.7.4 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".