Re: [FFmpeg-devel] [PATCH 4/8] libavfilter/dnn: determine dnn output during execute_model instead of set_input_output
> -Original Message-
> From: Steven Liu [mailto:l...@chinaffmpeg.org]
> Sent: Tuesday, April 16, 2019 8:04 PM
> To: FFmpeg development discussions and patches
> Cc: Steven Liu ; Guo, Yejun
> Subject: Re: [FFmpeg-devel] [PATCH 4/8] libavfilter/dnn: determine dnn output
> during execute_model instead of set_input_output
>
>
>
> > 在 2019年4月2日，22:29，Guo, Yejun 写道：
> >
> > Currently, within interface set_input_output, the dims/memory of the
> tensorflow
> > dnn model output is determined by executing the model with zero input,
> > actually, the output dims might vary with different input data for networks
> > such as object detection models faster-rcnn, ssd and yolo.
> >
> > This patch moves the logic from set_input_output to execute_model which
> > is suitable for all the cases. Since interface changed, and so
> dnn_backend_native
> > also changes.
> >
> > In vf_sr.c, it knows it's srcnn or espcn by executing the model with zero
> > input,
> > so execute_model has to be called in function config_props
> >
> > Signed-off-by: Guo, Yejun
> > ---
> > libavfilter/dnn_backend_native.c | 14 +-
> > libavfilter/dnn_backend_native.h | 2 +-
> > libavfilter/dnn_backend_tf.c | 55
> >
> > libavfilter/dnn_backend_tf.h | 2 +-
> > libavfilter/dnn_interface.h | 6 ++---
> > libavfilter/vf_sr.c | 20 ---
> > 6 files changed, 51 insertions(+), 48 deletions(-)
> >
> > diff --git a/libavfilter/dnn_backend_native.c
> b/libavfilter/dnn_backend_native.c
> > index fe43116..18735c0 100644
> > --- a/libavfilter/dnn_backend_native.c
> > +++ b/libavfilter/dnn_backend_native.c
> > @@ -25,7 +25,7 @@
> >
> > #include "dnn_backend_native.h"
> >
> > -static DNNReturnType set_input_output_native(void *model, DNNData
> *input, const char *input_name, DNNData *output, const char *output_name)
> > +static DNNReturnType set_input_output_native(void *model, DNNData
> *input, const char *input_name, const char *output_name)
> > {
> > ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
> > InputParams *input_params;
> > @@ -81,11 +81,6 @@ static DNNReturnType set_input_output_native(void
> *model, DNNData *input, const
> > }
> > }
> >
> > -output->data = network->layers[network->layers_num - 1].output;
> > -output->height = cur_height;
> > -output->width = cur_width;
> > -output->channels = cur_channels;
> > -
> > return DNN_SUCCESS;
> > }
> >
> > @@ -280,7 +275,7 @@ static void depth_to_space(const float *input, float
> *output, int block_size, in
> > }
> > }
> >
> > -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model)
> > +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model,
> DNNData *output)
> > {
> > ConvolutionalNetwork *network = (ConvolutionalNetwork
> *)model->model;
> > int cur_width, cur_height, cur_channels;
> > @@ -322,6 +317,11 @@ DNNReturnType
> ff_dnn_execute_model_native(const DNNModel *model)
> > }
> > }
> >
> > +output->data = network->layers[network->layers_num - 1].output;
> > +output->height = cur_height;
> > +output->width = cur_width;
> > +output->channels = cur_channels;
> > +
> > return DNN_SUCCESS;
> > }
> >
> > diff --git a/libavfilter/dnn_backend_native.h
> b/libavfilter/dnn_backend_native.h
> > index 51d4cac..adaf4a7 100644
> > --- a/libavfilter/dnn_backend_native.h
> > +++ b/libavfilter/dnn_backend_native.h
> > @@ -63,7 +63,7 @@ typedef struct ConvolutionalNetwork{
> >
> > DNNModel *ff_dnn_load_model_native(const char *model_filename);
> >
> > -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model);
> > +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model,
> DNNData *output);
> >
> > void ff_dnn_free_model_native(DNNModel **model);
> >
> > diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c
> > index a838907..7966688 100644
> > --- a/libavfilter/dnn_backend_tf.c
> > +++ b/libavfilter/dnn_backend_tf.c
> > @@ -35,7 +35,6 @@ typedef struct TFModel{
> > TF_Status *status;
> > TF_Output input, output;
> > TF_Tensor *input_tensor;
> > -DNNData *output_data;
> > } TFModel;
> >
> > static void free_buffer(void *data, size_t length)
> > @@ -76,13 +75
Re: [FFmpeg-devel] [PATCH 4/8] libavfilter/dnn: determine dnn output during execute_model instead of set_input_output
> 在 2019年4月2日，22:29，Guo, Yejun 写道：
>
> Currently, within interface set_input_output, the dims/memory of the
> tensorflow
> dnn model output is determined by executing the model with zero input,
> actually, the output dims might vary with different input data for networks
> such as object detection models faster-rcnn, ssd and yolo.
>
> This patch moves the logic from set_input_output to execute_model which
> is suitable for all the cases. Since interface changed, and so
> dnn_backend_native
> also changes.
>
> In vf_sr.c, it knows it's srcnn or espcn by executing the model with zero
> input,
> so execute_model has to be called in function config_props
>
> Signed-off-by: Guo, Yejun
> ---
> libavfilter/dnn_backend_native.c | 14 +-
> libavfilter/dnn_backend_native.h | 2 +-
> libavfilter/dnn_backend_tf.c | 55
> libavfilter/dnn_backend_tf.h | 2 +-
> libavfilter/dnn_interface.h | 6 ++---
> libavfilter/vf_sr.c | 20 ---
> 6 files changed, 51 insertions(+), 48 deletions(-)
>
> diff --git a/libavfilter/dnn_backend_native.c
> b/libavfilter/dnn_backend_native.c
> index fe43116..18735c0 100644
> --- a/libavfilter/dnn_backend_native.c
> +++ b/libavfilter/dnn_backend_native.c
> @@ -25,7 +25,7 @@
>
> #include "dnn_backend_native.h"
>
> -static DNNReturnType set_input_output_native(void *model, DNNData *input,
> const char *input_name, DNNData *output, const char *output_name)
> +static DNNReturnType set_input_output_native(void *model, DNNData *input,
> const char *input_name, const char *output_name)
> {
> ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
> InputParams *input_params;
> @@ -81,11 +81,6 @@ static DNNReturnType set_input_output_native(void *model,
> DNNData *input, const
> }
> }
>
> -output->data = network->layers[network->layers_num - 1].output;
> -output->height = cur_height;
> -output->width = cur_width;
> -output->channels = cur_channels;
> -
> return DNN_SUCCESS;
> }
>
> @@ -280,7 +275,7 @@ static void depth_to_space(const float *input, float
> *output, int block_size, in
> }
> }
>
> -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model)
> +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData
> *output)
> {
> ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
> int cur_width, cur_height, cur_channels;
> @@ -322,6 +317,11 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel
> *model)
> }
> }
>
> +output->data = network->layers[network->layers_num - 1].output;
> +output->height = cur_height;
> +output->width = cur_width;
> +output->channels = cur_channels;
> +
> return DNN_SUCCESS;
> }
>
> diff --git a/libavfilter/dnn_backend_native.h
> b/libavfilter/dnn_backend_native.h
> index 51d4cac..adaf4a7 100644
> --- a/libavfilter/dnn_backend_native.h
> +++ b/libavfilter/dnn_backend_native.h
> @@ -63,7 +63,7 @@ typedef struct ConvolutionalNetwork{
>
> DNNModel *ff_dnn_load_model_native(const char *model_filename);
>
> -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model);
> +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData
> *output);
>
> void ff_dnn_free_model_native(DNNModel **model);
>
> diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c
> index a838907..7966688 100644
> --- a/libavfilter/dnn_backend_tf.c
> +++ b/libavfilter/dnn_backend_tf.c
> @@ -35,7 +35,6 @@ typedef struct TFModel{
> TF_Status *status;
> TF_Output input, output;
> TF_Tensor *input_tensor;
> -DNNData *output_data;
> } TFModel;
>
> static void free_buffer(void *data, size_t length)
> @@ -76,13 +75,12 @@ static TF_Buffer *read_graph(const char *model_filename)
> return graph_buf;
> }
>
> -static DNNReturnType set_input_output_tf(void *model, DNNData *input, const
> char *input_name, DNNData *output, const char *output_name)
> +static DNNReturnType set_input_output_tf(void *model, DNNData *input, const
> char *input_name, const char *output_name)
> {
> TFModel *tf_model = (TFModel *)model;
> int64_t input_dims[] = {1, input->height, input->width, input->channels};
> TF_SessionOptions *sess_opts;
> const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph,
> "init");
> -TF_Tensor *output_tensor;
>
> // Input operation
> tf_model->input.oper = TF_GraphOperationByName(tf_model->graph,
> input_name);
> @@ -132,26 +130,6 @@ static DNNReturnType set_input_output_tf(void *model,
> DNNData *input, const char
> }
> }
>
> -// Execute network to get output height, width and number of channels
> -TF_SessionRun(tf_model->session, NULL,
> - _model->input, _model->input_tensor, 1,
> - _model->output, _tensor, 1,
> - NULL, 0, NULL, tf_model->status);
> -if (TF_GetCode(tf_model->status) !=
Re: [FFmpeg-devel] [PATCH 4/8] libavfilter/dnn: determine dnn output during execute_model instead of set_input_output
> 在 2019年4月2日，22:29，Guo, Yejun 写道：
>
> Currently, within interface set_input_output, the dims/memory of the
> tensorflow
> dnn model output is determined by executing the model with zero input,
> actually, the output dims might vary with different input data for networks
> such as object detection models faster-rcnn, ssd and yolo.
>
> This patch moves the logic from set_input_output to execute_model which
> is suitable for all the cases. Since interface changed, and so
> dnn_backend_native
> also changes.
>
> In vf_sr.c, it knows it's srcnn or espcn by executing the model with zero
> input,
> so execute_model has to be called in function config_props
>
> Signed-off-by: Guo, Yejun
> ---
> libavfilter/dnn_backend_native.c | 14 +-
> libavfilter/dnn_backend_native.h | 2 +-
> libavfilter/dnn_backend_tf.c | 55
> libavfilter/dnn_backend_tf.h | 2 +-
> libavfilter/dnn_interface.h | 6 ++---
> libavfilter/vf_sr.c | 20 ---
> 6 files changed, 51 insertions(+), 48 deletions(-)
>
> diff --git a/libavfilter/dnn_backend_native.c
> b/libavfilter/dnn_backend_native.c
> index fe43116..18735c0 100644
> --- a/libavfilter/dnn_backend_native.c
> +++ b/libavfilter/dnn_backend_native.c
> @@ -25,7 +25,7 @@
>
> #include "dnn_backend_native.h"
>
> -static DNNReturnType set_input_output_native(void *model, DNNData *input,
> const char *input_name, DNNData *output, const char *output_name)
> +static DNNReturnType set_input_output_native(void *model, DNNData *input,
> const char *input_name, const char *output_name)
> {
> ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
> InputParams *input_params;
> @@ -81,11 +81,6 @@ static DNNReturnType set_input_output_native(void *model,
> DNNData *input, const
> }
> }
>
> -output->data = network->layers[network->layers_num - 1].output;
> -output->height = cur_height;
> -output->width = cur_width;
> -output->channels = cur_channels;
> -
> return DNN_SUCCESS;
> }
>
> @@ -280,7 +275,7 @@ static void depth_to_space(const float *input, float
> *output, int block_size, in
> }
> }
>
> -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model)
> +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData
> *output)
> {
> ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
> int cur_width, cur_height, cur_channels;
> @@ -322,6 +317,11 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel
> *model)
> }
> }
>
> +output->data = network->layers[network->layers_num - 1].output;
> +output->height = cur_height;
> +output->width = cur_width;
> +output->channels = cur_channels;
> +
> return DNN_SUCCESS;
> }
>
> diff --git a/libavfilter/dnn_backend_native.h
> b/libavfilter/dnn_backend_native.h
> index 51d4cac..adaf4a7 100644
> --- a/libavfilter/dnn_backend_native.h
> +++ b/libavfilter/dnn_backend_native.h
> @@ -63,7 +63,7 @@ typedef struct ConvolutionalNetwork{
>
> DNNModel *ff_dnn_load_model_native(const char *model_filename);
>
> -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model);
> +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData
> *output);
>
> void ff_dnn_free_model_native(DNNModel **model);
>
> diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c
> index a838907..7966688 100644
> --- a/libavfilter/dnn_backend_tf.c
> +++ b/libavfilter/dnn_backend_tf.c
> @@ -35,7 +35,6 @@ typedef struct TFModel{
> TF_Status *status;
> TF_Output input, output;
> TF_Tensor *input_tensor;
> -DNNData *output_data;
> } TFModel;
>
> static void free_buffer(void *data, size_t length)
> @@ -76,13 +75,12 @@ static TF_Buffer *read_graph(const char *model_filename)
> return graph_buf;
> }
>
> -static DNNReturnType set_input_output_tf(void *model, DNNData *input, const
> char *input_name, DNNData *output, const char *output_name)
> +static DNNReturnType set_input_output_tf(void *model, DNNData *input, const
> char *input_name, const char *output_name)
> {
> TFModel *tf_model = (TFModel *)model;
> int64_t input_dims[] = {1, input->height, input->width, input->channels};
> TF_SessionOptions *sess_opts;
> const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph,
> "init");
> -TF_Tensor *output_tensor;
>
> // Input operation
> tf_model->input.oper = TF_GraphOperationByName(tf_model->graph,
> input_name);
> @@ -132,26 +130,6 @@ static DNNReturnType set_input_output_tf(void *model,
> DNNData *input, const char
> }
> }
>
> -// Execute network to get output height, width and number of channels
> -TF_SessionRun(tf_model->session, NULL,
> - _model->input, _model->input_tensor, 1,
> - _model->output, _tensor, 1,
> - NULL, 0, NULL, tf_model->status);
> -if (TF_GetCode(tf_model->status) !=
