Re: [FFmpeg-devel] [PATCH 4/8] libavfilter/dnn: determine dnn output during execute_model instead of set_input_output
> -Original Message-
> From: Steven Liu [mailto:l...@chinaffmpeg.org]
> Sent: Tuesday, April 16, 2019 8:04 PM
> To: FFmpeg development discussions and patches
> Cc: Steven Liu ; Guo, Yejun
> Subject: Re: [FFmpeg-devel] [PATCH 4/8] libavfilter/dnn: determine dnn output
> during execute_model instead of set_input_output
>
>
>
> > 在 2019年4月2日，22:29，Guo, Yejun 写道：
> >
> > Currently, within interface set_input_output, the dims/memory of the
> tensorflow
> > dnn model output is determined by executing the model with zero input,
> > actually, the output dims might vary with different input data for networks
> > such as object detection models faster-rcnn, ssd and yolo.
> >
> > This patch moves the logic from set_input_output to execute_model which
> > is suitable for all the cases. Since interface changed, and so
> dnn_backend_native
> > also changes.
> >
> > In vf_sr.c, it knows it's srcnn or espcn by executing the model with zero
> > input,
> > so execute_model has to be called in function config_props
> >
> > Signed-off-by: Guo, Yejun
> > ---
> > libavfilter/dnn_backend_native.c | 14 +-
> > libavfilter/dnn_backend_native.h | 2 +-
> > libavfilter/dnn_backend_tf.c | 55
> >
> > libavfilter/dnn_backend_tf.h | 2 +-
> > libavfilter/dnn_interface.h | 6 ++---
> > libavfilter/vf_sr.c | 20 ---
> > 6 files changed, 51 insertions(+), 48 deletions(-)
> >
> > diff --git a/libavfilter/dnn_backend_native.c
> b/libavfilter/dnn_backend_native.c
> > index fe43116..18735c0 100644
> > --- a/libavfilter/dnn_backend_native.c
> > +++ b/libavfilter/dnn_backend_native.c
> > @@ -25,7 +25,7 @@
> >
> > #include "dnn_backend_native.h"
> >
> > -static DNNReturnType set_input_output_native(void *model, DNNData
> *input, const char *input_name, DNNData *output, const char *output_name)
> > +static DNNReturnType set_input_output_native(void *model, DNNData
> *input, const char *input_name, const char *output_name)
> > {
> > ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
> > InputParams *input_params;
> > @@ -81,11 +81,6 @@ static DNNReturnType set_input_output_native(void
> *model, DNNData *input, const
> > }
> > }
> >
> > -output->data = network->layers[network->layers_num - 1].output;
> > -output->height = cur_height;
> > -output->width = cur_width;
> > -output->channels = cur_channels;
> > -
> > return DNN_SUCCESS;
> > }
> >
> > @@ -280,7 +275,7 @@ static void depth_to_space(const float *input, float
> *output, int block_size, in
> > }
> > }
> >
> > -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model)
> > +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model,
> DNNData *output)
> > {
> > ConvolutionalNetwork *network = (ConvolutionalNetwork
> *)model->model;
> > int cur_width, cur_height, cur_channels;
> > @@ -322,6 +317,11 @@ DNNReturnType
> ff_dnn_execute_model_native(const DNNModel *model)
> > }
> > }
> >
> > +output->data = network->layers[network->layers_num - 1].output;
> > +output->height = cur_height;
> > +output->width = cur_width;
> > +output->channels = cur_channels;
> > +
> > return DNN_SUCCESS;
> > }
> >
> > diff --git a/libavfilter/dnn_backend_native.h
> b/libavfilter/dnn_backend_native.h
> > index 51d4cac..adaf4a7 100644
> > --- a/libavfilter/dnn_backend_native.h
> > +++ b/libavfilter/dnn_backend_native.h
> > @@ -63,7 +63,7 @@ typedef struct ConvolutionalNetwork{
> >
> > DNNModel *ff_dnn_load_model_native(const char *model_filename);
> >
> > -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model);
> > +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model,
> DNNData *output);
> >
> > void ff_dnn_free_model_native(DNNModel **model);
> >
> > diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c
> > index a838907..7966688 100644
> > --- a/libavfilter/dnn_backend_tf.c
> > +++ b/libavfilter/dnn_backend_tf.c
> > @@ -35,7 +35,6 @@ typedef struct TFModel{
> > TF_Status *status;
> > TF_Output input, output;
> > TF_Tensor *input_tensor;
> > -DNNData *output_data;
> > } TFModel;
> >
> > static void free_buffer(void *data, size_t length)
> > @@ -76,13 +75
Re: [FFmpeg-devel] [PATCH 4/8] libavfilter/dnn: determine dnn output during execute_model instead of set_input_output
> 在 2019年4月2日，22:29，Guo, Yejun 写道：
>
> Currently, within interface set_input_output, the dims/memory of the
> tensorflow
> dnn model output is determined by executing the model with zero input,
> actually, the output dims might vary with different input data for networks
> such as object detection models faster-rcnn, ssd and yolo.
>
> This patch moves the logic from set_input_output to execute_model which
> is suitable for all the cases. Since interface changed, and so
> dnn_backend_native
> also changes.
>
> In vf_sr.c, it knows it's srcnn or espcn by executing the model with zero
> input,
> so execute_model has to be called in function config_props
>
> Signed-off-by: Guo, Yejun
> ---
> libavfilter/dnn_backend_native.c | 14 +-
> libavfilter/dnn_backend_native.h | 2 +-
> libavfilter/dnn_backend_tf.c | 55
> libavfilter/dnn_backend_tf.h | 2 +-
> libavfilter/dnn_interface.h | 6 ++---
> libavfilter/vf_sr.c | 20 ---
> 6 files changed, 51 insertions(+), 48 deletions(-)
>
> diff --git a/libavfilter/dnn_backend_native.c
> b/libavfilter/dnn_backend_native.c
> index fe43116..18735c0 100644
> --- a/libavfilter/dnn_backend_native.c
> +++ b/libavfilter/dnn_backend_native.c
> @@ -25,7 +25,7 @@
>
> #include "dnn_backend_native.h"
>
> -static DNNReturnType set_input_output_native(void *model, DNNData *input,
> const char *input_name, DNNData *output, const char *output_name)
> +static DNNReturnType set_input_output_native(void *model, DNNData *input,
> const char *input_name, const char *output_name)
> {
> ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
> InputParams *input_params;
> @@ -81,11 +81,6 @@ static DNNReturnType set_input_output_native(void *model,
> DNNData *input, const
> }
> }
>
> -output->data = network->layers[network->layers_num - 1].output;
> -output->height = cur_height;
> -output->width = cur_width;
> -output->channels = cur_channels;
> -
> return DNN_SUCCESS;
> }
>
> @@ -280,7 +275,7 @@ static void depth_to_space(const float *input, float
> *output, int block_size, in
> }
> }
>
> -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model)
> +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData
> *output)
> {
> ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
> int cur_width, cur_height, cur_channels;
> @@ -322,6 +317,11 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel
> *model)
> }
> }
>
> +output->data = network->layers[network->layers_num - 1].output;
> +output->height = cur_height;
> +output->width = cur_width;
> +output->channels = cur_channels;
> +
> return DNN_SUCCESS;
> }
>
> diff --git a/libavfilter/dnn_backend_native.h
> b/libavfilter/dnn_backend_native.h
> index 51d4cac..adaf4a7 100644
> --- a/libavfilter/dnn_backend_native.h
> +++ b/libavfilter/dnn_backend_native.h
> @@ -63,7 +63,7 @@ typedef struct ConvolutionalNetwork{
>
> DNNModel *ff_dnn_load_model_native(const char *model_filename);
>
> -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model);
> +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData
> *output);
>
> void ff_dnn_free_model_native(DNNModel **model);
>
> diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c
> index a838907..7966688 100644
> --- a/libavfilter/dnn_backend_tf.c
> +++ b/libavfilter/dnn_backend_tf.c
> @@ -35,7 +35,6 @@ typedef struct TFModel{
> TF_Status *status;
> TF_Output input, output;
> TF_Tensor *input_tensor;
> -DNNData *output_data;
> } TFModel;
>
> static void free_buffer(void *data, size_t length)
> @@ -76,13 +75,12 @@ static TF_Buffer *read_graph(const char *model_filename)
> return graph_buf;
> }
>
> -static DNNReturnType set_input_output_tf(void *model, DNNData *input, const
> char *input_name, DNNData *output, const char *output_name)
> +static DNNReturnType set_input_output_tf(void *model, DNNData *input, const
> char *input_name, const char *output_name)
> {
> TFModel *tf_model = (TFModel *)model;
> int64_t input_dims[] = {1, input->height, input->width, input->channels};
> TF_SessionOptions *sess_opts;
> const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph,
> "init");
> -TF_Tensor *output_tensor;
>
> // Input operation
> tf_model->input.oper = TF_GraphOperationByName(tf_model->graph,
> input_name);
> @@ -132,26 +130,6 @@ static DNNReturnType set_input_output_tf(void *model,
> DNNData *input, const char
> }
> }
>
> -// Execute network to get output height, width and number of channels
> -TF_SessionRun(tf_model->session, NULL,
> - _model->input, _model->input_tensor, 1,
> - _model->output, _tensor, 1,
> - NULL, 0, NULL, tf_model->status);
> -if (TF_GetCode(tf_model->status) !=
Re: [FFmpeg-devel] [PATCH 4/8] libavfilter/dnn: determine dnn output during execute_model instead of set_input_output
> 在 2019年4月2日，22:29，Guo, Yejun 写道：
>
> Currently, within interface set_input_output, the dims/memory of the
> tensorflow
> dnn model output is determined by executing the model with zero input,
> actually, the output dims might vary with different input data for networks
> such as object detection models faster-rcnn, ssd and yolo.
>
> This patch moves the logic from set_input_output to execute_model which
> is suitable for all the cases. Since interface changed, and so
> dnn_backend_native
> also changes.
>
> In vf_sr.c, it knows it's srcnn or espcn by executing the model with zero
> input,
> so execute_model has to be called in function config_props
>
> Signed-off-by: Guo, Yejun
> ---
> libavfilter/dnn_backend_native.c | 14 +-
> libavfilter/dnn_backend_native.h | 2 +-
> libavfilter/dnn_backend_tf.c | 55
> libavfilter/dnn_backend_tf.h | 2 +-
> libavfilter/dnn_interface.h | 6 ++---
> libavfilter/vf_sr.c | 20 ---
> 6 files changed, 51 insertions(+), 48 deletions(-)
>
> diff --git a/libavfilter/dnn_backend_native.c
> b/libavfilter/dnn_backend_native.c
> index fe43116..18735c0 100644
> --- a/libavfilter/dnn_backend_native.c
> +++ b/libavfilter/dnn_backend_native.c
> @@ -25,7 +25,7 @@
>
> #include "dnn_backend_native.h"
>
> -static DNNReturnType set_input_output_native(void *model, DNNData *input,
> const char *input_name, DNNData *output, const char *output_name)
> +static DNNReturnType set_input_output_native(void *model, DNNData *input,
> const char *input_name, const char *output_name)
> {
> ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
> InputParams *input_params;
> @@ -81,11 +81,6 @@ static DNNReturnType set_input_output_native(void *model,
> DNNData *input, const
> }
> }
>
> -output->data = network->layers[network->layers_num - 1].output;
> -output->height = cur_height;
> -output->width = cur_width;
> -output->channels = cur_channels;
> -
> return DNN_SUCCESS;
> }
>
> @@ -280,7 +275,7 @@ static void depth_to_space(const float *input, float
> *output, int block_size, in
> }
> }
>
> -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model)
> +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData
> *output)
> {
> ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
> int cur_width, cur_height, cur_channels;
> @@ -322,6 +317,11 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel
> *model)
> }
> }
>
> +output->data = network->layers[network->layers_num - 1].output;
> +output->height = cur_height;
> +output->width = cur_width;
> +output->channels = cur_channels;
> +
> return DNN_SUCCESS;
> }
>
> diff --git a/libavfilter/dnn_backend_native.h
> b/libavfilter/dnn_backend_native.h
> index 51d4cac..adaf4a7 100644
> --- a/libavfilter/dnn_backend_native.h
> +++ b/libavfilter/dnn_backend_native.h
> @@ -63,7 +63,7 @@ typedef struct ConvolutionalNetwork{
>
> DNNModel *ff_dnn_load_model_native(const char *model_filename);
>
> -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model);
> +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData
> *output);
>
> void ff_dnn_free_model_native(DNNModel **model);
>
> diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c
> index a838907..7966688 100644
> --- a/libavfilter/dnn_backend_tf.c
> +++ b/libavfilter/dnn_backend_tf.c
> @@ -35,7 +35,6 @@ typedef struct TFModel{
> TF_Status *status;
> TF_Output input, output;
> TF_Tensor *input_tensor;
> -DNNData *output_data;
> } TFModel;
>
> static void free_buffer(void *data, size_t length)
> @@ -76,13 +75,12 @@ static TF_Buffer *read_graph(const char *model_filename)
> return graph_buf;
> }
>
> -static DNNReturnType set_input_output_tf(void *model, DNNData *input, const
> char *input_name, DNNData *output, const char *output_name)
> +static DNNReturnType set_input_output_tf(void *model, DNNData *input, const
> char *input_name, const char *output_name)
> {
> TFModel *tf_model = (TFModel *)model;
> int64_t input_dims[] = {1, input->height, input->width, input->channels};
> TF_SessionOptions *sess_opts;
> const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph,
> "init");
> -TF_Tensor *output_tensor;
>
> // Input operation
> tf_model->input.oper = TF_GraphOperationByName(tf_model->graph,
> input_name);
> @@ -132,26 +130,6 @@ static DNNReturnType set_input_output_tf(void *model,
> DNNData *input, const char
> }
> }
>
> -// Execute network to get output height, width and number of channels
> -TF_SessionRun(tf_model->session, NULL,
> - _model->input, _model->input_tensor, 1,
> - _model->output, _tensor, 1,
> - NULL, 0, NULL, tf_model->status);
> -if (TF_GetCode(tf_model->status) !=
[FFmpeg-devel] [PATCH 4/8] libavfilter/dnn: determine dnn output during execute_model instead of set_input_output
Currently, within interface set_input_output, the dims/memory of the tensorflow
dnn model output is determined by executing the model with zero input,
actually, the output dims might vary with different input data for networks
such as object detection models faster-rcnn, ssd and yolo.
This patch moves the logic from set_input_output to execute_model which
is suitable for all the cases. Since interface changed, and so
dnn_backend_native
also changes.
In vf_sr.c, it knows it's srcnn or espcn by executing the model with zero input,
so execute_model has to be called in function config_props
Signed-off-by: Guo, Yejun
---
libavfilter/dnn_backend_native.c | 14 +-
libavfilter/dnn_backend_native.h | 2 +-
libavfilter/dnn_backend_tf.c | 55
libavfilter/dnn_backend_tf.h | 2 +-
libavfilter/dnn_interface.h | 6 ++---
libavfilter/vf_sr.c | 20 ---
6 files changed, 51 insertions(+), 48 deletions(-)
diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c
index fe43116..18735c0 100644
--- a/libavfilter/dnn_backend_native.c
+++ b/libavfilter/dnn_backend_native.c
@@ -25,7 +25,7 @@
#include "dnn_backend_native.h"
-static DNNReturnType set_input_output_native(void *model, DNNData *input,
const char *input_name, DNNData *output, const char *output_name)
+static DNNReturnType set_input_output_native(void *model, DNNData *input,
const char *input_name, const char *output_name)
{
ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
InputParams *input_params;
@@ -81,11 +81,6 @@ static DNNReturnType set_input_output_native(void *model,
DNNData *input, const
}
}
-output->data = network->layers[network->layers_num - 1].output;
-output->height = cur_height;
-output->width = cur_width;
-output->channels = cur_channels;
-
return DNN_SUCCESS;
}
@@ -280,7 +275,7 @@ static void depth_to_space(const float *input, float
*output, int block_size, in
}
}
-DNNReturnType ff_dnn_execute_model_native(const DNNModel *model)
+DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData
*output)
{
ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
int cur_width, cur_height, cur_channels;
@@ -322,6 +317,11 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel
*model)
}
}
+output->data = network->layers[network->layers_num - 1].output;
+output->height = cur_height;
+output->width = cur_width;
+output->channels = cur_channels;
+
return DNN_SUCCESS;
}
diff --git a/libavfilter/dnn_backend_native.h b/libavfilter/dnn_backend_native.h
index 51d4cac..adaf4a7 100644
--- a/libavfilter/dnn_backend_native.h
+++ b/libavfilter/dnn_backend_native.h
@@ -63,7 +63,7 @@ typedef struct ConvolutionalNetwork{
DNNModel *ff_dnn_load_model_native(const char *model_filename);
-DNNReturnType ff_dnn_execute_model_native(const DNNModel *model);
+DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData
*output);
void ff_dnn_free_model_native(DNNModel **model);
diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c
index a838907..7966688 100644
--- a/libavfilter/dnn_backend_tf.c
+++ b/libavfilter/dnn_backend_tf.c
@@ -35,7 +35,6 @@ typedef struct TFModel{
TF_Status *status;
TF_Output input, output;
TF_Tensor *input_tensor;
-DNNData *output_data;
} TFModel;
static void free_buffer(void *data, size_t length)
@@ -76,13 +75,12 @@ static TF_Buffer *read_graph(const char *model_filename)
return graph_buf;
}
-static DNNReturnType set_input_output_tf(void *model, DNNData *input, const
char *input_name, DNNData *output, const char *output_name)
+static DNNReturnType set_input_output_tf(void *model, DNNData *input, const
char *input_name, const char *output_name)
{
TFModel *tf_model = (TFModel *)model;
int64_t input_dims[] = {1, input->height, input->width, input->channels};
TF_SessionOptions *sess_opts;
const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph,
"init");
-TF_Tensor *output_tensor;
// Input operation
tf_model->input.oper = TF_GraphOperationByName(tf_model->graph,
input_name);
@@ -132,26 +130,6 @@ static DNNReturnType set_input_output_tf(void *model,
DNNData *input, const char
}
}
-// Execute network to get output height, width and number of channels
-TF_SessionRun(tf_model->session, NULL,
- _model->input, _model->input_tensor, 1,
- _model->output, _tensor, 1,
- NULL, 0, NULL, tf_model->status);
-if (TF_GetCode(tf_model->status) != TF_OK){
-return DNN_ERROR;
-}
-else{
-output->height = TF_Dim(output_tensor, 1);
-output->width = TF_Dim(output_tensor, 2);
-output->channels = TF_Dim(output_tensor, 3);
-output->data = av_malloc(output->height *
