Re: [FFmpeg-devel] [PATCH 12/18] avcodec/vvcdec: inter, wait reference with a diffrent resolution
On Sun, May 19, 2024 at 11:21 PM Jean-Baptiste Kempf
wrote:
> Careful about the typo on "different" on the title of the patch.
>
Changed locally.
Thank you, jb
>
> On Sun, 19 May 2024, at 13:27, Nuo Mi wrote:
> > For RPR, the current frame may reference a frame with a different
> resolution.
> > Therefore, we need to consider frame scaling when we wait for reference
> pixels.
> > ---
> > libavcodec/vvc/dec.c| 5 +
> > libavcodec/vvc/dec.h| 17 +
> > libavcodec/vvc/refs.c | 39 +++
> > libavcodec/vvc/thread.c | 10 +++---
> > 4 files changed, 68 insertions(+), 3 deletions(-)
> >
> > diff --git a/libavcodec/vvc/dec.c b/libavcodec/vvc/dec.c
> > index 3325133efb..584c7b219f 100644
> > --- a/libavcodec/vvc/dec.c
> > +++ b/libavcodec/vvc/dec.c
> > @@ -573,6 +573,11 @@ static int ref_frame(VVCFrame *dst, const VVCFrame
> *src)
> >
> > dst->poc = src->poc;
> > dst->ctb_count = src->ctb_count;
> > +
> > +dst->scaling_win = src->scaling_win;
> > +dst->ref_width = src->ref_width;
> > +dst->ref_height = src->ref_height;
> > +
> > dst->flags = src->flags;
> > dst->sequence = src->sequence;
> >
> > diff --git a/libavcodec/vvc/dec.h b/libavcodec/vvc/dec.h
> > index 6f14cc1860..1e0b76f283 100644
> > --- a/libavcodec/vvc/dec.h
> > +++ b/libavcodec/vvc/dec.h
> > @@ -46,6 +46,10 @@ typedef struct VVCRefPic {
> > struct VVCFrame *ref;
> > int poc;
> > int is_lt; // is long term reference
> > +
> > +// for RPR
> > +int is_scaled; ///< RprConstraintsActiveFlag
> > +int scale[2]; ///< RefPicScale[]
> > } VVCRefPic;
> >
> > typedef struct RefPicList {
> > @@ -57,6 +61,13 @@ typedef struct RefPicListTab {
> > RefPicList refPicList[2];
> > } RefPicListTab;
> >
> > +typedef struct VVCWindow {
> > +int16_t left_offset;
> > +int16_t right_offset;
> > +int16_t top_offset;
> > +int16_t bottom_offset;
> > +} VVCWindow;
> > +
> > typedef struct VVCFrame {
> > struct AVFrame *frame;
> >
> > @@ -71,6 +82,12 @@ typedef struct VVCFrame {
> >
> > int poc;
> >
> > +//for RPR
> > +VVCWindow scaling_win; ///<
> > pps_scaling_win_left_offset * SubWithC, pps_scaling_win_right_offset
> > * SubWithC,
> > +///<
> > pps_scaling_win_top_offset * SubHeigtC, pps_scaling_win_bottom_offset
> > * SubHiehgtC
> > +int ref_width; ///<
> > CurrPicScalWinWidthL
> > +int ref_height; ///<
> > CurrPicScalWinHeightL
> > +
> > struct VVCFrame *collocated_ref;
> >
> > struct FrameProgress *progress; ///< RefStruct reference
> > diff --git a/libavcodec/vvc/refs.c b/libavcodec/vvc/refs.c
> > index 954db4a8c8..fb42963034 100644
> > --- a/libavcodec/vvc/refs.c
> > +++ b/libavcodec/vvc/refs.c
> > @@ -114,10 +114,12 @@ static FrameProgress *alloc_progress(void)
> >
> > static VVCFrame *alloc_frame(VVCContext *s, VVCFrameContext *fc)
> > {
> > +const VVCSPS *sps = fc->ps.sps;
> > const VVCPPS *pps = fc->ps.pps;
> > for (int i = 0; i < FF_ARRAY_ELEMS(fc->DPB); i++) {
> > int ret;
> > VVCFrame *frame = >DPB[i];
> > +VVCWindow *win = >scaling_win;
> > if (frame->frame->buf[0])
> > continue;
> >
> > @@ -144,6 +146,13 @@ static VVCFrame *alloc_frame(VVCContext *s,
> > VVCFrameContext *fc)
> > for (int j = 0; j < frame->ctb_count; j++)
> > frame->rpl_tab[j] = frame->rpl;
> >
> > +win->left_offset = pps->r->pps_scaling_win_left_offset <<
> > sps->hshift[CHROMA];
> > +win->right_offset = pps->r->pps_scaling_win_right_offset <<
> > sps->hshift[CHROMA];
> > +win->top_offset= pps->r->pps_scaling_win_top_offset<<
> > sps->vshift[CHROMA];
> > +win->bottom_offset = pps->r->pps_scaling_win_bottom_offset <<
> > sps->vshift[CHROMA];
> > +frame->ref_width = pps->r->pps_pic_width_in_luma_samples -
> > win->left_offset - win->right_offset;
> > +frame->ref_height = pps->r->pps_pic_height_in_luma_samples -
> > win->bottom_offset - win->top_offset;
> > +
> > frame->progress = alloc_progress();
> > if (!frame->progress)
> > goto fail;
> > @@ -353,6 +362,24 @@ static VVCFrame *generate_missing_ref(VVCContext
> > *s, VVCFrameContext *fc, int po
> > return frame;
> > }
> >
> > +#define CHECK_MAX(d) (frame->ref_##d *
> > frame->sps->r->sps_pic_##d##_max_in_luma_samples >= ref->ref_##d *
> > (frame->pps->r->pps_pic_##d##_in_luma_samples - max))
> > +#define CHECK_SAMPLES(d) (frame->pps->r->pps_pic_##d##_in_luma_samples
> > == ref->pps->r->pps_pic_##d##_in_luma_samples)
> > +static int check_candidate_ref(const VVCFrame *frame, const VVCRefPic
> > *refp)
> > +{
> > +const VVCFrame *ref = refp->ref;
> > +
> > +if (refp->is_scaled) {
>
Re: [FFmpeg-devel] [PATCH 12/18] avcodec/vvcdec: inter, wait reference with a diffrent resolution
Careful about the typo on "different" on the title of the patch.
On Sun, 19 May 2024, at 13:27, Nuo Mi wrote:
> For RPR, the current frame may reference a frame with a different resolution.
> Therefore, we need to consider frame scaling when we wait for reference
> pixels.
> ---
> libavcodec/vvc/dec.c| 5 +
> libavcodec/vvc/dec.h| 17 +
> libavcodec/vvc/refs.c | 39 +++
> libavcodec/vvc/thread.c | 10 +++---
> 4 files changed, 68 insertions(+), 3 deletions(-)
>
> diff --git a/libavcodec/vvc/dec.c b/libavcodec/vvc/dec.c
> index 3325133efb..584c7b219f 100644
> --- a/libavcodec/vvc/dec.c
> +++ b/libavcodec/vvc/dec.c
> @@ -573,6 +573,11 @@ static int ref_frame(VVCFrame *dst, const VVCFrame *src)
>
> dst->poc = src->poc;
> dst->ctb_count = src->ctb_count;
> +
> +dst->scaling_win = src->scaling_win;
> +dst->ref_width = src->ref_width;
> +dst->ref_height = src->ref_height;
> +
> dst->flags = src->flags;
> dst->sequence = src->sequence;
>
> diff --git a/libavcodec/vvc/dec.h b/libavcodec/vvc/dec.h
> index 6f14cc1860..1e0b76f283 100644
> --- a/libavcodec/vvc/dec.h
> +++ b/libavcodec/vvc/dec.h
> @@ -46,6 +46,10 @@ typedef struct VVCRefPic {
> struct VVCFrame *ref;
> int poc;
> int is_lt; // is long term reference
> +
> +// for RPR
> +int is_scaled; ///< RprConstraintsActiveFlag
> +int scale[2]; ///< RefPicScale[]
> } VVCRefPic;
>
> typedef struct RefPicList {
> @@ -57,6 +61,13 @@ typedef struct RefPicListTab {
> RefPicList refPicList[2];
> } RefPicListTab;
>
> +typedef struct VVCWindow {
> +int16_t left_offset;
> +int16_t right_offset;
> +int16_t top_offset;
> +int16_t bottom_offset;
> +} VVCWindow;
> +
> typedef struct VVCFrame {
> struct AVFrame *frame;
>
> @@ -71,6 +82,12 @@ typedef struct VVCFrame {
>
> int poc;
>
> +//for RPR
> +VVCWindow scaling_win; ///<
> pps_scaling_win_left_offset * SubWithC, pps_scaling_win_right_offset
> * SubWithC,
> +///<
> pps_scaling_win_top_offset * SubHeigtC, pps_scaling_win_bottom_offset
> * SubHiehgtC
> +int ref_width; ///<
> CurrPicScalWinWidthL
> +int ref_height; ///<
> CurrPicScalWinHeightL
> +
> struct VVCFrame *collocated_ref;
>
> struct FrameProgress *progress; ///< RefStruct reference
> diff --git a/libavcodec/vvc/refs.c b/libavcodec/vvc/refs.c
> index 954db4a8c8..fb42963034 100644
> --- a/libavcodec/vvc/refs.c
> +++ b/libavcodec/vvc/refs.c
> @@ -114,10 +114,12 @@ static FrameProgress *alloc_progress(void)
>
> static VVCFrame *alloc_frame(VVCContext *s, VVCFrameContext *fc)
> {
> +const VVCSPS *sps = fc->ps.sps;
> const VVCPPS *pps = fc->ps.pps;
> for (int i = 0; i < FF_ARRAY_ELEMS(fc->DPB); i++) {
> int ret;
> VVCFrame *frame = >DPB[i];
> +VVCWindow *win = >scaling_win;
> if (frame->frame->buf[0])
> continue;
>
> @@ -144,6 +146,13 @@ static VVCFrame *alloc_frame(VVCContext *s,
> VVCFrameContext *fc)
> for (int j = 0; j < frame->ctb_count; j++)
> frame->rpl_tab[j] = frame->rpl;
>
> +win->left_offset = pps->r->pps_scaling_win_left_offset <<
> sps->hshift[CHROMA];
> +win->right_offset = pps->r->pps_scaling_win_right_offset <<
> sps->hshift[CHROMA];
> +win->top_offset= pps->r->pps_scaling_win_top_offset<<
> sps->vshift[CHROMA];
> +win->bottom_offset = pps->r->pps_scaling_win_bottom_offset <<
> sps->vshift[CHROMA];
> +frame->ref_width = pps->r->pps_pic_width_in_luma_samples -
> win->left_offset - win->right_offset;
> +frame->ref_height = pps->r->pps_pic_height_in_luma_samples -
> win->bottom_offset - win->top_offset;
> +
> frame->progress = alloc_progress();
> if (!frame->progress)
> goto fail;
> @@ -353,6 +362,24 @@ static VVCFrame *generate_missing_ref(VVCContext
> *s, VVCFrameContext *fc, int po
> return frame;
> }
>
> +#define CHECK_MAX(d) (frame->ref_##d *
> frame->sps->r->sps_pic_##d##_max_in_luma_samples >= ref->ref_##d *
> (frame->pps->r->pps_pic_##d##_in_luma_samples - max))
> +#define CHECK_SAMPLES(d) (frame->pps->r->pps_pic_##d##_in_luma_samples
> == ref->pps->r->pps_pic_##d##_in_luma_samples)
> +static int check_candidate_ref(const VVCFrame *frame, const VVCRefPic
> *refp)
> +{
> +const VVCFrame *ref = refp->ref;
> +
> +if (refp->is_scaled) {
> +const int max = FFMAX(8, frame->sps->min_cb_size_y);
> +return frame->ref_width * 2 >= ref->ref_width &&
> +frame->ref_height * 2 >= ref->ref_height &&
> +frame->ref_width <= ref->ref_width * 8 &&
> +frame->ref_height <= ref->ref_height * 8 &&
> +
[FFmpeg-devel] [PATCH 12/18] avcodec/vvcdec: inter, wait reference with a diffrent resolution
For RPR, the current frame may reference a frame with a different resolution.
Therefore, we need to consider frame scaling when we wait for reference pixels.
---
libavcodec/vvc/dec.c| 5 +
libavcodec/vvc/dec.h| 17 +
libavcodec/vvc/refs.c | 39 +++
libavcodec/vvc/thread.c | 10 +++---
4 files changed, 68 insertions(+), 3 deletions(-)
diff --git a/libavcodec/vvc/dec.c b/libavcodec/vvc/dec.c
index 3325133efb..584c7b219f 100644
--- a/libavcodec/vvc/dec.c
+++ b/libavcodec/vvc/dec.c
@@ -573,6 +573,11 @@ static int ref_frame(VVCFrame *dst, const VVCFrame *src)
dst->poc = src->poc;
dst->ctb_count = src->ctb_count;
+
+dst->scaling_win = src->scaling_win;
+dst->ref_width = src->ref_width;
+dst->ref_height = src->ref_height;
+
dst->flags = src->flags;
dst->sequence = src->sequence;
diff --git a/libavcodec/vvc/dec.h b/libavcodec/vvc/dec.h
index 6f14cc1860..1e0b76f283 100644
--- a/libavcodec/vvc/dec.h
+++ b/libavcodec/vvc/dec.h
@@ -46,6 +46,10 @@ typedef struct VVCRefPic {
struct VVCFrame *ref;
int poc;
int is_lt; // is long term reference
+
+// for RPR
+int is_scaled; ///< RprConstraintsActiveFlag
+int scale[2]; ///< RefPicScale[]
} VVCRefPic;
typedef struct RefPicList {
@@ -57,6 +61,13 @@ typedef struct RefPicListTab {
RefPicList refPicList[2];
} RefPicListTab;
+typedef struct VVCWindow {
+int16_t left_offset;
+int16_t right_offset;
+int16_t top_offset;
+int16_t bottom_offset;
+} VVCWindow;
+
typedef struct VVCFrame {
struct AVFrame *frame;
@@ -71,6 +82,12 @@ typedef struct VVCFrame {
int poc;
+//for RPR
+VVCWindow scaling_win; ///<
pps_scaling_win_left_offset * SubWithC, pps_scaling_win_right_offset *
SubWithC,
+///<
pps_scaling_win_top_offset * SubHeigtC, pps_scaling_win_bottom_offset *
SubHiehgtC
+int ref_width; ///< CurrPicScalWinWidthL
+int ref_height; ///< CurrPicScalWinHeightL
+
struct VVCFrame *collocated_ref;
struct FrameProgress *progress; ///< RefStruct reference
diff --git a/libavcodec/vvc/refs.c b/libavcodec/vvc/refs.c
index 954db4a8c8..fb42963034 100644
--- a/libavcodec/vvc/refs.c
+++ b/libavcodec/vvc/refs.c
@@ -114,10 +114,12 @@ static FrameProgress *alloc_progress(void)
static VVCFrame *alloc_frame(VVCContext *s, VVCFrameContext *fc)
{
+const VVCSPS *sps = fc->ps.sps;
const VVCPPS *pps = fc->ps.pps;
for (int i = 0; i < FF_ARRAY_ELEMS(fc->DPB); i++) {
int ret;
VVCFrame *frame = >DPB[i];
+VVCWindow *win = >scaling_win;
if (frame->frame->buf[0])
continue;
@@ -144,6 +146,13 @@ static VVCFrame *alloc_frame(VVCContext *s,
VVCFrameContext *fc)
for (int j = 0; j < frame->ctb_count; j++)
frame->rpl_tab[j] = frame->rpl;
+win->left_offset = pps->r->pps_scaling_win_left_offset <<
sps->hshift[CHROMA];
+win->right_offset = pps->r->pps_scaling_win_right_offset <<
sps->hshift[CHROMA];
+win->top_offset= pps->r->pps_scaling_win_top_offset<<
sps->vshift[CHROMA];
+win->bottom_offset = pps->r->pps_scaling_win_bottom_offset <<
sps->vshift[CHROMA];
+frame->ref_width = pps->r->pps_pic_width_in_luma_samples -
win->left_offset - win->right_offset;
+frame->ref_height = pps->r->pps_pic_height_in_luma_samples -
win->bottom_offset - win->top_offset;
+
frame->progress = alloc_progress();
if (!frame->progress)
goto fail;
@@ -353,6 +362,24 @@ static VVCFrame *generate_missing_ref(VVCContext *s,
VVCFrameContext *fc, int po
return frame;
}
+#define CHECK_MAX(d) (frame->ref_##d *
frame->sps->r->sps_pic_##d##_max_in_luma_samples >= ref->ref_##d *
(frame->pps->r->pps_pic_##d##_in_luma_samples - max))
+#define CHECK_SAMPLES(d) (frame->pps->r->pps_pic_##d##_in_luma_samples ==
ref->pps->r->pps_pic_##d##_in_luma_samples)
+static int check_candidate_ref(const VVCFrame *frame, const VVCRefPic *refp)
+{
+const VVCFrame *ref = refp->ref;
+
+if (refp->is_scaled) {
+const int max = FFMAX(8, frame->sps->min_cb_size_y);
+return frame->ref_width * 2 >= ref->ref_width &&
+frame->ref_height * 2 >= ref->ref_height &&
+frame->ref_width <= ref->ref_width * 8 &&
+frame->ref_height <= ref->ref_height * 8 &&
+CHECK_MAX(width) && CHECK_MAX(height);
+}
+return CHECK_SAMPLES(width) && CHECK_SAMPLES(height);
+}
+
+#define RPR_SCALE(f) (((ref->f << 14) + (fc->ref->f >> 1)) / fc->ref->f)
/* add a reference with the given poc to the list and mark it as used in DPB */
static int add_candidate_ref(VVCContext *s, VVCFrameContext *fc, RefPicList
*list,
int
