Re: [FFmpeg-devel] [PATCH 2/2] lavc/ccaption_dec: implement real_time option

[anshul . ffmpeg]

On Saturday, January 09, 2016 02:55:01 PM Aman Gupta wrote:
> From: Aman Gupta 
> 
> This new mode is useful for realtime decoding of closed captions so they
> can be display along with mpeg2 frames.
> 
> Closed caption streams contain two major types of captions:
> 
> - POPON captions, which are buffered off-screen and displayed
>   only after EOC (end of caption, aka display buffer)
> 
> - PAINTON/ROLLUP captions, which are written to the display as soon as
>   they arrive.
> 
> In a typical real-time eia608 decoder, commands like EOC (end of
> caption; display buffer), EDM (erase display memory) and EBM (erase
> buffered memory) perform their expected functions as soon as the
> commands are processed. This is implemented in the real_time branches
> added in this commit.
> 
> Before this commit, and in the !real_time branches after this commit,
> the decoder cleverly implements its own version of the decoder which is
> specifically geared towards buffered decoding. It does so by actively
> ignoring commands like EBM (erase buffered memory), and then re-using
> the non-display buffer to hold the previous caption while the new one is
> received. This is the opposite of the real-time decoder, which uses the
> non-display buffer to hold the new caption while the display buffer is
> still showing the current caption.
> 
> In addition to ignoring EBM, the buffered decoder also has custom
> implementations for EDM and EOC. An EDM (erase display memory) command
> flushes the existing contents before clearing the screen, and EOC
> similarly always flushes the active buffer (the previous subtitle)
> before flipping buffers.
> ---
>  libavcodec/ccaption_dec.c  | 85
> +- tests/fate/subtitles.mak   |
>  3 ++
>  tests/ref/fate/sub-cc-realtime | 42 +
>  3 files changed, 121 insertions(+), 9 deletions(-)
>  create mode 100644 tests/ref/fate/sub-cc-realtime
> 
> diff --git a/libavcodec/ccaption_dec.c b/libavcodec/ccaption_dec.c
> index 8bef771..8c26fcc 100644
> --- a/libavcodec/ccaption_dec.c
> +++ b/libavcodec/ccaption_dec.c
> @@ -116,6 +116,7 @@ struct Screen {
> 
>  typedef struct CCaptionSubContext {
>  AVClass *class;
> +int real_time;
>  struct Screen screen[2];
>  int active_screen;
>  uint8_t cursor_row;
> @@ -130,6 +131,8 @@ typedef struct CCaptionSubContext {
>  /* visible screen time */
>  int64_t startv_time;
>  int64_t end_time;
> +int screen_touched;
> +int64_t last_real_time;
>  char prev_cmd[2];
>  /* buffer to store pkt data */
>  AVBufferRef *pktbuf;
> @@ -180,7 +183,10 @@ static void flush_decoder(AVCodecContext *avctx)
>  ctx->cursor_color = 0;
>  ctx->active_screen = 0;
>  av_bprint_clear(>buffer);
> -ctx->buffer_changed = 0;
> +ctx->last_real_time = 0;
> +ctx->screen_touched = 0;
> +/* emit empty subtitle on seek in realtime mode */
> +ctx->buffer_changed = ctx->real_time ? 1 : 0;
>  }
> 
>  /**
> @@ -418,15 +424,33 @@ static void handle_edm(CCaptionSubContext *ctx,
> int64_t pts) {
>  struct Screen *screen = ctx->screen + ctx->active_screen;
> 
> -reap_screen(ctx, pts);
> +// In buffered mode, keep writing to screen until it is wiped.
> +// Before wiping the display, capture contents to emit subtitle.
> +if (!ctx->real_time)
> +reap_screen(ctx, pts);
> +
>  screen->row_used = 0;
> +
> +// In realtime mode, emit an empty caption so the last one doesn't
> +// stay on the screen.
> +if (ctx->real_time)
> +reap_screen(ctx, pts);
>  }
> 
>  static void handle_eoc(CCaptionSubContext *ctx, int64_t pts)
>  {
> -handle_edm(ctx,pts);
> +// In buffered mode, we wait til the *next* EOC and
> +// reap what was already on the screen since the last EOC.
> +if (!ctx->real_time)
> +handle_edm(ctx,pts);
> +
>  ctx->active_screen = !ctx->active_screen;
>  ctx->cursor_column = 0;
> +
> +// In realtime mode, we display the buffered contents (after
> +// flipping the buffer to active above) as soon as EOC arrives.
> +if (ctx->real_time)
> +reap_screen(ctx, pts);
>  }
> 
>  static void handle_delete_end_of_row(CCaptionSubContext *ctx, char hi, char
> lo) @@ -448,6 +472,9 @@ static void handle_char(CCaptionSubContext *ctx,
> char hi, char lo, int64_t pts) }
>  write_char(ctx, screen, 0);
> 
> +if (ctx->mode != CCMODE_POPON)
> +ctx->screen_touched = 1;
> +
>  /* reset prev command since character can repeat */
>  ctx->prev_cmd[0] = 0;
>  ctx->prev_cmd[1] = 0;
> @@ -497,10 +524,20 @@ static void process_cc608(CCaptionSubContext *ctx,
> int64_t pts, uint8_t hi, uint case 0x2d:
>  /* carriage return */
>  ff_dlog(ctx, "carriage return\n");
> -reap_screen(ctx, pts);
> +if (!ctx->real_time)
> +reap_screen(ctx, pts);
>  roll_up(ctx);
>  

[FFmpeg-devel] [PATCH 2/2] lavc/ccaption_dec: implement real_time option

[Aman Gupta]

From: Aman Gupta 

This new mode is useful for realtime decoding of closed captions so they
can be display along with mpeg2 frames.

Closed caption streams contain two major types of captions:

- POPON captions, which are buffered off-screen and displayed
  only after EOC (end of caption, aka display buffer)

- PAINTON/ROLLUP captions, which are written to the display as soon as
  they arrive.

In a typical real-time eia608 decoder, commands like EOC (end of
caption; display buffer), EDM (erase display memory) and EBM (erase
buffered memory) perform their expected functions as soon as the
commands are processed. This is implemented in the real_time branches
added in this commit.

Before this commit, and in the !real_time branches after this commit,
the decoder cleverly implements its own version of the decoder which is
specifically geared towards buffered decoding. It does so by actively
ignoring commands like EBM (erase buffered memory), and then re-using
the non-display buffer to hold the previous caption while the new one is
received. This is the opposite of the real-time decoder, which uses the
non-display buffer to hold the new caption while the display buffer is
still showing the current caption.

In addition to ignoring EBM, the buffered decoder also has custom
implementations for EDM and EOC. An EDM (erase display memory) command
flushes the existing contents before clearing the screen, and EOC
similarly always flushes the active buffer (the previous subtitle)
before flipping buffers.
---
 libavcodec/ccaption_dec.c  | 85 +-
 tests/fate/subtitles.mak   |  3 ++
 tests/ref/fate/sub-cc-realtime | 42 +
 3 files changed, 121 insertions(+), 9 deletions(-)
 create mode 100644 tests/ref/fate/sub-cc-realtime

diff --git a/libavcodec/ccaption_dec.c b/libavcodec/ccaption_dec.c
index 8bef771..8c26fcc 100644
--- a/libavcodec/ccaption_dec.c
+++ b/libavcodec/ccaption_dec.c
@@ -116,6 +116,7 @@ struct Screen {
 
 typedef struct CCaptionSubContext {
 AVClass *class;
+int real_time;
 struct Screen screen[2];
 int active_screen;
 uint8_t cursor_row;
@@ -130,6 +131,8 @@ typedef struct CCaptionSubContext {
 /* visible screen time */
 int64_t startv_time;
 int64_t end_time;
+int screen_touched;
+int64_t last_real_time;
 char prev_cmd[2];
 /* buffer to store pkt data */
 AVBufferRef *pktbuf;
@@ -180,7 +183,10 @@ static void flush_decoder(AVCodecContext *avctx)
 ctx->cursor_color = 0;
 ctx->active_screen = 0;
 av_bprint_clear(>buffer);
-ctx->buffer_changed = 0;
+ctx->last_real_time = 0;
+ctx->screen_touched = 0;
+/* emit empty subtitle on seek in realtime mode */
+ctx->buffer_changed = ctx->real_time ? 1 : 0;
 }
 
 /**
@@ -418,15 +424,33 @@ static void handle_edm(CCaptionSubContext *ctx, int64_t 
pts)
 {
 struct Screen *screen = ctx->screen + ctx->active_screen;
 
-reap_screen(ctx, pts);
+// In buffered mode, keep writing to screen until it is wiped.
+// Before wiping the display, capture contents to emit subtitle.
+if (!ctx->real_time)
+reap_screen(ctx, pts);
+
 screen->row_used = 0;
+
+// In realtime mode, emit an empty caption so the last one doesn't
+// stay on the screen.
+if (ctx->real_time)
+reap_screen(ctx, pts);
 }
 
 static void handle_eoc(CCaptionSubContext *ctx, int64_t pts)
 {
-handle_edm(ctx,pts);
+// In buffered mode, we wait til the *next* EOC and
+// reap what was already on the screen since the last EOC.
+if (!ctx->real_time)
+handle_edm(ctx,pts);
+
 ctx->active_screen = !ctx->active_screen;
 ctx->cursor_column = 0;
+
+// In realtime mode, we display the buffered contents (after
+// flipping the buffer to active above) as soon as EOC arrives.
+if (ctx->real_time)
+reap_screen(ctx, pts);
 }
 
 static void handle_delete_end_of_row(CCaptionSubContext *ctx, char hi, char lo)
@@ -448,6 +472,9 @@ static void handle_char(CCaptionSubContext *ctx, char hi, 
char lo, int64_t pts)
 }
 write_char(ctx, screen, 0);
 
+if (ctx->mode != CCMODE_POPON)
+ctx->screen_touched = 1;
+
 /* reset prev command since character can repeat */
 ctx->prev_cmd[0] = 0;
 ctx->prev_cmd[1] = 0;
@@ -497,10 +524,20 @@ static void process_cc608(CCaptionSubContext *ctx, 
int64_t pts, uint8_t hi, uint
 case 0x2d:
 /* carriage return */
 ff_dlog(ctx, "carriage return\n");
-reap_screen(ctx, pts);
+if (!ctx->real_time)
+reap_screen(ctx, pts);
 roll_up(ctx);
 ctx->cursor_column = 0;
 break;
+case 0x2e:
+/* erase buffered (non displayed) memory */
+// Only in realtime mode. In buffered mode, we re-use the inactive 
screen
+// for our own buffering.
+if (ctx->real_time) {
+