[issue803] libmp3lame: output buffer too small

[Aleksey Nogin]

Aleksey Nogin <ffm...@nogin.org> added the comment:

The problem is caused by the fact that ffmpeg keeps calling lame_encode_flush,
lame_encode_flush keeps adding 208 bytes, but on each iteration only 104 of
those 208 bytes are copied out of the buffer, so the buffer ends up being
overflown. Regardless of whether lame's behavior is unreasonable (I have no
idea), there is no reason not to fix this on the ffmpeg side by making sure we
never call lame_encode_flush more than once in a row and copying out the whole
buffer after any flush. My fix is attached.

P.S. I am not familiar with roundup - sorry if "analyzed" is the wrong status to
use.

----------
nosy: +nogin
status: closed -> open
substatus: invalid -> analyzed

________________________________________________
FFmpeg issue tracker <iss...@roundup.ffmpeg.org>
<https://roundup.ffmpeg.org/issue803>
________________________________________________
--- libavcodec/libmp3lame.c.buf	2009-01-31 18:00:19.000000000 -0800
+++ libavcodec/libmp3lame.c	2010-03-07 16:48:48.000000000 -0800
@@ -34,6 +34,7 @@
     int stereo;
     uint8_t buffer[BUFFER_SIZE];
     int buffer_index;
+    int flushed;
 } Mp3AudioContext;
 
 static av_cold int MP3lame_encode_init(AVCodecContext *avctx)
@@ -45,6 +46,8 @@
 
     s->stereo = avctx->channels > 1 ? 1 : 0;
 
+    s->flushed = 1;
+
     if ((s->gfp = lame_init()) == NULL)
         goto err;
     lame_set_in_samplerate(s->gfp, avctx->sample_rate);
@@ -148,6 +151,7 @@
     /* lame 3.91 dies on '1-channel interleaved' data */
 
     if(data){
+        s->flushed = 0;
         if (s->stereo) {
             lame_result = lame_encode_buffer_interleaved(
                 s->gfp,
@@ -167,11 +171,16 @@
                 );
         }
     }else{
-        lame_result= lame_encode_flush(
-                s->gfp,
-                s->buffer + s->buffer_index,
-                BUFFER_SIZE - s->buffer_index
-                );
+        if(!s->flushed){ 
+            lame_result= lame_encode_flush(
+                    s->gfp,
+                    s->buffer + s->buffer_index,
+                    BUFFER_SIZE - s->buffer_index
+                    );
+            s->flushed = 1;
+        }else{
+            lame_result = 0;
+        }
     }
 
     if(lame_result < 0){
@@ -187,9 +196,13 @@
     if(s->buffer_index<4)
         return 0;
 
-    len= mp3len(s->buffer, NULL, NULL);
+    if(s->flushed) {
+        len = s->buffer_index;
+    } else {
+        len= mp3len(s->buffer, NULL, NULL);
+    }
 //av_log(avctx, AV_LOG_DEBUG, "in:%d packet-len:%d index:%d\n", avctx->frame_size, len, s->buffer_index);
-    if(len <= s->buffer_index){
+    if((len > 0) && (len <= s->buffer_index)){
         memcpy(frame, s->buffer, len);
         s->buffer_index -= len;