Hi,
I'm experiencing a crash while decoding h264 videos. It seems to be much
easier to reproduce on win32, however valgrind is able to also identify the
problem on unix. The root cause of the problem is an invalid read out of the
buffer in one of the dsp util mmx/sse3 function.
This is one of the movie able to reproduce the issue:
http://movies.apple.com/movies/sony_pictures/hancock/hancock-tlr2r_h480p.mov
I attached the stack generated by GDB as well as the assembly where the crash
occurs.
When calling avcodec_decode_video(), the "put_h264_chroma_mc4_xxx" operation
can do an invalid read on the buffer 'src'. In my example, 'src' was first
initialized at "0x1bdf2bc" at the beginning of the function, then the last
value that it tried to read was "0x1be007c" and the last valid memory
location is at "0x1bdfffc". The loop was also on the last iteration when the
crash occured. It looks like this buffer should have been about 128 bytes
bigger to satisfy this function. This buffer is the one stored as
the 'ref_list' inside the H264Context struct. At the moment, I'm tracking it
down to see why it is too small for this assembly function.
(gdb) bt
#0 0x6bbc0ead in put_h264_chroma_mc4_mmx (
dst=0x66b96cc
"\215\215\214\212\212\212\211\211\210\210\210\211\211\212\212\213\213\213\214\213\
213\211\207\206\205\204\204\203\203\202\202\202\201\201\201\201\201\201\201\202\201\201\201\200\200\
177~}}}}}||||||||", '}' <repeats 19 times>, "~~~\177\177\200\200\200\200\200",
'\177' <repeats 35 times>, "||||||||", '~' <repeats 12 times>, "}}}}}}}}", '|'
<repeats 20 times>, '{' <repeats 28 times> ...,
src=0x1bdf2bc
"\215\215\215\214\211\211\220\222\221\214\204\202\201\200\200\200\217\217\217\220"
, '\221' <repeats 47 times>,
"\220\216\215\215\214\212\211\211\207\202\200\200\177~", '}' <repeats 1 1
times>, '~' <repeats 12 times>, "}}}}}}}}", '|' <repeats 28 times>, '{'
<repeats 32 times>, 'z' <repeats 12 times>, '{' <repeats 16 times>...,
stride=440, h=8, x=6, y=0)
at libavcodec/i386/dsputil_h264_template_mmx.c:190
#1 0x6bd7366a in hl_motion (h=0x62e0050,
dest_y=0x6fd5830
"¤c¬-°33±µ'''3¬c¥½_¿¿AAAÄÅÆÇEEÉEEEÆÄA_»1·'²±_«¦\233\216}lZH<::99:::::::999999887765555455555666555543343210/.,+***)('&%$$######\"\"\"\"\"!!!!!!!
!!!! !!", ' ' <repeats 16 times>,
"!!\"#%&((*,--./03357777666665566666665555"...,
dest_cb=0x666b688
"iiiijjjkjijjkkkkjjjjjjjjlosuwxxxxyyzz{zzzzzzz{{{||}}}}}}}~~~~", '\177'
<repeats 21 times>, "~~}}}|{zzzzyyyzzzzzzzz", '{' <repeats 16 times>,
"||||||||\200\200\200\200\200\200\200\200", '~' <repeats 12 times>, '\177'
<repeats 28 times>, '\200' <repeats 24 times>...,
dest_cr=0x66b96c8
"\217\217\217\216\215\215\214\212\212\212\211\211\210\210\210\211\211\212\212\213\213\213\214\213\213\211\207\206\205\204\204\203\203\202\202\202\201\201\201\201\201\201\201\202\201\201\201\200\200\177~}}}}}||||||||",
'}' <repeats 19 times>, "~~~\177\177\200\200\200\200\200", '\177' <repeats 35
times>, "||||||||", '~' <repeats 12 times>, "}}}}}}}}", '|' <repeats 20 times>,
'{' <repeats 24 times>...,
qpix_put=0x62e0e18, chroma_put=0x62e0df4,
qpix_avg=0x62e0f18, chroma_avg=0x62e0e0c,
weight_op=0x62e1218,
weight_avg=0x62e1240) at libavcodec/h264.c:1752
#2 0x01bdf2bc in ?? ()
#3 0x000001b8 in ?? ()
#4 0x00000008 in ?? ()
#5 0x00000006 in ?? ()
#6 0x00000000 in ?? ()
[ Registers ]
eax 0x2 2
ecx 0x1be007c 29229180
edx 0x1b8 440
ebx 0x66ba2d4 107717332
esp 0x175f7b8 0x175f7b8
ebp 0x62e0050 0x62e0050
esi 0x0 0
edi 0xc8 200
eip 0x6bbc0ead 0x6bbc0ead <put_h264_chroma_mc4_mmx+141>
eflags 0x216 [ PF AF IF ]
cs 0x1b 27
ss 0x23 35
ds 0x23 35
es 0x23 35
fs 0x3b 59
gs 0x0 0
[ Disasm ]
Dump of assembler code for function put_h264_chroma_mc4_mmx:
0x6bbc0e20 <put_h264_chroma_mc4_mmx+0>: push ebx
0x6bbc0e21 <put_h264_chroma_mc4_mmx+1>: mov ebx,DWORD PTR [esp+0x8]
0x6bbc0e25 <put_h264_chroma_mc4_mmx+5>: mov ecx,DWORD PTR [esp+0xc]
0x6bbc0e29 <put_h264_chroma_mc4_mmx+9>: mov edx,DWORD PTR [esp+0x10]
0x6bbc0e2d <put_h264_chroma_mc4_mmx+13>: mov eax,DWORD PTR [esp+0x14]
0x6bbc0e31 <put_h264_chroma_mc4_mmx+17>: pxor mm7,mm7
0x6bbc0e34 <put_h264_chroma_mc4_mmx+20>: movd mm2,DWORD PTR [esp+0x18]
0x6bbc0e39 <put_h264_chroma_mc4_mmx+25>: movd mm3,DWORD PTR [esp+0x1c]
0x6bbc0e3e <put_h264_chroma_mc4_mmx+30>: movq mm4,QWORD PTR
ds:0x6bec12f0
0x6bbc0e45 <put_h264_chroma_mc4_mmx+37>: movq mm5,QWORD PTR
ds:0x6bec12f0
0x6bbc0e4c <put_h264_chroma_mc4_mmx+44>: punpcklwd mm2,mm2
0x6bbc0e4f <put_h264_chroma_mc4_mmx+47>: punpcklwd mm3,mm3
0x6bbc0e52 <put_h264_chroma_mc4_mmx+50>: punpcklwd mm2,mm2
0x6bbc0e55 <put_h264_chroma_mc4_mmx+53>: punpcklwd mm3,mm3
0x6bbc0e58 <put_h264_chroma_mc4_mmx+56>: psubw mm4,mm2
0x6bbc0e5b <put_h264_chroma_mc4_mmx+59>: psubw mm5,mm3
0x6bbc0e5e <put_h264_chroma_mc4_mmx+62>: movd mm0,DWORD PTR [ecx]
0x6bbc0e61 <put_h264_chroma_mc4_mmx+65>: movd mm6,DWORD PTR [ecx+0x1]
0x6bbc0e65 <put_h264_chroma_mc4_mmx+69>: add ecx,edx
0x6bbc0e67 <put_h264_chroma_mc4_mmx+71>: punpcklbw mm0,mm7
0x6bbc0e6a <put_h264_chroma_mc4_mmx+74>: punpcklbw mm6,mm7
0x6bbc0e6d <put_h264_chroma_mc4_mmx+77>: pmullw mm0,mm4
0x6bbc0e70 <put_h264_chroma_mc4_mmx+80>: pmullw mm6,mm2
0x6bbc0e73 <put_h264_chroma_mc4_mmx+83>: paddw mm6,mm0
0x6bbc0e76 <put_h264_chroma_mc4_mmx+86>: movd mm0,DWORD PTR [ecx]
0x6bbc0e79 <put_h264_chroma_mc4_mmx+89>: movd mm1,DWORD PTR [ecx+0x1]
0x6bbc0e7d <put_h264_chroma_mc4_mmx+93>: add ecx,edx
0x6bbc0e7f <put_h264_chroma_mc4_mmx+95>: punpcklbw mm0,mm7
0x6bbc0e82 <put_h264_chroma_mc4_mmx+98>: punpcklbw mm1,mm7
0x6bbc0e85 <put_h264_chroma_mc4_mmx+101>: pmullw mm0,mm4
0x6bbc0e88 <put_h264_chroma_mc4_mmx+104>: pmullw mm1,mm2
0x6bbc0e8b <put_h264_chroma_mc4_mmx+107>: paddw mm1,mm0
0x6bbc0e8e <put_h264_chroma_mc4_mmx+110>: movq mm0,mm1
0x6bbc0e91 <put_h264_chroma_mc4_mmx+113>: pmullw mm6,mm5
0x6bbc0e94 <put_h264_chroma_mc4_mmx+116>: pmullw mm1,mm3
0x6bbc0e97 <put_h264_chroma_mc4_mmx+119>: paddw mm6,QWORD PTR
ds:0x6bec1330
0x6bbc0e9e <put_h264_chroma_mc4_mmx+126>: paddw mm1,mm6
0x6bbc0ea1 <put_h264_chroma_mc4_mmx+129>: psrlw mm1,0x6
0x6bbc0ea5 <put_h264_chroma_mc4_mmx+133>: packuswb mm1,mm1
0x6bbc0ea8 <put_h264_chroma_mc4_mmx+136>: movd DWORD PTR [ebx],mm1
0x6bbc0eab <put_h264_chroma_mc4_mmx+139>: add ebx,edx
0x6bbc0ead <put_h264_chroma_mc4_mmx+141>: movd mm6,DWORD PTR [ecx] #
<--- Crash
0x6bbc0eb0 <put_h264_chroma_mc4_mmx+144>: movd mm1,DWORD PTR [ecx+0x1]
0x6bbc0eb4 <put_h264_chroma_mc4_mmx+148>: add ecx,edx
0x6bbc0eb6 <put_h264_chroma_mc4_mmx+150>: punpcklbw mm6,mm7
0x6bbc0eb9 <put_h264_chroma_mc4_mmx+153>: punpcklbw mm1,mm7
0x6bbc0ebc <put_h264_chroma_mc4_mmx+156>: pmullw mm6,mm4
0x6bbc0ebf <put_h264_chroma_mc4_mmx+159>: pmullw mm1,mm2
0x6bbc0ec2 <put_h264_chroma_mc4_mmx+162>: paddw mm1,mm6
0x6bbc0ec5 <put_h264_chroma_mc4_mmx+165>: movq mm6,mm1
0x6bbc0ec8 <put_h264_chroma_mc4_mmx+168>: pmullw mm0,mm5
0x6bbc0ecb <put_h264_chroma_mc4_mmx+171>: pmullw mm1,mm3
0x6bbc0ece <put_h264_chroma_mc4_mmx+174>: paddw mm0,QWORD PTR
ds:0x6bec1330
0x6bbc0ed5 <put_h264_chroma_mc4_mmx+181>: paddw mm1,mm0
0x6bbc0ed8 <put_h264_chroma_mc4_mmx+184>: psrlw mm1,0x6
0x6bbc0edc <put_h264_chroma_mc4_mmx+188>: packuswb mm1,mm1
0x6bbc0edf <put_h264_chroma_mc4_mmx+191>: movd DWORD PTR [ebx],mm1
0x6bbc0ee2 <put_h264_chroma_mc4_mmx+194>: add ebx,edx
0x6bbc0ee4 <put_h264_chroma_mc4_mmx+196>: sub eax,0x2
0x6bbc0ee7 <put_h264_chroma_mc4_mmx+199>: jne 0x6bbc0e76
<put_h264_chroma_mc4_mmx+86>
0x6bbc0ee9 <put_h264_chroma_mc4_mmx+201>: pop ebx
0x6bbc0eea <put_h264_chroma_mc4_mmx+202>: ret
_______________________________________________
libav-user mailing list
[email protected]
https://lists.mplayerhq.hu/mailman/listinfo/libav-user
