#6838: avcodec:ff_prores_idct_put_10_sse2 segfault on decoding a mov -------------------------------------+------------------------------------- Reporter: j13r | Owner: Type: defect | Status: open Priority: important | Component: avcodec Version: git-master | Resolution: Keywords: prores | Blocked By: crash SIGSEGV regression | Reproduced by developer: 1 Blocking: | Analyzed by developer: 0 | -------------------------------------+------------------------------------- Changes (by cehoyos):
* keywords: => prores crash SIGSEGV regression * priority: normal => important * version: unspecified => git-master * status: new => open * reproduced: 0 => 1 Comment: Regression since bebaf4ea, reproducible with different compilers. {{{ (gdb) r -i fate-suite/prores/Sequence_1-Apple_ProRes_422.mov Starting program: ffmpeg_g -i fate- suite/prores/Sequence_1-Apple_ProRes_422.mov [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". ffmpeg version N-89061-g6d00905 Copyright (c) 2000-2017 the FFmpeg developers built with gcc 6.3.0 (GCC) configuration: --enable-small --toolchain=hardened --disable-avx libavutil 56. 0.100 / 56. 0.100 libavcodec 58. 3.102 / 58. 3.102 libavformat 58. 2.100 / 58. 2.100 libavdevice 58. 0.100 / 58. 0.100 libavfilter 7. 0.101 / 7. 0.101 libswscale 5. 0.101 / 5. 0.101 libswresample 3. 0.101 / 3. 0.101 Program received signal SIGSEGV, Segmentation fault. 0x0000555555de7c37 in ff_prores_idct_put_10_sse2 () (gdb) bt #0 0x0000555555de7c37 in ff_prores_idct_put_10_sse2 () #1 0x0000555555a4ef32 in decode_slice_luma (avctx=avctx@entry=0x555556dd8d60, dst=0x7ffff4d60f10, dst_stride=dst_stride@entry=7680, buf=buf@entry=0x555556de80a6 "\006\240\217\377\377\377\300\202?\377\202?\377\060\002", buf_size=<optimized out>, qmat=0x7fffffffcfb8, slice=<optimized out>) at libavcodec/proresdec2.c:389 #2 0x0000555555a5023b in decode_slice_thread (avctx=0x555556dd8d60, arg=<optimized out>, jobnr=<optimized out>, threadnr=<optimized out>) at libavcodec/proresdec2.c:581 #3 0x0000555555ac9bad in avcodec_default_execute2 (c=0x555556dd8d60, func=0x555555a4fff6 <decode_slice_thread>, arg=0x0, ret=0x0, count=<optimized out>) at libavcodec/utils.c:536 #4 0x0000555555a4fc19 in decode_picture (avctx=0x555556dd8d60) at libavcodec/proresdec2.c:625 #5 decode_frame (avctx=0x555556dd8d60, data=<optimized out>, got_frame=0x7fffffffd200, avpkt=0x555556ddd420) at libavcodec/proresdec2.c:677 #6 0x00005555558f5679 in decode_simple_internal (frame=0x555556ddd1a0, avctx=0x555556dd8d60) at libavcodec/decode.c:397 #7 decode_simple_receive_frame (frame=<optimized out>, avctx=<optimized out>) at libavcodec/decode.c:593 #8 decode_receive_frame_internal (avctx=avctx@entry=0x555556dd8d60, frame=0x555556ddd1a0) at libavcodec/decode.c:611 #9 0x00005555558f7789 in avcodec_send_packet (avctx=avctx@entry=0x555556dd8d60, avpkt=avpkt@entry=0x7fffffffd2d0) at libavcodec/decode.c:673 #10 0x000055555588f0aa in try_decode_frame (s=s@entry=0x555556dd71a0, st=st@entry=0x555556dd85d0, avpkt=avpkt@entry=0x7fffffffd418, options=<optimized out>) at libavformat/utils.c:3006 #11 0x000055555589543b in avformat_find_stream_info (ic=0x555556dd71a0, options=0x555556dd7cd0) at libavformat/utils.c:3831 #12 0x000055555569f30d in open_input_file (o=o@entry=0x7fffffffd6f8, filename=0x7fffffffe1cf "fate- suite/prores/Sequence_1-Apple_ProRes_422.mov") at fftools/ffmpeg_opt.c:1078 #13 0x000055555569d150 in open_files (l=0x555556dd7028, inout=inout@entry=0x55555614745d "input", open_file=open_file@entry=0x55555569eda3 <open_input_file>) at fftools/ffmpeg_opt.c:3281 #14 0x00005555556a3471 in ffmpeg_parse_options (argc=argc@entry=3, argv=argv@entry=0x7fffffffdd38) at fftools/ffmpeg_opt.c:3321 #15 0x000055555569ae6c in main (argc=3, argv=0x7fffffffdd38) at fftools/ffmpeg.c:4775 (gdb) disass $pc-23,$pc+32 Dump of assembler code from 0x555555de7c20 to 0x555555de7c57: 0x0000555555de7c20 <ff_prores_idct_put_10_sse2+0>: movdqa (%rdx),%xmm10 0x0000555555de7c25 <ff_prores_idct_put_10_sse2+5>: movdqa 0x20(%rdx),%xmm8 0x0000555555de7c2b <ff_prores_idct_put_10_sse2+11>: movdqa 0x40(%rdx),%xmm13 0x0000555555de7c31 <ff_prores_idct_put_10_sse2+17>: movdqa 0x60(%rdx),%xmm12 => 0x0000555555de7c37 <ff_prores_idct_put_10_sse2+23>: pmullw (%rcx),%xmm10 0x0000555555de7c3c <ff_prores_idct_put_10_sse2+28>: pmullw 0x20(%rcx),%xmm8 0x0000555555de7c42 <ff_prores_idct_put_10_sse2+34>: pmullw 0x40(%rcx),%xmm13 0x0000555555de7c48 <ff_prores_idct_put_10_sse2+40>: pmullw 0x60(%rcx),%xmm12 0x0000555555de7c4e <ff_prores_idct_put_10_sse2+46>: paddw 0x352f69(%rip),%xmm10 # 0x55555613abc0 <ff_pw_1> End of assembler dump. (gdb) info all-register rax 0x555556ddd4e0 93825017959648 rbx 0x1e00 7680 rcx 0x7fffffffcfb8 140737488342968 rdx 0x7fffffffbee0 140737488338656 rsi 0x1e00 7680 rdi 0x7ffff4d60f10 140737301057296 rbp 0x7fffffffcf30 0x7fffffffcf30 rsp 0x7fffffffbe58 0x7fffffffbe58 r8 0x0 0 r9 0x1f 31 r10 0x0 0 r11 0x0 0 r12 0x7fffffffbee0 140737488338656 r13 0xf000 61440 r14 0xf010 61456 r15 0x1 1 rip 0x555555de7c37 0x555555de7c37 <ff_prores_idct_put_10_sse2+23> eflags 0x10202 [ IF RF ] cs 0x33 51 ss 0x2b 43 ds 0x0 0 es 0x0 0 fs 0x0 0 gs 0x0 0 st0 0 (raw 0x00000000000000000000) st1 0 (raw 0x00000000000000000000) st2 0 (raw 0x00000000000000000000) st3 0 (raw 0x00000000000000000000) st4 0 (raw 0x00000000000000000000) st5 0 (raw 0x00000000000000000000) st6 0 (raw 0x00000000000000000000) st7 0 (raw 0x00000000000000000000) fctrl 0x37f 895 fstat 0x0 0 ftag 0xffff 65535 fiseg 0x0 0 fioff 0x0 0 foseg 0x0 0 fooff 0x0 0 fop 0x0 0 mxcsr 0x1fa0 [ PE IM DM ZM OM UM PM ] ymm0 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>}, v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = { 0x00000000000000000000000000000000, 0x00000000000000000000000000000000}} ymm1 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0xff, 0xff, 0xff, 0xff, 0x0 <repeats 28 times>}, v16_int16 = {0xffff, 0xffff, 0x0 <repeats 14 times>}, v8_int32 = {0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0xffffffff, 0x0, 0x0, 0x0}, v2_int128 = {0x000000000000000000000000ffffffff, 0x00000000000000000000000000000000}} ymm2 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x6e, 0x75, 0x6c, 0x6c, 0x0, 0x7f, 0x0, 0x0, 0x88, 0x6, 0x70, 0xf5, 0xff, 0x7f, 0x0 <repeats 18 times>}, v16_int16 = {0x756e, 0x6c6c, 0x7f00, 0x0, 0x688, 0xf570, 0x7fff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x6c6c756e, 0x7f00, 0xf5700688, 0x7fff, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x7f006c6c756e, 0x7ffff5700688, 0x0, 0x0}, v2_int128 = {0x00007ffff570068800007f006c6c756e, 0x00000000000000000000000000000000}} ymm3 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>}, v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = { 0x00000000000000000000000000000000, 0x00000000000000000000000000000000}} ymm4 {v8_float = {0x3, 0x351597c0, 0x0, 0x454c0000, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x8000000000000000, 0x8000000000000000, 0x0, 0x0}, v32_int8 = {0x0, 0x0, 0x40, 0x40, 0x5f, 0x56, 0x54, 0x4e, 0x52, 0x3d, 0x38, 0x0, 0x4c, 0x45, 0x53, 0x53, 0x0 <repeats 16 times>}, v16_int16 = {0x0, 0x4040, 0x565f, 0x4e54, 0x3d52, 0x38, 0x454c, 0x5353, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x40400000, 0x4e54565f, 0x383d52, 0x5353454c, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x4e54565f40400000, 0x5353454c00383d52, 0x0, 0x0}, v2_int128 = { 0x5353454c00383d524e54565f40400000, 0x00000000000000000000000000000000}} ymm5 {v8_float = {0x0, 0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x4c, 0x82, 0x9c, 0xe3, 0x92, 0x99, 0xd9, 0xbf, 0x0 <repeats 24 times>}, v16_int16 = {0x824c, 0xe39c, 0x9992, 0xbfd9, 0x0 <repeats 12 times>}, v8_int32 = { 0xe39c824c, 0xbfd99992, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0xbfd99992e39c824c, 0x0, 0x0, 0x0}, v2_int128 = {0x0000000000000000bfd99992e39c824c, 0x00000000000000000000000000000000}} ymm6 {v8_float = {0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x2540be400, 0x0, 0x0, 0x0}, v32_int8 = {0x0, 0x0, 0x0, 0x20, 0x5f, 0xa0, 0x2, 0x42, 0x0 <repeats 24 times>}, v16_int16 = {0x0, 0x2000, 0xa05f, 0x4202, 0x0 <repeats 12 times>}, v8_int32 = { 0x20000000, 0x4202a05f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x4202a05f20000000, 0x0, 0x0, 0x0}, v2_int128 = {0x00000000000000004202a05f20000000, 0x00000000000000000000000000000000}} ymm7 {v8_float = {0x0, 0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0xf9, 0xc8, 0xde, 0xfc, 0xd1, 0x21, 0x89, 0xbf, 0x0 <repeats 24 times>}, v16_int16 = {0xc8f9, 0xfcde, 0x21d1, 0xbf89, 0x0 <repeats 12 times>}, v8_int32 = { 0xfcdec8f9, 0xbf8921d1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0xbf8921d1fcdec8f9, 0x0, 0x0, 0x0}, v2_int128 = {0x0000000000000000bf8921d1fcdec8f9, 0x00000000000000000000000000000000}} ymm8 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>}, v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = { 0x00000000000000000000000000000000, 0x00000000000000000000000000000000}} ymm9 {v8_float = {0xba340000, 0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x8d, 0x2e, 0x44, 0x54, 0xfb, 0x21, 0x89, 0xbf, 0x0 <repeats 24 times>}, v16_int16 = {0x2e8d, 0x5444, 0x21fb, 0xbf89, 0x0 <repeats 12 times>}, v8_int32 = {0x54442e8d, 0xbf8921fb, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0xbf8921fb54442e8d, 0x0, 0x0, 0x0}, v2_int128 = {0x0000000000000000bf8921fb54442e8d, 0x00000000000000000000000000000000}} ymm10 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x40, 0x3, 0x0 <repeats 30 times>}, v16_int16 = {0x340, 0x0 <repeats 15 times>}, v8_int32 = {0x340, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x340, 0x0, 0x0, 0x0}, v2_int128 = {0x00000000000000000000000000000340, 0x00000000000000000000000000000000}} ymm11 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0xd9, 0x7e, 0x9a, 0x7b, 0xe2, 0x1d, 0xc7, 0x3e, 0x0 <repeats 24 times>}, v16_int16 = {0x7ed9, 0x7b9a, 0x1de2, 0x3ec7, 0x0 <repeats 12 times>}, v8_int32 = { 0x7b9a7ed9, 0x3ec71de2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x3ec71de27b9a7ed9, 0x0, 0x0, 0x0}, v2_int128 = {0x00000000000000003ec71de27b9a7ed9, 0x00000000000000000000000000000000}} ymm12 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>}, v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = { 0x00000000000000000000000000000000, 0x00000000000000000000000000000000}} ymm13 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>}, v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = { 0x00000000000000000000000000000000, 0x00000000000000000000000000000000}} ymm14 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x46, 0xb0, 0x42, 0xa4, 0x99, 0xe4, 0xd3, 0x3e, 0x0 <repeats 24 times>}, v16_int16 = {0xb046, 0xa442, 0xe499, 0x3ed3, 0x0 <repeats 12 times>}, v8_int32 = { 0xa442b046, 0x3ed3e499, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x3ed3e499a442b046, 0x0, 0x0, 0x0}, v2_int128 = {0x00000000000000003ed3e499a442b046, 0x00000000000000000000000000000000}} ymm15 {v8_float = {0x92854080, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x7f, 0xf5, 0xda, 0xce, 0xf0, 0x39, 0xc1, 0x3f, 0x0 <repeats 24 times>}, v16_int16 = {0xf57f, 0xceda, 0x39f0, 0x3fc1, 0x0 <repeats 12 times>}, v8_int32 = { 0xcedaf57f, 0x3fc139f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x3fc139f0cedaf57f, 0x0, 0x0, 0x0}, v2_int128 = {0x00000000000000003fc139f0cedaf57f, 0x00000000000000000000000000000000}} }}} -- Ticket URL: <https://trac.ffmpeg.org/ticket/6838#comment:2> FFmpeg <https://ffmpeg.org> FFmpeg issue tracker _______________________________________________ FFmpeg-trac mailing list FFmpeg-trac@avcodec.org http://ffmpeg.org/mailman/listinfo/ffmpeg-trac