when update to 1.3.1b20180925, error occurs when train ssd with coco: ---------------- train log and error log ------------------
INFO:root:Start training from [Epoch 0] [19:54:19] src/operator/nn/./cudnn/./cudnn_algoreg-inl.h:109: Running performance tests to find the best convolution algorithm, this can take a while... (setting env variable MXNET_CUDNN_AUTOTUNE_DEFAULT to 0 to disable) [19:54:28] src/operator/nn/./cudnn/./cudnn_algoreg-inl.h:109: Running performance tests to find the best convolution algorithm, this can take a while... (setting env variable MXNET_CUDNN_AUTOTUNE_DEFAULT to 0 to disable) python: malloc.c:3722: _int_malloc: Assertion `(unsigned long) (size) >= (unsigned long) (nb)' failed. *** Error in `python': malloc(): memory corruption: 0x00007fe3d29b3690 *** ======= Backtrace: ========= /lib/x86_64-linux-gnu/libc.so.6(+0x777e5)[0x7fe5b37c87e5] /lib/x86_64-linux-gnu/libc.so.6(+0x8213e)[0x7fe5b37d313e] /lib/x86_64-linux-gnu/libc.so.6(__libc_malloc+0x54)[0x7fe5b37d5184] /usr/lib/x86_64-linux-gnu/libstdc++.so.6(_Znwm+0x18)[0x7fe5af411e78] /home/liang/.local/lib/python2.7/site-packages/mxnet/libmxnet.so(+0x407eb0)[0x7fe52630beb0] /home/liang/.local/lib/python2.7/site-packages/mxnet/libmxnet.so(+0x40d7c9)[0x7fe5263117c9] /home/liang/.local/lib/python2.7/site-packages/mxnet/libmxnet.so(+0x2b88458)[0x7fe528a8c458] /home/liang/.local/lib/python2.7/site-packages/mxnet/libmxnet.so(+0x2adcb29)[0x7fe5289e0b29] /home/liang/.local/lib/python2.7/site-packages/mxnet/libmxnet.so(+0x2ae6544)[0x7fe5289ea544] /home/liang/.local/lib/python2.7/site-packages/mxnet/libmxnet.so(+0x2aea6c2)[0x7fe5289ee6c2] /home/liang/.local/lib/python2.7/site-packages/mxnet/libmxnet.so(+0x2ae6c64)[0x7fe5289eac64] /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xb8c80)[0x7fe5af43cc80] /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba)[0x7fe5b3b226ba] /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7fe5b385841d] ======= Memory map: ======== 00400000-006de000 r-xp 00000000 103:02 16254177 /usr/bin/python2.7 008dd000-008de000 r--p 002dd000 103:02 16254177 /usr/bin/python2.7 008de000-00955000 rw-p 002de000 103:02 16254177 /usr/bin/python2.7 00955000-00978000 rw-p 00000000 00:00 0 00c8d000-a94d5000 rw-p 00000000 00:00 0 [heap] a94d5000-a9806000 rw-p 00000000 00:00 0 [heap] 200000000-200200000 rw-s 00000000 00:06 456 /dev/nvidiactl 200200000-200400000 ---p 00000000 00:00 0 200400000-200404000 rw-s 00000000 00:06 456 /dev/nvidiactl 200404000-200600000 ---p 00000000 00:00 0 200600000-200a00000 rw-s 00000000 00:06 456 /dev/nvidiactl 200a00000-201800000 ---p 00000000 00:00 0 201800000-201804000 rw-s 00000000 00:06 456 /dev/nvidiactl 201804000-201a00000 ---p 00000000 00:00 0 201a00000-201e00000 rw-s 00000000 00:06 456 /dev/nvidiactl 201e00000-201e04000 rw-s 00000000 00:06 456 /dev/nvidiactl 201e04000-202000000 ---p 00000000 00:00 0 202000000-202400000 rw-s 00000000 00:06 456 /dev/nvidiactl 202400000-202404000 rw-s 00000000 00:06 456 /dev/nvidiactl 202404000-202600000 ---p 00000000 00:00 0 202600000-202a00000 rw-s 00000000 00:06 456 /dev/nvidiactl 202a00000-202a04000 rw-s 00000000 00:06 456 /dev/nvidiactl 202a04000-202c00000 ---p 00000000 00:00 0 202c00000-203000000 rw-s 00000000 00:06 456 /dev/nvidiactl 203000000-203004000 rw-s 00000000 00:06 456 /dev/nvidiactl 203004000-203200000 ---p 00000000 00:00 0 203200000-203600000 rw-s 00000000 00:06 456 /dev/nvidiactl 203600000-203604000 rw-s 00000000 00:06 456 /dev/nvidiactl 203604000-203800000 ---p 00000000 00:00 0 203800000-203c00000 rw-s 00000000 00:06 456 /dev/nvidiactl 203c00000-203c04000 rw-s 00000000 00:06 456 /dev/nvidiactl 203c04000-203e00000 ---p 00000000 00:00 0 203e00000-204200000 rw-s 00000000 00:06 456 /dev/nvidiactl 204200000-204204000 rw-s 00000000 00:06 456 /dev/nvidiactl 204204000-204400000 ---p 00000000 00:00 0 204400000-204800000 rw-s 00000000 00:06 456 /dev/nvidiactl 204800000-204804000 rw-s 00000000 00:06 456 /dev/nvidiactl 204804000-204a00000 ---p 00000000 00:00 0 204a00000-204e00000 rw-s 00000000 00:06 456 /dev/nvidiactl 204e00000-204e04000 rw-s 00000000 00:06 456 /dev/nvidiactl 204e04000-205000000 ---p 00000000 00:00 0 205000000-205400000 rw-s 00000000 00:06 456 /dev/nvidiactl 205400000-205404000 rw-s 00000000 00:06 456 /dev/nvidiactl 205404000-205600000 ---p 00000000 00:00 0 205600000-205a00000 rw-s 00000000 00:06 456 /dev/nvidiactl 205a00000-205a04000 rw-s 00000000 00:06 456 /dev/nvidiactl 205a04000-205c00000 ---p 00000000 00:00 0 205c00000-206000000 rw-s 00000000 00:06 456 /dev/nvidiactl 206000000-206004000 rw-s 00000000 00:06 456 /dev/nvidiactl 206004000-206200000 ---p 00000000 00:00 0 206200000-206600000 rw-s 00000000 00:06 456 /dev/nvidiactl 206600000-206604000 rw-s 00000000 00:06 456 /dev/nvidiactl 206604000-206800000 ---p 00000000 00:00 0 206800000-206c00000 rw-s 00000000 00:06 456 /dev/nvidiactl 206c00000-206c04000 rw-s 00000000 00:06 456 /dev/nvidiactl 206c04000-206e00000 ---p 00000000 00:00 0 206e00000-207200000 rw-s 00000000 00:06 456 /dev/nvidiactl 207200000-207400000 ---p 00000000 00:00 0 207400000-207600000 rw-s 00000000 00:06 456 /dev/nvidiactl 207600000-207800000 rw-s 00000000 00:06 456 /dev/nvidiactl 207800000-207a00000 ---p 00000000 00:00 0 207a00000-207a04000 rw-s 00000000 00:06 456 /dev/nvidiactl 207a04000-207c00000 ---p 00000000 00:00 0 207c00000-208000000 rw-s 00000000 00:06 456 /dev/nvidiactl 208000000-208e00000 ---p 00000000 00:00 0 208e00000-208e04000 rw-s 00000000 00:06 456 /dev/nvidiactl 208e04000-209000000 ---p 00000000 00:00 0 209000000-209400000 rw-s 00000000 00:06 456 /dev/nvidiactl 209400000-209404000 rw-s 00000000 00:06 456 /dev/nvidiactl 209404000-209600000 ---p 00000000 00:00 0 209600000-209a00000 rw-s 00000000 00:06 456 /dev/nvidiactl 209a00000-209a04000 rw-s 00000000 00:06 456 /dev/nvidiactl 209a04000-209c00000 ---p 00000000 00:00 0 209c00000-20a000000 rw-s 00000000 00:06 456 /dev/nvidiactl 20a000000-20a004000 rw-s 00000000 00:06 456 /dev/nvidiactl [ Full content available at: https://github.com/apache/incubator-mxnet/issues/12619 ] This message was relayed via gitbox.apache.org for [email protected]
