We run a multithreaded application that uses postgres 7.4 on Linux
2.4.18, dual cpu Xeon processor machine. We have occassional weird
crashes and have tried a lot of things to reproduce them in house, but
in vain. We do have coredumps and I have listed the backtraces and
their common characteristics here.
Briefly, the last frame is a call to a glibc (or rarely some other
shared lib) method.And the instruction pointer points to an indirect
jmp instruction to the shared lib method. Almost all coredumps show
this characteristic.
The dying process is postmaster. After these crashes the server is
missing from ps and no more new connections are possible. The backend
processes stay on till their connections close.
Any ideas appreciated.
core #1 (8706)
---
(gdb) bt
#0 0x0806f1c4 in snprintf ()
#1 0x081a7f50 in send_message_to_frontend (edata=0x826c1e0)
at /root/src/postgres/src/backend/utils/error/elog.c:1239
#2 0x081a6c85 in errfinish (dummy=0) at
/root/src/postgres/src/backend/utils/error/elog.c:359
#3 0x081a767e in elog_finish (elevel=20, fmt=0x8235680 invalid
memory alloc request size %lu)
at /root/src/postgres/src/backend/utils/error/elog.c:853
(gdb) disassemble
Dump of assembler code for function snprintf:
0x0806f1c4 snprintf+0:jmp*0x823f02c
0x0806f1ca snprintf+6:push $0x508
0x0806f1cf snprintf+11: jmp0x806e7a4 _init+24
End of assembler dump.
(gdb) x/i $pc
0x806f1c4 snprintf: jmp*0x823f02c
(gdb) x *0x823f02c
0x182de130 snprintf: push %ebp
(gdb) disassemble *0x823f02c
Dump of assembler code for function snprintf:
0x182de130 snprintf+0:push %ebp
0x182de131 snprintf+1:mov%esp,%ebp
0x182de133 snprintf+3:push %ebx
core #2 (5889)
---
(gdb) bt
#0 0x0806f0b4 in memcpy ()
#1 0x08103cee in pq_getbytes (s=0xbfffeb5c ., len=4) at
/root/src/postgres/src/backend/libpq/pqcomm.c:748
#2 0x08103e04 in pq_getmessage (s=0xbfffec10, maxlen=0) at
/root/src/postgres/src/backend/libpq/pqcomm.c:837
#3 0x0814c98b in SocketBackend (inBuf=0xbfffec10) at
/root/src/postgres/src/backend/tcop/postgres.c:377
(gdb) disassemble
Dump of assembler code for function memcpy:
0x0806f0b4 memcpy+0: jmp*0x823efe8
0x0806f0ba memcpy+6: push $0x480
0x0806f0bf memcpy+11: jmp0x806e7a4 _init+24
End of assembler dump.
(gdb) x/i $pc
0x806f0b4 memcpy: jmp*0x823efe8
(gdb) x *0x823efe8
0x18304f18 memcpy:push %ebp
(gdb) disassemble *0x823efe8
Dump of assembler code for function memcpy:
0x18304f18 memcpy+0: push %ebp
0x18304f19 memcpy+1: mov%esp,%ebp
0x18304f1b memcpy+3: mov0x10(%ebp),%eax
core #3 (32662)
---
(gdb) bt
#0 0x0806f3c4 in strncpy ()
#1 0x081b22fa in set_ps_display (activity=0x4 Address 0x4 out of
bounds)
at /root/src/postgres/src/backend/utils/misc/ps_status.c:282
#2 0x0814f3f5 in PostgresMain (argc=4, argv=0x8279838,
username=0x8279808 postgres)
at /root/src/postgres/src/backend/tcop/postgres.c:2805
#3 0x0812f24b in BackendFork (port=0x82877a8) at
/root/src/postgres/src/backend/postmaster/postmaster.c:2558
(gdb) x/i $pc
0x806f3c4 strncpy:jmp*0x823f0ac
(gdb) disassemble *0x823f0ac
Dump of assembler code for function strncpy:
0x183033c0 strncpy+0: push %ebp
0x183033c1 strncpy+1: mov%esp,%ebp
0x183033c3 strncpy+3: push %edi
core #4 (28335)
---
(gdb) bt
#0 0x0806f0c1 in memcpy ()
#1 0x08103cee in pq_getbytes (s=0xbfffeb5c \f, len=4) at
/root/src/postgres/src/backend/libpq/pqcomm.c:748
#2 0x08103e04 in pq_getmessage (s=0xbfffec10, maxlen=0) at
/root/src/postgres/src/backend/libpq/pqcomm.c:837
#3 0x0814c98b in SocketBackend (inBuf=0xbfffec10) at
/root/src/postgres/src/backend/tcop/postgres.c:377
(gdb) x/i $pc
0x806f0c1 memcpy+13: idiv %bh
(gdb) disassemble
Dump of assembler code for function memcpy:
0x0806f0b4 memcpy+0: jmp*0x823efe8
0x0806f0ba memcpy+6: push $0x480
0x0806f0bf memcpy+11: jmp0x806e7a4 _init+24
End of assembler dump.
(gdb) disassemble *0x823efe8
Dump of assembler code for function memcpy:
0x18304f18 memcpy+0: push %ebp
0x18304f19 memcpy+1: mov%esp,%ebp
0x18304f1b memcpy+3: mov0x10(%ebp),%eax
core #5 (22375)
(gdb) bt
#0 0x0806f32c in SSL_CTX_use_certificate_file ()
#1 0x08103cee in pq_getbytes (s=0xbfffeb5c \f, len=4) at
/root/src/postgres/src/backend/libpq/pqcomm.c:748
#2 0x08103e04 in pq_getmessage (s=0xbfffec10, maxlen=0) at
/root/src/postgres/src/backend/libpq/pqcomm.c:837
#3 0x0814c98b in SocketBackend (inBuf=0xbfffec10) at
/root/src/postgres/src/backend/tcop/postgres.c:377
(gdb) x/i $pc
0x806f32c SSL_CTX_use_certificate_file+8: add
$0x70e9,%eax
(gdb) disassemble
Dump of assembler code for function SSL_CTX_use_certificate_file:
0x0806f324 SSL_CTX_use_certificate_file+0:jmp*0x823f084
0x0806f32a SSL_CTX_use_certificate_file+6:push $0x5b8
0x0806f32f SSL_CTX_use_certificate_file+11: jmp0x806e7a4
_init+24
End of assembler dump.
(gdb) disassemble *0x823f084
Dump of
