Hi Aki,
Please see below:
Thank you
-Nick
# gdb /usr/libexec/dovecot/lmtp ./3445
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `dovecot/lmtp'.
Program terminated with signal 6, Aborted.
#0 0x00007f297814d1f7 in raise () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install
dovecot-2.3.0-8.cp1162.x86_64
(gdb) back
#0 0x00007f297814d1f7 in raise () from /lib64/libc.so.6
#1 0x00007f297814e8e8 in abort () from /lib64/libc.so.6
#2 0x00007f29785a3eab in default_fatal_finish (type=<optimized out>,
status=status@entry=0) at failures.c:228
#3 0x00007f29785a3f9e in i_internal_fatal_handler
(ctx=0x7ffe12c97a00, format=<optimized out>, args=<optimized out>) at
failures.c:718
#4 0x00007f2978517190 in i_panic (format=format@entry=0x7f29785df6a8
"file %s: line %d (%s): assertion failed: (%s)") at failures.c:306
#5 0x00007f29785bf6bb in event_unref
(_event=_event@entry=0x55ac2dab3ea8) at lib-event.c:148
#6 0x00007f29788a1b64 in mail_storage_service_user_unref
(_user=_user@entry=0x55ac2dab35a8) at mail-storage-service.c:1649
#7 0x000055ac2cbd03fa in lmtp_local_rcpt_deinit (rcpt=0x55ac2dab3570)
at lmtp-local.c:125
#8 0x000055ac2cbd08dc in lmtp_local_deinit
(_local=_local@entry=0x55ac2daa00f0) at lmtp-local.c:83
#9 0x000055ac2cbcfa5a in client_state_reset (client=0x55ac2daa0028)
at client.c:177
#10 0x00007f297852e96c in smtp_server_connection_reset_state
(conn=0x55ac2daaee40) at smtp-server-connection.c:1249
#11 0x00007f297852c31d in smtp_server_command_completed
(cmd=0x55ac2dab3998) at smtp-server-command.c:380
#12 0x00007f297852f848 in smtp_server_connection_next_reply
(conn=0x55ac2daaee40) at smtp-server-connection.c:623
#13 smtp_server_connection_send_replies (conn=0x55ac2daaee40) at
smtp-server-connection.c:682
#14 smtp_server_connection_output (conn=0x55ac2daaee40) at
smtp-server-connection.c:728
#15 0x00007f29785cb990 in stream_send_io (fstream=0x55ac2da75dd0) at
ostream-file.c:475
#16 0x00007f29785bb4d5 in io_loop_call_io (io=0x55ac2dab75b0) at
ioloop.c:614
#17 0x00007f29785bcdaf in io_loop_handler_run_internal
(ioloop=ioloop@entry=0x55ac2da6ac50) at ioloop-epoll.c:222
#18 0x00007f29785bb5d2 in io_loop_handler_run
(ioloop=ioloop@entry=0x55ac2da6ac50) at ioloop.c:666
#19 0x00007f29785bb7f8 in io_loop_run (ioloop=0x55ac2da6ac50) at
ioloop.c:639
#20 0x00007f2978539bb3 in master_service_run (service=0x55ac2da6aae0,
callback=callback@entry=0x55ac2cbcf3d0 <client_connected>) at
master-service.c:767
#21 0x000055ac2cbcf1a6 in main (argc=1, argv=0x55ac2da6a8a0) at
main.c:159
(gdb) bt full
#0 0x00007f297814d1f7 in raise () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007f297814e8e8 in abort () from /lib64/libc.so.6
No symbol table info available.
#2 0x00007f29785a3eab in default_fatal_finish (type=<optimized out>,
status=status@entry=0) at failures.c:228
backtrace = 0x55ac2da62088
"/usr/lib64/dovecot/libdovecot.so.0(+0xc8ee4) [0x7f29785a3ee4] ->
/usr/lib64/dovecot/libdovecot.so.0(+0xc8f9e) [0x7f29785a3f9e] ->
/usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7f2978517190] ->
/usr"...
recursed = 0
#3 0x00007f29785a3f9e in i_internal_fatal_handler
(ctx=0x7ffe12c97a00, format=<optimized out>, args=<optimized out>) at
failures.c:718
status = 0
#4 0x00007f2978517190 in i_panic (format=format@entry=0x7f29785df6a8
"file %s: line %d (%s): assertion failed: (%s)") at failures.c:306
ctx = {type = LOG_TYPE_PANIC, exit_status = 0, timestamp =
0x0, timestamp_usecs = 0, log_prefix = 0x0}
args = {{gp_offset = 40, fp_offset = 48, overflow_arg_area =
0x7ffe12c97af0, reg_save_area = 0x7ffe12c97a30}}
#5 0x00007f29785bf6bb in event_unref
(_event=_event@entry=0x55ac2dab3ea8) at lib-event.c:148
event = 0x55ac2dab8180
__func__ = "event_unref"
#6 0x00007f29788a1b64 in mail_storage_service_user_unref
(_user=_user@entry=0x55ac2dab35a8) at mail-storage-service.c:1649
user = 0x55ac2dab3e08
__func__ = "mail_storage_service_user_unref"
#7 0x000055ac2cbd03fa in lmtp_local_rcpt_deinit (rcpt=0x55ac2dab3570)
at lmtp-local.c:125
No locals.
#8 0x000055ac2cbd08dc in lmtp_local_deinit
(_local=_local@entry=0x55ac2daa00f0) at lmtp-local.c:83
rcptp_end = 0x55ac2da9fbb8
local = 0x55ac2dab8000
rcptp = 0x55ac2da9fbb0
#9 0x000055ac2cbcfa5a in client_state_reset (client=0x55ac2daa0028)
at client.c:177
No locals.
#10 0x00007f297852e96c in smtp_server_connection_reset_state
(conn=0x55ac2daaee40) at smtp-server-connection.c:1249
No locals.
#11 0x00007f297852c31d in smtp_server_command_completed
(cmd=0x55ac2dab3998) at smtp-server-command.c:380
hook_completed = <optimized out>
#12 0x00007f297852f848 in smtp_server_connection_next_reply
(conn=0x55ac2daaee40) at smtp-server-connection.c:623
cmd = 0x55ac2dab3998
error = 0x0
i = <optimized out>
#13 smtp_server_connection_send_replies (conn=0x55ac2daaee40) at
smtp-server-connection.c:682
No locals.
#14 smtp_server_connection_output (conn=0x55ac2daaee40) at
smtp-server-connection.c:728
ret = 1
#15 0x00007f29785cb990 in stream_send_io (fstream=0x55ac2da75dd0) at
ostream-file.c:475
ostream = 0x55ac2da75e60
use_cork = true
ret = <optimized out>
#16 0x00007f29785bb4d5 in io_loop_call_io (io=0x55ac2dab75b0) at
ioloop.c:614
ioloop = 0x55ac2da6ac50
t_id = 2
__func__ = "io_loop_call_io"
---Type <return> to continue, or q <return> to quit---
#17 0x00007f29785bcdaf in io_loop_handler_run_internal
(ioloop=ioloop@entry=0x55ac2da6ac50) at ioloop-epoll.c:222
ctx = 0x55ac2da761d0
events = <optimized out>
list = 0x55ac2da9c650
io = <optimized out>
tv = {tv_sec = 2147483, tv_usec = 0}
events_count = <optimized out>
msecs = <optimized out>
ret = 1
i = 0
call = <optimized out>
__func__ = "io_loop_handler_run_internal"
#18 0x00007f29785bb5d2 in io_loop_handler_run
(ioloop=ioloop@entry=0x55ac2da6ac50) at ioloop.c:666
__func__ = "io_loop_handler_run"
#19 0x00007f29785bb7f8 in io_loop_run (ioloop=0x55ac2da6ac50) at
ioloop.c:639
__func__ = "io_loop_run"
#20 0x00007f2978539bb3 in master_service_run (service=0x55ac2da6aae0,
callback=callback@entry=0x55ac2cbcf3d0 <client_connected>) at
master-service.c:767
No locals.
#21 0x000055ac2cbcf1a6 in main (argc=1, argv=0x55ac2da6a8a0) at
main.c:159
set_roots = {0x7f2978843fc0 <smtp_submit_setting_parser_info>,
0x7f2978da4820 <lda_setting_parser_info>, 0x55ac2cdd4560
<lmtp_setting_parser_info>, 0x0}
service_flags = <optimized out>
storage_service_flags = <optimized out>
tmp_base_dir = 0x55ac2da62040 "\003"
c = <optimized out>
error = 0x0
(gdb) p current_global_event
$1 = (struct event *) 0x55ac2dab8180
(gdb) p event
No symbol "event" in current context.
(gdb) p *current_global_event
$2 = {event_passthrough = {append_log_prefix = 0x7f29785bf830
<event_passthrough_set_append_log_prefix>, replace_log_prefix =
0x7f29785bf860 <event_passthrough_replace_log_prefix>,
set_name = 0x7f29785bf8d0 <event_passthrough_set_name>, set_source
= 0x7f29785bf930 <event_passthrough_set_source>, set_always_log_source
= 0x7f29785bf970 <event_passthrough_set_always_log_source>,
add_categories = 0x7f29785bfb00
<event_passthrough_add_categories>, add_category = 0x7f29785bfb70
<event_passthrough_add_category>, add_fields = 0x7f29785bfe80
<event_passthrough_add_fields>,
add_str = 0x7f29785bfcc0 <event_passthrough_add_str>, add_int =
0x7f29785bfd40 <event_passthrough_add_int>, add_timeval =
0x7f29785bfdd0 <event_passthrough_add_timeval>,
event = 0x7f29785bedd0 <event_passthrough_event>}, prev =
0x55ac2dac51a0, next = 0x55ac2da75be0, refcount = 0, pool =
0x55ac2dab8070, parent = 0x0, id = 2, log_prefix = 0x0,
log_prefix_from_system_pool = false, log_prefix_replace = false,
passthrough = false, forced_debug = false, always_log_source = false,
sending_debug_log = false, id_sent_to_stats = false,
call_free = false, tv_created = {tv_sec = 1517200513, tv_usec =
468845}, tv_last_sent = {tv_sec = 0, tv_usec = 0}, source_filename =
0x7f297894f95a "mail-storage-service.c", source_linenum = 1333,
sending_name = 0x0, categories = {arr = {buffer = 0x0, element_size
= 0}, v = 0x0, v_modifiable = 0x0}, fields = {arr = {buffer =
0x55ac2dab8278, element_size = 48}, v = 0x55ac2dab8278,
v_modifiable = 0x55ac2dab8278}}
On Jan 27, 2018, at 4:40 PM, Aki Tuomi <aki.tu...@dovecot.fi> wrote:
Hi!
This is a bug in the new event code. Can you get a full gdb backtrace?
Aki
On January 27, 2018 at 6:58 AM "J. Nick Koston" <n...@cpanel.net>
wrote:
Hi,
We are seeing a frequent assertion failure on LMTP delivery with
2.3.0. This only appears to happen on CentOS/RHEL 7.
Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Panic:
file lib-event.c: line 148 (event_unref): assertion failed: (event !=
current_global_event)
Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Error:
Raw backtrace: /usr/lib64/dovecot/libdovecot.so.0(+0xc8ec4)
[0x7f226d036ec4] -> /usr/lib64/dovecot/libdovecot.so.0(+0xc8f7e)
[0x7f226d036f7e] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0)
[0x7f226cfaa190] -> /usr/lib64/dovecot/libdovecot.so.0(+0xe469b)
[0x7f226d05269b] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mail_storage_service_user_unref+0xb4)
[0x7f226d334b64] -> dovecot/lmtp(+0x63fa) [0x56438f3673fa] ->
dovecot/lmtp(lmtp_local_deinit+0x5c) [0x56438f3678dc] ->
dovecot/lmtp(client_state_reset+0x1a) [0x56438f366a5a] ->
/usr/lib64/dovecot/libdovecot.so.0(smtp_server_connection_reset_state+0x3c)
[0x7f226cfc196c] ->
/usr/lib64/dovecot/libdovecot.so.0(smtp_server_command_completed+0x4d)
[0x7f226cfbf31d] -> /usr/lib64/dovecot/libdovecot.so.0(+0x54848)
[0x7f226cfc2848] -> /usr/lib64/dovecot/libdovecot.so.0(+0xf0970)
[0x7f226d05e970] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x65)
[0x7f226d04e4b5] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x10f)
[0x7f226d04fd8f] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x52)
[0x7f226d04e5b2] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7f226d04e7d8]
-> /usr/lib64/dovecot/libdovecot.so.0(master_service_run+0x13)
[0x7f226cfccbb3] -> dovecot/lmtp(main+0x226) [0x56438f3661a6] ->
/lib64/libc.so.6(__libc_start_main+0xf5) [0x7f226cbccc05] ->
dovecot/lmtp(+0x52d5) [0x56438f3662d5]
Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Fatal:
master: service(lmtp): child 29540 killed with signal 6 (core dumps
disabled)
From the exim log:
'LOG: MAIN cwd=/usr/local/cpanel 3 args: /usr/sbin/sendmail -v
bob@l56ob1nf.cptestLOG: MAIN <=
r...@smoker-devautomerge-c7-2.dev.cpanel.net U=root P=local S=3146285
T="Ignore this test from Cpanel-Email-Message-Retrieve.t"LOG: MAIN
cwd=/var/spool/exim 4 args: /usr/sbin/exim -v -Mc
1eemjH-0006uB-C2delivering 1eemjH-0006uB-C2 LMTP<< 220
smoker-devautomerge-c7-2.dev.cpanel.net Dovecot ready. LMTP>> LHLO
smoker-devautomerge-c7-2.dev.cpanel.net
LMTP<< 250-smoker-devautomerge-c7-2.dev.cpanel.net LMTP<<
250-8BITMIME LMTP<< 250-CHUNKING LMTP<< 250-ENHANCEDSTATUSCODES
LMTP<< 250-PIPELINING LMTP<< 250-STARTTLS LMTP<< 250-VRFY LMTP<<
250 XCLIENT ADDR PORT PROTO HELO LOGIN TTL TIMEOUT LMTP>> MAIL
FROM:<r...@smoker-devautomerge-c7-2.dev.cpanel.net>
LMTP<< 250 2.1.0 OK LMTP>> RCPT TO:<bob@l56ob1nf.cptest>
LMTP<< 250 2.1.5 OK LMTP>> DATA
LMTP<< 354 OK LMTP>> writing message and terminating "." LMTP>>
QUIT
LOG: MAIN == bob@l56ob1nf.cptest R=virtual_user
T=dovecot_virtual_delivery defer (-1): LMTP connection closed after
end of data’,
Jan 24 10:00:55 smoker-devautomerge-c7-1 dovecot: lmtp(30217): Panic:
file lib-event.c: line 148 (event_unref): assertion failed: (event !=
current_global_event)
Jan 24 10:00:55 smoker-devautomerge-c7-1 dovecot: lmtp(30233): Panic:
file lib-event.c: line 148 (event_unref): assertion failed: (event !=
current_global_event)
Jan 24 10:05:15 smoker-devautomerge-c7-1 dovecot: lmtp(32618): Panic:
file lib-event.c: line 148 (event_unref): assertion failed: (event !=
current_global_event)
Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(375): Panic:
file lib-event.c: line 148 (event_unref): assertion failed: (event !=
current_global_event)
Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(403): Panic:
file lib-event.c: line 148 (event_unref): assertion failed: (event !=
current_global_event)
Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(424): Panic:
file lib-event.c: line 148 (event_unref): assertion failed: (event !=
current_global_event)
Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(430): Panic:
file lib-event.c: line 148 (event_unref): assertion failed: (event !=
current_global_event)
Jan 24 12:32:16 smoker-devautomerge-c7-1 dovecot: lmtp(4435): Panic:
file lib-event.c: line 148 (event_unref): assertion failed: (event !=
current_global_event)
Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15232): Panic:
file lib-event.c: line 148 (event_unref): assertion failed: (event !=
current_global_event)
Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15238): Panic:
file lib-event.c: line 148 (event_unref): assertion failed: (event !=
current_global_event)
Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15244): Panic:
file lib-event.c: line 148 (event_unref): assertion failed: (event !=
current_global_event)
doveconf -n:
# 2.3.0 (c8b89eb): /etc/dovecot/dovecot.conf
# OS: Linux 3.10.0-693.2.2.el7.x86_64 x86_64 CentOS Linux release
7.4.1708 (Core)
auth_cache_size = 1 M
auth_mechanisms = plain login
auth_username_chars =
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$-=?^_{}~./@+%"
disable_plaintext_auth = no
first_valid_uid = 201
lda_mailbox_autocreate = yes
lmtp_rcpt_check_quota = yes
lmtp_save_to_detail_mailbox = yes
lmtp_user_concurrency_limit = 4
mail_access_groups = dovecot
mail_plugins = quota quota_clone zlib
mail_prefetch_count = 20
maildir_very_dirty_syncs = yes
namespace inbox {
inbox = yes
location =
mailbox Archive {
auto = create
special_use = \Archive
}
mailbox Archives {
auto = no
special_use = \Archive
}
mailbox Drafts {
auto = subscribe
special_use = \Drafts
}
mailbox Junk {
auto = create
special_use = \Junk
}
mailbox Sent {
auto = subscribe
special_use = \Sent
}
mailbox "Sent Messages" {
auto = no
special_use = \Sent
}
mailbox Trash {
auto = subscribe
special_use = \Trash
}
mailbox spam {
auto = create
special_use = \Junk
}
prefix = INBOX.
separator = .
type = private
}
passdb {
args = /usr/local/cpanel/etc/dovecot/cpauthd-dict.conf
driver = dict
result_failure = return-fail
}
passdb {
args = /usr/local/cpanel/bin/dovecot-wrap
driver = checkpassword
skip = authenticated
}
plugin {
acl = vfile:cache_secs=86400
quota_exceeded_message = Mailbox is full / Blocks limit exceeded /
Inode limit exceeded
}
protocols = lmtp imap pop3
service auth {
unix_listener auth-client {
mode = 0666
}
}
service config {
vsz_limit = 2 G
}
service dict {
unix_listener dict {
group = dovecot
mode = 0660
}
}
service imap-login {
client_limit = 500
inet_listener imap {
address = *,::
}
inet_listener imaps {
address = *,::
}
process_limit = 50
process_min_avail = 2
service_count = 0
vsz_limit = 128 M
}
service imap {
process_limit = 512
vsz_limit = 512 M
}
service lmtp {
client_limit = 1
process_limit = 500
unix_listener lmtp {
group = mail
mode = 0660
user = mailnull
}
vsz_limit = 512 M
}
service managesieve-login {
client_limit = 500
process_limit = 50
process_min_avail = 2
service_count = 0
vsz_limit = 128 M
}
service managesieve {
process_limit = 512
vsz_limit = 512 M
}
service pop3-login {
client_limit = 500
inet_listener pop3 {
address = *,::
}
inet_listener pop3s {
address = *,::
}
process_limit = 50
process_min_avail = 2
service_count = 0
vsz_limit = 128 M
}
service pop3 {
process_limit = 512
vsz_limit = 512 M
}
service quota-status {
executable = quota-status -p postfix
unix_listener quota-status {
mode = 0666
}
}
ssl_cert = </etc/dovecot/ssl/dovecot.crt
ssl_cipher_list =
ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
ssl_dh = # hidden, use -P to show it
ssl_key = # hidden, use -P to show it
userdb {
driver = prefetch
}
userdb {
args = /usr/local/cpanel/etc/dovecot/cpauthd-dict.conf
driver = dict
}
userdb {
args = /usr/local/cpanel/bin/dovecot-wrap
driver = checkpassword
}
protocol imap {
imap_capability = +NAMESPACE
imap_idle_notify_interval = 24 mins
imap_logout_format = in=%i, out=%o, bytes=%i/%o
mail_max_userip_connections = 20
mail_plugins = acl quota imap_quota zlib imap_zlib quota_clone
virtual
namespace sent {
hidden = yes
list = no
location =
virtual:/usr/local/cpanel/etc/dovecot/virtual/sent:INDEX=~/mail/virtual/%u/sent
prefix = sent
separator = .
}
namespace spam {
hidden = yes
list = no
location =
virtual:/usr/local/cpanel/etc/dovecot/virtual/spam:INDEX=~/mail/virtual/%u/spam
prefix = spam
separator = .
}
}
protocol pop3 {
mail_max_userip_connections = 3
mail_plugins = quota quota quota_clone virtual zlib
namespace sent {
hidden = yes
list = no
location =
virtual:/usr/local/cpanel/etc/dovecot/virtual/sent:INDEX=~/mail/virtual/%u/sent
prefix = sent
separator = .
}
namespace spam {
hidden = yes
list = no
location =
virtual:/usr/local/cpanel/etc/dovecot/virtual/spam:INDEX=~/mail/virtual/%u/spam
prefix = spam
separator = .
}
pop3_logout_format = top=%t/%p, retr=%r/%b, del=%d/%m, size=%s,
bytes=%i/%o
pop3_uidl_format = UID%u-%v
}
protocol lmtp {
mail_plugins = quota quota_clone zlib
postmaster_address = root
quota_full_tempfail = no
}
protocol lda {
mail_plugins = quota quota_clone zlib
postmaster_address = root
quota_full_tempfail = no
}
local_name smoker-devautomerge-c7-1.dev.cpanel.net {
ssl_cert = </etc/dovecot/ssl/dovecot.crt
ssl_key = # hidden, use -P to show it
}
