This is the second of several patches to limit the number of address
verification requests in the Postfix mail queue.
This patch implements the first part of the primary, before-queue,
enforcement mechanism that limits the number of probes in the active
queue to 1/4 of the active queue capacity. This part does not
consider the domain in an email address. Per-domain counters will
be added later.
During development of this patch it became clear that one address
verification request can result in multiple deliverability status
reports. If this is not taken into account, then the number of
pending requests will be under-estimated.
This problem does not exist with the earlier patch for the queue
manager. If you need to stop a flood of verification requests now,
use that patch.
The patch below eliminates one source of multiple deliverability
reports per address verification request. The workaround is to
report the deliverability only for the original address, and no
longer report the deliverability for the address that results from
rewriting or aliasing.
However, multiple deliverability reports per request also happen
with 1-to-many virtual alias expansion. For each virtual alias
expansion result, a deliverability status will be reported to the
verify(8) daemon.
To address this, the verify daemon will have to recognize that a
deliverability result arrives for an address before the automatic
refresh timer goes off. Such results are legitimate but they should
not affect the pending request counter.
Wietse
diff --exclude=man --exclude=html --exclude=README_FILES --exclude=INSTALL
--exclude=.indent.pro --exclude=Makefile.in -r -cr --exclude=util
--exclude=mantools --exclude=proto --exclude=oqmgr --exclude=qmgr
--exclude=mail_params.h /var/tmp/postfix-2.12-20140406/src/global/verify.c
./src/global/verify.c
*** /var/tmp/postfix-2.12-20140406/src/global/verify.c Tue Nov 1 17:43:13 2005
--- ./src/global/verify.c Fri Apr 25 19:12:42 2014
***************
*** 105,114 ****
--- 105,116 ----
if (var_verify_neg_cache || vrfy_stat == DEL_RCPT_STAT_OK) {
req_stat = verify_clnt_update(recipient->orig_addr, vrfy_stat,
my_dsn.reason);
+ #ifndef VAR_VRFY_PEND_LIMIT
if (req_stat == VRFY_STAT_OK && strcasecmp(recipient->address,
recipient->orig_addr) != 0)
req_stat = verify_clnt_update(recipient->address, vrfy_stat,
my_dsn.reason);
+ #endif
} else {
my_dsn.action = "undeliverable-but-not-cached";
req_stat = VRFY_STAT_OK;
diff --exclude=man --exclude=html --exclude=README_FILES --exclude=INSTALL
--exclude=.indent.pro --exclude=Makefile.in -r -cr --exclude=util
--exclude=mantools --exclude=proto --exclude=oqmgr --exclude=qmgr
--exclude=mail_params.h /var/tmp/postfix-2.12-20140406/src/verify/verify.c
./src/verify/verify.c
*** /var/tmp/postfix-2.12-20140406/src/verify/verify.c Sat Dec 3 19:48:05 2011
--- ./src/verify/verify.c Fri Apr 25 19:48:07 2014
***************
*** 148,153 ****
--- 148,160 ----
/* .IP "\fBaddress_verify_sender_dependent_default_transport_maps
($sender_dependent_default_transport_maps)\fR"
/* Overrides the sender_dependent_default_transport_maps parameter
/* setting for address verification probes.
+ /* SAFETY CONTROLS
+ /* .ad
+ /* .fi
+ /* Available in Postfix version 2.12 and later:
+ /* .IP "\fBaddress_verify_pending_request_limit (see 'postconf -d' output)\fR"
+ /* A safety limit that prevents address verification requests
+ /* from overwhelming the Postfix queue.
/* MISCELLANEOUS CONTROLS
/* .ad
/* .fi
***************
*** 246,251 ****
--- 253,259 ----
int var_verify_neg_exp;
int var_verify_neg_try;
int var_verify_scan_cache;
+ int var_vrfy_pend_limit;
/*
* State.
***************
*** 259,264 ****
--- 267,347 ----
#define STREQ(x,y) (strcmp(x,y) == 0)
/*
+ * Preventing verification requests from overwhelming the Postfix queue.
+ *
+ * First, the verify(8) daemon enforces a global (domain-independent) limit
on
+ * the number of pending verification requests. When this global limit is
+ * reached, the verify(8) daemon stops injecting new requests into the
+ * Postfix queue.
+ *
+ * By way of backup, the queue manager also enforces this global limit, to
+ * smooth out glitches due to program restarts. The queue manager enforces
+ * the global limit based on the content of the active queue, by tempfailing
+ * verification requests immediately.
+ *
+ * Later, the verify(8) daemon will maintain a fixed-size cache with
per-domain
+ * counters, so that it can suppress requests more selectively.
+ *
+ * Even with non-selective limits the impact on legitimate users will be
small.
+ * The verify(8) daemon proactively refreshes active addresses well before
+ * they expire. The non-selective limits will affect only addresses that are
+ * inactive (31 days by default) or addresses that are unknown.
+ */
+ static unsigned verify_pend_count;
+
+ /* verify_probe_queued - record that a probe has entered the mail queue */
+
+ static void verify_probe_queued(const char *addr)
+ {
+ const char myname[] = "verify_probe_queued";
+
+ verify_pend_count += 1;
+ if (msg_verbose)
+ msg_info("%s: address=%s pend_count=%u",
+ myname, addr, verify_pend_count);
+ }
+
+ /* verify_probe_done - chalk up another probe as completed */
+
+ static void verify_probe_done(const char *addr)
+ {
+ const char myname[] = "verify_probe_done";
+
+ /*
+ * Ignore the completion of requests that were issued before the current
+ * verify(8) process was started.
+ */
+ if (verify_pend_count == 0) {
+ if (msg_verbose)
+ msg_info("%s: ignoring address=%s without pending request count",
+ myname, addr);
+ } else {
+ verify_pend_count -= 1;
+ if (msg_verbose)
+ msg_info("%s: address=%s pend_count=%u",
+ myname, addr, verify_pend_count);
+ }
+ }
+
+ /* verify_probe_permit - permit or deny probe */
+
+ static int verify_probe_permit(const char *addr)
+ {
+ const char myname[] = "verify_probe_permit";
+
+ if (msg_verbose)
+ msg_info("%s: address=%s pend_count=%u pend_limit=%d",
+ myname, addr, verify_pend_count, var_vrfy_pend_limit);
+
+ if (verify_pend_count < var_vrfy_pend_limit) {
+ return (1);
+ } else {
+ msg_info("Dropping verification request for %s", addr);
+ return (0);
+ }
+ }
+
+ /*
* The address verification database consists of (address, data) tuples. The
* format of the data field is "status:probed:updated:text". The meaning of
* each field is:
***************
*** 387,392 ****
--- 470,476 ----
ATTR_TYPE_INT, MAIL_ATTR_STATUS, VRFY_STAT_OK,
ATTR_TYPE_END);
}
+ verify_probe_done(STR(addr));
}
vstring_free(buf);
vstring_free(addr);
***************
*** 395,409 ****
/* verify_post_mail_action - callback */
! static void verify_post_mail_action(VSTREAM *stream, void *unused_context)
{
/*
* Probe messages need no body content, because they are never delivered,
* deferred, or bounced.
*/
! if (stream != 0)
post_mail_fclose(stream);
}
/* verify_query_service - query address status */
--- 479,497 ----
/* verify_post_mail_action - callback */
! static void verify_post_mail_action(VSTREAM *stream, void *context)
{
+ char *addr = context;
/*
* Probe messages need no body content, because they are never delivered,
* deferred, or bounced.
*/
! if (stream != 0) {
! verify_probe_queued(addr);
! myfree(addr);
post_mail_fclose(stream);
+ }
}
/* verify_query_service - query address status */
***************
*** 418,423 ****
--- 506,513 ----
long probed;
long updated;
char *text;
+ int probe_permission_checked = 0;
+ int probe_permitted = 0;
if (attr_scan(client_stream, ATTR_FLAG_STRICT,
ATTR_TYPE_STR, MAIL_ATTR_ADDR, addr,
***************
*** 456,465 ****
|| (now - probed > PROBE_TTL /* safe to probe */
&& (POSITIVE_ENTRY_EXPIRED(addr_status, updated)
|| NEGATIVE_ENTRY_EXPIRED(addr_status, updated)))) {
! addr_status = DEL_RCPT_STAT_TODO;
! probed = 0;
! updated = 0;
! text = "Address verification in progress";
if (raw_data != 0 && var_verify_neg_cache == 0)
dict_cache_delete(verify_map, STR(addr));
}
--- 546,563 ----
|| (now - probed > PROBE_TTL /* safe to probe */
&& (POSITIVE_ENTRY_EXPIRED(addr_status, updated)
|| NEGATIVE_ENTRY_EXPIRED(addr_status, updated)))) {
! probe_permission_checked = 1;
! if ((probe_permitted = verify_probe_permit(STR(addr))) != 0) {
! addr_status = DEL_RCPT_STAT_TODO;
! probed = 0;
! updated = 0;
! text = "Address verification in progress";
! } else {
! addr_status = DEL_RCPT_STAT_DEFER;
! probed = 0;
! updated = 0;
! text = "Too many address verification requests";
! }
if (raw_data != 0 && var_verify_neg_cache == 0)
dict_cache_delete(verify_map, STR(addr));
}
***************
*** 494,500 ****
if (now - probed > PROBE_TTL
&& (POSITIVE_REFRESH_NEEDED(addr_status, updated)
! || NEGATIVE_REFRESH_NEEDED(addr_status, updated))) {
if (msg_verbose)
msg_info("PROBE %s status=%d probed=%ld updated=%ld",
STR(addr), addr_status, now, updated);
--- 592,600 ----
if (now - probed > PROBE_TTL
&& (POSITIVE_REFRESH_NEEDED(addr_status, updated)
! || NEGATIVE_REFRESH_NEEDED(addr_status, updated))
! && (probe_permission_checked ? probe_permitted
! : verify_probe_permit(STR(addr)))) {
if (msg_verbose)
msg_info("PROBE %s status=%d probed=%ld updated=%ld",
STR(addr), addr_status, now, updated);
***************
*** 503,509 ****
DEL_REQ_FLAG_MTA_VRFY,
(VSTRING *) 0,
verify_post_mail_action,
! (void *) 0);
if (updated != 0 || var_verify_neg_cache != 0) {
put_buf = vstring_alloc(10);
verify_make_entry(put_buf, addr_status, now, updated, text);
--- 603,609 ----
DEL_REQ_FLAG_MTA_VRFY,
(VSTRING *) 0,
verify_post_mail_action,
! mystrdup(STR(addr)));
if (updated != 0 || var_verify_neg_cache != 0) {
put_buf = vstring_alloc(10);
verify_make_entry(put_buf, addr_status, now, updated, text);
***************
*** 524,530 ****
/* verify_cache_validator - cache cleanup validator */
static int verify_cache_validator(const char *addr, const char *raw_data,
! char *context)
{
VSTRING *get_buf = (VSTRING *) context;
int addr_status;
--- 624,630 ----
/* verify_cache_validator - cache cleanup validator */
static int verify_cache_validator(const char *addr, const char *raw_data,
! char *context)
{
VSTRING *get_buf = (VSTRING *) context;
int addr_status;
***************
*** 709,714 ****
--- 809,818 ----
VAR_VERIFY_SENDER_TTL, DEF_VERIFY_SENDER_TTL, &var_verify_sender_ttl,
0, 0,
0,
};
+ static const CONFIG_INT_TABLE int_table[] = {
+ VAR_VRFY_PEND_LIMIT, DEF_VRFY_PEND_LIMIT, &var_vrfy_pend_limit, 1, 0,
+ 0,
+ };
/*
* Fingerprint executables and core dumps.
***************
*** 718,723 ****
--- 822,828 ----
multi_server_main(argc, argv, verify_service,
MAIL_SERVER_STR_TABLE, str_table,
MAIL_SERVER_TIME_TABLE, time_table,
+ MAIL_SERVER_INT_TABLE, int_table,
MAIL_SERVER_PRE_INIT, pre_jail_init,
MAIL_SERVER_POST_INIT, post_jail_init,
MAIL_SERVER_SOLITARY,
