I suggest updating your trunk version if using sa-compile, many metas depending on body rules might not have run at all..
Also check_rbl_sub rules not going occasionally missing will help.. On Fri, May 13, 2022 at 06:06:33AM -0000, h...@apache.org wrote: > Author: hege > Date: Fri May 13 06:06:33 2022 > New Revision: 1900849 > > URL: http://svn.apache.org/viewvc?rev=1900849&view=rev > Log: > - Bug 7987 > - fix body rules considered unrun when using sa-compile > - fix check_rbl_sub rules considered unrun and other DNSEval cleanups > - improve rule_pending/rule_ready/got_hit() logic > - rename $pms->get_pending_lookups to get_async_pending_rules > - other minor async cleanups > - test and documentation improvements > > Modified: > spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm > spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm > spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm > spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm > spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm > spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm > spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm > spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm > spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm > spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm > spamassassin/trunk/t/basic_meta2.t > spamassassin/trunk/t/basic_meta_net.t > spamassassin/trunk/t/dnsbl.t > spamassassin/trunk/t/sa_compile.t > spamassassin/trunk/t/uribl.t > > Modified: spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm (original) > +++ spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm Fri May 13 06:06:33 > 2022 > @@ -73,7 +73,7 @@ sub new { > queries_started => 0, > queries_completed => 0, > pending_lookups => { }, > - pending_rules => { }, > + pending_rules => { }, # maintain pending rules list for meta > evaluation > rules_for_key => { }, # record all rules used by a key for logging > timing_by_query => { }, > all_lookups => { }, # keyed by "rr_type/domain" > @@ -181,8 +181,17 @@ Hash of options. Only supported and requ > > =cut > > +sub start_queue { > + my($self) = @_; > + > + $self->{wait_queue} = 1; > +} > + > sub launch_queue { > my($self) = @_; > + > + delete $self->{wait_queue}; > + > if ($self->{bgsend_queue}) { > dbg("async: launching queued lookups"); > foreach (@{$self->{bgsend_queue}}) { > @@ -199,14 +208,17 @@ sub bgsend_and_start_lookup { > return if $self->{main}->{resolver}->{no_resolver}; > > # Waiting for priority -100 to launch? > - if ($self->{wait_launch}) { > + if ($self->{wait_queue}) { > push @{$self->{bgsend_queue}}, [@_]; > - dbg("async: dns priority not reached, queueing lookup: $domain/$type"); > + dbg("async: DNS priority not reached, queueing lookup: $domain/$type"); > return $ent; > } > > - if (!defined $ent->{rulename}) { > - info("async: bgsend_and_start_lookup called without rulename: > $domain/$type"); > + if (!defined $ent->{rulename} && !$self->{rulename_warned}++) { > + my($package, $filename, $line) = caller; > + warn "async: bgsend_and_start_lookup called without rulename, ". > + "from $package ($filename) line $line. You are likely using ". > + "a plugin that is not compatible with SpamAssasin 4.0.0."; > } > > $domain =~ s/\.+\z//s; # strip trailing dots, these sometimes still sneak > in > @@ -316,9 +328,10 @@ sub bgsend_and_start_lookup { > } > } > if ($blocked) { > - dbg("async: blocked by %s: %s", $blocked_by, $dnskey); > + dbg("async: blocked by %s: %s, rules: %s", $blocked_by, $dnskey, > + join(", ", @rulenames)); > } else { > - dbg("async: launching %s", $dnskey); > + dbg("async: launching %s, rules: %s", $dnskey, join(", ", @rulenames)); > $id = $self->{main}->{resolver}->bgsend($domain, $type, $class, sub { > my($pkt, $pkt_id, $timestamp) = @_; > # this callback sub is called from DnsResolver::poll_responses() > @@ -376,9 +389,12 @@ DIRECT USE DEPRECATED since 4.0.0, pleas > sub start_lookup { > my $self = shift; > > - warn "async: deprecated start_lookup called, please use > bgsend_and_start_lookup\n" > - if !$self->{start_lookup_warned}; > - $self->{start_lookup_warned} = 1; > + if (!$self->{start_lookup_warned}++) { > + my($package, $filename, $line) = caller; > + warn "async: deprecated start_lookup called, ". > + "from $package ($filename) line $line. You are likely using ". > + "a plugin that is not compatible with SpamAssasin 4.0.0."; > + } > > return if $self->{main}->{resolver}->{no_resolver}; > $self->_start_lookup(@_); > > Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm (original) > +++ spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm Fri May 13 06:06:33 2022 > @@ -110,6 +110,8 @@ sub do_rbl_lookup { > dbg("dns: launching rule %s, set %s, type %s, %s", $rule, $set, $type, > defined $subtest ? "subtest $subtest" : 'no subtest'); > > + $self->rule_pending($rule); # mark async > + > my $ent = { > rulename => $rule, > type => "DNSBL", > @@ -126,6 +128,8 @@ sub do_rbl_lookup { > sub do_dns_lookup { > my ($self, $rule, $type, $host) = @_; > > + $self->rule_pending($rule); # mark async > + > my $ent = { > rulename => $rule, > type => "DNSBL", > @@ -222,6 +226,20 @@ sub process_dnsbl_result { > my $question = ($pkt->question)[0]; > return if !$question; > > + my $rulename = $ent->{rulename}; > + > + # Mark rule ready for meta rules, but only if this was the last lookup > + # pending, rules can have many lookups launched for different IPs > + if (!$self->get_async_pending_rules($rulename)) { > + $self->rule_ready($rulename); > + # Mark depending check_rbl_sub rules too > + if (exists $self->{rbl_subs}{$ent->{set}}) { > + foreach (@{$self->{rbl_subs}{$ent->{set}}}) { > + $self->rule_ready($_->[1]); > + } > + } > + } > + > # DNSBL tests are here > foreach my $answer ($pkt->answer) { > next if !$answer; > @@ -256,17 +274,18 @@ sub process_dnsbl_result { > # check_rbl tests > if (defined $ent->{subtest}) { > if ($self->check_subtest($rdatastr, $ent->{subtest})) { > - $self->dnsbl_hit($ent->{rulename}, $question, $answer); > + $self->dnsbl_hit($rulename, $question, $answer); > } > } else { > - $self->dnsbl_hit($ent->{rulename}, $question, $answer); > + $self->dnsbl_hit($rulename, $question, $answer); > } > > # check_rbl_sub tests > - if (defined $self->{rbl_subs}{$ent->{set}}) { > + if (exists $self->{rbl_subs}{$ent->{set}}) { > $self->process_dnsbl_set($ent->{set}, $question, $answer, $rdatastr); > } > } > + > return 1; > } > > @@ -685,9 +704,9 @@ sub register_async_rule_finish {} > sub mark_all_async_rules_complete {} > sub is_rule_complete {} > > -# Return number of pending lookups for a rule, > +# Return number of pending DNS lookups for a rule, > # or list all of rules still pending > -sub get_pending_lookups { > +sub get_async_pending_rules { > my ($self, $rule) = @_; > if (defined $rule) { > return 0 if !exists $self->{async}->{pending_rules}{$rule}; > > Modified: spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm (original) > +++ spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm Fri May 13 > 06:06:33 2022 > @@ -3104,6 +3104,9 @@ sub got_hit { > $score = $params{defscore} if !defined $score; > } > > + # Make sure rule is marked ready in tests_pending also > + $self->{tests_pending}->{$rule} = 0; > + > # adding a hit does nothing if we don't have a score -- we probably > # shouldn't have run it in the first place > if (!$score) { > @@ -3170,20 +3173,25 @@ sub got_hit { > > =item $status->rule_pending ($rulename) > > -Register a pending rule. Must be called from rules eval-function, if the > -result can arrive later than when exiting the function (async lookups). > -This is not required if eval uses > $status->{async}->bgsend_and_start_lookup(), > -as it automatically registers a pending rule and also marks it ready when the > -result callback is run. > - > -$status->rule_ready($rulename) or $status->got_hit(...) must be called when > -the result has arrived. If these are not used, it can break dynamic meta > rule > -evaluation. This does not apply when bgsend_and_start_lookup() was used, as > -mentioned above. > +Register a pending rule. Must be called if the rules result is evaluated > +later than when exiting it's eval-function (e.g. async lookups). This is > +required for meta rules that depend on this rule to evaluate correctly. > + > +It is not strictly required when $status->{async}->bgsend_and_start_lookup() > +is used, as that automatically tracks pending DNS rules and also marks them > +ready when the result callback is run. But when a plugin has dummy stub > +eval-function (examples: check_uridnsbl, check_rbl_sub), then using > +rule_pending() and rule_ready() is necessary. Otherwise in some cases the > +eval-function could be run before any lookups are launched, and the rule > +would be marked ready too early. > + > +$status->rule_ready($rulename) or $status->got_hit($rulename, ...) must be > +called when the result has arrived. If these are not used, then any meta > +rules depending on the rule might not evaluate at all. > > =cut > > -# === Bug 7735 notes === > +# === Bug 7735 technical notes === > # > # Rule readiness status for dynamic meta rules evaluation (Plugins / Check / > # do_meta_tests) is always tracked by $pms->{tests_already_hit}->{$rule}. > @@ -3191,9 +3199,12 @@ mentioned above. > # > # The public rule_pending() / rule_ready() function pair is meant to be used > # for any async rules, and they additionally track rule status in > -# $pms->{tests_pending}->{$rule}, which do_meta_tests also checks. So any > -# rule_pending() call must always be accompanied later by rule_ready() or > -# got_hit(). These are not required when bgsend_and_start_lookup() is used. > +# $pms->{tests_pending}->{$rule}, which do_meta_tests also checks. Any > +# rule_pending() call _must_ be accompanied later by rule_ready() or > +# got_hit(). But as bgsend_and_start_lookup() automatically tracks rules > +# too, most of legacy plugins not using these should work correctly - only > +# in some corner cases where a dummy stub eval-function is called early by > +# accident (maybe due to adjusted rule priority) things might break. > # > # Any rule regardless of what it is must always be marked ready via > # $pms->{tests_already_hit} for meta evaluation to work, even if no > @@ -3201,38 +3212,43 @@ mentioned above. > # use $pms->{tests_already_hit} directly for marking, as is done for example > # by Check.pm for all basic body/header/etc rules (look for > # $hitsptr->{q{'.$rulename.'}} ||= 0). As such it's also always safe to > -# call rule_ready() without rule_pending(). > +# call rule_ready() without rule_pending(), it achieves the same. > > sub rule_pending { > my ($self, $rule) = @_; > > + # Ignore if called after rule_pending(), rule_ready() or got_hit(), they > + # are the only ones that make $self->{tests_pending}->{$rule} exist > + return if exists $self->{tests_pending}->{$rule}; > + > + # It's possible that tests_already_hit == 0, if some stub eval call was > + # run early. It's ok to clear this and pretend it was never run. We now > + # rely exclusively on tests_pending and wait for rule_ready() or got_hit() > + delete $self->{tests_already_hit}->{$rule} if > !$self->{tests_already_hit}->{$rule}; > $self->{tests_pending}->{$rule} = 1; > - > - if (exists $self->{tests_already_hit}->{$rule}) { > - # Only clear result if not hit > - if ($self->{tests_already_hit}->{$rule} == 0) { > - delete $self->{tests_already_hit}->{$rule}; > - } > - } > } > > =item $status->rule_ready ($rulename) > > -Mark a previously marked $status->rule_pending() rule ready. Alternatively > -$status->got_hit() will also mark rule ready. If these are not used, it can > -break dynamic meta rule evaluation. > +Mark a rule ready, so it can be considered for meta rule evaluation. Any > +rule regardless of type must always be marked ready when it's finished, > +otherwise any meta rules that depend on it might not evaluate. If > +$status->rule_pending() was called, then a $stats->rule_ready() call must be > +done, alternatively $status->got_hit() will also mark rule ready. > > =cut > > sub rule_ready { > my ($self, $rule) = @_; > > - if ($self->get_pending_lookups($rule)) { > - # Can't be ready if there are pending lookups, ignore for now. > + if ($self->get_async_pending_rules($rule)) { > + # Can't be ready if there are pending DNS lookups, ignore for now. > + # Make sure tests_pending exist, should not accept rule_pending() anymore > + $self->{tests_pending}->{$rule} ||= 0; > return; > } > > - delete $self->{tests_pending}->{$rule}; > + $self->{tests_pending}->{$rule} = 0; > $self->{tests_already_hit}->{$rule} ||= 0; > } > > > Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm (original) > +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm Fri May 13 > 06:06:33 2022 > @@ -433,13 +433,14 @@ sub launch_queries { > next; > } > dbg("askdns: launching query ($rulename): $query"); > - $pms->{async}->bgsend_and_start_lookup( > + my $ret = $pms->{async}->bgsend_and_start_lookup( > $query, $arule->{q_type}, undef, > { rulename => $rulename, type => 'AskDNS' }, > sub { my ($ent,$pkt) = @_; > $self->process_response_packet($pms, $ent, $pkt, $rulename) }, > master_deadline => $pms->{master_deadline} > ); > + $pms->rule_ready($rulename) if !$ret; # mark ready if nothing launched > } > } > > > Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm (original) > +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm Fri May 13 > 06:06:33 2022 > @@ -54,8 +54,8 @@ sub check_main { > my $pms = $args->{permsgstatus}; > $would_log_rules_all = would_log('dbg', 'rules-all') == 2; > > - # Make AsyncLoop wait permission for launching queries > - $pms->{async}->{wait_launch} = 1; > + # Make AsyncLoop wait launch_queue() for launching queries > + $pms->{async}->start_queue(); > > my $suppl_attrib = $pms->{msg}->{suppl_attrib}; > if (ref $suppl_attrib && ref $suppl_attrib->{rule_hits}) { > @@ -129,7 +129,6 @@ sub check_main { > # unneeded DNS queries. > if ($do_dns && !$rbls_running && $priority >= -100) { > $rbls_running = 1; > - $pms->{async}->{wait_launch} = 0; # permission granted > $pms->{async}->launch_queue(); # check if something was queued > $self->run_rbl_eval_tests($pms); > $self->{main}->call_plugins ("check_dnsbl", { permsgstatus => $pms }); > @@ -289,8 +288,8 @@ sub do_meta_tests { > my $h = $pms->{tests_already_hit}; > my $retry; > > - # Get pending async rule list > - my %pl = map { $_ => 1 } $pms->get_pending_lookups(); > + # Get pending DNS async rule list > + my %pl = map { $_ => 1 } $pms->get_async_pending_rules(); > > RULE: > foreach my $rulename (keys %$mp) { > @@ -323,6 +322,7 @@ sub finish_meta_tests { > return if $self->{am_compiling}; # nothing to compile here > > my $mp = $pms->{meta_pending}; > + my $tp = $pms->{tests_pending}; > my $md = $pms->{conf}->{meta_dependencies}; > my $mt = $pms->{conf}->{meta_tests}; > my $h = $pms->{tests_already_hit}; > @@ -334,7 +334,7 @@ RULE: > my %unrun; > # Meta is not ready if some dependency has not run yet > foreach my $deprule (@{$md->{$rulename}||[]}) { > - if (!exists $h->{$deprule}) { > + if (!exists $h->{$deprule} || $tp->{$deprule}) { > # Record all unrun deps for second meta evaluation > $unrun{$deprule} = 1; > } > > Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm (original) > +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm Fri May 13 > 06:06:33 2022 > @@ -150,13 +150,14 @@ sub set_config { > > sub finish_parsing_start { > my ($self, $opts) = @_; > + my $conf = $opts->{conf}; > > # Adjust priority -100 to launch early > # Find rulenames from eval_to_rule mappings > foreach my $evalfunc (@{$self->{'evalrules'}}) { > - foreach (@{$opts->{conf}->{eval_to_rule}->{$evalfunc}||[]}) { > + foreach (@{$conf->{eval_to_rule}->{$evalfunc}||[]}) { > dbg("dnseval: adjusting rule $_ priority to -100"); > - $opts->{conf}->{priority}->{$_} = -100; > + $conf->{priority}->{$_} = -100; > } > } > } > @@ -165,34 +166,33 @@ sub finish_parsing_start { > # directly as part of PMS > sub check_start { > my ($self, $opts) = @_; > + my $pms = $opts->{permsgstatus}; > > foreach(@{$self->{'evalrules'}}) { > - $opts->{'permsgstatus'}->register_plugin_eval_glue($_); > + $pms->register_plugin_eval_glue($_); > } > > # Initialize check_rbl_sub tests > - $self->init_rbl_subs($opts->{'permsgstatus'}); > + $self->_init_rbl_subs($pms); > } > > -sub init_rbl_subs { > +sub _init_rbl_subs { > my ($self, $pms) = @_; > - > - return if $pms->{rbl_subs}; > + my $conf = $pms->{conf}; > > # Very hacky stuff and direct rbl_evals usage for now, TODO rewrite > everything > - foreach my $rule (@{$pms->{conf}->{eval_to_rule}->{check_rbl_sub}}) { > - next if !exists $pms->{conf}->{rbl_evals}->{$rule}; > - next if !$pms->{conf}->{scores}->{$rule}; > + foreach my $rule (@{$conf->{eval_to_rule}->{check_rbl_sub}||[]}) { > + next if !exists $conf->{rbl_evals}->{$rule}; > + next if !$conf->{scores}->{$rule}; > # rbl_evals is [$function,[@args]] > - my $args = $pms->{conf}->{rbl_evals}->{$rule}->[1]; > - my $set = $args->[0]; > - my $subtest = $args->[1]; > + my $args = $conf->{rbl_evals}->{$rule}->[1]; > + my ($set, $subtest) = @$args; > if (!defined $subtest) { > warn("dnseval: missing subtest for rule $rule\n"); > next; > } > if ($subtest =~ /^sb:/) { > - info("dnseval: ignored $rule, SenderBase rules are deprecated"); > + warn("dnseval: ignored $rule, SenderBase rules are deprecated\n"); > next; > } > # Compile as regex if not pure ip/bitmask (same check in > process_dnsbl_result) > @@ -206,6 +206,7 @@ sub init_rbl_subs { > } > dbg("dnseval: initialize check_rbl_sub for rule $rule, set $set, subtest > $subtest"); > push @{$pms->{rbl_subs}{$set}}, [$subtest, $rule]; > + $pms->rule_pending($rule); # mark async, rule_ready() done in > Dns/process_dnsbl_result > } > } > > @@ -283,7 +284,7 @@ sub check_rbl_accreditor { > $self->message_accreditor_tag($pms); > } > if ($pms->{accreditor_tag}->{$accreditor}) { > - $self->check_rbl_backend($pms, $rule, $set, $rbl_server, 'A', $subtest); > + $self->_check_rbl_backend($pms, $rule, $set, $rbl_server, 'A', $subtest); > } > return 0; > } > @@ -324,7 +325,7 @@ sub message_accreditor_tag { > $pms->{accreditor_tag} = \%acctags; > } > > -sub check_rbl_backend { > +sub _check_rbl_backend { > my ($self, $pms, $rule, $set, $rbl_server, $type, $subtest) = @_; > > return if !exists $pms->{dnseval_ips}; # no untrusted ips > @@ -406,14 +407,12 @@ sub check_rbl_backend { > return 0; > } > > - $pms->rule_pending($rule); # mark async > - > dbg("dnseval: only inspecting the following IPs: ".join(", ", @ips)); > > foreach my $ip (@ips) { > - my $revip = reverse_ip_address($ip); > - $pms->do_rbl_lookup($rule, $set, $type, > - $revip.'.'.$rbl_server, $subtest) if defined $revip; > + if (defined(my $revip = reverse_ip_address($ip))) { > + $pms->do_rbl_lookup($rule, $set, $type, $revip.'.'.$rbl_server, > $subtest) > + } > } > > # note that results are not handled here, hits are handled directly > @@ -427,7 +426,7 @@ sub check_rbl { > return 0 if $self->{main}->{conf}->{skip_rbl_checks}; > return 0 if !$pms->is_dns_available(); > > - $self->check_rbl_backend($pms, $rule, $set, $rbl_server, 'A', $subtest); > + $self->_check_rbl_backend($pms, $rule, $set, $rbl_server, 'A', $subtest); > } > > sub check_rbl_txt { > @@ -436,12 +435,12 @@ sub check_rbl_txt { > return 0 if $self->{main}->{conf}->{skip_rbl_checks}; > return 0 if !$pms->is_dns_available(); > > - $self->check_rbl_backend($pms, $rule, $set, $rbl_server, 'TXT', $subtest); > + $self->_check_rbl_backend($pms, $rule, $set, $rbl_server, 'TXT', $subtest); > } > > sub check_rbl_sub { > my ($self, $pms, $rule, $set, $subtest) = @_; > - # just a dummy, check_start / init_rbl_subs handles the subs > + # just a dummy, _init_rbl_subs/do_rbl_lookup handles the subs > $pms->rule_pending($rule); # mark async > return 0; > } > @@ -454,8 +453,6 @@ sub check_rbl_from_host { > return 0 if $self->{main}->{conf}->{skip_rbl_checks}; > return 0 if !$pms->is_dns_available(); > > - $pms->rule_pending($rule); # mark async > - > $self->_check_rbl_addresses($pms, $rule, $set, $rbl_server, > $subtest, $pms->all_from_addrs()); > } > @@ -551,10 +548,11 @@ sub check_rbl_ns_from { > > dbg("dnseval: checking NS for host $domain"); > > - my $key = "NS:" . $domain; > + $pms->rule_pending($rule); # mark async > + > my $obj = { dom => $domain, rule => $rule, set => $set, rbl_server => > $rbl_server, subtest => $subtest }; > my $ent = { > - rulename => $rule, key => $key, zone => $domain, obj => $obj, type => > "URI-NS", > + rulename => $rule, zone => $domain, obj => $obj, type => "URI-NS", > }; > # dig $dom ns > $ent = $pms->{async}->bgsend_and_start_lookup( > > Modified: > spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm > (original) > +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm > Fri May 13 06:06:33 2022 > @@ -66,6 +66,21 @@ sub check_start { > } > } > > +sub check_cleanup { > + my ($self, $params) = @_; > + my $pms = $params->{permsgstatus}; > + my $hitsptr = $pms->{tests_already_hit}; > + my $scoresptr = $pms->{conf}->{scores}; > + > + # Force all body rules ready for meta rules. Need to do it here in > + # cleanup, because the body is scanned per line instead of per rule > + if ($pms->{conf}->{skip_body_rules}) { > + foreach (keys %{$pms->{conf}->{skip_body_rules}}) { > + $hitsptr->{$_} ||= 0 if $scoresptr->{$_}; > + } > + } > +} > + > ########################################################################### > > 1; > > Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm (original) > +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm Fri May 13 > 06:06:33 2022 > @@ -117,6 +117,11 @@ sub check_rules_at_priority { > $self->{one_line_body}->check_rules_at_priority($params); > } > > +sub check_cleanup { > + my ($self, $params) = @_; > + $self->{one_line_body}->check_cleanup($params); > +} > + > ########################################################################### > > sub run_body_fast_scan { > > Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm (original) > +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm Fri May 13 > 06:06:33 2022 > @@ -197,6 +197,11 @@ sub check_rules_at_priority { > $self->{one_line_body}->check_rules_at_priority($params); > } > > +sub check_cleanup { > + my ($self, $params) = @_; > + $self->{one_line_body}->check_cleanup($params); > +} > + > ########################################################################### > > sub run_body_fast_scan { > > Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm (original) > +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm Fri May 13 > 06:06:33 2022 > @@ -378,6 +378,8 @@ sub check_dnsbl { > foreach my $rulename (keys %{$conf->{uridnsbls}}) { > next if !$conf->{scores}->{$rulename}; > > + $pms->rule_pending($rulename); # mark async > + > my $rulecf = $conf->{uridnsbls}->{$rulename}; > my %tfl = map { ($_,1) } split(/\s+/, $conf->{tflags}->{$rulename}||''); > > > Modified: spamassassin/trunk/t/basic_meta2.t > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/t/basic_meta2.t?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/t/basic_meta2.t (original) > +++ spamassassin/trunk/t/basic_meta2.t Fri May 13 06:06:33 2022 > @@ -5,7 +5,7 @@ use lib 't'; > use SATest; sa_t_init("basic_meta2"); > > use Test::More; > -plan tests => 15; > +plan tests => 20; > > # --------------------------------------------------------------------------- > > @@ -19,6 +19,11 @@ plan tests => 15; > q{ TEST_META_7 } => '', > q{ TEST_META_A } => '', > q{ TEST_META_B } => '', > + q{ TEST_META_C } => '', > + q{ TEST_META_D } => '', > + q{ TEST_META_E } => '', > + q{ TEST_META_F } => '', > + q{ TEST_META_G } => '', > ); > > %anti_patterns = ( > @@ -80,6 +85,21 @@ tstlocalrules (qq{ > # local_tests_only > meta TEST_META_B NONEXISTINGRULE || local_tests_only > > + # complex metas with different priorities > + body __BAR_5 /a/ > + priority __BAR_5 -1000 > + body __BAR_6 /b/ > + priority __BAR_6 0 > + body __BAR_7 /c/ > + priority __BAR_7 1000 > + meta TEST_META_C __BAR_5 && __BAR_6 && __BAR_7 > + meta TEST_META_D __BAR_5 && __BAR_6 && TEST_META_C > + priority TEST_META_D -2000 > + meta TEST_META_E __BAR_6 && __BAR_7 && TEST_META_D > + meta TEST_META_F __BAR_5 && __BAR_7 && TEST_META_E > + priority TEST_META_F 2000 > + meta TEST_META_G TEST_META_C && TEST_META_D && TEST_META_E && TEST_META_F > + > }); > > sarun ("-L -t < data/nice/001 2>&1", \&patterns_run_cb); > > Modified: spamassassin/trunk/t/basic_meta_net.t > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/t/basic_meta_net.t?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/t/basic_meta_net.t (original) > +++ spamassassin/trunk/t/basic_meta_net.t Fri May 13 06:06:33 2022 > @@ -8,79 +8,87 @@ use Test::More; > plan skip_all => "Net tests disabled" unless > conf_bool('run_net_tests'); > plan skip_all => "Can't use Net::DNS Safely" unless > can_use_net_dns_safely(); > > -plan tests => 20; > +plan tests => 32; > > # --------------------------------------------------------------------------- > > > %patterns = ( > - q{ X_META_POS4 } => '', > + q{ 1.0 X_LOCAL_TESTS } => '', > ); > %anti_patterns = ( > - q{ X_URIBL_A } => '', > - q{ X_ASKDNS } => '', > - q{ X_META_POS1 } => '', > - q{ X_META_POS2 } => '', > - q{ X_META_POS3 } => '', > - q{ X_META_NEG1 } => '', > - q{ X_META_NEG2 } => '', > - q{ X_META_NEG3 } => '', > - q{ X_META_NEG4 } => '', > + q{ 1.0 X_URIBL_A } => '', > + q{ 1.0 X_ASKDNS } => '', > + q{ 1.0 X_DNSBL_TEST } => '', > + q{ 1.0 X_DNSBL_SUB } => '', > + q{ 1.0 X_META_POS1 } => '', > + q{ 1.0 X_META_POS2 } => '', > + q{ 1.0 X_META_POS3 } => '', > + q{ 1.0 X_META_POS4 } => '', > + q{ 1.0 X_META_POS5 } => '', > + q{ 1.0 X_META_NEG1 } => '', > + q{ 1.0 X_META_NEG2 } => '', > + q{ 1.0 X_META_NEG3 } => '', > + q{ 1.0 X_META_NEG4 } => '', > + q{ 1.0 X_META_NEG5 } => '', > + q{ 1.0 X_LOCAL_NEG } => '', > ); > > -# > -# Nothing should hit with a failed lookup > -# > - > -tstlocalrules (qq{ > - # Force DNS queries to fail/timeout > - rbl_timeout 2 1 > - dns_server 240.0.0.240 > - > +my $common_rules = q{ > urirhssub X_URIBL_A dnsbltest.spamassassin.org. A 2 > body X_URIBL_A eval:check_uridnsbl('X_URIBL_A') > tflags X_URIBL_A net > > askdns X_ASKDNS spamassassin.org TXT /./ > > + header X_DNSBL_TEST eval:check_rbl('test', > 'dnsbltest.spamassassin.org.') > + tflags X_DNSBL_TEST net > + > + header X_DNSBL_SUB eval:check_rbl_sub('test', '2') > + tflags X_DNSBL_SUB net > + > meta X_META_POS1 X_URIBL_A > meta X_META_POS2 X_ASKDNS > - meta X_META_POS3 X_URIBL_A || X_ASKDNS > + meta X_META_POS3 X_DNSBL_TEST > + meta X_META_POS4 X_DNSBL_SUB > + meta X_META_POS5 X_URIBL_A || X_ASKDNS || X_DNSBL_TEST || X_DNSBL_SUB > > meta X_META_NEG1 !X_URIBL_A > meta X_META_NEG2 !X_ASKDNS > - meta X_META_NEG3 !X_URIBL_A || !X_ASKDNS > + meta X_META_NEG3 !X_DNSBL_TEST > + meta X_META_NEG4 !X_DNSBL_SUB > + meta X_META_NEG5 !X_URIBL_A || !X_ASKDNS || !X_DNSBL_TEST || !X_DNSBL_SUB > +}; > + > +# > +# Nothing should hit with a timed out lookup > +# > + > +tstlocalrules (qq{ > + # Force DNS queries to fail/timeout > + rbl_timeout 2 1 > + dns_server 240.0.0.240 > + > + $common_rules > > # local_tests_only > - meta X_META_NEG4 local_tests_only > - meta X_META_POS4 !local_tests_only > + meta X_LOCAL_TESTS !local_tests_only > + meta X_LOCAL_NEG local_tests_only > }); > > -sarun ("-t < data/spam/dnsbl.eml 2>&1", \&patterns_run_cb); > +sarun ("-t < data/spam/dnsbl.eml", \&patterns_run_cb); > ok_all_patterns(); > > # > -# Local only, nothing should hit as nothing is queried > +# Local tests only, nothing should hit as nothing is queried > # > > tstlocalrules (qq{ > - urirhssub X_URIBL_A dnsbltest.spamassassin.org. A 2 > - body X_URIBL_A eval:check_uridnsbl('X_URIBL_A') > - tflags X_URIBL_A net > - > - askdns X_ASKDNS spamassassin.org TXT /./ > - > - meta X_META_POS1 X_URIBL_A > - meta X_META_POS2 X_ASKDNS > - meta X_META_POS3 X_URIBL_A || X_ASKDNS > - > - meta X_META_NEG1 !X_URIBL_A > - meta X_META_NEG2 !X_ASKDNS > - meta X_META_NEG3 !X_URIBL_A || !X_ASKDNS > + $common_rules > > # local_tests_only > - meta X_META_POS4 local_tests_only > - meta X_META_NEG4 !local_tests_only > + meta X_LOCAL_TESTS local_tests_only > + meta X_LOCAL_NEG !local_tests_only > }); > > sarun ("-t -L < data/spam/dnsbl.eml", \&patterns_run_cb); > > Modified: spamassassin/trunk/t/dnsbl.t > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/t/dnsbl.t?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/t/dnsbl.t (original) > +++ spamassassin/trunk/t/dnsbl.t Fri May 13 06:06:33 2022 > @@ -6,7 +6,10 @@ use SATest; sa_t_init("dnsbl"); > use Test::More; > plan skip_all => "Net tests disabled" unless conf_bool('run_net_tests'); > plan skip_all => "Can't use Net::DNS Safely" unless can_use_net_dns_safely(); > -plan tests => 21; > + > +# run many times to catch some random natured failures > +my $iterations = 5; > +plan tests => 22 * $iterations; > > # --------------------------------------------------------------------------- > # bind configuration currently used to support this test > @@ -71,6 +74,7 @@ EOF > q{'1.0 DNSBL_TXT_MISS'} => '', > q{'1.0 DNSBL_TEST_WHITELIST_MISS'} => '', > q{'14.35.17.212.untrusted.dnsbltest.spamassassin.org'} => '', > + q{/rules-all: unrun dependencies [^\n]+ (?:__|META_)?DNSBL_/} => '', > ); > > tstprefs(" > @@ -148,6 +152,9 @@ priority DNSBL_TEST_RELAY 2000 > > "); > > -sarun ("-t < data/spam/dnsbl.eml", \&patterns_run_cb); > -ok_all_patterns(); > +for (1 .. $iterations) { > + # rules-all debug needed for unrun check > + sarun ("-t -D rules-all < data/spam/dnsbl.eml 2>&1", \&patterns_run_cb); > + ok_all_patterns(); > +} > > > Modified: spamassassin/trunk/t/sa_compile.t > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/t/sa_compile.t?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/t/sa_compile.t (original) > +++ spamassassin/trunk/t/sa_compile.t Fri May 13 06:06:33 2022 > @@ -54,35 +54,36 @@ body FOO5 /éèæ©/ > body FOO6 /éè(?:xyz|æ©)/ > body FOO7 /\xe9\x87\x91\xe8\x9e\x8d\xe6\xa9\x9f/ > body FOO8 /.\x87(?:\x91|\x00)[\xe8\x00]\x9e\x8d\xe6\xa9\x9f/ > +# Test that meta rules work for sa-compiled body rules > +# (loosely related to Bug 7987) > +meta META1 FOO1 && FOO2 && FOO3 && FOO4 > +meta META2 FOO5 && FOO6 && FOO7 && FOO8 > '); > > # ensure we don't use compiled rules > untaint_system("rm -rf $instdir/var/spamassassin/compiled"); > > %patterns = ( > - '/ check: tests=FOO1,FOO2,FOO3,FOO4\n/', 'FOO', > + '/ check: tests=FOO1,FOO2,FOO3,FOO4,META1\n/', '', > ); > %anti_patterns = ( > '/ zoom: able to use /', '', > ); > -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 1' < > $cwd/data/spam/001 2>&1", \&patterns_run_cb); > +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 1' < > $cwd/data/spam/001 2>&1", \&patterns_run_cb); > ok_all_patterns(); > -clear_pattern_counters(); > -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 0' < > $cwd/data/spam/001 2>&1", \&patterns_run_cb); > +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 0' < > $cwd/data/spam/001 2>&1", \&patterns_run_cb); > ok_all_patterns(); > -clear_pattern_counters(); > + > %patterns = ( > - '/ check: tests=FOO4,FOO5,FOO6,FOO7,FOO8\n/', 'FOO', > + '/ check: tests=FOO4,FOO5,FOO6,FOO7,FOO8,META2\n/', '', > ); > %anti_patterns = ( > '/ zoom: able to use /', '', > ); > -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 1' < > $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb); > +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 1' < > $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb); > ok_all_patterns(); > -clear_pattern_counters(); > -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 0' < > $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb); > +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 0' < > $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb); > ok_all_patterns(); > -clear_pattern_counters(); > > # ------------------------------------------------------------------- > > @@ -92,27 +93,24 @@ $scr = "$instdir/$temp_binpath/spamassas > $scr_localrules_args = $scr_cf_args = ""; # use the default rules dir, > from our "install" > > %patterns = ( > - q{ zoom: able to use 5/5 'body_0' compiled rules }, 'able-to-use', > - '/ check: tests=FOO1,FOO2,FOO3,FOO4\n/', 'FOO', > + q{ zoom: able to use 5/5 'body_0' compiled rules }, '', > + '/ check: tests=FOO1,FOO2,FOO3,FOO4,META1\n/', '', > ); > %anti_patterns = (); > -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 1' < > $cwd/data/spam/001 2>&1", \&patterns_run_cb); > +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 1' < > $cwd/data/spam/001 2>&1", \&patterns_run_cb); > ok_all_patterns(); > -clear_pattern_counters(); > -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 0' < > $cwd/data/spam/001 2>&1", \&patterns_run_cb); > +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 0' < > $cwd/data/spam/001 2>&1", \&patterns_run_cb); > ok_all_patterns(); > -clear_pattern_counters(); > + > %patterns = ( > - q{ zoom: able to use 5/5 'body_0' compiled rules }, 'able-to-use', > - '/ check: tests=FOO4,FOO5,FOO6,FOO7,FOO8\n/', 'FOO', > + q{ zoom: able to use 5/5 'body_0' compiled rules }, '', > + '/ check: tests=FOO4,FOO5,FOO6,FOO7,FOO8,META2\n/', '', > ); > %anti_patterns = (); > -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 1' < > $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb); > +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 1' < > $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb); > ok_all_patterns(); > -clear_pattern_counters(); > -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 0' < > $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb); > +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 0' < > $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb); > ok_all_patterns(); > -clear_pattern_counters(); > > # ------------------------------------------------------------------- > > > Modified: spamassassin/trunk/t/uribl.t > URL: > http://svn.apache.org/viewvc/spamassassin/trunk/t/uribl.t?rev=1900849&r1=1900848&r2=1900849&view=diff > ============================================================================== > --- spamassassin/trunk/t/uribl.t (original) > +++ spamassassin/trunk/t/uribl.t Fri May 13 06:06:33 2022 > @@ -6,7 +6,10 @@ use SATest; sa_t_init("uribl"); > use Test::More; > plan skip_all => "Net tests disabled" unless > conf_bool('run_net_tests'); > plan skip_all => "Can't use Net::DNS Safely" unless > can_use_net_dns_safely(); > -plan tests => 10; > + > +# run many times to catch some random natured failures > +my $iterations = 5; > +plan tests => 10 * $iterations; > > # --------------------------------------------------------------------------- > > @@ -68,6 +71,8 @@ tstlocalrules(q{ > > }); > > -ok sarun ("-t < data/spam/dnsbl.eml", \&patterns_run_cb); > -ok_all_patterns(); > +for (1 .. $iterations) { > + ok sarun ("-t < data/spam/dnsbl.eml", \&patterns_run_cb); > + ok_all_patterns(); > +} > >
