Hi, cPanel has developed a native Perl Pyzor implementation for SpamAssassin and a diff against SpamAssassin 4.0 follows. Atm I am using it in production on a small server, more tests and opinions are welcome.
Original cPanel code is at https://metacpan.org/pod/Mail::Pyzor. Cheers Giovanni diff --git a/MANIFEST b/MANIFEST index 25d0192..2d9588c 100644 --- a/MANIFEST +++ b/MANIFEST @@ -126,6 +126,11 @@ lib/Mail/SpamAssassin/Plugin/WLBLEval.pm lib/Mail/SpamAssassin/Plugin/WhiteListSubject.pm lib/Mail/SpamAssassin/PluginHandler.pm lib/Mail/SpamAssassin/Plugin/URILocalBL.pm +lib/Mail/SpamAssassin/Pyzor/Client.pm +lib/Mail/SpamAssassin/Pyzor/Digest/Pieces.pm +lib/Mail/SpamAssassin/Pyzor/Digest/StripHtml.pm +lib/Mail/SpamAssassin/Pyzor/Digest.pm +lib/Mail/SpamAssassin/Pyzor.pm lib/Mail/SpamAssassin/RegistryBoundaries.pm lib/Mail/SpamAssassin/Reporter.pm lib/Mail/SpamAssassin/SQLBasedAddrList.pm diff --git a/lib/Mail/SpamAssassin/Plugin/Pyzor.pm b/lib/Mail/SpamAssassin/Plugin/Pyzor.pm index 3efd4b4..e4c9c05 100644 --- a/lib/Mail/SpamAssassin/Plugin/Pyzor.pm +++ b/lib/Mail/SpamAssassin/Plugin/Pyzor.pm @@ -36,17 +36,13 @@ package Mail::SpamAssassin::Plugin::Pyzor; use Mail::SpamAssassin::Plugin; use Mail::SpamAssassin::Logger; -use Mail::SpamAssassin::Timeout; -use Mail::SpamAssassin::Util qw(untaint_var untaint_file_path - proc_status_ok exit_status_str); +use Mail::SpamAssassin::Util qw(untaint_var); + use strict; use warnings; # use bytes; use re 'taint'; -use Storable; -use POSIX qw(PIPE_BUF WNOHANG _exit); - our @ISA = qw(Mail::SpamAssassin::Plugin); sub new { @@ -78,7 +74,7 @@ sub set_config { my ($self, $conf) = @_; my @cmds; -=head1 USER OPTIONS +=head1 ADMINISTRATOR OPTIONS =over 4 @@ -95,22 +91,7 @@ Whether to use Pyzor, if it is available. type => $Mail::SpamAssassin::Conf::CONF_TYPE_BOOL }); -=item pyzor_fork (0|1) (default: 0) - -Instead of running Pyzor synchronously, fork separate process for it and -read the results in later (similar to async DNS lookups). Increases -throughput. Experimental. - -=cut - - push(@cmds, { - setting => 'pyzor_fork', - is_admin => 1, - default => 0, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC, - }); - -=item pyzor_count_min NUMBER (default: 5) +=item pyzor_count_min NUMBER (default: 5) This option sets how often a message's body checksum must have been reported to the Pyzor server before SpamAssassin will consider the Pyzor @@ -128,54 +109,8 @@ set this to a relatively low value, e.g. C<5>. type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC }); - # Deprecated setting, the name makes no sense! - push (@cmds, { - setting => 'pyzor_max', - is_admin => 1, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC, - code => sub { - my ($self, $key, $value, $line) = @_; - warn("deprecated setting used, change pyzor_max to pyzor_count_min\n"); - if ($value !~ /^\d+$/) { - return $Mail::SpamAssassin::Conf::INVALID_VALUE; - } - $self->{pyzor_count_min} = $value; - } - }); - -=item pyzor_whitelist_min NUMBER (default: 10) - -This option sets how often a message's body checksum must have been -whitelisted to the Pyzor server for SpamAssassin to consider ignoring the -result. Final decision is made by pyzor_whitelist_factor. - -=cut - - push (@cmds, { - setting => 'pyzor_whitelist_min', - is_admin => 1, - default => 10, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC - }); - -=item pyzor_whitelist_factor NUMBER (default: 0.2) - -Ignore Pyzor result if REPORTCOUNT x NUMBER >= pyzor_whitelist_min. -For default setting this means: 50 reports requires 10 whitelistings. - -=cut - - push (@cmds, { - setting => 'pyzor_whitelist_factor', - is_admin => 1, - default => 0.2, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC - }); - =back -=head1 ADMINISTRATOR OPTIONS - =over 4 =item pyzor_timeout n (default: 5) @@ -210,478 +145,182 @@ removing one of them. type => $Mail::SpamAssassin::Conf::CONF_TYPE_DURATION }); -=item pyzor_options options +=item pyzor_whitelist_min NUMBER (default: 10) -Specify additional options to the pyzor(1) command. Please note that only -characters in the range [0-9A-Za-z =,._/-] are allowed for security reasons. +This option sets how often a message's body checksum must have been +whitelisted to the Pyzor server for SpamAssassin to consider ignoring the +result. Final decision is made by pyzor_whitelist_factor. =cut push (@cmds, { - setting => 'pyzor_options', + setting => 'pyzor_whitelist_min', is_admin => 1, - default => '', - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING, - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value !~ m{^([0-9A-Za-z =,._/-]+)$}) { - return $Mail::SpamAssassin::Conf::INVALID_VALUE; - } - $self->{pyzor_options} = $1; - } + default => 10, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC }); -=item pyzor_path STRING +=item pyzor_whitelist_factor NUMBER (default: 0.2) -This option tells SpamAssassin specifically where to find the C<pyzor> -client instead of relying on SpamAssassin to find it in the current -PATH. Note that if I<taint mode> is enabled in the Perl interpreter, -you should use this, as the current PATH will have been cleared. +Ignore Pyzor result if REPORTCOUNT x NUMBER >= pyzor_whitelist_min. +For default setting this means: 50 reports requires 10 whitelistings. =cut push (@cmds, { - setting => 'pyzor_path', + setting => 'pyzor_whitelist_factor', is_admin => 1, - default => undef, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING, - code => sub { - my ($self, $key, $value, $line) = @_; - if (!defined $value || !length $value) { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - $value = untaint_file_path($value); - if (!-x $value) { - info("config: pyzor_path \"$value\" isn't an executable"); - return $Mail::SpamAssassin::Conf::INVALID_VALUE; - } - - $self->{pyzor_path} = $value; - } + default => 0.2, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC }); $conf->{parser}->register_commands(\@cmds); } sub is_pyzor_available { - my ($self) = @_; + my ($self) = @_; - my $pyzor = $self->{main}->{conf}->{pyzor_path} || - Mail::SpamAssassin::Util::find_executable_in_env_path('pyzor'); - - unless ($pyzor && -x $pyzor) { - dbg("pyzor: no pyzor executable found"); - $self->{pyzor_available} = 0; - return 0; - } - - # remember any found pyzor - $self->{main}->{conf}->{pyzor_path} = $pyzor; - - dbg("pyzor: pyzor is available: $pyzor"); - return 1; + local $@; + eval { + require Mail::SpamAssassin::Pyzor::Digest; + require Mail::SpamAssassin::Pyzor::Client; + }; + return $@ ? 0 : 1; } -sub finish_parsing_start { - my ($self, $opts) = @_; +sub get_pyzor_interface { + my ($self) = @_; - # If forking, hard adjust priority -100 to launch early - # Find rulenames from eval_to_rule mappings - if ($opts->{conf}->{pyzor_fork}) { - foreach (@{$opts->{conf}->{eval_to_rule}->{check_pyzor}}) { - dbg("pyzor: adjusting rule $_ priority to -100"); - $opts->{conf}->{priority}->{$_} = -100; - } + if (!$self->{main}->{conf}->{use_pyzor}) { + dbg("pyzor: use_pyzor option not enabled, disabling Pyzor"); + $self->{pyzor_interface} = "disabled"; + $self->{pyzor_available} = 0; + } + elsif ($self->is_pyzor_available()) { + $self->{pyzor_interface} = "pyzor"; + $self->{pyzor_available} = 1; + } + else { + dbg("pyzor: no pyzor found, disabling Pyzor"); + $self->{pyzor_available} = 0; } } sub check_pyzor { - my ($self, $pms, $full) = @_; - - return 0 if !$self->{pyzor_available}; - return 0 if !$self->{main}->{conf}->{use_pyzor}; - - return 0 if $pms->{pyzor_running}; - $pms->{pyzor_running} = 1; - - return 0 if !$self->is_pyzor_available(); - - my $timer = $self->{main}->time_method("check_pyzor"); + my ($self, $permsgstatus, $full) = @_; # initialize valid tags - $pms->{tag_data}->{PYZOR} = ''; - - # create fulltext tmpfile now (before possible forking) - $pms->{pyzor_tmpfile} = $pms->create_fulltext_tmpfile(); - - ## non-forking method - - if (!$self->{main}->{conf}->{pyzor_fork}) { - my @results = $self->pyzor_lookup($pms); - return $self->_check_result($pms, \@results); - } - - ## forking method - - $pms->{pyzor_rulename} = $pms->get_current_eval_rule_name(); - $pms->rule_pending($pms->{pyzor_rulename}); # mark async - - # create socketpair for communication - $pms->{pyzor_backchannel} = Mail::SpamAssassin::SubProcBackChannel->new(); - my $back_selector = ''; - $pms->{pyzor_backchannel}->set_selector(\$back_selector); - eval { - $pms->{pyzor_backchannel}->setup_backchannel_parent_pre_fork(); - } or do { - dbg("pyzor: backchannel pre-setup failed: $@"); - delete $pms->{pyzor_backchannel}; - return 0; - }; + $permsgstatus->{tag_data}->{PYZOR} = ""; - my $pid = fork(); - if (!defined $pid) { - info("pyzor: child fork failed: $!"); - delete $pms->{pyzor_backchannel}; - return 0; - } - if (!$pid) { - $0 = "$0 (pyzor)"; - $SIG{CHLD} = 'DEFAULT'; - $SIG{PIPE} = 'IGNORE'; - $SIG{$_} = sub { - eval { dbg("pyzor: child process $$ caught signal $_[0]"); }; - _exit(6); # avoid END and destructor processing - kill('KILL',$$); # still kicking? die! - } foreach qw(INT HUP TERM TSTP QUIT USR1 USR2); - dbg("pyzor: child process $$ forked"); - $pms->{pyzor_backchannel}->setup_backchannel_child_post_fork(); - my @results = $self->pyzor_lookup($pms); - my $backmsg; - eval { - $backmsg = Storable::freeze(\@results); - }; - if ($@) { - dbg("pyzor: child return value freeze failed: $@"); - _exit(0); # avoid END and destructor processing - } - if (!syswrite($pms->{pyzor_backchannel}->{parent}, $backmsg)) { - dbg("pyzor: child backchannel write failed: $!"); - } - _exit(0); # avoid END and destructor processing - } - - $pms->{pyzor_pid} = $pid; + my $timer = $self->{main}->time_method("check_pyzor"); - eval { - $pms->{pyzor_backchannel}->setup_backchannel_parent_post_fork($pid); - } or do { - dbg("pyzor: backchannel post-setup failed: $@"); - delete $pms->{pyzor_backchannel}; - return 0; - }; + $self->get_pyzor_interface(); + return 0 unless $self->{pyzor_available}; - return 0; + return $self->pyzor_lookup($permsgstatus, $full); } sub pyzor_lookup { - my ($self, $pms) = @_; - - my $conf = $self->{main}->{conf}; - my $timeout = $conf->{pyzor_timeout}; - - # note: not really tainted, this came from system configuration file - my $path = untaint_file_path($conf->{pyzor_path}); - my $opts = untaint_var($conf->{pyzor_options}) || ''; - - $pms->enter_helper_run_mode(); - - my $pid; - my @resp; - my $timer = Mail::SpamAssassin::Timeout->new( - { secs => $timeout, deadline => $pms->{master_deadline} }); - my $err = $timer->run_and_catch(sub { - local $SIG{PIPE} = sub { die "__brokenpipe__ignore__\n" }; - - dbg("pyzor: opening pipe: ". - join(' ', $path, $opts, "check", "<".$pms->{pyzor_tmpfile})); - - $pid = Mail::SpamAssassin::Util::helper_app_pipe_open(*PYZOR, - $pms->{pyzor_tmpfile}, 1, $path, split(' ', $opts), "check"); - $pid or die "$!\n"; - - # read+split avoids a Perl I/O bug (Bug 5985) - my($inbuf, $nread); - my $resp = ''; - while ($nread = read(PYZOR, $inbuf, 8192)) { $resp .= $inbuf } - defined $nread or die "error reading from pipe: $!"; - @resp = split(/^/m, $resp, -1); - - my $errno = 0; - close PYZOR or $errno = $!; - if (proc_status_ok($?, $errno)) { - dbg("pyzor: [%s] finished successfully", $pid); - } elsif (proc_status_ok($?, $errno, 0, 1)) { # sometimes it exits with 1 - dbg("pyzor: [%s] finished: %s", $pid, exit_status_str($?, $errno)); - } else { - info("pyzor: [%s] error: %s", $pid, exit_status_str($?, $errno)); - } - - }); - - if (defined(fileno(*PYZOR))) { # still open - if ($pid) { - if (kill('TERM', $pid)) { - dbg("pyzor: killed stale helper [$pid]"); - } else { - dbg("pyzor: killing helper application [$pid] failed: $!"); - } - } - my $errno = 0; - close PYZOR or $errno = $!; - proc_status_ok($?, $errno) - or info("pyzor: [%s] error: %s", $pid, exit_status_str($?, $errno)); - } - - $pms->leave_helper_run_mode(); - - if ($timer->timed_out()) { - dbg("pyzor: check timed out after $timeout seconds"); - return (); - } elsif ($err) { - chomp $err; - info("pyzor: check failed: $err"); - return (); - } - - return @resp; -} - -sub check_tick { - my ($self, $opts) = @_; - $self->_check_forked_result($opts->{permsgstatus}, 0); -} - -sub check_cleanup { - my ($self, $opts) = @_; - $self->_check_forked_result($opts->{permsgstatus}, 1); -} - -sub _check_forked_result { - my ($self, $pms, $finish) = @_; - - return 0 if !$pms->{pyzor_backchannel}; - return 0 if !$pms->{pyzor_pid}; + my ( $self, $permsgstatus, $fulltext ) = @_; + my $conf = $self->{main}->{conf}; + my $timeout = $conf->{pyzor_timeout}; + + my $client = ( $self->{'_pyzor_client'} ||= Mail::SpamAssassin::Pyzor::Client->new( 'timeout' => $timeout ) ); + my $digest = Mail::SpamAssassin::Pyzor::Digest::get( $fulltext ); + + local $@; + my $ref = eval { $client->check($digest); }; + dbg("pyzor: got response: $client->{'_server_host'}"); + # $client reply must be an hash + return 0 if (not (ref $ref eq ref {})); + if ($@) { + my $err = $@; - my $timer = $self->{main}->time_method("check_pyzor"); + $err = eval { $err->get_message() } || $err; - $pms->{pyzor_abort} = $pms->{deadline_exceeded} || $pms->{shortcircuited}; - - my $kid_pid = $pms->{pyzor_pid}; - # if $finish, force waiting for the child - my $pid = waitpid($kid_pid, $finish && !$pms->{pyzor_abort} ? 0 : WNOHANG); - if ($pid == 0) { - #dbg("pyzor: child process $kid_pid not finished yet, trying later"); - if ($pms->{pyzor_abort}) { - dbg("pyzor: bailing out due to deadline/shortcircuit"); - kill('TERM', $kid_pid); - if (waitpid($kid_pid, WNOHANG) == 0) { - sleep(1); - if (waitpid($kid_pid, WNOHANG) == 0) { - dbg("pyzor: child process $kid_pid still alive, KILL"); - kill('KILL', $kid_pid); - waitpid($kid_pid, 0); + warn("pyzor: check failed: $err\n"); + return 0; + } elsif ( defined $ref->{'Code'} and $ref->{'Code'} ne 200 ) { + if(defined $ref->{'Code'} and defined $ref->{'Diag'}) { + dbg("pyzor: check failed with invalid code: $ref->{'Code'}: $ref->{'Diag'}"); + } else { + dbg("pyzor: check failed with undefined code"); } - } - delete $pms->{pyzor_pid}; - delete $pms->{pyzor_backchannel}; + return 0; } - return 0; - } elsif ($pid == -1) { - # child does not exist? - dbg("pyzor: child process $kid_pid already handled?"); - delete $pms->{pyzor_backchannel}; - return 0; - } - $pms->rule_ready($pms->{pyzor_rulename}); # mark rule ready for metas + my $pyzor_count = untaint_var($ref->{'Count'}) + 0; + my $pyzor_whitelisted = untaint_var($ref->{'WL-Count'}) + 0; + my $count_min = $conf->{pyzor_count_min}; + my $wl_min = $conf->{pyzor_whitelist_min}; - dbg("pyzor: child process $kid_pid finished, reading results"); + my $wl_limit = $pyzor_whitelisted >= $wl_min ? + $pyzor_count * $conf->{pyzor_whitelist_factor} : 0; - my $backmsg; - my $ret = sysread($pms->{pyzor_backchannel}->{latest_kid_fh}, $backmsg, PIPE_BUF); - if (!defined $ret || $ret == 0) { - dbg("pyzor: could not read result from child: ".($ret == 0 ? 0 : $!)); - delete $pms->{pyzor_backchannel}; - return 0; - } - - delete $pms->{pyzor_backchannel}; + $permsgstatus->set_tag('PYZOR', "Reported $pyzor_count times, whitelisted $pyzor_whitelisted times."); - my $results; - eval { - $results = Storable::thaw($backmsg); - }; - if ($@) { - dbg("pyzor: child return value thaw failed: $@"); - return; - } - - $self->_check_result($pms, $results); -} + dbg("pyzor: result: COUNT=$pyzor_count/$count_min WHITELIST=$pyzor_whitelisted/$wl_min/%.1f", + $wl_limit); -sub _check_result { - my ($self, $pms, $results) = @_; - - if (!@$results) { - dbg("pyzor: no response from server"); - return 0; - } - - my $count = 0; - my $count_wl = 0; - foreach my $res (@$results) { - chomp($res); - if ($res =~ /^Traceback/) { - info("pyzor: internal error, python traceback seen in response: $res"); + # Empty body etc results in same hash, we should skip very large numbers.. + if ($pyzor_count >= 1000000 || $pyzor_whitelisted >= 10000) { + dbg("pyzor: result exceeded hardcoded limits, ignoring: count/wl 1000000/10000"); return 0; } - dbg("pyzor: got response: $res"); - # this regexp is intended to be a little bit forgiving - if ($res =~ /^\S+\t.*?\t(\d+)\t(\d+)\s*$/) { - # until pyzor servers can sync their DBs, - # sum counts obtained from all servers - $count += untaint_var($1)+0; # crazy but needs untainting - $count_wl += untaint_var($2)+0; - } else { - # warn on failures to parse - info("pyzor: failure to parse response \"$res\""); - } - } - - my $conf = $self->{main}->{conf}; - - my $count_min = $conf->{pyzor_count_min}; - my $wl_min = $conf->{pyzor_whitelist_min}; - my $wl_limit = $count_wl >= $wl_min ? - $count * $conf->{pyzor_whitelist_factor} : 0; - - dbg("pyzor: result: COUNT=$count/$count_min WHITELIST=$count_wl/$wl_min/%.1f", - $wl_limit); - $pms->set_tag('PYZOR', "Reported $count times, whitelisted $count_wl times."); - - # Empty body etc results in same hash, we should skip very large numbers.. - if ($count >= 1000000 || $count_wl >= 10000) { - dbg("pyzor: result exceeded hardcoded limits, ignoring: count/wl 1000000/10000"); - return 0; - } - - # Whitelisted? - if ($wl_limit && $count_wl >= $wl_limit) { - dbg("pyzor: message whitelisted"); - return 0; - } + # Whitelisted? + if ($wl_limit && $pyzor_whitelisted >= $wl_limit) { + dbg("pyzor: message whitelisted"); + return 0; + } - if ($count >= $count_min) { - if ($conf->{pyzor_fork}) { - # forked needs to run got_hit() - $pms->got_hit($pms->{pyzor_rulename}, "", ruletype => 'eval'); + if ( $pyzor_count >= $count_min ) { + return 1; } - return 1; - } - return 0; + return 0; } sub plugin_report { my ($self, $options) = @_; - return if !$self->{pyzor_available}; - return if !$self->{main}->{conf}->{use_pyzor}; - return if $options->{report}->{options}->{dont_report_to_pyzor}; - return if !$self->is_pyzor_available(); - - # use temporary file: open2() is unreliable due to buffering under spamd - my $tmpf = $options->{report}->create_fulltext_tmpfile($options->{text}); - if ($self->pyzor_report($options, $tmpf)) { - $options->{report}->{report_available} = 1; - info("reporter: spam reported to Pyzor"); - $options->{report}->{report_return} = 1; - } - else { - info("reporter: could not report spam to Pyzor"); - } - $options->{report}->delete_fulltext_tmpfile($tmpf); + return unless $self->{pyzor_available}; + return unless $self->{main}->{conf}->{use_pyzor}; - return 1; + if (!$options->{report}->{options}->{dont_report_to_pyzor} && $self->is_pyzor_available()) + { + if ($self->pyzor_report($options)) { + $options->{report}->{report_available} = 1; + info("reporter: spam reported to Pyzor"); + $options->{report}->{report_return} = 1; + } + else { + info("reporter: could not report spam to Pyzor"); + } + } } sub pyzor_report { - my ($self, $options, $tmpf) = @_; - - # note: not really tainted, this came from system configuration file - my $path = untaint_file_path($options->{report}->{conf}->{pyzor_path}); - my $opts = untaint_var($options->{report}->{conf}->{pyzor_options}) || ''; + my ( $self, $options ) = @_; - my $timeout = $self->{main}->{conf}->{pyzor_timeout}; + my $timeout = $self->{main}->{conf}->{pyzor_timeout}; - $options->{report}->enter_helper_run_mode(); + my $client = ( $self->{'_pyzor_client'} ||= Mail::SpamAssassin::Pyzor::Client->new( 'timeout' => $timeout ) ); - my $timer = Mail::SpamAssassin::Timeout->new({ secs => $timeout }); - my $err = $timer->run_and_catch(sub { + my $digest = Mail::SpamAssassin::Pyzor::Digest::get( $options->{'text'} ); - local $SIG{PIPE} = sub { die "__brokenpipe__ignore__\n" }; - - dbg("pyzor: opening pipe: " . join(' ', $path, $opts, "report", "< $tmpf")); - - my $pid = Mail::SpamAssassin::Util::helper_app_pipe_open(*PYZOR, - $tmpf, 1, $path, split(' ', $opts), "report"); - $pid or die "$!\n"; - - my($inbuf,$nread,$nread_all); $nread_all = 0; - # response is ignored, just check its existence - while ( $nread=read(PYZOR,$inbuf,8192) ) { $nread_all += $nread } - defined $nread or die "error reading from pipe: $!"; - - dbg("pyzor: empty response") if $nread_all < 1; - - my $errno = 0; close PYZOR or $errno = $!; - # closing a pipe also waits for the process executing on the pipe to - # complete, no need to explicitly call waitpid - # my $child_stat = waitpid($pid,0) > 0 ? $? : undef; - if (proc_status_ok($?,$errno, 0)) { - dbg("pyzor: [%s] reporter finished successfully", $pid); - } else { - info("pyzor: [%s] reporter error: %s", $pid, exit_status_str($?,$errno)); + local $@; + my $ref = eval { $client->report($digest); }; + if ($@) { + warn("pyzor: report failed: $@"); + return 0; } - - }); - - $options->{report}->leave_helper_run_mode(); - - if ($timer->timed_out()) { - dbg("reporter: pyzor report timed out after $timeout seconds"); - return 0; - } - - if ($err) { - chomp $err; - if ($err eq '__brokenpipe__ignore__') { - dbg("reporter: pyzor report failed: broken pipe"); - } else { - warn("reporter: pyzor report failed: $err\n"); + elsif ( $ref->{'Code'} ne 200 ) { + dbg("pyzor: report failed with invalid code: $ref->{'Code'}: $ref->{'Diag'}"); + return 0; } - return 0; - } - return 1; + return 1; } -# Version features -sub has_fork { 1 } - 1; - -=back - -=cut diff --git a/lib/Mail/SpamAssassin/Pyzor.pm b/lib/Mail/SpamAssassin/Pyzor.pm new file mode 100644 index 0000000..8ac27f4 --- /dev/null +++ b/lib/Mail/SpamAssassin/Pyzor.pm @@ -0,0 +1,56 @@ +package Mail::SpamAssassin::Pyzor; + +# Copyright 2018 cPanel, LLC. +# All rights reserved. +# http://cpanel.net +# +# <@LICENSE> +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# </@LICENSE> +# + +use strict; +use warnings; + +our $VERSION = '0.06_01'; + +=encoding utf-8 + +=head1 NAME + +Mail::SpamAssassin::Pyzor - Pyzor spam filtering in Perl + +=head1 DESCRIPTION + +This distribution contains Perl implementations of parts of +L<Pyzor|http://pyzor.org>, a tool for use in spam email filtering. +It is intended for use with L<Mail::SpamAssassin> but may be useful +in other contexts. + +See the following modules for information on specific tools that +the distribution includes: + +=over + +=item * L<Mail::SpamAssassin::Pyzor::Client> + +=item * L<Mail::SpamAssassin::Pyzor::Digest> + +=back + +=cut + +1; diff --git a/lib/Mail/SpamAssassin/Pyzor/Client.pm b/lib/Mail/SpamAssassin/Pyzor/Client.pm new file mode 100644 index 0000000..ccff868 --- /dev/null +++ b/lib/Mail/SpamAssassin/Pyzor/Client.pm @@ -0,0 +1,415 @@ +package Mail::SpamAssassin::Pyzor::Client; + +# Copyright 2018 cPanel, LLC. +# All rights reserved. +# http://cpanel.net +# +# <@LICENSE> +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# </@LICENSE> +# + +use strict; +use warnings; + +=encoding utf-8 + +=head1 NAME + +Mail::SpamAssassin::Pyzor::Client - Pyzor client logic + +=head1 SYNOPSIS + + use Mail::SpamAssassin::Pyzor::Client (); + use Mail::SpamAssassin::Pyzor::Digest (); + + my $client = Mail::SpamAssassin::Pyzor::Client->new(); + + my $digest = Mail::SpamAssassin::Pyzor::Digest::get( $msg ); + + my $check_ref = $client->check($digest); + die $check_ref->{'Diag'} if $check_ref->{'Code'} ne '200'; + + my $report_ref = $client->report($digest); + die $report_ref->{'Diag'} if $report_ref->{'Code'} ne '200'; + +=head1 DESCRIPTION + +A bare-bones L<Pyzor|http://pyzor.org> client that currently only +implements the functionality needed for L<Mail::SpamAssassin>. + +=head1 PROTOCOL DETAILS + +The Pyzor protocol is not a published standard, and there appears to be +no meaningful public documentation. What follows is enough information, +largely gleaned through forum posts and reverse engineering, to facilitate +effective use of this module: + +Pyzor is an RPC-oriented, message-based protocol. Each message +is a simple dictionary of 7-bit ASCII keys and values. Server responses +always include at least the following: + +=over + +=item * C<Code> - Similar to HTTP status codes; anything besides C<200> +is an error. + +=item * C<Diag> - Similar to HTTP status reasons: a text description +of the status. + +=back + +(NB: There are additional standard response headers that are useful only for +the protocol itself and thus are not part of this module’s returns.) + +=head2 Reliability + +Pyzor uses UDP rather than TCP, so no message is guaranteed to reach its +destination. A transmission failure can happen in either the request or +the response; in either case, a timeout error will result. Such errors +are represented as thrown instances of L<Mail::Pyzor::X::Timeout>. + +=cut + +#---------------------------------------------------------------------- + +our $VERSION = '0.04'; + +our $DEFAULT_SERVER_HOST = 'public.pyzor.org'; +our $DEFAULT_SERVER_PORT = 24441; +our $DEFAULT_USERNAME = 'anonymous'; +our $DEFAULT_PASSWORD = ''; +our $DEFAULT_OP_SPEC = '20,3,60,3'; +our $PYZOR_PROTOCOL_VERSION = 2.1; +our $DEFAULT_TIMEOUT = 3.5; +our $READ_SIZE = 8192; + +use IO::Socket::INET (); +use Digest::SHA qw(sha1 sha1_hex); + +my @hash_order = ( 'Op', 'Op-Digest', 'Op-Spec', 'Thread', 'PV', 'User', 'Time', 'Sig' ); + +#---------------------------------------------------------------------- + +=head1 CONSTRUCTOR + +=head2 new(%OPTS) + +Create a new pyzor client. + +=over 2 + +=item Input + +%OPTS are (all optional): + +=over 3 + +=item * C<server_host> - The pyzor server host to connect to (default is +C<public.pyzor.org>) + +=item * C<server_port> - The pyzor server port to connect to (default is +24441) + +=item * C<username> - The username to present to the pyzor server (default +is C<anonymous>) + +=item * C<password> - The password to present to the pyzor server (default +is empty) + +=item * C<timeout> - The maximum time, in seconds, to wait for a response +from the pyzor server (defeault is 3.5) + +=back + +=item Output + +=over 3 + +Returns a L<Mail::SpamAssassin::Pyzor::Client> object. + +=back + +=back + +=cut + +sub new { + my ( $class, %OPTS ) = @_; + + return bless { + '_server_host' => $OPTS{'server_host'} || $DEFAULT_SERVER_HOST, + '_server_port' => $OPTS{'server_port'} || $DEFAULT_SERVER_PORT, + '_username' => $OPTS{'username'} || $DEFAULT_USERNAME, + '_password' => $OPTS{'password'} || $DEFAULT_PASSWORD, + '_op_spec' => $DEFAULT_OP_SPEC, + '_timeout' => $OPTS{'timeout'} || $DEFAULT_TIMEOUT, + }, $class; +} + +#---------------------------------------------------------------------- + +=head1 REQUEST METHODS + +=head2 report($digest) + +Report the digest of a spam message to the pyzor server. This function +will throw if a messaging failure or timeout happens. + +=over 2 + +=item Input + +=over 3 + +=item $digest C<SCALAR> + +The message digest to report, as given by +C<Mail::SpamAssassin::Pyzor::Digest::get()>. + +=back + +=item Output + +=over 3 + +=item C<HASHREF> + +Returns a hashref of the standard attributes noted above. + +=back + +=back + +=cut + +sub report { + my ( $self, $digest ) = @_; + + my $msg_ref = $self->_get_base_msg( 'report', $digest ); + + $msg_ref->{'Op-Spec'} = $self->{'_op_spec'}; + + return $self->_send_receive_msg($msg_ref); +} + +=head2 check($digest) + +Check the digest of a message to see if +the pyzor server has a report for it. This function +will throw if a messaging failure or timeout happens. + +=over 2 + +=item Input + +=over 3 + +=item $digest C<SCALAR> + +The message digest to check, as given by +C<Mail::SpamAssassin::Pyzor::Digest::get()>. + +=back + +=item Output + +=over 3 + +=item C<HASHREF> + +Returns a hashref of the standard attributes noted above +as well as the following: + +=over + +=item * C<Count> - The number of reports the server has received +for the given digest. + +=item * C<WL-Count> - The number of whitelist requests the server has received +for the given digest. + +=back + +=back + +=back + +=cut + +sub check { + my ( $self, $digest ) = @_; + + return $self->_send_receive_msg( $self->_get_base_msg( 'check', $digest ) ); +} + +# ---------------------------------------- + +sub _send_receive_msg { + my ( $self, $msg_ref ) = @_; + + my $thread_id = $msg_ref->{'Thread'} or warn 'No thread ID?'; + + $self->_sign_msg($msg_ref); + + return $self->_do_send_receive( + $self->_generate_packet_from_message($msg_ref) . "\n\n", + $thread_id, + ); +} + +sub _get_base_msg { + my ( $self, $op, $digest ) = @_; + + die "Implementor error: op is required" if !$op; + die "error: digest is required" if !$digest; + + return { + 'User' => $self->{'_username'}, + 'PV' => $PYZOR_PROTOCOL_VERSION, + 'Time' => time(), + 'Op' => $op, + 'Op-Digest' => $digest, + 'Thread' => $self->_generate_thread_id() + }; +} + +sub _do_send_receive { + my ( $self, $packet, $thread_id ) = @_; + + my $sock = $self->_get_connection_or_die(); + + $self->_send_packet( $sock, $packet ); + my $response = $self->_receive_packet( $sock, $thread_id ); + + return 0 if not defined $response; + + my $resp_hr = { map { ( split(m{: }) )[ 0, 1 ] } split( m{\n}, $response ) }; + + delete $resp_hr->{'Thread'}; + + my $response_pv = delete $resp_hr->{'PV'}; + + if ( $PYZOR_PROTOCOL_VERSION ne $response_pv ) { + warn "Unexpected protocol version ($response_pv) in Pyzor response!"; + } + + return $resp_hr; +} + +sub _receive_packet { + my ( $self, $sock, $thread_id ) = @_; + + my $timeout = $self->{'_timeout'} * 1000; + + my $end_time = time + $self->{'_timeout'}; + + $sock->blocking(0); + my $response = ''; + my $rout = ''; + my $rin = ''; + vec( $rin, fileno($sock), 1 ) = 1; + + while (1) { + my $time_left = $end_time - time; + + if ( $time_left <= 0 ) { + warn("Did not receive a response from the pyzor server $self->{'_server_host'}:$self->{'_server_port'} for $self->{'_timeout'} seconds!"); + return; + } + + my $bytes = sysread( $sock, $response, $READ_SIZE, length $response ); + if ( !defined($bytes) && !$!{'EAGAIN'} && !$!{'EWOULDBLOCK'} ) { + warn "read from socket: $!"; + } + + if ( index( $response, "\n\n" ) > -1 ) { + + # Reject the response unless its thread ID matches what we sent. + # This prevents confusion among concurrent Pyzor reqeusts. + if ( index( $response, "\nThread: $thread_id\n" ) != -1 ) { + last; + } + else { + $response = ''; + } + } + + my $found = select( $rout = $rin, undef, undef, $time_left ); + warn "select(): $!" if $found == -1; + } + + return $response; +} + +sub _send_packet { + my ( $self, $sock, $packet ) = @_; + + $sock->blocking(1); + syswrite( $sock, $packet ) or warn "write to socket: $!"; + + return; +} + +sub _get_connection_or_die { + my ($self) = @_; + + # clear the socket if the PID changes + if ( defined $self->{'_sock_pid'} && $self->{'_sock_pid'} != $$ ) { + undef $self->{'_sock_pid'}; + undef $self->{'_sock'}; + } + + $self->{'_sock_pid'} ||= $$; + $self->{'_sock'} ||= IO::Socket::INET->new( + 'PeerHost' => $self->{'_server_host'}, + 'PeerPort' => $self->{'_server_port'}, + 'Proto' => 'udp' + ) or die "Cannot connect to $self->{'_server_host'}:$self->{'_server_port'}: $@ $!"; + + return $self->{'_sock'}; +} + +sub _sign_msg { + my ( $self, $msg_ref ) = @_; + + $msg_ref->{'Sig'} = lc Digest::SHA::sha1_hex( + Digest::SHA::sha1( $self->_generate_packet_from_message($msg_ref) ) + ); + + return 1; +} + +sub _generate_packet_from_message { + my ( $self, $msg_ref ) = @_; + + return join( "\n", map { "$_: $msg_ref->{$_}" } grep { length $msg_ref->{$_} } @hash_order ); +} + +sub _generate_thread_id { + my $RAND_MAX = 2**16; + my $val = 0; + $val = int rand($RAND_MAX) while $val < 1024; + return $val; +} + +sub _get_user_pass_hash_key { + my ($self) = @_; + + return lc Digest::SHA::sha1_hex( $self->{'_username'} . ':' . $self->{'_password'} ); +} + +1; diff --git a/lib/Mail/SpamAssassin/Pyzor/Digest.pm b/lib/Mail/SpamAssassin/Pyzor/Digest.pm new file mode 100644 index 0000000..0e8a5ae --- /dev/null +++ b/lib/Mail/SpamAssassin/Pyzor/Digest.pm @@ -0,0 +1,103 @@ +package Mail::SpamAssassin::Pyzor::Digest; + +# Copyright 2018 cPanel, LLC. +# All rights reserved. +# http://cpanel.net +# +# <@LICENSE> +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# </@LICENSE> +# + +use strict; +use warnings; + +=encoding utf-8 + +=head1 NAME + +Mail::SpamAssassin::Pyzor::Digest + +=head1 SYNOPSIS + + my $digest = Mail::SpamAssassin::Pyzor::Digest::get( $mime_text ); + +=head1 DESCRIPTION + +A reimplementation of L<https://github.com/SpamExperts/pyzor/blob/master/pyzor/digest.py>. + +=cut + +#---------------------------------------------------------------------- + +use Email::MIME (); + +use Mail::SpamAssassin::Pyzor::Digest::Pieces (); +use Digest::SHA qw(sha1_hex); + +our $VERSION = '0.03'; + +#---------------------------------------------------------------------- + +=head1 FUNCTIONS + +=head2 $hex = get( $MSG ) + +This takes an email message in raw MIME text format (i.e., as saved in the +standard mbox format) and returns the message’s Pyzor digest in lower-case +hexadecimal. + +The output from this function should normally be identical to that of +the C<pyzor> script’s C<digest> command. It is suitable for use in +L<Mail::SpamAssassin::Pyzor::Client>’s request methods. + +=cut + +sub get { + my ($text) = @_; + return Digest::SHA::sha1_hex( ${ _get_predigest( $text ) } ); +} + +# NB: This is called from the test. +sub _get_predigest { ## no critic qw(RequireArgUnpacking) + my ($msg_text_sr) = @_; + + my $parsed = Email::MIME->new($$msg_text_sr); + + my @lines; + + my $payloads_ar = Mail::SpamAssassin::Pyzor::Digest::Pieces::digest_payloads($parsed); + + for my $payload (@$payloads_ar) { + my @p_lines = Mail::SpamAssassin::Pyzor::Digest::Pieces::splitlines($payload); + for my $line (@p_lines) { + Mail::SpamAssassin::Pyzor::Digest::Pieces::normalize($line); + + next if !Mail::SpamAssassin::Pyzor::Digest::Pieces::should_handle_line($line); + + # Make sure we have an octet string. + utf8::encode($line) if utf8::is_utf8($line); + + push @lines, $line; + } + } + + my $digest_sr = Mail::SpamAssassin::Pyzor::Digest::Pieces::assemble_lines( \@lines ); + + return $digest_sr; +} + +1; diff --git a/lib/Mail/SpamAssassin/Pyzor/Digest/Pieces.pm b/lib/Mail/SpamAssassin/Pyzor/Digest/Pieces.pm new file mode 100644 index 0000000..522accd --- /dev/null +++ b/lib/Mail/SpamAssassin/Pyzor/Digest/Pieces.pm @@ -0,0 +1,301 @@ +package Mail::SpamAssassin::Pyzor::Digest::Pieces; + +# Copyright 2018 cPanel, LLC. +# All rights reserved. +# http://cpanel.net +# +# <@LICENSE> +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# </@LICENSE> +# + +use strict; +use warnings; + +=encoding utf-8 + +=head1 NAME + +Mail::SpamAssassin::Pyzor::Digest::Pieces + +=head1 DESCRIPTION + +This module houses backend logic for L<Mail::SpamAssassin::Pyzor::Digest>. + +It reimplements logic found in pyzor’s F<digest.py> module +(L<https://github.com/SpamExperts/pyzor/blob/master/pyzor/digest.py>). + +=cut + +#---------------------------------------------------------------------- + +use Email::MIME::ContentType (); +use Encode (); + +our $VERSION = '0.03'; + +# each tuple is [ offset, length ] +use constant _HASH_SPEC => ( [ 20, 3 ], [ 60, 3 ] ); + +use constant { + _MIN_LINE_LENGTH => 8, + + _ATOMIC_NUM_LINES => 4, +}; + +#---------------------------------------------------------------------- + +=head1 FUNCTIONS + +=head2 $strings_ar = digest_payloads( $EMAIL_MIME ) + +This imitates the corresponding object method in F<digest.py>. +It returns a reference to an array of strings. Each string can be either +a byte string or a character string (e.g., UTF-8 decoded). + +NB: RFC 2822 stipulates that message bodies should use CRLF +line breaks, not plain LF (nor plain CR). L<Email::MIME::Encodings> +will thus convert any plain CRs in a quoted-printable message +body into CRLF. Python, though, doesn’t do this, so the output of +our implementation of C<digest_payloads()> diverges from that of the Python +original. It doesn’t ultimately make a difference since the line-ending +whitespace gets trimmed regardless, but it’s necessary to factor in when +comparing the output of our implementation with the Python output. + +=cut + +sub digest_payloads { + my ($parsed) = @_; + + my @subparts = $parsed->subparts(); + + my @payloads; + + if (@subparts) { + @payloads = map { @{ digest_payloads($_) } } $parsed->subparts(); + } + else { + my ( $main_type, $subtype, $encoding, $encode_check ) = parse_content_type( $parsed->content_type() ); + + my $payload; + + if ( $main_type eq 'text' ) { + + # Decode transfer encoding, but leave us as a byte string. + # Note that this is where Email::MIME converts plain LF to CRLF. + $payload = $parsed->body(); + + # This does the actual character decoding (i.e., “charset”). + $payload = Encode::decode( $encoding, $payload, $encode_check ); + + if ( $subtype eq 'html' ) { + require Mail::SpamAssassin::Pyzor::Digest::StripHtml; + $payload = Mail::SpamAssassin::Pyzor::Digest::StripHtml::strip($payload); + } + } + else { + + # This does no decoding, even of, e.g., quoted-printable or base64. + $payload = $parsed->body_raw(); + } + + push @payloads, $payload; + } + + return \@payloads; +} + +#---------------------------------------------------------------------- + +=head2 normalize( $STRING ) + +This imitates the corresponding object method in F<digest.py>. +It modifies C<$STRING> in-place. + +As with the original implementation, if C<$STRING> contains (decoded) +Unicode characters, those characters will be parsed accordingly. So: + + $str = "123\xc2\xa0"; # [ c2 a0 ] == \u00a0, non-breaking space + + normalize($str); + +The above will leave C<$str> alone, but this: + + utf8::decode($str); + + normalize($str); + +… will trim off the last two bytes from C<$str>. + +=cut + +sub normalize { ## no critic qw( Subroutines::RequireArgUnpacking ) + + # NULs are bad, mm-kay? + $_[0] =~ tr<\0><>d; + + # NB: Python’s \s without re.UNICODE is the same as Perl’s \s + # with the /a modifier. + # + # https://docs.python.org/2/library/re.html + # https://perldoc.perl.org/perlrecharclass.html#Backslash-sequences + + # Python: re.compile(r'\S{10,}') + $_[0] =~ s<\S{10,}><>ag; + + # Python: re.compile(r'\S+@\S+') + $_[0] =~ s<\S+ @ \S+><>agx; + + # Python: re.compile(r'[a-z]+:\S+', re.IGNORECASE) + $_[0] =~ s<[a-zA-Z]+ : \S+><>agx; + + # (from digest.py …) + # Make sure we do the whitespace last because some of the previous + # patterns rely on whitespace. + $_[0] =~ tr< \x09-\x0d><>d; + + # This is fun. digest.py’s normalize() does a non-UNICODE whitespace + # strip, then calls strip() on the string, which *will* strip Unicode + # whitespace from the ends. + $_[0] =~ s<\A\s+><>; + $_[0] =~ s<\s+\z><>; + + return; +} + +#---------------------------------------------------------------------- + +=head2 $yn = should_handle_line( $STRING ) + +This imitates the corresponding object method in F<digest.py>. +It returns a boolean. + +=cut + +sub should_handle_line { + return $_[0] && length( $_[0] ) >= _MIN_LINE_LENGTH(); +} + +#---------------------------------------------------------------------- + +=head2 $sr = assemble_lines( \@LINES ) + +This assembles a string buffer out of @LINES. The string is the buffer +of octets that will be hashed to produce the message digest. + +Each member of @LINES is expected to be an B<octet string>, not a +character string. + +=cut + +sub assemble_lines { + my ($lines_ar) = @_; + + if ( @$lines_ar <= _ATOMIC_NUM_LINES() ) { + + # cf. handle_atomic() in digest.py + return \join( q<>, @$lines_ar ); + } + + #---------------------------------------------------------------------- + # cf. handle_atomic() in digest.py + + my $str = q<>; + + for my $ofs_len ( _HASH_SPEC() ) { + my ( $offset, $length ) = @$ofs_len; + + for my $i ( 0 .. ( $length - 1 ) ) { + my $idx = int( $offset * @$lines_ar / 100 ) + $i; + + next if !defined $lines_ar->[$idx]; + + $str .= $lines_ar->[$idx]; + } + } + + return \$str; +} + +#---------------------------------------------------------------------- + +=head2 ($main, $sub, $encoding, $checkval) = parse_content_type( $CONTENT_TYPE ) + +=cut + +use constant _QUOTED_PRINTABLE_NAMES => ( + "quopri-codec", + "quopri", + "quoted-printable", + "quotedprintable", +); + +# Make Encode::decode() ignore anything that doesn’t fit the +# given encoding. +use constant _encode_check_ignore => q<>; + +sub parse_content_type { + my ($content_type) = @_; + + $Email::MIME::ContentType::STRICT_PARAMS = 0; + my $ct_parse = Email::MIME::ContentType::parse_content_type( + $content_type, + ); + + my $main = $ct_parse->{'type'} || q<>; + my $sub = $ct_parse->{'subtype'} || q<>; + + my $encoding = $ct_parse->{'attributes'}{'charset'}; + + my $checkval; + + if ($encoding) { + + # Lower-case everything, convert underscore to dash, and remove NUL. + $encoding =~ tr<A-Z_\0><a-z->d; + + # Apparently pyzor accommodates messages that put the transfer + # encoding in the Content-Type. + if ( grep { $_ eq $encoding } _QUOTED_PRINTABLE_NAMES() ) { + $checkval = Encode::FB_CROAK(); + } + } + else { + $encoding = 'ascii'; + } + + # Match Python .decode()’s 'ignore' behavior + $checkval ||= \&_encode_check_ignore; + + return ( $main, $sub, $encoding, $checkval ); +} + +#---------------------------------------------------------------------- + +=head2 @lines = splitlines( $TEXT ) + +Imitates C<str.splitlines()>. (cf. C<pydoc str>) + +Returns a plain list in list context. Returns the number of +items to be returned in scalar context. + +=cut + +sub splitlines { + return split m<\r\n?|\n>, $_[0]; +} + +1; diff --git a/lib/Mail/SpamAssassin/Pyzor/Digest/StripHtml.pm b/lib/Mail/SpamAssassin/Pyzor/Digest/StripHtml.pm new file mode 100644 index 0000000..2617b4a --- /dev/null +++ b/lib/Mail/SpamAssassin/Pyzor/Digest/StripHtml.pm @@ -0,0 +1,177 @@ +package Mail::SpamAssassin::Pyzor::Digest::StripHtml; + +# Copyright 2018 cPanel, LLC. +# All rights reserved. +# http://cpanel.net +# +# <@LICENSE> +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# </@LICENSE> +# + +use strict; +use warnings; + +=encoding utf-8 + +=head1 NAME + +Mail::SpamAssassin::Pyzor::Digest::StripHtml + +=head1 SYNOPSIS + + my $stripped = Mail::SpamAssassin::Pyzor::Digest::StripHtml::strip($html); + +=head1 DESCRIPTION + +This module attempts to duplicate pyzor’s HTML-stripping logic. + +=head1 ACCURACY + +This library cannot achieve 100%, bug-for-bug parity with pyzor +because to do so would require duplicating Python’s own HTML parsing +library. Since that library’s output has changed over time, and those +changes in turn affect pyzor, it’s literally impossible to arrive at +a single, fully-compatible reimplementation. + +That said, all known divergences between pyzor and this library involve +invalid HTML as input. + +Please open bug reports for any divergences you identify, particularly +if the input is valid HTML. + +=cut + +#---------------------------------------------------------------------- + +use HTML::Parser (); + +our $VERSION = '0.03'; + +#---------------------------------------------------------------------- + +=head1 FUNCTIONS + +=head2 $stripped = strip( $HTML ) + +Give it some HTML, and it’ll give back the stripped text. + +In B<general>, the stripping consists of removing tags as well as +C<E<lt>scriptE<gt>> and C<E<lt>styleE<gt>> elements; however, it also +removes HTML entities. + +This tries very hard to duplicate pyzor’s behavior with invalid HTML. + +=cut + +sub strip { + my ($html) = @_; + + $html =~ s<\A\s+><>; + $html =~ s<\s+\z><>; + + my $p = HTML::Parser->new( api_version => 3 ); + + my @pieces; + + my $accumulate = 1; + + $p->handler( + start => sub { + my ($tagname) = @_; + + $accumulate = 0 if $tagname eq 'script'; + $accumulate = 0 if $tagname eq 'style'; + + return; + }, + 'tagname', + ); + + $p->handler( + end => sub { + $accumulate = 1; + return; + } + ); + + $p->handler( + text => sub { + my ($copy) = @_; + + return if !$accumulate; + + # pyzor’s HTML parser discards HTML entities. On top of that, + # we need to match, as closely as possible, pyzor’s handling of + # invalid HTML entities … which is a function of Python’s + # standard HTML parsing library. This will probably never be + # fully compatible with the pyzor, but we can get it close. + + # The original is: + # + # re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') + # + # The parsing loop then “backs up” one byte if the last + # character isn’t a “;”. We use a look-ahead assertion to + # mimic that behavior. + $copy =~ s<\&\# (?:[0-9]+ | [xX][0-9a-fA-F]+) (?: ; | \z | (?=[^0-9a-fA-F]) )>< >gx; + + # The original is: + # + # re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') + # + # We again use a look-ahead assertion to mimic Python. + $copy =~ s<\& [a-zA-Z] [-.a-zA-Z0-9]* (?: ; | \z | (?=[^a-zA-Z0-9]) )>< >gx; + + # Python’s HTMLParser aborts its parsing loop when it encounters + # an invalid numeric reference. + $copy =~ s<\&\# + (?: + [^0-9xX] # anything but the expected first char + | + [0-9]+[a-fA-F] # hex within decimal + | + [xX][^0-9a-fA-F] + ) + (.*) + >< + ( -1 == index($1, ';') ) ? q<> : '&#' + >exs; + + # Python’s HTMLParser treats invalid entities as incomplete + $copy =~ s<(\&\#?)><$1 >gx; + + $copy =~ s<\A\s+><>; + $copy =~ s<\s+\z><>; + + push @pieces, \$copy if length $copy; + }, + 'text,tagname', + ); + + $p->parse($html); + $p->eof(); + + my $payload = join( q< >, map { $$_ } @pieces ); + + # Convert all sequences of whitespace OTHER THAN non-breaking spaces to + # plain spaces. + $payload =~ s<[^\S\x{a0}]+>< >g; + + return $payload; +} + +1; diff --git a/t/pyzor.t b/t/pyzor.t index 891f38d..e4ef83f 100755 --- a/t/pyzor.t +++ b/t/pyzor.t @@ -3,12 +3,9 @@ use lib '.'; use lib 't'; use SATest; sa_t_init("pyzor"); -use constant HAS_PYZOR => eval { $_ = untaint_cmd("which pyzor"); chomp; -x }; - use Test::More; plan skip_all => "Net tests disabled" unless conf_bool('run_net_tests'); -plan skip_all => "Pyzor executable not found in path" unless HAS_PYZOR; -plan tests => 8; +plan tests => 5; diag('Note: Failures may not be an SpamAssassin bug, as Pyzor tests can fail due to problems with the Pyzor servers.'); @@ -30,7 +27,7 @@ tstprefs (" sarun ("-t < data/spam/pyzor", \&patterns_run_cb); ok_all_patterns(); # Same with fork -sarun ("--cf='pyzor_fork 1' -t < data/spam/pyzor", \&patterns_run_cb); +sarun ("-t < data/spam/pyzor", \&patterns_run_cb); ok_all_patterns(); #TESTING FOR HAM @@ -44,7 +41,3 @@ ok_all_patterns(); sarun ("-D pyzor -t < data/nice/001 2>&1", \&patterns_run_cb); ok_all_patterns(); -# same with fork -sarun ("-D pyzor --cf='pyzor_fork 1' -t < data/nice/001 2>&1", \&patterns_run_cb); -ok_all_patterns(); -
signature.asc
Description: PGP signature
