Re: [Mimedefang] Wide character in print at /usr/bin/mimedefang.pl

[vieri]

On 2015-03-12 08:38, Steffen Kaiser wrote:
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
On Wed, 11 Mar 2015, vi...@openmailbox.org wrote:
On 2015-03-10 20:20, vi...@openmailbox.org wrote:
Hi,

I noticed that a decade ago some users detected "Wide character in
print" messages in their mimedefang logs.
It was supposedly fixed by adding

export LC_ALL=C

in the mimedefang init script.

(perl unicode issue)

However, my Linux distro (Gentoo) runs the following in the 
mimedefang
init script:

LC_ALL=C
export LC_ALL

but I'm occasionally getting the "Wide character in print" messages.

I'm running v 2.75 and Perl 5.18.2.

Any ideas?

Hi again,

Just wanted to add that I modified /usr/bin/mimedefang.pl for 
debugging purposes:

/usr/bin/mimedefang.pl:

sub action_notify_sender ($) {
[...]
   if (open(FILE, ">>NOTIFICATION")) {
<------>md_syslog('err', "action_notify_sender print FILE $msg");
[...]

If I check the logs, this is what I get (basically I have a custom 
filter which calls action_notify_sender with the name of each attached 
file):

action_notify_sender print FILE ComposicioÌ..pdf

So mimedefang and perl are choking on non-ASCII characters with "Wide 
character in print" messages.

Unfortunately the consequence of all this is that some e-mail messages 
aren't processed and are kept in the queue and not sent:

Mar 11 18:19:38 mail1 mimedefang-multiplexor[31444]: A0C4A326EB1: 
Slave 1 stderr: open body: Invalid argument at 
/usr/lib/perl5/vendor_perl/5.18.2/MIME/Entity.pm line 1878.

Could you post your filter?

It looks like the mailing list doesn't accept plain text attachments so 
I'm pasting the content of my mimedefang-filter file below (some leading 
# comments stripped out to reduce size).


$AdminAddress = 'postmas...@mydomain.org';
$AdminName = "IT";

$DaemonAddress = 'm...@mydomain.org';
$DaemonName = "MDIT";

$AddWarningsInline = 1;

md_graphdefang_log_enable('mail', 1);


$GeneralWarning = "NOTE: This e-mail has been modified automatically. 
Please follow the instructions below.\n\n";


$Stupidity{"NoMultipleInlines"} = 1;

detect_and_load_perl_modules();

sub filter_bad_filename  {
    my($entity) = @_;
    my($bad_exts, $re);

    $bad_exts = 
'(ade|adp|app|asd|asf|asx|bas|bat|chm|cmd|com|cpl|crt|dll|exe|fxp|hlp|hta|hto|inf|ini|ins|isp|jse?|lib|lnk|mdb|mde|msc|msi|msp|mst|ocx|pcd|pif|prg|reg|scr|sct|sh|shb|shs|sys|url|vb|vbe|vbs|vcs|vxd|wmd|wms|wmz|wsc|wsf|wsh|\{[^\}]+\})';

    $re = '\.' . $bad_exts . '\.*$';

    return 1 if (re_match($entity, $re));

    if (re_match($entity, '\.zip$') and
        $Features{"Archive::Zip"}) {
        my $bh = $entity->bodyhandle();
        if (defined($bh)) {
            my $path = $bh->path();
            if (defined($path)) {
                return re_match_in_zip_directory($path, $re);
            }
        }
    }
    return 0;
}

sub custom_action_replace_with_warning ($;$$$) {
    my($msg, $mtype, $mext, $mname) = @_;
    return 0 if 
(!in_filter_context("custom_action_replace_with_warning"));
    $Actions{'replace_with_warning'}++;
    $Action = "replace";
    $mtype = "text/plain" unless defined($mtype);
    $mext = "txt" unless defined($mext);
    $mname = "warning" unless defined($mname);
    $ReplacementEntity = MIME::Entity->build(Type => $mtype,
                                             Encoding => "-suggest",
                                             Data => [ "$msg\n" ]);
    $WarningCounter++;
    $ReplacementEntity->head->mime_attr("Content-Type.name" => 
"$mname$WarningCounter.$mext");
    $ReplacementEntity->head->mime_attr("Content-Disposition" => 
"inline");
    $ReplacementEntity->head->mime_attr("Content-Disposition.filename" 
=> "$mname$WarningCounter.$mext");
    return 1;
}

sub custom_action_replace_with_file ($$;$$$$$) {
    my($entity, $nfpath, $nfname, $nftype, $nfencode, $nfdispo) = @_;
    return 0 if (!in_filter_context("custom_action_replace_with_file"));
    $Actions{'replace_with_file'}++;
    $Action = "replace";
    $nftype = "application/octet-stream" unless defined($nftype);
    $nfname = "" unless defined($nfname);
    $nfencode = "base64" unless defined($nfencode);
    $nfdispo = "attachment" unless defined($nfdispo);
    $ReplacementEntity = MIME::Entity->build(Type => $nftype,
                                             Encoding => $nfencode,
                                             Path => $nfpath,
                                             Filename => $nfname,
                                             Disposition => $nfdispo);
    copy_or_link($nfpath, $entity->bodyhandle->path) or return 0;
    return 1;
}

sub custom_action_replace_with_url ($$$$;$$$$$) {
    my($entity, $doc_root, $base_url, $msg, $mtype, $mext, $mname, 
$cd_data, $salt) = @_;
    my($ctx);
    my($path);
    my($fname, $ext, $name, $url);
    my $extension = "";

    return 0 unless in_filter_context("custom_action_replace_with_url");
    return 0 unless defined($entity->bodyhandle);
    $path = $entity->bodyhandle->path;
    return 0 unless defined($path);
    open(IN, "<$path") or return 0;

    $ctx = Digest::SHA1->new;
    $ctx->addfile(*IN);
    $ctx->add($salt) if defined($salt);
    close(IN);

    $fname = takeStabAtFilename($entity);
    $fname = "" unless defined($fname);
    $extension = $1 if ($fname =~ /(\.[^.]*)$/);

    $extension = "" unless ($extension =~ /^\.[A-Za-z0-9_]*$/);

    $name = $ctx->hexdigest . $extension;
    $fname = $doc_root . "/" . $name;
    $url = $base_url . "/" . $name;

    if (-r $fname) {
        # If file exists, then this is either a duplicate or someone
        # has defeated SHA1.  Just update the mtime on the file.
        my($now);
        $now = time;
        utime($now, $now, $fname);
    } else {
        copy_or_link($path, $fname) or return 0;
        # In case umask is whacked...
        chmod 0644, $fname;
    }

    if (defined($cd_data) and ($cd_data ne "")) {
        $hasTab = 0 unless ($hasTab = index($cd_data, "\t"));
        if ($hasTab <= 0) {
            if ($extension eq '.xls') {
                $cd_data = "application/vnd.ms-excel\t" . $cd_data;
                }
            elsif ($extension eq '.ppt') {
                $cd_data = "application/vnd.ms-powerpoint\t" . $cd_data;
                }
            elsif ($extension eq '.pps') {
                $cd_data = "application/vnd.ms-powerpoint\t" . $cd_data;
                }
	    md_syslog('warning', "custom_action_replace_with_url MODIFIED 
cd_data: $cd_data - extension: $extension.");
            }
        if (open CDF, ">$doc_root/.$name") {
            print CDF $cd_data;
            close CDF;
            chmod 0644, "$doc_root/.$name";
        }
    }

    push(@custom_attachURLlist, $cd_data."\n".$url."\n");
    push(@custom_attachURLlist4html, $cd_data."\n<a 
href=\"".$url."\">".$url."</a>\n");

    $msg =~ s/_URL_/$url/g;
    if (defined($mtype) and ($mtype ne "") and defined($mext) and ($mext 
ne "") and defined($mname) and ($mname ne "")) {
        custom_action_replace_with_warning($msg, $mtype, $mext, $mname);
        }
    else {
        custom_action_replace_with_warning($msg);
        }

    return 1;
}

sub custom_list_replacement_urls ($;$$) {
    my($entity, $header, $footer) = @_;
    my $plain = $header.join("\n", @custom_attachURLlist).$footer;
    my $html = $header.join("\n", @custom_attachURLlist4html).$footer;
    $html =~ s/\n/<br>\n/g;
    append_text_boilerplate($entity, $plain, 0);
    append_html_boilerplate($entity, $html, 0);
}

sub filter_begin {
    my($entity) = @_;

    @custom_attachURLlist = ();
    @custom_attachURLlist4html = ();

    if ($SuspiciousCharsInHeaders) {
        md_graphdefang_log('suspicious_chars');
	# action_quarantine_entire_message("Message quarantined because of 
suspicious characters in headers");
        # Do NOT allow message to reach recipient(s)
        return action_discard();
    }

    md_copy_orig_msg_to_work_dir_as_mbox_file();

    my($code, $category, $action) = message_contains_virus();

    $FoundVirus = ($category eq "virus");


    md_graphdefang_log('virus check', $category, $action);

    if ($FoundVirus) {
        md_graphdefang_log('virus', $VirusName, $RelayAddr);
        md_syslog('warning', "Discarding because of virus $VirusName");
        return action_discard();
    }

    my $custom_blockcontent = 0;
    if ((open(MESSAGEHEADERS,"./HEADERS")) && 
(open(MESSAGEBODY,"./INPUTMSG"))) {

        # Let's try to get unwanted HEADER content from text file:
	if (open(UNWANTED, 
"</etc/mail/custom/custom_blacklist_content_headers")) {
        while(<UNWANTED>)
            {
            seek MESSAGEHEADERS, 0, 0;
            my $BANLINE = $_;
            chomp($BANLINE);
            # $BANLINE =~ tr/\015//d;
            # $BANLINE =~ s/\r\n//g;
            # $BANLINE =~ s/\n//g;
            $BANLINE =~ s/\r//g;

            if ((length($BANLINE) > 0) && (grep{/$BANLINE/} <MESSAGEHEADERS>)){
                $custom_blockcontent = 1;
		md_graphdefang_log('mail', "CUSTOM_BAN: Discarding because of SPAM 
marker [$BANLINE] in headers", 'ban_content');
                last;
                }
            }

        close(UNWANTED);
        } else { # if open UNWANTED
	    md_graphdefang_log('mail', 
"CUSTOM_BAN_ERROR_NO_UNWANTED_HEADERS_DATA", 'ban_content');
        }

        # Let's try to get unwanted BODY content from text file:
	if (($custom_blockcontent == 0) && (open(UNWANTED, 
"</etc/mail/custom/custom_blacklist_content_body"))) {
        while(<UNWANTED>)
            {
            seek MESSAGEBODY, 0, 0;
            my $BANLINE = $_;
            chomp($BANLINE);
            $BANLINE =~ s/\r//g;

            if ((length($BANLINE) > 0) && (grep{/$BANLINE/} <MESSAGEBODY>)){
                $custom_blockcontent = 1;
		md_graphdefang_log('mail', "CUSTOM_BAN: Discarding because of SPAM 
marker [$BANLINE] in body", 'ban_content');
                last;
                }
            }

        close(UNWANTED);
        } else { # if open UNWANTED
	    md_graphdefang_log('mail', 
"CUSTOM_BAN_ERROR_NO_UNWANTED_BODY_DATA_OR_ALREADY_BANNED", 
'ban_content');
        }

        # Let's try to get suspicious HEADER and BODY content from text file:
	if (($custom_blockcontent == 0) && (open(UNWANTED, 
"</etc/mail/custom/custom_alertlist_content"))) {
        while(<UNWANTED>)
            {
            seek MESSAGEHEADERS, 0, 0;
            seek MESSAGEBODY, 0, 0;
            my $BANLINE = $_;
            chomp($BANLINE);
            $BANLINE =~ s/\r//g;

	    if ((length($BANLINE) > 0) && ((grep{/$BANLINE/} <MESSAGEHEADERS>) 
|| (grep{/$BANLINE/} <MESSAGEBODY>))){
		md_graphdefang_log('mail', "CUSTOM_ALERT: Alerting because of 
SUSPICIOUS marker [$BANLINE] in message", 'alert_content');
                last;
                }
            }

        close(UNWANTED);
        } else { # if open UNWANTED
	    md_graphdefang_log('mail', 
"CUSTOM_ALERT_ERROR_NO_UNWANTED_DATA_OR_ALREADY_BANNED", 
'alert_content');
        }

        close(MESSAGEHEADERS);
        close(MESSAGEBODY);

        } else { # if open MESSAGEHEADERS and MESSAGEBODY
	    md_graphdefang_log('mail', "CUSTOM_BAN_ERROR_NO_MESSAGE_DATA", 
'ban_content');
        }

    return action_discard() if ($custom_blockcontent);

    if ($action eq "tempfail") {
	md_syslog('warning', "Problem running virus scanner: code=$code, 
category=$category, action=$action");
    }

    action_add_header("X-FHM-Filter", "Message from $Sender was filtered 
by FHM.");
}

sub filter {
    my($entity, $fname, $ext, $type) = @_;

    return if message_rejected(); # Avoid unnecessary work

    $NotifySenderSubject = "AVISO correo";

    if (lc($type) eq "message/partial") {
        md_graphdefang_log('message/partial');
        action_bounce("MIME type message/partial not accepted here");
        return action_discard();
    }

    if ( ($Sender eq '<>') && ($fname ne '') ) {
        md_graphdefang_log('mail', 'CUSTOM_UNKNOWN_SENDER', $fname);
	return action_drop_with_warning("Attachment removed due to unknown 
sender: $fname\n");
    }

    my($custom_applyfilter) = 1;

    my($custom_applysecurefilter) = 0;

    if ( (index($Sender, '@myotherdomain.org>') == -1) && 
(index($Sender, '@mydomain.org>') == -1) ) {
        $custom_applyfilter = 0;
        return action_accept();
        }

    if ($custom_applysecurefilter eq 1) {
        if ($fname ne '') {
            #action_notify_sender("Processed file: $fname.\n\n");
    	    return 
custom_action_replace_with_url($entity,"/var/www/localhost/htdocs/securedownloads/dkduueeksdfuh3447373","https://securedownloads.myotherdomain.org/securedownloads/dkduueeksdfuh3447373","You 
can download the file from here:<br>\n<br>\n<a 
href='_URL_'>_URL_</a><br>\n<br>\nWARNING: the attached file $fname will 
be available for a period of 30 
days.<br>\n","text/html","html","attachment",$fname);
            } else {
            return custom_action_replace_with_warning("My Org\n\n");
            #return 1;
            }
    }


    open(VIPUSERS, "</etc/mail/custom/custom_whitelist_senders");
    while(<VIPUSERS>)
    {
    my($VIPLINE) = $_;
    chomp($VIPLINE);
    $VIPLINE =~ s/\r//g;
    if ( $Sender =~ /<$VIPLINE@/i ) {
        $custom_applyfilter = 0;
	md_graphdefang_log('mail', 'CUSTOM_NOFILTER_SENDER_AUTO', 
'priv_senders');
    }
    }

    if ( ($fname eq '') || ($Sender eq '<>') ) {
        $custom_applyfilter = 0;
	md_graphdefang_log('mail', 'CUSTOM_NOFILTER_UNSPEC', 
'unspecified_fname_or_sender');
        }


    my($custom_size, $custom_bandwidth);
    my($custom_size_max, $custom_bandwidth_max);
    my($custom_size_max2, $custom_bandwidth_max2);
    $custom_bandwidth = (stat("./INPUTMSG"))[7]*scalar(@Recipients);
    md_graphdefang_log('mail', 'CUSTOM_BANDWIDTH', $custom_bandwidth);
    $custom_size = (stat($entity->bodyhandle->path))[7];
    md_graphdefang_log('mail', 'CUSTOM_MSG_PART_SIZE', $custom_size);
    $custom_size_max = 4*1024*1024;
    $custom_bandwidth_max = 8*1024*1024;
    $custom_size_max2 = 40*1024*1024;
    $custom_bandwidth_max2 = 150*1024*1024;
    if ( ($custom_size > $custom_size_max2) || ($custom_bandwidth > 
$custom_bandwidth_max2) ) {
        $custom_applyfilter = 1;
        }

    if ($custom_applyfilter eq 1) {

        # If it's not a compressed file then zip it, if possible
        my($comp_exts, $re);
        $comp_exts = '(zip|gz|tgz|bz2|Z|\{[^\}]+\})';
        $re = '\.' . $comp_exts . '\.*$';
        my $zipped;

	# Dec. 2010: do NOT create ZIP files because some recipients don't have 
uncompression software...
        #if ( ($Features{"Archive::Zip"}) && (!re_match($entity, $re)) ) {
        #    md_graphdefang_log('mail', 'CUSTOM_COMPRESS', $fname);
        #    my $zip = Archive::Zip->new();
        #    my $member;
        #    $member = $zip->addFile($entity->bodyhandle->path, $fname);
        #    # compress (DEFLATED) or not (STORED)
        #    $member->desiredCompressionMethod( COMPRESSION_DEFLATED );
	#    $member->desiredCompressionLevel( 
COMPRESSION_LEVEL_BEST_COMPRESSION );
        #    $member->setLastModFileDateTimeFromUnix( 318211200 );
        #    $fname = "$fname.zip" unless $fname =~ s/\.[^.]*$/\.zip/;
        #    $zip->writeToFileNamed("./Work/CUSTOM_$fname");
	#    custom_action_replace_with_file($entity, "./Work/CUSTOM_$fname", 
$fname);
        #    $zipped = 1;
        #}

	if ( ($custom_size > $custom_size_max) || ($custom_bandwidth > 
$custom_bandwidth_max) ) {
    	    action_notify_sender("The file $fname has been sent to the 
recipient as a web link.\n");
    	    return 
custom_action_replace_with_url($entity,"/var/www/localhost/htdocs/downloads/sdsfd67sf6dsfdsfd78","http://downloads.myotherdomain.org/downloads/sdsfd67sf6dsfdsfd78",";<i>You 
can download the attached file from here:</i><br>\n<br>\n<a 
href='_URL_'>_URL_</a><br>\n<br>\nNOTE: the file ($fname - $custom_size 
bytes) will be available for a period of 30 
days.\n","text/html","html","attachment",$fname);
        }

        if ($zipped) {
            return 1;
        }

    }

    return action_accept();
}

sub filter_multipart {
    my($entity, $fname, $ext, $type) = @_;

    return if message_rejected(); # Avoid unnecessary work

    md_graphdefang_log('CUSTOM_MULTIPART_DATA', $fname, $type);

    if (filter_bad_filename($entity)) {
        md_graphdefang_log('bad_filename', $fname, $type);
	action_notify_administrator("A MULTIPART attachment of type $type, 
named $fname was dropped.\n");
	return action_drop_with_warning("An attachment of type $type, named 
$fname was removed from this document as it\nconstituted a security 
hazard.  If you require this document, please contact\nthe sender and 
arrange an alternate means of receiving it.\n");
    }

    if (lc($type) eq "message/partial") {
        md_graphdefang_log('message/partial');
        action_bounce("MIME type message/partial not accepted here");
        return;
    }

    return action_accept();
}


sub defang_warning {
    my($oldfname, $fname) = @_;
    return
        "The attached file '$oldfname' was converted to '$fname'.\n" .
        "To restore it, right-click and Save As\n" .
        "'$oldfname'\n";
}

sub filter_end {
    my($entity) = @_;



    return if message_rejected();

    if ($Features{"SpamAssassin"}) {
        if (-s "./INPUTMSG" < 100*1024) {
            # Only scan messages smaller than 100kB.  Larger messages
            # are extremely unlikely to be spam, and SpamAssassin is
            # dreadfully slow on very large messages.
            my($hits, $req, $names, $report) = spam_assassin_check();
            my($score);
            if ($hits < 40) {
                $score = "*" x int($hits);
            } else {
                $score = "*" x 40;
            }
    	    md_graphdefang_log('CUSTOM_LOG_INFO_SPAM_HITS_SCORE', $hits, 
$score);
            # We add a header which looks like this:
            # X-Spam-Score: 6.8 (******) NAME_OF_TEST,NAME_OF_TEST
            # The number of asterisks in parens is the integer part
            # of the spam score clamped to a maximum of 40.
            # MUA filters can easily be written to trigger on a
            # minimum number of asterisks...
            if ($hits >= $req) {
                action_change_header("X-Spam-Score", "$hits ($score) $names");
                md_graphdefang_log('spam', $hits, $RelayAddr);

                action_change_header('Subject', "**MYORG-SPAM** $Subject");
                # If you find the SA report useful, add it, I guess...
                #action_add_part($entity, "text/plain", "-suggest",
                #                "$report\n",
                #               "SpamReport.txt", "inline");
            } else {
                # Delete any existing X-Spam-Score header?
                action_delete_header("X-Spam-Score");
            }
        }
    }



    md_graphdefang_log('mail_in');

    if (@custom_attachURLlist > 0) {
	&custom_list_replacement_urls($entity,"\n\n============================\n\nPlease 
click on each link to download.\n\n","\nData available during a period 
of 30 days.\n\n------- ------\n=== END MESSAGE ===\n")
    }

    if ($Sender eq '<>') {
	append_text_boilerplate($entity, "\n------\nPlease do not send the 
message again. Message ID $MessageID.", 0);
        my($custom_msgid) = $MessageID;
        $custom_msgid =~ s/\@/-/g;
        my($custom_okchars) = '-a-zA-Z0-9_.';
        $custom_msgid =~ s/[^$custom_okchars]//g;
        my($custom_recip) = $Recipients[0];
        $custom_recip =~ s/\@/-/g;
        $custom_recip =~ s/[^$custom_okchars]//g;
        $custom_recip = substr($custom_recip, 0, 30);
	my($custom_bkfname) = 
"/SAMBA/Trouble_email/msg_".$custom_msgid."_".$custom_recip."_".time()."_".int(rand(100)).".eml";
        copy_or_link("./INPUTMSG", $custom_bkfname);
        chmod 0666, $custom_bkfname;
    }



}

1;

_______________________________________________
NOTE: If there is a disclaimer or other legal boilerplate in the above
message, it is NULL AND VOID.  You may ignore it.

Visit http://www.mimedefang.org and http://www.roaringpenguin.com
MIMEDefang mailing list MIMEDefang@lists.roaringpenguin.com
http://lists.roaringpenguin.com/mailman/listinfo/mimedefang