This was kind of suggested when the SURBL came out.
Do you use the SURBL code.
I don't know if anyone is interested but I've got a batch file that goes through last
month's logs (it works on log level high) and pulls out all matches for a Body URL
filter. It can help trim the deadwood.
I've attached it renamed as a .txt file.
Scott Fisher
Director of IT
Farm Progress Companies
>>> [EMAIL PROTECTED] 06/11/04 01:12PM >>>
Would it be possible for declude to do DNS lookups on the urls in the body
of the email message and then run the IP address against an ipfile or a
filter file using remoteip? This would defeat the registering of tons of
domains that alot of times point back to the same web server. It is easy to
find the netblocks that the large discount web hosting companies use so
using the remoteip 0 cidr could be used better in the weighting system. For
example:
Servpath out of San Francisco has these netblocks, alot of legit (i hate
using that term here) email marketing spam comes from these netblocks (so
much that I block them out right because my users arent allowed to use their
email for non business purposes) but for the sake of this example weight
could be added to a message if a URL in the body translated to an IP in
these ranges.
remoteip 10 cidr 64.151.64.0/19
remoteip 10 cidr 69.59.128.0/18
It seems to me that it could be pretty effective, have it run with the DNS
tests and before the filters so it could be used in testsfailed end lines
My list of URLs is getting huge and I am sure alot of them are obsolete now.
What do you think? Doable?
Rick Davidson
National Systems Manager
North American Title Group
-
---
[This E-mail was scanned for viruses by Declude Virus (http://www.declude.com)]
---
This E-mail came from the Declude.JunkMail mailing list. To
unsubscribe, just send an E-mail to [EMAIL PROTECTED], and
type "unsubscribe Declude.JunkMail". The archives can be found
at http://www.mail-archive.com.
@echo off
rem
rem Credit for portions of this code go to [EMAIL PROTECTED]
rem
rem These settings must be done (SETTINGS section below) before the script is used:
rem v_path: path to this folder
rem v_logpath: path to the logs
rem v_maxweight: filter max weight (blank or 0 if no max weight should be used)
rem and filter entry weight (defaults to 0 if blank)
rem v_skipweight: filter skip weight (blank or 0 if filter never should be skipped)
rem v_filter: name of the Declude Filter as it appears in the log
set v_maxweight=80
set v_skipweight=240
set v_path=d:\imail\declude\fpfilters
set v_logpath=d:\logs\junkmail
set v_filter=BODYURL-KEYWORDS
rem --- Check settings and change current folder (or exit if path is incorrect): ---
set v_result=ok
if "%v_maxweight%"=="" set v_maxweight=0
if "%v_skipweight%"=="" set v_skipweight=0
if not exist %v_path%\nul (set v_result=path error) & (goto :s_end)
if not exist %v_logpath%\nul (set v_result=log path error) & (goto :s_end)
cd /d %v_path%
Rem --- Get the date for the Log
for /f "tokens=*" %%a in ('date /t') do set v_time=%%a
for /f "tokens=*" %%b in ('time /t') do set v_time=%v_time% %%b
Rem --- Get the previous month
for /f "tokens=1-2 delims=/ " %%a in ('date /t') do set v_Current_month=%%b
if "%V_current_month%"=="01" set v_Previous_month=12
if "%V_current_month%"=="02" set v_Previous_month=01
if "%V_current_month%"=="03" set v_Previous_month=02
if "%V_current_month%"=="04" set v_Previous_month=03
if "%V_current_month%"=="05" set v_Previous_month=04
if "%V_current_month%"=="06" set v_Previous_month=05
if "%V_current_month%"=="07" set v_Previous_month=06
if "%V_current_month%"=="08" set v_Previous_month=07
if "%V_current_month%"=="09" set v_Previous_month=08
if "%V_current_month%"=="10" set v_Previous_month=09
if "%V_current_month%"=="11" set v_Previous_month=10
if "%V_current_month%"=="12" set v_Previous_month=11
Rem --- Extract loglines containing Triggered Contains Filter (filter name)
if exist bodyurl.loglines.txt erase bodyurl.loglines.txt
Rem
Rem Previous Month's logs in the folder code
Rem
findstr /i Triggered.CONTAINS.Filter.%v_filter% %v_logpath%\dec%v_Previous_month%*.log
> bodyurl.loglines.txt
Rem
Rem All logs in the folder code
Rem
Rem findstr /i Triggered.CONTAINS.Filter.%v_filter% %v_logpath%\dec*.log >
bodyurl.loglines.txt
Rem --- Extract domain names from filter file
if exist bodyurl.domains.txt erase bodyurl.domains.txt
for /f "tokens=9 " %%i in ('findstr /i /r /V "FILTER-BYPASS" bodyurl.loglines.txt') do
echo %%i>> bodyurl.domains.txt
rem --- Sort the domain file
if exist bodyurl.sorted.txt erase bodyurl.sorted.txt
sort bodyurl.domains.txt /o bodyurl.sorted.txt
rem --- Dedup sorted file
if exist bodyurl.dedup.txt erase bodyurl.dedup.txt
setlocal
set infile=bodyurl.sorted.txt
set outfile=bodyurl.dedup.txt
type nul > %outfile%
for /f "tokens=1* delims=:" %%a in (
'type %infile%
^| sort
^| findstr /n /v /c:"CoLoRlEsS gReEn IdEaS"'
) do call :dedup %%a "%%b"
endlocal
goto :Makefilter
:dedup
set curr_rec=%2
if [%curr_rec%]==[""] set curr_rec=$$$blankline$$$
set curr_rec="""%curr_rec%"""
set curr_rec=%curr_rec:""""=%
set curr_rec=%curr_rec:"""=%
if not defined prev_rec goto :write
if "%curr_rec%" EQU "%prev_rec%" goto :EOF
:write
if "%curr_rec%" EQU "$$$blankline$$$" (
echo.>>%outfile%
) else (
echo>>%outfile% %curr_rec%
)
set prev_rec=%curr_rec%
goto :EOF
:EOF
:makefilter
rem --- Create filter file header: ---
if exist bodyurl.filter.txt erase bodyurl.filter.txt
echo # BodyURL filter updated %v_time%> bodyurl.filter.txt
if %v_maxweight%%v_skipweight% NEQ 0 echo # This filter will be:>> bodyurl.filter.txt
if %v_maxweight% NEQ 0 echo # - stopped at first match with the additional weight
%v_maxweight% >> bodyurl.filter.txt
if %v_skipweight% NEQ 0 echo # - skipped if the weight %v_skipweight% already has been
reached>> bodyurl.filter.txt
echo.>> bodyurl.filter.txt
if %v_maxweight% NEQ 0 echo MAXWEIGHT %v_maxweight% >> bodyurl.filter.txt
if %v_skipweight% NEQ 0 echo SKIPIFWEIGHT %v_skipweight% >> bodyurl.filter.txt
if %v_maxweight%%v_skipweight% NEQ 0 echo.>> bodyurl.filter.txt
rem ---
rem ---
Rem --- Add Testsfailed END tests here to bypass the filter if needed
rem ---
rem ---
echo #>> bodyurl.filter.txt
echo # End Filter Bypasses>> bodyurl.filter.txt
echo #>> bodyurl.filter.txt
echo TESTSFAILED END CONTAINS SCSURBL>> bodyurl.filter.txt
echo TESTSFAILED END CONTAINS FILTER-BYPASS>> bodyurl.filter.txt
echo #>> bodyurl.filter.txt
Rem --- Create Filter lines
for /f "tokens=1" %%i in ('findstr /i /r /V "#" bodyurl.dedup.txt') do echo BODY
%v_maxweight% CONTAINS %%i>> bodyurl.filter.txt
rem --- Count number of entries (and exit if filter file is empty)
for /f "tokens=1 delims=:" %%m in ('findstr /v /b /i /r "# $ MAXWEIGHT SKIPIFWEIGHT
TESTSFAILED" bodyurl.filter.txt ^| findstr /n /l "."') do set v_count=%%m
if "%v_count%"=="" (set v_result=no entries) & (goto :s_end)
:s_end
Rem --- Log the run
if not exist bodyurl_log.txt (echo # BodyURL filter log created %v_time%>
bodyurl_log.txt) & (echo.>> bodyurl_log.txt)
if "%v_result%"=="ok" (echo %v_time% Update successful [%v_result%]>> bodyurl_log.txt)
else (echo %v_time% Update failed [%v_result%]>> bodyurl_log.txt)
Rem --- Rename the file
if exist bodyurl.old erase bodyurl.old
if exist bodyurl.txt rename bodyurl.txt bodyurl.old
rename bodyurl.filter.txt bodyurl.txt
Rem --- Cleanup Temp files
if exist bodyurl.loglines.txt erase bodyurl.loglines.txt
if exist bodyurl.domains.txt erase bodyurl.domains.txt
if exist bodyurl.sorted.txt erase bodyurl.sorted.txt
if exist bodyurl.dedup.txt erase bodyurl.dedup.txt
if exist bodyurl.filter.txt erase bodyurl.filter.txt
