Thank you all for the suggestions regarding NAV, but NAV is not installed on the
client. As of this morning, I believe that I have it fixed by modifying the
ruleset (implying that it was pf), however, what I still lack is an
understanding of why the *new* ruleset works, while the old ruleset does not.
Additionally, I tested smtp through a vpn fw running 3.4 and it worked fine.
Also this fw is one of three identically (hardware + software) configured
machines taking advantage of the carp redundancy feature. All three xl2
interfaces from each machine are connected to a 4-port netgear hub for pfsync.
Here is everything - if more is needed, please let me know...
Here are my hostname.if files:
/etc/hostname.xl0:
inet 201.43.98.69 255.255.255.0 NONE media 100baseTX mediaopt
full-duplex
/etc/hostname.xl1:
inet 192.168.200.55 255.255.255.0 NONE media 100baseTX mediaopt
full-duplex
inet alias 192.168.202.9 255.255.255.0 NONE
/etc/hostname.xl2:
inet 192.168.254.254 255.255.255.0 NONE
/etc/hostname.pfsync0:
up syncif xl2
/etc/hostname.carp1:
up vhid 1 advskew 0 pass aa 201.43.98.87
/etc/hostname.carp2:
up vhid 2 advskew 0 pass ab 201.43.98.89
/etc/hostname.carp3
up vhid 3 advskew 0 pass ac 201.43.98.84
/etc/hostname.carp4
up vhid 4 advskew 0 pass ad 201.43.98.85
/etc/hostname.carp5
up vhid 5 advskew 0 pass ae 201.43.98.86
/etc/hostname.carp6
up vhid 6 advskew 0 pass af 201.43.98.91
/etc/hostname.carp7
up vhid 7 advskew 0 pass ag 201.43.98.130
/etc/hostname.carp8
up vhid 8 advskew 0 pass ah 201.43.98.132
/etc/hostname.carp9
up vhid 9 advskew 0 pass ai 201.43.98.134
/etc/hostname.carp10
up vhid 10 advskew 0 pass aj 201.43.98.136
/etc/hostname.carp11
up vhid 11 advskew 0 pass ak 201.43.98.138
/etc/hostname.carp12
up vhid 12 advskew 0 pass al 201.43.98.140
/etc/hostname.carp13
up vhid 13 advskew 0 pass am 201.43.98.142
/etc/hostname.carp14
up vhid 14 advskew 0 pass an 201.43.98.144
/etc/hostname.carp15
up vhid 15 advskew 0 pass ao 201.43.98.147
/etc/hostname.carp16
up vhid 16 advskew 0 pass ap 201.43.98.151
/etc/hostname.carp17
up vhid 17 advskew 0 pass aq 201.43.98.153
/etc/hostname.carp18
up vhid 18 advskew 0 pass ar 201.43.98.155
/etc/hostname.carp19
up vhid 19 advskew 0 pass as 201.43.98.83
/etc/hostname.carp20
up vhid 20 advskew 0 pass at 201.43.98.82
/etc/hostname.carp21
up vhid 21 advskew 0 pass au 201.43.98.81
/etc/hostname.carp22
up vhid 22 advskew 0 pass av 201.43.98.80
/etc/hostname.carp23
up vhid 23 advskew 0 pass aw 201.43.98.79
/etc/hostname.carp24
up vhid 24 advskew 0 pass ax 201.43.98.146
/etc/hostname.carp25
up vhid 25 advskew 0 pass ay 201.43.98.149
/etc/hostname.carp26
up vhid 26 advskew 0 pass az 201.43.98.150
/etc/hostname.carp27
up vhid 27 advskew 0 pass ba 192.168.200.1
/etc/hostname.carp28
up vhid 28 advskew 0 pass bb 201.43.98.73
/etc/hostname.carp29
up vhid 29 advskew 0 pass bc 201.43.98.74
/etc/hostname.carp30
up vhid 30 advskew 0 pass bd 201.43.98.66
..and here is the original ruleset through which large smtp transfers FAILED:
### BEGIN pf.conf ###
ext_if="xl0"
int_if="xl1"
pfsync_if="xl2"
table <reserved> const { 0.0.0.0/8, 1.0.0.0/8, 2.0.0.0/8, 5.0.0.0/8, 10.0.0.0/8,
14.0.0.0/8, 23.0.0.0/8, 27.0.0.0/8, 31.0.0.0/8, 36.0.0.0/7, 39.0.0.0/8,
41.0.0.0/8, 42.0.0.0/8, 49.0.0.0/8, 50.0.0.0/8, 58.0.0.0/7, 70.0.0.0/7,
72.0.0.0/5, 88.0.0.0/5, 96.0.0.0/3, 127.0.0.0/8, 172.16.0.0/12, 191.255.0.0/16,
192.0.2.0/24, 192.168.0.0/16, 197.0.0.0/8, 224.0.0.0/4, 240.0.0.0/4,
255.255.255.255/32 }
table <webservers> const { 192.168.202.20, 192.168.202.21, 192.168.202.100,
192.168.202.22, 192.168.202.23, 192.168.202.24, 192.168.202.25, 192.168.202.26,
192.168.202.27, 192.168.202.28, 192.168.202.29, 192.168.202.30, 192.168.202.31,
192.168.202.32, 192.168.202.33, 192.168.202.34, 192.168.202.35, 192.168.202.36,
192.168.202.37, 192.168.202.38, 192.168.202.39, 192.168.202.40, 192.168.202.41,
192.168.202.42, 192.168.202.43 }
set block-policy drop
scrub in on $ext_if all fragment reassemble
nat on $ext_if from $int_if:network to any -> $ext_if
binat on $ext_if from 192.168.200.11 to any -> 201.43.98.73
binat on $ext_if from 192.168.200.12 to any -> 201.43.98.74
# redirections for the web servers
rdr on $ext_if inet proto tcp from any to 201.43.98.84/32 port 80 ->
192.168.202.43 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.85/32 port 80 ->
192.168.202.23 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.86/32 port 80 ->
192.168.202.42 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.87/32 port 80 ->
192.168.202.24 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.89/32 port 80 ->
192.168.202.25 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.91/32 port 80 ->
192.168.202.38 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.130/32 port 80 ->
192.168.202.27 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.132/32 port 80 ->
192.168.202.31 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.134/32 port 80 ->
192.168.202.32 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.136/32 port 80 ->
192.168.202.34 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.138/32 port 80 ->
192.168.202.37 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.140/32 port 80 ->
192.168.202.26 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.142/32 port 80 ->
192.168.202.29 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.144/32 port 80 ->
192.168.202.35 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.147/32 port 80 ->
192.168.202.22 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.151/32 port 80 ->
192.168.202.33 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.153/32 port 80 ->
192.168.202.36 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.155/32 port 80 ->
192.168.202.30 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.83/32 port 80 ->
192.168.202.41 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.82/32 port 80 ->
192.168.202.40 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.79/32 port 80 ->
192.168.202.20 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.80/32 port 80 ->
192.168.202.39 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.81/32 port 80 ->
192.168.202.21 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.146/32 port 80 ->
192.168.202.100 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.146/32 port 443 ->
192.168.202.101 port 443 #
# redirections for the dns servers (includes pass and implied state addition)
rdr pass on $ext_if inet proto { tcp,udp } from any to 201.43.98.149/32 port 53
-> 192.168.200.9 port 53 #
rdr pass on $ext_if inet proto { tcp,udp } from any to 201.43.98.150/32 port 53
-> 192.168.200.10 port 53 #
# Misc. redirects
rdr pass on $ext_if inet proto tcp from 201.43.98.77/32 to 201.43.98.66/32 port
54982 -> 192.168.200.8 port 1433 #
rdr pass on $ext_if inet proto tcp from 201.43.98.77/32 to 201.43.98.66/32 port
54983 -> 192.168.200.7 port 1433 #
# DEFAULT BLOCK POLICY
block drop log all #default deny policy
#block in log quick from no-route to any
#block in log quick on $ext_if from <reserved> to any
pass in log quick on $ext_if inet proto icmp from 201.43.98.20/32 to any
icmp-type 8 code 0 keep state
# Allow mail servers to connect out
pass in log quick on $int_if inet proto tcp from {
192.168.200.11/32,192.168.200.12/32 } to any port 25 tag MAIL_OUT keep state
pass out log quick on $ext_if tagged MAIL_OUT keep state
### INCOMING ###
# All web connections
pass in log quick on $ext_if inet proto tcp from any to <webservers> port 8000
keep state
pass in log quick on $ext_if inet proto tcp from any to 192.168.202.101 port 443
keep state
# Allow mail server connections
pass in log quick on $ext_if inet proto tcp from any to { 192.168.200.11,
192.168.200.12 } port { 25 443 } tag MAIL_IN keep state
pass in log quick on $ext_if inet proto tcp from { 201.43.98.77/32,
201.43.98.75/32 } to { 192.168.200.11, 192.168.200.12 } port { 995 993 110 }
tag MAIL_IN keep state
pass out log quick on $int_if tag MAIL_IN keep state
### OUTGOING ###
# pass out non-authoritative dns lookups from the internal dns servers
pass in log quick on $int_if route-to ($ext_if 201.43.98.65/32) inet proto udp
from { 192.168.200.4/32,192.168.200.9/32,192.168.200.10/32 } to any port 53 tag
DNS keep state
pass out log quick on $ext_if tagged DNS keep state
pass in log-all quick on $int_if from 192.168.200.7 to any tag CHINOOK keep
state
pass out log-all quick on $ext_if tagged CHINOOK keep state
# allow everything to the localhost adapter
pass quick on { lo }
antispoof quick for { lo }
# allow ssh connections
pass in on $ext_if proto tcp to ($ext_if) port 2200 keep state
# allow the pfsync broadcasts across $pfsync_if
pass quick on { $pfsync_if } proto pfsync
# allow carp protocol across the $ext_if and $int_if
pass on { $ext_if $int_if } proto carp keep state
### END pf.conf ###
..and here is the new ruleset through which the smtp transfers are now
working:
### BEGIN pf.conf ###
ext_if="xl0"
int_if="xl1"
pfsync_if="xl2"
fc_monitor="201.43.98.20/32"
table <reserved> const { 0.0.0.0/8, 1.0.0.0/8, 2.0.0.0/8, 5.0.0.0/8, 10.0.0.0/8,
14.0.0.0/8, 23.0.0.0/8, 27.0.0.0/8, 31.0.0.0/8, 36.0.0.0/7, 39.0.0.0/8,
41.0.0.0/8, 42.0.0.0/8, 49.0.0.0/8, 50.0.0.0/8, 58.0.0.0/7, 70.0.0.0/7,
72.0.0.0/5, 88.0.0.0/5, 96.0.0.0/3, 127.0.0.0/8, 172.16.0.0/12, 191.255.0.0/16,
192.0.2.0/24, 192.168.0.0/16, 197.0.0.0/8, 224.0.0.0/4, 240.0.0.0/4,
255.255.255.255/32 }
table <webservers> const { 192.168.202.20, 192.168.202.21, 192.168.202.100,
192.168.202.22, 192.168.202.23, 192.168.202.24, 192.168.202.25, 192.168.202.26,
192.168.202.27, 192.168.202.28, 192.168.202.29, 192.168.202.30, 192.168.202.31,
192.168.202.32, 192.168.202.33, 192.168.202.34, 192.168.202.35, 192.168.202.36,
192.168.202.37, 192.168.202.38, 192.168.202.39, 192.168.202.40, 192.168.202.41,
192.168.202.42, 192.168.202.43 }
set block-policy drop
set loginterface $ext_if
scrub in all
###### TRANSLATION RULES ################
nat on $ext_if from $int_if:network to any -> $ext_if
binat on $ext_if from 192.168.200.11 to any -> 201.43.98.73
binat on $ext_if from 192.168.200.12 to any -> 201.43.98.74
# redirections for the web servers
rdr on $ext_if inet proto tcp from any to 201.43.98.84/32 port 80 ->
192.168.202.43 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.85/32 port 80 ->
192.168.202.23 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.86/32 port 80 ->
192.168.202.42 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.87/32 port 80 ->
192.168.202.24 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.89/32 port 80 ->
192.168.202.25 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.91/32 port 80 ->
192.168.202.38 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.130/32 port 80 ->
192.168.202.27 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.132/32 port 80 ->
192.168.202.31 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.134/32 port 80 ->
192.168.202.32 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.136/32 port 80 ->
192.168.202.34 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.138/32 port 80 ->
192.168.202.37 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.140/32 port 80 ->
192.168.202.26 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.142/32 port 80 ->
192.168.202.29 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.144/32 port 80 ->
192.168.202.35 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.147/32 port 80 ->
192.168.202.22 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.151/32 port 80 ->
192.168.202.33 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.153/32 port 80 ->
192.168.202.36 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.155/32 port 80 ->
192.168.202.30 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.83/32 port 80 ->
192.168.202.41 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.82/32 port 80 ->
192.168.202.40 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.79/32 port 80 ->
192.168.202.20 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.80/32 port 80 ->
192.168.202.39 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.81/32 port 80 ->
192.168.202.21 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.146/32 port 80 ->
192.168.202.100 port 8000 #
rdr on $ext_if inet proto tcp from any to 201.43.98.146/32 port 443 ->
192.168.202.101 port 443 #
# redirections for the dns servers (includes pass and implied state addition)
rdr pass on $ext_if inet proto { tcp,udp } from any to 201.43.98.149/32 port 53
-> 192.168.200.9 port 53 #
rdr pass on $ext_if inet proto { tcp,udp } from any to 201.43.98.150/32 port 53
-> 192.168.200.10 port 53 #
# Misc. redirects
rdr on $ext_if inet proto tcp from 201.43.98.77/32 to 201.43.98.66/32 port 34182
-> 192.168.200.8 port 1433 #
rdr on $ext_if inet proto tcp from 201.43.98.77/32 to 201.43.98.66/32 port 34183
-> 192.168.200.7 port 1433 #
###### FILTER RULES #################
block log all # default first condition
#block in log quick on $ext_if from <reserved> to any
block return-rst in quick on $ext_if proto tcp from any to any port 113
block drop in quick on $int_if inet proto igmp all
block drop in quick on $ext_if inet proto udp from any to 224.0.0.2 port 1985
block drop in quick inet proto udp from any to any port { 135, 137, 138, 139 }
# allow everything to the localhost adapter
pass quick on { lo0 }
antispoof quick for { lo0 }
# allow ssh connections
pass in quick on $ext_if proto tcp to ($ext_if) port 2200 keep state
# allow the pfsync broadcasts across $pfsync_if
pass quick on { $pfsync_if } proto pfsync
# allow carp protocol across the $ext_if and $int_if
pass quick on { $ext_if $int_if } proto carp keep state
### INCOMING ###
# FC Ping Requests
pass in log quick on $ext_if inet proto icmp from $fc_monitor to any icmp-type 8
code 0 keep state
# WEB
pass in quick on $ext_if inet proto tcp from any to <webservers> port 8000
synproxy state
pass in quick on $ext_if inet proto tcp from any to 192.168.202.101 port 443
flags S/SA modulate state
# DNS
pass in quick on $ext_if inet proto udp from any to { 192.168.200.9/32
192.168.200.10/32 } port 53 keep state
# Allow mail server connections
pass in log quick on $ext_if inet proto tcp from any to { 192.168.200.11,
192.168.200.12 } port { 25 443 } keep state
#pass in log quick on $ext_if inet proto tcp from any to { 192.168.200.11,
192.168.200.12 } port { 25 443 } keep state ( source-track rule, max-src-states
10) #might this reduce spam?
pass in quick on $ext_if inet proto tcp from { 201.43.98.77/32, 201.43.98.75/32
} to { 192.168.200.11, 192.168.200.12 } port { 995 993 110 } keep state
# DB
pass in log quick on $ext_if inet proto tcp from 201.43.98.77/32 to {
192.168.200.7/32 192.168.200.8/32 } port 1433 keep state
### OUTGOING ###
# Allow mail servers to connect out
pass in log on $int_if inet proto tcp from { 192.168.200.11/32,192.168.200.12/32
} to any port 25 keep state
# pass out non-authoritative dns lookups from the internal dns servers
pass in log on $int_if inet proto udp from { 192.168.200.4/32, 192.168.200.9/32,
192.168.200.10/32 } to any port 53 keep state
# chinook gets access to everything
pass in log on $int_if from 192.168.200.110/32 to any flags S/SA keep state
# mail server AV updates
pass in log on $int_if inet proto tcp from { 192.168.200.11/32,
192.168.200.12/32 } to 195.39.35.48/32 port 80 flags S/SA keep state
pass in on $int_if from { 192.168.200.0/24 192.168.202.0/24 } to any keep state
pass out on $int_if from any to { 192.168.200.0/24 192.168.202.0/24 } keep
state
pass out on $ext_if proto tcp all modulate state flags S/SA
pass out on $ext_if proto { udp, icmp } all keep state
### END pf.conf ###
Thank you!
-Justin
