Hello All:
We are using two pf-enabled devices in failover/standby (not load
balanced) on FreeBSD. We are using CARP and PFSync for their respective
tasks. Everything appears to be working on the CARP side and our
interface groups correctly fail together when one goes down.
Our problem is with state maintenance upon failover. It appears the
state tables are properly synced between the devices but, when we fail
to our secondary firewall, established connections through the firewalls
fail. We have replicated this behavior with port 25 and 110.
I've included our pf.conf file below. Any help would be greatly
appreciated.
Regards,
Mike
----
# $FreeBSD: src/etc/pf.conf,v 1.2.2.1 2006/04/04 20:31:20 mlaier
# $OpenBSD: pf.conf,v 1.21 2003/09/02 20:38:44 david Exp $
#
# See pf.conf(5) and /usr/share/examples/pf for syntax and examples.
# Required order: options, normalization, queueing, translation,
filtering.
# Macros and tables may be defined and used anywhere.
# Note that translation rules are first match while filter rules are
last match.
# Macros: define common values, so they can be referenced and changed
easily.
ext_if="em1" # replace with actual external interface name i.e., dc0
int_if="em2" # replace with actual internal interface name i.e., dc1
carp_ext_if="carp1" # replace with actual internal interface name
i.e., dc1
carp_int_if="carp2" # replace with actual internal interface name
i.e., dc1
pfsync_if="em3" # replace with actual internal interface name i.e., dc1
internal_net="10.211.0.0/16"
external_addr="216.211.128.166"
mail_ext_addr="216.211.128.164"
table <firewall> const { self }
table <infected_smtp> persist
table <infected_pop> persist
table <goodguys> persist file "/etc/pf.goodguys"
table <smtpd_reject_mining> persist file "/etc/pf.mining"
table <smtpd_reject_phishing> persist file "/etc/pf.phishing"
table <smtpd_reject_porn> persist file "/etc/pf.porn"
table <smtpd_reject_spam> persist file "/etc/pf.spam"
table <smtpd_reject_virus> persist file "/etc/pf.virus"
set limit { states 100000, src-nodes 100000, frags 60000 }
set optimization aggressive
set skip on lo0
set state-policy if-bound
#
#
# FIRST SET IP ADDRESSES OF OUTWARD-FACING, ROUND-ROBIN SERVICES
#
#
#Define Internal and External IP addresses of SMTP servers [NOTE
mail.adhost.com
is defined herein as mail_in01]
mail_in01_int="10.211.128.3"
mail_in01_ext="216.211.128.3"
mail_in02_int="10.211.128.4"
mail_in02_ext="216.211.128.129"
mail_in03_int="10.211.128.8"
mail_in03_ext="216.211.128.8"
mail_in04_int="10.211.128.9"
mail_in04_ext="216.211.128.131"
mail_in05_int="10.211.128.133"
mail_in05_ext="216.211.128.133"
mail_in06_int="10.211.128.134"
mail_in06_ext="216.211.128.134"
mail_in07_int="10.211.128.140"
mail_in07_ext="216.211.128.140"
mail_in08_int="10.211.128.141"
mail_in08_ext="216.211.128.141"
#Define table for SMTP Round Robin Servers
table <smtp_roundrobin> persist { \
$mail_in02_int, \
$mail_in03_int, \
$mail_in04_int, \
$mail_in05_int, \
$mail_in06_int, \
$mail_in07_int, \
$mail_in08_int \
}
#Define Internal and External IP addresses of POP/IMAP servers
pop_01_int="10.211.128.137"
pop_01_ext="216.211.128.137"
pop_02_int="10.211.128.138"
pop_02_ext="216.211.128.138"
pop_03_int="10.211.128.139"
pop_03_ext="216.211.128.139"
#Define table for POP Round Robin Servers
table <pop_roundrobin> persist { \
$mail_in01_int \
}
# $pop_01_int, \
# $pop_02_int, \
# $pop_03_int \
#Define Internal and External IP addresses of Reject servers
reject_01_int="10.211.128.200"
#Define table for REJECT Round Robin Servers
table <reject_roundrobin> persist { \
$reject_01_int \
}
#Define table for use in outward-bound NAT
table <nat_external_pool> persist { \
$mail_in02_ext, \
$mail_in03_ext, \
$mail_in04_ext, \
$mail_in05_ext, \
$mail_in06_ext, \
$mail_in07_ext, \
$mail_in08_ext, \
$pop_01_ext, \
$pop_02_ext, \
$pop_03_ext \
}
#
#
# SECOND SET IP ADDRESSES OF NON-OUTWARD-FACING, NON-ROUND-ROBIN
SERVICES
#
#
#Define Internal and External IP addresses of FILTER servers
filter_01_int="10.211.128.5"
filter_01_ext="216.211.128.5"
filter_02_int="10.211.128.130"
filter_02_ext="216.211.128.130"
filter_03_int="10.211.128.132"
filter_03_ext="216.211.128.132"
filter_04_int="10.211.128.135"
filter_04_ext="216.211.128.135"
#Defined Internal and External IP addresses of Deferred Queue server
defer_01_int="10.211.128.137"
#Defined Internal and External IP addresses of Disk Array server
disk_array_01_int="10.211.1.213"
disk_array_01_ext="216.211.128.142"
#Cleanup packets as they pass through
scrub all reassemble tcp fragment reassemble
#Setup NAT on the internal interface
nat-anchor "ftp-proxy/*"
#nat on $ext_if from $internal_net to any -> ($ext_if)
nat on $ext_if from $internal_net to any -> <nat_external_pool>
# Allow Passive FTP from behind Mailnat
rdr-anchor "ftp-proxy/*"
rdr on $int_if proto tcp from any to any port 21 -> 127.0.0.1 port 8021
#
#
# THIRD REDIRECT SMTP BASED ON MINING, SPAM, PORN, VIRUS AND PHISHING
LISTS
#
#
# Don't let known-good ip addresses (ip, client_ip_allow MySQL tables)
get redir
ected (redirect them first)
rdr on $ext_if proto tcp from <goodguys> to any port 25 ->
<smtp_roundrobin> rou
nd-robin sticky-address
# Redirect Mining entries
rdr on $ext_if proto tcp from <smtpd_reject_mining> to any port 25 ->
<reject_ro
undrobin> port 10026 round-robin
# Redirect Phishing entries
rdr on $ext_if proto tcp from <smtpd_reject_phishing> to any port 25 ->
<reject_
roundrobin> port 10027 round-robin
# Redirect Porn entries
rdr on $ext_if proto tcp from <smtpd_reject_porn> to any port 25 ->
<reject_roun
drobin> port 10028 round-robin
# Redirect Spam entries
rdr on $ext_if proto tcp from <smtpd_reject_spam> to any port 25 ->
<reject_roun
drobin> port 10029 round-robin
# Redirect Virus entries
rdr on $ext_if proto tcp from <smtpd_reject_virus> to any port 25 ->
<reject_rou
ndrobin> port 10030 round-robin
#
#
# FOURTH REDIRECT SMTP FOR ALL OTHER IPS USING ROUND-ROBIN
#
#
#Setup redirect from mail external port to clusters for anyone not
affected by t
he Above
# SMTP Servers PORT 25
rdr on $ext_if proto tcp from any to any port 25 -> <smtp_roundrobin>
round-robi
n sticky-address
#
#
# FIFTH REDIRECT POP FOR ALL IPS USING ROUND-ROBIN
#
#
# POP Servers PORT 110
rdr on $ext_if proto tcp from any to any port 110 -> <pop_roundrobin>
round-robi
n sticky-address
#
#
# SIXTH SETUP SSH CONNECTIONS TO INTERNAL SERVERS
#
#
rdr on $ext_if proto tcp from any to $mail_in01_ext port 22 ->
$mail_in01_int po
rt 22
rdr on $ext_if proto tcp from any to $mail_in02_ext port 22 ->
$mail_in02_int
rdr on $ext_if proto tcp from any to $mail_in03_ext port 22 ->
$mail_in03_int
rdr on $ext_if proto tcp from any to $mail_in04_ext port 22 ->
$mail_in04_int
rdr on $ext_if proto tcp from any to $mail_in05_ext port 22 ->
$mail_in05_int
rdr on $ext_if proto tcp from any to $mail_in06_ext port 22 ->
$mail_in06_int
rdr on $ext_if proto tcp from any to $mail_in07_ext port 22 ->
$mail_in07_int
rdr on $ext_if proto tcp from any to $mail_in08_ext port 22 ->
$mail_in08_int
#rdr on $ext_if proto tcp from any to $filter_01_ext port 22 ->
$filter_01_int
rdr on $ext_if proto tcp from any to $filter_02_ext port 22 ->
$filter_02_int
rdr on $ext_if proto tcp from any to $filter_03_ext port 22 ->
$filter_03_int
rdr on $ext_if proto tcp from any to $filter_04_ext port 22 ->
$filter_04_int
rdr on $ext_if proto tcp from any to $pop_01_ext port 22 -> $pop_01_int
rdr on $ext_if proto tcp from any to $pop_02_ext port 22 -> $pop_02_int
rdr on $ext_if proto tcp from any to $pop_03_ext port 22 -> $pop_03_int
rdr on $ext_if proto tcp from any to $disk_array_01_ext port 22 ->
$disk_array_0
1_int
#
#
# SEVENTH BLOCK ABUSERS MAKING TOO MANY CONNECTIONS TO SMTP/POP SERVERS
#
#
block in quick on $ext_if proto tcp from <infected_smtp> to any port 25
block in quick on $ext_if proto tcp from <infected_pop> to any port 110
#
#
# EIGHTH ESTABLISH ACCESS-CONTROL-LIST RULES FOR ALL OTHER TRAFFIC
#
#
pass quick on lo0 all
#pass quick on $pfsync_if proto pfsync keep state (no-sync)
pass quick on $pfsync_if proto pfsync keep state
#pass on {$carp_ext_if $carp_int_if } proto carp keep state
pass on { $ext_if $int_if } proto carp keep state
pass out log quick on $ext_if inet proto udp from any to any port {
53,123 } kee
p state
pass out log quick on $ext_if inet proto tcp from ($ext_if) to any flags
S/SA ke
ep state
pass out log quick on $ext_if inet proto { udp,icmp } from ($ext_if) to
any keep
state
pass in quick on $ext_if from 216.211.128.0/20 to any
pass in quick on $ext_if from 216.182.80.0/20 to any
pass in log quick on $ext_if proto tcp from any to any port { 22,25 }
keep state
(max-src-conn-rate 15/5, overload <infected_smtp>)
pass in log quick on $ext_if proto tcp from any to any port 110 keep
state (max-
src-conn-rate 100/60, overload <infected_pop>)
pass in log quick on $ext_if inet proto tcp from any to any port {
22,25,110 } f
lags S/SA keep state
#pass in log quick on $int_if inet proto tcp from $internal_net to
!<firewall> f
lags S/SA modulate state
pass in log quick on $int_if inet proto udp from $internal_net to
!<firewall> ke
ep state
pass in on $ext_if inet proto tcp from any to $ext_if user proxy keep
state
#
#
# NINTH ESTABLISH FTP PROXY ANCHOR
#
#
anchor "ftp-proxy/*"
