Hi,
I noticed a problem with the routing table that is easy to reproduce: put
multiple IPs on the same carp interface:
# ifconfig em0
em0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:0a:e4:31:9d:6e
index 1 priority 0 llprio 3
groups: egress
media: Ethernet autoselect (100baseTX full-duplex,rxpause,txpause)
status: active
inet 10.1.254.56 netmask 0xfffc0000 broadcast 10.3.255.255
# ifconfig carp0 up carpdev em0 vhid 3
# ifconfig carp0 inet alias 10.1.255.1 netmask 0xfffc0000
# ifconfig carp0 inet alias 10.1.255.2 netmask 0xfffc0000
# ifconfig carp0 inet alias 10.1.255.3 netmask 0xfffc0000
# route -n show -inet
Routing tables
Internet:
Destination Gateway Flags Refs Use Mtu Prio Iface
...
10.0/14 10.1.254.56 UCn 1 43573 - 4 em0
10.0/14 10.1.255.1 UCn 0 0 - 19 carp0
10.0/14 10.1.255.3 UCn 0 0 - 19 carp0
10.0/14 10.1.255.2 UCn 0 0 - 19 carp0
...
Now pull the cable on em0 and see what happens:
# route -n show -inet
Routing tables
Internet:
Destination Gateway Flags Refs Use Mtu Prio Iface
...
10.0/14 10.1.255.3 UCn 0 0 - 19 carp0
10.0/14 10.1.255.2 UCn 0 0 - 19 carp0
10.0/14 10.1.254.56 Cn 1 43763 - 4 em0
10.0/14 10.1.255.1 Cn 0 0 - 19 carp0
...
Some of the routes to 10.0/14 on carp0 still have the up ('U') flag set,
but not all of them.
The reason is that rt_if_linkstate_change() calls rtable_mpath_reprio()
and that function reorders the routing table while we're iterating over
it in rt_if_track().
Returning EAGAIN in case rtable_mpath_reprio() was called fixes the
problem.
Gerhard
Index: sys/net/route.c
===================================================================
RCS file: /cvs/src/sys/net/route.c,v
retrieving revision 1.366
diff -u -p -u -p -r1.366 route.c
--- sys/net/route.c 11 Aug 2017 21:24:19 -0000 1.366
+++ sys/net/route.c 4 Sep 2017 08:25:05 -0000
@@ -1550,14 +1550,15 @@ rt_if_linkstate_change(struct rtentry *r
{
struct ifnet *ifp = arg;
struct sockaddr_in6 sa_mask;
+ int ret = 0;
if (rt->rt_ifidx != ifp->if_index)
- return (0);
+ return (ret);
/* Local routes are always usable. */
if (rt->rt_flags & RTF_LOCAL) {
rt->rt_flags |= RTF_UP;
- return (0);
+ return (ret);
}
if (LINK_STATE_IS_UP(ifp->if_link_state) && ifp->if_flags & IFF_UP) {
@@ -1567,6 +1568,7 @@ rt_if_linkstate_change(struct rtentry *r
rtable_mpath_reprio(id, rt_key(rt),
rt_plen2mask(rt, &sa_mask),
rt->rt_priority & RTP_MASK, rt);
+ ret = EAGAIN;
}
} else {
if (rt->rt_flags & RTF_UP) {
@@ -1588,11 +1590,12 @@ rt_if_linkstate_change(struct rtentry *r
rtable_mpath_reprio(id, rt_key(rt),
rt_plen2mask(rt, &sa_mask),
rt->rt_priority | RTP_DOWN, rt);
+ ret = EAGAIN;
}
}
if_group_routechange(rt_key(rt), rt_plen2mask(rt, &sa_mask));
- return (0);
+ return (ret);
}
struct sockaddr *
[H[2J[H[H[2J[HScript started on Mon Sep 4 10:18:22 2017
bash: warning: setlocale: LC_TIME: cannot change locale (en_US.UTF-8)
bash: warning: setlocale: LC_NUMERIC: cannot change locale (en_US.UTF-8)
]0;gerhard@null: ~gerhard@null> ifconfig em0
em0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:0a:e4:31:9d:6e
index 1 priority 0 llprio 3
groups: egress
media: Ethernet autoselect (100baseTX full-duplex,rxpause,txpause)
status: active
inet 10.1.254.56 netmask 0xfffc0000 broadcast 10.3.255.255
]0;gerhard@null: ~gerhard@null> ifconfig carp0 up carpdev em0 vhid 3
ifconfig: SIOCGIFFLAGS: Device not configured
]0;gerhard@null: ~gerhard@null> ifconfig carp0 up carpdev em0 vhid
3[C[C[C[C[C[C[C[C[C[C[C[C[C[C[1@s[1@u[1@d[1@[5P^C
]0;gerhard@null: ~gerhard@null> sudo bash
bash-4.4# ifconfig carp0 up carpdev em0 vhid 3
bash-4.4# ifconfig carp0 inet alias 10.1.255.1 netmask 0xfffc0000
bash-4.4# ifconfig carp0 inet alias 10.1.255.[1P[1@2
bash-4.4# ifconfig carp0 inet alias 10.1.255.2 netmask
0xfffc0000[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[C[[1P[1@3
bash-4.4# route sho[[[[Kn [[[K-n show -inet
Routing tables
Internet:
[7mMPATH.before lines 1-23/71 30%[27m[K[K[H[2J[HDestination
Gateway Flags Refs Use Mtu Prio Iface
default 10.0.0.1 UGS 8 896 - 8 em0
224/4 127.0.0.1 URS 0 248 32768 8 lo0
10.0/14 10.1.254.56 UCn 1 43573 - 4 em0
10.0/14 10.1.255.1 UCn 0 0 - 19 carp0
10.0/14 10.1.255.3 UCn 0 0 - 19 carp0
10.0/14 10.1.255.2 UCn 0 0 - 19 carp0
10.0.0.1 88:f0:31:3f:48:46 UHLch 1 5090 - 3 em0
10.1.254.56 00:0a:e4:31:9d:6e UHLl 0 2 - 1 em0
10.1.255.1 00:00:5e:00:01:03 UHLl 0 0 - 1 carp0
10.1.255.2 00:00:5e:00:01:03 UHLl 0 0 - 1 carp0
10.1.255.3 00:00:5e:00:01:03 UHLl 0 0 - 1 carp0
10.3.255.255 10.1.254.56 UHb 0 6668 - 1 em0
10.3.255.255 10.1.255.1 UHb 0 0 - 1 carp0
10.3.255.255 10.1.255.3 UHb 0 0 - 1 carp0
10.3.255.255 10.1.255.2 UHb 0 0 - 1 carp0
127/8 127.0.0.1 UGRS 0 0 32768 8 lo0
127.0.0.1 127.0.0.1 UHhl 1 2 32768 1 lo0
== pull cable on em0
# route -n show -inet
Routing tables
Internet:
Destination Gateway Flags Refs Use Mtu Prio Iface
default 10.0.0.1 GS 6 898 - 8 em0
224/4 127.0.0.1 URS 0 290 32768 8 lo0
10.0/14 10.1.255.3 UCn 0 0 - 19 carp0
10.0/14 10.1.255.2 UCn 0 0 - 19 carp0
10.0/14 10.1.254.56 Cn 1 43763 - 4 em0
10.0/14 10.1.255.1 Cn 0 0 - 19 carp0
10.0.0.1 88:f0:31:3f:48:46 HLch 1 5110 - 3 em0
10.1.254.56 00:0a:e4:31:9d:6e UHLl 0 2 - 1 em0
10.1.255.1 00:00:5e:00:01:03 UHLl 0 0 - 1 carp0
10.1.255.2 00:00:5e:00:01:03 UHLl 0 0 - 1 carp0
10.1.255.3 00:00:5e:00:01:03 UHLl 0 0 - 1 carp0
10.3.255.255 10.1.255.3 UHb 0 0 - 1 carp0
10.3.255.255 10.1.255.2 UHb 0 0 - 1 carp0
10.3.255.255 10.1.254.56 Hb 0 6688 - 1 em0
10.3.255.255 10.1.255.1 Hb 0 0 - 1 carp0
127/8 127.0.0.1 UGRS 0 0 32768 8 lo0
127.0.0.1 127.0.0.1 UHhl 1 2 32768 1 lo0
== two routes on carp0 are still up!
