Generally packets are not transmitted by carp interfaces, but we have a
couple of things at work that mean we do send packets out on them.
Firstly, we have a dhcp relay implementation we run on carp interfaces,
and we sent dhcp replies fast enough to fill up the one slot on the
transmit queue and our relay got ENOBUFS unexpectedly. Letting carp use
the default ifq len would be enough to fix that, but we also have routes
to some networks that prefer carp interfaces (for failover reasons)
which I wanted to go fast, so I copied the vlan transmit semantics over.
carp can basically push packets onto it's parent without going through
the transmit queue or tx mitigation now.
I've been running this in production for most of a month, and it's been
nice and boring. Even more boring than before cos I see less ENOBUFS
complaints in our logs.
ok?
Index: ip_carp.c
===================================================================
RCS file: /cvs/src/sys/netinet/ip_carp.c,v
retrieving revision 1.343
diff -u -p -r1.343 ip_carp.c
--- ip_carp.c 29 Apr 2020 07:04:32 -0000 1.343
+++ ip_carp.c 29 Apr 2020 09:49:57 -0000
@@ -233,6 +233,8 @@ int carp_check_dup_vhids(struct carp_sof
void carp_ifgroup_ioctl(struct ifnet *, u_long, caddr_t);
void carp_ifgattr_ioctl(struct ifnet *, u_long, caddr_t);
void carp_start(struct ifnet *);
+int carp_enqueue(struct ifnet *, struct mbuf *);
+void carp_transmit(struct carp_softc *, struct ifnet *, struct mbuf *);
void carp_setrun_all(struct carp_softc *, sa_family_t);
void carp_setrun(struct carp_vhost_entry *, sa_family_t);
void carp_set_state_all(struct carp_softc *, int);
@@ -830,8 +809,8 @@ carp_clone_create(struct if_clone *ifc,
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_ioctl = carp_ioctl;
ifp->if_start = carp_start;
+ ifp->if_enqueue = carp_enqueue;
ifp->if_xflags = IFXF_CLONED;
- IFQ_SET_MAXLEN(&ifp->if_snd, 1);
if_counters_alloc(ifp);
if_attach(ifp);
ether_ifattach(ifp);
@@ -2263,65 +2226,87 @@ void
carp_start(struct ifnet *ifp)
{
struct carp_softc *sc = ifp->if_softc;
+ struct ifnet *ifp0 = sc->sc_carpdev;
struct mbuf *m;
- for (;;) {
- IFQ_DEQUEUE(&ifp->if_snd, m);
- if (m == NULL)
- break;
+ if (ifp0 == NULL) {
+ ifq_purge(&ifp->if_snd);
+ return;
+ }
-#if NBPFILTER > 0
- if (ifp->if_bpf)
- bpf_mtap_ether(ifp->if_bpf, m, BPF_DIRECTION_OUT);
-#endif /* NBPFILTER > 0 */
+ while ((m = ifq_dequeue(&ifp->if_snd)) != NULL)
+ carp_transmit(sc, ifp0, m);
+}
- if ((ifp->if_carpdev->if_flags & (IFF_UP|IFF_RUNNING)) !=
- (IFF_UP|IFF_RUNNING)) {
- ifp->if_oerrors++;
- m_freem(m);
- continue;
- }
+void
+carp_transmit(struct carp_softc *sc, struct ifnet *ifp0, struct mbuf *m)
+{
+ struct ifnet *ifp = &sc->sc_if;
- /*
- * Do not leak the multicast address when sending
- * advertisements in 'ip' and 'ip-stealth' balacing
- * modes.
- */
- if (sc->sc_balancing == CARP_BAL_IP ||
- sc->sc_balancing == CARP_BAL_IPSTEALTH) {
- struct ether_header *eh;
- uint8_t *esrc;
-
- eh = mtod(m, struct ether_header *);
- esrc = ((struct arpcom*)ifp->if_carpdev)->ac_enaddr;
- memcpy(eh->ether_shost, esrc, sizeof(eh->ether_shost));
+#if NBPFILTER > 0
+ {
+ caddr_t if_bpf = ifp->if_bpf;
+ if (if_bpf) {
+ if (bpf_mtap_ether(if_bpf, m, BPF_DIRECTION_OUT))
+ m_freem(m);
}
+ }
+#endif /* NBPFILTER > 0 */
- if (if_enqueue(ifp->if_carpdev, m)) {
- ifp->if_oerrors++;
- continue;
- }
- ifp->if_opackets++;
+ if (!ISSET(ifp0->if_flags, IFF_RUNNING)) {
+ counters_inc(ifp->if_counters, ifc_oerrors);
+ m_freem(m);
+ return;
}
+
+ /*
+ * Do not leak the multicast address when sending
+ * advertisements in 'ip' and 'ip-stealth' balacing
+ * modes.
+ */
+ if (sc->sc_balancing == CARP_BAL_IP ||
+ sc->sc_balancing == CARP_BAL_IPSTEALTH) {
+ struct ether_header *eh = mtod(m, struct ether_header *);
+ memcpy(eh->ether_shost, sc->sc_ac.ac_enaddr,
+ sizeof(eh->ether_shost));
+ }
+
+ if (if_enqueue(ifp0, m))
+ counters_inc(ifp->if_counters, ifc_oerrors);
}
int
-carp_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
- struct rtentry *rt)
+carp_enqueue(struct ifnet *ifp, struct mbuf *m)
{
- struct carp_softc *sc = ((struct carp_softc *)ifp->if_softc);
- struct carp_vhost_entry *vhe;
- struct srp_ref sr;
- int ismaster;
+ struct carp_softc *sc = ifp->if_softc;
+ struct ifnet *ifp0 = sc->sc_carpdev;
+
+ /* no ifq_is_priq, cos hfsc on carp doesn't make sense */
/*
* If the parent of this carp(4) got destroyed while
* `m' was being processed, silently drop it.
*/
- if (sc->sc_carpdev == NULL) {
+ if (ifp0 == NULL) {
m_freem(m);
return (0);
}
+
+ counters_pkt(ifp->if_counters,
+ ifc_opackets, ifc_obytes, m->m_pkthdr.len);
+ carp_transmit(sc, ifp0, m);
+
+ return (0);
+}
+
+int
+carp_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
+ struct rtentry *rt)
+{
+ struct carp_softc *sc = ((struct carp_softc *)ifp->if_softc);
+ struct carp_vhost_entry *vhe;
+ struct srp_ref sr;
+ int ismaster;
if (sc->cur_vhe == NULL) {
vhe = SRPL_FIRST(&sr, &sc->carp_vhosts);
