pfsync mutex mpfloor

[Alexander Bluhm]

Hi,

Hrvoje has hit a witness issue in pfsync.

panic: acquiring blockable sleep lock with spinlock or critical
section held (kernel_lock) &kernel_lock

panic(ffffffff81f45bb7) at panic+0xbf
witness_checkorder(ffffffff8246e970,9,0) at witness_checkorder+0xb61
__mp_lock(ffffffff8246e768) at __mp_lock+0x5f
kpageflttrap(ffff800020b26dc0,17) at kpageflttrap+0x173
kerntrap(ffff800020b26dc0) at kerntrap+0x91
alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b
pfsync_q_del(fffffd875f6336c0) at pfsync_q_del+0x70
pfsync_delete_state(fffffd875f6336c0) at pfsync_delete_state+0x118

pf and pfsync are running without kernel lock, so the mutexes
must have at least mpfloor spl protection.

ok?

bluhm

Index: net/if_pfsync.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/net/if_pfsync.c,v
retrieving revision 1.302
diff -u -p -r1.302 if_pfsync.c
--- net/if_pfsync.c     7 Apr 2022 13:38:54 -0000       1.302
+++ net/if_pfsync.c     11 Apr 2022 15:51:16 -0000
@@ -315,7 +315,7 @@ pfsyncattach(int npfsync)
 {
        if_clone_attach(&pfsync_cloner);
        pfsynccounters = counters_alloc(pfsyncs_ncounters);
-       mq_init(&pfsync_mq, 4096, IPL_SOFTNET);
+       mq_init(&pfsync_mq, 4096, IPL_MPFLOOR);
 }
 
 int
@@ -333,21 +333,21 @@ pfsync_clone_create(struct if_clone *ifc
        sc = malloc(sizeof(*pfsyncif), M_DEVBUF, M_WAITOK|M_ZERO);
        for (q = 0; q < PFSYNC_S_COUNT; q++)
                TAILQ_INIT(&sc->sc_qs[q]);
-       mtx_init_flags(&sc->sc_st_mtx, IPL_SOFTNET, "st_mtx", 0);
+       mtx_init(&sc->sc_st_mtx, IPL_MPFLOOR);
 
-       pool_init(&sc->sc_pool, PFSYNC_PLSIZE, 0, IPL_SOFTNET, 0, "pfsync",
+       pool_init(&sc->sc_pool, PFSYNC_PLSIZE, 0, IPL_MPFLOOR, 0, "pfsync",
            NULL);
        TAILQ_INIT(&sc->sc_upd_req_list);
-       mtx_init(&sc->sc_upd_req_mtx, IPL_SOFTNET);
+       mtx_init(&sc->sc_upd_req_mtx, IPL_MPFLOOR);
        TAILQ_INIT(&sc->sc_deferrals);
-       mtx_init(&sc->sc_deferrals_mtx, IPL_SOFTNET);
+       mtx_init(&sc->sc_deferrals_mtx, IPL_MPFLOOR);
        timeout_set_proc(&sc->sc_deferrals_tmo, pfsync_deferrals_tmo, sc);
        task_set(&sc->sc_ltask, pfsync_syncdev_state, sc);
        task_set(&sc->sc_dtask, pfsync_ifdetach, sc);
        sc->sc_deferred = 0;
 
        TAILQ_INIT(&sc->sc_tdb_q);
-       mtx_init(&sc->sc_tdb_mtx, IPL_SOFTNET);
+       mtx_init(&sc->sc_tdb_mtx, IPL_MPFLOOR);
 
        sc->sc_len = PFSYNC_MINPKT;
        sc->sc_maxupdates = 128;