This causes a panic when a NIC driver module is unloaded from a kernel
without VIMAGE.  if_vnet is NULL and if_detach_internal tries to
dereference it.

Regards,
Navdeep

On 06/21/2016 06:48, Bjoern A. Zeeb wrote:
> Author: bz
> Date: Tue Jun 21 13:48:49 2016
> New Revision: 302054
> URL: https://svnweb.freebsd.org/changeset/base/302054
> 
> Log:
>   Get closer to a VIMAGE network stack teardown from top to bottom rather
>   than removing the network interfaces first. This change is rather larger
>   and convoluted as the ordering requirements cannot be separated.
>   
>   Move the pfil(9) framework to SI_SUB_PROTO_PFIL, move Firewalls and
>   related modules to their own SI_SUB_PROTO_FIREWALL.
>   Move initialization of "physical" interfaces to SI_SUB_DRIVERS,
>   move virtual (cloned) interfaces to SI_SUB_PSEUDO.
>   Move Multicast to SI_SUB_PROTO_MC.
>   
>   Re-work parts of multicast initialisation and teardown, not taking the
>   huge amount of memory into account if used as a module yet.
>   
>   For interface teardown we try to do as many of them as we can on
>   SI_SUB_INIT_IF, but for some this makes no sense, e.g., when tunnelling
>   over a higher layer protocol such as IP. In that case the interface
>   has to go along (or before) the higher layer protocol is shutdown.
>   
>   Kernel hhooks need to go last on teardown as they may be used at various
>   higher layers and we cannot remove them before we cleaned up the higher
>   layers.
>   
>   For interface teardown there are multiple paths:
>   (a) a cloned interface is destroyed (inside a VIMAGE or in the base system),
>   (b) any interface is moved from a virtual network stack to a different
>   network stack ("vmove"), or (c) a virtual network stack is being shut down.
>   All code paths go through if_detach_internal() where we, depending on the
>   vmove flag or the vnet state, make a decision on how much to shut down;
>   in case we are destroying a VNET the individual protocol layers will
>   cleanup their own parts thus we cannot do so again for each interface as
>   we end up with, e.g., double-frees, destroying locks twice or acquiring
>   already destroyed locks.
>   When calling into protocol cleanups we equally have to tell them
>   whether they need to detach upper layer protocols ("ulp") or not
>   (e.g., in6_ifdetach()).
>   
>   Provide or enahnce helper functions to do proper cleanup at a protocol
>   rather than at an interface level.
>   
>   Approved by:                re (hrs)
>   Obtained from:              projects/vnet
>   Reviewed by:                gnn, jhb
>   Sponsored by:               The FreeBSD Foundation
>   MFC after:          2 weeks
>   Differential Revision:      https://reviews.freebsd.org/D6747
> 
> Modified:
>   head/sys/contrib/ipfilter/netinet/mlfk_ipl.c
>   head/sys/dev/usb/net/usb_ethernet.c
>   head/sys/kern/kern_hhook.c
>   head/sys/net/if.c
>   head/sys/net/if_bridge.c
>   head/sys/net/if_disc.c
>   head/sys/net/if_edsc.c
>   head/sys/net/if_enc.c
>   head/sys/net/if_epair.c
>   head/sys/net/if_lagg.c
>   head/sys/net/if_loop.c
>   head/sys/net/if_vlan.c
>   head/sys/net/pfil.c
>   head/sys/net/route.c
>   head/sys/net/vnet.c
>   head/sys/net/vnet.h
>   head/sys/netgraph/ng_eiface.c
>   head/sys/netgraph/ng_iface.c
>   head/sys/netinet/igmp.c
>   head/sys/netinet/in.c
>   head/sys/netinet/in_var.h
>   head/sys/netinet/ip_id.c
>   head/sys/netinet/ip_input.c
>   head/sys/netinet/ip_mroute.c
>   head/sys/netinet6/in6.c
>   head/sys/netinet6/in6_ifattach.c
>   head/sys/netinet6/in6_ifattach.h
>   head/sys/netinet6/ip6_input.c
>   head/sys/netinet6/ip6_mroute.c
>   head/sys/netinet6/mld6.c
>   head/sys/netinet6/nd6.c
>   head/sys/netinet6/nd6.h
>   head/sys/netipsec/ipsec.c
>   head/sys/netipsec/xform_tcp.c
>   head/sys/netpfil/ipfw/dn_sched.h
>   head/sys/netpfil/ipfw/ip_dummynet.c
>   head/sys/netpfil/ipfw/ip_fw2.c
>   head/sys/netpfil/ipfw/ip_fw_nat.c
>   head/sys/netpfil/pf/pf_ioctl.c
> 
> Modified: head/sys/contrib/ipfilter/netinet/mlfk_ipl.c
> ==============================================================================
> --- head/sys/contrib/ipfilter/netinet/mlfk_ipl.c      Tue Jun 21 07:05:49 
> 2016        (r302053)
> +++ head/sys/contrib/ipfilter/netinet/mlfk_ipl.c      Tue Jun 21 13:48:49 
> 2016        (r302054)
> @@ -287,7 +287,7 @@ static moduledata_t ipfiltermod = {
>  };
>  
>  
> -DECLARE_MODULE(ipfilter, ipfiltermod, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY);
> +DECLARE_MODULE(ipfilter, ipfiltermod, SI_SUB_PROTO_FIREWALL, SI_ORDER_ANY);
>  #ifdef       MODULE_VERSION
>  MODULE_VERSION(ipfilter, 1);
>  #endif
> 
> Modified: head/sys/dev/usb/net/usb_ethernet.c
> ==============================================================================
> --- head/sys/dev/usb/net/usb_ethernet.c       Tue Jun 21 07:05:49 2016        
> (r302053)
> +++ head/sys/dev/usb/net/usb_ethernet.c       Tue Jun 21 13:48:49 2016        
> (r302054)
> @@ -641,5 +641,9 @@ uether_rxflush(struct usb_ether *ue)
>       }
>  }
>  
> -DECLARE_MODULE(uether, uether_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
> +/*
> + * USB net drivers are run by DRIVER_MODULE() thus SI_SUB_DRIVERS,
> + * SI_ORDER_MIDDLE.  Run uether after that.
> + */
> +DECLARE_MODULE(uether, uether_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);
>  MODULE_VERSION(uether, 1);
> 
> Modified: head/sys/kern/kern_hhook.c
> ==============================================================================
> --- head/sys/kern/kern_hhook.c        Tue Jun 21 07:05:49 2016        
> (r302053)
> +++ head/sys/kern/kern_hhook.c        Tue Jun 21 13:48:49 2016        
> (r302054)
> @@ -510,7 +510,7 @@ hhook_vnet_uninit(const void *unused __u
>  /*
>   * When a vnet is created and being initialised, init the V_hhook_vhead_list.
>   */
> -VNET_SYSINIT(hhook_vnet_init, SI_SUB_MBUF, SI_ORDER_FIRST,
> +VNET_SYSINIT(hhook_vnet_init, SI_SUB_INIT_IF, SI_ORDER_FIRST,
>      hhook_vnet_init, NULL);
>  
>  /*
> @@ -518,5 +518,5 @@ VNET_SYSINIT(hhook_vnet_init, SI_SUB_MBU
>   * points to clean up on vnet tear down, but in case the KPI is misused,
>   * provide a function to clean up and free memory for a vnet being destroyed.
>   */
> -VNET_SYSUNINIT(hhook_vnet_uninit, SI_SUB_MBUF, SI_ORDER_ANY,
> +VNET_SYSUNINIT(hhook_vnet_uninit, SI_SUB_INIT_IF, SI_ORDER_FIRST,
>      hhook_vnet_uninit, NULL);
> 
> Modified: head/sys/net/if.c
> ==============================================================================
> --- head/sys/net/if.c Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/net/if.c Tue Jun 21 13:48:49 2016        (r302054)
> @@ -914,6 +914,16 @@ if_detach(struct ifnet *ifp)
>       CURVNET_RESTORE();
>  }
>  
> +/*
> + * The vmove flag, if set, indicates that we are called from a callpath
> + * that is moving an interface to a different vnet instance.
> + *
> + * The shutdown flag, if set, indicates that we are called in the
> + * process of shutting down a vnet instance.  Currently only the
> + * vnet_if_return SYSUNINIT function sets it.  Note: we can be called
> + * on a vnet instance shutdown without this flag being set, e.g., when
> + * the cloned interfaces are destoyed as first thing of teardown.
> + */
>  static int
>  if_detach_internal(struct ifnet *ifp, int vmove, struct if_clone **ifcp)
>  {
> @@ -921,8 +931,10 @@ if_detach_internal(struct ifnet *ifp, in
>       int i;
>       struct domain *dp;
>       struct ifnet *iter;
> -     int found = 0;
> +     int found = 0, shutdown;
>  
> +     shutdown = (ifp->if_vnet->vnet_state > SI_SUB_VNET &&
> +              ifp->if_vnet->vnet_state < SI_SUB_VNET_DONE) ? 1 : 0;
>       IFNET_WLOCK();
>       TAILQ_FOREACH(iter, &V_ifnet, if_link)
>               if (iter == ifp) {
> @@ -930,10 +942,6 @@ if_detach_internal(struct ifnet *ifp, in
>                       found = 1;
>                       break;
>               }
> -#ifdef VIMAGE
> -     if (found)
> -             curvnet->vnet_ifcnt--;
> -#endif
>       IFNET_WUNLOCK();
>       if (!found) {
>               /*
> @@ -951,19 +959,58 @@ if_detach_internal(struct ifnet *ifp, in
>  #endif
>       }
>  
> -     /* Check if this is a cloned interface or not. */
> +     /*
> +      * At this point we know the interface still was on the ifnet list
> +      * and we removed it so we are in a stable state.
> +      */
> +#ifdef VIMAGE
> +     curvnet->vnet_ifcnt--;
> +#endif
> +
> +     /*
> +      * In any case (destroy or vmove) detach us from the groups
> +      * and remove/wait for pending events on the taskq.
> +      * XXX-BZ in theory an interface could still enqueue a taskq change?
> +      */
> +     if_delgroups(ifp);
> +
> +     taskqueue_drain(taskqueue_swi, &ifp->if_linktask);
> +
> +     /*
> +      * Check if this is a cloned interface or not. Must do even if
> +      * shutting down as a if_vmove_reclaim() would move the ifp and
> +      * the if_clone_addgroup() will have a corrupted string overwise
> +      * from a gibberish pointer.
> +      */
>       if (vmove && ifcp != NULL)
>               *ifcp = if_clone_findifc(ifp);
>  
> +     if_down(ifp);
> +
>       /*
> -      * Remove/wait for pending events.
> +      * On VNET shutdown abort here as the stack teardown will do all
> +      * the work top-down for us.
> +      */
> +     if (shutdown) {
> +             /*
> +              * In case of a vmove we are done here without error.
> +              * If we would signal an error it would lead to the same
> +              * abort as if we did not find the ifnet anymore.
> +              * if_detach() calls us in void context and does not care
> +              * about an early abort notification, so life is splendid :)
> +              */
> +             goto finish_vnet_shutdown;
> +     }
> +
> +     /*
> +      * At this point we are not tearing down a VNET and are either
> +      * going to destroy or vmove the interface and have to cleanup
> +      * accordingly.
>        */
> -     taskqueue_drain(taskqueue_swi, &ifp->if_linktask);
>  
>       /*
>        * Remove routes and flush queues.
>        */
> -     if_down(ifp);
>  #ifdef ALTQ
>       if (ALTQ_IS_ENABLED(&ifp->if_snd))
>               altq_disable(&ifp->if_snd);
> @@ -1018,8 +1065,8 @@ if_detach_internal(struct ifnet *ifp, in
>       }
>  
>       rt_flushifroutes(ifp);
> -     if_delgroups(ifp);
>  
> +finish_vnet_shutdown:
>       /*
>        * We cannot hold the lock over dom_ifdetach calls as they might
>        * sleep, for example trying to drain a callout, thus open up the
> @@ -1048,7 +1095,7 @@ if_detach_internal(struct ifnet *ifp, in
>   * unused if_index in target vnet and calls if_grow() if necessary,
>   * and finally find an unused if_xname for the target vnet.
>   */
> -void
> +static void
>  if_vmove(struct ifnet *ifp, struct vnet *new_vnet)
>  {
>       struct if_clone *ifc;
> @@ -1115,6 +1162,7 @@ if_vmove_loan(struct thread *td, struct 
>  {
>       struct prison *pr;
>       struct ifnet *difp;
> +     int shutdown;
>  
>       /* Try to find the prison within our visibility. */
>       sx_slock(&allprison_lock);
> @@ -1135,12 +1183,22 @@ if_vmove_loan(struct thread *td, struct 
>       /* XXX Lock interfaces to avoid races. */
>       CURVNET_SET_QUIET(pr->pr_vnet);
>       difp = ifunit(ifname);
> -     CURVNET_RESTORE();
>       if (difp != NULL) {
> +             CURVNET_RESTORE();
>               prison_free(pr);
>               return (EEXIST);
>       }
>  
> +     /* Make sure the VNET is stable. */
> +     shutdown = (ifp->if_vnet->vnet_state > SI_SUB_VNET &&
> +              ifp->if_vnet->vnet_state < SI_SUB_VNET_DONE) ? 1 : 0;
> +     if (shutdown) {
> +             CURVNET_RESTORE();
> +             prison_free(pr);
> +             return (EBUSY);
> +     }
> +     CURVNET_RESTORE();
> +
>       /* Move the interface into the child jail/vnet. */
>       if_vmove(ifp, pr->pr_vnet);
>  
> @@ -1157,6 +1215,7 @@ if_vmove_reclaim(struct thread *td, char
>       struct prison *pr;
>       struct vnet *vnet_dst;
>       struct ifnet *ifp;
> +     int shutdown;
>  
>       /* Try to find the prison within our visibility. */
>       sx_slock(&allprison_lock);
> @@ -1184,6 +1243,15 @@ if_vmove_reclaim(struct thread *td, char
>               return (EEXIST);
>       }
>  
> +     /* Make sure the VNET is stable. */
> +     shutdown = (ifp->if_vnet->vnet_state > SI_SUB_VNET &&
> +              ifp->if_vnet->vnet_state < SI_SUB_VNET_DONE) ? 1 : 0;
> +     if (shutdown) {
> +             CURVNET_RESTORE();
> +             prison_free(pr);
> +             return (EBUSY);
> +     }
> +
>       /* Get interface back from child jail/vnet. */
>       if_vmove(ifp, vnet_dst);
>       CURVNET_RESTORE();
> @@ -2642,8 +2710,22 @@ ifioctl(struct socket *so, u_long cmd, c
>       struct ifreq *ifr;
>       int error;
>       int oif_flags;
> +#ifdef VIMAGE
> +     int shutdown;
> +#endif
>  
>       CURVNET_SET(so->so_vnet);
> +#ifdef VIMAGE
> +     /* Make sure the VNET is stable. */
> +     shutdown = (so->so_vnet->vnet_state > SI_SUB_VNET &&
> +              so->so_vnet->vnet_state < SI_SUB_VNET_DONE) ? 1 : 0;
> +     if (shutdown) {
> +             CURVNET_RESTORE();
> +             return (EBUSY);
> +     }
> +#endif
> +
> +
>       switch (cmd) {
>       case SIOCGIFCONF:
>               error = ifconf(cmd, data);
> 
> Modified: head/sys/net/if_bridge.c
> ==============================================================================
> --- head/sys/net/if_bridge.c  Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/net/if_bridge.c  Tue Jun 21 13:48:49 2016        (r302054)
> @@ -541,7 +541,7 @@ vnet_bridge_uninit(const void *unused __
>       V_bridge_cloner = NULL;
>       BRIDGE_LIST_LOCK_DESTROY();
>  }
> -VNET_SYSUNINIT(vnet_bridge_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
> +VNET_SYSUNINIT(vnet_bridge_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY,
>      vnet_bridge_uninit, NULL);
>  
>  static int
> 
> Modified: head/sys/net/if_disc.c
> ==============================================================================
> --- head/sys/net/if_disc.c    Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/net/if_disc.c    Tue Jun 21 13:48:49 2016        (r302054)
> @@ -137,7 +137,7 @@ vnet_disc_init(const void *unused __unus
>       V_disc_cloner = if_clone_simple(discname, disc_clone_create,
>           disc_clone_destroy, 0);
>  }
> -VNET_SYSINIT(vnet_disc_init, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
> +VNET_SYSINIT(vnet_disc_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
>      vnet_disc_init, NULL);
>  
>  static void
> @@ -146,7 +146,7 @@ vnet_disc_uninit(const void *unused __un
>  
>       if_clone_detach(V_disc_cloner);
>  }
> -VNET_SYSUNINIT(vnet_disc_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
> +VNET_SYSUNINIT(vnet_disc_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY,
>      vnet_disc_uninit, NULL);
>  
>  static int
> 
> Modified: head/sys/net/if_edsc.c
> ==============================================================================
> --- head/sys/net/if_edsc.c    Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/net/if_edsc.c    Tue Jun 21 13:48:49 2016        (r302054)
> @@ -336,7 +336,7 @@ vnet_edsc_uninit(const void *unused __un
>        */
>       if_clone_detach(V_edsc_cloner);
>  }
> -VNET_SYSUNINIT(vnet_edsc_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
> +VNET_SYSUNINIT(vnet_edsc_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY,
>      vnet_edsc_uninit, NULL);
>  
>  /*
> 
> Modified: head/sys/net/if_enc.c
> ==============================================================================
> --- head/sys/net/if_enc.c     Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/net/if_enc.c     Tue Jun 21 13:48:49 2016        (r302054)
> @@ -136,7 +136,6 @@ enc_clone_destroy(struct ifnet *ifp)
>       sc = ifp->if_softc;
>       KASSERT(sc == V_enc_sc, ("sc != ifp->if_softc"));
>  
> -     enc_remove_hhooks(sc);
>       bpfdetach(ifp);
>       if_detach(ifp);
>       if_free(ifp);
> @@ -170,10 +169,6 @@ enc_clone_create(struct if_clone *ifc, i
>       ifp->if_softc = sc;
>       if_attach(ifp);
>       bpfattach(ifp, DLT_ENC, sizeof(struct enchdr));
> -     if (enc_add_hhooks(sc) != 0) {
> -             enc_clone_destroy(ifp);
> -             return (ENXIO);
> -     }
>       return (0);
>  }
>  
> @@ -369,18 +364,44 @@ vnet_enc_init(const void *unused __unuse
>       V_enc_cloner = if_clone_simple(encname, enc_clone_create,
>           enc_clone_destroy, 1);
>  }
> -VNET_SYSINIT(vnet_enc_init, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
> +VNET_SYSINIT(vnet_enc_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
>      vnet_enc_init, NULL);
>  
>  static void
> +vnet_enc_init_proto(void *unused __unused)
> +{
> +     KASSERT(V_enc_sc != NULL, ("%s: V_enc_sc is %p\n", __func__, V_enc_sc));
> +
> +     if (enc_add_hhooks(V_enc_sc) != 0)
> +             enc_clone_destroy(V_enc_sc->sc_ifp);
> +}
> +VNET_SYSINIT(vnet_enc_init_proto, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
> +    vnet_enc_init_proto, NULL);
> +
> +static void
>  vnet_enc_uninit(const void *unused __unused)
>  {
> +     KASSERT(V_enc_sc != NULL, ("%s: V_enc_sc is %p\n", __func__, V_enc_sc));
>  
>       if_clone_detach(V_enc_cloner);
>  }
> -VNET_SYSUNINIT(vnet_enc_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
> +VNET_SYSUNINIT(vnet_enc_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY,
>      vnet_enc_uninit, NULL);
>  
> +/*
> + * The hhook consumer needs to go before ip[6]_destroy are called on
> + * SI_ORDER_THIRD.
> + */
> +static void
> +vnet_enc_uninit_hhook(const void *unused __unused)
> +{
> +     KASSERT(V_enc_sc != NULL, ("%s: V_enc_sc is %p\n", __func__, V_enc_sc));
> +
> +     enc_remove_hhooks(V_enc_sc);
> +}
> +VNET_SYSUNINIT(vnet_enc_uninit_hhook, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH,
> +    vnet_enc_uninit_hhook, NULL);
> +
>  static int
>  enc_modevent(module_t mod, int type, void *data)
>  {
> @@ -401,4 +422,4 @@ static moduledata_t enc_mod = {
>       0
>  };
>  
> -DECLARE_MODULE(if_enc, enc_mod, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY);
> +DECLARE_MODULE(if_enc, enc_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
> 
> Modified: head/sys/net/if_epair.c
> ==============================================================================
> --- head/sys/net/if_epair.c   Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/net/if_epair.c   Tue Jun 21 13:48:49 2016        (r302054)
> @@ -963,7 +963,7 @@ vnet_epair_init(const void *unused __unu
>       netisr_register_vnet(&epair_nh);
>  #endif
>  }
> -VNET_SYSINIT(vnet_epair_init, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
> +VNET_SYSINIT(vnet_epair_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
>      vnet_epair_init, NULL);
>  
>  static void
> @@ -975,7 +975,7 @@ vnet_epair_uninit(const void *unused __u
>  #endif
>       if_clone_detach(V_epair_cloner);
>  }
> -VNET_SYSUNINIT(vnet_epair_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
> +VNET_SYSUNINIT(vnet_epair_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY,
>      vnet_epair_uninit, NULL);
>  
>  static int
> @@ -1012,5 +1012,5 @@ static moduledata_t epair_mod = {
>       0
>  };
>  
> -DECLARE_MODULE(if_epair, epair_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
> +DECLARE_MODULE(if_epair, epair_mod, SI_SUB_PSEUDO, SI_ORDER_MIDDLE);
>  MODULE_VERSION(if_epair, 1);
> 
> Modified: head/sys/net/if_lagg.c
> ==============================================================================
> --- head/sys/net/if_lagg.c    Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/net/if_lagg.c    Tue Jun 21 13:48:49 2016        (r302054)
> @@ -271,7 +271,7 @@ vnet_lagg_uninit(const void *unused __un
>       if_clone_detach(V_lagg_cloner);
>       LAGG_LIST_LOCK_DESTROY();
>  }
> -VNET_SYSUNINIT(vnet_lagg_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
> +VNET_SYSUNINIT(vnet_lagg_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY,
>      vnet_lagg_uninit, NULL);
>  
>  static int
> 
> Modified: head/sys/net/if_loop.c
> ==============================================================================
> --- head/sys/net/if_loop.c    Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/net/if_loop.c    Tue Jun 21 13:48:49 2016        (r302054)
> @@ -156,7 +156,7 @@ vnet_loif_init(const void *unused __unus
>           1);
>  #endif
>  }
> -VNET_SYSINIT(vnet_loif_init, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
> +VNET_SYSINIT(vnet_loif_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
>      vnet_loif_init, NULL);
>  
>  #ifdef VIMAGE
> @@ -167,7 +167,7 @@ vnet_loif_uninit(const void *unused __un
>       if_clone_detach(V_lo_cloner);
>       V_loif = NULL;
>  }
> -VNET_SYSUNINIT(vnet_loif_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
> +VNET_SYSUNINIT(vnet_loif_uninit, SI_SUB_INIT_IF, SI_ORDER_SECOND,
>      vnet_loif_uninit, NULL);
>  #endif
>  
> 
> Modified: head/sys/net/if_vlan.c
> ==============================================================================
> --- head/sys/net/if_vlan.c    Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/net/if_vlan.c    Tue Jun 21 13:48:49 2016        (r302054)
> @@ -823,7 +823,7 @@ vnet_vlan_uninit(const void *unused __un
>  
>       if_clone_detach(V_vlan_cloner);
>  }
> -VNET_SYSUNINIT(vnet_vlan_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_FIRST,
> +VNET_SYSUNINIT(vnet_vlan_uninit, SI_SUB_INIT_IF, SI_ORDER_FIRST,
>      vnet_vlan_uninit, NULL);
>  #endif
>  
> 
> Modified: head/sys/net/pfil.c
> ==============================================================================
> --- head/sys/net/pfil.c       Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/net/pfil.c       Tue Jun 21 13:48:49 2016        (r302054)
> @@ -383,17 +383,14 @@ vnet_pfil_uninit(const void *unused __un
>       PFIL_LOCK_DESTROY_REAL(&V_pfil_lock);
>  }
>  
> -/* Define startup order. */
> -#define      PFIL_SYSINIT_ORDER      SI_SUB_PROTO_BEGIN
> -#define      PFIL_MODEVENT_ORDER     (SI_ORDER_FIRST) /* On boot slot in 
> here. */
> -#define      PFIL_VNET_ORDER         (PFIL_MODEVENT_ORDER + 2) /* Later 
> still. */
> -
>  /*
>   * Starting up.
>   *
>   * VNET_SYSINIT is called for each existing vnet and each new vnet.
> + * Make sure the pfil bits are first before any possible subsystem which
> + * might piggyback on the SI_SUB_PROTO_PFIL.
>   */
> -VNET_SYSINIT(vnet_pfil_init, PFIL_SYSINIT_ORDER, PFIL_VNET_ORDER,
> +VNET_SYSINIT(vnet_pfil_init, SI_SUB_PROTO_PFIL, SI_ORDER_FIRST,
>      vnet_pfil_init, NULL);
>   
>  /*
> @@ -401,5 +398,5 @@ VNET_SYSINIT(vnet_pfil_init, PFIL_SYSINI
>   *
>   * VNET_SYSUNINIT is called for each exiting vnet as it exits.
>   */
> -VNET_SYSUNINIT(vnet_pfil_uninit, PFIL_SYSINIT_ORDER, PFIL_VNET_ORDER,
> +VNET_SYSUNINIT(vnet_pfil_uninit, SI_SUB_PROTO_PFIL, SI_ORDER_FIRST,
>      vnet_pfil_uninit, NULL);
> 
> Modified: head/sys/net/route.c
> ==============================================================================
> --- head/sys/net/route.c      Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/net/route.c      Tue Jun 21 13:48:49 2016        (r302054)
> @@ -334,7 +334,7 @@ vnet_route_uninit(const void *unused __u
>       free(V_rt_tables, M_RTABLE);
>       uma_zdestroy(V_rtzone);
>  }
> -VNET_SYSUNINIT(vnet_route_uninit, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD,
> +VNET_SYSUNINIT(vnet_route_uninit, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST,
>      vnet_route_uninit, 0);
>  #endif
>  
> 
> Modified: head/sys/net/vnet.c
> ==============================================================================
> --- head/sys/net/vnet.c       Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/net/vnet.c       Tue Jun 21 13:48:49 2016        (r302054)
> @@ -331,8 +331,7 @@ vnet_init_done(void *unused __unused)
>  
>       curvnet = NULL;
>  }
> -
> -SYSINIT(vnet_init_done, SI_SUB_VNET_DONE, SI_ORDER_FIRST, vnet_init_done,
> +SYSINIT(vnet_init_done, SI_SUB_VNET_DONE, SI_ORDER_ANY, vnet_init_done,
>      NULL);
>  
>  /*
> 
> Modified: head/sys/net/vnet.h
> ==============================================================================
> --- head/sys/net/vnet.h       Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/net/vnet.h       Tue Jun 21 13:48:49 2016        (r302054)
> @@ -111,8 +111,8 @@ vnet_##name##_init(const void *unused)    \
>  {                                    \
>       VNET_PCPUSTAT_ALLOC(name, M_WAITOK);    \
>  }                                    \
> -VNET_SYSINIT(vnet_ ## name ## _init, SI_SUB_PROTO_IFATTACHDOMAIN,    \
> -    SI_ORDER_ANY, vnet_ ## name ## _init, NULL)
> +VNET_SYSINIT(vnet_ ## name ## _init, SI_SUB_INIT_IF,                 \
> +    SI_ORDER_FIRST, vnet_ ## name ## _init, NULL)
>  
>  #define      VNET_PCPUSTAT_SYSUNINIT(name)                                   
> \
>  static void                                                          \
> @@ -120,8 +120,8 @@ vnet_##name##_uninit(const void *unused)
>  {                                                                    \
>       VNET_PCPUSTAT_FREE(name);                                       \
>  }                                                                    \
> -VNET_SYSUNINIT(vnet_ ## name ## _uninit, SI_SUB_PROTO_IFATTACHDOMAIN,        
> \
> -    SI_ORDER_ANY, vnet_ ## name ## _uninit, NULL)
> +VNET_SYSUNINIT(vnet_ ## name ## _uninit, SI_SUB_INIT_IF,             \
> +    SI_ORDER_FIRST, vnet_ ## name ## _uninit, NULL)
>  
>  #ifdef SYSCTL_OID
>  #define      SYSCTL_VNET_PCPUSTAT(parent, nbr, name, type, array, desc)      
> \
> 
> Modified: head/sys/netgraph/ng_eiface.c
> ==============================================================================
> --- head/sys/netgraph/ng_eiface.c     Tue Jun 21 07:05:49 2016        
> (r302053)
> +++ head/sys/netgraph/ng_eiface.c     Tue Jun 21 13:48:49 2016        
> (r302054)
> @@ -679,5 +679,5 @@ vnet_ng_eiface_uninit(const void *unused
>  
>       delete_unrhdr(V_ng_eiface_unit);
>  }
> -VNET_SYSUNINIT(vnet_ng_eiface_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY,
> +VNET_SYSUNINIT(vnet_ng_eiface_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY,
>     vnet_ng_eiface_uninit, NULL);
> 
> Modified: head/sys/netgraph/ng_iface.c
> ==============================================================================
> --- head/sys/netgraph/ng_iface.c      Tue Jun 21 07:05:49 2016        
> (r302053)
> +++ head/sys/netgraph/ng_iface.c      Tue Jun 21 13:48:49 2016        
> (r302054)
> @@ -786,5 +786,5 @@ vnet_ng_iface_uninit(const void *unused)
>  
>       delete_unrhdr(V_ng_iface_unit);
>  }
> -VNET_SYSUNINIT(vnet_ng_iface_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY,
> +VNET_SYSUNINIT(vnet_ng_iface_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY,
>      vnet_ng_iface_uninit, NULL);
> 
> Modified: head/sys/netinet/igmp.c
> ==============================================================================
> --- head/sys/netinet/igmp.c   Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/netinet/igmp.c   Tue Jun 21 13:48:49 2016        (r302054)
> @@ -227,7 +227,8 @@ static VNET_DEFINE(int, current_state_ti
>  #define      V_state_change_timers_running   
> VNET(state_change_timers_running)
>  #define      V_current_state_timers_running  
> VNET(current_state_timers_running)
>  
> -static VNET_DEFINE(LIST_HEAD(, igmp_ifsoftc), igi_head);
> +static VNET_DEFINE(LIST_HEAD(, igmp_ifsoftc), igi_head) =
> +    LIST_HEAD_INITIALIZER(igi_head);
>  static VNET_DEFINE(struct igmpstat, igmpstat) = {
>       .igps_version = IGPS_VERSION_3,
>       .igps_len = sizeof(struct igmpstat),
> @@ -701,10 +702,6 @@ igi_delete_locked(const struct ifnet *if
>                       return;
>               }
>       }
> -
> -#ifdef INVARIANTS
> -     panic("%s: igmp_ifsoftc not found for ifp %p\n", __func__,  ifp);
> -#endif
>  }
>  
>  /*
> @@ -3595,57 +3592,28 @@ igmp_rec_type_to_str(const int type)
>  }
>  #endif
>  
> -static void
> -igmp_init(void *unused __unused)
> -{
> -
> -     CTR1(KTR_IGMPV3, "%s: initializing", __func__);
> -
> -     IGMP_LOCK_INIT();
> -
> -     m_raopt = igmp_ra_alloc();
> -
> -     netisr_register(&igmp_nh);
> -}
> -SYSINIT(igmp_init, SI_SUB_PSEUDO, SI_ORDER_MIDDLE, igmp_init, NULL);
> -
> -static void
> -igmp_uninit(void *unused __unused)
> -{
> -
> -     CTR1(KTR_IGMPV3, "%s: tearing down", __func__);
> -
> -     netisr_unregister(&igmp_nh);
> -
> -     m_free(m_raopt);
> -     m_raopt = NULL;
> -
> -     IGMP_LOCK_DESTROY();
> -}
> -SYSUNINIT(igmp_uninit, SI_SUB_PSEUDO, SI_ORDER_MIDDLE, igmp_uninit, NULL);
> -
> +#ifdef VIMAGE
>  static void
>  vnet_igmp_init(const void *unused __unused)
>  {
>  
> -     CTR1(KTR_IGMPV3, "%s: initializing", __func__);
> -
> -     LIST_INIT(&V_igi_head);
> +     netisr_register_vnet(&igmp_nh);
>  }
> -VNET_SYSINIT(vnet_igmp_init, SI_SUB_PSEUDO, SI_ORDER_ANY, vnet_igmp_init,
> -    NULL);
> +VNET_SYSINIT(vnet_igmp_init, SI_SUB_PROTO_MC, SI_ORDER_ANY,
> +    vnet_igmp_init, NULL);
>  
>  static void
>  vnet_igmp_uninit(const void *unused __unused)
>  {
>  
> +     /* This can happen when we shutdown the entire network stack. */
>       CTR1(KTR_IGMPV3, "%s: tearing down", __func__);
>  
> -     KASSERT(LIST_EMPTY(&V_igi_head),
> -         ("%s: igi list not empty; ifnets not detached?", __func__));
> +     netisr_unregister_vnet(&igmp_nh);
>  }
> -VNET_SYSUNINIT(vnet_igmp_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY,
> +VNET_SYSUNINIT(vnet_igmp_uninit, SI_SUB_PROTO_MC, SI_ORDER_ANY,
>      vnet_igmp_uninit, NULL);
> +#endif
>  
>  #ifdef DDB
>  DB_SHOW_COMMAND(igi_list, db_show_igi_list)
> @@ -3682,14 +3650,24 @@ static int
>  igmp_modevent(module_t mod, int type, void *unused __unused)
>  {
>  
> -    switch (type) {
> -    case MOD_LOAD:
> -    case MOD_UNLOAD:
> -     break;
> -    default:
> -     return (EOPNOTSUPP);
> -    }
> -    return (0);
> +     switch (type) {
> +     case MOD_LOAD:
> +             CTR1(KTR_IGMPV3, "%s: initializing", __func__);
> +             IGMP_LOCK_INIT();
> +             m_raopt = igmp_ra_alloc();
> +             netisr_register(&igmp_nh);
> +             break;
> +     case MOD_UNLOAD:
> +             CTR1(KTR_IGMPV3, "%s: tearing down", __func__);
> +             netisr_unregister(&igmp_nh);
> +             m_free(m_raopt);
> +             m_raopt = NULL;
> +             IGMP_LOCK_DESTROY();
> +             break;
> +     default:
> +             return (EOPNOTSUPP);
> +     }
> +     return (0);
>  }
>  
>  static moduledata_t igmp_mod = {
> @@ -3697,4 +3675,4 @@ static moduledata_t igmp_mod = {
>      igmp_modevent,
>      0
>  };
> -DECLARE_MODULE(igmp, igmp_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
> +DECLARE_MODULE(igmp, igmp_mod, SI_SUB_PROTO_MC, SI_ORDER_MIDDLE);
> 
> Modified: head/sys/netinet/in.c
> ==============================================================================
> --- head/sys/netinet/in.c     Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/netinet/in.c     Tue Jun 21 13:48:49 2016        (r302054)
> @@ -895,6 +895,39 @@ in_scrubprefix(struct in_ifaddr *target,
>  
>  #undef rtinitflags
>  
> +void
> +in_ifscrub_all(void)
> +{
> +     struct ifnet *ifp;
> +     struct ifaddr *ifa, *nifa;
> +     struct ifaliasreq ifr;
> +
> +     IFNET_RLOCK();
> +     TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
> +             /* Cannot lock here - lock recursion. */
> +             /* IF_ADDR_RLOCK(ifp); */
> +             TAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, nifa) {
> +                     if (ifa->ifa_addr->sa_family != AF_INET)
> +                             continue;
> +
> +                     /*
> +                      * This is ugly but the only way for legacy IP to
> +                      * cleanly remove addresses and everything attached.
> +                      */
> +                     bzero(&ifr, sizeof(ifr));
> +                     ifr.ifra_addr = *ifa->ifa_addr;
> +                     if (ifa->ifa_dstaddr)
> +                     ifr.ifra_broadaddr = *ifa->ifa_dstaddr;
> +                     (void)in_control(NULL, SIOCDIFADDR, (caddr_t)&ifr,
> +                         ifp, NULL);
> +             }
> +             /* IF_ADDR_RUNLOCK(ifp); */
> +             in_purgemaddrs(ifp);
> +             igmp_domifdetach(ifp);
> +     }
> +     IFNET_RUNLOCK();
> +}
> +
>  /*
>   * Return 1 if the address might be a local broadcast address.
>   */
> 
> Modified: head/sys/netinet/in_var.h
> ==============================================================================
> --- head/sys/netinet/in_var.h Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/netinet/in_var.h Tue Jun 21 13:48:49 2016        (r302054)
> @@ -376,6 +376,7 @@ int       in_control(struct socket *, u_long, 
>           struct thread *);
>  int  in_addprefix(struct in_ifaddr *, int);
>  int  in_scrubprefix(struct in_ifaddr *, u_int);
> +void in_ifscrub_all(void);
>  void ip_input(struct mbuf *);
>  void ip_direct_input(struct mbuf *);
>  void in_ifadown(struct ifaddr *ifa, int);
> 
> Modified: head/sys/netinet/ip_id.c
> ==============================================================================
> --- head/sys/netinet/ip_id.c  Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/netinet/ip_id.c  Tue Jun 21 13:48:49 2016        (r302054)
> @@ -294,4 +294,4 @@ ipid_sysuninit(void)
>       counter_u64_free(V_ip_id);
>       mtx_destroy(&V_ip_id_mtx);
>  }
> -VNET_SYSUNINIT(ip_id, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY, ipid_sysuninit, 
> NULL);
> +VNET_SYSUNINIT(ip_id, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, ipid_sysuninit, 
> NULL);
> 
> Modified: head/sys/netinet/ip_input.c
> ==============================================================================
> --- head/sys/netinet/ip_input.c       Tue Jun 21 07:05:49 2016        
> (r302053)
> +++ head/sys/netinet/ip_input.c       Tue Jun 21 13:48:49 2016        
> (r302054)
> @@ -370,6 +370,7 @@ ip_init(void)
>  static void
>  ip_destroy(void *unused __unused)
>  {
> +     struct ifnet *ifp;
>       int error;
>  
>  #ifdef       RSS
> @@ -393,11 +394,21 @@ ip_destroy(void *unused __unused)
>                   "type HHOOK_TYPE_IPSEC_OUT, id HHOOK_IPSEC_INET: "
>                   "error %d returned\n", __func__, error);
>       }
> -     /* Cleanup in_ifaddr hash table; should be empty. */
> -     hashdestroy(V_in_ifaddrhashtbl, M_IFADDR, V_in_ifaddrhmask);
> +
> +     /* Remove the IPv4 addresses from all interfaces. */
> +     in_ifscrub_all();
> +
> +     /* Make sure the IPv4 routes are gone as well. */
> +     IFNET_RLOCK();
> +     TAILQ_FOREACH(ifp, &V_ifnet, if_link)
> +             rt_flushifroutes_af(ifp, AF_INET);
> +     IFNET_RUNLOCK();
>  
>       /* Destroy IP reassembly queue. */
>       ipreass_destroy();
> +
> +     /* Cleanup in_ifaddr hash table; should be empty. */
> +     hashdestroy(V_in_ifaddrhashtbl, M_IFADDR, V_in_ifaddrhmask);
>  }
>  
>  VNET_SYSUNINIT(ip, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, ip_destroy, NULL);
> 
> Modified: head/sys/netinet/ip_mroute.c
> ==============================================================================
> --- head/sys/netinet/ip_mroute.c      Tue Jun 21 07:05:49 2016        
> (r302053)
> +++ head/sys/netinet/ip_mroute.c      Tue Jun 21 13:48:49 2016        
> (r302054)
> @@ -2822,7 +2822,7 @@ vnet_mroute_init(const void *unused __un
>       callout_init(&V_bw_meter_ch, 1);
>  }
>  
> -VNET_SYSINIT(vnet_mroute_init, SI_SUB_PSEUDO, SI_ORDER_ANY, vnet_mroute_init,
> +VNET_SYSINIT(vnet_mroute_init, SI_SUB_PROTO_MC, SI_ORDER_ANY, 
> vnet_mroute_init,
>       NULL);
>  
>  static void
> @@ -2833,7 +2833,7 @@ vnet_mroute_uninit(const void *unused __
>       V_nexpire = NULL;
>  }
>  
> -VNET_SYSUNINIT(vnet_mroute_uninit, SI_SUB_PSEUDO, SI_ORDER_MIDDLE, 
> +VNET_SYSUNINIT(vnet_mroute_uninit, SI_SUB_PROTO_MC, SI_ORDER_MIDDLE, 
>       vnet_mroute_uninit, NULL);
>  
>  static int
> @@ -2946,4 +2946,4 @@ static moduledata_t ip_mroutemod = {
>      0
>  };
>  
> -DECLARE_MODULE(ip_mroute, ip_mroutemod, SI_SUB_PSEUDO, SI_ORDER_MIDDLE);
> +DECLARE_MODULE(ip_mroute, ip_mroutemod, SI_SUB_PROTO_MC, SI_ORDER_MIDDLE);
> 
> Modified: head/sys/netinet6/in6.c
> ==============================================================================
> --- head/sys/netinet6/in6.c   Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/netinet6/in6.c   Tue Jun 21 13:48:49 2016        (r302054)
> @@ -2419,7 +2419,7 @@ in6_domifdetach(struct ifnet *ifp, void 
>  
>       mld_domifdetach(ifp);
>       scope6_ifdetach(ext->scope6_id);
> -     nd6_ifdetach(ext->nd_ifinfo);
> +     nd6_ifdetach(ifp, ext->nd_ifinfo);
>       lltable_free(ext->lltable);
>       COUNTER_ARRAY_FREE(ext->in6_ifstat,
>           sizeof(struct in6_ifstat) / sizeof(uint64_t));
> 
> Modified: head/sys/netinet6/in6_ifattach.c
> ==============================================================================
> --- head/sys/netinet6/in6_ifattach.c  Tue Jun 21 07:05:49 2016        
> (r302053)
> +++ head/sys/netinet6/in6_ifattach.c  Tue Jun 21 13:48:49 2016        
> (r302054)
> @@ -761,19 +761,30 @@ in6_ifattach(struct ifnet *ifp, struct i
>  
>  /*
>   * NOTE: in6_ifdetach() does not support loopback if at this moment.
> - * We don't need this function in bsdi, because interfaces are never removed
> - * from the ifnet list in bsdi.
> + *
> + * When shutting down a VNET we clean up layers top-down.  In that case
> + * upper layer protocols (ulp) are cleaned up already and locks are destroyed
> + * and we must not call into these cleanup functions anymore, thus purgeulp
> + * is set to 0 in that case by in6_ifdetach_destroy().
> + * The normal case of destroying a (cloned) interface still needs to cleanup
> + * everything related to the interface and will have purgeulp set to 1.
>   */
> -void
> -in6_ifdetach(struct ifnet *ifp)
> +static void
> +_in6_ifdetach(struct ifnet *ifp, int purgeulp)
>  {
>       struct ifaddr *ifa, *next;
>  
>       if (ifp->if_afdata[AF_INET6] == NULL)
>               return;
>  
> -     /* remove neighbor management table */
> -     nd6_purge(ifp);
> +     /*
> +      * Remove neighbor management table.
> +      * Enabling the nd6_purge will panic on vmove for interfaces on VNET
> +      * teardown as the IPv6 layer is cleaned up already and the locks
> +      * are destroyed.
> +      */
> +     if (purgeulp)
> +             nd6_purge(ifp);
>  
>       /*
>        * nuke any of IPv6 addresses we have
> @@ -784,9 +795,11 @@ in6_ifdetach(struct ifnet *ifp)
>                       continue;
>               in6_purgeaddr(ifa);
>       }
> -     in6_pcbpurgeif0(&V_udbinfo, ifp);
> -     in6_pcbpurgeif0(&V_ulitecbinfo, ifp);
> -     in6_pcbpurgeif0(&V_ripcbinfo, ifp);
> +     if (purgeulp) {
> +             in6_pcbpurgeif0(&V_udbinfo, ifp);
> +             in6_pcbpurgeif0(&V_ulitecbinfo, ifp);
> +             in6_pcbpurgeif0(&V_ripcbinfo, ifp);
> +     }
>       /* leave from all multicast groups joined */
>       in6_purgemaddrs(ifp);
>  
> @@ -798,7 +811,22 @@ in6_ifdetach(struct ifnet *ifp)
>        * prefixes after removing all addresses above.
>        * (Or can we just delay calling nd6_purge until at this point?)
>        */
> -     nd6_purge(ifp);
> +     if (purgeulp)
> +             nd6_purge(ifp);
> +}
> +
> +void
> +in6_ifdetach(struct ifnet *ifp)
> +{
> +
> +     _in6_ifdetach(ifp, 1);
> +}
> +
> +void
> +in6_ifdetach_destroy(struct ifnet *ifp)
> +{
> +
> +     _in6_ifdetach(ifp, 0);
>  }
>  
>  int
> 
> Modified: head/sys/netinet6/in6_ifattach.h
> ==============================================================================
> --- head/sys/netinet6/in6_ifattach.h  Tue Jun 21 07:05:49 2016        
> (r302053)
> +++ head/sys/netinet6/in6_ifattach.h  Tue Jun 21 13:48:49 2016        
> (r302054)
> @@ -37,6 +37,7 @@
>  void in6_ifattach(struct ifnet *, struct ifnet *);
>  void in6_ifattach_destroy(void);
>  void in6_ifdetach(struct ifnet *);
> +void in6_ifdetach_destroy(struct ifnet *);
>  int in6_get_tmpifid(struct ifnet *, u_int8_t *, const u_int8_t *, int);
>  void in6_tmpaddrtimer(void *);
>  int in6_get_hw_ifid(struct ifnet *, struct in6_addr *);
> 
> Modified: head/sys/netinet6/ip6_input.c
> ==============================================================================
> --- head/sys/netinet6/ip6_input.c     Tue Jun 21 07:05:49 2016        
> (r302053)
> +++ head/sys/netinet6/ip6_input.c     Tue Jun 21 13:48:49 2016        
> (r302054)
> @@ -113,6 +113,7 @@ __FBSDID("$FreeBSD$");
>  #include <netinet/icmp6.h>
>  #include <netinet6/scope6_var.h>
>  #include <netinet6/in6_ifattach.h>
> +#include <netinet6/mld6_var.h>
>  #include <netinet6/nd6.h>
>  #include <netinet6/in6_rss.h>
>  
> @@ -314,6 +315,8 @@ ip6proto_unregister(short ip6proto)
>  static void
>  ip6_destroy(void *unused __unused)
>  {
> +     struct ifaddr *ifa, *nifa;
> +     struct ifnet *ifp;
>       int error;
>  
>  #ifdef RSS
> @@ -336,9 +339,30 @@ ip6_destroy(void *unused __unused)
>                   "type HHOOK_TYPE_IPSEC_OUT, id HHOOK_IPSEC_INET6: "
>                   "error %d returned\n", __func__, error);
>       }
> -     hashdestroy(V_in6_ifaddrhashtbl, M_IFADDR, V_in6_ifaddrhmask);
> +
> +     /* Cleanup addresses. */
> +     IFNET_RLOCK();
> +     TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
> +             /* Cannot lock here - lock recursion. */
> +             /* IF_ADDR_LOCK(ifp); */
> +             TAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, nifa) {
> +
> +                     if (ifa->ifa_addr->sa_family != AF_INET6)
> +                             continue;
> +                     in6_purgeaddr(ifa);
> +             }
> +             /* IF_ADDR_UNLOCK(ifp); */
> +             in6_ifdetach_destroy(ifp);
> +             mld_domifdetach(ifp);
> +             /* Make sure any routes are gone as well. */
> +             rt_flushifroutes_af(ifp, AF_INET6);
> +     }
> +     IFNET_RUNLOCK();
> +
>       nd6_destroy();
>       in6_ifattach_destroy();
> +
> +     hashdestroy(V_in6_ifaddrhashtbl, M_IFADDR, V_in6_ifaddrhmask);
>  }
>  
>  VNET_SYSUNINIT(inet6, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, ip6_destroy, 
> NULL);
> 
> Modified: head/sys/netinet6/ip6_mroute.c
> ==============================================================================
> --- head/sys/netinet6/ip6_mroute.c    Tue Jun 21 07:05:49 2016        
> (r302053)
> +++ head/sys/netinet6/ip6_mroute.c    Tue Jun 21 13:48:49 2016        
> (r302054)
> @@ -1966,4 +1966,4 @@ static moduledata_t ip6_mroutemod = {
>       0
>  };
>  
> -DECLARE_MODULE(ip6_mroute, ip6_mroutemod, SI_SUB_PSEUDO, SI_ORDER_ANY);
> +DECLARE_MODULE(ip6_mroute, ip6_mroutemod, SI_SUB_PROTO_MC, SI_ORDER_ANY);
> 
> Modified: head/sys/netinet6/mld6.c
> ==============================================================================
> --- head/sys/netinet6/mld6.c  Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/netinet6/mld6.c  Tue Jun 21 13:48:49 2016        (r302054)
> @@ -612,9 +612,6 @@ mli_delete_locked(const struct ifnet *if
>                       return;
>               }
>       }
> -#ifdef INVARIANTS
> -     panic("%s: mld_ifsoftc not found for ifp %p\n", __func__,  ifp);
> -#endif
>  }
>  
>  /*
> @@ -3265,7 +3262,7 @@ mld_init(void *unused __unused)
>       mld_po.ip6po_prefer_tempaddr = IP6PO_TEMPADDR_NOTPREFER;
>       mld_po.ip6po_flags = IP6PO_DONTFRAG;
>  }
> -SYSINIT(mld_init, SI_SUB_PSEUDO, SI_ORDER_MIDDLE, mld_init, NULL);
> +SYSINIT(mld_init, SI_SUB_PROTO_MC, SI_ORDER_MIDDLE, mld_init, NULL);
>  
>  static void
>  mld_uninit(void *unused __unused)
> @@ -3274,7 +3271,7 @@ mld_uninit(void *unused __unused)
>       CTR1(KTR_MLD, "%s: tearing down", __func__);
>       MLD_LOCK_DESTROY();
>  }
> -SYSUNINIT(mld_uninit, SI_SUB_PSEUDO, SI_ORDER_MIDDLE, mld_uninit, NULL);
> +SYSUNINIT(mld_uninit, SI_SUB_PROTO_MC, SI_ORDER_MIDDLE, mld_uninit, NULL);
>  
>  static void
>  vnet_mld_init(const void *unused __unused)
> @@ -3284,19 +3281,17 @@ vnet_mld_init(const void *unused __unuse
>  
>       LIST_INIT(&V_mli_head);
>  }
> -VNET_SYSINIT(vnet_mld_init, SI_SUB_PSEUDO, SI_ORDER_ANY, vnet_mld_init,
> +VNET_SYSINIT(vnet_mld_init, SI_SUB_PROTO_MC, SI_ORDER_ANY, vnet_mld_init,
>      NULL);
>  
>  static void
>  vnet_mld_uninit(const void *unused __unused)
>  {
>  
> +     /* This can happen if we shutdown the network stack. */
>       CTR1(KTR_MLD, "%s: tearing down", __func__);
> -
> -     KASSERT(LIST_EMPTY(&V_mli_head),
> -         ("%s: mli list not empty; ifnets not detached?", __func__));
>  }
> -VNET_SYSUNINIT(vnet_mld_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, vnet_mld_uninit,
> +VNET_SYSUNINIT(vnet_mld_uninit, SI_SUB_PROTO_MC, SI_ORDER_ANY, 
> vnet_mld_uninit,
>      NULL);
>  
>  static int
> @@ -3318,4 +3313,4 @@ static moduledata_t mld_mod = {
>      mld_modevent,
>      0
>  };
> -DECLARE_MODULE(mld, mld_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
> +DECLARE_MODULE(mld, mld_mod, SI_SUB_PROTO_MC, SI_ORDER_ANY);
> 
> Modified: head/sys/netinet6/nd6.c
> ==============================================================================
> --- head/sys/netinet6/nd6.c   Tue Jun 21 07:05:49 2016        (r302053)
> +++ head/sys/netinet6/nd6.c   Tue Jun 21 13:48:49 2016        (r302054)
> @@ -292,8 +292,19 @@ nd6_ifattach(struct ifnet *ifp)
>  }
>  
>  void
> -nd6_ifdetach(struct nd_ifinfo *nd)
> +nd6_ifdetach(struct ifnet *ifp, struct nd_ifinfo *nd)
>  {
> +     struct ifaddr *ifa, *next;
> 
> *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
> 

_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"

Reply via email to