Hi again list. Here's little update about the issue.
Yes, there have been a few times when I have seen something like this, and 
issue when the driver is static and not when modulized. I have never been clear 
what to make of this. If anyone else here has any thoughts or enlightening 
ideas I'd appreciate it.
Fresh build, 8.2-STABLE #7: Tue Sep 13 14:25:45, problem is still here, regardless of igb included in the kernel or loaded as module from loader.conf, system panic with msix enabled.
Also if it does not even happen when using the GENERIC config then it would be 
interesting to see the diff between that and what does cause the panic.
Here it is:

   --- GENERIC  2011-04-10 19:55:25.304366901 +0200
   +++ PROLIANT 2011-06-18 16:03:32.728117341 +0200
   @@ -19,7 +19,7 @@
     # $FreeBSD: src/sys/amd64/conf/GENERIC,v 1.531.2.17 2011/04/09 12:04:35 bz 
Exp $

     cpu                HAMMER
   -ident               GENERIC
   +ident               PROLIANT

     # To statically compile in device wiring instead of /boot/device.hints
     #hints             "GENERIC.hints"               # Default places to look 
for devices.
   @@ -30,7 +30,7 @@
     #
     # env              "GENERIC.env"

   -makeoptions DEBUG=-g                # Build kernel with gdb(1) debug symbols
   +#makeoptions        DEBUG=-g                # Build kernel with gdb(1) 
debug symbols

     options    SCHED_ULE               # ULE scheduler
     options    PREEMPTION              # Enable kernel thread preemption
   @@ -44,9 +44,9 @@
     options    UFS_GJOURNAL            # Enable gjournal-based UFS journaling
     options    MD_ROOT                 # MD is a potential root device
     options    NFSCLIENT               # Network Filesystem Client
   -options     NFSSERVER               # Network Filesystem Server
   -options     NFSLOCKD                # Network Lock Manager
   -options     NFS_ROOT                # NFS usable as /, requires NFSCLIENT
   +#options    NFSSERVER               # Network Filesystem Server
   +#options    NFSLOCKD                # Network Lock Manager
   +#options    NFS_ROOT                # NFS usable as /, requires NFSCLIENT
     options    MSDOSFS                 # MSDOS Filesystem
     options    CD9660                  # ISO 9660 Filesystem
     options    PROCFS                  # Process filesystem (requires PSEUDOFS)
   @@ -81,10 +81,18 @@

     # Make an SMP-capable kernel by default
     options    SMP                     # Symmetric MultiProcessor Kernel
   +options         DEVICE_POLLING

     # CPU frequency control
     device             cpufreq

   +options         ALTQ
   +options         ALTQ_CBQ                # Class Bases Queuing (CBQ)
   +options         ALTQ_RED                # Random Early Detection (RED)
   +options         ALTQ_RIO                # RED In/Out
   +options         ALTQ_HFSC               # Hierarchical Packet Scheduler 
(HFSC)
   +options         ALTQ_PRIQ               # Priority Queuing (PRIQ)
   +
     # Bus support.
     device             acpi
     device             pci
   @@ -102,59 +110,59 @@
     options    ATA_STATIC_ID   # Static device numbering

     # SCSI Controllers
   -device              ahc             # AHA2940 and onboard AIC7xxx devices
   -options     AHC_REG_PRETTY_PRINT    # Print register bitfields in debug
   +#device             ahc             # AHA2940 and onboard AIC7xxx devices
   +#options    AHC_REG_PRETTY_PRINT    # Print register bitfields in debug
                                        # output.  Adds ~128k to driver.
   -device              ahd             # AHA39320/29320 and onboard AIC79xx 
devices
   -options     AHD_REG_PRETTY_PRINT    # Print register bitfields in debug
   +#device             ahd             # AHA39320/29320 and onboard AIC79xx 
devices
   +#options    AHD_REG_PRETTY_PRINT    # Print register bitfields in debug
                                        # output.  Adds ~215k to driver.
   -device              amd             # AMD 53C974 (Tekram DC-390(T))
   -device              hptiop          # Highpoint RocketRaid 3xxx series
   -device              isp             # Qlogic family
   +#device             amd             # AMD 53C974 (Tekram DC-390(T))
   +#device             hptiop          # Highpoint RocketRaid 3xxx series
   +#device             isp             # Qlogic family
     #device            ispfw           # Firmware for QLogic HBAs- normally a 
module
   -device              mpt             # LSI-Logic MPT-Fusion
   -device              mps             # LSI-Logic MPT-Fusion 2
   +#device             mpt             # LSI-Logic MPT-Fusion
   +#device             mps             # LSI-Logic MPT-Fusion 2
     #device            ncr             # NCR/Symbios Logic
   -device              sym             # NCR/Symbios Logic (newer chipsets + 
those of `ncr')
   -device              trm             # Tekram DC395U/UW/F DC315U adapters
   +#device             sym             # NCR/Symbios Logic (newer chipsets + 
those of `ncr')
   +#device             trm             # Tekram DC395U/UW/F DC315U adapters

   -device              adv             # Advansys SCSI adapters
   -device              adw             # Advansys wide SCSI adapters
   -device              aic             # Adaptec 15[012]x SCSI adapters, 
AIC-6[23]60.
   -device              bt              # Buslogic/Mylex MultiMaster SCSI 
adapters
   +#device             adv             # Advansys SCSI adapters
   +#device             adw             # Advansys wide SCSI adapters
   +#device             aic             # Adaptec 15[012]x SCSI adapters, 
AIC-6[23]60.
   +#device             bt              # Buslogic/Mylex MultiMaster SCSI 
adapters

     # SCSI peripherals
     device             scbus           # SCSI bus (required for SCSI)
   -device              ch              # SCSI media changers
   +#device             ch              # SCSI media changers
     device             da              # Direct Access (disks)
   -device              sa              # Sequential Access (tape etc)
   +#device             sa              # Squential Access (tape etc)
     device             cd              # CD
     device             pass            # Passthrough device (direct SCSI 
access)
   -device              ses             # SCSI Environmental Services (and 
SAF-TE)
   +#device             ses             # SCSI Environmental Services (and 
SAF-TE)

     # RAID controllers interfaced to the SCSI subsystem
   -device              amr             # AMI MegaRAID
   -device              arcmsr          # Areca SATA II RAID
   +#device             amr             # AMI MegaRAID
   +#device             arcmsr          # Areca SATA II RAID
     #XXX it is not 64-bit clean, -scottl
     #device            asr             # DPT SmartRAID V, VI and Adaptec SCSI 
RAID
   -device              ciss            # Compaq Smart RAID 5*
   -device              dpt             # DPT Smartcache III, IV - See NOTES 
for options
   -device              hptmv           # Highpoint RocketRAID 182x
   -device              hptrr           # Highpoint RocketRAID 17xx, 22xx, 
23xx, 25xx
   -device              iir             # Intel Integrated RAID
   -device              ips             # IBM (Adaptec) ServeRAID
   -device              mly             # Mylex AcceleRAID/eXtremeRAID
   -device              twa             # 3ware 9000 series PATA/SATA RAID
   +#device             ciss            # Compaq Smart RAID 5*
   +#device             dpt             # DPT Smartcache III, IV - See NOTES 
for options
   +#device             hptmv           # Highpoint RocketRAID 182x
   +#device             hptrr           # Highpoint RocketRAID 17xx, 22xx, 
23xx, 25xx
   +#device             iir             # Intel Integrated RAID
   +#device             ips             # IBM (Adaptec) ServeRAID
   +#device             mly             # Mylex AcceleRAID/eXtremeRAID
   +#device             twa             # 3ware 9000 series PATA/SATA RAID

     # RAID controllers
   -device              aac             # Adaptec FSA RAID
   -device              aacp            # SCSI passthrough for aac (requires 
CAM)
   -device              ida             # Compaq Smart RAID
   -device              mfi             # LSI MegaRAID SAS
   -device              mlx             # Mylex DAC960 family
   +#device             aac             # Adaptec FSA RAID
   +#device             aacp            # SCSI passthrough for aac (requires 
CAM)
   +#device             ida             # Compaq Smart RAID
   +#device             mfi             # LSI MegaRAID SAS
   +#device             mlx             # Mylex DAC960 family
     #XXX pointer/int warnings
     #device            pst             # Promise Supertrak SX6000
   -device              twe             # 3ware ATA RAID
   +#device             twe             # 3ware ATA RAID

     # atkbdc0 controls both the keyboard and the PS/2 mouse
     device             atkbdc          # AT keyboard controller
   @@ -170,13 +178,13 @@
     # syscons is the default console driver, resembling an SCO console
     device             sc

   -device              agp             # support several AGP chipsets
   +#device             agp             # support several AGP chipsets

     # PCCARD (PCMCIA) support
     # PCMCIA and cardbus bridge support
   -device              cbb             # cardbus (yenta) bridge
   -device              pccard          # PC Card (16-bit) bus
   -device              cardbus         # CardBus (32-bit) bus
   +#device             cbb             # cardbus (yenta) bridge
   +#device             pccard          # PC Card (16-bit) bus
   +#device             cardbus         # CardBus (32-bit) bus

     # Serial (COM) ports
     device             uart            # Generic UART driver
   @@ -195,76 +203,76 @@
     #device            puc

     # PCI Ethernet NICs.
   -device              de              # DEC/Intel DC21x4x (``Tulip'')
   -device              em              # Intel PRO/1000 Gigabit Ethernet Family
   -device              igb             # Intel PRO/1000 PCIE Server Gigabit 
Family
   -device              ixgbe           # Intel PRO/10GbE PCIE Ethernet Family
   -device              le              # AMD Am7900 LANCE and Am79C9xx PCnet
   -device              ti              # Alteon Networks Tigon I/II gigabit 
Ethernet
   -device              txp             # 3Com 3cR990 (``Typhoon'')
   -device              vx              # 3Com 3c590, 3c595 (``Vortex'')
   +#device             de              # DEC/Intel DC21x4x (``Tulip'')
   +#device             em              # Intel PRO/1000 Gigabit Ethernet Family
   +#device             igb             # Intel PRO/1000 PCIE Server Gigabit 
Family
   +#device             ixgbe           # Intel PRO/10GbE PCIE Ethernet Family
   +#device             le              # AMD Am7900 LANCE and Am79C9xx PCnet
   +#device             ti              # Alteon Networks Tigon I/II gigabit 
Ethernet
   +#device             txp             # 3Com 3cR990 (``Typhoon'')
   +#device             vx              # 3Com 3c590, 3c595 (``Vortex'')

     # PCI Ethernet NICs that use the common MII bus controller code.
     # NOTE: Be sure to keep the 'device miibus' line in order to use these 
NICs!
     device             miibus          # MII bus support
   -device              ae              # Attansic/Atheros L2 FastEthernet
   -device              age             # Attansic/Atheros L1 Gigabit Ethernet
   -device              alc             # Atheros AR8131/AR8132 Ethernet
   -device              ale             # Atheros AR8121/AR8113/AR8114 Ethernet
   -device              bce             # Broadcom BCM5706/BCM5708 Gigabit 
Ethernet
   -device              bfe             # Broadcom BCM440x 10/100 Ethernet
   -device              bge             # Broadcom BCM570xx Gigabit Ethernet
   -device              dc              # DEC/Intel 21143 and various workalikes
   -device              et              # Agere ET1310 10/100/Gigabit Ethernet
   -device              fxp             # Intel EtherExpress PRO/100B (82557, 
82558)
   -device              jme             # JMicron JMC250 Gigabit/JMC260 Fast 
Ethernet
   -device              lge             # Level 1 LXT1001 gigabit Ethernet
   -device              msk             # Marvell/SysKonnect Yukon II Gigabit 
Ethernet
   -device              nfe             # nVidia nForce MCP on-board Ethernet
   -device              nge             # NatSemi DP83820 gigabit Ethernet
   +#device             ae              # Attansic/Atheros L2 FastEthernet
   +#device             age             # Attansic/Atheros L1 Gigabit Ethernet
   +#device             alc             # Atheros AR8131/AR8132 Ethernet
   +#device             ale             # Atheros AR8121/AR8113/AR8114 Ethernet
   +#device             bce             # Broadcom BCM5706/BCM5708 Gigabit 
Ethernet
   +#device             bfe             # Broadcom BCM440x 10/100 Ethernet
   +#device             bge             # Broadcom BCM570xx Gigabit Ethernet
   +#device             dc              # DEC/Intel 21143 and various workalikes
   +#device             et              # Agere ET1310 10/100/Gigabit Ethernet
   +#device             fxp             # Intel EtherExpress PRO/100B (82557, 
82558)
   +#device             jme             # JMicron JMC250 Gigabit/JMC260 Fast 
Ethernet
   +#device             lge             # Level 1 LXT1001 gigabit Ethernet
   +#device             msk             # Marvell/SysKonnect Yukon II Gigabit 
Ethernet
   +#device             nfe             # nVidia nForce MCP on-board Ethernet
   +#device             nge             # NatSemi DP83820 gigabit Ethernet
     #device            nve             # nVidia nForce MCP on-board Ethernet 
Networking
   -device              pcn             # AMD Am79C97x PCI 10/100 (precedence 
over 'le')
   -device              re              # RealTek 8139C+/8169/8169S/8110S
   -device              rl              # RealTek 8129/8139
   -device              sf              # Adaptec AIC-6915 (``Starfire'')
   -device              sge             # Silicon Integrated Systems SiS190/191
   -device              sis             # Silicon Integrated Systems SiS 
900/SiS 7016
   -device              sk              # SysKonnect SK-984x&  SK-982x gigabit 
Ethernet
   -device              ste             # Sundance ST201 (D-Link DFE-550TX)
   -device              stge            # Sundance/Tamarack TC9021 gigabit 
Ethernet
   -device              tl              # Texas Instruments ThunderLAN
   -device              tx              # SMC EtherPower II (83c170 ``EPIC'')
   -device              vge             # VIA VT612x gigabit Ethernet
   -device              vr              # VIA Rhine, Rhine II
   -device              wb              # Winbond W89C840F
   -device              xl              # 3Com 3c90x (``Boomerang'', 
``Cyclone'')
   +#device             pcn             # AMD Am79C97x PCI 10/100 (precedence 
over 'le')
   +#device             re              # RealTek 8139C+/8169/8169S/8110S
   +#device             rl              # RealTek 8129/8139
   +#device             sf              # Adaptec AIC-6915 (``Starfire'')
   +#device             sge             # Silicon Integrated Systems SiS190/191
   +#device             sis             # Silicon Integrated Systems SiS 
900/SiS 7016
   +#device             sk              # SysKonnect SK-984x&  SK-982x gigabit 
Ethernet
   +#device             ste             # Sundance ST201 (D-Link DFE-550TX)
   +#device             stge            # Sundance/Tamarack TC9021 gigabit 
Ethernet
   +#device             tl              # Texas Instruments ThunderLAN
   +#device             tx              # SMC EtherPower II (83c170 ``EPIC'')
   +#device             vge             # VIA VT612x gigabit Ethernet
   +#device             vr              # VIA Rhine, Rhine II
   +#device             wb              # Winbond W89C840F
   +#device             xl              # 3Com 3c90x (``Boomerang'', 
``Cyclone'')

     # ISA Ethernet NICs.  pccard NICs included.
   -device              cs              # Crystal Semiconductor CS89x0 NIC
   +#device             cs              # Crystal Semiconductor CS89x0 NIC
     # 'device ed' requires 'device miibus'
   -device              ed              # NE[12]000, SMC Ultra, 3c503, DS8390 
cards
   -device              ex              # Intel EtherExpress Pro/10 and Pro/10+
   -device              ep              # Etherlink III based cards
   -device              fe              # Fujitsu MB8696x based cards
   -device              sn              # SMC's 9000 series of Ethernet chips
   -device              xe              # Xircom pccard Ethernet
   +#device             ed              # NE[12]000, SMC Ultra, 3c503, DS8390 
cards
   +#device             ex              # Intel EtherExpress Pro/10 and Pro/10+
   +#device             ep              # Etherlink III based cards
   +#device             fe              # Fujitsu MB8696x based cards
   +#device             sn              # SMC's 9000 series of Ethernet chips
   +#device             xe              # Xircom pccard Ethernet

     # Wireless NIC cards
   -device              wlan            # 802.11 support
   -options     IEEE80211_DEBUG # enable debug msgs
   -options     IEEE80211_AMPDU_AGE # age frames in AMPDU reorder q's
   -options     IEEE80211_SUPPORT_MESH  # enable 802.11s draft support
   -device              wlan_wep        # 802.11 WEP support
   -device              wlan_ccmp       # 802.11 CCMP support
   -device              wlan_tkip       # 802.11 TKIP support
   -device              wlan_amrr       # AMRR transmit rate control algorithm
   -device              an              # Aironet 4500/4800 802.11 wireless 
NICs.
   -device              ath             # Atheros pci/cardbus NIC's
   -device              ath_hal         # pci/cardbus chip support
   -options     AH_SUPPORT_AR5416       # enable AR5416 tx/rx descriptors
   -device              ath_rate_sample # SampleRate tx rate control for ath
   -device              ral             # Ralink Technology RT2500 wireless 
NICs.
   -device              wi              # WaveLAN/Intersil/Symbol 802.11 
wireless NICs.
   +#device             wlan            # 802.11 support
   +#options    IEEE80211_DEBUG # enable debug msgs
   +#options    IEEE80211_AMPDU_AGE # age frames in AMPDU reorder q's
   +#options    IEEE80211_SUPPORT_MESH  # enable 802.11s draft support
   +#device             wlan_wep        # 802.11 WEP support
   +#device             wlan_ccmp       # 802.11 CCMP support
   +#device             wlan_tkip       # 802.11 TKIP support
   +#device             wlan_amrr       # AMRR transmit rate control algorithm
   +#device             an              # Aironet 4500/4800 802.11 wireless 
NICs.
   +#device             ath             # Atheros pci/cardbus NIC's
   +#device             ath_hal         # pci/cardbus chip support
   +#options    AH_SUPPORT_AR5416       # enable AR5416 tx/rx descriptors
   +#device             ath_rate_sample # SampleRate tx rate control for ath
   +#device             ral             # Ralink Technology RT2500 wireless 
NICs.
   +#device             wi              # WaveLAN/Intersil/Symbol 802.11 
wireless NICs.

     # Pseudo devices.
     device             loop            # Network loopback
   @@ -295,34 +303,36 @@
     device             ulpt            # Printer
     device             umass           # Disks/Mass storage - Requires scbus 
and da
     device             ums             # Mouse
   -device              urio            # Diamond Rio 500 MP3 player
   +#device             urio            # Diamond Rio 500 MP3 player
     # USB Serial devices
   -device              uark            # Technologies ARK3116 based serial 
adapters
   -device              ubsa            # Belkin F5U103 and compatible serial 
adapters
   -device              uftdi           # For FTDI usb serial adapters
   -device              uipaq           # Some WinCE based devices
   -device              uplcom          # Prolific PL-2303 serial adapters
   -device              uslcom          # SI Labs CP2101/CP2102 serial adapters
   -device              uvisor          # Visor and Palm devices
   -device              uvscom          # USB serial support for DDI pocket's 
PHS
   +#device             uark            # Technologies ARK3116 based serial 
adapters
   +#device             ubsa            # Belkin F5U103 and compatible serial 
adapters
   +#device             uftdi           # For FTDI usb serial adapters
   +#device             uipaq           # Some WinCE based devices
   +#device             uplcom          # Prolific PL-2303 serial adapters
   +#device             uslcom          # SI Labs CP2101/CP2102 serial adapters
   +#device             uvisor          # Visor and Palm devices
   +#device             uvscom          # USB serial support for DDI pocket's 
PHS
     # USB Ethernet, requires miibus
   -device              aue             # ADMtek USB Ethernet
   -device              axe             # ASIX Electronics USB Ethernet
   -device              cdce            # Generic USB over Ethernet
   -device              cue             # CATC USB Ethernet
   -device              kue             # Kawasaki LSI USB Ethernet
   -device              rue             # RealTek RTL8150 USB Ethernet
   -device              udav            # Davicom DM9601E USB
   +#device             aue             # ADMtek USB Ethernet
   +#device             axe             # ASIX Electronics USB Ethernet
   +#device             cdce            # Generic USB over Ethernet
   +#device             cue             # CATC USB Ethernet
   +#device             kue             # Kawasaki LSI USB Ethernet
   +#device             rue             # RealTek RTL8150 USB Ethernet
   +#device             udav            # Davicom DM9601E USB
     # USB Wireless
   -device              rum             # Ralink Technology RT2501USB wireless 
NICs
   -device              uath            # Atheros AR5523 wireless NICs
   -device              ural            # Ralink Technology RT2500USB wireless 
NICs
   -device              zyd             # ZyDAS zb1211/zb1211b wireless NICs
   +#device             rum             # Ralink Technology RT2501USB wireless 
NICs
   +#device             uath            # Atheros AR5523 wireless NICs
   +#device             ural            # Ralink Technology RT2500USB wireless 
NICs
   +#device             zyd             # ZyDAS zb1211/zb1211b wireless NICs

     # FireWire support
   -device              firewire        # FireWire bus code
   +#device             firewire        # FireWire bus code
     #device            sbp             # SCSI over FireWire (Requires scbus 
and da)
   -device              fwe             # Ethernet over FireWire (non-standard!)
   -device              fwip            # IP over FireWire (RFC 2734,3146)
   -device              dcons           # Dumb console driver
   -device              dcons_crom      # Configuration ROM for dcons
   +#device             fwe             # Ethernet over FireWire (non-standard!)
   +#device             fwip            # IP over FireWire (RFC 2734,3146)
   +#device             dcons           # Dumb console driver
   +#device             dcons_crom      # Configuration ROM for dcons
   +
   +device          atapicam



-----Original Message-----
From: Bartosz Stec [mailto:[email protected]]
Sent: Friday, June 17, 2011 6:42 AM
To: Vogel, Jack
Cc: Jeremy Chadwick; FreeBSD Stable
Subject: Re: Panic during kernel booting on HP Proliant DL180G6 and latest 
STABLE

W dniu 2011-06-10 20:23, Vogel, Jack pisze:
Er, so what if you get rid of ZFS,  does your panic go away? It doesn't really 
matter what type adapter it is, the igb driver only requests standard size 
clusters, so memory is getting trashed somewhere I suspect.

Jack
Well, from my observations about this issue (which could be very wrong
because my lack of knowledge about BSD kernel) I don't suspect igb
driver directly, but rather an order which kernel is processing stuff
related to MSIX and hardware (so I suppose that  real cause of the
problem could be very hard to catch and repeat)? Here's why:

1. There's no panic when using GENERIC kernel. There's also nothing
unusual in my custom kernel (included in thread), neither in make.conf.

2. Before current build, panic was seen with igb driver included in
kernel, but no panic when using a module. Even better - no panic when
trying to load a module while igb driver is stil included in source. No
random memory corruption here - same scenario seen every boot with all
variants above. It's HP server with HP ECC memory by the way.

3. With current build kernel panics regardless if igb driver is a module
or included in kernel (unless i disable MSIX). But I found override - I
removed igb driver from kernel config, and a module from loader.conf.
Than booted in single user mode, and manually loaded igb driver. No
panic! Appareantly something gets wrong _only_ at boot time.


It seems that I was right about the relation of this issue with booting process. I enabled MSIX and, removed igb driver in loader.conf, and added if_igb to rc.conf uder ' kld_list="" '.
No panic if igb driver is loaded after kernel booting is complete:

   igb0: <Intel(R) PRO/1000 Network Connection version - 2.2.3> port
   0xe880-0xe89f mem
   0xfbe60000-0xfbe7ffff,0xfbe40000-0xfbe5ffff,0xfbeb8000-0xfbebbfff
   irq 28 at device 0.0 on pci7
   igb0: Using MSIX interrupts with 5 vectors
   igb0: [ITHREAD]
   igb0: [ITHREAD]
   igb0: [ITHREAD]
   igb0: [ITHREAD]
   igb0: [ITHREAD]
   igb0: Ethernet address: d8:d3:85:e3:e4:f6
   igb1: <Intel(R) PRO/1000 Network Connection version - 2.2.3> port
   0xec00-0xec1f mem
   0xfbee0000-0xfbefffff,0xfbec0000-0xfbedffff,0xfbebc000-0xfbebffff
   irq 40 at device 0.1 on pci7
   igb1: Using MSIX interrupts with 5 vectors
   igb1: [ITHREAD]
   igb1: [ITHREAD]
   igb1: [ITHREAD]
   igb1: [ITHREAD]
   igb1: [ITHREAD]
   igb1: Ethernet address: d8:d3:85:e3:e4:f7
   igb0: link state changed to UP
   igb1: link state changed to UP

kld_list feature looks like great problem override, because now I I have msix enabled and can reboot my server safelly, but it still far from actual fix.
Any new ideas?

--
Bartosz Stec

_______________________________________________
[email protected] mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-stable
To unsubscribe, send any mail to "[email protected]"

Reply via email to