On Fri, Oct 14, 2016 at 04:49:31PM +0900, YASUOKA Masahiko wrote:
> Hi,
> 
> I'm working on NEC Express5800/R110h-1 (dmesg is attached).  On this
> machine, our kernel panics with following message.
> 
>   cpu0 at mainbus0panic: cpu at apic id 0 already attached?
> 
> This seems to happen since x2APIC on the machine is enabled by BIOS
> and the kernel doesn't assume that.  The diff makes the kernel use
> x2APIC if it is enabled by BIOS.
> 
> ok?
> 

This should go in snaps, or wait for reports from tech@ with test results
before it should be committed, IMO. We don't have full support for x2apic,
and blindly enabling it like this hoping that the bios set everything up
right is bound to be a bad assumption on at least one machine out there.

-ml

> Index: sys/arch/amd64/amd64/lapic.c
> ===================================================================
> RCS file: /cvs/src/sys/arch/amd64/amd64/lapic.c,v
> retrieving revision 1.44
> diff -u -p -r1.44 lapic.c
> --- sys/arch/amd64/amd64/lapic.c      22 Jun 2016 01:12:38 -0000      1.44
> +++ sys/arch/amd64/amd64/lapic.c      14 Oct 2016 07:45:50 -0000
> @@ -170,60 +170,55 @@ lapic_map(paddr_t lapic_base)
>       int s;
>       pt_entry_t *pte;
>       vaddr_t va;
> +     u_int64_t msr;
>  
> -     /*
> -      * On real hardware, x2apic must only be enabled if interrupt remapping
> -      * is also enabled. See 10.12.7 of the SDM vol 3.
> -      * On hypervisors, this is not necessary. Hypervisors can implement
> -      * x2apic support even if the host CPU does not support it.
> -      * Until we support interrupt remapping, use x2apic only if the
> -      * hypervisor flag is also set.
> -      */
> -     if ((cpu_ecxfeature&CPUIDECX_X2APIC) && (cpu_ecxfeature&CPUIDECX_HV)) {
> -             u_int64_t msr;
> -
> -             disable_intr();
> -             s = lapic_tpr;
> -
> -             msr = rdmsr(MSR_APICBASE);
> -             msr |= APICBASE_ENABLE_X2APIC;
> -             wrmsr(MSR_APICBASE, msr);
> +     disable_intr();
> +     s = lapic_tpr;
> +
> +     msr = rdmsr(MSR_APICBASE);
>  
> +     if (ISSET(msr, APICBASE_ENABLE_X2APIC) ||
> +         (ISSET(cpu_ecxfeature, CPUIDECX_HV) &&
> +         ISSET(cpu_ecxfeature, CPUIDECX_X2APIC))) {
> +             /*
> +              * If x2APIC is enabled already, use it since it is enabled
> +              * by BIOS.  On hypervisor, use it if it exists.
> +              */
> +             if (!ISSET(msr, APICBASE_ENABLE_X2APIC)) {
> +                     msr |= APICBASE_ENABLE_X2APIC;
> +                     wrmsr(MSR_APICBASE, msr);
> +             }
>               lapic_readreg = x2apic_readreg;
>               lapic_writereg = x2apic_writereg;
>  #ifdef MULTIPROCESSOR
>               x86_ipi = x2apic_ipi;
>  #endif
>               x2apic_enabled = 1;
> -
>               codepatch_call(CPTAG_EOI, &x2apic_eoi);
>  
>               lapic_writereg(LAPIC_TPRI, s);
> -             enable_intr();
> +     } else {
> +             /*
> +              * Map local apic.  If we have a local apic, it's safe to
> +              * assume we're on a 486 or better and can use invlpg and
> +              * non-cacheable PTE's
> +              *
> +              * Whap the PTE "by hand" rather than calling pmap_kenter_pa
> +              * because the latter will attempt to invoke TLB shootdown
> +              * code just as we might have changed the value of
> +              * cpu_number()..
> +              */
> +             va = (vaddr_t)&local_apic;
> +             pte = kvtopte(va);
> +             *pte = lapic_base | PG_RW | PG_V | PG_N | PG_G | pg_nx;
> +             invlpg(va);
>  
> -             return;
> +             lapic_tpr = s;
>       }
>  
> -     va = (vaddr_t)&local_apic;
> -
> -     disable_intr();
> -     s = lapic_tpr;
> -
> -     /*
> -      * Map local apic.  If we have a local apic, it's safe to assume
> -      * we're on a 486 or better and can use invlpg and non-cacheable PTE's
> -      *
> -      * Whap the PTE "by hand" rather than calling pmap_kenter_pa because
> -      * the latter will attempt to invoke TLB shootdown code just as we
> -      * might have changed the value of cpu_number()..
> -      */
> -
> -     pte = kvtopte(va);
> -     *pte = lapic_base | PG_RW | PG_V | PG_N | PG_G | pg_nx;
> -     invlpg(va);
> -
> -     lapic_tpr = s;
>       enable_intr();
> +
> +     return;
>  }
>  
>  /*
> 
> OpenBSD 6.0-current (GENERIC.MP) #44: Fri Oct 14 16:32:03 JST 2016
>     
> yasu...@yasuoka-ob1.tokyo.iiji.jp:/source/yasuoka/openbsd/head/git/src/sys/arch/amd64/compile/GENERIC.MP
> real mem = 8207699968 (7827MB)
> avail mem = 7954391040 (7585MB)
> mpath0 at root
> scsibus0 at mpath0: 256 targets
> mainbus0 at root
> bios0 at mainbus0: SMBIOS rev. 3.0 @ 0x7f92b000 (60 entries)
> bios0: vendor American Megatrends Inc. version "5.0.4012" date 06/15/2016
> bios0: NEC Express5800/R110h-1 [N8100-2316Y]
> acpi0 at bios0: rev 2
> acpi0: sleep states S0 S4 S5
> acpi0: tables DSDT FACP APIC FIDT MCFG HPET SSDT SSDT SSDT SSDT SPCR DMAR BERT
> acpi0: wakeup devices PEGP(S4) PEG0(S4) PEG1(S4) PEG2(S4) PXSX(S4) RP17(S4) 
> PXSX(S4) RP18(S4) PXSX(S4) RP19(S4) PXSX(S4) RP20(S4) PXSX(S4) RP01(S4) 
> PXSX(S4) RP02(S4) [...]
> acpitimer0 at acpi0: 3579545 Hz, 24 bits
> acpimadt0 at acpi0 addr 0xfee00000: PC-AT compat
> cpu0 at mainbus0: apid 0 (boot processor)
> cpu0: Intel(R) Xeon(R) CPU E3-1220 v5 @ 3.00GHz, 3301.20 MHz
> cpu0: 
> FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,DS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM,PBE,SSE3,PCLMUL,DTES64,MWAIT,DS-CPL,VMX,SMX,EST,TM2,SSSE3,SDBG,FMA3,CX16,xTPR,PDCM,PCID,SSE4.1,SSE4.2,x2APIC,MOVBE,POPCNT,DEADLINE,AES,XSAVE,AVX,F16C,RDRAND,NXE,PAGE1GB,LONG,LAHF,ABM,3DNOWP,PERF,ITSC,FSGSBASE,SGX,BMI1,HLE,AVX2,SMEP,BMI2,ERMS,INVPCID,RTM,MPX,RDSEED,ADX,SMAP,CLFLUSHOPT,PT,SENSOR,ARAT
> cpu0: 256KB 64b/line 8-way L2 cache
> cpu0: smt 0, core 0, package 0
> mtrr: Pentium Pro MTRR support, 10 var ranges, 88 fixed ranges
> cpu0: apic clock running at 23MHz
> cpu0: mwait min=64, max=64, C-substates=0.2.1.2.4.1, IBE
> cpu1 at mainbus0: apid 2 (application processor)
> cpu1: Intel(R) Xeon(R) CPU E3-1220 v5 @ 3.00GHz, 3300.00 MHz
> cpu1: 
> FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,DS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM,PBE,SSE3,PCLMUL,DTES64,MWAIT,DS-CPL,VMX,SMX,EST,TM2,SSSE3,SDBG,FMA3,CX16,xTPR,PDCM,PCID,SSE4.1,SSE4.2,x2APIC,MOVBE,POPCNT,DEADLINE,AES,XSAVE,AVX,F16C,RDRAND,NXE,PAGE1GB,LONG,LAHF,ABM,3DNOWP,PERF,ITSC,FSGSBASE,SGX,BMI1,HLE,AVX2,SMEP,BMI2,ERMS,INVPCID,RTM,MPX,RDSEED,ADX,SMAP,CLFLUSHOPT,PT,SENSOR,ARAT
> cpu1: 256KB 64b/line 8-way L2 cache
> cpu1: smt 0, core 1, package 0
> cpu2 at mainbus0: apid 4 (application processor)
> cpu2: Intel(R) Xeon(R) CPU E3-1220 v5 @ 3.00GHz, 3300.00 MHz
> cpu2: 
> FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,DS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM,PBE,SSE3,PCLMUL,DTES64,MWAIT,DS-CPL,VMX,SMX,EST,TM2,SSSE3,SDBG,FMA3,CX16,xTPR,PDCM,PCID,SSE4.1,SSE4.2,x2APIC,MOVBE,POPCNT,DEADLINE,AES,XSAVE,AVX,F16C,RDRAND,NXE,PAGE1GB,LONG,LAHF,ABM,3DNOWP,PERF,ITSC,FSGSBASE,SGX,BMI1,HLE,AVX2,SMEP,BMI2,ERMS,INVPCID,RTM,MPX,RDSEED,ADX,SMAP,CLFLUSHOPT,PT,SENSOR,ARAT
> cpu2: 256KB 64b/line 8-way L2 cache
> cpu2: smt 0, core 2, package 0
> cpu3 at mainbus0: apid 6 (application processor)
> cpu3: Intel(R) Xeon(R) CPU E3-1220 v5 @ 3.00GHz, 3300.00 MHz
> cpu3: 
> FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,DS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM,PBE,SSE3,PCLMUL,DTES64,MWAIT,DS-CPL,VMX,SMX,EST,TM2,SSSE3,SDBG,FMA3,CX16,xTPR,PDCM,PCID,SSE4.1,SSE4.2,x2APIC,MOVBE,POPCNT,DEADLINE,AES,XSAVE,AVX,F16C,RDRAND,NXE,PAGE1GB,LONG,LAHF,ABM,3DNOWP,PERF,ITSC,FSGSBASE,SGX,BMI1,HLE,AVX2,SMEP,BMI2,ERMS,INVPCID,RTM,MPX,RDSEED,ADX,SMAP,CLFLUSHOPT,PT,SENSOR,ARAT
> cpu3: 256KB 64b/line 8-way L2 cache
> cpu3: smt 0, core 3, package 0
> ioapic0 at mainbus0: apid 2 pa 0xfec00000, version 20, 120 pins
> acpimcfg0 at acpi0 addr 0xe0000000, bus 0-255
> acpihpet0 at acpi0: 23999999 Hz
> acpiprt0 at acpi0: bus 0 (PCI0)
> acpiprt1 at acpi0: bus 1 (PEG0)
> acpiprt2 at acpi0: bus -1 (PEG1)
> acpiprt3 at acpi0: bus -1 (PEG2)
> acpiprt4 at acpi0: bus -1 (RP17)
> acpiprt5 at acpi0: bus 48 (RP18)
> acpiprt6 at acpi0: bus -1 (RP19)
> acpiprt7 at acpi0: bus 95 (RP20)
> acpiprt8 at acpi0: bus 96 (RP01)
> acpiprt9 at acpi0: bus -1 (RP02)
> acpiprt10 at acpi0: bus -1 (RP03)
> acpiprt11 at acpi0: bus 143 (RP04)
> acpiprt12 at acpi0: bus 144 (RP05)
> acpiprt13 at acpi0: bus -1 (RP06)
> acpiprt14 at acpi0: bus -1 (RP07)
> acpiprt15 at acpi0: bus -1 (RP08)
> acpiprt16 at acpi0: bus 192 (RP09)
> acpiprt17 at acpi0: bus -1 (RP10)
> acpiprt18 at acpi0: bus -1 (RP11)
> acpiprt19 at acpi0: bus -1 (RP12)
> acpiprt20 at acpi0: bus -1 (RP13)
> acpiprt21 at acpi0: bus -1 (RP14)
> acpiprt22 at acpi0: bus -1 (RP15)
> acpiprt23 at acpi0: bus -1 (RP16)
> acpicpu0 at acpi0: C2(350@120 mwait.1@0x20), C1(1000@1 mwait.1), PSS
> acpicpu1 at acpi0: C2(350@120 mwait.1@0x20), C1(1000@1 mwait.1), PSS
> acpicpu2 at acpi0: C2(350@120 mwait.1@0x20), C1(1000@1 mwait.1), PSS
> acpicpu3 at acpi0: C2(350@120 mwait.1@0x20), C1(1000@1 mwait.1), PSS
> acpipwrres0 at acpi0: PG00, resource for PEG0
> acpipwrres1 at acpi0: PG01, resource for PEG1
> acpipwrres2 at acpi0: PG02, resource for PEG2
> "MSFT0001" at acpi0 not configured
> "MSFT0003" at acpi0 not configured
> "PNP0501" at acpi0 not configured
> "PNP0501" at acpi0 not configured
> "INT3F0D" at acpi0 not configured
> "IPI0001" at acpi0 not configured
> "INT345D" at acpi0 not configured
> "PNP0C33" at acpi0 not configured
> acpibtn0 at acpi0: SLPB
> "INT33A1" at acpi0 not configured
> acpibtn1 at acpi0: PWRB
> "ACPI000D" at acpi0 not configured
> ipmi at mainbus0 not configured
> cpu0: Enhanced SpeedStep 3301 MHz: speeds: 3001, 3000, 2800, 2700, 2500, 
> 2400, 2200, 2100, 1900, 1700, 1600, 1400, 1300, 1100, 1000, 800 MHz
> pci0 at mainbus0 bus 0
> pchb0 at pci0 dev 0 function 0 "Intel Xeon E3-1200 v5 Host" rev 0x07
> ppb0 at pci0 dev 1 function 0 "Intel Core 6G PCIE" rev 0x07: msi
> pci1 at ppb0 bus 1
> bge0 at pci1 dev 0 function 0 "Broadcom BCM5718" rev 0x10, BCM5717 B0 
> (0x5717100): msi, address 8c:df:xx:xx:xx:xx
> brgphy0 at bge0 phy 1: BCM5717C 10/100/1000baseT PHY, rev. 0
> xhci0 at pci0 dev 20 function 0 "Intel 100 Series xHCI" rev 0x31: msi
> usb0 at xhci0: USB revision 3.0
> uhub0 at usb0 configuration 1 interface 0 "Intel xHCI root hub" rev 3.00/1.00 
> addr 1
> pchtemp0 at pci0 dev 20 function 2 "Intel 100 Series Thermal" rev 0x31
> "Intel 100 Series MEI" rev 0x31 at pci0 dev 22 function 0 not configured
> "Intel 100 Series MEI" rev 0x31 at pci0 dev 22 function 1 not configured
> ahci0 at pci0 dev 23 function 0 "Intel 100 Series AHCI" rev 0x31: msi, AHCI 
> 1.3.1
> scsibus1 at ahci0: 32 targets
> ppb1 at pci0 dev 27 function 0 "Intel 100 Series PCIE" rev 0xf1: msi
> pci2 at ppb1 bus 48
> ppb2 at pci0 dev 27 function 3 "Intel 100 Series PCIE" rev 0xf1: msi
> pci3 at ppb2 bus 95
> vga1 at pci3 dev 0 function 0 "Matrox MGA G200e" rev 0x00
> wsdisplay0 at vga1 mux 1: console (80x25, vt100 emulation)
> wsdisplay0: screen 1-5 added (80x25, vt100 emulation)
> ppb3 at pci0 dev 28 function 0 "Intel 100 Series PCIE" rev 0xf1: msi
> pci4 at ppb3 bus 96
> ppb4 at pci0 dev 28 function 3 "Intel 100 Series PCIE" rev 0xf1: msi
> pci5 at ppb4 bus 143
> bge1 at pci5 dev 0 function 0 "Broadcom BCM5720" rev 0x00, BCM5720 A0 
> (0x5720000), APE firmware NCSI 1.3.12.0: msi, address 40:8d:xx:xx:xx:xx
> brgphy1 at bge1 phy 1: BCM5720C 10/100/1000baseT PHY, rev. 0
> bge2 at pci5 dev 0 function 1 "Broadcom BCM5720" rev 0x00, BCM5720 A0 
> (0x5720000), APE firmware NCSI 1.3.12.0: msi, address 40:8d:xx:xx:xx:xx
> brgphy2 at bge2 phy 2: BCM5720C 10/100/1000baseT PHY, rev. 0
> ppb5 at pci0 dev 28 function 4 "Intel 100 Series PCIE" rev 0xf1: msi
> pci6 at ppb5 bus 144
> ppb6 at pci0 dev 29 function 0 "Intel 100 Series PCIE" rev 0xf1: msi
> pci7 at ppb6 bus 192
> mfii0 at pci7 dev 0 function 0 "Symbios Logic MegaRAID SAS3008" rev 0x02: msi
> mfii0: "MegaRAID SAS 9341-8i", firmware 24.2.1-0063
> scsibus2 at mfii0: 32 targets
> sd0 at scsibus2 targ 0 lun 0: <LSI, SAS3008-iMR, 4.22> SCSI3 0/direct fixed 
> naa.600605b00b5c56c01f0a3b561ceea3b2
> sd0: 476320MB, 512 bytes/sector, 975503360 sectors
> scsibus3 at mfii0: 256 targets
> pcib0 at pci0 dev 31 function 0 "Intel C236 LPC" rev 0x31
> "Intel 100 Series PMC" rev 0x31 at pci0 dev 31 function 2 not configured
> ichiic0 at pci0 dev 31 function 4 "Intel 100 Series SMBus" rev 0x31: apic 2 
> int 16
> iic0 at ichiic0
> isa0 at pcib0
> isadma0 at isa0
> com0 at isa0 port 0x3f8/8 irq 4: ns16550a, 16 byte fifo
> com1 at isa0 port 0x2f8/8 irq 3: ns16550a, 16 byte fifo
> com1: console
> pckbc0 at isa0 port 0x60/5 irq 1 irq 12
> pcppi0 at isa0 port 0x61
> spkr0 at pcppi0
> vmm0 at mainbus0: VMX/EPT
> uhidev0 at uhub0 port 10 configuration 1 interface 0 "American Megatrends 
> Inc. Virtual Keyboard and Mouse" rev 1.10/1.00 addr 2
> uhidev0: iclass 3/1
> ukbd0 at uhidev0: 8 variable keys, 6 key codes
> wskbd0 at ukbd0: console keyboard, using wsdisplay0
> uhidev1 at uhub0 port 10 configuration 1 interface 1 "American Megatrends 
> Inc. Virtual Keyboard and Mouse" rev 1.10/1.00 addr 2
> uhidev1: iclass 3/1
> ums0 at uhidev1: 3 buttons, Z dir
> wsmouse0 at ums0 mux 0
> vscsi0 at root
> scsibus4 at vscsi0: 256 targets
> softraid0 at root
> scsibus5 at softraid0: 256 targets
> 

Reply via email to