Hi all,
I have a bad bad problem with our brand new server!
The lengthy details are below but to cut the story short, on the same
hardware (3 x LSI 9240-8i, 20 x 3TB 6gb HDDs) I am getting ZFS
sequential writes of 1.4GB/s on Solaris 10 (20 disks, 10 mirrors) and
only 200-240MB/s on latest Solaris 11.11 (same zpool config). By
writing directly to raw disks I found that in S10 the speed is 140MB/s
sequential writes per disk (consistent with combined 1.4GB/s for my
zpool) whereas only 24MB/s in Solaris 11 (consistent with 240MB/s
zpool, 10 mirrors 24MB/s each).
This must be the controller drivers, right? I downloaded drivers
version 4.7 off LSI site (says "for Solaris 10 and later") - they
failed to attach on S11. Version 3.03 worked but the system would
randomly crash, so I moved my experiments off S11 to S10. However, S10
has only the old implementation if iSCSI which gives me other problems
so I decided to give S11 another go.
Would there be any advice in this community?
Many thanks!
Roman
==============
root@carbon:~# echo | format | grep Hitachi
1. c5t8d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
2. c5t9d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
3. c5t10d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
4. c5t11d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
5. c5t13d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
6. c5t14d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
7. c5t15d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
9. c6t9d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
10. c6t10d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
11. c6t11d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
12. c6t13d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
13. c6t14d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
14. c6t15d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
15. c7t8d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
17. c7t10d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
18. c7t11d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
19. c7t12d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
20. c7t13d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
21. c7t14d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
22. c7t15d1 <ATA-Hitachi HUA72303-A5C0-2.73TB>
Reading DD from all disks:
(dd of=/dev/null bs=1024kb if=/dev/rdsk/c7t9d1 &)
# Iostat –xznM 2
extended device statistics
r/s w/s Mr/s Mw/s wait actv wsvc_t asvc_t %w %b device
614.5 0.0 153.6 0.0 0.0 1.0 0.0 1.6 0 98 c5t8d1
595.5 0.0 148.9 0.0 0.0 1.0 0.0 1.7 0 99 c7t8d1
1566.5 0.0 391.6 0.0 0.0 1.0 0.0 0.6 1 96 c6t8d1 # (SSD)
618.5 0.0 154.6 0.0 0.0 1.0 0.0 1.6 0 99 c6t9d1
616.5 0.0 154.1 0.0 0.0 1.0 0.0 1.6 0 99 c5t9d1
1564.0 0.0 391.0 0.0 0.0 1.0 0.0 0.6 1 96 c7t9d1# (SSD)
616.0 0.0 154.0 0.0 0.0 1.0 0.0 1.6 0 98 c7t10d1
554.0 0.0 138.5 0.0 0.0 1.0 0.0 1.8 0 99 c6t10d1
598.5 0.0 149.6 0.0 0.0 1.0 0.0 1.7 0 99 c5t10d1
588.5 0.0 147.1 0.0 0.0 1.0 0.0 1.7 0 98 c6t11d1
590.5 0.0 147.6 0.0 0.0 1.0 0.0 1.7 0 98 c7t11d1
591.5 0.0 147.9 0.0 0.0 1.0 0.0 1.7 0 99 c5t11d1
600.5 0.0 150.1 0.0 0.0 1.0 0.0 1.6 0 98 c6t13d1
617.5 0.0 154.4 0.0 0.0 1.0 0.0 1.6 0 99 c7t12d1
611.0 0.0 152.8 0.0 0.0 1.0 0.0 1.6 0 99 c5t13d1
625.0 0.0 156.3 0.0 0.0 1.0 0.0 1.6 0 99 c6t14d1
592.5 0.0 148.1 0.0 0.0 1.0 0.0 1.7 0 99 c7t13d1
596.0 0.0 149.0 0.0 0.0 1.0 0.0 1.7 0 99 c5t14d1
598.5 0.0 149.6 0.0 0.0 1.0 0.0 1.6 0 98 c6t15d1
618.5 0.0 154.6 0.0 0.0 1.0 0.0 1.6 0 98 c7t14d1
606.5 0.0 151.6 0.0 0.0 1.0 0.0 1.6 0 98 c5t15d1
625.0 0.0 156.3 0.0 0.0 1.0 0.0 1.6 0 98 c7t15d1
extended device statistics
r/s w/s Mr/s Mw/s wait actv wsvc_t asvc_t %w %b device
620.5 0.0 155.1 0.0 0.0 1.0 0.0 1.6 0 99 c5t8d1
620.5 0.0 155.1 0.0 0.0 1.0 0.0 1.6 0 99 c7t8d1
1581.0 0.0 395.2 0.0 0.0 1.0 0.0 0.6 1 96 c6t8d1
611.5 0.0 152.9 0.0 0.0 1.0 0.0 1.6 0 99 c6t9d1
587.5 0.0 146.9 0.0 0.0 1.0 0.0 1.7 0 99 c5t9d1
1580.0 0.0 395.0 0.0 0.0 1.0 0.0 0.6 1 97 c7t9d1
593.0 0.0 148.2 0.0 0.0 1.0 0.0 1.7 0 99 c7t10d1
616.0 0.0 154.0 0.0 0.0 1.0 0.0 1.6 0 99 c6t10d1
601.0 0.0 150.2 0.0 0.0 1.0 0.0 1.6 0 99 c5t10d1
587.0 0.0 146.7 0.0 0.0 1.0 0.0 1.7 0 99 c6t11d1
578.5 0.0 144.6 0.0 0.0 1.0 0.0 1.7 0 99 c7t11d1
624.5 0.0 156.1 0.0 0.0 1.0 0.0 1.6 0 99 c5t11d1
604.5 0.0 151.1 0.0 0.0 1.0 0.0 1.6 0 99 c6t13d1
573.5 0.0 143.4 0.0 0.0 1.0 0.0 1.7 0 99 c7t12d1
609.0 0.0 152.2 0.0 0.0 1.0 0.0 1.6 0 99 c5t13d1
630.5 0.0 157.6 0.0 0.0 1.0 0.0 1.6 0 99 c6t14d1
618.5 0.0 154.6 0.0 0.0 1.0 0.0 1.6 0 99 c7t13d1
633.5 0.0 158.4 0.0 0.0 1.0 0.0 1.6 0 99 c5t14d1
602.5 0.0 150.6 0.0 0.0 1.0 0.0 1.6 0 99 c6t15d1
589.5 0.0 147.4 0.0 0.0 1.0 0.0 1.7 0 99 c7t14d1
603.0 0.0 150.7 0.0 0.0 1.0 0.0 1.6 0 99 c5t15d1
586.0 0.0 146.5 0.0 0.0 1.0 0.0 1.7 0 99 c7t15d1
WRITING ZEROS TO RAW DISKS
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c6t9d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c5t9d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c7t8d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c6t10d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c7t10d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c5t10d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c6t11d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c5t11d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c7t11d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c6t13d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c7t12d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c5t13d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c6t14d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c5t14d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c7t13d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c6t15d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c7t14d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c5t15d1 &
root@carbon:~# dd if=/dev/zero bs=1024kb of=/dev/rdsk/c7t15d1 &
extended device statistics
r/s w/s Mr/s Mw/s wait actv wsvc_t asvc_t %w %b device
0.0 99.5 0.0 24.9 0.0 1.0 0.0 10.0 0 100 c5t8d1
0.0 97.5 0.0 24.4 0.0 1.0 0.0 9.9 0 96 c7t8d1
0.0 98.0 0.0 24.5 0.0 1.0 0.0 10.2 0 100 c6t9d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c5t9d1
0.0 99.5 0.0 24.9 0.0 1.0 0.0 10.0 0 100 c7t10d1
0.0 99.0 0.0 24.8 0.0 1.0 0.0 10.1 0 100 c6t10d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c5t10d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c6t11d1
0.0 99.0 0.0 24.8 0.0 1.0 0.0 10.1 0 100 c7t11d1
0.0 97.5 0.0 24.4 0.0 1.0 0.0 10.2 0 100 c5t11d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c6t13d1
0.0 98.5 0.0 24.6 0.0 1.0 0.0 10.1 0 100 c7t12d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c5t13d1
0.0 101.0 0.0 25.2 0.0 1.0 0.0 9.9 0 100 c6t14d1
0.0 101.5 0.0 25.4 0.0 1.0 0.0 9.8 0 100 c7t13d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c5t14d1
0.0 99.5 0.0 24.9 0.0 1.0 0.0 10.0 0 100 c6t15d1
0.0 99.0 0.0 24.7 0.0 1.0 0.0 10.1 0 100 c7t14d1
0.0 100.5 0.0 25.1 0.0 1.0 0.0 9.9 0 100 c5t15d1
0.0 101.0 0.0 25.2 0.0 1.0 0.0 9.9 0 100 c7t15d1
extended device statistics
r/s w/s Mr/s Mw/s wait actv wsvc_t asvc_t %w %b device
0.0 100.5 0.0 25.1 0.0 1.0 0.0 9.9 0 100 c5t8d1
0.0 101.0 0.0 25.3 0.0 1.0 0.0 9.9 0 100 c7t8d1
0.0 98.0 0.0 24.5 0.0 1.0 0.0 10.2 0 100 c6t9d1
0.0 99.5 0.0 24.9 0.0 1.0 0.0 10.0 0 100 c5t9d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c7t10d1
0.0 99.5 0.0 24.9 0.0 1.0 0.0 10.0 0 100 c6t10d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c5t10d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c6t11d1
0.0 99.5 0.0 24.9 0.0 1.0 0.0 10.0 0 100 c7t11d1
0.0 94.5 0.0 23.6 0.0 1.0 0.0 10.2 0 96 c5t11d1
0.0 99.5 0.0 24.9 0.0 1.0 0.0 10.0 0 100 c6t13d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c7t12d1
0.0 99.5 0.0 24.9 0.0 1.0 0.0 10.0 0 100 c5t13d1
0.0 97.0 0.0 24.3 0.0 1.0 0.0 9.9 0 96 c6t14d1
0.0 100.5 0.0 25.1 0.0 1.0 0.0 9.9 0 100 c7t13d1
0.0 99.5 0.0 24.9 0.0 1.0 0.0 10.0 0 100 c5t14d1
0.0 100.5 0.0 25.1 0.0 1.0 0.0 9.9 0 100 c6t15d1
0.0 99.0 0.0 24.8 0.0 1.0 0.0 10.1 0 100 c7t14d1
0.0 96.5 0.0 24.1 0.0 1.0 0.0 10.0 0 96 c5t15d1
0.0 100.5 0.0 25.1 0.0 1.0 0.0 9.9 0 100 c7t15d1
extended device statistics
r/s w/s Mr/s Mw/s wait actv wsvc_t asvc_t %w %b device
0.0 101.5 0.0 25.4 0.0 1.0 0.0 9.8 0 100 c5t8d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c7t8d1
0.0 97.5 0.0 24.4 0.0 1.0 0.0 10.2 0 100 c6t9d1
0.0 100.5 0.0 25.1 0.0 1.0 0.0 9.9 0 100 c5t9d1
0.0 100.5 0.0 25.1 0.0 1.0 0.0 9.9 0 100 c7t10d1
0.0 101.5 0.0 25.4 0.0 1.0 0.0 9.8 0 100 c6t10d1
0.0 99.5 0.0 24.9 0.0 1.0 0.0 10.0 0 100 c5t10d1
0.0 100.5 0.0 25.1 0.0 1.0 0.0 9.9 0 100 c6t11d1
0.0 98.5 0.0 24.6 0.0 1.0 0.0 10.1 0 100 c7t11d1
0.0 98.0 0.0 24.5 0.0 1.0 0.0 10.2 0 100 c5t11d1
0.0 101.0 0.0 25.3 0.0 1.0 0.0 9.9 0 100 c6t13d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c7t12d1
0.0 101.0 0.0 25.3 0.0 1.0 0.0 9.9 0 100 c5t13d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c6t14d1
0.0 100.5 0.0 25.1 0.0 1.0 0.0 9.9 0 100 c7t13d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c5t14d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c6t15d1
0.0 99.5 0.0 24.9 0.0 1.0 0.0 10.0 0 100 c7t14d1
0.0 100.5 0.0 25.1 0.0 1.0 0.0 9.9 0 100 c5t15d1
0.0 100.0 0.0 25.0 0.0 1.0 0.0 10.0 0 100 c7t15d1
**************
Only 25MB/s!!!
**************
Testing ZPOOL
zpool create data mirror
zpool create data \
mirror c5t8d1 c6t9d1 mirror c5t9d1 c7t8d1 mirror c6t10d1 c7t10d1 \
mirror c5t10d1 c6t11d1 mirror c5t11d1 c7t11d1 mirror c6t13d1 c7t12d1 \
mirror c5t13d1 c6t14d1 mirror c5t14d1 c7t13d1 mirror c6t15d1 c7t14d1 \
mirror c5t15d1 c7t15d1
root@carbon:~#
root@carbon:~# zpool status data
pool: data
state: ONLINE
scan: none requested
config:
NAME STATE READ WRITE CKSUM
data ONLINE 0 0 0
mirror-0 ONLINE 0 0 0
c5t8d1 ONLINE 0 0 0
c6t9d1 ONLINE 0 0 0
mirror-1 ONLINE 0 0 0
c5t9d1 ONLINE 0 0 0
c7t8d1 ONLINE 0 0 0
mirror-2 ONLINE 0 0 0
c6t10d1 ONLINE 0 0 0
c7t10d1 ONLINE 0 0 0
mirror-3 ONLINE 0 0 0
c5t10d1 ONLINE 0 0 0
c6t11d1 ONLINE 0 0 0
mirror-4 ONLINE 0 0 0
c5t11d1 ONLINE 0 0 0
c7t11d1 ONLINE 0 0 0
mirror-5 ONLINE 0 0 0
c6t13d1 ONLINE 0 0 0
c7t12d1 ONLINE 0 0 0
mirror-6 ONLINE 0 0 0
c5t13d1 ONLINE 0 0 0
c6t14d1 ONLINE 0 0 0
mirror-7 ONLINE 0 0 0
c5t14d1 ONLINE 0 0 0
c7t13d1 ONLINE 0 0 0
mirror-8 ONLINE 0 0 0
c6t15d1 ONLINE 0 0 0
c7t14d1 ONLINE 0 0 0
mirror-9 ONLINE 0 0 0
c5t15d1 ONLINE 0 0 0
c7t15d1 ONLINE 0 0 0
errors: No known data errors
root@carbon:~#
==========================================================
Writing to file
==========================================================
root@carbon:~# dd if=/dev/zero of=/data/testfile bs=1024kb
root@carbon:~# dd if=/dev/zero of=/data/testfile bs=128kb
data 3.93M 27.2T 0 1.46K 102 179M
data 3.93M 27.2T 0 263 0 30.1M
data 1.02G 27.2T 0 456 0 56.3M
data 2.04G 27.2T 0 2.11K 0 263M
data 3.02G 27.2T 0 1.62K 0 202M
data 3.97G 27.2T 0 1.38K 0 171M
data 4.90G 27.2T 0 1.96K 0 245M
data 5.80G 27.2T 0 826 0 99.5M
data 5.80G 27.2T 0 755 0 92.9M
data 6.67G 27.2T 0 992 0 120M
data 7.53G 27.2T 0 1.49K 0 185M
data 8.36G 27.2T 0 1.13K 0 141M
********
250MB/s consistent with ten mirrors 25MB/s each (see individual disk
write test above).
********
#############################################################
SCSI CARD – LSI 9240-8i
SCSI CARD DRIVER (imraid_sas)
Solaris: SunOS carbon 5.11 11.0 i86pc i386 i86pc
#############################################################
root@carbon:~# grep 9240 /etc/path_to_inst
"/pci@0,0/pci15ad,7a0@15/pci1000,9240@0" 0 "imraid_sas"
"/pci@0,0/pci15ad,7a0@15/pci1000,9240@0/sd@8,1" 2 "sd"
"/pci@0,0/pci15ad,7a0@15/pci1000,9240@0/sd@9,1" 6 "sd"
"/pci@0,0/pci15ad,7a0@15/pci1000,9240@0/sd@a,1" 10 "sd"
"/pci@0,0/pci15ad,7a0@15/pci1000,9240@0/sd@b,1" 13 "sd"
"/pci@0,0/pci15ad,7a0@15/pci1000,9240@0/sd@d,1" 16 "sd"
"/pci@0,0/pci15ad,7a0@15/pci1000,9240@0/sd@e,1" 19 "sd"
"/pci@0,0/pci15ad,7a0@15/pci1000,9240@0/sd@f,1" 22 "sd"
"/pci@0,0/pci15ad,7a0@16/pci1000,9240@0" 1 "imraid_sas"
"/pci@0,0/pci15ad,7a0@16/pci1000,9240@0/sd@8,1" 4 "sd"
"/pci@0,0/pci15ad,7a0@16/pci1000,9240@0/sd@9,1" 5 "sd"
"/pci@0,0/pci15ad,7a0@16/pci1000,9240@0/sd@a,1" 9 "sd"
"/pci@0,0/pci15ad,7a0@16/pci1000,9240@0/sd@b,1" 11 "sd"
"/pci@0,0/pci15ad,7a0@16/pci1000,9240@0/sd@d,1" 14 "sd"
"/pci@0,0/pci15ad,7a0@16/pci1000,9240@0/sd@e,1" 17 "sd"
"/pci@0,0/pci15ad,7a0@16/pci1000,9240@0/sd@f,1" 20 "sd"
"/pci@0,0/pci15ad,7a0@17/pci1000,9240@0" 2 "imraid_sas"
"/pci@0,0/pci15ad,7a0@17/pci1000,9240@0/sd@8,1" 3 "sd"
"/pci@0,0/pci15ad,7a0@17/pci1000,9240@0/sd@9,1" 7 "sd"
"/pci@0,0/pci15ad,7a0@17/pci1000,9240@0/sd@a,1" 8 "sd"
"/pci@0,0/pci15ad,7a0@17/pci1000,9240@0/sd@b,1" 12 "sd"
"/pci@0,0/pci15ad,7a0@17/pci1000,9240@0/sd@c,1" 15 "sd"
"/pci@0,0/pci15ad,7a0@17/pci1000,9240@0/sd@d,1" 18 "sd"
"/pci@0,0/pci15ad,7a0@17/pci1000,9240@0/sd@e,1" 21 "sd"
"/pci@0,0/pci15ad,7a0@17/pci1000,9240@0/sd@f,1" 23 "sd"
root@carbon:~# grep imraid /etc/driver_aliases
imraid_sas "pciex1000,73"
#############################################################
_______________________________________________
storage-discuss mailing list
storage-discuss@opensolaris.org
http://mail.opensolaris.org/mailman/listinfo/storage-discuss
