I/O Error Test 6
================
Make sure that if two bcache devices share a cache device,
only one bcache device is offlined in case of I/O errors
in only one backing device.
Original
--------
# uname -rv
4.15.0-55-generic #60-Ubuntu SMP Tue Jul 2 18:22:20 UTC 2019
# ./setup-two-bcache-one-cache.sh >/dev/null 2>&1
[ 20.464988] bcache: register_bdev() registered backing device dm-0
[ 20.474082] bcache: register_bdev() registered backing device dm-1
[ 20.485464] bcache: run_cache_set() invalidating existing data
[ 20.496253] bcache: register_cache() registered cache device dm-2
[ 21.478688] bcache: bch_cached_dev_attach() Caching dm-0 as bcache0 on set
e247271f-eea4-46ea-8ffe-4b04299a7c24
[ 21.484183] bcache: bch_cached_dev_attach() Caching dm-1 as bcache1 on set
e247271f-eea4-46ea-8ffe-4b04299a7c24
# lsblk -e 252
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
loop0 7:0 0 1G 0 loop
└─fake-loop0 253:0 0 1024M 0 dm
└─bcache0 251:0 0 1024M 0 disk
loop1 7:1 0 1G 0 loop
└─fake-loop1 253:1 0 1024M 0 dm
└─bcache1 251:128 0 1024M 0 disk
loop2 7:2 0 1G 0 loop
└─fake-loop2 253:2 0 1024M 0 dm
├─bcache0 251:0 0 1024M 0 disk
└─bcache1 251:128 0 1024M 0 disk
# echo writeback | tee /sys/block/dm-*/bcache/cache_mode
writeback
# cat /sys/block/dm-*/bcache/cache_mode
writethrough [writeback] writearound none
writethrough [writeback] writearound none
# ./dm_fake_dev.sh /dev/loop0 bad
[ 56.142318] Buffer I/O error on dev dm-0, logical block 262128, async page
read
[ 56.151754] Buffer I/O error on dev dm-0, logical block 262128, async page
read
[ 56.158706] bcache: register_bcache() error /dev/dm-0: device already
registered (emitting change event)
[ 56.161631] Buffer I/O error on dev bcache0, logical block 262112, async
page read
[ 56.169723] Buffer I/O error on dev bcache0, logical block 262112, async
page read
[ 56.172303] Buffer I/O error on dev bcache0, logical block 1, async page read
root@guest-bcache:~# dd if=/dev/zero of=/dev/bcache1 bs=4k & dd if=/dev/zero
of=/dev/bcache0 bs=4k &
[1] 1400
[2] 1401
# [ 85.400054] Buffer I/O error on dev bcache0, logical block 0, lost async
page write
[ 85.403138] Buffer I/O error on dev bcache0, logical block 1, lost async
page write
[ 85.417864] Buffer I/O error on dev bcache0, logical block 2, lost async
page write
[ 85.419699] Buffer I/O error on dev bcache0, logical block 3, lost async
page write
[ 85.421510] Buffer I/O error on dev bcache0, logical block 4, lost async
page write
[ 85.423301] Buffer I/O error on dev bcache0, logical block 5, lost async
page write
[ 85.450407] Buffer I/O error on dev bcache0, logical block 6, lost async
page write
[ 85.452248] Buffer I/O error on dev bcache0, logical block 7, lost async
page write
[ 85.453980] Buffer I/O error on dev bcache0, logical block 8, lost async
page write
[ 85.455722] Buffer I/O error on dev bcache0, logical block 9, lost async
page write
dd: error writing '/dev/bcache0': No space left on device
dd: error writing '/dev/bcache1': No space left on device
262142+0 records in
262141+0 records out
1073729536 bytes (1.1 GB, 1.0 GiB) copied, 4.23916 s, 253 MB/s
[2]+ Exit 1 dd if=/dev/zero of=/dev/bcache0 bs=4k
# 262142+0 records in
262141+0 records out
1073729536 bytes (1.1 GB, 1.0 GiB) copied, 6.5355 s, 164 MB/s
[1]+ Exit 1 dd if=/dev/zero of=/dev/bcache1 bs=4k
# lsblk -e 252
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
loop0 7:0 0 1G 0 loop
loop1 7:1 0 1G 0 loop
└─fake-loop1 253:1 0 1024M 0 dm
└─bcache1 251:128 0 1024M 0 disk
loop2 7:2 0 1G 0 loop
└─fake-loop2 253:2 0 1024M 0 dm
├─bcache0 251:0 0 1024M 0 disk
└─bcache1 251:128 0 1024M 0 disk
fake-loop0 253:0 0 1G 0 dm
└─bcache0 251:0 0 1024M 0 disk
Nothing is removed, since no I/O errors are detected.
Modified
--------
# uname -rv
4.15.0-55-generic #60+test20190703build1bcache1-Ubuntu SMP Wed Jul 3 21:41:37
UTC
# ./setup-two-bcache-one-cache.sh >/dev/null 2>&1
[ 29.341483] bcache: register_bdev() registered backing device dm-0
[ 29.351963] bcache: run_cache_set() invalidating existing data
[ 29.365256] bcache: register_cache() registered cache device dm-2
[ 29.365566] bcache: register_bdev() registered backing device dm-1
[ 30.357267] bcache: bch_cached_dev_attach() Caching dm-0 as bcache0 on set
ee57b847-da70-4b65-888b-e5795ecf6c46
[ 30.363117] bcache: bch_cached_dev_attach() Caching dm-1 as bcache1 on set
ee57b847-da70-4b65-888b-e5795ecf6c46
# lsblk -e 252
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
loop0 7:0 0 1G 0 loop
└─fake-loop0 253:0 0 1024M 0 dm
└─bcache0 251:0 0 1024M 0 disk
loop1 7:1 0 1G 0 loop
└─fake-loop1 253:1 0 1024M 0 dm
└─bcache1 251:128 0 1024M 0 disk
loop2 7:2 0 1G 0 loop
└─fake-loop2 253:2 0 1024M 0 dm
├─bcache0 251:0 0 1024M 0 disk
└─bcache1 251:128 0 1024M 0 disk
# echo writeback | tee /sys/block/dm-*/bcache/cache_mode
writeback
# cat /sys/block/dm-*/bcache/cache_mode
writethrough [writeback] writearound none
writethrough [writeback] writearound none
# ./dm_fake_dev.sh /dev/loop0 bad
[ 63.680438] Buffer I/O error on dev dm-0, logical block 262128, async page
read
[ 63.686730] Buffer I/O error on dev dm-0, logical block 262128, async page
read
[ 63.693292] bcache: register_bcache() error /dev/dm-0: device already
registered (emitting change event)
[ 63.695221] bcache: bch_count_backing_io_errors() dm-0: IO error on backing
device, unrecoverable
[ 63.695248] bcache: bch_count_backing_io_errors() dm-0: IO error on backing
device, unrecoverable
[ 63.695253] Buffer I/O error on dev bcache0, logical block 262112, async
page read
[ 63.697374] bcache: bch_count_backing_io_errors() dm-0: IO error on backing
device, unrecoverable
[ 63.697389] Buffer I/O error on dev bcache0, logical block 262112, async
page read
[ 63.697414] bcache: bch_count_backing_io_errors() dm-0: IO error on backing
device, unrecoverable
[ 63.697430] bcache: bch_count_backing_io_errors() dm-0: IO error on backing
device, unrecoverable
[ 63.697433] Buffer I/O error on dev bcache0, logical block 1, async page read
root@guest-bcache:~# dd if=/dev/zero of=/dev/bcache1 bs=4k & dd if=/dev/zero
of=/dev/bcache0 bs=4k &
[1] 1391
[2] 1392
# [ 75.160405] bcache: bch_count_backing_io_errors() dm-0: IO error on
backing device, unrecoverable
[ 75.200618] Buffer I/O error on dev bcache0, logical block 0, lost async
page write
[ 75.204141] bcache: bch_count_backing_io_errors() dm-0: IO error on backing
device, unrecoverable
[ 75.207546] Buffer I/O error on dev bcache0, logical block 1, lost async
page write
[ 75.210500] bcache: bch_count_backing_io_errors() dm-0: IO error on backing
device, unrecoverable
[ 75.212927] Buffer I/O error on dev bcache0, logical block 2, lost async
page write
[ 75.214696] bcache: bch_count_backing_io_errors() dm-0: IO error on backing
device, unrecoverable
[ 75.216750] Buffer I/O error on dev bcache0, logical block 3, lost async
page write
[ 75.218637] bcache: bch_count_backing_io_errors() dm-0: IO error on backing
device, unrecoverable
[ 75.220841] Buffer I/O error on dev bcache0, logical block 4, lost async
page write
[ 75.222737] bcache: bch_count_backing_io_errors() dm-0: IO error on backing
device, unrecoverable
...
[ 75.397159] bcache: bch_count_backing_io_errors() dm-0: IO error on backing
device, unrecoverable
[ 75.399609] bcache: bch_cached_dev_error() stop bcache0: too many IO errors
on backing device dm-0
[ 75.399609]
dd: error writing '/dev/bcache0': No space left on device
262142+0 records in[ 76.558623] bcache: bcache_device_free() bcache0 stopped
262141+0 records out
1073729536 bytes (1.1 GB, 1.0 GiB) copied, 2.15178 s, 499 MB/s
dd: error writing '/dev/bcache1': No space left on device
[2]+ Exit 1 dd if=/dev/zero of=/dev/bcache0 bs=4k
# 262142+0 records in
262141+0 records out
1073729536 bytes (1.1 GB, 1.0 GiB) copied, 4.65434 s, 231 MB/s
[1]+ Exit 1 dd if=/dev/zero of=/dev/bcache1 bs=4k
#
# lsblk -e 252
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
loop0 7:0 0 1G 0 loop
loop1 7:1 0 1G 0 loop
└─fake-loop1 253:1 0 1024M 0 dm
└─bcache1 251:128 0 1024M 0 disk
loop2 7:2 0 1G 0 loop
└─fake-loop2 253:2 0 1024M 0 dm
└─bcache1 251:128 0 1024M 0 disk
fake-loop0 253:0 0 1G 0 dm
Only bcache0 was stopped.
Note bcache1 remains working.
# reboot
# ./setup-two-bcache-one-cache.reboot.sh >/dev/null 2>&1
[ 40.814007] bcache: register_bdev() registered backing device dm-0
[ 40.890096] bcache: register_bdev() registered backing device dm-1
[ 41.007100] bcache: bch_journal_replay() journal replay done, 63545 keys in
37 entries, seq 148
[ 41.012823] bcache: bch_cached_dev_attach() Caching dm-1 as bcache1 on set
ee57b847-da70-4b65-888b-e5795ecf6c46
[ 41.021751] bcache: bch_cached_dev_attach() Caching dm-0 as bcache0 on set
ee57b847-da70-4b65-888b-e5795ecf6c46
[ 41.025195] bcache: register_cache() registered cache device dm-2
# lsblk -e 252
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
loop0 7:0 0 1G 0 loop
└─fake-loop0 253:0 0 1024M 0 dm
└─bcache0 251:0 0 1024M 0 disk
loop1 7:1 0 1G 0 loop
└─fake-loop1 253:1 0 1024M 0 dm
└─bcache1 251:128 0 1024M 0 disk
loop2 7:2 0 1G 0 loop
└─fake-loop2 253:2 0 1024M 0 dm
├─bcache0 251:0 0 1024M 0 disk
└─bcache1 251:128 0 1024M 0 disk
#
After reboot, the bcache devices were reattached successfully.
--
You received this bug notification because you are a member of Ubuntu
Bugs, which is subscribed to Ubuntu.
https://bugs.launchpad.net/bugs/1829563
Title:
bcache: risk of data loss on I/O errors in backing or caching devices
To manage notifications about this bug go to:
https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1829563/+subscriptions
--
ubuntu-bugs mailing list
[email protected]
https://lists.ubuntu.com/mailman/listinfo/ubuntu-bugs
