[PATCH v2] fstests: btrfs/149 make it sectorsize independent

2018-09-09 Thread Anand Jain 
Originally this test case was designed to work with only 4K sectorsize.
Now enhance it to work with any sector sizes and makes the following
changes:
Output file not to contain any traces of sector size.
Use max_inline=0 mount option so that it meets the requisite of non inline
regular extent.
Don't log the md5sum results to the output file as the data size vary by
the sectorsize.

Signed-off-by: Anand Jain 
---
v1->v2: rename _scratch_sectorsize() to _scratch_btrfs_sectorsize()
add _require_btrfs_command inspect-internal dump-super

 common/btrfs|  7 +++
 common/filter   |  5 +
 tests/btrfs/149 | 30 +-
 tests/btrfs/149.out | 12 ++--
 4 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/common/btrfs b/common/btrfs
index 79c687f73376..26dc0bb9600f 100644
--- a/common/btrfs
+++ b/common/btrfs
@@ -367,3 +367,10 @@ _run_btrfs_balance_start()
 
run_check $BTRFS_UTIL_PROG balance start $bal_opt $*
 }
+
+#return the sector size of the btrfs scratch fs
+_scratch_btrfs_sectorsize()
+{
+   $BTRFS_UTIL_PROG inspect-internal dump-super $SCRATCH_DEV |\
+   grep sectorsize | awk '{print $2}'
+}
diff --git a/common/filter b/common/filter
index 3965c2eb752b..e87740ddda3f 100644
--- a/common/filter
+++ b/common/filter
@@ -271,6 +271,11 @@ _filter_xfs_io_pages_modified()
_filter_xfs_io_units_modified "Page" $PAGE_SIZE
 }
 
+_filter_xfs_io_numbers()
+{
+_filter_xfs_io | sed -E 's/[0-9]+//g'
+}
+
 _filter_test_dir()
 {
# TEST_DEV may be a prefix of TEST_DIR (e.g. /mnt, /mnt/ovl-mnt)
diff --git a/tests/btrfs/149 b/tests/btrfs/149
index 3e955a305e0f..9b0f00b0ed81 100755
--- a/tests/btrfs/149
+++ b/tests/btrfs/149
@@ -36,6 +36,7 @@ _require_test
 _require_scratch
 _require_scratch_reflink
 _require_odirect
+_require_btrfs_command inspect-internal dump-super
 
 send_files_dir=$TEST_DIR/btrfs-test-$seq
 
@@ -44,21 +45,27 @@ rm -fr $send_files_dir
 mkdir $send_files_dir
 
 _scratch_mkfs >>$seqres.full 2>&1
-_scratch_mount "-o compress"
+# On 64K pagesize systems the compression is more efficient, so max_inline
+# helps to create regular (non inline) extent irrespective of the final
+# write size.
+_scratch_mount "-o compress -o max_inline=0"
 
 # Write to our file using direct IO, so that this way the write ends up not
 # getting compressed, that is, we get a regular extent which is neither
 # inlined nor compressed.
 # Alternatively, we could have mounted the fs without compression enabled,
 # which would result as well in an uncompressed regular extent.
-$XFS_IO_PROG -f -d -c "pwrite -S 0xab 0 4K" $SCRATCH_MNT/foobar | 
_filter_xfs_io
+sectorsize=$(_scratch_btrfs_sectorsize)
+$XFS_IO_PROG -f -d -c "pwrite -S 0xab 0 $sectorsize" $SCRATCH_MNT/foobar |\
+   _filter_xfs_io_numbers
 
 $BTRFS_UTIL_PROG subvolume snapshot -r $SCRATCH_MNT \
$SCRATCH_MNT/mysnap1 > /dev/null
 
 # Clone the regular (not inlined) extent.
-$XFS_IO_PROG -c "reflink $SCRATCH_MNT/foobar 0 8K 4K" $SCRATCH_MNT/foobar \
-   | _filter_xfs_io
+$XFS_IO_PROG -c \
+   "reflink $SCRATCH_MNT/foobar 0 $((2 * $sectorsize)) $sectorsize" \
+   $SCRATCH_MNT/foobar | _filter_xfs_io_numbers
 
 $BTRFS_UTIL_PROG subvolume snapshot -r $SCRATCH_MNT \
$SCRATCH_MNT/mysnap2 > /dev/null
@@ -76,21 +83,26 @@ $BTRFS_UTIL_PROG send -p $SCRATCH_MNT/mysnap1 -f 
$send_files_dir/2.snap \
 $SCRATCH_MNT/mysnap2 2>&1 >/dev/null | _filter_scratch
 
 echo "File digests in the original filesystem:"
-md5sum $SCRATCH_MNT/mysnap1/foobar | _filter_scratch
-md5sum $SCRATCH_MNT/mysnap2/foobar | _filter_scratch
+sum_src_snap1=$(md5sum $SCRATCH_MNT/mysnap1/foobar | awk '{print $1}')
+sum_src_snap2=$(md5sum $SCRATCH_MNT/mysnap2/foobar | awk '{print $1}')
+echo "src checksum created"
 
 # Now recreate the filesystem by receiving both send streams and verify we get
 # the same file content that the original filesystem had.
 _scratch_unmount
 _scratch_mkfs >>$seqres.full 2>&1
-_scratch_mount "-o compress"
+_scratch_mount "-o compress,max_inline=0"
 
 $BTRFS_UTIL_PROG receive -f $send_files_dir/1.snap $SCRATCH_MNT > /dev/null
 $BTRFS_UTIL_PROG receive -f $send_files_dir/2.snap $SCRATCH_MNT > /dev/null
 
 echo "File digests in the new filesystem:"
-md5sum $SCRATCH_MNT/mysnap1/foobar | _filter_scratch
-md5sum $SCRATCH_MNT/mysnap2/foobar | _filter_scratch
+sum_dest_snap1=$(md5sum $SCRATCH_MNT/mysnap1/foobar | awk '{print $1}')
+sum_dest_snap2=$(md5sum $SCRATCH_MNT/mysnap2/foobar | awk '{print $1}')
+echo "dest checksum created"
+
+[[ $sum_src_snap1 == $sum_dest_snap1 ]] && echo "src and dest checksum matched"
+[[ $sum_src_snap2 == $sum_dest_snap2 ]] && echo "src and dest checksum matched"
 
 status=0
 exit
diff --git a/tests/btrfs/149.out b/tests/btrfs/149.out
index 303de928d35a..6ba251799ff2 100644
--- a/tests/btrfs/149.out
+++ b/tests/btrfs/149.out
@@ -1,14 +1,14 @@
 QA output created by 149
-wrote 4096/4096 bytes at offset 0
+wrote

Re: [PATCH] fstests: btrfs/149 make it sectorsize independent

2018-09-09 Thread anand . jain 




On 09/09/2018 11:24 PM, Eryu Guan wrote:

On Wed, Sep 05, 2018 at 02:35:12PM +0800, Anand Jain wrote:

Originally this test case was designed to work with only 4K sectorsize.
Now enhance it to work with any sector sizes and makes the following
changes:
Output file not to contain any traces of sector size.
Use max_inline=0 mount option so that it meets the requisite of non inline
regular extent.
Don't log the md5sum results to the output file as the data size vary by
the sectorsize.

Signed-off-by: Anand Jain 
---
  common/btrfs|  7 +++
  common/filter   |  5 +
  tests/btrfs/149 | 29 -
  tests/btrfs/149.out | 12 ++--
  4 files changed, 38 insertions(+), 15 deletions(-)

diff --git a/common/btrfs b/common/btrfs
index 79c687f73376..e6a218d6b63a 100644
--- a/common/btrfs
+++ b/common/btrfs
@@ -367,3 +367,10 @@ _run_btrfs_balance_start()
  
  	run_check $BTRFS_UTIL_PROG balance start $bal_opt $*

  }
+
+#return the sector size of the btrfs scratch fs
+_scratch_sectorsize()


The function name would indicate that it's a general helper function and
works for all filesystems, but it only works for btrfs. I'd be better to
name it as "_scratch_btrfs_sectorsize" or similar.


 Ah. Will rename.


+{
+   $BTRFS_UTIL_PROG inspect-internal dump-super $SCRATCH_DEV |\
+   grep sectorsize | awk '{print $2}'
+}
diff --git a/common/filter b/common/filter
index 3965c2eb752b..e87740ddda3f 100644
--- a/common/filter
+++ b/common/filter
@@ -271,6 +271,11 @@ _filter_xfs_io_pages_modified()
_filter_xfs_io_units_modified "Page" $PAGE_SIZE
  }
  
+_filter_xfs_io_numbers()

+{
+_filter_xfs_io | sed -E 's/[0-9]+//g'
+}
+
  _filter_test_dir()
  {
# TEST_DEV may be a prefix of TEST_DIR (e.g. /mnt, /mnt/ovl-mnt)
diff --git a/tests/btrfs/149 b/tests/btrfs/149
index 3e955a305e0f..3958fa844c8b 100755
--- a/tests/btrfs/149
+++ b/tests/btrfs/149
@@ -44,21 +44,27 @@ rm -fr $send_files_dir
  mkdir $send_files_dir
  
  _scratch_mkfs >>$seqres.full 2>&1

-_scratch_mount "-o compress"
+# On 64K pagesize systems the compression is more efficient, so max_inline
+# helps to create regular (non inline) extent irrespective of the final
+# write size.
+_scratch_mount "-o compress -o max_inline=0"
  
  # Write to our file using direct IO, so that this way the write ends up not

  # getting compressed, that is, we get a regular extent which is neither
  # inlined nor compressed.
  # Alternatively, we could have mounted the fs without compression enabled,
  # which would result as well in an uncompressed regular extent.
-$XFS_IO_PROG -f -d -c "pwrite -S 0xab 0 4K" $SCRATCH_MNT/foobar | 
_filter_xfs_io
+sectorsize=$(_scratch_sectorsize)


This new helper takes use of "btrfs inspect-internal dump-super", so the
test needs a new _require rule:

_require_btrfs_command inspect-internal dump-super


Will add.

Thanks, Anand


Otherwise the patch looks fine to me. But I'd be great if other btrfs
folks could help review the change and/or help actually test the changes
with 64k sectorsize.

Thanks,
Eryu


+$XFS_IO_PROG -f -d -c "pwrite -S 0xab 0 $sectorsize" $SCRATCH_MNT/foobar |\
+   _filter_xfs_io_numbers
  
  $BTRFS_UTIL_PROG subvolume snapshot -r $SCRATCH_MNT \

$SCRATCH_MNT/mysnap1 > /dev/null
  
  # Clone the regular (not inlined) extent.

-$XFS_IO_PROG -c "reflink $SCRATCH_MNT/foobar 0 8K 4K" $SCRATCH_MNT/foobar \
-   | _filter_xfs_io
+$XFS_IO_PROG -c \
+   "reflink $SCRATCH_MNT/foobar 0 $((2 * $sectorsize)) $sectorsize" \
+   $SCRATCH_MNT/foobar | _filter_xfs_io_numbers
  
  $BTRFS_UTIL_PROG subvolume snapshot -r $SCRATCH_MNT \

$SCRATCH_MNT/mysnap2 > /dev/null
@@ -76,21 +82,26 @@ $BTRFS_UTIL_PROG send -p $SCRATCH_MNT/mysnap1 -f 
$send_files_dir/2.snap \
 $SCRATCH_MNT/mysnap2 2>&1 >/dev/null | _filter_scratch
  
  echo "File digests in the original filesystem:"

-md5sum $SCRATCH_MNT/mysnap1/foobar | _filter_scratch
-md5sum $SCRATCH_MNT/mysnap2/foobar | _filter_scratch
+sum_src_snap1=$(md5sum $SCRATCH_MNT/mysnap1/foobar | awk '{print $1}')
+sum_src_snap2=$(md5sum $SCRATCH_MNT/mysnap2/foobar | awk '{print $1}')
+echo "src checksum created"
  
  # Now recreate the filesystem by receiving both send streams and verify we get

  # the same file content that the original filesystem had.
  _scratch_unmount
  _scratch_mkfs >>$seqres.full 2>&1
-_scratch_mount "-o compress"
+_scratch_mount "-o compress,max_inline=0"
  
  $BTRFS_UTIL_PROG receive -f $send_files_dir/1.snap $SCRATCH_MNT > /dev/null

  $BTRFS_UTIL_PROG receive -f $send_files_dir/2.snap $SCRATCH_MNT > /dev/null
  
  echo "File digests in the new filesystem:"

-md5sum $SCRATCH_MNT/mysnap1/foobar | _filter_scratch
-md5sum $SCRATCH_MNT/mysnap2/foobar | _filter_scratch
+sum_dest_snap1=$(md5sum $SCRATCH_MNT/mysnap1/foobar | awk '{print $1}')
+sum_dest_snap2=$(md5sum $SCRATCH_MNT/mysnap2/foobar | awk '{print $1}')
+echo "dest checksum created"
+
+[[

Re: btrfs send hung in pipe_wait

2018-09-09 Thread Chris Murphy 
On Sun, Sep 9, 2018 at 2:16 PM, Stefan Loewen  wrote:
> I'm not sure about the exact definition of "blocked" here, but I was
> also surprised that there were no blocked tasks listed since I'm
> definitely unable to kill (SIGKILL) that process.
> On the other hand it wakes up hourly to transfer a few bytes.
> The problem is definitely not, that I issued the sysrq too early. I
> think it was after about 45min of no IO.

Another one the devs have asked for in cases where things get slow or
hang, but without explicit blocked task messages, is sysrq + t. But
I'm throwing spaghetti at a wall at this point, none of it will fix
the problem, and I haven't learned how to read these outputs.



> So there is some problem with this "original" subvol. Maybe I should
> describe how that came into existence.
> Initially I had my data on a NTFS formatted drive. I then created a
> btrfs partition on my second drive and rsynced all my stuff over into
> the root subvol.
> Then I noticed that having all my data in the root subvol was a bad
> idea and created a "data" subvol and reflinked everything into it.
> I deleted the data from the root subvol, made a snapshot of the "data"
> subvol, tried sending that and ran into the problem we're discussing
> here.

That is interesting and useful information. I see nothing invalid
about it at all. However, just for future reference it is possible to
snapshot the top level (default) subvolume.

By default, the top level subvolume (sometimes referred to as
subvolid=5 or subvolid=0) is what is mounted if you haven't used
'btrfs sub set-default' to change it. You can snapshot that subvolume
by snapshotting the mount point. e.g.

mount /dev/sda1 /mnt
btrfs sub snap /mnt/subvolume1

So now you have a readwrite subvolume called "subvolume1" which
contains everything that was in the top level, which you can now
delete if you're trying to keep things tidy and just have subvolumes
and snapshots in the top level.

Anyway, what you did is possibly relevant to the problem. But if it
turns out it's the cause of the problem, it's definitely a bug.


>
> btrfs check in lowmem mode did not find any errors either:
>
> $ sudo btrfs check --mode=lowmem --progress /dev/sdb1
> Opening filesystem to check...
> Checking filesystem on /dev/sdb1
> UUID: cd786597-3816-40e7-bf6c-d585265ad372
> [1/7] checking root items  (0:00:30 elapsed,
> 1047408 items checked)
> [2/7] checking extents (0:03:55 elapsed,
> 309170 items checked)
> cache and super generation don't match, space cache will be invalidated
> [3/7] checking free space cache(0:00:00 elapsed)
> [4/7] checking fs roots(0:04:07 elapsed, 85373
> items checked)
> [5/7] checking csums (without verifying data)  (0:00:00 elapsed,
> 253106 items checked)
> [6/7] checking root refs done with fs roots in lowmem mode, skipping
> [7/7] checking quota groups skipped (not enabled on this FS)
> found 708354711552 bytes used, no error found
> total csum bytes: 689206904
> total tree bytes: 2423865344
> total fs tree bytes: 1542914048
> total extent tree bytes: 129843200
> btree space waste bytes: 299191292
> file data blocks allocated: 31709967417344
> referenced 928531877888

OK good to know.


-- 
Chris Murphy

Re: [PATCH] fstests: btrfs/149 make it sectorsize independent

2018-09-09 Thread Eryu Guan 
On Wed, Sep 05, 2018 at 02:35:12PM +0800, Anand Jain wrote:
> Originally this test case was designed to work with only 4K sectorsize.
> Now enhance it to work with any sector sizes and makes the following
> changes:
> Output file not to contain any traces of sector size.
> Use max_inline=0 mount option so that it meets the requisite of non inline
> regular extent.
> Don't log the md5sum results to the output file as the data size vary by
> the sectorsize.
> 
> Signed-off-by: Anand Jain 
> ---
>  common/btrfs|  7 +++
>  common/filter   |  5 +
>  tests/btrfs/149 | 29 -
>  tests/btrfs/149.out | 12 ++--
>  4 files changed, 38 insertions(+), 15 deletions(-)
> 
> diff --git a/common/btrfs b/common/btrfs
> index 79c687f73376..e6a218d6b63a 100644
> --- a/common/btrfs
> +++ b/common/btrfs
> @@ -367,3 +367,10 @@ _run_btrfs_balance_start()
>  
>   run_check $BTRFS_UTIL_PROG balance start $bal_opt $*
>  }
> +
> +#return the sector size of the btrfs scratch fs
> +_scratch_sectorsize()

The function name would indicate that it's a general helper function and
works for all filesystems, but it only works for btrfs. I'd be better to
name it as "_scratch_btrfs_sectorsize" or similar.

> +{
> + $BTRFS_UTIL_PROG inspect-internal dump-super $SCRATCH_DEV |\
> + grep sectorsize | awk '{print $2}'
> +}
> diff --git a/common/filter b/common/filter
> index 3965c2eb752b..e87740ddda3f 100644
> --- a/common/filter
> +++ b/common/filter
> @@ -271,6 +271,11 @@ _filter_xfs_io_pages_modified()
>   _filter_xfs_io_units_modified "Page" $PAGE_SIZE
>  }
>  
> +_filter_xfs_io_numbers()
> +{
> +_filter_xfs_io | sed -E 's/[0-9]+//g'
> +}
> +
>  _filter_test_dir()
>  {
>   # TEST_DEV may be a prefix of TEST_DIR (e.g. /mnt, /mnt/ovl-mnt)
> diff --git a/tests/btrfs/149 b/tests/btrfs/149
> index 3e955a305e0f..3958fa844c8b 100755
> --- a/tests/btrfs/149
> +++ b/tests/btrfs/149
> @@ -44,21 +44,27 @@ rm -fr $send_files_dir
>  mkdir $send_files_dir
>  
>  _scratch_mkfs >>$seqres.full 2>&1
> -_scratch_mount "-o compress"
> +# On 64K pagesize systems the compression is more efficient, so max_inline
> +# helps to create regular (non inline) extent irrespective of the final
> +# write size.
> +_scratch_mount "-o compress -o max_inline=0"
>  
>  # Write to our file using direct IO, so that this way the write ends up not
>  # getting compressed, that is, we get a regular extent which is neither
>  # inlined nor compressed.
>  # Alternatively, we could have mounted the fs without compression enabled,
>  # which would result as well in an uncompressed regular extent.
> -$XFS_IO_PROG -f -d -c "pwrite -S 0xab 0 4K" $SCRATCH_MNT/foobar | 
> _filter_xfs_io
> +sectorsize=$(_scratch_sectorsize)

This new helper takes use of "btrfs inspect-internal dump-super", so the
test needs a new _require rule:

_require_btrfs_command inspect-internal dump-super

Otherwise the patch looks fine to me. But I'd be great if other btrfs
folks could help review the change and/or help actually test the changes
with 64k sectorsize.

Thanks,
Eryu

> +$XFS_IO_PROG -f -d -c "pwrite -S 0xab 0 $sectorsize" $SCRATCH_MNT/foobar |\
> + _filter_xfs_io_numbers
>  
>  $BTRFS_UTIL_PROG subvolume snapshot -r $SCRATCH_MNT \
>   $SCRATCH_MNT/mysnap1 > /dev/null
>  
>  # Clone the regular (not inlined) extent.
> -$XFS_IO_PROG -c "reflink $SCRATCH_MNT/foobar 0 8K 4K" $SCRATCH_MNT/foobar \
> - | _filter_xfs_io
> +$XFS_IO_PROG -c \
> + "reflink $SCRATCH_MNT/foobar 0 $((2 * $sectorsize)) $sectorsize" \
> + $SCRATCH_MNT/foobar | _filter_xfs_io_numbers
>  
>  $BTRFS_UTIL_PROG subvolume snapshot -r $SCRATCH_MNT \
>   $SCRATCH_MNT/mysnap2 > /dev/null
> @@ -76,21 +82,26 @@ $BTRFS_UTIL_PROG send -p $SCRATCH_MNT/mysnap1 -f 
> $send_files_dir/2.snap \
>$SCRATCH_MNT/mysnap2 2>&1 >/dev/null | _filter_scratch
>  
>  echo "File digests in the original filesystem:"
> -md5sum $SCRATCH_MNT/mysnap1/foobar | _filter_scratch
> -md5sum $SCRATCH_MNT/mysnap2/foobar | _filter_scratch
> +sum_src_snap1=$(md5sum $SCRATCH_MNT/mysnap1/foobar | awk '{print $1}')
> +sum_src_snap2=$(md5sum $SCRATCH_MNT/mysnap2/foobar | awk '{print $1}')
> +echo "src checksum created"
>  
>  # Now recreate the filesystem by receiving both send streams and verify we 
> get
>  # the same file content that the original filesystem had.
>  _scratch_unmount
>  _scratch_mkfs >>$seqres.full 2>&1
> -_scratch_mount "-o compress"
> +_scratch_mount "-o compress,max_inline=0"
>  
>  $BTRFS_UTIL_PROG receive -f $send_files_dir/1.snap $SCRATCH_MNT > /dev/null
>  $BTRFS_UTIL_PROG receive -f $send_files_dir/2.snap $SCRATCH_MNT > /dev/null
>  
>  echo "File digests in the new filesystem:"
> -md5sum $SCRATCH_MNT/mysnap1/foobar | _filter_scratch
> -md5sum $SCRATCH_MNT/mysnap2/foobar | _filter_scratch
> +sum_dest_snap1=$(md5sum $SCRATCH_MNT/mysnap1/foobar | awk '{print $1}')
> +sum_dest_snap2=$(md5sum $SCRATCH_MNT/mysnap2/foobar | awk