Re: Announcing btrfs-dedupe

[James Pharaoh]

Perhaps the complexity of doing this efficiently makes it inappropriate 
for inclusion in the tool itself, whereas I believe the core 
implementation's focus is on in-band deduplication, automatic and behind 
the scenes.


On 08/11/16 03:40, Christoph Anton Mitterer wrote:

On Mon, 2016-11-07 at 15:02 +0100, David Sterba wrote:

I think adding a whole-file dedup mode to duperemove would be better
(from user's POV) than writing a whole new tool


What would IMO be really good from a user's POV was, if one of the
tools, deemed to be the "best", would be added to the btrfs-progs and
simply become "the official" one.

Cheers,
Chris.


--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: Announcing btrfs-dedupe

[Christoph Anton Mitterer]

On Mon, 2016-11-07 at 15:02 +0100, David Sterba wrote:
> I think adding a whole-file dedup mode to duperemove would be better
> (from user's POV) than writing a whole new tool

What would IMO be really good from a user's POV was, if one of the
tools, deemed to be the "best", would be added to the btrfs-progs and
simply become "the official" one.

Cheers,
Chris.

smime.p7s
Description: S/MIME cryptographic signature

[PATCH v3 2/3] btrfs: Add trace point for qgroup reserved space

[Qu Wenruo]

Introduce the following trace points:
qgroup_update_reserve
qgroup_meta_reserve

These trace points are handy to trace qgroup reserve space related
problems.

Signed-off-by: Qu Wenruo 
---
Although I hope these trace points will not be used again.

v2:
  None
v3:
  Separate from trace point timing modification patch.
---
 fs/btrfs/qgroup.c| 15 +++
 include/trace/events/btrfs.h | 43 +++
 2 files changed, 58 insertions(+)

diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c
index fc0c64e..c00f962 100644
--- a/fs/btrfs/qgroup.c
+++ b/fs/btrfs/qgroup.c
@@ -1068,6 +1068,8 @@ static int __qgroup_excl_accounting(struct btrfs_fs_info 
*fs_info,
qgroup->excl += sign * num_bytes;
qgroup->excl_cmpr += sign * num_bytes;
if (sign > 0) {
+   trace_qgroup_update_reserve(fs_info, qgroup->qgroupid,
+   qgroup->reserved, (s64)-num_bytes);
if (WARN_ON(qgroup->reserved < num_bytes))
report_reserved_underflow(fs_info, qgroup,
  num_bytes);
@@ -1094,6 +1096,9 @@ static int __qgroup_excl_accounting(struct btrfs_fs_info 
*fs_info,
WARN_ON(sign < 0 && qgroup->excl < num_bytes);
qgroup->excl += sign * num_bytes;
if (sign > 0) {
+   trace_qgroup_update_reserve(fs_info, qgroup->qgroupid,
+   qgroup->reserved,
+   (s64)-num_bytes);
if (WARN_ON(qgroup->reserved < num_bytes))
report_reserved_underflow(fs_info, qgroup,
  num_bytes);
@@ -2178,6 +2183,8 @@ static int qgroup_reserve(struct btrfs_root *root, u64 
num_bytes)
 
qg = u64_to_ptr(unode->aux);
 
+   trace_qgroup_update_reserve(fs_info, qg->qgroupid,
+   qg->reserved, num_bytes);
qg->reserved += num_bytes;
}
 
@@ -2223,6 +2230,8 @@ void btrfs_qgroup_free_refroot(struct btrfs_fs_info 
*fs_info,
 
qg = u64_to_ptr(unode->aux);
 
+   trace_qgroup_update_reserve(fs_info, qg->qgroupid,
+   qg->reserved, (s64)-num_bytes);
if (WARN_ON(qgroup->reserved < num_bytes))
report_reserved_underflow(fs_info, qgroup, num_bytes);
else
@@ -2706,6 +2715,8 @@ int btrfs_qgroup_reserve_meta(struct btrfs_root *root, 
int num_bytes)
return 0;
 
BUG_ON(num_bytes != round_down(num_bytes, root->nodesize));
+   trace_qgroup_meta_reserve(root->fs_info, root->objectid,
+ (s64)num_bytes);
ret = qgroup_reserve(root, num_bytes);
if (ret < 0)
return ret;
@@ -2724,6 +2735,8 @@ void btrfs_qgroup_free_meta_all(struct btrfs_root *root)
reserved = atomic_xchg(>qgroup_meta_rsv, 0);
if (reserved == 0)
return;
+   trace_qgroup_meta_reserve(root->fs_info, root->objectid,
+ (s64)-reserved);
qgroup_free(root, reserved);
 }
 
@@ -2736,6 +2749,8 @@ void btrfs_qgroup_free_meta(struct btrfs_root *root, int 
num_bytes)
BUG_ON(num_bytes != round_down(num_bytes, root->nodesize));
WARN_ON(atomic_read(>qgroup_meta_rsv) < num_bytes);
atomic_sub(num_bytes, >qgroup_meta_rsv);
+   trace_qgroup_meta_reserve(root->fs_info, root->objectid,
+ (s64)-num_bytes);
qgroup_free(root, num_bytes);
 }
 
diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h
index e030d6f..fb3cb6c 100644
--- a/include/trace/events/btrfs.h
+++ b/include/trace/events/btrfs.h
@@ -1468,6 +1468,49 @@ TRACE_EVENT(qgroup_update_counters,
  __entry->cur_new_count)
 );
 
+TRACE_EVENT(qgroup_update_reserve,
+
+   TP_PROTO(struct btrfs_fs_info *fs_info, u64 qgid, u64 cur_reserved,
+s64 diff),
+
+   TP_ARGS(fs_info, qgid, cur_reserved, diff),
+
+   TP_STRUCT__entry_btrfs(
+   __field(u64,qgid)
+   __field(u64,cur_reserved)
+   __field(s64,diff)
+   ),
+
+   TP_fast_assign_btrfs(fs_info,
+   __entry->qgid   = qgid;
+   __entry->cur_reserved   = cur_reserved;
+   __entry->diff   = diff;
+   ),
+
+   TP_printk_btrfs("qgid = %llu, cur_reserved = %llu, diff = %lld",
+   __entry->qgid, __entry->cur_reserved, __entry->diff)
+);
+
+TRACE_EVENT(qgroup_meta_reserve,
+
+   TP_PROTO(struct btrfs_fs_info *fs_info, u64 refroot, s64 diff),
+
+   TP_ARGS(fs_info, 

[PATCH v3 3/3] btrfs: qgroup: Re-arrange tracepoint timing to co-operate with reserved space tracepoint

[Qu Wenruo]

Newly introduced qgroup reserved space trace points are normally nested
into several common qgroup operations.

While some other trace points are not well placed to co-operate with
them, causing confusing output.

This patch re-arrange trace_btrfs_qgroup_release_data() and
trace_btrfs_qgroup_free_delayed_ref() trace points so they are triggered
before reserved space ones.

Signed-off-by: Qu Wenruo 
---
v3:
  Separated from new trace points patch.
---
 fs/btrfs/qgroup.c | 6 +++---
 fs/btrfs/qgroup.h | 6 +-
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c
index c00f962..aad34314 100644
--- a/fs/btrfs/qgroup.c
+++ b/fs/btrfs/qgroup.c
@@ -2660,12 +2660,12 @@ static int __btrfs_qgroup_release_data(struct inode 
*inode, u64 start, u64 len,
if (ret < 0)
goto out;
 
-   if (free) {
-   qgroup_free(BTRFS_I(inode)->root, changeset.bytes_changed);
+   if (free)
trace_op = QGROUP_FREE;
-   }
trace_btrfs_qgroup_release_data(inode, start, len,
changeset.bytes_changed, trace_op);
+   if (free)
+   qgroup_free(BTRFS_I(inode)->root, changeset.bytes_changed);
 out:
ulist_free(changeset.range_changed);
return ret;
diff --git a/fs/btrfs/qgroup.h b/fs/btrfs/qgroup.h
index 1bc64c8..93b222b 100644
--- a/fs/btrfs/qgroup.h
+++ b/fs/btrfs/qgroup.h
@@ -107,15 +107,11 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans,
 struct btrfs_qgroup_inherit *inherit);
 void btrfs_qgroup_free_refroot(struct btrfs_fs_info *fs_info,
   u64 ref_root, u64 num_bytes);
-/*
- * TODO: Add proper trace point for it, as btrfs_qgroup_free() is
- * called by everywhere, can't provide good trace for delayed ref case.
- */
 static inline void btrfs_qgroup_free_delayed_ref(struct btrfs_fs_info *fs_info,
 u64 ref_root, u64 num_bytes)
 {
-   btrfs_qgroup_free_refroot(fs_info, ref_root, num_bytes);
trace_btrfs_qgroup_free_delayed_ref(fs_info, ref_root, num_bytes);
+   btrfs_qgroup_free_refroot(fs_info, ref_root, num_bytes);
 }
 void assert_qgroups_uptodate(struct btrfs_trans_handle *trans);
 
-- 
2.10.2



--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v3 1/3] btrfs: Add WARN_ON for qgroup reserved underflow

[Qu Wenruo]

Goldwyn Rodrigues has exposed and fixed a bug which underflows btrfs
qgroup reserved space, and leads to non-writable fs.

This reminds us that we don't have enough underflow check for qgroup
reserved space.

For underflow case, we should not really underflow the numbers but warn
and keeps qgroup still work.

So add more check on qgroup reserved space and add WARN_ON() and
btrfs_warn() for any underflow case.

Signed-off-by: Qu Wenruo 
Reviewed-by: David Sterba 
---
v2:
  Add btrfs_warn() output for fsid, qgroupid, original reserved space and
  num_bytes to reduce, for end-user to locate the subvolume causing the
  problem. Suggested by David.
v3:
  None
---
 fs/btrfs/qgroup.c | 32 +++-
 1 file changed, 27 insertions(+), 5 deletions(-)

diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c
index 11f4fff..fc0c64e 100644
--- a/fs/btrfs/qgroup.c
+++ b/fs/btrfs/qgroup.c
@@ -1031,6 +1031,15 @@ static void qgroup_dirty(struct btrfs_fs_info *fs_info,
list_add(>dirty, _info->dirty_qgroups);
 }
 
+static void report_reserved_underflow(struct btrfs_fs_info *fs_info,
+ struct btrfs_qgroup *qgroup,
+ u64 num_bytes)
+{
+   btrfs_warn(fs_info,
+   "qgroup %llu reserved space underflow, have: %llu, to free: 
%llu",
+   qgroup->qgroupid, qgroup->reserved, num_bytes);
+   qgroup->reserved = 0;
+}
 /*
  * The easy accounting, if we are adding/removing the only ref for an extent
  * then this qgroup and all of the parent qgroups get their reference and
@@ -1058,8 +1067,13 @@ static int __qgroup_excl_accounting(struct btrfs_fs_info 
*fs_info,
WARN_ON(sign < 0 && qgroup->excl < num_bytes);
qgroup->excl += sign * num_bytes;
qgroup->excl_cmpr += sign * num_bytes;
-   if (sign > 0)
-   qgroup->reserved -= num_bytes;
+   if (sign > 0) {
+   if (WARN_ON(qgroup->reserved < num_bytes))
+   report_reserved_underflow(fs_info, qgroup,
+ num_bytes);
+   else
+   qgroup->reserved -= num_bytes;
+   }
 
qgroup_dirty(fs_info, qgroup);
 
@@ -1079,8 +1093,13 @@ static int __qgroup_excl_accounting(struct btrfs_fs_info 
*fs_info,
qgroup->rfer_cmpr += sign * num_bytes;
WARN_ON(sign < 0 && qgroup->excl < num_bytes);
qgroup->excl += sign * num_bytes;
-   if (sign > 0)
-   qgroup->reserved -= num_bytes;
+   if (sign > 0) {
+   if (WARN_ON(qgroup->reserved < num_bytes))
+   report_reserved_underflow(fs_info, qgroup,
+ num_bytes);
+   else
+   qgroup->reserved -= num_bytes;
+   }
qgroup->excl_cmpr += sign * num_bytes;
qgroup_dirty(fs_info, qgroup);
 
@@ -2204,7 +2223,10 @@ void btrfs_qgroup_free_refroot(struct btrfs_fs_info 
*fs_info,
 
qg = u64_to_ptr(unode->aux);
 
-   qg->reserved -= num_bytes;
+   if (WARN_ON(qgroup->reserved < num_bytes))
+   report_reserved_underflow(fs_info, qgroup, num_bytes);
+   else
+   qgroup->reserved -= num_bytes;
 
list_for_each_entry(glist, >groups, next_group) {
ret = ulist_add(fs_info->qgroup_ulist,
-- 
2.10.2



--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: Announcing btrfs-dedupe

[Darrick J. Wong]

On Mon, Nov 07, 2016 at 09:54:09PM +0100, Adam Borowski wrote:
> On Mon, Nov 07, 2016 at 09:48:41AM -0800, Mark Fasheh wrote:
> > also on XFS with the dedupe ioctl (I believe this should be out with
> > Linux-4.9).
> 
> It's already there in 4.9-rc1, although you need a special version of
> xfsprogs (possibly already released, I didn't check).  It's an experimental
> feature that needs to be enabled with "-m reflink=1".

The code will be available in xfsprogs 4.9, due out after Linux 4.9.

You'll still have to pass '-m reflink=1' to enable reflink until we
declare the feature stable, however.

> Despite that experimental status, I'd strongly recommend James to test his
> tool on xfs as well, as it's the second major implementation of this API[1].

Agreed. :)

> Mark has already included XFS in documentation of duperemove, all that looks
> amiss is btrfs-extent-same having an obsolete name.  But then, I never did
> any non-superficial tests on XFS, beyond "seems to work".

/me wonders if ocfs2 will ever catch up to the reflink/dedupe party. ;)

--Darrick

> 
> 
> Meow!
> 
> [1]. For some reasons zfs-on-linux guys didn't implement this yet, despite
> it being an obvious thing on ZFS.
> -- 
> A MAP07 (Dead Simple) raspberry tincture recipe: 0.5l 95% alcohol, 1kg
> raspberries, 0.4kg sugar; put into a big jar for 1 month.  Filter out and
> throw away the fruits (can dump them into a cake, etc), let the drink age
> at least 3-6 months.
> --
> To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
> the body of a message to majord...@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: btrfs support for filesystems >8TB on 32bit architectures

[Marc MERLIN]

On Tue, Nov 08, 2016 at 08:43:34AM +0800, Qu Wenruo wrote:
> That's strange, balance is done completely in kernel space.
> 
> Unless we're calling vfs_* function we won't go through the extra check.
> 
> What's the error reported?

See below. Note however that is may be because btrfs received messed up the
filesystem first.

BTRFS info (device dm-0): use zlib compression
BTRFS info (device dm-0): disk space caching is enabled
BTRFS info (device dm-0): has skinny extents
BTRFS info (device dm-0): bdev /dev/mapper/crypt_bcache0 errs: wr 0, rd 0, 
flush 0, corrupt 512, gen 0
BTRFS info (device dm-0): detected SSD devices, enabling SSD mode
BTRFS info (device dm-0): continuing balance
BTRFS info (device dm-0): The free space cache file (1593999097856) is invalid. 
skip it

BTRFS info (device dm-0): The free space cache file (1671308509184) is invalid. 
skip it

BTRFS info (device dm-0): relocating block group 13835461197824 flags 34
[ cut here ]
WARNING: CPU: 0 PID: 22825 at fs/btrfs/disk-io.c:520 
btree_csum_one_bio.isra.39+0xf7/0x100
Modules linked in: bcache configs rc_hauppauge ir_kbd_i2c cpufreq_userspace 
cpufreq_powersave cpufreq_conservative autofs4 snd_hda_codec_hdmi joydev 
snd_hda_codec_realtek snd_hda_codec_generic tuner_simple tuner_types tda9887 
snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep tda8290 coretemp snd_pcm_oss 
snd_mixer_oss tuner snd_pcm msp3400 snd_seq_midi snd_seq_midi_event 
firewire_sbp2 saa7127 snd_rawmidi hwmon_vid dm_crypt dm_mod saa7115 snd_seq 
bttv hid_generic snd_seq_device snd_timer ehci_pci ivtv tea575x videobuf_dma_sg 
rc_core videobuf_core input_leds tveeprom cx2341x v4l2_common ehci_hcd videodev 
media acpi_cpufreq tpm_tis tpm_tis_core gpio_ich snd soundcore tpm psmouse 
lpc_ich evdev asus_atk0110 serio_raw lp parport raid456 async_raid6_recov 
async_pq async_xor async_memcpy async_tx multipath usbhid hid sr_mod cdrom sg 
firewire_ohci firewire_core floppy crc_itu_t i915 atl1 fjes mii uhci_hcd 
usbcore usb_common
CPU: 0 PID: 22825 Comm: kworker/u9:2 Tainted: GW   
4.8.5-ia32-20161028 #2
Hardware name: System manufacturer P5E-VM HDMI/P5E-VM HDMI, BIOS 0604
07/16/2008
Workqueue: btrfs-worker-high btrfs_worker_helper
 00200286 00200286 d3d81e48 df414827  dfa12da5 d3d81e78 df05677a
 df9ed884  5929 dfa12da5 0208 df2cf067 0208 f7463fa0
 f401a080  d3d81e8c df05684a 0009   d3d81eb4
Call Trace:
 [] dump_stack+0x58/0x81
 [] __warn+0xea/0x110
 [] ? btree_csum_one_bio.isra.39+0xf7/0x100
 [] warn_slowpath_null+0x2a/0x30
 [] btree_csum_one_bio.isra.39+0xf7/0x100
 [] __btree_submit_bio_start+0x15/0x20
 [] run_one_async_start+0x30/0x40
 [] btrfs_scrubparity_helper+0xcd/0x2d0
 [] ? run_one_async_free+0x20/0x20
 [] btrfs_worker_helper+0xd/0x10
 [] process_one_work+0x10b/0x400
 [] worker_thread+0x37/0x4b0
 [] ? process_one_work+0x400/0x400
 [] kthread+0x9b/0xb0
 [] ret_from_kernel_thread+0xe/0x24
 [] ? kthread_stop+0x100/0x100
---[ end trace f461faff989bf258 ]---
BTRFS: error (device dm-0) in btrfs_commit_transaction:2232: errno=-5 IO 
failure (Error while writing out transaction)
BTRFS info (device dm-0): forced readonly
BTRFS warning (device dm-0): Skipping commit of aborted transaction.
[ cut here ]
WARNING: CPU: 0 PID: 22318 at fs/btrfs/transaction.c:1854 
btrfs_commit_transaction+0x2f5/0xcc0
BTRFS: Transaction aborted (error -5)
Modules linked in: bcache configs rc_hauppauge ir_kbd_i2c cpufreq_userspace 
cpufreq_powersave cpufreq_conservative autofs4 snd_hda_codec_hdmi joydev 
snd_hda_codec_realtek snd_hda_codec_generic tuner_simple tuner_types tda9887 
snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep tda8290 coretemp snd_pcm_oss 
snd_mixer_oss tuner snd_pcm msp3400 snd_seq_midi snd_seq_midi_event 
firewire_sbp2 saa7127 snd_rawmidi hwmon_vid dm_crypt dm_mod saa7115 snd_seq 
bttv hid_generic snd_seq_device snd_timer ehci_pci ivtv tea575x videobuf_dma_sg 
rc_core videobuf_core input_leds tveeprom cx2341x v4l2_common ehci_hcd videodev 
media acpi_cpufreq tpm_tis tpm_tis_core gpio_ich snd soundcore tpm psmouse 
lpc_ich evdev asus_atk0110 serio_raw lp parport raid456 async_raid6_recov 
async_pq async_xor async_memcpy async_tx multipath usbhid hid sr_mod cdrom sg 
firewire_ohci firewire_core floppy crc_itu_t i915 atl1 fjes mii uhci_hcd 
usbcore usb_common
CPU: 0 PID: 22318 Comm: btrfs-balance Tainted: GW   
4.8.5-ia32-20161028 #2
Hardware name: System manufacturer P5E-VM HDMI/P5E-VM HDMI, BIOS 0604
07/16/2008
 0286 0286 d74a3ca4 df414827 d74a3ce8 dfa132ab d74a3cd4 df05677a
 dfa075cc d74a3d04 572e dfa132ab 073e df2d7de5 073e f698dc00
 e9173e70 fffb d74a3cf0 df0567db 0009  d74a3ce8 dfa075cc
Call Trace:
 [] dump_stack+0x58/0x81
 [] __warn+0xea/0x110
 [] ? btrfs_commit_transaction+0x2f5/0xcc0
 [] warn_slowpath_fmt+0x3b/0x40
 [] btrfs_commit_transaction+0x2f5/0xcc0
 [] ? prepare_to_wait_event+0xd0/0xd0
 [] 

Re: btrfs support for filesystems >8TB on 32bit architectures

[Qu Wenruo]

At 11/08/2016 09:06 AM, Marc MERLIN wrote:

On Tue, Nov 08, 2016 at 08:43:34AM +0800, Qu Wenruo wrote:

That's strange, balance is done completely in kernel space.

Unless we're calling vfs_* function we won't go through the extra check.

What's the error reported?


See below. Note however that is may be because btrfs received messed up the
filesystem first.


If receive can easily screw up the fs, then fsstress can also screw up 
btrfs easily.


So I didn't think that's the case. (Several years ago it's possible)



BTRFS info (device dm-0): use zlib compression
BTRFS info (device dm-0): disk space caching is enabled
BTRFS info (device dm-0): has skinny extents
BTRFS info (device dm-0): bdev /dev/mapper/crypt_bcache0 errs: wr 0, rd 0, 
flush 0, corrupt 512, gen 0
BTRFS info (device dm-0): detected SSD devices, enabling SSD mode
BTRFS info (device dm-0): continuing balance
BTRFS info (device dm-0): The free space cache file (1593999097856) is invalid. 
skip it

BTRFS info (device dm-0): The free space cache file (1671308509184) is invalid. 
skip it

BTRFS info (device dm-0): relocating block group 13835461197824 flags 34
[ cut here ]
WARNING: CPU: 0 PID: 22825 at fs/btrfs/disk-io.c:520 
btree_csum_one_bio.isra.39+0xf7/0x100


Dirty tree block's bytenr doesn't match with page's logical.
It seems that the tree block is not up-to-date, maybe corrupted.

Seems not related to the 8T limit.

Could you please add pr_info() to print out the 'found_start' and 'start'?
Also I'm not familiar with this code, the number may has a clue to show 
what's going wrong.


Thanks,
Qu


Modules linked in: bcache configs rc_hauppauge ir_kbd_i2c cpufreq_userspace 
cpufreq_powersave cpufreq_conservative autofs4 snd_hda_codec_hdmi joydev 
snd_hda_codec_realtek snd_hda_codec_generic tuner_simple tuner_types tda9887 
snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep tda8290 coretemp snd_pcm_oss 
snd_mixer_oss tuner snd_pcm msp3400 snd_seq_midi snd_seq_midi_event 
firewire_sbp2 saa7127 snd_rawmidi hwmon_vid dm_crypt dm_mod saa7115 snd_seq 
bttv hid_generic snd_seq_device snd_timer ehci_pci ivtv tea575x videobuf_dma_sg 
rc_core videobuf_core input_leds tveeprom cx2341x v4l2_common ehci_hcd videodev 
media acpi_cpufreq tpm_tis tpm_tis_core gpio_ich snd soundcore tpm psmouse 
lpc_ich evdev asus_atk0110 serio_raw lp parport raid456 async_raid6_recov 
async_pq async_xor async_memcpy async_tx multipath usbhid hid sr_mod cdrom sg 
firewire_ohci firewire_core floppy crc_itu_t i915 atl1 fjes mii uhci_hcd 
usbcore usb_common
CPU: 0 PID: 22825 Comm: kworker/u9:2 Tainted: GW   
4.8.5-ia32-20161028 #2
Hardware name: System manufacturer P5E-VM HDMI/P5E-VM HDMI, BIOS 0604
07/16/2008
Workqueue: btrfs-worker-high btrfs_worker_helper
 00200286 00200286 d3d81e48 df414827  dfa12da5 d3d81e78 df05677a
 df9ed884  5929 dfa12da5 0208 df2cf067 0208 f7463fa0
 f401a080  d3d81e8c df05684a 0009   d3d81eb4
Call Trace:
 [] dump_stack+0x58/0x81
 [] __warn+0xea/0x110
 [] ? btree_csum_one_bio.isra.39+0xf7/0x100
 [] warn_slowpath_null+0x2a/0x30
 [] btree_csum_one_bio.isra.39+0xf7/0x100
 [] __btree_submit_bio_start+0x15/0x20
 [] run_one_async_start+0x30/0x40
 [] btrfs_scrubparity_helper+0xcd/0x2d0
 [] ? run_one_async_free+0x20/0x20
 [] btrfs_worker_helper+0xd/0x10
 [] process_one_work+0x10b/0x400
 [] worker_thread+0x37/0x4b0
 [] ? process_one_work+0x400/0x400
 [] kthread+0x9b/0xb0
 [] ret_from_kernel_thread+0xe/0x24
 [] ? kthread_stop+0x100/0x100
---[ end trace f461faff989bf258 ]---
BTRFS: error (device dm-0) in btrfs_commit_transaction:2232: errno=-5 IO 
failure (Error while writing out transaction)
BTRFS info (device dm-0): forced readonly
BTRFS warning (device dm-0): Skipping commit of aborted transaction.
[ cut here ]
WARNING: CPU: 0 PID: 22318 at fs/btrfs/transaction.c:1854 
btrfs_commit_transaction+0x2f5/0xcc0
BTRFS: Transaction aborted (error -5)
Modules linked in: bcache configs rc_hauppauge ir_kbd_i2c cpufreq_userspace 
cpufreq_powersave cpufreq_conservative autofs4 snd_hda_codec_hdmi joydev 
snd_hda_codec_realtek snd_hda_codec_generic tuner_simple tuner_types tda9887 
snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep tda8290 coretemp snd_pcm_oss 
snd_mixer_oss tuner snd_pcm msp3400 snd_seq_midi snd_seq_midi_event 
firewire_sbp2 saa7127 snd_rawmidi hwmon_vid dm_crypt dm_mod saa7115 snd_seq 
bttv hid_generic snd_seq_device snd_timer ehci_pci ivtv tea575x videobuf_dma_sg 
rc_core videobuf_core input_leds tveeprom cx2341x v4l2_common ehci_hcd videodev 
media acpi_cpufreq tpm_tis tpm_tis_core gpio_ich snd soundcore tpm psmouse 
lpc_ich evdev asus_atk0110 serio_raw lp parport raid456 async_raid6_recov 
async_pq async_xor async_memcpy async_tx multipath usbhid hid sr_mod cdrom sg 
firewire_ohci firewire_core floppy crc_itu_t i915 atl1 fjes mii uhci_hcd 
usbcore usb_common
CPU: 0 PID: 22318 Comm: btrfs-balance Tainted: GW   

Re: [PATCH v4 1/4] btrfs-progs: utils: Introduce function to escape characters

[Qu Wenruo]

At 11/07/2016 06:20 PM, David Sterba wrote:

On Thu, Nov 03, 2016 at 12:07:31PM +0800, Qu Wenruo wrote:

Introduce new function, escape_string_inplace(), to escape specified
characters in place.

Signed-off-by: Qu Wenruo 
---
 utils.c | 24 
 utils.h | 14 ++
 2 files changed, 38 insertions(+)

diff --git a/utils.c b/utils.c
index 3f54245..3c50d84 100644
--- a/utils.c
+++ b/utils.c
@@ -4251,3 +4251,27 @@ unsigned int rand_range(unsigned int upper)
 */
return (unsigned int)(jrand48(rand_seed) % upper);
 }
+
+static void escape_one_char(char *restrict string, char escape)
+{
+   int i = 0;
+   int j = 0;
+
+   while (string[j] != '\0') {
+   if (string[j] != escape) {
+   string[i] = string[j];
+   i++;
+   }


So how does this actually escape the characters? I don't see anything
inserted. I think we don't have a common understanding of what's
supposed to be done here. I mean C-string-like escaping. And this cannot
be done inplace, as it increases string length. I've implemented it in a
different way, so this patch is left out. The final stream dump output
is preserved.



And further more.

The new string escaping is screwing up original output.

Operation lile mkfile lacks the final '\n'.
String over the width will lack the ending ' ' to seperate later 
key=value pairs.


It would be better to output the escaped string into a buffer and still 
use printf() format to output them out as it used to be.


Thanks,
Qu


--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: btrfs support for filesystems >8TB on 32bit architectures

[Qu Wenruo]

At 11/08/2016 08:39 AM, Marc MERLIN wrote:

On Tue, Nov 08, 2016 at 08:35:54AM +0800, Qu Wenruo wrote:

Understood. One big thing (for me) I forgot to confirm:
1) btrfs receive


Unfortunately, receive is completely done in userspace.
Only send works inside kernel.


right, I've confirmed that btrfs receive fails.
It looks like btrfs balance is also failing, which is more surprising.
Isn't that one in the kernel?


That's strange, balance is done completely in kernel space.

Unless we're calling vfs_* function we won't go through the extra check.

What's the error reported?

Thanks,
Qu


Marc




--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: btrfs support for filesystems >8TB on 32bit architectures

[Marc MERLIN]

On Tue, Nov 08, 2016 at 08:35:54AM +0800, Qu Wenruo wrote:
> >Understood. One big thing (for me) I forgot to confirm:
> >1) btrfs receive
> 
> Unfortunately, receive is completely done in userspace.
> Only send works inside kernel.
 
right, I've confirmed that btrfs receive fails.
It looks like btrfs balance is also failing, which is more surprising.
Isn't that one in the kernel?

Marc
-- 
"A mouse is a device used to point at the xterm you want to type in" - A.S.R.
Microsoft is to operating systems 
   what McDonalds is to gourmet cooking
Home page: http://marc.merlins.org/  
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v4 1/4] btrfs-progs: utils: Introduce function to escape characters

[Qu Wenruo]

At 11/07/2016 06:20 PM, David Sterba wrote:

On Thu, Nov 03, 2016 at 12:07:31PM +0800, Qu Wenruo wrote:

Introduce new function, escape_string_inplace(), to escape specified
characters in place.

Signed-off-by: Qu Wenruo 
---
 utils.c | 24 
 utils.h | 14 ++
 2 files changed, 38 insertions(+)

diff --git a/utils.c b/utils.c
index 3f54245..3c50d84 100644
--- a/utils.c
+++ b/utils.c
@@ -4251,3 +4251,27 @@ unsigned int rand_range(unsigned int upper)
 */
return (unsigned int)(jrand48(rand_seed) % upper);
 }
+
+static void escape_one_char(char *restrict string, char escape)
+{
+   int i = 0;
+   int j = 0;
+
+   while (string[j] != '\0') {
+   if (string[j] != escape) {
+   string[i] = string[j];
+   i++;
+   }


So how does this actually escape the characters? I don't see anything
inserted. I think we don't have a common understanding of what's
supposed to be done here. I mean C-string-like escaping. And this cannot
be done inplace, as it increases string length. I've implemented it in a
different way, so this patch is left out. The final stream dump output
is preserved.



Oh, sorry.

I misunderstand the term "string escaping". I though it's just deleting 
special characters.


I checked your print_path_escaped() and finally got what string-escaping 
should do.


Sorry for the extra trouble.

While still some small comment.
+ if (!isprint(c)) {
+ printf("\\%c%c%c",
+ '0' + ((c & 0300) >> 6),
+ '0' + ((c & 070) >> 3),
+ '0' + (c & 07));

For non-printable chars, isn't it more common to print it as hex other 
than octal?


Thanks,
Qu


--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: btrfs support for filesystems >8TB on 32bit architectures

[Qu Wenruo]

At 11/07/2016 10:55 PM, Marc MERLIN wrote:

On Mon, Nov 07, 2016 at 02:16:37PM +0800, Qu Wenruo wrote:



At 11/07/2016 01:36 PM, Marc MERLIN wrote:

(sorry for the bad subject line from the mdadm list on the previous mail)

On Mon, Nov 07, 2016 at 12:18:10PM +0800, Qu Wenruo wrote:

I'm totally wrong here.

DirectIO needs the 'buf' parameter of read()/pread() to be 512 bytes
aligned.

While we are using a lot of stack memory() and normal malloc()/calloc()
allocated memory, which are seldom aligned to 512 bytes.

So to *workaround* the problem in btrfs-progs, we may need to change any
pread() caller to use aligned memory allocation.

I really don't think David will accept such huge change for a workdaround...


Thanks for looking into it.
So basically should we just document that btrfs filesystems past 8TB in
size are not supported on 32bit architectures?
(as in you can mount them and use them I believe, but you cannot create,
or repair them)

Marc


Add David to this thread.

For create, it should be OK. As at create time, we hardly write beyond 3G.
So it won't be a big problem.

For repair, we do have a possibility that btrfsck can't handle it.

Anyway, I'd like to see how David thinks what we should do the handle the
problem.


Understood. One big thing (for me) I forgot to confirm:
1) btrfs receive


Unfortunately, receive is completely done in userspace.
Only send works inside kernel.

So, receive will fail to reconstruct any file larger beyond 8T.
Despite that, any other normal file smaller than 8T is not affected.


2) btrfs scrub


Scrub does work in kernel, so it's unaffected.

Thanks,
Qu


should both be able to work because the IO operations are done directly
inside the kernel and not from user space, correct?

Thanks,
Marc




--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] btrfs-progs: send-test: add checking of clone-src option

[Tsutomu Itoh]

On 2016/11/08 0:16, David Sterba wrote:
> On Fri, Nov 04, 2016 at 05:35:18PM +0900, Tsutomu Itoh wrote:
>> +before_size=`ls -l "$here"/send.stream.before | awk '{print $5}'`
>> +after_size=`ls -l "$here"/send.stream.after | awk '{print $5}'`
> 
> Thanks for the test. Minor fixes: "stat --format=%s" instead of the
> above, and /dev/zero instead of urandom and stream redirection so it
> works in my NFS setup.

Thanks for the fixing.
But following command should be old btrfs. So please use 'btrfs' instead
of '$TOP/btrfs'.

   run_check $SUDO_HELPER $TOP/btrfs send -f "$here"/send.stream.before \
   -c subv-snap1_1 -c subv-snap2_1 subv-snap1_[23] subv-snap2_[23]

Thanks,
Tsutomu

> --
> To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
> the body of a message to majord...@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v2] btrfs: make block group flags in balance printks human-readable

[Adam Borowski]

They're not even documented anywhere, letting users with no recourse but
to RTFS.  It's no big burden to output the bitfield as words.

Also, display unknown flags as hex.

Signed-off-by: Adam Borowski 
---
 fs/btrfs/relocation.c | 53 ---
 1 file changed, 50 insertions(+), 3 deletions(-)

diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c
index c4af0cd..b5d2a00 100644
--- a/fs/btrfs/relocation.c
+++ b/fs/btrfs/relocation.c
@@ -4333,6 +4333,42 @@ static struct reloc_control *alloc_reloc_control(struct 
btrfs_fs_info *fs_info)
 }
 
 /*
+ * explain bit flags, prefixed by a '|' that'll be dropped
+ */
+static char *describe_block_group_flags(char *buf, u64 flags)
+{
+#define BUF_SIZE 128
+   char *buf0 = buf = kmalloc(BUF_SIZE, GFP_NOFS);
+
+   if (!buf)
+   return 0;
+
+   if (!flags) {
+   strcpy(buf, "|NONE");
+   return buf0;
+   }
+#define DESCRIBE_FLAG(f, d) \
+   if (flags & BTRFS_BLOCK_GROUP_##f) { \
+   buf += snprintf(buf, buf0 - buf + BUF_SIZE, "|%s", d); \
+   flags &= ~BTRFS_BLOCK_GROUP_##f; \
+   }
+   DESCRIBE_FLAG(DATA, "data");
+   DESCRIBE_FLAG(SYSTEM,   "system");
+   DESCRIBE_FLAG(METADATA, "metadata");
+   DESCRIBE_FLAG(RAID0,"raid0");
+   DESCRIBE_FLAG(RAID1,"raid1");
+   DESCRIBE_FLAG(DUP,  "dup");
+   DESCRIBE_FLAG(RAID10,   "raid10");
+   DESCRIBE_FLAG(RAID5,"raid5");
+   DESCRIBE_FLAG(RAID6,"raid6");
+   if (flags)
+   buf += snprintf(buf, buf0 - buf + BUF_SIZE, "|0x%llx", flags);
+   return buf0;
+#undef DESCRIBE_FLAG
+#undef BUF_SIZE
+}
+
+/*
  * function to relocate all extents in a block group.
  */
 int btrfs_relocate_block_group(struct btrfs_root *extent_root, u64 group_start)
@@ -4344,6 +4380,7 @@ int btrfs_relocate_block_group(struct btrfs_root 
*extent_root, u64 group_start)
int ret;
int rw = 0;
int err = 0;
+   char *flags_str;
 
rc = alloc_reloc_control(fs_info);
if (!rc)
@@ -4388,9 +4425,19 @@ int btrfs_relocate_block_group(struct btrfs_root 
*extent_root, u64 group_start)
goto out;
}
 
-   btrfs_info(extent_root->fs_info,
-  "relocating block group %llu flags %llu",
-  rc->block_group->key.objectid, rc->block_group->flags);
+   if ((flags_str = describe_block_group_flags(flags_str,
+  rc->block_group->flags))) {
+   btrfs_info(extent_root->fs_info,
+  "relocating block group %llu flags %s",
+  rc->block_group->key.objectid,
+  flags_str+1);
+   kfree(flags_str);
+   } else {
+   btrfs_info(extent_root->fs_info,
+  "relocating block group %llu flags %llx",
+  rc->block_group->key.objectid,
+  rc->block_group->flags);
+   }
 
btrfs_wait_block_group_reservations(rc->block_group);
btrfs_wait_nocow_writers(rc->block_group);
-- 
2.10.2

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] btrfs: make block group flags in balance printks human-readable

[Adam Borowski]

On Mon, Nov 07, 2016 at 05:58:41PM +0100, David Sterba wrote:
> On Fri, Nov 04, 2016 at 08:26:54AM +0100, Adam Borowski wrote:
> > They're not even documented anywhere, letting users with no recourse but
> > to RTFS.  It's no big burden to output the bitfield as words.
> 
> Ok. Below are some comments (and style nitpicks).

[...]

> > +   char flags_str[128];
> 
> This could be a problem. The relocation can trigger deep call chains
> when doing IO, that need stack. I'd rather avoid the static buffer,
> please switch it to kmalloc.

Are the chains callers or callees of relocation?  I believe exclusively the
latter, as balance is a long process that wouldn't make sense inside a deep
chain -- am I right?  In that case, moving the printk to the display
function would localize the buffer, avoiding all complications of an alloc.

(v2 does alloc)

> As this is not a critical allocation, the fallback would be to print just
> the raw value as now.

If the kernel is that low on memory, is it reasonable to print an
unimportant message?  As I see, lots of btrfs code would just fall over and
die horribly, with BUG_ON or aborting; perhaps skipping the message would
be safer?

(v2 has a fallback message)

> Also, please use snprintf. This would need extra variable to track
> number of already printed chars, but should be easy

M'kay.  Even with all flags on (most are mutually exclusive) we won't get
anywhere close to 128 characters, but paranoia can't hurt.


Meow!
-- 
A MAP07 (Dead Simple) raspberry tincture recipe: 0.5l 95% alcohol, 1kg
raspberries, 0.4kg sugar; put into a big jar for 1 month.  Filter out and
throw away the fruits (can dump them into a cake, etc), let the drink age
at least 3-6 months.
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: Announcing btrfs-dedupe

[Adam Borowski]

On Mon, Nov 07, 2016 at 09:48:41AM -0800, Mark Fasheh wrote:
> also on XFS with the dedupe ioctl (I believe this should be out with
> Linux-4.9).

It's already there in 4.9-rc1, although you need a special version of
xfsprogs (possibly already released, I didn't check).  It's an experimental
feature that needs to be enabled with "-m reflink=1".

Despite that experimental status, I'd strongly recommend James to test his
tool on xfs as well, as it's the second major implementation of this API[1].


Mark has already included XFS in documentation of duperemove, all that looks
amiss is btrfs-extent-same having an obsolete name.  But then, I never did
any non-superficial tests on XFS, beyond "seems to work".


Meow!

[1]. For some reasons zfs-on-linux guys didn't implement this yet, despite
it being an obvious thing on ZFS.
-- 
A MAP07 (Dead Simple) raspberry tincture recipe: 0.5l 95% alcohol, 1kg
raspberries, 0.4kg sugar; put into a big jar for 1 month.  Filter out and
throw away the fruits (can dump them into a cake, etc), let the drink age
at least 3-6 months.
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

kernel BUG at fs/btrfs/delayed-inode.c

[Juan Orti Alcaine]

Hi, today I got this bug after a power failure.

One file that was being written during the power failure, appeared
after reboot as:

$ ls -la data/0/3833
?-. 1 root root 0 ene  1  1970 data/0/3833

I decided to delete it, but I got this bug after a few seconds and the
system halted, I had to reboot it with the magic SysRq keys.

After the reboot, I saw this file as:

# ls -la
ls: no se puede acceder a '3833': No such file or directory
total 0
drwx--. 1 backup backup 4 nov  7 19:48 .
drwx--. 1 backup backup 2 nov  7 09:05 ..
-?? ? ?  ?  ?? 3833

I've already deleted the affected subvolume, and I have not detected
any more problems. Should I worry?

My versions are:

# uname -a
Linux xenon 4.8.6-300.fc25.x86_64 #1 SMP Tue Nov 1 12:36:38 UTC 2016
x86_64 x86_64 x86_64 GNU/Linux
# rpm -q btrfs-progs
btrfs-progs-4.8.2-2.fc25.x86_64

I've also reported the bug in the Fedora bugzilla:

https://bugzilla.redhat.com/show_bug.cgi?id=1392557


nov 07 19:24:43 xenon kernel: BTRFS error (device sdb7): err add
delayed dir index item(index: 50) into the deletion tree of the
delayed node(root id: 28557, inode id: 265, errno: -17)
nov 07 19:24:43 xenon kernel: [ cut here ]
nov 07 19:24:43 xenon kernel: kernel BUG at fs/btrfs/delayed-inode.c:1561!
nov 07 19:24:43 xenon kernel: invalid opcode:  [#1] SMP
nov 07 19:24:43 xenon kernel: Modules linked in: xfs libcrc32c rfcomm
fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun xt_set
xt_multiport ip_set_hash_ip nf_conntrack_netbios_ns
nf_conntrack_broadcast
nov 07 19:24:43 xenon kernel:  intel_cstate intel_uncore xor
snd_hda_codec_realtek snd_hda_codec_generic snd_hda_codec_hdmi
intel_rapl_perf snd_hda_intel snd_hda_codec uvcvideo snd_usb_audio
snd_hda_core snd_usb
nov 07 19:24:43 xenon kernel: CPU: 0 PID: 10691 Comm: rm Not tainted
4.8.6-300.fc25.x86_64 #1
nov 07 19:24:43 xenon kernel: Hardware name: System manufacturer
System Product Name/P8Z68-V LE, BIOS 4102 09/09/2013
nov 07 19:24:43 xenon kernel: task: 94ccac7a3d00 task.stack:
94cc0bf14000
nov 07 19:24:43 xenon kernel: RIP: 0010:[]
[] btrfs_delete_delayed_dir_index+0x222/0x2c0
[btrfs]
nov 07 19:24:43 xenon kernel: RSP: 0018:94cc0bf17d18  EFLAGS: 00010282
nov 07 19:24:43 xenon kernel: RAX:  RBX:
94ccba0dbdb0 RCX: 
nov 07 19:24:43 xenon kernel: RDX:  RSI:
94cd1ec0e048 RDI: 94cd1ec0e048
nov 07 19:24:43 xenon kernel: RBP: 94cc0bf17d88 R08:
00098b7f R09: 0005
nov 07 19:24:43 xenon kernel: R10: 94cd0703c400 R11:
0483 R12: 94ccba0dbdf8
nov 07 19:24:43 xenon kernel: R13: 94cd071b2800 R14:
0032 R15: 
nov 07 19:24:43 xenon kernel: FS:  7fb8228b5700()
GS:94cd1ec0() knlGS:
nov 07 19:24:43 xenon kernel: CS:  0010 DS:  ES:  CR0: 80050033
nov 07 19:24:43 xenon kernel: CR2: 7f512fa32000 CR3:
00030bd31000 CR4: 000406f0
nov 07 19:24:43 xenon kernel: Stack:
nov 07 19:24:43 xenon kernel:  94cd0ac81a20 94cc0bf17d58
0004 94cd0703c400
nov 07 19:24:43 xenon kernel:  94cd0703c400 0900
6001 0032
nov 07 19:24:43 xenon kernel:  6dbfa5b9 94cc4266da88
94cc6c073d00 3272
nov 07 19:24:43 xenon kernel: Call Trace:
nov 07 19:24:43 xenon kernel:  []
__btrfs_unlink_inode+0x1b1/0x4b0 [btrfs]
nov 07 19:24:43 xenon kernel:  []
btrfs_unlink_inode+0x1c/0x40 [btrfs]
nov 07 19:24:43 xenon kernel:  []
btrfs_unlink+0x6b/0xc0 [btrfs]
nov 07 19:24:43 xenon kernel:  [] vfs_unlink+0x108/0x190
nov 07 19:24:43 xenon kernel:  [] do_unlinkat+0x277/0x2f0
nov 07 19:24:43 xenon kernel:  [] SyS_unlinkat+0x1b/0x30
nov 07 19:24:43 xenon kernel:  []
entry_SYSCALL_64_fastpath+0x1a/0xa4
nov 07 19:24:43 xenon kernel: Code: eb fe ff ff 48 8b 53 10 49 8b bd
f0 01 00 00 41 89 c1 4c 8b 03 48 c7 c6 d0 1b db c0 48 8b 8a 48 03 00
00 4c 89 f2 e8 fe f5 f6 ff <0f> 0b 65 8b 05 8d 96 28 3f 89 c0 48 0f a3
05
nov 07 19:24:43 xenon kernel: RIP  []
btrfs_delete_delayed_dir_index+0x222/0x2c0 [btrfs]
nov 07 19:24:43 xenon kernel:  RSP 
nov 07 19:24:43 xenon kernel: ---[ end trace fb2b0363e7e89d6e ]---


-- 
Juan Orti
https://apuntesderootblog.wordpress.com/
GPG: 61F0 8272 6882 BCA6 3A35  88F6 B630 4B72 DEEB D08B
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: Announcing btrfs-dedupe

[James Pharaoh]

FWIW I have updated my comments about duperemove and also the "caveat" 
section you mentioned in your other mail in the readme.


http://btrfs-dedupe.com

James

On 07/11/16 19:49, James Pharaoh wrote:

Annoyingly I can't find this now, but I definitely remember reading
someone, apparently someone knowledgable, claim that the latest version
of the kernel which I was using at the time, still suffered from issues
regarding the dedupe code.

This was a while ago, and I would be very pleased to hear that there is
high confidence in the current implementation! I'll post a link if I
manage to find the comments.

James

On 07/11/16 18:59, Mark Fasheh wrote:

Hi James,

Re the following text on your project page:

"IMPORTANT CAVEAT — I have read that there are race and/or error
conditions which can cause filesystem corruption in the kernel
implementation of the deduplication ioctl."

Can you expound on that? I'm not aware of any bugs right now but if
there is any it'd absolutely be worth having that info on the btrfs
list.

Thanks,
--Mark


On Sun, Nov 6, 2016 at 7:30 AM, James Pharaoh
 wrote:

Hi all,

I'm pleased to announce my btrfs deduplication utility, written in Rust.
This operates on whole files, is fast, and I believe complements the
existing utilities (duperemove, bedup), which exist currently.

Please visit the homepage for more information:

http://btrfs-dedupe.com

James Pharaoh
--
To unsubscribe from this list: send the line "unsubscribe
linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: Announcing btrfs-dedupe

[James Pharaoh]

Annoyingly I can't find this now, but I definitely remember reading 
someone, apparently someone knowledgable, claim that the latest version 
of the kernel which I was using at the time, still suffered from issues 
regarding the dedupe code.


This was a while ago, and I would be very pleased to hear that there is 
high confidence in the current implementation! I'll post a link if I 
manage to find the comments.


James

On 07/11/16 18:59, Mark Fasheh wrote:

Hi James,

Re the following text on your project page:

"IMPORTANT CAVEAT — I have read that there are race and/or error
conditions which can cause filesystem corruption in the kernel
implementation of the deduplication ioctl."

Can you expound on that? I'm not aware of any bugs right now but if
there is any it'd absolutely be worth having that info on the btrfs
list.

Thanks,
--Mark


On Sun, Nov 6, 2016 at 7:30 AM, James Pharaoh
 wrote:

Hi all,

I'm pleased to announce my btrfs deduplication utility, written in Rust.
This operates on whole files, is fast, and I believe complements the
existing utilities (duperemove, bedup), which exist currently.

Please visit the homepage for more information:

http://btrfs-dedupe.com

James Pharaoh
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v14 02/15] btrfs: fix false enospc for compression

[David Sterba]

On Fri, Nov 04, 2016 at 09:32:51AM +0800, Qu Wenruo wrote:
>  int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end,
> -   struct extent_state **cached_state, int dedupe);
> +   struct extent_state **cached_state, int flag);

During a test merge I've noticed that the 'flag' gets assigned the
btrfs_metadata_reserve_type, please change this function accordingly,
and btrfs_metadata_reserve_type as well. Thanks.

>  int btrfs_set_extent_defrag(struct inode *inode, u64 start, u64 end,
> - struct extent_state **cached_state);
> + struct extent_state **cached_state, int flag);

I'd recommend to separate the btrfs_metadata_reserve_type from "btrfs:
fix false enospc for compression", so we can merge it in advance.
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 0/4] Qgroup comment enhance and balance fix

[David Sterba]

On Tue, Oct 18, 2016 at 09:31:25AM +0800, Qu Wenruo wrote:
> The patchset does the following things:
> 1) Enhance comment for qgroup, rename 2 functions
>Explain the how qgroup works, so new developers won't waste too much
>time digging into the boring codes.
> 
>The qgroup work flow is split into 3 main phrases:
>Reverse, Trace, Account.
>And rename functions like btrfs_qgroup_insert_dirty_extent_record()
>to btrfs_qgroup_trace_extent(), to follow the "Trace" phrase.
> 
>Other function name already follows such schema before.
> 
> 2) Move account_shared_subtree() and account_leaf_items() to qgroup.c
>Such functions are only used by qgroup, so move them to qgroup.c and
>rename them to follow "trace" schema.
> 
> 3) Fix the long standing qgroup balance corruption
>Commit 62b99540a1d91e4 doesn't fix the problem completely.
>It can only handle case that merge_reloc_roots() are all done in one
>transaction.
> 
>If transaction commits during merge_reloc_roots(), the data extents
>will leak again.
> 
>The tree fix is to info qgroup to trace both subtree(tree reloc tree
>and destination fs tree), at replace_path() time.
>Inside  replace_path(), there is one transaction start and end, so we
>must make qgroup to trace both subtrees.
> 
>Thanks for previous work, now we can easily trace subtree, so the fix
>is quite simple now.
> 
>And the cause also makes it easier to create pinpoint test case for
>this bug.
> 
> Qu Wenruo (4):
>   btrfs: qgroup: Add comments explaining how btrfs qgroup works
>   btrfs: qgroup: Rename functions to make it follow
> reserve,trace,account steps
>   btrfs: Expoert and move leaf/subtree qgroup helpers to qgroup.c
>   btrfs: qgroup: Fix qgroup data leaking by using subtree tracing

Added to next queue.
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: Announcing btrfs-dedupe

[Mark Fasheh]

Hi James,

Re the following text on your project page:

"IMPORTANT CAVEAT — I have read that there are race and/or error
conditions which can cause filesystem corruption in the kernel
implementation of the deduplication ioctl."

Can you expound on that? I'm not aware of any bugs right now but if
there is any it'd absolutely be worth having that info on the btrfs
list.

Thanks,
--Mark


On Sun, Nov 6, 2016 at 7:30 AM, James Pharaoh
 wrote:
> Hi all,
>
> I'm pleased to announce my btrfs deduplication utility, written in Rust.
> This operates on whole files, is fast, and I believe complements the
> existing utilities (duperemove, bedup), which exist currently.
>
> Please visit the homepage for more information:
>
> http://btrfs-dedupe.com
>
> James Pharaoh
> --
> To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
> the body of a message to majord...@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v2 1/2] btrfs: Add WARN_ON for qgroup reserved underflow

[David Sterba]

On Thu, Oct 20, 2016 at 10:28:41AM +0800, Qu Wenruo wrote:
> Goldwyn Rodrigues has exposed and fixed a bug which underflows btrfs
> qgroup reserved space, and leads to non-writable fs.
> 
> This reminds us that we don't have enough underflow check for qgroup
> reserved space.
> 
> For underflow case, we should not really underflow the numbers but warn
> and keeps qgroup still work.
> 
> So add more check on qgroup reserved space and add WARN_ON() and
> btrfs_warn() for any underflow case.
> 
> Signed-off-by: Qu Wenruo 

Reviewed-by: David Sterba 
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 2/2] btrfs: Add trace point for qgroup reserved space

[David Sterba]

On Thu, Oct 20, 2016 at 10:28:42AM +0800, Qu Wenruo wrote:
> Introduce the following trace points:
> qgroup_update_reserve
> qgroup_meta_reserve
> 
> And modify the timing of btrfs_qgroup_free_delayed_ref() and
> btrfs_qgroup_release_data() events, to work with qgroup_update_reserve()
> event.

I think this should go to a separate patch, as it's a functional change
(and not obvious to me) unlike adding the tracepoints.
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: Announcing btrfs-dedupe

[Mark Fasheh]

Hi David and James,

On Mon, Nov 7, 2016 at 6:02 AM, David Sterba  wrote:
> On Sun, Nov 06, 2016 at 02:30:52PM +0100, James Pharaoh wrote:
>> I'm pleased to announce my btrfs deduplication utility, written in Rust.
>> This operates on whole files, is fast, and I believe complements the
>> existing utilities (duperemove, bedup), which exist currently.
>
> Mark can correct me if I'm wrong, but AFAIK, duperemove can consume
> output of fdupes, which does the whole file scanning for duplicates. And
> I think adding a whole-file dedup mode to duperemove would be better
> (from user's POV) than writing a whole new tool, eg. because of existing
> availability of duperemove in the distros.

Yeah you are correct - fdupes -r /foo | duperemove --fdupes  will get
you the same effect.

There's been a request for us to do all of that internally so that the
whole file dedupe works with the mtime checking code. This is entirely
doable. I would probably either add a field to the files table or add
a new table to hold whole-file hashes. We can then squeeze down our
existing block hashes into one big one or just rehash the whole file.


> Also looking to your roadmap, some of the items are implemented in
> duperemove: database of existing csums, cross filesystem boundary,
> mtime-based speedups).

Yeah, rescanning based on mtime was a huge speedup for Duperemove as
was keeping checksums in a db. We do all this today, also on XFS with
the dedupe ioctl (I believe this should be out with Linux-4.9).

Btw, there's lots of little details and bug fixes which I feel add up
to a relatively complete (though far from perfect!) tool. For example,
the dedupe code can handle multiple kernel versions including old
kernels which couldn't dedupe on non aligned block boundaries. Every
major step in duperemove is threaded at this point too which has also
been an enormous performance increase (which new features benefit
from).

Thanks,
--Mark

-- 
"When the going gets weird, the weird turn pro."
Hunter S. Thompson
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v2 02/14] btrfs-progs: check: introduce function to find dir_item

[David Sterba]

On Thu, Nov 03, 2016 at 09:58:21AM +0800, Qu Wenruo wrote:
> 
> 
> At 11/02/2016 11:21 PM, David Sterba wrote:
> > On Wed, Sep 21, 2016 at 11:15:52AM +0800, Qu Wenruo wrote:
> >> From: Lu Fengqi 
> >>
> >> Introduce a new function find_dir_item() to find DIR_ITEM for the given
> >> key, and check it with the specified INODE_REF/INODE_EXTREF match.
> >>
> >> Signed-off-by: Lu Fengqi 
> >> Signed-off-by: Qu Wenruo 
> >> ---
> >>  cmds-check.c | 140 
> >> +++
> >>  1 file changed, 140 insertions(+)
> >>
> >> diff --git a/cmds-check.c b/cmds-check.c
> >> index 998ba63..4e25804 100644
> >> --- a/cmds-check.c
> >> +++ b/cmds-check.c
> >> @@ -3848,6 +3848,146 @@ out:
> >>return err;
> >>  }
> >>
> >> +#define ROOT_DIR_ERROR(1<<1)  /* bad root_dir */
> >> +#define DIR_ITEM_MISSING  (1<<2)  /* DIR_ITEM not found */
> >> +#define DIR_ITEM_MISMATCH (1<<3)  /* DIR_ITEM found but not match */
> >
> > What's the reason for another definition of the error codes? They're
> > mostly copied from te I_ERR_* counterparts. I'd rather have one set of
> > error codes.
> 
> The main reason is, in lowmem fsck mode, we are not checking inodes or 
> ref/backref in batch.
> 
> If using I_ERR and REF_ERR, we can mixing them up as they share the same 
> bits.
> 
> So we introduced the new error bitmap, to make sure all error bits won't 
> cover each other.
> 
> It may be better if we rearrange I_ERR/REF_ERR to avoid conflicts.
> E.g, let REF_ERR_ starts from lowest bit and let I_ERR_ starts from high 
> bits.

Yes please. Third namespace for existing error bits is not a good
option. Move the I_ERR bits to start from 32 and use them in the low-mem
code that's been merged to devel.
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] btrfs: make block group flags in balance printks human-readable

[David Sterba]

On Fri, Nov 04, 2016 at 08:26:54AM +0100, Adam Borowski wrote:
> They're not even documented anywhere, letting users with no recourse but
> to RTFS.  It's no big burden to output the bitfield as words.

Ok. Below are some comments (and style nitpicks).

> Also, display unknown flags as hex.
> 
> Signed-off-by: Adam Borowski 
> ---
>  fs/btrfs/relocation.c | 34 --
>  1 file changed, 32 insertions(+), 2 deletions(-)
> 
> diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c
> index 0ec8ffa..388216f 100644
> --- a/fs/btrfs/relocation.c
> +++ b/fs/btrfs/relocation.c
> @@ -4326,6 +4326,34 @@ static struct reloc_control 
> *alloc_reloc_control(struct btrfs_fs_info *fs_info)
>  }
>  
>  /*
> + * explain bit flags, prefixed by a '|' that'll be dropped
> + */
> +static void describe_block_group_flags(char *buf, u64 flags)
> +{
> + if (!flags)

if (!flags) {

> + *buf += sprintf(buf, "|NONE");

Should it be just 'buf'? Otherwise this stores 5 after 'E', which gets
overwritten by 0 at the end but still. Unless I'm missing something.

} else {

> + else {
> +#define DESCRIBE_FLAG(f, d) \
> + if (flags & BTRFS_BLOCK_GROUP_##f) { \
> + buf += sprintf(buf, "|%s", d); \
> + flags &= ~BTRFS_BLOCK_GROUP_##f; \
> + }
> + DESCRIBE_FLAG(DATA, "data");
> + DESCRIBE_FLAG(SYSTEM,   "system");
> + DESCRIBE_FLAG(METADATA, "metadata");
> + DESCRIBE_FLAG(RAID0,"raid0");
> + DESCRIBE_FLAG(RAID1,"raid1");
> + DESCRIBE_FLAG(DUP,  "dup");
> + DESCRIBE_FLAG(RAID10,   "raid10");
> + DESCRIBE_FLAG(RAID5,"raid5");
> + DESCRIBE_FLAG(RAID6,"raid6");

An #undef should go here.

> + if (flags)
> + buf += sprintf(buf, "|0x%llx", flags);
> + }
> + *buf = 0;
> +}
> +
> +/*
>   * function to relocate all extents in a block group.
>   */
>  int btrfs_relocate_block_group(struct btrfs_root *extent_root, u64 
> group_start)
> @@ -4337,6 +4365,7 @@ int btrfs_relocate_block_group(struct btrfs_root 
> *extent_root, u64 group_start)
>   int ret;
>   int rw = 0;
>   int err = 0;
> + char flags_str[128];

This could be a problem. The relocation can trigger deep call chains
when doing IO, that need stack. I'd rather avoid the static buffer,
please switch it to kmalloc. As this is not a critical allocation, the
fallback would be to print just the raw value as now.

Also, please use snprintf. This would need extra variable to track
number of already printed chars, but should be easy, schematically like:

remaining = BUFFERSIZE;

ret = snprintf(buf, remaining, "...");
remaining -= ret;
buf += ret;

>  
>   rc = alloc_reloc_control(fs_info);
>   if (!rc)
> @@ -4381,9 +4410,10 @@ int btrfs_relocate_block_group(struct btrfs_root 
> *extent_root, u64 group_start)
>   goto out;
>   }
>  
> + describe_block_group_flags(flags_str, rc->block_group->flags);
>   btrfs_info(extent_root->fs_info,
> -"relocating block group %llu flags %llu",
> -rc->block_group->key.objectid, rc->block_group->flags);
> +"relocating block group %llu flags %s",
> +rc->block_group->key.objectid, flags_str+1);
>  
>   btrfs_wait_block_group_reservations(rc->block_group);
>   btrfs_wait_nocow_writers(rc->block_group);
> -- 
> 2.10.2
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
> the body of a message to majord...@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH V2] btrfs: Remove some dead code

[David Sterba]

On Fri, Nov 04, 2016 at 08:02:35AM +0100, Christophe JAILLET wrote:
> 'btrfs_iget()' can not return NULL, so this test can be removed.
> 
> Signed-off-by: Christophe JAILLET 

Patch replaced, I've edited the subject to be a bit more specific.
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v2 3/3] btrfs-progs: send: fix handling of -c option

[David Sterba]

On Fri, Nov 04, 2016 at 05:33:58PM +0900, Tsutomu Itoh wrote:
> When two or more -c options are specified, cannot find a suitable
> parent. So, output stream is bigger than correct one.
> 
> [before]
> # btrfs send -f /tmp/data1 -c Snap0 -c ../SnapX Snap[12] ../SnapY
> At subvol Snap1
> At subvol Snap2
> At subvol ../SnapY
> # ls -l /tmp/data1
> -rw--- 1 root root 3153 Oct 19 10:37 /tmp/data1
> #
> 
> [after]
> # btrfs send -f /tmp/data1 -c Snap0 -c ../SnapX Snap[12] ../SnapY
> At subvol Snap1
> At subvol Snap2
> At subvol ../SnapY
> # ls -l /tmp/data1
> -rw--- 1 root root 1492 Oct 19 10:39 /tmp/data1
> #
> 
> Signed-off-by: Tsutomu Itoh 

Applied, thanks.
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] btrfs-progs: send-test: add checking of clone-src option

[David Sterba]

On Fri, Nov 04, 2016 at 05:35:18PM +0900, Tsutomu Itoh wrote:
> +before_size=`ls -l "$here"/send.stream.before | awk '{print $5}'`
> +after_size=`ls -l "$here"/send.stream.after | awk '{print $5}'`

Thanks for the test. Minor fixes: "stat --format=%s" instead of the
above, and /dev/zero instead of urandom and stream redirection so it
works in my NFS setup.
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: btrfs support for filesystems >8TB on 32bit architectures

[Marc MERLIN]

On Mon, Nov 07, 2016 at 02:16:37PM +0800, Qu Wenruo wrote:
> 
> 
> At 11/07/2016 01:36 PM, Marc MERLIN wrote:
> > (sorry for the bad subject line from the mdadm list on the previous mail)
> > 
> > On Mon, Nov 07, 2016 at 12:18:10PM +0800, Qu Wenruo wrote:
> > > I'm totally wrong here.
> > > 
> > > DirectIO needs the 'buf' parameter of read()/pread() to be 512 bytes
> > > aligned.
> > > 
> > > While we are using a lot of stack memory() and normal malloc()/calloc()
> > > allocated memory, which are seldom aligned to 512 bytes.
> > > 
> > > So to *workaround* the problem in btrfs-progs, we may need to change any
> > > pread() caller to use aligned memory allocation.
> > > 
> > > I really don't think David will accept such huge change for a 
> > > workdaround...
> > 
> > Thanks for looking into it.
> > So basically should we just document that btrfs filesystems past 8TB in
> > size are not supported on 32bit architectures?
> > (as in you can mount them and use them I believe, but you cannot create,
> > or repair them)
> > 
> > Marc
> > 
> Add David to this thread.
> 
> For create, it should be OK. As at create time, we hardly write beyond 3G.
> So it won't be a big problem.
> 
> For repair, we do have a possibility that btrfsck can't handle it.
> 
> Anyway, I'd like to see how David thinks what we should do the handle the
> problem.

Understood. One big thing (for me) I forgot to confirm:
1) btrfs receive
2) btrfs scrub
should both be able to work because the IO operations are done directly
inside the kernel and not from user space, correct?

Thanks,
Marc
-- 
"A mouse is a device used to point at the xterm you want to type in" - A.S.R.
Microsoft is to operating systems 
   what McDonalds is to gourmet cooking
Home page: http://marc.merlins.org/ | PGP 1024R/763BE901
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] btrfs: make block group flags in balance printks human-readable

[Holger Hoffstätte]

On 11/04/16 08:26, Adam Borowski wrote:
> They're not even documented anywhere, letting users with no recourse but
> to RTFS.  It's no big burden to output the bitfield as words.
> 
> Also, display unknown flags as hex.
> 
> Signed-off-by: Adam Borowski 

Very helpful and works (for me) as advertised.

Tested-by: Holger Hoffstätte 

> ---
>  fs/btrfs/relocation.c | 34 --
>  1 file changed, 32 insertions(+), 2 deletions(-)
> 
> diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c
> index 0ec8ffa..388216f 100644
> --- a/fs/btrfs/relocation.c
> +++ b/fs/btrfs/relocation.c
> @@ -4326,6 +4326,34 @@ static struct reloc_control 
> *alloc_reloc_control(struct btrfs_fs_info *fs_info)
>  }
>  
>  /*
> + * explain bit flags, prefixed by a '|' that'll be dropped
> + */
> +static void describe_block_group_flags(char *buf, u64 flags)
> +{
> + if (!flags)
> + *buf += sprintf(buf, "|NONE");
> + else {
> +#define DESCRIBE_FLAG(f, d) \
> + if (flags & BTRFS_BLOCK_GROUP_##f) { \
> + buf += sprintf(buf, "|%s", d); \
> + flags &= ~BTRFS_BLOCK_GROUP_##f; \
> + }
> + DESCRIBE_FLAG(DATA, "data");
> + DESCRIBE_FLAG(SYSTEM,   "system");
> + DESCRIBE_FLAG(METADATA, "metadata");
> + DESCRIBE_FLAG(RAID0,"raid0");
> + DESCRIBE_FLAG(RAID1,"raid1");
> + DESCRIBE_FLAG(DUP,  "dup");
> + DESCRIBE_FLAG(RAID10,   "raid10");
> + DESCRIBE_FLAG(RAID5,"raid5");
> + DESCRIBE_FLAG(RAID6,"raid6");
> + if (flags)
> + buf += sprintf(buf, "|0x%llx", flags);
> + }
> + *buf = 0;
> +}
> +
> +/*
>   * function to relocate all extents in a block group.
>   */
>  int btrfs_relocate_block_group(struct btrfs_root *extent_root, u64 
> group_start)
> @@ -4337,6 +4365,7 @@ int btrfs_relocate_block_group(struct btrfs_root 
> *extent_root, u64 group_start)
>   int ret;
>   int rw = 0;
>   int err = 0;
> + char flags_str[128];
>  
>   rc = alloc_reloc_control(fs_info);
>   if (!rc)
> @@ -4381,9 +4410,10 @@ int btrfs_relocate_block_group(struct btrfs_root 
> *extent_root, u64 group_start)
>   goto out;
>   }
>  
> + describe_block_group_flags(flags_str, rc->block_group->flags);
>   btrfs_info(extent_root->fs_info,
> -"relocating block group %llu flags %llu",
> -rc->block_group->key.objectid, rc->block_group->flags);
> +"relocating block group %llu flags %s",
> +rc->block_group->key.objectid, flags_str+1);
>  
>   btrfs_wait_block_group_reservations(rc->block_group);
>   btrfs_wait_nocow_writers(rc->block_group);
> 

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: Announcing btrfs-dedupe

[David Sterba]

On Sun, Nov 06, 2016 at 02:30:52PM +0100, James Pharaoh wrote:
> I'm pleased to announce my btrfs deduplication utility, written in Rust. 
> This operates on whole files, is fast, and I believe complements the 
> existing utilities (duperemove, bedup), which exist currently.

Mark can correct me if I'm wrong, but AFAIK, duperemove can consume
output of fdupes, which does the whole file scanning for duplicates. And
I think adding a whole-file dedup mode to duperemove would be better
(from user's POV) than writing a whole new tool, eg. because of existing
availability of duperemove in the distros.

Also looking to your roadmap, some of the items are implemented in
duperemove: database of existing csums, cross filesystem boundary,
mtime-based speedups).
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v4 2/4] btrfs-progs: introduce new send-dump object

[David Sterba]

On Thu, Nov 03, 2016 at 12:07:32PM +0800, Qu Wenruo wrote:
> +/*
> + * Disable format zero length warning since we use zero length format
> + * for unified function parameters.
> + */
> +#pragma GCC diagnostic ignored "-Wformat-zero-length"

No such tricks should be necessary, NULL instead of "" works.
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v4 1/4] btrfs-progs: utils: Introduce function to escape characters

[David Sterba]

On Thu, Nov 03, 2016 at 12:07:31PM +0800, Qu Wenruo wrote:
> Introduce new function, escape_string_inplace(), to escape specified
> characters in place.
> 
> Signed-off-by: Qu Wenruo 
> ---
>  utils.c | 24 
>  utils.h | 14 ++
>  2 files changed, 38 insertions(+)
> 
> diff --git a/utils.c b/utils.c
> index 3f54245..3c50d84 100644
> --- a/utils.c
> +++ b/utils.c
> @@ -4251,3 +4251,27 @@ unsigned int rand_range(unsigned int upper)
>*/
>   return (unsigned int)(jrand48(rand_seed) % upper);
>  }
> +
> +static void escape_one_char(char *restrict string, char escape)
> +{
> + int i = 0;
> + int j = 0;
> +
> + while (string[j] != '\0') {
> + if (string[j] != escape) {
> + string[i] = string[j];
> + i++;
> + }

So how does this actually escape the characters? I don't see anything
inserted. I think we don't have a common understanding of what's
supposed to be done here. I mean C-string-like escaping. And this cannot
be done inplace, as it increases string length. I've implemented it in a
different way, so this patch is left out. The final stream dump output
is preserved.
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH 2/2] btrfs: increase tickets_id even for failed metadata request

[Wang Xiaoguang]

Not functional change, it just makes codes logic more reasonable, then
at least tickets_id can reflect the number of metadata requests we already
handled.

Signed-off-by: Wang Xiaoguang 
---
 fs/btrfs/extent-tree.c | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c
index 4607af3..d63bf40 100644
--- a/fs/btrfs/extent-tree.c
+++ b/fs/btrfs/extent-tree.c
@@ -4949,14 +4949,16 @@ static inline int need_do_async_reclaim(struct 
btrfs_space_info *space_info,
  >fs_info->fs_state));
 }
 
-static void wake_all_tickets(struct list_head *head)
+static void wake_all_tickets(struct btrfs_space_info *space_info)
 {
struct reserve_ticket *ticket;
+   struct list_head *head = _info->tickets;
 
while (!list_empty(head)) {
ticket = list_first_entry(head, struct reserve_ticket, list);
list_del_init(>list);
ticket->error = -ENOSPC;
+   space_info->tickets_id++;
wake_up(>wait);
}
 }
@@ -5018,7 +5020,7 @@ static void btrfs_async_reclaim_metadata_space(struct 
work_struct *work)
if (flush_state > COMMIT_TRANS) {
commit_cycles++;
if (commit_cycles > 2) {
-   wake_all_tickets(_info->tickets);
+   wake_all_tickets(space_info);
space_info->flush = 0;
} else {
flush_state = FLUSH_DELAYED_ITEMS_NR;
-- 
2.5.0



--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH 1/2] btrfs: add necessary comments about tickets_id

[Wang Xiaoguang]

Tickets_id's name may result in some misunderstandings,  it just indicates
the next ticket will be handled and is not stored per ticket.

Fixes: ce12965 ("btrfs: introduce tickets_id to determine whether
asynchronous metadata reclaim work makes progress")
Signed-off-by: Wang Xiaoguang 
---
 fs/btrfs/ctree.h | 4 
 1 file changed, 4 insertions(+)

diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index 0b8ce2b..9307084 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -429,6 +429,10 @@ struct btrfs_space_info {
struct list_head ro_bgs;
struct list_head priority_tickets;
struct list_head tickets;
+   /*
+* tickets_id just indicates the next ticket will be handled, so note
+* it's not stored per ticket.
+*/
u64 tickets_id;
 
struct rw_semaphore groups_sem;
-- 
2.5.0



--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html