Memory errors are at least somewhat more likely on disaggregated memory than on-board memory. This commit registers to be notified by fsdev_dax in the event that a memory failure is detected.
When a file access resolves to a daxdev with memory errors, it will fail with an appropriate error. If a daxdev failed fs_dax_get(), we set dd->dax_err. If a daxdev called our notify_failure(), set dd->error. When any of the above happens, set (file)->error and stop allowing access. In general, the recovery from memory errors is to unmount the file system and re-initialize the memory, but there may be usable degraded modes of operation - particularly in the future when famfs supports file systems backed by more than one daxdev. In those cases, accessing data that is on a working daxdev can still work. For now, return errors for any file that has encountered a memory or dax error. Signed-off-by: John Groves <[email protected]> --- fs/fuse/famfs.c | 115 +++++++++++++++++++++++++++++++++++++++--- fs/fuse/famfs_kfmap.h | 3 +- 2 files changed, 109 insertions(+), 9 deletions(-) diff --git a/fs/fuse/famfs.c b/fs/fuse/famfs.c index c02b14789c6e..4eb87c5c628e 100644 --- a/fs/fuse/famfs.c +++ b/fs/fuse/famfs.c @@ -20,6 +20,26 @@ #include "famfs_kfmap.h" #include "fuse_i.h" +static void famfs_set_daxdev_err( + struct fuse_conn *fc, struct dax_device *dax_devp); + +static int +famfs_dax_notify_failure(struct dax_device *dax_devp, u64 offset, + u64 len, int mf_flags) +{ + struct fuse_conn *fc = dax_holder(dax_devp); + + famfs_set_daxdev_err(fc, dax_devp); + + return 0; +} + +static const struct dax_holder_operations famfs_fuse_dax_holder_ops = { + .notify_failure = famfs_dax_notify_failure, +}; + +/*****************************************************************************/ + /* * famfs_teardown() * @@ -48,9 +68,12 @@ famfs_teardown(struct fuse_conn *fc) if (!dd->valid) continue; - /* Release reference from dax_dev_get() */ - if (dd->devp) + /* Only call fs_put_dax if fs_dax_get succeeded */ + if (dd->devp) { + if (!dd->dax_err) + fs_put_dax(dd->devp, fc); put_dax(dd->devp); + } kfree(dd->name); } @@ -174,6 +197,17 @@ famfs_fuse_get_daxdev(struct fuse_mount *fm, const u64 index) goto out; } + err = fs_dax_get(daxdev->devp, fc, &famfs_fuse_dax_holder_ops); + if (err) { + /* If fs_dax_get() fails, we don't attempt recovery; + * We mark the daxdev valid with dax_err + */ + daxdev->dax_err = 1; + pr_err("%s: fs_dax_get(%lld) failed\n", + __func__, (u64)daxdev->devno); + err = -EBUSY; + } + daxdev->name = kstrdup(daxdev_out.name, GFP_KERNEL); wmb(); /* all daxdev fields must be visible before marking it valid */ daxdev->valid = 1; @@ -254,6 +288,38 @@ famfs_update_daxdev_table( return 0; } +static void +famfs_set_daxdev_err( + struct fuse_conn *fc, + struct dax_device *dax_devp) +{ + int i; + + /* Gotta search the list by dax_devp; + * read lock because we're not adding or removing daxdev entries + */ + down_read(&fc->famfs_devlist_sem); + for (i = 0; i < fc->dax_devlist->nslots; i++) { + if (fc->dax_devlist->devlist[i].valid) { + struct famfs_daxdev *dd = &fc->dax_devlist->devlist[i]; + + if (dd->devp != dax_devp) + continue; + + dd->error = true; + up_read(&fc->famfs_devlist_sem); + + pr_err("%s: memory error on daxdev %s (%d)\n", + __func__, dd->name, i); + goto done; + } + } + up_read(&fc->famfs_devlist_sem); + pr_err("%s: memory err on unrecognized daxdev\n", __func__); + +done: +} + /***************************************************************************/ void @@ -611,6 +677,26 @@ famfs_file_init_dax( static ssize_t famfs_file_bad(struct inode *inode); +static int famfs_dax_err(struct famfs_daxdev *dd) +{ + if (!dd->valid) { + pr_err("%s: daxdev=%s invalid\n", + __func__, dd->name); + return -EIO; + } + if (dd->dax_err) { + pr_err("%s: daxdev=%s dax_err\n", + __func__, dd->name); + return -EIO; + } + if (dd->error) { + pr_err("%s: daxdev=%s memory error\n", + __func__, dd->name); + return -EHWPOISON; + } + return 0; +} + static int famfs_interleave_fileofs_to_daxofs(struct inode *inode, struct iomap *iomap, loff_t file_offset, off_t len, unsigned int flags) @@ -648,6 +734,7 @@ famfs_interleave_fileofs_to_daxofs(struct inode *inode, struct iomap *iomap, /* Is the data is in this striped extent? */ if (local_offset < ext_size) { + struct famfs_daxdev *dd; u64 chunk_num = local_offset / chunk_size; u64 chunk_offset = local_offset % chunk_size; u64 stripe_num = chunk_num / nstrips; @@ -656,6 +743,7 @@ famfs_interleave_fileofs_to_daxofs(struct inode *inode, struct iomap *iomap, u64 strip_offset = chunk_offset + (stripe_num * chunk_size); u64 strip_dax_ofs = fei->ie_strips[strip_num].ext_offset; u64 strip_devidx = fei->ie_strips[strip_num].dev_index; + int rc; if (strip_devidx >= fc->dax_devlist->nslots) { pr_err("%s: strip_devidx %llu >= nslots %d\n", @@ -670,6 +758,15 @@ famfs_interleave_fileofs_to_daxofs(struct inode *inode, struct iomap *iomap, goto err_out; } + dd = &fc->dax_devlist->devlist[strip_devidx]; + + rc = famfs_dax_err(dd); + if (rc) { + /* Shut down access to this file */ + meta->error = true; + return rc; + } + iomap->addr = strip_dax_ofs + strip_offset; iomap->offset = file_offset; iomap->length = min_t(loff_t, len, chunk_remainder); @@ -767,6 +864,7 @@ famfs_fileofs_to_daxofs(struct inode *inode, struct iomap *iomap, if (local_offset < dax_ext_len) { loff_t ext_len_remainder = dax_ext_len - local_offset; struct famfs_daxdev *dd; + int rc; if (daxdev_idx >= fc->dax_devlist->nslots) { pr_err("%s: daxdev_idx %llu >= nslots %d\n", @@ -777,11 +875,11 @@ famfs_fileofs_to_daxofs(struct inode *inode, struct iomap *iomap, dd = &fc->dax_devlist->devlist[daxdev_idx]; - if (!dd->valid || dd->error) { - pr_err("%s: daxdev=%lld %s\n", __func__, - daxdev_idx, - dd->valid ? "error" : "invalid"); - goto err_out; + rc = famfs_dax_err(dd); + if (rc) { + /* Shut down access to this file */ + meta->error = true; + return rc; } /* @@ -966,7 +1064,8 @@ famfs_file_bad(struct inode *inode) return -EIO; } if (meta->error) { - pr_debug("%s: previously detected metadata errors\n", __func__); + pr_debug("%s: previously detected metadata errors\n", + __func__); return -EIO; } if (i_size != meta->file_size) { diff --git a/fs/fuse/famfs_kfmap.h b/fs/fuse/famfs_kfmap.h index e76b9057a1e0..6a6420bdff48 100644 --- a/fs/fuse/famfs_kfmap.h +++ b/fs/fuse/famfs_kfmap.h @@ -73,7 +73,8 @@ struct famfs_file_meta { struct famfs_daxdev { /* Include dev uuid? */ bool valid; - bool error; + bool error; /* Dax has reported a memory error (probably poison) */ + bool dax_err; /* fs_dax_get() failed */ dev_t devno; struct dax_device *devp; char *name; -- 2.49.0
