Hi Steven,
JSON output is a feature I have waited for a long time. Thank you! Just
my two cents: why not create your own struct btrfs_send_ops object and
your own set of print_* functions? I think that may be more OOP design.
Regards,
Sheng
On Sat, Jan 16, 2021 at 03:56:06PM +0000, Steven Davies wrote:
> I have seen a few requests from users on other forums for a way to find a
> diff between two snapshots and some scripts have been created[1] which parse
> a btrfs-send stream to get this information.
>
> This patch adds native JSON format output support to the btrfs-receive dump
> command which would make writing such scripts much easier as they can use
> JSON parsing libraries rather than string or binary decoding.
>
> Usage: btrfs send --no-data <subvolume> | btrfs receive --dump-json
>
> Sample output:
> [
> {"operation": "subvol", "path": "./2021-01-16T03:01:36+00:00", "uuid":
> "4f6756a2-c8e8-614b-beaa-483de9a7ed13", "transid": 47864957},
> {"operation": "chown", "path": "./2021-01-16T03:01:36+00:00/", "gid": 0,
> "uid": 0},
> {"operation": "chmod", "path": "./2021-01-16T03:01:36+00:00/", "mode":
> "755"},
> {"operation": "utimes", "path": "./2021-01-16T03:01:36+00:00/", "atime":
> "2021-01-16T03:00:30+0000", "mtime": "1970-01-01T01:00:00+0100", "ctime":
> "2021-01-13T03:00:38+0000"},
> {"operation": "mkdir", "path": "./2021-01-16T03:01:36+00:00/o257-2524-0"},
> {"operation": "rename", "path": "./2021-01-16T03:01:36+00:00/o257-2524-0",
> "dest": "./2021-01-16T03:01:36+00:00/grub"},
> {}
> ]
>
> Patch is against current btrfs-progs master.
>
> [1] https://github.com/sysnux/btrfs-snapshots-diff
>
> Signed-off-by: Steven Davies <[email protected]>
> ---
> cmds/receive-dump.c | 268
> +++++++++++++++++++++++++++++++++++++++++-----------
> cmds/receive-dump.h | 1 +
> cmds/receive.c | 20 +++-
> 3 files changed, 234 insertions(+), 55 deletions(-)
>
> diff --git a/cmds/receive-dump.c b/cmds/receive-dump.c
> index 648d9314..2c1af22e 100644
> --- a/cmds/receive-dump.c
> +++ b/cmds/receive-dump.c
> @@ -51,7 +51,7 @@
> * Returns the length of the escaped characters. Unprintable characters are
> * escaped as octals.
> */
> -static int print_path_escaped(const char *path)
> +static int print_path_escaped(const char *path, int json)
> {
> size_t i;
> size_t path_len = strlen(path);
> @@ -61,27 +61,44 @@ static int print_path_escaped(const char *path)
> char c = path[i];
>
> len++;
> - switch (c) {
> - case '\a': putchar('\\'); putchar('a'); len++; break;
> - case '\b': putchar('\\'); putchar('b'); len++; break;
> - case '\e': putchar('\\'); putchar('e'); len++; break;
> - case '\f': putchar('\\'); putchar('f'); len++; break;
> - case '\n': putchar('\\'); putchar('n'); len++; break;
> - case '\r': putchar('\\'); putchar('r'); len++; break;
> - case '\t': putchar('\\'); putchar('t'); len++; break;
> - case '\v': putchar('\\'); putchar('v'); len++; break;
> - case ' ': putchar('\\'); putchar(' '); len++; break;
> - case '\\': putchar('\\'); putchar('\\'); len++; break;
> - default:
> - if (!isprint(c)) {
> - printf("\\%c%c%c",
> - '0' + ((c & 0300) >> 6),
> - '0' + ((c & 070) >> 3),
> - '0' + (c & 07));
> - len += 3;
> - } else {
> - putchar(c);
> - }
> + if (!json) {
> + switch (c) {
> + case '\a': putchar('\\'); putchar('a'); len++; break;
> + case '\b': putchar('\\'); putchar('b'); len++; break;
> + case '\e': putchar('\\'); putchar('e'); len++; break;
> + case '\f': putchar('\\'); putchar('f'); len++; break;
> + case '\n': putchar('\\'); putchar('n'); len++; break;
> + case '\r': putchar('\\'); putchar('r'); len++; break;
> + case '\t': putchar('\\'); putchar('t'); len++; break;
> + case '\v': putchar('\\'); putchar('v'); len++; break;
> + case ' ': putchar('\\'); putchar(' '); len++; break;
> + case '\\': putchar('\\'); putchar('\\'); len++; break;
> + default:
> + if (!isprint(c)) {
> + printf("\\%c%c%c",
> + '0' + ((c & 0300) >>
> 6),
> + '0' + ((c & 070) >>
> 3),
> + '0' + (c & 07));
> + len += 3;
> + } else {
> + putchar(c);
> + }
> + }
> + } else {
> + if (c < 0x20) {
> + printf("\\u%04x", c);
> + len += 5;
> + } else if (c == '\\') {
> + putchar('\\');
> + putchar('\\');
> + len++;
> + } else if (c == '"') {
> + putchar('\\');
> + putchar('"');
> + len++;
> + } else {
> + putchar(c);
> + }
> }
> }
> return len;
> @@ -109,22 +126,41 @@ static int __print_dump(int subvol, void *user, const
> char *path,
> out_path = full_path;
> }
>
> - /* Unified header */
> - printf("%-16s", title);
> - ret = print_path_escaped(out_path);
> - if (!fmt) {
> +
> + if (!r->json) {
> + /* Unified header */
> + printf("%-16s", title);
> + ret = print_path_escaped(out_path, r->json);
> + if (!fmt) {
> + putchar('\n');
> + return 0;
> + }
> + /* Short paths are aligned to 32 chars; longer paths get a
> single space */
> + do {
> + putchar(' ');
> + } while (++ret < 32);
> + va_start(args, fmt);
> + /* Operation specified ones */
> + vprintf(fmt, args);
> + va_end(args);
> putchar('\n');
> - return 0;
> - }
> - /* Short paths are aligned to 32 chars; longer paths get a single space
> */
> - do {
> + } else {
> + /* Unified header */
> + printf(" {\"operation\": \"%s\", \"path\": \"", title);
> + ret = print_path_escaped(out_path, r->json);
> + putchar('"');
> + if (!fmt) {
> + printf("},\n");
> + return 0;
> + }
> + putchar(',');
> putchar(' ');
> - } while (++ret < 32);
> - va_start(args, fmt);
> - /* Operation specified ones */
> - vprintf(fmt, args);
> - va_end(args);
> - putchar('\n');
> + va_start(args, fmt);
> + /* Operation specified ones */
> + vprintf(fmt, args);
> + va_end(args);
> + printf("},\n");
> + }
> return 0;
> }
>
> @@ -140,11 +176,18 @@ static int print_subvol(const char *path, const u8
> *uuid, u64 ctransid,
> void *user)
> {
> char uuid_str[BTRFS_UUID_UNPARSED_SIZE];
> + struct btrfs_dump_send_args *r = user;
>
> uuid_unparse(uuid, uuid_str);
>
> - return PRINT_DUMP_SUBVOL(user, path, "subvol", "uuid=%s transid=%llu",
> - uuid_str, ctransid);
> + if (!r->json) {
> + return PRINT_DUMP_SUBVOL(user, path, "subvol",
> + "uuid=%s transid=%llu", uuid_str, ctransid);
> + } else {
> + return PRINT_DUMP_SUBVOL(user, path, "subvol",
> + "\"uuid\": \"%s\", \"transid\": %llu",
> + uuid_str, ctransid);
> + }
> }
>
> static int print_snapshot(const char *path, const u8 *uuid, u64 ctransid,
> @@ -154,14 +197,23 @@ static int print_snapshot(const char *path, const u8
> *uuid, u64 ctransid,
> char uuid_str[BTRFS_UUID_UNPARSED_SIZE];
> char parent_uuid_str[BTRFS_UUID_UNPARSED_SIZE];
> int ret;
> + struct btrfs_dump_send_args *r = user;
>
> uuid_unparse(uuid, uuid_str);
> uuid_unparse(parent_uuid, parent_uuid_str);
>
> - ret = PRINT_DUMP_SUBVOL(user, path, "snapshot",
> - "uuid=%s transid=%llu parent_uuid=%s parent_transid=%llu",
> + if (r->json) {
> + ret = PRINT_DUMP_SUBVOL(user, path, "snapshot",
> + "uuid=%s transid=%llu parent_uuid=%s
> parent_transid=%llu",
> uuid_str, ctransid, parent_uuid_str,
> parent_ctransid);
> + } else {
> + ret = PRINT_DUMP_SUBVOL(user, path, "snapshot",
> + "\"uuid\": \"%s\", \"transid\": %llu, \"parent_uuid\":
> \"%s\", \
> + \"parent_transid\": %llu",
> + uuid_str, ctransid, parent_uuid_str,
> + parent_ctransid);
> + }
> return ret;
> }
>
> @@ -177,8 +229,16 @@ static int print_mkdir(const char *path, void *user)
>
> static int print_mknod(const char *path, u64 mode, u64 dev, void *user)
> {
> - return PRINT_DUMP(user, path, "mknod", "mode=%llo dev=0x%llx", mode,
> - dev);
> + struct btrfs_dump_send_args *r = user;
> +
> + if (!r->json) {
> + return PRINT_DUMP(user, path, "mknod", "mode=%llo dev=0x%llx",
> + mode, dev);
> + } else {
> + return PRINT_DUMP(user, path,
> + "mknod", "\"mode\": \"%llo\", \"dev\": \"0x%llx\"",
> + mode, dev);
> + }
> }
>
> static int print_mkfifo(const char *path, void *user)
> @@ -203,12 +263,26 @@ static int print_rename(const char *from, const char
> *to, void *user)
> int ret;
>
> PATH_CAT_OR_RET("rename", full_to, r->full_subvol_path, to, ret);
> - return PRINT_DUMP(user, from, "rename", "dest=%s", full_to);
> + if (!r->json) {
> + ret = PRINT_DUMP(user, from, "rename", "dest=%s", full_to);
> + } else {
> + ret = PRINT_DUMP(user, from, "rename", "\"dest\": \"%s\"",
> + full_to);
> + }
> + return ret;
> }
>
> static int print_link(const char *path, const char *lnk, void *user)
> {
> - return PRINT_DUMP(user, path, "link", "dest=%s", lnk);
> + struct btrfs_dump_send_args *r = user;
> + int ret;
> +
> + if (!r->json) {
> + ret = PRINT_DUMP(user, path, "link", "dest=%s", lnk);
> + } else {
> + ret = PRINT_DUMP(user, path, "link", "\"dest\": \"%s\"", lnk);
> + }
> + return ret;
> }
>
> static int print_unlink(const char *path, void *user)
> @@ -224,8 +298,18 @@ static int print_rmdir(const char *path, void *user)
> static int print_write(const char *path, const void *data, u64 offset,
> u64 len, void *user)
> {
> - return PRINT_DUMP(user, path, "write", "offset=%llu len=%llu",
> + struct btrfs_dump_send_args *r = user;
> + int ret;
> +
> + if (!r->json) {
> + ret = PRINT_DUMP(user, path, "write", "offset=%llu len=%llu",
> + offset, len);
> + } else {
> + ret = PRINT_DUMP(user, path,
> + "write", "\"offset\": %llu, \"len\": %llu",
> offset, len);
> + }
> + return ret;
> }
>
> static int print_clone(const char *path, u64 offset, u64 len,
> @@ -239,37 +323,93 @@ static int print_clone(const char *path, u64 offset,
> u64 len,
>
> PATH_CAT_OR_RET("clone", full_path, r->full_subvol_path, clone_path,
> ret);
> - return PRINT_DUMP(user, path, "clone",
> +
> + if (!r->json) {
> + ret = PRINT_DUMP(user, path, "clone",
> "offset=%llu len=%llu from=%s clone_offset=%llu",
> offset, len, full_path, clone_offset);
> + } else {
> + ret = PRINT_DUMP(user, path, "clone",
> + "\"offset\": %llu, \"len\": %llu, \"from\": \"%s\""
> + ", \"clone_offset\": %llu",
> + offset, len, full_path, clone_offset);
> + }
> + return ret;
> }
>
> static int print_set_xattr(const char *path, const char *name,
> const void *data, int len, void *user)
> {
> - return PRINT_DUMP(user, path, "set_xattr", "name=%s data=%.*s len=%d",
> + struct btrfs_dump_send_args *r = user;
> + int ret;
> +
> + if (!r->json) {
> + ret = PRINT_DUMP(user, path,
> + "set_xattr", "name=%s data=%.*s len=%d",
> + name, len, (char *)data, len);
> + } else {
> + ret = PRINT_DUMP(user, path,
> + "set_xattr", "\"name\": \"%s\""
> + ", \"data\": \"%.*s\", \"len\": %d",
> name, len, (char *)data, len);
> + }
> + return ret;
> }
>
> static int print_remove_xattr(const char *path, const char *name, void *user)
> {
> + struct btrfs_dump_send_args *r = user;
> + int ret;
>
> - return PRINT_DUMP(user, path, "remove_xattr", "name=%s", name);
> + if (!r->json) {
> + ret = PRINT_DUMP(user, path, "remove_xattr", "name=%s", name);
> + } else {
> + ret = PRINT_DUMP(user, path, "remove_xattr",
> + "\"name\": \"%s\"", name);
> + }
> + return ret;
> }
>
> static int print_truncate(const char *path, u64 size, void *user)
> {
> - return PRINT_DUMP(user, path, "truncate", "size=%llu", size);
> + struct btrfs_dump_send_args *r = user;
> + int ret;
> +
> + if (!r->json) {
> + ret = PRINT_DUMP(user, path, "truncate", "size=%llu", size);
> + } else {
> + ret = PRINT_DUMP(user, path, "truncate", "\"size\": %llu",
> size);
> + }
> + return ret;
> }
>
> static int print_chmod(const char *path, u64 mode, void *user)
> {
> - return PRINT_DUMP(user, path, "chmod", "mode=%llo", mode);
> + struct btrfs_dump_send_args *r = user;
> + int ret;
> +
> + if (!r->json) {
> + ret = PRINT_DUMP(user, path, "chmod", "mode=%llo", mode);
> + } else {
> + ret = PRINT_DUMP(user, path, "chmod", "\"mode\": \"%llo\"",
> mode);
> + }
> + return ret;
> }
>
> static int print_chown(const char *path, u64 uid, u64 gid, void *user)
> {
> - return PRINT_DUMP(user, path, "chown", "gid=%llu uid=%llu", gid, uid);
> + struct btrfs_dump_send_args *r = user;
> + int ret;
> +
> + if (!r->json) {
> + ret = PRINT_DUMP(user, path, "chown", "gid=%llu uid=%llu",
> + gid, uid);
> + } else {
> + ret = PRINT_DUMP(user, path, "chown",
> + "\"gid\": %llu, \"uid\": %llu",
> + gid, uid);
> + }
> + return ret;
> }
>
> static int sprintf_timespec(struct timespec *ts, char *dest, int max_size)
> @@ -297,6 +437,8 @@ static int print_utimes(const char *path, struct timespec
> *at,
> struct timespec *mt, struct timespec *ct,
> void *user)
> {
> + struct btrfs_dump_send_args *r = user;
> + int ret;
> char at_str[TIME_STRING_MAX];
> char mt_str[TIME_STRING_MAX];
> char ct_str[TIME_STRING_MAX];
> @@ -305,15 +447,33 @@ static int print_utimes(const char *path, struct
> timespec *at,
> sprintf_timespec(mt, mt_str, TIME_STRING_MAX - 1) < 0 ||
> sprintf_timespec(ct, ct_str, TIME_STRING_MAX - 1) < 0)
> return -EINVAL;
> - return PRINT_DUMP(user, path, "utimes", "atime=%s mtime=%s ctime=%s",
> + if (!r->json) {
> + ret = PRINT_DUMP(user, path,
> + "utimes", "atime=%s mtime=%s ctime=%s",
> + at_str, mt_str, ct_str);
> + } else {
> + ret = PRINT_DUMP(user, path, "utimes",
> + "\"atime\": \"%s\", \"mtime\": \"%s\", " \
> + "\"ctime\": \"%s\"",
> at_str, mt_str, ct_str);
> + }
> + return ret;
> }
>
> static int print_update_extent(const char *path, u64 offset, u64 len,
> void *user)
> {
> - return PRINT_DUMP(user, path, "update_extent", "offset=%llu len=%llu",
> - offset, len);
> + struct btrfs_dump_send_args *r = user;
> + int ret;
> +
> + if (!r->json) {
> + ret = PRINT_DUMP(user, path, "update_extent",
> + "offset=%llu len=%llu", offset, len);
> + } else {
> + ret = PRINT_DUMP(user, path, "update_extent",
> + "\"offset\": %llu, \"len\": %llu", offset, len);
> + }
> + return ret;
> }
>
> struct btrfs_send_ops btrfs_print_send_ops = {
> diff --git a/cmds/receive-dump.h b/cmds/receive-dump.h
> index 06a61085..d62462d0 100644
> --- a/cmds/receive-dump.h
> +++ b/cmds/receive-dump.h
> @@ -22,6 +22,7 @@
> struct btrfs_dump_send_args {
> char full_subvol_path[PATH_MAX];
> char root_path[PATH_MAX];
> + int json;
> };
>
> extern struct btrfs_send_ops btrfs_print_send_ops;
> diff --git a/cmds/receive.c b/cmds/receive.c
> index 2aaba3ff..e977e9db 100644
> --- a/cmds/receive.c
> +++ b/cmds/receive.c
> @@ -1244,6 +1244,8 @@ static const char * const cmd_receive_usage[] = {
> " this file system is mounted.",
> "--dump dump stream metadata, one line per operation,",
> " does not require the MOUNT parameter",
> + "--dump-json dump stream metadata in JSON format,",
> + " does not require the MOUNT parameter",
> "-v deprecated, alias for global -v option",
> HELPINFO_INSERT_GLOBALS,
> HELPINFO_INSERT_VERBOSE,
> @@ -1260,6 +1262,7 @@ static int cmd_receive(const struct cmd_struct *cmd,
> int argc, char **argv)
> int receive_fd = fileno(stdin);
> u64 max_errors = 1;
> int dump = 0;
> + int dump_json = 0;
> int ret = 0;
>
> memset(&rctx, 0, sizeof(rctx));
> @@ -1285,11 +1288,13 @@ static int cmd_receive(const struct cmd_struct *cmd,
> int argc, char **argv)
> optind = 0;
> while (1) {
> int c;
> - enum { GETOPT_VAL_DUMP = 257 };
> + enum { GETOPT_VAL_DUMP = 257,
> + GETOPT_VAL_DUMP_JSON = 258 };
> static const struct option long_opts[] = {
> { "max-errors", required_argument, NULL, 'E' },
> { "chroot", no_argument, NULL, 'C' },
> { "dump", no_argument, NULL, GETOPT_VAL_DUMP },
> + { "dump-json", no_argument, NULL, GETOPT_VAL_DUMP_JSON
> },
> { "quiet", no_argument, NULL, 'q' },
> { NULL, 0, NULL, 0 }
> };
> @@ -1333,6 +1338,10 @@ static int cmd_receive(const struct cmd_struct *cmd,
> int argc, char **argv)
> case GETOPT_VAL_DUMP:
> dump = 1;
> break;
> + case GETOPT_VAL_DUMP_JSON:
> + dump = 1;
> + dump_json = 1;
> + break;
> default:
> usage_unknown_option(cmd, argv);
> }
> @@ -1360,12 +1369,21 @@ static int cmd_receive(const struct cmd_struct *cmd,
> int argc, char **argv)
> dump_args.root_path[1] = '\0';
> dump_args.full_subvol_path[0] = '.';
> dump_args.full_subvol_path[1] = '\0';
> + dump_args.json = dump_json;
> + if (dump_json) {
> + putchar('[');
> + putchar('\n');
> + }
> ret = btrfs_read_and_process_send_stream(receive_fd,
> &btrfs_print_send_ops, &dump_args, 0, max_errors);
> if (ret < 0) {
> errno = -ret;
> error("failed to dump the send stream: %m");
> }
> + if (dump_json) {
> + //Add an empty record so there isn't a trailing ,
> + printf(" {}\n]");
> + }
> } else {
> ret = do_receive(&rctx, tomnt, realmnt, receive_fd, max_errors);
> }
