Add a new command 'daxctl-reconfigure-device'. This is used to switch the mode of a dax device between regular 'device_dax' and 'system-memory'. The command also uses the memory hotplug sysfs interfaces to online the newly available memory when converting to 'system-ram', and to attempt to offline the memory when converting back to a DAX device.
Cc: Pavel Tatashin <[email protected]> Cc: Dave Hansen <[email protected]> Cc: Dan Williams <[email protected]> Signed-off-by: Vishal Verma <[email protected]> --- daxctl/Makefile.am | 2 + daxctl/builtin.h | 1 + daxctl/daxctl.c | 1 + daxctl/device.c | 503 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 507 insertions(+) create mode 100644 daxctl/device.c diff --git a/daxctl/Makefile.am b/daxctl/Makefile.am index 94f73f9..66dcc7f 100644 --- a/daxctl/Makefile.am +++ b/daxctl/Makefile.am @@ -15,10 +15,12 @@ daxctl_SOURCES =\ daxctl.c \ list.c \ migrate.c \ + device.c \ ../util/json.c daxctl_LDADD =\ lib/libdaxctl.la \ ../libutil.a \ $(UUID_LIBS) \ + $(KMOD_LIBS) \ $(JSON_LIBS) diff --git a/daxctl/builtin.h b/daxctl/builtin.h index 00ef5e9..756ba2a 100644 --- a/daxctl/builtin.h +++ b/daxctl/builtin.h @@ -6,4 +6,5 @@ struct daxctl_ctx; int cmd_list(int argc, const char **argv, struct daxctl_ctx *ctx); int cmd_migrate(int argc, const char **argv, struct daxctl_ctx *ctx); +int cmd_reconfig_device(int argc, const char **argv, struct daxctl_ctx *ctx); #endif /* _DAXCTL_BUILTIN_H_ */ diff --git a/daxctl/daxctl.c b/daxctl/daxctl.c index 2e41747..e1ba7b8 100644 --- a/daxctl/daxctl.c +++ b/daxctl/daxctl.c @@ -71,6 +71,7 @@ static struct cmd_struct commands[] = { { "list", .d_fn = cmd_list }, { "help", .d_fn = cmd_help }, { "migrate-device-model", .d_fn = cmd_migrate }, + { "reconfigure-device", .d_fn = cmd_reconfig_device }, }; int main(int argc, const char **argv) diff --git a/daxctl/device.c b/daxctl/device.c new file mode 100644 index 0000000..2629804 --- /dev/null +++ b/daxctl/device.c @@ -0,0 +1,503 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2019 Intel Corporation. All rights reserved. */ +#include <stdio.h> +#include <errno.h> +#include <stdlib.h> +#include <syslog.h> +#include <unistd.h> +#include <limits.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/sysmacros.h> +#include <util/json.h> +#include <util/filter.h> +#include <json-c/json.h> +#include <daxctl/libdaxctl.h> +#include <util/parse-options.h> +#include <ccan/array_size/array_size.h> + +static struct { + const char *dev; + const char *mode; + int region_id; + bool no_online; + bool force; + bool human; + bool verbose; +} param = { + .region_id = -1, +}; + +enum dev_mode { + DAXCTL_DEV_MODE_UNKNOWN, + DAXCTL_DEV_MODE_DEVDAX, + DAXCTL_DEV_MODE_RAM, +}; + +static enum dev_mode reconfig_mode = DAXCTL_DEV_MODE_UNKNOWN; +static unsigned long flags; + +enum device_action { + ACTION_RECONFIG, +}; + +#define BASE_OPTIONS() \ +OPT_INTEGER('r', "region", ¶m.region_id, "restrict to the given region"), \ +OPT_BOOLEAN('u', "human", ¶m.human, "use human friendly number formats"), \ +OPT_BOOLEAN('v', "verbose", ¶m.verbose, "emit more debug messages") + +#define RECONFIG_OPTIONS() \ +OPT_STRING('m', "mode", ¶m.mode, "mode", "mode to switch the device to"), \ +OPT_BOOLEAN('N', "no-online", ¶m.no_online, \ + "don't auto-online memory sections"), \ +OPT_BOOLEAN('f', "force", ¶m.force, \ + "attempt to offline memory sections before reconfiguration") + +static const struct option reconfig_options[] = { + BASE_OPTIONS(), + RECONFIG_OPTIONS(), + OPT_END(), +}; + +static const char *parse_device_options(int argc, const char **argv, + enum device_action action, const struct option *options, + const char *usage, struct daxctl_ctx *ctx) +{ + const char * const u[] = { + usage, + NULL + }; + int i, rc = 0; + + argc = parse_options(argc, argv, options, u, 0); + + /* Handle action-agnostic non-option arguments */ + if (argc == 0) { + char *action_string; + + switch (action) { + case ACTION_RECONFIG: + action_string = "reconfigure"; + break; + default: + action_string = "<>"; + break; + } + fprintf(stderr, "specify a device to %s, or \"all\"\n", + action_string); + rc = -EINVAL; + } + for (i = 1; i < argc; i++) { + fprintf(stderr, "unknown extra parameter \"%s\"\n", argv[i]); + rc = -EINVAL; + } + + if (rc) { + usage_with_options(u, options); + return NULL; + } + + /* Handle action-agnostic options */ + if (param.verbose) + daxctl_set_log_priority(ctx, LOG_DEBUG); + if (param.human) + flags |= UTIL_JSON_HUMAN; + + /* Handle action-specific options */ + switch (action) { + case ACTION_RECONFIG: + if (!param.mode) { + fprintf(stderr, "error: a 'mode' option is required\n"); + usage_with_options(u, reconfig_options); + rc = -EINVAL; + } + if (strcmp(param.mode, "system-ram") == 0) { + reconfig_mode = DAXCTL_DEV_MODE_RAM; + } else if (strcmp(param.mode, "devdax") == 0) { + reconfig_mode = DAXCTL_DEV_MODE_DEVDAX; + if (param.no_online) { + fprintf(stderr, + "--no-online is incompatible with --mode=devdax\n"); + rc = -EINVAL; + } + } + break; + } + if (rc) { + usage_with_options(u, options); + return NULL; + } + + return argv[0]; +} + +static int dev_online_memory(struct daxctl_dev *dev) +{ + struct daxctl_memory *mem = daxctl_dev_get_memory(dev); + const char *devname = daxctl_dev_get_devname(dev); + int num_sections, num_on, rc; + + if (!mem) { + fprintf(stderr, "%s: failed to get the memory object\n", + devname); + return -ENXIO; + } + + /* get total number of sections and sections already online */ + num_sections = daxctl_memory_num_sections(mem); + if (num_sections < 0) { + fprintf(stderr, "%s: failed to get number of memory sections\n", + devname); + return num_sections; + } + + num_on = daxctl_memory_is_online(mem); + if (num_on < 0) { + fprintf(stderr, "%s: failed to determine online state: %s\n", + devname, strerror(-num_on)); + return num_on; + } + if (num_on == num_sections) { + fprintf(stderr, "%s: all memory sections (%d) already online\n", + devname, num_on); + return 1; + } + if (num_on > 0) + fprintf(stderr, "%s: %d memory section%s already online\n", + devname, num_on, + num_on == 1 ? "" : "s"); + + /* online the remaining sections */ + rc = daxctl_memory_online(mem); + if (rc < 0) { + fprintf(stderr, "%s: failed to online memory: %s\n", + devname, strerror(-rc)); + return rc; + } + if (param.verbose) + fprintf(stderr, "%s: %d memory section%s onlined\n", devname, rc, + rc == 1 ? "" : "s"); + + /* all sections should now be online */ + num_on = daxctl_memory_is_online(mem); + if (num_on < 0) { + fprintf(stderr, "%s: failed to determine online state: %s\n", + devname, strerror(-num_on)); + return num_on; + } + if (num_on < num_sections) { + fprintf(stderr, "%s: failed to online %d memory sections\n", + devname, num_sections - num_on); + return -ENXIO; + } + + return 0; +} + +static int dev_offline_memory(struct daxctl_dev *dev) +{ + struct daxctl_memory *mem = daxctl_dev_get_memory(dev); + const char *devname = daxctl_dev_get_devname(dev); + int num_sections, num_on, num_off, rc; + + if (!mem) { + fprintf(stderr, "%s: failed to get the memory object\n", + devname); + return -ENXIO; + } + + /* get total number of sections and sections already offline */ + num_sections = daxctl_memory_num_sections(mem); + if (num_sections < 0) { + fprintf(stderr, "%s: failed to get number of memory sections\n", + devname); + return num_sections; + } + + num_on = daxctl_memory_is_online(mem); + if (num_on < 0) { + fprintf(stderr, "%s: failed to determine online state: %s\n", + devname, strerror(-num_on)); + return num_on; + } + + num_off = num_sections - num_on; + if (num_off == num_sections) { + fprintf(stderr, "%s: all memory sections (%d) already offline\n", + devname, num_off); + return 1; + } + if (num_off) + fprintf(stderr, "%s: %d memory section%s already offline\n", + devname, num_off, + num_off == 1 ? "" : "s"); + + /* offline the remaining sections */ + rc = daxctl_memory_offline(mem); + if (rc < 0) { + fprintf(stderr, "%s: failed to offline memory: %s\n", + devname, strerror(-rc)); + return rc; + } + if (param.verbose) + fprintf(stderr, "%s: %d memory section%s offlined\n", devname, rc, + rc == 1 ? "" : "s"); + + /* all sections should now be ofline */ + num_on = daxctl_memory_is_online(mem); + if (num_on < 0) { + fprintf(stderr, "%s: failed to determine online state: %s\n", + devname, strerror(-num_on)); + return num_on; + } + if (num_on) { + fprintf(stderr, "%s: failed to offline %d memory sections\n", + devname, num_on); + return -ENXIO; + } + + return 0; +} + +static int disable_devdax_device(struct daxctl_dev *dev) +{ + struct daxctl_memory *mem = daxctl_dev_get_memory(dev); + const char *devname = daxctl_dev_get_devname(dev); + int rc; + + if (mem) { + fprintf(stderr, "%s was already in system-ram mode\n", + devname); + return 1; + } + rc = daxctl_dev_disable(dev); + if (rc) { + fprintf(stderr, "%s: disable failed: %s\n", + daxctl_dev_get_devname(dev), strerror(-rc)); + return rc; + } + return 0; +} + +static int reconfig_mode_system_ram(struct daxctl_dev *dev) +{ + int rc, skip_enable = 0; + + if (daxctl_dev_is_enabled(dev)) { + rc = disable_devdax_device(dev); + if (rc < 0) + return rc; + if (rc > 0) + skip_enable = 1; + } + + if (!skip_enable) { + rc = daxctl_dev_enable_ram(dev); + if (rc) + return rc; + } + + if (param.no_online) + return 0; + + return dev_online_memory(dev); +} + +static int disable_system_ram_device(struct daxctl_dev *dev) +{ + struct daxctl_memory *mem = daxctl_dev_get_memory(dev); + const char *devname = daxctl_dev_get_devname(dev); + int rc; + + if (!mem) { + fprintf(stderr, "%s was already in devdax mode\n", devname); + return 1; + } + + if (param.force) { + rc = dev_offline_memory(dev); + if (rc < 0) + return rc; + } + + rc = daxctl_memory_is_online(mem); + if (rc < 0) { + fprintf(stderr, "%s: failed to determine online state: %s\n", + devname, strerror(-rc)); + return rc; + } + if (rc > 0) { + if (param.verbose) { + fprintf(stderr, "%s: found %d memory sections online\n", + devname, rc); + fprintf(stderr, "%s: refusing to change modes\n", + devname); + } + return -EBUSY; + } + rc = daxctl_dev_disable(dev); + if (rc) { + fprintf(stderr, "%s: disable failed: %s\n", + daxctl_dev_get_devname(dev), strerror(-rc)); + return rc; + } + return 0; +} + +static int reconfig_mode_devdax(struct daxctl_dev *dev) +{ + int rc; + + if (daxctl_dev_is_enabled(dev)) { + rc = disable_system_ram_device(dev); + if (rc) + return rc; + } + + rc = daxctl_dev_enable_devdax(dev); + if (rc) + return rc; + + return 0; +} + +static int verify_dax_bus_model(struct daxctl_dev *dev) +{ + const char *devname = daxctl_dev_get_devname(dev); + char *dev_path, *subsys_path, *resolved; + struct stat sb; + int rc; + + if (asprintf(&dev_path, "/dev/%s", devname) < 0) + return -ENOMEM; + + rc = lstat(dev_path, &sb); + if (rc < 0) { + rc = -errno; + fprintf(stderr, "%s: stat for %s failed: %s\n", + devname, dev_path, strerror(-rc)); + goto out_dev;; + } + + if (asprintf(&subsys_path, "/sys/dev/char/%d:%d/subsystem", + major(sb.st_rdev), minor(sb.st_rdev)) < 0) { + rc = -ENOMEM; + goto out_dev; + } + + resolved = realpath(subsys_path, NULL); + if (!resolved) { + rc = -errno; + fprintf(stderr, "%s: unable to determine subsys: %s\n", + devname, strerror(errno)); + goto out_subsys; + } + + if (strcmp(resolved, "/sys/bus/dax") == 0) + rc = 0; + else + rc = -ENXIO; + + free(resolved); +out_subsys: + free(subsys_path); +out_dev: + free(dev_path); + return rc; +} + +static int do_reconfig(struct daxctl_dev *dev, enum dev_mode mode, + struct json_object **jdevs) +{ + const char *devname = daxctl_dev_get_devname(dev); + struct json_object *jdev; + int rc = 0; + + if (verify_dax_bus_model(dev) != 0) { + fprintf(stderr, "%s: dax-class device model unsupported\n", + devname); + return -ENXIO; + } + + switch (mode) { + case DAXCTL_DEV_MODE_RAM: + rc = reconfig_mode_system_ram(dev); + break; + case DAXCTL_DEV_MODE_DEVDAX: + rc = reconfig_mode_devdax(dev); + break; + default: + fprintf(stderr, "%s: unknown mode requested: %d\n", + devname, mode); + rc = -EINVAL; + } + + *jdevs = json_object_new_array(); + if (*jdevs) { + jdev = util_daxctl_dev_to_json(dev, flags); + if (jdev) + json_object_array_add(*jdevs, jdev); + } + + return rc; +} + +static int do_xaction_device(const char *device, enum device_action action, + struct daxctl_ctx *ctx, int *processed) +{ + struct json_object *jdevs = NULL; + struct daxctl_region *region; + struct daxctl_dev *dev; + int rc = -ENXIO; + + *processed = 0; + + daxctl_region_foreach(ctx, region) { + if (param.region_id >= 0 && param.region_id + != daxctl_region_get_id(region)) + continue; + + daxctl_dev_foreach(region, dev) { + if (!util_daxctl_dev_filter(dev, device)) + continue; + + switch (action) { + case ACTION_RECONFIG: + rc = do_reconfig(dev, reconfig_mode, &jdevs); + if (rc == 0) + (*processed)++; + break; + default: + rc = -EINVAL; + break; + } + } + } + + /* + * jdevs is the containing json array for all devices we are reporting + * on. It therefore needs to be outside the region/device iterators, + * and passed in to the do_<action> functions to add their objects to + */ + if (jdevs) + util_display_json_array(stdout, jdevs, flags); + + return rc; +} + +int cmd_reconfig_device(int argc, const char **argv, struct daxctl_ctx *ctx) +{ + char *usage = "daxctl reconfigure-device <device> [<options>]"; + const char *device = parse_device_options(argc, argv, ACTION_RECONFIG, + reconfig_options, usage, ctx); + int processed, rc; + + rc = do_xaction_device(device, ACTION_RECONFIG, ctx, &processed); + if (rc < 0) + fprintf(stderr, "error reconfiguring devices: %s\n", + strerror(-rc)); + + fprintf(stderr, "reconfigured %d device%s\n", processed, + processed == 1 ? "" : "s"); + return rc; +} -- 2.20.1 _______________________________________________ Linux-nvdimm mailing list [email protected] https://lists.01.org/mailman/listinfo/linux-nvdimm
