iorequest_cnt and iodone_cnt are updated on every command dispatch and completion, often from different CPUs on high queue depth workloads. Using adjacent atomic_t fields causes cache line contention between the submission and completion paths.
Extend the same treatment to ioerr_cnt and iotmo_cnt so all four iostat counters in struct scsi_device use struct percpu_counter. Suggested-by: John Garry <[email protected]> Signed-off-by: Sumit Saxena <[email protected]> --- drivers/scsi/scsi_error.c | 4 ++-- drivers/scsi/scsi_lib.c | 10 +++++----- drivers/scsi/scsi_scan.c | 8 ++++++++ drivers/scsi/scsi_sysfs.c | 23 ++++++++++++++--------- drivers/scsi/sd.c | 2 +- include/scsi/scsi_device.h | 9 +++++---- 6 files changed, 35 insertions(+), 21 deletions(-) diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c index 147127fb4db9..b1aa7da2ba7c 100644 --- a/drivers/scsi/scsi_error.c +++ b/drivers/scsi/scsi_error.c @@ -349,7 +349,7 @@ enum blk_eh_timer_return scsi_timeout(struct request *req) trace_scsi_dispatch_cmd_timeout(scmd); scsi_log_completion(scmd, TIMEOUT_ERROR); - atomic_inc(&scmd->device->iotmo_cnt); + percpu_counter_inc(&scmd->device->iotmo_cnt); if (host->eh_deadline != -1 && !host->last_reset) host->last_reset = jiffies; @@ -370,7 +370,7 @@ enum blk_eh_timer_return scsi_timeout(struct request *req) */ if (test_and_set_bit(SCMD_STATE_COMPLETE, &scmd->state)) return BLK_EH_DONE; - atomic_inc(&scmd->device->iodone_cnt); + percpu_counter_inc(&scmd->device->iodone_cnt); if (scsi_abort_command(scmd) != SUCCESS) { set_host_byte(scmd, DID_TIME_OUT); scsi_eh_scmd_add(scmd); diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 6e8c7a42603e..979fdace33ac 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1554,9 +1554,9 @@ static void scsi_complete(struct request *rq) INIT_LIST_HEAD(&cmd->eh_entry); - atomic_inc(&cmd->device->iodone_cnt); + percpu_counter_inc(&cmd->device->iodone_cnt); if (cmd->result) - atomic_inc(&cmd->device->ioerr_cnt); + percpu_counter_inc(&cmd->device->ioerr_cnt); disposition = scsi_decide_disposition(cmd); if (disposition != SUCCESS && scsi_cmd_runtime_exceeced(cmd)) @@ -1592,7 +1592,7 @@ static enum scsi_qc_status scsi_dispatch_cmd(struct scsi_cmnd *cmd) struct Scsi_Host *host = cmd->device->host; int rtn = 0; - atomic_inc(&cmd->device->iorequest_cnt); + percpu_counter_inc(&cmd->device->iorequest_cnt); /* check if the device is still usable */ if (unlikely(cmd->device->sdev_state == SDEV_DEL)) { @@ -1614,7 +1614,7 @@ static enum scsi_qc_status scsi_dispatch_cmd(struct scsi_cmnd *cmd) */ SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd, "queuecommand : device blocked\n")); - atomic_dec(&cmd->device->iorequest_cnt); + percpu_counter_dec(&cmd->device->iorequest_cnt); return SCSI_MLQUEUE_DEVICE_BUSY; } @@ -1647,7 +1647,7 @@ static enum scsi_qc_status scsi_dispatch_cmd(struct scsi_cmnd *cmd) trace_scsi_dispatch_cmd_start(cmd); rtn = host->hostt->queuecommand(host, cmd); if (rtn) { - atomic_dec(&cmd->device->iorequest_cnt); + percpu_counter_dec(&cmd->device->iorequest_cnt); trace_scsi_dispatch_cmd_error(cmd, rtn); if (rtn != SCSI_MLQUEUE_DEVICE_BUSY && rtn != SCSI_MLQUEUE_TARGET_BUSY) diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index 121a14d5fdb8..bc885c72f01e 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -350,6 +350,14 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget, scsi_sysfs_device_initialize(sdev); + if (percpu_counter_init(&sdev->iorequest_cnt, 0, GFP_KERNEL) || + percpu_counter_init(&sdev->iodone_cnt, 0, GFP_KERNEL) || + percpu_counter_init(&sdev->ioerr_cnt, 0, GFP_KERNEL) || + percpu_counter_init(&sdev->iotmo_cnt, 0, GFP_KERNEL)) { + ret = -ENOMEM; + goto out_device_destroy; + } + if (scsi_device_is_pseudo_dev(sdev)) return sdev; diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c index dfc3559e7e04..f652edd16497 100644 --- a/drivers/scsi/scsi_sysfs.c +++ b/drivers/scsi/scsi_sysfs.c @@ -516,6 +516,10 @@ static void scsi_device_dev_release(struct device *dev) if (vpd_pgb7) kfree_rcu(vpd_pgb7, rcu); kfree(sdev->inquiry); + percpu_counter_destroy(&sdev->iotmo_cnt); + percpu_counter_destroy(&sdev->ioerr_cnt); + percpu_counter_destroy(&sdev->iodone_cnt); + percpu_counter_destroy(&sdev->iorequest_cnt); kfree(sdev); if (parent) @@ -936,26 +940,27 @@ static ssize_t show_iostat_counterbits(struct device *dev, struct device_attribute *attr, char *buf) { - return snprintf(buf, 20, "%d\n", (int)sizeof(atomic_t) * 8); + /* iostat counters are per-CPU sums (s64). Report width for tools. */ + return sysfs_emit(buf, "%zu\n", sizeof(s64) * 8); } static DEVICE_ATTR(iocounterbits, S_IRUGO, show_iostat_counterbits, NULL); -#define show_sdev_iostat(field) \ +#define show_sdev_iostat_percpu(field) \ static ssize_t \ show_iostat_##field(struct device *dev, struct device_attribute *attr, \ char *buf) \ { \ struct scsi_device *sdev = to_scsi_device(dev); \ - unsigned long long count = atomic_read(&sdev->field); \ - return snprintf(buf, 20, "0x%llx\n", count); \ + unsigned long long count = percpu_counter_sum(&sdev->field); \ + return sysfs_emit(buf, "0x%llx\n", count); \ } \ -static DEVICE_ATTR(field, S_IRUGO, show_iostat_##field, NULL) +static DEVICE_ATTR(field, 0444, show_iostat_##field, NULL) -show_sdev_iostat(iorequest_cnt); -show_sdev_iostat(iodone_cnt); -show_sdev_iostat(ioerr_cnt); -show_sdev_iostat(iotmo_cnt); +show_sdev_iostat_percpu(iorequest_cnt); +show_sdev_iostat_percpu(iodone_cnt); +show_sdev_iostat_percpu(ioerr_cnt); +show_sdev_iostat_percpu(iotmo_cnt); static ssize_t sdev_show_modalias(struct device *dev, struct device_attribute *attr, char *buf) diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index adc3fa55ca2c..b7ce01de17b3 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -4043,7 +4043,7 @@ static int sd_probe(struct scsi_device *sdp) sdkp->index = index; sdkp->max_retries = SD_MAX_RETRIES; atomic_set(&sdkp->openers, 0); - atomic_set(&sdkp->device->ioerr_cnt, 0); + percpu_counter_set(&sdkp->device->ioerr_cnt, 0); if (!sdp->request_queue->rq_timeout) { if (sdp->type != TYPE_MOD) diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h index 029f5115b2ea..4be36bf2a475 100644 --- a/include/scsi/scsi_device.h +++ b/include/scsi/scsi_device.h @@ -9,6 +9,7 @@ #include <scsi/scsi.h> #include <scsi/scsi_common.h> #include <linux/atomic.h> +#include <linux/percpu_counter.h> #include <linux/sbitmap.h> struct bsg_device; @@ -272,10 +273,10 @@ struct scsi_device { unsigned int max_device_blocked; /* what device_blocked counts down from */ #define SCSI_DEFAULT_DEVICE_BLOCKED 3 - atomic_t iorequest_cnt; - atomic_t iodone_cnt; - atomic_t ioerr_cnt; - atomic_t iotmo_cnt; + struct percpu_counter iorequest_cnt; + struct percpu_counter iodone_cnt; + struct percpu_counter ioerr_cnt; + struct percpu_counter iotmo_cnt; struct device sdev_gendev, sdev_dev; -- 2.43.7
