On Wed, Jun 17, 2015 at 11:37:53AM +0530, Nagarajkumar Narayanan wrote:
> Problem Description:
>
> https://bugzilla.kernel.org/show_bug.cgi?id=95101
>
> Due to lack of synchronization between ioctl, BRM status access, pci
> resource removal kernel oops happen as ioctl path and BRM status
> access path still tries to access the removed resources
>
> kernel: BUG: unable to handle kernel paging request at ffffc900171e0000
>
> Oops: 0000 [#1] SMP
>
>
> Patch Description:
>
> Two locks added to provide syncrhonization
>
> 1. pci_access_mutex: Mutex to synchronize ioctl,sysfs show path and
> pci resource handling. PCI resource freeing will lead to free
> vital hardware/memory resource, which might be in use by cli/sysfs
> path functions resulting in Null pointer reference followed by kernel
> crash. To avoid the above race condition we use mutex syncrhonization
> which ensures the syncrhonization between cli/sysfs_show path
>
> 2. spinlock on list operations over IOCs
>
> Case: when multiple warpdrive cards(IOCs) are in use
> Each IOC will added to the ioc list stucture on initialization.
> Watchdog threads run at regular intervals to check IOC for any
> fault conditions which will trigger the dead_ioc thread to
> deallocate pci resource, resulting deleting the IOC netry from list,
> this deletion need to protected by spinlock to enusre that
> ioc removal is syncrhonized, if not synchronized it might lead to
> list_del corruption as the ioc list is traversed in cli path
>
>
>
> please find the patch to apply on scsi/mpt2sas driver
>
> http://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/
>
>
> From ba692140278e6e2b660896c32206b26dac98d215 Mon Sep 17 00:00:00 2001
> From: Nagarajkumar Narayanan <[email protected]>
> Date: Thu, 19 Mar 2015 12:02:07 +0530
> Subject: [PATCH] mpt2sas setpci kernel oops fix
>
> Signed-off-by: Nagarajkumar Narayanan <[email protected]>
> ---
> drivers/scsi/mpt2sas/mpt2sas_base.c | 10 +++++++
> drivers/scsi/mpt2sas/mpt2sas_base.h | 20 +++++++++++++-
> drivers/scsi/mpt2sas/mpt2sas_ctl.c | 48 +++++++++++++++++++++++++++++----
> drivers/scsi/mpt2sas/mpt2sas_scsih.c | 32 ++++++++++++++++++++++-
> 4 files changed, 102 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/scsi/mpt2sas/mpt2sas_base.c
> b/drivers/scsi/mpt2sas/mpt2sas_base.c
> index 11248de..d2a498c 100644
> --- a/drivers/scsi/mpt2sas/mpt2sas_base.c
> +++ b/drivers/scsi/mpt2sas/mpt2sas_base.c
> @@ -108,13 +108,18 @@ _scsih_set_fwfault_debug(const char *val, struct
> kernel_param *kp)
> {
> int ret = param_set_int(val, kp);
> struct MPT2SAS_ADAPTER *ioc;
> + unsigned long flags;
>
> if (ret)
> return ret;
>
> + /* global ioc spinlock to protect controller list on list operations */
> + mpt2sas_initialize_gioc_lock();
> printk(KERN_INFO "setting fwfault_debug(%d)\n", mpt2sas_fwfault_debug);
> + spin_lock_irqsave(&gioc_lock, flags);
> list_for_each_entry(ioc, &mpt2sas_ioc_list, list)
> ioc->fwfault_debug = mpt2sas_fwfault_debug;
> + spin_unlock_irqrestore(&gioc_lock, flags);
> return 0;
> }
>
> @@ -4436,6 +4441,9 @@ mpt2sas_base_free_resources(struct MPT2SAS_ADAPTER *ioc)
> __func__));
>
> if (ioc->chip_phys && ioc->chip) {
> + /* synchronizing freeing resource with pci_access_mutex lock */
> + if (ioc->is_warpdrive)
> + mutex_lock(&ioc->pci_access_mutex);
> _base_mask_interrupts(ioc);
> ioc->shost_recovery = 1;
> _base_make_ioc_ready(ioc, CAN_SLEEP, SOFT_RESET);
> @@ -4454,6 +4462,8 @@ mpt2sas_base_free_resources(struct MPT2SAS_ADAPTER *ioc)
> pci_disable_pcie_error_reporting(pdev);
> pci_disable_device(pdev);
> }
> + if (ioc->is_warpdrive)
> + mutex_unlock(&ioc->pci_access_mutex);
> return;
> }
>
> diff --git a/drivers/scsi/mpt2sas/mpt2sas_base.h
> b/drivers/scsi/mpt2sas/mpt2sas_base.h
> index caff8d1..a0d26f0 100644
> --- a/drivers/scsi/mpt2sas/mpt2sas_base.h
> +++ b/drivers/scsi/mpt2sas/mpt2sas_base.h
> @@ -799,6 +799,12 @@ typedef void (*MPT2SAS_FLUSH_RUNNING_CMDS)(struct
> MPT2SAS_ADAPTER *ioc);
> * @delayed_tr_list: target reset link list
> * @delayed_tr_volume_list: volume target reset link list
> * @@temp_sensors_count: flag to carry the number of temperature sensors
> + * @pci_access_mutex: Mutex to synchronize ioctl,sysfs show path and
> + * pci resource handling. PCI resource freeing will lead to free
> + * vital hardware/memory resource, which might be in use by cli/sysfs
> + * path functions resulting in Null pointer reference followed by kernel
> + * crash. To avoid the above race condition we use mutex syncrhonization
> + * which ensures the syncrhonization between cli/sysfs_show path
> */
> struct MPT2SAS_ADAPTER {
> struct list_head list;
> @@ -1015,6 +1021,7 @@ struct MPT2SAS_ADAPTER {
> u8 mfg_pg10_hide_flag;
> u8 hide_drives;
>
> + struct mutex pci_access_mutex;
> };
>
> typedef u8 (*MPT_CALLBACK)(struct MPT2SAS_ADAPTER *ioc, u16 smid, u8
> msix_index,
> @@ -1023,6 +1030,17 @@ typedef u8 (*MPT_CALLBACK)(struct
> MPT2SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
>
> /* base shared API */
> extern struct list_head mpt2sas_ioc_list;
> +/* spinlock on list operations over IOCs
> ++ * Case: when multiple warpdrive cards(IOCs) are in use
> ++ * Each IOC will added to the ioc list stucture on initialization.
> ++ * Watchdog threads run at regular intervals to check IOC for any
> ++ * fault conditions which will trigger the dead_ioc thread to
> ++ * deallocate pci resource, resulting deleting the IOC netry from list,
> ++ * this deletion need to protected by spinlock to enusre that
> ++ * ioc removal is syncrhonized, if not synchronized it might lead to
> ++ * list_del corruption as the ioc list is traversed in cli path
> ++ */
> +extern spinlock_t gioc_lock;
> void mpt2sas_base_start_watchdog(struct MPT2SAS_ADAPTER *ioc);
> void mpt2sas_base_stop_watchdog(struct MPT2SAS_ADAPTER *ioc);
>
> @@ -1099,7 +1117,7 @@ struct _sas_device
> *mpt2sas_scsih_sas_device_find_by_sas_address(
> struct MPT2SAS_ADAPTER *ioc, u64 sas_address);
>
> void mpt2sas_port_enable_complete(struct MPT2SAS_ADAPTER *ioc);
> -
> +void mpt2sas_initialize_gioc_lock(void);
> void mpt2sas_scsih_reset_handler(struct MPT2SAS_ADAPTER *ioc, int
> reset_phase);
>
> /* config shared API */
> diff --git a/drivers/scsi/mpt2sas/mpt2sas_ctl.c
> b/drivers/scsi/mpt2sas/mpt2sas_ctl.c
> index 4e50960..5345368 100644
> --- a/drivers/scsi/mpt2sas/mpt2sas_ctl.c
> +++ b/drivers/scsi/mpt2sas/mpt2sas_ctl.c
> @@ -427,13 +427,17 @@ static int
> _ctl_verify_adapter(int ioc_number, struct MPT2SAS_ADAPTER **iocpp)
> {
> struct MPT2SAS_ADAPTER *ioc;
> -
> + unsigned long flags;
> + /* global ioc lock to protect controller on list operations */
> + spin_lock_irqsave(&gioc_lock, flags);
> list_for_each_entry(ioc, &mpt2sas_ioc_list, list) {
> if (ioc->id != ioc_number)
> continue;
> + spin_unlock_irqrestore(&gioc_lock, flags);
> *iocpp = ioc;
> return ioc_number;
> }
> + spin_unlock_irqrestore(&gioc_lock, flags);
> *iocpp = NULL;
> return -1;
> }
> @@ -519,13 +523,19 @@ static unsigned int
> _ctl_poll(struct file *filep, poll_table *wait)
> {
> struct MPT2SAS_ADAPTER *ioc;
> + unsigned long flags;
>
> poll_wait(filep, &ctl_poll_wait, wait);
>
> + /* global ioc lock to protect controller on list operations */
> + spin_lock_irqsave(&gioc_lock, flags);
> list_for_each_entry(ioc, &mpt2sas_ioc_list, list) {
> - if (ioc->aen_event_read_flag)
> + if (ioc->aen_event_read_flag) {
> + spin_unlock_irqrestore(&gioc_lock, flags);
> return POLLIN | POLLRDNORM;
> + }
> }
> + spin_unlock_irqrestore(&gioc_lock, flags);
> return 0;
> }
>
> @@ -2168,15 +2178,30 @@ _ctl_ioctl_main(struct file *file, unsigned
> int cmd, void __user *arg,
>
> if (_ctl_verify_adapter(ioctl_header.ioc_number, &ioc) == -1 || !ioc)
> return -ENODEV;
> - if (ioc->shost_recovery || ioc->pci_error_recovery ||
> - ioc->is_driver_loading)
> - return -EAGAIN;
> + if (!ioc->is_warpdrive) {
> + if (ioc->shost_recovery || ioc->pci_error_recovery ||
> + ioc->is_driver_loading)
> + return -EAGAIN;
> + } else {
> + /* pci_access_mutex lock acquired by ioctl path */
> + mutex_lock(&ioc->pci_access_mutex);
> + if (ioc->shost_recovery || ioc->pci_error_recovery ||
> + ioc->is_driver_loading || ioc->remove_host) {
> + mutex_unlock(&ioc->pci_access_mutex);
> + return -EAGAIN;
> + }
> + }
>
> state = (file->f_flags & O_NONBLOCK) ? NON_BLOCKING : BLOCKING;
> if (state == NON_BLOCKING) {
> - if (!mutex_trylock(&ioc->ctl_cmds.mutex))
> + if (!mutex_trylock(&ioc->ctl_cmds.mutex)) {
> + if (ioc->is_warpdrive)
> + mutex_unlock(&ioc->pci_access_mutex);
> return -EAGAIN;
> + }
> } else if (mutex_lock_interruptible(&ioc->ctl_cmds.mutex)) {
> + if (ioc->is_warpdrive)
> + mutex_unlock(&ioc->pci_access_mutex);
> return -ERESTARTSYS;
> }
>
> @@ -2258,6 +2283,8 @@ _ctl_ioctl_main(struct file *file, unsigned int
> cmd, void __user *arg,
> }
>
> mutex_unlock(&ioc->ctl_cmds.mutex);
> + if (ioc->is_warpdrive)
> + mutex_unlock(&ioc->pci_access_mutex);
> return ret;
> }
>
> @@ -2710,6 +2737,13 @@ _ctl_BRM_status_show(struct device *cdev,
> struct device_attribute *attr,
> printk(MPT2SAS_ERR_FMT "%s: BRM attribute is only for"\
> "warpdrive\n", ioc->name, __func__);
> goto out;
> + } else {
> + /* pci_access_mutex lock acquired by sysfs show path */
> + mutex_lock(&ioc->pci_access_mutex);
> + if (ioc->pci_error_recovery || ioc->remove_host) {
> + mutex_unlock(&ioc->pci_access_mutex);
> + return 0;
> + }
> }
>
> /* allocate upto GPIOVal 36 entries */
> @@ -2749,6 +2783,8 @@ _ctl_BRM_status_show(struct device *cdev, struct
> device_attribute *attr,
>
> out:
> kfree(io_unit_pg3);
> + if (ioc->is_warpdrive)
> + mutex_unlock(&ioc->pci_access_mutex);
> return rc;
> }
> static DEVICE_ATTR(BRM_status, S_IRUGO, _ctl_BRM_status_show, NULL);
> diff --git a/drivers/scsi/mpt2sas/mpt2sas_scsih.c
> b/drivers/scsi/mpt2sas/mpt2sas_scsih.c
> index 3f26147..ef20ed3 100644
> --- a/drivers/scsi/mpt2sas/mpt2sas_scsih.c
> +++ b/drivers/scsi/mpt2sas/mpt2sas_scsih.c
> @@ -79,7 +79,8 @@ static int _scsih_scan_finished(struct Scsi_Host
> *shost, unsigned long time);
>
> /* global parameters */
> LIST_HEAD(mpt2sas_ioc_list);
> -
> +/* global ioc lock for list operations */
> +spinlock_t gioc_lock;
> /* local parameters */
> static u8 scsi_io_cb_idx = -1;
> static u8 tm_cb_idx = -1;
> @@ -279,6 +280,20 @@ static struct pci_device_id scsih_pci_table[] = {
> MODULE_DEVICE_TABLE(pci, scsih_pci_table);
>
> /**
> + * mpt2sas_initialize_gioc_lock - initialize the gobal ioc lock
> + */
> +void
> +mpt2sas_initialize_gioc_lock(void)
> +{
> + static int gioc_lock_initialize;
> +
> + if (!gioc_lock_initialize) {
> + spin_lock_init(&gioc_lock);
> + gioc_lock_initialize = 1;
> + }
> +}
> +
> +/**
> * _scsih_set_debug_level - global setting of ioc->logging_level.
> *
> * Note: The logging levels are defined in mpt2sas_debug.h.
> @@ -288,13 +303,17 @@ _scsih_set_debug_level(const char *val, struct
> kernel_param *kp)
> {
> int ret = param_set_int(val, kp);
> struct MPT2SAS_ADAPTER *ioc;
> + unsigned long flags;
>
> if (ret)
> return ret;
>
> + mpt2sas_initialize_gioc_lock();
> printk(KERN_INFO "setting logging_level(0x%08x)\n", logging_level);
> + spin_lock_irqsave(&gioc_lock, flags);
> list_for_each_entry(ioc, &mpt2sas_ioc_list, list)
> ioc->logging_level = logging_level;
> + spin_unlock_irqrestore(&gioc_lock, flags);
> return 0;
> }
> module_param_call(logging_level, _scsih_set_debug_level, param_get_int,
> @@ -7867,7 +7886,9 @@ _scsih_remove(struct pci_dev *pdev)
> sas_remove_host(shost);
> scsi_remove_host(shost);
> mpt2sas_base_detach(ioc);
> + spin_lock_irqsave(&gioc_lock, flags);
> list_del(&ioc->list);
> + spin_unlock_irqrestore(&gioc_lock, flags);
> scsi_host_put(shost);
> }
>
> @@ -8132,6 +8153,7 @@ _scsih_probe(struct pci_dev *pdev, const struct
> pci_device_id *id)
> struct MPT2SAS_ADAPTER *ioc;
> struct Scsi_Host *shost;
> int rv;
> + unsigned long flags;
>
> shost = scsi_host_alloc(&scsih_driver_template,
> sizeof(struct MPT2SAS_ADAPTER));
> @@ -8142,7 +8164,9 @@ _scsih_probe(struct pci_dev *pdev, const struct
> pci_device_id *id)
> ioc = shost_priv(shost);
> memset(ioc, 0, sizeof(struct MPT2SAS_ADAPTER));
> INIT_LIST_HEAD(&ioc->list);
> + spin_lock_irqsave(&gioc_lock, flags);
> list_add_tail(&ioc->list, &mpt2sas_ioc_list);
> + spin_unlock_irqrestore(&gioc_lock, flags);
> ioc->shost = shost;
> ioc->id = mpt_ids++;
> sprintf(ioc->name, "%s%d", MPT2SAS_DRIVER_NAME, ioc->id);
> @@ -8167,6 +8191,9 @@ _scsih_probe(struct pci_dev *pdev, const struct
> pci_device_id *id)
> ioc->schedule_dead_ioc_flush_running_cmds = &_scsih_flush_running_cmds;
> /* misc semaphores and spin locks */
> mutex_init(&ioc->reset_in_progress_mutex);
> + /* initializing pci_access_mutex lock */
> + if (ioc->is_warpdrive)
> + mutex_init(&ioc->pci_access_mutex);
> spin_lock_init(&ioc->ioc_reset_in_progress_lock);
> spin_lock_init(&ioc->scsi_lookup_lock);
> spin_lock_init(&ioc->sas_device_lock);
> @@ -8269,7 +8296,9 @@ _scsih_probe(struct pci_dev *pdev, const struct
> pci_device_id *id)
> out_attach_fail:
> destroy_workqueue(ioc->firmware_event_thread);
> out_thread_fail:
> + spin_lock_irqsave(&gioc_lock, flags);
> list_del(&ioc->list);
> + spin_unlock_irqrestore(&gioc_lock, flags);
> scsi_host_put(shost);
> return rv;
> }
> @@ -8506,6 +8535,7 @@ _scsih_init(void)
> return -ENODEV;
> }
>
> + mpt2sas_initialize_gioc_lock();
> mpt2sas_base_initialize_callback_handler();
>
> /* queuecommand callback hander */
> --
> 1.7.1
>
>
> Regards,
> Nagarajkumar Narayanan
> --
> To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html
Hi Nagarajkumar,
Thanks for the patch, but can you please format the patch according to
Documentation/SubmittingPatches and re-send.
Thanks,
Johannes
--
Johannes Thumshirn Storage
[email protected] +49 911 74053 689
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: F. Imendörffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton
HRB 21284 (AG Nürnberg)
--
To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
