RE: [RESEND][PATCH 7/8][SCSI]mpt3sas: Added Reply Descriptor Post Queue (RDPQ) Array support

2014-08-12 Thread Sreekanth Reddy 
Sending this patch once again using git send-email.

Up to now, Driver allocates a single contiguous block of memory
pool for all reply queues and passes down a single address in the
ReplyDescriptorPostQueueAddress field of the IOC Init Request
Message to the firmware.

When firmware receives this address, it will program each of the
Reply Descriptor Post Queue registers, as each reply queue has its
own register. Thus the firmware, starting from a base address it
determines the starting address of the subsequent reply queues
through some simple arithmetic calculations.

The size of this contiguous block of memory pool is directly proportional
to number of MSI-X vectors and the HBA queue depth. For example higher
MSIX vectors requires larger contiguous block of memory pool.

But some of the OS kernels are unable to allocate this larger
contiguous block of memory pool.

So, the proposal is to allocate memory independently for each
Reply Queue and pass down all of the addresses to the firmware.
Then the firmware will just take each address and program the value
into the correct register.

When HBAs with older firmware(i.e. without RDPQ capability) is used
with this new driver then the max_msix_vectors value would be set
to 8 by default.

Change set in v2:

1. Declared the _base_get_ioc_facts() functions at the beginning of the 
mpt3sas_base.c file
instead of moving all these functions before mpt3sas_base_map_resources() 
function
a. _base_wait_for_doorbell_int()
b. _base_wait_for_doorbell_ack()
c. _base_wait_for_doorbell_not_used()
d. _base_handshake_req_reply_wait()
e. _base_get_ioc_facts()

2. Initially set the consistent DMA mask to 32 bit and then change it to 64 bit 
mask
after allocating RDPQ pools by calling the function 
_base_change_consistent_dma_mask.
This is to ensure that all the upper 32 bits of RDPQ entries's base address to 
be same.

3. Reduced the redundancy between the RDPQ and non-RDPQ support in these 
following functions
a. _base_release_memory_pools()
b. _base_allocate_memory_pools()
c. _base_send_ioc_init()
d. _base_make_ioc_operational()

Signed-off-by: Sreekanth Reddy sreekanth.re...@avagotech.com
---
 drivers/scsi/mpt3sas/mpt3sas_base.c | 231 +++-
 drivers/scsi/mpt3sas/mpt3sas_base.h |  18 ++-
 2 files changed, 189 insertions(+), 60 deletions(-)

diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c 
b/drivers/scsi/mpt3sas/mpt3sas_base.c
index d71f135..408eb81 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -92,6 +92,11 @@ MODULE_PARM_DESC(mpt3sas_fwfault_debug,
 enable detection of firmware fault and halt firmware - (default=0));
 
 
+static int dma_mask;
+
+static int
+_base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc, int sleep_flag);
+
 /**
  * _scsih_set_fwfault_debug - global setting of ioc-fwfault_debug.
  *
@@ -1482,17 +1487,23 @@ static int
 _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev)
 {
struct sysinfo s;
-   char *desc = NULL;
+   u64 consistent_dma_mask;
+
+   if (dma_mask)
+   consistent_dma_mask = DMA_BIT_MASK(64);
+   else
+   consistent_dma_mask = DMA_BIT_MASK(32);
 
if (sizeof(dma_addr_t)  4) {
const uint64_t required_mask =
dma_get_required_mask(pdev-dev);
if ((required_mask  DMA_BIT_MASK(32)) 
!pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) 
-   !pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
+   !pci_set_consistent_dma_mask(pdev,
+ DMA_BIT_MASK(consistent_dma_mask))) {
ioc-base_add_sg_single = _base_add_sg_single_64;
ioc-sge_size = sizeof(Mpi2SGESimple64_t);
-   desc = 64;
+   dma_mask = 64;
goto out;
}
}
@@ -1501,19 +1512,30 @@ _base_config_dma_addressing(struct MPT3SAS_ADAPTER 
*ioc, struct pci_dev *pdev)
 !pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32))) {
ioc-base_add_sg_single = _base_add_sg_single_32;
ioc-sge_size = sizeof(Mpi2SGESimple32_t);
-   desc = 32;
+   dma_mask = 32;
} else
return -ENODEV;
 
  out:
si_meminfo(s);
pr_info(MPT3SAS_FMT
-   %s BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n,
-   ioc-name, desc, convert_to_kb(s.totalram));
+   %d BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n,
+   ioc-name, dma_mask, convert_to_kb(s.totalram));
 
return 0;
 }
 
+static int
+_base_change_consistent_dma_mask(struct MPT3SAS_ADAPTER *ioc,
+ struct pci_dev *pdev)
+{
+   if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
+

Re: [RESEND][PATCH 7/8][SCSI]mpt3sas: Added Reply Descriptor Post Queue (RDPQ) Array support

2014-08-06 Thread Tomas Henzl 
On 08/05/2014 06:41 PM, Sreekanth Reddy wrote:
 Hi Tomas,

 Can you please review this updated patch,

Hi Sreekanth,
the patch is mangled so a resend is needed anyway and I think you will
probably also want implement the changes you were asked for in the
mpt2sas sibling of this patch.
Thanks, Tomas


 Up to now, Driver allocates a single contiguous block of memory
 pool for all reply queues and passes down a single address in the
 ReplyDescriptorPostQueueAddress field of the IOC Init Request
 Message to the firmware.

 When firmware receives this address, it will program each of the
 Reply Descriptor Post Queue registers, as each reply queue has its
 own register. Thus the firmware, starting from a base address it
 determines the starting address of the subsequent reply queues
 through some simple arithmetic calculations.

 The size of this contiguous block of memory pool is directly proportional
 to number of MSI-X vectors and the HBA queue depth. For example higher
 MSIX vectors requires larger contiguous block of memory pool.

 But some of the OS kernels are unable to allocate this larger
 contiguous block of memory pool.

 So, the proposal is to allocate memory independently for each
 Reply Queue and pass down all of the addresses to the firmware.
 Then the firmware will just take each address and program the value
 into the correct register.

 When HBAs with older firmware(i.e. without RDPQ capability) is used
 with this new driver then the max_msix_vectors value would be set
 to 8 by default.

 Change set in v2:

 1. Declared the following functions at the beginning of the mpt3sas_base.c
 file instead of moving all these functions before
 mpt3sas_base_map_resources() function
 a. _base_wait_for_doorbell_int()
 b. _base_wait_for_doorbell_ack()
 c. _base_wait_for_doorbell_not_used()
 d. _base_handshake_req_reply_wait()
 e. _base_get_ioc_facts()

 2. Initially set the consistent DMA mask to 32 bit and then change it to 64
 bit mask after allocating RDPQ pools by calling the function
 _base_change_consistent_dma_mask. This is to ensure that all the upper 32
 bits of RDPQ entries's base address to be same.

 3. Reduced the redundancy between the RDPQ and non-RDPQ support in these
 following functions
 a. _base_release_memory_pools()
 b. _base_allocate_memory_pools()
 c. _base_send_ioc_init()
 d. _base_make_ioc_operational()

 Signed-off-by: Sreekanth Reddy sreekanth.re...@avagotech.com
 ---
  drivers/scsi/mpt3sas/mpt3sas_base.c | 246
 
  drivers/scsi/mpt3sas/mpt3sas_base.h |  18 ++-
  2 files changed, 204 insertions(+), 60 deletions(-)

 diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c
 b/drivers/scsi/mpt3sas/mpt3sas_base.c
 index 153b2c1..20c2c7b 100644
 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c
 +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
 @@ -92,6 +92,22 @@ MODULE_PARM_DESC(mpt3sas_fwfault_debug,
   enable detection of firmware fault and halt firmware - (default=0));


 +static int dma_mask;
 +
 +static int
 +_base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout,
 +int sleep_flag);
 +static int
 +_base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout,
 +int sleep_flag);
 +static int
 +_base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout,
 + int sleep_flag);
 +static int
 +_base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int
 request_bytes,
 +u32 *request, int reply_bytes, u16 *reply, int timeout, int
 sleep_flag);
 +static int
 +_base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc, int sleep_flag);
  /**
   * _scsih_set_fwfault_debug - global setting of ioc-fwfault_debug.
   *
 @@ -1482,17 +1498,23 @@ static int
  _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev
 *pdev)
  {
  struct sysinfo s;
 -char *desc = NULL;
 +u64 consistent_dma_mask;
 +
 +if (dma_mask)
 +consistent_dma_mask = DMA_BIT_MASK(64);
 +else
 +consistent_dma_mask = DMA_BIT_MASK(32);

  if (sizeof(dma_addr_t)  4) {
  const uint64_t required_mask =
  dma_get_required_mask(pdev-dev);
  if ((required_mask  DMA_BIT_MASK(32)) 
  !pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) 
 -!pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
 +!pci_set_consistent_dma_mask(pdev,
 +  DMA_BIT_MASK(consistent_dma_mask))) {
  ioc-base_add_sg_single = _base_add_sg_single_64;
  ioc-sge_size = sizeof(Mpi2SGESimple64_t);
 -desc = 64;
 +dma_mask = 64;
  goto out;
  }
  }
 @@ -1501,19 +1523,30 @@ _base_config_dma_addressing(struct MPT3SAS_ADAPTER
 *ioc, struct pci_dev *pdev)
   !pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32))) {
  ioc-base_add_sg_single = _base_add_sg_single_32;
  ioc-sge_size =

Re: [RESEND][PATCH 7/8][SCSI]mpt3sas: Added Reply Descriptor Post Queue (RDPQ) Array support

2014-08-05 Thread Tomas Henzl 
On 06/25/2014 12:41 PM, Reddy, Sreekanth wrote:
 Up to now, Driver allocates a single contiguous block of memory
 pool for all reply queues and passes down a single address in the
 ReplyDescriptorPostQueueAddress field of the IOC Init Request
 Message to the firmware.

 When firmware receives this address, it will program each of the
 Reply Descriptor Post Queue registers, as each reply queue has its
 own register. Thus the firmware, starting from a base address it
 determines the starting address of the subsequent reply queues
 through some simple arithmetic calculations.

 The size of this contiguous block of memory pool is directly proportional
 to number of MSI-X vectors and the HBA queue depth. For example higher
 MSIX vectors requires larger contiguous block of memory pool.

 But some of the OS kernels are unable to allocate this larger
 contiguous block of memory pool.

 So, the proposal is to allocate memory independently for each
 Reply Queue and pass down all of the addresses to the firmware.
 Then the firmware will just take each address and program the value
 into the correct register.

 When HBAs with older firmware(i.e. without RDPQ capability) is used
 with this new driver then the max_msix_vectors value would be set
 to 8 by default.

 Signed-off-by: Sreekanth Reddy sreekanth.re...@avagotech.com
 ---
  drivers/scsi/mpt3sas/mpt3sas_base.c |  916 
 ---
  drivers/scsi/mpt3sas/mpt3sas_base.h |   19 +-
  2 files changed, 543 insertions(+), 392 deletions(-)

...
 +static int
 +_base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int 
 request_bytes,
 + u32 *request, int reply_bytes, u16 *reply, int timeout, int sleep_flag)
 +{
 + MPI2DefaultReply_t *default_reply = (MPI2DefaultReply_t *)reply;
 + int i;
 + u8 failed;
 + u16 dummy;
 + __le32 *mfp;
 +
 + /* make sure doorbell is not in use */
 + if ((readl(ioc-chip-Doorbell)  MPI2_DOORBELL_USED)) {

if (readl(ioc-chip-Doorbell)  MPI2_DOORBELL_USED) {
I think it's equal and better looking with less parentheses, but 
it's a personal preference so you can ignore it.

 + pr_err(MPT3SAS_FMT doorbell is in use (line=%d)\n,
 + ioc-name, __LINE__);
 + return -EFAULT;
 + }
 +
 + /* clear pending doorbell interrupts from previous state changes */
 + if (readl(ioc-chip-HostInterruptStatus) 
 + MPI2_HIS_IOC2SYS_DB_STATUS)
 + writel(0, ioc-chip-HostInterruptStatus);
 +
 + /* send message to ioc */
 + writel(((MPI2_FUNCTION_HANDSHAKEMPI2_DOORBELL_FUNCTION_SHIFT) |
 + ((request_bytes/4)MPI2_DOORBELL_ADD_DWORDS_SHIFT)),
 + ioc-chip-Doorbell);
 +
 + if ((_base_wait_for_doorbell_int(ioc, 5, NO_SLEEP))) {

Most likely not a problem, but why NO_SLEEP and not the sleep_flag ?

 + pr_err(MPT3SAS_FMT doorbell handshake int failed (line=%d)\n,
 +ioc-name, __LINE__);
 + return -EFAULT;
 + }
 + writel(0, ioc-chip-HostInterruptStatus);
 +
 + if ((_base_wait_for_doorbell_ack(ioc, 5, sleep_flag))) {
 + pr_err(MPT3SAS_FMT doorbell handshake ack failed (line=%d)\n,
 + ioc-name, __LINE__);
 + return -EFAULT;
 + }
 +
 + /* send message 32-bits at a time */
 + for (i = 0, failed = 0; i  request_bytes/4  !failed; i++) {
 + writel(cpu_to_le32(request[i]), ioc-chip-Doorbell);
 + if ((_base_wait_for_doorbell_ack(ioc, 5, sleep_flag)))
 + failed = 1;
 + }
...

@@ -2945,39 +3307,82 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER 
*ioc,  int sleep_flag)
ioc-name, (unsigned long long)ioc-reply_free_dma));
total_sz += sz;
 
-   /* reply post queue, 16 byte align */
-   reply_post_free_sz = ioc-reply_post_queue_depth *
-   sizeof(Mpi2DefaultReplyDescriptor_t);
-   if (_base_is_controller_msix_enabled(ioc))
-   sz = reply_post_free_sz * ioc-reply_queue_count;
-   else
+   if (ioc-rdpq_array_enable) {
+   ioc-reply_post = kcalloc(ioc-reply_queue_count,
+   sizeof(struct reply_post_struct), GFP_KERNEL);
+   /* reply post queue, 16 byte align */
+   reply_post_free_sz = ioc-reply_post_queue_depth *
+   sizeof(Mpi2DefaultReplyDescriptor_t);
sz = reply_post_free_sz;
-   ioc-reply_post_free_dma_pool = pci_pool_create(reply_post_free pool,
-   ioc-pdev, sz, 16, 0);
-   if (!ioc-reply_post_free_dma_pool) {
-   pr_err(MPT3SAS_FMT
-   reply_post_free pool: pci_pool_create failed\n,
-   ioc-name);
-   goto out;
-   }
-   ioc-reply_post_free = pci_pool_alloc(ioc-reply_post_free_dma_pool ,
-   GFP_KERNEL, ioc-reply_post_free_dma);
-   if (!ioc-reply_post_free) {
-   pr_err(MPT3SAS_FMT
-   reply_post_free

[RESEND][PATCH 7/8][SCSI]mpt3sas: Added Reply Descriptor Post Queue (RDPQ) Array support

2014-06-25 Thread Reddy, Sreekanth 
Up to now, Driver allocates a single contiguous block of memory
pool for all reply queues and passes down a single address in the
ReplyDescriptorPostQueueAddress field of the IOC Init Request
Message to the firmware.

When firmware receives this address, it will program each of the
Reply Descriptor Post Queue registers, as each reply queue has its
own register. Thus the firmware, starting from a base address it
determines the starting address of the subsequent reply queues
through some simple arithmetic calculations.

The size of this contiguous block of memory pool is directly proportional
to number of MSI-X vectors and the HBA queue depth. For example higher
MSIX vectors requires larger contiguous block of memory pool.

But some of the OS kernels are unable to allocate this larger
contiguous block of memory pool.

So, the proposal is to allocate memory independently for each
Reply Queue and pass down all of the addresses to the firmware.
Then the firmware will just take each address and program the value
into the correct register.

When HBAs with older firmware(i.e. without RDPQ capability) is used
with this new driver then the max_msix_vectors value would be set
to 8 by default.

Signed-off-by: Sreekanth Reddy sreekanth.re...@avagotech.com
---
 drivers/scsi/mpt3sas/mpt3sas_base.c |  916 ---
 drivers/scsi/mpt3sas/mpt3sas_base.h |   19 +-
 2 files changed, 543 insertions(+), 392 deletions(-)

diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c 
b/drivers/scsi/mpt3sas/mpt3sas_base.c
index f1406cc..483785b 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -82,10 +82,10 @@ static int msix_disable = -1;
 module_param(msix_disable, int, 0);
 MODULE_PARM_DESC(msix_disable,  disable msix routed interrupts (default=0));
 
-static int max_msix_vectors = 8;
+static int max_msix_vectors = -1;
 module_param(max_msix_vectors, int, 0);
 MODULE_PARM_DESC(max_msix_vectors,
-max msix vectors - (default=8));
+max msix vectors);
 
 static int mpt3sas_fwfault_debug;
 MODULE_PARM_DESC(mpt3sas_fwfault_debug,
@@ -1728,6 +1728,9 @@ _base_enable_msix(struct MPT3SAS_ADAPTER *ioc)
ioc-reply_queue_count = min_t(int, ioc-cpu_count,
ioc-msix_vector_count);
 
+   if (!ioc-rdpq_array_enable  max_msix_vectors == -1)
+   max_msix_vectors = 8;
+
printk(MPT3SAS_FMT MSI-X vectors supported: %d, no of cores
  : %d, max_msix_vectors: %d\n, ioc-name, ioc-msix_vector_count,
  ioc-cpu_count, max_msix_vectors);
@@ -1782,6 +1785,334 @@ _base_enable_msix(struct MPT3SAS_ADAPTER *ioc)
 }
 
 /**
+ * _base_wait_for_doorbell_int - waiting for controller interrupt(generated by
+ * a write to the doorbell)
+ * @ioc: per adapter object
+ * @timeout: timeout in second
+ * @sleep_flag: CAN_SLEEP or NO_SLEEP
+ *
+ * Returns 0 for success, non-zero for failure.
+ *
+ * Notes: MPI2_HIS_IOC2SYS_DB_STATUS - set to one when IOC writes to doorbell.
+ */
+static int
+_base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout,
+   int sleep_flag)
+{
+   u32 cntdn, count;
+   u32 int_status;
+
+   count = 0;
+   cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout;
+   do {
+   int_status = readl(ioc-chip-HostInterruptStatus);
+   if (int_status  MPI2_HIS_IOC2SYS_DB_STATUS) {
+   dhsprintk(ioc, pr_info(MPT3SAS_FMT
+   %s: successfull count(%d), timeout(%d)\n,
+   ioc-name, __func__, count, timeout));
+   return 0;
+   }
+   if (sleep_flag == CAN_SLEEP)
+   msleep(1);
+   else
+   udelay(500);
+   count++;
+   } while (--cntdn);
+
+   pr_err(MPT3SAS_FMT
+   %s: failed due to timeout count(%d), int_status(%x)!\n,
+   ioc-name, __func__, count, int_status);
+   return -EFAULT;
+}
+
+/**
+ * _base_wait_for_doorbell_ack - waiting for controller to read the doorbell.
+ * @ioc: per adapter object
+ * @timeout: timeout in second
+ * @sleep_flag: CAN_SLEEP or NO_SLEEP
+ *
+ * Returns 0 for success, non-zero for failure.
+ *
+ * Notes: MPI2_HIS_SYS2IOC_DB_STATUS - set to one when host writes to
+ * doorbell.
+ */
+static int
+_base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout,
+   int sleep_flag)
+{
+   u32 cntdn, count;
+   u32 int_status;
+   u32 doorbell;
+
+   count = 0;
+   cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout;
+   do {
+   int_status = readl(ioc-chip-HostInterruptStatus);
+   if (!(int_status  MPI2_HIS_SYS2IOC_DB_STATUS)) {
+   dhsprintk(ioc, pr_info(MPT3SAS_FMT
+   %s: successfull count(%d), timeout(%d)\n,
+   ioc-name, __func__, count, timeout));
+   return 0;
+