If a lot of QPs fall into Error state at once and the EQ of the respective
HCA is too small, it might overrun, causing the eHCA driver to stop
processing completion events and call application software's completion
handlers, effectively causing traffic to stop.
Fix this by limiting available QPs and CQs to a customizable max count,
and determining EQ size based on these counts and a worst-case assumption.
Signed-off-by: Stefan Roscher stefan.roscher at de.ibm.com
---
Reposted based on Roland's comments:
- use atomic_add_unless instead of atomic_read
- inf% changelog increase ;)
drivers/infiniband/hw/ehca/ehca_classes.h |5
drivers/infiniband/hw/ehca/ehca_cq.c | 11 +
drivers/infiniband/hw/ehca/ehca_main.c| 36 +++-
drivers/infiniband/hw/ehca/ehca_qp.c | 26 +++-
4 files changed, 74 insertions(+), 4 deletions(-)
diff --git a/drivers/infiniband/hw/ehca/ehca_classes.h
b/drivers/infiniband/hw/ehca/ehca_classes.h
index 3d6d946..00bab60 100644
--- a/drivers/infiniband/hw/ehca/ehca_classes.h
+++ b/drivers/infiniband/hw/ehca/ehca_classes.h
@@ -66,6 +66,7 @@ struct ehca_av;
#include ehca_irq.h
#define EHCA_EQE_CACHE_SIZE 20
+#define EHCA_MAX_NUM_QUEUES 0x
struct ehca_eqe_cache_entry {
struct ehca_eqe *eqe;
@@ -127,6 +128,8 @@ struct ehca_shca {
/* MR pgsize: bit 0-3 means 4K, 64K, 1M, 16M respectively */
u32 hca_cap_mr_pgsize;
int max_mtu;
+ atomic_t num_cqs;
+ atomic_t num_qps;
};
struct ehca_pd {
@@ -344,6 +347,8 @@ extern int ehca_use_hp_mr;
extern int ehca_scaling_code;
extern int ehca_lock_hcalls;
extern int ehca_nr_ports;
+extern int ehca_max_cq;
+extern int ehca_max_qp;
struct ipzu_queue_resp {
u32 qe_size; /* queue entry size */
diff --git a/drivers/infiniband/hw/ehca/ehca_cq.c
b/drivers/infiniband/hw/ehca/ehca_cq.c
index ec0cfcf..5540b27 100644
--- a/drivers/infiniband/hw/ehca/ehca_cq.c
+++ b/drivers/infiniband/hw/ehca/ehca_cq.c
@@ -132,10 +132,19 @@ struct ib_cq *ehca_create_cq(struct ib_device *device,
int cqe, int comp_vector,
if (cqe = 0x - 64 - additional_cqe)
return ERR_PTR(-EINVAL);
+ if (!atomic_add_unless(shca-num_cqs, 1, ehca_max_cq)) {
+ ehca_err(device, Unable to create CQ, max number of %i
+ CQs reached., ehca_max_cq);
+ ehca_err(device, To increase the maximum number of CQs
+ use the number_of_cqs module parameter.\n);
+ return ERR_PTR(-ENOSPC);
+ }
+
my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL);
if (!my_cq) {
ehca_err(device, Out of memory for ehca_cq struct device=%p,
device);
+ atomic_dec(shca-num_cqs);
return ERR_PTR(-ENOMEM);
}
@@ -305,6 +314,7 @@ create_cq_exit2:
create_cq_exit1:
kmem_cache_free(cq_cache, my_cq);
+ atomic_dec(shca-num_cqs);
return cq;
}
@@ -359,6 +369,7 @@ int ehca_destroy_cq(struct ib_cq *cq)
ipz_queue_dtor(NULL, my_cq-ipz_queue);
kmem_cache_free(cq_cache, my_cq);
+ atomic_dec(shca-num_cqs);
return 0;
}
diff --git a/drivers/infiniband/hw/ehca/ehca_main.c
b/drivers/infiniband/hw/ehca/ehca_main.c
index 6504897..482103e 100644
--- a/drivers/infiniband/hw/ehca/ehca_main.c
+++ b/drivers/infiniband/hw/ehca/ehca_main.c
@@ -68,6 +68,8 @@ int ehca_port_act_time = 30;
int ehca_static_rate = -1;
int ehca_scaling_code = 0;
int ehca_lock_hcalls = -1;
+int ehca_max_cq= -1;
+int ehca_max_qp= -1;
module_param_named(open_aqp1, ehca_open_aqp1, bool, S_IRUGO);
module_param_named(debug_level, ehca_debug_level, int, S_IRUGO);
@@ -79,6 +81,8 @@ module_param_named(poll_all_eqs, ehca_poll_all_eqs, bool,
S_IRUGO);
module_param_named(static_rate, ehca_static_rate, int, S_IRUGO);
module_param_named(scaling_code, ehca_scaling_code, bool, S_IRUGO);
module_param_named(lock_hcalls, ehca_lock_hcalls, bool, S_IRUGO);
+module_param_named(number_of_cqs, ehca_max_cq,int, S_IRUGO);
+module_param_named(number_of_qps, ehca_max_qp,int, S_IRUGO);
MODULE_PARM_DESC(open_aqp1,
Open AQP1 on startup (default: no));
@@ -104,6 +108,12 @@ MODULE_PARM_DESC(scaling_code,
MODULE_PARM_DESC(lock_hcalls,
Serialize all hCalls made by the driver
(default: autodetect));
+MODULE_PARM_DESC(number_of_cqs,
+ Max number of CQs which can be allocated
+ (default: autodetect));
+MODULE_PARM_DESC(number_of_qps,
+ Max number of QPs which can be allocated
+ (default: autodetect));
DEFINE_RWLOCK(ehca_qp_idr_lock);
DEFINE_RWLOCK(ehca_cq_idr_lock);
@@ -355,6 +365,25 @@ static int ehca_sense_attributes(struct ehca_shca *shca)
if (rblock-memory_page_size_supported pgsize_map[i])
