Re: [PATCH 3/9] IB: Convert from tasklet to BH workqueue
在 2024/3/27 17:03, Allen Pais 写道:
The only generic interface to execute asynchronously in the BH context is
tasklet; however, it's marked deprecated and has some design flaws. To
replace tasklets, BH workqueue support was recently added. A BH workqueue
behaves similarly to regular workqueues except that the queued work items
are executed in the BH context.
This patch converts drivers/infiniband/* from tasklet to BH workqueue.
Based on the work done by Tejun Heo
Branch: https://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git for-6.10
I made simple tests. And I can not find the difference on latency and
throughput on RoCEv2 devices.
Anyone also made tests with these patches on IB? Any difference on
Latency and throughput?
Thanks,
Zhu Yanjun
Signed-off-by: Allen Pais
---
drivers/infiniband/hw/bnxt_re/bnxt_re.h| 3 +-
drivers/infiniband/hw/bnxt_re/qplib_fp.c | 21 ++--
drivers/infiniband/hw/bnxt_re/qplib_fp.h | 2 +-
drivers/infiniband/hw/bnxt_re/qplib_rcfw.c | 25 ---
drivers/infiniband/hw/bnxt_re/qplib_rcfw.h | 2 +-
drivers/infiniband/hw/erdma/erdma.h| 3 +-
drivers/infiniband/hw/erdma/erdma_eq.c | 11 ---
drivers/infiniband/hw/hfi1/rc.c| 2 +-
drivers/infiniband/hw/hfi1/sdma.c | 37 +++---
drivers/infiniband/hw/hfi1/sdma.h | 9 +++---
drivers/infiniband/hw/hfi1/tid_rdma.c | 6 ++--
drivers/infiniband/hw/irdma/ctrl.c | 2 +-
drivers/infiniband/hw/irdma/hw.c | 24 +++---
drivers/infiniband/hw/irdma/main.h | 5 +--
drivers/infiniband/hw/qib/qib.h| 7 ++--
drivers/infiniband/hw/qib/qib_iba7322.c| 9 +++---
drivers/infiniband/hw/qib/qib_rc.c | 16 +-
drivers/infiniband/hw/qib/qib_ruc.c| 4 +--
drivers/infiniband/hw/qib/qib_sdma.c | 11 ---
drivers/infiniband/sw/rdmavt/qp.c | 2 +-
20 files changed, 106 insertions(+), 95 deletions(-)
diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
index 9dca451ed522..f511c8415806 100644
--- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
+++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
@@ -42,6 +42,7 @@
#include
#include "hw_counters.h"
#include
+#include
#define ROCE_DRV_MODULE_NAME "bnxt_re"
#define BNXT_RE_DESC "Broadcom NetXtreme-C/E RoCE Driver"
@@ -162,7 +163,7 @@ struct bnxt_re_dev {
u8 cur_prio_map;
/* FP Notification Queue (CQ & SRQ) */
- struct tasklet_struct nq_task;
+ struct work_struct nq_work;
/* RCFW Channel */
struct bnxt_qplib_rcfw rcfw;
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 439d0c7c5d0c..052906982cdf 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -46,6 +46,7 @@
#include
#include
#include
+#include
#include
#include "roce_hsi.h"
@@ -294,9 +295,9 @@ static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq,
u16 cnq_events)
}
}
-static void bnxt_qplib_service_nq(struct tasklet_struct *t)
+static void bnxt_qplib_service_nq(struct work_struct *t)
{
- struct bnxt_qplib_nq *nq = from_tasklet(nq, t, nq_tasklet);
+ struct bnxt_qplib_nq *nq = from_work(nq, t, nq_work);
struct bnxt_qplib_hwq *hwq = >hwq;
struct bnxt_qplib_cq *cq;
int budget = nq->budget;
@@ -394,7 +395,7 @@ void bnxt_re_synchronize_nq(struct bnxt_qplib_nq *nq)
int budget = nq->budget;
nq->budget = nq->hwq.max_elements;
- bnxt_qplib_service_nq(>nq_tasklet);
+ bnxt_qplib_service_nq(>nq_work);
nq->budget = budget;
}
@@ -409,7 +410,7 @@ static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
prefetch(bnxt_qplib_get_qe(hwq, sw_cons, NULL));
/* Fan out to CPU affinitized kthreads? */
- tasklet_schedule(>nq_tasklet);
+ queue_work(system_bh_wq, >nq_work);
return IRQ_HANDLED;
}
@@ -430,8 +431,8 @@ void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool
kill)
nq->name = NULL;
if (kill)
- tasklet_kill(>nq_tasklet);
- tasklet_disable(>nq_tasklet);
+ cancel_work_sync(>nq_work);
+ disable_work_sync(>nq_work);
}
void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
@@ -465,9 +466,9 @@ int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int
nq_indx,
nq->msix_vec = msix_vector;
if (need_init)
- tasklet_setup(>nq_tasklet, bnxt_qplib_service_nq);
+ INIT_WORK(>nq_work, bnxt_qplib_service_nq);
else
- tasklet_enable(>nq_tasklet);
+ enable_and_queue_work(system_bh_wq, >nq_work);
nq->name = kasprintf(GFP_KERNEL, "bnxt_re-nq-%d@pci:%s",
nq_indx, pci_name(res->pdev));
@@
[PATCH 3/9] IB: Convert from tasklet to BH workqueue
The only generic interface to execute asynchronously in the BH context is
tasklet; however, it's marked deprecated and has some design flaws. To
replace tasklets, BH workqueue support was recently added. A BH workqueue
behaves similarly to regular workqueues except that the queued work items
are executed in the BH context.
This patch converts drivers/infiniband/* from tasklet to BH workqueue.
Based on the work done by Tejun Heo
Branch: https://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git for-6.10
Signed-off-by: Allen Pais
---
drivers/infiniband/hw/bnxt_re/bnxt_re.h| 3 +-
drivers/infiniband/hw/bnxt_re/qplib_fp.c | 21 ++--
drivers/infiniband/hw/bnxt_re/qplib_fp.h | 2 +-
drivers/infiniband/hw/bnxt_re/qplib_rcfw.c | 25 ---
drivers/infiniband/hw/bnxt_re/qplib_rcfw.h | 2 +-
drivers/infiniband/hw/erdma/erdma.h| 3 +-
drivers/infiniband/hw/erdma/erdma_eq.c | 11 ---
drivers/infiniband/hw/hfi1/rc.c| 2 +-
drivers/infiniband/hw/hfi1/sdma.c | 37 +++---
drivers/infiniband/hw/hfi1/sdma.h | 9 +++---
drivers/infiniband/hw/hfi1/tid_rdma.c | 6 ++--
drivers/infiniband/hw/irdma/ctrl.c | 2 +-
drivers/infiniband/hw/irdma/hw.c | 24 +++---
drivers/infiniband/hw/irdma/main.h | 5 +--
drivers/infiniband/hw/qib/qib.h| 7 ++--
drivers/infiniband/hw/qib/qib_iba7322.c| 9 +++---
drivers/infiniband/hw/qib/qib_rc.c | 16 +-
drivers/infiniband/hw/qib/qib_ruc.c| 4 +--
drivers/infiniband/hw/qib/qib_sdma.c | 11 ---
drivers/infiniband/sw/rdmavt/qp.c | 2 +-
20 files changed, 106 insertions(+), 95 deletions(-)
diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
index 9dca451ed522..f511c8415806 100644
--- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
+++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
@@ -42,6 +42,7 @@
#include
#include "hw_counters.h"
#include
+#include
#define ROCE_DRV_MODULE_NAME "bnxt_re"
#define BNXT_RE_DESC "Broadcom NetXtreme-C/E RoCE Driver"
@@ -162,7 +163,7 @@ struct bnxt_re_dev {
u8 cur_prio_map;
/* FP Notification Queue (CQ & SRQ) */
- struct tasklet_struct nq_task;
+ struct work_struct nq_work;
/* RCFW Channel */
struct bnxt_qplib_rcfw rcfw;
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 439d0c7c5d0c..052906982cdf 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -46,6 +46,7 @@
#include
#include
#include
+#include
#include
#include "roce_hsi.h"
@@ -294,9 +295,9 @@ static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq,
u16 cnq_events)
}
}
-static void bnxt_qplib_service_nq(struct tasklet_struct *t)
+static void bnxt_qplib_service_nq(struct work_struct *t)
{
- struct bnxt_qplib_nq *nq = from_tasklet(nq, t, nq_tasklet);
+ struct bnxt_qplib_nq *nq = from_work(nq, t, nq_work);
struct bnxt_qplib_hwq *hwq = >hwq;
struct bnxt_qplib_cq *cq;
int budget = nq->budget;
@@ -394,7 +395,7 @@ void bnxt_re_synchronize_nq(struct bnxt_qplib_nq *nq)
int budget = nq->budget;
nq->budget = nq->hwq.max_elements;
- bnxt_qplib_service_nq(>nq_tasklet);
+ bnxt_qplib_service_nq(>nq_work);
nq->budget = budget;
}
@@ -409,7 +410,7 @@ static irqreturn_t bnxt_qplib_nq_irq(int irq, void
*dev_instance)
prefetch(bnxt_qplib_get_qe(hwq, sw_cons, NULL));
/* Fan out to CPU affinitized kthreads? */
- tasklet_schedule(>nq_tasklet);
+ queue_work(system_bh_wq, >nq_work);
return IRQ_HANDLED;
}
@@ -430,8 +431,8 @@ void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool
kill)
nq->name = NULL;
if (kill)
- tasklet_kill(>nq_tasklet);
- tasklet_disable(>nq_tasklet);
+ cancel_work_sync(>nq_work);
+ disable_work_sync(>nq_work);
}
void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
@@ -465,9 +466,9 @@ int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int
nq_indx,
nq->msix_vec = msix_vector;
if (need_init)
- tasklet_setup(>nq_tasklet, bnxt_qplib_service_nq);
+ INIT_WORK(>nq_work, bnxt_qplib_service_nq);
else
- tasklet_enable(>nq_tasklet);
+ enable_and_queue_work(system_bh_wq, >nq_work);
nq->name = kasprintf(GFP_KERNEL, "bnxt_re-nq-%d@pci:%s",
nq_indx, pci_name(res->pdev));
@@ -477,7 +478,7 @@ int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int
nq_indx,
if (rc) {
kfree(nq->name);
nq->name = NULL;
- tasklet_disable(>nq_tasklet);
+ disable_work_sync(>nq_work);
