Mike,
I'm still investigating the performance problems in open-iscsi over iSER, and I
think that I see the cause of the problem:
I've added an outstanding commands counter. It's incremented when
iscsi_queuecommand is called and decremented when iscsi_complete_command is
called. After decrementing the counter, I save it's value in some global array.
I print this array when iscsi_session_teardown is called.
>From what I see, in ~25% of command completions, the command queue is empty
>(i.e. outstanding_cmds = 0). This can explain the low throughput that I see
>with iSER.
What could be the cause for that? Is there anything not well configured in the
scsi_host_template that we use? Anything else?
Here's the patch that I used:
diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c
b/drivers/infiniband/ulp/iser/iser_initiator.c
index 47f716c..e96252a 100644
--- a/drivers/infiniband/ulp/iser/iser_initiator.c
+++ b/drivers/infiniband/ulp/iser/iser_initiator.c
@@ -570,6 +570,7 @@ void iser_rcv_completion(struct iser_desc *rx_desc,
iser_dbg("itt %d ctask %p\n",itt,ctask);
iser_ctask->status = ISER_TASK_STATUS_COMPLETED;
iser_ctask_rdma_finalize(iser_ctask);
+ ctask->stats.iser_rcv_completion = jiffies;
}
iser_dto_buffs_release(dto);
diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
index c76bd5c..61bcf6b 100644
--- a/drivers/scsi/libiscsi.c
+++ b/drivers/scsi/libiscsi.c
@@ -38,6 +38,9 @@
#include <scsi/scsi_transport_iscsi.h>
#include <scsi/libiscsi.h>
+int cmd_arr_idx = 0;
+struct cmd_stats cmd_arr[CMD_ARR_LEN];
+
struct iscsi_session *
class_to_transport_session(struct iscsi_cls_session *cls_session)
{
@@ -254,6 +257,14 @@ static void iscsi_complete_command(struct iscsi_cmd_task
*ctask)
struct iscsi_session *session = conn->session;
struct scsi_cmnd *sc = ctask->sc;
+ session->outstanding_cmds--;
+
+ if (cmd_arr_idx < CMD_ARR_LEN) {
+ memcpy(&cmd_arr[cmd_arr_idx], &ctask->stats, sizeof(struct
cmd_stats));
+ cmd_arr[cmd_arr_idx].outstanding_cmds =
session->outstanding_cmds;
+ cmd_arr_idx++;
+ }
+
ctask->state = ISCSI_TASK_COMPLETED;
ctask->sc = NULL;
/* SCSI eh reuses commands to verify us */
@@ -1093,9 +1104,14 @@ int iscsi_queuecommand(struct scsi_cmnd *sc, void
(*done)(struct scsi_cmnd *))
ctask->state = ISCSI_TASK_PENDING;
ctask->conn = conn;
ctask->sc = sc;
+
+ ctask->stats.itt = ctask->itt;
+ ctask->stats.qeueucommand = jiffies;
+
INIT_LIST_HEAD(&ctask->running);
list_add_tail(&ctask->running, &conn->xmitqueue);
+ session->outstanding_cmds++;
spin_unlock(&session->lock);
scsi_queue_work(host, &conn->xmitwork);
@@ -1708,6 +1724,11 @@ iscsi_session_setup(struct iscsi_transport *iscsit,
struct iscsi_session *session;
struct iscsi_cls_session *cls_session;
int cmd_i;
+ int i;
+
+ for (i=0 ; i<CMD_ARR_LEN; i++) {
+ memset(&cmd_arr[i],0,sizeof(struct cmd_stats));
+ }
if (qdepth > ISCSI_MAX_CMD_PER_LUN || qdepth < 1) {
if (qdepth != 0)
@@ -1760,6 +1781,8 @@ iscsi_session_setup(struct iscsi_transport *iscsit,
session->tt = iscsit;
mutex_init(&session->eh_mutex);
+ session->outstanding_cmds = 0;
+
/* initialize SCSI PDU commands pool */
if (iscsi_pool_init(&session->cmdpool, session->cmds_max,
(void***)&session->cmds,
@@ -1834,6 +1857,16 @@ void iscsi_session_teardown(struct iscsi_cls_session
*cls_session)
struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
struct module *owner = cls_session->transport->owner;
+ int i;
+ struct cmd_stats *p_stats;
+
+ printk("%s: printing cmd_arr:\n", __FUNCTION__);
+ for (i=0 ; i<cmd_arr_idx ; i++) {
+ p_stats = &cmd_arr[i];
+ printk("%d , %u , %lu , %lu , %d\n", i,
+ p_stats->itt, p_stats->qeueucommand,
+ p_stats->iser_rcv_completion, p_stats->outstanding_cmds);
+ }
iscsi_remove_session(cls_session);
scsi_remove_host(shost);
diff --git a/include/scsi/libiscsi.h b/include/scsi/libiscsi.h
index 5daa83c..b16969e 100644
--- a/include/scsi/libiscsi.h
+++ b/include/scsi/libiscsi.h
@@ -84,6 +84,8 @@ enum {
ISCSI_DIGEST_SIZE = sizeof(__u32),
};
+#define CMD_ARR_LEN 1000
+
struct iscsi_mgmt_task {
/*
* Becuae LLDs allocate their hdr differently, this is a pointer to
@@ -103,6 +105,15 @@ enum {
ISCSI_TASK_RUNNING,
};
+struct cmd_stats {
+ uint32_t itt;
+
+ unsigned long qeueucommand;
+ unsigned long iser_rcv_completion;
+
+ int outstanding_cmds;
+};
+
struct iscsi_cmd_task {
/*
* Because LLDs allocate their hdr differently, this is a pointer
@@ -114,6 +125,8 @@ struct iscsi_cmd_task {
unsigned short hdr_len; /* accumulated size of hdr used
*/
int itt; /* this ITT */
+ struct cmd_stats stats;
+
uint32_t unsol_datasn;
unsigned imm_count; /* imm-data (bytes) */
unsigned unsol_count; /* unsolicited (bytes)*/
@@ -254,6 +267,8 @@ struct iscsi_session {
*/
struct mutex eh_mutex;
+ int outstanding_cmds;
+
/* iSCSI session-wide sequencing */
uint32_t cmdsn;
uint32_t exp_cmdsn;
--~--~---------~--~----~------------~-------~--~----~
You received this message because you are subscribed to the Google Groups
"open-iscsi" group.
To post to this group, send email to open-iscsi@googlegroups.com
To unsubscribe from this group, send email to [EMAIL PROTECTED]
For more options, visit this group at http://groups.google.com/group/open-iscsi
-~----------~----~----~----~------~----~------~--~---
