On 09/24/2014 03:01 AM, Jamie Clark wrote:
> On machines running Ubuntu 12.04 open-iscsi initiator we have noticed
> intermittent "pdu ... rejected due to DataDigest error. (data digests
> configured and negotiated ok on initiator and target).
>
> After extensive wiresharking (the problem only affects something like 1
> in 100000 write PDUs) I captured a failed write from the initiator and
> found that the CRC32C value following the payload was incorrect, while
> surrounding writes were OK.
>
> The target then rejects this bad pdu and follows that with an RST on the
> TCP connection. This is expected behaviour. This seems to repeat about
> every 10 minutes or so.
>
> The writes are all 4k blocks originating from iscsi-backed libvirt
> managed KVM virtual machines. The particular failure I captured appeared
> to be syslog data. The first 100 bytes or so of the 4k data was a syslog
> entry, the remainder was zeroed out.
>
> Just wondering if digest calculation still has the potential failure
> modes described back
> here https://groups.google.com/d/msg/open-iscsi/OCYS5wUIYhc/Gno7edWCyH0J
>
What FS are you using and what kernel?
The problem was fixed in the upper layers by default a while back for
man file systems. Later, it was changed to a opt-in type of feature
since it affects performance. iSCSI was not setting the correct bits to
turn it on. I attached a patch for the current upstream kernel which
should tell a FS to do stable pages when using iscsi data digests.
Let me know if you need it for a different kernel.
--
You received this message because you are subscribed to the Google Groups
"open-iscsi" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
Visit this group at http://groups.google.com/group/open-iscsi.
For more options, visit https://groups.google.com/d/optout.
diff --git a/block/blk-settings.c b/block/blk-settings.c
index f1a1795..7e543dd 100644
--- a/block/blk-settings.c
+++ b/block/blk-settings.c
@@ -20,6 +20,27 @@ EXPORT_SYMBOL(blk_max_low_pfn);
unsigned long blk_max_pfn;
/**
+ * blk_queue_stable_writes_required - indicate stable writes are required
+ * @q: queue
+ */
+void blk_queue_stable_writes_required(struct request_queue *q)
+{
+ q->backing_dev_info.capabilities |= BDI_CAP_STABLE_WRITES;
+}
+EXPORT_SYMBOL(blk_queue_stable_writes_required);
+
+
+/**
+ * blk_queue_stable_writes_optional - indicate stable writes are not required
+ * @q: queue
+ */
+void blk_queue_stable_writes_optional(struct request_queue *q)
+{
+ q->backing_dev_info.capabilities &= ~BDI_CAP_STABLE_WRITES;
+}
+EXPORT_SYMBOL(blk_queue_stable_writes_optional);
+
+/**
* blk_queue_prep_rq - set a prepare_request function for queue
* @q: queue
* @pfn: prepare_request function
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index a669f2d..6ecf9cc 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -688,6 +688,23 @@ free_socket:
return err;
}
+
+static void iscsi_sw_tcp_stable_writes_optional(struct Scsi_Host *shost)
+{
+ struct scsi_device *sdev;
+
+ shost_for_each_device(sdev, shost)
+ blk_queue_stable_writes_optional(sdev->request_queue);
+}
+
+static void iscsi_sw_tcp_stable_writes_required(struct Scsi_Host *shost)
+{
+ struct scsi_device *sdev;
+
+ shost_for_each_device(sdev, shost)
+ blk_queue_stable_writes_required(sdev->request_queue);
+}
+
static int iscsi_sw_tcp_conn_set_param(struct iscsi_cls_conn *cls_conn,
enum iscsi_param param, char *buf,
int buflen)
@@ -702,8 +719,13 @@ static int iscsi_sw_tcp_conn_set_param(struct iscsi_cls_conn *cls_conn,
break;
case ISCSI_PARAM_DATADGST_EN:
iscsi_set_param(cls_conn, param, buf, buflen);
- tcp_sw_conn->sendpage = conn->datadgst_en ?
- sock_no_sendpage : tcp_sw_conn->sock->ops->sendpage;
+ if (conn->datadgst_en) {
+ tcp_sw_conn->sendpage = sock_no_sendpage;
+ iscsi_sw_tcp_stable_writes_required(conn->session->host);
+ } else {
+ tcp_sw_conn->sendpage = tcp_sw_conn->sock->ops->sendpage;
+ iscsi_sw_tcp_stable_writes_optional(conn->session->host);
+ }
break;
case ISCSI_PARAM_MAX_R2T:
return iscsi_tcp_set_max_r2t(conn, buf);
@@ -937,8 +959,17 @@ static int iscsi_sw_tcp_slave_alloc(struct scsi_device *sdev)
static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev)
{
+ struct iscsi_cls_session *cls_session =
+ starget_to_session(scsi_target(sdev));
+ struct iscsi_session *session = cls_session->dd_data;
+ struct iscsi_conn *conn = session->leadconn;
+
blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY);
blk_queue_dma_alignment(sdev->request_queue, 0);
+
+ if (conn->datadgst_en)
+ blk_queue_stable_writes_required(sdev->request_queue);
+
return 0;
}
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 518b465..811038d 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -1048,6 +1048,8 @@ extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
extern void blk_queue_flush(struct request_queue *q, unsigned int flush);
extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev);
+extern void blk_queue_stable_writes_required(struct request_queue *q);
+extern void blk_queue_stable_writes_optional(struct request_queue *q);
extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
extern int blk_bio_map_sg(struct request_queue *q, struct bio *bio,
