Here's the complete (untested) libibverbs patch I am working on now... I
think I got everything needed...
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 0db083a..b503a28 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -306,7 +306,10 @@ struct ibv_kern_wc {
__u32 opcode;
__u32 vendor_err;
__u32 byte_len;
- __u32 imm_data;
+ union {
+ __u32 imm_data;
+ __u32 invalidate_rkey;
+ };
__u32 qp_num;
__u32 src_qp;
__u32 wc_flags;
@@ -572,7 +575,10 @@ struct ibv_kern_send_wr {
__u32 num_sge;
__u32 opcode;
__u32 send_flags;
- __u32 imm_data;
+ union {
+ __u32 imm_data;
+ __u32 invalidate_rkey;
+ };
union {
struct {
__u64 remote_addr;
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index a04cc62..e6e2b10 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -92,7 +92,17 @@ enum ibv_device_cap_flags {
IBV_DEVICE_SYS_IMAGE_GUID = 1 << 11,
IBV_DEVICE_RC_RNR_NAK_GEN = 1 << 12,
IBV_DEVICE_SRQ_RESIZE = 1 << 13,
- IBV_DEVICE_N_NOTIFY_CQ = 1 << 14
+ IBV_DEVICE_N_NOTIFY_CQ = 1 << 14,
+ /*
+ * IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS is used by libibverbs to
+ * signal to low-level driver libraries that the kernel set
+ * the "send with invalidate" capaibility bit. Applications
+ * should only test IBV_DEVICE_MEM_MGT_EXTENSIONS and never
+ * look at IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS.
+ */
+ IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS = 1 << 16,
+ IBV_DEVICE_MEM_WINDOW = 1 << 17,
+ IBV_DEVICE_MEM_MGT_EXTENSIONS = 1 << 21,
};
enum ibv_atomic_cap {
@@ -257,7 +267,8 @@ enum ibv_wc_opcode {
enum ibv_wc_flags {
IBV_WC_GRH = 1 << 0,
- IBV_WC_WITH_IMM = 1 << 1
+ IBV_WC_WITH_IMM = 1 << 1,
+ IBV_WC_WITH_INVALIDATE = 1 << 2,
};
struct ibv_wc {
@@ -266,7 +277,10 @@ struct ibv_wc {
enum ibv_wc_opcode opcode;
uint32_t vendor_err;
uint32_t byte_len;
- uint32_t imm_data; /* in network byte order */
+ union {
+ uint32_t imm_data; /* in network byte order */
+ uint32_t invalidate_rkey;
+ };
uint32_t qp_num;
uint32_t src_qp;
enum ibv_wc_flags wc_flags;
@@ -486,7 +500,11 @@ enum ibv_wr_opcode {
IBV_WR_SEND_WITH_IMM,
IBV_WR_RDMA_READ,
IBV_WR_ATOMIC_CMP_AND_SWP,
- IBV_WR_ATOMIC_FETCH_AND_ADD
+ IBV_WR_ATOMIC_FETCH_AND_ADD,
+ IBV_WR_LSO,
+ IBV_WR_SEND_WITH_INV,
+ IBV_WR_RDMA_READ_WITH_INV,
+ IBV_WR_LOCAL_INV,
};
enum ibv_send_flags {
@@ -509,7 +527,10 @@ struct ibv_send_wr {
int num_sge;
enum ibv_wr_opcode opcode;
enum ibv_send_flags send_flags;
- uint32_t imm_data; /* in network byte order */
+ union {
+ uint32_t imm_data; /* in network byte order */
+ uint32_t invalidate_rkey;
+ };
union {
struct {
uint64_t remote_addr;
diff --git a/src/cmd.c b/src/cmd.c
index 66d7134..1945143 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -159,6 +159,18 @@ int ibv_cmd_query_device(struct ibv_context *context,
device_attr->local_ca_ack_delay = resp.local_ca_ack_delay;
device_attr->phys_port_cnt = resp.phys_port_cnt;
+ /*
+ * If the kernel driver says that it supports memory
+ * management extensions, then move the flag to
+ * IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS so that the low-level
+ * driver needs to move the flag back to show it supports the
+ * operations as well.
+ */
+ if (device_attr->device_cap_flags & IBV_DEVICE_MEM_MGT_EXTENSIONS) {
+ device_attr->device_cap_flags &= ~IBV_DEVICE_MEM_MGT_EXTENSIONS;
+ device_attr->device_cap_flags |=
IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS;
+ }
+
return 0;
}
diff --git a/src/compat-1_0.c b/src/compat-1_0.c
index 459ade9..0df8b68 100644
--- a/src/compat-1_0.c
+++ b/src/compat-1_0.c
@@ -535,7 +535,18 @@ symver(__ibv_ack_async_event_1_0, ibv_ack_async_event,
IBVERBS_1.0);
int __ibv_query_device_1_0(struct ibv_context_1_0 *context,
struct ibv_device_attr *device_attr)
{
- return ibv_query_device(context->real_context, device_attr);
+ int ret;
+
+ ret = ibv_query_device(context->real_context, device_attr);
+
+ /*
+ * ABI 1.0 consumers are never expecting memory management
+ * extension support.
+ */
+ device_attr->device_cap_flags &= ~(IBV_DEVICE_MEM_MGT_EXTENSIONS |
+ IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS);
+
+ return ret;
}
symver(__ibv_query_device_1_0, ibv_query_device, IBVERBS_1.0);
diff --git a/src/verbs.c b/src/verbs.c
index 9e370ce..4ea342f 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -79,7 +79,13 @@ enum ibv_rate mult_to_ibv_rate(int mult)
int __ibv_query_device(struct ibv_context *context,
struct ibv_device_attr *device_attr)
{
- return context->ops.query_device(context, device_attr);
+ int ret;
+
+ ret = context->ops.query_device(context, device_attr);
+
+ device_attr->device_cap_flags &= ~IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS;
+
+ return ret;
}
default_symver(__ibv_query_device, ibv_query_device);
_______________________________________________
general mailing list
[email protected]
http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
