Hello! Here is an API proposal for support of the SRC (scalable reliable connected) protocol extension in libibverbs.
This adds APIs to: - manage SRC domains - share SRC domains between processes, by means of creating a 1:1 association between an SRC domain and a file. Notes: - The file is specified by means of a file descriptor, this makes it possible for the user to manage file creation/deletion in the most flexible manner (e.g. tmpfile can be used). - I envision implementing this sharing mechanism in kernel by means of a per-device tree, with inode as a key and domain object as a value. Please comment. Signed-off-by: Michael S. Tsirkin <[EMAIL PROTECTED]> --- diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h index acc1b82..503f201 100644 --- a/include/infiniband/verbs.h +++ b/include/infiniband/verbs.h @@ -370,6 +370,11 @@ struct ibv_ah_attr { uint8_t port_num; }; +struct ibv_src_domain { + struct ibv_context *context; + uint32_t handle; +}; + enum ibv_srq_attr_mask { IBV_SRQ_MAX_WR = 1 << 0, IBV_SRQ_LIMIT = 1 << 1 @@ -389,7 +394,8 @@ struct ibv_srq_init_attr { enum ibv_qp_type { IBV_QPT_RC = 2, IBV_QPT_UC, - IBV_QPT_UD + IBV_QPT_UD, + IBV_QPT_SRC }; struct ibv_qp_cap { @@ -408,6 +414,7 @@ struct ibv_qp_init_attr { struct ibv_qp_cap cap; enum ibv_qp_type qp_type; int sq_sig_all; + struct ibv_src_domain *src_domain; }; enum ibv_qp_attr_mask { @@ -526,6 +533,7 @@ struct ibv_send_wr { uint32_t remote_qkey; } ud; } wr; + uint32_t src_remote_srq_num; }; struct ibv_recv_wr { @@ -553,6 +561,10 @@ struct ibv_srq { pthread_mutex_t mutex; pthread_cond_t cond; uint32_t events_completed; + + uint32_t src_srq_num; + struct ibv_src_domain *src_domain; + struct ibv_cq *src_cq; }; struct ibv_qp { @@ -570,6 +582,8 @@ struct ibv_qp { pthread_mutex_t mutex; pthread_cond_t cond; uint32_t events_completed; + + struct ibv_src_domain *src_domain; }; struct ibv_comp_channel { @@ -912,6 +926,25 @@ struct ibv_srq *ibv_create_srq(struct ibv_pd *pd, struct ibv_srq_init_attr *srq_init_attr); /** + * ibv_create_src_srq - Creates a SRQ associated with the specified protection + * domain and src domain. + * @pd: The protection domain associated with the SRQ. + * @src_domain: The SRC domain associated with the SRQ. + * @src_cq: CQ to report completions for SRC packets on. + * + * @srq_init_attr: A list of initial attributes required to create the SRQ. + * + * srq_attr->max_wr and srq_attr->max_sge are read the determine the + * requested size of the SRQ, and set to the actual values allocated + * on return. If ibv_create_srq() succeeds, then max_wr and max_sge + * will always be at least as large as the requested values. + */ +struct ibv_srq *ibv_create_src_srq(struct ibv_pd *pd, + struct ibv_src_domain *src_domain, + struct ibv_cq *src_cq, + struct ibv_srq_init_attr *srq_init_attr); + +/** * ibv_modify_srq - Modifies the attributes for the specified SRQ. * @srq: The SRQ to modify. * @srq_attr: On input, specifies the SRQ attributes to modify. On output, @@ -1074,6 +1107,44 @@ int ibv_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); */ int ibv_fork_init(void); +/** + * ibv_alloc_src_domain - Allocate an SRC domain + * Returns a reference to an SRC domain. + * Use ibv_put_src_domain to free the reference. + * @context: Device context + */ +struct ibv_src_domain *ibv_get_new_src_domain(struct ibv_context *context); + +/** + * ibv_share_src_domain - associate the src domain with a file. + * Establishes a connection between an SRC domain object and a file descriptor. + * + * @d: SRC domain to share + * @fd: descriptor for a file to associate with the domain + */ +int ibv_share_src_domain(struct ibv_src_domain *d, int fd); + +/** + * ibv_unshare_src_domain - disassociate the src domain from a file. + * Subsequent calls to ibv_get_shared_src_domain will fail. + * @d: SRC domain to unshare + */ +int ibv_unshare_src_domain(struct ibv_src_domain *d); + +/** + * ibv_get_src_domain - get a reference to shared SRC domain + * @context: Device context + * @fd: descriptor for a file associated with the domain + */ +struct ibv_src_domain *ibv_get_shared_src_domain(struct ibv_context *context, int fd); + +/** + * ibv_put_src_domain - destroy a reference to an SRC domain + * If this is the last reference, destroys the domain. + * @d: reference to SRC domain to put + */ +int ibv_put_src_domain(struct ibv_src_domain *d); + END_C_DECLS # undef __attribute_const -- MST _______________________________________________ ewg mailing list ewg@lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg