On Sun, Jul 29, 2007 at 05:04:31PM +0300, Michael S. Tsirkin wrote:
> Hello!
> Here is an API proposal for support of the SRC
> (scalable reliable connected) protocol extension in libibverbs.
>
> This adds APIs to:
> - manage SRC domains
>
> - share SRC domains between processes,
> by means of creating a 1:1 association
> between an SRC domain and a file.
>
> Notes:
> - The file is specified by means of a file descriptor,
> this makes it possible for the user to manage file
> creation/deletion in the most flexible manner
> (e.g. tmpfile can be used).
>
> - I envision implementing this sharing mechanism in kernel by means
> of a per-device tree, with inode as a key and domain object
> as a value.
>
> Please comment.
Can you provide a pseudo code of an application using this API?
Especially QP sharing part.
>
> Signed-off-by: Michael S. Tsirkin <[EMAIL PROTECTED]>
>
> ---
>
> diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
> index acc1b82..503f201 100644
> --- a/include/infiniband/verbs.h
> +++ b/include/infiniband/verbs.h
> @@ -370,6 +370,11 @@ struct ibv_ah_attr {
> uint8_t port_num;
> };
>
> +struct ibv_src_domain {
> + struct ibv_context *context;
> + uint32_t handle;
> +};
> +
> enum ibv_srq_attr_mask {
> IBV_SRQ_MAX_WR = 1 << 0,
> IBV_SRQ_LIMIT = 1 << 1
> @@ -389,7 +394,8 @@ struct ibv_srq_init_attr {
> enum ibv_qp_type {
> IBV_QPT_RC = 2,
> IBV_QPT_UC,
> - IBV_QPT_UD
> + IBV_QPT_UD,
> + IBV_QPT_SRC
> };
>
> struct ibv_qp_cap {
> @@ -408,6 +414,7 @@ struct ibv_qp_init_attr {
> struct ibv_qp_cap cap;
> enum ibv_qp_type qp_type;
> int sq_sig_all;
> + struct ibv_src_domain *src_domain;
> };
>
> enum ibv_qp_attr_mask {
> @@ -526,6 +533,7 @@ struct ibv_send_wr {
> uint32_t remote_qkey;
> } ud;
> } wr;
> + uint32_t src_remote_srq_num;
> };
>
> struct ibv_recv_wr {
> @@ -553,6 +561,10 @@ struct ibv_srq {
> pthread_mutex_t mutex;
> pthread_cond_t cond;
> uint32_t events_completed;
> +
> + uint32_t src_srq_num;
> + struct ibv_src_domain *src_domain;
> + struct ibv_cq *src_cq;
> };
>
> struct ibv_qp {
> @@ -570,6 +582,8 @@ struct ibv_qp {
> pthread_mutex_t mutex;
> pthread_cond_t cond;
> uint32_t events_completed;
> +
> + struct ibv_src_domain *src_domain;
> };
>
> struct ibv_comp_channel {
> @@ -912,6 +926,25 @@ struct ibv_srq *ibv_create_srq(struct ibv_pd *pd,
> struct ibv_srq_init_attr *srq_init_attr);
>
> /**
> + * ibv_create_src_srq - Creates a SRQ associated with the specified
> protection
> + * domain and src domain.
> + * @pd: The protection domain associated with the SRQ.
> + * @src_domain: The SRC domain associated with the SRQ.
> + * @src_cq: CQ to report completions for SRC packets on.
> + *
> + * @srq_init_attr: A list of initial attributes required to create the SRQ.
> + *
> + * srq_attr->max_wr and srq_attr->max_sge are read the determine the
> + * requested size of the SRQ, and set to the actual values allocated
> + * on return. If ibv_create_srq() succeeds, then max_wr and max_sge
> + * will always be at least as large as the requested values.
> + */
> +struct ibv_srq *ibv_create_src_srq(struct ibv_pd *pd,
> + struct ibv_src_domain *src_domain,
> + struct ibv_cq *src_cq,
> + struct ibv_srq_init_attr *srq_init_attr);
> +
> +/**
> * ibv_modify_srq - Modifies the attributes for the specified SRQ.
> * @srq: The SRQ to modify.
> * @srq_attr: On input, specifies the SRQ attributes to modify. On output,
> @@ -1074,6 +1107,44 @@ int ibv_detach_mcast(struct ibv_qp *qp, union ibv_gid
> *gid, uint16_t lid);
> */
> int ibv_fork_init(void);
>
> +/**
> + * ibv_alloc_src_domain - Allocate an SRC domain
> + * Returns a reference to an SRC domain.
> + * Use ibv_put_src_domain to free the reference.
> + * @context: Device context
> + */
> +struct ibv_src_domain *ibv_get_new_src_domain(struct ibv_context *context);
> +
> +/**
> + * ibv_share_src_domain - associate the src domain with a file.
> + * Establishes a connection between an SRC domain object and a file
> descriptor.
> + *
> + * @d: SRC domain to share
> + * @fd: descriptor for a file to associate with the domain
> + */
> +int ibv_share_src_domain(struct ibv_src_domain *d, int fd);
> +
> +/**
> + * ibv_unshare_src_domain - disassociate the src domain from a file.
> + * Subsequent calls to ibv_get_shared_src_domain will fail.
> + * @d: SRC domain to unshare
> + */
> +int ibv_unshare_src_domain(struct ibv_src_domain *d);
> +
> +/**
> + * ibv_get_src_domain - get a reference to shared SRC domain
> + * @context: Device context
> + * @fd: descriptor for a file associated with the domain
> + */
> +struct ibv_src_domain *ibv_get_shared_src_domain(struct ibv_context
> *context, int fd);
> +
> +/**
> + * ibv_put_src_domain - destroy a reference to an SRC domain
> + * If this is the last reference, destroys the domain.
> + * @d: reference to SRC domain to put
> + */
> +int ibv_put_src_domain(struct ibv_src_domain *d);
> +
> END_C_DECLS
>
> # undef __attribute_const
>
>
>
> --
> MST
> _______________________________________________
> general mailing list
> [EMAIL PROTECTED]
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
>
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
--
Gleb.
_______________________________________________
ewg mailing list
ewg@lists.openfabrics.org
http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg
