Add an end-user library with simple interfaces for communicating with the ib_acm service.
The linux and windows specific files for the library are simple and not shown for this review Signed-off-by: Sean Hefty <[email protected]> --- ib_acm.h: defines library interfaces. These are the end-user application interfaces to the ib acm. /* * Copyright (c) 2009 Intel Corporation. All rights reserved. * * This software is available to you under the OpenFabrics.org BSD license * below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #if !defined(IB_ACM_H) #define IB_ACM_H #include <infiniband/verbs.h> #if defined(_WIN32) #define LIB_EXPORT __declspec(dllexport) #else #define LIB_EXPORT #endif #ifdef __cplusplus extern "C" { #endif struct ib_acm_dev_addr { uint64_t guid; uint16_t pkey_index; uint8_t port_num; uint8_t reserved[5]; }; struct ib_acm_resolve_data { uint32_t reserved1; uint8_t init_depth; uint8_t resp_resources; uint8_t packet_lifetime; uint8_t mtu; uint8_t reserved2[8]; }; /** * ib_acm_resolve_name - Resolve path data between the specified names. * Description: * Discover path information, including identifying the local device, * between the given the source and destination names. * Notes: * The source and destination names should match entries in acm_addr.cfg * configuration files on their respective systems. Typically, the * source and destination names will refer to system host names * assigned to an Infiniband port. */ LIB_EXPORT int ib_acm_resolve_name(char *src, char *dest, struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data); /** * ib_acm_resolve_ip - Resolve path data between the specified addresses. * Description: * Discover path information, including identifying the local device, * between the given the source and destination addresses. * Notes: * The source and destination addresses should match entries in acm_addr.cfg * configuration files on their respective systems. Typically, the * source and destination addresses will refer to IP addresses assigned * to an IPoIB instance. */ LIB_EXPORT int ib_acm_resolve_ip(struct sockaddr *src, struct sockaddr *dest, struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data); #define IB_PATH_RECORD_REVERSIBLE 0x80 struct ib_path_record { uint64_t service_id; union ibv_gid dgid; union ibv_gid sgid; uint16_t dlid; uint16_t slid; uint32_t flowlabel_hoplimit; /* resv-31:28 flow label-27:8 hop limit-7:0*/ uint8_t tclass; uint8_t reversible_numpath; /* reversible-7:7 num path-6:0 */ uint16_t pkey; uint16_t qosclass_sl; /* qos class-15:4 sl-3:0 */ uint8_t mtu; /* mtu selector-7:6 mtu-5:0 */ uint8_t rate; /* rate selector-7:6 rate-5:0 */ uint8_t packetlifetime; /* lifetime selector-7:6 lifetime-5:0 */ uint8_t preference; uint8_t reserved[6]; }; /** * ib_acm_resolve_path - Resolve path data meeting specified restrictions * Description: * Discover path information using the provided path record to * restrict the discovery. * Notes: * Uses the provided path record as input into an query for path * information. If successful, fills in any missing information. The * caller must provide at least the source and destination LIDs as input. */ LIB_EXPORT int ib_acm_resolve_path(struct ib_path_record *path); /** * ib_acm_query_path - Resolve path data meeting specified restrictions * Description: * Queries the IB SA for a path record using the provided path record to * restrict the query. * Notes: * Uses the provided path record as input into an SA query for path * information. If successful, fills in any missing information. The * caller must provide at least the source and destination LIDs as input. * Use of this call always results in sending a query to the IB SA. */ LIB_EXPORT int ib_acm_query_path(struct ib_path_record *path); /** * ib_acm_convert_to_path - Convert resolved path data to a path record * Description: * Converts path information returned from resolving a host name or address * to the format of an IB path record. */ LIB_EXPORT int ib_acm_convert_to_path(struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data, struct ib_path_record *path); #ifdef __cplusplus } #endif #endif /* IB_ACM_H */ acm.h: defines the client/application side protocol of the ib acm service /* * Copyright (c) 2009 Intel Corporation. All rights reserved. * * This software is available to you under the OpenFabrics.org BSD license * below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #if !defined(ACM_H) #define ACM_H #include <infiniband/ib_acm.h> #define ACM_VERSION 1 #define ACM_OP_MASK 0x0F #define ACM_OP_RESOLVE 0x01 #define ACM_OP_QUERY 0x02 //#define ACM_OP_CM 0x03 //#define ACM_OP_ACK_REQ 0x40 /* optional ack is required */ #define ACM_OP_ACK 0x80 #define ACM_STATUS_SUCCESS 0 #define ACM_STATUS_ENOMEM 1 #define ACM_STATUS_EINVAL 2 #define ACM_STATUS_ENODATA 3 #define ACM_STATUS_ENOTCONN 5 #define ACM_STATUS_ETIMEDOUT 6 #define ACM_STATUS_ESRCADDR 7 #define ACM_STATUS_ESRCTYPE 8 #define ACM_STATUS_EDESTADDR 9 #define ACM_STATUS_EDESTTYPE 10 struct acm_hdr { uint8_t version; uint8_t opcode; uint8_t status; uint8_t param; uint8_t dest_type; uint8_t src_type; uint8_t reserved[2]; uint64_t tid; }; #define ACM_EP_TYPE_NAME 0x01 #define ACM_EP_TYPE_ADDRESS_IP 0x02 #define ACM_EP_TYPE_ADDRESS_IP6 0x03 #define ACM_EP_TYPE_DEVICE 0x10 #define ACM_EP_TYPE_AV 0x20 #define ACM_MAX_ADDRESS 32 union acm_ep_addr { uint8_t addr[ACM_MAX_ADDRESS]; uint8_t name[ACM_MAX_ADDRESS]; struct ib_acm_dev_addr dev; struct ibv_ah_attr av; }; struct acm_resolve_msg { struct acm_hdr hdr; union acm_ep_addr src; union acm_ep_addr dest; struct ib_acm_resolve_data data; }; //struct acm_cm_param //{ // uint32_t qpn; // uint8_t init_depth; // uint8_t resp_resources; // uint8_t retry_cnt; // uint8_t rnr_retry_cnt; // uint16_t src_port; // uint16_t dest_port; // uint8_t reserved[4]; //}; //struct acm_cm_msg //{ // struct acm_hdr hdr; // union acm_ep_addr src; // union acm_ep_addr dest; // struct acm_cm_param param; //}; #define ACM_QUERY_PATH_RECORD 0x01 #define ACM_QUERY_SA 0x80 #define ACM_EP_TYPE_LID 0x01 #define ACM_EP_TYPE_GID 0x02 union acm_query_data { struct ib_path_record path; }; struct acm_query_msg { struct acm_hdr hdr; union acm_query_data data; uint8_t reserved[16]; }; #define ACM_MSG_DATA_SIZE 80 struct acm_msg { struct acm_hdr hdr; uint8_t data[ACM_MSG_DATA_SIZE]; }; #endif /* ACM_H */ libibacm.c: brain dead implementation that handles the client side protocol of the ib acm for the user /* * Copyright (c) 2009 Intel Corporation. All rights reserved. * * This software is available to you under the OpenIB.org BSD license * below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include <osd.h> #include <infiniband/ib_acm.h> #include <infiniband/acm.h> #include <stdio.h> extern lock_t lock; static SOCKET sock = INVALID_SOCKET; static short server_port = 6125; static int ready; static int acm_init(void) { struct sockaddr_in addr; int ret; ret = osd_init(); if (ret) return ret; sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); if (sock == INVALID_SOCKET) { ret = socket_errno(); goto err1; } memset(&addr, 0, sizeof addr); addr.sin_family = AF_INET; addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); addr.sin_port = htons(server_port); ret = connect(sock, (struct sockaddr *) &addr, sizeof(addr)); if (ret) goto err2; ready = 1; return 0; err2: closesocket(sock); sock = INVALID_SOCKET; err1: osd_close(); return ret; } void LIB_DESTRUCTOR acm_cleanup(void) { if (sock != INVALID_SOCKET) { shutdown(sock, SHUT_RDWR); closesocket(sock); } } static int acm_resolve(uint8_t *src, uint8_t *dest, uint8_t type, struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data) { struct acm_resolve_msg msg; int ret; lock_acquire(&lock); if (!ready && (ret = acm_init())) goto out; memset(&msg, 0, sizeof msg); msg.hdr.version = ACM_VERSION; msg.hdr.opcode = ACM_OP_RESOLVE; msg.hdr.dest_type = type; msg.hdr.src_type = type; switch (type) { case ACM_EP_TYPE_NAME: strncpy((char *) msg.src.name, (char *) src, ACM_MAX_ADDRESS); strncpy((char *) msg.dest.name, (char *) dest, ACM_MAX_ADDRESS); break; case ACM_EP_TYPE_ADDRESS_IP: memcpy(msg.src.addr, &((struct sockaddr_in *) src)->sin_addr, 4); memcpy(msg.dest.addr, &((struct sockaddr_in *) dest)->sin_addr, 4); break; case ACM_EP_TYPE_ADDRESS_IP6: memcpy(msg.src.addr, &((struct sockaddr_in6 *) src)->sin6_addr, 16); memcpy(msg.dest.addr, &((struct sockaddr_in *) dest)->sin_addr, 16); break; case ACM_EP_TYPE_AV: memcpy(&msg.src.av, src, sizeof(msg.src.av)); memcpy(&msg.dest.av, dest, sizeof(msg.dest.av)); break; default: ret = -1; goto out; } ret = send(sock, (char *) &msg, sizeof msg, 0); if (ret != sizeof msg) goto out; ret = recv(sock, (char *) &msg, sizeof msg, 0); if (ret != sizeof msg) goto out; memcpy(dev_addr, &msg.src.dev, sizeof(*dev_addr)); *ah = msg.dest.av; memcpy(data, &msg.data, sizeof(*data)); ret = 0; out: lock_release(&lock); return ret; } LIB_EXPORT int ib_acm_resolve_name(char *src, char *dest, struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data) { return acm_resolve((uint8_t *) src, (uint8_t *) dest, ACM_EP_TYPE_NAME, dev_addr, ah, data); } LIB_EXPORT int ib_acm_resolve_ip(struct sockaddr *src, struct sockaddr *dest, struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data) { if (((struct sockaddr *) dest)->sa_family == AF_INET) { return acm_resolve((uint8_t *) src, (uint8_t *) dest, ACM_EP_TYPE_ADDRESS_IP, dev_addr, ah, data); } else { return acm_resolve((uint8_t *) src, (uint8_t *) dest, ACM_EP_TYPE_ADDRESS_IP6, dev_addr, ah, data); } } static int acm_query_path(struct ib_path_record *path, uint8_t query_sa) { struct acm_query_msg msg; int ret; lock_acquire(&lock); if (!ready && (ret = acm_init())) goto out; memset(&msg, 0, sizeof msg); msg.hdr.version = ACM_VERSION; msg.hdr.opcode = ACM_OP_QUERY; msg.hdr.param = ACM_QUERY_PATH_RECORD | query_sa; if (path->dgid.global.interface_id || path->dgid.global.subnet_prefix) { msg.hdr.dest_type = ACM_EP_TYPE_GID; } else if (path->dlid) { msg.hdr.dest_type = ACM_EP_TYPE_LID; } else { ret = -1; goto out; } if (path->sgid.global.interface_id || path->sgid.global.subnet_prefix) { msg.hdr.src_type = ACM_EP_TYPE_GID; } else if (path->slid) { msg.hdr.src_type = ACM_EP_TYPE_LID; } else { ret = -1; goto out; } msg.data.path = *path; ret = send(sock, (char *) &msg, sizeof msg, 0); if (ret != sizeof msg) goto out; ret = recv(sock, (char *) &msg, sizeof msg, 0); if (ret != sizeof msg) goto out; *path = msg.data.path; ret = msg.hdr.status; out: lock_release(&lock); return ret; } LIB_EXPORT int ib_acm_query_path(struct ib_path_record *path) { return acm_query_path(path, ACM_QUERY_SA); } LIB_EXPORT int ib_acm_resolve_path(struct ib_path_record *path) { return acm_query_path(path, 0); } static struct ibv_context *acm_open_device(uint64_t guid) { struct ibv_device **dev_array; struct ibv_context *verbs = NULL; int i, cnt; dev_array = ibv_get_device_list(&cnt); if (!dev_array) return NULL; for (i = 0; i < cnt; i++) { if (guid == ibv_get_device_guid(dev_array[i])) { verbs = ibv_open_device(dev_array[i]); break; } } ibv_free_device_list(dev_array); return verbs; } LIB_EXPORT int ib_acm_convert_to_path(struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data, struct ib_path_record *path) { struct ibv_context *verbs; struct ibv_port_attr attr; int ret; verbs = acm_open_device(dev_addr->guid); if (!verbs) return -1; if (ah->is_global) { path->dgid = ah->grh.dgid; ret = ibv_query_gid(verbs, dev_addr->port_num, ah->grh.sgid_index, &path->sgid); if (ret) goto out; path->flowlabel_hoplimit = htonl(ah->grh.flow_label << 8 | (uint32_t) ah->grh.hop_limit); path->tclass = ah->grh.traffic_class; } path->dlid = htons(ah->dlid); ret = ibv_query_port(verbs, dev_addr->port_num, &attr); if (ret) goto out; path->slid = htons(attr.lid | ah->src_path_bits); path->reversible_numpath = IB_PATH_RECORD_REVERSIBLE | 1; ret = ibv_query_pkey(verbs, dev_addr->port_num, dev_addr->pkey_index, &path->pkey); if (ret) goto out; path->pkey = htons(path->pkey); path->qosclass_sl = htons((uint16_t) ah->sl); path->mtu = (2 << 6) | data->mtu; path->rate = (2 << 6) | ah->static_rate; path->packetlifetime = (2 << 6) | data->packet_lifetime; out: ibv_close_device(verbs); return ret; } _______________________________________________ ofw mailing list [email protected] http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ofw
