Added multi queue support to netmap pktio.
Signed-off-by: Matias Elo <[email protected]>
---
platform/linux-generic/include/odp_packet_netmap.h | 27 +-
platform/linux-generic/pktio/netmap.c | 367 ++++++++++++++++++---
2 files changed, 341 insertions(+), 53 deletions(-)
diff --git a/platform/linux-generic/include/odp_packet_netmap.h
b/platform/linux-generic/include/odp_packet_netmap.h
index a088d23..27efccb 100644
--- a/platform/linux-generic/include/odp_packet_netmap.h
+++ b/platform/linux-generic/include/odp_packet_netmap.h
@@ -9,21 +9,44 @@
#include <odp/packet_io.h>
#include <odp/pool.h>
+#include <odp/ticketlock.h>
#include <linux/if_ether.h>
#include <net/if.h>
+#define NM_MAX_DESC 32
+
+/** Ring for mapping pktin/pktout queues to netmap descriptors */
+typedef struct {
+ unsigned first; /**< Index of first netmap descriptor */
+ unsigned last; /**< Index of last netmap descriptor */
+ unsigned cur; /**< Index of current netmap descriptor */
+ unsigned num; /**< Number of netmap descriptors*/
+} netmap_desc_ring_t;
+
/** Packet socket using netmap mmaped rings for both Rx and Tx */
typedef struct {
odp_pool_t pool; /**< pool to alloc packets from */
size_t max_frame_len; /**< buf_size - sizeof(pkt_hdr) */
- struct nm_desc *rx_desc; /**< netmap meta-data for the device */
- struct nm_desc *tx_desc; /**< netmap meta-data for the device */
+ /** mapping of pktin queues to netmap rx descriptors */
+ netmap_desc_ring_t rx_desc_ring[NM_MAX_DESC];
+ /** mapping of pktout queues to netmap tx descriptors */
+ netmap_desc_ring_t tx_desc_ring[NM_MAX_DESC];
+ /** netmap metadata for the device, used only for rx */
+ struct nm_desc *rx_desc[NM_MAX_DESC];
+ /** netmap metadata for the device, used only for tx */
+ struct nm_desc *tx_desc[NM_MAX_DESC];
+ odp_ticketlock_t rx_lock[NM_MAX_DESC]; /**< netmap rx locks */
+ odp_ticketlock_t tx_lock[NM_MAX_DESC]; /**< netmap tx locks */
uint32_t if_flags; /**< interface flags */
int sockfd; /**< control socket */
unsigned char if_mac[ETH_ALEN]; /**< eth mac address */
char nm_name[IF_NAMESIZE + 7]; /**< netmap:<ifname> */
odp_pktio_capability_t capa; /**< interface capabilities */
+ unsigned num_rx_queues; /**< number of pktin queues */
+ unsigned num_tx_queues; /**< number of pktout queues */
+ odp_bool_t lockless_rx; /**< no locking for rx */
+ odp_bool_t lockless_tx; /**< no locking for tx */
} pkt_netmap_t;
#endif
diff --git a/platform/linux-generic/pktio/netmap.c
b/platform/linux-generic/pktio/netmap.c
index 313bcbe..36d7467 100644
--- a/platform/linux-generic/pktio/netmap.c
+++ b/platform/linux-generic/pktio/netmap.c
@@ -75,15 +75,116 @@ done:
return err;
}
+/**
+ * Map netmap rings to pktin/pktout queues
+ *
+ * @param rings Array of netmap descriptor rings
+ * @param num_queues Number of pktin/pktout queues
+ * @param num_rings Number of matching netmap rings
+ */
+static inline void map_netmap_rings(netmap_desc_ring_t *rings,
+ unsigned num_queues, unsigned num_rings)
+{
+ netmap_desc_ring_t *desc_ring;
+ unsigned rings_per_queue;
+ unsigned remainder;
+ unsigned mapped_rings;
+ unsigned i;
+ unsigned desc_id = 0;
+
+ rings_per_queue = num_rings / num_queues;
+ remainder = num_rings % num_queues;
+
+ if (remainder)
+ ODP_DBG("WARNING: Netmap rings mapped unevenly to queues\n");
+
+ for (i = 0; i < num_queues; i++) {
+ desc_ring = &rings[i];
+ if (i < remainder)
+ mapped_rings = rings_per_queue + 1;
+ else
+ mapped_rings = rings_per_queue;
+
+ desc_ring->first = desc_id;
+ desc_ring->cur = desc_id;
+ desc_ring->last = desc_ring->first + mapped_rings - 1;
+ desc_ring->num = mapped_rings;
+
+ desc_id = desc_ring->last + 1;
+ }
+}
+
+static int netmap_input_queues_config(pktio_entry_t *pktio_entry,
+ const odp_pktio_input_queue_param_t *p)
+{
+ struct pktio_entry *pktio = &pktio_entry->s;
+ pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm;
+ unsigned num_queues = p->num_queues;
+ unsigned i;
+
+ if (num_queues <= 0 || num_queues > pkt_nm->capa.max_input_queues ||
+ num_queues > NM_MAX_DESC) {
+ ODP_ERR("Invalid input queue count: %u\n", num_queues);
+ return -1;
+ }
+ /* Map pktin queues to netmap rings */
+ map_netmap_rings(pkt_nm->rx_desc_ring, num_queues,
+ pkt_nm->capa.max_input_queues);
+
+ if (p->hash_enable) {
+ if (rss_conf_set_fd(pktio_entry->s.pkt_nm.sockfd,
+ pktio_entry->s.name, &p->hash_proto)) {
+ ODP_ERR("Failed to configure input hash\n");
+ return -1;
+ }
+ }
+ for (i = 0; i < num_queues; i++) {
+ pktio->in_queue[i].queue = ODP_QUEUE_INVALID;
+ pktio->in_queue[i].pktin.index = i;
+ pktio->in_queue[i].pktin.pktio = pktio_entry->s.handle;
+ }
+ pkt_nm->lockless_rx = p->single_user;
+ pkt_nm->num_rx_queues = num_queues;
+ return 0;
+}
+
+static int netmap_output_queues_config(pktio_entry_t *pktio_entry,
+ const odp_pktio_output_queue_param_t *p)
+{
+ struct pktio_entry *pktio = &pktio_entry->s;
+ pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm;
+ unsigned num_queues = p->num_queues;
+ unsigned i;
+
+ if (num_queues <= 0 || num_queues > pkt_nm->capa.max_output_queues ||
+ num_queues > NM_MAX_DESC) {
+ ODP_ERR("Invalid output queue count: %u\n", num_queues);
+ return -1;
+ }
+
+ /* Enough to map only one netmap tx ring per pktout queue */
+ map_netmap_rings(pkt_nm->tx_desc_ring, num_queues, num_queues);
+
+ for (i = 0; i < num_queues; i++) {
+ pktio->out_queue[i].pktout.index = i;
+ pktio->out_queue[i].pktout.pktio = pktio_entry->s.handle;
+ }
+ pkt_nm->lockless_tx = p->single_user;
+ pkt_nm->num_tx_queues = num_queues;
+ return 0;
+}
+
static int netmap_close(pktio_entry_t *pktio_entry)
{
+ int i;
pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm;
- if (pkt_nm->rx_desc != NULL)
- nm_close(pkt_nm->rx_desc);
- if (pkt_nm->tx_desc != NULL)
- nm_close(pkt_nm->tx_desc);
-
+ for (i = 0; i < NM_MAX_DESC; i++) {
+ if (pkt_nm->rx_desc[i] != NULL)
+ nm_close(pkt_nm->rx_desc[i]);
+ if (pkt_nm->tx_desc[i] != NULL)
+ nm_close(pkt_nm->tx_desc[i]);
+ }
if (pkt_nm->sockfd != -1 && close(pkt_nm->sockfd) != 0) {
__odp_errno = errno;
ODP_ERR("close(sockfd): %s\n", strerror(errno));
@@ -95,6 +196,7 @@ static int netmap_close(pktio_entry_t *pktio_entry)
static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry,
const char *netdev, odp_pool_t pool)
{
+ int i;
int err;
int sockfd;
pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm;
@@ -121,7 +223,8 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED,
pktio_entry_t *pktio_entry,
snprintf(pkt_nm->nm_name, sizeof(pkt_nm->nm_name), "netmap:%s",
netdev);
- /* Dummy open here to check if netmap module is available */
+ /* Dummy open here to check if netmap module is available and to read
+ * capability info. */
desc = nm_open(pkt_nm->nm_name, NULL, 0, NULL);
if (desc == NULL) {
ODP_ERR("nm_open(%s) failed\n", pkt_nm->nm_name);
@@ -148,6 +251,11 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED,
pktio_entry_t *pktio_entry,
if (err)
goto error;
+ for (i = 0; i < NM_MAX_DESC; i++) {
+ odp_ticketlock_init(&pktio_entry->s.pkt_nm.rx_lock[i]);
+ odp_ticketlock_init(&pktio_entry->s.pkt_nm.tx_lock[i]);
+ }
+
return 0;
error:
@@ -158,19 +266,67 @@ error:
static int netmap_start(pktio_entry_t *pktio_entry)
{
pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm;
+ struct nm_desc base_desc;
int err;
unsigned i;
- const char *ifname = pkt_nm->nm_name;
- pkt_nm->rx_desc = nm_open(ifname, NULL, NETMAP_NO_TX_POLL, NULL);
- pkt_nm->tx_desc = nm_open(ifname, NULL, NM_OPEN_NO_MMAP,
- pkt_nm->rx_desc);
+ /* If no pktin/pktout queues have been configured. Configure one
+ * for each direction. */
+ if (!pkt_nm->num_rx_queues) {
+ odp_pktio_input_queue_param_t param;
- if (pkt_nm->rx_desc == NULL || pkt_nm->tx_desc == NULL) {
- ODP_ERR("nm_open(%s) failed\n", ifname);
+ memset(¶m, 0, sizeof(odp_pktio_input_queue_param_t));
+ param.num_queues = 1;
+ if (netmap_input_queues_config(pktio_entry, ¶m))
+ return -1;
+ }
+ if (!pkt_nm->num_tx_queues) {
+ odp_pktio_output_queue_param_t param;
+
+ memset(¶m, 0, sizeof(odp_pktio_output_queue_param_t));
+ param.num_queues = 1;
+ if (netmap_output_queues_config(pktio_entry, ¶m))
+ return -1;
+ }
+
+ base_desc.self = &base_desc;
+ base_desc.mem = NULL;
+ memcpy(base_desc.req.nr_name, pktio_entry->s.name,
+ sizeof(pktio_entry->s.name));
+ base_desc.req.nr_flags &= ~NR_REG_MASK;
+ base_desc.req.nr_flags |= NR_REG_ONE_NIC;
+ base_desc.req.nr_ringid = 0;
+
+ /* Only the first rx descriptor does mmap */
+ pkt_nm->rx_desc[0] = nm_open(pkt_nm->nm_name, NULL, NM_OPEN_IFNAME |
+ NETMAP_NO_TX_POLL, &base_desc);
+ if (pkt_nm->rx_desc[0] == NULL) {
+ ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name);
goto error;
}
-
+ /* Open rest of the rx descriptors (one per netmap ring) */
+ for (i = 1; i < pkt_nm->capa.max_input_queues; i++) {
+ base_desc.req.nr_ringid = i;
+ pkt_nm->rx_desc[i] = nm_open(pkt_nm->nm_name, NULL,
+ NM_OPEN_IFNAME | NM_OPEN_NO_MMAP |
+ NETMAP_NO_TX_POLL, &base_desc);
+ if (pkt_nm->rx_desc[i] == NULL) {
+ ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name);
+ goto error;
+ }
+ }
+ /* Open tx descriptors. Enough to use one netmap tx ring per pktout
+ * queue. */
+ for (i = 0; i < pkt_nm->num_tx_queues; i++) {
+ base_desc.req.nr_ringid = i;
+ pkt_nm->tx_desc[i] = nm_open(pkt_nm->nm_name, NULL,
+ NM_OPEN_IFNAME | NM_OPEN_NO_MMAP,
+ &base_desc);
+ if (pkt_nm->tx_desc[i] == NULL) {
+ ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name);
+ goto error;
+ }
+ }
/* Wait for the link to come up */
for (i = 0; i < NM_OPEN_RETRIES; i++) {
err = netmap_do_ioctl(pktio_entry, SIOCETHTOOL, ETHTOOL_GLINK);
@@ -237,29 +393,33 @@ static inline int netmap_pkt_to_odp(pktio_entry_t
*pktio_entry,
return 0;
}
-static int netmap_recv(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[],
- unsigned num)
+static int netmap_recv_queue(pktio_entry_t *pktio_entry, int index,
+ odp_packet_t pkt_table[], int num)
{
+ char *buf;
struct netmap_ring *ring;
- struct nm_desc *desc = pktio_entry->s.pkt_nm.rx_desc;
+ struct nm_desc *desc;
struct pollfd polld;
- char *buf;
+ pkt_netmap_t *pktio_nm = &pktio_entry->s.pkt_nm;
+ unsigned first_desc_id = pktio_nm->rx_desc_ring[index].first;
+ unsigned last_desc_id = pktio_nm->rx_desc_ring[index].last;
+ unsigned desc_id;
+ int num_desc = pktio_nm->rx_desc_ring[index].num;
int i;
- int num_rings = desc->last_rx_ring - desc->first_rx_ring + 1;
- int ring_id = desc->cur_rx_ring;
- unsigned num_rx = 0;
+ int num_rx = 0;
uint32_t slot_id;
- polld.fd = desc->fd;
- polld.events = POLLIN;
+ if (!pktio_nm->lockless_rx)
+ odp_ticketlock_lock(&pktio_nm->rx_lock[index]);
- for (i = 0; i < num_rings && num_rx != num; i++) {
- ring_id = desc->cur_rx_ring + i;
+ desc_id = pktio_nm->rx_desc_ring[index].cur;
- if (ring_id > desc->last_rx_ring)
- ring_id = desc->first_rx_ring;
+ for (i = 0; i < num_desc && num_rx != num; i++) {
+ if (desc_id > last_desc_id)
+ desc_id = first_desc_id;
- ring = NETMAP_RXRING(desc->nifp, ring_id);
+ desc = pktio_entry->s.pkt_nm.rx_desc[desc_id];
+ ring = NETMAP_RXRING(desc->nifp, desc->cur_rx_ring);
while (!nm_ring_empty(ring) && num_rx != num) {
slot_id = ring->cur;
@@ -274,51 +434,114 @@ static int netmap_recv(pktio_entry_t *pktio_entry,
odp_packet_t pkt_table[],
ring->cur = nm_ring_next(ring, slot_id);
ring->head = ring->cur;
}
- }
- desc->cur_rx_ring = ring_id;
- if (num_rx == 0) {
- if (odp_unlikely(poll(&polld, 1, 0) < 0))
- ODP_ERR("RX: poll error\n");
+ if (num_rx == 0) {
+ polld.fd = desc->fd;
+ polld.events = POLLIN;
+ if (odp_unlikely(poll(&polld, 1, 0) < 0))
+ ODP_ERR("RX: poll error\n");
+ }
+ desc_id++;
}
+ pktio_nm->rx_desc_ring[index].cur = desc_id;
+
+ if (!pktio_nm->lockless_rx)
+ odp_ticketlock_unlock(&pktio_nm->rx_lock[index]);
+
return num_rx;
}
-static int netmap_send(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[],
+static int netmap_recv(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[],
unsigned num)
{
+ return netmap_recv_queue(pktio_entry, 0, pkt_table, num);
+}
+
+/**
+ * Inject ODP packet to a netmap TX ring
+ *
+ * @param desc Netmap device descriptor
+ * @param pkt ODP packet handle
+ *
+ * @retval 0 on success
+ * @retval <0 on failure
+ */
+static inline int netmap_inject(struct nm_desc *desc, odp_packet_t pkt)
+{
+ struct netmap_ring *ring;
+ unsigned i;
+ uint32_t pkt_len = odp_packet_len(pkt);
+ uint32_t offset = 0;
+ char *buf;
+
+ ring = NETMAP_TXRING(desc->nifp, desc->cur_tx_ring);
+
+ if (nm_ring_empty(ring))
+ return -1;
+
+ if (pkt_len > ring->nr_buf_size) {
+ __odp_errno = -EMSGSIZE;
+ return -1;
+ }
+
+ i = ring->cur;
+ ring->slot[i].flags = 0;
+ ring->slot[i].len = pkt_len;
+
+ buf = NETMAP_BUF(ring, ring->slot[i].buf_idx);
+
+ if (odp_packet_copydata_out(pkt, offset, pkt_len, buf))
+ return -1;
+
+ ring->cur = nm_ring_next(ring, i);
+ ring->head = ring->cur;
+
+ return 0;
+}
+
+static int netmap_send_queue(pktio_entry_t *pktio_entry, int index,
+ odp_packet_t pkt_table[], int num)
+{
+ pkt_netmap_t *pktio_nm = &pktio_entry->s.pkt_nm;
struct pollfd polld;
- struct nm_desc *nm_desc = pktio_entry->s.pkt_nm.tx_desc;
- unsigned i, nb_tx;
- uint8_t *frame;
- uint32_t frame_len;
+ struct nm_desc *nm_desc;
+ int i;
+ int nb_tx;
+
+ if (!pktio_nm->lockless_tx)
+ odp_ticketlock_lock(&pktio_nm->tx_lock[index]);
+
+ nm_desc = pktio_nm->tx_desc[pktio_nm->tx_desc_ring[index].cur];
polld.fd = nm_desc->fd;
polld.events = POLLOUT;
for (nb_tx = 0; nb_tx < num; nb_tx++) {
- frame_len = 0;
- frame = odp_packet_l2_ptr(pkt_table[nb_tx], &frame_len);
for (i = 0; i < NM_INJECT_RETRIES; i++) {
- if (nm_inject(nm_desc, frame, frame_len) == 0)
+ if (netmap_inject(nm_desc, pkt_table[nb_tx]))
poll(&polld, 1, 0);
else
break;
}
- if (odp_unlikely(i == NM_INJECT_RETRIES)) {
- ioctl(nm_desc->fd, NIOCTXSYNC, NULL);
- break;
- }
}
/* Send pending packets */
poll(&polld, 1, 0);
+ if (!pktio_nm->lockless_tx)
+ odp_ticketlock_unlock(&pktio_nm->tx_lock[index]);
+
for (i = 0; i < nb_tx; i++)
odp_packet_free(pkt_table[i]);
return nb_tx;
}
+static int netmap_send(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[],
+ unsigned num)
+{
+ return netmap_send_queue(pktio_entry, 0, pkt_table, num);
+}
+
static int netmap_mac_addr_get(pktio_entry_t *pktio_entry, void *mac_addr)
{
memcpy(mac_addr, pktio_entry->s.pkt_nm.if_mac, ETH_ALEN);
@@ -350,6 +573,48 @@ static int netmap_capability(pktio_entry_t *pktio_entry,
return 0;
}
+static int netmap_in_queues(pktio_entry_t *pktio_entry, odp_queue_t queues[],
+ int num)
+{
+ int i;
+ int num_rx_queues = pktio_entry->s.pkt_nm.num_rx_queues;
+
+ if (queues && num > 0) {
+ for (i = 0; i < num && i < num_rx_queues; i++)
+ queues[i] = pktio_entry->s.in_queue[i].queue;
+ }
+
+ return pktio_entry->s.pkt_nm.num_rx_queues;
+}
+
+static int netmap_pktin_queues(pktio_entry_t *pktio_entry,
+ odp_pktin_queue_t queues[], int num)
+{
+ int i;
+ int num_rx_queues = pktio_entry->s.pkt_nm.num_rx_queues;
+
+ if (queues && num > 0) {
+ for (i = 0; i < num && i < num_rx_queues; i++)
+ queues[i] = pktio_entry->s.in_queue[i].pktin;
+ }
+
+ return pktio_entry->s.pkt_nm.num_rx_queues;
+}
+
+static int netmap_pktout_queues(pktio_entry_t *pktio_entry,
+ odp_pktout_queue_t queues[], int num)
+{
+ int i;
+ int num_tx_queues = pktio_entry->s.pkt_nm.num_tx_queues;
+
+ if (queues && num > 0) {
+ for (i = 0; i < num && i < num_tx_queues; i++)
+ queues[i] = pktio_entry->s.out_queue[i].pktout;
+ }
+
+ return pktio_entry->s.pkt_nm.num_tx_queues;
+}
+
const pktio_if_ops_t netmap_pktio_ops = {
.name = "netmap",
.init = NULL,
@@ -365,13 +630,13 @@ const pktio_if_ops_t netmap_pktio_ops = {
.promisc_mode_get = netmap_promisc_mode_get,
.mac_get = netmap_mac_addr_get,
.capability = netmap_capability,
- .input_queues_config = NULL,
- .output_queues_config = NULL,
- .in_queues = NULL,
- .pktin_queues = NULL,
- .pktout_queues = NULL,
- .recv_queue = NULL,
- .send_queue = NULL
+ .input_queues_config = netmap_input_queues_config,
+ .output_queues_config = netmap_output_queues_config,
+ .in_queues = netmap_in_queues,
+ .pktin_queues = netmap_pktin_queues,
+ .pktout_queues = netmap_pktout_queues,
+ .recv_queue = netmap_recv_queue,
+ .send_queue = netmap_send_queue
};
#endif /* ODP_NETMAP */
--
1.9.1
_______________________________________________
lng-odp mailing list
[email protected]
https://lists.linaro.org/mailman/listinfo/lng-odp
