subject:"\[ovs\-dev\] \[RFC\] dpdk\: support multiple queues in vhost"

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

2015-08-11 Thread Traynor, Kevin


 -Original Message-
 From: dev [mailto:dev-boun...@openvswitch.org] On Behalf Of Flavio Leitner
 Sent: Friday, July 31, 2015 11:30 PM
 To: dev@openvswitch.org
 Cc: Flavio Leitner
 Subject: [ovs-dev] [RFC] dpdk: support multiple queues in vhost
 
 This RFC is based on the vhost multiple queues work on
 dpdk-dev: http://dpdk.org/ml/archives/dev/2015-June/019345.html

Hi Flavio - the patch looks good, one minor comment below.

 
 Signed-off-by: Flavio Leitner f...@redhat.com
 ---
  lib/netdev-dpdk.c | 61 -
 --
  1 file changed, 40 insertions(+), 21 deletions(-)
 
 diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
 index 5ae805e..493172c 100644
 --- a/lib/netdev-dpdk.c
 +++ b/lib/netdev-dpdk.c
 @@ -215,12 +215,9 @@ struct netdev_dpdk {
   * If the numbers match, 'txq_needs_locking' is false, otherwise it is
   * true and we will take a spinlock on transmission */
  int real_n_txq;
 +int real_n_rxq;
  bool txq_needs_locking;
 
 -/* Spinlock for vhost transmission.  Other DPDK devices use spinlocks in
 - * dpdk_tx_queue */
 -rte_spinlock_t vhost_tx_lock;
 -
  /* virtio-net structure for vhost device */
  OVSRCU_TYPE(struct virtio_net *) virtio_dev;
 
 @@ -602,13 +599,10 @@ dpdk_dev_parse_name(const char dev_name[], const char
 prefix[],
  static int
  vhost_construct_helper(struct netdev *netdev_) OVS_REQUIRES(dpdk_mutex)
  {
 -struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
 -
  if (rte_eal_init_ret) {
  return rte_eal_init_ret;
  }
 
 -rte_spinlock_init(netdev-vhost_tx_lock);
  return netdev_dpdk_init(netdev_, -1, DPDK_DEV_VHOST);
  }
 
 @@ -791,9 +785,16 @@ netdev_dpdk_vhost_set_multiq(struct netdev *netdev_,
 unsigned int n_txq,
  ovs_mutex_lock(dpdk_mutex);
  ovs_mutex_lock(netdev-mutex);
 
 +rte_free(netdev-tx_q);
 +/* FIXME: the number of vqueues needs to match */
  netdev-up.n_txq = n_txq;
 -netdev-real_n_txq = 1;
 -netdev-up.n_rxq = 1;
 +netdev-up.n_rxq = n_rxq;
 +
 +/* vring has txq = rxq */
 +netdev-real_n_txq = n_rxq;
 +netdev-real_n_rxq = n_rxq;
 +netdev-txq_needs_locking = netdev-real_n_txq != netdev-up.n_txq;
 +netdev_dpdk_alloc_txq(netdev, netdev-up.n_txq);
 
  ovs_mutex_unlock(netdev-mutex);
  ovs_mutex_unlock(dpdk_mutex);
 @@ -904,14 +905,14 @@ netdev_dpdk_vhost_rxq_recv(struct netdev_rxq *rxq_,
  struct netdev *netdev = rx-up.netdev;
  struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
  struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
 -int qid = 1;
 +int qid = rxq_-queue_id;
  uint16_t nb_rx = 0;
 
  if (OVS_UNLIKELY(!is_vhost_running(virtio_dev))) {
  return EAGAIN;
  }
 
 -nb_rx = rte_vhost_dequeue_burst(virtio_dev, qid,
 +nb_rx = rte_vhost_dequeue_burst(virtio_dev, VIRTIO_TXQ + qid * 2,
  vhost_dev-dpdk_mp-mp,
  (struct rte_mbuf **)packets,
  NETDEV_MAX_BURST);
 @@ -958,8 +959,9 @@ netdev_dpdk_rxq_recv(struct netdev_rxq *rxq_, struct
 dp_packet **packets,
  }
 
  static void
 -__netdev_dpdk_vhost_send(struct netdev *netdev, struct dp_packet **pkts,
 - int cnt, bool may_steal)
 +__netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
 + struct dp_packet **pkts, int cnt,
 + bool may_steal)
  {
  struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
  struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
 @@ -974,13 +976,16 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct
 dp_packet **pkts,
  goto out;
  }
 
 -/* There is vHost TX single queue, So we need to lock it for TX. */
 -rte_spinlock_lock(vhost_dev-vhost_tx_lock);
 +if (vhost_dev-txq_needs_locking) {
 +qid = qid % vhost_dev-real_n_txq;
 +rte_spinlock_lock(vhost_dev-tx_q[qid].tx_lock);
 +}
 
  do {
 +int vhost_qid = VIRTIO_RXQ + qid * VIRTIO_QNUM;
  unsigned int tx_pkts;
 
 -tx_pkts = rte_vhost_enqueue_burst(virtio_dev, VIRTIO_RXQ,
 +tx_pkts = rte_vhost_enqueue_burst(virtio_dev, vhost_qid,
cur_pkts, cnt);
  if (OVS_LIKELY(tx_pkts)) {
  /* Packets have been sent.*/
 @@ -999,7 +1004,7 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct
 dp_packet **pkts,
   * Unable to enqueue packets to vhost interface.
   * Check available entries before retrying.
   */
 -while (!rte_vring_available_entries(virtio_dev, VIRTIO_RXQ)) {
 +while (!rte_vring_available_entries(virtio_dev, vhost_qid)) {
  if (OVS_UNLIKELY((rte_get_timer_cycles() - start) 
 timeout)) {
  expired = 1;
  break;
 @@ -1011,7 +1016,10

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

2015-08-11 Thread Flavio Leitner

On Tue, Aug 11, 2015 at 04:24:01PM +, Traynor, Kevin wrote:
 
  -Original Message-
  From: dev [mailto:dev-boun...@openvswitch.org] On Behalf Of Flavio Leitner
  Sent: Friday, July 31, 2015 11:30 PM
  To: dev@openvswitch.org
  Cc: Flavio Leitner
  Subject: [ovs-dev] [RFC] dpdk: support multiple queues in vhost
  
  This RFC is based on the vhost multiple queues work on
  dpdk-dev: http://dpdk.org/ml/archives/dev/2015-June/019345.html
 
 Hi Flavio - the patch looks good, one minor comment below.
 
  
  Signed-off-by: Flavio Leitner f...@redhat.com
  ---
   lib/netdev-dpdk.c | 61 
  -
  --
   1 file changed, 40 insertions(+), 21 deletions(-)
  
  diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
  index 5ae805e..493172c 100644
  --- a/lib/netdev-dpdk.c
  +++ b/lib/netdev-dpdk.c
  @@ -215,12 +215,9 @@ struct netdev_dpdk {
* If the numbers match, 'txq_needs_locking' is false, otherwise it is
* true and we will take a spinlock on transmission */
   int real_n_txq;
  +int real_n_rxq;
   bool txq_needs_locking;
  
  -/* Spinlock for vhost transmission.  Other DPDK devices use spinlocks 
  in
  - * dpdk_tx_queue */
  -rte_spinlock_t vhost_tx_lock;
  -
   /* virtio-net structure for vhost device */
   OVSRCU_TYPE(struct virtio_net *) virtio_dev;
  
  @@ -602,13 +599,10 @@ dpdk_dev_parse_name(const char dev_name[], const char
  prefix[],
   static int
   vhost_construct_helper(struct netdev *netdev_) OVS_REQUIRES(dpdk_mutex)
   {
  -struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
  -
   if (rte_eal_init_ret) {
   return rte_eal_init_ret;
   }
  
  -rte_spinlock_init(netdev-vhost_tx_lock);
   return netdev_dpdk_init(netdev_, -1, DPDK_DEV_VHOST);
   }
  
  @@ -791,9 +785,16 @@ netdev_dpdk_vhost_set_multiq(struct netdev *netdev_,
  unsigned int n_txq,
   ovs_mutex_lock(dpdk_mutex);
   ovs_mutex_lock(netdev-mutex);
  
  +rte_free(netdev-tx_q);
  +/* FIXME: the number of vqueues needs to match */
   netdev-up.n_txq = n_txq;
  -netdev-real_n_txq = 1;
  -netdev-up.n_rxq = 1;
  +netdev-up.n_rxq = n_rxq;
  +
  +/* vring has txq = rxq */
  +netdev-real_n_txq = n_rxq;
  +netdev-real_n_rxq = n_rxq;
  +netdev-txq_needs_locking = netdev-real_n_txq != netdev-up.n_txq;
  +netdev_dpdk_alloc_txq(netdev, netdev-up.n_txq);
  
   ovs_mutex_unlock(netdev-mutex);
   ovs_mutex_unlock(dpdk_mutex);
  @@ -904,14 +905,14 @@ netdev_dpdk_vhost_rxq_recv(struct netdev_rxq *rxq_,
   struct netdev *netdev = rx-up.netdev;
   struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
   struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
  -int qid = 1;
  +int qid = rxq_-queue_id;
   uint16_t nb_rx = 0;
  
   if (OVS_UNLIKELY(!is_vhost_running(virtio_dev))) {
   return EAGAIN;
   }
  
  -nb_rx = rte_vhost_dequeue_burst(virtio_dev, qid,
  +nb_rx = rte_vhost_dequeue_burst(virtio_dev, VIRTIO_TXQ + qid * 2,
   vhost_dev-dpdk_mp-mp,
   (struct rte_mbuf **)packets,
   NETDEV_MAX_BURST);
  @@ -958,8 +959,9 @@ netdev_dpdk_rxq_recv(struct netdev_rxq *rxq_, struct
  dp_packet **packets,
   }
  
   static void
  -__netdev_dpdk_vhost_send(struct netdev *netdev, struct dp_packet **pkts,
  - int cnt, bool may_steal)
  +__netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
  + struct dp_packet **pkts, int cnt,
  + bool may_steal)
   {
   struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
   struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
  @@ -974,13 +976,16 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct
  dp_packet **pkts,
   goto out;
   }
  
  -/* There is vHost TX single queue, So we need to lock it for TX. */
  -rte_spinlock_lock(vhost_dev-vhost_tx_lock);
  +if (vhost_dev-txq_needs_locking) {
  +qid = qid % vhost_dev-real_n_txq;
  +rte_spinlock_lock(vhost_dev-tx_q[qid].tx_lock);
  +}
  
   do {
  +int vhost_qid = VIRTIO_RXQ + qid * VIRTIO_QNUM;
   unsigned int tx_pkts;
  
  -tx_pkts = rte_vhost_enqueue_burst(virtio_dev, VIRTIO_RXQ,
  +tx_pkts = rte_vhost_enqueue_burst(virtio_dev, vhost_qid,
 cur_pkts, cnt);
   if (OVS_LIKELY(tx_pkts)) {
   /* Packets have been sent.*/
  @@ -999,7 +1004,7 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct
  dp_packet **pkts,
* Unable to enqueue packets to vhost interface.
* Check available entries before retrying.
*/
  -while (!rte_vring_available_entries(virtio_dev, VIRTIO_RXQ)) {
  +while (!rte_vring_available_entries

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

2015-08-07 Thread Flavio Leitner

On Fri, Aug 07, 2015 at 08:43:23AM -0400, Thomas F Herbert wrote:
 On 8/6/15 5:29 PM, Flavio Leitner wrote:
 On Thu, Aug 06, 2015 at 03:24:29PM -0400, Thomas F Herbert wrote:
 On 8/6/15 1:40 PM, Flavio Leitner wrote:
 On Thu, Aug 06, 2015 at 12:39:29PM -0400, Thomas F Herbert wrote:
 On 7/31/15 6:30 PM, Flavio Leitner wrote:
 This RFC is based on the vhost multiple queues work on
 dpdk-dev: http://dpdk.org/ml/archives/dev/2015-June/019345.html
 
 Signed-off-by: Flavio Leitner f...@redhat.com
 ---
   lib/netdev-dpdk.c | 61 
  ---
   1 file changed, 40 insertions(+), 21 deletions(-)
 
 diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
 index 5ae805e..493172c 100644
 --- a/lib/netdev-dpdk.c
 +++ b/lib/netdev-dpdk.c
 @@ -215,12 +215,9 @@ struct netdev_dpdk {
* If the numbers match, 'txq_needs_locking' is false, otherwise 
  it is
* true and we will take a spinlock on transmission */
   int real_n_txq;
 +int real_n_rxq;
   bool txq_needs_locking;
 
 -/* Spinlock for vhost transmission.  Other DPDK devices use 
 spinlocks in
 - * dpdk_tx_queue */
 -rte_spinlock_t vhost_tx_lock;
 -
   /* virtio-net structure for vhost device */
   OVSRCU_TYPE(struct virtio_net *) virtio_dev;
 
 @@ -602,13 +599,10 @@ dpdk_dev_parse_name(const char dev_name[], const 
 char prefix[],
   static int
   vhost_construct_helper(struct netdev *netdev_) OVS_REQUIRES(dpdk_mutex)
   {
 -struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
 -
   if (rte_eal_init_ret) {
   return rte_eal_init_ret;
   }
 
 -rte_spinlock_init(netdev-vhost_tx_lock);
   return netdev_dpdk_init(netdev_, -1, DPDK_DEV_VHOST);
   }
 
 @@ -791,9 +785,16 @@ netdev_dpdk_vhost_set_multiq(struct netdev 
 *netdev_, unsigned int n_txq,
   ovs_mutex_lock(dpdk_mutex);
   ovs_mutex_lock(netdev-mutex);
 
 +rte_free(netdev-tx_q);
 +/* FIXME: the number of vqueues needs to match */
 Do you still need this FIXME? Isn't the code you added below freeing and
 re-allocating the correct number of tx queues?
 
 Yes, because that is about virtual queues provided by qemu.
 Thanks,
 fbl
 I understand this is an RFC but I think your patch is in the right
 direction. I know the merging is complex and requires upstream changes to
 DPDK and Qemu. I ack this patch is an important step that moves the ball
 forward toward vhost user performance of DPDK accelerated OVS.

Thanks for your review
fbl

___
dev mailing list
dev@openvswitch.org
http://openvswitch.org/mailman/listinfo/dev

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

2015-08-07 Thread Thomas F Herbert


On 8/6/15 5:29 PM, Flavio Leitner wrote:

On Thu, Aug 06, 2015 at 03:24:29PM -0400, Thomas F Herbert wrote:

On 8/6/15 1:40 PM, Flavio Leitner wrote:

On Thu, Aug 06, 2015 at 12:39:29PM -0400, Thomas F Herbert wrote:

On 7/31/15 6:30 PM, Flavio Leitner wrote:

This RFC is based on the vhost multiple queues work on
dpdk-dev: http://dpdk.org/ml/archives/dev/2015-June/019345.html

Signed-off-by: Flavio Leitner f...@redhat.com
---
  lib/netdev-dpdk.c | 61 ---
  1 file changed, 40 insertions(+), 21 deletions(-)

diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index 5ae805e..493172c 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -215,12 +215,9 @@ struct netdev_dpdk {
   * If the numbers match, 'txq_needs_locking' is false, otherwise it is
   * true and we will take a spinlock on transmission */
  int real_n_txq;
+int real_n_rxq;
  bool txq_needs_locking;

-/* Spinlock for vhost transmission.  Other DPDK devices use spinlocks in
- * dpdk_tx_queue */
-rte_spinlock_t vhost_tx_lock;
-
  /* virtio-net structure for vhost device */
  OVSRCU_TYPE(struct virtio_net *) virtio_dev;

@@ -602,13 +599,10 @@ dpdk_dev_parse_name(const char dev_name[], const char 
prefix[],
  static int
  vhost_construct_helper(struct netdev *netdev_) OVS_REQUIRES(dpdk_mutex)
  {
-struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
-
  if (rte_eal_init_ret) {
  return rte_eal_init_ret;
  }

-rte_spinlock_init(netdev-vhost_tx_lock);
  return netdev_dpdk_init(netdev_, -1, DPDK_DEV_VHOST);
  }

@@ -791,9 +785,16 @@ netdev_dpdk_vhost_set_multiq(struct netdev *netdev_, 
unsigned int n_txq,
  ovs_mutex_lock(dpdk_mutex);
  ovs_mutex_lock(netdev-mutex);

+rte_free(netdev-tx_q);
+/* FIXME: the number of vqueues needs to match */

Do you still need this FIXME? Isn't the code you added below freeing and
re-allocating the correct number of tx queues?


Yes, because that is about virtual queues provided by qemu.
Thanks,
fbl
I understand this is an RFC but I think your patch is in the right 
direction. I know the merging is complex and requires upstream changes 
to DPDK and Qemu. I ack this patch is an important step that moves the 
ball forward toward vhost user performance of DPDK accelerated OVS.


--TFH



  netdev-up.n_txq = n_txq;
-netdev-real_n_txq = 1;
-netdev-up.n_rxq = 1;
+netdev-up.n_rxq = n_rxq;
+
+/* vring has txq = rxq */
+netdev-real_n_txq = n_rxq;
+netdev-real_n_rxq = n_rxq;
+netdev-txq_needs_locking = netdev-real_n_txq != netdev-up.n_txq;
+netdev_dpdk_alloc_txq(netdev, netdev-up.n_txq);

  ovs_mutex_unlock(netdev-mutex);
  ovs_mutex_unlock(dpdk_mutex);
@@ -904,14 +905,14 @@ netdev_dpdk_vhost_rxq_recv(struct netdev_rxq *rxq_,
  struct netdev *netdev = rx-up.netdev;
  struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
  struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
-int qid = 1;
+int qid = rxq_-queue_id;
  uint16_t nb_rx = 0;

  if (OVS_UNLIKELY(!is_vhost_running(virtio_dev))) {
  return EAGAIN;
  }

-nb_rx = rte_vhost_dequeue_burst(virtio_dev, qid,
+nb_rx = rte_vhost_dequeue_burst(virtio_dev, VIRTIO_TXQ + qid * 2,
  vhost_dev-dpdk_mp-mp,
  (struct rte_mbuf **)packets,
  NETDEV_MAX_BURST);
@@ -958,8 +959,9 @@ netdev_dpdk_rxq_recv(struct netdev_rxq *rxq_, struct 
dp_packet **packets,
  }

  static void
-__netdev_dpdk_vhost_send(struct netdev *netdev, struct dp_packet **pkts,
- int cnt, bool may_steal)
+__netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
+ struct dp_packet **pkts, int cnt,
+ bool may_steal)
  {
  struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
  struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
@@ -974,13 +976,16 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct 
dp_packet **pkts,
  goto out;
  }

-/* There is vHost TX single queue, So we need to lock it for TX. */
-rte_spinlock_lock(vhost_dev-vhost_tx_lock);
+if (vhost_dev-txq_needs_locking) {
+qid = qid % vhost_dev-real_n_txq;
+rte_spinlock_lock(vhost_dev-tx_q[qid].tx_lock);
+}

  do {
+int vhost_qid = VIRTIO_RXQ + qid * VIRTIO_QNUM;
  unsigned int tx_pkts;

-tx_pkts = rte_vhost_enqueue_burst(virtio_dev, VIRTIO_RXQ,
+tx_pkts = rte_vhost_enqueue_burst(virtio_dev, vhost_qid,
cur_pkts, cnt);
  if (OVS_LIKELY(tx_pkts)) {
  /* Packets have been sent.*/
@@ -999,7 +1004,7 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct 
dp_packet **pkts,
   * Unable to enqueue packets to vhost interface.
   * Check available

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

2015-08-06 Thread Thomas F Herbert


On 7/31/15 6:30 PM, Flavio Leitner wrote:

This RFC is based on the vhost multiple queues work on
dpdk-dev: http://dpdk.org/ml/archives/dev/2015-June/019345.html

Signed-off-by: Flavio Leitner f...@redhat.com
---
  lib/netdev-dpdk.c | 61 ---
  1 file changed, 40 insertions(+), 21 deletions(-)

diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index 5ae805e..493172c 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -215,12 +215,9 @@ struct netdev_dpdk {
   * If the numbers match, 'txq_needs_locking' is false, otherwise it is
   * true and we will take a spinlock on transmission */
  int real_n_txq;
+int real_n_rxq;
  bool txq_needs_locking;

-/* Spinlock for vhost transmission.  Other DPDK devices use spinlocks in
- * dpdk_tx_queue */
-rte_spinlock_t vhost_tx_lock;
-
  /* virtio-net structure for vhost device */
  OVSRCU_TYPE(struct virtio_net *) virtio_dev;

@@ -602,13 +599,10 @@ dpdk_dev_parse_name(const char dev_name[], const char 
prefix[],
  static int
  vhost_construct_helper(struct netdev *netdev_) OVS_REQUIRES(dpdk_mutex)
  {
-struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
-
  if (rte_eal_init_ret) {
  return rte_eal_init_ret;
  }

-rte_spinlock_init(netdev-vhost_tx_lock);
  return netdev_dpdk_init(netdev_, -1, DPDK_DEV_VHOST);
  }

@@ -791,9 +785,16 @@ netdev_dpdk_vhost_set_multiq(struct netdev *netdev_, 
unsigned int n_txq,
  ovs_mutex_lock(dpdk_mutex);
  ovs_mutex_lock(netdev-mutex);

+rte_free(netdev-tx_q);
+/* FIXME: the number of vqueues needs to match */
  netdev-up.n_txq = n_txq;
-netdev-real_n_txq = 1;
-netdev-up.n_rxq = 1;
+netdev-up.n_rxq = n_rxq;
+
+/* vring has txq = rxq */
+netdev-real_n_txq = n_rxq;
+netdev-real_n_rxq = n_rxq;
+netdev-txq_needs_locking = netdev-real_n_txq != netdev-up.n_txq;
+netdev_dpdk_alloc_txq(netdev, netdev-up.n_txq);

  ovs_mutex_unlock(netdev-mutex);
  ovs_mutex_unlock(dpdk_mutex);
@@ -904,14 +905,14 @@ netdev_dpdk_vhost_rxq_recv(struct netdev_rxq *rxq_,
  struct netdev *netdev = rx-up.netdev;
  struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
  struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
-int qid = 1;
+int qid = rxq_-queue_id;
  uint16_t nb_rx = 0;

  if (OVS_UNLIKELY(!is_vhost_running(virtio_dev))) {
  return EAGAIN;
  }

-nb_rx = rte_vhost_dequeue_burst(virtio_dev, qid,
+nb_rx = rte_vhost_dequeue_burst(virtio_dev, VIRTIO_TXQ + qid * 2,
  vhost_dev-dpdk_mp-mp,
  (struct rte_mbuf **)packets,
  NETDEV_MAX_BURST);
@@ -958,8 +959,9 @@ netdev_dpdk_rxq_recv(struct netdev_rxq *rxq_, struct 
dp_packet **packets,
  }

  static void
-__netdev_dpdk_vhost_send(struct netdev *netdev, struct dp_packet **pkts,
- int cnt, bool may_steal)
+__netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
+ struct dp_packet **pkts, int cnt,
+ bool may_steal)
  {
  struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
  struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
@@ -974,13 +976,16 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct 
dp_packet **pkts,
  goto out;
  }

-/* There is vHost TX single queue, So we need to lock it for TX. */
-rte_spinlock_lock(vhost_dev-vhost_tx_lock);
+if (vhost_dev-txq_needs_locking) {
+qid = qid % vhost_dev-real_n_txq;
+rte_spinlock_lock(vhost_dev-tx_q[qid].tx_lock);
+}

  do {
+int vhost_qid = VIRTIO_RXQ + qid * VIRTIO_QNUM;
  unsigned int tx_pkts;

-tx_pkts = rte_vhost_enqueue_burst(virtio_dev, VIRTIO_RXQ,
+tx_pkts = rte_vhost_enqueue_burst(virtio_dev, vhost_qid,
cur_pkts, cnt);
  if (OVS_LIKELY(tx_pkts)) {
  /* Packets have been sent.*/
@@ -999,7 +1004,7 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct 
dp_packet **pkts,
   * Unable to enqueue packets to vhost interface.
   * Check available entries before retrying.
   */
-while (!rte_vring_available_entries(virtio_dev, VIRTIO_RXQ)) {
+while (!rte_vring_available_entries(virtio_dev, vhost_qid)) {
  if (OVS_UNLIKELY((rte_get_timer_cycles() - start)  timeout)) 
{
  expired = 1;
  break;
@@ -1011,7 +1016,10 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct 
dp_packet **pkts,
  }
  }
  } while (cnt);
-rte_spinlock_unlock(vhost_dev-vhost_tx_lock);
+
+if (vhost_dev-txq_needs_locking) {
+rte_spinlock_unlock(vhost_dev-tx_q[qid].tx_lock);
+}

  rte_spinlock_lock(vhost_dev-stats_lock);

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

2015-08-06 Thread Flavio Leitner

On Thu, Aug 06, 2015 at 03:24:29PM -0400, Thomas F Herbert wrote:
 On 8/6/15 1:40 PM, Flavio Leitner wrote:
 On Thu, Aug 06, 2015 at 12:39:29PM -0400, Thomas F Herbert wrote:
 On 7/31/15 6:30 PM, Flavio Leitner wrote:
 This RFC is based on the vhost multiple queues work on
 dpdk-dev: http://dpdk.org/ml/archives/dev/2015-June/019345.html
 
 Signed-off-by: Flavio Leitner f...@redhat.com
 ---
   lib/netdev-dpdk.c | 61 
  ---
   1 file changed, 40 insertions(+), 21 deletions(-)
 
 diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
 index 5ae805e..493172c 100644
 --- a/lib/netdev-dpdk.c
 +++ b/lib/netdev-dpdk.c
 @@ -215,12 +215,9 @@ struct netdev_dpdk {
* If the numbers match, 'txq_needs_locking' is false, otherwise it 
  is
* true and we will take a spinlock on transmission */
   int real_n_txq;
 +int real_n_rxq;
   bool txq_needs_locking;
 
 -/* Spinlock for vhost transmission.  Other DPDK devices use spinlocks 
 in
 - * dpdk_tx_queue */
 -rte_spinlock_t vhost_tx_lock;
 -
   /* virtio-net structure for vhost device */
   OVSRCU_TYPE(struct virtio_net *) virtio_dev;
 
 @@ -602,13 +599,10 @@ dpdk_dev_parse_name(const char dev_name[], const 
 char prefix[],
   static int
   vhost_construct_helper(struct netdev *netdev_) OVS_REQUIRES(dpdk_mutex)
   {
 -struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
 -
   if (rte_eal_init_ret) {
   return rte_eal_init_ret;
   }
 
 -rte_spinlock_init(netdev-vhost_tx_lock);
   return netdev_dpdk_init(netdev_, -1, DPDK_DEV_VHOST);
   }
 
 @@ -791,9 +785,16 @@ netdev_dpdk_vhost_set_multiq(struct netdev *netdev_, 
 unsigned int n_txq,
   ovs_mutex_lock(dpdk_mutex);
   ovs_mutex_lock(netdev-mutex);
 
 +rte_free(netdev-tx_q);
 +/* FIXME: the number of vqueues needs to match */
 Do you still need this FIXME? Isn't the code you added below freeing and
 re-allocating the correct number of tx queues?

Yes, because that is about virtual queues provided by qemu.
Thanks,
fbl

   netdev-up.n_txq = n_txq;
 -netdev-real_n_txq = 1;
 -netdev-up.n_rxq = 1;
 +netdev-up.n_rxq = n_rxq;
 +
 +/* vring has txq = rxq */
 +netdev-real_n_txq = n_rxq;
 +netdev-real_n_rxq = n_rxq;
 +netdev-txq_needs_locking = netdev-real_n_txq != netdev-up.n_txq;
 +netdev_dpdk_alloc_txq(netdev, netdev-up.n_txq);
 
   ovs_mutex_unlock(netdev-mutex);
   ovs_mutex_unlock(dpdk_mutex);
 @@ -904,14 +905,14 @@ netdev_dpdk_vhost_rxq_recv(struct netdev_rxq *rxq_,
   struct netdev *netdev = rx-up.netdev;
   struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
   struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
 -int qid = 1;
 +int qid = rxq_-queue_id;
   uint16_t nb_rx = 0;
 
   if (OVS_UNLIKELY(!is_vhost_running(virtio_dev))) {
   return EAGAIN;
   }
 
 -nb_rx = rte_vhost_dequeue_burst(virtio_dev, qid,
 +nb_rx = rte_vhost_dequeue_burst(virtio_dev, VIRTIO_TXQ + qid * 2,
   vhost_dev-dpdk_mp-mp,
   (struct rte_mbuf **)packets,
   NETDEV_MAX_BURST);
 @@ -958,8 +959,9 @@ netdev_dpdk_rxq_recv(struct netdev_rxq *rxq_, struct 
 dp_packet **packets,
   }
 
   static void
 -__netdev_dpdk_vhost_send(struct netdev *netdev, struct dp_packet **pkts,
 - int cnt, bool may_steal)
 +__netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
 + struct dp_packet **pkts, int cnt,
 + bool may_steal)
   {
   struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
   struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
 @@ -974,13 +976,16 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, 
 struct dp_packet **pkts,
   goto out;
   }
 
 -/* There is vHost TX single queue, So we need to lock it for TX. */
 -rte_spinlock_lock(vhost_dev-vhost_tx_lock);
 +if (vhost_dev-txq_needs_locking) {
 +qid = qid % vhost_dev-real_n_txq;
 +rte_spinlock_lock(vhost_dev-tx_q[qid].tx_lock);
 +}
 
   do {
 +int vhost_qid = VIRTIO_RXQ + qid * VIRTIO_QNUM;
   unsigned int tx_pkts;
 
 -tx_pkts = rte_vhost_enqueue_burst(virtio_dev, VIRTIO_RXQ,
 +tx_pkts = rte_vhost_enqueue_burst(virtio_dev, vhost_qid,
 cur_pkts, cnt);
   if (OVS_LIKELY(tx_pkts)) {
   /* Packets have been sent.*/
 @@ -999,7 +1004,7 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, 
 struct dp_packet **pkts,
* Unable to enqueue packets to vhost interface.
* Check available entries before retrying.
*/
 -while (!rte_vring_available_entries(virtio_dev, VIRTIO_RXQ)) {
 +while (!rte_vring_available_entries(virtio_dev, vhost_qid)) {

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

2015-08-06 Thread Karthick, A.R.

Just to top-post briefly:
 qemu reverted the vhost-user multiqueue support sometime back.
 So this patch isn't really usable.

Regards,
-Karthick

 commit f73ca7363440240b7ee5ee7f7ddb1c64751efb54
Merge: 7135847 f9d6dbf
Author: Peter Maydell peter.mayd...@linaro.org
Date:   Mon Jul 20 13:25:28 2015 +0100

Merge remote-tracking branch 'remotes/mst/tags/for_upstream' into
staging

virtio, vhost, pc fixes for 2.4

The only notable thing here is vhost-user multiqueue
revert. We'll work on making it stable in 2.5,
reverting now means we won't have to maintain
bug for bug compability forever.

Signed-off-by: Michael S. Tsirkin m...@redhat.com

# gpg: Signature made Mon Jul 20 12:24:00 2015 BST using RSA key ID
D28D5469
# gpg: Good signature from Michael S. Tsirkin m...@kernel.org
# gpg: aka Michael S. Tsirkin m...@redhat.com

* remotes/mst/tags/for_upstream:
  virtio-net: remove virtio queues if the guest doesn't support
multiqueue
  virtio-net: Flush incoming queues when DRIVER_OK is being set
  pci_add_capability: remove duplicate comments
  virtio-net: unbreak any layout
  Revert vhost-user: add multi queue support
  ich9: fix skipped vmstate_memhp_state subsection

Signed-off-by: Peter Maydell peter.mayd...@linaro.org


On Thu, Aug 6, 2015 at 10:40 AM, Flavio Leitner f...@sysclose.org wrote:

 On Thu, Aug 06, 2015 at 12:39:29PM -0400, Thomas F Herbert wrote:
  On 7/31/15 6:30 PM, Flavio Leitner wrote:
  This RFC is based on the vhost multiple queues work on
  dpdk-dev: http://dpdk.org/ml/archives/dev/2015-June/019345.html
  
  Signed-off-by: Flavio Leitner f...@redhat.com
  ---
lib/netdev-dpdk.c | 61
 ---
1 file changed, 40 insertions(+), 21 deletions(-)
  
  diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
  index 5ae805e..493172c 100644
  --- a/lib/netdev-dpdk.c
  +++ b/lib/netdev-dpdk.c
  @@ -215,12 +215,9 @@ struct netdev_dpdk {
 * If the numbers match, 'txq_needs_locking' is false, otherwise
 it is
 * true and we will take a spinlock on transmission */
int real_n_txq;
  +int real_n_rxq;
bool txq_needs_locking;
  
  -/* Spinlock for vhost transmission.  Other DPDK devices use
 spinlocks in
  - * dpdk_tx_queue */
  -rte_spinlock_t vhost_tx_lock;
  -
/* virtio-net structure for vhost device */
OVSRCU_TYPE(struct virtio_net *) virtio_dev;
  
  @@ -602,13 +599,10 @@ dpdk_dev_parse_name(const char dev_name[], const
 char prefix[],
static int
vhost_construct_helper(struct netdev *netdev_)
 OVS_REQUIRES(dpdk_mutex)
{
  -struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
  -
if (rte_eal_init_ret) {
return rte_eal_init_ret;
}
  
  -rte_spinlock_init(netdev-vhost_tx_lock);
return netdev_dpdk_init(netdev_, -1, DPDK_DEV_VHOST);
}
  
  @@ -791,9 +785,16 @@ netdev_dpdk_vhost_set_multiq(struct netdev
 *netdev_, unsigned int n_txq,
ovs_mutex_lock(dpdk_mutex);
ovs_mutex_lock(netdev-mutex);
  
  +rte_free(netdev-tx_q);
  +/* FIXME: the number of vqueues needs to match */
netdev-up.n_txq = n_txq;
  -netdev-real_n_txq = 1;
  -netdev-up.n_rxq = 1;
  +netdev-up.n_rxq = n_rxq;
  +
  +/* vring has txq = rxq */
  +netdev-real_n_txq = n_rxq;
  +netdev-real_n_rxq = n_rxq;
  +netdev-txq_needs_locking = netdev-real_n_txq != netdev-up.n_txq;
  +netdev_dpdk_alloc_txq(netdev, netdev-up.n_txq);
  
ovs_mutex_unlock(netdev-mutex);
ovs_mutex_unlock(dpdk_mutex);
  @@ -904,14 +905,14 @@ netdev_dpdk_vhost_rxq_recv(struct netdev_rxq
 *rxq_,
struct netdev *netdev = rx-up.netdev;
struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
  -int qid = 1;
  +int qid = rxq_-queue_id;
uint16_t nb_rx = 0;
  
if (OVS_UNLIKELY(!is_vhost_running(virtio_dev))) {
return EAGAIN;
}
  
  -nb_rx = rte_vhost_dequeue_burst(virtio_dev, qid,
  +nb_rx = rte_vhost_dequeue_burst(virtio_dev, VIRTIO_TXQ + qid * 2,
vhost_dev-dpdk_mp-mp,
(struct rte_mbuf **)packets,
NETDEV_MAX_BURST);
  @@ -958,8 +959,9 @@ netdev_dpdk_rxq_recv(struct netdev_rxq *rxq_,
 struct dp_packet **packets,
}
  
static void
  -__netdev_dpdk_vhost_send(struct netdev *netdev, struct dp_packet
 **pkts,
  - int cnt, bool may_steal)
  +__netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
  + struct dp_packet **pkts, int cnt,
  + bool may_steal)
{
struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
  @@ -974,13 +976,16

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

2015-08-06 Thread Flavio Leitner

On Thu, Aug 06, 2015 at 10:54:16AM -0700, Karthick, A.R. wrote:
 Just to top-post briefly:
  qemu reverted the vhost-user multiqueue support sometime back.
  So this patch isn't really usable.

That's correct.  There is a new proposal enhancing the protocol
which will require work on the DPDK proposal as well.  Since OVS
uses DPDK vhost-user library, those changes shouldn't affect much
this patch.

Sorry for not being clear before. The goal here is to find out
if the OVS side is in the right direction.

Thanks for your feedback,
fbl


 
 Regards,
 -Karthick
 
  commit f73ca7363440240b7ee5ee7f7ddb1c64751efb54
 Merge: 7135847 f9d6dbf
 Author: Peter Maydell peter.mayd...@linaro.org
 Date:   Mon Jul 20 13:25:28 2015 +0100
 
 Merge remote-tracking branch 'remotes/mst/tags/for_upstream' into
 staging
 
 virtio, vhost, pc fixes for 2.4
 
 The only notable thing here is vhost-user multiqueue
 revert. We'll work on making it stable in 2.5,
 reverting now means we won't have to maintain
 bug for bug compability forever.
 
 Signed-off-by: Michael S. Tsirkin m...@redhat.com
 
 # gpg: Signature made Mon Jul 20 12:24:00 2015 BST using RSA key ID
 D28D5469
 # gpg: Good signature from Michael S. Tsirkin m...@kernel.org
 # gpg: aka Michael S. Tsirkin m...@redhat.com
 
 * remotes/mst/tags/for_upstream:
   virtio-net: remove virtio queues if the guest doesn't support
 multiqueue
   virtio-net: Flush incoming queues when DRIVER_OK is being set
   pci_add_capability: remove duplicate comments
   virtio-net: unbreak any layout
   Revert vhost-user: add multi queue support
   ich9: fix skipped vmstate_memhp_state subsection
 
 Signed-off-by: Peter Maydell peter.mayd...@linaro.org
 

___
dev mailing list
dev@openvswitch.org
http://openvswitch.org/mailman/listinfo/dev

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

2015-08-06 Thread Flavio Leitner

On Thu, Aug 06, 2015 at 12:39:29PM -0400, Thomas F Herbert wrote:
 On 7/31/15 6:30 PM, Flavio Leitner wrote:
 This RFC is based on the vhost multiple queues work on
 dpdk-dev: http://dpdk.org/ml/archives/dev/2015-June/019345.html
 
 Signed-off-by: Flavio Leitner f...@redhat.com
 ---
   lib/netdev-dpdk.c | 61 
  ---
   1 file changed, 40 insertions(+), 21 deletions(-)
 
 diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
 index 5ae805e..493172c 100644
 --- a/lib/netdev-dpdk.c
 +++ b/lib/netdev-dpdk.c
 @@ -215,12 +215,9 @@ struct netdev_dpdk {
* If the numbers match, 'txq_needs_locking' is false, otherwise it is
* true and we will take a spinlock on transmission */
   int real_n_txq;
 +int real_n_rxq;
   bool txq_needs_locking;
 
 -/* Spinlock for vhost transmission.  Other DPDK devices use spinlocks in
 - * dpdk_tx_queue */
 -rte_spinlock_t vhost_tx_lock;
 -
   /* virtio-net structure for vhost device */
   OVSRCU_TYPE(struct virtio_net *) virtio_dev;
 
 @@ -602,13 +599,10 @@ dpdk_dev_parse_name(const char dev_name[], const char 
 prefix[],
   static int
   vhost_construct_helper(struct netdev *netdev_) OVS_REQUIRES(dpdk_mutex)
   {
 -struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
 -
   if (rte_eal_init_ret) {
   return rte_eal_init_ret;
   }
 
 -rte_spinlock_init(netdev-vhost_tx_lock);
   return netdev_dpdk_init(netdev_, -1, DPDK_DEV_VHOST);
   }
 
 @@ -791,9 +785,16 @@ netdev_dpdk_vhost_set_multiq(struct netdev *netdev_, 
 unsigned int n_txq,
   ovs_mutex_lock(dpdk_mutex);
   ovs_mutex_lock(netdev-mutex);
 
 +rte_free(netdev-tx_q);
 +/* FIXME: the number of vqueues needs to match */
   netdev-up.n_txq = n_txq;
 -netdev-real_n_txq = 1;
 -netdev-up.n_rxq = 1;
 +netdev-up.n_rxq = n_rxq;
 +
 +/* vring has txq = rxq */
 +netdev-real_n_txq = n_rxq;
 +netdev-real_n_rxq = n_rxq;
 +netdev-txq_needs_locking = netdev-real_n_txq != netdev-up.n_txq;
 +netdev_dpdk_alloc_txq(netdev, netdev-up.n_txq);
 
   ovs_mutex_unlock(netdev-mutex);
   ovs_mutex_unlock(dpdk_mutex);
 @@ -904,14 +905,14 @@ netdev_dpdk_vhost_rxq_recv(struct netdev_rxq *rxq_,
   struct netdev *netdev = rx-up.netdev;
   struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
   struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
 -int qid = 1;
 +int qid = rxq_-queue_id;
   uint16_t nb_rx = 0;
 
   if (OVS_UNLIKELY(!is_vhost_running(virtio_dev))) {
   return EAGAIN;
   }
 
 -nb_rx = rte_vhost_dequeue_burst(virtio_dev, qid,
 +nb_rx = rte_vhost_dequeue_burst(virtio_dev, VIRTIO_TXQ + qid * 2,
   vhost_dev-dpdk_mp-mp,
   (struct rte_mbuf **)packets,
   NETDEV_MAX_BURST);
 @@ -958,8 +959,9 @@ netdev_dpdk_rxq_recv(struct netdev_rxq *rxq_, struct 
 dp_packet **packets,
   }
 
   static void
 -__netdev_dpdk_vhost_send(struct netdev *netdev, struct dp_packet **pkts,
 - int cnt, bool may_steal)
 +__netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
 + struct dp_packet **pkts, int cnt,
 + bool may_steal)
   {
   struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
   struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
 @@ -974,13 +976,16 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct 
 dp_packet **pkts,
   goto out;
   }
 
 -/* There is vHost TX single queue, So we need to lock it for TX. */
 -rte_spinlock_lock(vhost_dev-vhost_tx_lock);
 +if (vhost_dev-txq_needs_locking) {
 +qid = qid % vhost_dev-real_n_txq;
 +rte_spinlock_lock(vhost_dev-tx_q[qid].tx_lock);
 +}
 
   do {
 +int vhost_qid = VIRTIO_RXQ + qid * VIRTIO_QNUM;
   unsigned int tx_pkts;
 
 -tx_pkts = rte_vhost_enqueue_burst(virtio_dev, VIRTIO_RXQ,
 +tx_pkts = rte_vhost_enqueue_burst(virtio_dev, vhost_qid,
 cur_pkts, cnt);
   if (OVS_LIKELY(tx_pkts)) {
   /* Packets have been sent.*/
 @@ -999,7 +1004,7 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct 
 dp_packet **pkts,
* Unable to enqueue packets to vhost interface.
* Check available entries before retrying.
*/
 -while (!rte_vring_available_entries(virtio_dev, VIRTIO_RXQ)) {
 +while (!rte_vring_available_entries(virtio_dev, vhost_qid)) {
   if (OVS_UNLIKELY((rte_get_timer_cycles() - start)  
  timeout)) {
   expired = 1;
   break;
 @@ -1011,7 +1016,10 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, 
 struct dp_packet **pkts,
   }
   }
   } while (cnt);
 -

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

2015-08-06 Thread Thomas F Herbert


On 8/6/15 1:40 PM, Flavio Leitner wrote:

On Thu, Aug 06, 2015 at 12:39:29PM -0400, Thomas F Herbert wrote:

On 7/31/15 6:30 PM, Flavio Leitner wrote:

This RFC is based on the vhost multiple queues work on
dpdk-dev: http://dpdk.org/ml/archives/dev/2015-June/019345.html

Signed-off-by: Flavio Leitner f...@redhat.com
---
  lib/netdev-dpdk.c | 61 ---
  1 file changed, 40 insertions(+), 21 deletions(-)

diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index 5ae805e..493172c 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -215,12 +215,9 @@ struct netdev_dpdk {
   * If the numbers match, 'txq_needs_locking' is false, otherwise it is
   * true and we will take a spinlock on transmission */
  int real_n_txq;
+int real_n_rxq;
  bool txq_needs_locking;

-/* Spinlock for vhost transmission.  Other DPDK devices use spinlocks in
- * dpdk_tx_queue */
-rte_spinlock_t vhost_tx_lock;
-
  /* virtio-net structure for vhost device */
  OVSRCU_TYPE(struct virtio_net *) virtio_dev;

@@ -602,13 +599,10 @@ dpdk_dev_parse_name(const char dev_name[], const char 
prefix[],
  static int
  vhost_construct_helper(struct netdev *netdev_) OVS_REQUIRES(dpdk_mutex)
  {
-struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
-
  if (rte_eal_init_ret) {
  return rte_eal_init_ret;
  }

-rte_spinlock_init(netdev-vhost_tx_lock);
  return netdev_dpdk_init(netdev_, -1, DPDK_DEV_VHOST);
  }

@@ -791,9 +785,16 @@ netdev_dpdk_vhost_set_multiq(struct netdev *netdev_, 
unsigned int n_txq,
  ovs_mutex_lock(dpdk_mutex);
  ovs_mutex_lock(netdev-mutex);

+rte_free(netdev-tx_q);
+/* FIXME: the number of vqueues needs to match */
Do you still need this FIXME? Isn't the code you added below freeing 
and re-allocating the correct number of tx queues?

  netdev-up.n_txq = n_txq;
-netdev-real_n_txq = 1;
-netdev-up.n_rxq = 1;
+netdev-up.n_rxq = n_rxq;
+
+/* vring has txq = rxq */
+netdev-real_n_txq = n_rxq;
+netdev-real_n_rxq = n_rxq;
+netdev-txq_needs_locking = netdev-real_n_txq != netdev-up.n_txq;
+netdev_dpdk_alloc_txq(netdev, netdev-up.n_txq);

  ovs_mutex_unlock(netdev-mutex);
  ovs_mutex_unlock(dpdk_mutex);
@@ -904,14 +905,14 @@ netdev_dpdk_vhost_rxq_recv(struct netdev_rxq *rxq_,
  struct netdev *netdev = rx-up.netdev;
  struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
  struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
-int qid = 1;
+int qid = rxq_-queue_id;
  uint16_t nb_rx = 0;

  if (OVS_UNLIKELY(!is_vhost_running(virtio_dev))) {
  return EAGAIN;
  }

-nb_rx = rte_vhost_dequeue_burst(virtio_dev, qid,
+nb_rx = rte_vhost_dequeue_burst(virtio_dev, VIRTIO_TXQ + qid * 2,
  vhost_dev-dpdk_mp-mp,
  (struct rte_mbuf **)packets,
  NETDEV_MAX_BURST);
@@ -958,8 +959,9 @@ netdev_dpdk_rxq_recv(struct netdev_rxq *rxq_, struct 
dp_packet **packets,
  }

  static void
-__netdev_dpdk_vhost_send(struct netdev *netdev, struct dp_packet **pkts,
- int cnt, bool may_steal)
+__netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
+ struct dp_packet **pkts, int cnt,
+ bool may_steal)
  {
  struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
  struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
@@ -974,13 +976,16 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct 
dp_packet **pkts,
  goto out;
  }

-/* There is vHost TX single queue, So we need to lock it for TX. */
-rte_spinlock_lock(vhost_dev-vhost_tx_lock);
+if (vhost_dev-txq_needs_locking) {
+qid = qid % vhost_dev-real_n_txq;
+rte_spinlock_lock(vhost_dev-tx_q[qid].tx_lock);
+}

  do {
+int vhost_qid = VIRTIO_RXQ + qid * VIRTIO_QNUM;
  unsigned int tx_pkts;

-tx_pkts = rte_vhost_enqueue_burst(virtio_dev, VIRTIO_RXQ,
+tx_pkts = rte_vhost_enqueue_burst(virtio_dev, vhost_qid,
cur_pkts, cnt);
  if (OVS_LIKELY(tx_pkts)) {
  /* Packets have been sent.*/
@@ -999,7 +1004,7 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct 
dp_packet **pkts,
   * Unable to enqueue packets to vhost interface.
   * Check available entries before retrying.
   */
-while (!rte_vring_available_entries(virtio_dev, VIRTIO_RXQ)) {
+while (!rte_vring_available_entries(virtio_dev, vhost_qid)) {
  if (OVS_UNLIKELY((rte_get_timer_cycles() - start)  timeout)) 
{
  expired = 1;
  break;
@@ -1011,7 +1016,10 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct 
dp_packet **pkts,
  }
  }
  } while

[ovs-dev] [RFC] dpdk: support multiple queues in vhost

2015-07-31 Thread Flavio Leitner

This RFC is based on the vhost multiple queues work on
dpdk-dev: http://dpdk.org/ml/archives/dev/2015-June/019345.html

Signed-off-by: Flavio Leitner f...@redhat.com
---
 lib/netdev-dpdk.c | 61 ---
 1 file changed, 40 insertions(+), 21 deletions(-)

diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index 5ae805e..493172c 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -215,12 +215,9 @@ struct netdev_dpdk {
  * If the numbers match, 'txq_needs_locking' is false, otherwise it is
  * true and we will take a spinlock on transmission */
 int real_n_txq;
+int real_n_rxq;
 bool txq_needs_locking;
 
-/* Spinlock for vhost transmission.  Other DPDK devices use spinlocks in
- * dpdk_tx_queue */
-rte_spinlock_t vhost_tx_lock;
-
 /* virtio-net structure for vhost device */
 OVSRCU_TYPE(struct virtio_net *) virtio_dev;
 
@@ -602,13 +599,10 @@ dpdk_dev_parse_name(const char dev_name[], const char 
prefix[],
 static int
 vhost_construct_helper(struct netdev *netdev_) OVS_REQUIRES(dpdk_mutex)
 {
-struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
-
 if (rte_eal_init_ret) {
 return rte_eal_init_ret;
 }
 
-rte_spinlock_init(netdev-vhost_tx_lock);
 return netdev_dpdk_init(netdev_, -1, DPDK_DEV_VHOST);
 }
 
@@ -791,9 +785,16 @@ netdev_dpdk_vhost_set_multiq(struct netdev *netdev_, 
unsigned int n_txq,
 ovs_mutex_lock(dpdk_mutex);
 ovs_mutex_lock(netdev-mutex);
 
+rte_free(netdev-tx_q);
+/* FIXME: the number of vqueues needs to match */
 netdev-up.n_txq = n_txq;
-netdev-real_n_txq = 1;
-netdev-up.n_rxq = 1;
+netdev-up.n_rxq = n_rxq;
+
+/* vring has txq = rxq */
+netdev-real_n_txq = n_rxq;
+netdev-real_n_rxq = n_rxq;
+netdev-txq_needs_locking = netdev-real_n_txq != netdev-up.n_txq;
+netdev_dpdk_alloc_txq(netdev, netdev-up.n_txq);
 
 ovs_mutex_unlock(netdev-mutex);
 ovs_mutex_unlock(dpdk_mutex);
@@ -904,14 +905,14 @@ netdev_dpdk_vhost_rxq_recv(struct netdev_rxq *rxq_,
 struct netdev *netdev = rx-up.netdev;
 struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
 struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
-int qid = 1;
+int qid = rxq_-queue_id;
 uint16_t nb_rx = 0;
 
 if (OVS_UNLIKELY(!is_vhost_running(virtio_dev))) {
 return EAGAIN;
 }
 
-nb_rx = rte_vhost_dequeue_burst(virtio_dev, qid,
+nb_rx = rte_vhost_dequeue_burst(virtio_dev, VIRTIO_TXQ + qid * 2,
 vhost_dev-dpdk_mp-mp,
 (struct rte_mbuf **)packets,
 NETDEV_MAX_BURST);
@@ -958,8 +959,9 @@ netdev_dpdk_rxq_recv(struct netdev_rxq *rxq_, struct 
dp_packet **packets,
 }
 
 static void
-__netdev_dpdk_vhost_send(struct netdev *netdev, struct dp_packet **pkts,
- int cnt, bool may_steal)
+__netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
+ struct dp_packet **pkts, int cnt,
+ bool may_steal)
 {
 struct netdev_dpdk *vhost_dev = netdev_dpdk_cast(netdev);
 struct virtio_net *virtio_dev = netdev_dpdk_get_virtio(vhost_dev);
@@ -974,13 +976,16 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct 
dp_packet **pkts,
 goto out;
 }
 
-/* There is vHost TX single queue, So we need to lock it for TX. */
-rte_spinlock_lock(vhost_dev-vhost_tx_lock);
+if (vhost_dev-txq_needs_locking) {
+qid = qid % vhost_dev-real_n_txq;
+rte_spinlock_lock(vhost_dev-tx_q[qid].tx_lock);
+}
 
 do {
+int vhost_qid = VIRTIO_RXQ + qid * VIRTIO_QNUM;
 unsigned int tx_pkts;
 
-tx_pkts = rte_vhost_enqueue_burst(virtio_dev, VIRTIO_RXQ,
+tx_pkts = rte_vhost_enqueue_burst(virtio_dev, vhost_qid,
   cur_pkts, cnt);
 if (OVS_LIKELY(tx_pkts)) {
 /* Packets have been sent.*/
@@ -999,7 +1004,7 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct 
dp_packet **pkts,
  * Unable to enqueue packets to vhost interface.
  * Check available entries before retrying.
  */
-while (!rte_vring_available_entries(virtio_dev, VIRTIO_RXQ)) {
+while (!rte_vring_available_entries(virtio_dev, vhost_qid)) {
 if (OVS_UNLIKELY((rte_get_timer_cycles() - start)  timeout)) {
 expired = 1;
 break;
@@ -1011,7 +1016,10 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, struct 
dp_packet **pkts,
 }
 }
 } while (cnt);
-rte_spinlock_unlock(vhost_dev-vhost_tx_lock);
+
+if (vhost_dev-txq_needs_locking) {
+rte_spinlock_unlock(vhost_dev-tx_q[qid].tx_lock);
+}
 
 rte_spinlock_lock(vhost_dev-stats_lock);
 vhost_dev-stats.tx_packets += (total_pkts - cnt);
@@ -1116,7 +1124,7 @@ dpdk_do_tx_copy(struct

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

Re: [ovs-dev] [RFC] dpdk: support multiple queues in vhost

[ovs-dev] [RFC] dpdk: support multiple queues in vhost

11 matches

Site Navigation

Mail list logo

Footer information