This RFC patch provides a first draft implementation of the modifications required in netdev-dpdk.c and vswitch.xml to allow for a DPDK user space QoS algorithm. QoS has not been implemented fully in this patch, but the aim is to solicit feedback early from the community on the initial design steps taken.
This patch adds a QoS configuration structure for netdev-dpdk, structures to represent a QoS queue 'dpdk_qos_queue' and expected QoS operations 'dpdk_qos_ops'. Various helper functions are also supplied. Also included are the modifications required for vswitch.xml to allow a new QoS implementation for netdev-dpdk devices. This includes a new QoS type as well as its expected QoS and Queue table entries. This patch does not include a specific QoS algorithm implementation for DPDK devices. This will be implemented at a later stage. The algorithm that will be implemented will be user space strict priority queuing or 'us-spq' for short. The netdev-dpdk QoS functions will also be implemented at this stage. Signed-off-by: Ian Stokes <[email protected]> --- lib/netdev-dpdk.c | 247 ++++++++++++++++++++++++++++++++++++++++++++++++++ vswitchd/vswitch.xml | 25 +++++- 2 files changed, 271 insertions(+), 1 deletions(-) diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c index 0ede200..7bdb018 100644 --- a/lib/netdev-dpdk.c +++ b/lib/netdev-dpdk.c @@ -47,11 +47,15 @@ #include "timeval.h" #include "unixctl.h" #include "openvswitch/vlog.h" +#include "hash.h" +#include "hmap.h" +#include "smap.h" VLOG_DEFINE_THIS_MODULE(dpdk); static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20); #define DPDK_PORT_WATCHDOG_INTERVAL 5 +#define DPDK_MAX_QOS_NAME_SIZE 10 #define OVS_CACHE_LINE_SIZE CACHE_LINE_SIZE #define OVS_VPORT_DPDK "ovs_dpdk" @@ -133,6 +137,156 @@ static int rte_eal_init_ret = ENODEV; static struct ovs_mutex dpdk_mutex = OVS_MUTEX_INITIALIZER; +/* Quality of Service */ + +/* An instance of a QoS configuration. Always associated with a particular + * network device. + * + * Each QoS implementation subclasses this with whatever additional data it + * needs. + */ +struct qos_conf{ + const struct dpdk_qos_ops *ops; + struct hmap queues; +}; + +/* A DPDK QoS queue. + * + * Each DPDK QoS queue implementation subclasses this with whatever additional + * data it needs. + */ +struct dpdk_qos_queue{ + struct hmap_node hmap_node; /* In struct dpdk_qos "queues" hmap. */ + unsigned int queue_id; /* Queue ID of dpdk_qos_queue */ + void *queue_p; /* Pointer to QoS queue used in dataplane */ +}; + +/* A particular implementation of dpdk QoS operations. + * + * The functions below return 0 if successful or a positive errno value on + * failure, except where otherwise noted. All of them must be provided, except + * where otherwise noted. + */ +struct dpdk_qos_ops{ + + /* Name of the QoS type */ + const char *qos_name; + + /* Number of supported QoS queues, 0 for QoS implementations that have no + * queues. The queues are numbered 0 through n_queues - 1. + */ + unsigned int n_queues; + + /* Called to construct the QoS implementation on 'netdev'. The + * implementation should make the appropriate calls to configure QoS + * according to 'details'. The implementation may assume that any current + * QoS configuration already installed should be destroyed before + * constructing the new configuration. + * + * The contents of 'details' should be documented as valid for 'ovs_name' + * in the "other_config" column in the "QoS" table in vswitchd/vswitch.xml + * (which is built as ovs-vswitchd.conf.db(8)). + * + * This function must return 0 if and only if it sets 'netdev->qos_conf' + * to an initialized 'struct qos_conf'. + * + * For all QoS implementations it should always be nonnull. + */ + int (*qos_construct)(struct netdev *netdev, const struct smap *details); + + /* Destroys the data structures allocated by the implementation as part of + * 'qos_conf'. (This includes destroying 'qos_conf->queues' by calling + * qos_destruct(qos_conf). + * + * This function may be null if 'qos_conf' is trivial. + */ + void (*qos_destruct)(struct qos_conf *conf); + + /* Retrieves details of 'netdev->qos_conf' configuration into 'details'. + * + * The contents of 'details' should be documented as valid for 'ovs_name' + * in the "other_config" column in the "QoS" table in vswitchd/vswitch.xml + * (which is built as ovs-vswitchd.conf.db(8)). + */ + int (*qos_get)(const struct netdev *netdev, struct smap *details); + + /* Reconfigures 'netdev->qos_conf' according to 'details', performing any + * required calls to complete the reconfiguration. + * + * The contents of 'details' should be documented as valid for 'ovs_name' + * in the "other_config" column in the "QoS" table in vswitchd/vswitch.xml + * (which is built as ovs-vswitchd.conf.db(8)). + * + * This function may be null if 'qos_conf' is not configurable. + */ + int (*qos_set)(struct netdev *netdev, const struct smap *details); + + /* Retrieves details of 'queue' on 'netdev->qos_conf' into 'details'. + * 'queue' is one of the 'struct dpdk_qos_queue's within + * 'netdev->qos_conf->queues'. + * + * The contents of 'details' should be documented as valid for 'ovs_name' + * in the "other_config" column in the "Queue" table in + * vswitchd/vswitch.xml (which is built as ovs-vswitchd.conf.db(8)). + * + * This function may be null if 'qos_conf' does not have queues + * ('n_queues' is 0). + */ + int (*queue_get)(const struct netdev *netdev, const struct dpdk_qos_queue + *queue, struct smap *details); + + /* Configures or reconfigures 'queue_id' on 'netdev->qos_conf' according to + * 'details', perfoming any required calls to complete the reconfiguration. + * The caller ensures that 'queue_id' is less than 'n_queues'. + * + * The contents of 'details' should be documented as valid for 'ovs_name' + * in the "other_config" column in the "Queue" table in + * vswitchd/vswitch.xml (which is built as ovs-vswitchd.conf.db(8)). + * + * This function may be null if 'qos_conf' does not have queues or its + * queues are not configurable. + */ + int (*queue_set)(struct netdev *netdev, unsigned int queue_id, + const struct smap *details); + + /* Deletes 'queue' from 'netdev->qos_conf'. 'queue' is one of the 'struct + * dpdk_qos_queue's within 'netdev->qos_conf->queues'. This function should + * also remove the dataplane queue created when qos_construct is called. + * + * This function may be null if 'qos_conf' does not have queues or its + * queues cannot be deleted. + */ + int (*queue_delete)(struct netdev *netdev, struct dpdk_qos_queue *queue); + + /* Obtains stats for 'queue' from 'netdev->qos_conf'. 'queue' is one of the + * 'struct dpdk_qos_queue's within 'netdev->qos_conf->queues'. + * + * On success, initializes '*stats'. + * + * This function may be null if 'qos_conf' does not have queues or if it + * cannot report queue statistics. + */ + int (*queue_get_stats)(const struct netdev *netdev, + const struct dpdk_qos_queue *queue, + struct netdev_queue_stats *stats); + + /* Gather queue stats and passes them to 'cb' along with 'aux'. + * + * This function may be null if 'qos_conf' does not have queues or if it + * cannot report queue statistics. + */ + int (*queue_dump_stats)(const struct netdev *netdev, + netdev_dump_queue_stats_cb *cb, void *aux); +}; + +/* + * Array of dpdk_qos_ops, contains pointer to all supported QoS + * operations. + */ +static const struct dpdk_qos_ops *const qos_confs[] = { + NULL +}; + /* Contains all 'struct dpdk_dev's. */ static struct ovs_list dpdk_list OVS_GUARDED_BY(dpdk_mutex) = OVS_LIST_INITIALIZER(&dpdk_list); @@ -203,6 +357,9 @@ struct netdev_dpdk { /* In dpdk_list. */ struct ovs_list list_node OVS_GUARDED_BY(dpdk_mutex); rte_spinlock_t dpdkr_tx_lock; + + /* QoS configuration for the device */ + struct qos_conf *qos_conf; }; struct netdev_rxq_dpdk { @@ -511,6 +668,9 @@ netdev_dpdk_init(struct netdev *netdev_, unsigned int port_no) goto unlock; } + /* Initialise QoS configuration to NULL */ + netdev->qos_conf = NULL; + netdev_->n_txq = NR_QUEUE; netdev_->n_rxq = NR_QUEUE; err = dpdk_eth_dev_init(netdev); @@ -1406,6 +1566,93 @@ unlock_dpdk: return err; } +/* QoS Functions */ + +/* + * Initialise QoS configuration operations and queue HMAP for conf. + */ +static void +qos_conf_init(struct qos_conf *conf, const struct dpdk_qos_ops *ops) +{ + conf->ops = ops; + hmap_init(&conf->queues); +} + +/* + * Destroy HMAP associated with qos_conf. + */ +static void +qos_conf_destruct(struct qos_conf *conf) +{ + hmap_destroy(&conf->queues); +} + +/* + * Search existing qos operations in qos_ops and compare each set of operations + * qos_name to name. Return a dpdk_qos_ops pointer to a match, else return + * NULL + */ +static const struct dpdk_qos_ops * +qos_lookup_name(const char *name) +{ + const struct dpdk_qos_ops *const *opsp; + + for (opsp = qos_confs; *opsp != NULL; opsp++) { + const struct dpdk_qos_ops *ops = *opsp; + if (!strncmp(name, ops->qos_name,DPDK_MAX_QOS_NAME_SIZE)) { + return ops; + } + } + return NULL; +} + +/* + * Search the hash map associated with a netdevs qos configuration and return + * a pointer to the dpdk_qos_queue with queue_id. Return NULL if not found. + */ +static struct dpdk_qos_queue * +qos_conf_find_queue__(const struct netdev *netdev_, unsigned int queue_id, + size_t hash) +{ + struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_); + struct dpdk_qos_queue *queue = NULL; + + HMAP_FOR_EACH_IN_BUCKET (queue, hmap_node, hash, + &netdev->qos_conf->queues) { + if (queue->queue_id == queue_id) { + return queue; + } + } + return NULL; +} + +/* + * Search for dpdk_qos_queue associated with queue_id. Calls + * qos_conf_find_queue__ with calculated hash value for queue_id. + */ +static struct dpdk_qos_queue * +qos_conf_find_queue(const struct netdev *netdev_,unsigned int queue_id) +{ + return qos_conf_find_queue__(netdev_, queue_id, hash_int(queue_id, 0)); +} + +/* + * Call qos_destruct to clean up dpdk_qos_queues associated with the netdevs + * qos_conf. Set netdevs qos_conf to NULL. + */ +static void +qos_delete_conf(struct netdev *netdev_) +{ + struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_); + + if (netdev->qos_conf) { + if (netdev->qos_conf->ops->qos_destruct) { + netdev->qos_conf->ops->qos_destruct(netdev->qos_conf); + } + netdev->qos_conf = NULL; + } +} + #define NETDEV_DPDK_CLASS(NAME, INIT, CONSTRUCT, MULTIQ, SEND) \ { \ NAME, \ diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml index 37a33a6..3afbe89 100644 --- a/vswitchd/vswitch.xml +++ b/vswitchd/vswitch.xml @@ -3006,6 +3006,12 @@ information on how this classifier works. </dd> </dl> + <dl> + <dt><code>us-spq</code></dt> + <dd> + Userspace "Strict Priority Queuing" QoS algorithm. + </dd> + </dl> </column> <column name="queues"> @@ -3041,7 +3047,7 @@ is currently 100 Mbps. </column> </group> - + <group title="Common Columns"> The overall purpose of these columns is described under <code>Common Columns</code> at the beginning of this document. @@ -3122,6 +3128,23 @@ limit. </column> </group> + + <group title="Configuration for us-spq QoS"> + <p> + <ref table="QoS"/> <ref table="QoS" column="type"/> + <code>us-spq</code> may use <code>queue_id</code>s less than the maximum + queue supported for the QoS type. + It has the following key-value pairs defined. + </p> + + <column name="other_config" key="priority" + type='{"type": "integer", "minInteger": 0, "maxInteger": 7}'> + A queue with a smaller <code>priority</code> will receive all the + excess bandwidth that it can use before a queue with a larger value + receives any. Specific priority values are unimportant; only relative + ordering matters. Defaults to 0 if unspecified. + </column> + </group> <group title="Common Columns"> The overall purpose of these columns is described under <code>Common -- 1.7.4.1 _______________________________________________ dev mailing list [email protected] http://openvswitch.org/mailman/listinfo/dev
