On Mon, Jun 02, 2025 at 10:38:03PM +0000, Wathsala Vithanage wrote:
> Extend the ethdev library to enable the stashing of different data
> objects, such as the ones listed below, into CPU caches directly
> from the NIC.
>
> - Rx/Tx queue descriptors
> - Rx packets
> - Packet headers
> - packet payloads
> - Data of a packet at an offset from the start of the packet
>
> The APIs are designed in a hardware/vendor agnostic manner such that
> supporting PMDs could use any capabilities available in the underlying
> hardware for fine-grained stashing of data objects into a CPU cache
>
> The API provides an interface to query the availability of stashing
> capabilities, i.e., platform/NIC support, stashable object types, etc,
> via the rte_eth_dev_stashing_capabilities_get interface.
>
> The function pair rte_eth_dev_stashing_rx_config_set and
> rte_eth_dev_stashing_tx_config_set sets the stashing hint (the CPU,
> cache level, and data object types) on the Rx and Tx queues.
>
> PMDs that support stashing must register their implementations with the
> following eth_dev_ops callbacks, which are invoked by the ethdev
> functions listed above.
>
> - stashing_capabilities_get
> - stashing_rx_hints_set
> - stashing_tx_hints_set
>
> Signed-off-by: Wathsala Vithanage <wathsala.vithan...@arm.com>
> Reviewed-by: Honnappa Nagarahalli <honnappa.nagaraha...@arm.com>
> Reviewed-by: Dhruv Tripathi <dhruv.tripa...@arm.com>
> ---
Few small comments inline below
/Bruce
> lib/ethdev/ethdev_driver.h | 66 ++++++++++++++++
> lib/ethdev/rte_ethdev.c | 149 ++++++++++++++++++++++++++++++++++
> lib/ethdev/rte_ethdev.h | 158 +++++++++++++++++++++++++++++++++++++
> 3 files changed, 373 insertions(+)
>
> diff --git a/lib/ethdev/ethdev_driver.h b/lib/ethdev/ethdev_driver.h
> index 2b4d2ae9c3..8a4012db08 100644
> --- a/lib/ethdev/ethdev_driver.h
> +++ b/lib/ethdev/ethdev_driver.h
> @@ -1376,6 +1376,68 @@ enum rte_eth_dev_operation {
> typedef uint64_t (*eth_get_restore_flags_t)(struct rte_eth_dev *dev,
> enum rte_eth_dev_operation op);
>
> +/**
> + * @internal
> + * Set cache stashing hints in Rx queue.
> + *
> + * @param dev
> + * Port (ethdev) handle.
> + * @param queue_id
> + * Rx queue.
> + * @param config
> + * Stashing hints configuration for the queue.
> + *
> + * @return
> + * -ENOTSUP if the device or the platform does not support cache stashing.
> + * -ENOSYS if the underlying PMD hasn't implemented cache stashing
> feature.
> + * -EINVAL on invalid arguments.
> + * 0 on success.
> + */
> +typedef int (*eth_stashing_rx_hints_set_t)(struct rte_eth_dev *dev, uint16_t
> queue_id,
> + struct rte_eth_stashing_config
> *config);
> +
> +/**
> + * @internal
> + * Set cache stashing hints in Tx queue.
> + *
> + * @param dev
> + * Port (ethdev) handle.
> + * @param queue_id
> + * Tx queue.
> + * @param config
> + * Stashing hints configuration for the queue.
> + *
> + * @return
> + * -ENOTSUP if the device or the platform does not support cache stashing.
> + * -ENOSYS if the underlying PMD hasn't implemented cache stashing
> feature.
> + * -EINVAL on invalid arguments.
> + * 0 on success.
What about on failure of the underlying ioctl call?
> + */
> +typedef int (*eth_stashing_tx_hints_set_t)(struct rte_eth_dev *dev, uint16_t
> queue_id,
> + struct rte_eth_stashing_config
> *config);
> +
> +/**
> + * @internal
> + * Get cache stashing object types supported in the ethernet device.
> + * The return value indicates availability of stashing hints support
> + * in the hardware and the PMD.
> + *
> + * @param dev
> + * Port (ethdev) handle.
> + * @param objects
> + * PMD sets supported bits on return.
> + *
> + * @return
> + * -ENOTSUP if the device or the platform does not support cache stashing.
> + * -ENOSYS if the underlying PMD hasn't implemented cache stashing
> feature.
> + * -EINVAL on NULL values for types or hints parameters.
> + * On return, types and hints parameters will have bits set for supported
> + * object types and hints.
> + * 0 on success.
> + */
> +typedef int (*eth_stashing_capabilities_get_t)(struct rte_eth_dev *dev,
> + uint16_t *objects);
> +
> /**
> * @internal A structure containing the functions exported by an Ethernet
> driver.
> */
> @@ -1402,6 +1464,10 @@ struct eth_dev_ops {
> eth_mac_addr_remove_t mac_addr_remove; /**< Remove MAC address */
> eth_mac_addr_add_t mac_addr_add; /**< Add a MAC address */
> eth_mac_addr_set_t mac_addr_set; /**< Set a MAC address */
> + eth_stashing_rx_hints_set_t stashing_rx_hints_set; /**< Set Rx cache
> stashing*/
> + eth_stashing_tx_hints_set_t stashing_tx_hints_set; /**< Set Tx cache
> stashing*/
> + /** Get supported stashing hints*/
> + eth_stashing_capabilities_get_t stashing_capabilities_get;
> /** Set list of multicast addresses */
> eth_set_mc_addr_list_t set_mc_addr_list;
> mtu_set_t mtu_set; /**< Set MTU */
> diff --git a/lib/ethdev/rte_ethdev.c b/lib/ethdev/rte_ethdev.c
> index d4197322a0..ae666c370b 100644
> --- a/lib/ethdev/rte_ethdev.c
> +++ b/lib/ethdev/rte_ethdev.c
> @@ -158,6 +158,7 @@ static const struct {
> {RTE_ETH_DEV_CAPA_RXQ_SHARE, "RXQ_SHARE"},
> {RTE_ETH_DEV_CAPA_FLOW_RULE_KEEP, "FLOW_RULE_KEEP"},
> {RTE_ETH_DEV_CAPA_FLOW_SHARED_OBJECT_KEEP, "FLOW_SHARED_OBJECT_KEEP"},
> + {RTE_ETH_DEV_CAPA_CACHE_STASHING, "CACHE_STASHING"},
> };
>
> enum {
> @@ -7419,5 +7420,153 @@ int rte_eth_dev_map_aggr_tx_affinity(uint16_t
> port_id, uint16_t tx_queue_id,
> return ret;
> }
>
> +RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_eth_dev_validate_stashing_config, 25.07)
> +int
> +rte_eth_dev_validate_stashing_config(uint16_t port_id, uint16_t queue_id,
> + uint8_t queue_direction,
> + struct rte_eth_stashing_config *config)
> +{
> + struct rte_eth_dev *dev;
> + struct rte_eth_dev_info dev_info;
> + int ret = 0;
> + uint16_t nb_queues;
> +
> + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
> +
> + if (!config) {
> + RTE_ETHDEV_LOG_LINE(ERR, "Invalid stashing configuration");
> + ret = -EINVAL;
> + goto out;
> + }
> +
> + /*
> + * Check for invalid objects
> + */
> + if (!RTE_ETH_DEV_STASH_OBJECTS_VALID(config->objects)) {
> + RTE_ETHDEV_LOG_LINE(ERR, "Invalid stashing objects");
> + ret = -EINVAL;
> + goto out;
> + }
> +
> + dev = &rte_eth_devices[port_id];
> +
> + nb_queues = (queue_direction == RTE_ETH_DEV_RX_QUEUE) ?
> + dev->data->nb_rx_queues :
> + dev->data->nb_tx_queues;
> +
> + if (queue_id >= nb_queues) {
> + RTE_ETHDEV_LOG_LINE(ERR, "Invalid Rx queue_id=%u", queue_id);
> + ret = -EINVAL;
> + goto out;
> + }
> +
> + ret = rte_eth_dev_info_get(port_id, &dev_info);
> + if (ret < 0)
> + goto out;
> +
> + if ((dev_info.dev_capa & RTE_ETH_DEV_CAPA_CACHE_STASHING) !=
> + RTE_ETH_DEV_CAPA_CACHE_STASHING) {
Nit: check if all this can fit on one line under 100 chars. I think it can.
> + ret = -ENOTSUP;
> + goto out;
> + }
> +
> + if (*dev->dev_ops->stashing_rx_hints_set == NULL ||
> + *dev->dev_ops->stashing_tx_hints_set == NULL) {
> + RTE_ETHDEV_LOG_LINE(ERR, "Stashing hints are not implemented "
> + "in %s for %s", dev_info.driver_name,
Don't split error messages across lines. Text strings are allowed to go
over the 100 char limit if necessary to avoid splitting.
> + dev_info.device->name);
> + ret = -ENOSYS;
> + }
> +
> +out:
> + return ret;
> +}
> +
> +RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_eth_dev_stashing_rx_config_set, 25.07)
> +int
> +rte_eth_dev_stashing_rx_config_set(uint16_t port_id, uint16_t queue_id,
> + struct rte_eth_stashing_config *config)
> +{
> + struct rte_eth_dev *dev;
> + int ret = 0;
> +
> + ret = rte_eth_dev_validate_stashing_config(port_id, queue_id,
> + RTE_ETH_DEV_RX_QUEUE,
> + config);
> + if (ret < 0)
> + goto out;
> +
> + dev = &rte_eth_devices[port_id];
> +
> + ret = eth_err(port_id,
> + (*dev->dev_ops->stashing_rx_hints_set)(dev, queue_id,
> + config));
> +out:
> + return ret;
> +}
> +
> +RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_eth_dev_stashing_tx_config_set, 25.07)
> +int
> +rte_eth_dev_stashing_tx_config_set(uint16_t port_id, uint16_t queue_id,
> + struct rte_eth_stashing_config *config)
> +{
> + struct rte_eth_dev *dev;
> + int ret = 0;
> +
> + ret = rte_eth_dev_validate_stashing_config(port_id, queue_id,
> + RTE_ETH_DEV_TX_QUEUE,
> + config);
> + if (ret < 0)
> + goto out;
> +
> + dev = &rte_eth_devices[port_id];
> +
> + ret = eth_err(port_id,
> + (*dev->dev_ops->stashing_rx_hints_set) (dev, queue_id,
> + config));
> +out:
> + return ret;
> +}
> +
> +RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_eth_dev_stashing_capabilities_get, 25.07)
> +int
> +rte_eth_dev_stashing_capabilities_get(uint16_t port_id, uint16_t *objects)
> +{
> + struct rte_eth_dev *dev;
> + struct rte_eth_dev_info dev_info;
> + int ret = 0;
> +
> + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
> +
> + if (!objects) {
> + ret = -EINVAL;
> + goto out;
> + }
> +
> + dev = &rte_eth_devices[port_id];
> +
> + ret = rte_eth_dev_info_get(port_id, &dev_info);
> + if (ret < 0)
> + goto out;
> +
> + if ((dev_info.dev_capa & RTE_ETH_DEV_CAPA_CACHE_STASHING) !=
> + RTE_ETH_DEV_CAPA_CACHE_STASHING) {
> + ret = -ENOTSUP;
> + goto out;
> + }
> +
> + if (*dev->dev_ops->stashing_capabilities_get == NULL) {
> + RTE_ETHDEV_LOG_LINE(ERR, "Stashing hints are not implemented "
> + "in %s for %s", dev_info.driver_name,
> + dev_info.device->name);
> + ret = -ENOSYS;
> + goto out;
> + }
> + ret = eth_err(port_id,
> + (*dev->dev_ops->stashing_capabilities_get)(dev, objects));
> +out:
> + return ret;
> +}
> +
> RTE_EXPORT_SYMBOL(rte_eth_dev_logtype)
> RTE_LOG_REGISTER_DEFAULT(rte_eth_dev_logtype, INFO);
> diff --git a/lib/ethdev/rte_ethdev.h b/lib/ethdev/rte_ethdev.h
> index ea7f8c4a1a..1398f8c837 100644
> --- a/lib/ethdev/rte_ethdev.h
> +++ b/lib/ethdev/rte_ethdev.h
> @@ -1667,6 +1667,9 @@ struct rte_eth_conf {
> #define RTE_ETH_DEV_CAPA_FLOW_SHARED_OBJECT_KEEP RTE_BIT64(4)
> /**@}*/
>
> +/** Device supports stashing to CPU/system caches. */
> +#define RTE_ETH_DEV_CAPA_CACHE_STASHING RTE_BIT64(5)
> +
> /*
> * Fallback default preferred Rx/Tx port parameters.
> * These are used if an application requests default parameters
> @@ -1838,6 +1841,7 @@ struct rte_eth_dev_info {
> struct rte_eth_dev_portconf default_txportconf;
> /** Generic device capabilities (RTE_ETH_DEV_CAPA_). */
> uint64_t dev_capa;
> + uint16_t stashing_capa;
> /**
> * Switching information for ports on a device with a
> * embedded managed interconnect/switch.
> @@ -6173,6 +6177,160 @@ int rte_eth_cman_config_set(uint16_t port_id, const
> struct rte_eth_cman_config *
> __rte_experimental
> int rte_eth_cman_config_get(uint16_t port_id, struct rte_eth_cman_config
> *config);
>
> +
> +/** Queue type is RX. */
> +#define RTE_ETH_DEV_RX_QUEUE 0
> +/** Queue type is TX. */
> +#define RTE_ETH_DEV_TX_QUEUE 1
> +
I'd prefer an enum for these.
Why are the necessary since we have separate rx and tx functions for the
caching hints.
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this structure may change, or be removed, without prior
> notice
> + *
> + * A structure used for configuring the cache stashing hints.
> + */
> +struct rte_eth_stashing_config {
> + /**
> + * lcore_id of the processor the stashing hints are applied to.
> + */
> + uint32_t lcore_id;
> + /**
> + * Zero based cache level relative to the CPU.
> + * E.g. l1d = 0, l2d = 1,...
> + */
> + uint32_t cache_level;
> + /**
> + * Object types the configuration is applied to
> + */
> + uint16_t objects;
What are the objects? That needs to be covered by the docs, or make this an
enum type so that it's clear from the typesystem what it applies to [though
that would require an array of objects and count, it may be clearer for
user].
> + /**
> + * The offset if RTE_ETH_DEV_STASH_OBJECT_OFFSET bit is set
> + * in objects
> + */
> + int offset;
> +};
> +
> +/**@{@name Stashable Rx/Tx queue object types supported by the ethernet
> device
> + *@see rte_eth_dev_stashing_capabilities_get
> + *@see rte_eth_dev_stashing_rx_config_set
> + *@see rte_eth_dev_stashing_tx_config_set
> + */
> +
> +/**
> + * Apply stashing hint to data at a given offset from the start of a
> + * received packet.
> + */
> +#define RTE_ETH_DEV_STASH_OBJECT_OFFSET 0x0001
> +
> +/** Apply stashing hint to an rx descriptor. */
> +#define RTE_ETH_DEV_STASH_OBJECT_DESC 0x0002
> +
> +/** Apply stashing hint to a header of a received packet. */
> +#define RTE_ETH_DEV_STASH_OBJECT_HEADER 0x0004
> +
> +/** Apply stashing hint to a payload of a received packet. */
> +#define RTE_ETH_DEV_STASH_OBJECT_PAYLOAD 0x0008
> +
> +#define __RTE_ETH_DEV_STASH_OBJECT_MASK 0x000f
> +/**@}*/
> +
> +#define RTE_ETH_DEV_STASH_OBJECTS_VALID(t) \
> + ((!((t) & (~__RTE_ETH_DEV_STASH_OBJECT_MASK))) && (t))
> +
> +/**
> + *
> + * @warning
> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
> + *
> + * @internal
> + * Helper function to validate stashing hints configuration.
> + */
> +__rte_experimental
> +int rte_eth_dev_validate_stashing_config(uint16_t port_id, uint16_t queue_id,
> + uint8_t queue_direction,
> + struct rte_eth_stashing_config
> *config);
> +
> +/**
> + *
> + * @warning
> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
> + *
> + * Provide cache stashing hints for improved memory access latencies for
> + * packets received by the NIC.
> + * This feature is available only in supported NICs and platforms.
> + *
> + * @param port_id
> + * The port identifier of the Ethernet device.
> + * @param queue_id
> + * The index of the receive queue to which hints are applied.
> + * @param config
> + * Stashing configuration.
> + * @return
> + * - (-ENODEV) on incorrect port_ids.
> + * - (-EINVAL) if both RX and TX object types used in conjuection in objects
> + * parameter.
> + * - (-EINVAL) on invalid queue_id.
> + * - (-ENOTSUP) if RTE_ETH_DEV_CAPA_CACHE_STASHING capability is
> unavailable.
> + * - (-ENOSYS) if PMD does not implement cache stashing hints.
> + * - (0) on Success.
> + */
> +__rte_experimental
> +int rte_eth_dev_stashing_rx_config_set(uint16_t port_id, uint16_t queue_id,
> + struct rte_eth_stashing_config *config);
> +
> +/**
> + *
> + * @warning
> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
> + *
> + * Configure cache stashing for improved memory access latencies for Tx
> + * queue completion descriptors being sent to host system by the NIC.
> + * This feature is available only in supported NICs and platforms.
> + *
> + * @param port_id
> + * The port identifier of the Ethernet device.
> + * @param queue_id
> + * The index of the receive queue to which hints are applied.
> + * @param config
> + * Stashing configuration.
> + * @return
> + * - (-ENODEV) on incorrect port_ids.
> + * - (-EINVAL) if both RX and TX object types are used in conjuection in
> objects
> + * parameter.
> + * - (-EINVAL) if hints are incompatible with TX queues.
> + * - (-EINVAL) on invalid queue_id.
> + * - (-ENOTSUP) if RTE_ETH_DEV_CAPA_CACHE_STASHING capability is
> unavailable.
> + * - (-ENOSYS) if PMD does not implement cache stashing hints.
> + * - (0) on Success.
> + */
> +__rte_experimental
> +int rte_eth_dev_stashing_tx_config_set(uint16_t port_id, uint16_t queue_id,
> + struct rte_eth_stashing_config *config);
> +
> +/**
> + *
> + * @warning
> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
> + *
> + * Discover cache stashing objects supported in the ethernet device.
> + *
> + * @param port_id
> + * The port identifier of the Ethernet device.
> + * @param objects
> + * Supported objects vector set by the ethernet device.
> + * @return
> + * On return types and hints parameters will have bits set for supported
> + * object types.
> + * - (-ENOTSUP) if the device or the platform does not support cache
> stashing.
> + * - (-ENOSYS) if the underlying PMD hasn't implemented cache stashing
> + * feature.
> + * - (-EINVAL) on NULL values for types or hints parameters.
> + * - (0) on success.
> + */
> +__rte_experimental
> +int rte_eth_dev_stashing_capabilities_get(uint16_t port_id, uint16_t
> *objects);
> +
> #ifdef __cplusplus
> }
> #endif
> --
> 2.43.0
>
