> On Mon, Jun 02, 2025 at 10:38:03PM +0000, Wathsala Vithanage wrote:
> > Extend the ethdev library to enable the stashing of different data
> > objects, such as the ones listed below, into CPU caches directly from
> > the NIC.
> >
> > - Rx/Tx queue descriptors
> > - Rx packets
> > - Packet headers
> > - packet payloads
> > - Data of a packet at an offset from the start of the packet
> >
> > The APIs are designed in a hardware/vendor agnostic manner such that
> > supporting PMDs could use any capabilities available in the underlying
> > hardware for fine-grained stashing of data objects into a CPU cache
> >
> > The API provides an interface to query the availability of stashing
> > capabilities, i.e., platform/NIC support, stashable object types, etc,
> > via the rte_eth_dev_stashing_capabilities_get interface.
> >
> > The function pair rte_eth_dev_stashing_rx_config_set and
> > rte_eth_dev_stashing_tx_config_set sets the stashing hint (the CPU,
> > cache level, and data object types) on the Rx and Tx queues.
> >
> > PMDs that support stashing must register their implementations with
> > the following eth_dev_ops callbacks, which are invoked by the ethdev
> > functions listed above.
> >
> > - stashing_capabilities_get
> > - stashing_rx_hints_set
> > - stashing_tx_hints_set
> >
> > Signed-off-by: Wathsala Vithanage <wathsala.vithan...@arm.com>
> > Reviewed-by: Honnappa Nagarahalli <honnappa.nagaraha...@arm.com>
> > Reviewed-by: Dhruv Tripathi <dhruv.tripa...@arm.com>
> > ---
>
> Few small comments inline below
>
Thank you, Bruce. I will address them in the next version. As asked by
@stephen@networkplumber I will only send it only if kernel piece makes it
into Linus branch.
--wathsala
> /Bruce
>
> > lib/ethdev/ethdev_driver.h | 66 ++++++++++++++++
> > lib/ethdev/rte_ethdev.c | 149 ++++++++++++++++++++++++++++++++++
> > lib/ethdev/rte_ethdev.h | 158 +++++++++++++++++++++++++++++++++++++
> > 3 files changed, 373 insertions(+)
> >
> > diff --git a/lib/ethdev/ethdev_driver.h b/lib/ethdev/ethdev_driver.h
> > index 2b4d2ae9c3..8a4012db08 100644
> > --- a/lib/ethdev/ethdev_driver.h
> > +++ b/lib/ethdev/ethdev_driver.h
> > @@ -1376,6 +1376,68 @@ enum rte_eth_dev_operation { typedef uint64_t
> > (*eth_get_restore_flags_t)(struct rte_eth_dev *dev,
> > enum rte_eth_dev_operation op);
> >
> > +/**
> > + * @internal
> > + * Set cache stashing hints in Rx queue.
> > + *
> > + * @param dev
> > + * Port (ethdev) handle.
> > + * @param queue_id
> > + * Rx queue.
> > + * @param config
> > + * Stashing hints configuration for the queue.
> > + *
> > + * @return
> > + * -ENOTSUP if the device or the platform does not support cache
> > stashing.
> > + * -ENOSYS if the underlying PMD hasn't implemented cache stashing
> feature.
> > + * -EINVAL on invalid arguments.
> > + * 0 on success.
> > + */
> > +typedef int (*eth_stashing_rx_hints_set_t)(struct rte_eth_dev *dev,
> > uint16_t
> queue_id,
> > + struct rte_eth_stashing_config
> *config);
> > +
> > +/**
> > + * @internal
> > + * Set cache stashing hints in Tx queue.
> > + *
> > + * @param dev
> > + * Port (ethdev) handle.
> > + * @param queue_id
> > + * Tx queue.
> > + * @param config
> > + * Stashing hints configuration for the queue.
> > + *
> > + * @return
> > + * -ENOTSUP if the device or the platform does not support cache
> > stashing.
> > + * -ENOSYS if the underlying PMD hasn't implemented cache stashing
> feature.
> > + * -EINVAL on invalid arguments.
> > + * 0 on success.
>
> What about on failure of the underlying ioctl call?
>
> > + */
> > +typedef int (*eth_stashing_tx_hints_set_t)(struct rte_eth_dev *dev,
> > uint16_t
> queue_id,
> > + struct rte_eth_stashing_config
> *config);
> > +
> > +/**
> > + * @internal
> > + * Get cache stashing object types supported in the ethernet device.
> > + * The return value indicates availability of stashing hints support
> > + * in the hardware and the PMD.
> > + *
> > + * @param dev
> > + * Port (ethdev) handle.
> > + * @param objects
> > + * PMD sets supported bits on return.
> > + *
> > + * @return
> > + * -ENOTSUP if the device or the platform does not support cache
> > stashing.
> > + * -ENOSYS if the underlying PMD hasn't implemented cache stashing
> feature.
> > + * -EINVAL on NULL values for types or hints parameters.
> > + * On return, types and hints parameters will have bits set for supported
> > + * object types and hints.
> > + * 0 on success.
> > + */
> > +typedef int (*eth_stashing_capabilities_get_t)(struct rte_eth_dev *dev,
> > + uint16_t *objects);
> > +
> > /**
> > * @internal A structure containing the functions exported by an Ethernet
> driver.
> > */
> > @@ -1402,6 +1464,10 @@ struct eth_dev_ops {
> > eth_mac_addr_remove_t mac_addr_remove; /**< Remove MAC
> address */
> > eth_mac_addr_add_t mac_addr_add; /**< Add a MAC address */
> > eth_mac_addr_set_t mac_addr_set; /**< Set a MAC address */
> > + eth_stashing_rx_hints_set_t stashing_rx_hints_set; /**< Set Rx cache
> stashing*/
> > + eth_stashing_tx_hints_set_t stashing_tx_hints_set; /**< Set Tx cache
> stashing*/
> > + /** Get supported stashing hints*/
> > + eth_stashing_capabilities_get_t stashing_capabilities_get;
> > /** Set list of multicast addresses */
> > eth_set_mc_addr_list_t set_mc_addr_list;
> > mtu_set_t mtu_set; /**< Set MTU */
> > diff --git a/lib/ethdev/rte_ethdev.c b/lib/ethdev/rte_ethdev.c index
> > d4197322a0..ae666c370b 100644
> > --- a/lib/ethdev/rte_ethdev.c
> > +++ b/lib/ethdev/rte_ethdev.c
> > @@ -158,6 +158,7 @@ static const struct {
> > {RTE_ETH_DEV_CAPA_RXQ_SHARE, "RXQ_SHARE"},
> > {RTE_ETH_DEV_CAPA_FLOW_RULE_KEEP, "FLOW_RULE_KEEP"},
> > {RTE_ETH_DEV_CAPA_FLOW_SHARED_OBJECT_KEEP,
> > "FLOW_SHARED_OBJECT_KEEP"},
> > + {RTE_ETH_DEV_CAPA_CACHE_STASHING, "CACHE_STASHING"},
> > };
> >
> > enum {
> > @@ -7419,5 +7420,153 @@ int rte_eth_dev_map_aggr_tx_affinity(uint16_t
> port_id, uint16_t tx_queue_id,
> > return ret;
> > }
> >
> >
> +RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_eth_dev_validate_stashing_config,
> > +25.07) int rte_eth_dev_validate_stashing_config(uint16_t port_id,
> > +uint16_t queue_id,
> > + uint8_t queue_direction,
> > + struct rte_eth_stashing_config *config) {
> > + struct rte_eth_dev *dev;
> > + struct rte_eth_dev_info dev_info;
> > + int ret = 0;
> > + uint16_t nb_queues;
> > +
> > + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
> > +
> > + if (!config) {
> > + RTE_ETHDEV_LOG_LINE(ERR, "Invalid stashing configuration");
> > + ret = -EINVAL;
> > + goto out;
> > + }
> > +
> > + /*
> > + * Check for invalid objects
> > + */
> > + if (!RTE_ETH_DEV_STASH_OBJECTS_VALID(config->objects)) {
> > + RTE_ETHDEV_LOG_LINE(ERR, "Invalid stashing objects");
> > + ret = -EINVAL;
> > + goto out;
> > + }
> > +
> > + dev = &rte_eth_devices[port_id];
> > +
> > + nb_queues = (queue_direction == RTE_ETH_DEV_RX_QUEUE) ?
> > + dev->data->nb_rx_queues :
> > + dev->data->nb_tx_queues;
> > +
> > + if (queue_id >= nb_queues) {
> > + RTE_ETHDEV_LOG_LINE(ERR, "Invalid Rx queue_id=%u",
> queue_id);
> > + ret = -EINVAL;
> > + goto out;
> > + }
> > +
> > + ret = rte_eth_dev_info_get(port_id, &dev_info);
> > + if (ret < 0)
> > + goto out;
> > +
> > + if ((dev_info.dev_capa & RTE_ETH_DEV_CAPA_CACHE_STASHING) !=
> > + RTE_ETH_DEV_CAPA_CACHE_STASHING) {
>
> Nit: check if all this can fit on one line under 100 chars. I think it can.
>
> > + ret = -ENOTSUP;
> > + goto out;
> > + }
> > +
> > + if (*dev->dev_ops->stashing_rx_hints_set == NULL ||
> > + *dev->dev_ops->stashing_tx_hints_set == NULL) {
> > + RTE_ETHDEV_LOG_LINE(ERR, "Stashing hints are not
> implemented "
> > + "in %s for %s", dev_info.driver_name,
>
> Don't split error messages across lines. Text strings are allowed to go over
> the 100
> char limit if necessary to avoid splitting.
>
> > + dev_info.device->name);
> > + ret = -ENOSYS;
> > + }
> > +
> > +out:
> > + return ret;
> > +}
> > +
> > +RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_eth_dev_stashing_rx_config_set,
> > +25.07) int rte_eth_dev_stashing_rx_config_set(uint16_t port_id,
> > +uint16_t queue_id,
> > + struct rte_eth_stashing_config *config) {
> > + struct rte_eth_dev *dev;
> > + int ret = 0;
> > +
> > + ret = rte_eth_dev_validate_stashing_config(port_id, queue_id,
> > + RTE_ETH_DEV_RX_QUEUE,
> > + config);
> > + if (ret < 0)
> > + goto out;
> > +
> > + dev = &rte_eth_devices[port_id];
> > +
> > + ret = eth_err(port_id,
> > + (*dev->dev_ops->stashing_rx_hints_set)(dev, queue_id,
> > + config));
> > +out:
> > + return ret;
> > +}
> > +
> > +RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_eth_dev_stashing_tx_config_set,
> > +25.07) int rte_eth_dev_stashing_tx_config_set(uint16_t port_id,
> > +uint16_t queue_id,
> > + struct rte_eth_stashing_config *config) {
> > + struct rte_eth_dev *dev;
> > + int ret = 0;
> > +
> > + ret = rte_eth_dev_validate_stashing_config(port_id, queue_id,
> > + RTE_ETH_DEV_TX_QUEUE,
> > + config);
> > + if (ret < 0)
> > + goto out;
> > +
> > + dev = &rte_eth_devices[port_id];
> > +
> > + ret = eth_err(port_id,
> > + (*dev->dev_ops->stashing_rx_hints_set) (dev, queue_id,
> > + config));
> > +out:
> > + return ret;
> > +}
> > +
> >
> +RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_eth_dev_stashing_capabilities_get,
> > +25.07) int rte_eth_dev_stashing_capabilities_get(uint16_t port_id,
> > +uint16_t *objects) {
> > + struct rte_eth_dev *dev;
> > + struct rte_eth_dev_info dev_info;
> > + int ret = 0;
> > +
> > + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
> > +
> > + if (!objects) {
> > + ret = -EINVAL;
> > + goto out;
> > + }
> > +
> > + dev = &rte_eth_devices[port_id];
> > +
> > + ret = rte_eth_dev_info_get(port_id, &dev_info);
> > + if (ret < 0)
> > + goto out;
> > +
> > + if ((dev_info.dev_capa & RTE_ETH_DEV_CAPA_CACHE_STASHING) !=
> > + RTE_ETH_DEV_CAPA_CACHE_STASHING) {
> > + ret = -ENOTSUP;
> > + goto out;
> > + }
> > +
> > + if (*dev->dev_ops->stashing_capabilities_get == NULL) {
> > + RTE_ETHDEV_LOG_LINE(ERR, "Stashing hints are not
> implemented "
> > + "in %s for %s", dev_info.driver_name,
> > + dev_info.device->name);
> > + ret = -ENOSYS;
> > + goto out;
> > + }
> > + ret = eth_err(port_id,
> > + (*dev->dev_ops->stashing_capabilities_get)(dev, objects));
> > +out:
> > + return ret;
> > +}
> > +
> > RTE_EXPORT_SYMBOL(rte_eth_dev_logtype)
> > RTE_LOG_REGISTER_DEFAULT(rte_eth_dev_logtype, INFO); diff --git
> > a/lib/ethdev/rte_ethdev.h b/lib/ethdev/rte_ethdev.h index
> > ea7f8c4a1a..1398f8c837 100644
> > --- a/lib/ethdev/rte_ethdev.h
> > +++ b/lib/ethdev/rte_ethdev.h
> > @@ -1667,6 +1667,9 @@ struct rte_eth_conf { #define
> > RTE_ETH_DEV_CAPA_FLOW_SHARED_OBJECT_KEEP RTE_BIT64(4) /**@}*/
> >
> > +/** Device supports stashing to CPU/system caches. */ #define
> > +RTE_ETH_DEV_CAPA_CACHE_STASHING RTE_BIT64(5)
> > +
> > /*
> > * Fallback default preferred Rx/Tx port parameters.
> > * These are used if an application requests default parameters @@
> > -1838,6 +1841,7 @@ struct rte_eth_dev_info {
> > struct rte_eth_dev_portconf default_txportconf;
> > /** Generic device capabilities (RTE_ETH_DEV_CAPA_). */
> > uint64_t dev_capa;
> > + uint16_t stashing_capa;
> > /**
> > * Switching information for ports on a device with a
> > * embedded managed interconnect/switch.
> > @@ -6173,6 +6177,160 @@ int rte_eth_cman_config_set(uint16_t port_id,
> > const struct rte_eth_cman_config * __rte_experimental int
> > rte_eth_cman_config_get(uint16_t port_id, struct rte_eth_cman_config
> > *config);
> >
> > +
> > +/** Queue type is RX. */
> > +#define RTE_ETH_DEV_RX_QUEUE 0
> > +/** Queue type is TX. */
> > +#define RTE_ETH_DEV_TX_QUEUE 1
> > +
>
> I'd prefer an enum for these.
> Why are the necessary since we have separate rx and tx functions for the
> caching
> hints.
>
> > +
> > +/**
> > + * @warning
> > + * @b EXPERIMENTAL: this structure may change, or be removed, without
> > +prior notice
> > + *
> > + * A structure used for configuring the cache stashing hints.
> > + */
> > +struct rte_eth_stashing_config {
> > + /**
> > + * lcore_id of the processor the stashing hints are applied to.
> > + */
> > + uint32_t lcore_id;
> > + /**
> > + * Zero based cache level relative to the CPU.
> > + * E.g. l1d = 0, l2d = 1,...
> > + */
> > + uint32_t cache_level;
> > + /**
> > + * Object types the configuration is applied to
> > + */
> > + uint16_t objects;
>
> What are the objects? That needs to be covered by the docs, or make this an
> enum type so that it's clear from the typesystem what it applies to [though
> that
> would require an array of objects and count, it may be clearer for user].
>
> > + /**
> > + * The offset if RTE_ETH_DEV_STASH_OBJECT_OFFSET bit is set
> > + * in objects
> > + */
> > + int offset;
> > +};
> > +
> > +/**@{@name Stashable Rx/Tx queue object types supported by the
> > +ethernet device *@see rte_eth_dev_stashing_capabilities_get
> > + *@see rte_eth_dev_stashing_rx_config_set
> > + *@see rte_eth_dev_stashing_tx_config_set
> > + */
> > +
> > +/**
> > + * Apply stashing hint to data at a given offset from the start of a
> > + * received packet.
> > + */
> > +#define RTE_ETH_DEV_STASH_OBJECT_OFFSET 0x0001
> > +
> > +/** Apply stashing hint to an rx descriptor. */
> > +#define RTE_ETH_DEV_STASH_OBJECT_DESC 0x0002
> > +
> > +/** Apply stashing hint to a header of a received packet. */
> > +#define RTE_ETH_DEV_STASH_OBJECT_HEADER 0x0004
> > +
> > +/** Apply stashing hint to a payload of a received packet. */
> > +#define RTE_ETH_DEV_STASH_OBJECT_PAYLOAD 0x0008
> > +
> > +#define __RTE_ETH_DEV_STASH_OBJECT_MASK 0x000f
> > +/**@}*/
> > +
> > +#define RTE_ETH_DEV_STASH_OBJECTS_VALID(t)
> \
> > + ((!((t) & (~__RTE_ETH_DEV_STASH_OBJECT_MASK))) && (t))
> > +
> > +/**
> > + *
> > + * @warning
> > + * @b EXPERIMENTAL: this API may change, or be removed, without prior
> > +notice
> > + *
> > + * @internal
> > + * Helper function to validate stashing hints configuration.
> > + */
> > +__rte_experimental
> > +int rte_eth_dev_validate_stashing_config(uint16_t port_id, uint16_t
> > queue_id,
> > + uint8_t queue_direction,
> > + struct rte_eth_stashing_config
> *config);
> > +
> > +/**
> > + *
> > + * @warning
> > + * @b EXPERIMENTAL: this API may change, or be removed, without prior
> > +notice
> > + *
> > + * Provide cache stashing hints for improved memory access latencies
> > +for
> > + * packets received by the NIC.
> > + * This feature is available only in supported NICs and platforms.
> > + *
> > + * @param port_id
> > + * The port identifier of the Ethernet device.
> > + * @param queue_id
> > + * The index of the receive queue to which hints are applied.
> > + * @param config
> > + * Stashing configuration.
> > + * @return
> > + * - (-ENODEV) on incorrect port_ids.
> > + * - (-EINVAL) if both RX and TX object types used in conjuection in
> > +objects
> > + * parameter.
> > + * - (-EINVAL) on invalid queue_id.
> > + * - (-ENOTSUP) if RTE_ETH_DEV_CAPA_CACHE_STASHING capability is
> unavailable.
> > + * - (-ENOSYS) if PMD does not implement cache stashing hints.
> > + * - (0) on Success.
> > + */
> > +__rte_experimental
> > +int rte_eth_dev_stashing_rx_config_set(uint16_t port_id, uint16_t queue_id,
> > + struct rte_eth_stashing_config *config);
> > +
> > +/**
> > + *
> > + * @warning
> > + * @b EXPERIMENTAL: this API may change, or be removed, without prior
> > +notice
> > + *
> > + * Configure cache stashing for improved memory access latencies for
> > +Tx
> > + * queue completion descriptors being sent to host system by the NIC.
> > + * This feature is available only in supported NICs and platforms.
> > + *
> > + * @param port_id
> > + * The port identifier of the Ethernet device.
> > + * @param queue_id
> > + * The index of the receive queue to which hints are applied.
> > + * @param config
> > + * Stashing configuration.
> > + * @return
> > + * - (-ENODEV) on incorrect port_ids.
> > + * - (-EINVAL) if both RX and TX object types are used in
> > +conjuection in objects
> > + * parameter.
> > + * - (-EINVAL) if hints are incompatible with TX queues.
> > + * - (-EINVAL) on invalid queue_id.
> > + * - (-ENOTSUP) if RTE_ETH_DEV_CAPA_CACHE_STASHING capability is
> unavailable.
> > + * - (-ENOSYS) if PMD does not implement cache stashing hints.
> > + * - (0) on Success.
> > + */
> > +__rte_experimental
> > +int rte_eth_dev_stashing_tx_config_set(uint16_t port_id, uint16_t queue_id,
> > + struct rte_eth_stashing_config *config);
> > +
> > +/**
> > + *
> > + * @warning
> > + * @b EXPERIMENTAL: this API may change, or be removed, without prior
> > +notice
> > + *
> > + * Discover cache stashing objects supported in the ethernet device.
> > + *
> > + * @param port_id
> > + * The port identifier of the Ethernet device.
> > + * @param objects
> > + * Supported objects vector set by the ethernet device.
> > + * @return
> > + * On return types and hints parameters will have bits set for
> > +supported
> > + * object types.
> > + * - (-ENOTSUP) if the device or the platform does not support cache
> > stashing.
> > + * - (-ENOSYS) if the underlying PMD hasn't implemented cache
> > +stashing
> > + * feature.
> > + * - (-EINVAL) on NULL values for types or hints parameters.
> > + * - (0) on success.
> > + */
> > +__rte_experimental
> > +int rte_eth_dev_stashing_capabilities_get(uint16_t port_id, uint16_t
> > +*objects);
> > +
> > #ifdef __cplusplus
> > }
> > #endif
> > --
> > 2.43.0
> >
