Count per-queue RX stats in poll, replenish, and the IRQ handler: packets, bytes, polls, large_packets, invalid_buffers, no_buffer_drops, and interrupts. Track transmit counters per TX queue to avoid cache line contention in the xmit hot path.
Expose counters via ethtool -S (rxN_* and txN_*), get_stats64() aggregation for ip -s and /proc/net/dev, and hcall_stats counters. Fix get_channels() reporting: max_rx is IBMVETH_MAX_RX_QUEUES only when MQ firmware is enabled, rx_count tracks adapter->num_rx_queues. Signed-off-by: Mingming Cao <[email protected]> Reviewed-by: Dave Marquardt <[email protected]> --- drivers/net/ethernet/ibm/ibmveth.c | 269 ++++++++++++++++++++++++++--- drivers/net/ethernet/ibm/ibmveth.h | 13 ++ 2 files changed, 259 insertions(+), 23 deletions(-) diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c index 863e5c68b42c..4e3f49b6346f 100644 --- a/drivers/net/ethernet/ibm/ibmveth.c +++ b/drivers/net/ethernet/ibm/ibmveth.c @@ -98,7 +98,15 @@ static struct ibmveth_stat ibmveth_stats[] = { { "fw_enabled_ipv6_csum", IBMVETH_STAT_OFF(fw_ipv6_csum_support) }, { "tx_large_packets", IBMVETH_STAT_OFF(tx_large_packets) }, { "rx_large_packets", IBMVETH_STAT_OFF(rx_large_packets) }, - { "fw_enabled_large_send", IBMVETH_STAT_OFF(fw_large_send_support) } + { "fw_enabled_large_send", IBMVETH_STAT_OFF(fw_large_send_support) }, + { "hcall_reg_lan_queue", IBMVETH_STAT_OFF(hcall_stats.reg_lan_queue) }, + { "hcall_reg_lan", IBMVETH_STAT_OFF(hcall_stats.reg_lan) }, + { "hcall_add_bufs_queue", IBMVETH_STAT_OFF(hcall_stats.add_bufs_queue) }, + { "hcall_add_bufs", IBMVETH_STAT_OFF(hcall_stats.add_bufs) }, + { "hcall_add_buf", IBMVETH_STAT_OFF(hcall_stats.add_buf) }, + { "hcall_free_lan_queue", IBMVETH_STAT_OFF(hcall_stats.free_lan_queue) }, + { "hcall_free_lan", IBMVETH_STAT_OFF(hcall_stats.free_lan) }, + { "hcall_send_lan", IBMVETH_STAT_OFF(hcall_stats.send_lan) }, }; /* simple methods of getting data from the current rxq entry */ @@ -244,6 +252,33 @@ static void ibmveth_free_rx_qstats(struct ibmveth_adapter *adapter) adapter->rx_qstats = NULL; } +/** + * ibmveth_alloc_tx_qstats - Allocate per-queue TX statistics + * @adapter: ibmveth adapter structure + * + * Return: 0 on success, -ENOMEM on failure + */ +static int ibmveth_alloc_tx_qstats(struct ibmveth_adapter *adapter) +{ + adapter->tx_qstats = kcalloc(IBMVETH_MAX_QUEUES, + sizeof(struct ibmveth_tx_queue_stats), + GFP_KERNEL); + if (!adapter->tx_qstats) + return -ENOMEM; + + return 0; +} + +/** + * ibmveth_free_tx_qstats - Free per-queue TX statistics + * @adapter: ibmveth adapter structure + */ +static void ibmveth_free_tx_qstats(struct ibmveth_adapter *adapter) +{ + kfree(adapter->tx_qstats); + adapter->tx_qstats = NULL; +} + /** * ibmveth_alloc_rx_queues - Allocate per-queue RX resources * @adapter: ibmveth adapter structure @@ -847,6 +882,8 @@ static void ibmveth_update_rx_no_buffer(struct ibmveth_adapter *adapter) __be64 *p = adapter->buffer_list_addr[i] + 4096 - 8; u64 drops = be64_to_cpup(p); + if (adapter->rx_qstats) + adapter->rx_qstats[i].no_buffer_drops = drops; if (i == 0) adapter->rx_no_buffer = drops; } @@ -1618,6 +1655,10 @@ static int ibmveth_open(struct net_device *netdev) if (rc) goto out_cleanup_rx_interrupts; + rc = ibmveth_alloc_tx_qstats(adapter); + if (rc) + goto out_free_tx_resources; + netif_tx_start_all_queues(netdev); netdev_dbg(netdev, "open complete\n"); @@ -1658,6 +1699,7 @@ static int ibmveth_close(struct net_device *netdev) } } + ibmveth_free_tx_qstats(adapter); ibmveth_free_tx_resources(adapter); ibmveth_cleanup_rx_interrupts(adapter); ibmveth_update_rx_no_buffer(adapter); @@ -1925,22 +1967,107 @@ static int ibmveth_set_features(struct net_device *dev, return rc1 ? rc1 : rc2; } +/** + * ibmveth_aggregate_rx_qstats - Sum per-queue RX stats into globals + * @adapter: ibmveth adapter + * + * Cold path only (ethtool). Keeps legacy global counters meaningful for + * tools that read the adapter-level fields in ibmveth_stats[]. + */ +static void ibmveth_aggregate_rx_qstats(struct ibmveth_adapter *adapter) +{ + u64 total_invalid = 0; + u64 total_large = 0; + int i; + + if (!adapter->rx_qstats) + return; + + for (i = 0; i < adapter->num_rx_queues; i++) { + total_invalid += adapter->rx_qstats[i].invalid_buffers; + total_large += adapter->rx_qstats[i].large_packets; + } + + adapter->rx_invalid_buffer = total_invalid; + adapter->rx_large_packets = total_large; +} + +/** + * ibmveth_aggregate_tx_qstats - Sum per-queue TX stats into globals + * @adapter: ibmveth adapter + * + * Cold path only (ethtool). Keeps legacy global counters meaningful for + * tools that read the adapter-level fields in ibmveth_stats[]. + */ +static void ibmveth_aggregate_tx_qstats(struct ibmveth_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + u64 total_large = 0; + u64 total_send_failed = 0; + int i; + + if (!adapter->tx_qstats) + return; + + for (i = 0; i < netdev->real_num_tx_queues; i++) { + total_large += adapter->tx_qstats[i].large_packets; + total_send_failed += adapter->tx_qstats[i].send_failures; + } + + adapter->tx_large_packets = total_large; + adapter->tx_send_failed = total_send_failed; +} + static void ibmveth_get_strings(struct net_device *dev, u32 stringset, u8 *data) { + struct ibmveth_adapter *adapter = netdev_priv(dev); + u8 *p = data; int i; if (stringset != ETH_SS_STATS) return; - for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++, data += ETH_GSTRING_LEN) - memcpy(data, ibmveth_stats[i].name, ETH_GSTRING_LEN); + for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++) { + memcpy(p, ibmveth_stats[i].name, ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } + + for (i = 0; i < adapter->num_rx_queues; i++) { + ethtool_sprintf(&p, "rx%d_packets", i); + ethtool_sprintf(&p, "rx%d_bytes", i); + ethtool_sprintf(&p, "rx%d_interrupts", i); + ethtool_sprintf(&p, "rx%d_polls", i); + ethtool_sprintf(&p, "rx%d_large_packets", i); + ethtool_sprintf(&p, "rx%d_invalid_buffers", i); + ethtool_sprintf(&p, "rx%d_no_buffer_drops", i); + } + + for (i = 0; i < dev->real_num_tx_queues; i++) { + ethtool_sprintf(&p, "tx%d_packets", i); + ethtool_sprintf(&p, "tx%d_bytes", i); + ethtool_sprintf(&p, "tx%d_large_packets", i); + ethtool_sprintf(&p, "tx%d_dropped_packets", i); + ethtool_sprintf(&p, "tx%d_send_failures", i); + ethtool_sprintf(&p, "tx%d_checksum_offload", i); + } + + for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { + ethtool_sprintf(&p, "pool%d_size", i); + ethtool_sprintf(&p, "pool%d_active", i); + ethtool_sprintf(&p, "pool%d_available", i); + } } static int ibmveth_get_sset_count(struct net_device *dev, int sset) { + struct ibmveth_adapter *adapter = netdev_priv(dev); + switch (sset) { case ETH_SS_STATS: - return ARRAY_SIZE(ibmveth_stats); + return ARRAY_SIZE(ibmveth_stats) + + adapter->num_rx_queues * IBMVETH_NUM_RX_QSTATS + + dev->real_num_tx_queues * IBMVETH_NUM_TX_QSTATS + + IBMVETH_NUM_BUFF_POOLS * 3; default: return -EOPNOTSUPP; } @@ -1949,21 +2076,62 @@ static int ibmveth_get_sset_count(struct net_device *dev, int sset) static void ibmveth_get_ethtool_stats(struct net_device *dev, struct ethtool_stats *stats, u64 *data) { - int i; struct ibmveth_adapter *adapter = netdev_priv(dev); + int i, j; + + ibmveth_aggregate_rx_qstats(adapter); + ibmveth_aggregate_tx_qstats(adapter); for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++) data[i] = IBMVETH_GET_STAT(adapter, ibmveth_stats[i].offset); + + for (j = 0; j < adapter->num_rx_queues; j++) { + if (adapter->rx_qstats) { + data[i++] = adapter->rx_qstats[j].packets; + data[i++] = adapter->rx_qstats[j].bytes; + data[i++] = adapter->rx_qstats[j].interrupts; + data[i++] = adapter->rx_qstats[j].polls; + data[i++] = adapter->rx_qstats[j].large_packets; + data[i++] = adapter->rx_qstats[j].invalid_buffers; + data[i++] = adapter->rx_qstats[j].no_buffer_drops; + } else { + i += IBMVETH_NUM_RX_QSTATS; + } + } + + for (j = 0; j < dev->real_num_tx_queues; j++) { + if (adapter->tx_qstats) { + data[i++] = adapter->tx_qstats[j].packets; + data[i++] = adapter->tx_qstats[j].bytes; + data[i++] = adapter->tx_qstats[j].large_packets; + data[i++] = adapter->tx_qstats[j].dropped_packets; + data[i++] = adapter->tx_qstats[j].send_failures; + data[i++] = adapter->tx_qstats[j].checksum_offload; + } else { + i += IBMVETH_NUM_TX_QSTATS; + } + } + + for (j = 0; j < IBMVETH_NUM_BUFF_POOLS; j++) { + data[i++] = adapter->rx_buff_pool[0][j].size; + data[i++] = adapter->rx_buff_pool[0][j].active; + data[i++] = atomic_read(&adapter->rx_buff_pool[0][j].available); + } } static void ibmveth_get_channels(struct net_device *netdev, struct ethtool_channels *channels) { + struct ibmveth_adapter *adapter = netdev_priv(netdev); + channels->max_tx = ibmveth_real_max_tx_queues(); channels->tx_count = netdev->real_num_tx_queues; - channels->max_rx = netdev->real_num_rx_queues; - channels->rx_count = netdev->real_num_rx_queues; + if (adapter->multi_queue) + channels->max_rx = IBMVETH_MAX_RX_QUEUES; + else + channels->max_rx = 1; + channels->rx_count = adapter->num_rx_queues; } static int ibmveth_set_channels(struct net_device *netdev, @@ -2061,12 +2229,15 @@ static int ibmveth_send(struct ibmveth_adapter *adapter, return 1; } + adapter->hcall_stats.send_lan++; return 0; } static int ibmveth_is_packet_unsupported(struct sk_buff *skb, - struct net_device *netdev) + struct ibmveth_adapter *adapter, + int queue_num) { + struct net_device *netdev = adapter->netdev; struct ethhdr *ether_header; int ret = 0; @@ -2074,7 +2245,8 @@ static int ibmveth_is_packet_unsupported(struct sk_buff *skb, if (ether_addr_equal(ether_header->h_dest, netdev->dev_addr)) { netdev_dbg(netdev, "veth doesn't support loopback packets, dropping packet.\n"); - netdev->stats.tx_dropped++; + if (adapter->tx_qstats) + adapter->tx_qstats[queue_num].dropped_packets++; ret = -EOPNOTSUPP; } @@ -2090,7 +2262,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb, int i, queue_num = skb_get_queue_mapping(skb); unsigned long mss = 0; - if (ibmveth_is_packet_unsupported(skb, netdev)) + if (ibmveth_is_packet_unsupported(skb, adapter, queue_num)) goto out; /* veth can't checksum offload UDP */ if (skb->ip_summed == CHECKSUM_PARTIAL && @@ -2101,7 +2273,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb, skb_checksum_help(skb)) { netdev_err(netdev, "tx: failed to checksum packet\n"); - netdev->stats.tx_dropped++; + adapter->tx_qstats[queue_num].dropped_packets++; goto out; } @@ -2113,6 +2285,8 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb, desc_flags |= (IBMVETH_BUF_NO_CSUM | IBMVETH_BUF_CSUM_GOOD); + adapter->tx_qstats[queue_num].checksum_offload++; + /* Need to zero out the checksum */ buf[0] = 0; buf[1] = 0; @@ -2124,7 +2298,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb, if (skb->ip_summed == CHECKSUM_PARTIAL && skb_is_gso(skb)) { if (adapter->fw_large_send_support) { mss = (unsigned long)skb_shinfo(skb)->gso_size; - adapter->tx_large_packets++; + adapter->tx_qstats[queue_num].large_packets++; } else if (!skb_is_gso_v6(skb)) { /* Put -1 in the IP checksum to tell phyp it * is a largesend packet. Put the mss in @@ -2133,7 +2307,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb, ip_hdr(skb)->check = 0xffff; tcp_hdr(skb)->check = cpu_to_be16(skb_shinfo(skb)->gso_size); - adapter->tx_large_packets++; + adapter->tx_qstats[queue_num].large_packets++; } } @@ -2141,7 +2315,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb, if (unlikely(skb->len > adapter->tx_ltb_size)) { netdev_err(adapter->netdev, "tx: packet size (%u) exceeds ltb (%u)\n", skb->len, adapter->tx_ltb_size); - netdev->stats.tx_dropped++; + adapter->tx_qstats[queue_num].dropped_packets++; goto out; } memcpy(adapter->tx_ltb_ptr[queue_num], skb->data, skb_headlen(skb)); @@ -2158,7 +2332,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb, if (unlikely(total_bytes != skb->len)) { netdev_err(adapter->netdev, "tx: incorrect packet len copied into ltb (%u != %u)\n", skb->len, total_bytes); - netdev->stats.tx_dropped++; + adapter->tx_qstats[queue_num].dropped_packets++; goto out; } desc.fields.flags_len = desc_flags | skb->len; @@ -2167,11 +2341,11 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb, dma_wmb(); if (ibmveth_send(adapter, desc.desc, mss)) { - adapter->tx_send_failed++; - netdev->stats.tx_dropped++; + adapter->tx_qstats[queue_num].send_failures++; + adapter->tx_qstats[queue_num].dropped_packets++; } else { - netdev->stats.tx_packets++; - netdev->stats.tx_bytes += skb->len; + adapter->tx_qstats[queue_num].packets++; + adapter->tx_qstats[queue_num].bytes += skb->len; } out: @@ -2311,6 +2485,9 @@ static int ibmveth_poll(struct napi_struct *napi, int budget) if (WARN_ON(queue_index < 0 || queue_index >= adapter->num_rx_queues)) return 0; + if (adapter->rx_qstats) + adapter->rx_qstats[queue_index].polls++; + restart_poll: while (frames_processed < budget) { if (!ibmveth_rxq_pending_buffer(adapter, queue_index)) @@ -2319,7 +2496,10 @@ static int ibmveth_poll(struct napi_struct *napi, int budget) smp_rmb(); if (!ibmveth_rxq_buffer_valid(adapter, queue_index)) { wmb(); /* suggested by larson1 */ - adapter->rx_invalid_buffer++; + if (adapter->rx_qstats) + adapter->rx_qstats[queue_index].invalid_buffers++; + else + adapter->rx_invalid_buffer++; netdev_dbg(netdev, "recycling invalid buffer\n"); rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, true); if (unlikely(rc)) @@ -2384,7 +2564,10 @@ static int ibmveth_poll(struct napi_struct *napi, int budget) if ((length > netdev->mtu + ETH_HLEN) || lrg_pkt || iph_check == 0xffff) { ibmveth_rx_mss_helper(skb, mss, lrg_pkt); - adapter->rx_large_packets++; + if (adapter->rx_qstats) + adapter->rx_qstats[queue_index].large_packets++; + else + adapter->rx_large_packets++; } if (csum_good) { @@ -2394,8 +2577,11 @@ static int ibmveth_poll(struct napi_struct *napi, int budget) napi_gro_receive(napi, skb); /* send it up */ - netdev->stats.rx_packets++; - netdev->stats.rx_bytes += length; + if (adapter->rx_qstats) { + adapter->rx_qstats[queue_index].packets++; + adapter->rx_qstats[queue_index].bytes += length; + } + frames_processed++; } } @@ -2444,6 +2630,9 @@ static irqreturn_t ibmveth_interrupt(int irq, void *dev_instance) if (WARN_ON(qindex < 0 || qindex >= adapter->num_rx_queues)) return IRQ_NONE; + if (adapter->rx_qstats) + adapter->rx_qstats[qindex].interrupts++; + if (napi_schedule_prep(napi)) { lpar_rc = ibmveth_disable_irq(adapter, qindex); WARN_ON(lpar_rc != H_SUCCESS); @@ -2656,6 +2845,39 @@ static netdev_features_t ibmveth_features_check(struct sk_buff *skb, return vlan_features_check(skb, features); } +/** + * ibmveth_get_stats64 - Return aggregated per-queue statistics + * @dev: network device + * @stats: rtnl link statistics storage + * + * Sums per-queue rx_qstats and tx_qstats into the rtnl counters. + * Callers use ndo_get_stats64(); avoid updating netdev->stats on the + * xmit/poll paths to keep per-queue counters off the hot cache line. + */ +static void ibmveth_get_stats64(struct net_device *dev, + struct rtnl_link_stats64 *stats) +{ + struct ibmveth_adapter *adapter = netdev_priv(dev); + int i; + + if (adapter->rx_qstats) { + for (i = 0; i < adapter->num_rx_queues; i++) { + stats->rx_packets += adapter->rx_qstats[i].packets; + stats->rx_bytes += adapter->rx_qstats[i].bytes; + } + } + + if (adapter->tx_qstats) { + for (i = 0; i < dev->real_num_tx_queues; i++) { + stats->tx_packets += adapter->tx_qstats[i].packets; + stats->tx_bytes += adapter->tx_qstats[i].bytes; + stats->tx_dropped += adapter->tx_qstats[i].dropped_packets; + } + } + + stats->tx_errors = dev->stats.tx_errors; +} + static const struct net_device_ops ibmveth_netdev_ops = { .ndo_open = ibmveth_open, .ndo_stop = ibmveth_close, @@ -2668,6 +2890,7 @@ static const struct net_device_ops ibmveth_netdev_ops = { .ndo_validate_addr = eth_validate_addr, .ndo_set_mac_address = ibmveth_set_mac_addr, .ndo_features_check = ibmveth_features_check, + .ndo_get_stats64 = ibmveth_get_stats64, #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = ibmveth_poll_controller, #endif diff --git a/drivers/net/ethernet/ibm/ibmveth.h b/drivers/net/ethernet/ibm/ibmveth.h index f7b20fd01acb..390c660af979 100644 --- a/drivers/net/ethernet/ibm/ibmveth.h +++ b/drivers/net/ethernet/ibm/ibmveth.h @@ -316,9 +316,21 @@ struct ibmveth_rx_queue_stats { u64 no_buffer_drops; }; +struct ibmveth_tx_queue_stats { + u64 packets; + u64 bytes; + u64 large_packets; + u64 dropped_packets; + u64 send_failures; + u64 checksum_offload; +}; + #define IBMVETH_NUM_RX_QSTATS \ (sizeof(struct ibmveth_rx_queue_stats) / sizeof(u64)) +#define IBMVETH_NUM_TX_QSTATS \ + (sizeof(struct ibmveth_tx_queue_stats) / sizeof(u64)) + struct ibmveth_buff_pool { u32 size; u32 index; @@ -386,6 +398,7 @@ struct ibmveth_adapter { /* Multi-queue statistics */ struct ibmveth_hcall_stats hcall_stats; struct ibmveth_rx_queue_stats *rx_qstats; + struct ibmveth_tx_queue_stats *tx_qstats; /* Ethtool settings */ u8 duplex; -- 2.39.3 (Apple Git-146)
