rt_flush_dev() currently walks every per-CPU uncached route list for each device being removed. This repeatedly examines unrelated routes and makes teardown increasingly expensive as the number of devices grows.
Replace each per-CPU list with 64 hash buckets keyed by the route's netdevice. Keep the owning-list pointer in dst_entry so route removal remains unchanged, while device teardown only walks the matching bucket on each CPU. Hash collisions are filtered by the existing device comparison. Signed-off-by: Chris J Arges <[email protected]> --- net/ipv4/route.c | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 604cc51dfd9b..3f9bc1ec72cc 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -74,6 +74,7 @@ #include <linux/init.h> #include <linux/skbuff.h> #include <linux/inetdevice.h> +#include <linux/hash.h> #include <linux/igmp.h> #include <linux/pkt_sched.h> #include <linux/mroute.h> @@ -1552,11 +1553,22 @@ struct uncached_list { struct list_head head; }; -static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list); +#define RT_UNCACHED_HASH_BITS 6 +#define RT_UNCACHED_HASH_SIZE BIT(RT_UNCACHED_HASH_BITS) + +struct uncached_table { + struct uncached_list buckets[RT_UNCACHED_HASH_SIZE]; +}; + +static DEFINE_PER_CPU_ALIGNED(struct uncached_table, rt_uncached_table); void rt_add_uncached_list(struct rtable *rt) { - struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list); + struct uncached_table *table = raw_cpu_ptr(&rt_uncached_table); + struct uncached_list *ul; + + ul = &table->buckets[hash_ptr(dst_dev(&rt->dst), + RT_UNCACHED_HASH_BITS)]; rt->dst.rt_uncached_list = ul; @@ -1588,14 +1600,18 @@ void rt_flush_dev(struct net_device *dev) int cpu; for_each_possible_cpu(cpu) { - struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); + struct uncached_table *table; + struct uncached_list *ul; + + table = per_cpu_ptr(&rt_uncached_table, cpu); + ul = &table->buckets[hash_ptr(dev, RT_UNCACHED_HASH_BITS)]; if (list_empty(&ul->head)) continue; spin_lock_bh(&ul->lock); list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) { - if (rt->dst.dev != dev) + if (dst_dev(&rt->dst) != dev) continue; rcu_assign_pointer(rt->dst.dev_rcu, blackhole_netdev); netdev_ref_replace(dev, blackhole_netdev, @@ -3771,10 +3787,16 @@ int __init ip_rt_init(void) ip_tstamps = idents_hash + (ip_idents_mask + 1) * sizeof(*ip_idents); for_each_possible_cpu(cpu) { - struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); + struct uncached_table *table; + int bucket; + + table = per_cpu_ptr(&rt_uncached_table, cpu); + for (bucket = 0; bucket < RT_UNCACHED_HASH_SIZE; bucket++) { + struct uncached_list *ul = &table->buckets[bucket]; - INIT_LIST_HEAD(&ul->head); - spin_lock_init(&ul->lock); + INIT_LIST_HEAD(&ul->head); + spin_lock_init(&ul->lock); + } } #ifdef CONFIG_IP_ROUTE_CLASSID ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct)); -- 2.43.0
