Implement functions to migrate objects. This is based on
initial code by Matthew Wilcox and was modified to work with
slab object migration.
Signed-off-by: Christoph Lameter <c...@linux.com>
Index: linux/lib/radix-tree.c
===================================================================
--- linux.orig/lib/radix-tree.c
+++ linux/lib/radix-tree.c
@@ -1613,6 +1613,18 @@ static int radix_tree_cpu_dead(unsigned
return 0;
}
+
+extern void xa_object_migrate(void *tree_node, int numa_node);
+
+static void radix_tree_migrate(struct kmem_cache *s, void **objects, int nr,
+ int node, void *private)
+{
+ int i;
+
+ for (i=0; i<nr; i++)
+ xa_object_migrate(objects[i], node);
+}
+
void __init radix_tree_init(void)
{
int ret;
@@ -1627,4 +1639,7 @@ void __init radix_tree_init(void)
ret = cpuhp_setup_state_nocalls(CPUHP_RADIX_DEAD, "lib/radix:dead",
NULL, radix_tree_cpu_dead);
WARN_ON(ret < 0);
+ kmem_cache_setup_mobility(radix_tree_node_cachep,
+ NULL,
+ radix_tree_migrate);
}
Index: linux/lib/xarray.c
===================================================================
--- linux.orig/lib/xarray.c
+++ linux/lib/xarray.c
@@ -1934,6 +1934,51 @@ void xa_destroy(struct xarray *xa)
}
EXPORT_SYMBOL(xa_destroy);
+void xa_object_migrate(struct xa_node *node, int numa_node)
+{
+ struct xarray *xa = READ_ONCE(node->array);
+ void __rcu **slot;
+ struct xa_node *new_node;
+ int i;
+
+ /* Freed or not yet in tree then skip */
+ if (!xa || xa == XA_FREE_MARK)
+ return;
+
+ new_node = kmem_cache_alloc_node(radix_tree_node_cachep, GFP_KERNEL,
numa_node);
+
+ xa_lock_irq(xa);
+
+ /* Check again..... */
+ if (xa != node->array || !list_empty(&node->private_list)) {
+ node = new_node;
+ goto unlock;
+ }
+
+ memcpy(new_node, node, sizeof(struct xa_node));
+
+ /* Move pointers to new node */
+ INIT_LIST_HEAD(&new_node->private_list);
+ for (i = 0; i < XA_CHUNK_SIZE; i++) {
+ void *x = xa_entry_locked(xa, new_node, i);
+
+ if (xa_is_node(x))
+ rcu_assign_pointer(xa_to_node(x)->parent, new_node);
+ }
+ if (!new_node->parent)
+ slot = &xa->xa_head;
+ else
+ slot = &xa_parent_locked(xa, new_node)->slots[new_node->offset];
+ rcu_assign_pointer(*slot, xa_mk_node(new_node));
+
+unlock:
+ xa_unlock_irq(xa);
+ xa_node_free(node);
+ rcu_barrier();
+ return;
+
+}
+
#ifdef XA_DEBUG
void xa_dump_node(const struct xa_node *node)
{
