From: Honglei Huang <[email protected]> Add amdgpu_svm_attr.h with the SVM attribute types and tree infrastructure together with the helper implementation in amdgpu_svm_attr.c that uses them. The header is squashed into the implementation patch so it does not stand alone as declarations without an implementation.
amdgpu_svm_attr.h: - Internal flag bitmask definitions mapped from UAPI attr types - PTE_FLAG_MASK and MAPPING_FLAG_MASK for change detection - struct amdgpu_svm_attrs: user set attribute range - struct amdgpu_svm_attr_range: interval tree node with attrs - struct amdgpu_svm_attr_tree: mutex protected RB tree for store and search - enum amdgpu_svm_attr_change_trigger: change flags of user attribute changes amdgpu_svm_attr.c: - Default attribute initialization: amdgpu_svm_attr_set_default - Device memory and VRAM preference helpers - VMA validity checker: amdgpu_svm_check_vma - Attribute equality comparison: attr_equal - Interval tree CRUD operations: find, get_bounds, alloc, insert, and remove - attr_set_interval helper for range boundary updates Signed-off-by: Honglei Huang <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c | 235 +++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.h | 183 +++++++++++++++ 2 files changed, 418 insertions(+) create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.h diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c new file mode 100644 index 0000000000000..9d3519776c9c8 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright 2026 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "amdgpu_svm.h" +#include "amdgpu_svm_attr.h" + +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/gfp.h> +#include <linux/lockdep.h> +#include <linux/minmax.h> +#include <linux/mm.h> +#include <linux/slab.h> + +struct attr_set_ctx { + struct amdgpu_svm_attrs old_attrs; + struct amdgpu_svm_attrs new_attrs; + unsigned long start_page; + unsigned long last_page; +}; + +struct attr_get_ctx { + int32_t preferred_loc; + int32_t prefetch_loc; + enum amdgpu_ioctl_svm_access access; + uint32_t granularity; + uint32_t flags_and; + bool has_range; +}; + +bool amdgpu_svm_attr_prefer_vram(const struct amdgpu_svm_attrs *attrs) +{ + if (attrs->preferred_loc != AMDGPU_SVM_LOCATION_UNDEFINED && + attrs->preferred_loc != AMDGPU_SVM_LOCATION_SYSMEM) + return true; + + if (attrs->prefetch_loc != AMDGPU_SVM_LOCATION_UNDEFINED && + attrs->prefetch_loc != AMDGPU_SVM_LOCATION_SYSMEM) + return true; + + return false; +} + +struct vm_area_struct *amdgpu_svm_check_vma(struct mm_struct *mm, + unsigned long addr) +{ + const unsigned long unsupported_vm_flags = VM_IO | VM_PFNMAP | + VM_MIXEDMAP; + struct vm_area_struct *vma = vma_lookup(mm, addr); + + if (!vma) + return ERR_PTR(-EFAULT); + + if (vma->vm_flags & unsupported_vm_flags) + return ERR_PTR(-EOPNOTSUPP); + + return vma; +} + +static void attr_set_interval(struct amdgpu_svm_attr_range *range, + unsigned long start_page, + unsigned long last_page) +{ + range->it_node.start = start_page; + range->it_node.last = last_page; +} + +void amdgpu_svm_attr_set_default(struct amdgpu_svm *svm, + struct amdgpu_svm_attrs *attrs) +{ + attrs->preferred_loc = AMDGPU_SVM_LOCATION_UNDEFINED; + attrs->prefetch_loc = AMDGPU_SVM_LOCATION_UNDEFINED; + attrs->granularity = svm->default_granularity; + attrs->flags = AMDGPU_SVM_ATTR_BIT_HOST_ACCESS | AMDGPU_SVM_ATTR_BIT_COHERENT; + attrs->access = svm->xnack_enabled ? + AMDGPU_SVM_ACCESS_ALLOW_MIGRATE : AMDGPU_SVM_ACCESS_INACCESSIBLE; +} + +struct amdgpu_svm_attr_range * +amdgpu_svm_attr_find_locked(struct amdgpu_svm_attr_tree *attr_tree, + unsigned long page) +{ + struct interval_tree_node *node; + + node = interval_tree_iter_first(&attr_tree->tree, page, page); + if (node) + return container_of(node, struct amdgpu_svm_attr_range, it_node); + + return NULL; +} + +/** + * amdgpu_svm_attr_get_bounds_locked() - Find attributes or surrounding bounds + * @attr_tree: attribute tree to search. + * @page: page index to look up. + * @start_page: out, first page of the returned range or gap. + * @last_page: out, last page of the returned range or gap. + * + * If @page is covered by an attribute range, return that range and report its + * bounds. Otherwise return NULL and fill @start_page/@last_page with the bounds + * of the default attribute gap around @page, clamped by neighboring explicit + * ranges or [0, ULONG_MAX] when none. + * + * Return: the covering range, or NULL if @page falls in a gap. + */ +struct amdgpu_svm_attr_range * +amdgpu_svm_attr_get_bounds_locked(struct amdgpu_svm_attr_tree *attr_tree, + unsigned long page, + unsigned long *start_page, + unsigned long *last_page) +{ + struct amdgpu_svm_attr_range *attr_range; + struct interval_tree_node *node; + struct rb_node *rb; + + attr_range = amdgpu_svm_attr_find_locked(attr_tree, page); + if (attr_range) { + *start_page = amdgpu_svm_attr_start_page(attr_range); + *last_page = amdgpu_svm_attr_last_page(attr_range); + return attr_range; + } + + *start_page = 0; + *last_page = ULONG_MAX; + + if (page == ULONG_MAX) + return NULL; + + node = interval_tree_iter_first(&attr_tree->tree, page + 1, ULONG_MAX); + if (node) { + if (node->start > page) + *last_page = node->start - 1; + + rb = rb_prev(&node->rb); + if (rb) { + node = container_of(rb, struct interval_tree_node, rb); + if (node->last < page) + *start_page = node->last + 1; + } + } else { + rb = rb_last(&attr_tree->tree.rb_root); + + if (rb) { + node = container_of(rb, struct interval_tree_node, rb); + if (node->last < page) + *start_page = node->last + 1; + } + } + + return NULL; +} + +static bool attr_equal(const struct amdgpu_svm_attrs *a, + const struct amdgpu_svm_attrs *b) +{ + return a->flags == b->flags && + a->preferred_loc == b->preferred_loc && + a->prefetch_loc == b->prefetch_loc && + a->granularity == b->granularity && + a->access == b->access; +} + +struct amdgpu_svm_attr_range * +amdgpu_svm_attr_range_alloc(unsigned long start_page, + unsigned long last_page, + const struct amdgpu_svm_attrs *attrs) +{ + struct amdgpu_svm_attr_range *range; + + range = kzalloc(sizeof(*range), GFP_KERNEL); + if (!range) + return NULL; + + INIT_LIST_HEAD(&range->list); + attr_set_interval(range, start_page, last_page); + range->attrs = *attrs; + return range; +} + +void amdgpu_svm_attr_range_insert_locked(struct amdgpu_svm_attr_tree *attr_tree, + struct amdgpu_svm_attr_range *range) +{ + struct interval_tree_node *node; + struct amdgpu_svm_attr_range *next; + + lockdep_assert_held(&attr_tree->lock); + + /* + * Keep @range_list ordered by start page: insert before the first range + * starting at or after @range, then add to the interval tree. + */ + node = interval_tree_iter_first(&attr_tree->tree, amdgpu_svm_attr_start_page(range), + ULONG_MAX); + if (node) { + next = container_of(node, struct amdgpu_svm_attr_range, it_node); + list_add_tail(&range->list, &next->list); + } else { + list_add_tail(&range->list, &attr_tree->range_list); + } + + interval_tree_insert(&range->it_node, &attr_tree->tree); +} + +static void attr_remove_range_locked(struct amdgpu_svm_attr_tree *attr_tree, + struct amdgpu_svm_attr_range *range, + bool free_range) +{ + lockdep_assert_held(&attr_tree->lock); + + interval_tree_remove(&range->it_node, &attr_tree->tree); + list_del_init(&range->list); + if (free_range) + kfree(range); +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.h new file mode 100644 index 0000000000000..0f712536a5dc1 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.h @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright 2026 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_SVM_ATTR_H__ +#define __AMDGPU_SVM_ATTR_H__ + +#include <drm/amdgpu_drm.h> +#include <linux/interval_tree.h> +#include <linux/list.h> +#include <linux/mutex.h> +#include <linux/rbtree.h> +#include <linux/types.h> + +/* Internal SVM attribute bitmask flags mapped from UAPI ioctl definitions */ +#define AMDGPU_SVM_ATTR_BIT_HOST_ACCESS (1u << 0) +#define AMDGPU_SVM_ATTR_BIT_COHERENT (1u << 1) +#define AMDGPU_SVM_ATTR_BIT_EXT_COHERENT (1u << 2) +#define AMDGPU_SVM_ATTR_BIT_HIVE_LOCAL (1u << 3) +#define AMDGPU_SVM_ATTR_BIT_GPU_RO (1u << 4) +#define AMDGPU_SVM_ATTR_BIT_GPU_EXEC (1u << 5) +#define AMDGPU_SVM_ATTR_BIT_GPU_READ_MOSTLY (1u << 6) + +#define AMDGPU_SVM_PTE_FLAG_MASK \ + (AMDGPU_SVM_ATTR_BIT_COHERENT | AMDGPU_SVM_ATTR_BIT_EXT_COHERENT | \ + AMDGPU_SVM_ATTR_BIT_GPU_RO | AMDGPU_SVM_ATTR_BIT_GPU_EXEC) + +#define AMDGPU_SVM_MAPPING_FLAG_MASK \ + (AMDGPU_SVM_ATTR_BIT_HOST_ACCESS | AMDGPU_SVM_ATTR_BIT_HIVE_LOCAL | \ + AMDGPU_SVM_ATTR_BIT_GPU_READ_MOSTLY) + +/** + * struct amdgpu_svm_attrs - SVM attributes for an address range + * @preferred_loc: Preferred backing location, using AMDGPU_SVM_LOCATION_*. + * @prefetch_loc: Target location for prefetch requests, using + * AMDGPU_SVM_LOCATION_*. + * @flags: Internal AMDGPU_SVM_ATTR_BIT_* flags mapped from the UAPI. + * @granularity: Mapping granularity encoded as a page order. + * @access: CPU/GPU access policy from the SVM UAPI. + */ +struct amdgpu_svm_attrs { + int32_t preferred_loc; + int32_t prefetch_loc; + uint32_t flags; + uint32_t granularity; + enum amdgpu_ioctl_svm_access access; +}; + +/** + * struct amdgpu_svm_attr_range - a range of user attributes + * @it_node: interval tree node keyed by [start, last] page index. + * @list: links the range into amdgpu_svm_attr_tree.range_list in address order. + * @attrs: the attributes applied to this range. + */ +struct amdgpu_svm_attr_range { + struct interval_tree_node it_node; + struct list_head list; + struct amdgpu_svm_attrs attrs; +}; + +static inline unsigned long +amdgpu_svm_attr_start_page(const struct amdgpu_svm_attr_range *range) +{ + return range->it_node.start; +} + +static inline unsigned long +amdgpu_svm_attr_last_page(const struct amdgpu_svm_attr_range *range) +{ + return range->it_node.last; +} + +static inline unsigned long +amdgpu_svm_attr_start(const struct amdgpu_svm_attr_range *range) +{ + return range->it_node.start << PAGE_SHIFT; +} + +static inline unsigned long +amdgpu_svm_attr_end(const struct amdgpu_svm_attr_range *range) +{ + return (range->it_node.last + 1) << PAGE_SHIFT; +} + +struct amdgpu_svm; +struct mm_struct; +struct vm_area_struct; + +static inline bool +amdgpu_svm_attr_has_access(enum amdgpu_ioctl_svm_access access) +{ + return access == AMDGPU_SVM_ACCESS_ALLOW_MIGRATE || + access == AMDGPU_SVM_ACCESS_IN_PLACE; +} + +/** + * struct amdgpu_svm_attr_tree - per-SVM store of user attribute ranges + * @lock: protects @tree and @range_list. + * @tree: interval tree of struct amdgpu_svm_attr_range for fast lookup. + * @range_list: address-ordered list of the same ranges, kept in sync with + * @tree to allow ordered traversal during modification. + * @svm: back pointer to the owning SVM instance. + */ +struct amdgpu_svm_attr_tree { + struct mutex lock; + struct rb_root_cached tree; + struct list_head range_list; + struct amdgpu_svm *svm; +}; + +/** + * enum amdgpu_svm_attr_change_trigger - effects caused by an attribute change + * @AMDGPU_SVM_ATTR_TRIGGER_ACCESS_CHANGE: Access policy changed. + * @AMDGPU_SVM_ATTR_TRIGGER_PTE_FLAG_CHANGE: GPU PTE permission/cache bits changed. + * @AMDGPU_SVM_ATTR_TRIGGER_MAPPING_FLAG_CHANGE: Mapping policy bits changed. + * @AMDGPU_SVM_ATTR_TRIGGER_LOCATION_CHANGE: Preferred or prefetch location changed. + * @AMDGPU_SVM_ATTR_TRIGGER_GRANULARITY_CHANGE: Range granularity changed. + * @AMDGPU_SVM_ATTR_TRIGGER_PREFETCH: New attributes request a VRAM prefetch. + * + * Bitmask describing what an attribute update touched. It drives whether the + * existing GPU mappings must be invalidated and/or remapped. + */ +enum amdgpu_svm_attr_change_trigger { + AMDGPU_SVM_ATTR_TRIGGER_ACCESS_CHANGE = (1U << 0), + AMDGPU_SVM_ATTR_TRIGGER_PTE_FLAG_CHANGE = (1U << 1), + AMDGPU_SVM_ATTR_TRIGGER_MAPPING_FLAG_CHANGE = (1U << 2), + AMDGPU_SVM_ATTR_TRIGGER_LOCATION_CHANGE = (1U << 3), + AMDGPU_SVM_ATTR_TRIGGER_GRANULARITY_CHANGE = (1U << 4), + AMDGPU_SVM_ATTR_TRIGGER_PREFETCH = (1U << 5), +}; + +/* + * Attribute changes that require invalidating existing GPU mappings. + * A granularity-only change does not, so it is intentionally excluded. + */ +#define AMDGPU_SVM_ATTR_TRIGGER_NEED_INVALIDATE \ + (AMDGPU_SVM_ATTR_TRIGGER_ACCESS_CHANGE | \ + AMDGPU_SVM_ATTR_TRIGGER_PTE_FLAG_CHANGE | \ + AMDGPU_SVM_ATTR_TRIGGER_MAPPING_FLAG_CHANGE | \ + AMDGPU_SVM_ATTR_TRIGGER_LOCATION_CHANGE) + +struct amdgpu_svm_attr_range * +amdgpu_svm_attr_find_locked(struct amdgpu_svm_attr_tree *attr_tree, + unsigned long page); +struct amdgpu_svm_attr_range * +amdgpu_svm_attr_get_bounds_locked(struct amdgpu_svm_attr_tree *attr_tree, + unsigned long page, + unsigned long *start_page, + unsigned long *last_page); +void amdgpu_svm_attr_set_default(struct amdgpu_svm *svm, + struct amdgpu_svm_attrs *attrs); + +struct amdgpu_svm_attr_range * +amdgpu_svm_attr_range_alloc(unsigned long start_page, + unsigned long last_page, + const struct amdgpu_svm_attrs *attrs); +void amdgpu_svm_attr_range_insert_locked(struct amdgpu_svm_attr_tree *attr_tree, + struct amdgpu_svm_attr_range *range); +bool amdgpu_svm_attr_prefer_vram(const struct amdgpu_svm_attrs *attrs); +struct vm_area_struct *amdgpu_svm_check_vma(struct mm_struct *mm, + unsigned long addr); + +#endif /* __AMDGPU_SVM_ATTR_H__ */ -- 2.53.0
