On systems with discontiguous NUMA node IDs, the GPU driver fails to load with the following errors:
[ 5.609767] amdgpu 0038:12:00.0: Error parsing VCRAT [ 5.609781] kfd kfd: Error adding device to topology [ 5.609848] kfd kfd: Error initializing KFD node In the current KFD code, proximity domains are created contiguously when generating the VCRAT. With discontiguous NUMA node IDs, this creates a mismatch between the proximity domain ID and the NUMA node ID. For example, if NUMA nodes 0, 2 and 3 are online, the current code assigns proximity domains 0, 1 and 2 to these nodes respectively, instead of domains 0, 2 and 3. The KFD topology uses the NUMA node and proximity domain information to create topology devices representing the CPU and memory resources associated with each NUMA node. This topology is exposed to userspace through sysfs, where the NUMA node and proximity domain information are used to identify the topology devices. The mismatch between the NUMA node ID and proximity domain therefore results in an inconsistent KFD topology and causes topology parsing to fail. Make the proximity domain ID correspond directly to the NUMA node ID by iterating through the complete NUMA node ID range up to the last online node. Account for every node ID when assigning proximity domains, while skipping resource entry creation for offline nodes. This preserves the NUMA-node-to-proximity-domain mapping and allows the KFD topology and its sysfs representation to correctly reflect discontiguous NUMA topologies. A NUMA node may also be CPU-less, memory-less, or both. Only create a Compute Unit CRAT entry if the node has CPUs, and only create a Memory CRAT entry if the node has memory. Additionally, update kfd_parse_subtype_mem() to use simd_count instead of cpu_cores_count to distinguish host nodes from GPU nodes. This ensures that memory on CPU-less host NUMA nodes is correctly classified as system memory (HSA_MEM_HEAP_TYPE_SYSTEM) rather than private GPU framebuffer memory. The placeholder domains created for offline node IDs have no IO links, so skip offline and CPU-less nodes when building the indirect GPU to CPU links. Co-developed-by: Donet Tom <[email protected]> Signed-off-by: Donet Tom <[email protected]> Signed-off-by: Dhruv B <[email protected]> --- drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 59 +++++++++++++---------- drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 3 ++ 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index 99b0c7f825fe..bebbc6715b2f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -1103,8 +1103,10 @@ static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem, mem->proximity_domain); list_for_each_entry(dev, device_list, list) { if (mem->proximity_domain == dev->proximity_domain) { - /* We're on GPU node */ - if (dev->node_props.cpu_cores_count == 0) { + if (dev->node_props.simd_count == 0) { + heap_type = HSA_MEM_HEAP_TYPE_SYSTEM; + } else { + /* We're on GPU node */ /* APU */ if (mem->visibility_type == 0) heap_type = @@ -1112,8 +1114,7 @@ static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem, /* dGPU */ else heap_type = mem->visibility_type; - } else - heap_type = HSA_MEM_HEAP_TYPE_SYSTEM; + } if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE) flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE; @@ -1899,7 +1900,7 @@ static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size) acpi_status status; struct crat_subtype_generic *sub_type_hdr; int avail_size = *size; - int numa_node_id; + int numa_node_id, last_node; #if defined(CONFIG_X86_64) && !defined(CONFIG_UML) uint32_t entries = 0; #endif @@ -1935,34 +1936,41 @@ static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size) crat_table->num_domains = 0; sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1); + last_node = find_last_bit(node_online_map.bits, MAX_NUMNODES); - for_each_online_node(numa_node_id) { - if (kfd_numa_node_to_apic_id(numa_node_id) == -1) + for (numa_node_id = 0; numa_node_id <= last_node; numa_node_id++) { + if (!node_online(numa_node_id)) { + crat_table->num_domains++; continue; + } /* Fill in Subtype: Compute Unit */ - ret = kfd_fill_cu_for_cpu(numa_node_id, &avail_size, - crat_table->num_domains, - (struct crat_subtype_computeunit *)sub_type_hdr); - if (ret < 0) - return ret; - crat_table->length += sub_type_hdr->length; - crat_table->total_entries++; + if (kfd_numa_node_to_apic_id(numa_node_id) != -1) { + ret = kfd_fill_cu_for_cpu(numa_node_id, &avail_size, + crat_table->num_domains, + (struct crat_subtype_computeunit *)sub_type_hdr); + if (ret < 0) + return ret; + crat_table->length += sub_type_hdr->length; + crat_table->total_entries++; - sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr + - sub_type_hdr->length); + sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr + + sub_type_hdr->length); + } /* Fill in Subtype: Memory */ - ret = kfd_fill_mem_info_for_cpu(numa_node_id, &avail_size, - crat_table->num_domains, - (struct crat_subtype_memory *)sub_type_hdr); - if (ret < 0) - return ret; - crat_table->length += sub_type_hdr->length; - crat_table->total_entries++; + if (node_state(numa_node_id, N_MEMORY)) { + ret = kfd_fill_mem_info_for_cpu(numa_node_id, &avail_size, + crat_table->num_domains, + (struct crat_subtype_memory *)sub_type_hdr); + if (ret < 0) + return ret; + crat_table->length += sub_type_hdr->length; + crat_table->total_entries++; - sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr + - sub_type_hdr->length); + sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr + + sub_type_hdr->length); + } /* Fill in Subtype: IO Link */ #if defined(CONFIG_X86_64) && !defined(CONFIG_UML) @@ -1982,7 +1990,6 @@ static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size) #else pr_info("IO link not available for non x86 platforms\n"); #endif - crat_table->num_domains++; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index b38d12f3da84..d2fbe73e57ef 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -1419,6 +1419,9 @@ static int kfd_create_indirect_link_prop(struct kfd_topology_device *kdev, int g struct kfd_iolink_properties, list); for (i = 0; i < num_cpu; i++) { + if (!node_online(i) || !node_state(i, N_CPU)) + continue; + /* CPU <--> GPU */ if (gpu_link->node_to == i) continue; -- 2.56.0
