https://gcc.gnu.org/g:7ad97ed9a70d18f518a2623af3113178fa6f8d79
commit 7ad97ed9a70d18f518a2623af3113178fa6f8d79 Author: supers1ngular <[email protected]> Date: Mon Jul 13 21:37:12 2026 -0700 libgomp: Runtime USM This patch introduces support for runtime USM, meaning that libgomp can now automatically detect and set Unified Shared Memory capabilities. It can be controlled via a new environment variable, 'GOMP_RUNTIME_USM', which can be set to 'enabled', 'auto', or 'disabled'. Documentation for the feature has also been added. libgomp/ChangeLog: * env.c (enum gomp_runtime_usm_t): New environment variable. (parse_rt_usm): Parse new environment variable. (initialize_env): Behavior handling for new variable. * libgomp-plugin.h (GOMP_OFFLOAD_CAP_AUTO_USM): New capability. (GOMP_OFFLOAD_get_dev_caps): New function for enumeration. * libgomp.h (enum gomp_device_num): Fix whitespace. (enum gomp_runtime_usm_t): Define. (struct gomp_device_descr): Add 'get_dev_caps_func'. * libgomp.texi: Update documentation. * plugin/plugin-gcn.c (get_memory_region): Fix whitespace. (process_reverse_offload): Ditto. (gcn_exec): Remove preprocessor elision. (is_apu_with_xnack): New function. (GCN_DEVICE): Query XNACK support. (GOMP_OFFLOAD_get_dev_caps): New function. * plugin/plugin-nvptx.c (GOMP_OFFLOAD_get_dev_caps): Ditto. * target.c (gomp_init_device): Logic for deciding USM behavior. (gomp_load_plugin_for_device): Get symbol. (gomp_target_init): New comment, and fix style. Diff: --- libgomp/env.c | 71 ++++++++++++++++++++++++++++++++++++++++++- libgomp/libgomp-plugin.h | 2 ++ libgomp/libgomp.h | 13 ++++++-- libgomp/libgomp.texi | 20 ++++++++++++ libgomp/plugin/plugin-gcn.c | 56 +++++++++++++++++++++++++++++++--- libgomp/plugin/plugin-nvptx.c | 19 ++++++++++++ libgomp/target.c | 29 +++++++++++++++--- 7 files changed, 199 insertions(+), 11 deletions(-) diff --git a/libgomp/env.c b/libgomp/env.c index 07de99b48977..3e5f557c5a29 100644 --- a/libgomp/env.c +++ b/libgomp/env.c @@ -121,6 +121,8 @@ int gomp_teams_thread_limit_var; bool gomp_display_affinity_var; char *gomp_affinity_format_var = "level %L thread %i affinity %A"; size_t gomp_affinity_format_len; +enum gomp_runtime_usm_t gomp_runtime_usm_var + = GOMP_RUNTIME_USM_DISABLED; char *goacc_device_type; int goacc_device_num; int goacc_default_dims[GOMP_DIM_MAX]; @@ -1057,6 +1059,53 @@ parse_spincount (const char *name, unsigned long long *pvalue) return false; } +static bool +parse_rt_usm (const char *name, enum gomp_runtime_usm_t *val) +{ + char *env, *end; + env = getenv (name); + if (env == NULL) + return false; + end = env; + while (isspace ((unsigned char) *env)) + ++env; + if (*env == '\0') + { + gomp_error ("Invalid value for environment variable %s", name); + return false; + } + enum gomp_runtime_usm_t store_state = GOMP_RUNTIME_USM_DISABLED; + if (strncasecmp (env, "disabled", 8) == 0) + { + store_state = GOMP_RUNTIME_USM_DISABLED; + end += 8; + } + else if (strncasecmp (env, "auto", 4) == 0) + { + store_state = GOMP_RUNTIME_USM_AUTO; + end += 4; + } + else if (strncasecmp (env, "enabled", 7) == 0) + { + store_state = GOMP_RUNTIME_USM_ENABLED; + end += 7; + } + else + { + gomp_error ("Invalid value for environment variable %s", name); + return false; + } + while (isspace ((unsigned char) *end)) + ++end; + if (*end != '\0') + { + gomp_error ("Invalid value for environment variable %s", name); + return false; + } + *val = store_state; + return true; +} + /* Parse a boolean value for environment variable NAME and store the result in VALUE. Return true if one was present and it was successfully parsed. */ @@ -1975,8 +2024,21 @@ omp_display_env (int verbose) fprintf (stderr, " [host] GOMP_SPINCOUNT = '%lu'\n", (unsigned long) gomp_spin_count_var); #endif + fputs (" [device] GOMP_RUNTIME_USM = '", stderr); + switch (gomp_runtime_usm_var) + { + case GOMP_RUNTIME_USM_DISABLED: + fputs ("DISABLED", stderr); + break; + case GOMP_RUNTIME_USM_AUTO: + fputs ("AUTO", stderr); + break; + case GOMP_RUNTIME_USM_ENABLED: + fputs ("ENABLED", stderr); + break; + } + fputs ("'\n", stderr); } - fputs ("OPENMP DISPLAY ENVIRONMENT END\n", stderr); } ialias (omp_display_env) @@ -2456,6 +2518,13 @@ initialize_env (void) if (gomp_throttled_spin_count_var > gomp_spin_count_var) gomp_throttled_spin_count_var = gomp_spin_count_var; + /* If we fail, we currently default to 'disabled', both here and on + initialization of 'gomp_runtime_usm_var'. Eventually, it may + bear consideration if we wish to rather default to 'auto' - but + until then, we do not wish to surprise. */ + if (!parse_rt_usm ("GOMP_RUNTIME_USM", &gomp_runtime_usm_var)) + gomp_runtime_usm_var = GOMP_RUNTIME_USM_DISABLED; + /* Not strictly environment related, but ordering constructors is tricky. */ pthread_attr_init (&gomp_thread_attr); diff --git a/libgomp/libgomp-plugin.h b/libgomp/libgomp-plugin.h index 5c770c80202e..051666617874 100644 --- a/libgomp/libgomp-plugin.h +++ b/libgomp/libgomp-plugin.h @@ -50,6 +50,7 @@ extern "C" { #define GOMP_OFFLOAD_CAP_NATIVE_EXEC (1 << 1) #define GOMP_OFFLOAD_CAP_OPENMP_400 (1 << 2) #define GOMP_OFFLOAD_CAP_OPENACC_200 (1 << 3) +#define GOMP_OFFLOAD_CAP_AUTO_USM (1 << 4) /* Type of offload target device. Keep in sync with include/gomp-constants.h. */ enum offload_target_type @@ -164,6 +165,7 @@ extern int GOMP_OFFLOAD_supported_teams_dim (int, int); extern int GOMP_OFFLOAD_supported_threads_dim (int, int); extern unsigned int GOMP_OFFLOAD_get_caps (void); +extern unsigned int GOMP_OFFLOAD_get_dev_caps (int); extern int GOMP_OFFLOAD_get_type (void); extern int GOMP_OFFLOAD_get_num_devices (unsigned int); extern bool GOMP_OFFLOAD_init_device (int); diff --git a/libgomp/libgomp.h b/libgomp/libgomp.h index af771be090e1..9f13f9e32c2a 100644 --- a/libgomp/libgomp.h +++ b/libgomp/libgomp.h @@ -494,7 +494,7 @@ enum gomp_device_num section 2.3.1. Those described as having one copy per task are stored within the structure; those described as having one copy for the whole program are (naturally) global variables. */ - + struct gomp_task_icv { unsigned long nthreads_var; @@ -590,6 +590,13 @@ enum gomp_target_offload_t GOMP_TARGET_OFFLOAD_DISABLED }; +enum gomp_runtime_usm_t +{ + GOMP_RUNTIME_USM_DISABLED, + GOMP_RUNTIME_USM_AUTO, + GOMP_RUNTIME_USM_ENABLED +}; + #define gomp_supported_active_levels UCHAR_MAX extern struct gomp_task_icv gomp_global_icv; @@ -620,6 +627,7 @@ extern const size_t gomp_omp_allocator_data_size; extern const struct gomp_default_icv gomp_default_icv_values; extern struct gomp_icv_list *gomp_initial_icv_list; extern struct gomp_offload_icv_list *gomp_offload_icv_list; +extern enum gomp_runtime_usm_t gomp_runtime_usm_var; extern int goacc_device_num; extern char *goacc_device_type; extern int goacc_default_dims[GOMP_DIM_MAX]; @@ -1396,7 +1404,7 @@ typedef struct acc_dispatch_t __typeof (GOMP_OFFLOAD_openacc_create_thread_data) *create_thread_data_func; __typeof (GOMP_OFFLOAD_openacc_destroy_thread_data) *destroy_thread_data_func; - + struct { /* Once created and put into the "active" list, asyncqueues are then never destructed and removed from the "active" list, other than if the TODO @@ -1469,6 +1477,7 @@ struct gomp_device_descr __typeof (GOMP_OFFLOAD_supported_teams_dim) *supported_teams_dim_func; __typeof (GOMP_OFFLOAD_supported_threads_dim) *supported_threads_dim_func; __typeof (GOMP_OFFLOAD_get_caps) *get_caps_func; + __typeof (GOMP_OFFLOAD_get_dev_caps) *get_dev_caps_func; __typeof (GOMP_OFFLOAD_get_type) *get_type_func; __typeof (GOMP_OFFLOAD_get_num_devices) *get_num_devices_func; __typeof (GOMP_OFFLOAD_init_device) *init_device_func; diff --git a/libgomp/libgomp.texi b/libgomp/libgomp.texi index e12d4488a5ab..526c1ea3ab9b 100644 --- a/libgomp/libgomp.texi +++ b/libgomp/libgomp.texi @@ -4569,6 +4569,7 @@ variable is not set. * OMP_WAIT_POLICY:: How waiting threads are handled * GOMP_CPU_AFFINITY:: Bind threads to specific CPUs * GOMP_DEBUG:: Enable debugging output +* GOMP_RUNTIME_USM:: Control runtime enabling of Unified Shared Memory * GOMP_STACKSIZE:: Set default thread stack size * GOMP_SPINCOUNT:: Set the busy-wait spin count * GOMP_RTEMS_THREAD_POOLS:: Set the RTEMS specific thread pools @@ -5199,6 +5200,25 @@ This is currently not specified in more detail, and subject to change. +@node GOMP_RUNTIME_USM +@section @env{GOMP_RUNTIME_USM} -- Control runtime enabling of Unified Shared Memory +@cindex Environment Variable +@table @asis +@item @emph{Description}: +Control the behavior of automatic Unified Shared Memory (USM) in the runtime. +The default is @code{disabled}, meaning that Unified Shared Memory will +only be used if specified by the @code{requires} clause. The variable +may also be set to @code{auto}, in which the runtime will decide whether +or not to use USM based on safety and performance considerations. More +specifically, it will only utilize USM if it detects XNACK support and +an APU. Lastly, one can set the variable to @code{enabled}, which will +unconditionally trust the device's self-report. This can potentially +cause instability, as some devices report capabilities that they may not +actually have. +@end table + + + @node GOMP_STACKSIZE @section @env{GOMP_STACKSIZE} -- Set default thread stack size @cindex Environment Variable diff --git a/libgomp/plugin/plugin-gcn.c b/libgomp/plugin/plugin-gcn.c index 2a532b1141dc..adf1691eca1d 100644 --- a/libgomp/plugin/plugin-gcn.c +++ b/libgomp/plugin/plugin-gcn.c @@ -1747,7 +1747,7 @@ get_memory_region (hsa_region_t region, hsa_region_t *retval, } /* Callback of hsa_agent_iterate_regions. - + Selects a kernargs memory region. */ static hsa_status_t @@ -2239,7 +2239,7 @@ process_reverse_offload (uint64_t fn, uint64_t mapnum, uint64_t hostaddrs, We print all entries from the last item printed to the next entry without a "written" flag. If the "final" flag is set then it'll continue right to the end. - + The print buffer is circular, but the from and to locations don't wrap when the buffer does, so the output limit is UINT_MAX. The target blocks on output when the buffer is full. */ @@ -3648,7 +3648,6 @@ gcn_exec (struct kernel_info *kernel, struct gomp_offload_session *session, /* }}} */ /* {{{ Generic Plugin API */ -#if 0 /* TODO: Use to enable self-mapping/USM automatically. */ /* FIXME: The auto-self-map feature depends on still mapping 'declare target' variables, even if ignoring all other mappings. Cf. PR 115279. */ @@ -3701,7 +3700,37 @@ is_integrated_apu (struct agent_info *agent, bool check_xnack) } return is_apu; } -#endif + +static bool +is_apu_with_xnack (struct agent_info *agent) +{ + /* We do not care for non-APU targets, at the moment. + If this ever changes, we can just elide the below check. + + Another point is that we are assuming the gfx902 does not + incur performance penalties, as it reports USM and XNACK+. + If it is later shown that enabling USM by default on this + platform incurs performance issues, then we need to add + an additional carveout here. */ + if (!is_integrated_apu (agent, false)) + return false; + enum { + HSACO_ATTR_UNSUPPORTED, + HSACO_ATTR_OFF, + HSACO_ATTR_ON, + HSACO_ATTR_ANY, + HSACO_ATTR_DEFAULT + }; + + switch (agent->device_isa) + { +#define GCN_DEVICE(name, NAME, ELF, ISA, XNACK, ...) \ + case ELF: return (XNACK == HSACO_ATTR_ANY); +#include "../../gcc/config/gcn/gcn-devices.def" + default: return false; + } + return false; +} /* Return the name of the accelerator, which is "gcn". */ @@ -3811,6 +3840,25 @@ GOMP_OFFLOAD_get_caps (void) | GOMP_OFFLOAD_CAP_OPENACC_200; } + + +unsigned int +GOMP_OFFLOAD_get_dev_caps (int ord) +{ + struct agent_info *agent = get_agent_info (ord); + bool claims_usm_p = 0; + unsigned int flags = GOMP_OFFLOAD_CAP_OPENMP_400 + | GOMP_OFFLOAD_CAP_OPENACC_200; + hsa_system_info_t type = HSA_AMD_SYSTEM_INFO_SVM_ACCESSIBLE_BY_DEFAULT; + hsa_status_t status = hsa_fns.hsa_system_get_info_fn (type, &claims_usm_p); + if (status != HSA_STATUS_SUCCESS) + GOMP_PLUGIN_error ("Could not fetch SVM_ACCESSIBLE_BY_DEFAULT"); + if (claims_usm_p) + flags |= GOMP_OFFLOAD_CAP_SHARED_MEM; + if (claims_usm_p && is_apu_with_xnack (agent)) + flags |= GOMP_OFFLOAD_CAP_AUTO_USM; + return flags; +} /* Identify as GCN accelerator. */ int diff --git a/libgomp/plugin/plugin-nvptx.c b/libgomp/plugin/plugin-nvptx.c index 0be45c51d35c..7e5e044eb74f 100644 --- a/libgomp/plugin/plugin-nvptx.c +++ b/libgomp/plugin/plugin-nvptx.c @@ -1434,6 +1434,25 @@ GOMP_OFFLOAD_get_caps (void) return GOMP_OFFLOAD_CAP_OPENACC_200 | GOMP_OFFLOAD_CAP_OPENMP_400; } +/* We duplicate functionality here for consistency, as this function + in plugin-gcn.c is for runtime USM. The gcn function determines + the case for auto-usm as well, however, here we only concern + ourselves for the enabled and disabled case. */ + +unsigned int +GOMP_OFFLOAD_get_dev_caps (int ord) +{ + unsigned int flags = GOMP_OFFLOAD_CAP_OPENACC_200 + | GOMP_OFFLOAD_CAP_OPENMP_400; + int pi = 0; + CUresult r; + r = CUDA_CALL_NOCHECK (cuDeviceGetAttribute, &pi, + CU_DEVICE_ATTRIBUTE_UNIFIED_ADDRESSING, ord); + if (r == CUDA_SUCCESS && pi) + flags |= GOMP_OFFLOAD_CAP_SHARED_MEM; + return flags; +} + int GOMP_OFFLOAD_get_type (void) { diff --git a/libgomp/target.c b/libgomp/target.c index c3d395103b46..859e2de18471 100644 --- a/libgomp/target.c +++ b/libgomp/target.c @@ -3359,6 +3359,24 @@ gomp_init_device (struct gomp_device_descr *devicep) gomp_fatal ("device initialization failed"); } + /* Now that we have initialized the device, we can evaluate auto USM. + If the user has explicitly requested USM, we can just skip the check. */ + if (!(devicep->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)) + { + devicep->capabilities = devicep->get_dev_caps_func (devicep->target_id); + if ((gomp_runtime_usm_var == GOMP_RUNTIME_USM_AUTO) + && !(devicep->capabilities & GOMP_OFFLOAD_CAP_AUTO_USM)) + devicep->capabilities &= ~GOMP_OFFLOAD_CAP_SHARED_MEM; + if (gomp_runtime_usm_var == GOMP_RUNTIME_USM_DISABLED) + devicep->capabilities &= ~GOMP_OFFLOAD_CAP_SHARED_MEM; + } + /* Peel off the GOMP_OFFLOAD_CAP_AUTO_USM, if it was set by the plugin, as + we no longer need it. */ + devicep->capabilities &= ~GOMP_OFFLOAD_CAP_AUTO_USM; + /* We can now set the requires mask based on the capabilities. + This makes it so the runtime treats it as if the user requested USM. */ + if (devicep->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) + omp_requires_mask |= GOMP_REQUIRES_UNIFIED_SHARED_MEMORY; /* Load to device all images registered by the moment. */ for (i = 0; i < num_offload_images; i++) { @@ -3372,6 +3390,7 @@ gomp_init_device (struct gomp_device_descr *devicep) /* Initialize OpenACC asynchronous queues. */ goacc_init_asyncqueues (devicep); + gomp_debug (0, "capabilities: %d\n", devicep->capabilities); devicep->state = GOMP_DEVICE_INITIALIZED; } @@ -6728,6 +6747,7 @@ gomp_load_plugin_for_device (struct gomp_device_descr *device, DLSYM_OPT (supported_threads_dim, supported_threads_dim); DLSYM_OPT (supported_teams_dim, supported_teams_dim); DLSYM (get_caps); + DLSYM (get_dev_caps); DLSYM (get_type); DLSYM (get_num_devices); DLSYM (init_device); @@ -6753,7 +6773,8 @@ gomp_load_plugin_for_device (struct gomp_device_descr *device, DLSYM (get_interop_str); DLSYM (get_interop_type_desc); } - + /* Returns offloading capabilities, but does not say anything about + auto USM yet. */ device->capabilities = device->get_caps_func (); device->session.size = 0; if (device->capabilities & GOMP_OFFLOAD_CAP_OPENMP_400) @@ -6921,10 +6942,10 @@ gomp_target_init (void) { /* Augment DEVICES and NUM_DEVICES. */ - /* If USM has been requested and is supported by all devices - of this type, set the capability accordingly. */ + /* If USM has been requested, set the capability. */ if (omp_requires_mask - & (GOMP_REQUIRES_UNIFIED_SHARED_MEMORY | GOMP_REQUIRES_SELF_MAPS)) + & (GOMP_REQUIRES_UNIFIED_SHARED_MEMORY + | GOMP_REQUIRES_SELF_MAPS)) current_device.capabilities |= GOMP_OFFLOAD_CAP_SHARED_MEM; devs = realloc (devs, (num_devs + new_num_devs)
