https://gcc.gnu.org/g:486b381b7ae6c18d70b81da30b3913155e1a7960
commit r17-3803-g486b381b7ae6c18d70b81da30b3913155e1a7960 Author: supers1ngular <[email protected]> Date: Tue Sep 1 00:37:19 2026 -0700 libgomp: Add Controls For Runtime USM This patch introduces the 'GOMP_RUNTIME_USM' environment variable, enabling one to control if the runtime is allowed to choose the memory paradigm to utilize. It can be set to 'disabled', 'auto', or 'enabled', which does nothing, conditionally enables the USM model if the device is an integrated GPU ('APU'), or attempts to use whatever model the hardware reports itself as capable of running, respectively. The documentation for the feature has also been included, and it has been added to the verbose output for 'omp_display_env'. libgomp/ChangeLog: * env.c (enum gomp_runtime_usm_t): New variable. (parse_rt_usm): New function to parse environment variable. (initialize_env): Enforce default 'disabled'. * libgomp.h (struct gomp_team_state): Whitespace. (enum gomp_device_num): Ditto. (enum gomp_runtime_usm_t): New enum. (gomp_init_device): Update signature. * libgomp.texi: Add relevant documentation. * oacc-init.c (acc_init_1): Update function call. (goacc_attach_host_thread_to_device): Whitespace. (acc_init): Ditto. (acc_set_device_type): Update function call. (acc_set_device_num): Ditto. (get_property_any): Ditto. (goacc_restore_bind): Whitespace. * target.c (resolve_device): Update function call. (gomp_load_image_to_device): Allow copying if the device capability is set. (gomp_init_device): Update signature, and add capability logic. Also add a debugging output. (gomp_page_locked_host_alloc): Update function call. Diff: --- libgomp/env.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++- libgomp/libgomp.h | 20 ++++++++++----- libgomp/libgomp.texi | 44 +++++++++++++++++++++++++++++++++ libgomp/oacc-init.c | 26 +++++++++---------- libgomp/target.c | 45 +++++++++++++++++++++++++++------ 5 files changed, 178 insertions(+), 27 deletions(-) diff --git a/libgomp/env.c b/libgomp/env.c index ff0044b42e20..1d659ad81bdb 100644 --- a/libgomp/env.c +++ b/libgomp/env.c @@ -121,6 +121,7 @@ int gomp_teams_thread_limit_var; bool gomp_display_affinity_var; char *gomp_affinity_format_var = "level %L thread %i affinity %A"; size_t gomp_affinity_format_len; +enum gomp_runtime_usm_t gomp_runtime_usm_var = GOMP_RUNTIME_USM_DISABLED; char *goacc_device_type; int goacc_device_num; int goacc_default_dims[GOMP_DIM_MAX]; @@ -1056,6 +1057,53 @@ parse_spincount (const char *name, unsigned long long *pvalue) return false; } +static bool +parse_rt_usm (const char *name, enum gomp_runtime_usm_t *val) +{ + char *env, *end; + env = getenv (name); + if (env == NULL) + return false; + end = env; + while (isspace ((unsigned char) *env)) + ++env; + if (*env == '\0') + { + gomp_error ("Invalid value for environment variable %s", name); + return false; + } + enum gomp_runtime_usm_t store_state = GOMP_RUNTIME_USM_DISABLED; + if (strncasecmp (env, "disabled", 8) == 0) + { + store_state = GOMP_RUNTIME_USM_DISABLED; + end += 8; + } + else if (strncasecmp (env, "auto", 4) == 0) + { + store_state = GOMP_RUNTIME_USM_AUTO; + end += 4; + } + else if (strncasecmp (env, "enabled", 7) == 0) + { + store_state = GOMP_RUNTIME_USM_ENABLED; + end += 7; + } + else + { + gomp_error ("Invalid value for environment variable %s", name); + return false; + } + while (isspace ((unsigned char) *end)) + ++end; + if (*end != '\0') + { + gomp_error ("Invalid value for environment variable %s", name); + return false; + } + *val = store_state; + return true; +} + /* Parse a boolean value for environment variable NAME and store the result in VALUE. Return true if one was present and it was successfully parsed. */ @@ -1974,8 +2022,21 @@ omp_display_env (int verbose) fprintf (stderr, " [host] GOMP_SPINCOUNT = '%lu'\n", (unsigned long) gomp_spin_count_var); #endif + fputs (" [all] GOMP_RUNTIME_USM = '", stderr); + switch (gomp_runtime_usm_var) + { + case GOMP_RUNTIME_USM_DISABLED: + fputs ("DISABLED", stderr); + break; + case GOMP_RUNTIME_USM_AUTO: + fputs ("AUTO", stderr); + break; + case GOMP_RUNTIME_USM_ENABLED: + fputs ("ENABLED", stderr); + break; + } + fputs ("'\n", stderr); } - fputs ("OPENMP DISPLAY ENVIRONMENT END\n", stderr); } ialias (omp_display_env) @@ -2455,6 +2516,13 @@ initialize_env (void) if (gomp_throttled_spin_count_var > gomp_spin_count_var) gomp_throttled_spin_count_var = gomp_spin_count_var; + /* If unset or we fail, we currently default to 'disabled', both here and on + initialization of 'gomp_runtime_usm_var'. Eventually, it may bear + consideration if we wish to rather default to 'auto' - but until then, + we do not wish to surprise. */ + if (!parse_rt_usm ("GOMP_RUNTIME_USM", &gomp_runtime_usm_var)) + gomp_runtime_usm_var = GOMP_RUNTIME_USM_DISABLED; + /* Not strictly environment related, but ordering constructors is tricky. */ pthread_attr_init (&gomp_thread_attr); diff --git a/libgomp/libgomp.h b/libgomp/libgomp.h index 19d4909316eb..c4bab1097368 100644 --- a/libgomp/libgomp.h +++ b/libgomp/libgomp.h @@ -33,7 +33,7 @@ that are part of the external ABI, and the lower case prefix "gomp" is used group items that are completely private to the library. */ -#ifndef LIBGOMP_H +#ifndef LIBGOMP_H #define LIBGOMP_H 1 #ifndef _LIBGOMP_CHECKING_ @@ -405,7 +405,7 @@ extern char gomp_workshare_struct_check1 extern char gomp_workshare_struct_check2 [offsetof (struct gomp_work_share, lock) == 64 ? 1 : -1]; -/* This structure contains all of the thread-local data associated with +/* This structure contains all of the thread-local data associated with a thread team. This is the data that must be saved when a thread encounters a nested PARALLEL construct. */ @@ -415,7 +415,7 @@ struct gomp_team_state struct gomp_team *team; /* This is the work share construct which this thread is currently - processing. Recall that with NOWAIT, not all threads may be + processing. Recall that with NOWAIT, not all threads may be processing the same construct. */ struct gomp_work_share *work_share; @@ -494,7 +494,7 @@ enum gomp_device_num section 2.3.1. Those described as having one copy per task are stored within the structure; those described as having one copy for the whole program are (naturally) global variables. */ - + struct gomp_task_icv { unsigned long nthreads_var; @@ -590,6 +590,13 @@ enum gomp_target_offload_t GOMP_TARGET_OFFLOAD_DISABLED }; +enum gomp_runtime_usm_t +{ + GOMP_RUNTIME_USM_DISABLED, + GOMP_RUNTIME_USM_AUTO, + GOMP_RUNTIME_USM_ENABLED +}; + #define gomp_supported_active_levels UCHAR_MAX extern struct gomp_task_icv gomp_global_icv; @@ -620,6 +627,7 @@ extern const size_t gomp_omp_allocator_data_size; extern const struct gomp_default_icv gomp_default_icv_values; extern struct gomp_icv_list *gomp_initial_icv_list; extern struct gomp_offload_icv_list *gomp_offload_icv_list; +extern enum gomp_runtime_usm_t gomp_runtime_usm_var; extern int goacc_device_num; extern char *goacc_device_type; extern int goacc_default_dims[GOMP_DIM_MAX]; @@ -1380,7 +1388,7 @@ typedef struct acc_dispatch_t __typeof (GOMP_OFFLOAD_openacc_create_thread_data) *create_thread_data_func; __typeof (GOMP_OFFLOAD_openacc_destroy_thread_data) *destroy_thread_data_func; - + struct { /* Once created and put into the "active" list, asyncqueues are then never destructed and removed from the "active" list, other than if the TODO @@ -1557,7 +1565,7 @@ extern struct target_mem_desc *goacc_map_vars (struct gomp_device_descr *, struct gomp_offload_session *); extern void goacc_unmap_vars (struct target_mem_desc *, bool, struct goacc_asyncqueue *); -extern void gomp_init_device (struct gomp_device_descr *); +extern void gomp_init_device (struct gomp_device_descr *, bool); extern bool gomp_fini_device (struct gomp_device_descr *); extern void gomp_unload_device (struct gomp_device_descr *); extern bool gomp_remove_var (struct gomp_device_descr *, splay_tree_key); diff --git a/libgomp/libgomp.texi b/libgomp/libgomp.texi index 564a1df2091a..82f0a65bd503 100644 --- a/libgomp/libgomp.texi +++ b/libgomp/libgomp.texi @@ -4611,6 +4611,7 @@ variable is not set. * OMP_WAIT_POLICY:: How waiting threads are handled * GOMP_CPU_AFFINITY:: Bind threads to specific CPUs * GOMP_DEBUG:: Enable debugging output +* GOMP_RUNTIME_USM:: Control runtime Unified Shared Memory application * GOMP_STACKSIZE:: Set default thread stack size * GOMP_SPINCOUNT:: Set the busy-wait spin count * GOMP_RTEMS_THREAD_POOLS:: Set the RTEMS specific thread pools @@ -5240,6 +5241,44 @@ This is currently not specified in more detail, and subject to change. +@node GOMP_RUNTIME_USM +@section @env{GOMP_RUNTIME_USM} -- Control runtime Unified Shared Memory application +@cindex Environment Variable +@table @asis +@item @emph{Description}: +Control the behavior of devices which can access host moemory, specifically +whether data mapping copies data or uses self-mapping, such that the device +directly accesses the corresponding host memory. Setting the environment +variable to @code{DISABLED} does not make use of Unified Shared Memory by +default. Setting it to @code{AUTO} conditionally enables Unified Shared +Memory on specific integrated GPUs (APUs, in the verbiage of AMD). In most +workloads on these types of GPUs, self-mapping is expected to reduce runtime +overhead. Lastly, setting it to @code{ENABLED} unconditionally attempts +to make use of Unified Shared Memory if the device claims to support it. Of +additional note, specifying the @code{unified_shared_memory} or +@code{self_maps} clauses with the @code{requires} directive causes that +behavior to take precedence over the behavior set by this environment +variable. For caveats around Unified Shared Memory, please consult +the relevant sections in @ref{Offload-Target Specifics}. + +When Unified Shared Memory is enabled, whether by a @code{requires} directive +or by this environment variable being set to @code{AUTO} or @code{ENABLED}, +global static variables that appear in a @code{declare target} directive are +not updated between device and host. That is, data copying under this model +for global static variables is ignored. + +OpenACC is unaffected by the value of this environment variable, however, +hybrid usage of OpenMP and OpenACC offloading results in the first used +framework taking precedent. As an example, if one uses OpenMP offloading and +then later OpenACC offloading, the capabilities the device possesses reflect +the original OpenMP model. + +@item @emph{See also}: +@ref{Offload-Target Specifics}, @ref{AMD Radeon} +@end table + + + @node GOMP_STACKSIZE @section @env{GOMP_STACKSIZE} -- Set default thread stack size @cindex Environment Variable @@ -7715,6 +7754,11 @@ The implementation remark: available devices, possibly leaving only the initial device (``host fallback''); consider using @code{ROCR_VISIBLE_DEVICES} to only enable devices supporting unified-shared memory. +@item APU (integrated GPU) systems that do not support XNACK, but make use of + Unified Shared Memory, may encounter issues, especially related to + device-side page-fault handling. +@item The MI100 (gfx908) and gfx902 are not supported with the + @code{GOMP_RUNTIME_USM} environment variable. @item The available stack size can be changed using the @code{GCN_STACK_SIZE} environment variable; the default is 32 kiB per thread. @item Low-latency memory (@code{omp_low_lat_mem_space}) is supported when the diff --git a/libgomp/oacc-init.c b/libgomp/oacc-init.c index b9e6d405b0b6..9c5b8774c9ca 100644 --- a/libgomp/oacc-init.c +++ b/libgomp/oacc-init.c @@ -323,7 +323,7 @@ acc_init_1 (acc_device_t d, acc_construct_t parent_construct, int implicit, gomp_fatal ("device already active"); } - gomp_init_device (acc_dev); + gomp_init_device (acc_dev, false); gomp_mutex_unlock (&acc_dev->lock); if (profiling_p) @@ -548,13 +548,13 @@ goacc_attach_host_thread_to_device (int ord) struct goacc_thread *thr = goacc_thread (); struct gomp_device_descr *acc_dev = NULL, *base_dev = NULL; int num_devices; - + if (thr && thr->dev && (thr->dev->target_id == ord || ord < 0)) return; - + if (ord < 0) ord = goacc_device_num; - + /* Decide which type of device to use. If the current thread has a device type already (e.g. set by acc_set_device_type), use that, else use the global default. */ @@ -565,15 +565,15 @@ goacc_attach_host_thread_to_device (int ord) assert (cached_base_dev); base_dev = cached_base_dev; } - + num_devices = base_dev->get_num_devices_func (); if (num_devices <= 0 || ord >= num_devices) acc_dev_num_out_of_range (acc_device_type (base_dev->type), ord, num_devices); - + if (!thr) thr = goacc_new_thread (); - + thr->base_dev = base_dev; thr->dev = acc_dev = &base_dev[ord]; thr->saved_bound_dev = NULL; @@ -704,7 +704,7 @@ acc_init (acc_device_t d) gomp_mutex_lock (&acc_device_lock); cached_base_dev = acc_init_1 (d, acc_construct_runtime_api, 0, -1); gomp_mutex_unlock (&acc_device_lock); - + goacc_attach_host_thread_to_device (-1); } @@ -779,7 +779,7 @@ acc_set_device_type (acc_device_t d) gomp_mutex_lock (&acc_dev->lock); if (acc_dev->state == GOMP_DEVICE_UNINITIALIZED) - gomp_init_device (acc_dev); + gomp_init_device (acc_dev, false); gomp_mutex_unlock (&acc_dev->lock); gomp_mutex_unlock (&acc_device_lock); @@ -925,14 +925,14 @@ acc_set_device_num (int ord, acc_device_t d) gomp_mutex_lock (&acc_dev->lock); if (acc_dev->state == GOMP_DEVICE_UNINITIALIZED) - gomp_init_device (acc_dev); + gomp_init_device (acc_dev, false); gomp_mutex_unlock (&acc_dev->lock); gomp_mutex_unlock (&acc_device_lock); goacc_attach_host_thread_to_device (ord); } - + goacc_device_num = ord; } @@ -958,7 +958,7 @@ get_property_any (int ord, acc_device_t d, acc_device_property_t prop) gomp_mutex_lock (&dev->lock); if (dev->state == GOMP_DEVICE_UNINITIALIZED) - gomp_init_device (dev); + gomp_init_device (dev, false); gomp_mutex_unlock (&dev->lock); gomp_mutex_unlock (&acc_device_lock); @@ -1060,7 +1060,7 @@ goacc_restore_bind (void) } /* This is called from any OpenACC support function that may need to implicitly - initialize the libgomp runtime, either globally or from a new host thread. + initialize the libgomp runtime, either globally or from a new host thread. On exit "goacc_thread" will return a valid & populated thread block. */ attribute_hidden void diff --git a/libgomp/target.c b/libgomp/target.c index 2abb364244e7..cd41c6552a2d 100644 --- a/libgomp/target.c +++ b/libgomp/target.c @@ -187,7 +187,7 @@ resolve_device (int device_id, bool remapped) gomp_mutex_lock (&devices[device_id].lock); if (devices[device_id].state == GOMP_DEVICE_UNINITIALIZED) - gomp_init_device (&devices[device_id]); + gomp_init_device (&devices[device_id], true); else if (devices[device_id].state == GOMP_DEVICE_FINALIZED) { gomp_mutex_unlock (&devices[device_id].lock); @@ -2826,8 +2826,9 @@ gomp_load_image_to_device (struct gomp_device_descr *devicep, unsigned version, array++; if (is_link_var - && (omp_requires_mask - & (GOMP_REQUIRES_UNIFIED_SHARED_MEMORY | GOMP_REQUIRES_SELF_MAPS))) + && ((omp_requires_mask + & (GOMP_REQUIRES_UNIFIED_SHARED_MEMORY | GOMP_REQUIRES_SELF_MAPS)) + || (devicep->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM))) gomp_copy_host2dev (devicep, NULL, (void *) target_var->start, &k->host_start, sizeof (void *), false, NULL); } @@ -3115,10 +3116,11 @@ GOMP_offload_unregister (const void *host_table, int target_type, } /* This function initializes the target device, specified by DEVICEP. DEVICEP - must be locked on entry, and remains locked on return. */ + must be locked on entry, and remains locked on return. 'openmp_p' must be + set if called from an OpenMP context, and must not be set for OpenACC. */ attribute_hidden void -gomp_init_device (struct gomp_device_descr *devicep) +gomp_init_device (struct gomp_device_descr *devicep, bool openmp_p) { int i; if (!devicep->init_device_func (devicep->target_id)) @@ -3127,6 +3129,33 @@ gomp_init_device (struct gomp_device_descr *devicep) gomp_fatal ("device initialization failed"); } + /* Evaluate device capabilities and determine if we want USM enabled + in accordance with the environment variable 'GOMP_RUNTIME_USM'. + If we explicitly requested USM, skip the check. + We also check to make sure we're in OpenMP - if we are in OpenACC, + we need to skip this. + See also gomp_target_init for the generic requires USM/self_maps + handling. */ + if (!(omp_requires_mask + & (GOMP_REQUIRES_UNIFIED_SHARED_MEMORY | GOMP_REQUIRES_SELF_MAPS)) + && openmp_p) + { + /* If we have not used the 'requires' directive, then we + can safely modify the device capabilities without losing that + information. We should not be in here if we have required + USM or self maps, anyways. */ + devicep->capabilities |= devicep->get_dev_caps_func (devicep->target_id); + if ((gomp_runtime_usm_var == GOMP_RUNTIME_USM_AUTO) + && !(devicep->capabilities & GOMP_OFFLOAD_CAP_APU_SHARED_MEM)) + devicep->capabilities &= ~GOMP_OFFLOAD_CAP_SHARED_MEM; + if (gomp_runtime_usm_var == GOMP_RUNTIME_USM_DISABLED) + devicep->capabilities &= ~GOMP_OFFLOAD_CAP_SHARED_MEM; + } + /* Since we only query runtime device capabilities from the OpenMP side, + OpenACC initialization will never hit that path, meaning we don't have + to worry about accidentally automatically enabling USM de facto. */ + + /* Load to device all images registered by the moment. */ for (i = 0; i < num_offload_images; i++) { @@ -3140,6 +3169,7 @@ gomp_init_device (struct gomp_device_descr *devicep) /* Initialize OpenACC asynchronous queues. */ goacc_init_asyncqueues (devicep); + gomp_debug (0, "capabilities: %d\n", devicep->capabilities); devicep->state = GOMP_DEVICE_INITIALIZED; } @@ -4976,7 +5006,7 @@ gomp_page_locked_host_alloc (void **ptr, size_t size) { gomp_mutex_lock (&device->lock); if (device->state == GOMP_DEVICE_UNINITIALIZED) - gomp_init_device (device); + gomp_init_device (device, true); else if (device->state == GOMP_DEVICE_FINALIZED) { gomp_mutex_unlock (&device->lock); @@ -6495,7 +6525,8 @@ gomp_target_init (void) /* Skip the device (= remove from available devices) if a requirement cannot be fulfilled. For USM/self_maps, set SHARED_MEM capability for the - device. */ + device. See also GOMP_RUNTIME_USM handling in + gomp_init_device. */ int dev_caps = current_device.capabilities; if (current_device.get_dev_caps_func) dev_caps |= current_device.get_dev_caps_func (i);
