Module: Mesa Branch: main Commit: 2b128c570b9848d9f9b8621278b2365ff690f170 URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=2b128c570b9848d9f9b8621278b2365ff690f170
Author: Jordan Justen <[email protected]> Date: Thu Mar 30 00:50:10 2023 -0700 intel/clflush: Add support for clflushopt instruction Rework: * Split clflushopt into a separate file as recommended by Ken. If we enable -mclflush on all driver source compilation, then gcc may insert uses of it on processors that don't support it. * Add uintptr_t casting to cpu_caps->cacheline usage Signed-off-by: Jordan Justen <[email protected]> Reviewed-by: Lionel Landwerlin <[email protected]> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/22379> --- src/intel/common/intel_clflushopt.c | 47 +++++++++++++++++++++++++++++++++++++ src/intel/common/intel_mem.c | 21 +++++++++++++++++ src/intel/common/intel_mem.h | 3 --- src/intel/common/meson.build | 15 +++++++++++- 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/src/intel/common/intel_clflushopt.c b/src/intel/common/intel_clflushopt.c new file mode 100644 index 00000000000..fb71c51c3ac --- /dev/null +++ b/src/intel/common/intel_clflushopt.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2023 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "intel_mem.h" +#include "util/u_cpu_detect.h" + +#ifndef HAVE___BUILTIN_IA32_CLFLUSHOPT +#error "Compiler doesn't support clflushopt!" +#endif + +void intel_clflushopt_range(void *start, size_t size); + +void +intel_clflushopt_range(void *start, size_t size) +{ + const struct util_cpu_caps_t *cpu_caps = util_get_cpu_caps(); + assert(cpu_caps->has_clflushopt); + assert(cpu_caps->cacheline > 0); + void *p = (void *) (((uintptr_t) start) & + ~((uintptr_t)cpu_caps->cacheline - 1)); + void *end = start + size; + + while (p < end) { + __builtin_ia32_clflushopt(p); + p += cpu_caps->cacheline; + } +} diff --git a/src/intel/common/intel_mem.c b/src/intel/common/intel_mem.c index f0ae944dd2b..c2e09a44371 100644 --- a/src/intel/common/intel_mem.c +++ b/src/intel/common/intel_mem.c @@ -22,10 +22,19 @@ */ #include "intel_mem.h" +#include "util/u_cpu_detect.h" #include <stdint.h> +#define CACHELINE_SIZE 64 +#define CACHELINE_MASK 63 + #ifdef SUPPORT_INTEL_INTEGRATED_GPUS + +#ifdef HAVE___BUILTIN_IA32_CLFLUSHOPT +void intel_clflushopt_range(void *start, size_t size); +#endif + static void intel_clflush_range(void *start, size_t size) { @@ -41,6 +50,13 @@ intel_clflush_range(void *start, size_t size) void intel_flush_range_no_fence(void *start, size_t size) { +#ifdef HAVE___BUILTIN_IA32_CLFLUSHOPT + const struct util_cpu_caps_t *cpu_caps = util_get_cpu_caps(); + if (cpu_caps->has_clflushopt) { + intel_clflushopt_range(start, size); + return; + } +#endif intel_clflush_range(start, size); } @@ -49,6 +65,11 @@ intel_flush_range(void *start, size_t size) { __builtin_ia32_mfence(); intel_flush_range_no_fence(start, size); +#ifdef HAVE___BUILTIN_IA32_CLFLUSHOPT + /* clflushopt doesn't include an mfence like clflush */ + if (util_get_cpu_caps()->has_clflushopt) + __builtin_ia32_mfence(); +#endif } void diff --git a/src/intel/common/intel_mem.h b/src/intel/common/intel_mem.h index 4ec4469647d..976694b30c2 100644 --- a/src/intel/common/intel_mem.h +++ b/src/intel/common/intel_mem.h @@ -30,9 +30,6 @@ extern "C" { #endif -#define CACHELINE_SIZE 64 -#define CACHELINE_MASK 63 - #ifdef SUPPORT_INTEL_INTEGRATED_GPUS void intel_flush_range(void *start, size_t size); void intel_flush_range_no_fence(void *start, size_t size); diff --git a/src/intel/common/meson.build b/src/intel/common/meson.build index bc71a27dd18..76b928e97ad 100644 --- a/src/intel/common/meson.build +++ b/src/intel/common/meson.build @@ -60,6 +60,19 @@ files_batch_decoder = files( 'intel_decoder.c', ) +libintel_common_links = [libisl] + +if with_clflushopt + libintel_clflushopt = static_library( + 'intel_clflushopt', + ['intel_clflushopt.c'], + include_directories : [inc_include, inc_src], + c_args : [no_override_init_args] + clflushopt_args, + gnu_symbol_visibility : 'hidden', + ) + libintel_common_links += libintel_clflushopt +endif + batch_decoder_dependencies = [] if not dep_expat.found() files_libintel_common += 'intel_batch_decoder_stub.c' @@ -74,7 +87,7 @@ libintel_common = static_library( include_directories : [inc_include, inc_src, inc_intel], c_args : [no_override_init_args], gnu_symbol_visibility : 'hidden', - link_with : [libisl], + link_with : libintel_common_links, dependencies : [batch_decoder_dependencies, dep_libdrm, dep_thread, idep_genxml, idep_mesautil, idep_intel_dev], )
