https://github.com/wenju-he created https://github.com/llvm/llvm-project/pull/168329
None >From 95a742a639368d501f2caf9afd32c72090851522 Mon Sep 17 00:00:00 2001 From: Wenju He <[email protected]> Date: Mon, 17 Nov 2025 08:51:36 +0100 Subject: [PATCH] [libclc] Add atomic_init, atomic_flag_clear and atomic_flag_test_and_set --- .../clc/atomic/clc_atomic_flag_clear.h | 24 +++++++ .../clc/atomic/clc_atomic_flag_test_and_set.h | 24 +++++++ libclc/clc/lib/generic/SOURCES | 2 + .../generic/atomic/clc_atomic_flag_clear.cl | 25 +++++++ .../atomic/clc_atomic_flag_test_and_set.cl | 25 +++++++ .../clc/opencl/atomic/atomic_flag_clear.h | 46 +++++++++++++ .../opencl/atomic/atomic_flag_test_and_set.h | 50 ++++++++++++++ .../include/clc/opencl/atomic/atomic_init.h | 22 ++++++ .../include/clc/opencl/atomic/atomic_init.inc | 46 +++++++++++++ libclc/opencl/include/clc/opencl/types.h | 48 +++++++++++++ libclc/opencl/include/clc/opencl/utils.h | 35 ++++++++++ libclc/opencl/lib/generic/SOURCES | 3 + .../lib/generic/atomic/atomic_flag_clear.cl | 61 +++++++++++++++++ .../atomic/atomic_flag_test_and_set.cl | 67 +++++++++++++++++++ .../opencl/lib/generic/atomic/atomic_init.cl | 17 +++++ .../opencl/lib/generic/atomic/atomic_init.inc | 48 +++++++++++++ 16 files changed, 543 insertions(+) create mode 100644 libclc/clc/include/clc/atomic/clc_atomic_flag_clear.h create mode 100644 libclc/clc/include/clc/atomic/clc_atomic_flag_test_and_set.h create mode 100644 libclc/clc/lib/generic/atomic/clc_atomic_flag_clear.cl create mode 100644 libclc/clc/lib/generic/atomic/clc_atomic_flag_test_and_set.cl create mode 100644 libclc/opencl/include/clc/opencl/atomic/atomic_flag_clear.h create mode 100644 libclc/opencl/include/clc/opencl/atomic/atomic_flag_test_and_set.h create mode 100644 libclc/opencl/include/clc/opencl/atomic/atomic_init.h create mode 100644 libclc/opencl/include/clc/opencl/atomic/atomic_init.inc create mode 100644 libclc/opencl/include/clc/opencl/types.h create mode 100644 libclc/opencl/include/clc/opencl/utils.h create mode 100644 libclc/opencl/lib/generic/atomic/atomic_flag_clear.cl create mode 100644 libclc/opencl/lib/generic/atomic/atomic_flag_test_and_set.cl create mode 100644 libclc/opencl/lib/generic/atomic/atomic_init.cl create mode 100644 libclc/opencl/lib/generic/atomic/atomic_init.inc diff --git a/libclc/clc/include/clc/atomic/clc_atomic_flag_clear.h b/libclc/clc/include/clc/atomic/clc_atomic_flag_clear.h new file mode 100644 index 0000000000000..40a3295dcdd9a --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_flag_clear.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_FLAG_CLEAR_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_FLAG_CLEAR_H__ + +#include <clc/internal/clc.h> + +#define __CLC_DECLARE_ATOMIC_FLAG_CLEAR(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL void __clc_atomic_flag_clear( \ + volatile ADDRSPACE int *Ptr, int MemoryOrder, int MemoryScope); + +__CLC_DECLARE_ATOMIC_FLAG_CLEAR(global) +__CLC_DECLARE_ATOMIC_FLAG_CLEAR(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DECLARE_ATOMIC_FLAG_CLEAR() +#endif + +#endif // __CLC_ATOMIC_CLC_ATOMIC_FLAG_CLEAR_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_flag_test_and_set.h b/libclc/clc/include/clc/atomic/clc_atomic_flag_test_and_set.h new file mode 100644 index 0000000000000..055852b8d7fed --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_flag_test_and_set.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_FLAG_TEST_AND_SET_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_FLAG_TEST_AND_SET_H__ + +#include <clc/internal/clc.h> + +#define __CLC_DECLARE_ATOMIC_FLAG_TEST_AND_SET(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL bool __clc_atomic_flag_test_and_set( \ + volatile ADDRSPACE int *Ptr, int MemoryOrder, int MemoryScope); + +__CLC_DECLARE_ATOMIC_FLAG_TEST_AND_SET(global) +__CLC_DECLARE_ATOMIC_FLAG_TEST_AND_SET(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DECLARE_ATOMIC_FLAG_TEST_AND_SET() +#endif + +#endif // __CLC_ATOMIC_CLC_ATOMIC_FLAG_TEST_AND_SET_H__ diff --git a/libclc/clc/lib/generic/SOURCES b/libclc/clc/lib/generic/SOURCES index ee4f771799e8e..64fc6b4827a1a 100644 --- a/libclc/clc/lib/generic/SOURCES +++ b/libclc/clc/lib/generic/SOURCES @@ -9,6 +9,8 @@ atomic/clc_atomic_fetch_min.cl atomic/clc_atomic_fetch_or.cl atomic/clc_atomic_fetch_sub.cl atomic/clc_atomic_fetch_xor.cl +atomic/clc_atomic_flag_clear.cl +atomic/clc_atomic_flag_test_and_set.cl atomic/clc_atomic_inc.cl atomic/clc_atomic_load.cl atomic/clc_atomic_store.cl diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_flag_clear.cl b/libclc/clc/lib/generic/atomic/clc_atomic_flag_clear.cl new file mode 100644 index 0000000000000..7cc245dfcb6a3 --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_flag_clear.cl @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_flag_clear.h> +#include <clc/atomic/clc_atomic_store.h> + +#define __CLC_ATOMIC_FLAG_FALSE 0 + +#define __CLC_DEFINE_ATOMIC_FLAG_CLEAR(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF void __clc_atomic_flag_clear( \ + volatile ADDRSPACE int *Ptr, int MemoryOrder, int MemoryScope) { \ + __clc_atomic_store(Ptr, __CLC_ATOMIC_FLAG_FALSE, MemoryOrder, \ + MemoryScope); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_CLEAR(global) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DEFINE_ATOMIC_FLAG_CLEAR() +#endif diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_flag_test_and_set.cl b/libclc/clc/lib/generic/atomic/clc_atomic_flag_test_and_set.cl new file mode 100644 index 0000000000000..a34e0328a7c8d --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_flag_test_and_set.cl @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_exchange.h> +#include <clc/atomic/clc_atomic_flag_test_and_set.h> + +#define __CLC_ATOMIC_FLAG_TRUE 1 + +#define __CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF bool __clc_atomic_flag_test_and_set( \ + volatile ADDRSPACE int *Ptr, int MemoryOrder, int MemoryScope) { \ + return (bool)__clc_atomic_exchange(Ptr, __CLC_ATOMIC_FLAG_TRUE, \ + MemoryOrder, MemoryScope); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(global) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET() +#endif diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_flag_clear.h b/libclc/opencl/include/clc/opencl/atomic/atomic_flag_clear.h new file mode 100644 index 0000000000000..2fcd3eef43a65 --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_flag_clear.h @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_CLEAR_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_CLEAR_H__ + +#include <clc/opencl/opencl-base.h> +#include <clc/opencl/types.h> + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) +_CLC_OVERLOAD _CLC_DECL void atomic_flag_clear(volatile __global atomic_flag *); +_CLC_OVERLOAD _CLC_DECL void atomic_flag_clear(volatile __local atomic_flag *); +#if defined(__opencl_c_generic_address_space) +_CLC_OVERLOAD _CLC_DECL void atomic_flag_clear(volatile atomic_flag *); +#endif // defined(__opencl_c_generic_address_space) +#endif + +#if defined(__opencl_c_atomic_scope_device) +_CLC_OVERLOAD _CLC_DECL void +atomic_flag_clear_explicit(volatile __global atomic_flag *, memory_order); +_CLC_OVERLOAD _CLC_DECL void +atomic_flag_clear_explicit(volatile __local atomic_flag *, memory_order); +#if defined(__opencl_c_generic_address_space) +_CLC_OVERLOAD _CLC_DECL void atomic_flag_clear_explicit(volatile atomic_flag *, + memory_order); +#endif // defined(__opencl_c_generic_address_space) +#endif + +_CLC_OVERLOAD _CLC_DECL void +atomic_flag_clear_explicit(volatile __global atomic_flag *, memory_order, + memory_scope); +_CLC_OVERLOAD _CLC_DECL void +atomic_flag_clear_explicit(volatile __local atomic_flag *, memory_order, + memory_scope); +#if defined(__opencl_c_generic_address_space) +_CLC_OVERLOAD _CLC_DECL void +atomic_flag_clear_explicit(volatile atomic_flag *, memory_order, memory_scope); +#endif // defined(__opencl_c_generic_address_space) + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_CLEAR_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_flag_test_and_set.h b/libclc/opencl/include/clc/opencl/atomic/atomic_flag_test_and_set.h new file mode 100644 index 0000000000000..6e3a8e403d5da --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_flag_test_and_set.h @@ -0,0 +1,50 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_TEST_AND_SET_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_TEST_AND_SET_H__ + +#include <clc/opencl/opencl-base.h> +#include <clc/opencl/types.h> + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set(volatile __global atomic_flag *); +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set(volatile __local atomic_flag *); +#if defined(__opencl_c_generic_address_space) +_CLC_OVERLOAD _CLC_DECL bool atomic_flag_test_and_set(volatile atomic_flag *); +#endif // defined(__opencl_c_generic_address_space) +#endif + +#if defined(__opencl_c_atomic_scope_device) +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set_explicit(volatile __global atomic_flag *, + memory_order); +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set_explicit(volatile __local atomic_flag *, memory_order); +#if defined(__opencl_c_generic_address_space) +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set_explicit(volatile atomic_flag *, memory_order); +#endif // defined(__opencl_c_generic_address_space) +#endif + +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set_explicit(volatile __global atomic_flag *, memory_order, + memory_scope); +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set_explicit(volatile __local atomic_flag *, memory_order, + memory_scope); +#if defined(__opencl_c_generic_address_space) +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set_explicit(volatile atomic_flag *, memory_order, + memory_scope); +#endif // defined(__opencl_c_generic_address_space) + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_TEST_AND_SET_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_init.h b/libclc/opencl/include/clc/opencl/atomic/atomic_init.h new file mode 100644 index 0000000000000..355c1453a28cb --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_init.h @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_INIT_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_INIT_H__ + +#include <clc/opencl/opencl-base.h> + +#define __CLC_BODY <atomic_init.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <atomic_init.inc> +#include <clc/math/gentype.inc> + +#undef __CLC_FUNCTION + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_INIT_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_init.inc b/libclc/opencl/include/clc/opencl/atomic/atomic_init.inc new file mode 100644 index 0000000000000..22a303127dfd6 --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_init.inc @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifdef __CLC_SCALAR + +#if defined(__opencl_c_fp64) && (defined(cl_khr_int64_base_atomics) && \ + defined(cl_khr_int64_extended_atomics)) +#define __CLC_HAVE_64_ATOMIC +#endif +#if defined(__CLC_FPSIZE) && \ + (__CLC_FPSIZE < 64 || defined(__CLC_HAVE_64_ATOMIC)) +#define __CLC_HAVE_FP_ATOMIC +#endif +#if defined(__CLC_GENSIZE) && \ + ((__CLC_GENSIZE == 32) || \ + (__CLC_GENSIZE == 64 && defined(__CLC_HAVE_64_ATOMIC))) +#define __CLC_HAVE_INT_ATOMIC +#endif +#if defined(__CLC_HAVE_FP_ATOMIC) || defined(__CLC_HAVE_INT_ATOMIC) + +#define __CLC_ATOMIC_GENTYPE __CLC_XCONCAT(atomic_, __CLC_GENTYPE) + +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL void atomic_init( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value); + +__CLC_DECL_ATOMIC(global) +__CLC_DECL_ATOMIC(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DECL_ATOMIC() +#endif + +#undef __CLC_DECL_ATOMIC + +#endif // __CLC_HAVE_FP_ATOMIC || __CLC_HAVE_INT_ATOMIC + +#undef __CLC_HAVE_INT_ATOMIC +#undef __CLC_HAVE_FP_ATOMIC +#undef __CLC_HAVE_64_ATOMIC + +#endif // __CLC_SCALAR diff --git a/libclc/opencl/include/clc/opencl/types.h b/libclc/opencl/include/clc/opencl/types.h new file mode 100644 index 0000000000000..b1be88f21bdaa --- /dev/null +++ b/libclc/opencl/include/clc/opencl/types.h @@ -0,0 +1,48 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_TYPES_H__ +#define __CLC_OPENCL_TYPES_H__ + +// Copied from clang/lib/Headers/opencl-c-base.h + +typedef enum memory_scope { + memory_scope_work_item = __OPENCL_MEMORY_SCOPE_WORK_ITEM, + memory_scope_work_group = __OPENCL_MEMORY_SCOPE_WORK_GROUP, + memory_scope_device = __OPENCL_MEMORY_SCOPE_DEVICE, +#if defined(__opencl_c_atomic_scope_all_devices) + memory_scope_all_svm_devices = __OPENCL_MEMORY_SCOPE_ALL_SVM_DEVICES, +#if (__OPENCL_C_VERSION__ >= CL_VERSION_3_0 || __OPENCL_CPP_VERSION__ >= 202100) + memory_scope_all_devices = memory_scope_all_svm_devices, +#endif // (__OPENCL_C_VERSION__ >= CL_VERSION_3_0 || __OPENCL_CPP_VERSION__ >= + // 202100) +#endif // defined(__opencl_c_atomic_scope_all_devices) +/** + * Subgroups have different requirements on forward progress, so just test + * all the relevant macros. + * CL 3.0 sub-groups "they are not guaranteed to make independent forward + * progress" KHR subgroups "Subgroups within a workgroup are independent, make + * forward progress with respect to each other" + */ +#if defined(cl_intel_subgroups) || defined(cl_khr_subgroups) || \ + defined(__opencl_c_subgroups) + memory_scope_sub_group = __OPENCL_MEMORY_SCOPE_SUB_GROUP +#endif +} memory_scope; + +typedef enum memory_order { + memory_order_relaxed = __ATOMIC_RELAXED, + memory_order_acquire = __ATOMIC_ACQUIRE, + memory_order_release = __ATOMIC_RELEASE, + memory_order_acq_rel = __ATOMIC_ACQ_REL, +#if defined(__opencl_c_atomic_order_seq_cst) + memory_order_seq_cst = __ATOMIC_SEQ_CST +#endif +} memory_order; + +#endif // __CLC_OPENCL_TYPES_H__ diff --git a/libclc/opencl/include/clc/opencl/utils.h b/libclc/opencl/include/clc/opencl/utils.h new file mode 100644 index 0000000000000..42b948b8d30d2 --- /dev/null +++ b/libclc/opencl/include/clc/opencl/utils.h @@ -0,0 +1,35 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_UTILS_H__ +#define __CLC_OPENCL_UTILS_H__ + +#include <clc/internal/clc.h> +#include <clc/opencl/types.h> + +// INTEL_FEATURE_PISA +static _CLC_INLINE int __opencl_get_clang_memory_scope(memory_scope scope) { + switch (scope) { + case __OPENCL_MEMORY_SCOPE_WORK_ITEM: + return __MEMORY_SCOPE_SINGLE; +#if defined(cl_intel_subgroups) || defined(cl_khr_subgroups) || \ + defined(__opencl_c_subgroups) + case __OPENCL_MEMORY_SCOPE_SUB_GROUP: + return __MEMORY_SCOPE_WVFRNT; +#endif + case __OPENCL_MEMORY_SCOPE_WORK_GROUP: + return __MEMORY_SCOPE_WRKGRP; + case __OPENCL_MEMORY_SCOPE_DEVICE: + return __MEMORY_SCOPE_DEVICE; + default: + return __MEMORY_SCOPE_SYSTEM; + } +} +// end INTEL_FEATURE_PISA + +#endif // __CLC_OPENCL_UTILS_H__ diff --git a/libclc/opencl/lib/generic/SOURCES b/libclc/opencl/lib/generic/SOURCES index 61757efbcaad7..94a333e765b18 100644 --- a/libclc/opencl/lib/generic/SOURCES +++ b/libclc/opencl/lib/generic/SOURCES @@ -22,7 +22,10 @@ atomic/atomic_fetch_min.cl atomic/atomic_fetch_or.cl atomic/atomic_fetch_sub.cl atomic/atomic_fetch_xor.cl +atomic/atomic_flag_clear.cl +atomic/atomic_flag_test_and_set.cl atomic/atomic_inc.cl +atomic/atomic_init.cl atomic/atomic_load.cl atomic/atomic_max.cl atomic/atomic_min.cl diff --git a/libclc/opencl/lib/generic/atomic/atomic_flag_clear.cl b/libclc/opencl/lib/generic/atomic/atomic_flag_clear.cl new file mode 100644 index 0000000000000..d7c927a0fe4e8 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_flag_clear.cl @@ -0,0 +1,61 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_flag_clear.h> +#include <clc/opencl/atomic/atomic_flag_clear.h> +#include <clc/opencl/utils.h> + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#define __CLC_DEFINE_ATOMIC_FLAG_CLEAR(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF void atomic_flag_clear( \ + volatile ADDRSPACE atomic_flag *object) { \ + __clc_atomic_flag_clear((volatile ADDRSPACE int *)object, \ + __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_CLEAR(global) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR(local) +#if defined(__opencl_c_generic_address_space) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR() +#endif + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) + +#if defined(__opencl_c_atomic_scope_device) + +#define __CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF void atomic_flag_clear_explicit( \ + volatile ADDRSPACE atomic_flag *object, memory_order order) { \ + __clc_atomic_flag_clear((volatile ADDRSPACE int *)object, order, \ + __MEMORY_SCOPE_DEVICE); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER(global) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER(local) +#if defined(__opencl_c_generic_address_space) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER() +#endif + +#endif // defined(__opencl_c_atomic_scope_device) + +#define __CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER_SCOPE(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF void atomic_flag_clear_explicit( \ + volatile ADDRSPACE atomic_flag *object, memory_order order, \ + memory_scope scope) { \ + __clc_atomic_flag_clear((volatile ADDRSPACE int *)object, order, \ + __opencl_get_clang_memory_scope(scope)); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER_SCOPE(global) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER_SCOPE(local) +#if defined(__opencl_c_generic_address_space) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER_SCOPE() +#endif diff --git a/libclc/opencl/lib/generic/atomic/atomic_flag_test_and_set.cl b/libclc/opencl/lib/generic/atomic/atomic_flag_test_and_set.cl new file mode 100644 index 0000000000000..4369a380fabfb --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_flag_test_and_set.cl @@ -0,0 +1,67 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_flag_test_and_set.h> +#include <clc/opencl/atomic/atomic_flag_test_and_set.h> +#include <clc/opencl/utils.h> + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#define __CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF bool atomic_flag_test_and_set( \ + volatile ADDRSPACE atomic_flag *object) { \ + return __clc_atomic_flag_test_and_set((volatile ADDRSPACE int *)object, \ + __ATOMIC_SEQ_CST, \ + __MEMORY_SCOPE_DEVICE); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(global) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(local) +#if defined(__opencl_c_generic_address_space) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET() +#endif + +#undef __CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) + +#if defined(__opencl_c_atomic_scope_device) + +#define __CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF bool atomic_flag_test_and_set_explicit( \ + volatile ADDRSPACE atomic_flag *object, memory_order order) { \ + return __clc_atomic_flag_test_and_set((volatile ADDRSPACE int *)object, \ + order, __MEMORY_SCOPE_DEVICE); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(global) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(local) +#if defined(__opencl_c_generic_address_space) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET() +#endif + +#undef __CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET + +#endif // defined(__opencl_c_atomic_scope_device) + +#define __CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF bool atomic_flag_test_and_set_explicit( \ + volatile ADDRSPACE atomic_flag *object, memory_order order, \ + memory_scope scope) { \ + return __clc_atomic_flag_test_and_set( \ + (volatile ADDRSPACE int *)object, order, \ + __opencl_get_clang_memory_scope(scope)); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(global) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(local) +#if defined(__opencl_c_generic_address_space) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET() +#endif diff --git a/libclc/opencl/lib/generic/atomic/atomic_init.cl b/libclc/opencl/lib/generic/atomic/atomic_init.cl new file mode 100644 index 0000000000000..af69e751a30b4 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_init.cl @@ -0,0 +1,17 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_fetch_sub.h> +#include <clc/opencl/atomic/atomic_fetch_sub.h> +#include <clc/opencl/utils.h> + +#define __CLC_BODY <atomic_init.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <atomic_init.inc> +#include <clc/math/gentype.inc> diff --git a/libclc/opencl/lib/generic/atomic/atomic_init.inc b/libclc/opencl/lib/generic/atomic/atomic_init.inc new file mode 100644 index 0000000000000..45348dbf49235 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_init.inc @@ -0,0 +1,48 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifdef __CLC_SCALAR + +#if defined(__opencl_c_fp64) && (defined(cl_khr_int64_base_atomics) && \ + defined(cl_khr_int64_extended_atomics)) +#define __CLC_HAVE_64_ATOMIC +#endif +#if defined(__CLC_FPSIZE) && \ + (__CLC_FPSIZE < 64 || defined(__CLC_HAVE_64_ATOMIC)) +#define __CLC_HAVE_FP_ATOMIC +#endif +#if defined(__CLC_GENSIZE) && \ + ((__CLC_GENSIZE == 32) || \ + (__CLC_GENSIZE == 64 && defined(__CLC_HAVE_64_ATOMIC))) +#define __CLC_HAVE_INT_ATOMIC +#endif +#if defined(__CLC_HAVE_FP_ATOMIC) || defined(__CLC_HAVE_INT_ATOMIC) + +#define __CLC_ATOMIC_GENTYPE __CLC_XCONCAT(atomic_, __CLC_GENTYPE) + +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF void atomic_init( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value) { \ + *(volatile ADDRSPACE __CLC_GENTYPE *)Ptr = Value; \ + } + +__CLC_DEFINE_ATOMIC(global) +__CLC_DEFINE_ATOMIC(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DEFINE_ATOMIC() +#endif + +#undef __CLC_DEFINE_ATOMIC + +#endif // __CLC_HAVE_FP_ATOMIC || __CLC_HAVE_INT_ATOMIC + +#undef __CLC_HAVE_INT_ATOMIC +#undef __CLC_HAVE_FP_ATOMIC +#undef __CLC_HAVE_64_ATOMIC + +#endif // __CLC_SCALAR _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
