https://github.com/arsenm created https://github.com/llvm/llvm-project/pull/186082
Use a more straightforward version which allows optimizations to delete the edge case checks, and also codegens better. Implement in terms of new nextup and nextdown helper functions, which are IEEE functions, and usable in other functions. >From 1606180ebe4e739802a5f273b8b4580b86e056a5 Mon Sep 17 00:00:00 2001 From: Matt Arsenault <[email protected]> Date: Thu, 12 Mar 2026 10:03:02 +0100 Subject: [PATCH] libclc: Replace nextafter implementation Use a more straightforward version which allows optimizations to delete the edge case checks, and also codegens better. Implement in terms of new nextup and nextdown helper functions, which are IEEE functions, and usable in other functions. --- libclc/clc/include/clc/float/definitions.h | 3 + libclc/clc/include/clc/math/clc_nextdown.h | 19 +++++ libclc/clc/include/clc/math/clc_nextup.h | 19 +++++ libclc/clc/include/clc/math/gentype.inc | 6 ++ libclc/clc/lib/generic/CMakeLists.txt | 2 + libclc/clc/lib/generic/math/clc_nextafter.cl | 81 ++----------------- libclc/clc/lib/generic/math/clc_nextafter.inc | 17 ++++ libclc/clc/lib/generic/math/clc_nextdown.cl | 15 ++++ libclc/clc/lib/generic/math/clc_nextdown.inc | 17 ++++ libclc/clc/lib/generic/math/clc_nextup.cl | 14 ++++ libclc/clc/lib/generic/math/clc_nextup.inc | 15 ++++ 11 files changed, 135 insertions(+), 73 deletions(-) create mode 100644 libclc/clc/include/clc/math/clc_nextdown.h create mode 100644 libclc/clc/include/clc/math/clc_nextup.h create mode 100644 libclc/clc/lib/generic/math/clc_nextafter.inc create mode 100644 libclc/clc/lib/generic/math/clc_nextdown.cl create mode 100644 libclc/clc/lib/generic/math/clc_nextdown.inc create mode 100644 libclc/clc/lib/generic/math/clc_nextup.cl create mode 100644 libclc/clc/lib/generic/math/clc_nextup.inc diff --git a/libclc/clc/include/clc/float/definitions.h b/libclc/clc/include/clc/float/definitions.h index 25f35a4ef68ec..6d94bdea44bd7 100644 --- a/libclc/clc/include/clc/float/definitions.h +++ b/libclc/clc/include/clc/float/definitions.h @@ -14,6 +14,7 @@ #define FLT_MIN_EXP -125 #define FLT_RADIX 2 #define FLT_MIN 0x1.0p-126f +#define FLT_TRUE_MIN 0x1p-149f #define FLT_EPSILON 0x1.0p-23f #define FLT_NAN __builtin_nanf("") @@ -29,6 +30,7 @@ #define DBL_MIN_EXP -1021 #define DBL_MAX 0x1.fffffffffffffp1023 #define DBL_MIN 0x1.0p-1022 +#define DBL_TRUE_MIN 0x1p-1074 #define DBL_EPSILON 0x1.0p-52 #define DBL_NAN __builtin_nan("") @@ -56,6 +58,7 @@ #define HALF_MAX_EXP +16 #define HALF_MIN_10_EXP -4 #define HALF_MIN_EXP -13 +#define HALF_TRUE_MIN 0x1p-24h #define HALF_RADIX 2 #define HALF_NAN __builtin_nanf16("") diff --git a/libclc/clc/include/clc/math/clc_nextdown.h b/libclc/clc/include/clc/math/clc_nextdown.h new file mode 100644 index 0000000000000..418c30031271a --- /dev/null +++ b/libclc/clc/include/clc/math/clc_nextdown.h @@ -0,0 +1,19 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_MATH_CLC_NEXTDOWN_H__ +#define __CLC_MATH_CLC_NEXTDOWN_H__ + +#define __CLC_BODY <clc/shared/unary_decl.inc> +#define __CLC_FUNCTION __clc_nextdown + +#include <clc/math/gentype.inc> + +#undef __CLC_FUNCTION + +#endif // __CLC_MATH_CLC_NEXTDOWN_H__ diff --git a/libclc/clc/include/clc/math/clc_nextup.h b/libclc/clc/include/clc/math/clc_nextup.h new file mode 100644 index 0000000000000..5b62b8aba929f --- /dev/null +++ b/libclc/clc/include/clc/math/clc_nextup.h @@ -0,0 +1,19 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_MATH_CLC_NEXTUP_H__ +#define __CLC_MATH_CLC_NEXTUP_H__ + +#define __CLC_BODY <clc/shared/unary_decl.inc> +#define __CLC_FUNCTION __clc_nextup + +#include <clc/math/gentype.inc> + +#undef __CLC_FUNCTION + +#endif // __CLC_MATH_CLC_NEXTUP_H__ diff --git a/libclc/clc/include/clc/math/gentype.inc b/libclc/clc/include/clc/math/gentype.inc index d5664a249af33..bee877c562123 100644 --- a/libclc/clc/include/clc/math/gentype.inc +++ b/libclc/clc/include/clc/math/gentype.inc @@ -75,6 +75,7 @@ #define __CLC_FP_LIT(x) (__CLC_GENTYPE) x##F #define __CLC_GENTYPE_NAN (__CLC_GENTYPE) FLT_NAN #define __CLC_GENTYPE_INF (__CLC_GENTYPE) INFINITY +#define __CLC_GENTYPE_TRUE_MIN (__CLC_GENTYPE) FLT_TRUE_MIN #define __CLC_S_GENTYPE __CLC_XCONCAT(int, __CLC_VECSIZE) #define __CLC_U_GENTYPE __CLC_XCONCAT(uint, __CLC_VECSIZE) @@ -139,6 +140,7 @@ #undef __CLC_U_GENTYPE #undef __CLC_S_GENTYPE #undef __CLC_GENTYPE_NAN +#undef __CLC_GENTYPE_TRUE_MIN #undef __CLC_FP_LIT #undef __CLC_FPSIZE #undef __CLC_SCALAR_GENTYPE @@ -153,6 +155,7 @@ #define __CLC_FPSIZE 64 #define __CLC_FP_LIT(x) (__CLC_GENTYPE)(x) #define __CLC_GENTYPE_NAN (__CLC_GENTYPE) DBL_NAN +#define __CLC_GENTYPE_TRUE_MIN (__CLC_GENTYPE) DBL_TRUE_MIN #define __CLC_S_GENTYPE __CLC_XCONCAT(long, __CLC_VECSIZE) #define __CLC_U_GENTYPE __CLC_XCONCAT(ulong, __CLC_VECSIZE) @@ -217,6 +220,7 @@ #undef __CLC_U_GENTYPE #undef __CLC_S_GENTYPE #undef __CLC_GENTYPE_NAN +#undef __CLC_GENTYPE_TRUE_MIN #undef __CLC_FP_LIT #undef __CLC_FPSIZE #undef __CLC_SCALAR_GENTYPE @@ -231,6 +235,7 @@ #define __CLC_FPSIZE 16 #define __CLC_FP_LIT(x) (__CLC_GENTYPE) x##H #define __CLC_GENTYPE_NAN (__CLC_GENTYPE) HALF_NAN +#define __CLC_GENTYPE_TRUE_MIN (__CLC_GENTYPE) HALF_TRUE_MIN #define __CLC_S_GENTYPE __CLC_XCONCAT(short, __CLC_VECSIZE) #define __CLC_U_GENTYPE __CLC_XCONCAT(ushort, __CLC_VECSIZE) @@ -295,6 +300,7 @@ #undef __CLC_U_GENTYPE #undef __CLC_S_GENTYPE #undef __CLC_GENTYPE_NAN +#undef __CLC_GENTYPE_TRUE_MIN #undef __CLC_FP_LIT #undef __CLC_FPSIZE #undef __CLC_SCALAR_GENTYPE diff --git a/libclc/clc/lib/generic/CMakeLists.txt b/libclc/clc/lib/generic/CMakeLists.txt index 1be28882ddf82..259a660451295 100644 --- a/libclc/clc/lib/generic/CMakeLists.txt +++ b/libclc/clc/lib/generic/CMakeLists.txt @@ -137,6 +137,8 @@ libclc_configure_source_list(CLC_GENERIC_SOURCES math/clc_native_sqrt.cl math/clc_native_tan.cl math/clc_nextafter.cl + math/clc_nextup.cl + math/clc_nextdown.cl math/clc_pow.cl math/clc_pown.cl math/clc_powr.cl diff --git a/libclc/clc/lib/generic/math/clc_nextafter.cl b/libclc/clc/lib/generic/math/clc_nextafter.cl index 48e28c08edf75..e68b95f0b6ac2 100644 --- a/libclc/clc/lib/generic/math/clc_nextafter.cl +++ b/libclc/clc/lib/generic/math/clc_nextafter.cl @@ -6,76 +6,11 @@ // //===----------------------------------------------------------------------===// -#include <clc/internal/clc.h> -#include <clc/math/clc_fabs.h> -#include <clc/relational/clc_isnan.h> - -// This file provides OpenCL C implementations of __clc_nextafter for -// targets that don't support the clang builtin. - -#define __CLC_CLC_AS_TYPE(x) __clc_as_##x - -#define __CLC_NEXTAFTER(FLOAT_TYPE, UINT_TYPE, INT_TYPE, INT_TYPE_SCALAR) \ - _CLC_OVERLOAD _CLC_DEF FLOAT_TYPE __clc_nextafter(FLOAT_TYPE x, \ - FLOAT_TYPE y) { \ - const UINT_TYPE sign_bit = (UINT_TYPE)1 \ - << (sizeof(INT_TYPE_SCALAR) * 8 - 1); \ - UINT_TYPE ix = __CLC_CLC_AS_TYPE(UINT_TYPE)(x); \ - FLOAT_TYPE absx = __clc_fabs(x); \ - UINT_TYPE mxu = sign_bit - ix; \ - INT_TYPE mx = __CLC_CLC_AS_TYPE(INT_TYPE)(mxu); \ - mx = __CLC_CLC_AS_TYPE(INT_TYPE)(ix) < (INT_TYPE)0 \ - ? mx \ - : __CLC_CLC_AS_TYPE(INT_TYPE)(ix); \ - UINT_TYPE iy = __CLC_CLC_AS_TYPE(UINT_TYPE)(y); \ - FLOAT_TYPE absy = __clc_fabs(y); \ - UINT_TYPE myu = sign_bit - iy; \ - INT_TYPE my = __CLC_CLC_AS_TYPE(INT_TYPE)(myu); \ - my = __CLC_CLC_AS_TYPE(INT_TYPE)(iy) < (INT_TYPE)0 \ - ? my \ - : __CLC_CLC_AS_TYPE(INT_TYPE)(iy); \ - INT_TYPE t = mx + (mx < my ? (INT_TYPE)1 : (INT_TYPE) - 1); \ - UINT_TYPE r = sign_bit - __CLC_CLC_AS_TYPE(UINT_TYPE)(t); \ - r = (t < (INT_TYPE)0 || (t == (INT_TYPE)0 && mx < my)) \ - ? r \ - : __CLC_CLC_AS_TYPE(UINT_TYPE)(t); \ - r = __clc_isnan(x) ? ix : r; \ - r = __clc_isnan(y) ? iy : r; \ - r = ((__CLC_CLC_AS_TYPE(UINT_TYPE)(absx) | \ - __CLC_CLC_AS_TYPE(UINT_TYPE)(absy)) == (UINT_TYPE)0 || \ - ix == iy) \ - ? iy \ - : r; \ - return __CLC_CLC_AS_TYPE(FLOAT_TYPE)(r); \ - } - -__CLC_NEXTAFTER(float, uint, int, int) -__CLC_NEXTAFTER(float2, uint2, int2, int) -__CLC_NEXTAFTER(float3, uint3, int3, int) -__CLC_NEXTAFTER(float4, uint4, int4, int) -__CLC_NEXTAFTER(float8, uint8, int8, int) -__CLC_NEXTAFTER(float16, uint16, int16, int) - -#ifdef cl_khr_fp64 -#pragma OPENCL EXTENSION cl_khr_fp64 : enable - -__CLC_NEXTAFTER(double, ulong, long, long) -__CLC_NEXTAFTER(double2, ulong2, long2, long) -__CLC_NEXTAFTER(double3, ulong3, long3, long) -__CLC_NEXTAFTER(double4, ulong4, long4, long) -__CLC_NEXTAFTER(double8, ulong8, long8, long) -__CLC_NEXTAFTER(double16, ulong16, long16, long) - -#endif - -#ifdef cl_khr_fp16 -#pragma OPENCL EXTENSION cl_khr_fp16 : enable - -__CLC_NEXTAFTER(half, ushort, short, short) -__CLC_NEXTAFTER(half2, ushort2, short2, short) -__CLC_NEXTAFTER(half3, ushort3, short3, short) -__CLC_NEXTAFTER(half4, ushort4, short4, short) -__CLC_NEXTAFTER(half8, ushort8, short8, short) -__CLC_NEXTAFTER(half16, ushort16, short16, short) - -#endif +#include "clc/clc_convert.h" +#include "clc/float/definitions.h" +#include "clc/math/clc_nextdown.h" +#include "clc/math/clc_nextup.h" +#include "clc/relational/clc_isunordered.h" + +#define __CLC_BODY <clc_nextafter.inc> +#include <clc/math/gentype.inc> diff --git a/libclc/clc/lib/generic/math/clc_nextafter.inc b/libclc/clc/lib/generic/math/clc_nextafter.inc new file mode 100644 index 0000000000000..ccc8c36c204fd --- /dev/null +++ b/libclc/clc/lib/generic/math/clc_nextafter.inc @@ -0,0 +1,17 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +_CLC_OVERLOAD _CLC_CONST _CLC_DEF __CLC_GENTYPE +__clc_nextafter(__CLC_GENTYPE x, __CLC_GENTYPE y) { + __CLC_GENTYPE up = __clc_nextup(x); + __CLC_GENTYPE down = __clc_nextdown(x); + __CLC_GENTYPE ret = x < y ? up : y; + ret = x > y ? down : ret; + + return __clc_isunordered(x, y) ? __CLC_GENTYPE_NAN : ret; +} diff --git a/libclc/clc/lib/generic/math/clc_nextdown.cl b/libclc/clc/lib/generic/math/clc_nextdown.cl new file mode 100644 index 0000000000000..ed8728e25e595 --- /dev/null +++ b/libclc/clc/lib/generic/math/clc_nextdown.cl @@ -0,0 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clc/clc_convert.h" +#include "clc/float/definitions.h" +#include "clc/math/clc_nextdown.h" +#include "clc/relational/clc_isnan.h" + +#define __CLC_BODY <clc_nextdown.inc> +#include <clc/math/gentype.inc> diff --git a/libclc/clc/lib/generic/math/clc_nextdown.inc b/libclc/clc/lib/generic/math/clc_nextdown.inc new file mode 100644 index 0000000000000..4c16f96fdde29 --- /dev/null +++ b/libclc/clc/lib/generic/math/clc_nextdown.inc @@ -0,0 +1,17 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +_CLC_OVERLOAD _CLC_CONST _CLC_DEF __CLC_GENTYPE +__clc_nextdown(__CLC_GENTYPE x) { + __CLC_S_GENTYPE increment = + x > __CLC_FP_LIT(0.0) ? (__CLC_S_GENTYPE)-1 : (__CLC_S_GENTYPE)1; + __CLC_S_GENTYPE ix = __CLC_AS_S_GENTYPE(x) + increment; + __CLC_GENTYPE y = + x == __CLC_FP_LIT(0.0) ? -__CLC_GENTYPE_TRUE_MIN : __CLC_AS_GENTYPE(ix); + return __clc_isnan(x) || x == -__CLC_GENTYPE_INF ? x : y; +} diff --git a/libclc/clc/lib/generic/math/clc_nextup.cl b/libclc/clc/lib/generic/math/clc_nextup.cl new file mode 100644 index 0000000000000..bc86665923e20 --- /dev/null +++ b/libclc/clc/lib/generic/math/clc_nextup.cl @@ -0,0 +1,14 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clc/clc_convert.h" +#include "clc/math/clc_nextup.h" +#include "clc/relational/clc_isnan.h" + +#define __CLC_BODY <clc_nextup.inc> +#include <clc/math/gentype.inc> diff --git a/libclc/clc/lib/generic/math/clc_nextup.inc b/libclc/clc/lib/generic/math/clc_nextup.inc new file mode 100644 index 0000000000000..a75deda82933a --- /dev/null +++ b/libclc/clc/lib/generic/math/clc_nextup.inc @@ -0,0 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +_CLC_OVERLOAD _CLC_CONST _CLC_DEF __CLC_GENTYPE __clc_nextup(__CLC_GENTYPE x) { + __CLC_S_GENTYPE y = __CLC_AS_S_GENTYPE(x + __CLC_FP_LIT(0.0)); + __CLC_S_GENTYPE ix = + y + (y >= (__CLC_S_GENTYPE)0 ? (__CLC_S_GENTYPE)1 : (__CLC_S_GENTYPE)-1); + __CLC_GENTYPE converted = __CLC_AS_GENTYPE(ix); + return __clc_isnan(x) || x == __CLC_GENTYPE_INF ? x : converted; +} _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
