The branch main has been updated by kib: URL: https://cgit.FreeBSD.org/src/commit/?id=b08e6a3882b44a09fbda84dd315a431c7a585219
commit b08e6a3882b44a09fbda84dd315a431c7a585219 Author: Steve Kargl <[email protected]> AuthorDate: 2026-09-10 23:18:58 +0000 Commit: Konstantin Belousov <[email protected]> CommitDate: 2026-09-11 17:53:08 +0000 lib/msun: use the same algorithm for sw fmaf(3) as for fma(3) This fixes rounding at the last bit for subnormals. PR: 298260 Reviewed by: kib MFC after: 1 week Differential revision: https://reviews.freebsd.org/D59579 --- lib/msun/src/s_fmaf.c | 254 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 230 insertions(+), 24 deletions(-) diff --git a/lib/msun/src/s_fmaf.c b/lib/msun/src/s_fmaf.c index 666d2bb26d91..27fbe462a3d8 100644 --- a/lib/msun/src/s_fmaf.c +++ b/lib/msun/src/s_fmaf.c @@ -27,17 +27,132 @@ */ #include <fenv.h> +#include <float.h> /* Needed for FLT_MANT_DIG. */ #include "math.h" #include "math_private.h" +#pragma STDC FENV_ACCESS ON + #ifdef USE_BUILTIN_FMAF float -fmaf(float x, float y, float z) +#ifdef _RENAME_FMAF +fmaf_sw +#else +fmaf +#endif +(float x, float y, float z) { return (__builtin_fmaf(x, y, z)); } #else + +#define _CC (0x1p12F + 1) /* Needed by _SLOW2SUM() and _MUL() below */ + +/* + * A struct dd represents a floating-point number with twice the precision + * of a float. We maintain the invariant that "hi" stores the 24 high-order + * bits of the result. + */ +struct dd { + float hi; + float lo; +}; + +/* + * Compute a+b exactly, returning the exact result in a struct dd. We assume + * that both a and b are finite, but make no assumptions about their relative + * magnitudes. + */ +static inline struct dd +dd_add(float a, float b) +{ + struct dd ret; + _SLOW2SUM(a, b, ret.hi, ret.lo); + return (ret); +} + +/* + * Compute a+b, with a small tweak: The least significant bit of the + * result is adjusted into a sticky bit summarizing all the bits that + * were lost to rounding. This adjustment negates the effects of double + * rounding when the result is added to another number with a higher + * exponent. For an explanation of round and sticky bits, see any reference + * on FPU design, e.g., + * + * J. Coonen. An Implementation Guide to a Proposed Standard for + * Floating-Point Arithmetic. Computer, vol. 13, no. 1, Jan 1980. + */ +static inline float +add_adjusted(float a, float b) +{ + struct dd sum; + uint32_t hibits, lobits; + + sum = dd_add(a, b); + if (sum.lo != 0) { + GET_FLOAT_WORD(hibits, sum.hi); + if ((hibits & 1) == 0) { + /* hibits += (int)copysignf(1.0f, sum.hi * sum.lo); */ + GET_FLOAT_WORD(lobits, sum.lo); + hibits += 1 - ((hibits ^ lobits) >> 30); + SET_FLOAT_WORD(sum.hi, hibits); + } + } + return (sum.hi); +} + +/* + * Compute ldexp(a+b, scale) with a single rounding error. It is assumed + * that the result will be subnormal, and care is taken to ensure that + * double rounding does not occur. + */ + +static inline float +add_and_denormalize(float a, float b, int scale) +{ + struct dd sum; + uint32_t hibits, lobits; + int bits_lost; + + sum = dd_add(a, b); + + /* + * If we are losing at least two bits of accuracy to denormalization, + * then the first lost bit becomes a round bit, and we adjust the + * lowest bit of sum.hi to make it a sticky bit summarizing all the + * bits in sum.lo. With the sticky bit adjusted, the hardware will + * break any ties in the correct direction. + * + * If we are losing only one bit to denormalization, however, we must + * break the ties manually. + */ + if (sum.lo != 0) { + GET_FLOAT_WORD(hibits, sum.hi); + bits_lost = -((int)(hibits >> 24) & 0x7f) - scale + 1; + if ((bits_lost != 1) ^ (int)(hibits & 1)) { + /* hibits += (int)copysign(1.0, sum.hi * sum.lo) */ + GET_FLOAT_WORD(lobits, sum.lo); + hibits += 1 - (((hibits ^ lobits) >> 30) & 2); + SET_FLOAT_WORD(sum.hi, hibits); + } + } + return (ldexpf(sum.hi, scale)); +} + +/* + * Compute a*b exactly, returning the exact result in a struct dd. We assume + * that both a and b are normalized, so no underflow or overflow will occur. + * The current rounding mode must be round-to-nearest. + */ +static inline struct dd +dd_mul(float a, float b) +{ + struct dd ret; + _MUL(a, b, ret.hi, ret.lo); + return (ret); +} + #ifdef _RENAME_FMAF float fmaf_sw(float, float, float); #endif @@ -45,9 +160,19 @@ float fmaf_sw(float, float, float); /* * Fused multiply-add: Compute x * y + z with a single rounding error. * - * A double has more than twice as much precision than a float, so - * direct double-precision arithmetic suffices, except where double - * rounding occurs. + * We use scaling to avoid overflow/underflow, along with the + * canonical precision-doubling technique adapted from: + * + * Dekker, T. A Floating-Point Technique for Extending the + * Available Precision. Numer. Math. 18, 224-242 (1971). + * + * This algorithm is sensitive to the rounding precision. FPUs such + * as the i387 must be set in double-precision mode if variables are + * to be stored in FP registers in order to avoid incorrect results. + * This is the default on FreeBSD, but not on many other systems. + * + * Hardware instructions should be used on architectures that support it, + * since this implementation will likely be several times slower. */ float #ifdef _RENAME_FMAF @@ -57,29 +182,110 @@ fmaf #endif (float x, float y, float z) { - double xy, result; - uint32_t hr, lr; - - xy = (double)x * y; - result = xy + z; - EXTRACT_WORDS(hr, lr, result); - /* Common case: The double precision result is fine. */ - if ((lr & 0x1fffffff) != 0x10000000 || /* not a halfway case */ - (hr & 0x7ff00000) == 0x7ff00000 || /* NaN */ - result - xy == z || /* exact */ - fegetround() != FE_TONEAREST) /* not round-to-nearest */ - return (result); + float xs, ys, zs, adj; + struct dd xy, r; + int oround; + int ex, ey, ez; + int spread; /* - * If result is inexact, and exactly halfway between two float values, - * we need to adjust the low-order bit in the direction of the error. + * Handle special cases. The order of operations and the particular + * return values here are crucial in handling special cases involving + * infinities, NaNs, overflows, and signed zeroes correctly. */ - fesetround(FE_TOWARDZERO); - volatile double vxy = xy; /* XXX work around gcc CSE bug */ - double adjusted_result = vxy + z; + if (x == 0 || y == 0) + return (x * y + z); + if (z == 0) + return (x * y); + if (!isfinite(x) || !isfinite(y)) + return (x * y + z); + if (!isfinite(z)) + return (z); + + xs = frexpf(x, &ex); + ys = frexpf(y, &ey); + zs = frexpf(z, &ez); + oround = fegetround(); + spread = ex + ey - ez; + + /* + * If x * y and z are many orders of magnitude apart, the scaling + * will overflow, so we handle these cases specially. Rounding + * modes other than FE_TONEAREST are painful. + */ + if (spread < -FLT_MANT_DIG) { + feraiseexcept(FE_INEXACT); + if (!isnormal(z)) + feraiseexcept(FE_UNDERFLOW); + switch (oround) { + case FE_TONEAREST: + return (z); + case FE_TOWARDZERO: + if ((x > 0) ^ (y < 0) ^ (z < 0)) + return (z); + else + return (nextafterf(z, 0)); + case FE_DOWNWARD: + if ((x > 0) ^ (y < 0)) + return (z); + else + return (nextafterf(z, -INFINITY)); + default: /* FE_UPWARD */ + if ((x > 0) ^ (y < 0)) + return (nextafterf(z, INFINITY)); + else + return (z); + } + } + if (spread <= FLT_MANT_DIG * 2) + zs = ldexpf(zs, -spread); + else + zs = copysignf(FLT_MIN, zs); + fesetround(FE_TONEAREST); - if (result == adjusted_result) - SET_LOW_WORD(adjusted_result, lr + 1); - return (adjusted_result); + /* work around clang issue #8472 */ + volatile float vxs = xs; + + /* + * Basic approach for round-to-nearest: + * + * (xy.hi, xy.lo) = x * y (exact) + * (r.hi, r.lo) = xy.hi + z (exact) + * adj = xy.lo + r.lo (inexact; low bit is sticky) + * result = r.hi + adj (correctly rounded) + */ + xy = dd_mul(vxs, ys); + r = dd_add(xy.hi, zs); + + spread = ex + ey; + + if (r.hi == 0 && xy.lo == 0) { + /* + * When the addends cancel to 0, ensure that the result has + * the correct sign. + */ + fesetround(oround); + volatile float vzs = zs; /* XXX gcc CSE bug workaround */ + return (xy.hi + vzs); + } + + if (oround != FE_TONEAREST) { + /* + * There is no need to worry about double rounding in directed + * rounding modes. + */ + fesetround(oround); + /* work around clang issue #8472 */ + volatile float vrlo = r.lo; + adj = vrlo + xy.lo; + return (ldexpf(r.hi + adj, spread)); + } + + adj = add_adjusted(r.lo, xy.lo); + + if (spread + ilogbf(r.hi) > -127) + return (ldexpf(r.hi + adj, spread)); + else + return (add_and_denormalize(r.hi, adj, spread)); } #endif /* !USE_BUILTIN_FMAF */
