================
@@ -2714,12 +2714,440 @@ static void ClearPadding(CodeGenFunction &CGF, Address
Src,
}
}
+/// Lower builtins that operate on ARM64EC's software x87 long double. The
+/// ones that are required to be exact (fabs, copysign, signbit, isnan, isinf,
+/// isfinite, isnormal, fpclassify) are implemented with integer operations on
+/// the image itself; everything else is evaluated in `double` exactly like the
+/// C library does for its own long double entry points. Return false when the
+/// builtin does not involve such a value.
+bool CodeGenFunction::EmitSoftX87LongDoubleBuiltin(unsigned BuiltinID,
+ const CallExpr *E,
+ RValue &Result) {
+ // The 'nan' and 'inf' families produce a `long double`, so they do not
+ // consume one but do need the integer image constructed below: they are the
+ // only builtins that yield a constant. Match the bare name, since some of
+ // them are also reachable with a `__builtin_` prefix.
+ std::string Name = getContext().BuiltinInfo.getName(BuiltinID);
+ StringRef BaseName = Name;
+ BaseName.consume_front("__builtin_");
+ bool ProducesLongDouble = false;
+ for (const char *Str : {"nanl", "nansl", "infl", "huge_vall"})
+ ProducesLongDouble |= BaseName == Str;
+
+ // Everything below rewrites a call into its binary64 equivalent, which only
+ // makes sense when a `long double` is *consumed*.
+ bool ConsumesLongDouble = false;
+ for (const Expr *Arg : E->arguments()) {
+ QualType Ty = Arg->getType();
+ if (const auto *CTy = Ty->getAs<ComplexType>())
+ Ty = CTy->getElementType();
+ ConsumesLongDouble |= Ty->isSpecificBuiltinType(BuiltinType::LongDouble);
+ }
+
+ if (!ProducesLongDouble && !ConsumesLongDouble)
+ return false;
+
+ // `fabsl`: Set the sign bit to zero.
+ if (BaseName == "fabsl") {
+ if (!E->getType()->isSpecificBuiltinType(BuiltinType::LongDouble))
+ return false;
+ llvm::Value *V = EmitScalarExpr(E->getArg(0));
+ llvm::Constant *MagnitudeMask = llvm::ConstantInt::get(
+ V->getType(), ~llvm::APInt::getOneBitSet(128, 79));
+ Result = RValue::get(Builder.CreateAnd(V, MagnitudeMask));
+ return true;
+ }
+
+ // `copysignl`: Take the magnitude from the first operand and the sign bit
+ // from the second one.
+ if (BaseName == "copysignl") {
+ if (!E->getType()->isSpecificBuiltinType(BuiltinType::LongDouble))
+ return false;
+ llvm::Value *X = EmitScalarExpr(E->getArg(0));
+ llvm::Value *Y = EmitScalarExpr(E->getArg(1));
+ llvm::Constant *MagnitudeMask = llvm::ConstantInt::get(
+ X->getType(), ~llvm::APInt::getOneBitSet(128, 79));
+ llvm::Value *Magnitude = Builder.CreateAnd(X, MagnitudeMask);
+ llvm::Value *Sign = Builder.CreateAnd(Y, Builder.CreateNot(MagnitudeMask));
+ Result = RValue::get(Builder.CreateOr(Magnitude, Sign));
+ return true;
+ }
+
+ // `signbit`: Take the sign bit and normalize it to boolean.
+ if (Name == "__builtin_signbit" || BaseName == "signbitl") {
+ if (!E->getArg(0)->getType()->isSpecificBuiltinType(
+ BuiltinType::LongDouble))
+ return false;
+ llvm::Value *V = EmitScalarExpr(E->getArg(0));
+ llvm::Value *Sign =
+ Builder.CreateTrunc(Builder.CreateLShr(V, 79), Builder.getInt1Ty());
+ Result = RValue::get(Builder.CreateZExt(Sign, ConvertType(E->getType())));
+ return true;
+ }
+
+ // `__builtin_isnan`: Check if the value is a NaN. An unnormal or pseudo
+ // denormal image is undefined behavior, since it will never be produced in a
+ // constant expression.
+ if (Name == "__builtin_isnan") {
+ if (!E->getArg(0)->getType()->isSpecificBuiltinType(
+ BuiltinType::LongDouble))
+ return false;
+ llvm::Value *V = EmitScalarExpr(E->getArg(0));
+ llvm::Constant *MagnitudeMask = llvm::ConstantInt::get(
+ V->getType(), ~llvm::APInt::getOneBitSet(128, 79));
+ llvm::Value *Magnitude = Builder.CreateAnd(V, MagnitudeMask);
+ llvm::APInt InfinityImage(128, 0x8000000000000000ull);
+ InfinityImage |= llvm::APInt(128, 0x7fffull) << 64;
+ llvm::Constant *Infinity =
+ llvm::ConstantInt::get(V->getType(), InfinityImage);
+ llvm::Value *IsNan = Builder.CreateICmpUGT(Magnitude, Infinity);
+ Result = RValue::get(Builder.CreateZExt(IsNan, ConvertType(E->getType())));
+ return true;
+ }
+
+ // `__builtin_isinf`: Check if the value is a (positive or negative)
infinity.
+ if (Name == "__builtin_isinf") {
+ if (!E->getArg(0)->getType()->isSpecificBuiltinType(
+ BuiltinType::LongDouble))
+ return false;
+ llvm::Value *V = EmitScalarExpr(E->getArg(0));
+ llvm::Constant *MagnitudeMask = llvm::ConstantInt::get(
+ V->getType(), ~llvm::APInt::getOneBitSet(128, 79));
+ llvm::Value *Magnitude = Builder.CreateAnd(V, MagnitudeMask);
+ llvm::APInt InfinityImage(128, 0x8000000000000000ull);
+ InfinityImage |= llvm::APInt(128, 0x7fffull) << 64;
+ llvm::Constant *Infinity =
+ llvm::ConstantInt::get(V->getType(), InfinityImage);
+ llvm::Value *IsInf = Builder.CreateICmpEQ(Magnitude, Infinity);
+ Result = RValue::get(Builder.CreateZExt(IsInf, ConvertType(E->getType())));
+ return true;
+ }
+
+ // `__builtin_isinf_sign`: Return 1 or -1 depending on the sign of an
+ // infinity, and zero for everything else.
+ if (Name == "__builtin_isinf_sign") {
+ if (!E->getArg(0)->getType()->isSpecificBuiltinType(
+ BuiltinType::LongDouble))
+ return false;
+ llvm::Value *V = EmitScalarExpr(E->getArg(0));
+ llvm::Constant *MagnitudeMask = llvm::ConstantInt::get(
+ V->getType(), ~llvm::APInt::getOneBitSet(128, 79));
+ llvm::Value *Magnitude = Builder.CreateAnd(V, MagnitudeMask);
+ llvm::APInt InfinityImage(128, 0x8000000000000000ull);
+ InfinityImage |= llvm::APInt(128, 0x7fffull) << 64;
+ llvm::Constant *Infinity =
+ llvm::ConstantInt::get(V->getType(), InfinityImage);
+ llvm::Type *IntTy = ConvertType(E->getType());
+ llvm::Value *Sign =
+ Builder.CreateTrunc(Builder.CreateLShr(V, 79), Builder.getInt1Ty());
+ llvm::Value *Signed = Builder.CreateSelect(
+ Sign, llvm::ConstantInt::get(IntTy, -1, /*isSigned=*/true),
+ llvm::ConstantInt::get(IntTy, 1));
+ llvm::Value *IsInf = Builder.CreateICmpEQ(Magnitude, Infinity);
+ Result = RValue::get(
+ Builder.CreateSelect(IsInf, Signed, llvm::ConstantInt::get(IntTy, 0)));
+ return true;
+ }
+
+ // `__builtin_isfinite`: Check if the value is neither an infinity nor a NaN.
+ if (Name == "__builtin_isfinite") {
+ if (!E->getArg(0)->getType()->isSpecificBuiltinType(
+ BuiltinType::LongDouble))
+ return false;
+ llvm::Value *V = EmitScalarExpr(E->getArg(0));
+ llvm::Constant *MagnitudeMask = llvm::ConstantInt::get(
+ V->getType(), ~llvm::APInt::getOneBitSet(128, 79));
+ llvm::Value *Magnitude = Builder.CreateAnd(V, MagnitudeMask);
+ llvm::APInt InfinityImage(128, 0x8000000000000000ull);
+ InfinityImage |= llvm::APInt(128, 0x7fffull) << 64;
+ llvm::Constant *Infinity =
+ llvm::ConstantInt::get(V->getType(), InfinityImage);
+ llvm::Value *IsFinite = Builder.CreateICmpULT(Magnitude, Infinity);
+ Result =
+ RValue::get(Builder.CreateZExt(IsFinite, ConvertType(E->getType())));
+ return true;
+ }
+
+ // `__builtin_isnormal`: Check if the value is finite and nonzero. A
+ // canonical image has the explicit integer bit set when the exponent is
+ // nonzero, so the exponent alone decides; an unnormal is undefined
behaviour.
+ if (Name == "__builtin_isnormal") {
+ if (!E->getArg(0)->getType()->isSpecificBuiltinType(
+ BuiltinType::LongDouble))
+ return false;
+ llvm::Value *V = EmitScalarExpr(E->getArg(0));
+ llvm::Value *Exp = Builder.CreateAnd(
+ Builder.CreateTrunc(Builder.CreateLShr(V, 64), Builder.getInt16Ty()),
+ Builder.getInt16(0x7FFF));
+ llvm::Value *IsNormal =
+ Builder.CreateAnd(Builder.CreateICmpNE(Exp, Builder.getInt16(0)),
+ Builder.CreateICmpNE(Exp, Builder.getInt16(0x7FFF)));
+ Result =
+ RValue::get(Builder.CreateZExt(IsNormal, ConvertType(E->getType())));
+ return true;
+ }
+
+ // `__builtin_fpclassify`: Pick the first five arguments that correspond to a
+ // NaN, an infinity, a normal, a subnormal and a zero respectively.
+ if (Name == "__builtin_fpclassify") {
+ if (!E->getArg(5)->getType()->isSpecificBuiltinType(
+ BuiltinType::LongDouble))
+ return false;
+ llvm::Value *V = EmitScalarExpr(E->getArg(5));
+ llvm::Value *Exp = Builder.CreateAnd(
+ Builder.CreateTrunc(Builder.CreateLShr(V, 64), Builder.getInt16Ty()),
+ Builder.getInt16(0x7FFF));
+ llvm::Value *Man = Builder.CreateTrunc(V, Builder.getInt64Ty());
+ llvm::Value *ExpZero = Builder.CreateICmpEQ(Exp, Builder.getInt16(0));
+ llvm::Value *ExpAllOnes =
+ Builder.CreateICmpEQ(Exp, Builder.getInt16(0x7FFF));
+ llvm::Value *ManZero = Builder.CreateICmpEQ(Man, Builder.getInt64(0));
+ // Infinity is the all-ones exponent with nothing but the integer bit set;
+ // any other value with that exponent is a NaN. An unnormal, whose integer
+ // bit is clear, is undefined behaviour.
+ llvm::Value *IsInf = Builder.CreateAnd(
+ ExpAllOnes,
+ Builder.CreateICmpEQ(Man, Builder.getInt64(0x8000000000000000ull)));
+ llvm::Value *IsNaN =
+ Builder.CreateAnd(ExpAllOnes, Builder.CreateNot(IsInf));
+ llvm::Value *IsNormal = Builder.CreateAnd(Builder.CreateNot(ExpZero),
+ Builder.CreateNot(ExpAllOnes));
+ llvm::Value *IsSubnormal =
+ Builder.CreateAnd(ExpZero, Builder.CreateNot(ManZero));
+ llvm::Value *R = EmitScalarExpr(E->getArg(4)); // zero
+ R = Builder.CreateSelect(IsSubnormal, EmitScalarExpr(E->getArg(3)), R);
+ R = Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)), R);
+ R = Builder.CreateSelect(IsInf, EmitScalarExpr(E->getArg(1)), R);
+ R = Builder.CreateSelect(IsNaN, EmitScalarExpr(E->getArg(0)), R);
+ Result = RValue::get(R);
+ return true;
+ }
+
+ // `infl` and `huge_vall`: Infinity has the significand 0x8000000000000000
+ // with an all-ones exponent, and the six padding bytes are zeroes. They
+ // take no argument at all, so there is nothing to check.
+ if (BaseName == "infl" || BaseName == "huge_vall") {
+ llvm::APInt Image(128, 0x8000000000000000ull);
+ Image |= llvm::APInt(128, 0x7fffull) << 64;
+ llvm::Type *ImageTy = llvm::IntegerType::get(getLLVMContext(), 128);
+ Result = RValue::get(llvm::ConstantInt::get(ImageTy, Image));
+ return true;
+ }
+
+ // `nanl` and `nansl`: The quiet NaN has the significand 0xc000000000000000
+ // and the signalling one has 0xa000000000000000, both with an all-ones
+ // exponent. The payload of the tag argument is not propagated, but the
+ // argument is still evaluated, since it may have side effects.
+ if (BaseName == "nanl" || BaseName == "nansl") {
+ if (!E->getArg(0)->getType()->isPointerType())
+ return false;
+ EmitScalarExpr(E->getArg(0));
+ llvm::APInt Image(128, BaseName == "nansl" ? 0xa000000000000000ull
+ : 0xc000000000000000ull);
+ Image |= llvm::APInt(128, 0x7fffull) << 64;
+ llvm::Type *ImageTy = llvm::IntegerType::get(getLLVMContext(), 128);
+ Result = RValue::get(llvm::ConstantInt::get(ImageTy, Image));
+ return true;
+ }
+
+ // `__builtin_isunordered`: True if either operand is a NaN.
+ // `__builtin_isgreater`, `__builtin_isgreaterequal`, `__builtin_isless` and
+ // `__builtin_islessequal`: Perform ordered comparisons of two operands.
+ // `__builtin_islessgreater`: True if the operands are ordered and unequal;
+ // that is, if either of `x < y` and `x > y` holds.
+ struct BuiltinComparison {
+ const char *Name;
+ CmpInst::Predicate Predicate;
+ };
+ static constexpr BuiltinComparison BuiltinComparisons[] = {
+ {"__builtin_isunordered", CmpInst::FCMP_UNO},
+ {"__builtin_isgreater", CmpInst::FCMP_OGT},
+ {"__builtin_isgreaterequal", CmpInst::FCMP_OGE},
+ {"__builtin_isless", CmpInst::FCMP_OLT},
+ {"__builtin_islessequal", CmpInst::FCMP_OLE},
+ {"__builtin_islessgreater", CmpInst::FCMP_ONE},
+ };
+ for (const auto &BC : BuiltinComparisons)
+ if (Name == BC.Name) {
+ if (!E->getArg(0)->getType()->isSpecificBuiltinType(
+ BuiltinType::LongDouble) ||
+ !E->getArg(1)->getType()->isSpecificBuiltinType(
+ BuiltinType::LongDouble))
+ return false;
+ llvm::Value *A = EmitScalarExpr(E->getArg(0));
+ llvm::Value *B = EmitScalarExpr(E->getArg(1));
+ llvm::Value *Cmp = EmitSoftX87LongDoubleComparison(A, B, BC.Predicate);
+ Result = RValue::get(Builder.CreateZExt(Cmp, ConvertType(E->getType())));
+ return true;
+ }
+
+ // `__builtin_isfpclass`: Test the image against a mask of class bits. The
+ // second argument is an integer constant expression built out of the
+ // `__FPCLASS_*` values, whose bits number the classes in this order:
+ // signalling NaN, quiet NaN, -inf, negative normal, negative subnormal,
+ // -0, +0, positive subnormal, positive normal and +inf. So the value only
+ // has to be mapped to the ordinal of the class it belongs to, and the bit
+ // of the mask with that index decides.
+ if (Name == "__builtin_isfpclass") {
+ if (!E->getArg(0)->getType()->isSpecificBuiltinType(
+ BuiltinType::LongDouble))
+ return false;
+ Expr::EvalResult MaskResult;
+ if (!E->getArg(1)->EvaluateAsInt(MaskResult, getContext()))
+ return false;
+ const uint64_t Mask = MaskResult.Val.getInt().getLimitedValue();
+ llvm::Value *V = EmitScalarExpr(E->getArg(0));
+ llvm::Value *Exp = Builder.CreateAnd(
+ Builder.CreateTrunc(Builder.CreateLShr(V, 64), Builder.getInt16Ty()),
+ Builder.getInt16(0x7FFF));
+ llvm::Value *Man = Builder.CreateTrunc(V, Builder.getInt64Ty());
+ llvm::Value *Negative =
+ Builder.CreateTrunc(Builder.CreateLShr(V, 79), Builder.getInt1Ty());
+ llvm::Value *ExpZero = Builder.CreateICmpEQ(Exp, Builder.getInt16(0));
+ llvm::Value *ExpAllOnes =
+ Builder.CreateICmpEQ(Exp, Builder.getInt16(0x7FFF));
+ llvm::Value *ManZero = Builder.CreateICmpEQ(Man, Builder.getInt64(0));
+ // Infinity is the all-ones exponent with nothing but the integer bit
+ // set, and a NaN is any other image with that exponent, quiet when bit
+ // 62 of the significand is set. An unnormal, whose integer bit is
+ // clear, is undefined behaviour.
+ llvm::Value *Infinity = Builder.CreateAnd(
+ ExpAllOnes,
+ Builder.CreateICmpEQ(Man, Builder.getInt64(0x8000000000000000ull)));
+ llvm::Value *Quiet = Builder.CreateICmpNE(
+ Builder.CreateAnd(Man, Builder.getInt64(0x4000000000000000ull)),
+ Builder.getInt64(0));
+ // The magnitudes are numbered 2 to 5 from the largest downwards, and the
+ // positive classes continue the sequence in the reverse order, which makes
+ // them `11 - Magnitude`. A NaN takes 0 or 1 instead of a magnitude.
+ llvm::Value *Magnitude = Builder.getInt64(2);
+ Magnitude = Builder.CreateSelect(Builder.CreateNot(ExpAllOnes),
+ Builder.getInt64(3), Magnitude);
+ Magnitude = Builder.CreateSelect(ExpZero, Builder.getInt64(4), Magnitude);
+ Magnitude = Builder.CreateSelect(Builder.CreateAnd(ExpZero, ManZero),
+ Builder.getInt64(5), Magnitude);
+ Magnitude = Builder.CreateSelect(
+ Negative, Magnitude,
+ Builder.CreateSub(Builder.getInt64(11), Magnitude));
+ llvm::Value *Ordinal =
+ Builder.CreateSelect(Quiet, Builder.getInt64(1), Builder.getInt64(0));
+ Ordinal = Builder.CreateSelect(
+ Builder.CreateAnd(ExpAllOnes, Builder.CreateNot(Infinity)), Ordinal,
+ Magnitude);
+ llvm::Value *Bit =
+ Builder.CreateAnd(Builder.CreateLShr(Builder.getInt64(Mask), Ordinal),
+ Builder.getInt64(1));
+ Result = RValue::get(
+ Builder.CreateZExt(Builder.CreateICmpNE(Bit, Builder.getInt64(0)),
+ ConvertType(E->getType())));
+ return true;
+ }
+
+ // `__builtin_complex`: Assemble a complex value out of a real and an
+ // imaginary part, which become the low and high halves of the image pair.
+ // Its operands are "custom type checked", so they are not promoted: both
+ // have the same real floating type already, and they have to be the `long
+ // double` one for this lowering to apply.
+ if (Name == "__builtin_complex") {
+ if (!E->getArg(0)->getType()->isSpecificBuiltinType(
+ BuiltinType::LongDouble) ||
+ !E->getArg(1)->getType()->isSpecificBuiltinType(
+ BuiltinType::LongDouble))
+ return false;
+ llvm::Value *Real = EmitScalarExpr(E->getArg(0));
+ llvm::Value *Imag = EmitScalarExpr(E->getArg(1));
+ Result = RValue::getComplex(Real, Imag);
+ return true;
+ }
+
+ // `conjl`: Conjugation only flips the sign bit of the imaginary part, which
+ // is the high half of the image pair, and leaves the real part alone.
+ if (BaseName == "conjl") {
+ const auto *CTy = E->getArg(0)->getType()->getAs<ComplexType>();
+ if (!CTy ||
+ !CTy->getElementType()->isSpecificBuiltinType(BuiltinType::LongDouble))
+ return false;
+ ComplexPairTy Z = EmitComplexExpr(E->getArg(0));
+ llvm::Constant *SignMask = llvm::ConstantInt::get(
+ Z.second->getType(), llvm::APInt::getOneBitSet(128, 79));
+ llvm::Value *Imag = Builder.CreateXor(Z.second, SignMask);
+ Result = RValue::getComplex(Z.first, Imag);
+ return true;
+ }
+
+ // `creall` and `cimagl`: They only pick the low or the high half of the
+ // image pair, which is the real or the imaginary part.
+ if (BaseName == "creall" || BaseName == "cimagl") {
+ const auto *CTy = E->getArg(0)->getType()->getAs<ComplexType>();
+ if (!CTy ||
+ !CTy->getElementType()->isSpecificBuiltinType(BuiltinType::LongDouble))
+ return false;
+ ComplexPairTy Z = EmitComplexExpr(E->getArg(0));
+ Result = RValue::get(BaseName == "creall" ? Z.first : Z.second);
+ return true;
+ }
+
+ // Everything else is a function of the C library, which on this target works
+ // on the 80-bit image itself: a `long double` *is* the image, so its own
+ // entry point takes and returns it untouched, and the builtin is nothing
+ // but a call to that entry point. Nothing is evaluated in `double` and
+ // rounded back, so that not a single bit is lost. The variadic `printf`
+ // and `scanf` families go the same way: their ellipsis arguments are
+ // forwarded untouched, since the x64 calling convention that ARM64EC
+ // reproduces passes a 16-byte `long double` by reference.
+
+ // `powil` is the only one left that has no entry point of that spelling in
+ // the C library: there is no `powi` function, and `llvm.powi` only accepts
+ // real floating-point values. mingw-w64 CRT provides `__powil` for it.
+ if (Name == "__builtin_powil") {
+ llvm::Value *X = EmitScalarExpr(E->getArg(0));
+ llvm::Value *N = EmitScalarExpr(E->getArg(1));
+ llvm::FunctionType *FT = llvm::FunctionType::get(
+ ConvertType(E->getType()), {X->getType(), N->getType()}, false);
+ Result = RValue::get(
+ Builder.CreateCall(CGM.CreateRuntimeFunction(FT, "__powil"), {X, N}));
+ return true;
+ }
+
+ // Both spellings arrive here: `__builtin_sinl` and the `sinl` that the
+ // header declares are separate builtin identifiers, and they resolve to the
+ // same entry point.
+ const FunctionDecl *FD = E->getDirectCallee();
+ if (FD && getContext().BuiltinInfo.isLibFunction(BuiltinID)) {
+ Result =
+ emitLibraryCall(*this, FD, E, CGM.getBuiltinLibFunction(FD,
BuiltinID));
+ return true;
+ }
+
+ if (FD && getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) {
+ Result = emitLibraryCall(*this, FD, E, CGM.getRawFunctionPointer(FD));
+ return true;
+ }
+
+ return false;
----------------
lhmouse wrote:
Add in 295fe6ac475e3b80ddcbbf2281f8984fcd052f67
https://github.com/llvm/llvm-project/pull/237057
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
