https://github.com/tbaederr updated https://github.com/llvm/llvm-project/pull/173756
>From 899c6ed3c6bd109681f6d5b684ffc88e5d7e72d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= <[email protected]> Date: Fri, 26 Dec 2025 09:06:45 +0100 Subject: [PATCH] musttail --- clang/lib/AST/ByteCode/Compiler.cpp | 11 +- clang/lib/AST/ByteCode/EvalEmitter.cpp | 5 + clang/lib/AST/ByteCode/Interp.cpp | 196 ++++++++++--------- clang/lib/AST/ByteCode/Interp.h | 48 +++-- clang/lib/AST/ByteCode/InterpState.h | 3 + clang/lib/AST/ByteCode/Opcodes.td | 3 + clang/utils/TableGen/ClangOpcodesEmitter.cpp | 126 +++++++----- 7 files changed, 239 insertions(+), 153 deletions(-) diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp index 67980676dcd30..030faf157bb52 100644 --- a/clang/lib/AST/ByteCode/Compiler.cpp +++ b/clang/lib/AST/ByteCode/Compiler.cpp @@ -2529,7 +2529,7 @@ bool Compiler<Emitter>::VisitAbstractConditionalOperator( LabelTy LabelFalse = this->getLabel(); // Label for the false expr. if (IsBcpCall) { - if (!this->emitStartSpeculation(E)) + if (!this->emitPushIgnoreDiags(E)) return false; } @@ -2560,8 +2560,11 @@ bool Compiler<Emitter>::VisitAbstractConditionalOperator( this->fallthrough(LabelEnd); this->emitLabel(LabelEnd); - if (IsBcpCall) - return this->emitEndSpeculation(E); + if (IsBcpCall) { + if (!this->emitPopIgnoreDiags(E)) + return false; + } + return true; } @@ -5081,9 +5084,9 @@ bool Compiler<Emitter>::VisitBuiltinCallExpr(const CallExpr *E, LabelTy EndLabel = this->getLabel(); if (!this->speculate(E, EndLabel)) return false; - this->fallthrough(EndLabel); if (!this->emitEndSpeculation(E)) return false; + this->fallthrough(EndLabel); if (DiscardResult) return this->emitPop(classifyPrim(E), E); return true; diff --git a/clang/lib/AST/ByteCode/EvalEmitter.cpp b/clang/lib/AST/ByteCode/EvalEmitter.cpp index cf3cc1b17133c..2e7eed285a76a 100644 --- a/clang/lib/AST/ByteCode/EvalEmitter.cpp +++ b/clang/lib/AST/ByteCode/EvalEmitter.cpp @@ -11,6 +11,7 @@ #include "IntegralAP.h" #include "Interp.h" #include "clang/AST/DeclCXX.h" +#include "llvm/ADT/ScopeExit.h" using namespace clang; using namespace clang::interp; @@ -157,6 +158,10 @@ bool EvalEmitter::fallthrough(const LabelTy &Label) { bool EvalEmitter::speculate(const CallExpr *E, const LabelTy &EndLabel) { if (!isActive()) return true; + + PushIgnoreDiags(S, OpPC); + auto _ = llvm::make_scope_exit([&]() { PopIgnoreDiags(S, OpPC); }); + size_t StackSizeBefore = S.Stk.size(); const Expr *Arg = E->getArg(0); if (!this->visit(Arg)) { diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp index 889ac1e1a9a7e..857b2bcf372c5 100644 --- a/clang/lib/AST/ByteCode/Interp.cpp +++ b/clang/lib/AST/ByteCode/Interp.cpp @@ -28,7 +28,17 @@ using namespace clang; using namespace clang::interp; -static bool RetValue(InterpState &S, CodePtr &Pt) { +#if defined(_MSC_VER) && !defined(__clang__) +#ifdef NDEBUG +#define MUSTTAIL [[msvc::musttail]] +#else +#define MUSTTAIL +#endif +#else +#define MUSTTAIL [[clang::musttail]] +#endif + +PRESERVE_NONE static bool RetValue(InterpState &S, CodePtr &Pt) { llvm::report_fatal_error("Interpreter cannot return values"); } @@ -55,76 +65,6 @@ static bool Jf(InterpState &S, CodePtr &PC, int32_t Offset) { return true; } -// https://github.com/llvm/llvm-project/issues/102513 -#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG) -#pragma optimize("", off) -#endif -// FIXME: We have the large switch over all opcodes here again, and in -// Interpret(). -static bool BCP(InterpState &S, CodePtr &RealPC, int32_t Offset, PrimType PT) { - [[maybe_unused]] CodePtr PCBefore = RealPC; - size_t StackSizeBefore = S.Stk.size(); - - auto SpeculativeInterp = [&S, RealPC]() -> bool { - const InterpFrame *StartFrame = S.Current; - CodePtr PC = RealPC; - - for (;;) { - auto Op = PC.read<Opcode>(); - if (Op == OP_EndSpeculation) - return true; - CodePtr OpPC = PC; - - switch (Op) { -#define GET_INTERP -#include "Opcodes.inc" -#undef GET_INTERP - } - } - llvm_unreachable("We didn't see an EndSpeculation op?"); - }; - - if (SpeculativeInterp()) { - if (PT == PT_Ptr) { - const auto &Ptr = S.Stk.pop<Pointer>(); - assert(S.Stk.size() == StackSizeBefore); - S.Stk.push<Integral<32, true>>( - Integral<32, true>::from(CheckBCPResult(S, Ptr))); - } else { - // Pop the result from the stack and return success. - TYPE_SWITCH(PT, S.Stk.pop<T>();); - assert(S.Stk.size() == StackSizeBefore); - S.Stk.push<Integral<32, true>>(Integral<32, true>::from(1)); - } - } else { - if (!S.inConstantContext()) - return Invalid(S, RealPC); - - S.Stk.clearTo(StackSizeBefore); - S.Stk.push<Integral<32, true>>(Integral<32, true>::from(0)); - } - - // RealPC should not have been modified. - assert(*RealPC == *PCBefore); - - // Jump to end label. This is a little tricker than just RealPC += Offset - // because our usual jump instructions don't have any arguments, to the offset - // we get is a little too much and we need to subtract the size of the - // bool and PrimType arguments again. - int32_t ParamSize = align(sizeof(PrimType)); - assert(Offset >= ParamSize); - RealPC += Offset - ParamSize; - - [[maybe_unused]] CodePtr PCCopy = RealPC; - assert(PCCopy.read<Opcode>() == OP_EndSpeculation); - - return true; -} -// https://github.com/llvm/llvm-project/issues/102513 -#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG) -#pragma optimize("", on) -#endif - static void diagnoseMissingInitializer(InterpState &S, CodePtr OpPC, const ValueDecl *VD) { const SourceInfo &E = S.Current->getSource(OpPC); @@ -253,6 +193,9 @@ static bool CheckGlobal(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { namespace clang { namespace interp { +PRESERVE_NONE static bool BCP(InterpState &S, CodePtr &RealPC, int32_t Offset, + PrimType PT); + static void popArg(InterpState &S, const Expr *Arg) { PrimType Ty = S.getContext().classify(Arg).value_or(PT_Ptr); TYPE_SWITCH(Ty, S.Stk.discard<T>()); @@ -2324,38 +2267,113 @@ bool FinishInitGlobal(InterpState &S, CodePtr OpPC) { return true; } -// https://github.com/llvm/llvm-project/issues/102513 -#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG) -#pragma optimize("", off) -#endif +PRESERVE_NONE static bool InterpNext(InterpState &S, CodePtr &PC); + bool Interpret(InterpState &S) { // The current stack frame when we started Interpret(). // This is being used by the ops to determine wheter // to return from this function and thus terminate // interpretation. - const InterpFrame *StartFrame = S.Current; assert(!S.Current->isRoot()); CodePtr PC = S.Current->getPC(); - // Empty program. - if (!PC) - return true; + return InterpNext(S, PC); +} - for (;;) { - auto Op = PC.read<Opcode>(); - CodePtr OpPC = PC; +// The dispatcher functions read the opcode arguments from the +// bytecode and call the implementation function. +#define GET_INTERPFN_DISPATCHERS +#include "Opcodes.inc" +#undef GET_INTERPFN_DISPATCHERS + +using InterpFn = bool (*)(InterpState &, CodePtr &PC) PRESERVE_NONE; - switch (Op) { -#define GET_INTERP +// Array of the dispatcher functions defined above. +const InterpFn InterpFunctions[] = { +#define GET_INTERPFN_LIST #include "Opcodes.inc" -#undef GET_INTERP +#undef GET_INTERPFN_LIST +}; + +// Read the next opcode and call the dispatcher function. +PRESERVE_NONE static bool InterpNext(InterpState &S, CodePtr &PC) { + auto Op = PC.read<Opcode>(); + auto Fn = InterpFunctions[Op]; + [[clang::musttail]] return Fn(S, PC); +} + +/// This is used to implement speculative execution via __builtin_constant_p +/// when we generate bytecode. +/// The setup here is that we use the same tailcall mechanism for speculative +/// evaluation that we use for the regular one. +/// Since each speculative execution ends with an EndSpeculation opcode, +/// that one does NOT call InterpNext() but simply returns true. +/// This way, we return back to this function when we see an EndSpeculation, +/// OR (of course), when we encounter an error and one of the opcodes +/// returns false. +PRESERVE_NONE static bool BCP(InterpState &S, CodePtr &RealPC, int32_t Offset, + PrimType PT) { + [[maybe_unused]] CodePtr PCBefore = RealPC; + size_t StackSizeBefore = S.Stk.size(); + + // Speculation depth must be at least 1 here, since we must have + // passed a StartSpeculation op before. + [[maybe_unused]] unsigned DepthBefore = S.SpeculationDepth; + assert(DepthBefore >= 1); + + CodePtr PC = RealPC; + auto SpeculativeInterp = [&S, &PC]() -> bool { + // Ignore diagnostics during speculative execution. + PushIgnoreDiags(S, PC); + auto _ = llvm::make_scope_exit([&]() { PopIgnoreDiags(S, PC); }); + + auto Op = PC.read<Opcode>(); + auto Fn = InterpFunctions[Op]; + return Fn(S, PC); + }; + + if (SpeculativeInterp()) { + // Speculation must've ended naturally via a EndSpeculation opcode. + assert(S.SpeculationDepth == DepthBefore - 1); + if (PT == PT_Ptr) { + const auto &Ptr = S.Stk.pop<Pointer>(); + assert(S.Stk.size() == StackSizeBefore); + S.Stk.push<Integral<32, true>>( + Integral<32, true>::from(CheckBCPResult(S, Ptr))); + } else { + // Pop the result from the stack and return success. + TYPE_SWITCH(PT, S.Stk.discard<T>();); + assert(S.Stk.size() == StackSizeBefore); + S.Stk.push<Integral<32, true>>(Integral<32, true>::from(1)); } + } else { + // End the speculation manually since we didn't call EndSpeculation + // naturally. + EndSpeculation(S, RealPC); + + if (!S.inConstantContext()) + return Invalid(S, RealPC); + + S.Stk.clearTo(StackSizeBefore); + S.Stk.push<Integral<32, true>>(Integral<32, true>::from(0)); } + + // RealPC should not have been modified. + assert(*RealPC == *PCBefore); + + // We have already evaluated this speculation's EndSpeculation opcode. + assert(S.SpeculationDepth == DepthBefore - 1); + + // Jump to end label. This is a little tricker than just RealPC += Offset + // because our usual jump instructions don't have any arguments, to the offset + // we get is a little too much and we need to subtract the size of the + // bool and PrimType arguments again. + int32_t ParamSize = align(sizeof(PrimType)); + assert(Offset >= ParamSize); + RealPC += Offset - ParamSize; + + return true; } -// https://github.com/llvm/llvm-project/issues/102513 -#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG) -#pragma optimize("", on) -#endif } // namespace interp } // namespace clang diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h index 9accbbc1605a9..0e63c721f12af 100644 --- a/clang/lib/AST/ByteCode/Interp.h +++ b/clang/lib/AST/ByteCode/Interp.h @@ -33,8 +33,15 @@ #include "clang/AST/Expr.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/APSInt.h" +#include "llvm/ADT/ScopeExit.h" #include <type_traits> +#ifdef __clang__ +#define PRESERVE_NONE [[clang::preserve_none]] +#else +#define PRESERVE_NONE +#endif + namespace clang { namespace interp { @@ -221,7 +228,7 @@ void cleanupAfterFunctionCall(InterpState &S, CodePtr OpPC, const Function *Func); template <PrimType Name, class T = typename PrimConv<Name>::T> -bool Ret(InterpState &S, CodePtr &PC) { +bool Ret(InterpState &S, CodePtr &PC) PRESERVE_NONE { const T &Ret = S.Stk.pop<T>(); assert(S.Current); @@ -243,7 +250,7 @@ bool Ret(InterpState &S, CodePtr &PC) { return true; } -inline bool RetVoid(InterpState &S, CodePtr &PC) { +inline bool RetVoid(InterpState &S, CodePtr &PC) PRESERVE_NONE { assert(S.Current->getFrameOffset() == S.Stk.size() && "Invalid frame"); if (!S.checkingPotentialConstantExpression() || S.Current->Caller) @@ -3039,8 +3046,7 @@ static inline bool ShiftFixedPoint(InterpState &S, CodePtr OpPC, bool Left) { //===----------------------------------------------------------------------===// // NoRet //===----------------------------------------------------------------------===// - -inline bool NoRet(InterpState &S, CodePtr OpPC) { +inline bool NoRet(InterpState &S, CodePtr OpPC) PRESERVE_NONE { SourceLocation EndLoc = S.Current->getCallee()->getEndLoc(); S.FFDiag(EndLoc, diag::note_constexpr_no_return); return false; @@ -3258,26 +3264,44 @@ inline bool Unsupported(InterpState &S, CodePtr OpPC) { return false; } -inline bool StartSpeculation(InterpState &S, CodePtr OpPC) { - ++S.SpeculationDepth; - if (S.SpeculationDepth != 1) +inline bool PushIgnoreDiags(InterpState &S, CodePtr OpPC) { + ++S.DiagIgnoreDepth; + if (S.DiagIgnoreDepth != 1) return true; - assert(S.PrevDiags == nullptr); S.PrevDiags = S.getEvalStatus().Diag; S.getEvalStatus().Diag = nullptr; + assert(!S.diagnosing()); return true; } -inline bool EndSpeculation(InterpState &S, CodePtr OpPC) { - assert(S.SpeculationDepth != 0); - --S.SpeculationDepth; - if (S.SpeculationDepth == 0) { + +inline bool PopIgnoreDiags(InterpState &S, CodePtr OpPC) { + --S.DiagIgnoreDepth; + if (S.DiagIgnoreDepth == 0) { S.getEvalStatus().Diag = S.PrevDiags; S.PrevDiags = nullptr; } return true; } +inline bool StartSpeculation(InterpState &S, CodePtr OpPC) { +#ifndef NDEBUG + ++S.SpeculationDepth; +#endif + return true; +} + +// This is special-cased in the tablegen opcode emitter. +// Its dispatch function will NOT call InterpNext +// and instead simply return true. +inline bool EndSpeculation(InterpState &S, CodePtr OpPC) { +#ifndef NDEBUG + assert(S.SpeculationDepth != 0); + --S.SpeculationDepth; +#endif + return true; +} + inline bool PushCC(InterpState &S, CodePtr OpPC, bool Value) { S.ConstantContextOverride = Value; return true; diff --git a/clang/lib/AST/ByteCode/InterpState.h b/clang/lib/AST/ByteCode/InterpState.h index e2e4d5c985f93..1591d90715131 100644 --- a/clang/lib/AST/ByteCode/InterpState.h +++ b/clang/lib/AST/ByteCode/InterpState.h @@ -182,7 +182,10 @@ class InterpState final : public State, public SourceMapper { const VarDecl *EvaluatingDecl = nullptr; /// Things needed to do speculative execution. SmallVectorImpl<PartialDiagnosticAt> *PrevDiags = nullptr; +#ifndef NDEBUG unsigned SpeculationDepth = 0; +#endif + unsigned DiagIgnoreDepth = 0; std::optional<bool> ConstantContextOverride; llvm::SmallVector< diff --git a/clang/lib/AST/ByteCode/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td index 6e768793fcfcf..81e7bc3813a4e 100644 --- a/clang/lib/AST/ByteCode/Opcodes.td +++ b/clang/lib/AST/ByteCode/Opcodes.td @@ -178,6 +178,9 @@ def Jt : JumpOpcode; // [Bool] -> [], jumps if false. def Jf : JumpOpcode; +def PushIgnoreDiags : Opcode; +def PopIgnoreDiags : Opcode; + def StartSpeculation : Opcode; def EndSpeculation : Opcode; def BCP : Opcode { diff --git a/clang/utils/TableGen/ClangOpcodesEmitter.cpp b/clang/utils/TableGen/ClangOpcodesEmitter.cpp index d26122aca46bd..4dc21f4c6c383 100644 --- a/clang/utils/TableGen/ClangOpcodesEmitter.cpp +++ b/clang/utils/TableGen/ClangOpcodesEmitter.cpp @@ -34,8 +34,8 @@ class ClangOpcodesEmitter { /// The name is obtained by concatenating the name with the list of types. void EmitEnum(raw_ostream &OS, StringRef N, const Record *R); - /// Emits the switch case and the invocation in the interpreter. - void EmitInterp(raw_ostream &OS, StringRef N, const Record *R); + void EmitInterpFnList(raw_ostream &OS, StringRef N, const Record *R); + void EmitInterpFnDispatchers(raw_ostream &OS, StringRef N, const Record *R); /// Emits the disassembler. void EmitDisasm(raw_ostream &OS, StringRef N, const Record *R); @@ -91,7 +91,8 @@ void ClangOpcodesEmitter::run(raw_ostream &OS) { N = Opcode->getName(); EmitEnum(OS, N, Opcode); - EmitInterp(OS, N, Opcode); + EmitInterpFnList(OS, N, Opcode); + EmitInterpFnDispatchers(OS, N, Opcode); EmitDisasm(OS, N, Opcode); EmitProto(OS, N, Opcode); EmitGroup(OS, N, Opcode); @@ -109,60 +110,89 @@ void ClangOpcodesEmitter::EmitEnum(raw_ostream &OS, StringRef N, OS << "#endif\n"; } -void ClangOpcodesEmitter::EmitInterp(raw_ostream &OS, StringRef N, - const Record *R) { - OS << "#ifdef GET_INTERP\n"; +void ClangOpcodesEmitter::EmitInterpFnDispatchers(raw_ostream &OS, StringRef N, + const Record *R) { + OS << "#ifdef GET_INTERPFN_DISPATCHERS\n"; + Enumerate(R, N, [&](ArrayRef<const Record *> TS, const Twine &ID) { + OS << "PRESERVE_NONE\nstatic bool Interp_" << ID + << "(InterpState &S, CodePtr &PC) {\n"; - Enumerate(R, N, - [this, R, &OS, &N](ArrayRef<const Record *> TS, const Twine &ID) { - bool CanReturn = R->getValueAsBit("CanReturn"); - bool ChangesPC = R->getValueAsBit("ChangesPC"); - const auto &Args = R->getValueAsListOfDefs("Args"); + if (ID.str() == "EndSpeculation") { + OS << " return EndSpeculation(S, PC);\n"; + OS << "}\n"; + return; + } - OS << "case OP_" << ID << ": {\n"; + bool CanReturn = R->getValueAsBit("CanReturn"); + const auto &Args = R->getValueAsListOfDefs("Args"); + bool ChangesPC = R->getValueAsBit("ChangesPC"); - if (CanReturn) - OS << " bool DoReturn = (S.Current == StartFrame);\n"; + if (Args.empty()) { + if (CanReturn) { + OS << " MUSTTAIL return " << N; + PrintTypes(OS, TS); + OS << "(S, PC);\n"; + OS << "}\n"; + return; + } - // Emit calls to read arguments. - for (size_t I = 0, N = Args.size(); I < N; ++I) { - const auto *Arg = Args[I]; - bool AsRef = Arg->getValueAsBit("AsRef"); + OS << " if (!" << N; + PrintTypes(OS, TS); + OS << "(S, PC))\n"; + OS << " return false;\n"; + OS << " MUSTTAIL return InterpNext(S, PC);\n"; + OS << "}\n"; + return; + } - if (AsRef) - OS << " const auto &V" << I; - else - OS << " const auto V" << I; - OS << " = "; - OS << "ReadArg<" << Arg->getValueAsString("Name") - << ">(S, PC);\n"; - } + OS << " {\n"; - // Emit a call to the template method and pass arguments. - OS << " if (!" << N; - PrintTypes(OS, TS); - OS << "(S"; - if (ChangesPC) - OS << ", PC"; - else - OS << ", OpPC"; - for (size_t I = 0, N = Args.size(); I < N; ++I) - OS << ", V" << I; - OS << "))\n"; - OS << " return false;\n"; + if (!ChangesPC) + OS << " CodePtr OpPC = PC;\n"; - // Bail out if interpreter returned. - if (CanReturn) { - OS << " if (!S.Current || S.Current->isRoot())\n"; - OS << " return true;\n"; + // Emit calls to read arguments. + for (size_t I = 0, N = Args.size(); I < N; ++I) { + const auto *Arg = Args[I]; + bool AsRef = Arg->getValueAsBit("AsRef"); - OS << " if (DoReturn)\n"; - OS << " return true;\n"; - } + if (AsRef) + OS << " const auto &V" << I; + else + OS << " const auto V" << I; + OS << " = "; + OS << "ReadArg<" << Arg->getValueAsString("Name") << ">(S, PC);\n"; + } - OS << " continue;\n"; - OS << "}\n"; - }); + OS << " if (!" << N; + PrintTypes(OS, TS); + OS << "(S"; + if (ChangesPC) + OS << ", PC"; + else + OS << ", OpPC"; + for (size_t I = 0, N = Args.size(); I < N; ++I) + OS << ", V" << I; + OS << "))\n"; + OS << " return false;\n"; + + OS << " }\n"; + + if (!CanReturn) + OS << " MUSTTAIL return InterpNext(S, PC);\n"; + else + OS << " return true;\n"; + + OS << "}\n"; + }); + OS << "#endif\n"; +} + +void ClangOpcodesEmitter::EmitInterpFnList(raw_ostream &OS, StringRef N, + const Record *R) { + OS << "#ifdef GET_INTERPFN_LIST\n"; + Enumerate(R, N, [&OS](ArrayRef<const Record *>, const Twine &ID) { + OS << "&Interp_" << ID << ",\n"; + }); OS << "#endif\n"; } _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
