https://github.com/erichkeane created https://github.com/llvm/llvm-project/pull/232975
Classic codegen has a lot of infrastructure to emit the loop-info-stack, we can do it as a pretty simple attribute. This handles lowering to CIR for these hints, after extracting the stuff that determines the values to a separate type in CodeGenUtils. Flattening the loops/emitting this to LLVM-IR is left to a follow up. AI Notice: I made claude help me with the tests, so it did much of that to make sure we got reasonable test coverage. >From 350c195622afab60464a5e53cbfd175b24486d19 Mon Sep 17 00:00:00 2001 From: erichkeane <[email protected]> Date: Fri, 16 Oct 2026 13:51:27 -0700 Subject: [PATCH] [CIR] Implement 'loop hint' lowering Classic codegen has a lot of infrastructure to emit the loop-info-stack, we can do it as a pretty simple attribute. This handles lowering to CIR for these hints, after extracting the stuff that determines the values to a separate type in CodeGenUtils. Flattening the loops/emitting this to LLVM-IR is left to a follow up. AI Notice: I made claude help me with the tests, so it did much of that to make sure we got reasonable test coverage. --- .../include/clang/CIR/Dialect/IR/CIRAttrs.td | 134 ++++++++ clang/include/clang/CIR/Dialect/IR/CIROps.td | 15 +- .../clang/CodeGenUtils/LoopAttributes.h | 98 ++++++ clang/lib/CIR/CodeGen/CIRGenFunction.h | 14 +- clang/lib/CIR/CodeGen/CIRGenStmt.cpp | 118 ++++++-- clang/lib/CodeGen/CGLoopInfo.cpp | 264 +--------------- clang/lib/CodeGen/CGLoopInfo.h | 60 +--- clang/lib/CodeGenUtils/CMakeLists.txt | 1 + clang/lib/CodeGenUtils/LoopAttributes.cpp | 285 ++++++++++++++++++ clang/test/CIR/CodeGen/loop-hints-stds.cpp | 84 ++++++ clang/test/CIR/CodeGen/loop-hints.c | 52 ++++ clang/test/CIR/CodeGen/loop-hints.cpp | 139 +++++++++ clang/test/CIR/CodeGenHLSL/loop-hints.hlsl | 16 + clang/test/CIR/CodeGenOpenACC/loop.cpp | 2 +- clang/test/CIR/CodeGenOpenCL/loop-hints.cl | 18 ++ clang/test/CIR/IR/invalid-loop-hints.cir | 73 +++++ clang/test/CIR/IR/loop-hints.cir | 40 +++ 17 files changed, 1070 insertions(+), 343 deletions(-) create mode 100644 clang/include/clang/CodeGenUtils/LoopAttributes.h create mode 100644 clang/lib/CodeGenUtils/LoopAttributes.cpp create mode 100644 clang/test/CIR/CodeGen/loop-hints-stds.cpp create mode 100644 clang/test/CIR/CodeGen/loop-hints.c create mode 100644 clang/test/CIR/CodeGen/loop-hints.cpp create mode 100644 clang/test/CIR/CodeGenHLSL/loop-hints.hlsl create mode 100644 clang/test/CIR/CodeGenOpenCL/loop-hints.cl create mode 100644 clang/test/CIR/IR/invalid-loop-hints.cir create mode 100644 clang/test/CIR/IR/loop-hints.cir diff --git a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td index aa9562586d2f5..b3cc021989fde 100644 --- a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td +++ b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td @@ -953,6 +953,140 @@ def CIR_FenvAttr : CIR_Attr<"Fenv", "fenv"> { let canHaveIllegalCXXABIType = 0; } +//===----------------------------------------------------------------------===// +// CIR_FlagParam +//===----------------------------------------------------------------------===// + +/// A simple UnitAttr wrapper so that an attributes parameter can print JUST its +/// name and be used with the ($keyword^)? syntax in assemblyFormat. +class CIR_FlagParam<string keyword> : OptionalParameter<"mlir::UnitAttr"> { + let printer = "$_printer << \"" # keyword # "\""; + let parser = "[&]() -> ::mlir::FailureOr<::mlir::UnitAttr> {\n" + " if (::mlir::succeeded($_parser.parseOptionalKeyword(\"" + # keyword # "\")))\n" + " return ::mlir::UnitAttr::get($_ctxt);\n" + " return ::mlir::UnitAttr();\n" + "}()"; +} + +//===----------------------------------------------------------------------===// +// LoopUnrollMode +//===----------------------------------------------------------------------===// + +def CIR_LoopUnrollMode : CIR_I32Enum< + "LoopUnrollMode", "loop unrolling mode", [ + I32EnumCase<"Enable", 0, "enable">, + I32EnumCase<"Disable", 1, "disable">, + I32EnumCase<"Full", 2, "full"> +]> { + let description = [{ + How a loop is unrolled (or unrolled and jammed): `enable` asks for the + optimizer to unroll the loop, `disable` forbids it, and `full` asks for it + to be unrolled completely. + }]; +} + +//===----------------------------------------------------------------------===// +// LoopHintAttr +//===----------------------------------------------------------------------===// + +def CIR_LoopHintAttr : CIR_Attr<"LoopHint", "loop_hint"> { + let summary = "Optimizer hints and guarantees attached to a loop"; + let description = [{ + The `#cir.loop_hint` attribute carries per-loop information for the + optimizer: the forward-progress guarantee and the hints from + `#pragma clang loop`, `#pragma unroll`, `[[clang::code_align]]` and their + OpenCL and HLSL spellings. It is attached to the loop operations + (`cir.for`, `cir.while`, `cir.do`). + + - `must_progress`: the loop must eventually terminate or have an + observable side effect (the forward-progress guarantee of C++11 and + later, and C11 loops with a non-constant condition). + - `vectorize_enable`: force vectorization on. + - `vectorize_width`: the vectorization factor. A width of 1 disables + vectorization. + - `vectorize_scalable`: whether the vectorization factor is + scalable, or explicitly fixed. + - `vectorize_predicate`: enable or disable predicated vectorization. + - `interleave_count`: the interleave count. A count of 1 disables + interleaving. + - `unroll`: The unrolling mode: enable, disable, or full. + - `unroll_count`: the unroll factor. + - `unroll_and_jam`: the unroll-and-jam mode: enable, disable, or full. + - `distribute`: enable or disable loop distribution. + - `pipeline_disabled`: disable software pipelining. + - `pipeline_initiation_interval`: the pipelining initiation interval. + - `licm_disabled`: disable loop-invariant code motion. + - `parallel`: the loop's iterations are independent, so its memory + accesses are parallel accesses. + - `code_align`: the alignment of the loop, in bytes. + + Example: + ``` + #cir.loop_hint<> + #cir.loop_hint< must_progress> + #cir.loop_hint< must_progress vectorize_enable interleave_count = 2 + unroll_count = 4> + ``` + }]; + + let parameters = (ins + CIR_FlagParam<"must_progress">:$must_progress, + CIR_FlagParam<"vectorize_enable">:$vectorize_enable, + OptionalParameter<"uint32_t">:$vectorize_width, + OptionalParameter<"mlir::BoolAttr">:$vectorize_scalable, + OptionalParameter<"mlir::BoolAttr">:$vectorize_predicate, + OptionalParameter<"uint32_t">:$interleave_count, + OptionalParameter<"std::optional<cir::LoopUnrollMode>">:$unroll, + OptionalParameter<"uint32_t">:$unroll_count, + OptionalParameter<"std::optional<cir::LoopUnrollMode>">:$unroll_and_jam, + OptionalParameter<"uint32_t">:$unroll_and_jam_count, + OptionalParameter<"mlir::BoolAttr">:$distribute, + CIR_FlagParam<"pipeline_disabled">:$pipeline_disabled, + OptionalParameter<"uint32_t">:$pipeline_initiation_interval, + CIR_FlagParam<"licm_disabled">:$licm_disabled, + CIR_FlagParam<"parallel">:$parallel, + OptionalParameter<"uint32_t">:$code_align); + + let assemblyFormat = [{ + `<` + (` ` $must_progress^)? + (` ` $vectorize_enable^)? + (` ` `vectorize_width` `=` $vectorize_width^)? + (` ` `vectorize_scalable` `=` $vectorize_scalable^)? + (` ` `vectorize_predicate` `=` $vectorize_predicate^)? + (` ` `interleave_count` `=` $interleave_count^)? + (` ` `unroll` `=` $unroll^)? + (` ` `unroll_count` `=` $unroll_count^)? + (` ` `unroll_and_jam` `=` $unroll_and_jam^)? + (` ` `unroll_and_jam_count` `=` $unroll_and_jam_count^)? + (` ` `distribute` `=` $distribute^)? + (` ` $pipeline_disabled^)? + (` ` `pipeline_initiation_interval` `=` $pipeline_initiation_interval^)? + (` ` $licm_disabled^)? + (` ` $parallel^)? + (` ` `code_align` `=` $code_align^)? + `>` + }]; + + let extraClassDeclaration = [{ + /// Returns true if no hint is set, in which case the attribute has no + /// information and there is no reason to use it. Every parameter is + /// unset when it is null, empty or 0, so this must name all of them. + bool isEmpty() const { + return !getMustProgress() && !getVectorizeEnable() && + !getVectorizeWidth() && !getVectorizeScalable() && + !getVectorizePredicate() && !getInterleaveCount() && + !getUnroll() && !getUnrollCount() && !getUnrollAndJam() && + !getUnrollAndJamCount() && !getDistribute() && + !getPipelineDisabled() && !getPipelineInitiationInterval() && + !getLicmDisabled() && !getParallel() && !getCodeAlign(); + } + }]; + + let canHaveIllegalCXXABIType = 0; +} + //===----------------------------------------------------------------------===// // FastMathFlagsAttr //===----------------------------------------------------------------------===// diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td index 63d50761806a6..533039f44c3f2 100644 --- a/clang/include/clang/CIR/Dialect/IR/CIROps.td +++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td @@ -2369,11 +2369,13 @@ class CIR_WhileOpBase<string mnemonic> : CIR_LoopOpBase<mnemonic> { } def CIR_WhileOp : CIR_WhileOpBase<"while"> { - let arguments = (ins OptionalAttr<CIR_CleanupKindAttr>:$cleanupKind); + let arguments = (ins OptionalAttr<CIR_CleanupKindAttr>:$cleanupKind, + OptionalAttr<CIR_LoopHintAttr>:$loop_hints); let regions = (region MinSizedRegion<1>:$cond, MinSizedRegion<1>:$body, MaxSizedRegion<1>:$cleanup); let assemblyFormat = [{ - $cond `do` $body (`cleanup` enum($cleanupKind) $cleanup^)? attr-dict + $cond `do` $body (`cleanup` enum($cleanupKind) $cleanup^)? + (`loop_hints` $loop_hints^)? attr-dict }]; let description = [{ @@ -2449,8 +2451,11 @@ def CIR_WhileOp : CIR_WhileOpBase<"while"> { } def CIR_DoWhileOp : CIR_WhileOpBase<"do"> { + let arguments = (ins OptionalAttr<CIR_LoopHintAttr>:$loop_hints); let regions = (region MinSizedRegion<1>:$body, MinSizedRegion<1>:$cond); - let assemblyFormat = " $body `while` $cond attr-dict"; + let assemblyFormat = [{ + $body `while` $cond (`loop_hints` $loop_hints^)? attr-dict + }]; let extraClassDeclaration = [{ mlir::Region &getEntry() { return getBody(); } @@ -2524,7 +2529,8 @@ def CIR_ForOp : CIR_LoopOpBase<"for"> { ``` }]; - let arguments = (ins OptionalAttr<CIR_CleanupKindAttr>:$cleanupKind); + let arguments = (ins OptionalAttr<CIR_CleanupKindAttr>:$cleanupKind, + OptionalAttr<CIR_LoopHintAttr>:$loop_hints); let regions = (region MinSizedRegion<1>:$cond, MinSizedRegion<1>:$body, MinSizedRegion<1>:$step, @@ -2534,6 +2540,7 @@ def CIR_ForOp : CIR_LoopOpBase<"for"> { `body` $body `step` $step (`cleanup` enum($cleanupKind) $cleanup^)? + (`loop_hints` $loop_hints^)? attr-dict }]; diff --git a/clang/include/clang/CodeGenUtils/LoopAttributes.h b/clang/include/clang/CodeGenUtils/LoopAttributes.h new file mode 100644 index 0000000000000..49c1bbfcacfba --- /dev/null +++ b/clang/include/clang/CodeGenUtils/LoopAttributes.h @@ -0,0 +1,98 @@ +//===--- LoopAttributes.h - Shared loop hint state ---------------*- C++ +//-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file holds the state of the attributes and hints that may be specified +// on a loop, and the logic to determine it from the loop's statement +// attributes and the command line options. Both classic CodeGen and CIR +// CodeGen use it, and each translates the state into its own representation. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_CODEGENUTILS_LOOPATTRIBUTES_H +#define LLVM_CLANG_CODEGENUTILS_LOOPATTRIBUTES_H + +#include "llvm/ADT/ArrayRef.h" + +namespace clang { +class ASTContext; +class Attr; +class CodeGenOptions; +} // namespace clang + +namespace clang::CodeGenUtils { + +/// Attributes that may be specified on loops. +struct LoopAttributes { + explicit LoopAttributes(bool IsParallel = false); + void clear(); + + /// Generate llvm.loop.parallel metadata for loads and stores. + bool IsParallel; + + /// State of loop vectorization or unrolling. + enum LVEnableState { Unspecified, Enable, Disable, Full }; + + /// Value for llvm.loop.vectorize.enable metadata. + LVEnableState VectorizeEnable; + + /// Value for llvm.loop.unroll.* metadata (enable, disable, or full). + LVEnableState UnrollEnable; + + /// Value for llvm.loop.unroll_and_jam.* metadata (enable, disable, or full). + LVEnableState UnrollAndJamEnable; + + /// Value for llvm.loop.vectorize.predicate metadata + LVEnableState VectorizePredicateEnable; + + /// Value for llvm.loop.vectorize.width metadata. + unsigned VectorizeWidth; + + // Value for llvm.loop.vectorize.scalable.enable + LVEnableState VectorizeScalable; + + /// Value for llvm.loop.interleave.count metadata. + unsigned InterleaveCount; + + /// llvm.unroll. + unsigned UnrollCount; + + /// llvm.unroll. + unsigned UnrollAndJamCount; + + /// Value for llvm.loop.distribute.enable metadata. + LVEnableState DistributeEnable; + + /// Value for llvm.loop.pipeline.disable metadata. + bool PipelineDisabled; + + /// Value for llvm.licm.disable metadata. + bool LICMDisabled; + + /// Value for llvm.loop.pipeline.iicount metadata. + unsigned PipelineInitiationInterval; + + /// Value for 'llvm.loop.align' metadata. + unsigned CodeAlign; + + /// Value for whether the loop is required to make progress. + bool MustProgress; +}; + +/// Update \p Attrs with the loop hints in \p StmtAttrs (`#pragma clang loop`, +/// `#pragma unroll`, `[[clang::code_align]]`, the OpenCL and HLSL spellings), +/// the command line options that adjust every loop (-falign-loops and +/// -fno-unroll-loops), and whether the loop must make progress. +void updateLoopAttributes(LoopAttributes &Attrs, ASTContext &Ctx, + const CodeGenOptions &CGOpts, + llvm::ArrayRef<const Attr *> StmtAttrs, + bool MustProgress); + +} // namespace clang::CodeGenUtils + +#endif // LLVM_CLANG_CODEGENUTILS_LOOPATTRIBUTES_H diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h index 90da139c0f19c..ee7a1e38f0017 100644 --- a/clang/lib/CIR/CodeGen/CIRGenFunction.h +++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h @@ -2240,7 +2240,8 @@ class CIRGenFunction : public CIRGenTypeCache { void emitDeleteCall(const FunctionDecl *deleteFD, mlir::Value ptr, QualType deleteTy); - mlir::LogicalResult emitDoStmt(const clang::DoStmt &s); + mlir::LogicalResult emitDoStmt(const clang::DoStmt &s, + llvm::ArrayRef<const Attr *> attrs = {}); mlir::Value emitCXXTypeidExpr(const CXXTypeidExpr *e); mlir::Value emitDynamicCast(Address thisAddr, const CXXDynamicCastExpr *dce); @@ -2302,7 +2303,8 @@ class CIRGenFunction : public CIRGenTypeCache { mlir::LogicalResult emitSimpleStmt(const clang::Stmt *s, bool useCurrentScope); - mlir::LogicalResult emitForStmt(const clang::ForStmt &s); + mlir::LogicalResult emitForStmt(const clang::ForStmt &s, + llvm::ArrayRef<const Attr *> attrs = {}); void emitForwardingCallToLambda(const CXXMethodDecl *lambdaCallOperator, CallArgList &callArgs); @@ -2585,7 +2587,13 @@ class CIRGenFunction : public CIRGenTypeCache { void emitVariablyModifiedType(QualType ty); - mlir::LogicalResult emitWhileStmt(const clang::WhileStmt &s); + mlir::LogicalResult emitWhileStmt(const clang::WhileStmt &s, + llvm::ArrayRef<const Attr *> attrs = {}); + + /// Build the hints that get attached to a loop, or a null attribute + /// if there is nothing to say about it. + cir::LoopHintAttr getLoopHints(llvm::ArrayRef<const Attr *> attrs, + bool mustProgress); std::optional<mlir::Value> emitRISCVBuiltinExpr(unsigned builtinID, const CallExpr *expr); diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp index 9e14ac420aa9f..5f308751f6f54 100644 --- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp @@ -22,6 +22,7 @@ #include "clang/AST/StmtOpenMP.h" #include "clang/AST/StmtSYCL.h" #include "clang/CIR/MissingFeatures.h" +#include "clang/CodeGenUtils/LoopAttributes.h" #include "clang/CodeGenUtils/StmtUtils.h" #include "llvm/Support/SaveAndRestore.h" @@ -256,11 +257,11 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s, case Stmt::SwitchStmtClass: return emitSwitchStmt(cast<SwitchStmt>(*s)); case Stmt::ForStmtClass: - return emitForStmt(cast<ForStmt>(*s)); + return emitForStmt(cast<ForStmt>(*s), attr); case Stmt::WhileStmtClass: - return emitWhileStmt(cast<WhileStmt>(*s)); + return emitWhileStmt(cast<WhileStmt>(*s), attr); case Stmt::DoStmtClass: - return emitDoStmt(cast<DoStmt>(*s)); + return emitDoStmt(cast<DoStmt>(*s), attr); case Stmt::CXXTryStmtClass: return emitCXXTryStmt(cast<CXXTryStmt>(*s)); case Stmt::CXXForRangeStmtClass: @@ -991,12 +992,83 @@ mlir::LogicalResult CIRGenFunction::emitSwitchCase(const SwitchCase &s, llvm_unreachable("expect case or default stmt"); } +cir::LoopHintAttr CIRGenFunction::getLoopHints(ArrayRef<const Attr *> attrs, + bool mustProgress) { + // Determine the state of the hints, the same way classic codegen does. + CodeGenUtils::LoopAttributes loopAttrs; + CodeGenUtils::updateLoopAttributes(loopAttrs, getContext(), + cgm.getCodeGenOpts(), attrs, mustProgress); + + // Memory accesses in a `parallel` loop are tagged as parallel accesses when + // CIR is lowered. + assert(!cir::MissingFeatures::loopInfoStack()); + mlir::MLIRContext *ctx = &getMLIRContext(); + + auto convertFlag = [&](bool set) { + return set ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr(); + }; + auto convertStateToBool = + [&](CodeGenUtils::LoopAttributes::LVEnableState state) { + switch (state) { + case CodeGenUtils::LoopAttributes::LVEnableState::Unspecified: + return mlir::BoolAttr(); + case CodeGenUtils::LoopAttributes::LVEnableState::Enable: + return mlir::BoolAttr::get(ctx, true); + case CodeGenUtils::LoopAttributes::LVEnableState::Disable: + return mlir::BoolAttr::get(ctx, false); + case CodeGenUtils::LoopAttributes::LVEnableState::Full: + llvm_unreachable("'full' only applies to unrolling"); + } + llvm_unreachable("unknown bool state"); + }; + + auto convertUnrollMode = [](CodeGenUtils::LoopAttributes::LVEnableState state) + -> std::optional<cir::LoopUnrollMode> { + switch (state) { + case CodeGenUtils::LoopAttributes::LVEnableState::Unspecified: + return std::nullopt; + case CodeGenUtils::LoopAttributes::LVEnableState::Enable: + return cir::LoopUnrollMode::Enable; + case CodeGenUtils::LoopAttributes::LVEnableState::Disable: + return cir::LoopUnrollMode::Disable; + case CodeGenUtils::LoopAttributes::LVEnableState::Full: + return cir::LoopUnrollMode::Full; + } + + llvm_unreachable("unknown unroll state"); + }; + + // Only OMP seems to generate this, so leave as an NYI for now. + if (loopAttrs.VectorizeEnable == + CodeGenUtils::LoopAttributes::LVEnableState::Disable || + loopAttrs.VectorizeEnable == + CodeGenUtils::LoopAttributes::LVEnableState::Full) + cgm.errorNYI(SourceLocation{}, "unsupported vector-enable loop state"); + + bool vectorizeEnable = loopAttrs.VectorizeEnable == + CodeGenUtils::LoopAttributes::LVEnableState::Enable; + auto hints = cir::LoopHintAttr::get( + ctx, convertFlag(loopAttrs.MustProgress), convertFlag(vectorizeEnable), + loopAttrs.VectorizeWidth, convertStateToBool(loopAttrs.VectorizeScalable), + convertStateToBool(loopAttrs.VectorizePredicateEnable), + loopAttrs.InterleaveCount, convertUnrollMode(loopAttrs.UnrollEnable), + loopAttrs.UnrollCount, convertUnrollMode(loopAttrs.UnrollAndJamEnable), + loopAttrs.UnrollAndJamCount, + convertStateToBool(loopAttrs.DistributeEnable), + convertFlag(loopAttrs.PipelineDisabled), + loopAttrs.PipelineInitiationInterval, convertFlag(loopAttrs.LICMDisabled), + convertFlag(loopAttrs.IsParallel), loopAttrs.CodeAlign); + + if (hints.isEmpty()) + return {}; + return hints; +} + mlir::LogicalResult CIRGenFunction::emitCXXForRangeStmt(const CXXForRangeStmt &s, ArrayRef<const Attr *> forAttrs) { cir::ForOp forOp; - // TODO(cir): pass in array of attributes. auto forStmtBuilder = [&]() -> mlir::LogicalResult { mlir::LogicalResult loopRes = mlir::success(); // Evaluate the first pieces before the loop. @@ -1010,7 +1082,10 @@ CIRGenFunction::emitCXXForRangeStmt(const CXXForRangeStmt &s, if (emitStmt(s.getEndStmt(), /*useCurrentScope=*/true).failed()) return mlir::failure(); - assert(!cir::MissingFeatures::loopInfoStack()); + // FIXME: Classic codegen never sets 'must progress' on these loops, but + // probably could? + cir::LoopHintAttr loopHints = + getLoopHints(forAttrs, /*mustProgress=*/false); forOp = builder.createFor( getLoc(s.getSourceRange()), @@ -1041,6 +1116,7 @@ CIRGenFunction::emitCXXForRangeStmt(const CXXForRangeStmt &s, loopRes = mlir::failure(); builder.createYield(loc); }); + forOp.setLoopHintsAttr(loopHints); return loopRes; }; @@ -1064,18 +1140,19 @@ CIRGenFunction::emitCXXForRangeStmt(const CXXForRangeStmt &s, return mlir::success(); } -mlir::LogicalResult CIRGenFunction::emitForStmt(const ForStmt &s) { +mlir::LogicalResult CIRGenFunction::emitForStmt(const ForStmt &s, + ArrayRef<const Attr *> attrs) { cir::ForOp forOp; - // TODO: pass in an array of attributes. auto forStmtBuilder = [&]() -> mlir::LogicalResult { mlir::LogicalResult loopRes = mlir::success(); // Evaluate the first part before the loop. if (s.getInit()) if (emitStmt(s.getInit(), /*useCurrentScope=*/true).failed()) return mlir::failure(); - assert(!cir::MissingFeatures::loopInfoStack()); - checkIfLoopMustProgress(s.getCond(), CodeGenUtils::hasEmptyLoopBody(s)); + bool mustProgress = + checkIfLoopMustProgress(s.getCond(), CodeGenUtils::hasEmptyLoopBody(s)); + cir::LoopHintAttr loopHints = getLoopHints(attrs, mustProgress); // A condition variable's lifetime is a single iteration, so capture its // destructor and lifetime-end cleanups, and those of the temporaries that @@ -1143,6 +1220,8 @@ mlir::LogicalResult CIRGenFunction::emitForStmt(const ForStmt &s) { forOp = builder.createFor(getLoc(s.getSourceRange()), condBuilder, bodyBuilder, stepBuilder); } + + forOp.setLoopHintsAttr(loopHints); return loopRes; }; @@ -1162,14 +1241,15 @@ mlir::LogicalResult CIRGenFunction::emitForStmt(const ForStmt &s) { return mlir::success(); } -mlir::LogicalResult CIRGenFunction::emitDoStmt(const DoStmt &s) { +mlir::LogicalResult CIRGenFunction::emitDoStmt(const DoStmt &s, + ArrayRef<const Attr *> attrs) { cir::DoWhileOp doWhileOp; - // TODO: pass in array of attributes. auto doStmtBuilder = [&]() -> mlir::LogicalResult { mlir::LogicalResult loopRes = mlir::success(); - assert(!cir::MissingFeatures::loopInfoStack()); - checkIfLoopMustProgress(s.getCond(), CodeGenUtils::hasEmptyLoopBody(s)); + bool mustProgress = + checkIfLoopMustProgress(s.getCond(), CodeGenUtils::hasEmptyLoopBody(s)); + cir::LoopHintAttr loopHints = getLoopHints(attrs, mustProgress); doWhileOp = builder.createDoWhile( getLoc(s.getSourceRange()), @@ -1191,6 +1271,7 @@ mlir::LogicalResult CIRGenFunction::emitDoStmt(const DoStmt &s) { loopRes = mlir::failure(); emitStopPoint(&s); }); + doWhileOp.setLoopHintsAttr(loopHints); return loopRes; }; @@ -1210,14 +1291,16 @@ mlir::LogicalResult CIRGenFunction::emitDoStmt(const DoStmt &s) { return mlir::success(); } -mlir::LogicalResult CIRGenFunction::emitWhileStmt(const WhileStmt &s) { +mlir::LogicalResult +CIRGenFunction::emitWhileStmt(const WhileStmt &s, + ArrayRef<const Attr *> attrs) { cir::WhileOp whileOp; - // TODO: pass in array of attributes. auto whileStmtBuilder = [&]() -> mlir::LogicalResult { mlir::LogicalResult loopRes = mlir::success(); - assert(!cir::MissingFeatures::loopInfoStack()); - checkIfLoopMustProgress(s.getCond(), CodeGenUtils::hasEmptyLoopBody(s)); + bool mustProgress = + checkIfLoopMustProgress(s.getCond(), CodeGenUtils::hasEmptyLoopBody(s)); + cir::LoopHintAttr loopHints = getLoopHints(attrs, mustProgress); // A condition variable's lifetime is a single iteration, so capture its // destructor and lifetime-end cleanups, and those of the temporaries that @@ -1272,6 +1355,7 @@ mlir::LogicalResult CIRGenFunction::emitWhileStmt(const WhileStmt &s) { whileOp = builder.createWhile(getLoc(s.getSourceRange()), condBuilder, bodyBuilder); } + whileOp.setLoopHintsAttr(loopHints); return loopRes; }; diff --git a/clang/lib/CodeGen/CGLoopInfo.cpp b/clang/lib/CodeGen/CGLoopInfo.cpp index 14e13af69b011..61cb14793d268 100644 --- a/clang/lib/CodeGen/CGLoopInfo.cpp +++ b/clang/lib/CodeGen/CGLoopInfo.cpp @@ -432,36 +432,6 @@ SmallVector<Metadata *, 4> clang::CodeGen::LoopInfo::createMetadata( return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms); } -LoopAttributes::LoopAttributes(bool IsParallel) - : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), - UnrollEnable(LoopAttributes::Unspecified), - UnrollAndJamEnable(LoopAttributes::Unspecified), - VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0), - VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0), - UnrollCount(0), UnrollAndJamCount(0), - DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false), - LICMDisabled(false), PipelineInitiationInterval(0), CodeAlign(0), - MustProgress(false) {} - -void LoopAttributes::clear() { - IsParallel = false; - VectorizeWidth = 0; - VectorizeScalable = LoopAttributes::Unspecified; - InterleaveCount = 0; - UnrollCount = 0; - UnrollAndJamCount = 0; - VectorizeEnable = LoopAttributes::Unspecified; - UnrollEnable = LoopAttributes::Unspecified; - UnrollAndJamEnable = LoopAttributes::Unspecified; - VectorizePredicateEnable = LoopAttributes::Unspecified; - DistributeEnable = LoopAttributes::Unspecified; - PipelineDisabled = false; - LICMDisabled = false; - PipelineInitiationInterval = 0; - CodeAlign = 0; - MustProgress = false; -} - clang::CodeGen::LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, const llvm::DebugLoc &StartLoc, @@ -598,239 +568,9 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, ArrayRef<const clang::Attr *> Attrs, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, bool MustProgress) { - // Identify loop hint attributes from Attrs. - for (const auto *Attr : Attrs) { - const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); - const OpenCLUnrollHintAttr *OpenCLHint = - dyn_cast<OpenCLUnrollHintAttr>(Attr); - const HLSLLoopHintAttr *HLSLLoopHint = dyn_cast<HLSLLoopHintAttr>(Attr); - // Skip non loop hint attributes - if (!LH && !OpenCLHint && !HLSLLoopHint) { - continue; - } - - LoopHintAttr::OptionType Option = LoopHintAttr::Unroll; - LoopHintAttr::LoopHintState State = LoopHintAttr::Disable; - unsigned ValueInt = 1; - // Translate opencl_unroll_hint attribute argument to - // equivalent LoopHintAttr enums. - // OpenCL v2.0 s6.11.5: - // 0 - enable unroll (no argument). - // 1 - disable unroll. - // other positive integer n - unroll by n. - if (OpenCLHint) { - ValueInt = OpenCLHint->getUnrollHint(); - if (ValueInt == 0) { - State = LoopHintAttr::Enable; - } else if (ValueInt != 1) { - Option = LoopHintAttr::UnrollCount; - State = LoopHintAttr::Numeric; - } - } else if (HLSLLoopHint) { - ValueInt = HLSLLoopHint->getDirective(); - if (HLSLLoopHint->getSemanticSpelling() == - HLSLLoopHintAttr::Spelling::Microsoft_unroll) { - if (ValueInt == 0) - State = LoopHintAttr::Enable; - if (ValueInt > 0) { - Option = LoopHintAttr::UnrollCount; - State = LoopHintAttr::Numeric; - } - } - } else if (LH) { - auto *ValueExpr = LH->getValue(); - if (ValueExpr) { - llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx); - ValueInt = ValueAPS.getSExtValue(); - } - - Option = LH->getOption(); - State = LH->getState(); - } - switch (State) { - case LoopHintAttr::Disable: - switch (Option) { - case LoopHintAttr::Vectorize: - // Disable vectorization by specifying a width of 1 and disabling - // scalable vectorization explicitly. - setVectorizeWidth(1); - setVectorizeScalable(LoopAttributes::Disable); - break; - case LoopHintAttr::Interleave: - // Disable interleaving by speciyfing a count of 1. - setInterleaveCount(1); - break; - case LoopHintAttr::Unroll: - setUnrollState(LoopAttributes::Disable); - break; - case LoopHintAttr::UnrollAndJam: - setUnrollAndJamState(LoopAttributes::Disable); - break; - case LoopHintAttr::VectorizePredicate: - setVectorizePredicateState(LoopAttributes::Disable); - break; - case LoopHintAttr::Distribute: - setDistributeState(false); - break; - case LoopHintAttr::PipelineDisabled: - setPipelineDisabled(true); - break; - case LoopHintAttr::LICMDisabled: - setLICMDisabled(true); - break; - case LoopHintAttr::UnrollCount: - case LoopHintAttr::UnrollAndJamCount: - case LoopHintAttr::VectorizeWidth: - case LoopHintAttr::InterleaveCount: - case LoopHintAttr::PipelineInitiationInterval: - llvm_unreachable("Options cannot be disabled."); - break; - } - break; - case LoopHintAttr::Enable: - switch (Option) { - case LoopHintAttr::Vectorize: - case LoopHintAttr::Interleave: - setVectorizeEnable(true); - break; - case LoopHintAttr::Unroll: - setUnrollState(LoopAttributes::Enable); - break; - case LoopHintAttr::UnrollAndJam: - setUnrollAndJamState(LoopAttributes::Enable); - break; - case LoopHintAttr::VectorizePredicate: - setVectorizePredicateState(LoopAttributes::Enable); - break; - case LoopHintAttr::Distribute: - setDistributeState(true); - break; - case LoopHintAttr::UnrollCount: - case LoopHintAttr::UnrollAndJamCount: - case LoopHintAttr::VectorizeWidth: - case LoopHintAttr::InterleaveCount: - case LoopHintAttr::PipelineDisabled: - case LoopHintAttr::PipelineInitiationInterval: - case LoopHintAttr::LICMDisabled: - llvm_unreachable("Options cannot enabled."); - break; - } - break; - case LoopHintAttr::AssumeSafety: - switch (Option) { - case LoopHintAttr::Vectorize: - case LoopHintAttr::Interleave: - // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. - setParallel(true); - setVectorizeEnable(true); - break; - case LoopHintAttr::Unroll: - case LoopHintAttr::UnrollAndJam: - case LoopHintAttr::VectorizePredicate: - case LoopHintAttr::UnrollCount: - case LoopHintAttr::UnrollAndJamCount: - case LoopHintAttr::VectorizeWidth: - case LoopHintAttr::InterleaveCount: - case LoopHintAttr::Distribute: - case LoopHintAttr::PipelineDisabled: - case LoopHintAttr::PipelineInitiationInterval: - case LoopHintAttr::LICMDisabled: - llvm_unreachable("Options cannot be used to assume mem safety."); - break; - } - break; - case LoopHintAttr::Full: - switch (Option) { - case LoopHintAttr::Unroll: - setUnrollState(LoopAttributes::Full); - break; - case LoopHintAttr::UnrollAndJam: - setUnrollAndJamState(LoopAttributes::Full); - break; - case LoopHintAttr::Vectorize: - case LoopHintAttr::Interleave: - case LoopHintAttr::UnrollCount: - case LoopHintAttr::UnrollAndJamCount: - case LoopHintAttr::VectorizeWidth: - case LoopHintAttr::InterleaveCount: - case LoopHintAttr::Distribute: - case LoopHintAttr::PipelineDisabled: - case LoopHintAttr::PipelineInitiationInterval: - case LoopHintAttr::VectorizePredicate: - case LoopHintAttr::LICMDisabled: - llvm_unreachable("Options cannot be used with 'full' hint."); - break; - } - break; - case LoopHintAttr::FixedWidth: - case LoopHintAttr::ScalableWidth: - switch (Option) { - case LoopHintAttr::VectorizeWidth: - setVectorizeScalable(State == LoopHintAttr::ScalableWidth - ? LoopAttributes::Enable - : LoopAttributes::Disable); - if (LH->getValue()) - setVectorizeWidth(ValueInt); - break; - default: - llvm_unreachable("Options cannot be used with 'scalable' hint."); - break; - } - break; - case LoopHintAttr::Numeric: - switch (Option) { - case LoopHintAttr::InterleaveCount: - setInterleaveCount(ValueInt); - break; - case LoopHintAttr::UnrollCount: - setUnrollCount(ValueInt); - break; - case LoopHintAttr::UnrollAndJamCount: - setUnrollAndJamCount(ValueInt); - break; - case LoopHintAttr::PipelineInitiationInterval: - setPipelineInitiationInterval(ValueInt); - break; - case LoopHintAttr::Unroll: - case LoopHintAttr::UnrollAndJam: - case LoopHintAttr::VectorizePredicate: - case LoopHintAttr::Vectorize: - case LoopHintAttr::VectorizeWidth: - case LoopHintAttr::Interleave: - case LoopHintAttr::Distribute: - case LoopHintAttr::PipelineDisabled: - case LoopHintAttr::LICMDisabled: - llvm_unreachable("Options cannot be assigned a value."); - break; - } - break; - } - } - - // Identify loop attribute 'code_align' from Attrs. - // For attribute code_align: - // n - 'llvm.loop.align i32 n' metadata will be emitted. - // A source-level [[clang::code_align]] attribute takes precedence over the - // -falign-loops=N command-line default. - if (const auto *CodeAlign = getSpecificAttr<CodeAlignAttr>(Attrs)) { - const auto *CE = cast<ConstantExpr>(CodeAlign->getAlignment()); - llvm::APSInt ArgVal = CE->getResultAsAPSInt(); - setCodeAlign(ArgVal.getSExtValue()); - } else if (CGOpts.LoopAlignment) { - setCodeAlign(CGOpts.LoopAlignment); - } - - setMustProgress(MustProgress); - - if (CGOpts.OptimizationLevel > 0) - // Disable unrolling for the loop, if unrolling is disabled (via - // -fno-unroll-loops) and no pragmas override the decision. - if (!CGOpts.UnrollLoops && - (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified && - StagedAttrs.UnrollCount == 0)) - setUnrollState(LoopAttributes::Disable); - /// Stage the attributes. + clang::CodeGenUtils::updateLoopAttributes(StagedAttrs, Ctx, CGOpts, Attrs, + MustProgress); push(Header, StartLoc, EndLoc); } diff --git a/clang/lib/CodeGen/CGLoopInfo.h b/clang/lib/CodeGen/CGLoopInfo.h index 065d489fb9bb2..d287ec52ed52e 100644 --- a/clang/lib/CodeGen/CGLoopInfo.h +++ b/clang/lib/CodeGen/CGLoopInfo.h @@ -14,6 +14,7 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_CGLOOPINFO_H #define LLVM_CLANG_LIB_CODEGEN_CGLOOPINFO_H +#include "clang/CodeGenUtils/LoopAttributes.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/DebugLoc.h" @@ -32,62 +33,9 @@ class ASTContext; class CodeGenOptions; namespace CodeGen { -/// Attributes that may be specified on loops. -struct LoopAttributes { - explicit LoopAttributes(bool IsParallel = false); - void clear(); - - /// Generate llvm.loop.parallel metadata for loads and stores. - bool IsParallel; - - /// State of loop vectorization or unrolling. - enum LVEnableState { Unspecified, Enable, Disable, Full }; - - /// Value for llvm.loop.vectorize.enable metadata. - LVEnableState VectorizeEnable; - - /// Value for llvm.loop.unroll.* metadata (enable, disable, or full). - LVEnableState UnrollEnable; - - /// Value for llvm.loop.unroll_and_jam.* metadata (enable, disable, or full). - LVEnableState UnrollAndJamEnable; - - /// Value for llvm.loop.vectorize.predicate metadata - LVEnableState VectorizePredicateEnable; - - /// Value for llvm.loop.vectorize.width metadata. - unsigned VectorizeWidth; - - // Value for llvm.loop.vectorize.scalable.enable - LVEnableState VectorizeScalable; - - /// Value for llvm.loop.interleave.count metadata. - unsigned InterleaveCount; - - /// llvm.unroll. - unsigned UnrollCount; - - /// llvm.unroll. - unsigned UnrollAndJamCount; - - /// Value for llvm.loop.distribute.enable metadata. - LVEnableState DistributeEnable; - - /// Value for llvm.loop.pipeline.disable metadata. - bool PipelineDisabled; - - /// Value for llvm.licm.disable metadata. - bool LICMDisabled; - - /// Value for llvm.loop.pipeline.iicount metadata. - unsigned PipelineInitiationInterval; - - /// Value for 'llvm.loop.align' metadata. - unsigned CodeAlign; - - /// Value for whether the loop is required to make progress. - bool MustProgress; -}; +/// Attributes that may be specified on loops. The state, and how it is +/// determined from the loop's attributes, is shared with CIR CodeGen. +using LoopAttributes = clang::CodeGenUtils::LoopAttributes; /// Information used when generating a structured loop. class LoopInfo { diff --git a/clang/lib/CodeGenUtils/CMakeLists.txt b/clang/lib/CodeGenUtils/CMakeLists.txt index dc701f8dbc72f..1a836bcab29c1 100644 --- a/clang/lib/CodeGenUtils/CMakeLists.txt +++ b/clang/lib/CodeGenUtils/CMakeLists.txt @@ -13,6 +13,7 @@ add_clang_library(clangCodeGenUtils ExprUtils.cpp FunctionUtils.cpp ItaniumCXXABIUtils.cpp + LoopAttributes.cpp ModuleUtils.cpp RecordLayoutUtils.cpp StmtUtils.cpp diff --git a/clang/lib/CodeGenUtils/LoopAttributes.cpp b/clang/lib/CodeGenUtils/LoopAttributes.cpp new file mode 100644 index 0000000000000..d8dd0dad9eca4 --- /dev/null +++ b/clang/lib/CodeGenUtils/LoopAttributes.cpp @@ -0,0 +1,285 @@ +//===--- LoopAttributes.cpp - Shared loop hint state ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/CodeGenUtils/LoopAttributes.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/Attr.h" +#include "clang/AST/Expr.h" +#include "clang/Basic/CodeGenOptions.h" +#include "llvm/Support/ErrorHandling.h" + +namespace clang::CodeGenUtils { + +LoopAttributes::LoopAttributes(bool IsParallel) + : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), + UnrollEnable(LoopAttributes::Unspecified), + UnrollAndJamEnable(LoopAttributes::Unspecified), + VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0), + VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0), + UnrollCount(0), UnrollAndJamCount(0), + DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false), + LICMDisabled(false), PipelineInitiationInterval(0), CodeAlign(0), + MustProgress(false) {} + +void LoopAttributes::clear() { + IsParallel = false; + VectorizeWidth = 0; + VectorizeScalable = LoopAttributes::Unspecified; + InterleaveCount = 0; + UnrollCount = 0; + UnrollAndJamCount = 0; + VectorizeEnable = LoopAttributes::Unspecified; + UnrollEnable = LoopAttributes::Unspecified; + UnrollAndJamEnable = LoopAttributes::Unspecified; + VectorizePredicateEnable = LoopAttributes::Unspecified; + DistributeEnable = LoopAttributes::Unspecified; + PipelineDisabled = false; + LICMDisabled = false; + PipelineInitiationInterval = 0; + CodeAlign = 0; + MustProgress = false; +} + +void updateLoopAttributes(LoopAttributes &Attrs, ASTContext &Ctx, + const CodeGenOptions &CGOpts, + llvm::ArrayRef<const Attr *> StmtAttrs, + bool MustProgress) { + // Identify loop hint attributes from Attrs. + for (const auto *Attr : StmtAttrs) { + const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); + const OpenCLUnrollHintAttr *OpenCLHint = + dyn_cast<OpenCLUnrollHintAttr>(Attr); + const HLSLLoopHintAttr *HLSLLoopHint = dyn_cast<HLSLLoopHintAttr>(Attr); + // Skip non loop hint attributes + if (!LH && !OpenCLHint && !HLSLLoopHint) { + continue; + } + + LoopHintAttr::OptionType Option = LoopHintAttr::Unroll; + LoopHintAttr::LoopHintState State = LoopHintAttr::Disable; + unsigned ValueInt = 1; + // Translate opencl_unroll_hint attribute argument to + // equivalent LoopHintAttr enums. + // OpenCL v2.0 s6.11.5: + // 0 - enable unroll (no argument). + // 1 - disable unroll. + // other positive integer n - unroll by n. + if (OpenCLHint) { + ValueInt = OpenCLHint->getUnrollHint(); + if (ValueInt == 0) { + State = LoopHintAttr::Enable; + } else if (ValueInt != 1) { + Option = LoopHintAttr::UnrollCount; + State = LoopHintAttr::Numeric; + } + } else if (HLSLLoopHint) { + ValueInt = HLSLLoopHint->getDirective(); + if (HLSLLoopHint->getSemanticSpelling() == + HLSLLoopHintAttr::Spelling::Microsoft_unroll) { + if (ValueInt == 0) + State = LoopHintAttr::Enable; + if (ValueInt > 0) { + Option = LoopHintAttr::UnrollCount; + State = LoopHintAttr::Numeric; + } + } + } else if (LH) { + auto *ValueExpr = LH->getValue(); + if (ValueExpr) { + llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx); + ValueInt = ValueAPS.getSExtValue(); + } + + Option = LH->getOption(); + State = LH->getState(); + } + switch (State) { + case LoopHintAttr::Disable: + switch (Option) { + case LoopHintAttr::Vectorize: + // Disable vectorization by specifying a width of 1 and disabling + // scalable vectorization explicitly. + Attrs.VectorizeWidth = 1; + Attrs.VectorizeScalable = LoopAttributes::Disable; + break; + case LoopHintAttr::Interleave: + // Disable interleaving by speciyfing a count of 1. + Attrs.InterleaveCount = 1; + break; + case LoopHintAttr::Unroll: + Attrs.UnrollEnable = LoopAttributes::Disable; + break; + case LoopHintAttr::UnrollAndJam: + Attrs.UnrollAndJamEnable = LoopAttributes::Disable; + break; + case LoopHintAttr::VectorizePredicate: + Attrs.VectorizePredicateEnable = LoopAttributes::Disable; + break; + case LoopHintAttr::Distribute: + Attrs.DistributeEnable = LoopAttributes::Disable; + break; + case LoopHintAttr::PipelineDisabled: + Attrs.PipelineDisabled = true; + break; + case LoopHintAttr::LICMDisabled: + Attrs.LICMDisabled = true; + break; + case LoopHintAttr::UnrollCount: + case LoopHintAttr::UnrollAndJamCount: + case LoopHintAttr::VectorizeWidth: + case LoopHintAttr::InterleaveCount: + case LoopHintAttr::PipelineInitiationInterval: + llvm_unreachable("Options cannot be disabled."); + break; + } + break; + case LoopHintAttr::Enable: + switch (Option) { + case LoopHintAttr::Vectorize: + case LoopHintAttr::Interleave: + Attrs.VectorizeEnable = LoopAttributes::Enable; + break; + case LoopHintAttr::Unroll: + Attrs.UnrollEnable = LoopAttributes::Enable; + break; + case LoopHintAttr::UnrollAndJam: + Attrs.UnrollAndJamEnable = LoopAttributes::Enable; + break; + case LoopHintAttr::VectorizePredicate: + Attrs.VectorizePredicateEnable = LoopAttributes::Enable; + break; + case LoopHintAttr::Distribute: + Attrs.DistributeEnable = LoopAttributes::Enable; + break; + case LoopHintAttr::UnrollCount: + case LoopHintAttr::UnrollAndJamCount: + case LoopHintAttr::VectorizeWidth: + case LoopHintAttr::InterleaveCount: + case LoopHintAttr::PipelineDisabled: + case LoopHintAttr::PipelineInitiationInterval: + case LoopHintAttr::LICMDisabled: + llvm_unreachable("Options cannot enabled."); + break; + } + break; + case LoopHintAttr::AssumeSafety: + switch (Option) { + case LoopHintAttr::Vectorize: + case LoopHintAttr::Interleave: + // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. + Attrs.IsParallel = true; + Attrs.VectorizeEnable = LoopAttributes::Enable; + break; + case LoopHintAttr::Unroll: + case LoopHintAttr::UnrollAndJam: + case LoopHintAttr::VectorizePredicate: + case LoopHintAttr::UnrollCount: + case LoopHintAttr::UnrollAndJamCount: + case LoopHintAttr::VectorizeWidth: + case LoopHintAttr::InterleaveCount: + case LoopHintAttr::Distribute: + case LoopHintAttr::PipelineDisabled: + case LoopHintAttr::PipelineInitiationInterval: + case LoopHintAttr::LICMDisabled: + llvm_unreachable("Options cannot be used to assume mem safety."); + break; + } + break; + case LoopHintAttr::Full: + switch (Option) { + case LoopHintAttr::Unroll: + Attrs.UnrollEnable = LoopAttributes::Full; + break; + case LoopHintAttr::UnrollAndJam: + Attrs.UnrollAndJamEnable = LoopAttributes::Full; + break; + case LoopHintAttr::Vectorize: + case LoopHintAttr::Interleave: + case LoopHintAttr::UnrollCount: + case LoopHintAttr::UnrollAndJamCount: + case LoopHintAttr::VectorizeWidth: + case LoopHintAttr::InterleaveCount: + case LoopHintAttr::Distribute: + case LoopHintAttr::PipelineDisabled: + case LoopHintAttr::PipelineInitiationInterval: + case LoopHintAttr::VectorizePredicate: + case LoopHintAttr::LICMDisabled: + llvm_unreachable("Options cannot be used with 'full' hint."); + break; + } + break; + case LoopHintAttr::FixedWidth: + case LoopHintAttr::ScalableWidth: + switch (Option) { + case LoopHintAttr::VectorizeWidth: + Attrs.VectorizeScalable = State == LoopHintAttr::ScalableWidth + ? LoopAttributes::Enable + : LoopAttributes::Disable; + if (LH->getValue()) + Attrs.VectorizeWidth = ValueInt; + break; + default: + llvm_unreachable("Options cannot be used with 'scalable' hint."); + break; + } + break; + case LoopHintAttr::Numeric: + switch (Option) { + case LoopHintAttr::InterleaveCount: + Attrs.InterleaveCount = ValueInt; + break; + case LoopHintAttr::UnrollCount: + Attrs.UnrollCount = ValueInt; + break; + case LoopHintAttr::UnrollAndJamCount: + Attrs.UnrollAndJamCount = ValueInt; + break; + case LoopHintAttr::PipelineInitiationInterval: + Attrs.PipelineInitiationInterval = ValueInt; + break; + case LoopHintAttr::Unroll: + case LoopHintAttr::UnrollAndJam: + case LoopHintAttr::VectorizePredicate: + case LoopHintAttr::Vectorize: + case LoopHintAttr::VectorizeWidth: + case LoopHintAttr::Interleave: + case LoopHintAttr::Distribute: + case LoopHintAttr::PipelineDisabled: + case LoopHintAttr::LICMDisabled: + llvm_unreachable("Options cannot be assigned a value."); + break; + } + break; + } + } + + // Identify loop attribute 'code_align' from Attrs. + // For attribute code_align: + // n - 'llvm.loop.align i32 n' metadata will be emitted. + // A source-level [[clang::code_align]] attribute takes precedence over the + // -falign-loops=N command-line default. + if (const auto *CodeAlign = getSpecificAttr<CodeAlignAttr>(StmtAttrs)) { + const auto *CE = cast<ConstantExpr>(CodeAlign->getAlignment()); + llvm::APSInt ArgVal = CE->getResultAsAPSInt(); + Attrs.CodeAlign = ArgVal.getSExtValue(); + } else if (CGOpts.LoopAlignment) { + Attrs.CodeAlign = CGOpts.LoopAlignment; + } + + Attrs.MustProgress = MustProgress; + + if (CGOpts.OptimizationLevel > 0) + // Disable unrolling for the loop, if unrolling is disabled (via + // -fno-unroll-loops) and no pragmas override the decision. + if (!CGOpts.UnrollLoops && + (Attrs.UnrollEnable == LoopAttributes::Unspecified && + Attrs.UnrollCount == 0)) + Attrs.UnrollEnable = LoopAttributes::Disable; +} + +} // namespace clang::CodeGenUtils diff --git a/clang/test/CIR/CodeGen/loop-hints-stds.cpp b/clang/test/CIR/CodeGen/loop-hints-stds.cpp new file mode 100644 index 0000000000000..f23e8076041cb --- /dev/null +++ b/clang/test/CIR/CodeGen/loop-hints-stds.cpp @@ -0,0 +1,84 @@ +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir +// RUN: FileCheck --input-file=%t.cir %s --check-prefixes=CIR,CXX17 +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t-98.cir +// RUN: FileCheck --input-file=%t-98.cir %s --check-prefixes=CIR,CXX98 + +// C++17 has different mustprogress guarantees that dont' exist before this, so +// mustprogress changes behavior. + +void use(int); + +// CIR-LABEL: cir.func {{.*}} @_Z5hintsi +void hints(int n) { +// CXX17: } loop_hints < must_progress unroll_count = 4> +// CXX98: } loop_hints < unroll_count = 4> +#pragma GCC unroll 4 + for (int i = 0; i < n; ++i) + use(i); +#pragma clang loop unroll(disable) + for (int i = 0; i < n; ++i) + use(i); +// CXX17: } loop_hints < must_progress unroll = disable> +// CXX98: } loop_hints < unroll = disable> +#pragma clang loop unroll(enable) + for (int i = 0; i < n; ++i) + use(i); +// CXX17: } loop_hints < must_progress unroll = enable> +// CXX98: } loop_hints < unroll = enable> +#pragma clang loop interleave(enable) + for (int i = 0; i < n; ++i) + use(i); +// CXX17: } loop_hints < must_progress vectorize_enable> +// CXX98: } loop_hints < vectorize_enable> +#pragma clang loop vectorize_width(4) unroll_count(2) + for (int i = 0; i < n; ++i) + use(i); +// CXX17: } loop_hints < must_progress vectorize_width = 4 vectorize_scalable = false unroll_count = 2> +// CXX98: } loop_hints < vectorize_width = 4 vectorize_scalable = false unroll_count = 2> + for (int i = 0; i < n; ++i) + use(i); +// CXX17: } loop_hints < must_progress> +// CXX98-NOT: loop_hints + +// CIR: cir.return +} + + +// CIR-LABEL: cir.func {{.*}} @_Z6nestedi +void nested(int n) { +#pragma clang loop unroll(full) + for (int i = 0; i < n; ++i) { +#pragma clang loop vectorize(enable) + for (int j = 0; j < n; ++j) + use(j); +// CXX17: } loop_hints < must_progress vectorize_enable> +// CXX98: } loop_hints < vectorize_enable> + } +// CXX17: } loop_hints < must_progress unroll = full> +// CXX98: } loop_hints < unroll = full> +} + + +template <typename T> void tmpl(T n) { +#pragma clang loop unroll_count(3) + while (n--) + use(n); +} + +void instantiate() { + tmpl(4); +// CIR-LABEL: cir.func {{.*}} @_Z4tmplIiEvT_ +// CXX17: } loop_hints < must_progress unroll_count = 3> +// CXX98: } loop_hints < unroll_count = 3> + auto l = [](int n) { +#pragma nounroll + do { + use(n); + } while (--n); + }; + l(2); +// CIR-LABEL: cir.func {{.*}} @_ZZ11instantiatevENK3$_0clEi +// CXX17: } loop_hints < must_progress unroll = disable> +// CXX98: } loop_hints < unroll = disable> +} + diff --git a/clang/test/CIR/CodeGen/loop-hints.c b/clang/test/CIR/CodeGen/loop-hints.c new file mode 100644 index 0000000000000..98b9e9291b296 --- /dev/null +++ b/clang/test/CIR/CodeGen/loop-hints.c @@ -0,0 +1,52 @@ +// RUN: %clang_cc1 -std=c99 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t-c99.cir +// RUN: FileCheck --input-file=%t-c99.cir %s --check-prefixes=CIR,C99 +// RUN: %clang_cc1 -std=c11 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t-c11.cir +// RUN: FileCheck --input-file=%t-c11.cir %s --check-prefixes=CIR,C11 +// RUN: %clang_cc1 -std=c99 -ffinite-loops -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t-finite.cir +// RUN: FileCheck --input-file=%t-finite.cir %s --check-prefixes=CIR,FINITE +// RUN: %clang_cc1 -std=c11 -fno-finite-loops -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t-nofinite.cir +// RUN: FileCheck --input-file=%t-nofinite.cir %s --check-prefixes=CIR,NOFINITE + +// In C11 a loop must make progress unless its controlling expression is a +// constant. C99 loops don't, unless -ffinite-loops says so, and +// -fno-finite-loops turns it off. A trivial infinite loop never does. + +void use(int); + +void f(int n) { + for (int i = 0; i < n; ++i) + use(i); + while (n) + --n; + do { + --n; + } while (n > 0); + while (1) + use(n); + for (;;) + use(n); + while (1) { + } +} + +// CIR-LABEL: cir.func {{.*}} @f +// CIR: cir.for +// C99-NOT: loop_hints +// C11: } loop_hints < must_progress> +// FINITE: } loop_hints < must_progress> +// NOFINITE-NOT: loop_hints +// CIR: cir.while +// C11: } loop_hints < must_progress> +// FINITE: } loop_hints < must_progress> +// CIR: cir.do +// C11: } loop_hints < must_progress> +// FINITE: } loop_hints < must_progress> +// With a constant condition, only -ffinite-loops marks the loops, unless they +// are trivially empty. +// CIR: cir.while +// FINITE: } loop_hints < must_progress> +// CIR: cir.for +// FINITE: } loop_hints < must_progress> +// CIR: cir.while +// CIR-NOT: loop_hints +// CIR: cir.return diff --git a/clang/test/CIR/CodeGen/loop-hints.cpp b/clang/test/CIR/CodeGen/loop-hints.cpp new file mode 100644 index 0000000000000..706dec0baf11a --- /dev/null +++ b/clang/test/CIR/CodeGen/loop-hints.cpp @@ -0,0 +1,139 @@ +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir +// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CIR +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -fclangir -O2 -fno-unroll-loops -falign-loops=32 -emit-cir %s -o %t-opts.cir +// RUN: FileCheck --input-file=%t-opts.cir %s --check-prefix=OPTS + +void use(int); +int arr[3]; + +// C++11 and later loops are 'mustprogress', unless they are trivial infinite +// loops. +void must_progress(int n) { + for (int i = 0; i < n; ++i) + use(i); + while (n--) + use(n); + do { + use(n); + } while (n > 0); + // Like classic codegen, range-for loops are not 'mustprogress'. + for (int x : arr) + use(x); + for (;;) { + } +} + +// CIR-LABEL: cir.func {{.*}} @_Z13must_progressi +// CIR: cir.for +// CIR: } loop_hints < must_progress> +// CIR: cir.while +// CIR: } loop_hints < must_progress> +// CIR: cir.do +// CIR: } loop_hints < must_progress> +// CIR: cir.for +// CIR-NOT: loop_hints +// CIR: cir.for +// CIR-NOT: loop_hints +// CIR: cir.return + +void pragmas(int n) { +#pragma clang loop unroll_count(4) vectorize(enable) interleave_count(2) + for (int i = 0; i < n; ++i) + use(i); +#pragma nounroll + while (n) + --n; +#pragma unroll + do { + --n; + } while (n > 0); +#pragma unroll 8 + for (int i = 0; i < n; ++i) + use(i); +#pragma clang loop unroll(full) + for (int i = 0; i < 4; ++i) + use(i); +#pragma clang loop vectorize(disable) interleave(disable) + for (int i = 0; i < n; ++i) + use(i); +#pragma clang loop vectorize_width(8) + for (int x : arr) + use(x); +#pragma clang loop vectorize_width(4, scalable) + for (int i = 0; i < n; ++i) + use(i); +#pragma clang loop vectorize_width(4, fixed) + for (int i = 0; i < n; ++i) + use(i); +#pragma clang loop vectorize(assume_safety) + for (int i = 0; i < n; ++i) + use(i); +#pragma clang loop vectorize_predicate(enable) + for (int i = 0; i < n; ++i) + use(i); +#pragma clang loop vectorize_predicate(disable) + for (int i = 0; i < n; ++i) + use(i); +#pragma clang loop distribute(enable) + for (int i = 0; i < n; ++i) + use(i); +#pragma clang loop distribute(disable) + for (int i = 0; i < n; ++i) + use(i); +#pragma unroll_and_jam + for (int i = 0; i < n; ++i) + use(i); +#pragma nounroll_and_jam + for (int i = 0; i < n; ++i) + use(i); +#pragma unroll_and_jam(4) + for (int i = 0; i < n; ++i) + use(i); +#pragma clang loop pipeline(disable) + for (int i = 0; i < n; ++i) + use(i); +#pragma clang loop pipeline_initiation_interval(3) + for (int i = 0; i < n; ++i) + use(i); +#pragma clang loop licm(disable) + for (int i = 0; i < n; ++i) + use(i); + [[clang::code_align(16)]] + for (int i = 0; i < n; ++i) + use(i); +} + +// CIR-LABEL: cir.func {{.*}} @_Z7pragmasi +// CIR: } loop_hints < must_progress vectorize_enable interleave_count = 2 unroll_count = 4> +// CIR: } loop_hints < must_progress unroll = disable> +// CIR: } loop_hints < must_progress unroll = enable> +// CIR: } loop_hints < must_progress unroll_count = 8> +// CIR: } loop_hints < must_progress unroll = full> +// CIR: } loop_hints < must_progress vectorize_width = 1 vectorize_scalable = false interleave_count = 1> +// CIR: } loop_hints < vectorize_width = 8 vectorize_scalable = false> +// CIR: } loop_hints < must_progress vectorize_width = 4 vectorize_scalable = true> +// CIR: } loop_hints < must_progress vectorize_width = 4 vectorize_scalable = false> +// CIR: } loop_hints < must_progress vectorize_enable parallel> +// CIR: } loop_hints < must_progress vectorize_predicate = true> +// CIR: } loop_hints < must_progress vectorize_predicate = false> +// CIR: } loop_hints < must_progress distribute = true> +// CIR: } loop_hints < must_progress distribute = false> +// CIR: } loop_hints < must_progress unroll_and_jam = enable> +// CIR: } loop_hints < must_progress unroll_and_jam = disable> +// CIR: } loop_hints < must_progress unroll_and_jam_count = 4> +// CIR: } loop_hints < must_progress pipeline_disabled> +// CIR: } loop_hints < must_progress pipeline_initiation_interval = 3> +// CIR: } loop_hints < must_progress licm_disabled> +// CIR: } loop_hints < must_progress code_align = 16> + +// -falign-loops applies to every loop, unless [[clang::code_align]] overrides +// it. -fno-unroll-loops disables unrolling of every loop that no pragma +// overrides. +// OPTS-LABEL: cir.func {{.*}} @_Z7pragmasi +// OPTS: } loop_hints < must_progress vectorize_enable interleave_count = 2 unroll_count = 4 code_align = 32> +// OPTS: } loop_hints < must_progress unroll = disable code_align = 32> +// OPTS: } loop_hints < must_progress unroll = enable code_align = 32> +// OPTS: } loop_hints < must_progress unroll_count = 8 code_align = 32> +// OPTS: } loop_hints < must_progress unroll = full code_align = 32> +// OPTS: } loop_hints < must_progress vectorize_width = 1 vectorize_scalable = false interleave_count = 1 unroll = disable code_align = 32> +// OPTS: } loop_hints < must_progress unroll = disable code_align = 16> diff --git a/clang/test/CIR/CodeGenHLSL/loop-hints.hlsl b/clang/test/CIR/CodeGenHLSL/loop-hints.hlsl new file mode 100644 index 0000000000000..4aa683b00dd5e --- /dev/null +++ b/clang/test/CIR/CodeGenHLSL/loop-hints.hlsl @@ -0,0 +1,16 @@ +// RUN: %clang_cc1 -x hlsl -triple spirv-unknown-vulkan-library -fclangir -emit-cir %s -o %t.cir +// RUN: FileCheck --input-file=%t.cir %s + +void use(int); + +[numthreads(1, 1, 1)] void main() { + int n = 4; + [unroll] for (int i = 0; i < n; ++i) use(i); + [unroll(4)] for (int i = 0; i < n; ++i) use(i); + [loop] for (int i = 0; i < n; ++i) use(i); +} + +// CHECK-LABEL: cir.func {{.*}} @_Z4mainv +// CHECK: } loop_hints < must_progress unroll = enable> +// CHECK: } loop_hints < must_progress unroll_count = 4> +// CHECK: } loop_hints < must_progress unroll = disable> diff --git a/clang/test/CIR/CodeGenOpenACC/loop.cpp b/clang/test/CIR/CodeGenOpenACC/loop.cpp index 65ab68c5adb6b..b0feb82387a50 100644 --- a/clang/test/CIR/CodeGenOpenACC/loop.cpp +++ b/clang/test/CIR/CodeGenOpenACC/loop.cpp @@ -26,7 +26,7 @@ extern "C" void acc_loop(int *A, int *B, int *C, int N) { // CHECK-NEXT: cir.yield // CHECK-NEXT: } step { // CHECK: cir.yield - // CHECK-NEXT: } loc + // CHECK-NEXT: } loop_hints < must_progress> loc // CHECK-NEXT: } loc // CHECK-NEXT: acc.yield // CHECK-NEXT: } independent loc diff --git a/clang/test/CIR/CodeGenOpenCL/loop-hints.cl b/clang/test/CIR/CodeGenOpenCL/loop-hints.cl new file mode 100644 index 0000000000000..dcf1ec86bc68e --- /dev/null +++ b/clang/test/CIR/CodeGenOpenCL/loop-hints.cl @@ -0,0 +1,18 @@ +// RUN: %clang_cc1 -x cl -triple spir64 -cl-std=CL2.0 -fclangir -emit-cir %s -o %t.cir +// RUN: FileCheck --input-file=%t.cir %s + +void use(int); + +kernel void k(int n) { + __attribute__((opencl_unroll_hint)) for (int i = 0; i < n; ++i) + use(i); + __attribute__((opencl_unroll_hint(1))) for (int i = 0; i < n; ++i) + use(i); + __attribute__((opencl_unroll_hint(8))) for (int i = 0; i < n; ++i) + use(i); +} + +// CHECK-LABEL: cir.func {{.*}} @k +// CHECK: } loop_hints < unroll = enable> +// CHECK: } loop_hints < unroll = disable> +// CHECK: } loop_hints < unroll_count = 8> diff --git a/clang/test/CIR/IR/invalid-loop-hints.cir b/clang/test/CIR/IR/invalid-loop-hints.cir new file mode 100644 index 0000000000000..2f09d8a26d96e --- /dev/null +++ b/clang/test/CIR/IR/invalid-loop-hints.cir @@ -0,0 +1,73 @@ +// RUN: cir-opt %s -verify-diagnostics -split-input-file + +cir.func @bad_bool(%c: !cir.bool) { + cir.while { + cir.condition(%c) + } do { + cir.yield + // expected-error@+2 {{expected attribute value}} + // expected-error@+1 {{failed to parse CIR_LoopHintAttr parameter 'vectorize_scalable'}} + } loop_hints < vectorize_scalable = maybe> + cir.return +} + +// ----- + +cir.func @bad_mode(%c: !cir.bool) { + cir.while { + cir.condition(%c) + } do { + cir.yield + // expected-error@+1 {{expected '>'}} + } loop_hints < unroll = sometimes> + cir.return +} + +// ----- + +cir.func @missing_value(%c: !cir.bool) { + cir.while { + cir.condition(%c) + } do { + cir.yield + // expected-error@+2 {{expected integer value}} + // expected-error@+1 {{failed to parse CIR_LoopHintAttr parameter 'vectorize_width'}} + } loop_hints < vectorize_width = > + cir.return +} + +// ----- + +cir.func @unknown_entry(%c: !cir.bool) { + cir.while { + cir.condition(%c) + } do { + cir.yield + // expected-error@+1 {{expected '>'}} + } loop_hints < bogus> + cir.return +} + +// ----- + +cir.func @out_of_order(%c: !cir.bool) { + cir.while { + cir.condition(%c) + } do { + cir.yield + // expected-error@+1 {{expected '>'}} + } loop_hints < unroll_count = 4 must_progress> + cir.return +} + +// ----- + +cir.func @repeated(%c: !cir.bool) { + cir.while { + cir.condition(%c) + } do { + cir.yield + // expected-error@+1 {{expected '>'}} + } loop_hints < must_progress must_progress> + cir.return +} diff --git a/clang/test/CIR/IR/loop-hints.cir b/clang/test/CIR/IR/loop-hints.cir new file mode 100644 index 0000000000000..3653d7d1b172b --- /dev/null +++ b/clang/test/CIR/IR/loop-hints.cir @@ -0,0 +1,40 @@ +// RUN: cir-opt %s | cir-opt | FileCheck %s + +cir.func @loops(%c: !cir.bool) { + // CHECK: cir.while { + cir.while { + cir.condition(%c) + } do { + cir.yield + // CHECK: } loop_hints < must_progress> + } loop_hints < must_progress> + + // CHECK: cir.do { + cir.do { + cir.yield + } while { + cir.condition(%c) + // CHECK: } loop_hints <> + } loop_hints <> + + // CHECK: cir.for : cond { + cir.for : cond { + cir.condition(%c) + } body { + cir.yield + } step { + cir.yield + // CHECK: } loop_hints < must_progress vectorize_enable vectorize_width = 4 vectorize_scalable = true vectorize_predicate = false interleave_count = 2 unroll = enable unroll_count = 8 distribute = true> + } loop_hints < must_progress vectorize_enable vectorize_width = 4 vectorize_scalable = true vectorize_predicate = false interleave_count = 2 unroll = enable unroll_count = 8 distribute = true> + + // CHECK: cir.for : cond { + cir.for : cond { + cir.condition(%c) + } body { + cir.yield + } step { + cir.yield + // CHECK: } loop_hints < unroll = disable unroll_and_jam = full unroll_and_jam_count = 2 pipeline_disabled pipeline_initiation_interval = 3 licm_disabled parallel code_align = 64> + } loop_hints < unroll = disable unroll_and_jam = full unroll_and_jam_count = 2 pipeline_disabled pipeline_initiation_interval = 3 licm_disabled parallel code_align = 64> + cir.return +} _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
