https://github.com/jhuber6 updated https://github.com/llvm/llvm-project/pull/225780
>From ad7ad53cfa780bd081934983f2eea969e1ef1ac7 Mon Sep 17 00:00:00 2001 From: Joseph Huber <[email protected]> Date: Mon, 21 Sep 2026 17:34:50 -0500 Subject: [PATCH 1/6] [LLVM] Add a ConcurrencySanitizer LLVM pass Summary: Add an LLVM ConcurrencySanitizer pass to instrument memory and atomic operations for a sampling race detector. Fundamentally the pass iterates through every candidate memory operation and adds a check above it. Unlike TSAN, we do not replace any operations, which heavily simplifies the instrumentation. We do not re-use the TSAN instrumentaiton because the appraoches are different enough and the passes simple enough that the extra code is not a large concern. More differences will be added in the future, so keeping them separate is easlier. --- .../Instrumentation/ConcurrencySanitizer.h | 29 + llvm/lib/Passes/PassBuilder.cpp | 1 + llvm/lib/Passes/PassRegistry.def | 2 + .../Transforms/Instrumentation/CMakeLists.txt | 1 + .../Instrumentation/ConcurrencySanitizer.cpp | 614 ++++++++++++++++++ .../ConcurrencySanitizer/address_space.ll | 106 +++ .../ConcurrencySanitizer/atomic.ll | 86 +++ .../ConcurrencySanitizer/basic.ll | 49 ++ .../ConcurrencySanitizer/callbacks.ll | 72 ++ .../ConcurrencySanitizer/function_control.ll | 59 ++ .../ConcurrencySanitizer/memintrinsics.ll | 46 ++ .../ConcurrencySanitizer/module.ll | 12 + .../ConcurrencySanitizer/no_sanitize.ll | 57 ++ .../ConcurrencySanitizer/selection.ll | 131 ++++ .../ConcurrencySanitizer/shared_alloca.ll | 23 + .../ConcurrencySanitizer/unusual_sizes.ll | 33 + 16 files changed, 1321 insertions(+) create mode 100644 llvm/include/llvm/Transforms/Instrumentation/ConcurrencySanitizer.h create mode 100644 llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp create mode 100644 llvm/test/Instrumentation/ConcurrencySanitizer/address_space.ll create mode 100644 llvm/test/Instrumentation/ConcurrencySanitizer/atomic.ll create mode 100644 llvm/test/Instrumentation/ConcurrencySanitizer/basic.ll create mode 100644 llvm/test/Instrumentation/ConcurrencySanitizer/callbacks.ll create mode 100644 llvm/test/Instrumentation/ConcurrencySanitizer/function_control.ll create mode 100644 llvm/test/Instrumentation/ConcurrencySanitizer/memintrinsics.ll create mode 100644 llvm/test/Instrumentation/ConcurrencySanitizer/module.ll create mode 100644 llvm/test/Instrumentation/ConcurrencySanitizer/no_sanitize.ll create mode 100644 llvm/test/Instrumentation/ConcurrencySanitizer/selection.ll create mode 100644 llvm/test/Instrumentation/ConcurrencySanitizer/shared_alloca.ll create mode 100644 llvm/test/Instrumentation/ConcurrencySanitizer/unusual_sizes.ll diff --git a/llvm/include/llvm/Transforms/Instrumentation/ConcurrencySanitizer.h b/llvm/include/llvm/Transforms/Instrumentation/ConcurrencySanitizer.h new file mode 100644 index 00000000000000..9bf3b8bde6a13b --- /dev/null +++ b/llvm/include/llvm/Transforms/Instrumentation/ConcurrencySanitizer.h @@ -0,0 +1,29 @@ +//===- ConcurrencySanitizer.h - ConcurrencySanitizer ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_CONCURRENCYSANITIZER_H +#define LLVM_TRANSFORMS_INSTRUMENTATION_CONCURRENCYSANITIZER_H + +#include "llvm/IR/PassManager.h" +#include "llvm/Support/Compiler.h" + +namespace llvm { + +struct ConcurrencySanitizerPass + : public RequiredPassInfoMixin<ConcurrencySanitizerPass> { + LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); +}; + +struct ModuleConcurrencySanitizerPass + : public RequiredPassInfoMixin<ModuleConcurrencySanitizerPass> { + LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + +} // namespace llvm + +#endif // LLVM_TRANSFORMS_INSTRUMENTATION_CONCURRENCYSANITIZER_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 30a6f75a1b86d5..773ac2ae632521 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -274,6 +274,7 @@ #include "llvm/Transforms/Instrumentation/AllocToken.h" #include "llvm/Transforms/Instrumentation/BoundsChecking.h" #include "llvm/Transforms/Instrumentation/CGProfile.h" +#include "llvm/Transforms/Instrumentation/ConcurrencySanitizer.h" #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h" #include "llvm/Transforms/Instrumentation/CopyProf.h" #include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index af4ce5029551db..c5b7001d550c0f 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -67,6 +67,7 @@ MODULE_PASS("coro-cleanup", CoroCleanupPass()) MODULE_PASS("coro-early", CoroEarlyPass()) MODULE_PASS("lower-comment-string", LowerCommentStringPass()) MODULE_PASS("cross-dso-cfi", CrossDSOCFIPass()) +MODULE_PASS("csan-module", ModuleConcurrencySanitizerPass()) MODULE_PASS("ctx-instr-gen", PGOInstrumentationGen(PGOInstrumentationType::CTXPROF)) MODULE_PASS("ctx-prof-flatten", @@ -429,6 +430,7 @@ FUNCTION_PASS("copyprof-stores", CopyProfStoresPass()) FUNCTION_PASS("coro-elide", CoroElidePass()) FUNCTION_PASS("correlated-propagation", CorrelatedValuePropagationPass()) FUNCTION_PASS("count-visits", CountVisitsPass()) +FUNCTION_PASS("csan", ConcurrencySanitizerPass()) FUNCTION_PASS("dce", DCEPass()) FUNCTION_PASS("declare-to-assign", llvm::AssignmentTrackingPass()) FUNCTION_PASS("dfa-jump-threading", DFAJumpThreadingPass()) diff --git a/llvm/lib/Transforms/Instrumentation/CMakeLists.txt b/llvm/lib/Transforms/Instrumentation/CMakeLists.txt index 7d2e658acb5c15..4d08d0fa42184b 100644 --- a/llvm/lib/Transforms/Instrumentation/CMakeLists.txt +++ b/llvm/lib/Transforms/Instrumentation/CMakeLists.txt @@ -25,6 +25,7 @@ add_llvm_component_library(LLVMInstrumentation SanitizerBinaryMetadata.cpp ValueProfileCollector.cpp ThreadSanitizer.cpp + ConcurrencySanitizer.cpp TypeSanitizer.cpp HWAddressSanitizer.cpp RealtimeSanitizer.cpp diff --git a/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp new file mode 100644 index 00000000000000..0a14d67f9f5bc1 --- /dev/null +++ b/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp @@ -0,0 +1,614 @@ +//===- ConcurrencySanitizer.cpp - watchpoint race detector ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements instrumentation for ConcurrencySanitizer, a sampling +// data-race detector. Memory accesses are preceded by runtime probes while the +// original operations remain unchanged. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Instrumentation/ConcurrencySanitizer.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/bit.h" +#include "llvm/Analysis/CaptureTracking.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Module.h" +#include "llvm/ProfileData/InstrProf.h" +#include "llvm/Support/AMDGPUAddrSpace.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/NVPTXAddrSpace.h" +#include "llvm/TargetParser/Triple.h" +#include "llvm/Transforms/Utils/EscapeEnumerator.h" +#include "llvm/Transforms/Utils/Instrumentation.h" +#include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" + +using namespace llvm; + +#define DEBUG_TYPE "csan" + +static cl::opt<bool> ClInstrumentMemoryAccesses( + "csan-instrument-memory-accesses", cl::init(true), + cl::desc("Instrument memory accesses"), cl::Hidden); +static cl::opt<bool> + ClInstrumentFuncEntryExit("csan-instrument-func-entry-exit", cl::init(true), + cl::desc("Instrument function entry and exit"), + cl::Hidden); +static cl::opt<bool> ClHandleCxxExceptions( + "csan-handle-cxx-exceptions", cl::init(true), + cl::desc("Handle C++ exceptions (insert cleanup blocks for unwinding)"), + cl::Hidden); +static cl::opt<bool> ClInstrumentAtomics("csan-instrument-atomics", + cl::init(true), + cl::desc("Instrument atomics"), + cl::Hidden); +static cl::opt<bool> ClInstrumentMemIntrinsics( + "csan-instrument-memintrinsics", cl::init(true), + cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); +static cl::opt<bool> ClDistinguishVolatile( + "csan-distinguish-volatile", cl::init(false), + cl::desc("Emit special instrumentation for accesses to volatiles"), + cl::Hidden); +STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); +STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); +STATISTIC(NumOmittedReadsBeforeWrite, + "Number of reads ignored due to following writes"); + +static constexpr char kCsanModuleCtorName[] = "csan.module_ctor"; +static constexpr char kCsanInitName[] = "__csan_init"; + +namespace { + +/// Must match CSAN_ACCESS_* in compiler-rt/lib/csan/csan_defs.h. +enum AccessFlags : unsigned { + AF_None = 0, + AF_Atomic = 1u << 0, + AF_Compound = 1u << 1, +}; + +static bool isAtomicMemoryAccess(const Instruction *I) { + auto SSID = getAtomicSyncScopeID(I); + if (!SSID) + return false; + if (isa<LoadInst>(I) || isa<StoreInst>(I)) + return *SSID != SyncScope::SingleThread; + return true; +} + +static Value *getCallbackAddress(IRBuilderBase &IRB, Value *Addr) { + return IRB.CreateAddrSpaceCast(Addr, IRB.getPtrTy()); +} + +static ConstantInt *createOrdering(IRBuilderBase &IRB, AtomicOrdering Ord) { + uint32_t Value = 0; + switch (Ord) { + case AtomicOrdering::NotAtomic: + llvm_unreachable("unexpected atomic ordering"); + case AtomicOrdering::Unordered: + case AtomicOrdering::Monotonic: + Value = 0; + break; + case AtomicOrdering::Acquire: + Value = 2; + break; + case AtomicOrdering::Release: + Value = 3; + break; + case AtomicOrdering::AcquireRelease: + Value = 4; + break; + case AtomicOrdering::SequentiallyConsistent: + Value = 5; + break; + } + return IRB.getInt32(Value); +} + +static int getAccessSizeIndex(Type *Ty, const DataLayout &DL) { + assert(Ty->isSized()); + if (Ty->isScalableTy()) + return -1; + uint32_t TypeSize = DL.getTypeStoreSizeInBits(Ty); + if (TypeSize != 8 && TypeSize != 16 && TypeSize != 32 && TypeSize != 64 && + TypeSize != 128) + return -1; + unsigned Idx = llvm::countr_zero(TypeSize / 8); + return static_cast<int>(Idx); +} + +static bool addressSpaceMayRace(const Triple &T, unsigned AS) { + if (T.isAMDGPU()) + // GDS and buffer fat pointers cannot form a generic watchpoint key. + return AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS || + AS == AMDGPUAS::LOCAL_ADDRESS; + if (T.isNVPTX()) + return AS == NVPTXAS::ADDRESS_SPACE_GENERIC || + AS == NVPTXAS::ADDRESS_SPACE_GLOBAL || + AS == NVPTXAS::ADDRESS_SPACE_SHARED || + AS == NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER; + if (T.isSPIRV()) + // FIXME: No exposed address spaces for SPIR-V. + return false; + return AS == 0; +} + +struct ConcurrencySanitizer { + bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI); + +private: + struct MemoryAccessLists { + SmallVector<Instruction *, 8> LoadsAndStores; + SmallVector<Instruction *, 8> AtomicAccesses; + SmallVector<MemIntrinsic *, 8> MemIntrinCalls; + bool HasCalls = false; + }; + + void initialize(Module &M, const TargetLibraryInfo &TLI); + void collectMemoryAccesses(Function &F, MemoryAccessLists &Out); + bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); + bool instrumentAtomic(Instruction *I, const DataLayout &DL); + bool instrumentMemIntrinsic(MemIntrinsic *M); + bool insertAccessProbe(Instruction *I, Value *Addr, Type *AccessTy, + const DataLayout &DL, bool IsWrite, bool IsCompound, + bool IsAtomic); + bool shouldInstrumentAddress(Value *Addr) const; + bool shouldInstrumentAccess(Instruction *I) const; + void insertFuncEntryExit(Function &F); + void insertRuntimeIgnores(Function &F); + + Module *Mod = nullptr; + Type *IntptrTy = nullptr; + IntegerType *FlagsTy = nullptr; + + // Accesses sizes are powers of two: 1, 2, 4, 8, 16. + static const size_t kNumAccessSizes = 5; + // void __csan_readN(ptr, i32); + FunctionCallee CsanRead[kNumAccessSizes]; + // void __csan_writeN(ptr, i32); + FunctionCallee CsanWrite[kNumAccessSizes]; + // void __csan_unaligned_readN(ptr, i32); + FunctionCallee CsanUnalignedRead[kNumAccessSizes]; + // void __csan_unaligned_writeN(ptr, i32); + FunctionCallee CsanUnalignedWrite[kNumAccessSizes]; + // void __csan_volatile_readN(ptr, i32); + FunctionCallee CsanVolatileRead[kNumAccessSizes]; + // void __csan_volatile_writeN(ptr, i32); + FunctionCallee CsanVolatileWrite[kNumAccessSizes]; + // void __csan_unaligned_volatile_readN(ptr, i32); + FunctionCallee CsanUnalignedVolatileRead[kNumAccessSizes]; + // void __csan_unaligned_volatile_writeN(ptr, i32); + FunctionCallee CsanUnalignedVolatileWrite[kNumAccessSizes]; + // void __csan_read_writeN(ptr, i32); + FunctionCallee CsanCompoundRW[kNumAccessSizes]; + // void __csan_unaligned_read_writeN(ptr, i32); + FunctionCallee CsanUnalignedCompoundRW[kNumAccessSizes]; + // void __csan_func_entry(ptr); + FunctionCallee CsanFuncEntry; + // void __csan_func_exit(); + FunctionCallee CsanFuncExit; + // void __csan_ignore_thread_begin(); + FunctionCallee CsanIgnoreBegin; + // void __csan_ignore_thread_end(); + FunctionCallee CsanIgnoreEnd; + // void __csan_read_range(ptr, intptr_t, i32); + FunctionCallee CsanReadRange; + // void __csan_write_range(ptr, intptr_t, i32); + FunctionCallee CsanWriteRange; + // void __csan_atomic_thread_fence(i32); + FunctionCallee CsanAtomicThreadFence; + // void __csan_atomic_signal_fence(i32); + FunctionCallee CsanAtomicSignalFence; +}; + +void insertModuleCtor(Module &M) { + getOrCreateSanitizerCtorAndInitFunctions( + M, kCsanModuleCtorName, kCsanInitName, /*InitArgTypes=*/{}, + /*InitArgs=*/{}, + [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, Ctor, 0); }); +} + +} // namespace + +PreservedAnalyses ConcurrencySanitizerPass::run(Function &F, + FunctionAnalysisManager &FAM) { + ConcurrencySanitizer CSan; + if (CSan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F))) + return PreservedAnalyses::none(); + return PreservedAnalyses::all(); +} + +PreservedAnalyses ModuleConcurrencySanitizerPass::run(Module &M, + ModuleAnalysisManager &) { + if (checkIfAlreadyInstrumented(M, "nosanitize_concurrency")) + return PreservedAnalyses::all(); + insertModuleCtor(M); + return PreservedAnalyses::none(); +} + +void ConcurrencySanitizer::initialize(Module &M, const TargetLibraryInfo &TLI) { + LLVMContext &Ctx = M.getContext(); + Mod = &M; + IntptrTy = M.getDataLayout().getIntPtrType(Ctx); + FlagsTy = Type::getInt32Ty(Ctx); + + AttributeList Attr = AttributeList().addFnAttribute(Ctx, Attribute::NoUnwind); + IRBuilder<> IRB(Ctx); + Type *VoidTy = IRB.getVoidTy(); + Type *PtrTy = IRB.getPtrTy(); + CsanFuncEntry = + M.getOrInsertFunction("__csan_func_entry", Attr, VoidTy, PtrTy); + CsanFuncExit = M.getOrInsertFunction("__csan_func_exit", Attr, VoidTy); + CsanIgnoreBegin = + M.getOrInsertFunction("__csan_ignore_thread_begin", Attr, VoidTy); + CsanIgnoreEnd = + M.getOrInsertFunction("__csan_ignore_thread_end", Attr, VoidTy); + for (unsigned I = 0; I < kNumAccessSizes; ++I) { + std::string ByteSize = utostr(1U << I); + auto AccessFn = [&](const Twine &Name) { + return M.getOrInsertFunction(("__csan_" + Name).str(), Attr, VoidTy, + PtrTy, FlagsTy); + }; + CsanRead[I] = AccessFn("read" + ByteSize); + CsanWrite[I] = AccessFn("write" + ByteSize); + CsanUnalignedRead[I] = AccessFn("unaligned_read" + ByteSize); + CsanUnalignedWrite[I] = AccessFn("unaligned_write" + ByteSize); + CsanVolatileRead[I] = AccessFn("volatile_read" + ByteSize); + CsanVolatileWrite[I] = AccessFn("volatile_write" + ByteSize); + CsanUnalignedVolatileRead[I] = + AccessFn("unaligned_volatile_read" + ByteSize); + CsanUnalignedVolatileWrite[I] = + AccessFn("unaligned_volatile_write" + ByteSize); + CsanCompoundRW[I] = AccessFn("read_write" + ByteSize); + CsanUnalignedCompoundRW[I] = AccessFn("unaligned_read_write" + ByteSize); + } + IntegerType *OrdTy = IRB.getInt32Ty(); + CsanReadRange = M.getOrInsertFunction("__csan_read_range", Attr, VoidTy, + PtrTy, IntptrTy, FlagsTy); + CsanWriteRange = M.getOrInsertFunction("__csan_write_range", Attr, VoidTy, + PtrTy, IntptrTy, FlagsTy); + CsanAtomicThreadFence = M.getOrInsertFunction( + "__csan_atomic_thread_fence", + TLI.getAttrList(&Ctx, {0}, /*Signed=*/true, /*Ret=*/false, Attr), VoidTy, + OrdTy); + CsanAtomicSignalFence = M.getOrInsertFunction( + "__csan_atomic_signal_fence", + TLI.getAttrList(&Ctx, {0}, /*Signed=*/true, /*Ret=*/false, Attr), VoidTy, + OrdTy); +} + +bool ConcurrencySanitizer::sanitizeFunction(Function &F, + const TargetLibraryInfo &TLI) { + // This is required to prevent instrumenting call to __csan_init from within + // the module constructor. + if (F.getName() == kCsanModuleCtorName) + return false; + // Naked functions can not have prologue/epilogue + // (__csan_func_entry/__csan_func_exit) generated, so don't instrument them at + // all. + if (F.hasFnAttribute(Attribute::Naked)) + return false; + + // __attribute__(disable_sanitizer_instrumentation) prevents all kinds of + // instrumentation. + if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation)) + return false; + + Mod = F.getParent(); + MemoryAccessLists Acc; + collectMemoryAccesses(F, Acc); + + const bool SuppressChecking = + F.hasFnAttribute("sanitize_concurrency_no_checking_at_run_time"); + const bool SanitizeFunction = + F.hasFnAttribute(Attribute::SanitizeConcurrency) && !SuppressChecking; + const bool MayInstrument = + SanitizeFunction && + ((ClInstrumentMemoryAccesses && !Acc.LoadsAndStores.empty()) || + (ClInstrumentAtomics && !Acc.AtomicAccesses.empty()) || + (ClInstrumentMemIntrinsics && !Acc.MemIntrinCalls.empty())); + const bool NeedsRuntimeIgnores = SuppressChecking && Acc.HasCalls; + const bool NeedsFuncEntryExit = ClInstrumentFuncEntryExit && Acc.HasCalls; + if (!MayInstrument && !NeedsRuntimeIgnores && !NeedsFuncEntryExit) + return false; + + initialize(*Mod, TLI); + bool Res = false; + const DataLayout &DL = F.getDataLayout(); + + if (ClInstrumentMemoryAccesses && SanitizeFunction) + for (Instruction *I : Acc.LoadsAndStores) + Res |= instrumentLoadOrStore(I, DL); + + if (ClInstrumentAtomics && SanitizeFunction) + for (Instruction *I : Acc.AtomicAccesses) + Res |= instrumentAtomic(I, DL); + + if (ClInstrumentMemIntrinsics && SanitizeFunction) + for (MemIntrinsic *MI : Acc.MemIntrinCalls) + Res |= instrumentMemIntrinsic(MI); + + if (NeedsRuntimeIgnores) { + insertRuntimeIgnores(F); + Res = true; + } + + if ((Res || Acc.HasCalls) && ClInstrumentFuncEntryExit) { + insertFuncEntryExit(F); + Res = true; + } + // Callback declarations may have changed the module. + return true; +} + +bool ConcurrencySanitizer::shouldInstrumentAddress(Value *Addr) const { + Value *BaseAddr = Addr->stripInBoundsOffsets(); + if (auto *GV = dyn_cast<GlobalVariable>(BaseAddr)) { + if (GV->hasSection()) { + StringRef SectionName = GV->getSection(); + auto OF = Mod->getTargetTriple().getObjectFormat(); + if (SectionName.ends_with( + getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false))) + return false; + } + } + + Type *PtrTy = cast<PointerType>(Addr->getType()->getScalarType()); + unsigned AS = PtrTy->getPointerAddressSpace(); + if (Mod->getDataLayout().getPointerSizeInBits(AS) > 64) + return false; + return addressSpaceMayRace(Mod->getTargetTriple(), AS); +} + +bool ConcurrencySanitizer::shouldInstrumentAccess(Instruction *I) const { + const bool IsWrite = isa<StoreInst>(I); + Value *Addr = getLoadStorePointerOperand(I); + if (!shouldInstrumentAddress(Addr)) + return false; + + if (!IsWrite) + if (auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(Addr))) + if (GV->isConstant()) + return false; + + const AllocaInst *AI = findAllocaForValue(Addr); + if (!AI || PointerMayBeCaptured(AI, /*ReturnCaptures=*/true)) + return true; + + const Triple &T = AI->getModule()->getTargetTriple(); + return T.isGPU() && addressSpaceMayRace(T, AI->getAddressSpace()); +} + +void ConcurrencySanitizer::collectMemoryAccesses(Function &F, + MemoryAccessLists &Out) { + SmallVector<Instruction *, 8> LocalLoadsAndStores; + auto FlushLocalAccesses = [&] { + DenseSet<Value *> WriteTargets; + for (Instruction *Inst : reverse(LocalLoadsAndStores)) { + const bool IsWrite = isa<StoreInst>(Inst); + Value *Addr = getLoadStorePointerOperand(Inst); + if (!IsWrite && WriteTargets.contains(Addr)) { + ++NumOmittedReadsBeforeWrite; + continue; + } + + Out.LoadsAndStores.push_back(Inst); + if (IsWrite) + WriteTargets.insert(Addr); + } + LocalLoadsAndStores.clear(); + }; + + for (BasicBlock &BB : F) { + for (Instruction &Inst : BB) { + // Skip instructions inserted by another instrumentation. + if (Inst.hasMetadata(LLVMContext::MD_nosanitize)) + continue; + if (isAtomicMemoryAccess(&Inst)) + Out.AtomicAccesses.push_back(&Inst); + else if ((isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) && + shouldInstrumentAccess(&Inst)) + LocalLoadsAndStores.push_back(&Inst); + else if (isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) { + FlushLocalAccesses(); + if (auto *MI = dyn_cast<MemIntrinsic>(&Inst)) + Out.MemIntrinCalls.push_back(MI); + Out.HasCalls = true; + } + } + FlushLocalAccesses(); + } +} + +bool ConcurrencySanitizer::instrumentLoadOrStore(Instruction *I, + const DataLayout &DL) { + const bool IsWrite = isa<StoreInst>(I); + Value *Addr = getLoadStorePointerOperand(I); + if (!insertAccessProbe(I, Addr, getLoadStoreType(I), DL, IsWrite, + /*IsCompound=*/false, /*IsAtomic=*/false)) + return false; + if (IsWrite) + ++NumInstrumentedWrites; + else + ++NumInstrumentedReads; + return true; +} + +bool ConcurrencySanitizer::instrumentAtomic(Instruction *I, + const DataLayout &DL) { + if (auto *FI = dyn_cast<FenceInst>(I)) { + InstrumentationIRBuilder IRB(I); + FunctionCallee Fn = FI->getSyncScopeID() == SyncScope::SingleThread + ? CsanAtomicSignalFence + : CsanAtomicThreadFence; + IRB.CreateCall(Fn, createOrdering(IRB, FI->getOrdering())); + return true; + } + + Value *Addr = nullptr; + Type *AccessTy = nullptr; + bool IsWrite = true; + bool IsCompound = false; + if (auto *LI = dyn_cast<LoadInst>(I)) { + Addr = LI->getPointerOperand(); + AccessTy = LI->getType(); + IsWrite = false; + } else if (auto *SI = dyn_cast<StoreInst>(I)) { + Addr = SI->getPointerOperand(); + AccessTy = SI->getValueOperand()->getType(); + } else if (auto *RMW = dyn_cast<AtomicRMWInst>(I)) { + Addr = RMW->getPointerOperand(); + AccessTy = RMW->getValOperand()->getType(); + IsCompound = true; + } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(I)) { + Addr = CAS->getPointerOperand(); + AccessTy = CAS->getNewValOperand()->getType(); + IsCompound = true; + } else { + return false; + } + + if (!shouldInstrumentAddress(Addr)) + return false; + if (!insertAccessProbe(I, Addr, AccessTy, DL, IsWrite, IsCompound, + /*IsAtomic=*/true)) + return false; + if (IsCompound || IsWrite) + ++NumInstrumentedWrites; + if (IsCompound || !IsWrite) + ++NumInstrumentedReads; + return true; +} + +bool ConcurrencySanitizer::instrumentMemIntrinsic(MemIntrinsic *M) { + if (auto *MS = dyn_cast<MemSetInst>(M)) { + if (!shouldInstrumentAddress(MS->getRawDest())) + return false; + InstrumentationIRBuilder IRB(M); + Value *Len = IRB.CreateIntCast(M->getLength(), IntptrTy, false); + IRB.CreateCall(CsanWriteRange, {getCallbackAddress(IRB, MS->getRawDest()), + Len, ConstantInt::get(FlagsTy, AF_None)}); + ++NumInstrumentedWrites; + return true; + } + + auto *MT = cast<MemTransferInst>(M); + bool InstrumentRead = shouldInstrumentAddress(MT->getRawSource()); + bool InstrumentWrite = shouldInstrumentAddress(MT->getRawDest()); + if (!InstrumentRead && !InstrumentWrite) + return false; + + InstrumentationIRBuilder IRB(M); + Value *Len = IRB.CreateIntCast(M->getLength(), IntptrTy, false); + if (InstrumentRead) { + IRB.CreateCall(CsanReadRange, {getCallbackAddress(IRB, MT->getRawSource()), + Len, ConstantInt::get(FlagsTy, AF_None)}); + ++NumInstrumentedReads; + } + if (InstrumentWrite) { + IRB.CreateCall(CsanWriteRange, {getCallbackAddress(IRB, MT->getRawDest()), + Len, ConstantInt::get(FlagsTy, AF_None)}); + ++NumInstrumentedWrites; + } + return true; +} + +bool ConcurrencySanitizer::insertAccessProbe(Instruction *I, Value *Addr, + Type *AccessTy, + const DataLayout &DL, bool IsWrite, + bool IsCompound, bool IsAtomic) { + if (Addr->isSwiftError()) + return false; + unsigned Flags = + (IsAtomic ? AF_Atomic : AF_None) | (IsCompound ? AF_Compound : AF_None); + int Idx = getAccessSizeIndex(AccessTy, DL); + if (Idx < 0) { + if (IsCompound) + return false; + InstrumentationIRBuilder IRB(I); + Value *Len = IRB.CreateTypeSize(IntptrTy, DL.getTypeStoreSize(AccessTy)); + IRB.CreateCall( + IsWrite ? CsanWriteRange : CsanReadRange, + {getCallbackAddress(IRB, Addr), Len, ConstantInt::get(FlagsTy, Flags)}); + return true; + } + + Align Alignment = Align(1); + bool IsVolatile = false; + if (auto *LI = dyn_cast<LoadInst>(I)) { + Alignment = LI->getAlign(); + IsVolatile = LI->isVolatile(); + } else if (auto *SI = dyn_cast<StoreInst>(I)) { + Alignment = SI->getAlign(); + IsVolatile = SI->isVolatile(); + } else if (auto *RMW = dyn_cast<AtomicRMWInst>(I)) { + Alignment = RMW->getAlign(); + IsVolatile = RMW->isVolatile(); + } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(I)) { + Alignment = CAS->getAlign(); + IsVolatile = CAS->isVolatile(); + } + IsVolatile &= ClDistinguishVolatile; + + uint32_t TypeSize = DL.getTypeStoreSizeInBits(AccessTy); + bool Unaligned = + Alignment < Align(8) && Alignment.value() % (TypeSize / 8) != 0; + FunctionCallee Callback; + if (IsCompound) + Callback = Unaligned ? CsanUnalignedCompoundRW[Idx] : CsanCompoundRW[Idx]; + else if (IsVolatile && Unaligned) + Callback = IsWrite ? CsanUnalignedVolatileWrite[Idx] + : CsanUnalignedVolatileRead[Idx]; + else if (IsVolatile) + Callback = IsWrite ? CsanVolatileWrite[Idx] : CsanVolatileRead[Idx]; + else if (Unaligned) + Callback = IsWrite ? CsanUnalignedWrite[Idx] : CsanUnalignedRead[Idx]; + else + Callback = IsWrite ? CsanWrite[Idx] : CsanRead[Idx]; + + InstrumentationIRBuilder IRB(I); + IRB.CreateCall(Callback, {getCallbackAddress(IRB, Addr), + ConstantInt::get(FlagsTy, Flags)}); + return true; +} + +void ConcurrencySanitizer::insertFuncEntryExit(Function &F) { + const DataLayout &DL = F.getDataLayout(); + InstrumentationIRBuilder IRB(&F.getEntryBlock(), + F.getEntryBlock().getFirstNonPHIIt()); + Type *ProgramAsPtrTy = + PointerType::get(F.getContext(), DL.getProgramAddressSpace()); + Value *ReturnAddress = IRB.CreateIntrinsic(Intrinsic::returnaddress, + {ProgramAsPtrTy}, IRB.getInt32(0)); + IRB.CreateCall(CsanFuncEntry, ReturnAddress); + + EscapeEnumerator EE(F, "csan_cleanup", ClHandleCxxExceptions); + while (IRBuilder<> *AtExit = EE.Next()) { + InstrumentationIRBuilder::ensureDebugInfo(*AtExit, F); + AtExit->CreateCall(CsanFuncExit); + } +} + +void ConcurrencySanitizer::insertRuntimeIgnores(Function &F) { + InstrumentationIRBuilder IRB(&F.getEntryBlock(), + F.getEntryBlock().getFirstNonPHIIt()); + IRB.CreateCall(CsanIgnoreBegin); + EscapeEnumerator EE(F, "csan_ignore_cleanup", ClHandleCxxExceptions); + while (IRBuilder<> *AtExit = EE.Next()) { + InstrumentationIRBuilder::ensureDebugInfo(*AtExit, F); + AtExit->CreateCall(CsanIgnoreEnd); + } +} diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/address_space.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/address_space.ll new file mode 100644 index 00000000000000..c0190cf74b5c60 --- /dev/null +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/address_space.ll @@ -0,0 +1,106 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt < %s -passes='function(csan)' -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s --check-prefixes=CHECK + +define void @load_as1(ptr addrspace(1) %p) sanitize_concurrency { +; CHECK-LABEL: define void @load_as1( +; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[TMP0:%.*]] = call ptr @llvm.returnaddress.p0(i32 0) +; CHECK-NEXT: call void @__csan_func_entry(ptr [[TMP0]]) +; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(1) [[P]] to ptr +; CHECK-NEXT: call void @__csan_read4(ptr [[TMP1]], i32 0) +; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[P]], align 4 +; CHECK-NEXT: call void @__csan_func_exit() +; CHECK-NEXT: ret void +; +entry: + %v = load i32, ptr addrspace(1) %p, align 4 + ret void +} + +define void @load_lds(ptr addrspace(3) %p) sanitize_concurrency { +; CHECK-LABEL: define void @load_lds( +; CHECK-SAME: ptr addrspace(3) [[P:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[TMP0:%.*]] = call ptr @llvm.returnaddress.p0(i32 0) +; CHECK-NEXT: call void @__csan_func_entry(ptr [[TMP0]]) +; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(3) [[P]] to ptr +; CHECK-NEXT: call void @__csan_read4(ptr [[TMP1]], i32 0) +; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(3) [[P]], align 4 +; CHECK-NEXT: call void @__csan_func_exit() +; CHECK-NEXT: ret void +; +entry: + %v = load i32, ptr addrspace(3) %p, align 4 + ret void +} + +define void @load_private(ptr addrspace(5) %p) sanitize_concurrency { +; CHECK-LABEL: define void @load_private( +; CHECK-SAME: ptr addrspace(5) [[P:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4 +; CHECK-NEXT: ret void +; +entry: + %v = load i32, ptr addrspace(5) %p, align 4 + ret void +} + +define void @load_gds(ptr addrspace(2) %p) sanitize_concurrency { +; CHECK-LABEL: define void @load_gds( +; CHECK-SAME: ptr addrspace(2) [[P:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(2) [[P]], align 4 +; CHECK-NEXT: ret void +; +entry: + %v = load i32, ptr addrspace(2) %p, align 4 + ret void +} + +define void @load_buffer_fat(ptr addrspace(7) %p) sanitize_concurrency { +; CHECK-LABEL: define void @load_buffer_fat( +; CHECK-SAME: ptr addrspace(7) [[P:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(7) [[P]], align 4 +; CHECK-NEXT: ret void +; +entry: + %v = load i32, ptr addrspace(7) %p, align 4 + ret void +} + +define void @memcpy_private(ptr addrspace(5) %dst, ptr addrspace(5) %src, i64 %n) sanitize_concurrency { +; CHECK-LABEL: define void @memcpy_private( +; CHECK-SAME: ptr addrspace(5) [[DST:%.*]], ptr addrspace(5) [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[TMP0:%.*]] = call ptr @llvm.returnaddress.p0(i32 0) +; CHECK-NEXT: call void @__csan_func_entry(ptr [[TMP0]]) +; CHECK-NEXT: call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) [[DST]], ptr addrspace(5) [[SRC]], i64 [[N]], i1 false) +; CHECK-NEXT: call void @__csan_func_exit() +; CHECK-NEXT: ret void +; +entry: + call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) %dst, ptr addrspace(5) %src, i64 %n, i1 false) + ret void +} + +define i32 @scoped_atomic(ptr addrspace(1) %p) sanitize_concurrency { +; CHECK-LABEL: define i32 @scoped_atomic( +; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[TMP0:%.*]] = call ptr @llvm.returnaddress.p0(i32 0) +; CHECK-NEXT: call void @__csan_func_entry(ptr [[TMP0]]) +; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(1) [[P]] to ptr +; CHECK-NEXT: call void @__csan_read4(ptr [[TMP1]], i32 1) +; CHECK-NEXT: [[V:%.*]] = load atomic i32, ptr addrspace(1) [[P]] syncscope("agent") seq_cst, align 4 +; CHECK-NEXT: call void @__csan_func_exit() +; CHECK-NEXT: ret i32 [[V]] +; +entry: + %v = load atomic i32, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4 + ret i32 %v +} + +declare void @llvm.memcpy.p5.p5.i64(ptr addrspace(5), ptr addrspace(5), i64, i1) diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/atomic.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/atomic.ll new file mode 100644 index 00000000000000..ac2e0c2f78835e --- /dev/null +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/atomic.ll @@ -0,0 +1,86 @@ +; RUN: opt < %s -passes='function(csan)' -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s +; RUN: opt < %s -passes='function(csan)' -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s + +define i32 @atomic_load(ptr %a) sanitize_concurrency { +entry: + %v = load atomic i32, ptr %a seq_cst, align 4 + ret i32 %v +} +; CHECK-LABEL: @atomic_load( +; CHECK: call void @__csan_read4(ptr %a, i32 1) +; CHECK-NEXT: %v = load atomic i32, ptr %a seq_cst, align 4 + +define void @atomic_store(ptr %a, i32 %v) sanitize_concurrency { +entry: + store atomic i32 %v, ptr %a release, align 4 + ret void +} +; CHECK-LABEL: @atomic_store( +; CHECK: call void @__csan_write4(ptr %a, i32 1) +; CHECK-NEXT: store atomic i32 %v, ptr %a release, align 4 + +define i32 @atomic_rmw(ptr %a, i32 %v) sanitize_concurrency { +entry: + %old = atomicrmw add ptr %a, i32 %v seq_cst + ret i32 %old +} +; CHECK-LABEL: @atomic_rmw( +; CHECK: call void @__csan_read_write4(ptr %a, i32 3) +; CHECK-NEXT: %old = atomicrmw add ptr %a, i32 %v seq_cst + +define i32 @atomic_cas(ptr %a, i32 %cmp, i32 %new) sanitize_concurrency { +entry: + %pair = cmpxchg ptr %a, i32 %cmp, i32 %new seq_cst seq_cst + %old = extractvalue { i32, i1 } %pair, 0 + ret i32 %old +} +; CHECK-LABEL: @atomic_cas( +; CHECK: call void @__csan_read_write4(ptr %a, i32 3) +; CHECK-NEXT: %pair = cmpxchg ptr %a, i32 %cmp, i32 %new seq_cst seq_cst + +define void @atomic_fence() sanitize_concurrency { +entry: + fence seq_cst + ret void +} +; CHECK-LABEL: @atomic_fence( +; CHECK: call void @__csan_atomic_thread_fence(i32 5) +; CHECK-NEXT: fence seq_cst + +define void @atomic_signal_fence() sanitize_concurrency { +entry: + fence syncscope("singlethread") seq_cst + ret void +} +; CHECK-LABEL: @atomic_signal_fence( +; CHECK: call void @__csan_atomic_signal_fence(i32 5) +; CHECK-NEXT: fence syncscope("singlethread") seq_cst + +define void @atomic_fence_orderings() sanitize_concurrency { +entry: + fence acquire + fence release + fence acq_rel + ret void +} +; CHECK-LABEL: @atomic_fence_orderings( +; CHECK: call void @__csan_atomic_thread_fence(i32 2) +; CHECK-NEXT: fence acquire +; CHECK: call void @__csan_atomic_thread_fence(i32 3) +; CHECK-NEXT: fence release +; CHECK: call void @__csan_atomic_thread_fence(i32 4) +; CHECK-NEXT: fence acq_rel + +define i32 @suppressed(ptr %a) { +entry: + %v = load atomic i32, ptr %a seq_cst, align 4 + ret i32 %v +} +; CHECK-LABEL: @suppressed( +; CHECK-NEXT: entry: +; CHECK-NEXT: %v = load atomic i32, ptr %a seq_cst, align 4 +; CHECK-NEXT: ret i32 %v + +; CHECK-NOT: @__tsan +; CHECK-NOT: @__csan_atomic8 +; CHECK-NOT: @__csan_atomic32 diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/basic.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/basic.ll new file mode 100644 index 00000000000000..fa1cc65c709892 --- /dev/null +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/basic.ll @@ -0,0 +1,49 @@ +; RUN: opt < %s -passes='module(csan-module),function(csan)' -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s +; RUN: opt < %s -passes='module(csan-module),function(csan)' -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s + +define void @read_write(ptr %a, ptr %b) sanitize_concurrency { +entry: + %v = load i32, ptr %a, align 4 + store i32 %v, ptr %b, align 4 + ret void +} +; CHECK-LABEL: @read_write( +; CHECK: call void @__csan_func_entry +; CHECK: call void @__csan_read4(ptr %a, i32 0) +; CHECK-NEXT: %v = load i32, ptr %a, align 4 +; CHECK: call void @__csan_write4(ptr %b, i32 0) +; CHECK-NEXT: store i32 %v, ptr %b, align 4 +; CHECK: call void @__csan_func_exit + +define i16 @unaligned(ptr %a) sanitize_concurrency { +entry: + %v = load i16, ptr %a, align 1 + ret i16 %v +} +; CHECK-LABEL: @unaligned( +; CHECK: call void @__csan_unaligned_read2(ptr %a, i32 0) +; CHECK-NEXT: %v = load i16, ptr %a, align 1 + +define void @memintrinsics(ptr %dst, ptr %src, i64 %n) sanitize_concurrency { +entry: + call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %n, i1 false) + call void @llvm.memmove.p0.p0.i64(ptr %dst, ptr %src, i64 %n, i1 false) + call void @llvm.memset.p0.i64(ptr %dst, i8 0, i64 %n, i1 false) + ret void +} +; CHECK-LABEL: @memintrinsics( +; CHECK: call void @__csan_read_range(ptr %src, i64 %n, i32 0) +; CHECK: call void @__csan_write_range(ptr %dst, i64 %n, i32 0) +; CHECK: call void @llvm.memcpy +; CHECK: call void @llvm.memmove +; CHECK: call void @llvm.memset + +; CHECK: define internal void @csan.module_ctor +; CHECK: call void @__csan_init() +; CHECK-NOT: @__tsan +; CHECK-NOT: @__csan_atomic8 +; CHECK-NOT: @__csan_memcpy + +declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1) +declare void @llvm.memmove.p0.p0.i64(ptr, ptr, i64, i1) +declare void @llvm.memset.p0.i64(ptr, i8, i64, i1) diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/callbacks.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/callbacks.ll new file mode 100644 index 00000000000000..d7d8269b081c17 --- /dev/null +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/callbacks.ll @@ -0,0 +1,72 @@ +; RUN: opt < %s -passes='function(csan)' -csan-distinguish-volatile -S | FileCheck %s + +define void @access_sizes(ptr %p1, ptr %p2, ptr %p4, ptr %p8, ptr %p16) sanitize_concurrency { +entry: + %v1 = load i8, ptr %p1, align 1 + %v2 = load i16, ptr %p2, align 2 + %v4 = load i32, ptr %p4, align 4 + %v8 = load i64, ptr %p8, align 8 + %v16 = load i128, ptr %p16, align 16 + store i8 %v1, ptr %p1, align 1 + store i16 %v2, ptr %p2, align 2 + store i32 %v4, ptr %p4, align 4 + store i64 %v8, ptr %p8, align 8 + store i128 %v16, ptr %p16, align 16 + ret void +} +; CHECK-LABEL: @access_sizes( +; CHECK-DAG: call void @__csan_write1(ptr %p1, i32 0) +; CHECK-DAG: call void @__csan_write2(ptr %p2, i32 0) +; CHECK-DAG: call void @__csan_write4(ptr %p4, i32 0) +; CHECK-DAG: call void @__csan_write8(ptr %p8, i32 0) +; CHECK-DAG: call void @__csan_write16(ptr %p16, i32 0) + +define i128 @read_sizes(ptr %p1, ptr %p8, ptr %p16) sanitize_concurrency { +entry: + %v1 = load i8, ptr %p1, align 1 + %v8 = load i64, ptr %p8, align 8 + %v16 = load i128, ptr %p16, align 16 + %v1.ext = zext i8 %v1 to i128 + %v8.ext = zext i64 %v8 to i128 + %sum1 = add i128 %v1.ext, %v8.ext + %sum2 = add i128 %sum1, %v16 + ret i128 %sum2 +} +; CHECK-LABEL: @read_sizes( +; CHECK: call void @__csan_read1(ptr %p1, i32 0) +; CHECK: call void @__csan_read8(ptr %p8, i32 0) +; CHECK: call void @__csan_read16(ptr %p16, i32 0) + +define i32 @volatile_read(ptr %p) sanitize_concurrency { +entry: + %v = load volatile i32, ptr %p, align 4 + ret i32 %v +} +; CHECK-LABEL: @volatile_read( +; CHECK: call void @__csan_volatile_read4(ptr %p, i32 0) + +define void @unaligned_volatile_write(ptr %p) sanitize_concurrency { +entry: + store volatile i64 1, ptr %p, align 1 + ret void +} +; CHECK-LABEL: @unaligned_volatile_write( +; CHECK: call void @__csan_unaligned_volatile_write8(ptr %p, i32 0) + +define i32 @unaligned_atomic_rmw(ptr %p) sanitize_concurrency { +entry: + %v = atomicrmw add ptr %p, i32 1 seq_cst, align 1 + ret i32 %v +} +; CHECK-LABEL: @unaligned_atomic_rmw( +; CHECK: call void @__csan_unaligned_read_write4(ptr %p, i32 3) +; CHECK-NEXT: %v = atomicrmw add ptr %p, i32 1 seq_cst, align 1 + +define float @atomic_float(ptr %p) sanitize_concurrency { +entry: + %v = load atomic float, ptr %p monotonic, align 4 + ret float %v +} +; CHECK-LABEL: @atomic_float( +; CHECK: call void @__csan_read4(ptr %p, i32 1) +; CHECK-NEXT: %v = load atomic float, ptr %p monotonic, align 4 diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/function_control.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/function_control.ll new file mode 100644 index 00000000000000..69636563b24754 --- /dev/null +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/function_control.ll @@ -0,0 +1,59 @@ +; RUN: opt < %s -passes='function(csan)' -S | FileCheck %s --check-prefixes=CHECK,EH +; RUN: opt < %s -passes='function(csan)' -csan-handle-cxx-exceptions=0 -S | FileCheck %s --check-prefixes=CHECK,NOEH +; RUN: opt < %s -passes='function(csan)' -csan-instrument-func-entry-exit=0 -S | FileCheck %s --check-prefix=NOENTRY + +declare void @may_throw() + +; Entry and exit callbacks must remain balanced when a call unwinds. +define void @exception_exit() sanitize_concurrency { +entry: + call void @may_throw() + ret void +} +; EH-LABEL: @exception_exit( +; EH: invoke void @may_throw() +; EH: csan_cleanup: +; EH: call void @__csan_func_exit() +; EH-NEXT: resume +; NOEH-LABEL: @exception_exit( +; NOEH: call void @may_throw() +; NOEH: call void @__csan_func_exit() + +define void @unattributed_call() { +entry: + call void @may_throw() + ret void +} +; CHECK-LABEL: @unattributed_call( +; CHECK: call void @__csan_func_entry +; CHECK: {{call|invoke}} void @may_throw() +; CHECK: call void @__csan_func_exit() + +define i32 @musttail_callee(ptr %p) sanitize_concurrency { +entry: + %v = load i32, ptr %p + ret i32 %v +} + +; A musttail call must remain immediately before its return. +define i32 @musttail_caller(ptr %p) sanitize_concurrency { +entry: + %v = musttail call i32 @musttail_callee(ptr %p) + ret i32 %v +} +; CHECK-LABEL: @musttail_caller( +; CHECK: call void @__csan_func_exit() +; CHECK-NEXT: %v = musttail call i32 @musttail_callee(ptr %p) +; CHECK-NEXT: ret i32 %v + +define void @ignored() "sanitize_concurrency_no_checking_at_run_time" { +entry: + call void @may_throw() + ret void +} +; NOENTRY-LABEL: @ignored( +; NOENTRY-NOT: __csan_func_ +; NOENTRY: call void @__csan_ignore_thread_begin() +; NOENTRY: invoke void @may_throw() +; NOENTRY: call void @__csan_ignore_thread_end() +; NOENTRY: ret void diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/memintrinsics.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/memintrinsics.ll new file mode 100644 index 00000000000000..35216aa84091a8 --- /dev/null +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/memintrinsics.ll @@ -0,0 +1,46 @@ +; RUN: opt < %s -passes='function(csan)' -csan-instrument-func-entry-exit=0 -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s + +define void @instrument_source(ptr addrspace(5) %dst, ptr addrspace(1) %src, i32 %n) sanitize_concurrency { +entry: + call void @llvm.memcpy.p5.p1.i32(ptr addrspace(5) %dst, ptr addrspace(1) %src, i32 %n, i1 false) + ret void +} +; CHECK-LABEL: @instrument_source( +; CHECK: %[[LEN:.*]] = zext i32 %n to i64 +; CHECK: %[[SRC:.*]] = addrspacecast ptr addrspace(1) %src to ptr +; CHECK: call void @__csan_read_range(ptr %[[SRC]], i64 %[[LEN]], i32 0) +; CHECK-NOT: __csan_write_range +; CHECK: call void @llvm.memcpy + +define void @instrument_dest(ptr addrspace(1) %dst, ptr addrspace(5) %src, i32 %n) sanitize_concurrency { +entry: + call void @llvm.memcpy.p1.p5.i32(ptr addrspace(1) %dst, ptr addrspace(5) %src, i32 %n, i1 false) + ret void +} +; CHECK-LABEL: @instrument_dest( +; CHECK: %[[LEN:.*]] = zext i32 %n to i64 +; CHECK-NOT: __csan_read_range +; CHECK: %[[DST:.*]] = addrspacecast ptr addrspace(1) %dst to ptr +; CHECK: call void @__csan_write_range(ptr %[[DST]], i64 %[[LEN]], i32 0) +; CHECK: call void @llvm.memcpy + +define void @private_copy(ptr addrspace(5) %dst, ptr addrspace(5) %src, i32 %n) sanitize_concurrency { +entry: + call void @llvm.memcpy.p5.p5.i32(ptr addrspace(5) %dst, ptr addrspace(5) %src, i32 %n, i1 false) + ret void +} +; CHECK-LABEL: @private_copy( +; CHECK-NEXT: entry: +; CHECK-NEXT: call void @llvm.memcpy.p5.p5.i32 +; CHECK-NEXT: ret void + +define void @instrument_memset(ptr addrspace(1) %dst, i32 %n) sanitize_concurrency { +entry: + call void @llvm.memset.p1.i32(ptr addrspace(1) %dst, i8 0, i32 %n, i1 false) + ret void +} +; CHECK-LABEL: @instrument_memset( +; CHECK: %[[LEN:.*]] = zext i32 %n to i64 +; CHECK: %[[DST:.*]] = addrspacecast ptr addrspace(1) %dst to ptr +; CHECK: call void @__csan_write_range(ptr %[[DST]], i64 %[[LEN]], i32 0) +; CHECK: call void @llvm.memset diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/module.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/module.ll new file mode 100644 index 00000000000000..b0ae86ad53bc88 --- /dev/null +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/module.ll @@ -0,0 +1,12 @@ +; RUN: opt < %s -passes='csan-module,csan-module' -S | FileCheck %s + +; CHECK: @llvm.used = appending global [1 x ptr] [ptr @csan.module_ctor] +; CHECK: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @csan.module_ctor, ptr null }] + +define void @f() sanitize_concurrency { + ret void +} + +; CHECK-LABEL: define internal void @csan.module_ctor() +; CHECK: call void @__csan_init() +; CHECK: !{i32 4, !"nosanitize_concurrency", i32 1} diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/no_sanitize.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/no_sanitize.ll new file mode 100644 index 00000000000000..506ec9f380d130 --- /dev/null +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/no_sanitize.ll @@ -0,0 +1,57 @@ +; RUN: opt < %s -passes='function(csan)' -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s +; RUN: opt < %s -passes='function(csan)' -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s + +define i32 @unattributed(ptr %a) { +entry: + %v = load i32, ptr %a, align 4 + ret i32 %v +} +; CHECK-LABEL: @unattributed( +; CHECK-NEXT: entry: +; CHECK-NEXT: %v = load i32, ptr %a, align 4 +; CHECK-NEXT: ret i32 %v + +define i32 @thread_only(ptr %a) sanitize_thread { +entry: + %v = load i32, ptr %a, align 4 + ret i32 %v +} +; CHECK-LABEL: @thread_only( +; CHECK-NEXT: entry: +; CHECK-NEXT: %v = load i32, ptr %a, align 4 +; CHECK-NEXT: ret i32 %v + +define i32 @instrumented(ptr %a) sanitize_concurrency { +entry: + %v = load i32, ptr %a, align 4 + ret i32 %v +} +; CHECK-LABEL: @instrumented( +; CHECK: call void @__csan_read4(ptr %a, i32 0) + +define i32 @disabled(ptr %a) sanitize_concurrency disable_sanitizer_instrumentation { +entry: + %v = load i32, ptr %a, align 4 + ret i32 %v +} +; CHECK-LABEL: @disabled( +; CHECK-NEXT: entry: +; CHECK-NEXT: %v = load i32, ptr %a, align 4 +; CHECK-NEXT: ret i32 %v + +declare void @callee() + +define i32 @checking_suppressed(ptr %a) sanitize_concurrency "sanitize_concurrency_no_checking_at_run_time" { +entry: + %v = load i32, ptr %a, align 4 + call void @callee() + ret i32 %v +} +; CHECK-LABEL: @checking_suppressed( +; CHECK: call void @__csan_func_entry +; CHECK: call void @__csan_ignore_thread_begin() +; CHECK-NOT: call void @__csan_read +; CHECK: %v = load i32, ptr %a, align 4 +; CHECK: {{call|invoke}} void @callee() +; CHECK: call void @__csan_ignore_thread_end() +; CHECK: call void @__csan_func_exit() diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/selection.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/selection.ll new file mode 100644 index 00000000000000..eb56ab4111e018 --- /dev/null +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/selection.ll @@ -0,0 +1,131 @@ +; RUN: opt < %s -passes='function(csan)' -S | FileCheck %s --check-prefix=DEFAULT + +@constant = constant i32 0 +@mutable = global i32 0 + +declare void @capture(ptr) + +define void @uncaptured_alloca() sanitize_concurrency { +entry: + %p = alloca i32, align 4 + store i32 1, ptr %p, align 4 + ret void +} +; DEFAULT-LABEL: @uncaptured_alloca( +; DEFAULT-NOT: __csan_write +; DEFAULT: ret void + +define void @captured_alloca() sanitize_concurrency { +entry: + %p = alloca i32, align 4 + call void @capture(ptr %p) + store i32 1, ptr %p, align 4 + ret void +} +; DEFAULT-LABEL: @captured_alloca( +; DEFAULT: {{call|invoke}} void @capture(ptr %p) +; DEFAULT: call void @__csan_write4(ptr %p, i32 0) + +define void @read_before_write(ptr %p) sanitize_concurrency { +entry: + %v = load i32, ptr %p, align 4 + store i32 %v, ptr %p, align 4 + ret void +} +; DEFAULT-LABEL: @read_before_write( +; DEFAULT-NOT: call void @__csan_read4(ptr %p, i32 0) +; DEFAULT: call void @__csan_write4(ptr %p, i32 0) + +define void @call_between_accesses(ptr %p) sanitize_concurrency { +entry: + %v = load i32, ptr %p, align 4 + call void @capture(ptr %p) + store i32 %v, ptr %p, align 4 + ret void +} +; DEFAULT-LABEL: @call_between_accesses( +; DEFAULT: call void @__csan_read4(ptr %p, i32 0) +; DEFAULT: {{call|invoke}} void @capture(ptr %p) +; DEFAULT: call void @__csan_write4(ptr %p, i32 0) + +define i32 @constant_global() sanitize_concurrency { +entry: + %v = load i32, ptr @constant, align 4 + ret i32 %v +} +; DEFAULT-LABEL: @constant_global( +; DEFAULT-NOT: __csan_read +; DEFAULT: ret i32 + +define i32 @mutable_global() sanitize_concurrency { +entry: + %v = load i32, ptr @mutable, align 4 + ret i32 %v +} +; DEFAULT-LABEL: @mutable_global( +; DEFAULT: call void @__csan_read4(ptr @mutable, i32 0) + +define void @volatile_read_before_write(ptr %p) sanitize_concurrency { +entry: + %v = load volatile i32, ptr %p, align 4 + store volatile i32 %v, ptr %p, align 4 + ret void +} +; DEFAULT-LABEL: @volatile_read_before_write( +; DEFAULT-NOT: call void @__csan_read4(ptr %p, i32 0) +; DEFAULT: call void @__csan_write4(ptr %p, i32 0) + +define void @no_sanitize_metadata(ptr %p) sanitize_concurrency { +entry: + %v = load i32, ptr %p, align 4, !nosanitize !0 + store i32 %v, ptr %p, align 4, !nosanitize !0 + ret void +} +; DEFAULT-LABEL: @no_sanitize_metadata( +; DEFAULT-NOT: __csan_read +; DEFAULT-NOT: __csan_write +; DEFAULT: ret void + +define void @unusual_size(ptr %p) sanitize_concurrency { +entry: + %v = load i24, ptr %p, align 4 + store i24 %v, ptr %p, align 4 + ret void +} +; DEFAULT-LABEL: @unusual_size( +; DEFAULT-NOT: call void @__csan_read_range(ptr %p, i64 3, i32 0) +; DEFAULT: call void @__csan_write_range(ptr %p, i64 3, i32 0) +; DEFAULT: ret void + +define void @scalable(ptr %p) sanitize_concurrency { +entry: + %v = load <vscale x 4 x i32>, ptr %p, align 16 + store <vscale x 4 x i32> %v, ptr %p, align 16 + ret void +} +; DEFAULT-LABEL: @scalable( +; DEFAULT-NOT: call void @__csan_read_range(ptr %p, i64 %{{.*}}, i32 0) +; DEFAULT: call void @__csan_write_range(ptr %p, i64 %{{.*}}, i32 0) +; DEFAULT: ret void + +define void @swifterror(ptr swifterror %p) sanitize_concurrency { +entry: + %v = load ptr, ptr %p + store ptr null, ptr %p + ret void +} +; DEFAULT-LABEL: @swifterror( +; DEFAULT-NOT: __csan_read +; DEFAULT-NOT: __csan_write +; DEFAULT: ret void + +define i32 @naked() naked sanitize_concurrency { +entry: + %v = load i32, ptr @mutable + ret i32 %v +} +; DEFAULT-LABEL: @naked( +; DEFAULT-NEXT: entry: +; DEFAULT-NEXT: %v = load i32, ptr @mutable, align 4 + +!0 = !{} diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/shared_alloca.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/shared_alloca.ll new file mode 100644 index 00000000000000..694720f1f31bd8 --- /dev/null +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/shared_alloca.ll @@ -0,0 +1,23 @@ +; RUN: opt < %s -passes='function(csan)' -S -mtriple=nvptx64-nvidia-cuda | FileCheck %s + +define void @shared_alloca() sanitize_concurrency { +entry: + %p = alloca i32, align 4, addrspace(3) + store i32 0, ptr addrspace(3) %p, align 4 + ret void +} +; CHECK-LABEL: @shared_alloca( +; CHECK: call void @__csan_write4(ptr %{{.*}}, i32 0) +; CHECK: store i32 0, ptr addrspace(3) %p, align 4 + +define void @local_alloca() sanitize_concurrency { +entry: + %p = alloca i32, align 4, addrspace(5) + store i32 0, ptr addrspace(5) %p, align 4 + ret void +} +; CHECK-LABEL: @local_alloca( +; CHECK-NEXT: entry: +; CHECK-NEXT: %p = alloca i32, align 4, addrspace(5) +; CHECK-NEXT: store i32 0, ptr addrspace(5) %p, align 4 +; CHECK-NEXT: ret void diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/unusual_sizes.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/unusual_sizes.ll new file mode 100644 index 00000000000000..cd8f320a20155b --- /dev/null +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/unusual_sizes.ll @@ -0,0 +1,33 @@ +; RUN: opt < %s -passes='function(csan)' -S | FileCheck %s + +target datalayout = "e-p:64:64" + +define void @three_byte_access(ptr %src, ptr %dst) sanitize_concurrency { +entry: + %value = load i24, ptr %src, align 1 + store i24 %value, ptr %dst, align 1 + ret void +} +; CHECK-LABEL: @three_byte_access( +; CHECK: call void @__csan_read_range(ptr %src, i64 3, i32 0) +; CHECK: call void @__csan_write_range(ptr %dst, i64 3, i32 0) + +define void @large_access(ptr %src, ptr %dst) sanitize_concurrency { +entry: + %value = load i256, ptr %src, align 32 + store i256 %value, ptr %dst, align 32 + ret void +} +; CHECK-LABEL: @large_access( +; CHECK: call void @__csan_read_range(ptr %src, i64 32, i32 0) +; CHECK: call void @__csan_write_range(ptr %dst, i64 32, i32 0) + +define void @scalable_access(ptr %src, ptr %dst) sanitize_concurrency { +entry: + %value = load <vscale x 8 x i32>, ptr %src, align 32 + store <vscale x 8 x i32> %value, ptr %dst, align 32 + ret void +} +; CHECK-LABEL: @scalable_access( +; CHECK: call void @__csan_read_range(ptr %src, i64 %{{.*}}, i32 0) +; CHECK: call void @__csan_write_range(ptr %dst, i64 %{{.*}}, i32 0) >From 2c0f1123e8838bfd4316e2982f7bdb74983d5f80 Mon Sep 17 00:00:00 2001 From: Joseph Huber <[email protected]> Date: Thu, 1 Oct 2026 16:19:09 -0500 Subject: [PATCH 2/6] triples, comments, and volatile fix to flags --- .../Instrumentation/ConcurrencySanitizer.cpp | 28 ++++--------------- .../ConcurrencySanitizer/address_space.ll | 2 +- .../ConcurrencySanitizer/atomic.ll | 2 +- .../ConcurrencySanitizer/basic.ll | 2 +- .../ConcurrencySanitizer/callbacks.ll | 25 +++++++++++++++-- .../ConcurrencySanitizer/memintrinsics.ll | 2 +- .../ConcurrencySanitizer/no_sanitize.ll | 2 +- 7 files changed, 34 insertions(+), 29 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp index 0a14d67f9f5bc1..76b87c0d8b538a 100644 --- a/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp @@ -61,8 +61,7 @@ static cl::opt<bool> ClInstrumentMemIntrinsics( cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); static cl::opt<bool> ClDistinguishVolatile( "csan-distinguish-volatile", cl::init(false), - cl::desc("Emit special instrumentation for accesses to volatiles"), - cl::Hidden); + cl::desc("Mark volatile accesses in the access flags"), cl::Hidden); STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); STATISTIC(NumOmittedReadsBeforeWrite, @@ -78,6 +77,8 @@ enum AccessFlags : unsigned { AF_None = 0, AF_Atomic = 1u << 0, AF_Compound = 1u << 1, + // 1u << 2 is reserved for the runtime's CSAN_ACCESS_WRITE. + AF_Volatile = 1u << 3, }; static bool isAtomicMemoryAccess(const Instruction *I) { @@ -184,14 +185,6 @@ struct ConcurrencySanitizer { FunctionCallee CsanUnalignedRead[kNumAccessSizes]; // void __csan_unaligned_writeN(ptr, i32); FunctionCallee CsanUnalignedWrite[kNumAccessSizes]; - // void __csan_volatile_readN(ptr, i32); - FunctionCallee CsanVolatileRead[kNumAccessSizes]; - // void __csan_volatile_writeN(ptr, i32); - FunctionCallee CsanVolatileWrite[kNumAccessSizes]; - // void __csan_unaligned_volatile_readN(ptr, i32); - FunctionCallee CsanUnalignedVolatileRead[kNumAccessSizes]; - // void __csan_unaligned_volatile_writeN(ptr, i32); - FunctionCallee CsanUnalignedVolatileWrite[kNumAccessSizes]; // void __csan_read_writeN(ptr, i32); FunctionCallee CsanCompoundRW[kNumAccessSizes]; // void __csan_unaligned_read_writeN(ptr, i32); @@ -266,12 +259,6 @@ void ConcurrencySanitizer::initialize(Module &M, const TargetLibraryInfo &TLI) { CsanWrite[I] = AccessFn("write" + ByteSize); CsanUnalignedRead[I] = AccessFn("unaligned_read" + ByteSize); CsanUnalignedWrite[I] = AccessFn("unaligned_write" + ByteSize); - CsanVolatileRead[I] = AccessFn("volatile_read" + ByteSize); - CsanVolatileWrite[I] = AccessFn("volatile_write" + ByteSize); - CsanUnalignedVolatileRead[I] = - AccessFn("unaligned_volatile_read" + ByteSize); - CsanUnalignedVolatileWrite[I] = - AccessFn("unaligned_volatile_write" + ByteSize); CsanCompoundRW[I] = AccessFn("read_write" + ByteSize); CsanUnalignedCompoundRW[I] = AccessFn("unaligned_read_write" + ByteSize); } @@ -474,6 +461,7 @@ bool ConcurrencySanitizer::instrumentAtomic(Instruction *I, AccessTy = RMW->getValOperand()->getType(); IsCompound = true; } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(I)) { + // A failed CAS is still considered a write as the output is unkonwn. Addr = CAS->getPointerOperand(); AccessTy = CAS->getNewValOperand()->getType(); IsCompound = true; @@ -561,19 +549,15 @@ bool ConcurrencySanitizer::insertAccessProbe(Instruction *I, Value *Addr, Alignment = CAS->getAlign(); IsVolatile = CAS->isVolatile(); } - IsVolatile &= ClDistinguishVolatile; uint32_t TypeSize = DL.getTypeStoreSizeInBits(AccessTy); bool Unaligned = Alignment < Align(8) && Alignment.value() % (TypeSize / 8) != 0; + if (ClDistinguishVolatile && IsVolatile) + Flags |= AF_Volatile; FunctionCallee Callback; if (IsCompound) Callback = Unaligned ? CsanUnalignedCompoundRW[Idx] : CsanCompoundRW[Idx]; - else if (IsVolatile && Unaligned) - Callback = IsWrite ? CsanUnalignedVolatileWrite[Idx] - : CsanUnalignedVolatileRead[Idx]; - else if (IsVolatile) - Callback = IsWrite ? CsanVolatileWrite[Idx] : CsanVolatileRead[Idx]; else if (Unaligned) Callback = IsWrite ? CsanUnalignedWrite[Idx] : CsanUnalignedRead[Idx]; else diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/address_space.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/address_space.ll index c0190cf74b5c60..1682e67ae1c090 100644 --- a/llvm/test/Instrumentation/ConcurrencySanitizer/address_space.ll +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/address_space.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 -; RUN: opt < %s -passes='function(csan)' -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s --check-prefixes=CHECK +; RUN: opt < %s -passes='function(csan)' -S -mtriple=amdgpu-amd-amdhsa | FileCheck %s --check-prefixes=CHECK define void @load_as1(ptr addrspace(1) %p) sanitize_concurrency { ; CHECK-LABEL: define void @load_as1( diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/atomic.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/atomic.ll index ac2e0c2f78835e..15648fe92d5656 100644 --- a/llvm/test/Instrumentation/ConcurrencySanitizer/atomic.ll +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/atomic.ll @@ -1,5 +1,5 @@ ; RUN: opt < %s -passes='function(csan)' -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s -; RUN: opt < %s -passes='function(csan)' -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s +; RUN: opt < %s -passes='function(csan)' -S -mtriple=amdgpu-amd-amdhsa | FileCheck %s define i32 @atomic_load(ptr %a) sanitize_concurrency { entry: diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/basic.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/basic.ll index fa1cc65c709892..6d38e39b50e0f1 100644 --- a/llvm/test/Instrumentation/ConcurrencySanitizer/basic.ll +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/basic.ll @@ -1,5 +1,5 @@ ; RUN: opt < %s -passes='module(csan-module),function(csan)' -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s -; RUN: opt < %s -passes='module(csan-module),function(csan)' -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s +; RUN: opt < %s -passes='module(csan-module),function(csan)' -S -mtriple=amdgpu-amd-amdhsa | FileCheck %s define void @read_write(ptr %a, ptr %b) sanitize_concurrency { entry: diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/callbacks.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/callbacks.ll index d7d8269b081c17..7934c81425ecd4 100644 --- a/llvm/test/Instrumentation/ConcurrencySanitizer/callbacks.ll +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/callbacks.ll @@ -43,7 +43,7 @@ entry: ret i32 %v } ; CHECK-LABEL: @volatile_read( -; CHECK: call void @__csan_volatile_read4(ptr %p, i32 0) +; CHECK: call void @__csan_read4(ptr %p, i32 8) define void @unaligned_volatile_write(ptr %p) sanitize_concurrency { entry: @@ -51,7 +51,28 @@ entry: ret void } ; CHECK-LABEL: @unaligned_volatile_write( -; CHECK: call void @__csan_unaligned_volatile_write8(ptr %p, i32 0) +; CHECK: call void @__csan_unaligned_write8(ptr %p, i32 8) + +define i128 @large_volatile_read(ptr %p) sanitize_concurrency { +entry: + %v = load volatile i128, ptr %p, align 16 + ret i128 %v +} +; CHECK-LABEL: @large_volatile_read( +; CHECK: call void @__csan_read16(ptr %p, i32 8) + +define void @volatile_range(ptr %src, ptr %dst, i64 %n) sanitize_concurrency { +entry: + %v = load volatile i24, ptr %src, align 4 + store volatile i24 %v, ptr %dst, align 4 + call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %src, i64 8, i1 true) + ret void +} +; CHECK-LABEL: @volatile_range( +; CHECK: call void @__csan_read_range(ptr %src, i64 3, i32 0) +; CHECK: call void @__csan_write_range(ptr %dst, i64 3, i32 0) +; CHECK: call void @__csan_read_range(ptr %src, i64 8, i32 0) +; CHECK: call void @__csan_write_range(ptr %dst, i64 8, i32 0) define i32 @unaligned_atomic_rmw(ptr %p) sanitize_concurrency { entry: diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/memintrinsics.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/memintrinsics.ll index 35216aa84091a8..b28cdef8b55948 100644 --- a/llvm/test/Instrumentation/ConcurrencySanitizer/memintrinsics.ll +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/memintrinsics.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -passes='function(csan)' -csan-instrument-func-entry-exit=0 -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s +; RUN: opt < %s -passes='function(csan)' -csan-instrument-func-entry-exit=0 -S -mtriple=amdgpu-amd-amdhsa | FileCheck %s define void @instrument_source(ptr addrspace(5) %dst, ptr addrspace(1) %src, i32 %n) sanitize_concurrency { entry: diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/no_sanitize.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/no_sanitize.ll index 506ec9f380d130..84201ac1a53185 100644 --- a/llvm/test/Instrumentation/ConcurrencySanitizer/no_sanitize.ll +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/no_sanitize.ll @@ -1,5 +1,5 @@ ; RUN: opt < %s -passes='function(csan)' -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s -; RUN: opt < %s -passes='function(csan)' -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s +; RUN: opt < %s -passes='function(csan)' -S -mtriple=amdgpu-amd-amdhsa | FileCheck %s define i32 @unattributed(ptr %a) { entry: >From 8739f3f7e52443743f8f31cca3564b84d2101b59 Mon Sep 17 00:00:00 2001 From: Joseph Huber <[email protected]> Date: Thu, 1 Oct 2026 17:02:01 -0500 Subject: [PATCH 3/6] Apply the volatile policy in the instrumentation --- .../Instrumentation/ConcurrencySanitizer.cpp | 21 ++++++++++++------- .../ConcurrencySanitizer/callbacks.ll | 10 ++++++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp index 76b87c0d8b538a..bf417f36ffc502 100644 --- a/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp @@ -61,7 +61,8 @@ static cl::opt<bool> ClInstrumentMemIntrinsics( cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); static cl::opt<bool> ClDistinguishVolatile( "csan-distinguish-volatile", cl::init(false), - cl::desc("Mark volatile accesses in the access flags"), cl::Hidden); + cl::desc("Treat aligned volatile accesses up to 8 bytes as atomic"), + cl::Hidden); STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); STATISTIC(NumOmittedReadsBeforeWrite, @@ -77,8 +78,6 @@ enum AccessFlags : unsigned { AF_None = 0, AF_Atomic = 1u << 0, AF_Compound = 1u << 1, - // 1u << 2 is reserved for the runtime's CSAN_ACCESS_WRITE. - AF_Volatile = 1u << 3, }; static bool isAtomicMemoryAccess(const Instruction *I) { @@ -553,8 +552,6 @@ bool ConcurrencySanitizer::insertAccessProbe(Instruction *I, Value *Addr, uint32_t TypeSize = DL.getTypeStoreSizeInBits(AccessTy); bool Unaligned = Alignment < Align(8) && Alignment.value() % (TypeSize / 8) != 0; - if (ClDistinguishVolatile && IsVolatile) - Flags |= AF_Volatile; FunctionCallee Callback; if (IsCompound) Callback = Unaligned ? CsanUnalignedCompoundRW[Idx] : CsanCompoundRW[Idx]; @@ -564,8 +561,18 @@ bool ConcurrencySanitizer::insertAccessProbe(Instruction *I, Value *Addr, Callback = IsWrite ? CsanWrite[Idx] : CsanRead[Idx]; InstrumentationIRBuilder IRB(I); - IRB.CreateCall(Callback, {getCallbackAddress(IRB, Addr), - ConstantInt::get(FlagsTy, Flags)}); + Value *CallbackAddr = getCallbackAddress(IRB, Addr); + Value *FlagsVal = ConstantInt::get(FlagsTy, Flags); + if (ClDistinguishVolatile && IsVolatile && TypeSize <= 64) { + // The declared alignment may understate the pointer's real alignment. + Value *IsAligned = IRB.getTrue(); + if (Unaligned) + IsAligned = IRB.CreateIsNull(IRB.CreateAnd( + IRB.CreatePtrToInt(CallbackAddr, IntptrTy), TypeSize / 8 - 1)); + FlagsVal = IRB.CreateSelect( + IsAligned, ConstantInt::get(FlagsTy, Flags | AF_Atomic), FlagsVal); + } + IRB.CreateCall(Callback, {CallbackAddr, FlagsVal}); return true; } diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/callbacks.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/callbacks.ll index 7934c81425ecd4..49e9d4fe885010 100644 --- a/llvm/test/Instrumentation/ConcurrencySanitizer/callbacks.ll +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/callbacks.ll @@ -43,7 +43,7 @@ entry: ret i32 %v } ; CHECK-LABEL: @volatile_read( -; CHECK: call void @__csan_read4(ptr %p, i32 8) +; CHECK: call void @__csan_read4(ptr %p, i32 1) define void @unaligned_volatile_write(ptr %p) sanitize_concurrency { entry: @@ -51,7 +51,11 @@ entry: ret void } ; CHECK-LABEL: @unaligned_volatile_write( -; CHECK: call void @__csan_unaligned_write8(ptr %p, i32 8) +; CHECK: %[[INT:.*]] = ptrtoint ptr %p to i64 +; CHECK: %[[LOW:.*]] = and i64 %[[INT]], 7 +; CHECK: %[[ALIGNED:.*]] = icmp eq i64 %[[LOW]], 0 +; CHECK: %[[FLAGS:.*]] = select i1 %[[ALIGNED]], i32 1, i32 0 +; CHECK: call void @__csan_unaligned_write8(ptr %p, i32 %[[FLAGS]]) define i128 @large_volatile_read(ptr %p) sanitize_concurrency { entry: @@ -59,7 +63,7 @@ entry: ret i128 %v } ; CHECK-LABEL: @large_volatile_read( -; CHECK: call void @__csan_read16(ptr %p, i32 8) +; CHECK: call void @__csan_read16(ptr %p, i32 0) define void @volatile_range(ptr %src, ptr %dst, i64 %n) sanitize_concurrency { entry: >From d35beeff2480160fe3429b18c46a3afccee8fe76 Mon Sep 17 00:00:00 2001 From: Joseph Huber <[email protected]> Date: Thu, 1 Oct 2026 17:17:21 -0500 Subject: [PATCH 4/6] Instrument atomic and pattern memory intrinsics --- .../Instrumentation/ConcurrencySanitizer.cpp | 56 ++++++++++--------- .../ConcurrencySanitizer/memintrinsics.ll | 54 ++++++++++++++++++ 2 files changed, 83 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp index bf417f36ffc502..9c8e534690d326 100644 --- a/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp @@ -153,7 +153,7 @@ struct ConcurrencySanitizer { struct MemoryAccessLists { SmallVector<Instruction *, 8> LoadsAndStores; SmallVector<Instruction *, 8> AtomicAccesses; - SmallVector<MemIntrinsic *, 8> MemIntrinCalls; + SmallVector<IntrinsicInst *, 8> MemIntrinCalls; bool HasCalls = false; }; @@ -161,7 +161,7 @@ struct ConcurrencySanitizer { void collectMemoryAccesses(Function &F, MemoryAccessLists &Out); bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); bool instrumentAtomic(Instruction *I, const DataLayout &DL); - bool instrumentMemIntrinsic(MemIntrinsic *M); + bool instrumentMemIntrinsic(IntrinsicInst *II, const DataLayout &DL); bool insertAccessProbe(Instruction *I, Value *Addr, Type *AccessTy, const DataLayout &DL, bool IsWrite, bool IsCompound, bool IsAtomic); @@ -324,8 +324,8 @@ bool ConcurrencySanitizer::sanitizeFunction(Function &F, Res |= instrumentAtomic(I, DL); if (ClInstrumentMemIntrinsics && SanitizeFunction) - for (MemIntrinsic *MI : Acc.MemIntrinCalls) - Res |= instrumentMemIntrinsic(MI); + for (IntrinsicInst *II : Acc.MemIntrinCalls) + Res |= instrumentMemIntrinsic(II, DL); if (NeedsRuntimeIgnores) { insertRuntimeIgnores(F); @@ -410,8 +410,8 @@ void ConcurrencySanitizer::collectMemoryAccesses(Function &F, LocalLoadsAndStores.push_back(&Inst); else if (isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) { FlushLocalAccesses(); - if (auto *MI = dyn_cast<MemIntrinsic>(&Inst)) - Out.MemIntrinCalls.push_back(MI); + if (isa<AnyMemIntrinsic, MemSetPatternInst>(Inst)) + Out.MemIntrinCalls.push_back(cast<IntrinsicInst>(&Inst)); Out.HasCalls = true; } } @@ -480,34 +480,36 @@ bool ConcurrencySanitizer::instrumentAtomic(Instruction *I, return true; } -bool ConcurrencySanitizer::instrumentMemIntrinsic(MemIntrinsic *M) { - if (auto *MS = dyn_cast<MemSetInst>(M)) { - if (!shouldInstrumentAddress(MS->getRawDest())) - return false; - InstrumentationIRBuilder IRB(M); - Value *Len = IRB.CreateIntCast(M->getLength(), IntptrTy, false); - IRB.CreateCall(CsanWriteRange, {getCallbackAddress(IRB, MS->getRawDest()), - Len, ConstantInt::get(FlagsTy, AF_None)}); - ++NumInstrumentedWrites; - return true; - } - - auto *MT = cast<MemTransferInst>(M); - bool InstrumentRead = shouldInstrumentAddress(MT->getRawSource()); - bool InstrumentWrite = shouldInstrumentAddress(MT->getRawDest()); +bool ConcurrencySanitizer::instrumentMemIntrinsic(IntrinsicInst *II, + const DataLayout &DL) { + auto *MI = dyn_cast<AnyMemIntrinsic>(II); + auto *MSP = dyn_cast<MemSetPatternInst>(II); + Value *Dst = MI ? MI->getRawDest() : MSP->getRawDest(); + Value *Src = nullptr; + if (auto *MT = dyn_cast<AnyMemTransferInst>(II)) + Src = MT->getRawSource(); + + bool InstrumentRead = Src && shouldInstrumentAddress(Src); + bool InstrumentWrite = shouldInstrumentAddress(Dst); if (!InstrumentRead && !InstrumentWrite) return false; - InstrumentationIRBuilder IRB(M); - Value *Len = IRB.CreateIntCast(M->getLength(), IntptrTy, false); + InstrumentationIRBuilder IRB(II); + Value *Len = IRB.CreateIntCast(MI ? MI->getLength() : MSP->getLength(), + IntptrTy, false); + // The length of a pattern memset counts patterns, not bytes. + if (MSP) { + TypeSize Size = DL.getTypeAllocSize(MSP->getValue()->getType()); + Len = IRB.CreateMul(Len, IRB.CreateTypeSize(IntptrTy, Size)); + } + Constant *Flags = + ConstantInt::get(FlagsTy, MI && MI->isAtomic() ? AF_Atomic : AF_None); if (InstrumentRead) { - IRB.CreateCall(CsanReadRange, {getCallbackAddress(IRB, MT->getRawSource()), - Len, ConstantInt::get(FlagsTy, AF_None)}); + IRB.CreateCall(CsanReadRange, {getCallbackAddress(IRB, Src), Len, Flags}); ++NumInstrumentedReads; } if (InstrumentWrite) { - IRB.CreateCall(CsanWriteRange, {getCallbackAddress(IRB, MT->getRawDest()), - Len, ConstantInt::get(FlagsTy, AF_None)}); + IRB.CreateCall(CsanWriteRange, {getCallbackAddress(IRB, Dst), Len, Flags}); ++NumInstrumentedWrites; } return true; diff --git a/llvm/test/Instrumentation/ConcurrencySanitizer/memintrinsics.ll b/llvm/test/Instrumentation/ConcurrencySanitizer/memintrinsics.ll index b28cdef8b55948..dd2360abdce23a 100644 --- a/llvm/test/Instrumentation/ConcurrencySanitizer/memintrinsics.ll +++ b/llvm/test/Instrumentation/ConcurrencySanitizer/memintrinsics.ll @@ -44,3 +44,57 @@ entry: ; CHECK: %[[DST:.*]] = addrspacecast ptr addrspace(1) %dst to ptr ; CHECK: call void @__csan_write_range(ptr %[[DST]], i64 %[[LEN]], i32 0) ; CHECK: call void @llvm.memset + +define void @atomic_memcpy(ptr addrspace(1) %dst, ptr addrspace(1) %src, i32 %n) sanitize_concurrency { +entry: + call void @llvm.memcpy.element.unordered.atomic.p1.p1.i32(ptr addrspace(1) align 4 %dst, ptr addrspace(1) align 4 %src, i32 %n, i32 4) + ret void +} +; CHECK-LABEL: @atomic_memcpy( +; CHECK: %[[LEN:.*]] = zext i32 %n to i64 +; CHECK: %[[SRC:.*]] = addrspacecast ptr addrspace(1) %src to ptr +; CHECK: call void @__csan_read_range(ptr %[[SRC]], i64 %[[LEN]], i32 1) +; CHECK: %[[DST:.*]] = addrspacecast ptr addrspace(1) %dst to ptr +; CHECK: call void @__csan_write_range(ptr %[[DST]], i64 %[[LEN]], i32 1) +; CHECK: call void @llvm.memcpy.element.unordered.atomic + +define void @atomic_memmove(ptr addrspace(1) %dst, ptr addrspace(5) %src, i32 %n) sanitize_concurrency { +entry: + call void @llvm.memmove.element.unordered.atomic.p1.p5.i32(ptr addrspace(1) align 4 %dst, ptr addrspace(5) align 4 %src, i32 %n, i32 4) + ret void +} +; CHECK-LABEL: @atomic_memmove( +; CHECK-NOT: __csan_read_range +; CHECK: call void @__csan_write_range(ptr %{{.*}}, i64 %{{.*}}, i32 1) +; CHECK: call void @llvm.memmove.element.unordered.atomic + +define void @atomic_memset(ptr addrspace(1) %dst, i32 %n) sanitize_concurrency { +entry: + call void @llvm.memset.element.unordered.atomic.p1.i32(ptr addrspace(1) align 4 %dst, i8 0, i32 %n, i32 4) + ret void +} +; CHECK-LABEL: @atomic_memset( +; CHECK: call void @__csan_write_range(ptr %{{.*}}, i64 %{{.*}}, i32 1) +; CHECK: call void @llvm.memset.element.unordered.atomic + +define void @memset_pattern(ptr addrspace(1) %dst, i32 %n) sanitize_concurrency { +entry: + call void @llvm.experimental.memset.pattern.p1.i64.i32(ptr addrspace(1) %dst, i64 7, i32 %n, i1 false) + ret void +} +; CHECK-LABEL: @memset_pattern( +; CHECK: %[[COUNT:.*]] = zext i32 %n to i64 +; CHECK: %[[LEN:.*]] = mul i64 %[[COUNT]], 8 +; CHECK: %[[DST:.*]] = addrspacecast ptr addrspace(1) %dst to ptr +; CHECK: call void @__csan_write_range(ptr %[[DST]], i64 %[[LEN]], i32 0) +; CHECK: call void @llvm.experimental.memset.pattern + +define void @memset_pattern_private(ptr addrspace(5) %dst, i32 %n) sanitize_concurrency { +entry: + call void @llvm.experimental.memset.pattern.p5.i64.i32(ptr addrspace(5) %dst, i64 7, i32 %n, i1 false) + ret void +} +; CHECK-LABEL: @memset_pattern_private( +; CHECK-NEXT: entry: +; CHECK-NEXT: call void @llvm.experimental.memset.pattern +; CHECK-NEXT: ret void >From 1ddebe88544e75caf6bc83fda96d9039b720ad21 Mon Sep 17 00:00:00 2001 From: Joseph Huber <[email protected]> Date: Thu, 1 Oct 2026 19:43:17 -0500 Subject: [PATCH 5/6] enum class and return Res --- .../Instrumentation/ConcurrencySanitizer.cpp | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp index 9c8e534690d326..3c1d6af7cd512e 100644 --- a/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Instrumentation/ConcurrencySanitizer.h" +#include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" @@ -74,10 +75,11 @@ static constexpr char kCsanInitName[] = "__csan_init"; namespace { /// Must match CSAN_ACCESS_* in compiler-rt/lib/csan/csan_defs.h. -enum AccessFlags : unsigned { - AF_None = 0, - AF_Atomic = 1u << 0, - AF_Compound = 1u << 1, +enum class AccessFlags : unsigned { + None = 0, + Atomic = 1u << 0, + Compound = 1u << 1, + LLVM_MARK_AS_BITMASK_ENUM(Compound), }; static bool isAtomicMemoryAccess(const Instruction *I) { @@ -169,6 +171,9 @@ struct ConcurrencySanitizer { bool shouldInstrumentAccess(Instruction *I) const; void insertFuncEntryExit(Function &F); void insertRuntimeIgnores(Function &F); + Constant *getFlags(AccessFlags Flags) const { + return ConstantInt::get(FlagsTy, to_underlying(Flags)); + } Module *Mod = nullptr; Type *IntptrTy = nullptr; @@ -336,8 +341,7 @@ bool ConcurrencySanitizer::sanitizeFunction(Function &F, insertFuncEntryExit(F); Res = true; } - // Callback declarations may have changed the module. - return true; + return Res; } bool ConcurrencySanitizer::shouldInstrumentAddress(Value *Addr) const { @@ -503,7 +507,7 @@ bool ConcurrencySanitizer::instrumentMemIntrinsic(IntrinsicInst *II, Len = IRB.CreateMul(Len, IRB.CreateTypeSize(IntptrTy, Size)); } Constant *Flags = - ConstantInt::get(FlagsTy, MI && MI->isAtomic() ? AF_Atomic : AF_None); + getFlags(MI && MI->isAtomic() ? AccessFlags::Atomic : AccessFlags::None); if (InstrumentRead) { IRB.CreateCall(CsanReadRange, {getCallbackAddress(IRB, Src), Len, Flags}); ++NumInstrumentedReads; @@ -521,17 +525,19 @@ bool ConcurrencySanitizer::insertAccessProbe(Instruction *I, Value *Addr, bool IsCompound, bool IsAtomic) { if (Addr->isSwiftError()) return false; - unsigned Flags = - (IsAtomic ? AF_Atomic : AF_None) | (IsCompound ? AF_Compound : AF_None); + AccessFlags Flags = AccessFlags::None; + if (IsAtomic) + Flags |= AccessFlags::Atomic; + if (IsCompound) + Flags |= AccessFlags::Compound; int Idx = getAccessSizeIndex(AccessTy, DL); if (Idx < 0) { if (IsCompound) return false; InstrumentationIRBuilder IRB(I); Value *Len = IRB.CreateTypeSize(IntptrTy, DL.getTypeStoreSize(AccessTy)); - IRB.CreateCall( - IsWrite ? CsanWriteRange : CsanReadRange, - {getCallbackAddress(IRB, Addr), Len, ConstantInt::get(FlagsTy, Flags)}); + IRB.CreateCall(IsWrite ? CsanWriteRange : CsanReadRange, + {getCallbackAddress(IRB, Addr), Len, getFlags(Flags)}); return true; } @@ -564,7 +570,7 @@ bool ConcurrencySanitizer::insertAccessProbe(Instruction *I, Value *Addr, InstrumentationIRBuilder IRB(I); Value *CallbackAddr = getCallbackAddress(IRB, Addr); - Value *FlagsVal = ConstantInt::get(FlagsTy, Flags); + Value *FlagsVal = getFlags(Flags); if (ClDistinguishVolatile && IsVolatile && TypeSize <= 64) { // The declared alignment may understate the pointer's real alignment. Value *IsAligned = IRB.getTrue(); @@ -572,7 +578,7 @@ bool ConcurrencySanitizer::insertAccessProbe(Instruction *I, Value *Addr, IsAligned = IRB.CreateIsNull(IRB.CreateAnd( IRB.CreatePtrToInt(CallbackAddr, IntptrTy), TypeSize / 8 - 1)); FlagsVal = IRB.CreateSelect( - IsAligned, ConstantInt::get(FlagsTy, Flags | AF_Atomic), FlagsVal); + IsAligned, getFlags(Flags | AccessFlags::Atomic), FlagsVal); } IRB.CreateCall(Callback, {CallbackAddr, FlagsVal}); return true; >From 50a872f29400035f61ee02bd08bacbd4c3eb8760 Mon Sep 17 00:00:00 2001 From: Joseph Huber <[email protected]> Date: Fri, 2 Oct 2026 09:54:01 -0500 Subject: [PATCH 6/6] names and braces --- .../Instrumentation/ConcurrencySanitizer.cpp | 56 +++++++++++++------ 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp index 3c1d6af7cd512e..14e2da74698a45 100644 --- a/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/ConcurrencySanitizer.cpp @@ -17,7 +17,6 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" -#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/bit.h" #include "llvm/Analysis/CaptureTracking.h" #include "llvm/Analysis/TargetLibraryInfo.h" @@ -72,6 +71,30 @@ STATISTIC(NumOmittedReadsBeforeWrite, static constexpr char kCsanModuleCtorName[] = "csan.module_ctor"; static constexpr char kCsanInitName[] = "__csan_init"; +// Accesses sizes are powers of two: 1, 2, 4, 8, 16. +static constexpr size_t kNumAccessSizes = 5; +static constexpr StringLiteral kCsanReadNames[kNumAccessSizes] = { + "__csan_read1", "__csan_read2", "__csan_read4", "__csan_read8", + "__csan_read16"}; +static constexpr StringLiteral kCsanWriteNames[kNumAccessSizes] = { + "__csan_write1", "__csan_write2", "__csan_write4", "__csan_write8", + "__csan_write16"}; +static constexpr StringLiteral kCsanUnalignedReadNames[kNumAccessSizes] = { + "__csan_unaligned_read1", "__csan_unaligned_read2", + "__csan_unaligned_read4", "__csan_unaligned_read8", + "__csan_unaligned_read16"}; +static constexpr StringLiteral kCsanUnalignedWriteNames[kNumAccessSizes] = { + "__csan_unaligned_write1", "__csan_unaligned_write2", + "__csan_unaligned_write4", "__csan_unaligned_write8", + "__csan_unaligned_write16"}; +static constexpr StringLiteral kCsanCompoundRWNames[kNumAccessSizes] = { + "__csan_read_write1", "__csan_read_write2", "__csan_read_write4", + "__csan_read_write8", "__csan_read_write16"}; +static constexpr StringLiteral kCsanUnalignedCompoundRWNames[kNumAccessSizes] = + {"__csan_unaligned_read_write1", "__csan_unaligned_read_write2", + "__csan_unaligned_read_write4", "__csan_unaligned_read_write8", + "__csan_unaligned_read_write16"}; + namespace { /// Must match CSAN_ACCESS_* in compiler-rt/lib/csan/csan_defs.h. @@ -133,18 +156,21 @@ static int getAccessSizeIndex(Type *Ty, const DataLayout &DL) { } static bool addressSpaceMayRace(const Triple &T, unsigned AS) { - if (T.isAMDGPU()) + if (T.isAMDGPU()) { // GDS and buffer fat pointers cannot form a generic watchpoint key. return AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::LOCAL_ADDRESS; - if (T.isNVPTX()) + } + if (T.isNVPTX()) { return AS == NVPTXAS::ADDRESS_SPACE_GENERIC || AS == NVPTXAS::ADDRESS_SPACE_GLOBAL || AS == NVPTXAS::ADDRESS_SPACE_SHARED || AS == NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER; - if (T.isSPIRV()) + } + if (T.isSPIRV()) { // FIXME: No exposed address spaces for SPIR-V. return false; + } return AS == 0; } @@ -179,8 +205,6 @@ struct ConcurrencySanitizer { Type *IntptrTy = nullptr; IntegerType *FlagsTy = nullptr; - // Accesses sizes are powers of two: 1, 2, 4, 8, 16. - static const size_t kNumAccessSizes = 5; // void __csan_readN(ptr, i32); FunctionCallee CsanRead[kNumAccessSizes]; // void __csan_writeN(ptr, i32); @@ -253,18 +277,16 @@ void ConcurrencySanitizer::initialize(Module &M, const TargetLibraryInfo &TLI) { M.getOrInsertFunction("__csan_ignore_thread_begin", Attr, VoidTy); CsanIgnoreEnd = M.getOrInsertFunction("__csan_ignore_thread_end", Attr, VoidTy); + auto AccessFn = [&](StringRef Name) { + return M.getOrInsertFunction(Name, Attr, VoidTy, PtrTy, FlagsTy); + }; for (unsigned I = 0; I < kNumAccessSizes; ++I) { - std::string ByteSize = utostr(1U << I); - auto AccessFn = [&](const Twine &Name) { - return M.getOrInsertFunction(("__csan_" + Name).str(), Attr, VoidTy, - PtrTy, FlagsTy); - }; - CsanRead[I] = AccessFn("read" + ByteSize); - CsanWrite[I] = AccessFn("write" + ByteSize); - CsanUnalignedRead[I] = AccessFn("unaligned_read" + ByteSize); - CsanUnalignedWrite[I] = AccessFn("unaligned_write" + ByteSize); - CsanCompoundRW[I] = AccessFn("read_write" + ByteSize); - CsanUnalignedCompoundRW[I] = AccessFn("unaligned_read_write" + ByteSize); + CsanRead[I] = AccessFn(kCsanReadNames[I]); + CsanWrite[I] = AccessFn(kCsanWriteNames[I]); + CsanUnalignedRead[I] = AccessFn(kCsanUnalignedReadNames[I]); + CsanUnalignedWrite[I] = AccessFn(kCsanUnalignedWriteNames[I]); + CsanCompoundRW[I] = AccessFn(kCsanCompoundRWNames[I]); + CsanUnalignedCompoundRW[I] = AccessFn(kCsanUnalignedCompoundRWNames[I]); } IntegerType *OrdTy = IRB.getInt32Ty(); CsanReadRange = M.getOrInsertFunction("__csan_read_range", Attr, VoidTy, _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
