https://github.com/gbossu updated https://github.com/llvm/llvm-project/pull/226948
From 1e4754ca8cb7dba85f40613feec889ca09eac761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <[email protected]> Date: Mon, 28 Sep 2026 10:28:14 +0000 Subject: [PATCH 1/3] [LV] -scalable-vectorization=preferred should not override UserVF=1 For loops using #pragma clang loop vectorize(disable), one would expect vectorisation to be disabled regardless of -scalable-vectorization flags. However, LV used to turn a UserVF=1 (vectorisation disabled) into UserVF=vscale x 1 when -scalable-vectorization=preferred was passed. --- .../Vectorize/LoopVectorizationLegality.cpp | 7 +++- .../disable-with-prefer-scalable.ll | 37 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 llvm/test/Transforms/LoopVectorize/disable-with-prefer-scalable.ll diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp index 862470ce0de6d..fe68a03ca7e40 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -132,8 +132,11 @@ LoopVectorizeHints::LoopVectorizeHints(const Loop *L, // If the flag is set to force any use of scalable vectors, override the loop // hints. - if (ForceScalableVectorization.getValue() != - LoopVectorizeHints::SK_Unspecified) + // However: A preference must not turn a UserVF of 1, used by e.g. + // vectorize(disable) pragmas, into a vscale x 1 VF. + if (ForceScalableVectorization.getValue() != SK_Unspecified && + (Width.Value != 1 || + ForceScalableVectorization.getValue() != SK_PreferScalable)) Scalable = ForceScalableVectorization.getValue(); // If force-vector-width is scalable, force scalable vectorization. diff --git a/llvm/test/Transforms/LoopVectorize/disable-with-prefer-scalable.ll b/llvm/test/Transforms/LoopVectorize/disable-with-prefer-scalable.ll new file mode 100644 index 0000000000000..084bebf4a1202 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/disable-with-prefer-scalable.ll @@ -0,0 +1,37 @@ +; RUN: opt -force-target-supports-scalable-vectors -scalable-vectorization=preferred -passes=loop-vectorize -S %s | FileCheck %s +; +; Clang lowers "#pragma clang loop vectorize(disable) interleave_count(1)" to +; width 1 and interleave count 1. + +; Ensure the scalable preference does not override the usual fixed-width meaning +; of that width hint: +; A UserVF of 1 should disable vectorization. +; -scalable-vectorization=preferred should not yield a VF of vscale x 1. +; +; CHECK-LABEL: define void @repro( +; CHECK-NOT: vector.body: +define void @repro(ptr %out, ptr %in, i32 %tc) { +entry: + %start = zext i32 %tc to i64 + br label %loop + +loop: + %iv = phi i64 [ %start, %entry ], [ %next, %loop ] + %in.ptr = getelementptr inbounds i8, ptr %in, i64 %iv + %value = load i8, ptr %in.ptr, align 1 + %sum = add i8 %value, 10 + %out.ptr = getelementptr inbounds i8, ptr %out, i64 %iv + store i8 %sum, ptr %out.ptr, align 1 + %next = add nsw i64 %iv, -1 + %next32 = and i64 %next, 4294967295 + %done = icmp eq i64 %next32, 0 + br i1 %done, label %exit, label %loop, !llvm.loop !0 + +exit: + ret void +} + +!0 = distinct !{!0, !1, !2, !3} +!1 = !{!"llvm.loop.mustprogress"} +!2 = !{!"llvm.loop.vectorize.width", i32 1} +!3 = !{!"llvm.loop.interleave.count", i32 1} From 8d7a63a5b03e01c8eb24ab964e8187d9d104bd8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <[email protected]> Date: Tue, 29 Sep 2026 09:47:14 +0000 Subject: [PATCH 2/3] -scalable-vectorization should not override !loop MD --- .../Vectorize/LoopVectorizationLegality.cpp | 29 ++++++-------- .../disable-with-prefer-scalable.ll | 7 ++-- .../scalable-width-one-with-off.ll | 38 +++++++++++++++++++ 3 files changed, 53 insertions(+), 21 deletions(-) create mode 100644 llvm/test/Transforms/LoopVectorize/scalable-width-one-with-off.ll diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp index fe68a03ca7e40..26f7e024ced78 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -112,32 +112,25 @@ LoopVectorizeHints::LoopVectorizeHints(const Loop *L, if (VectorizerParams::isInterleaveForced()) Interleave.Value = VectorizerParams::VectorizationInterleave; - // If the metadata doesn't explicitly specify whether to enable scalable - // vectorization, then decide based on the following criteria (increasing - // level of priority): - // - Target default - // - Metadata width - // - Force option (always overrides) + // Scalable vectorization is decided based on the following criteria + // (increasing level of priority): + // 1. Target TTI default + // 2. A UserVF (!loop or cl::opt) implies SK_FixedWidthOnly + // 3. -scalable-vectorization cl::opt + // 4. !loop.vectorize.scalable enable/disable metadata + // 5. A scalable -force-vector-width cl::opt implies SK_AlwaysScalable if ((LoopVectorizeHints::ScalableForceKind)Scalable == SK_Unspecified) { if (TTI) Scalable = TTI->enableScalableVectorization() ? SK_PreferScalable : SK_FixedWidthOnly; if (Width.Value) - // If the width is set, but the metadata says nothing about the scalable - // property, then assume it concerns only a fixed-width UserVF. - // If width is not set, the flag takes precedence. Scalable = SK_FixedWidthOnly; - } - // If the flag is set to force any use of scalable vectors, override the loop - // hints. - // However: A preference must not turn a UserVF of 1, used by e.g. - // vectorize(disable) pragmas, into a vscale x 1 VF. - if (ForceScalableVectorization.getValue() != SK_Unspecified && - (Width.Value != 1 || - ForceScalableVectorization.getValue() != SK_PreferScalable)) - Scalable = ForceScalableVectorization.getValue(); + auto ForcedScalable = ForceScalableVectorization.getValue(); + if (ForcedScalable != SK_Unspecified) + Scalable = ForcedScalable; + } // If force-vector-width is scalable, force scalable vectorization. if (VectorizerParams::VectorizationFactor.isScalable()) diff --git a/llvm/test/Transforms/LoopVectorize/disable-with-prefer-scalable.ll b/llvm/test/Transforms/LoopVectorize/disable-with-prefer-scalable.ll index 084bebf4a1202..b11375da54b69 100644 --- a/llvm/test/Transforms/LoopVectorize/disable-with-prefer-scalable.ll +++ b/llvm/test/Transforms/LoopVectorize/disable-with-prefer-scalable.ll @@ -1,7 +1,7 @@ ; RUN: opt -force-target-supports-scalable-vectors -scalable-vectorization=preferred -passes=loop-vectorize -S %s | FileCheck %s ; ; Clang lowers "#pragma clang loop vectorize(disable) interleave_count(1)" to -; width 1 and interleave count 1. +; width 1, scalable.disable and interleave count 1. ; Ensure the scalable preference does not override the usual fixed-width meaning ; of that width hint: @@ -31,7 +31,8 @@ exit: ret void } -!0 = distinct !{!0, !1, !2, !3} +!0 = distinct !{!0, !1, !2, !3, !4} !1 = !{!"llvm.loop.mustprogress"} !2 = !{!"llvm.loop.vectorize.width", i32 1} -!3 = !{!"llvm.loop.interleave.count", i32 1} +!3 = !{!"llvm.loop.vectorize.scalable.disable"} +!4 = !{!"llvm.loop.interleave.count", i32 1} diff --git a/llvm/test/Transforms/LoopVectorize/scalable-width-one-with-off.ll b/llvm/test/Transforms/LoopVectorize/scalable-width-one-with-off.ll new file mode 100644 index 0000000000000..7681748d61060 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/scalable-width-one-with-off.ll @@ -0,0 +1,38 @@ +; RUN: opt -force-target-supports-scalable-vectors -scalable-vectorization=off -passes=loop-vectorize -S %s | FileCheck %s +; +; Clang lowers "#pragma clang loop vectorize_width(1, scalable)" and +; "interleave_count(1)" to width 1, scalable.enable, and interleave count 1. +; The explicit scalable width must remain vscale x 1 even with the global +; scalable-vectorization option set to off. +; +; CHECK-LABEL: define void @repro( +; CHECK: vector.body: +; CHECK: load <vscale x 1 x i8> +; CHECK: store <vscale x 1 x i8> + +define void @repro(ptr %out, ptr %in, i32 %tc) { +entry: + %start = zext i32 %tc to i64 + br label %loop + +loop: + %iv = phi i64 [ %start, %entry ], [ %next, %loop ] + %in.ptr = getelementptr inbounds i8, ptr %in, i64 %iv + %value = load i8, ptr %in.ptr, align 1 + %sum = add i8 %value, 10 + %out.ptr = getelementptr inbounds i8, ptr %out, i64 %iv + store i8 %sum, ptr %out.ptr, align 1 + %next = add nsw i64 %iv, -1 + %next32 = and i64 %next, 4294967295 + %done = icmp eq i64 %next32, 0 + br i1 %done, label %exit, label %loop, !llvm.loop !0 + +exit: + ret void +} + +!0 = distinct !{!0, !1, !2, !3, !4} +!1 = !{!"llvm.loop.mustprogress"} +!2 = !{!"llvm.loop.vectorize.width", i32 1} +!3 = !{!"llvm.loop.interleave.count", i32 1} +!4 = !{!"llvm.loop.vectorize.scalable.enable"} From 6be4cf0ee8eafb52a03464ec7b4cce158a1f63ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <[email protected]> Date: Tue, 29 Sep 2026 09:53:56 +0000 Subject: [PATCH 3/3] Also document precedence order for vector width and IC --- .../Vectorize/LoopVectorizationLegality.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp index 26f7e024ced78..9d2347602f411 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -108,7 +108,16 @@ LoopVectorizeHints::LoopVectorizeHints(const Loop *L, // Populate values with existing loop metadata. getHintsFromMetadata(); - // force-vector-interleave overrides DisableInterleaving. + // The vector width is selected in increasing order of priority: + // 1. -force-vector-width + // 2. llvm.loop.vectorize.width metadata + + // The interleave count is selected in increasing order of priority: + // 1. InterleaveOnlyWhenForced initializes IC to 1 + // 2. llvm.loop.interleave.count metadata + // 3. -force-vector-interleave + // Note: If no IC is set, getInterleave() returns 1 when loop unrolling is + // disabled. if (VectorizerParams::isInterleaveForced()) Interleave.Value = VectorizerParams::VectorizationInterleave; _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
