Author: Yazhou Tang Date: 2026-10-22T20:10:36-07:00 New Revision: 0df807454b93f4037575e0b9f7ac1814609f7b8c
URL: https://github.com/llvm/llvm-project/commit/0df807454b93f4037575e0b9f7ac1814609f7b8c DIFF: https://github.com/llvm/llvm-project/commit/0df807454b93f4037575e0b9f7ac1814609f7b8c.diff LOG: [BPF] Add UHMUL and SHMUL instruction variants with -mcpu=v5 (#234076) This patch adds LLVM support for the unsigned and signed high-half multiply instructions (UHMUL and SHMUL) proposed for the BPF ISA [1]. The instructions use the existing `BPF_MUL` opcode with the offset field distinguishing the variants: - `off = 1`: UHMUL, returning the upper 64 bits of an unsigned 64×64-bit product. - `off = 2`: SHMUL, returning the upper 64 bits of a signed 64×64-bit product. Both register and immediate forms are supported. For example: ```asm r1 uh*= r2 r3 sh*= r4 r5 uh*= -2 r6 sh*= -3 ``` The backend selects these instructions for 64-bit `ISD::MULHU` and `ISD::MULHS` operations. ### CPU v5 As suggested by Alexei Starovoitov [2], this patch introduces `-mcpu=v5` instead of extending `-mcpu=v4`. Since UHMUL and SHMUL are new ISA extensions, enabling them under v4 would change the instruction set available to existing v4 targets and could generate instructions unsupported by those targets. CPU v5 inherits all v4 features and additionally enables UHMUL and SHMUL. The existing v4 code generation remains unchanged. Clang also defines `__BPF_FEATURE_HMUL` for v5, and the `-disable-hmul` option allows the backend to disable generation of these instructions. The patch includes assembly/disassembly tests, code generation tests, and Clang CPU feature tests. It also verifies that earlier CPU versions and v5 with `-disable-hmul` retain the existing lowering behavior. The corresponding Linux kernel changes are still under review. [1] Kernel patch series v3: https://lore.kernel.org/bpf/[email protected]/ [2] https://lore.kernel.org/bpf/[email protected]/ --- ### AI Tool Use Codex was used to assist with code development for this PR. I reviewed the resulting changes and take responsibility for the submitted code. --------- Co-authored-by: Tianci Cao <[email protected]> Co-authored-by: Shenghao Yuan <[email protected]> Added: llvm/test/CodeGen/BPF/assembler-disassembler-v5.s llvm/test/CodeGen/BPF/div-to-hmul.ll llvm/test/CodeGen/BPF/hmul.ll llvm/test/MC/BPF/bad-hmul.s Modified: clang/docs/ReleaseNotes.md clang/lib/Basic/Targets/BPF.cpp clang/lib/Basic/Targets/BPF.h clang/test/Misc/target-invalid-cpu-note/bpf.c clang/test/Preprocessor/bpf-predefined-macros.c llvm/docs/ReleaseNotes.md llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp llvm/lib/Target/BPF/BPF.td llvm/lib/Target/BPF/BPFISelLowering.cpp llvm/lib/Target/BPF/BPFInstrInfo.td llvm/lib/Target/BPF/BPFSubtarget.cpp llvm/lib/Target/BPF/BPFSubtarget.h llvm/test/CodeGen/BPF/sdiv_smod.ll Removed: ################################################################################ diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md index 1ae4ec0d6b6d5a..0299ea8e799227 100644 --- a/clang/docs/ReleaseNotes.md +++ b/clang/docs/ReleaseNotes.md @@ -1196,6 +1196,11 @@ documentation](CIR/index.md) for more information regarding ClangIR's status and - `__builtin_amdgcn_fcmp` - `__builtin_amdgcn_fcmpf` +#### BPF Support + +- Added `-mcpu=v5` and the `__BPF_FEATURE_HMUL` predefined macro for unsigned + and signed high-half multiply instructions. + #### DirectX Support - `clang-dxc` and HLSL support are now enabled by default, following the diff --git a/clang/lib/Basic/Targets/BPF.cpp b/clang/lib/Basic/Targets/BPF.cpp index 29e4c86ad88621..2c37d56186ce52 100644 --- a/clang/lib/Basic/Targets/BPF.cpp +++ b/clang/lib/Basic/Targets/BPF.cpp @@ -79,10 +79,14 @@ void BPFTargetInfo::getTargetDefines(const LangOptions &Opts, Builder.defineMacro("__BPF_FEATURE_LOAD_ACQ_STORE_REL"); Builder.defineMacro("__BPF_FEATURE_GOTOX"); } + + if (CpuVerNum >= 5) { + Builder.defineMacro("__BPF_FEATURE_HMUL"); + } } -static constexpr llvm::StringLiteral ValidCPUNames[] = {"generic", "v1", "v2", - "v3", "v4", "probe"}; +static constexpr llvm::StringLiteral ValidCPUNames[] = { + "generic", "v1", "v2", "v3", "v4", "v5", "probe"}; bool BPFTargetInfo::isValidCPUName(StringRef Name) const { return llvm::is_contained(ValidCPUNames, Name); diff --git a/clang/lib/Basic/Targets/BPF.h b/clang/lib/Basic/Targets/BPF.h index e51f37b851d481..73ab60339a311a 100644 --- a/clang/lib/Basic/Targets/BPF.h +++ b/clang/lib/Basic/Targets/BPF.h @@ -100,7 +100,7 @@ class LLVM_LIBRARY_VISIBILITY BPFTargetInfo : public TargetInfo { void fillValidCPUList(SmallVectorImpl<StringRef> &Values) const override; bool setCPU(StringRef Name) override { - if (Name == "v3" || Name == "v4") { + if (Name == "v3" || Name == "v4" || Name == "v5") { HasAlu32 = true; } diff --git a/clang/test/Misc/target-invalid-cpu-note/bpf.c b/clang/test/Misc/target-invalid-cpu-note/bpf.c index fe925f86cdd137..7f90e64ab16f16 100644 --- a/clang/test/Misc/target-invalid-cpu-note/bpf.c +++ b/clang/test/Misc/target-invalid-cpu-note/bpf.c @@ -10,6 +10,7 @@ // CHECK-SAME: {{^}}, v2 // CHECK-SAME: {{^}}, v3 // CHECK-SAME: {{^}}, v4 +// CHECK-SAME: {{^}}, v5 // CHECK-SAME: {{^}}, probe // CHECK-SAME: {{$}} diff --git a/clang/test/Preprocessor/bpf-predefined-macros.c b/clang/test/Preprocessor/bpf-predefined-macros.c index d3d59a979e06bf..f89e04ae6327af 100644 --- a/clang/test/Preprocessor/bpf-predefined-macros.c +++ b/clang/test/Preprocessor/bpf-predefined-macros.c @@ -4,6 +4,7 @@ // RUN: %clang -E -target bpfel -mcpu=v2 -x c -o - %s | FileCheck -check-prefix=CHECK -check-prefix=CPU_V2 %s // RUN: %clang -E -target bpfel -mcpu=v3 -x c -o - %s | FileCheck -check-prefix=CHECK -check-prefix=CPU_V3 %s // RUN: %clang -E -target bpfel -mcpu=v4 -x c -o - %s | FileCheck -check-prefix=CHECK -check-prefix=CPU_V4 %s +// RUN: %clang -E -target bpfel -mcpu=v5 -x c -o - %s | FileCheck -check-prefix=CHECK -check-prefix=CPU_V5 %s // RUN: %clang -E -target bpfel -mcpu=generic -x c -o - %s | FileCheck -check-prefix=CHECK -check-prefix=CPU_GENERIC %s // RUN: %clang -E -target bpfel -mcpu=probe -x c -o - %s | FileCheck -check-prefix=CHECK -check-prefix=CPU_PROBE %s @@ -34,6 +35,9 @@ int i; #if __BPF_CPU_VERSION__ == 4 int j; #endif +#if __BPF_CPU_VERSION__ == 5 +int aa; +#endif #ifdef __BPF_FEATURE_JMP_EXT int k; #endif @@ -82,6 +86,12 @@ int y; #ifdef __BPF_FEATURE_UNWIND int z; #endif +#ifdef __BPF_FEATURE_HMUL +int ab; +#endif +#if __BPF_CPU_VERSION__ < 5 && defined(__BPF_FEATURE_HMUL) +#error HMUL requires CPU v5 or later +#endif // CHECK: int b; // CHECK: int c; @@ -111,33 +121,53 @@ int z; // CPU_V4: int r; // CPU_V4: int s; +// CPU_V5: int aa; +// CPU_V5: int k; +// CPU_V5: int l; +// CPU_V5: int m; +// CPU_V5: int n; +// CPU_V5: int o; +// CPU_V5: int p; +// CPU_V5: int q; +// CPU_V5: int r; +// CPU_V5: int s; + // CPU_V1: int t; // CPU_V2: int t; // CPU_V3: int t; // CPU_V4: int t; +// CPU_V5: int t; // CPU_V1: int u; // CPU_V2: int u; // CPU_V3: int u; // CPU_V4: int u; +// CPU_V5: int u; // CPU_V4: int v; // CPU_V4: int w; +// CPU_V5: int v; +// CPU_V5: int w; // CPU_V1: int x; // CPU_V2: int x; // CPU_V3: int x; // CPU_V4: int x; +// CPU_V5: int x; // CPU_V1: int y; // CPU_V2: int y; // CPU_V3: int y; // CPU_V4: int y; +// CPU_V5: int y; // CPU_V1: int z; // CPU_V2: int z; // CPU_V3: int z; // CPU_V4: int z; +// CPU_V5: int z; + +// CPU_V5: int ab; // CPU_GENERIC: int g; diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md index a32825e3855a64..943fad8f36147f 100644 --- a/llvm/docs/ReleaseNotes.md +++ b/llvm/docs/ReleaseNotes.md @@ -311,6 +311,12 @@ Makes programs 10x faster by doing Special New Thing. ### Changes to the AVR Backend +### Changes to the BPF Backend + +* Added `-mcpu=v5`, which includes the v4 instruction set and enables unsigned + and signed high-half multiply instructions (`UHMUL` and `SHMUL`). Use + `-disable-hmul` to disable their generation. + ### Changes to the DirectX Backend * The DirectX backend has been promoted from experimental to an official, diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp index ec167a680cdc1b..6483d02aa6f87b 100644 --- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp +++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp @@ -268,6 +268,8 @@ struct BPFOperand : public MCParsedAsmOperand { .Case("ll", true) .Case("skb", true) .Case("s", true) + .Case("uh", true) + .Case("sh", true) .Case("atomic_fetch_add", true) .Case("atomic_fetch_and", true) .Case("atomic_fetch_or", true) diff --git a/llvm/lib/Target/BPF/BPF.td b/llvm/lib/Target/BPF/BPF.td index 1fc364dad9988f..f52c8eadf60fd2 100644 --- a/llvm/lib/Target/BPF/BPF.td +++ b/llvm/lib/Target/BPF/BPF.td @@ -39,6 +39,7 @@ def : Proc<"v1", []>; def : Proc<"v2", []>; def : Proc<"v3", [ALU32]>; def : Proc<"v4", [ALU32]>; +def : Proc<"v5", [ALU32]>; def : Proc<"probe", []>; def BPFInstPrinter : AsmWriter { diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp index 435f0e15a25c59..204905a3a1c526 100644 --- a/llvm/lib/Target/BPF/BPFISelLowering.cpp +++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp @@ -127,8 +127,10 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, setOperationAction(ISD::SDIV, VT, Custom); setOperationAction(ISD::SREM, VT, Custom); } - setOperationAction(ISD::MULHU, VT, Expand); - setOperationAction(ISD::MULHS, VT, Expand); + if (VT == MVT::i32 || !STI.hasHmul()) { + setOperationAction(ISD::MULHU, VT, Expand); + setOperationAction(ISD::MULHS, VT, Expand); + } setOperationAction(ISD::UMUL_LOHI, VT, Expand); setOperationAction(ISD::SMUL_LOHI, VT, Expand); setOperationAction(ISD::ROTR, VT, Expand); diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td index 8c3e09009a5e50..0e5d7dce3e4da3 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.td +++ b/llvm/lib/Target/BPF/BPFInstrInfo.td @@ -71,6 +71,7 @@ def BPFHasLdsx : Predicate<"Subtarget->hasLdsx()">; def BPFHasMovsx : Predicate<"Subtarget->hasMovsx()">; def BPFHasBswap : Predicate<"Subtarget->hasBswap()">; def BPFHasSdivSmod : Predicate<"Subtarget->hasSdivSmod()">; +def BPFHasHmul : Predicate<"Subtarget->hasHmul()">; def BPFNoMovsx : Predicate<"!Subtarget->hasMovsx()">; def BPFNoBswap : Predicate<"!Subtarget->hasBswap()">; def BPFHasStoreImm : Predicate<"Subtarget->hasStoreImm()">; @@ -360,7 +361,7 @@ class ALU_RR<BPFOpClass Class, BPFArithOp Opc, int off, let BPFClass = Class; } -multiclass ALU<BPFArithOp Opc, int off, string OpcodeStr, SDNode OpNode> { +multiclass ALU64<BPFArithOp Opc, int off, string OpcodeStr, SDNode OpNode> { def _rr : ALU_RR<BPF_ALU64, Opc, off, (outs GPR:$dst), (ins GPR:$src2, GPR:$src), @@ -371,6 +372,10 @@ multiclass ALU<BPFArithOp Opc, int off, string OpcodeStr, SDNode OpNode> { (ins GPR:$src2, i64imm:$imm), "$dst "#OpcodeStr#" $imm", [(set GPR:$dst, (OpNode GPR:$src2, i64immSExt32:$imm))]>; +} + +multiclass ALU<BPFArithOp Opc, int off, string OpcodeStr, SDNode OpNode> + : ALU64<Opc, off, OpcodeStr, OpNode> { def _rr_32 : ALU_RR<BPF_ALU, Opc, off, (outs GPR32:$dst), (ins GPR32:$src2, GPR32:$src), @@ -402,6 +407,14 @@ let isAsCheapAsAMove = 1 in { defm SDIV : ALU<BPF_DIV, 1, "s/=", sdiv>; defm SMOD : ALU<BPF_MOD, 1, "s%=", srem>; } + + // High-half multiply variants encoded via insn->off on BPF_MUL: + // off == 1 UHMUL: dst = ((u128)dst * (u128)src) >> 64 + // off == 2 SHMUL: dst = ((i128)dst * (i128)src) >> 64 + let Predicates = [BPFHasHmul] in { + defm UHMUL : ALU64<BPF_MUL, 1, "uh*=", mulhu>; + defm SHMUL : ALU64<BPF_MUL, 2, "sh*=", mulhs>; + } } class NEG_RR<BPFOpClass Class, BPFArithOp Opc, diff --git a/llvm/lib/Target/BPF/BPFSubtarget.cpp b/llvm/lib/Target/BPF/BPFSubtarget.cpp index 726f8f4b39827f..c9990589925573 100644 --- a/llvm/lib/Target/BPF/BPFSubtarget.cpp +++ b/llvm/lib/Target/BPF/BPFSubtarget.cpp @@ -45,6 +45,9 @@ static cl::opt<bool> Disable_load_acq_store_rel( cl::desc("Disable load-acquire and store-release insns")); static cl::opt<bool> Disable_gotox("disable-gotox", cl::Hidden, cl::init(false), cl::desc("Disable gotox insn")); +static cl::opt<bool> + Disable_hmul("disable-hmul", cl::Hidden, cl::init(false), + cl::desc("Disable high-half multiply (UHMUL/SHMUL) insns")); void BPFSubtarget::anchor() {} @@ -69,6 +72,7 @@ void BPFSubtarget::initializeEnvironment() { HasStoreImm = false; HasLoadAcqStoreRel = false; HasGotox = false; + HasHmul = false; AllowsMisalignedMemAccess = false; } @@ -89,7 +93,7 @@ void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) { HasAlu32 = true; return; } - if (CPU == "v4") { + if (CPU == "v4" || CPU == "v5") { HasJmpExt = true; HasJmp32 = true; HasAlu32 = true; @@ -101,6 +105,7 @@ void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) { HasStoreImm = !Disable_StoreImm; HasLoadAcqStoreRel = !Disable_load_acq_store_rel; HasGotox = !Disable_gotox; + HasHmul = CPU == "v5" && !Disable_hmul; return; } } diff --git a/llvm/lib/Target/BPF/BPFSubtarget.h b/llvm/lib/Target/BPF/BPFSubtarget.h index 24eff862224b04..f7808facf0822e 100644 --- a/llvm/lib/Target/BPF/BPFSubtarget.h +++ b/llvm/lib/Target/BPF/BPFSubtarget.h @@ -70,6 +70,9 @@ class BPFSubtarget : public BPFGenSubtargetInfo { bool HasLdsx, HasMovsx, HasBswap, HasSdivSmod, HasGotol, HasStoreImm, HasLoadAcqStoreRel, HasGotox; + // whether cpu v5 insns are enabled. + bool HasHmul; + std::unique_ptr<CallLowering> CallLoweringInfo; std::unique_ptr<InstructionSelector> InstSelector; std::unique_ptr<LegalizerInfo> Legalizer; @@ -101,6 +104,7 @@ class BPFSubtarget : public BPFGenSubtargetInfo { bool hasStoreImm() const { return HasStoreImm; } bool hasLoadAcqStoreRel() const { return HasLoadAcqStoreRel; } bool hasGotox() const { return HasGotox; } + bool hasHmul() const { return HasHmul; } bool isLittleEndian() const { return IsLittleEndian; } diff --git a/llvm/test/CodeGen/BPF/assembler-disassembler-v5.s b/llvm/test/CodeGen/BPF/assembler-disassembler-v5.s new file mode 100644 index 00000000000000..e51e0c7f3f6964 --- /dev/null +++ b/llvm/test/CodeGen/BPF/assembler-disassembler-v5.s @@ -0,0 +1,12 @@ +// RUN: llvm-mc -triple bpfel --mcpu=v5 --assemble --filetype=obj %s \ +// RUN: | llvm-objdump -d --mattr=+alu32 - \ +// RUN: | FileCheck %s + +// CHECK: 2f 21 01 00 00 00 00 00 r1 uh*= r2 +// CHECK: 2f 43 02 00 00 00 00 00 r3 sh*= r4 +// CHECK: 27 05 01 00 fe ff ff ff r5 uh*= -0x2 +// CHECK: 27 06 02 00 fd ff ff ff r6 sh*= -0x3 +r1 uh*= r2 +r3 sh*= r4 +r5 uh*= -2 +r6 sh*= -3 diff --git a/llvm/test/CodeGen/BPF/div-to-hmul.ll b/llvm/test/CodeGen/BPF/div-to-hmul.ll new file mode 100644 index 00000000000000..c1f4442093a027 --- /dev/null +++ b/llvm/test/CodeGen/BPF/div-to-hmul.ll @@ -0,0 +1,68 @@ +; RUN: llc -mtriple=bpfel -mcpu=v5 -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=V5,COMMON --implicit-check-not="/=" +; RUN: llc -mtriple=bpfel -mcpu=v4 -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=NO-HMUL,COMMON --implicit-check-not="uh*=" --implicit-check-not="sh*=" +; RUN: llc -mtriple=bpfel -mcpu=v5 -disable-hmul -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=NO-HMUL,COMMON --implicit-check-not="uh*=" --implicit-check-not="sh*=" + +; Constant division can introduce high-half multiplication even when the IR +; contains no multiply. Only v5 with HMUL enabled may use the new instructions. + +define i64 @udiv_by_10(i64 %a) { +; V5-LABEL: udiv_by_10: +; V5: r[[REG1:[0-9]+]] = -3689348814741910323 ll +; V5: r0 uh*= r[[REG1]] +; V5-NEXT: r0 >>= 3 +; V5-NEXT: exit +; NO-HMUL-LABEL: udiv_by_10: +; NO-HMUL: r0 = r1 +; NO-HMUL-NEXT: r0 /= 10 +; NO-HMUL-NEXT: exit + %result = udiv i64 %a, 10 + ret i64 %result +} + +; Signed division needs a correction so negative quotients round toward zero. +define i64 @sdiv_by_10(i64 %a) { +; V5-LABEL: sdiv_by_10: +; V5: r[[REG1:[0-9]+]] = 7378697629483820647 ll +; V5: r0 sh*= r[[REG1]] +; V5-DAG: r[[REG2:[0-9]+]] = r0 +; V5-DAG: r[[REG2]] >>= 63 +; V5-DAG: r0 s>>= 2 +; V5-NEXT: r0 += r[[REG2]] +; V5-NEXT: exit +; NO-HMUL-LABEL: sdiv_by_10: +; NO-HMUL: r0 = r1 +; NO-HMUL-NEXT: r0 s/= 10 +; NO-HMUL-NEXT: exit + %result = sdiv i64 %a, 10 + ret i64 %result +} + +define i64 @sdiv_by_neg10(i64 %a) { +; V5-LABEL: sdiv_by_neg10: +; V5: r[[REG1:[0-9]+]] = -7378697629483820647 ll +; V5: r0 sh*= r[[REG1]] +; V5-DAG: r[[REG2:[0-9]+]] = r0 +; V5-DAG: r[[REG2]] >>= 63 +; V5-DAG: r0 s>>= 2 +; V5-NEXT: r0 += r[[REG2]] +; V5-NEXT: exit +; NO-HMUL-LABEL: sdiv_by_neg10: +; NO-HMUL: r0 = r1 +; NO-HMUL-NEXT: r0 s/= -10 +; NO-HMUL-NEXT: exit + %result = sdiv i64 %a, -10 + ret i64 %result +} + +; Division by a power of two remains a shift in all configurations. +define i64 @udiv_by_8(i64 %a) { +; COMMON-LABEL: udiv_by_8: +; COMMON: r0 = r1 +; COMMON-NEXT: r0 >>= 3 +; COMMON-NEXT: exit + %result = udiv i64 %a, 8 + ret i64 %result +} diff --git a/llvm/test/CodeGen/BPF/hmul.ll b/llvm/test/CodeGen/BPF/hmul.ll new file mode 100644 index 00000000000000..cad9d403a7b47b --- /dev/null +++ b/llvm/test/CodeGen/BPF/hmul.ll @@ -0,0 +1,163 @@ +; RUN: llc -mtriple=bpfel -mcpu=v5 -verify-machineinstrs -show-mc-encoding < %s | FileCheck %s --check-prefixes=V5,I32 +; RUN: llc -mtriple=bpfel -mcpu=v3 -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=NO-HMUL,I32 --implicit-check-not="uh*=" --implicit-check-not="sh*=" +; RUN: llc -mtriple=bpfel -mcpu=v4 -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=NO-HMUL,I32 --implicit-check-not="uh*=" --implicit-check-not="sh*=" +; RUN: llc -mtriple=bpfel -mcpu=v5 -disable-hmul -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=NO-HMUL,I32 --implicit-check-not="uh*=" --implicit-check-not="sh*=" + +declare i64 @llvm.umulh.i64(i64, i64) +declare i64 @llvm.smulh.i64(i64, i64) +declare i32 @llvm.umulh.i32(i32, i32) +declare i32 @llvm.smulh.i32(i32, i32) + +; --- 64-bit high-half multiply instruction selection --- +; Select UHMUL/SHMUL on v5, covering register operands and immediate boundaries. + +define i64 @uhmul_reg(i64 %a, i64 %b) { +; V5-LABEL: uhmul_reg: +; V5: r0 = r1 +; V5-NEXT: r0 uh*= r2 # encoding: [0x2f,0x20,0x01,0x00,0x00,0x00,0x00,0x00] +; V5-NEXT: exit +; NO-HMUL-LABEL: uhmul_reg: +; NO-HMUL: exit + %result = call i64 @llvm.umulh.i64(i64 %a, i64 %b) + ret i64 %result +} + +define i64 @uhmul_imm_neg2(i64 %a) { +; V5-LABEL: uhmul_imm_neg2: +; V5: r0 = r1 +; V5-NEXT: r0 uh*= -2 # encoding: [0x27,0x00,0x01,0x00,0xfe,0xff,0xff,0xff] +; V5-NEXT: exit +; NO-HMUL-LABEL: uhmul_imm_neg2: +; NO-HMUL: exit + %result = call i64 @llvm.umulh.i64(i64 %a, i64 -2) + ret i64 %result +} + +; The immediate operand is sign-extended from 32 to 64 bits. +define i64 @uhmul_imm_s32_max(i64 %a) { +; V5-LABEL: uhmul_imm_s32_max: +; V5: r0 uh*= 2147483647 +; V5-NEXT: exit +; NO-HMUL-LABEL: uhmul_imm_s32_max: +; NO-HMUL: exit + %result = call i64 @llvm.umulh.i64(i64 %a, i64 2147483647) + ret i64 %result +} + +define i64 @uhmul_imm_s32_min(i64 %a) { +; V5-LABEL: uhmul_imm_s32_min: +; V5: r0 uh*= -2147483648 +; V5-NEXT: exit +; NO-HMUL-LABEL: uhmul_imm_s32_min: +; NO-HMUL: exit + %result = call i64 @llvm.umulh.i64(i64 %a, i64 -2147483648) + ret i64 %result +} + +; 0x00000000ffffffff cannot be represented by a sign-extended imm32. +; Materialize the full value and use the register form instead of imm32 -1. +define i64 @uhmul_imm_u32_max(i64 %a) { +; V5-LABEL: uhmul_imm_u32_max: +; V5: r{{[0-9]+}} = 4294967295 ll +; V5: r0 uh*= r{{[0-9]+}} +; V5-NEXT: exit +; NO-HMUL-LABEL: uhmul_imm_u32_max: +; NO-HMUL: exit + %result = call i64 @llvm.umulh.i64(i64 %a, i64 4294967295) + ret i64 %result +} + +define i64 @shmul_reg(i64 %a, i64 %b) { +; V5-LABEL: shmul_reg: +; V5: r0 = r1 +; V5-NEXT: r0 sh*= r2 # encoding: [0x2f,0x20,0x02,0x00,0x00,0x00,0x00,0x00] +; V5-NEXT: exit +; NO-HMUL-LABEL: shmul_reg: +; NO-HMUL: exit + %result = call i64 @llvm.smulh.i64(i64 %a, i64 %b) + ret i64 %result +} + +define i64 @shmul_imm_neg2(i64 %a) { +; V5-LABEL: shmul_imm_neg2: +; V5: r0 = r1 +; V5-NEXT: r0 sh*= -2 # encoding: [0x27,0x00,0x02,0x00,0xfe,0xff,0xff,0xff] +; V5-NEXT: exit +; NO-HMUL-LABEL: shmul_imm_neg2: +; NO-HMUL: exit + %result = call i64 @llvm.smulh.i64(i64 %a, i64 -2) + ret i64 %result +} + +; The immediate operand is sign-extended from 32 to 64 bits. +define i64 @shmul_imm_s32_max(i64 %a) { +; V5-LABEL: shmul_imm_s32_max: +; V5: r0 sh*= 2147483647 +; V5-NEXT: exit +; NO-HMUL-LABEL: shmul_imm_s32_max: +; NO-HMUL: exit + %result = call i64 @llvm.smulh.i64(i64 %a, i64 2147483647) + ret i64 %result +} + +define i64 @shmul_imm_s32_min(i64 %a) { +; V5-LABEL: shmul_imm_s32_min: +; V5: r0 sh*= -2147483648 +; V5-NEXT: exit +; NO-HMUL-LABEL: shmul_imm_s32_min: +; NO-HMUL: exit + %result = call i64 @llvm.smulh.i64(i64 %a, i64 -2147483648) + ret i64 %result +} + +; 0x00000000ffffffff cannot be represented by a sign-extended imm32. +; Materialize the full value and use the register form instead of imm32 -1. +define i64 @shmul_imm_u32_max(i64 %a) { +; V5-LABEL: shmul_imm_u32_max: +; V5: r{{[0-9]+}} = 4294967295 ll +; V5: r0 sh*= r{{[0-9]+}} +; V5-NEXT: exit +; NO-HMUL-LABEL: shmul_imm_u32_max: +; NO-HMUL: exit + %result = call i64 @llvm.smulh.i64(i64 %a, i64 4294967295) + ret i64 %result +} + +; --- 32-bit high-half multiply expansion --- +; There are no ALU32 high-half multiply instructions, even on v5. +; Expand i32 MULHU/MULHS to a full i64 product and extract bits 32..63. + +define i32 @uhmul_i32(i32 %a, i32 %b) { +; I32-LABEL: uhmul_i32: +; I32: r0 *= r2 +; I32-NEXT: r0 >>= 32 +; I32: exit + %result = call i32 @llvm.umulh.i32(i32 %a, i32 %b) + ret i32 %result +} + +define i32 @shmul_i32(i32 %a, i32 %b) { +; I32-LABEL: shmul_i32: +; I32: r0 *= r2 +; I32-NEXT: r0 >>= 32 +; I32: exit + %result = call i32 @llvm.smulh.i32(i32 %a, i32 %b) + ret i32 %result +} + +; --- Low-half multiply regression --- +; Ordinary multiplication must still select MUL with offset 0, including on v5. + +define i64 @mul_low(i64 %a, i64 %b) { +; V5-LABEL: mul_low: +; V5: r0 = r1 +; V5-NEXT: r0 *= r2 # encoding: [0x2f,0x20,0x00,0x00,0x00,0x00,0x00,0x00] +; V5-NEXT: exit +; NO-HMUL-LABEL: mul_low: +; NO-HMUL: r0 *= r2 + %product = mul i64 %a, %b + ret i64 %product +} diff --git a/llvm/test/CodeGen/BPF/sdiv_smod.ll b/llvm/test/CodeGen/BPF/sdiv_smod.ll index 5fb23d03f172db..6d921d683c66c3 100644 --- a/llvm/test/CodeGen/BPF/sdiv_smod.ll +++ b/llvm/test/CodeGen/BPF/sdiv_smod.ll @@ -1,4 +1,5 @@ ; RUN: llc -mtriple=bpfel -mcpu=v4 -verify-machineinstrs -show-mc-encoding < %s | FileCheck %s +; RUN: llc -mtriple=bpfel -mcpu=v5 -verify-machineinstrs -show-mc-encoding < %s | FileCheck %s ; Source: ; int foo(int a, int b, int c) { ; return a/b + a%c; diff --git a/llvm/test/MC/BPF/bad-hmul.s b/llvm/test/MC/BPF/bad-hmul.s new file mode 100644 index 00000000000000..ae163662eefea7 --- /dev/null +++ b/llvm/test/MC/BPF/bad-hmul.s @@ -0,0 +1,5 @@ +# RUN: not llvm-mc -mcpu=v5 -triple bpfel < %s 2>&1 \ +# RUN: | grep 'error: invalid operand for instruction' \ +# RUN: | count 2 + w1 uh*= w2 + w3 sh*= -2 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
