From: Pritam Gharat <[email protected]>
Some logical operations (AND/IOR/XOR) use an immediate that is not a legal
AArch64 bitmask immediate and therefore previously required the constant to
be materialised in a register (MOV/MOVK) followed by a register-form logical
operation.
For values that can be expressed as the combination of exactly two legal
bitmask immediates, emit two immediate-form logical operations instead,
avoiding the constant materialisation. aarch64_split_logical_imm ()
computes the two halves and verifies that both are legal bitmask immediates;
a new define_insn_and_split pattern performs the transformation after reload,
and aarch64_emit_split_logical () emits the two logical instructions.
For AND the two halves overlap and their intersection is the original value;
for IOR/XOR the halves are disjoint and their union/xor is the original
value. In both cases "(src OP p1) OP p2" reproduces "src OP val".
gcc/ChangeLog:
* config/aarch64/aarch64-protos.h (aarch64_split_logical_imm): Declare.
(aarch64_emit_split_logical): Declare.
* config/aarch64/aarch64.cc (aarch64_split_logical_imm): New function.
(aarch64_emit_split_logical): New function.
* config/aarch64/aarch64.md (*aarch64_<optab><mode>3_split_bitmask):
New define_insn_and_split.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/split_logical_imm_si.c: New test.
* gcc.target/aarch64/split_logical_imm_di.c: New test.
Signed-off-by: Pritam Gharat <[email protected]>
---
gcc/config/aarch64/aarch64-protos.h | 12 ++
gcc/config/aarch64/aarch64.cc | 114 ++++++++++++
gcc/config/aarch64/aarch64.md | 20 ++
.../gcc.target/aarch64/split_logical_imm_di.c | 175 ++++++++++++++++++
.../gcc.target/aarch64/split_logical_imm_si.c | 168 +++++++++++++++++
5 files changed, 489 insertions(+)
create mode 100644 gcc/testsuite/gcc.target/aarch64/split_logical_imm_di.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/split_logical_imm_si.c
diff --git a/gcc/config/aarch64/aarch64-protos.h
b/gcc/config/aarch64/aarch64-protos.h
index bccdbffc194..62260d396d0 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -1317,4 +1317,16 @@ extern unsigned aarch64_stack_alignment (const_tree exp,
unsigned align);
extern rtx aarch64_gen_compare_zero_and_branch (rtx_code code, rtx x,
rtx_code_label *label);
+bool
+aarch64_split_logical_imm (unsigned HOST_WIDE_INT val, machine_mode mode,
+ enum rtx_code code,
+ unsigned HOST_WIDE_INT *part1,
+ unsigned HOST_WIDE_INT *part2);
+
+void
+aarch64_emit_split_logical (rtx dst, rtx src, machine_mode mode,
+ enum rtx_code op,
+ unsigned HOST_WIDE_INT p1,
+ unsigned HOST_WIDE_INT p2);
+
#endif /* GCC_AARCH64_PROTOS_H */
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 5b1c78462a3..c06bcfd9007 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -251,6 +251,120 @@ inline simd_immediate_info
u.pattern = pattern_in;
}
+/* Try to express VAL (interpreted in MODE) as the combination of two
+ AArch64 "logical immediate" (bitmask) values that can each be encoded
+ directly in an ORR/EOR/AND instruction, so that
+
+ ORR: x | VAL == (x | PART1) | PART2
+ EOR: x ^ VAL == (x ^ PART1) ^ PART2 (PART1 & PART2 == 0)
+ AND: x & VAL == (x & ~M1) & ~M2 (VAL == ~(M1 | M2))
+
+ For ORR/EOR we split VAL itself into two disjoint bitmask pieces.
+ For AND we split the *inverse* (~VAL), because AND clears bits; each
+ emitted immediate is then the bitwise-NOT of a bitmask piece and must
+ itself be a valid bitmask.
+
+ On success, store the two immediates to emit in *PART1 and *PART2 and
+ return true. PART1/PART2 may be NULL when the caller only wants a
+ feasibility test (for example, from an insn condition). */
+
+bool
+aarch64_split_logical_imm (unsigned HOST_WIDE_INT val, machine_mode mode,
+ enum rtx_code code,
+ unsigned HOST_WIDE_INT *part1,
+ unsigned HOST_WIDE_INT *part2)
+{
+ gcc_assert (mode == SImode || mode == DImode);
+
+ /* Mask VAL to the mode width so all bit tricks stay within the
+ relevant bits (important for SImode on a 64-bit HOST_WIDE_INT). */
+ unsigned HOST_WIDE_INT mask
+ = (mode == SImode) ? 0xffffffffULL : ~(unsigned HOST_WIDE_INT) 0;
+ val &= mask;
+
+ /* Only worth it when VAL isn't already a single bitmask and isn't a
+ cheap MOV. */
+ if (aarch64_bitmask_imm (val, mode)
+ || aarch64_move_imm (val, mode))
+ return false;
+
+ unsigned HOST_WIDE_INT part_a = 0, part_b = 0;
+
+ if (code == AND)
+ {
+ /* AND clears bits, so decompose the *zero-bit* runs of VAL, i.e.
+ the set-bit runs of the inverse, then invert each half back.
+ The two resulting masks OVERLAP; their intersection is VAL. */
+ unsigned HOST_WIDE_INT inv_val = (~val) & mask;
+
+ /* Isolate the lowest contiguous run of zero-bits. */
+ unsigned HOST_WIDE_INT low_bit = inv_val & -inv_val;
+ unsigned HOST_WIDE_INT low_run = ((inv_val + low_bit) ^ inv_val) & mask;
+ unsigned HOST_WIDE_INT inv_part_a = inv_val & low_run;
+
+ /* All remaining zero-bits. */
+ unsigned HOST_WIDE_INT inv_part_b = inv_val ^ inv_part_a;
+
+ part_a = (~inv_part_a) & mask;
+ part_b = (~inv_part_b) & mask;
+ }
+ else if (code == IOR || code == XOR)
+ {
+ /* OR/XOR set bits, so decompose the set-bit runs of VAL directly.
+ The two halves are DISJOINT, so both '|' and '^' reproduce VAL. */
+ unsigned HOST_WIDE_INT low_bit = val & -val;
+ unsigned HOST_WIDE_INT low_run = ((val + low_bit) ^ val) & mask;
+ part_a = val & low_run;
+
+ /* All remaining set-bits. */
+ part_b = val ^ part_a;
+ }
+ else
+ /* Unsupported bitwise operation. */
+ return false;
+
+ /* Hardware check: BOTH halves must be encodable as logical immediates.
+ part_b may still hold several bit-runs (VAL had >2 runs); in that
+ case it fails this test and we correctly bail out. */
+ if (aarch64_bitmask_imm (part_a, mode)
+ && aarch64_bitmask_imm (part_b, mode))
+ {
+ if (part1 != nullptr && part2 != nullptr)
+ {
+ *part1=part_a;
+ *part2=part_b;
+ }
+
+ return true;
+ }
+
+ return false;
+}
+
+void
+aarch64_emit_split_logical (rtx dst, rtx src, machine_mode mode,
+ enum rtx_code op,
+ unsigned HOST_WIDE_INT p1,
+ unsigned HOST_WIDE_INT p2)
+{
+ /* This helper is only valid for the bitwise logical operations that
+ aarch64_split_logical_imm () handles. For AND the two halves overlap
+ and satisfy (p1 & p2) == val; for IOR/XOR they are disjoint and satisfy
+ (p1 | p2) == val / (p1 ^ p2) == val. In every case
+ (src OP p1) OP p2 reproduces src OP val. */
+ gcc_assert (op == AND || op == IOR || op == XOR);
+ gcc_assert (mode == SImode || mode == DImode);
+
+ rtx imm1 = gen_int_mode (p1, mode);
+ rtx imm2 = gen_int_mode (p2, mode);
+
+ /* 1st instruction: dst = src OP p1. */
+ emit_insn (gen_rtx_SET (dst, gen_rtx_fmt_ee (op, mode, src, imm1)));
+
+ /* 2nd instruction: dst = dst OP p2. */
+ emit_insn (gen_rtx_SET (dst, gen_rtx_fmt_ee (op, mode, dst, imm2)));
+}
+
namespace {
/* Describes types that map to Pure Scalable Types (PSTs) in the AAPCS64. */
diff --git a/gcc/config/aarch64/aarch64.md b/gcc/config/aarch64/aarch64.md
index 8d0e50db272..0897af24551 100644
--- a/gcc/config/aarch64/aarch64.md
+++ b/gcc/config/aarch64/aarch64.md
@@ -3030,6 +3030,26 @@ (define_insn "add<mode>3_compare0"
}
)
+(define_insn_and_split "*aarch64_<optab><mode>3_split_bitmask"
+ [(set (match_operand:GPI 0 "register_operand" "=r")
+ (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "r")
+ (match_operand:GPI 2 "const_int_operand" "n")))]
+ "!aarch64_logical_operand (operands[2], <MODE>mode)
+ && aarch64_split_logical_imm (UINTVAL (operands[2]), <MODE>mode,
+ <CODE>, nullptr, nullptr)"
+ "#"
+ "&& reload_completed"
+ [(const_int 0)]
+ {
+ unsigned HOST_WIDE_INT p1=0, p2=0;
+ aarch64_split_logical_imm (UINTVAL (operands[2]), <MODE>mode,
+ <CODE>, &p1, &p2);
+ aarch64_emit_split_logical (operands[0], operands[1], <MODE>mode,
+ <CODE>, p1, p2);
+ DONE;
+ }
+)
+
;; zero_extend version of above
(define_insn "*addsi3_compare0_uxtw"
[(set (reg:CC_NZ CC_REGNUM)
diff --git a/gcc/testsuite/gcc.target/aarch64/split_logical_imm_di.c
b/gcc/testsuite/gcc.target/aarch64/split_logical_imm_di.c
new file mode 100644
index 00000000000..0c0397d9b69
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/split_logical_imm_di.c
@@ -0,0 +1,175 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-schedule-insns -fno-schedule-insns2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/* =====================================================================
+ aarch64_split_logical_imm () - DImode (64-bit, "x" registers)
+ 4 scenarios x {AND, IOR (|), XOR (^)}.
+
+ The expected code for each function is verified with check-function-bodies
+ (the "** <insn>" block immediately preceding each function): this pins the
+ exact, ordered instruction sequence - order, contents, and the absence of
+ stray instructions - which scan-assembler-times cannot do.
+
+ Representative immediates (verified identical behaviour for AND/IOR/XOR):
+ S1 valid : 0x000000000000ffff (= 65535)
+ S2 -> 2 valid : 0xffff0000000000f0
+ S3 -> >2 valid : 0x00000000000f0f0f
+ S4 unsplittable : 0x123456789abcdef0
+ ===================================================================== */
+
+
+/* =====================================================================
+ SCENARIO 1 (DImode): valid bitmask immediate -> NO split required.
+
+ 0x000000000000ffff is a single contiguous run of 16 ones - a legal
+ AArch64 64-bit logical bitmask immediate. aarch64_split_logical_imm ()
+ returns false early (aarch64_bitmask_imm () is true), so a SINGLE
+ immediate-form logical instruction is emitted and no MOVK is needed.
+ ===================================================================== */
+
+/*
+** s1_di_and:
+** and x0, x0, 65535
+** ret
+*/
+unsigned long s1_di_and (unsigned long x) { return x & 0x000000000000ffffUL; }
+
+/*
+** s1_di_ior:
+** orr x0, x0, 65535
+** ret
+*/
+unsigned long s1_di_ior (unsigned long x) { return x | 0x000000000000ffffUL; }
+
+/*
+** s1_di_xor:
+** eor x0, x0, 65535
+** ret
+*/
+unsigned long s1_di_xor (unsigned long x) { return x ^ 0x000000000000ffffUL; }
+
+
+/* =====================================================================
+ SCENARIO 2 (DImode): invalid bitmask, splits into EXACTLY TWO valid
+ bitmask immediates -> TWO immediate-form logical ops, no MOVK.
+
+ 0xffff0000000000f0 is not a legal bitmask immediate but is the
+ combination of two legal bitmask immediates. The split differs per
+ opcode:
+ IOR/XOR (disjoint): part_a = 0x00000000000000f0 (240),
+ part_b = 0xffff000000000000 (-281474976710656)
+ AND (overlap) : part_a = 0xfffffffffffffff0 (-16),
+ part_b = 0xffff0000000000ff (-281474976710401)
+ In every case (x OP part_a) OP part_b reproduces x OP 0xffff0000000000f0.
+ ===================================================================== */
+
+/*
+** s2_di_and:
+** and x0, x0, -16
+** and x0, x0, -281474976710401
+** ret
+*/
+unsigned long s2_di_and (unsigned long x) { return x & 0xffff0000000000f0UL; }
+
+/*
+** s2_di_ior:
+** orr x0, x0, 240
+** orr x0, x0, -281474976710656
+** ret
+*/
+unsigned long s2_di_ior (unsigned long x) { return x | 0xffff0000000000f0UL; }
+
+/*
+** s2_di_xor:
+** eor x0, x0, 240
+** eor x0, x0, -281474976710656
+** ret
+*/
+unsigned long s2_di_xor (unsigned long x) { return x ^ 0xffff0000000000f0UL; }
+
+
+/* =====================================================================
+ SCENARIO 3 (DImode): invalid bitmask, needs MORE THAN TWO valid
+ bitmask immediates.
+
+ 0x00000000000f0f0f has THREE disjoint bit-runs, each individually a
+ legal bitmask, but it CANNOT be grouped into just two legal bitmask
+ immediates. aarch64_split_logical_imm () performs only a 2-way split;
+ here part_b still contains two runs and fails aarch64_bitmask_imm (), so
+ the function returns FALSE. The constant is therefore materialised
+ (MOV/MOVK) and a REGISTER-form op is used. (The scratch register is
+ RA-chosen; matched with a backreference.)
+ ===================================================================== */
+
+/*
+** s3_di_and:
+** mov w([0-9]+), 3855
+** movk x\1, 0xf, lsl 16
+** and x0, x0, x\1
+** ret
+*/
+unsigned long s3_di_and (unsigned long x) { return x & 0x00000000000f0f0fUL; }
+
+/*
+** s3_di_ior:
+** mov w([0-9]+), 3855
+** movk x\1, 0xf, lsl 16
+** orr x0, x0, x\1
+** ret
+*/
+unsigned long s3_di_ior (unsigned long x) { return x | 0x00000000000f0f0fUL; }
+
+/*
+** s3_di_xor:
+** mov w([0-9]+), 3855
+** movk x\1, 0xf, lsl 16
+** eor x0, x0, x\1
+** ret
+*/
+unsigned long s3_di_xor (unsigned long x) { return x ^ 0x00000000000f0f0fUL; }
+
+
+/* =====================================================================
+ SCENARIO 4 (DImode): invalid bitmask, CANNOT be split into valid
+ bitmask immediates.
+
+ 0x123456789abcdef0 has many scattered runs and cannot be expressed as a
+ combination of two legal bitmask immediates at all.
+ aarch64_split_logical_imm () returns FALSE, so the full 64-bit constant
+ is materialised (MOV + 3x MOVK) and a single REGISTER-form logical op is
+ used. (Same KIND of code as Scenario 3; distinguished by the value.)
+ ===================================================================== */
+
+/*
+** s4_di_and:
+** mov w([0-9]+), 57072
+** movk x\1, 0x9abc, lsl 16
+** movk x\1, 0x5678, lsl 32
+** movk x\1, 0x1234, lsl 48
+** and x0, x0, x\1
+** ret
+*/
+unsigned long s4_di_and (unsigned long x) { return x & 0x123456789abcdef0UL; }
+
+/*
+** s4_di_ior:
+** mov w([0-9]+), 57072
+** movk x\1, 0x9abc, lsl 16
+** movk x\1, 0x5678, lsl 32
+** movk x\1, 0x1234, lsl 48
+** orr x0, x0, x\1
+** ret
+*/
+unsigned long s4_di_ior (unsigned long x) { return x | 0x123456789abcdef0UL; }
+
+/*
+** s4_di_xor:
+** mov w([0-9]+), 57072
+** movk x\1, 0x9abc, lsl 16
+** movk x\1, 0x5678, lsl 32
+** movk x\1, 0x1234, lsl 48
+** eor x0, x0, x\1
+** ret
+*/
+unsigned long s4_di_xor (unsigned long x) { return x ^ 0x123456789abcdef0UL; }
diff --git a/gcc/testsuite/gcc.target/aarch64/split_logical_imm_si.c
b/gcc/testsuite/gcc.target/aarch64/split_logical_imm_si.c
new file mode 100644
index 00000000000..ad41b900f34
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/split_logical_imm_si.c
@@ -0,0 +1,168 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-schedule-insns -fno-schedule-insns2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/* =====================================================================
+ aarch64_split_logical_imm () - SImode (32-bit, "w" registers)
+ 4 scenarios x {AND, IOR (|), XOR (^)}.
+
+ The expected code for each function is verified with check-function-bodies
+ (the "** <insn>" block immediately preceding each function): this pins the
+ exact, ordered instruction sequence - order, contents, and the absence of
+ stray instructions - which scan-assembler-times cannot do.
+
+ Representative immediates (verified identical behaviour for AND/IOR/XOR):
+ S1 valid : 0x0f0f0f0f (= 252645135)
+ S2 -> 2 valid : 0x00f0000f
+ S3 -> >2 valid : 0x00f0f00f
+ S4 unsplittable : 0x12345678
+ ===================================================================== */
+
+
+/* =====================================================================
+ SCENARIO 1 (SImode): valid bitmask immediate -> NO split required.
+
+ 0x0f0f0f0f is a legal AArch64 logical bitmask immediate (element 0x0f
+ replicated every 8 bits). aarch64_split_logical_imm () returns false
+ early because aarch64_bitmask_imm () is true, so a SINGLE immediate-form
+ logical instruction is emitted and no constant is materialised.
+ ===================================================================== */
+
+/*
+** s1_si_and:
+** and w0, w0, 252645135
+** ret
+*/
+unsigned int s1_si_and (unsigned int x) { return x & 0x0f0f0f0fU; }
+
+/*
+** s1_si_ior:
+** orr w0, w0, 252645135
+** ret
+*/
+unsigned int s1_si_ior (unsigned int x) { return x | 0x0f0f0f0fU; }
+
+/*
+** s1_si_xor:
+** eor w0, w0, 252645135
+** ret
+*/
+unsigned int s1_si_xor (unsigned int x) { return x ^ 0x0f0f0f0fU; }
+
+
+/* =====================================================================
+ SCENARIO 2 (SImode): invalid bitmask, splits into EXACTLY TWO valid
+ bitmask immediates -> TWO immediate-form logical ops, no MOVK.
+
+ 0x00f0000f is not a legal bitmask immediate but is the combination of
+ two legal bitmask immediates. The split differs per opcode:
+ IOR/XOR (disjoint): part_a = 0x0000000f (15),
+ part_b = 0x00f00000 (15728640)
+ AND (overlap) : part_a = 0xfff0000f (-1048561),
+ part_b = 0x00ffffff (16777215)
+ In every case (x OP part_a) OP part_b reproduces x OP 0x00f0000f.
+ ===================================================================== */
+
+/*
+** s2_si_and:
+** and w0, w0, 16777215
+** and w0, w0, -1048561
+** ret
+*/
+unsigned int s2_si_and (unsigned int x) { return x & 0x00f0000fU; }
+
+/*
+** s2_si_ior:
+** orr w0, w0, 15
+** orr w0, w0, 15728640
+** ret
+*/
+unsigned int s2_si_ior (unsigned int x) { return x | 0x00f0000fU; }
+
+/*
+** s2_si_xor:
+** eor w0, w0, 15
+** eor w0, w0, 15728640
+** ret
+*/
+unsigned int s2_si_xor (unsigned int x) { return x ^ 0x00f0000fU; }
+
+
+/* =====================================================================
+ SCENARIO 3 (SImode): invalid bitmask, needs MORE THAN TWO valid
+ bitmask immediates.
+
+ 0x00f0f00f has THREE disjoint bit-runs, each individually a legal
+ bitmask, but it CANNOT be grouped into just two legal bitmask
+ immediates. aarch64_split_logical_imm () performs only a 2-way split;
+ here part_b (0x00f0f000) still contains two runs and fails
+ aarch64_bitmask_imm (), so the function returns FALSE. The constant is
+ therefore materialised (MOV/MOVK) and a REGISTER-form op is used.
+ (The scratch register is RA-chosen; matched with a backreference.)
+ ===================================================================== */
+
+/*
+** s3_si_and:
+** mov (w[0-9]+), 61455
+** movk \1, 0xf0, lsl 16
+** and w0, w0, \1
+** ret
+*/
+unsigned int s3_si_and (unsigned int x) { return x & 0x00f0f00fU; }
+
+/*
+** s3_si_ior:
+** mov (w[0-9]+), 61455
+** movk \1, 0xf0, lsl 16
+** orr w0, w0, \1
+** ret
+*/
+unsigned int s3_si_ior (unsigned int x) { return x | 0x00f0f00fU; }
+
+/*
+** s3_si_xor:
+** mov (w[0-9]+), 61455
+** movk \1, 0xf0, lsl 16
+** eor w0, w0, \1
+** ret
+*/
+unsigned int s3_si_xor (unsigned int x) { return x ^ 0x00f0f00fU; }
+
+
+/* =====================================================================
+ SCENARIO 4 (SImode): invalid bitmask, CANNOT be split into valid
+ bitmask immediates.
+
+ 0x12345678 has many scattered runs and cannot be expressed as a
+ combination of two legal bitmask immediates at all.
+ aarch64_split_logical_imm () returns FALSE, so the constant is fully
+ materialised (MOV + MOVK) and a single REGISTER-form logical op is used.
+ (Same KIND of code as Scenario 3; distinguished by the input value.)
+ ===================================================================== */
+
+/*
+** s4_si_and:
+** mov (w[0-9]+), 22136
+** movk \1, 0x1234, lsl 16
+** and w0, w0, \1
+** ret
+*/
+unsigned int s4_si_and (unsigned int x) { return x & 0x12345678U; }
+
+/*
+** s4_si_ior:
+** mov (w[0-9]+), 22136
+** movk \1, 0x1234, lsl 16
+** orr w0, w0, \1
+** ret
+*/
+unsigned int s4_si_ior (unsigned int x) { return x | 0x12345678U; }
+
+/*
+** s4_si_xor:
+** mov (w[0-9]+), 22136
+** movk \1, 0x1234, lsl 16
+** eor w0, w0, \1
+** ret
+*/
+unsigned int s4_si_xor (unsigned int x) { return x ^ 0x12345678U; }
--
2.34.1
