https://gcc.gnu.org/g:8244aefef4c483ed66fa2bbd9eace076cd6fa757
commit r17-4977-g8244aefef4c483ed66fa2bbd9eace076cd6fa757 Author: Stefan Schulze Frielinghaus <[email protected]> Date: Thu Oct 8 08:43:36 2026 +0200 s390: Add vector multiply-and-add over integers This patch adds insn pattern for instruction VMAL which matches PLUS(MULT(a, b), c) over integer vector modes and TI mode. The 64- and 128-bit element modes become available with VXE3/z17. In order to ensure that the pattern is selected, RTX costing has to be tweaked. First of all constant vectors never end up as immediates. Costing for loading those via a single instruction (vzero, vone, vgm, vgbm, vrepi) is set to COSTS_N_INSNS(1) and otherwise they have to be loaded from constant pool and are therefore slightly penalized reflected by costs COSTS_N_INSNS(2). Costs for multiply-and-add are the same as for multiply only. gcc/ChangeLog: * config/s390/s390.cc (legitimate_reload_vector_constant_p): Move. (s390_rtx_costs): Add costs for VML and VMAL. (s390_legitimate_constant_p): Utilize legitimate_reload_vector_constant_p. * config/s390/vector.md (*vmal<mode>): New insn pattern. gcc/testsuite/ChangeLog: * gcc.target/s390/vector/vmal-1.c: New test. * gcc.target/s390/vector/vmal-2.c: New test. Diff: --- gcc/config/s390/s390.cc | 87 +++++++++---- gcc/config/s390/vector.md | 16 +++ gcc/testsuite/gcc.target/s390/vector/vmal-1.c | 85 ++++++++++++ gcc/testsuite/gcc.target/s390/vector/vmal-2.c | 181 ++++++++++++++++++++++++++ 4 files changed, 344 insertions(+), 25 deletions(-) diff --git a/gcc/config/s390/s390.cc b/gcc/config/s390/s390.cc index 36a49e5d22b6..ba576680229a 100644 --- a/gcc/config/s390/s390.cc +++ b/gcc/config/s390/s390.cc @@ -3966,6 +3966,21 @@ s390_insn_cost (rtx_insn *insn, bool speed) return pattern_cost (PATTERN (insn), speed); } +/* Returns true if the constant value OP is a legitimate vector operand + during and after reload. + This function accepts all constants which can be loaded directly + into an VR. */ + +static bool +legitimate_reload_vector_constant_p (rtx op) +{ + return TARGET_VX && (satisfies_constraint_j00 (op) + || satisfies_constraint_jm1 (op) + || satisfies_constraint_jzz (op) + || satisfies_constraint_jxx (op) + || satisfies_constraint_jyy (op)); +} + /* Compute a (partial) cost for rtx X. Return true if the complete cost has been computed, and false if subexpressions should be scanned. In either case, *TOTAL contains the cost result. The @@ -3978,6 +3993,7 @@ s390_rtx_costs (rtx x, machine_mode mode, int outer_code, int opno ATTRIBUTE_UNUSED, int *total, bool speed ATTRIBUTE_UNUSED) { + constexpr int vml_vmal_costs = COSTS_N_INSNS (5); int code = GET_CODE (x); switch (code) { @@ -3990,6 +4006,16 @@ s390_rtx_costs (rtx x, machine_mode mode, int outer_code, case MEM: *total = 0; return true; + + case CONST_VECTOR: + /* Constant vectors never end up as immediates. Penalize vectors which + cannot be loaded by a single instruction (vzero, vone, vgm, vgbm, + vrepi) and therefore go through the constant pool. */ + *total = COSTS_N_INSNS (1); + if (!legitimate_reload_vector_constant_p (x)) + *total += COSTS_N_INSNS (1); + return true; + case SET: { rtx dst = SET_DEST (x); @@ -4161,12 +4187,43 @@ s390_rtx_costs (rtx x, machine_mode mode, int outer_code, case XOR: case NEG: case NOT: - case PLUS: case MINUS: *total = COSTS_N_INSNS (1); return false; + case PLUS: + /* Check for VECTOR MULTIPLY AND ADD. */ + if (TARGET_VX + && (GET_MODE_CLASS (mode) == MODE_VECTOR_INT || mode == TImode) + /* VMALG/Q become available with z17. */ + && (GET_MODE_UNIT_SIZE (mode) < 8 || TARGET_VXE3) + && GET_CODE (XEXP (x, 0)) == MULT) + { + *total = vml_vmal_costs; + *total += rtx_cost (XEXP (x, 1), mode, PLUS, 1, speed); + *total += rtx_cost (XEXP (XEXP (x, 0), 0), mode, MULT, 0, speed); + *total += rtx_cost (XEXP (XEXP (x, 0), 1), mode, MULT, 1, speed); + return true; + } + *total = COSTS_N_INSNS (1); + return false; + case MULT: + if (TARGET_VX + && (GET_MODE_CLASS (mode) == MODE_VECTOR_INT || mode == TImode) + /* VMLG/Q become available with z17. */ + && (GET_MODE_UNIT_SIZE (mode) < 8 || TARGET_VXE3)) + { + /* During expand_mult we may emit a synthetic multiplication + depending on the costs. For example, for mode V16QI and + multiplicand 123 the sequence VREPI+VML competes with + VESL+VS+VESL+VS. The downside of the synthetic multiplication is + that on RTL level it is hard to reconstruct the fact that the + sequence was a multiplication which may prohibit the combination + with a PLUS into a VMAL. */ + *total = vml_vmal_costs; + return false; + } switch (mode) { case E_SImode: @@ -4509,15 +4566,10 @@ legitimate_pic_operand_p (rtx op) static bool s390_legitimate_constant_p (machine_mode mode, rtx op) { - if (TARGET_VX && GET_CODE (op) == CONST_VECTOR) - { - if (!satisfies_constraint_j00 (op) - && !satisfies_constraint_jm1 (op) - && !satisfies_constraint_jzz (op) - && !satisfies_constraint_jxx (op) - && !satisfies_constraint_jyy (op)) - return 0; - } + if (TARGET_VX + && GET_CODE (op) == CONST_VECTOR + && !legitimate_reload_vector_constant_p (op)) + return 0; if (mode == HFmode) return satisfies_constraint_j00 (op); @@ -4700,21 +4752,6 @@ legitimate_reload_fp_constant_p (rtx op) return false; } -/* Returns true if the constant value OP is a legitimate vector operand - during and after reload. - This function accepts all constants which can be loaded directly - into an VR. */ - -static bool -legitimate_reload_vector_constant_p (rtx op) -{ - return TARGET_VX && (satisfies_constraint_j00 (op) - || satisfies_constraint_jm1 (op) - || satisfies_constraint_jzz (op) - || satisfies_constraint_jxx (op) - || satisfies_constraint_jyy (op)); -} - /* Given an rtx OP being reloaded into a reg required to be in class RCLASS, return the class of reg to actually use. */ diff --git a/gcc/config/s390/vector.md b/gcc/config/s390/vector.md index acca6c4ea340..f3b9636e3988 100644 --- a/gcc/config/s390/vector.md +++ b/gcc/config/s390/vector.md @@ -67,6 +67,11 @@ ; All integer vector modes supported in a vector register + TImode (define_mode_iterator VIT [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI V1SI V2SI V4SI V1DI V2DI V1TI TI]) (define_mode_iterator VIT_VXE3 [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI V1SI V2SI V4SI V1DI V2DI (V1TI "TARGET_VXE3") (TI "TARGET_VXE3")]) +(define_mode_iterator VIT_VXE3_DT [V1QI V2QI V4QI V8QI V16QI + V1HI V2HI V4HI V8HI + V1SI V2SI V4SI + (V1DI "TARGET_VXE3") (V2DI "TARGET_VXE3") + (V1TI "TARGET_VXE3") (TI "TARGET_VXE3")]) (define_mode_iterator VI [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI V1SI V2SI V4SI V1DI V2DI]) (define_mode_iterator VI_VXE3 [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI V1SI V2SI V4SI V1DI (V2DI "TARGET_VXE3")]) (define_mode_iterator VI_QHS [V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI V1SI V2SI V4SI]) @@ -2020,6 +2025,17 @@ operands[5] = s390_expand_merge_perm_const (<vec_double>mode, true); }) +; vmalb, vmalhw, vmalf, vmalg, vmalq +(define_insn "*vmal<mode>" + [(set (match_operand:VIT_VXE3_DT 0 "register_operand" "=v") + (plus:VIT_VXE3_DT (mult:VIT_VXE3_DT + (match_operand:VIT_VXE3_DT 1 "register_operand" "v") + (match_operand:VIT_VXE3_DT 2 "general_operand" "v")) + (match_operand:VIT_VXE3_DT 3 "general_operand" "v")))] + "TARGET_VX" + "vmal<bhfgq><w>\t%v0,%v1,%v2,%v3" + [(set_attr "op_type" "VRR")]) + ; vec_widen_ushiftl_hi ; vec_widen_ushiftl_lo ; vec_widen_sshiftl_hi diff --git a/gcc/testsuite/gcc.target/s390/vector/vmal-1.c b/gcc/testsuite/gcc.target/s390/vector/vmal-1.c new file mode 100644 index 000000000000..a2a25850f49d --- /dev/null +++ b/gcc/testsuite/gcc.target/s390/vector/vmal-1.c @@ -0,0 +1,85 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=z13" } */ +/* { dg-final { check-function-bodies "**" "" "" } } */ + +/* Basic tests for VMAL where all operands are in registers (except for the + scalar TI mode test). VMALG/Q become available with TARGET_VXE3/z17. */ + +typedef signed char v16qi __attribute__ ((vector_size (16))); +typedef short v8hi __attribute__ ((vector_size (16))); +typedef int v4si __attribute__ ((vector_size (16))); +typedef long v2di __attribute__ ((vector_size (16))); +typedef __int128 v1ti __attribute__ ((vector_size (16))); + +/* +** test_v16qi: +** vmalb %v24,%v26,%v28,%v24 +** br %r14 +*/ + +v16qi test_v16qi (v16qi a, v16qi b, v16qi c) +{ + return a + b * c; +} + +/* +** test_v8hi: +** vmalhw %v24,%v26,%v28,%v24 +** br %r14 +*/ + +v8hi test_v8hi (v8hi a, v8hi b, v8hi c) +{ + return a + b * c; +} + +/* +** test_v4si: +** vmalf %v24,%v26,%v28,%v24 +** br %r14 +*/ + +v4si test_v4si (v4si a, v4si b, v4si c) +{ + return a + b * c; +} + +/* +** test_v2di: +** vmalg %v24,%v26,%v28,%v24 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +v2di test_v2di (v2di a, v2di b, v2di c) +{ + return a + b * c; +} + +/* +** test_v1ti: +** vmalq %v24,%v26,%v28,%v24 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +v1ti test_v1ti (v1ti a, v1ti b, v1ti c) +{ + return a + b * c; +} + +/* +** test_ti: +** vl .* +** vl .* +** vl .* +** vmalq .* +** vst .* +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +__int128 test_ti (__int128 a, __int128 b, __int128 c) +{ + return a + b * c; +} diff --git a/gcc/testsuite/gcc.target/s390/vector/vmal-2.c b/gcc/testsuite/gcc.target/s390/vector/vmal-2.c new file mode 100644 index 000000000000..f84534fd8edc --- /dev/null +++ b/gcc/testsuite/gcc.target/s390/vector/vmal-2.c @@ -0,0 +1,181 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=z13" } */ +/* { dg-final { check-function-bodies "**" "" "" } } */ + +/* Tests for VMAL with a constant multiplicand. If the constant is small, then + we compete with a synthetic multiplication which in turn means those tests + are subject to costing and therefore may be a bit fragile. */ + +typedef signed char v16qi __attribute__ ((vector_size (16))); +typedef short v8hi __attribute__ ((vector_size (16))); +typedef int v4si __attribute__ ((vector_size (16))); +typedef long v2di __attribute__ ((vector_size (16))); +typedef unsigned long uv2di __attribute__ ((vector_size (16))); +typedef __int128 v1ti __attribute__ ((vector_size (16))); + +#define CST_VEC (uv2di){0xBADC0FFEE0DDF00D, 0xBADC0FFEE0DDF00D} + +/* +** test_v16qi_plus_vrepi: +** veslb (%v[0-9]+),%v26,5 +** vsb (%v[0-9]+),\1,%v26 +** veslb (%v[0-9]+),\2,2 +** vsb (%v[0-9]+),\3,%v26 +** vab %v24,\4,%v24 +** br %r14 +*/ + +v16qi test_v16qi_plus_vrepi (v16qi a, v16qi b) +{ + return a + b * 123; +} + +/* +** test_v16qi_plus_constpool: +** larl .* +** vl (%v[0-9]+),.* +** vmalb %v24,%v26,\1,%v24 +** br %r14 +*/ + +v16qi test_v16qi_plus_constpool (v16qi a, v16qi b) +{ + return a + b * (v16qi)CST_VEC; +} + +/* +** test_v8hi_plus_vrepi: +** vrepi (%v[0-9]+),1234,1 +** vmalhw %v24,%v26,\1,%v24 +** br %r14 +*/ + +v8hi test_v8hi_plus_vrepi (v8hi a, v8hi b) +{ + return a + b * 1234; +} + +/* +** test_v8hi_plus_constpool: +** larl .* +** vl (%v[0-9]+),.* +** vmalhw %v24,%v26,\1,%v24 +** br %r14 +*/ + +v8hi test_v8hi_plus_constpool (v8hi a, v8hi b) +{ + return a + b * (v8hi)CST_VEC; +} + +/* +** test_v4si_plus_vrepi: +** vrepi (%v[0-9]+),1234,2 +** vmalf %v24,%v26,\1,%v24 +** br %r14 +*/ + +v4si test_v4si_plus_vrepi (v4si a, v4si b) +{ + return a + b * 1234; +} + +/* +** test_v4si_plus_constpool: +** larl .* +** vl (%v[0-9]+),.* +** vmalf %v24,%v26,\1,%v24 +** br %r14 +*/ + +v4si test_v4si_plus_constpool (v4si a, v4si b) +{ + return a + b * (v4si)CST_VEC; +} + +/* +** test_v2di_plus_vrepi: +** vrepi (%v[0-9]+),1234,3 +** vmalg %v24,%v26,\1,%v24 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +v2di test_v2di_plus_vrepi (v2di a, v2di b) +{ + return a + b * 1234; +} + +/* +** test_v2di_plus_constpool: +** larl .* +** vl (%v[0-9]+),.* +** vmalg %v24,%v26,\1,%v24 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +v2di test_v2di_plus_constpool (v2di a, v2di b) +{ + return a + b * (v2di)CST_VEC; +} + +/* +** test_v1ti_plus_vrepi: +** vrepi (%v[0-9]+),-86,0 +** vmalq %v24,%v26,\1,%v24 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +v1ti test_v1ti_plus_vrepi (v1ti a, v1ti b) +{ + return a + b * ((__int128)0xaaaaaaaaaaaaaaaa << 64 | (__int128)0xaaaaaaaaaaaaaaaa); +} + +/* +** test_v1ti_plus_constpool: +** larl .* +** vl (%v[0-9]+),.* +** vmalq %v24,%v26,\1,%v24 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +v1ti test_v1ti_plus_constpool (v1ti a, v1ti b) +{ + return a + b * ((__int128)0xBADC0FFEE0DDF00D << 64 | (__int128)0xBADC0FFEE0DDF00D); +} + +/* +** test_int128_plus_vrepi: +** (?:vl|vrepi) .* +** (?:vl|vrepi) .* +** (?:vl|vrepi) .* +** vmalq (%v[0-9]+),.* +** vst \1,0\(%r2\),3 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +__int128 test_int128_plus_vrepi (__int128 a, __int128 b) +{ + return a + b * ((__int128)0xaaaaaaaaaaaaaaaa << 64 | (__int128)0xaaaaaaaaaaaaaaaa); +} + +/* +** test_int128_plus_constpool: +** larl .* +** vl .* +** vl .* +** vl .* +** vmalq (%v[0-9]+),.* +** vst \1,0\(%r2\),3 +** br %r14 +*/ + +__attribute__ ((target ("arch=z17"))) +__int128 test_int128_plus_constpool (__int128 a, __int128 b) +{ + return a + b * ((__int128)0xBADC0FFEE0DDF00D << 64 | (__int128)0xBADC0FFEE0DDF00D); +}
