Module: Mesa Branch: main Commit: 3328714295c23c367d4be991a16ce407c88ac967 URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=3328714295c23c367d4be991a16ce407c88ac967
Author: Caio Oliveira <[email protected]> Date: Wed Nov 16 23:58:27 2022 -0800 nir/lower_subgroups: Add option lower_rotate_to_shuffle Reviewed-by: Ian Romanick <[email protected]> Reviewed-by: Timur Kristóf <[email protected]> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/19797> --- src/compiler/nir/nir.h | 1 + src/compiler/nir/nir_lower_subgroups.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index 872f5156567..12ff307db18 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -5094,6 +5094,7 @@ typedef struct nir_lower_subgroups_options { bool lower_quad_broadcast_dynamic_to_const:1; bool lower_elect:1; bool lower_read_invocation_to_cond:1; + bool lower_rotate_to_shuffle:1; } nir_lower_subgroups_options; bool nir_lower_subgroups(nir_shader *shader, diff --git a/src/compiler/nir/nir_lower_subgroups.c b/src/compiler/nir/nir_lower_subgroups.c index 0c6f3a2b28c..75a8ecd22b8 100644 --- a/src/compiler/nir/nir_lower_subgroups.c +++ b/src/compiler/nir/nir_lower_subgroups.c @@ -281,6 +281,23 @@ lower_to_shuffle(nir_builder *b, nir_intrinsic_instr *intrin, case nir_intrinsic_quad_swap_diagonal: index = nir_ixor(b, index, nir_imm_int(b, 0x3)); break; + case nir_intrinsic_rotate: { + nir_ssa_def *delta = intrin->src[1].ssa; + nir_ssa_def *local_id = nir_load_subgroup_invocation(b); + const unsigned cluster_size = nir_intrinsic_cluster_size(intrin); + + nir_ssa_def *rotation_group_mask = + cluster_size > 0 ? nir_imm_int(b, (int)(cluster_size - 1)) : + nir_iadd_imm(b, nir_load_subgroup_size(b), -1); + + index = nir_iand(b, nir_iadd(b, local_id, delta), + rotation_group_mask); + if (cluster_size > 0) { + index = nir_iadd(b, index, + nir_iand(b, local_id, nir_inot(b, rotation_group_mask))); + } + break; + } default: unreachable("Invalid intrinsic"); } @@ -828,6 +845,17 @@ lower_subgroups_instr(nir_builder *b, nir_instr *instr, void *_options) return lower_subgroup_op_to_scalar(b, intrin, false); break; + case nir_intrinsic_rotate: + if (nir_intrinsic_execution_scope(intrin) == NIR_SCOPE_SUBGROUP) { + if (options->lower_rotate_to_shuffle) + return lower_to_shuffle(b, intrin, options); + else if (options->lower_to_scalar && intrin->num_components > 1) + return lower_subgroup_op_to_scalar(b, intrin, options->lower_shuffle_to_32bit); + else if (options->lower_shuffle_to_32bit && intrin->src[0].ssa->bit_size == 64) + return lower_subgroup_op_to_32bit(b, intrin); + } + break; + default: break; }
