Module: Mesa Branch: main Commit: 4f892ecc1e143c42a41d32450262e799d4d152af URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=4f892ecc1e143c42a41d32450262e799d4d152af
Author: Karol Herbst <kher...@redhat.com> Date: Tue Oct 24 19:34:10 2023 +0200 zink: emit MemoryAccess flags for coherent global load/stores Makes global load/stores coherent on a device level if requested by the shader. Signed-off-by: Karol Herbst <kher...@redhat.com> Acked-by: Mike Blumenkrantz <michael.blumenkra...@gmail.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/25937> --- .../drivers/zink/nir_to_spirv/nir_to_spirv.c | 6 +++-- .../drivers/zink/nir_to_spirv/spirv_builder.c | 30 +++++++++++++++++----- .../drivers/zink/nir_to_spirv/spirv_builder.h | 4 +-- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c b/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c index 3c8d2a53af3..c30426aba83 100644 --- a/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c +++ b/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c @@ -2539,6 +2539,7 @@ emit_load_push_const(struct ntv_context *ctx, nir_intrinsic_instr *intr) static void emit_load_global(struct ntv_context *ctx, nir_intrinsic_instr *intr) { + bool coherent = ctx->sinfo->have_vulkan_memory_model && nir_intrinsic_access(intr) & ACCESS_COHERENT; spirv_builder_emit_cap(&ctx->builder, SpvCapabilityPhysicalStorageBufferAddresses); SpvId dest_type = get_def_type(ctx, &intr->def, nir_type_uint); SpvId pointer_type = spirv_builder_type_pointer(&ctx->builder, @@ -2546,13 +2547,14 @@ emit_load_global(struct ntv_context *ctx, nir_intrinsic_instr *intr) dest_type); nir_alu_type atype; SpvId ptr = emit_bitcast(ctx, pointer_type, get_src(ctx, &intr->src[0], &atype)); - SpvId result = spirv_builder_emit_load_aligned(&ctx->builder, dest_type, ptr, intr->def.bit_size / 8); + SpvId result = spirv_builder_emit_load_aligned(&ctx->builder, dest_type, ptr, intr->def.bit_size / 8, coherent); store_def(ctx, &intr->def, result, nir_type_uint); } static void emit_store_global(struct ntv_context *ctx, nir_intrinsic_instr *intr) { + bool coherent = ctx->sinfo->have_vulkan_memory_model && nir_intrinsic_access(intr) & ACCESS_COHERENT; spirv_builder_emit_cap(&ctx->builder, SpvCapabilityPhysicalStorageBufferAddresses); unsigned bit_size = nir_src_bit_size(intr->src[0]); SpvId dest_type = get_uvec_type(ctx, bit_size, 1); @@ -2564,7 +2566,7 @@ emit_store_global(struct ntv_context *ctx, nir_intrinsic_instr *intr) if (atype != nir_type_uint) param = emit_bitcast(ctx, dest_type, param); SpvId ptr = emit_bitcast(ctx, pointer_type, get_src(ctx, &intr->src[1], &atype)); - spirv_builder_emit_store_aligned(&ctx->builder, ptr, param, bit_size / 8); + spirv_builder_emit_store_aligned(&ctx->builder, ptr, param, bit_size / 8, coherent); } static void diff --git a/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.c b/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.c index 68a55ee730c..60b14114415 100644 --- a/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.c +++ b/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.c @@ -470,9 +470,14 @@ spirv_builder_emit_load(struct spirv_builder *b, SpvId result_type, } SpvId -spirv_builder_emit_load_aligned(struct spirv_builder *b, SpvId result_type, SpvId pointer, unsigned alignment) +spirv_builder_emit_load_aligned(struct spirv_builder *b, SpvId result_type, SpvId pointer, unsigned alignment, bool coherent) { - return spirv_builder_emit_triop(b, SpvOpLoad, result_type, pointer, SpvMemoryAccessAlignedMask, alignment); + if (coherent) { + SpvId scope = spirv_builder_const_int(b, 32, SpvScopeDevice); + return spirv_builder_emit_quadop(b, SpvOpLoad, result_type, pointer, SpvMemoryAccessAlignedMask | SpvMemoryAccessNonPrivatePointerMask | SpvMemoryAccessMakePointerVisibleMask, alignment, scope); + } else { + return spirv_builder_emit_triop(b, SpvOpLoad, result_type, pointer, SpvMemoryAccessAlignedMask, alignment); + } } void @@ -485,14 +490,27 @@ spirv_builder_emit_store(struct spirv_builder *b, SpvId pointer, SpvId object) } void -spirv_builder_emit_store_aligned(struct spirv_builder *b, SpvId pointer, SpvId object, unsigned alignment) +spirv_builder_emit_store_aligned(struct spirv_builder *b, SpvId pointer, SpvId object, unsigned alignment, bool coherent) { - spirv_buffer_prepare(&b->instructions, b->mem_ctx, 5); - spirv_buffer_emit_word(&b->instructions, SpvOpStore | (5 << 16)); + unsigned size = 5; + SpvMemoryAccessMask mask = SpvMemoryAccessAlignedMask; + + if (coherent) { + mask |= SpvMemoryAccessNonPrivatePointerMask | SpvMemoryAccessMakePointerAvailableMask; + size++; + } + + spirv_buffer_prepare(&b->instructions, b->mem_ctx, size); + spirv_buffer_emit_word(&b->instructions, SpvOpStore | (size << 16)); spirv_buffer_emit_word(&b->instructions, pointer); spirv_buffer_emit_word(&b->instructions, object); - spirv_buffer_emit_word(&b->instructions, SpvMemoryAccessAlignedMask); + spirv_buffer_emit_word(&b->instructions, mask); spirv_buffer_emit_word(&b->instructions, alignment); + + if (coherent) { + SpvId scope = spirv_builder_const_int(b, 32, SpvScopeDevice); + spirv_buffer_emit_word(&b->instructions, scope); + } } void diff --git a/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.h b/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.h index dabf6f71440..2b9cac6fc2e 100644 --- a/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.h +++ b/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.h @@ -188,7 +188,7 @@ spirv_builder_emit_load(struct spirv_builder *b, SpvId result_type, SpvId pointer); SpvId -spirv_builder_emit_load_aligned(struct spirv_builder *b, SpvId result_type, SpvId pointer, unsigned alignment); +spirv_builder_emit_load_aligned(struct spirv_builder *b, SpvId result_type, SpvId pointer, unsigned alignment, bool coherent); void spirv_builder_emit_atomic_store(struct spirv_builder *b, SpvId pointer, SpvScope scope, SpvMemorySemanticsMask semantics, SpvId object); @@ -196,7 +196,7 @@ spirv_builder_emit_atomic_store(struct spirv_builder *b, SpvId pointer, SpvScope void spirv_builder_emit_store(struct spirv_builder *b, SpvId pointer, SpvId object); void -spirv_builder_emit_store_aligned(struct spirv_builder *b, SpvId pointer, SpvId object, unsigned alignment); +spirv_builder_emit_store_aligned(struct spirv_builder *b, SpvId pointer, SpvId object, unsigned alignment, bool coherent); SpvId spirv_builder_emit_access_chain(struct spirv_builder *b, SpvId result_type,
