Module: Mesa
Branch: master
Commit: 922cd38172b8a2bc286bd082fde9cad4e278765b
URL:    
http://cgit.freedesktop.org/mesa/mesa/commit/?id=922cd38172b8a2bc286bd082fde9cad4e278765b

Author: Samuel Pitoiset <samuel.pitoi...@gmail.com>
Date:   Wed Mar 28 19:03:00 2018 +0200

radv: implement out-of-order rasterization when it's safe on VI+

Disabled by default for now, it can be enabled with
RADV_PERFTEST=outoforder.

No CTS regressions on Polaris, and all Vulkan games I tested
look good as well.

Expect small performance improvements for applications where
out-of-order rasterization can be enabled by the driver.

Loosely based on RadeonSI.

Signed-off-by: Samuel Pitoiset <samuel.pitoi...@gmail.com>
Reviewed-by: Bas Nieuwenhuizen <b...@basnieuwenhuizen.nl>

---

 src/amd/vulkan/radv_cmd_buffer.c  |  28 +++++
 src/amd/vulkan/radv_debug.h       |   1 +
 src/amd/vulkan/radv_device.c      |   7 ++
 src/amd/vulkan/radv_extensions.py |   2 +-
 src/amd/vulkan/radv_pipeline.c    | 240 +++++++++++++++++++++++++++++++++++++-
 src/amd/vulkan/radv_private.h     |   4 +
 6 files changed, 279 insertions(+), 3 deletions(-)

diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c
index b3d6fc484e..b18718458f 100644
--- a/src/amd/vulkan/radv_cmd_buffer.c
+++ b/src/amd/vulkan/radv_cmd_buffer.c
@@ -1171,10 +1171,24 @@ radv_emit_index_buffer(struct radv_cmd_buffer 
*cmd_buffer)
 
 void radv_set_db_count_control(struct radv_cmd_buffer *cmd_buffer)
 {
+       struct radv_pipeline *pipeline = cmd_buffer->state.pipeline;
+       uint32_t pa_sc_mode_cntl_1 =
+               pipeline ? pipeline->graphics.ms.pa_sc_mode_cntl_1 : 0;
        uint32_t db_count_control;
 
        if(!cmd_buffer->state.active_occlusion_queries) {
                if (cmd_buffer->device->physical_device->rad_info.chip_class >= 
CIK) {
+                       if 
(G_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(pa_sc_mode_cntl_1) &&
+                           
pipeline->graphics.disable_out_of_order_rast_for_occlusion) {
+                               /* Re-enable out-of-order rasterization if the
+                                * bound pipeline supports it and if it's has
+                                * been disabled before starting occlusion
+                                * queries.
+                                */
+                               radeon_set_context_reg(cmd_buffer->cs,
+                                                      
R_028A4C_PA_SC_MODE_CNTL_1,
+                                                      pa_sc_mode_cntl_1);
+                       }
                        db_count_control = 0;
                } else {
                        db_count_control = S_028004_ZPASS_INCREMENT_DISABLE(1);
@@ -1189,6 +1203,20 @@ void radv_set_db_count_control(struct radv_cmd_buffer 
*cmd_buffer)
                                S_028004_ZPASS_ENABLE(1) |
                                S_028004_SLICE_EVEN_ENABLE(1) |
                                S_028004_SLICE_ODD_ENABLE(1);
+
+                       if 
(G_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(pa_sc_mode_cntl_1) &&
+                           
pipeline->graphics.disable_out_of_order_rast_for_occlusion) {
+                               /* If the bound pipeline has enabled
+                                * out-of-order rasterization, we should
+                                * disable it before starting occlusion
+                                * queries.
+                                */
+                               pa_sc_mode_cntl_1 &= 
C_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE;
+
+                               radeon_set_context_reg(cmd_buffer->cs,
+                                                      
R_028A4C_PA_SC_MODE_CNTL_1,
+                                                      pa_sc_mode_cntl_1);
+                       }
                } else {
                        db_count_control = S_028004_PERFECT_ZPASS_COUNTS(1) |
                                S_028004_SAMPLE_RATE(sample_rate);
diff --git a/src/amd/vulkan/radv_debug.h b/src/amd/vulkan/radv_debug.h
index 08877676b5..f35991fa4e 100644
--- a/src/amd/vulkan/radv_debug.h
+++ b/src/amd/vulkan/radv_debug.h
@@ -50,6 +50,7 @@ enum {
        RADV_PERFTEST_SISCHED        =   0x2,
        RADV_PERFTEST_LOCAL_BOS      =   0x4,
        RADV_PERFTEST_BINNING     =   0x8,
+       RADV_PERFTEST_OUT_OF_ORDER   =  0x10,
 };
 
 bool
diff --git a/src/amd/vulkan/radv_device.c b/src/amd/vulkan/radv_device.c
index 65727571a3..41f8242754 100644
--- a/src/amd/vulkan/radv_device.c
+++ b/src/amd/vulkan/radv_device.c
@@ -307,6 +307,12 @@ radv_physical_device_init(struct radv_physical_device 
*device,
        device->has_scissor_bug = device->rad_info.family == CHIP_VEGA10 ||
                                  device->rad_info.family == CHIP_RAVEN;
 
+       /* Out-of-order primitive rasterization. */
+       device->has_out_of_order_rast = device->rad_info.chip_class >= VI &&
+                                       device->rad_info.max_se >= 2;
+       device->out_of_order_rast_allowed = device->has_out_of_order_rast &&
+                                           (device->instance->perftest_flags & 
RADV_PERFTEST_OUT_OF_ORDER);
+
        radv_physical_device_init_mem_types(device);
        radv_fill_device_extension_table(device, &device->supported_extensions);
 
@@ -391,6 +397,7 @@ static const struct debug_control radv_perftest_options[] = 
{
        {"sisched", RADV_PERFTEST_SISCHED},
        {"localbos", RADV_PERFTEST_LOCAL_BOS},
        {"binning", RADV_PERFTEST_BINNING},
+       {"outoforderrast", RADV_PERFTEST_OUT_OF_ORDER},
        {NULL, 0}
 };
 
diff --git a/src/amd/vulkan/radv_extensions.py 
b/src/amd/vulkan/radv_extensions.py
index 8cb0b7aa7d..bc63a34896 100644
--- a/src/amd/vulkan/radv_extensions.py
+++ b/src/amd/vulkan/radv_extensions.py
@@ -95,7 +95,7 @@ EXTENSIONS = [
     Extension('VK_EXT_shader_viewport_index_layer',       1, True),
     Extension('VK_AMD_draw_indirect_count',               1, True),
     Extension('VK_AMD_gcn_shader',                        1, True),
-    Extension('VK_AMD_rasterization_order',               1, 
'device->rad_info.chip_class >= VI && device->rad_info.max_se >= 2'),
+    Extension('VK_AMD_rasterization_order',               1, 
'device->has_out_of_order_rast'),
     Extension('VK_AMD_shader_info',                       1, True),
     Extension('VK_AMD_shader_trinary_minmax',             1, True),
 ]
diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c
index c6828329fd..08abb9dbc4 100644
--- a/src/amd/vulkan/radv_pipeline.c
+++ b/src/amd/vulkan/radv_pipeline.c
@@ -63,10 +63,25 @@ struct radv_blend_state {
        uint32_t cb_shader_mask;
        uint32_t db_alpha_to_mask;
 
+       uint32_t commutative_4bit;
+
        bool single_cb_enable;
        bool mrt0_is_dual_src;
 };
 
+struct radv_dsa_order_invariance {
+       /* Whether the final result in Z/S buffers is guaranteed to be
+        * invariant under changes to the order in which fragments arrive.
+        */
+       bool zs;
+
+       /* Whether the set of fragments that pass the combined Z/S test is
+        * guaranteed to be invariant under changes to the order in which
+        * fragments arrive.
+        */
+       bool pass_set;
+};
+
 struct radv_tessellation_state {
        uint32_t ls_hs_config;
        unsigned num_patches;
@@ -530,6 +545,40 @@ radv_pipeline_compute_get_int_clamp(const 
VkGraphicsPipelineCreateInfo *pCreateI
        }
 }
 
+static void
+radv_blend_check_commutativity(struct radv_blend_state *blend,
+                              VkBlendOp op, VkBlendFactor src,
+                              VkBlendFactor dst, unsigned chanmask)
+{
+       /* Src factor is allowed when it does not depend on Dst. */
+       static const uint32_t src_allowed =
+               (1u << VK_BLEND_FACTOR_ONE) |
+               (1u << VK_BLEND_FACTOR_SRC_COLOR) |
+               (1u << VK_BLEND_FACTOR_SRC_ALPHA) |
+               (1u << VK_BLEND_FACTOR_SRC_ALPHA_SATURATE) |
+               (1u << VK_BLEND_FACTOR_CONSTANT_COLOR) |
+               (1u << VK_BLEND_FACTOR_CONSTANT_ALPHA) |
+               (1u << VK_BLEND_FACTOR_SRC1_COLOR) |
+               (1u << VK_BLEND_FACTOR_SRC1_ALPHA) |
+               (1u << VK_BLEND_FACTOR_ZERO) |
+               (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR) |
+               (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA) |
+               (1u << VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR) |
+               (1u << VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA) |
+               (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR) |
+               (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA);
+
+       if (dst == VK_BLEND_FACTOR_ONE &&
+           (src_allowed && (1u << src))) {
+               /* Addition is commutative, but floating point addition isn't
+                * associative: subtle changes can be introduced via different
+                * rounding. Be conservative, only enable for min and max.
+                */
+               if (op == VK_BLEND_OP_MAX || op == VK_BLEND_OP_MIN)
+                       blend->commutative_4bit |= chanmask;
+       }
+}
+
 static struct radv_blend_state
 radv_pipeline_init_blend_state(struct radv_pipeline *pipeline,
                               const VkGraphicsPipelineCreateInfo *pCreateInfo,
@@ -600,6 +649,11 @@ radv_pipeline_init_blend_state(struct radv_pipeline 
*pipeline,
                        dstA = VK_BLEND_FACTOR_ONE;
                }
 
+               radv_blend_check_commutativity(&blend, eqRGB, srcRGB, dstRGB,
+                                              0x7 << (4 * i));
+               radv_blend_check_commutativity(&blend, eqA, srcA, dstA,
+                                              0x8 << (4 * i));
+
                /* Blending optimizations for RB+.
                 * These transformations don't change the behavior.
                 *
@@ -750,13 +804,182 @@ static uint8_t radv_pipeline_get_ps_iter_samples(const 
VkPipelineMultisampleStat
        return ps_iter_samples;
 }
 
+static bool
+radv_is_depth_write_enabled(const VkPipelineDepthStencilStateCreateInfo 
*pCreateInfo)
+{
+       return pCreateInfo->depthTestEnable &&
+              pCreateInfo->depthWriteEnable &&
+              pCreateInfo->depthCompareOp != VK_COMPARE_OP_NEVER;
+}
+
+static bool
+radv_writes_stencil(const VkStencilOpState *state)
+{
+       return state->writeMask &&
+              (state->failOp != VK_STENCIL_OP_KEEP ||
+               state->passOp != VK_STENCIL_OP_KEEP ||
+               state->depthFailOp != VK_STENCIL_OP_KEEP);
+}
+
+static bool
+radv_is_stencil_write_enabled(const VkPipelineDepthStencilStateCreateInfo 
*pCreateInfo)
+{
+       return pCreateInfo->stencilTestEnable &&
+              (radv_writes_stencil(&pCreateInfo->front) ||
+               radv_writes_stencil(&pCreateInfo->back));
+}
+
+static bool
+radv_is_ds_write_enabled(const VkPipelineDepthStencilStateCreateInfo 
*pCreateInfo)
+{
+       return radv_is_depth_write_enabled(pCreateInfo) ||
+              radv_is_stencil_write_enabled(pCreateInfo);
+}
+
+static bool
+radv_order_invariant_stencil_op(VkStencilOp op)
+{
+       /* REPLACE is normally order invariant, except when the stencil
+        * reference value is written by the fragment shader. Tracking this
+        * interaction does not seem worth the effort, so be conservative.
+        */
+       return op != VK_STENCIL_OP_INCREMENT_AND_CLAMP &&
+              op != VK_STENCIL_OP_DECREMENT_AND_CLAMP &&
+              op != VK_STENCIL_OP_REPLACE;
+}
+
+static bool
+radv_order_invariant_stencil_state(const VkStencilOpState *state)
+{
+       /* Compute whether, assuming Z writes are disabled, this stencil state
+        * is order invariant in the sense that the set of passing fragments as
+        * well as the final stencil buffer result does not depend on the order
+        * of fragments.
+        */
+       return !state->writeMask ||
+              /* The following assumes that Z writes are disabled. */
+              (state->compareOp == VK_COMPARE_OP_ALWAYS &&
+               radv_order_invariant_stencil_op(state->passOp) &&
+               radv_order_invariant_stencil_op(state->depthFailOp)) ||
+              (state->compareOp == VK_COMPARE_OP_NEVER &&
+               radv_order_invariant_stencil_op(state->failOp));
+}
+
+static bool
+radv_pipeline_out_of_order_rast(struct radv_pipeline *pipeline,
+                               struct radv_blend_state *blend,
+                               const VkGraphicsPipelineCreateInfo *pCreateInfo)
+{
+       RADV_FROM_HANDLE(radv_render_pass, pass, pCreateInfo->renderPass);
+       struct radv_subpass *subpass = pass->subpasses + pCreateInfo->subpass;
+       unsigned colormask = blend->cb_target_enabled_4bit;
+
+       if (!pipeline->device->physical_device->out_of_order_rast_allowed)
+               return false;
+
+       /* Be conservative if a logic operation is enabled with color buffers. 
*/
+       if (colormask && pCreateInfo->pColorBlendState->logicOpEnable)
+               return false;
+
+       /* Default depth/stencil invariance when no attachment is bound. */
+       struct radv_dsa_order_invariance dsa_order_invariant = {
+               .zs = true, .pass_set = true
+       };
+
+       if (pCreateInfo->pDepthStencilState &&
+           subpass->depth_stencil_attachment.attachment != 
VK_ATTACHMENT_UNUSED) {
+               const VkPipelineDepthStencilStateCreateInfo *vkds =
+                       pCreateInfo->pDepthStencilState;
+               struct radv_render_pass_attachment *attachment =
+                       pass->attachments + 
subpass->depth_stencil_attachment.attachment;
+               bool has_stencil = vk_format_is_stencil(attachment->format);
+               struct radv_dsa_order_invariance order_invariance[2];
+               struct radv_shader_variant *ps =
+                       pipeline->shaders[MESA_SHADER_FRAGMENT];
+
+               /* Compute depth/stencil order invariance in order to know if
+                * it's safe to enable out-of-order.
+                */
+               bool zfunc_is_ordered =
+                       vkds->depthCompareOp == VK_COMPARE_OP_NEVER ||
+                       vkds->depthCompareOp == VK_COMPARE_OP_LESS ||
+                       vkds->depthCompareOp == VK_COMPARE_OP_LESS_OR_EQUAL ||
+                       vkds->depthCompareOp == VK_COMPARE_OP_GREATER ||
+                       vkds->depthCompareOp == VK_COMPARE_OP_GREATER_OR_EQUAL;
+
+               bool nozwrite_and_order_invariant_stencil =
+                       !radv_is_ds_write_enabled(vkds) ||
+                       (!radv_is_depth_write_enabled(vkds) &&
+                        radv_order_invariant_stencil_state(&vkds->front) &&
+                        radv_order_invariant_stencil_state(&vkds->back));
+
+               order_invariance[1].zs =
+                       nozwrite_and_order_invariant_stencil ||
+                       (!radv_is_stencil_write_enabled(vkds) &&
+                        zfunc_is_ordered);
+               order_invariance[0].zs =
+                       !radv_is_depth_write_enabled(vkds) || zfunc_is_ordered;
+
+               order_invariance[1].pass_set =
+                       nozwrite_and_order_invariant_stencil ||
+                       (!radv_is_stencil_write_enabled(vkds) &&
+                        (vkds->depthCompareOp == VK_COMPARE_OP_ALWAYS ||
+                         vkds->depthCompareOp == VK_COMPARE_OP_NEVER));
+               order_invariance[0].pass_set =
+                       !radv_is_depth_write_enabled(vkds) ||
+                       (vkds->depthCompareOp == VK_COMPARE_OP_ALWAYS ||
+                        vkds->depthCompareOp == VK_COMPARE_OP_NEVER);
+
+               dsa_order_invariant = order_invariance[has_stencil];
+               if (!dsa_order_invariant.zs)
+                       return false;
+
+               /* The set of PS invocations is always order invariant,
+                * except when early Z/S tests are requested.
+                */
+               if (ps &&
+                   ps->info.info.ps.writes_memory &&
+                   ps->info.fs.early_fragment_test &&
+                   !dsa_order_invariant.pass_set)
+                       return false;
+
+               /* Determine if out-of-order rasterization should be disabled
+                * when occlusion queries are used.
+                */
+               pipeline->graphics.disable_out_of_order_rast_for_occlusion =
+                       !dsa_order_invariant.pass_set;
+       }
+
+       /* No color buffers are enabled for writing. */
+       if (!colormask)
+               return true;
+
+       unsigned blendmask = colormask & blend->blend_enable_4bit;
+
+       if (blendmask) {
+               /* Only commutative blending. */
+               if (blendmask & ~blend->commutative_4bit)
+                       return false;
+
+               if (!dsa_order_invariant.pass_set)
+                       return false;
+       }
+
+       if (colormask & ~blendmask)
+               return false;
+
+       return true;
+}
+
 static void
 radv_pipeline_init_multisample_state(struct radv_pipeline *pipeline,
+                                    struct radv_blend_state *blend,
                                     const VkGraphicsPipelineCreateInfo 
*pCreateInfo)
 {
        const VkPipelineMultisampleStateCreateInfo *vkms = 
pCreateInfo->pMultisampleState;
        struct radv_multisample_state *ms = &pipeline->graphics.ms;
        unsigned num_tile_pipes = 
pipeline->device->physical_device->rad_info.num_tile_pipes;
+       bool out_of_order_rast = false;
        int ps_iter_samples = 1;
        uint32_t mask = 0xffff;
 
@@ -808,8 +1031,21 @@ radv_pipeline_init_multisample_state(struct radv_pipeline 
*pipeline,
        const struct VkPipelineRasterizationStateRasterizationOrderAMD 
*raster_order =
                vk_find_struct_const(pCreateInfo->pRasterizationState->pNext, 
PIPELINE_RASTERIZATION_STATE_RASTERIZATION_ORDER_AMD);
        if (raster_order && raster_order->rasterizationOrder == 
VK_RASTERIZATION_ORDER_RELAXED_AMD) {
+               /* Out-of-order rasterization is explicitly enabled by the
+                * application.
+                */
+               out_of_order_rast = true;
+       } else {
+               /* Determine if the driver can enable out-of-order
+                * rasterization internally.
+                */
+               out_of_order_rast =
+                       radv_pipeline_out_of_order_rast(pipeline, blend, 
pCreateInfo);
+       }
+
+       if (out_of_order_rast) {
                ms->pa_sc_mode_cntl_1 |= 
S_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(1) |
-                                       S_028A4C_OUT_OF_ORDER_WATER_MARK(0x7);
+                                        S_028A4C_OUT_OF_ORDER_WATER_MARK(0x7);
        }
 
        if (vkms && vkms->pSampleMask) {
@@ -3094,7 +3330,7 @@ radv_pipeline_init(struct radv_pipeline *pipeline,
                            pStages);
 
        pipeline->graphics.spi_baryc_cntl = S_0286E0_FRONT_FACE_ALL_BITS(1);
-       radv_pipeline_init_multisample_state(pipeline, pCreateInfo);
+       radv_pipeline_init_multisample_state(pipeline, &blend, pCreateInfo);
        uint32_t gs_out;
        uint32_t prim = 
si_translate_prim(pCreateInfo->pInputAssemblyState->topology);
 
diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h
index a5cbad6cfe..4485efaa09 100644
--- a/src/amd/vulkan/radv_private.h
+++ b/src/amd/vulkan/radv_private.h
@@ -291,6 +291,9 @@ struct radv_physical_device {
        bool cpdma_prefetch_writes_memory;
        bool has_scissor_bug;
 
+       bool has_out_of_order_rast;
+       bool out_of_order_rast_allowed;
+
        /* This is the drivers on-disk cache used as a fallback as opposed to
         * the pipeline cache defined by apps.
         */
@@ -1232,6 +1235,7 @@ struct radv_pipeline {
                        struct radv_prim_vertex_count prim_vertex_count;
                        bool can_use_guardband;
                        uint32_t needed_dynamic_state;
+                       bool disable_out_of_order_rast_for_occlusion;
                } graphics;
        };
 

_______________________________________________
mesa-commit mailing list
mesa-commit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/mesa-commit

Reply via email to