Module: Mesa Branch: main Commit: c733c166d7c572e6ad2917452dc32bea442fa4ce URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=c733c166d7c572e6ad2917452dc32bea442fa4ce
Author: Samuel Pitoiset <[email protected]> Date: Wed Jul 26 13:01:42 2023 +0200 radv: add radv_graphics_shaders_compile() to compile graphics shaders Similar to radv_compile_cs() but for all graphics stages. Signed-off-by: Samuel Pitoiset <[email protected]> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24423> --- src/amd/vulkan/radv_pipeline_graphics.c | 243 ++++++++++++++++++-------------- 1 file changed, 134 insertions(+), 109 deletions(-) diff --git a/src/amd/vulkan/radv_pipeline_graphics.c b/src/amd/vulkan/radv_pipeline_graphics.c index c930140dfa2..783b3a94eec 100644 --- a/src/amd/vulkan/radv_pipeline_graphics.c +++ b/src/amd/vulkan/radv_pipeline_graphics.c @@ -2491,101 +2491,15 @@ radv_skip_graphics_pipeline_compile(const struct radv_device *device, const stru return binary_stages == pipeline->active_stages; } -static VkResult -radv_graphics_pipeline_compile(struct radv_graphics_pipeline *pipeline, const VkGraphicsPipelineCreateInfo *pCreateInfo, - struct radv_pipeline_layout *pipeline_layout, struct radv_device *device, - struct vk_pipeline_cache *cache, const struct radv_pipeline_key *pipeline_key, - VkGraphicsPipelineLibraryFlagBitsEXT lib_flags, bool fast_linking_enabled) +static void +radv_graphics_shaders_compile(struct radv_device *device, struct vk_pipeline_cache *cache, + struct radv_shader_stage *stages, const struct radv_pipeline_key *pipeline_key, + struct radv_pipeline_layout *pipeline_layout, bool keep_executable_info, + bool keep_statistic_info, bool is_internal, + struct radv_retained_shaders *retained_shaders, bool noop_fs, + struct radv_shader **shaders, struct radv_shader_binary **binaries, + struct radv_shader **gs_copy_shader, struct radv_shader_binary **gs_copy_binary) { - struct radv_shader_binary *binaries[MESA_VULKAN_SHADER_STAGES] = {NULL}; - struct radv_shader_binary *gs_copy_binary = NULL; - struct radv_shader_part_binary *ps_epilog_binary = NULL; - unsigned char hash[20]; - bool keep_executable_info = radv_pipeline_capture_shaders(device, pCreateInfo->flags); - bool keep_statistic_info = radv_pipeline_capture_shader_stats(device, pCreateInfo->flags); - struct radv_shader_stage stages[MESA_VULKAN_SHADER_STAGES]; - const VkPipelineCreationFeedbackCreateInfo *creation_feedback = - vk_find_struct_const(pCreateInfo->pNext, PIPELINE_CREATION_FEEDBACK_CREATE_INFO); - VkPipelineCreationFeedback pipeline_feedback = { - .flags = VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT, - }; - bool skip_shaders_cache = false; - VkResult result = VK_SUCCESS; - const bool retain_shaders = !!(pCreateInfo->flags & VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT); - - int64_t pipeline_start = os_time_get_nano(); - - for (unsigned i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) { - stages[i].entrypoint = NULL; - stages[i].nir = NULL; - stages[i].spirv.size = 0; - } - - for (uint32_t i = 0; i < pCreateInfo->stageCount; i++) { - const VkPipelineShaderStageCreateInfo *sinfo = &pCreateInfo->pStages[i]; - gl_shader_stage stage = vk_to_mesa_shader_stage(sinfo->stage); - - /* Ignore graphics shader stages that don't need to be imported. */ - if (!(shader_stage_to_pipeline_library_flags(sinfo->stage) & lib_flags)) - continue; - - radv_shader_stage_init(sinfo, &stages[stage], stage); - } - - radv_pipeline_load_retained_shaders(device, pipeline, pCreateInfo, stages); - - if (!fast_linking_enabled) { - radv_hash_shaders(hash, stages, MESA_VULKAN_SHADER_STAGES, pipeline_layout, pipeline_key, - radv_get_hash_flags(device, keep_statistic_info)); - - pipeline->base.pipeline_hash = *(uint64_t *)hash; - } - - /* Skip the shaders cache when any of the below are true: - * - fast-linking is enabled because it's useless to cache unoptimized pipelines - * - shaders are captured because it's for debugging purposes - * - graphics pipeline libraries are created with the RETAIN_LINK_TIME_OPTIMIZATION flag and - * module identifiers are used (ie. no SPIR-V provided). - */ - if (fast_linking_enabled || keep_executable_info) { - skip_shaders_cache = true; - } else if ((pCreateInfo->flags & VK_PIPELINE_CREATE_LIBRARY_BIT_KHR) && retain_shaders) { - for (uint32_t i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) { - if (stages[i].entrypoint && !stages[i].spirv.size) { - skip_shaders_cache = true; - break; - } - } - } - - bool found_in_application_cache = true; - if (!skip_shaders_cache && - radv_pipeline_cache_search(device, cache, &pipeline->base, hash, &found_in_application_cache)) { - if (found_in_application_cache) - pipeline_feedback.flags |= VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT; - - if (retain_shaders) { - /* For graphics pipeline libraries created with the RETAIN_LINK_TIME_OPTIMIZATION flag, we - * need to retain the stage info because we can't know if the LTO pipelines will - * be find in the shaders cache. - */ - struct radv_graphics_lib_pipeline *gfx_pipeline_lib = radv_pipeline_to_graphics_lib(&pipeline->base); - - gfx_pipeline_lib->stages = radv_copy_shader_stage_create_info(device, pCreateInfo->stageCount, - pCreateInfo->pStages, gfx_pipeline_lib->mem_ctx); - if (!gfx_pipeline_lib->stages) - return VK_ERROR_OUT_OF_HOST_MEMORY; - - gfx_pipeline_lib->stage_count = pCreateInfo->stageCount; - } - - result = VK_SUCCESS; - goto done; - } - - if (pCreateInfo->flags & VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT) - return VK_PIPELINE_COMPILE_REQUIRED; - for (unsigned s = 0; s < MESA_VULKAN_SHADER_STAGES; s++) { if (!stages[s].entrypoint) continue; @@ -2594,15 +2508,14 @@ radv_graphics_pipeline_compile(struct radv_graphics_pipeline *pipeline, const Vk /* NIR might already have been imported from a library. */ if (!stages[s].nir) { - stages[s].nir = radv_shader_spirv_to_nir(device, &stages[s], pipeline_key, pipeline->base.is_internal); + stages[s].nir = radv_shader_spirv_to_nir(device, &stages[s], pipeline_key, is_internal); } stages[s].feedback.duration += os_time_get_nano() - stage_start; } - if (retain_shaders) { - struct radv_graphics_lib_pipeline *gfx_pipeline_lib = radv_pipeline_to_graphics_lib(&pipeline->base); - radv_pipeline_retain_shaders(&gfx_pipeline_lib->retained_shaders, stages); + if (retained_shaders) { + radv_pipeline_retain_shaders(retained_shaders, stages); } VkShaderStageFlagBits active_nir_stages = 0; @@ -2636,10 +2549,14 @@ radv_graphics_pipeline_compile(struct radv_graphics_pipeline *pipeline, const Vk } /* Remove all varyings when the fragment shader is a noop. */ - const bool noop_fs = radv_pipeline_needs_noop_fs(pipeline, pipeline_key); - if (noop_fs && pipeline->last_vgt_api_stage != MESA_SHADER_NONE) { - nir_shader *nir = stages[pipeline->last_vgt_api_stage].nir; - radv_remove_varyings(nir); + if (noop_fs) { + radv_foreach_stage(i, active_nir_stages) + { + if (radv_is_last_vgt_stage(&stages[i])) { + radv_remove_varyings(stages[i].nir); + break; + } + } } radv_graphics_shaders_link(device, pipeline_key, stages); @@ -2667,7 +2584,7 @@ radv_graphics_pipeline_compile(struct radv_graphics_pipeline *pipeline, const Vk radv_nir_lower_poly_line_smooth(stages[MESA_SHADER_FRAGMENT].nir, pipeline_key); } - radv_fill_shader_info(device, pipeline->base.type, pipeline_layout, pipeline_key, stages, active_nir_stages); + radv_fill_shader_info(device, RADV_PIPELINE_GRAPHICS, pipeline_layout, pipeline_key, stages, active_nir_stages); radv_declare_pipeline_args(device, stages, pipeline_key, active_nir_stages); @@ -2685,15 +2602,12 @@ radv_graphics_pipeline_compile(struct radv_graphics_pipeline *pipeline, const Vk /* Compile NIR shaders to AMD assembly. */ radv_graphics_shaders_nir_to_asm(device, cache, stages, pipeline_key, pipeline_layout, keep_executable_info, - keep_statistic_info, active_nir_stages, pipeline->base.shaders, binaries, - &pipeline->base.gs_copy_shader, &gs_copy_binary); - - if (!radv_pipeline_create_ps_epilog(device, pipeline, pipeline_key, lib_flags, &ps_epilog_binary)) - return VK_ERROR_OUT_OF_DEVICE_MEMORY; + keep_statistic_info, active_nir_stages, shaders, binaries, gs_copy_shader, + gs_copy_binary); if (keep_executable_info) { for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { - struct radv_shader *shader = pipeline->base.shaders[i]; + struct radv_shader *shader = shaders[i]; if (!shader) continue; @@ -2705,6 +2619,117 @@ radv_graphics_pipeline_compile(struct radv_graphics_pipeline *pipeline, const Vk shader->spirv_size = stages[i].spirv.size; } } +} + +static VkResult +radv_graphics_pipeline_compile(struct radv_graphics_pipeline *pipeline, const VkGraphicsPipelineCreateInfo *pCreateInfo, + struct radv_pipeline_layout *pipeline_layout, struct radv_device *device, + struct vk_pipeline_cache *cache, const struct radv_pipeline_key *pipeline_key, + VkGraphicsPipelineLibraryFlagBitsEXT lib_flags, bool fast_linking_enabled) +{ + struct radv_shader_binary *binaries[MESA_VULKAN_SHADER_STAGES] = {NULL}; + struct radv_shader_binary *gs_copy_binary = NULL; + struct radv_shader_part_binary *ps_epilog_binary = NULL; + unsigned char hash[20]; + bool keep_executable_info = radv_pipeline_capture_shaders(device, pCreateInfo->flags); + bool keep_statistic_info = radv_pipeline_capture_shader_stats(device, pCreateInfo->flags); + struct radv_shader_stage stages[MESA_VULKAN_SHADER_STAGES]; + const VkPipelineCreationFeedbackCreateInfo *creation_feedback = + vk_find_struct_const(pCreateInfo->pNext, PIPELINE_CREATION_FEEDBACK_CREATE_INFO); + VkPipelineCreationFeedback pipeline_feedback = { + .flags = VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT, + }; + bool skip_shaders_cache = false; + VkResult result = VK_SUCCESS; + const bool retain_shaders = !!(pCreateInfo->flags & VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT); + struct radv_retained_shaders *retained_shaders = NULL; + + int64_t pipeline_start = os_time_get_nano(); + + for (unsigned i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) { + stages[i].entrypoint = NULL; + stages[i].nir = NULL; + stages[i].spirv.size = 0; + } + + for (uint32_t i = 0; i < pCreateInfo->stageCount; i++) { + const VkPipelineShaderStageCreateInfo *sinfo = &pCreateInfo->pStages[i]; + gl_shader_stage stage = vk_to_mesa_shader_stage(sinfo->stage); + + /* Ignore graphics shader stages that don't need to be imported. */ + if (!(shader_stage_to_pipeline_library_flags(sinfo->stage) & lib_flags)) + continue; + + radv_shader_stage_init(sinfo, &stages[stage], stage); + } + + radv_pipeline_load_retained_shaders(device, pipeline, pCreateInfo, stages); + + if (!fast_linking_enabled) { + radv_hash_shaders(hash, stages, MESA_VULKAN_SHADER_STAGES, pipeline_layout, pipeline_key, + radv_get_hash_flags(device, keep_statistic_info)); + + pipeline->base.pipeline_hash = *(uint64_t *)hash; + } + + /* Skip the shaders cache when any of the below are true: + * - fast-linking is enabled because it's useless to cache unoptimized pipelines + * - shaders are captured because it's for debugging purposes + * - graphics pipeline libraries are created with the RETAIN_LINK_TIME_OPTIMIZATION flag and + * module identifiers are used (ie. no SPIR-V provided). + */ + if (fast_linking_enabled || keep_executable_info) { + skip_shaders_cache = true; + } else if ((pCreateInfo->flags & VK_PIPELINE_CREATE_LIBRARY_BIT_KHR) && retain_shaders) { + for (uint32_t i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) { + if (stages[i].entrypoint && !stages[i].spirv.size) { + skip_shaders_cache = true; + break; + } + } + } + + bool found_in_application_cache = true; + if (!skip_shaders_cache && + radv_pipeline_cache_search(device, cache, &pipeline->base, hash, &found_in_application_cache)) { + if (found_in_application_cache) + pipeline_feedback.flags |= VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT; + + if (retain_shaders) { + /* For graphics pipeline libraries created with the RETAIN_LINK_TIME_OPTIMIZATION flag, we + * need to retain the stage info because we can't know if the LTO pipelines will + * be find in the shaders cache. + */ + struct radv_graphics_lib_pipeline *gfx_pipeline_lib = radv_pipeline_to_graphics_lib(&pipeline->base); + + gfx_pipeline_lib->stages = radv_copy_shader_stage_create_info(device, pCreateInfo->stageCount, + pCreateInfo->pStages, gfx_pipeline_lib->mem_ctx); + if (!gfx_pipeline_lib->stages) + return VK_ERROR_OUT_OF_HOST_MEMORY; + + gfx_pipeline_lib->stage_count = pCreateInfo->stageCount; + } + + result = VK_SUCCESS; + goto done; + } + + if (pCreateInfo->flags & VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT) + return VK_PIPELINE_COMPILE_REQUIRED; + + if (retain_shaders) { + struct radv_graphics_lib_pipeline *gfx_pipeline_lib = radv_pipeline_to_graphics_lib(&pipeline->base); + retained_shaders = &gfx_pipeline_lib->retained_shaders; + } + + const bool noop_fs = radv_pipeline_needs_noop_fs(pipeline, pipeline_key); + + radv_graphics_shaders_compile(device, cache, stages, pipeline_key, pipeline_layout, keep_executable_info, + keep_statistic_info, pipeline->base.is_internal, retained_shaders, noop_fs, + pipeline->base.shaders, binaries, &pipeline->base.gs_copy_shader, &gs_copy_binary); + + if (!radv_pipeline_create_ps_epilog(device, pipeline, pipeline_key, lib_flags, &ps_epilog_binary)) + return VK_ERROR_OUT_OF_DEVICE_MEMORY; if (!skip_shaders_cache) { radv_pipeline_cache_insert(device, cache, &pipeline->base, ps_epilog_binary, hash);
