Module: Mesa Branch: main Commit: 86e4fcd9a92962007adaf8fe1554637c624cb05c URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=86e4fcd9a92962007adaf8fe1554637c624cb05c
Author: Mike Blumenkrantz <[email protected]> Date: Thu Jun 23 17:07:12 2022 -0400 zink: add a graphics pipeline library implementation this is only for driver debugging/validation and will not activate unless ZINK_PIPELINE_LIBRARY_FORCE=1 is specified since it can only ever add overhead and slow things down ...for now Reviewed-by: Dave Airlie <[email protected]> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17225> --- src/gallium/drivers/zink/zink_context.h | 2 + src/gallium/drivers/zink/zink_pipeline.c | 387 +++++++++++++++++++++++++++++++ src/gallium/drivers/zink/zink_pipeline.h | 17 +- src/gallium/drivers/zink/zink_program.c | 181 ++++++++++++++- src/gallium/drivers/zink/zink_program.h | 34 +++ src/gallium/drivers/zink/zink_screen.c | 12 +- src/gallium/drivers/zink/zink_screen.h | 1 + 7 files changed, 625 insertions(+), 9 deletions(-) diff --git a/src/gallium/drivers/zink/zink_context.h b/src/gallium/drivers/zink/zink_context.h index 25d2370ae50..5800f292e88 100644 --- a/src/gallium/drivers/zink/zink_context.h +++ b/src/gallium/drivers/zink/zink_context.h @@ -252,6 +252,8 @@ struct zink_context { struct hash_table program_cache[8]; uint32_t gfx_hash; struct zink_gfx_program *curr_program; + struct set gfx_inputs; + struct set gfx_outputs; struct zink_descriptor_data *dd; diff --git a/src/gallium/drivers/zink/zink_pipeline.c b/src/gallium/drivers/zink/zink_pipeline.c index 26d531cb823..a6b6b8e6a77 100644 --- a/src/gallium/drivers/zink/zink_pipeline.c +++ b/src/gallium/drivers/zink/zink_pipeline.c @@ -426,3 +426,390 @@ zink_create_compute_pipeline(struct zink_screen *screen, struct zink_compute_pro return pipeline; } + +VkPipeline +zink_create_gfx_pipeline_output(struct zink_screen *screen, struct zink_gfx_pipeline_state *state) +{ + VkGraphicsPipelineLibraryCreateInfoEXT gplci = { + VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT, + &state->rendering_info, + VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT, + }; + + VkPipelineColorBlendAttachmentState blend_att[PIPE_MAX_COLOR_BUFS]; + VkPipelineColorBlendStateCreateInfo blend_state = {0}; + blend_state.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO; + if (state->blend_state) { + unsigned num_attachments = state->rendering_info.colorAttachmentCount; + if (state->void_alpha_attachments) { + for (unsigned i = 0; i < num_attachments; i++) { + blend_att[i] = state->blend_state->attachments[i]; + if (state->void_alpha_attachments & BITFIELD_BIT(i)) { + blend_att[i].dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO; + blend_att[i].srcColorBlendFactor = clamp_void_blend_factor(blend_att[i].srcColorBlendFactor); + blend_att[i].dstColorBlendFactor = clamp_void_blend_factor(blend_att[i].dstColorBlendFactor); + } + } + blend_state.pAttachments = blend_att; + } else + blend_state.pAttachments = state->blend_state->attachments; + blend_state.attachmentCount = num_attachments; + blend_state.logicOpEnable = state->blend_state->logicop_enable; + blend_state.logicOp = state->blend_state->logicop_func; + } + + VkPipelineMultisampleStateCreateInfo ms_state = {0}; + ms_state.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO; + ms_state.rasterizationSamples = state->rast_samples + 1; + if (state->blend_state) { + ms_state.alphaToCoverageEnable = state->blend_state->alpha_to_coverage; + if (state->blend_state->alpha_to_one && !screen->info.feats.features.alphaToOne) { + static bool warned = false; + warn_missing_feature(warned, "alphaToOne"); + } + ms_state.alphaToOneEnable = state->blend_state->alpha_to_one; + } + /* "If pSampleMask is NULL, it is treated as if the mask has all bits set to 1." + * - Chapter 27. Rasterization + * + * thus it never makes sense to leave this as NULL since gallium will provide correct + * data here as long as sample_mask is initialized on context creation + */ + ms_state.pSampleMask = &state->sample_mask; + if (state->force_persample_interp) { + ms_state.sampleShadingEnable = VK_TRUE; + ms_state.minSampleShading = 1.0; + } + + VkDynamicState dynamicStateEnables[30] = { + VK_DYNAMIC_STATE_BLEND_CONSTANTS, + }; + unsigned state_count = 1; + if (screen->info.have_EXT_extended_dynamic_state) { + if (state->sample_locations_enabled) + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_SAMPLE_LOCATIONS_EXT; + } + if (!screen->driver_workarounds.color_write_missing) + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_COLOR_WRITE_ENABLE_EXT; + assert(state_count < ARRAY_SIZE(dynamicStateEnables)); + + VkPipelineDynamicStateCreateInfo pipelineDynamicStateCreateInfo = {0}; + pipelineDynamicStateCreateInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO; + pipelineDynamicStateCreateInfo.pDynamicStates = dynamicStateEnables; + pipelineDynamicStateCreateInfo.dynamicStateCount = state_count; + + VkGraphicsPipelineCreateInfo pci = {0}; + pci.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO; + pci.pNext = &gplci; + pci.flags = VK_PIPELINE_CREATE_LIBRARY_BIT_KHR; + pci.pColorBlendState = &blend_state; + pci.pMultisampleState = &ms_state; + pci.pDynamicState = &pipelineDynamicStateCreateInfo; + + VkPipeline pipeline; + if (VKSCR(CreateGraphicsPipelines)(screen->dev, VK_NULL_HANDLE, 1, &pci, + NULL, &pipeline) != VK_SUCCESS) { + mesa_loge("ZINK: vkCreateGraphicsPipelines failed"); + return VK_NULL_HANDLE; + } + + return pipeline; +} + +VkPipeline +zink_create_gfx_pipeline_input(struct zink_screen *screen, + struct zink_gfx_pipeline_state *state, + const uint8_t *binding_map, + VkPrimitiveTopology primitive_topology) +{ + VkGraphicsPipelineLibraryCreateInfoEXT gplci = { + VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT, + NULL, + VK_GRAPHICS_PIPELINE_LIBRARY_VERTEX_INPUT_INTERFACE_BIT_EXT + }; + + VkPipelineVertexInputStateCreateInfo vertex_input_state; + memset(&vertex_input_state, 0, sizeof(vertex_input_state)); + vertex_input_state.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO; + if (!screen->info.have_EXT_vertex_input_dynamic_state || !state->uses_dynamic_stride) { + vertex_input_state.pVertexBindingDescriptions = state->element_state->b.bindings; + vertex_input_state.vertexBindingDescriptionCount = state->element_state->num_bindings; + vertex_input_state.pVertexAttributeDescriptions = state->element_state->attribs; + vertex_input_state.vertexAttributeDescriptionCount = state->element_state->num_attribs; + if (!state->uses_dynamic_stride) { + for (int i = 0; i < state->element_state->num_bindings; ++i) { + const unsigned buffer_id = binding_map[i]; + VkVertexInputBindingDescription *binding = &state->element_state->b.bindings[i]; + binding->stride = state->vertex_strides[buffer_id]; + } + } + } + + VkPipelineVertexInputDivisorStateCreateInfoEXT vdiv_state; + if (!screen->info.have_EXT_vertex_input_dynamic_state && state->element_state->b.divisors_present) { + memset(&vdiv_state, 0, sizeof(vdiv_state)); + vertex_input_state.pNext = &vdiv_state; + vdiv_state.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_DIVISOR_STATE_CREATE_INFO_EXT; + vdiv_state.vertexBindingDivisorCount = state->element_state->b.divisors_present; + vdiv_state.pVertexBindingDivisors = state->element_state->b.divisors; + } + + VkPipelineInputAssemblyStateCreateInfo primitive_state = {0}; + primitive_state.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO; + primitive_state.topology = primitive_topology; + assert(screen->info.have_EXT_extended_dynamic_state2); + + VkDynamicState dynamicStateEnables[30]; + unsigned state_count = 0; + if (screen->info.have_EXT_vertex_input_dynamic_state) + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_VERTEX_INPUT_EXT; + else if (state->uses_dynamic_stride) + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_VERTEX_INPUT_BINDING_STRIDE_EXT; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_PRIMITIVE_RESTART_ENABLE; + assert(state_count < ARRAY_SIZE(dynamicStateEnables)); + + VkPipelineDynamicStateCreateInfo pipelineDynamicStateCreateInfo = {0}; + pipelineDynamicStateCreateInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO; + pipelineDynamicStateCreateInfo.pDynamicStates = dynamicStateEnables; + pipelineDynamicStateCreateInfo.dynamicStateCount = state_count; + + VkGraphicsPipelineCreateInfo pci = {0}; + pci.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO; + pci.pNext = &gplci; + pci.flags = VK_PIPELINE_CREATE_LIBRARY_BIT_KHR; + pci.pVertexInputState = &vertex_input_state; + pci.pInputAssemblyState = &primitive_state; + pci.pDynamicState = &pipelineDynamicStateCreateInfo; + + VkPipeline pipeline; + if (VKSCR(CreateGraphicsPipelines)(screen->dev, VK_NULL_HANDLE, 1, &pci, + NULL, &pipeline) != VK_SUCCESS) { + mesa_loge("ZINK: vkCreateGraphicsPipelines failed"); + return VK_NULL_HANDLE; + } + + return pipeline; +} + +VkPipeline +zink_create_gfx_pipeline_library(struct zink_screen *screen, struct zink_gfx_program *prog, + struct zink_rasterizer_hw_state *hw_rast_state, bool line) +{ + assert(screen->info.have_EXT_extended_dynamic_state && screen->info.have_EXT_extended_dynamic_state2); + VkPipelineRenderingCreateInfo rendering_info; + rendering_info.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO; + rendering_info.pNext = NULL; + rendering_info.viewMask = 0; + VkGraphicsPipelineLibraryCreateInfoEXT gplci = { + VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT, + &rendering_info, + VK_GRAPHICS_PIPELINE_LIBRARY_PRE_RASTERIZATION_SHADERS_BIT_EXT | VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT + }; + + VkPipelineViewportStateCreateInfo viewport_state = {0}; + VkPipelineViewportDepthClipControlCreateInfoEXT clip = { + VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_DEPTH_CLIP_CONTROL_CREATE_INFO_EXT, + NULL, + VK_TRUE + }; + viewport_state.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO; + viewport_state.viewportCount = 0; + viewport_state.pViewports = NULL; + viewport_state.scissorCount = 0; + viewport_state.pScissors = NULL; + if (!screen->driver_workarounds.depth_clip_control_missing && !hw_rast_state->clip_halfz) + viewport_state.pNext = &clip; + + VkPipelineRasterizationStateCreateInfo rast_state = {0}; + rast_state.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO; + + rast_state.depthClampEnable = VK_TRUE; + rast_state.polygonMode = hw_rast_state->polygon_mode; + + rast_state.depthBiasEnable = VK_TRUE; + rast_state.depthBiasConstantFactor = 0.0; + rast_state.depthBiasClamp = 0.0; + rast_state.depthBiasSlopeFactor = 0.0; + + VkPipelineRasterizationDepthClipStateCreateInfoEXT depth_clip_state = {0}; + depth_clip_state.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_DEPTH_CLIP_STATE_CREATE_INFO_EXT; + depth_clip_state.depthClipEnable = hw_rast_state->depth_clip; + if (screen->info.have_EXT_depth_clip_enable) { + depth_clip_state.pNext = rast_state.pNext; + rast_state.pNext = &depth_clip_state; + } else { + static bool warned = false; + warn_missing_feature(warned, "VK_EXT_depth_clip_enable"); + rast_state.depthClampEnable = !hw_rast_state->depth_clip; + } + + VkPipelineRasterizationProvokingVertexStateCreateInfoEXT pv_state; + pv_state.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_PROVOKING_VERTEX_STATE_CREATE_INFO_EXT; + pv_state.provokingVertexMode = hw_rast_state->pv_last ? + VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT : + VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT; + if (screen->info.have_EXT_provoking_vertex && hw_rast_state->pv_last) { + pv_state.pNext = rast_state.pNext; + rast_state.pNext = &pv_state; + } + + VkPipelineDepthStencilStateCreateInfo depth_stencil_state = {0}; + depth_stencil_state.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO; + + VkDynamicState dynamicStateEnables[30] = { + VK_DYNAMIC_STATE_LINE_WIDTH, + VK_DYNAMIC_STATE_DEPTH_BIAS, + VK_DYNAMIC_STATE_STENCIL_REFERENCE, + }; + unsigned state_count = 3; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_VIEWPORT_WITH_COUNT; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_SCISSOR_WITH_COUNT; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_DEPTH_BOUNDS; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_DEPTH_BOUNDS_TEST_ENABLE; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_DEPTH_COMPARE_OP; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_DEPTH_TEST_ENABLE; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_DEPTH_WRITE_ENABLE; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_STENCIL_WRITE_MASK; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_STENCIL_OP; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_STENCIL_TEST_ENABLE; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_FRONT_FACE; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_CULL_MODE; + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_RASTERIZER_DISCARD_ENABLE; + if (screen->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints) + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_PATCH_CONTROL_POINTS_EXT; + + VkPipelineRasterizationLineStateCreateInfoEXT rast_line_state; + if (screen->info.have_EXT_line_rasterization) { + rast_line_state.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_LINE_STATE_CREATE_INFO_EXT; + rast_line_state.pNext = rast_state.pNext; + rast_line_state.stippledLineEnable = VK_FALSE; + rast_line_state.lineRasterizationMode = VK_LINE_RASTERIZATION_MODE_DEFAULT_EXT; + + bool check_warn = line; + if (prog->nir[PIPE_SHADER_TESS_EVAL]) { + check_warn |= !prog->nir[PIPE_SHADER_TESS_EVAL]->info.tess.point_mode && + prog->nir[PIPE_SHADER_TESS_EVAL]->info.tess._primitive_mode == TESS_PRIMITIVE_ISOLINES; + } + if (prog->nir[PIPE_SHADER_GEOMETRY]) { + switch (prog->nir[PIPE_SHADER_GEOMETRY]->info.gs.output_primitive) { + case SHADER_PRIM_LINES: + case SHADER_PRIM_LINE_LOOP: + case SHADER_PRIM_LINE_STRIP: + case SHADER_PRIM_LINES_ADJACENCY: + case SHADER_PRIM_LINE_STRIP_ADJACENCY: + check_warn = true; + break; + default: break; + } + } + + if (check_warn) { + const char *features[4][2] = { + [VK_LINE_RASTERIZATION_MODE_DEFAULT_EXT] = {"",""}, + [VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT] = {"rectangularLines", "stippledRectangularLines"}, + [VK_LINE_RASTERIZATION_MODE_BRESENHAM_EXT] = {"bresenhamLines", "stippledBresenhamLines"}, + [VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT] = {"smoothLines", "stippledSmoothLines"}, + }; + static bool warned[6] = {0}; + const VkPhysicalDeviceLineRasterizationFeaturesEXT *line_feats = &screen->info.line_rast_feats; + /* line features can be represented as an array VkBool32[6], + * with the 3 base features preceding the 3 (matching) stippled features + */ + const VkBool32 *feat = &line_feats->rectangularLines; + unsigned mode_idx = hw_rast_state->line_mode - VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT; + /* add base mode index, add 3 if stippling is enabled */ + mode_idx += hw_rast_state->line_stipple_enable * 3; + if (*(feat + mode_idx)) + rast_line_state.lineRasterizationMode = hw_rast_state->line_mode; + else + warn_missing_feature(warned[mode_idx], features[hw_rast_state->line_mode][hw_rast_state->line_stipple_enable]); + } + + if (hw_rast_state->line_stipple_enable) { + dynamicStateEnables[state_count++] = VK_DYNAMIC_STATE_LINE_STIPPLE_EXT; + rast_line_state.stippledLineEnable = VK_TRUE; + } + + rast_state.pNext = &rast_line_state; + } + + assert(state_count < ARRAY_SIZE(dynamicStateEnables)); + + VkPipelineDynamicStateCreateInfo pipelineDynamicStateCreateInfo = {0}; + pipelineDynamicStateCreateInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO; + pipelineDynamicStateCreateInfo.pDynamicStates = dynamicStateEnables; + pipelineDynamicStateCreateInfo.dynamicStateCount = state_count; + + VkGraphicsPipelineCreateInfo pci = {0}; + pci.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO; + pci.pNext = &gplci; + pci.flags = VK_PIPELINE_CREATE_LIBRARY_BIT_KHR; + pci.layout = prog->base.layout; + pci.pRasterizationState = &rast_state; + pci.pViewportState = &viewport_state; + pci.pDepthStencilState = &depth_stencil_state; + pci.pDynamicState = &pipelineDynamicStateCreateInfo; + + VkPipelineTessellationStateCreateInfo tci = {0}; + VkPipelineTessellationDomainOriginStateCreateInfo tdci = {0}; + if (prog->shaders[PIPE_SHADER_TESS_CTRL] && prog->shaders[PIPE_SHADER_TESS_EVAL]) { + tci.sType = VK_STRUCTURE_TYPE_PIPELINE_TESSELLATION_STATE_CREATE_INFO; + tci.patchControlPoints = 3; //this is a wild guess; pray for extendedDynamicState2PatchControlPoints + pci.pTessellationState = &tci; + tci.pNext = &tdci; + tdci.sType = VK_STRUCTURE_TYPE_PIPELINE_TESSELLATION_DOMAIN_ORIGIN_STATE_CREATE_INFO; + tdci.domainOrigin = VK_TESSELLATION_DOMAIN_ORIGIN_LOWER_LEFT; + } + + VkPipelineShaderStageCreateInfo shader_stages[ZINK_SHADER_COUNT]; + uint32_t num_stages = 0; + for (int i = 0; i < ZINK_SHADER_COUNT; ++i) { + if (!prog->modules[i]) + continue; + + VkPipelineShaderStageCreateInfo stage = {0}; + stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + stage.stage = zink_shader_stage(i); + stage.module = prog->modules[i]->shader; + stage.pName = "main"; + shader_stages[num_stages++] = stage; + } + assert(num_stages > 0); + + pci.pStages = shader_stages; + pci.stageCount = num_stages; + + VkPipeline pipeline; + if (VKSCR(CreateGraphicsPipelines)(screen->dev, prog->base.pipeline_cache, 1, &pci, + NULL, &pipeline) != VK_SUCCESS) { + mesa_loge("ZINK: vkCreateGraphicsPipelines failed"); + return VK_NULL_HANDLE; + } + + return pipeline; +} + +VkPipeline +zink_create_gfx_pipeline_combined(struct zink_screen *screen, struct zink_gfx_program *prog, VkPipeline input, VkPipeline library, VkPipeline output) +{ + VkPipeline libraries[] = {input, library, output}; + VkPipelineLibraryCreateInfoKHR libstate = {0}; + libstate.sType = VK_STRUCTURE_TYPE_PIPELINE_LIBRARY_CREATE_INFO_KHR; + libstate.libraryCount = 3; + libstate.pLibraries = libraries; + + VkGraphicsPipelineCreateInfo pci = {0}; + pci.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO; + pci.pNext = &libstate; + + VkPipeline pipeline; + if (VKSCR(CreateGraphicsPipelines)(screen->dev, prog->base.pipeline_cache, 1, &pci, + NULL, &pipeline) != VK_SUCCESS) { + mesa_loge("ZINK: vkCreateGraphicsPipelines failed"); + return VK_NULL_HANDLE; + } + + return pipeline; +} diff --git a/src/gallium/drivers/zink/zink_pipeline.h b/src/gallium/drivers/zink/zink_pipeline.h index 8fd869e329d..79f307cb48c 100644 --- a/src/gallium/drivers/zink/zink_pipeline.h +++ b/src/gallium/drivers/zink/zink_pipeline.h @@ -72,9 +72,7 @@ struct zink_gfx_pipeline_state { struct zink_pipeline_dynamic_state2 dyn_state2; -#if VK_USE_64_BIT_PTR_DEFINES uint32_t _pad; -#endif uint32_t gkey; //for pipeline library lookups VkShaderModule modules[PIPE_SHADER_TYPES - 1]; bool modules_changed; @@ -83,9 +81,7 @@ struct zink_gfx_pipeline_state { uint32_t final_hash; -#if VK_USE_64_BIT_PTR_DEFINES uint32_t _pad2; -#endif /* order matches zink_gfx_input_key */ union { struct { @@ -142,4 +138,17 @@ zink_create_gfx_pipeline(struct zink_screen *screen, VkPipeline zink_create_compute_pipeline(struct zink_screen *screen, struct zink_compute_program *comp, struct zink_compute_pipeline_state *state); + +VkPipeline +zink_create_gfx_pipeline_input(struct zink_screen *screen, + struct zink_gfx_pipeline_state *state, + const uint8_t *binding_map, + VkPrimitiveTopology primitive_topology); +VkPipeline +zink_create_gfx_pipeline_library(struct zink_screen *screen, struct zink_gfx_program *prog, + struct zink_rasterizer_hw_state *hw_rast_state, bool line); +VkPipeline +zink_create_gfx_pipeline_output(struct zink_screen *screen, struct zink_gfx_pipeline_state *state); +VkPipeline +zink_create_gfx_pipeline_combined(struct zink_screen *screen, struct zink_gfx_program *prog, VkPipeline input, VkPipeline library, VkPipeline output); #endif diff --git a/src/gallium/drivers/zink/zink_program.c b/src/gallium/drivers/zink/zink_program.c index b40d2accf01..0f1830097b5 100644 --- a/src/gallium/drivers/zink/zink_program.c +++ b/src/gallium/drivers/zink/zink_program.c @@ -224,6 +224,18 @@ update_gfx_shader_modules(struct zink_context *ctx, } } +static uint32_t +hash_pipeline_lib(const void *key) +{ + return _mesa_hash_data(key, sizeof(struct zink_rasterizer_hw_state)); +} + +static bool +equals_pipeline_lib(const void *a, const void *b) +{ + return !memcmp(a, b, offsetof(struct zink_gfx_library_key, pipeline)); +} + static uint32_t hash_gfx_pipeline_state(const void *key) { @@ -279,6 +291,60 @@ equals_gfx_pipeline_state(const void *a, const void *b) !memcmp(a, b, offsetof(struct zink_gfx_pipeline_state, hash)); } + +static uint32_t +hash_gfx_input_dynamic(const void *key) +{ + const struct zink_gfx_input_key *ikey = key; + return ikey->idx; +} + +static bool +equals_gfx_input_dynamic(const void *a, const void *b) +{ + const struct zink_gfx_input_key *ikey_a = a; + const struct zink_gfx_input_key *ikey_b = b; + return ikey_a->idx == ikey_b->idx; +} + +static uint32_t +hash_gfx_input(const void *key) +{ + const struct zink_gfx_input_key *ikey = key; + if (ikey->uses_dynamic_stride) + return ikey->input; + return _mesa_hash_data(key, offsetof(struct zink_gfx_input_key, pipeline)); +} + +static bool +equals_gfx_input(const void *a, const void *b) +{ + const struct zink_gfx_input_key *ikey_a = a; + const struct zink_gfx_input_key *ikey_b = b; + if (ikey_a->uses_dynamic_stride) + return ikey_a->element_state == ikey_b->element_state && + !memcmp(a, b, offsetof(struct zink_gfx_input_key, vertex_buffers_enabled_mask)); + return !memcmp(a, b, offsetof(struct zink_gfx_input_key, pipeline)); +} + +static uint32_t +hash_gfx_output(const void *key) +{ + const uint8_t *data = key; + const struct zink_gfx_output_key *okey = key; + /* manually add in force_persample_interp */ + return okey->force_persample_interp ^ + _mesa_hash_data(data + sizeof(uint16_t), offsetof(struct zink_gfx_output_key, pipeline) - sizeof(uint16_t)); +} + +static bool +equals_gfx_output(const void *a, const void *b) +{ + const uint8_t *da = a; + const uint8_t *db = b; + return !memcmp(da + sizeof(uint16_t), db + sizeof(uint16_t), offsetof(struct zink_gfx_output_key, pipeline) - sizeof(uint16_t)); +} + void zink_update_gfx_program(struct zink_context *ctx, struct zink_gfx_program *prog) { @@ -473,6 +539,9 @@ zink_create_gfx_program(struct zink_context *ctx, break; } + for (unsigned i = 0; i < ARRAY_SIZE(prog->libs); i++) + _mesa_set_init(&prog->libs[i], prog, hash_pipeline_lib, equals_pipeline_lib); + struct mesa_sha1 sctx; _mesa_sha1_init(&sctx); for (int i = 0; i < ZINK_SHADER_COUNT; ++i) { @@ -700,6 +769,13 @@ zink_destroy_gfx_program(struct zink_context *ctx, ralloc_free(prog->nir[i]); } + for (unsigned i = 0; i < ARRAY_SIZE(prog->libs); i++) { + set_foreach_remove(&prog->libs[i], he) { + struct zink_gfx_library_key *gkey = (void*)he->key; + VKSCR(DestroyPipeline)(screen->dev, gkey->pipeline, NULL); + } + } + unsigned max_idx = ARRAY_SIZE(prog->pipelines); if (screen->info.have_EXT_extended_dynamic_state) { /* only need first 3/4 for point/line/tri/patch */ @@ -772,6 +848,17 @@ get_primtype_idx(enum pipe_prim_type mode) } } +static void +create_pipeline_lib(struct zink_screen *screen, struct zink_gfx_program *prog, struct zink_gfx_pipeline_state *state, enum pipe_prim_type mode) +{ + struct zink_gfx_library_key *gkey = rzalloc(prog, struct zink_gfx_library_key); + gkey->hw_rast_state = state->rast_state; + memcpy(gkey->modules, state->modules, sizeof(gkey->modules)); + bool line = u_reduced_prim(mode) == PIPE_PRIM_LINES; + gkey->pipeline = zink_create_gfx_pipeline_library(screen, prog, (struct zink_rasterizer_hw_state*)state, line); + _mesa_set_add(&prog->libs[get_primtype_idx(mode)], gkey); +} + static unsigned get_pipeline_idx(bool have_EXT_extended_dynamic_state, enum pipe_prim_type mode, VkPrimitiveTopology vkmode) { @@ -786,6 +873,54 @@ get_pipeline_idx(bool have_EXT_extended_dynamic_state, enum pipe_prim_type mode, return vkmode; } +static struct zink_gfx_input_key * +find_or_create_input_dynamic(struct zink_context *ctx, VkPrimitiveTopology vkmode) +{ + uint32_t hash = hash_gfx_input_dynamic(&ctx->gfx_pipeline_state.input); + struct set_entry *he = _mesa_set_search_pre_hashed(&ctx->gfx_inputs, hash, &ctx->gfx_pipeline_state.input); + if (!he) { + struct zink_gfx_input_key *ikey = rzalloc(ctx, struct zink_gfx_input_key); + ikey->idx = ctx->gfx_pipeline_state.idx; + ikey->pipeline = zink_create_gfx_pipeline_input(zink_screen(ctx->base.screen), &ctx->gfx_pipeline_state, ctx->element_state->binding_map, vkmode); + he = _mesa_set_add_pre_hashed(&ctx->gfx_inputs, hash, ikey); + } + return (void*)he->key; +} + +static struct zink_gfx_input_key * +find_or_create_input(struct zink_context *ctx, VkPrimitiveTopology vkmode) +{ + uint32_t hash = hash_gfx_input(&ctx->gfx_pipeline_state.input); + struct set_entry *he = _mesa_set_search_pre_hashed(&ctx->gfx_inputs, hash, &ctx->gfx_pipeline_state.input); + if (!he) { + struct zink_gfx_input_key *ikey = rzalloc(ctx, struct zink_gfx_input_key); + if (ctx->gfx_pipeline_state.uses_dynamic_stride) { + memcpy(ikey, &ctx->gfx_pipeline_state.input, offsetof(struct zink_gfx_input_key, vertex_buffers_enabled_mask)); + ikey->element_state = ctx->gfx_pipeline_state.element_state; + } else { + memcpy(ikey, &ctx->gfx_pipeline_state.input, offsetof(struct zink_gfx_input_key, pipeline)); + } + ikey->pipeline = zink_create_gfx_pipeline_input(zink_screen(ctx->base.screen), &ctx->gfx_pipeline_state, ctx->element_state->binding_map, vkmode); + he = _mesa_set_add_pre_hashed(&ctx->gfx_inputs, hash, ikey); + } + return (void*)he->key; +} + +static struct zink_gfx_output_key * +find_or_create_output(struct zink_context *ctx) +{ + uint32_t hash = hash_gfx_output(&ctx->gfx_pipeline_state); + struct set_entry *he = _mesa_set_search_pre_hashed(&ctx->gfx_outputs, hash, &ctx->gfx_pipeline_state); + if (!he) { + struct zink_gfx_output_key *okey = rzalloc(ctx, struct zink_gfx_output_key); + memcpy(okey, &ctx->gfx_pipeline_state, offsetof(struct zink_gfx_output_key, pipeline)); + okey->_pad = 0; + okey->pipeline = zink_create_gfx_pipeline_output(zink_screen(ctx->base.screen), &ctx->gfx_pipeline_state); + he = _mesa_set_add_pre_hashed(&ctx->gfx_outputs, hash, okey); + } + return (void*)he->key; +} + /* VUID-vkCmdBindVertexBuffers2-pStrides-06209 If pStrides is not NULL each element of pStrides must be either 0 or greater than or equal @@ -868,9 +1003,33 @@ zink_get_gfx_pipeline(struct zink_context *ctx, if (!entry) { util_queue_fence_wait(&prog->base.cache_fence); - VkPipeline pipeline = zink_create_gfx_pipeline(screen, prog, state, - ctx->element_state->binding_map, - vkmode); + VkPipeline pipeline = VK_NULL_HANDLE; + if (screen->info.have_EXT_graphics_pipeline_library && + /* TODO: if there's ever a dynamic render extension with input attachments */ + !ctx->gfx_pipeline_state.render_pass) { + ctx->gfx_pipeline_state.gkey = ctx->gfx_pipeline_state.rast_state; + struct set_entry *he = NULL; + /* TODO: this will eventually be pre-populated by async shader compile */ + //struct set_entry *he = _mesa_set_search(&prog->libs[idx], &ctx->gfx_pipeline_state.gkey); + if (!he && screen->driver_workarounds.force_pipeline_library) { + create_pipeline_lib(screen, prog, &ctx->gfx_pipeline_state, mode); + he = _mesa_set_search(&prog->libs[idx], &ctx->gfx_pipeline_state.gkey); + assert(he); + } + if (he) { + struct zink_gfx_library_key *gkey = (void*)he->key; + struct zink_gfx_input_key *ikey = have_EXT_vertex_input_dynamic_state ? + find_or_create_input_dynamic(ctx, vkmode) : + find_or_create_input(ctx, vkmode); + struct zink_gfx_output_key *okey = find_or_create_output(ctx); + pipeline = zink_create_gfx_pipeline_combined(screen, prog, ikey->pipeline, gkey->pipeline, okey->pipeline); + } + } + if (!pipeline) { + pipeline = zink_create_gfx_pipeline(screen, prog, state, + ctx->element_state->binding_map, + vkmode); + } if (pipeline == VK_NULL_HANDLE) return VK_NULL_HANDLE; @@ -1209,6 +1368,22 @@ zink_program_init(struct zink_context *ctx) ctx->base.create_compute_state = zink_create_cs_state; ctx->base.bind_compute_state = zink_bind_cs_state; ctx->base.delete_compute_state = zink_delete_shader_state; + + if (zink_screen(ctx->base.screen)->info.have_EXT_vertex_input_dynamic_state) + _mesa_set_init(&ctx->gfx_inputs, ctx, hash_gfx_input_dynamic, equals_gfx_input_dynamic); + else + _mesa_set_init(&ctx->gfx_inputs, ctx, hash_gfx_input, equals_gfx_input); + _mesa_set_init(&ctx->gfx_outputs, ctx, hash_gfx_output, equals_gfx_output); + /* validate struct packing */ + STATIC_ASSERT(offsetof(struct zink_gfx_pipeline_state, vertex_buffers_enabled_mask) - offsetof(struct zink_gfx_pipeline_state, input) == + offsetof(struct zink_gfx_input_key, vertex_buffers_enabled_mask) - offsetof(struct zink_gfx_input_key, input)); + STATIC_ASSERT(offsetof(struct zink_gfx_pipeline_state, vertex_strides) - offsetof(struct zink_gfx_pipeline_state, input) == + offsetof(struct zink_gfx_input_key, vertex_strides) - offsetof(struct zink_gfx_input_key, input)); + STATIC_ASSERT(offsetof(struct zink_gfx_pipeline_state, element_state) - offsetof(struct zink_gfx_pipeline_state, input) == + offsetof(struct zink_gfx_input_key, element_state) - offsetof(struct zink_gfx_input_key, input)); + + STATIC_ASSERT(offsetof(struct zink_gfx_pipeline_state, modules) - offsetof(struct zink_gfx_pipeline_state, gkey) == + offsetof(struct zink_gfx_library_key, modules) - offsetof(struct zink_gfx_library_key, hw_rast_state)); } bool diff --git a/src/gallium/drivers/zink/zink_program.h b/src/gallium/drivers/zink/zink_program.h index ab2e72288ee..1ad2b51c035 100644 --- a/src/gallium/drivers/zink/zink_program.h +++ b/src/gallium/drivers/zink/zink_program.h @@ -95,6 +95,38 @@ struct zink_program { #define ZINK_MAX_INLINED_VARIANTS 5 +struct zink_gfx_library_key { + uint32_t hw_rast_state; + VkShaderModule modules[ZINK_SHADER_COUNT]; + VkPipeline pipeline; +}; + +struct zink_gfx_input_key { + union { + struct { + unsigned idx:8; + bool uses_dynamic_stride; + }; + uint32_t input; + }; + uint32_t vertex_buffers_enabled_mask; + uint32_t vertex_strides[PIPE_MAX_ATTRIBS]; + struct zink_vertex_elements_hw_state *element_state; + VkPipeline pipeline; +}; + +struct zink_gfx_output_key { + uint32_t _pad:15; + uint32_t force_persample_interp:1; + uint32_t rast_samples:8; + uint32_t void_alpha_attachments:PIPE_MAX_COLOR_BUFS; + VkSampleMask sample_mask; + + unsigned rp_state; + uint32_t blend_id; + VkPipeline pipeline; +}; + struct zink_gfx_program { struct zink_program base; @@ -112,6 +144,8 @@ struct zink_gfx_program { struct hash_table pipelines[11]; // number of draw modes we support uint32_t default_variant_hash; uint32_t last_variant_hash; + + struct set libs[4]; //zink_gfx_library_key[primtype] -> VkPipeline }; struct zink_compute_program { diff --git a/src/gallium/drivers/zink/zink_screen.c b/src/gallium/drivers/zink/zink_screen.c index 037d0b318b6..f073de687b5 100644 --- a/src/gallium/drivers/zink/zink_screen.c +++ b/src/gallium/drivers/zink/zink_screen.c @@ -2083,6 +2083,7 @@ static void init_driver_workarounds(struct zink_screen *screen) { /* enable implicit sync for all non-mesa drivers */ + screen->driver_workarounds.force_pipeline_library = debug_get_bool_option("ZINK_PIPELINE_LIBRARY_FORCE", false); screen->driver_workarounds.implicit_sync = true; switch (screen->info.driver_props.driverID) { case VK_DRIVER_ID_MESA_RADV: @@ -2097,11 +2098,18 @@ init_driver_workarounds(struct zink_screen *screen) default: break; } - + if (screen->info.have_EXT_graphics_pipeline_library) + screen->info.have_EXT_graphics_pipeline_library = screen->info.have_EXT_extended_dynamic_state && + screen->info.have_EXT_extended_dynamic_state2 && + screen->info.have_KHR_dynamic_rendering && + (screen->info.gpl_props.graphicsPipelineLibraryFastLinking || + screen->is_cpu || + screen->driver_workarounds.force_pipeline_library); + if (!screen->driver_workarounds.force_pipeline_library) + screen->info.have_EXT_graphics_pipeline_library = false; screen->driver_workarounds.color_write_missing = !screen->info.have_EXT_color_write_enable || !screen->info.cwrite_feats.colorWriteEnable; - screen->driver_workarounds.depth_clip_control_missing = !screen->info.have_EXT_depth_clip_control; if (screen->info.driver_props.driverID == VK_DRIVER_ID_AMD_PROPRIETARY) /* this completely breaks xfb somehow */ diff --git a/src/gallium/drivers/zink/zink_screen.h b/src/gallium/drivers/zink/zink_screen.h index 1f2a279054f..789f054fdf3 100644 --- a/src/gallium/drivers/zink/zink_screen.h +++ b/src/gallium/drivers/zink/zink_screen.h @@ -210,6 +210,7 @@ struct zink_screen { bool color_write_missing; bool depth_clip_control_missing; bool implicit_sync; + bool force_pipeline_library; } driver_workarounds; };
