Module: Mesa Branch: main Commit: 10c2180a929f4a06bd81693f86bdc11050536368 URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=10c2180a929f4a06bd81693f86bdc11050536368
Author: Mike Blumenkrantz <[email protected]> Date: Mon Oct 16 15:45:41 2023 -0400 zink: add automatic swapchain readback using heuristics in cases where apps (stupidly) do swapbuffers->blitframebuffer, there's no functional way to (legitimately) perform readback on the just-presented vk image, which leads to the existing acquire+present loop dance this adds a counter threshold which, when exceeded, begins copying the scanout image for swapchains to provide local readback on images without the massive perf penalty of roundtrips Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/25754> --- src/gallium/drivers/zink/zink_blit.c | 27 ++++++++++------- src/gallium/drivers/zink/zink_context.c | 18 +++++++----- src/gallium/drivers/zink/zink_kopper.c | 51 +++++++++++++++++++++++++++++++-- src/gallium/drivers/zink/zink_kopper.h | 9 +++++- 4 files changed, 84 insertions(+), 21 deletions(-) diff --git a/src/gallium/drivers/zink/zink_blit.c b/src/gallium/drivers/zink/zink_blit.c index 85c18411f07..1fe9a27459f 100644 --- a/src/gallium/drivers/zink/zink_blit.c +++ b/src/gallium/drivers/zink/zink_blit.c @@ -52,6 +52,7 @@ blit_resolve(struct zink_context *ctx, const struct pipe_blit_info *info, bool * return false; struct zink_resource *src = zink_resource(info->src.resource); + struct zink_resource *use_src = src; struct zink_resource *dst = zink_resource(info->dst.resource); struct zink_screen *screen = zink_screen(ctx->base.screen); @@ -67,16 +68,16 @@ blit_resolve(struct zink_context *ctx, const struct pipe_blit_info *info, bool * zink_fb_clears_apply_region(ctx, info->src.resource, zink_rect_from_box(&info->src.box)); if (src->obj->dt) - *needs_present_readback = zink_kopper_acquire_readback(ctx, src); + *needs_present_readback = zink_kopper_acquire_readback(ctx, src, &use_src); struct zink_batch *batch = &ctx->batch; - zink_resource_setup_transfer_layouts(ctx, src, dst); + zink_resource_setup_transfer_layouts(ctx, use_src, dst); VkCommandBuffer cmdbuf = *needs_present_readback ? ctx->batch.state->cmdbuf : zink_get_cmdbuf(ctx, src, dst); if (cmdbuf == ctx->batch.state->cmdbuf) zink_flush_dgc_if_enabled(ctx); - zink_batch_reference_resource_rw(batch, src, false); + zink_batch_reference_resource_rw(batch, use_src, false); zink_batch_reference_resource_rw(batch, dst, true); bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "blit_resolve(%s->%s, %dx%d->%dx%d)", @@ -121,7 +122,7 @@ blit_resolve(struct zink_context *ctx, const struct pipe_blit_info *info, bool * region.extent.width = info->dst.box.width; region.extent.height = info->dst.box.height; region.extent.depth = info->dst.box.depth; - VKCTX(CmdResolveImage)(cmdbuf, src->obj->image, src->layout, + VKCTX(CmdResolveImage)(cmdbuf, use_src->obj->image, src->layout, dst->obj->image, dst->layout, 1, ®ion); zink_cmd_debug_marker_end(ctx, cmdbuf, marker); @@ -151,6 +152,7 @@ blit_native(struct zink_context *ctx, const struct pipe_blit_info *info, bool *n return false; struct zink_resource *src = zink_resource(info->src.resource); + struct zink_resource *use_src = src; struct zink_resource *dst = zink_resource(info->dst.resource); struct zink_screen *screen = zink_screen(ctx->base.screen); @@ -261,16 +263,16 @@ blit_native(struct zink_context *ctx, const struct pipe_blit_info *info, bool *n zink_fb_clears_apply_region(ctx, info->src.resource, zink_rect_from_box(&info->src.box)); if (src->obj->dt) - *needs_present_readback = zink_kopper_acquire_readback(ctx, src); + *needs_present_readback = zink_kopper_acquire_readback(ctx, src, &use_src); struct zink_batch *batch = &ctx->batch; - zink_resource_setup_transfer_layouts(ctx, src, dst); + zink_resource_setup_transfer_layouts(ctx, use_src, dst); VkCommandBuffer cmdbuf = *needs_present_readback ? ctx->batch.state->cmdbuf : zink_get_cmdbuf(ctx, src, dst); if (cmdbuf == ctx->batch.state->cmdbuf) zink_flush_dgc_if_enabled(ctx); - zink_batch_reference_resource_rw(batch, src, false); + zink_batch_reference_resource_rw(batch, use_src, false); zink_batch_reference_resource_rw(batch, dst, true); bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "blit_native(%s->%s, %dx%d->%dx%d)", @@ -279,7 +281,7 @@ blit_native(struct zink_context *ctx, const struct pipe_blit_info *info, bool *n info->src.box.width, info->src.box.height, info->dst.box.width, info->dst.box.height); - VKCTX(CmdBlitImage)(cmdbuf, src->obj->image, src->layout, + VKCTX(CmdBlitImage)(cmdbuf, use_src->obj->image, src->layout, dst->obj->image, dst->layout, 1, ®ion, zink_filter(info->filter)); @@ -317,6 +319,7 @@ zink_blit(struct pipe_context *pctx, const struct util_format_description *dst_desc = util_format_description(info->dst.format); struct zink_resource *src = zink_resource(info->src.resource); + struct zink_resource *use_src = src; struct zink_resource *dst = zink_resource(info->dst.resource); bool needs_present_readback = false; if (zink_is_swapchain(dst)) { @@ -365,7 +368,7 @@ zink_blit(struct pipe_context *pctx, if (src->obj->dt) { zink_fb_clears_apply_region(ctx, info->src.resource, zink_rect_from_box(&info->src.box)); - needs_present_readback = zink_kopper_acquire_readback(ctx, src); + needs_present_readback = zink_kopper_acquire_readback(ctx, src, &use_src); } /* this is discard_only because we're about to start a renderpass that will @@ -426,7 +429,7 @@ zink_blit(struct pipe_context *pctx, zink_resource_object_init_mutable(ctx, src); if (zink_format_needs_mutable(info->dst.format, info->dst.resource->format)) zink_resource_object_init_mutable(ctx, dst); - zink_blit_barriers(ctx, src, dst, whole); + zink_blit_barriers(ctx, use_src, dst, whole); ctx->blitting = true; if (stencil_blit) { @@ -449,7 +452,9 @@ zink_blit(struct pipe_context *pctx, pipe_surface_release(pctx, &dst_view); } else { - util_blitter_blit(ctx->blitter, info); + struct pipe_blit_info new_info = *info; + new_info.src.resource = &use_src->base.b; + util_blitter_blit(ctx->blitter, &new_info); } ctx->blitting = false; ctx->rp_clears_enabled = rp_clears_enabled; diff --git a/src/gallium/drivers/zink/zink_context.c b/src/gallium/drivers/zink/zink_context.c index 82db90a911e..e83b94875e0 100644 --- a/src/gallium/drivers/zink/zink_context.c +++ b/src/gallium/drivers/zink/zink_context.c @@ -3777,8 +3777,10 @@ zink_flush(struct pipe_context *pctx, p_atomic_inc(&screen->renderdoc_frame); #endif if (ctx->needs_present && ctx->needs_present->obj->image && - zink_is_swapchain(ctx->needs_present)) + zink_is_swapchain(ctx->needs_present)) { + zink_kopper_readback_update(ctx, ctx->needs_present); screen->image_barrier(ctx, ctx->needs_present, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, 0, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT); + } ctx->needs_present = NULL; } @@ -4062,6 +4064,7 @@ zink_flush_resource(struct pipe_context *pctx, if (res->obj->dt) { if (zink_kopper_acquired(res->obj->dt, res->obj->dt_idx)) { zink_batch_no_rp_safe(ctx); + zink_kopper_readback_update(ctx, res); zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, 0, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT); zink_batch_reference_resource_rw(&ctx->batch, res, true); } else { @@ -4447,6 +4450,7 @@ zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, stru unsigned src_level, const struct pipe_box *src_box, enum pipe_map_flags map_flags) { struct zink_resource *img = dst->base.b.target == PIPE_BUFFER ? src : dst; + struct zink_resource *use_img = img; struct zink_resource *buf = dst->base.b.target == PIPE_BUFFER ? dst : src; struct zink_batch *batch = &ctx->batch; bool needs_present_readback = false; @@ -4466,8 +4470,8 @@ zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, stru zink_screen(ctx->base.screen)->buffer_barrier(ctx, buf, VK_ACCESS_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); } else { if (zink_is_swapchain(img)) - needs_present_readback = zink_kopper_acquire_readback(ctx, img); - zink_screen(ctx->base.screen)->image_barrier(ctx, img, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, 0, 0); + needs_present_readback = zink_kopper_acquire_readback(ctx, img, &use_img); + zink_screen(ctx->base.screen)->image_barrier(ctx, use_img, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, 0, 0); zink_resource_buffer_transfer_dst_barrier(ctx, buf, dstx, src_box->width); } @@ -4513,8 +4517,8 @@ zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, stru /* never promote to unordered if swapchain was acquired */ VkCommandBuffer cmdbuf = needs_present_readback ? ctx->batch.state->cmdbuf : - buf2img ? zink_get_cmdbuf(ctx, buf, img) : zink_get_cmdbuf(ctx, img, buf); - zink_batch_reference_resource_rw(batch, img, buf2img); + buf2img ? zink_get_cmdbuf(ctx, buf, use_img) : zink_get_cmdbuf(ctx, use_img, buf); + zink_batch_reference_resource_rw(batch, use_img, buf2img); zink_batch_reference_resource_rw(batch, buf, !buf2img); /* we're using u_transfer_helper_deinterleave, which means we'll be getting PIPE_MAP_* usage @@ -4563,14 +4567,14 @@ zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, stru region.imageExtent.width, region.imageExtent.height, MAX2(region.imageSubresource.layerCount, region.imageExtent.depth)); - VKCTX(CmdCopyBufferToImage)(cmdbuf, buf->obj->buffer, img->obj->image, img->layout, 1, ®ion); + VKCTX(CmdCopyBufferToImage)(cmdbuf, buf->obj->buffer, use_img->obj->image, use_img->layout, 1, ®ion); } else { marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "copy_image2buffer(%s, %dx%dx%d)", util_format_short_name(src->base.b.format), region.imageExtent.width, region.imageExtent.height, MAX2(region.imageSubresource.layerCount, region.imageExtent.depth)); - VKCTX(CmdCopyImageToBuffer)(cmdbuf, img->obj->image, img->layout, buf->obj->buffer, 1, ®ion); + VKCTX(CmdCopyImageToBuffer)(cmdbuf, use_img->obj->image, use_img->layout, buf->obj->buffer, 1, ®ion); } zink_cmd_debug_marker_end(ctx, cmdbuf, marker); } diff --git a/src/gallium/drivers/zink/zink_kopper.c b/src/gallium/drivers/zink/zink_kopper.c index 115737463c6..d1e9a57b20f 100644 --- a/src/gallium/drivers/zink/zink_kopper.c +++ b/src/gallium/drivers/zink/zink_kopper.c @@ -151,6 +151,7 @@ destroy_swapchain(struct zink_screen *screen, struct kopper_swapchain *cswap) simple_mtx_lock(&screen->semaphores_lock); util_dynarray_append(&screen->semaphores, VkSemaphore, cswap->images[i].acquire); simple_mtx_unlock(&screen->semaphores_lock); + pipe_resource_reference(&cswap->images[i].readback, NULL); } free(cswap->images); hash_table_foreach(cswap->presents, he) { @@ -555,6 +556,8 @@ kopper_acquire(struct zink_screen *screen, struct zink_resource *res, uint64_t t } cdt->swapchain->images[res->obj->dt_idx].acquire = acquire; + if (cdt->swapchain->images[res->obj->dt_idx].readback) + zink_resource(cdt->swapchain->images[res->obj->dt_idx].readback)->valid = false; res->obj->image = cdt->swapchain->images[res->obj->dt_idx].image; cdt->swapchain->images[res->obj->dt_idx].acquired = false; if (!cdt->swapchain->images[res->obj->dt_idx].init) { @@ -827,8 +830,23 @@ zink_kopper_present_queue(struct zink_screen *screen, struct zink_resource *res) res->obj->dt_idx = UINT32_MAX; } +static void +kopper_ensure_readback(struct zink_screen *screen, struct zink_resource *res) +{ + struct kopper_displaytarget *cdt = res->obj->dt; + struct kopper_swapchain *cswap = cdt->swapchain; + + for (unsigned i = 0; i < cswap->num_images; i++) { + if (cswap->images[i].readback) + return; + struct pipe_resource templ = res->base.b; + templ.bind = PIPE_BIND_RENDER_TARGET | PIPE_BIND_SAMPLER_VIEW; + cswap->images[i].readback = screen->base.resource_create(&screen->base, &templ); + } +} + bool -zink_kopper_acquire_readback(struct zink_context *ctx, struct zink_resource *res) +zink_kopper_acquire_readback(struct zink_context *ctx, struct zink_resource *res, struct zink_resource **readback) { struct zink_screen *screen = zink_screen(ctx->base.screen); assert(res->obj->dt); @@ -836,10 +854,22 @@ zink_kopper_acquire_readback(struct zink_context *ctx, struct zink_resource *res const struct kopper_swapchain *cswap = cdt->swapchain; uint32_t last_dt_idx = res->obj->last_dt_idx; VkResult ret = VK_SUCCESS; + /* if this hasn't been presented or if it has data, use this as the readback target */ if (res->obj->last_dt_idx == UINT32_MAX || - (zink_kopper_acquired(cdt, res->obj->dt_idx) && cdt->swapchain->images[res->obj->dt_idx].dt_has_data)) + (zink_kopper_acquired(cdt, res->obj->dt_idx) && cdt->swapchain->images[res->obj->dt_idx].dt_has_data)) { + *readback = res; return false; + } + if (cswap->images[last_dt_idx].readback) { + struct zink_resource *rb = zink_resource(cswap->images[res->obj->last_dt_idx].readback); + if (rb->valid) { + *readback = rb; + return false; + } + } + if (++cdt->readback_counter >= ZINK_READBACK_THRESHOLD) + kopper_ensure_readback(screen, res); while (res->obj->dt_idx != last_dt_idx) { if (res->obj->dt_idx != UINT32_MAX && !zink_kopper_present_readback(ctx, res)) break; @@ -848,6 +878,7 @@ zink_kopper_acquire_readback(struct zink_context *ctx, struct zink_resource *res } while (!is_swapchain_kill(ret) && (ret == VK_NOT_READY || ret == VK_TIMEOUT)); if (is_swapchain_kill(ret)) { kill_swapchain(ctx, res); + *readback = NULL; return false; } } @@ -857,6 +888,7 @@ zink_kopper_acquire_readback(struct zink_context *ctx, struct zink_resource *res res->base.b.height0 = ctx->swapchain_size.height; } zink_batch_usage_set(&cdt->swapchain->batch_uses, ctx->batch.state); + *readback = res; return true; } @@ -865,6 +897,7 @@ zink_kopper_present_readback(struct zink_context *ctx, struct zink_resource *res { struct zink_screen *screen = zink_screen(ctx->base.screen); VkSubmitInfo si = {0}; + assert(zink_is_swapchain(res)); if (res->obj->last_dt_idx == UINT32_MAX) return true; if (res->layout != VK_IMAGE_LAYOUT_PRESENT_SRC_KHR) { @@ -904,6 +937,20 @@ zink_kopper_present_readback(struct zink_context *ctx, struct zink_resource *res return zink_screen_handle_vkresult(screen, error); } +void +zink_kopper_readback_update(struct zink_context *ctx, struct zink_resource *res) +{ + assert(res->obj->dt); + struct kopper_displaytarget *cdt = res->obj->dt; + struct kopper_swapchain *cswap = cdt->swapchain; + assert(res->obj->dt_idx != UINT32_MAX); + struct pipe_resource *readback = cswap->images[res->obj->dt_idx].readback; + struct pipe_box box = {0, 0, 0, res->base.b.width0, res->base.b.height0, res->base.b.depth0}; + + if (readback) + ctx->base.resource_copy_region(&ctx->base, readback, 0, 0, 0, 0, &res->base.b, 0, &box); +} + bool zink_kopper_update(struct pipe_screen *pscreen, struct pipe_resource *pres, int *w, int *h) { diff --git a/src/gallium/drivers/zink/zink_kopper.h b/src/gallium/drivers/zink/zink_kopper.h index f1062b657a5..f930f282e2e 100644 --- a/src/gallium/drivers/zink/zink_kopper.h +++ b/src/gallium/drivers/zink/zink_kopper.h @@ -36,12 +36,16 @@ extern "C" { struct zink_batch_usage; +/* number of times a swapchain can be read without forcing readback mode */ +#define ZINK_READBACK_THRESHOLD 3 + struct kopper_swapchain_image { bool init; bool acquired; bool dt_has_data; int age; VkImage image; + struct pipe_resource *readback; VkSemaphore acquire; VkImageLayout layout; }; @@ -90,6 +94,7 @@ struct kopper_displaytarget enum kopper_type type; bool is_kill; VkPresentModeKHR present_mode; + unsigned readback_counter; }; struct zink_context; @@ -132,10 +137,12 @@ zink_kopper_present(struct zink_screen *screen, struct zink_resource *res); void zink_kopper_present_queue(struct zink_screen *screen, struct zink_resource *res); bool -zink_kopper_acquire_readback(struct zink_context *ctx, struct zink_resource *res); +zink_kopper_acquire_readback(struct zink_context *ctx, struct zink_resource *res, struct zink_resource **readback); bool zink_kopper_present_readback(struct zink_context *ctx, struct zink_resource *res); void +zink_kopper_readback_update(struct zink_context *ctx, struct zink_resource *res); +void zink_kopper_deinit_displaytarget(struct zink_screen *screen, struct kopper_displaytarget *cdt); bool zink_kopper_update(struct pipe_screen *pscreen, struct pipe_resource *pres, int *w, int *h);
