Module: Mesa Branch: main Commit: ad9c61c2925627520b0b54c4b12d6cdcc6ed1839 URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=ad9c61c2925627520b0b54c4b12d6cdcc6ed1839
Author: Karmjit Mahil <[email protected]> Date: Tue Nov 22 16:28:29 2022 +0000 pvr: Add SPM scratch buffer infrastructure. Signed-off-by: Karmjit Mahil <[email protected]> Reviewed-by: Frank Binns <[email protected]> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/21102> --- src/imagination/vulkan/meson.build | 1 + src/imagination/vulkan/pvr_device.c | 3 + src/imagination/vulkan/pvr_private.h | 3 + src/imagination/vulkan/pvr_spm.c | 206 +++++++++++++++++++++++++++++++++++ src/imagination/vulkan/pvr_spm.h | 73 +++++++++++++ 5 files changed, 286 insertions(+) diff --git a/src/imagination/vulkan/meson.build b/src/imagination/vulkan/meson.build index e69e844a091..ba4538239fe 100644 --- a/src/imagination/vulkan/meson.build +++ b/src/imagination/vulkan/meson.build @@ -61,6 +61,7 @@ pvr_files = files( 'pvr_query_compute.c', 'pvr_queue.c', 'pvr_shader.c', + 'pvr_spm.c', 'pvr_tex_state.c', 'pvr_wsi.c', ) diff --git a/src/imagination/vulkan/pvr_device.c b/src/imagination/vulkan/pvr_device.c index d7bb421ecb8..ab88b2ed448 100644 --- a/src/imagination/vulkan/pvr_device.c +++ b/src/imagination/vulkan/pvr_device.c @@ -1777,6 +1777,8 @@ VkResult pvr_CreateDevice(VkPhysicalDevice physicalDevice, pvr_device_init_default_sampler_state(device); + pvr_spm_init_scratch_buffer_store(device); + if (pCreateInfo->pEnabledFeatures) memcpy(&device->features, pCreateInfo->pEnabledFeatures, @@ -1846,6 +1848,7 @@ void pvr_DestroyDevice(VkDevice _device, { PVR_FROM_HANDLE(pvr_device, device, _device); + pvr_spm_finish_scratch_buffer_store(device); pvr_queues_destroy(device); pvr_device_finish_tile_buffer_state(device); pvr_device_finish_graphics_static_clear_state(device); diff --git a/src/imagination/vulkan/pvr_private.h b/src/imagination/vulkan/pvr_private.h index b8b345fc77f..592b92906a2 100644 --- a/src/imagination/vulkan/pvr_private.h +++ b/src/imagination/vulkan/pvr_private.h @@ -47,6 +47,7 @@ #include "pvr_limits.h" #include "pvr_pds.h" #include "pvr_shader_factory.h" +#include "pvr_spm.h" #include "pvr_types.h" #include "pvr_winsys.h" #include "rogue/rogue.h" @@ -373,6 +374,8 @@ struct pvr_device { uint32_t buffer_count; } tile_buffer_state; + struct pvr_spm_scratch_buffer_store spm_scratch_buffer_store; + VkPhysicalDeviceFeatures features; struct pvr_bo_store *bo_store; diff --git a/src/imagination/vulkan/pvr_spm.c b/src/imagination/vulkan/pvr_spm.c new file mode 100644 index 00000000000..2bc6aeab14f --- /dev/null +++ b/src/imagination/vulkan/pvr_spm.c @@ -0,0 +1,206 @@ +/* + * Copyright © 2023 Imagination Technologies Ltd. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <stdint.h> +#include <stddef.h> +#include <vulkan/vulkan_core.h> + +#include "c11/threads.h" +#include "hwdef/rogue_hw_utils.h" +#include "pvr_bo.h" +#include "pvr_private.h" +#include "pvr_spm.h" +#include "util/simple_mtx.h" +#include "util/u_atomic.h" +#include "vk_alloc.h" +#include "vk_log.h" + +struct pvr_spm_scratch_buffer { + uint32_t ref_count; + struct pvr_bo *bo; + uint64_t size; +}; + +void pvr_spm_init_scratch_buffer_store(struct pvr_device *device) +{ + struct pvr_spm_scratch_buffer_store *store = + &device->spm_scratch_buffer_store; + + simple_mtx_init(&store->mtx, mtx_plain); + store->head_ref = NULL; +} + +void pvr_spm_finish_scratch_buffer_store(struct pvr_device *device) +{ + struct pvr_spm_scratch_buffer_store *store = + &device->spm_scratch_buffer_store; + + /* Either a framebuffer was never created so no scratch buffer was ever + * created or all framebuffers have been freed so only the store's reference + * remains. + */ + assert(!store->head_ref || p_atomic_read(&store->head_ref->ref_count) == 1); + + simple_mtx_destroy(&store->mtx); + + if (store->head_ref) { + pvr_bo_free(device, store->head_ref->bo); + vk_free(&device->vk.alloc, store->head_ref); + } +} + +static VkResult +pvr_spm_scratch_buffer_alloc(struct pvr_device *device, + uint64_t size, + struct pvr_spm_scratch_buffer **const buffer_out) +{ + const uint32_t cache_line_size = + rogue_get_slc_cache_line_size(&device->pdevice->dev_info); + struct pvr_spm_scratch_buffer *scratch_buffer; + struct pvr_bo *bo; + VkResult result; + + result = pvr_bo_alloc(device, + device->heaps.general_heap, + size, + cache_line_size, + 0, + &bo); + if (result != VK_SUCCESS) { + *buffer_out = NULL; + return result; + } + + scratch_buffer = vk_alloc(&device->vk.alloc, + sizeof(*scratch_buffer), + 4, + VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + if (!scratch_buffer) { + pvr_bo_free(device, bo); + *buffer_out = NULL; + return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + } + + *scratch_buffer = (struct pvr_spm_scratch_buffer){ + .bo = bo, + .size = size, + }; + + *buffer_out = scratch_buffer; + + return VK_SUCCESS; +} + +static void +pvr_spm_scratch_buffer_release_locked(struct pvr_device *device, + struct pvr_spm_scratch_buffer *buffer) +{ + struct pvr_spm_scratch_buffer_store *store = + &device->spm_scratch_buffer_store; + + simple_mtx_assert_locked(&store->mtx); + + if (p_atomic_dec_zero(&buffer->ref_count)) { + pvr_bo_free(device, buffer->bo); + vk_free(&device->vk.alloc, buffer); + } +} + +void pvr_spm_scratch_buffer_release(struct pvr_device *device, + struct pvr_spm_scratch_buffer *buffer) +{ + struct pvr_spm_scratch_buffer_store *store = + &device->spm_scratch_buffer_store; + + simple_mtx_lock(&store->mtx); + + pvr_spm_scratch_buffer_release_locked(device, buffer); + + simple_mtx_unlock(&store->mtx); +} + +static void pvr_spm_scratch_buffer_store_set_head_ref_locked( + struct pvr_spm_scratch_buffer_store *store, + struct pvr_spm_scratch_buffer *buffer) +{ + simple_mtx_assert_locked(&store->mtx); + assert(!store->head_ref); + + p_atomic_inc(&buffer->ref_count); + store->head_ref = buffer; +} + +static void pvr_spm_scratch_buffer_store_release_head_ref_locked( + struct pvr_device *device, + struct pvr_spm_scratch_buffer_store *store) +{ + simple_mtx_assert_locked(&store->mtx); + + pvr_spm_scratch_buffer_release_locked(device, store->head_ref); + + store->head_ref = NULL; +} + +VkResult pvr_spm_scratch_buffer_get_buffer( + struct pvr_device *device, + uint64_t size, + struct pvr_spm_scratch_buffer **const buffer_out) +{ + struct pvr_spm_scratch_buffer_store *store = + &device->spm_scratch_buffer_store; + struct pvr_spm_scratch_buffer *buffer; + + simple_mtx_lock(&store->mtx); + + /* When a render requires a PR the fw will wait for other renders to end, + * free the PB space, unschedule any other vert/frag jobs and solely run the + * PR on the whole device until completion. + * Thus we can safely use the same scratch buffer across multiple + * framebuffers as the scratch buffer is only used during PRs and only one PR + * can ever be executed at any one time. + */ + if (store->head_ref && store->head_ref->size <= size) { + buffer = store->head_ref; + } else { + VkResult result; + + if (store->head_ref) + pvr_spm_scratch_buffer_store_release_head_ref_locked(device, store); + + result = pvr_spm_scratch_buffer_alloc(device, size, &buffer); + if (result != VK_SUCCESS) { + simple_mtx_unlock(&store->mtx); + *buffer_out = NULL; + + return result; + } + + pvr_spm_scratch_buffer_store_set_head_ref_locked(store, buffer); + } + + p_atomic_inc(&buffer->ref_count); + simple_mtx_unlock(&store->mtx); + *buffer_out = buffer; + + return VK_SUCCESS; +} diff --git a/src/imagination/vulkan/pvr_spm.h b/src/imagination/vulkan/pvr_spm.h new file mode 100644 index 00000000000..fe0854f33e3 --- /dev/null +++ b/src/imagination/vulkan/pvr_spm.h @@ -0,0 +1,73 @@ +/* + * Copyright © 2023 Imagination Technologies Ltd. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef PVR_SPM_H +#define PVR_SPM_H + +/** + * \file pvr_spm.h + * + * \brief Smart Parameter Management. + * + * With large amounts of geometry the device can run out of Parameter Buffer + * (PB) as no more free pages are left in the freelist to allow the PB to grow. + * In such cases the render is split into multiple partial renders (PRs) to fit + * within the memory constraints. Each PR produces intermediary results until + * they have all completed, producing the final scene equivalent to what would + * have been produced by the original render. + * + * SPM comprises all the necessary work required of the driver to manage the PB. + */ + +#include <stdint.h> +#include <vulkan/vulkan_core.h> + +#include "util/simple_mtx.h" + +struct pvr_device; +struct pvr_spm_scratch_buffer; + +struct pvr_spm_scratch_buffer_store { + simple_mtx_t mtx; + struct pvr_spm_scratch_buffer *head_ref; +}; + +void pvr_spm_init_scratch_buffer_store(struct pvr_device *device); +void pvr_spm_finish_scratch_buffer_store(struct pvr_device *device); + +/* A scratch buffer is required in various situations: + * + * - An MSAA workload which needs saving to a larger buffer than the output for + * PRs. + * - To store transient results during a PR with read only attachments (i.e. + * VK_ATTACHMENT_STORE_OP_NONE, not currently supported) or lazily allocated + * attachments with no backing. + */ +VkResult pvr_spm_scratch_buffer_get_buffer( + struct pvr_device *device, + uint64_t size, + struct pvr_spm_scratch_buffer **const buffer_out); +void pvr_spm_scratch_buffer_release(struct pvr_device *device, + struct pvr_spm_scratch_buffer *buffer); + +#endif /* PVR_SPM_H */
