Module: Mesa Branch: master Commit: ea3db9f5960c50da34730cb08023947810b47aaa URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=ea3db9f5960c50da34730cb08023947810b47aaa
Author: Connor Abbott <[email protected]> Date: Fri Oct 30 16:31:05 2020 +0100 freedreno/a6xx: Implement private memory Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/7386> --- src/gallium/drivers/freedreno/a6xx/fd6_compute.c | 7 +- src/gallium/drivers/freedreno/a6xx/fd6_emit.c | 2 - src/gallium/drivers/freedreno/a6xx/fd6_program.c | 92 +++++++++++++++-------- src/gallium/drivers/freedreno/a6xx/fd6_program.h | 3 +- src/gallium/drivers/freedreno/freedreno_context.c | 5 ++ src/gallium/drivers/freedreno/freedreno_context.h | 21 ++++++ 6 files changed, 91 insertions(+), 39 deletions(-) diff --git a/src/gallium/drivers/freedreno/a6xx/fd6_compute.c b/src/gallium/drivers/freedreno/a6xx/fd6_compute.c index 6faf2de8f40..aa6e4996661 100644 --- a/src/gallium/drivers/freedreno/a6xx/fd6_compute.c +++ b/src/gallium/drivers/freedreno/a6xx/fd6_compute.c @@ -74,7 +74,8 @@ fd6_delete_compute_state(struct pipe_context *pctx, void *hwcso) /* maybe move to fd6_program? */ static void -cs_program_emit(struct fd_ringbuffer *ring, struct ir3_shader_variant *v) +cs_program_emit(struct fd_context *ctx, struct fd_ringbuffer *ring, + struct ir3_shader_variant *v) { const struct ir3_info *i = &v->info; enum a3xx_threadsize thrsz = FOUR_QUADS; @@ -128,7 +129,7 @@ cs_program_emit(struct fd_ringbuffer *ring, struct ir3_shader_variant *v) OUT_RELOC(ring, v->bo, 0, 0, 0); /* SP_CS_OBJ_START_LO/HI */ if (v->instrlen > 0) - fd6_emit_shader(ring, v); + fd6_emit_shader(ctx, ring, v); } static void @@ -145,7 +146,7 @@ fd6_launch_grid(struct fd_context *ctx, const struct pipe_grid_info *info) return; if (ctx->dirty_shader[PIPE_SHADER_COMPUTE] & FD_DIRTY_SHADER_PROG) - cs_program_emit(ring, v); + cs_program_emit(ctx, ring, v); fd6_emit_cs_state(ctx, ring, v); fd6_emit_cs_consts(v, ring, ctx, info); diff --git a/src/gallium/drivers/freedreno/a6xx/fd6_emit.c b/src/gallium/drivers/freedreno/a6xx/fd6_emit.c index bbd9e682f41..83afd1c6575 100644 --- a/src/gallium/drivers/freedreno/a6xx/fd6_emit.c +++ b/src/gallium/drivers/freedreno/a6xx/fd6_emit.c @@ -1217,8 +1217,6 @@ fd6_emit_restore(struct fd_batch *batch, struct fd_ringbuffer *ring) WRITE(REG_A6XX_PC_MULTIVIEW_CNTL, 0); - WRITE(REG_A6XX_SP_VS_OBJ_FIRST_EXEC_OFFSET, 0); - WRITE(REG_A6XX_SP_UNKNOWN_B183, 0); WRITE(REG_A6XX_GRAS_UNKNOWN_8099, 0); diff --git a/src/gallium/drivers/freedreno/a6xx/fd6_program.c b/src/gallium/drivers/freedreno/a6xx/fd6_program.c index 95dbdd2b675..6d7f7e45232 100644 --- a/src/gallium/drivers/freedreno/a6xx/fd6_program.c +++ b/src/gallium/drivers/freedreno/a6xx/fd6_program.c @@ -42,37 +42,38 @@ #include "fd6_pack.h" void -fd6_emit_shader(struct fd_ringbuffer *ring, const struct ir3_shader_variant *so) +fd6_emit_shader(struct fd_context *ctx, struct fd_ringbuffer *ring, + const struct ir3_shader_variant *so) { enum a6xx_state_block sb = fd6_stage2shadersb(so->type); - uint32_t obj_start = 0; + uint32_t first_exec_offset = 0; uint32_t instrlen = 0; switch (so->type) { case MESA_SHADER_VERTEX: - obj_start = REG_A6XX_SP_VS_OBJ_START_LO; + first_exec_offset = REG_A6XX_SP_VS_OBJ_FIRST_EXEC_OFFSET; instrlen = REG_A6XX_SP_VS_INSTRLEN; break; case MESA_SHADER_TESS_CTRL: - obj_start = REG_A6XX_SP_HS_OBJ_START_LO; + first_exec_offset = REG_A6XX_SP_HS_OBJ_FIRST_EXEC_OFFSET; instrlen = REG_A6XX_SP_HS_INSTRLEN; break; case MESA_SHADER_TESS_EVAL: - obj_start = REG_A6XX_SP_DS_OBJ_START_LO; + first_exec_offset = REG_A6XX_SP_DS_OBJ_FIRST_EXEC_OFFSET; instrlen = REG_A6XX_SP_DS_INSTRLEN; break; case MESA_SHADER_GEOMETRY: - obj_start = REG_A6XX_SP_GS_OBJ_START_LO; + first_exec_offset = REG_A6XX_SP_GS_OBJ_FIRST_EXEC_OFFSET; instrlen = REG_A6XX_SP_GS_INSTRLEN; break; case MESA_SHADER_FRAGMENT: - obj_start = REG_A6XX_SP_FS_OBJ_START_LO; + first_exec_offset = REG_A6XX_SP_FS_OBJ_FIRST_EXEC_OFFSET; instrlen = REG_A6XX_SP_FS_INSTRLEN; break; case MESA_SHADER_COMPUTE: case MESA_SHADER_KERNEL: - obj_start = REG_A6XX_SP_CS_OBJ_START_LO; + first_exec_offset = REG_A6XX_SP_CS_OBJ_FIRST_EXEC_OFFSET; instrlen = REG_A6XX_SP_CS_INSTRLEN; break; case MESA_SHADER_TASK: @@ -95,11 +96,42 @@ fd6_emit_shader(struct fd_ringbuffer *ring, const struct ir3_shader_variant *so) fd_emit_string5(ring, name, strlen(name)); #endif + uint32_t fibers_per_sp = ctx->screen->info.fibers_per_sp; + uint32_t num_sp_cores = ctx->screen->info.num_sp_cores; + + uint32_t per_fiber_size = ALIGN(so->pvtmem_size, 512); + if (per_fiber_size > ctx->pvtmem[so->pvtmem_per_wave].per_fiber_size) { + if (ctx->pvtmem[so->pvtmem_per_wave].bo) + fd_bo_del(ctx->pvtmem[so->pvtmem_per_wave].bo); + ctx->pvtmem[so->pvtmem_per_wave].per_fiber_size = per_fiber_size; + uint32_t total_size = ALIGN(per_fiber_size * fibers_per_sp, 1 << 12) + * num_sp_cores; + ctx->pvtmem[so->pvtmem_per_wave].bo = + fd_bo_new(ctx->screen->dev, total_size, + DRM_FREEDRENO_GEM_TYPE_KMEM, "pvtmem_%s_%d", + so->pvtmem_per_wave ? "per_wave" : "per_fiber", + per_fiber_size); + } else { + per_fiber_size = ctx->pvtmem[so->pvtmem_per_wave].per_fiber_size; + } + + uint32_t per_sp_size = ALIGN(per_fiber_size * fibers_per_sp, 1 << 12); + OUT_PKT4(ring, instrlen, 1); OUT_RING(ring, so->instrlen); - OUT_PKT4(ring, obj_start, 2); - OUT_RELOC(ring, so->bo, 0, 0, 0); + OUT_PKT4(ring, first_exec_offset, 7); + OUT_RING(ring, 0); /* SP_xS_OBJ_FIRST_EXEC_OFFSET */ + OUT_RELOC(ring, so->bo, 0, 0, 0); /* SP_xS_OBJ_START_LO */ + OUT_RING(ring, A6XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM(per_fiber_size)); + if (so->pvtmem_size > 0) { /* SP_xS_PVT_MEM_ADDR */ + OUT_RELOC(ring, ctx->pvtmem[so->pvtmem_per_wave].bo, 0, 0, 0); + } else { + OUT_RING(ring, 0); + OUT_RING(ring, 0); + } + OUT_RING(ring, A6XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(per_sp_size) | + COND(so->pvtmem_per_wave, A6XX_SP_VS_PVT_MEM_SIZE_PERWAVEMEMLAYOUT)); OUT_PKT7(ring, fd6_stage2opcode(so->type), 3); OUT_RING(ring, CP_LOAD_STATE6_0_DST_OFF(0) | @@ -311,7 +343,7 @@ next_regid(uint32_t reg, uint32_t increment) } static void -setup_stateobj(struct fd_ringbuffer *ring, struct fd_screen *screen, +setup_stateobj(struct fd_ringbuffer *ring, struct fd_context *ctx, struct fd6_program_state *state, const struct ir3_shader_key *key, bool binning_pass) { @@ -430,9 +462,6 @@ setup_stateobj(struct fd_ringbuffer *ring, struct fd_screen *screen, * emitted if frag-prog is dirty vs if vert-prog is dirty.. */ - OUT_PKT4(ring, REG_A6XX_SP_HS_OBJ_FIRST_EXEC_OFFSET, 1); - OUT_RING(ring, 0x0); - OUT_PKT4(ring, REG_A6XX_SP_FS_PREFETCH_CNTL, 1 + fs->num_sampler_prefetch); OUT_RING(ring, A6XX_SP_FS_PREFETCH_CNTL_COUNT(fs->num_sampler_prefetch) | A6XX_SP_FS_PREFETCH_CNTL_UNK4(regid(63, 0)) | @@ -474,8 +503,8 @@ setup_stateobj(struct fd_ringbuffer *ring, struct fd_screen *screen, A6XX_SP_VS_CTRL_REG0_BRANCHSTACK(vs->branchstack) | COND(vs->need_pixlod, A6XX_SP_VS_CTRL_REG0_PIXLODENABLE)); - fd6_emit_shader(ring, vs); - fd6_emit_immediates(screen, vs, ring); + fd6_emit_shader(ctx, ring, vs); + fd6_emit_immediates(ctx->screen, vs, ring); struct ir3_shader_linkage l = {0}; const struct ir3_shader_variant *last_shader = fd6_last_shader(state); @@ -594,9 +623,9 @@ setup_stateobj(struct fd_ringbuffer *ring, struct fd_screen *screen, A6XX_SP_HS_CTRL_REG0_BRANCHSTACK(hs->branchstack) | COND(hs->need_pixlod, A6XX_SP_HS_CTRL_REG0_PIXLODENABLE)); - fd6_emit_shader(ring, hs); - fd6_emit_immediates(screen, hs, ring); - fd6_emit_link_map(screen, vs, hs, ring); + fd6_emit_shader(ctx, ring, hs); + fd6_emit_immediates(ctx->screen, hs, ring); + fd6_emit_link_map(ctx->screen, vs, hs, ring); OUT_PKT4(ring, REG_A6XX_SP_DS_CTRL_REG0, 1); OUT_RING(ring, A6XX_SP_DS_CTRL_REG0_THREADSIZE(TWO_QUADS) | @@ -606,9 +635,9 @@ setup_stateobj(struct fd_ringbuffer *ring, struct fd_screen *screen, A6XX_SP_DS_CTRL_REG0_BRANCHSTACK(ds->branchstack) | COND(ds->need_pixlod, A6XX_SP_DS_CTRL_REG0_PIXLODENABLE)); - fd6_emit_shader(ring, ds); - fd6_emit_immediates(screen, ds, ring); - fd6_emit_link_map(screen, hs, ds, ring); + fd6_emit_shader(ctx, ring, ds); + fd6_emit_immediates(ctx->screen, ds, ring); + fd6_emit_link_map(ctx->screen, hs, ds, ring); shader_info *hs_info = &hs->shader->nir->info; OUT_PKT4(ring, REG_A6XX_PC_TESS_NUM_VERTEX, 1); @@ -723,9 +752,6 @@ setup_stateobj(struct fd_ringbuffer *ring, struct fd_screen *screen, A6XX_SP_FS_CTRL_REG0_BRANCHSTACK(fs->branchstack) | COND(fs->need_pixlod, A6XX_SP_FS_CTRL_REG0_PIXLODENABLE)); - OUT_PKT4(ring, REG_A6XX_SP_FS_OBJ_FIRST_EXEC_OFFSET, 1); - OUT_RING(ring, 0); - OUT_PKT4(ring, REG_A6XX_VPC_VS_LAYER_CNTL, 1); OUT_RING(ring, 0x0000ffff); /* XXX */ @@ -797,12 +823,12 @@ setup_stateobj(struct fd_ringbuffer *ring, struct fd_screen *screen, A6XX_SP_GS_CTRL_REG0_BRANCHSTACK(gs->branchstack) | COND(gs->need_pixlod, A6XX_SP_GS_CTRL_REG0_PIXLODENABLE)); - fd6_emit_shader(ring, gs); - fd6_emit_immediates(screen, gs, ring); + fd6_emit_shader(ctx, ring, gs); + fd6_emit_immediates(ctx->screen, gs, ring); if (ds) - fd6_emit_link_map(screen, ds, gs, ring); + fd6_emit_link_map(ctx->screen, ds, gs, ring); else - fd6_emit_link_map(screen, vs, gs, ring); + fd6_emit_link_map(ctx->screen, vs, gs, ring); OUT_PKT4(ring, REG_A6XX_VPC_GS_PACK, 1); OUT_RING(ring, A6XX_VPC_GS_PACK_POSITIONLOC(pos_loc) | @@ -894,7 +920,7 @@ setup_stateobj(struct fd_ringbuffer *ring, struct fd_screen *screen, OUT_RING(ring, 0); if (fs->instrlen) - fd6_emit_shader(ring, fs); + fd6_emit_shader(ctx, ring, fs); OUT_REG(ring, A6XX_PC_PRIMID_PASSTHRU(primid_passthru)); @@ -932,7 +958,7 @@ setup_stateobj(struct fd_ringbuffer *ring, struct fd_screen *screen, COND(primid_passthru, A6XX_VFD_CONTROL_6_PRIMID_PASSTHRU)); /* VFD_CONTROL_6 */ if (!binning_pass) - fd6_emit_immediates(screen, fs, ring); + fd6_emit_immediates(ctx->screen, fs, ring); } static void emit_interp_state(struct fd_ringbuffer *ring, struct ir3_shader_variant *fs, @@ -1080,8 +1106,8 @@ fd6_program_create(void *data, struct ir3_shader_variant *bs, #endif setup_config_stateobj(state->config_stateobj, state); - setup_stateobj(state->binning_stateobj, ctx->screen, state, key, true); - setup_stateobj(state->stateobj, ctx->screen, state, key, false); + setup_stateobj(state->binning_stateobj, ctx, state, key, true); + setup_stateobj(state->stateobj, ctx, state, key, false); state->interp_stateobj = create_interp_stateobj(ctx, state); return &state->base; diff --git a/src/gallium/drivers/freedreno/a6xx/fd6_program.h b/src/gallium/drivers/freedreno/a6xx/fd6_program.h index 54a7c1fb52c..3790c62348d 100644 --- a/src/gallium/drivers/freedreno/a6xx/fd6_program.h +++ b/src/gallium/drivers/freedreno/a6xx/fd6_program.h @@ -68,7 +68,8 @@ fd6_last_shader(const struct fd6_program_state *state) return state->vs; } -void fd6_emit_shader(struct fd_ringbuffer *ring, const struct ir3_shader_variant *so); +void fd6_emit_shader(struct fd_context *ctx, struct fd_ringbuffer *ring, + const struct ir3_shader_variant *so); struct fd_ringbuffer * fd6_program_interp_state(struct fd6_emit *emit); diff --git a/src/gallium/drivers/freedreno/freedreno_context.c b/src/gallium/drivers/freedreno/freedreno_context.c index b8149513d7a..775e0e151c2 100644 --- a/src/gallium/drivers/freedreno/freedreno_context.c +++ b/src/gallium/drivers/freedreno/freedreno_context.c @@ -307,6 +307,11 @@ fd_context_destroy(struct pipe_context *pctx) if (ctx->in_fence_fd != -1) close(ctx->in_fence_fd); + for (i = 0; i < ARRAY_SIZE(ctx->pvtmem); i++) { + if (ctx->pvtmem[i].bo) + fd_bo_del(ctx->pvtmem[i].bo); + } + util_copy_framebuffer_state(&ctx->framebuffer, NULL); fd_batch_reference(&ctx->batch, NULL); /* unref current batch */ fd_bc_invalidate_context(ctx); diff --git a/src/gallium/drivers/freedreno/freedreno_context.h b/src/gallium/drivers/freedreno/freedreno_context.h index 948350a4f48..564c0ba66f2 100644 --- a/src/gallium/drivers/freedreno/freedreno_context.h +++ b/src/gallium/drivers/freedreno/freedreno_context.h @@ -370,6 +370,27 @@ struct fd_context { bool cond_cond; /* inverted rendering condition */ uint cond_mode; + /* Private memory is a memory space where each fiber gets its own piece of + * memory, in addition to registers. It is backed by a buffer which needs + * to be large enough to hold the contents of every possible wavefront in + * every core of the GPU. Because it allocates space via the internal + * wavefront ID which is shared between all currently executing shaders, + * the same buffer can be reused by all shaders, as long as all shaders + * sharing the same buffer use the exact same configuration. There are two + * inputs to the configuration, the amount of per-fiber space and whether + * to use the newer per-wave or older per-fiber layout. We only ever + * increase the size, and shaders with a smaller size requirement simply + * use the larger existing buffer, so that we only need to keep track of + * one buffer and its size, but we still need to keep track of per-fiber + * and per-wave buffers separately so that we never use the same buffer + * for different layouts. pvtmem[0] is for per-fiber, and pvtmem[1] is for + * per-wave. + */ + struct { + struct fd_bo *bo; + uint32_t per_fiber_size; + } pvtmem[2]; + struct pipe_debug_callback debug; /* Called on rebind_resource() for any per-gen cleanup required: */ _______________________________________________ mesa-commit mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/mesa-commit
