Instead of open coding it via the inefficient amdgpu_ring_write(), which which the compiler is not able to optimise much, we can add a new amdgpu_ring_fill() helper which pads using memset32. We do this by moving the existing implementation from amdgpu_ring_insert_nop(), which is used by the GFX rings, and then individually convert all the SDMA insert_nop implementations.
For SDMA this should have a lesser benefit than for GFX (only SDMA v4.0 uses the 256 byte ring padding while the rest use 16), but it does not harm to be consistent. Signed-off-by: Tvrtko Ursulin <[email protected]> Cc: Timur Kristóf <[email protected]> --- v2: * Unconfused commit message. v3: * Handle zero padding. --- drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 17 +--------------- drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 26 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c | 15 +++++++------- drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 15 +++++++------- drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c | 15 +++++++------- drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c | 15 +++++++------- drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c | 15 +++++++------- drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c | 15 +++++++------- drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c | 15 +++++++------- drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c | 15 +++++++------- drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c | 15 +++++++------- 11 files changed, 90 insertions(+), 88 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c index 686c92e96025..38434a4c3566 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c @@ -117,22 +117,7 @@ int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned int ndw) */ void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) { - uint32_t occupied, chunk1, chunk2; - - occupied = ring->wptr & ring->buf_mask; - chunk1 = ring->buf_mask + 1 - occupied; - chunk1 = (chunk1 >= count) ? count : chunk1; - chunk2 = count - chunk1; - - if (chunk1) - memset32(&ring->ring[occupied], ring->funcs->nop, chunk1); - - if (chunk2) - memset32(ring->ring, ring->funcs->nop, chunk2); - - ring->wptr += count; - ring->wptr &= ring->ptr_mask; - ring->count_dw -= count; + amdgpu_ring_fill(ring, ring->funcs->nop, count); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h index 6b6ee4083c8d..2b1d3956cdd3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h @@ -522,6 +522,32 @@ static inline void amdgpu_ring_write_multiple(struct amdgpu_ring *ring, ring->count_dw -= count_dw; } +static inline void amdgpu_ring_fill(struct amdgpu_ring *ring, + u32 val, u32 count) +{ + const u32 buf_mask = ring->buf_mask; + u32 occupied, chunk1, chunk2; + u64 wptr = ring->wptr; + + if (count == 0) + return; + + occupied = wptr & buf_mask; + chunk1 = buf_mask + 1 - occupied; + chunk1 = (chunk1 >= count) ? count : chunk1; + chunk2 = count - chunk1; + + if (chunk1) + memset32(&ring->ring[occupied], val, chunk1); + + if (chunk2) + memset32(ring->ring, val, chunk2); + + wptr += count; + ring->wptr = wptr & ring->ptr_mask; + ring->count_dw -= count; +} + static inline unsigned int amdgpu_ring_get_dw_distance(struct amdgpu_ring *ring, u64 start_wptr, u64 end_wptr) { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index 5da75852e5bf..790d8eba1b3c 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -224,15 +224,14 @@ static void sdma_v2_4_ring_set_wptr(struct amdgpu_ring *ring) static void sdma_v2_4_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) { struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); - const bool burst_nop = sdma->burst_nop; - int i; + const u32 nop = ring->funcs->nop; - for (i = 0; i < count; i++) - if (i == 0 && burst_nop) - amdgpu_ring_write(ring, ring->funcs->nop | - SDMA_PKT_NOP_HEADER_COUNT(count - 1)); - else - amdgpu_ring_write(ring, ring->funcs->nop); + if (count && sdma->burst_nop) { + --count; + amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count)); + } + + amdgpu_ring_fill(ring, nop, count); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index 380967852e78..d658e512f5d5 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -401,15 +401,14 @@ static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring) static void sdma_v3_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) { struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); - const bool burst_nop = sdma->burst_nop; - int i; + const u32 nop = ring->funcs->nop; - for (i = 0; i < count; i++) - if (i == 0 && burst_nop) - amdgpu_ring_write(ring, ring->funcs->nop | - SDMA_PKT_NOP_HEADER_COUNT(count - 1)); - else - amdgpu_ring_write(ring, ring->funcs->nop); + if (count && sdma->burst_nop) { + --count; + amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count)); + } + + amdgpu_ring_fill(ring, nop, count); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c index fe0966ab92ac..f2a358084468 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c @@ -784,15 +784,14 @@ static void sdma_v4_0_page_ring_set_wptr(struct amdgpu_ring *ring) static void sdma_v4_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) { struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); - const bool burst_nop = sdma->burst_nop; - int i; + const u32 nop = ring->funcs->nop; - for (i = 0; i < count; i++) - if (i == 0 && burst_nop) - amdgpu_ring_write(ring, ring->funcs->nop | - SDMA_PKT_NOP_HEADER_COUNT(count - 1)); - else - amdgpu_ring_write(ring, ring->funcs->nop); + if (count && sdma->burst_nop) { + --count; + amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count)); + } + + amdgpu_ring_fill(ring, nop, count); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c index 332536454aee..585c8816d67a 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c @@ -350,15 +350,14 @@ static void sdma_v4_4_2_page_ring_set_wptr(struct amdgpu_ring *ring) static void sdma_v4_4_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) { struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); - const bool burst_nop = sdma->burst_nop; - int i; + const u32 nop = ring->funcs->nop; - for (i = 0; i < count; i++) - if (i == 0 && burst_nop) - amdgpu_ring_write(ring, ring->funcs->nop | - SDMA_PKT_NOP_HEADER_COUNT(count - 1)); - else - amdgpu_ring_write(ring, ring->funcs->nop); + if (count && sdma->burst_nop) { + --count; + amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count)); + } + + amdgpu_ring_fill(ring, nop, count); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c index 33a7ce87cc3a..e41956c663c1 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c @@ -407,15 +407,14 @@ static void sdma_v5_0_ring_set_wptr(struct amdgpu_ring *ring) static void sdma_v5_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) { struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); - const bool burst_nop = sdma->burst_nop; - int i; + const u32 nop = ring->funcs->nop; - for (i = 0; i < count; i++) - if (i == 0 && burst_nop) - amdgpu_ring_write(ring, ring->funcs->nop | - SDMA_PKT_NOP_HEADER_COUNT(count - 1)); - else - amdgpu_ring_write(ring, ring->funcs->nop); + if (count && sdma->burst_nop) { + --count; + amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count)); + } + + amdgpu_ring_fill(ring, nop, count); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c index 0b2da12acf23..94b6ed9c0819 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c @@ -255,15 +255,14 @@ static void sdma_v5_2_ring_set_wptr(struct amdgpu_ring *ring) static void sdma_v5_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) { struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); - const bool burst_nop = sdma->burst_nop; - int i; + const u32 nop = ring->funcs->nop; - for (i = 0; i < count; i++) - if (i == 0 && burst_nop) - amdgpu_ring_write(ring, ring->funcs->nop | - SDMA_PKT_NOP_HEADER_COUNT(count - 1)); - else - amdgpu_ring_write(ring, ring->funcs->nop); + if (count && sdma->burst_nop) { + --count; + amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count)); + } + + amdgpu_ring_fill(ring, nop, count); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c index 800fd9370229..5d0c0c5ebb7d 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c @@ -243,15 +243,14 @@ static void sdma_v6_0_ring_set_wptr(struct amdgpu_ring *ring) static void sdma_v6_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) { struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); - const bool burst_nop = sdma->burst_nop; - int i; + const u32 nop = ring->funcs->nop; - for (i = 0; i < count; i++) - if (i == 0 && burst_nop) - amdgpu_ring_write(ring, ring->funcs->nop | - SDMA_PKT_NOP_HEADER_COUNT(count - 1)); - else - amdgpu_ring_write(ring, ring->funcs->nop); + if (count && sdma->burst_nop) { + --count; + amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count)); + } + + amdgpu_ring_fill(ring, nop, count); } /* diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c index da21791fc585..0b4791424749 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c @@ -245,15 +245,14 @@ static void sdma_v7_0_ring_set_wptr(struct amdgpu_ring *ring) static void sdma_v7_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) { struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); - const bool burst_nop = sdma->burst_nop; - int i; + const u32 nop = ring->funcs->nop; - for (i = 0; i < count; i++) - if (i == 0 && burst_nop) - amdgpu_ring_write(ring, ring->funcs->nop | - SDMA_PKT_NOP_HEADER_COUNT(count - 1)); - else - amdgpu_ring_write(ring, ring->funcs->nop); + if (count && sdma->burst_nop) { + --count; + amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count)); + } + + amdgpu_ring_fill(ring, nop, count); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c index 934e84cb81e2..321c67e01443 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c @@ -239,15 +239,14 @@ static void sdma_v7_1_ring_set_wptr(struct amdgpu_ring *ring) static void sdma_v7_1_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) { struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); - const bool burst_nop = sdma->burst_nop; - int i; + const u32 nop = ring->funcs->nop; - for (i = 0; i < count; i++) - if (i == 0 && burst_nop) - amdgpu_ring_write(ring, ring->funcs->nop | - SDMA_PKT_NOP_HEADER_COUNT(count - 1)); - else - amdgpu_ring_write(ring, ring->funcs->nop); + if (count && sdma->burst_nop) { + --count; + amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count)); + } + + amdgpu_ring_fill(ring, nop, count); } /** -- 2.55.0
