From: Harish Kasiviswanathan <[email protected]> GFX12.1 CP to improve performance has removed updating event_id into signal mailbox. In future, this optimization can be extended to older ASICs. Update driver code to handle this case.
Signed-off-by: Harish Kasiviswanathan <[email protected]> Reviewed-by: Felix Kuehling <[email protected]> Signed-off-by: Alex Deucher <[email protected]> --- drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c | 6 +++--- drivers/gpu/drm/amd/amdkfd/kfd_events.c | 12 ++++++++---- drivers/gpu/drm/amd/amdkfd/kfd_events.h | 3 ++- drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c | 6 +++--- drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c | 6 +++--- drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c | 6 +++--- drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c | 6 +++--- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 2 +- 8 files changed, 26 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c index 73acbe0b7c212..b799c70f5742a 100644 --- a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c +++ b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c @@ -97,11 +97,11 @@ static void cik_event_interrupt_wq(struct kfd_node *dev, return; if (ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE) - kfd_signal_event_interrupt(pasid, context_id, 28); + kfd_signal_event_interrupt(pasid, context_id, 28, true); else if (ihre->source_id == CIK_INTSRC_SDMA_TRAP) - kfd_signal_event_interrupt(pasid, context_id, 28); + kfd_signal_event_interrupt(pasid, context_id, 28, true); else if (ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG) - kfd_signal_event_interrupt(pasid, context_id & 0xff, 8); + kfd_signal_event_interrupt(pasid, context_id & 0xff, 8, true); else if (ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE) kfd_signal_hw_exception_event(pasid); else if (ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT || diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index 13416bff77636..ee70eb4b63193 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -142,6 +142,7 @@ static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id) * @p: Pointer to struct kfd_process * @id: ID to look up * @bits: Number of valid bits in @id + * @signal_mailbox_updated: flag indicates if FW updates signal mailbox entry * * Finds the first signaled event with a matching partial ID. If no * matching signaled event is found, returns NULL. In that case the @@ -155,7 +156,8 @@ static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id) * driver. */ static struct kfd_event *lookup_signaled_event_by_partial_id( - struct kfd_process *p, uint32_t id, uint32_t bits) + struct kfd_process *p, uint32_t id, uint32_t bits, + bool signal_mailbox_updated) { struct kfd_event *ev; @@ -166,7 +168,8 @@ static struct kfd_event *lookup_signaled_event_by_partial_id( * and we only need a single lookup. */ if (bits > 31 || (1U << bits) >= KFD_SIGNAL_EVENT_LIMIT) { - if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT) + if (signal_mailbox_updated && + page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT) return NULL; return idr_find(&p->event_idr, id); @@ -724,7 +727,7 @@ static void set_event_from_interrupt(struct kfd_process *p, } void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id, - uint32_t valid_id_bits) + uint32_t valid_id_bits, bool signal_mailbox_updated) { struct kfd_event *ev = NULL; @@ -742,7 +745,8 @@ void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id, if (valid_id_bits) ev = lookup_signaled_event_by_partial_id(p, partial_id, - valid_id_bits); + valid_id_bits, + signal_mailbox_updated); if (ev) { set_event_from_interrupt(p, ev); } else if (p->signal_page) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.h b/drivers/gpu/drm/amd/amdkfd/kfd_events.h index 52ccfd397c2b3..1dc21c13833b7 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.h @@ -85,6 +85,7 @@ struct kfd_event { #define KFD_EVENT_TYPE_MEMORY 8 extern void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id, - uint32_t valid_id_bits); + uint32_t valid_id_bits, + bool signal_mailbox_updated); #endif diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c index 3e1ad89747979..19406ab92c5bb 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c @@ -211,7 +211,7 @@ static void event_interrupt_wq_v10(struct kfd_node *dev, client_id == SOC15_IH_CLIENTID_SE2SH || client_id == SOC15_IH_CLIENTID_SE3SH) { if (source_id == SOC15_INTSRC_CP_END_OF_PIPE) - kfd_signal_event_interrupt(pasid, context_id0, 32); + kfd_signal_event_interrupt(pasid, context_id0, 32, true); else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) { encoding = REG_GET_FIELD(context_id1, SQ_INTERRUPT_WORD_WAVE_CTXID1, ENCODING); @@ -324,7 +324,7 @@ static void event_interrupt_wq_v10(struct kfd_node *dev, default: break; } - kfd_signal_event_interrupt(pasid, context_id0 & 0x7fffff, 23); + kfd_signal_event_interrupt(pasid, context_id0 & 0x7fffff, 23, true); } else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE && KFD_DBG_EC_TYPE_IS_PACKET(KFD_DEBUG_CP_BAD_OP_ECODE(context_id0))) { kfd_set_dbg_ev_from_interrupt(dev, pasid, @@ -344,7 +344,7 @@ static void event_interrupt_wq_v10(struct kfd_node *dev, client_id == SOC15_IH_CLIENTID_SDMA6 || client_id == SOC15_IH_CLIENTID_SDMA7) { if (source_id == SOC15_INTSRC_SDMA_TRAP) { - kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28); + kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28, true); } } else if (client_id == SOC15_IH_CLIENTID_VMC || client_id == SOC15_IH_CLIENTID_VMC1 || diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c index 2788a52714d10..12d81abed7481 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c @@ -353,7 +353,7 @@ static void event_interrupt_wq_v11(struct kfd_node *dev, /* CP */ if (source_id == SOC15_INTSRC_CP_END_OF_PIPE) - kfd_signal_event_interrupt(pasid, context_id0, 32); + kfd_signal_event_interrupt(pasid, context_id0, 32, true); else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE && KFD_DBG_EC_TYPE_IS_PACKET(KFD_CTXID0_CP_BAD_OP_ECODE(context_id0))) { u32 doorbell_id = KFD_CTXID0_DOORBELL_ID(context_id0); @@ -366,7 +366,7 @@ static void event_interrupt_wq_v11(struct kfd_node *dev, /* SDMA */ else if (source_id == SOC21_INTSRC_SDMA_TRAP) - kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28); + kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28, true); else if (source_id == SOC21_INTSRC_SDMA_ECC) { event_interrupt_poison_consumption_v11(dev, pasid, source_id); return; @@ -404,7 +404,7 @@ static void event_interrupt_wq_v11(struct kfd_node *dev, default: break; } - kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24); + kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24, true); } } else if (KFD_IRQ_IS_FENCE(client_id, source_id)) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c index 230e57dbcec58..5d8b5aa194da8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c @@ -326,7 +326,7 @@ static void event_interrupt_wq_v12_1(struct kfd_node *node, /* CP */ if (source_id == SOC15_INTSRC_CP_END_OF_PIPE) - kfd_signal_event_interrupt(pasid, context_id0, 32); + kfd_signal_event_interrupt(pasid, context_id0, 32, false); else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE && KFD_DBG_EC_TYPE_IS_PACKET(KFD_CTXID0_CP_BAD_OP_ECODE(context_id0))) { u32 doorbell_id = KFD_CTXID0_DOORBELL_ID(context_id0); @@ -339,7 +339,7 @@ static void event_interrupt_wq_v12_1(struct kfd_node *node, /* SDMA */ else if (source_id == SOC21_INTSRC_SDMA_TRAP) - kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28); + kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28, true); else if (source_id == SOC21_INTSRC_SDMA_ECC) { event_interrupt_poison_consumption_v12_1(node, pasid, source_id); return; @@ -377,7 +377,7 @@ static void event_interrupt_wq_v12_1(struct kfd_node *node, default: break; } - kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24); + kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24, true); } } else if (KFD_IRQ_IS_FENCE(client_id, source_id)) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c index d76fb61869c75..1688d8e595f2f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c @@ -379,7 +379,7 @@ static void event_interrupt_wq_v9(struct kfd_node *dev, client_id == SOC15_IH_CLIENTID_SE2SH || client_id == SOC15_IH_CLIENTID_SE3SH) { if (source_id == SOC15_INTSRC_CP_END_OF_PIPE) - kfd_signal_event_interrupt(pasid, context_id0, 32); + kfd_signal_event_interrupt(pasid, context_id0, 32, true); else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) { sq_int_data = KFD_CONTEXT_ID_GET_SQ_INT_DATA(context_id0, context_id1); encoding = REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, ENCODING); @@ -513,7 +513,7 @@ static void event_interrupt_wq_v9(struct kfd_node *dev, default: break; } - kfd_signal_event_interrupt(pasid, sq_int_data, 24); + kfd_signal_event_interrupt(pasid, sq_int_data, 24, true); } else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE && KFD_DBG_EC_TYPE_IS_PACKET(KFD_DEBUG_CP_BAD_OP_ECODE(context_id0))) { kfd_set_dbg_ev_from_interrupt(dev, pasid, @@ -530,7 +530,7 @@ static void event_interrupt_wq_v9(struct kfd_node *dev, client_id == SOC15_IH_CLIENTID_SDMA6 || client_id == SOC15_IH_CLIENTID_SDMA7) { if (source_id == SOC15_INTSRC_SDMA_TRAP) { - kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28); + kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28, true); } else if (source_id == SOC15_INTSRC_SDMA_ECC) { event_interrupt_poison_consumption_v9(dev, pasid, client_id); return; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 9849b54f54ba0..27e4859e4ad7a 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -1525,7 +1525,7 @@ int kfd_wait_on_events(struct kfd_process *p, bool all, uint32_t *user_timeout_ms, uint32_t *wait_result); void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id, - uint32_t valid_id_bits); + uint32_t valid_id_bits, bool signal_mailbox_updated); void kfd_signal_hw_exception_event(u32 pasid); int kfd_set_event(struct kfd_process *p, uint32_t event_id); int kfd_reset_event(struct kfd_process *p, uint32_t event_id); -- 2.53.0
