Attention is currently required from: Matt Sinclair, Matthew Poremba.
Hello kokoro, Matt Sinclair, Matthew Poremba,
I'd like you to do a code review.
Please visit
https://gem5-review.googlesource.com/c/public/gem5/+/48803
to review the following change.
Change subject: arch-gcn3: Implement LDS accesses in Flat instructions
......................................................................
arch-gcn3: Implement LDS accesses in Flat instructions
Add support for LDS accesses by allowing Flat instructions to dispatch
into the local memory pipeline if the requested address is in the group
aperture.
This requires implementing LDS accesses in the Flat initMemRead/Write
functions, in a similar fashion to the DS functions of the same name.
Because we now can potentially dispatch to the local memory pipeline,
this change also adds a check to regain any tokens we requested as a
flat instruction.
Change-Id: Id26191f7ee43291a5e5ca5f39af06af981ec23ab
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/48343
Reviewed-by: Matt Sinclair <[email protected]>
Reviewed-by: Matthew Poremba <[email protected]>
Maintainer: Matt Sinclair <[email protected]>
Tested-by: kokoro <[email protected]>
---
M src/arch/amdgpu/gcn3/insts/instructions.cc
M src/arch/amdgpu/gcn3/insts/op_encodings.hh
M src/gpu-compute/gpu_dyn_inst.cc
M src/gpu-compute/local_memory_pipeline.cc
4 files changed, 184 insertions(+), 32 deletions(-)
diff --git a/src/arch/amdgpu/gcn3/insts/instructions.cc
b/src/arch/amdgpu/gcn3/insts/instructions.cc
index 79af7ac..65d008b 100644
--- a/src/arch/amdgpu/gcn3/insts/instructions.cc
+++ b/src/arch/amdgpu/gcn3/insts/instructions.cc
@@ -36314,7 +36314,7 @@
gpuDynInst->computeUnit()->globalMemoryPipe.
issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -36363,7 +36363,7 @@
gpuDynInst->computeUnit()->globalMemoryPipe.
issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
void
@@ -39384,8 +39384,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe
.issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
} // execute
@@ -39448,8 +39451,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe
.issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -39511,8 +39517,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe
.issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -39603,8 +39612,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe
.issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -39667,8 +39679,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe
.issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -39731,8 +39746,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe
.issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -39804,8 +39822,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe
.issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -39889,8 +39910,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe
.issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
} // execute
@@ -39952,8 +39976,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe
.issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -40015,8 +40042,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe
.issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -40079,8 +40109,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe
.issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -40151,8 +40184,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe
.issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -40227,8 +40263,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe
.issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -40294,8 +40333,11 @@
"Flats to private aperture not tested yet\n");
gpuDynInst->computeUnit()->globalMemoryPipe.
issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
ConstVecOperandU32 data(gpuDynInst, extData.DATA);
@@ -40408,8 +40450,11 @@
"Flats to private aperture not tested yet\n");
gpuDynInst->computeUnit()->globalMemoryPipe.
issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -40492,8 +40537,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe.
issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -40576,8 +40624,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe.
issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
void
@@ -40834,8 +40885,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe.
issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -40918,8 +40972,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe.
issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -41044,8 +41101,11 @@
"Flats to private aperture not tested yet\n");
gpuDynInst->computeUnit()->globalMemoryPipe.
issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -41129,8 +41189,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe.
issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -41215,8 +41278,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe.
issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -41483,8 +41549,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe.
issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
@@ -41570,8 +41639,11 @@
if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
gpuDynInst->computeUnit()->globalMemoryPipe.
issueRequest(gpuDynInst);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ gpuDynInst->computeUnit()->localMemoryPipe
+ .issueRequest(gpuDynInst);
} else {
- fatal("Non global flat instructions not implemented yet.\n");
+ fatal("Unsupported scope for flat instruction.\n");
}
}
diff --git a/src/arch/amdgpu/gcn3/insts/op_encodings.hh
b/src/arch/amdgpu/gcn3/insts/op_encodings.hh
index a061285..27b9b99 100644
--- a/src/arch/amdgpu/gcn3/insts/op_encodings.hh
+++ b/src/arch/amdgpu/gcn3/insts/op_encodings.hh
@@ -799,35 +799,107 @@
void
initMemRead(GPUDynInstPtr gpuDynInst)
{
- initMemReqHelper<T, 1>(gpuDynInst, MemCmd::ReadReq);
+ if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
+ initMemReqHelper<T, 1>(gpuDynInst, MemCmd::ReadReq);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ Wavefront *wf = gpuDynInst->wavefront();
+ for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
+ if (gpuDynInst->exec_mask[lane]) {
+ Addr vaddr = gpuDynInst->addr[lane];
+ (reinterpret_cast<T*>(gpuDynInst->d_data))[lane]
+ = wf->ldsChunk->read<T>(vaddr);
+ }
+ }
+ }
}
template<int N>
void
initMemRead(GPUDynInstPtr gpuDynInst)
{
- initMemReqHelper<VecElemU32, N>(gpuDynInst, MemCmd::ReadReq);
+ if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
+ initMemReqHelper<VecElemU32, N>(gpuDynInst,
MemCmd::ReadReq);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ Wavefront *wf = gpuDynInst->wavefront();
+ for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
+ if (gpuDynInst->exec_mask[lane]) {
+ Addr vaddr = gpuDynInst->addr[lane];
+ for (int i = 0; i < N; ++i) {
+ (reinterpret_cast<VecElemU32*>(
+ gpuDynInst->d_data))[lane * N + i]
+ = wf->ldsChunk->read<VecElemU32>(
+ vaddr + i*sizeof(VecElemU32));
+ }
+ }
+ }
+ }
}
template<typename T>
void
initMemWrite(GPUDynInstPtr gpuDynInst)
{
- initMemReqHelper<T, 1>(gpuDynInst, MemCmd::WriteReq);
+ if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
+ initMemReqHelper<T, 1>(gpuDynInst, MemCmd::WriteReq);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ Wavefront *wf = gpuDynInst->wavefront();
+ for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
+ if (gpuDynInst->exec_mask[lane]) {
+ Addr vaddr = gpuDynInst->addr[lane];
+ wf->ldsChunk->write<T>(vaddr,
+
(reinterpret_cast<T*>(gpuDynInst->d_data))[lane]);
+ }
+ }
+ }
}
template<int N>
void
initMemWrite(GPUDynInstPtr gpuDynInst)
{
- initMemReqHelper<VecElemU32, N>(gpuDynInst, MemCmd::WriteReq);
+ if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
+ initMemReqHelper<VecElemU32, N>(gpuDynInst,
MemCmd::WriteReq);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ Wavefront *wf = gpuDynInst->wavefront();
+ for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
+ if (gpuDynInst->exec_mask[lane]) {
+ Addr vaddr = gpuDynInst->addr[lane];
+ for (int i = 0; i < N; ++i) {
+ wf->ldsChunk->write<VecElemU32>(
+ vaddr + i*sizeof(VecElemU32),
+ (reinterpret_cast<VecElemU32*>(
+ gpuDynInst->d_data))[lane * N + i]);
+ }
+ }
+ }
+ }
}
template<typename T>
void
initAtomicAccess(GPUDynInstPtr gpuDynInst)
{
- initMemReqHelper<T, 1>(gpuDynInst, MemCmd::SwapReq, true);
+ if (gpuDynInst->executedAs() == enums::SC_GLOBAL) {
+ initMemReqHelper<T, 1>(gpuDynInst, MemCmd::SwapReq, true);
+ } else if (gpuDynInst->executedAs() == enums::SC_GROUP) {
+ Wavefront *wf = gpuDynInst->wavefront();
+ for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
+ if (gpuDynInst->exec_mask[lane]) {
+ Addr vaddr = gpuDynInst->addr[lane];
+ AtomicOpFunctor* amo_op =
+ gpuDynInst->makeAtomicOpFunctor<T>(
+ &(reinterpret_cast<T*>(
+ gpuDynInst->a_data))[lane],
+ &(reinterpret_cast<T*>(
+ gpuDynInst->x_data))[lane]).get();
+
+ T tmp = wf->ldsChunk->read<T>(vaddr);
+ (*amo_op)(reinterpret_cast<uint8_t *>(&tmp));
+ wf->ldsChunk->write<T>(vaddr, tmp);
+ (reinterpret_cast<T*>(gpuDynInst->d_data))[lane] =
tmp;
+ }
+ }
+ }
}
void
diff --git a/src/gpu-compute/gpu_dyn_inst.cc
b/src/gpu-compute/gpu_dyn_inst.cc
index fb9bf07..937e572 100644
--- a/src/gpu-compute/gpu_dyn_inst.cc
+++ b/src/gpu-compute/gpu_dyn_inst.cc
@@ -834,7 +834,10 @@
if (mask[lane]) {
// flat address calculation goes here.
// addr[lane] = segmented address
- panic("Flat group memory operation is unimplemented!\n");
+ addr[lane] = addr[lane] -
+ wavefront()->computeUnit->shader->ldsApe().base;
+ assert(addr[lane] <
+
wavefront()->computeUnit->getLds().getAddrRange().size());
}
}
wavefront()->execUnitId = wavefront()->flatLmUnitId;
diff --git a/src/gpu-compute/local_memory_pipeline.cc
b/src/gpu-compute/local_memory_pipeline.cc
index 995ea75..c99be00 100644
--- a/src/gpu-compute/local_memory_pipeline.cc
+++ b/src/gpu-compute/local_memory_pipeline.cc
@@ -76,6 +76,11 @@
lmReturnedRequests.pop();
w = m->wavefront();
+ if (m->isFlat() && !m->isMemSync() && !m->isEndOfKernel()
+ && m->allLanesZero()) {
+ computeUnit.getTokenManager()->recvTokens(1);
+ }
+
DPRINTF(GPUMem, "CU%d: WF[%d][%d]: Completing local mem
instr %s\n",
m->cu_id, m->simdId, m->wfSlotId, m->disassemble());
m->completeAcc(m);
--
To view, visit https://gem5-review.googlesource.com/c/public/gem5/+/48803
To unsubscribe, or for help writing mail filters, visit
https://gem5-review.googlesource.com/settings
Gerrit-Project: public/gem5
Gerrit-Branch: develop
Gerrit-Change-Id: Id26191f7ee43291a5e5ca5f39af06af981ec23ab
Gerrit-Change-Number: 48803
Gerrit-PatchSet: 1
Gerrit-Owner: Bobby R. Bruce <[email protected]>
Gerrit-Reviewer: Matt Sinclair <[email protected]>
Gerrit-Reviewer: Matthew Poremba <[email protected]>
Gerrit-Reviewer: kokoro <[email protected]>
Gerrit-CC: Kyle Roarty <[email protected]>
Gerrit-Attention: Matt Sinclair <[email protected]>
Gerrit-Attention: Matthew Poremba <[email protected]>
Gerrit-MessageType: newchange
_______________________________________________
gem5-dev mailing list -- [email protected]
To unsubscribe send an email to [email protected]
%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s
