---
.../gpu/drm/amd/amdkfd/cwsr_trap_handler.h | 356 +++++++++++-------
.../amd/amdkfd/cwsr_trap_handler_gfx12.asm | 217 ++++++++++-
2 files changed, 432 insertions(+), 141 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
index 54fa76f374c9..259b2a4a1940 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
@@ -3644,7 +3644,7 @@ static const uint32_t cwsr_trap_gfx9_4_3_hex[] = {
};
static const uint32_t cwsr_trap_gfx12_hex[] = {
- 0xbfa00001, 0xbfa00239,
+ 0xbfa00001, 0xbfa00273,
0xb0804009, 0xb8eef81a,
0xbf880000, 0xb980081a,
0x00000000, 0xb8f8f804,
@@ -3707,7 +3707,10 @@ static const uint32_t cwsr_trap_gfx12_hex[] = {
0x000c0000, 0x00000000,
0x7e000280, 0xbefe007a,
0xbeff007b, 0xb8fb0742,
- 0x847b997b, 0xb8fa3b05,
+ 0x847b997b, 0xb8faf802,
+ 0xbf0d9e7a, 0xbfa10003,
+ 0xbefa00ff, 0x00000800,
+ 0xbfa00007, 0xb8fa3b05,
0x807a817a, 0xbf0d997b,
0xbfa20002, 0x847a897a,
0xbfa00001, 0x847a8a7a,
@@ -3748,6 +3751,9 @@ static const uint32_t cwsr_trap_gfx12_hex[] = {
0xee0a4074, 0x010c0000,
0x00020000, 0xee0a4074,
0x018c0000, 0x00030000,
+ 0xb8f0f802, 0xbf0d9e70,
+ 0xbfa10003, 0xbef000ff,
+ 0x00000800, 0xbfa00007,
0xb8f03b05, 0x80708170,
0xbf0d9973, 0xbfa20002,
0x84708970, 0xbfa00001,
@@ -3790,7 +3796,10 @@ static const uint32_t cwsr_trap_gfx12_hex[] = {
0xbeff0080, 0x80767074,
0x82778075, 0xee0a4076,
0x010c0000, 0x00000000,
- 0xbefe00c1, 0xb8f03b05,
+ 0xbefe00c1, 0xb8f0f802,
+ 0xbf0d9e70, 0xbfa10003,
+ 0xbef000ff, 0x00000800,
+ 0xbfa00007, 0xb8f03b05,
0x80708170, 0xbf0d9973,
0xbfa20002, 0x84708970,
0xbfa00001, 0x84708a70,
@@ -3859,9 +3868,12 @@ static const uint32_t cwsr_trap_gfx12_hex[] = {
0xbf06817d, 0xbfa20002,
0xbeff0080, 0xbfa00001,
0xbeff00c1, 0xb8fb4306,
- 0x8b7bc17b, 0xbfa10042,
+ 0x8b7bc17b, 0xbfa10048,
0x8b7aff6d, 0x80000000,
- 0xbfa1003f, 0x847b897b,
+ 0xbfa10045, 0x847b897b,
+ 0xb8f0f802, 0xbf0d9e70,
+ 0xbfa10003, 0xbef000ff,
+ 0x00000800, 0xbfa00007,
0xb8f03b05, 0x80708170,
0xbf0d9973, 0xbfa20002,
0x84708970, 0xbfa00001,
@@ -3902,20 +3914,37 @@ static const uint32_t cwsr_trap_gfx12_hex[] = {
0xb8fb3b05, 0x807b817b,
0x847b827b, 0x857d9973,
0x8b7d817d, 0xbf06817d,
- 0xbfa2001b, 0xbefd0084,
- 0xbf0a7b7d, 0xbfa10032,
- 0x7e008700, 0x7e028701,
- 0x7e048702, 0x7e068703,
- 0x80767074, 0x82778075,
- 0xee0a4076, 0x000c0000,
- 0x00000000, 0xee0a4076,
- 0x008c0000, 0x00008000,
- 0xee0a4076, 0x010c0000,
- 0x00010000, 0xee0a4076,
- 0x018c0000, 0x00018000,
- 0x807d847d, 0x8070ff70,
- 0x00000200, 0xbf0a7b7d,
- 0xbfa2ffe9, 0xbfa0001a,
+ 0xbfa2003d, 0xbefd0084,
+ 0xb8faf802, 0xbf0a7b7d,
+ 0xbfa10053, 0x7e008700,
+ 0x7e028701, 0x7e048702,
+ 0x7e068703, 0x80767074,
+ 0x82778075, 0xee0a4076,
+ 0x000c0000, 0x00000000,
+ 0xee0a4076, 0x008c0000,
+ 0x00008000, 0xee0a4076,
+ 0x010c0000, 0x00010000,
+ 0xee0a4076, 0x018c0000,
+ 0x00018000, 0x807d847d,
+ 0xbf0d9e7a, 0xbfa10002,
+ 0xbf0a907d, 0xbfa10005,
+ 0x8070ff70, 0x00000200,
+ 0xbf0a7b7d, 0xbfa2ffe5,
+ 0xbfa00037, 0xbf0d906d,
+ 0xbfa20035, 0xbef00080,
+ 0x7e00027b, 0xed068070,
+ 0x00000000, 0x00000000,
+ 0xbef00084, 0xbf097b7d,
+ 0xbfa2002d, 0x7e008700,
+ 0x7e028701, 0x7e048702,
+ 0x7e068703, 0xed068070,
+ 0x00000000, 0x00000000,
+ 0xed068070, 0x00800000,
+ 0x00000400, 0xed068070,
+ 0x01000000, 0x00000800,
+ 0xed068070, 0x01800000,
+ 0x00000c00, 0x807d847d,
+ 0x80709070, 0xbfa0ffeb,
0xbefd0084, 0xbf0a7b7d,
0xbfa10017, 0x7e008700,
0x7e028701, 0x7e048702,
@@ -3929,17 +3958,20 @@ static const uint32_t cwsr_trap_gfx12_hex[] = {
0x00030000, 0x807d847d,
0x8070ff70, 0x00000400,
0xbf0a7b7d, 0xbfa2ffe9,
- 0xbfa0014c, 0xbef4007e,
+ 0xbfa001ab, 0xbef4007e,
0x8b75ff7f, 0x0000ffff,
0xbef1007f, 0xb8f20742,
0x84729972, 0x8b6eff7f,
- 0x04000000, 0xbfa10044,
+ 0x04000000, 0xbfa1004a,
0xbefe00c1, 0x857d9972,
0x8b7d817d, 0xbf06817d,
0xbfa20002, 0xbeff0080,
0xbfa00001, 0xbeff00c1,
0xb8ef4306, 0x8b6fc16f,
- 0xbfa10039, 0x846f896f,
+ 0xbfa1003f, 0x846f896f,
+ 0xb8f8f802, 0xbf0d9e78,
+ 0xbfa10003, 0xbef800ff,
+ 0x00000800, 0xbfa00007,
0xb8f83b05, 0x80788178,
0xbf0d9972, 0xbfa20002,
0x84788978, 0xbfa00001,
@@ -3972,134 +4004,178 @@ static const uint32_t cwsr_trap_gfx12_hex[] = {
0x857d9972, 0x8b7d817d,
0xbf06817d, 0xbfa20002,
0xbeff0080, 0xbfa00001,
- 0xbeff00c1, 0xb8ef3b05,
- 0x806f816f, 0x846f826f,
- 0x857d9972, 0x8b7d817d,
- 0xbf06817d, 0xbfa2002c,
- 0xbeee0078, 0x8078ff78,
- 0x00000200, 0xbefd0084,
- 0x80767874, 0x82778075,
- 0xee0a0076, 0x000c0000,
- 0x00000000, 0xee0a0076,
- 0x000c0001, 0x00008000,
- 0xee0a0076, 0x000c0002,
- 0x00010000, 0xee0a0076,
- 0x000c0003, 0x00018000,
- 0xbf8a0000, 0x7e008500,
- 0x7e028501, 0x7e048502,
- 0x7e068503, 0x807d847d,
+ 0xbeff00c1, 0xb8ecf802,
+ 0xbf0d9e6c, 0xbfa10002,
+ 0xbeef0090, 0xbfa00003,
+ 0xb8ef3b05, 0x806f816f,
+ 0x846f826f, 0x857d9972,
+ 0x8b7d817d, 0xbf06817d,
+ 0xbfa2002c, 0xbeee0078,
0x8078ff78, 0x00000200,
- 0xbf0a6f7d, 0xbfa2ffe8,
- 0x80766e74, 0x82778075,
- 0xee0a0076, 0x000c0000,
- 0x00000000, 0xee0a0076,
- 0x000c0001, 0x00008000,
- 0xee0a0076, 0x000c0002,
- 0x00010000, 0xee0a0076,
- 0x000c0003, 0x00018000,
- 0xbf8a0000, 0xbfa0002d,
- 0xbeee0078, 0x8078ff78,
- 0x00000400, 0xbefd0084,
- 0xbf0a6f7d, 0xbfa10018,
- 0x80767874, 0x82778075,
- 0xee0a0076, 0x000c0000,
- 0x00000000, 0xee0a0076,
- 0x000c0001, 0x00010000,
- 0xee0a0076, 0x000c0002,
- 0x00020000, 0xee0a0076,
- 0x000c0003, 0x00030000,
- 0xbf8a0000, 0x7e008500,
- 0x7e028501, 0x7e048502,
- 0x7e068503, 0x807d847d,
+ 0xbefd0084, 0x80767874,
+ 0x82778075, 0xee0a0076,
+ 0x000c0000, 0x00000000,
+ 0xee0a0076, 0x000c0001,
+ 0x00008000, 0xee0a0076,
+ 0x000c0002, 0x00010000,
+ 0xee0a0076, 0x000c0003,
+ 0x00018000, 0xbf8a0000,
+ 0x7e008500, 0x7e028501,
+ 0x7e048502, 0x7e068503,
+ 0x807d847d, 0x8078ff78,
+ 0x00000200, 0xbf0a6f7d,
+ 0xbfa2ffe8, 0x80766e74,
+ 0x82778075, 0xee0a0076,
+ 0x000c0000, 0x00000000,
+ 0xee0a0076, 0x000c0001,
+ 0x00008000, 0xee0a0076,
+ 0x000c0002, 0x00010000,
+ 0xee0a0076, 0x000c0003,
+ 0x00018000, 0xbf8a0000,
+ 0xbfa0002d, 0xbeee0078,
0x8078ff78, 0x00000400,
- 0xbf0a6f7d, 0xbfa2ffe8,
- 0x80766e74, 0x82778075,
- 0xee0a0076, 0x000c0000,
- 0x00000000, 0xee0a0076,
- 0x000c0001, 0x00010000,
- 0xee0a0076, 0x000c0002,
- 0x00020000, 0xee0a0076,
- 0x000c0003, 0x00030000,
- 0xbf8a0000, 0xb8f83b05,
- 0x80788178, 0xbf0d9972,
- 0xbfa20002, 0x84788978,
- 0xbfa00001, 0x84788a78,
- 0x8078ff78, 0x00000200,
- 0x80f8ff78, 0x00000060,
- 0x80767874, 0x82778075,
- 0xbefd00ff, 0x0000006c,
- 0xf460403b, 0xf8000000,
- 0xbf8a0000, 0x80fd847d,
- 0xbf800000, 0xbe804300,
- 0xbe824302, 0x80f6a076,
- 0x82f78077, 0xf460603b,
+ 0xbefd0084, 0xbf0a6f7d,
+ 0xbfa10018, 0x80767874,
+ 0x82778075, 0xee0a0076,
+ 0x000c0000, 0x00000000,
+ 0xee0a0076, 0x000c0001,
+ 0x00010000, 0xee0a0076,
+ 0x000c0002, 0x00020000,
+ 0xee0a0076, 0x000c0003,
+ 0x00030000, 0xbf8a0000,
+ 0x7e008500, 0x7e028501,
+ 0x7e048502, 0x7e068503,
+ 0x807d847d, 0x8078ff78,
+ 0x00000400, 0xbf0a6f7d,
+ 0xbfa2ffe8, 0x80766e74,
+ 0x82778075, 0xee0a0076,
+ 0x000c0000, 0x00000000,
+ 0xee0a0076, 0x000c0001,
+ 0x00010000, 0xee0a0076,
+ 0x000c0002, 0x00020000,
+ 0xee0a0076, 0x000c0003,
+ 0x00030000, 0xbf8a0000,
+ 0xb8f8f802, 0xbf0d9e78,
+ 0xbfa10003, 0xbef800ff,
+ 0x00000800, 0xbfa00007,
+ 0xb8f83b05, 0x80788178,
+ 0xbf0d9972, 0xbfa20002,
+ 0x84788978, 0xbfa00001,
+ 0x84788a78, 0x8078ff78,
+ 0x00000200, 0x80f8ff78,
+ 0x00000060, 0x80767874,
+ 0x82778075, 0xbefd00ff,
+ 0x0000006c, 0xf460403b,
0xf8000000, 0xbf8a0000,
- 0x80fd887d, 0xbf800000,
+ 0x80fd847d, 0xbf800000,
0xbe804300, 0xbe824302,
- 0xbe844304, 0xbe864306,
- 0x80f6c076, 0x82f78077,
- 0xf460803b, 0xf8000000,
- 0xbf8a0000, 0x80fd907d,
+ 0x80f6a076, 0x82f78077,
+ 0xf460603b, 0xf8000000,
+ 0xbf8a0000, 0x80fd887d,
0xbf800000, 0xbe804300,
0xbe824302, 0xbe844304,
- 0xbe864306, 0xbe884308,
- 0xbe8a430a, 0xbe8c430c,
- 0xbe8e430e, 0xbf06807d,
- 0xbfa1ffef, 0xb980f801,
- 0x00000000, 0xb8f83b05,
- 0x80788178, 0xbf0d9972,
- 0xbfa20002, 0x84788978,
- 0xbfa00001, 0x84788a78,
- 0x8078ff78, 0x00000200,
- 0x80767874, 0x82778075,
- 0xbeff0071, 0xf4601bfb,
- 0xf8000000, 0xf4601b3b,
- 0xf8000004, 0xf4601b7b,
- 0xf8000008, 0xf4601c3b,
- 0xf800000c, 0xf4601c7b,
- 0xf8000010, 0xf4601ebb,
- 0xf8000014, 0xf4601efb,
- 0xf8000018, 0xf4601e7b,
- 0xf800001c, 0xf4601cfb,
- 0xf8000020, 0xf4601bbb,
- 0xf8000024, 0xbf8a0000,
- 0xb96ef814, 0xf4601bbb,
- 0xf8000028, 0xbf8a0000,
- 0xb96ef815, 0xf4601bbb,
- 0xf800002c, 0xbf8a0000,
- 0xb96ef812, 0xf4601bbb,
- 0xf8000030, 0xbf8a0000,
- 0xb96ef813, 0x8b6eff7f,
- 0x04000000, 0xbfa1000b,
- 0xf4601bbb, 0xf8000038,
- 0xbf8a0000, 0xbf0d806e,
- 0xbfa10006, 0x856e906e,
- 0x8b6e6e6e, 0xbfa10003,
- 0xbe804ec1, 0x816ec16e,
- 0xbfa0fffb, 0xbefd006f,
- 0xbefe0070, 0xbeff0071,
- 0xb97b2011, 0x857b867b,
- 0xb97b0191, 0x857b827b,
- 0xb97bba11, 0xb973f801,
- 0xb8ee3b05, 0x806e816e,
+ 0xbe864306, 0x80f6c076,
+ 0x82f78077, 0xf460803b,
+ 0xf8000000, 0xbf8a0000,
+ 0x80fd907d, 0xbf800000,
+ 0xbe804300, 0xbe824302,
+ 0xbe844304, 0xbe864306,
+ 0xbe884308, 0xbe8a430a,
+ 0xbe8c430c, 0xbe8e430e,
+ 0xbf06807d, 0xbfa1ffef,
+ 0xb980f801, 0x00000000,
+ 0xb8f8f802, 0xbf0d9e78,
+ 0xbfa10003, 0xbef800ff,
+ 0x00000800, 0xbfa00007,
+ 0xb8f83b05, 0x80788178,
0xbf0d9972, 0xbfa20002,
- 0x846e896e, 0xbfa00001,
- 0x846e8a6e, 0x806eff6e,
- 0x00000240, 0x806e746e,
- 0x826f8075, 0xf4605c37,
- 0xf8000010, 0xf4605d37,
- 0xf8000020, 0xf4601e77,
- 0xf8000034, 0xbf8a0000,
- 0x8b6dff6d, 0x0000ffff,
- 0x8bfe7e7e, 0x8bea6a6a,
- 0x936eff77, 0x0002001a,
- 0xb96ef81a, 0xb97af804,
+ 0x84788978, 0xbfa00001,
+ 0x84788a78, 0x8078ff78,
+ 0x00000200, 0x80767874,
+ 0x82778075, 0xbeff0071,
+ 0xf4601bfb, 0xf8000000,
+ 0xf4601b3b, 0xf8000004,
+ 0xf4601b7b, 0xf8000008,
+ 0xf4601c3b, 0xf800000c,
+ 0xf4601c7b, 0xf8000010,
+ 0xf4601ebb, 0xf8000014,
+ 0xf4601efb, 0xf8000018,
+ 0xf4601e7b, 0xf800001c,
+ 0xf4601cfb, 0xf8000020,
+ 0xf4601bbb, 0xf8000024,
+ 0xbf8a0000, 0xb96ef814,
+ 0xf4601bbb, 0xf8000028,
+ 0xbf8a0000, 0xb96ef815,
+ 0xf4601bbb, 0xf800002c,
+ 0xbf8a0000, 0xb96ef812,
+ 0xf4601bbb, 0xf8000030,
+ 0xbf8a0000, 0xb96ef813,
+ 0x8b6eff7f, 0x04000000,
+ 0xbfa1000b, 0xf4601bbb,
+ 0xf8000038, 0xbf8a0000,
+ 0xbf0d806e, 0xbfa10006,
+ 0x856e906e, 0x8b6e6e6e,
+ 0xbfa10003, 0xbe804ec1,
+ 0x816ec16e, 0xbfa0fffb,
+ 0xbef80080, 0xb8f6f802,
+ 0xbf0d9e76, 0xbfa10038,
+ 0xbefe00c1, 0xbef80080,
+ 0xed050078, 0x00000000,
+ 0x00000000, 0xbfc00000,
+ 0xd7600077, 0x00010100,
+ 0xbef60077, 0xbe805376,
+ 0xbfa20006, 0xb8f6f811,
+ 0x8b76a076, 0xbf830010,
+ 0xbfa1fff9, 0xbef80081,
+ 0xbfa0001a, 0xbefe00c1,
+ 0xbefd0090, 0xbef80084,
+ 0xbf0a777d, 0xbfa10015,
+ 0xed050078, 0x00000000,
+ 0x00000000, 0xed050078,
+ 0x00000001, 0x00000400,
+ 0xed050078, 0x00000002,
+ 0x00000800, 0xed050078,
+ 0x00000003, 0x00000c00,
+ 0xbfc00000, 0x7e008500,
+ 0x7e028501, 0x7e048502,
+ 0x7e068503, 0x807d847d,
+ 0x80789078, 0xbf0a777d,
+ 0xbfa2ffeb, 0xee0a0074,
+ 0x000c0000, 0x00000000,
+ 0xee0a0074, 0x000c0001,
+ 0x00008000, 0xee0a0074,
+ 0x000c0002, 0x00010000,
+ 0xee0a0074, 0x000c0003,
+ 0x00018000, 0xbf8a0000,
+ 0xbefd006f, 0xbefe0070,
+ 0xbeff0071, 0xb97b2011,
+ 0x857b867b, 0xb97b0191,
+ 0x857b827b, 0xb97bba11,
+ 0xb973f801, 0xb8eef802,
+ 0xbf0d9e6e, 0xbfa10003,
+ 0xbeee00ff, 0x00000800,
+ 0xbfa00007, 0xb8ee3b05,
+ 0x806e816e, 0xbf0d9972,
+ 0xbfa20002, 0x846e896e,
+ 0xbfa00001, 0x846e8a6e,
+ 0x806eff6e, 0x00000240,
+ 0x806e746e, 0x826f8075,
+ 0xf4605c37, 0xf8000010,
+ 0xf4605d37, 0xf8000020,
+ 0xf4601e77, 0xf8000034,
+ 0xbf8a0000, 0x8b6dff6d,
+ 0x0000ffff, 0x8bfe7e7e,
+ 0x8bea6a6a, 0xbf068178,
+ 0xbfa20007, 0x936eff77,
+ 0x0002001a, 0xb96ef81a,
+ 0xb97af804, 0xbe804ec2,
+ 0xbf94fffe, 0xbe804a6c,
+ 0xbef8007a, 0x8c6dff6d,
+ 0x00010000, 0xbfa0fc47,
0xbe804ec2, 0xbf94fffe,
- 0xbe804a6c, 0xbe804ec2,
- 0xbf94fffe, 0xbfb10000,
+ 0xbfb10000, 0xbf9f0000,
0xbf9f0000, 0xbf9f0000,
0xbf9f0000, 0xbf9f0000,
- 0xbf9f0000, 0x00000000,
};
static const uint32_t cwsr_trap_gfx9_5_0_hex[] = {
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm
b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm
index d38ff404277b..355b5175f141 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm
@@ -38,6 +38,7 @@
#define CLUSTER_BARRIER_SERIALIZE_WORKAROUND (ASIC_FAMILY == CHIP_GC_12_0_3)
#define RELAXED_SCHEDULING_IN_TRAP (ASIC_FAMILY == CHIP_GFX12)
#define HAVE_INSTRUCTION_FIXUP (ASIC_FAMILY == CHIP_GC_12_0_3)
+#define HAVE_DYNAMIC_VGPRS (ASIC_FAMILY == CHIP_GFX12)
#define SINGLE_STEP_MISSED_WORKAROUND 1 //workaround for lost TRAP_AFTER_INST exception when SAVECTX raised
#define HAVE_VALU_SGPR_HAZARD (ASIC_FAMILY == CHIP_GFX12)
@@ -60,16 +61,20 @@ var SQ_WAVE_STATE_PRIV_POISON_ERR_SHIFT = 15
var SQ_WAVE_STATUS_WAVE64_SHIFT = 29
var SQ_WAVE_STATUS_WAVE64_SIZE = 1
var SQ_WAVE_STATUS_NO_VGPRS_SHIFT = 24
+var SQ_WAVE_STATUS_DYN_VGPR_EN_SHIFT = 30
var SQ_WAVE_STATUS_IN_WG_SHIFT = 11
var SQ_WAVE_STATE_PRIV_ALWAYS_CLEAR_MASK =
SQ_WAVE_STATE_PRIV_SYS_PRIO_MASK|SQ_WAVE_STATE_PRIV_POISON_ERR_MASK
var S_SAVE_PC_HI_TRAP_ID_MASK = 0xF0000000
var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT = 12
var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE = 9
+var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_MASK = 0xFF000
var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE = 8
var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 12
#if ASIC_FAMILY < CHIP_GC_12_0_3
+var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT = 24
+var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE = 4
var SQ_WAVE_LDS_ALLOC_GRANULARITY = 9
#else
var SQ_WAVE_LDS_ALLOC_GRANULARITY = 10
@@ -185,6 +190,34 @@ var S_RESTORE_SPI_INIT_FIRST_WAVE_MASK =
0x04000000
var S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT = 26
var S_WAVE_SIZE = 25
+#if HAVE_DYNAMIC_VGPRS
+var WAVE_MIN_NUM_DVGPRS = 16 // Defined by
SQ_DYN_VGPR.block_size. Assume default value for now.
+
+// The total VGPR count of a DVGPR wave is persisted in the wave's own
scratch, in a small
+// region reserved at the front of the dynamic-VGPR backing store (ahead of
the dynamic
+// VGPRs themselves).
+//
+// NOTE: the driver-side dynamic-VGPR scratch allocation must include these
reserved bytes
+// per lane in addition to the space for the dynamic VGPRs.
+var S_SAVE_DVGPR_COUNT_SCRATCH_OFFSET = 0 // count location
within the reserved region
+var S_SAVE_DVGPR_SCRATCH_RESERVE = 4 // bytes reserved per
lane at scratch offset 0
+
+// DVGPR restore-allocation bypass. If the restore-time s_alloc_vgpr fails
while a
+// context-save is pending, the wave cannot restore its dynamic VGPRs. Rather
than
+// dropping them (corruption) or spinning forever, the wave re-enters the save
path
+// (L_SAVE_BYPASS) so it is re-saved and retried on a later restore when pool
space frees.
+//
+// On such a re-save the live allocation has collapsed to the minimum block,
so the save
+// must NOT overwrite the durable scratch count (that would lose the real
count and strand
+// the dynamic VGPRs still sitting in scratch). A freshly dispatched wave
never passes
+// through restore, so a "this is a bypass re-save" marker cannot live in
scratch (its first
+// save would read it uninitialized); it is instead carried for the single
trap execution in
+// a free PC_HI bit. On GFX12 the PC occupies PC_HI[15:0] (48-bit address)
and there are no
+// banked-VGPR bits, so PC_HI[30:16] are free; the bit is masked off before
PC_HI is written
+// to memory, so it is never persisted.
+var S_SAVE_PC_HI_DVGPR_BYPASS_SHIFT = 16
+var S_SAVE_PC_HI_DVGPR_BYPASS_MASK = 0x00010000
+#endif
var s_restore_spi_init_lo = exec_lo
var s_restore_spi_init_hi = exec_hi
var s_restore_mem_offset = ttmp12
@@ -196,6 +229,8 @@ var s_restore_mode = ttmp7
var s_restore_flat_scratch = s_restore_tmp
var s_restore_pc_lo = ttmp0
var s_restore_pc_hi = ttmp1
+// Scratch register, valid only before L_RESTORE_HWREG loads the saved PC.
+var s_restore_tmp2 = s_restore_pc_lo
var s_restore_exec_lo = ttmp4
var s_restore_exec_hi = ttmp5
var s_restore_state_priv = ttmp14
@@ -210,6 +245,20 @@ var s_restore_ttmps_lo =
s_restore_tmp
var s_restore_ttmps_hi = s_restore_alloc_size
var s_restore_spi_init_hi_save = s_restore_exec_hi
+#if HAVE_DYNAMIC_VGPRS
+// Temporaries for the dynamic-VGPR restore, which runs after L_RESTORE_HWREG
(scratch is not
+// addressable before SCRATCH_BASE is restored).
+var s_restore_dvgpr_tmp = s_restore_addr_lo
+var s_restore_dvgpr_count = s_restore_addr_hi
+
+// s_restore_mem_offset (ttmp12) is dead as a scratch offset once the
dynamic-VGPR restore
+// finishes, so it is reused to carry the rfe-vs-resave decision to the end of
restore:
+// 0 == resume the shader normally, 1 == bypass to the save path (re-save).
The dynamic-VGPR
+// restore loop only ever leaves the register 0 or >=
S_SAVE_DVGPR_SCRATCH_RESERVE (its
+// starting offset, which it only increments), so 1 is an unambiguous marker.
+var s_restore_dvgpr_bypass = s_restore_mem_offset
+#endif
+
#if SAVE_TTMPS_IN_SGPR_BLOCK
var TTMP_SR_OFFSET_FROM_HWREG = -0x40
#else
@@ -409,6 +458,15 @@ L_SAVE:
s_endpgm
L_HAVE_VGPRS:
s_and_b32 s_save_pc_hi, s_save_pc_hi, ADDRESS_HI32_MASK
+#if HAVE_DYNAMIC_VGPRS
+// Re-entry point for the DVGPR restore-allocation bypass (see
L_DYN_VGPR_ALLOC_LOOP).
+// On entry from restore s_save_state_priv has been populated from
s_restore_state_priv and
+// the bypass marker bit (S_SAVE_PC_HI_DVGPR_BYPASS_MASK) has been set in
PC_HI -- this entry
+// skips the PC_HI mask at L_HAVE_VGPRS, so the marker survives to the
dynamic-VGPR save,
+// where it suppresses the durable scratch count write (the count and the
dynamic VGPRs are
+// already intact in scratch from the previous save).
+L_SAVE_BYPASS:
+#endif
s_mov_b32 s_save_tmp, 0
s_setreg_b32 hwreg(HW_REG_WAVE_EXCP_FLAG_PRIV,
SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_SHIFT, 1), s_save_tmp //clear saveCtx
bit
@@ -798,6 +856,9 @@ L_SAVE_VGPR_NORMAL:
// VGPR store using dw burst
s_mov_b32 m0, 0x4
//VGPR initial index value =4
+#if HAVE_DYNAMIC_VGPRS
+ s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_STATUS)
//save to check DYN_VGPR_EN later
+#endif
s_cmp_lt_u32 m0, s_save_alloc_size
s_cbranch_scc0 L_SAVE_VGPR_END
@@ -815,12 +876,51 @@ L_SAVE_VGPR_W32_LOOP:
global_store_addtid_b32 v3, [s_save_addr_lo, s_save_addr_hi]
scope:SCOPE_SYS offset:128*3
s_add_u32 m0, m0, 4 //next vgpr index
+#if HAVE_DYNAMIC_VGPRS
+ s_bitcmp1_b32 s_save_tmp, SQ_WAVE_STATUS_DYN_VGPR_EN_SHIFT
+ s_cbranch_scc0 L_SAVE_FIXED_VGPR_END_LOOP_CHECK
+ s_cmp_lt_u32 m0, WAVE_MIN_NUM_DVGPRS
+ s_cbranch_scc0 L_SAVE_DYN_VGPR_SAVE
+
+L_SAVE_FIXED_VGPR_END_LOOP_CHECK:
+#endif
s_add_u32 s_save_mem_offset, s_save_mem_offset, 128*4
//every buffer_store_dword does 128 bytes
s_cmp_lt_u32 m0, s_save_alloc_size
//scc = (m0 < s_save_alloc_size) ? 1 : 0
s_cbranch_scc1 L_SAVE_VGPR_W32_LOOP
//VGPR save is complete?
-
s_branch L_SAVE_VGPR_END
+#if HAVE_DYNAMIC_VGPRS
+L_SAVE_DYN_VGPR_SAVE:
+ s_bitcmp1_b32 s_save_pc_hi, S_SAVE_PC_HI_DVGPR_BYPASS_SHIFT
+ s_cbranch_scc1 L_SAVE_VGPR_END
+
+ // Persist the total VGPR count in the reserved scratch slot (durable
per wave).
+ // s_save_alloc_size holds the live total VGPR count
(HW_REG_WAVE_GPR_ALLOC).
+ s_mov_b32 s_save_mem_offset, S_SAVE_DVGPR_COUNT_SCRATCH_OFFSET
+ v_mov_b32 v0, s_save_alloc_size
+ scratch_store_b32 off, v0, s_save_mem_offset offset: 0
+
+ // m0 already holds WAVE_MIN_NUM_DVGPRS: the fixed-VGPR loop steps by 4
and branches here
+ // on the first index at or past the block boundary (must match
RESTORE).
+ s_mov_b32 s_save_mem_offset, S_SAVE_DVGPR_SCRATCH_RESERVE
+
+L_SAVE_DYN_VGPR_SAVE_LOOP:
+ s_cmp_ge_u32 m0, s_save_alloc_size
+ s_cbranch_scc1 L_SAVE_VGPR_END
+
+ v_movrels_b32 v0, v0
//v0 = v[0+m0]
+ v_movrels_b32 v1, v1
//v1 = v[1+m0]
+ v_movrels_b32 v2, v2
//v2 = v[2+m0]
+ v_movrels_b32 v3, v3
//v3 = v[3+m0]
+ scratch_store_b32 off, v0, s_save_mem_offset offset: 0
+ scratch_store_b32 off, v1, s_save_mem_offset offset: 4
+ scratch_store_b32 off, v2, s_save_mem_offset offset: 8
+ scratch_store_b32 off, v3, s_save_mem_offset offset: 12
+ s_add_u32 m0, m0, 4
+ s_add_u32 s_save_mem_offset, s_save_mem_offset, 16
+ s_branch L_SAVE_DYN_VGPR_SAVE_LOOP
+#endif
+
L_SAVE_VGPR_WAVE64:
// VGPR store using dw burst
s_mov_b32 m0, 0x4
//VGPR initial index value =4
@@ -934,9 +1034,23 @@ L_RESTORE_VGPR:
L_ENABLE_RESTORE_VGPR_EXEC_HI:
s_mov_b32 exec_hi, 0xFFFFFFFF
L_RESTORE_VGPR_NORMAL:
+#if HAVE_DYNAMIC_VGPRS
+ // For DVGPR waves this loop restores only the fixed VGPR block, so cap
the loop bound at
+ // WAVE_MIN_NUM_DVGPRS rather than taking the live GPR_ALLOC size. The
dynamic VGPRs are
+ // restored later, from scratch, once SCRATCH_BASE has been restored
(it is not yet
+ // restored at this point) and their count is known.
+ s_getreg_b32 s_restore_tmp2, hwreg(HW_REG_WAVE_STATUS)
+ s_bitcmp1_b32 s_restore_tmp2, SQ_WAVE_STATUS_DYN_VGPR_EN_SHIFT
+ s_cbranch_scc0 L_RESTORE_ALLOC_SIZE_FROM_HW // Not DVGPR, use HW
register
+ s_mov_b32 s_restore_alloc_size, WAVE_MIN_NUM_DVGPRS
+ s_branch L_RESTORE_ALLOC_SIZE_DONE
