Module: Mesa Branch: main Commit: 6f76f4db9f500d37a8666223ae6c8d108cdd9c91 URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=6f76f4db9f500d37a8666223ae6c8d108cdd9c91
Author: Faith Ekstrand <faith.ekstr...@collabora.com> Date: Mon Nov 20 14:39:04 2023 -0600 nak: Make MemAddrType a part of MemSpace This is similar to what we did with scopes and ordering. Shared and local memory can only have an A32 address type. The only times we can get A64 is for global or generic. Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/26291> --- src/nouveau/compiler/nak_encode_sm70.rs | 17 ++++++----------- src/nouveau/compiler/nak_from_nir.rs | 22 ++++++---------------- src/nouveau/compiler/nak_ir.rs | 26 +++++++++++++++++--------- src/nouveau/compiler/nak_lower_copy_swap.rs | 2 -- 4 files changed, 29 insertions(+), 38 deletions(-) diff --git a/src/nouveau/compiler/nak_encode_sm70.rs b/src/nouveau/compiler/nak_encode_sm70.rs index f11ade5622a..4e1ce63fd9f 100644 --- a/src/nouveau/compiler/nak_encode_sm70.rs +++ b/src/nouveau/compiler/nak_encode_sm70.rs @@ -1263,7 +1263,7 @@ impl SM70Instr { fn set_mem_access(&mut self, access: &MemAccess) { self.set_field( 72..73, - match access.addr_type { + match access.space.addr_type() { MemAddrType::A32 => 0_u8, MemAddrType::A64 => 1_u8, }, @@ -1291,7 +1291,6 @@ impl SM70Instr { self.set_reg_src(24..32, op.addr); self.set_field(40..64, op.offset); - assert!(op.access.addr_type == MemAddrType::A32); self.set_mem_type(73..76, op.access.mem_type); assert!(op.access.order == MemOrder::Strong(MemScope::CTA)); assert!(op.access.eviction_priority == MemEvictionPriority::Normal); @@ -1304,7 +1303,6 @@ impl SM70Instr { self.set_reg_src(24..32, op.addr); self.set_field(40..64, op.offset); - assert!(op.access.addr_type == MemAddrType::A32); self.set_mem_type(73..76, op.access.mem_type); assert!(op.access.order == MemOrder::Strong(MemScope::CTA)); assert!(op.access.eviction_priority == MemEvictionPriority::Normal); @@ -1314,7 +1312,7 @@ impl SM70Instr { fn encode_ld(&mut self, op: &OpLd) { match op.access.space { - MemSpace::Global => self.encode_ldg(op), + MemSpace::Global(_) => self.encode_ldg(op), MemSpace::Local => self.encode_ldl(op), MemSpace::Shared => self.encode_lds(op), } @@ -1351,7 +1349,6 @@ impl SM70Instr { self.set_reg_src(32..40, op.data); self.set_field(40..64, op.offset); - assert!(op.access.addr_type == MemAddrType::A32); self.set_mem_type(73..76, op.access.mem_type); assert!(op.access.order == MemOrder::Strong(MemScope::CTA)); assert!(op.access.eviction_priority == MemEvictionPriority::Normal); @@ -1364,7 +1361,6 @@ impl SM70Instr { self.set_reg_src(32..40, op.data); self.set_field(40..64, op.offset); - assert!(op.access.addr_type == MemAddrType::A32); self.set_mem_type(73..76, op.access.mem_type); assert!(op.access.order == MemOrder::Strong(MemScope::CTA)); assert!(op.access.eviction_priority == MemEvictionPriority::Normal); @@ -1372,7 +1368,7 @@ impl SM70Instr { fn encode_st(&mut self, op: &OpSt) { match op.access.space { - MemSpace::Global => self.encode_stg(op), + MemSpace::Global(_) => self.encode_stg(op), MemSpace::Local => self.encode_stl(op), MemSpace::Shared => self.encode_sts(op), } @@ -1434,7 +1430,7 @@ impl SM70Instr { self.set_field( 72..73, - match op.addr_type { + match op.mem_space.addr_type() { MemAddrType::A32 => 0_u8, MemAddrType::A64 => 1_u8, }, @@ -1463,7 +1459,6 @@ impl SM70Instr { self.set_reg_src(24..32, op.addr); self.set_field(40..64, op.addr_offset); - assert!(op.addr_type == MemAddrType::A32); assert!(op.mem_order == MemOrder::Strong(MemScope::CTA)); assert!(op.mem_eviction_priority == MemEvictionPriority::Normal); @@ -1472,7 +1467,7 @@ impl SM70Instr { fn encode_atom(&mut self, op: &OpAtom) { match op.mem_space { - MemSpace::Global => self.encode_atomg(op), + MemSpace::Global(_) => self.encode_atomg(op), MemSpace::Local => panic!("Atomics do not support local"), MemSpace::Shared => self.encode_atoms(op), } @@ -1564,7 +1559,7 @@ impl SM70Instr { } fn encode_cctl(&mut self, op: &OpCCtl) { - assert!(op.mem_space == MemSpace::Global); + assert!(matches!(op.mem_space, MemSpace::Global(_))); self.set_opcode(0x98f); self.set_reg_src(24..32, op.addr); diff --git a/src/nouveau/compiler/nak_from_nir.rs b/src/nouveau/compiler/nak_from_nir.rs index 40550924b37..f48860e6a00 100644 --- a/src/nouveau/compiler/nak_from_nir.rs +++ b/src/nouveau/compiler/nak_from_nir.rs @@ -1527,9 +1527,8 @@ impl<'a> ShaderFromNir<'a> { data: data, atom_op: atom_op, atom_type: atom_type, - addr_type: MemAddrType::A64, addr_offset: offset, - mem_space: MemSpace::Global, + mem_space: MemSpace::Global(MemAddrType::A64), mem_order: MemOrder::Strong(MemScope::System), mem_eviction_priority: MemEvictionPriority::Normal, // Note: no intrinic access }); @@ -1553,9 +1552,8 @@ impl<'a> ShaderFromNir<'a> { data: data, atom_op: AtomOp::CmpExch, atom_type: atom_type, - addr_type: MemAddrType::A64, addr_offset: offset, - mem_space: MemSpace::Global, + mem_space: MemSpace::Global(MemAddrType::A64), mem_order: MemOrder::Strong(MemScope::System), mem_eviction_priority: MemEvictionPriority::Normal, // Note: no intrinic access }); @@ -1634,9 +1632,8 @@ impl<'a> ShaderFromNir<'a> { MemOrder::Strong(MemScope::System) }; let access = MemAccess { - addr_type: MemAddrType::A64, mem_type: MemType::from_size(size_B, false), - space: MemSpace::Global, + space: MemSpace::Global(MemAddrType::A64), order: order, eviction_priority: self .get_eviction_priority(intrin.access()), @@ -1748,7 +1745,6 @@ impl<'a> ShaderFromNir<'a> { (intrin.def.bit_size() / 8) * intrin.def.num_components(); assert!(u32::from(size_B) <= intrin.align()); let access = MemAccess { - addr_type: MemAddrType::A32, mem_type: MemType::from_size(size_B, false), space: MemSpace::Local, order: MemOrder::Strong(MemScope::CTA), @@ -1770,7 +1766,6 @@ impl<'a> ShaderFromNir<'a> { (intrin.def.bit_size() / 8) * intrin.def.num_components(); assert!(u32::from(size_B) <= intrin.align()); let access = MemAccess { - addr_type: MemAddrType::A32, mem_type: MemType::from_size(size_B, false), space: MemSpace::Shared, order: MemOrder::Strong(MemScope::CTA), @@ -1845,7 +1840,7 @@ impl<'a> ShaderFromNir<'a> { { b.push_op(OpCCtl { op: CCtlOp::WBAll, - mem_space: MemSpace::Global, + mem_space: MemSpace::Global(MemAddrType::A64), addr: 0.into(), addr_offset: 0, }); @@ -1875,7 +1870,7 @@ impl<'a> ShaderFromNir<'a> { { b.push_op(OpCCtl { op: CCtlOp::IVAll, - mem_space: MemSpace::Global, + mem_space: MemSpace::Global(MemAddrType::A64), addr: 0.into(), addr_offset: 0, }); @@ -1957,7 +1952,6 @@ impl<'a> ShaderFromNir<'a> { data: data, atom_op: atom_op, atom_type: atom_type, - addr_type: MemAddrType::A32, addr_offset: offset, mem_space: MemSpace::Shared, mem_order: MemOrder::Strong(MemScope::CTA), @@ -1983,7 +1977,6 @@ impl<'a> ShaderFromNir<'a> { data: data, atom_op: AtomOp::CmpExch, atom_type: atom_type, - addr_type: MemAddrType::A32, addr_offset: offset, mem_space: MemSpace::Shared, mem_order: MemOrder::Strong(MemScope::CTA), @@ -1997,9 +1990,8 @@ impl<'a> ShaderFromNir<'a> { (srcs[0].bit_size() / 8) * srcs[0].num_components(); assert!(u32::from(size_B) <= intrin.align()); let access = MemAccess { - addr_type: MemAddrType::A64, mem_type: MemType::from_size(size_B, false), - space: MemSpace::Global, + space: MemSpace::Global(MemAddrType::A64), order: MemOrder::Strong(MemScope::System), eviction_priority: self .get_eviction_priority(intrin.access()), @@ -2035,7 +2027,6 @@ impl<'a> ShaderFromNir<'a> { (srcs[0].bit_size() / 8) * srcs[0].num_components(); assert!(u32::from(size_B) <= intrin.align()); let access = MemAccess { - addr_type: MemAddrType::A32, mem_type: MemType::from_size(size_B, false), space: MemSpace::Local, order: MemOrder::Strong(MemScope::CTA), @@ -2056,7 +2047,6 @@ impl<'a> ShaderFromNir<'a> { (srcs[0].bit_size() / 8) * srcs[0].num_components(); assert!(u32::from(size_B) <= intrin.align()); let access = MemAccess { - addr_type: MemAddrType::A32, mem_type: MemType::from_size(size_B, false), space: MemSpace::Shared, order: MemOrder::Strong(MemScope::CTA), diff --git a/src/nouveau/compiler/nak_ir.rs b/src/nouveau/compiler/nak_ir.rs index 4aee9455799..4d2f3dff7bc 100644 --- a/src/nouveau/compiler/nak_ir.rs +++ b/src/nouveau/compiler/nak_ir.rs @@ -1851,15 +1851,25 @@ impl fmt::Display for MemScope { #[derive(Clone, Copy, Eq, Hash, PartialEq)] pub enum MemSpace { - Global, + Global(MemAddrType), Local, Shared, } +impl MemSpace { + pub fn addr_type(&self) -> MemAddrType { + match self { + MemSpace::Global(t) => *t, + MemSpace::Local => MemAddrType::A32, + MemSpace::Shared => MemAddrType::A32, + } + } +} + impl fmt::Display for MemSpace { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - MemSpace::Global => write!(f, ".global"), + MemSpace::Global(t) => write!(f, ".global{t}"), MemSpace::Local => write!(f, ".local"), MemSpace::Shared => write!(f, ".shared"), } @@ -1888,7 +1898,6 @@ impl fmt::Display for MemEvictionPriority { #[derive(Clone)] pub struct MemAccess { - pub addr_type: MemAddrType, pub mem_type: MemType, pub space: MemSpace, pub order: MemOrder, @@ -1899,8 +1908,7 @@ impl fmt::Display for MemAccess { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "{}{}{}{}{}", - self.addr_type, + "{}{}{}{}", self.mem_type, self.space, self.order, @@ -3290,7 +3298,6 @@ pub struct OpAtom { pub atom_op: AtomOp, pub atom_type: AtomType, - pub addr_type: MemAddrType, pub addr_offset: i32, pub mem_space: MemSpace, @@ -3302,9 +3309,10 @@ impl DisplayOp for OpAtom { fn fmt_op(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "atom{}{}{}{}", + "atom{}{}{}{}{}", self.atom_op, self.atom_type, + self.mem_space, self.mem_order, self.mem_eviction_priority, )?; @@ -4655,8 +4663,8 @@ impl Instr { pub fn writes_global_mem(&self) -> bool { match &self.op { - Op::Atom(op) => op.mem_space == MemSpace::Global, - Op::St(op) => op.access.space == MemSpace::Global, + Op::Atom(op) => matches!(op.mem_space, MemSpace::Global(_)), + Op::St(op) => matches!(op.access.space, MemSpace::Global(_)), Op::SuAtom(_) | Op::SuSt(_) => true, _ => false, } diff --git a/src/nouveau/compiler/nak_lower_copy_swap.rs b/src/nouveau/compiler/nak_lower_copy_swap.rs index 292dacfc2ae..476310cc457 100644 --- a/src/nouveau/compiler/nak_lower_copy_swap.rs +++ b/src/nouveau/compiler/nak_lower_copy_swap.rs @@ -45,7 +45,6 @@ impl LowerCopySwap { } RegFile::Mem => { let access = MemAccess { - addr_type: MemAddrType::A32, mem_type: MemType::B32, space: MemSpace::Local, order: MemOrder::Strong(MemScope::CTA), @@ -101,7 +100,6 @@ impl LowerCopySwap { SrcRef::Reg(src_reg) => match src_reg.file() { RegFile::GPR => { let access = MemAccess { - addr_type: MemAddrType::A32, mem_type: MemType::B32, space: MemSpace::Local, order: MemOrder::Strong(MemScope::CTA),
