Daecheol You has uploaded this change for review. (
https://gem5-review.googlesource.com/c/public/gem5/+/56610 )
Change subject: mem-ruby: Memory range configuration for NUMA system
......................................................................
mem-ruby: Memory range configuration for NUMA system
When system is configured for NUMA, it has multiple memory ranges,
and each memory range is mapped to a corresponding NUMA node.
However, existing memory configuration maps each LLC/memory
controller to all memory ranges.
The change enables the CHI protocol based system to be configured as NUMA.
Two main changes below:
1. numa_nodes attribute in CHI_Node's NoC_Params
- It describes NUMA node ID where each CHI node belongs
- Simple NoC configuration example was added
2. Memory range mapping only to a corresponding NUMA node
- LLC/memory controllers get only a memory range of the
NUMA node where they belong
Jira Issue:https://gem5.atlassian.net/browse/GEM5-1187
Change-Id: If4a8f3ba9aac9f74125970f63410883d2ad32f01
---
A configs/example/noc_config/2x4_numa.py
M configs/ruby/CHI.py
M configs/ruby/CHI_config.py
M configs/ruby/Ruby.py
M src/sim/System.py
5 files changed, 217 insertions(+), 22 deletions(-)
diff --git a/configs/example/noc_config/2x4_numa.py
b/configs/example/noc_config/2x4_numa.py
new file mode 100644
index 0000000..5560c26
--- /dev/null
+++ b/configs/example/noc_config/2x4_numa.py
@@ -0,0 +1,86 @@
+# Copyright (c) 2022 Samsung Electronics Co., Ltd.
+# Copyright (c) 2021 ARM Limited
+# All rights reserved.
+#
+# The license below extends only to copyright in the software and shall
+# not be construed as granting a license to any other intellectual
+# property including but not limited to intellectual property relating
+# to a hardware implementation of the functionality of the software
+# licensed hereunder. You may use the software subject to the license
+# terms below provided that you ensure that this notice is replicated
+# unmodified and in its entirety in all distributions of the software,
+# modified or unmodified, in source code or in binary form.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from ruby import CHI_config
+
+# CustomMesh parameters for a 2x4 mesh. Routers will have the following
layout:
+#
+# 0 --- 1 --- 2 --- 3
+# | | | |
+# 4 --- 5 --- 6 --- 7
+#
+# 2x4 mesh is split into four numa nodes.
+# Numa node0 consists of router 0, 1 (HNF0, SNF0)
+# Numa node1 consists of router 2, 3 (HNF1, SNF1)
+# Numa node2 consists of router 4, 5 (HNF2, SNF2)
+# Numa node3 consists of router 6, 7 (HNF3, SNF3)
+#
+# Default parameter are configs/ruby/CHI_config.py
+#
+class NoC_Params(CHI_config.NoC_Params):
+ num_rows = 2
+ num_cols = 4
+
+# Specialization of nodes to define bindings for each CHI node type
+# needed by CustomMesh.
+# The default types are defined in CHI_Node and their derivatives in
+# configs/ruby/CHI_config.py
+
+class CHI_RNF(CHI_config.CHI_RNF):
+ class NoC_Params(CHI_config.CHI_RNF.NoC_Params):
+ router_list = [1, 2, 5, 6]
+
+class CHI_HNF(CHI_config.CHI_HNF):
+ class NoC_Params(CHI_config.CHI_HNF.NoC_Params):
+ router_list = [1, 2, 5, 6]
+ numa_nodes = [0, 1, 2, 3]
+
+class CHI_SNF_MainMem(CHI_config.CHI_SNF_MainMem):
+ class NoC_Params(CHI_config.CHI_SNF_MainMem.NoC_Params):
+ router_list = [0, 3, 4, 7]
+ numa_nodes = [0, 1, 2, 3]
+
+class CHI_SNF_BootMem(CHI_config.CHI_SNF_BootMem):
+ class NoC_Params(CHI_config.CHI_SNF_BootMem.NoC_Params):
+ router_list = [0]
+
+class CHI_RNI_DMA(CHI_config.CHI_RNI_DMA):
+ class NoC_Params(CHI_config.CHI_RNI_DMA.NoC_Params):
+ router_list = [1, 2, 5, 6]
+
+class CHI_RNI_IO(CHI_config.CHI_RNI_IO):
+ class NoC_Params(CHI_config.CHI_RNI_IO.NoC_Params):
+ router_list = [1, 2, 5 , 6]
diff --git a/configs/ruby/CHI.py b/configs/ruby/CHI.py
index e4a2477..f961e8a 100644
--- a/configs/ruby/CHI.py
+++ b/configs/ruby/CHI.py
@@ -1,3 +1,4 @@
+# Copyright (c) 2022 Samsung Electronics Co., Ltd.
# Copyright (c) 2021 ARM Limited
# All rights reserved.
#
@@ -156,8 +157,42 @@
for m in other_memories:
sysranges.append(m.range)
- CHI_HNF.createAddrRanges(sysranges, system.cache_line_size.value,
- options.num_l3caches)
+ # Check NUMA configuration is set
+ if isinstance(CHI_HNF.NoC_Params.numa_nodes, list):
+ num_numa_nodes = max(CHI_HNF.NoC_Params.numa_nodes) + 1
+ assert(num_numa_nodes == len(system.mem_ranges))
+ assert(num_numa_nodes ==
+ max(CHI_SNF_MainMem.NoC_Params.numa_nodes) + 1)
+ numa_config = True
+
+ if numa_config:
+ numa_nodes = CHI_HNF.NoC_Params.numa_nodes
+ hnf_index = 0
+ # Create address range for each NUMA node
+ for i in range(num_numa_nodes):
+ # Assumes node numbers with the same value
+ # are described consecutively
+ # For example, numa_nodes = [0, 0, 1, 1, ...]
+ node_index = numa_nodes[hnf_index]
+ hnf_list = []
+ while (hnf_index < options.num_l3caches and
+ node_index == numa_nodes[hnf_index]):
+ hnf_list.append(hnf_index)
+ hnf_index = hnf_index + 1
+
+ mem_ranges = []
+ mem_ranges.append(system.mem_ranges[node_index])
+ # 'other_memories' is assigned to node 0
+ if node_index == 0:
+ for m in other_memories:
+ mem_ranges.append(m.range)
+ CHI_HNF.createAddrRanges(mem_ranges,
+ system.cache_line_size.value,
+ hnf_list)
+ else:
+ hnf_list = [i for i in range(options.num_l3caches)]
+ CHI_HNF.createAddrRanges(sysranges, system.cache_line_size.value,
+ hnf_list)
ruby_system.hnf = [ CHI_HNF(i, ruby_system, HNFCache, None)
for i in range(options.num_l3caches) ]
@@ -172,6 +207,9 @@
# Notice we don't define a Directory_Controller type so we don't use
# create_directories shared by other protocols.
+ system.numa_config = numa_config
+ system.numa_nodes = (CHI_SNF_MainMem.NoC_Params.numa_nodes
+ if numa_config else [])
ruby_system.snf = [ CHI_SNF_MainMem(ruby_system, None, None)
for i in range(options.num_dirs) ]
for snf in ruby_system.snf:
diff --git a/configs/ruby/CHI_config.py b/configs/ruby/CHI_config.py
index 097f367..94a26c0 100644
--- a/configs/ruby/CHI_config.py
+++ b/configs/ruby/CHI_config.py
@@ -107,6 +107,7 @@
'''
num_nodes_per_router = None
router_list = None
+ numa_nodes = None
def __init__(self, ruby_system):
super(CHI_Node, self).__init__()
@@ -488,13 +489,12 @@
_addr_ranges = []
@classmethod
- def createAddrRanges(cls, sys_mem_ranges, cache_line_size, num_hnfs):
+ def createAddrRanges(cls, sys_mem_ranges, cache_line_size, hnfs):
# Create the HNFs interleaved addr ranges
block_size_bits = int(math.log(cache_line_size, 2))
- cls._addr_ranges = []
- llc_bits = int(math.log(num_hnfs, 2))
+ llc_bits = int(math.log(len(hnfs), 2))
numa_bit = block_size_bits + llc_bits - 1
- for i in range(num_hnfs):
+ for i, hnf in enumerate(hnfs):
ranges = []
for r in sys_mem_ranges:
addr_range = AddrRange(r.start, size = r.size(),
@@ -502,7 +502,7 @@
intlvBits = llc_bits,
intlvMatch = i)
ranges.append(addr_range)
- cls._addr_ranges.append((ranges, numa_bit, i))
+ cls._addr_ranges.append((ranges, numa_bit, hnf))
@classmethod
def getAddrRanges(cls, hnf_idx):
diff --git a/configs/ruby/Ruby.py b/configs/ruby/Ruby.py
index 631c65c..b87965e 100644
--- a/configs/ruby/Ruby.py
+++ b/configs/ruby/Ruby.py
@@ -1,3 +1,4 @@
+# Copyright (c) 2022 Samsung Electronics Co., Ltd.
# Copyright (c) 2012, 2017-2018, 2021 ARM Limited
# All rights reserved.
#
@@ -99,6 +100,26 @@
eval("%s.define_options(parser)" % protocol)
Network.define_options(parser)
+def create_memory_controller(mem_range, num_dirs, index, intlv_size,
options):
+ mem_type = ObjectList.mem_list.get(options.mem_type)
+ dram_intf = MemConfig.create_mem_intf(mem_type, mem_range, index,
+ int(math.log(num_dirs, 2)),
+ intlv_size, options.xor_low_bit)
+ if issubclass(mem_type, DRAMInterface):
+ mem_ctrl = m5.objects.MemCtrl(dram = dram_intf)
+ else:
+ mem_ctrl = dram_intf
+
+ if options.access_backing_store:
+ dram_intf.kvm_map=False
+
+ # Enable low-power DRAM states if option is set
+ if issubclass(mem_type, DRAMInterface):
+ mem_ctrl.dram.enable_dram_powerdown = \
+ options.enable_dram_powerdown
+
+ return mem_ctrl, dram_intf
+
def setup_memory_controllers(system, ruby, dir_cntrls, options):
if (options.numa_high_bit):
block_size_bits = options.numa_high_bit + 1 - \
@@ -113,6 +134,13 @@
mem_ctrls = []
crossbars = []
+ # NUMA configuration doesn't support numa_high_bit
+ # since number of dir controllers might be different among nodes
+ if system.numa_config:
+ assert(not options.numa_high_bit)
+ index_in_node = 0
+ num_dirs_in_node = 0
+
if options.numa_high_bit:
dir_bits = int(math.log(options.num_dirs, 2))
intlv_size = 2 ** (options.numa_high_bit - dir_bits + 1)
@@ -127,25 +155,38 @@
# contiguous address range as of now.
for dir_cntrl in dir_cntrls:
crossbar = None
- if len(system.mem_ranges) > 1:
+ # If NUMA configuration is used,
+ # each mem range is assigned only to the corresponding NUMA node
+ if not system.numa_config and len(system.mem_ranges) > 1:
crossbar = IOXBar()
crossbars.append(crossbar)
dir_cntrl.memory = crossbar.cpu_side_ports
dir_ranges = []
for r in system.mem_ranges:
- mem_type = ObjectList.mem_list.get(options.mem_type)
- dram_intf = MemConfig.create_mem_intf(mem_type, r, index,
- int(math.log(options.num_dirs, 2)),
- intlv_size, options.xor_low_bit)
- if issubclass(mem_type, DRAMInterface):
- mem_ctrl = m5.objects.MemCtrl(dram = dram_intf)
+ # Pick the corresponding mem range if NUMA configuration is
used
+ if system.numa_config:
+ node_index = system.numa_nodes[index]
+ r = system.mem_ranges[node_index]
+ # Set the number of dirs in the node
+ if index_in_node == 0:
+ num_dirs_in_node = system.numa_nodes.count(node_index)
+ num_dirs = num_dirs_in_node
+ dir_index = index_in_node
+
+ # Reset the index after all dirs in the node are populated
+ # Assumes node numbers with the same value
+ # are described consecutively
+ # For example, numa_nodes = [0, 0, 1, 1, ...]
+ index_in_node = index_in_node + 1
+ if index_in_node == num_dirs_in_node:
+ index_in_node = 0
else:
- mem_ctrl = dram_intf
+ num_dirs= options.num_dirs
+ dir_index = index
- if options.access_backing_store:
- dram_intf.kvm_map=False
-
+ mem_ctrl, dram_intf= create_memory_controller(r, num_dirs,
+ dir_index, intlv_size, options)
mem_ctrls.append(mem_ctrl)
dir_ranges.append(dram_intf.range)
@@ -154,10 +195,9 @@
else:
mem_ctrl.port = dir_cntrl.memory
- # Enable low-power DRAM states if option is set
- if issubclass(mem_type, DRAMInterface):
- mem_ctrl.dram.enable_dram_powerdown = \
- options.enable_dram_powerdown
+ # We only create a memory controller for the matched mem range
+ if system.numa_config:
+ break
index += 1
dir_cntrl.addr_ranges = dir_ranges
diff --git a/src/sim/System.py b/src/sim/System.py
index 499cf9b..d3ce7ed 100644
--- a/src/sim/System.py
+++ b/src/sim/System.py
@@ -80,6 +80,13 @@
# I/O bridge or cache
mem_ranges = VectorParam.AddrRange([], "Ranges that constitute main
memory")
+ # If set, each memory range is limited to the corresponding NUMA node
+ numa_config = Param.Bool(False, "NUMA configuration is used")
+
+ # Corresponding NUMA node list for each memory controller
+ numa_nodes = VectorParam.UInt32([], "List of NUMA node id where each "
+ "memory controller belongs")
+
# The ranges backed by a shadowed ROM
shadow_rom_ranges = VectorParam.AddrRange([], "Ranges backed by a " \
"shadowed ROM")
--
To view, visit https://gem5-review.googlesource.com/c/public/gem5/+/56610
To unsubscribe, or for help writing mail filters, visit
https://gem5-review.googlesource.com/settings
Gerrit-Project: public/gem5
Gerrit-Branch: develop
Gerrit-Change-Id: If4a8f3ba9aac9f74125970f63410883d2ad32f01
Gerrit-Change-Number: 56610
Gerrit-PatchSet: 1
Gerrit-Owner: Daecheol You <daecheol....@samsung.com>
Gerrit-MessageType: newchange
_______________________________________________
gem5-dev mailing list -- gem5-dev@gem5.org
To unsubscribe send an email to gem5-dev-le...@gem5.org
%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s
