Matthew Poremba has uploaded this change for review. (
https://gem5-review.googlesource.com/c/public/gem5/+/27927 )
Change subject: mem-ruby,mem-garnet: Multiple networks per RubySystem
......................................................................
mem-ruby,mem-garnet: Multiple networks per RubySystem
Add support for multiple networks per RubySystem. This is done by
introducing local IDs to each network and translating from a global ID
passed around through Ruby and SLICC code. The local IDs represents the
NodeID of a MachineType in the network and are ordered the same way
that NodeIDs are ordered using MachineType_base_number. If there are
not multiple networks in a RubySystem the local and global IDs are the
same value.
This is useful in cases where multiple isolated networks are needed to
support devices with Ruby caches which do not interact with other
networks. For example, a dGPU device will have a cache hierarchy that
will not interact with the CPU cache hierachy.
Change-Id: I33a917b3a394eec84b16fbf001c3c2c44c047f66
JIRA: https://gem5.atlassian.net/browse/GEM5-445
---
M src/mem/ruby/network/Network.cc
M src/mem/ruby/network/Network.hh
M src/mem/ruby/network/Network.py
M src/mem/ruby/network/garnet2.0/GarnetNetwork.cc
M src/mem/ruby/network/simple/SimpleNetwork.cc
M src/mem/ruby/system/RubySystem.cc
M src/mem/ruby/system/RubySystem.hh
7 files changed, 95 insertions(+), 34 deletions(-)
diff --git a/src/mem/ruby/network/Network.cc
b/src/mem/ruby/network/Network.cc
index 57834f2..4f7c51e 100644
--- a/src/mem/ruby/network/Network.cc
+++ b/src/mem/ruby/network/Network.cc
@@ -57,13 +57,44 @@
// Total nodes/controllers in network
// Must make sure this is called after the State Machine constructors
- m_nodes = MachineType_base_number(MachineType_NUM);
+ if (p->local_nodes) {
+ m_nodes = p->local_nodes;
+ } else {
+ m_nodes = MachineType_base_number(MachineType_NUM);
+ }
+
assert(m_nodes != 0);
assert(m_virtual_networks != 0);
m_topology_ptr = new Topology(p->routers.size(), p->ext_links,
p->int_links);
+ // Populate localNodeVersions with the version of each MachineType in
+ // this network. This will be used to compute a global to local ID.
+ // Do this by looking at the ext_node for each ext_link. There is one
+ // ext_node per ext_link and it points to an AbstractController.
+ // For RubySystems with one network global and local ID are the same.
+ std::unordered_map<MachineType, std::vector<NodeID>> localNodeVersions;
+ for (auto &it : params()->ext_links) {
+ AbstractController *cntrl = it->params()->ext_node;
+ localNodeVersions[cntrl->getType()].push_back(cntrl->getVersion());
+ }
+
+ // Compute a local ID for each MachineType using the same order as
SLICC
+ NodeID local_node_id = 0;
+ for (int i = 0; i < MachineType_base_level(MachineType_NUM); ++i) {
+ MachineType mach = static_cast<MachineType>(i);
+ if (localNodeVersions.count(mach)) {
+ for (auto &ver : localNodeVersions.at(mach)) {
+ // Get the global ID Ruby will pass around
+ NodeID global_node_id = MachineType_base_number(mach) +
ver;
+ globalToLocalMap.insert(
+ std::make_pair(global_node_id, local_node_id));
+ ++local_node_id;
+ }
+ }
+ }
+
// Allocate to and from queues
// Queues that are getting messages from protocol
m_toNetQueues.resize(m_nodes);
@@ -158,11 +189,11 @@
}
void
-Network::checkNetworkAllocation(NodeID id, bool ordered,
+Network::checkNetworkAllocation(NodeID local_id, bool ordered,
int network_num,
std::string vnet_type)
{
- fatal_if(id >= m_nodes, "Node ID is out of range");
+ fatal_if(local_id >= m_nodes, "Node ID is out of range");
fatal_if(network_num >= m_virtual_networks, "Network id is out of
range");
if (ordered) {
@@ -174,25 +205,31 @@
void
-Network::setToNetQueue(NodeID id, bool ordered, int network_num,
+Network::setToNetQueue(NodeID global_id, bool ordered, int network_num,
std::string vnet_type, MessageBuffer *b)
{
- checkNetworkAllocation(id, ordered, network_num, vnet_type);
- while (m_toNetQueues[id].size() <= network_num) {
- m_toNetQueues[id].push_back(nullptr);
+ assert(globalToLocalMap.count(global_id));
+ NodeID local_id = globalToLocalMap[global_id];
+ checkNetworkAllocation(local_id, ordered, network_num, vnet_type);
+
+ while (m_toNetQueues[local_id].size() <= network_num) {
+ m_toNetQueues[local_id].push_back(nullptr);
}
- m_toNetQueues[id][network_num] = b;
+ m_toNetQueues[local_id][network_num] = b;
}
void
-Network::setFromNetQueue(NodeID id, bool ordered, int network_num,
+Network::setFromNetQueue(NodeID global_id, bool ordered, int network_num,
std::string vnet_type, MessageBuffer *b)
{
- checkNetworkAllocation(id, ordered, network_num, vnet_type);
- while (m_fromNetQueues[id].size() <= network_num) {
- m_fromNetQueues[id].push_back(nullptr);
+ assert(globalToLocalMap.count(global_id));
+ NodeID local_id = globalToLocalMap[global_id];
+ checkNetworkAllocation(local_id, ordered, network_num, vnet_type);
+
+ while (m_fromNetQueues[local_id].size() <= network_num) {
+ m_fromNetQueues[local_id].push_back(nullptr);
}
- m_fromNetQueues[id][network_num] = b;
+ m_fromNetQueues[local_id][network_num] = b;
}
NodeID
diff --git a/src/mem/ruby/network/Network.hh
b/src/mem/ruby/network/Network.hh
index 606e670..226658b 100644
--- a/src/mem/ruby/network/Network.hh
+++ b/src/mem/ruby/network/Network.hh
@@ -157,6 +157,9 @@
std::vector<std::vector<MessageBuffer*> > m_fromNetQueues;
std::vector<bool> m_ordered;
+ // Global NodeID to local node map
+ std::unordered_map<NodeID, NodeID> globalToLocalMap;
+
private:
//! Callback class used for collating statistics from all the
//! controller of this type.
diff --git a/src/mem/ruby/network/Network.py
b/src/mem/ruby/network/Network.py
index 5acad60..06742ba 100644
--- a/src/mem/ruby/network/Network.py
+++ b/src/mem/ruby/network/Network.py
@@ -51,3 +51,7 @@
slave = VectorSlavePort("CPU slave port")
master = VectorMasterPort("CPU master port")
+
+ local_nodes = Param.Int(0, "For multiple network per RubySystem this "
+ "represents the number of nodes local to
this "
+ "network")
diff --git a/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc
b/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc
index 1eff921..a1b0216 100644
--- a/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc
+++ b/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc
@@ -146,10 +146,12 @@
*/
void
-GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link,
+GarnetNetwork::makeExtInLink(NodeID global_src, SwitchID dest, BasicLink*
link,
const NetDest& routing_table_entry)
{
- assert(src < m_nodes);
+ assert(globalToLocalMap.count(global_src));
+ NodeID local_src = globalToLocalMap[global_src];
+ assert(local_src < m_nodes);
GarnetExtLink* garnet_link = safe_cast<GarnetExtLink*>(link);
@@ -163,7 +165,7 @@
PortDirection dst_inport_dirn = "Local";
m_routers[dest]->addInPort(dst_inport_dirn, net_link, credit_link);
- m_nis[src]->addOutPort(net_link, credit_link, dest);
+ m_nis[local_src]->addOutPort(net_link, credit_link, dest);
}
/*
@@ -173,10 +175,13 @@
*/
void
-GarnetNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link,
- const NetDest& routing_table_entry)
+GarnetNetwork::makeExtOutLink(SwitchID src, NodeID global_dest,
+ BasicLink* link,
+ const NetDest& routing_table_entry)
{
- assert(dest < m_nodes);
+ assert(globalToLocalMap.count(global_dest));
+ NodeID local_dest = globalToLocalMap[global_dest];
+ assert(local_dest < m_nodes);
assert(src < m_routers.size());
assert(m_routers[src] != NULL);
@@ -194,7 +199,7 @@
m_routers[src]->addOutPort(src_outport_dirn, net_link,
routing_table_entry,
link->m_weight, credit_link);
- m_nis[dest]->addInPort(net_link, credit_link);
+ m_nis[local_dest]->addInPort(net_link, credit_link);
}
/*
@@ -233,9 +238,12 @@
// Get ID of router connected to a NI.
int
-GarnetNetwork::get_router_id(int ni)
+GarnetNetwork::get_router_id(int global_ni)
{
- return m_nis[ni]->get_router_id();
+ assert(globalToLocalMap.count(global_ni));
+ NodeID local_ni = globalToLocalMap[global_ni];
+
+ return m_nis[local_ni]->get_router_id();
}
void
diff --git a/src/mem/ruby/network/simple/SimpleNetwork.cc
b/src/mem/ruby/network/simple/SimpleNetwork.cc
index 51d4dae..2a2e70c 100644
--- a/src/mem/ruby/network/simple/SimpleNetwork.cc
+++ b/src/mem/ruby/network/simple/SimpleNetwork.cc
@@ -71,27 +71,32 @@
// From a switch to an endpoint node
void
-SimpleNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link,
- const NetDest& routing_table_entry)
+SimpleNetwork::makeExtOutLink(SwitchID src, NodeID global_dest,
+ BasicLink* link,
+ const NetDest& routing_table_entry)
{
- assert(dest < m_nodes);
+ assert(globalToLocalMap.count(global_dest));
+ NodeID local_dest = globalToLocalMap[global_dest];
+ assert(local_dest < m_nodes);
assert(src < m_switches.size());
assert(m_switches[src] != NULL);
SimpleExtLink *simple_link = safe_cast<SimpleExtLink*>(link);
- m_switches[src]->addOutPort(m_fromNetQueues[dest], routing_table_entry,
- simple_link->m_latency,
+ m_switches[src]->addOutPort(m_fromNetQueues[local_dest],
+ routing_table_entry,
simple_link->m_latency,
simple_link->m_bw_multiplier);
}
// From an endpoint node to a switch
void
-SimpleNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link,
+SimpleNetwork::makeExtInLink(NodeID global_src, SwitchID dest, BasicLink*
link,
const NetDest& routing_table_entry)
{
- assert(src < m_nodes);
- m_switches[dest]->addInPort(m_toNetQueues[src]);
+ assert(globalToLocalMap.count(global_src));
+ NodeID local_src = globalToLocalMap[global_src];
+ assert(local_src < m_nodes);
+ m_switches[dest]->addInPort(m_toNetQueues[local_src]);
}
// From a switch to a switch
diff --git a/src/mem/ruby/system/RubySystem.cc
b/src/mem/ruby/system/RubySystem.cc
index 83fa4c7..5a7fddd 100644
--- a/src/mem/ruby/system/RubySystem.cc
+++ b/src/mem/ruby/system/RubySystem.cc
@@ -80,7 +80,7 @@
void
RubySystem::registerNetwork(Network* network_ptr)
{
- m_network = network_ptr;
+ m_networks.push_back(network_ptr);
}
void
@@ -94,7 +94,9 @@
RubySystem::~RubySystem()
{
- delete m_network;
+ for (int i = 0; i < m_networks.size(); ++i) {
+ delete m_networks[i];
+ }
delete m_profiler;
}
@@ -508,7 +510,9 @@
}
}
- num_functional_writes += m_network->functionalWrite(pkt);
+ for (int i = 0; i < m_networks.size(); ++i) {
+ num_functional_writes += m_networks[i]->functionalWrite(pkt);
+ }
DPRINTF(RubySystem, "Messages written = %u\n", num_functional_writes);
return true;
diff --git a/src/mem/ruby/system/RubySystem.hh
b/src/mem/ruby/system/RubySystem.hh
index 5d10991..9a0df07 100644
--- a/src/mem/ruby/system/RubySystem.hh
+++ b/src/mem/ruby/system/RubySystem.hh
@@ -130,7 +130,7 @@
SimpleMemory *m_phys_mem;
const bool m_access_backing_store;
- Network* m_network;
+ std::vector<Network *> m_networks;
std::vector<AbstractController *> m_abs_cntrl_vec;
Cycles m_start_cycle;
--
To view, visit https://gem5-review.googlesource.com/c/public/gem5/+/27927
To unsubscribe, or for help writing mail filters, visit
https://gem5-review.googlesource.com/settings
Gerrit-Project: public/gem5
Gerrit-Branch: develop
Gerrit-Change-Id: I33a917b3a394eec84b16fbf001c3c2c44c047f66
Gerrit-Change-Number: 27927
Gerrit-PatchSet: 1
Gerrit-Owner: Matthew Poremba <matthew.pore...@amd.com>
Gerrit-MessageType: newchange
_______________________________________________
gem5-dev mailing list
gem5-dev@gem5.org
http://m5sim.org/mailman/listinfo/gem5-dev
