changeset 5fe05690d03d in /z/repo/gem5
details: http://repo.gem5.org/gem5?cmd=changeset;node=5fe05690d03d
description:
dev: add support for multi gem5 runs
Multi gem5 is an extension to gem5 to enable parallel simulation of a
distributed system (e.g. simulation of a pool of machines
connected by Ethernet links). A multi gem5 run consists of seperate gem5
processes running in parallel (potentially on different hosts/slots on
a cluster). Each gem5 process executes the simulation of a component of
the
simulated distributed system (e.g. a multi-core board with an Ethernet
NIC).
The patch implements the "distributed" Ethernet link device
(dev/src/multi_etherlink.[hh.cc]). This device will send/receive
(simulated) Ethernet packets to/from peer gem5 processes. The interface
to talk to the peer gem5 processes is defined in dev/src/multi_iface.hh
and
in tcp_iface.hh.
There is also a central message server process
(util/multi/tcp_server.[hh,cc])
which acts like an Ethernet switch and transfers messages among the
gem5 peers.
A multi gem5 simulations can be kicked off by the
util/multi/gem5-multi.sh
wrapper script.
Checkpoints are supported by multi-gem5. The checkpoint must be
initiated by a single gem5 process. E.g., the gem5 process with rank 0
can take a checkpoint from the bootscript just before it invokes
'mpirun' to launch an MPI test. The message server process will notify
all the other peer gem5 processes and make them take a checkpoint, too
(after completing a global synchronisation to ensure that there are no
inflight messages among gem5).
diffstat:
src/dev/Ethernet.py | 26 +
src/dev/SConscript | 6 +
src/dev/etherpkt.cc | 18 +
src/dev/etherpkt.hh | 15 +
src/dev/multi_etherlink.cc | 266 +++++++++++++++++++
src/dev/multi_etherlink.hh | 235 +++++++++++++++++
src/dev/multi_iface.cc | 622 +++++++++++++++++++++++++++++++++++++++++++++
src/dev/multi_iface.hh | 492 +++++++++++++++++++++++++++++++++++
src/dev/multi_packet.cc | 100 +++++++
src/dev/multi_packet.hh | 130 +++++++++
src/dev/tcp_iface.cc | 158 +++++++++++
src/dev/tcp_iface.hh | 134 +++++++++
util/multi/Makefile | 63 ++++
util/multi/bootscript.rcS | 122 ++++++++
util/multi/gem5-multi.sh | 275 +++++++++++++++++++
util/multi/tcp_server.cc | 463 +++++++++++++++++++++++++++++++++
util/multi/tcp_server.hh | 254 ++++++++++++++++++
17 files changed, 3379 insertions(+), 0 deletions(-)
diffs (truncated from 3488 to 300 lines):
diff -r 5ee72f4b2931 -r 5fe05690d03d src/dev/Ethernet.py
--- a/src/dev/Ethernet.py Mon Jul 13 08:46:28 2015 -0400
+++ b/src/dev/Ethernet.py Wed Jul 15 19:53:50 2015 -0500
@@ -1,3 +1,15 @@
+# Copyright (c) 2015 ARM Limited
+# All rights reserved.
+#
+# The license below extends only to copyright in the software and shall
+# not be construed as granting a license to any other intellectual
+# property including but not limited to intellectual property relating
+# to a hardware implementation of the functionality of the software
+# licensed hereunder. You may use the software subject to the license
+# terms below provided that you ensure that this notice is replicated
+# unmodified and in its entirety in all distributions of the software,
+# modified or unmodified, in source code or in binary form.
+#
# Copyright (c) 2005-2007 The Regents of The University of Michigan
# All rights reserved.
#
@@ -46,6 +58,20 @@
speed = Param.NetworkBandwidth('1Gbps', "link speed")
dump = Param.EtherDump(NULL, "dump object")
+class MultiEtherLink(EtherObject):
+ type = 'MultiEtherLink'
+ cxx_header = "dev/multi_etherlink.hh"
+ int0 = SlavePort("interface 0")
+ delay = Param.Latency('0us', "packet transmit delay")
+ delay_var = Param.Latency('0ns', "packet transmit delay variability")
+ speed = Param.NetworkBandwidth('1Gbps', "link speed")
+ dump = Param.EtherDump(NULL, "dump object")
+ multi_rank = Param.UInt32('0', "Rank of the this gem5 process (multi
run)")
+ sync_start = Param.Latency('5200000000000t', "first multi sync barrier")
+ sync_repeat = Param.Latency('10us', "multi sync barrier repeat")
+ server_name = Param.String('localhost', "Message server name")
+ server_port = Param.UInt32('2200', "Message server port")
+
class EtherBus(EtherObject):
type = 'EtherBus'
cxx_header = "dev/etherbus.hh"
diff -r 5ee72f4b2931 -r 5fe05690d03d src/dev/SConscript
--- a/src/dev/SConscript Mon Jul 13 08:46:28 2015 -0400
+++ b/src/dev/SConscript Wed Jul 15 19:53:50 2015 -0500
@@ -60,6 +60,10 @@
Source('etherdump.cc')
Source('etherint.cc')
Source('etherlink.cc')
+Source('multi_packet.cc')
+Source('multi_iface.cc')
+Source('multi_etherlink.cc')
+Source('tcp_iface.cc')
Source('etherpkt.cc')
Source('ethertap.cc')
Source('i2cbus.cc')
@@ -85,6 +89,8 @@
DebugFlag('DMA')
DebugFlag('DMACopyEngine')
DebugFlag('Ethernet')
+DebugFlag('MultiEthernet')
+DebugFlag('MultiEthernetPkt')
DebugFlag('EthernetCksum')
DebugFlag('EthernetDMA')
DebugFlag('EthernetData')
diff -r 5ee72f4b2931 -r 5fe05690d03d src/dev/etherpkt.cc
--- a/src/dev/etherpkt.cc Mon Jul 13 08:46:28 2015 -0400
+++ b/src/dev/etherpkt.cc Wed Jul 15 19:53:50 2015 -0500
@@ -31,6 +31,7 @@
#include <iostream>
#include "base/misc.hh"
+#include "base/inet.hh"
#include "dev/etherpkt.hh"
#include "sim/serialize.hh"
@@ -50,3 +51,20 @@
if (length)
arrayParamIn(cp, base + ".data", data, length);
}
+
+void
+EthPacketData::packAddress(uint8_t *src_addr,
+ uint8_t *dst_addr,
+ unsigned &nbytes)
+{
+ Net::EthHdr *hdr = (Net::EthHdr *)data;
+ assert(hdr->src().size() == hdr->dst().size());
+ if (nbytes < hdr->src().size())
+ panic("EthPacketData::packAddress() Buffer overflow");
+
+ memcpy(dst_addr, hdr->dst().bytes(), hdr->dst().size());
+ memcpy(src_addr, hdr->src().bytes(), hdr->src().size());
+
+ nbytes = hdr->src().size();
+}
+
diff -r 5ee72f4b2931 -r 5fe05690d03d src/dev/etherpkt.hh
--- a/src/dev/etherpkt.hh Mon Jul 13 08:46:28 2015 -0400
+++ b/src/dev/etherpkt.hh Wed Jul 15 19:53:50 2015 -0500
@@ -71,8 +71,23 @@
~EthPacketData() { if (data) delete [] data; }
public:
+ /**
+ * This function pulls out the MAC source and destination addresses from
+ * the packet data and stores them in the caller specified buffers.
+ *
+ * @param src_addr The buffer to store the source MAC address.
+ * @param dst_addr The buffer to store the destination MAC address.
+ * @param length This is an inout parameter. The caller stores in this
+ * the size of the address buffers. On return, this will contain the
+ * actual address size stored in the buffers. (We assume that source
+ * address size is equal to that of the destination address.)
+ */
+ void packAddress(uint8_t *src_addr, uint8_t *dst_addr, unsigned &length);
+
void serialize(const std::string &base, CheckpointOut &cp) const;
void unserialize(const std::string &base, CheckpointIn &cp);
+
+ unsigned size() const { return length; }
};
typedef std::shared_ptr<EthPacketData> EthPacketPtr;
diff -r 5ee72f4b2931 -r 5fe05690d03d src/dev/multi_etherlink.cc
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/dev/multi_etherlink.cc Wed Jul 15 19:53:50 2015 -0500
@@ -0,0 +1,266 @@
+/*
+ * Copyright (c) 2015 ARM Limited
+ * All rights reserved
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder. You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met: redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer;
+ * redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution;
+ * neither the name of the copyright holders nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Authors: Gabor Dozsa
+ */
+
+/* @file
+ * Device module for a full duplex ethernet link for multi gem5 simulations.
+ */
+
+#include "dev/multi_etherlink.hh"
+
+#include <arpa/inet.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <cmath>
+#include <deque>
+#include <string>
+#include <vector>
+
+#include "base/random.hh"
+#include "base/trace.hh"
+#include "debug/EthernetData.hh"
+#include "debug/MultiEthernet.hh"
+#include "debug/MultiEthernetPkt.hh"
+#include "dev/etherdump.hh"
+#include "dev/etherint.hh"
+#include "dev/etherlink.hh"
+#include "dev/etherobject.hh"
+#include "dev/etherpkt.hh"
+#include "dev/multi_iface.hh"
+#include "dev/tcp_iface.hh"
+#include "params/EtherLink.hh"
+#include "sim/core.hh"
+#include "sim/serialize.hh"
+#include "sim/system.hh"
+
+using namespace std;
+
+MultiEtherLink::MultiEtherLink(const Params *p)
+ : EtherObject(p)
+{
+ DPRINTF(MultiEthernet,"MultiEtherLink::MultiEtherLink() "
+ "link delay:%llu\n", p->delay);
+
+ txLink = new TxLink(name() + ".link0", this, p->speed, p->delay_var,
+ p->dump);
+ rxLink = new RxLink(name() + ".link1", this, p->delay, p->dump);
+
+ // create the multi (TCP) interface to talk to the peer gem5 processes.
+ multiIface = new TCPIface(p->server_name, p->server_port, p->multi_rank,
+ p->sync_start, p->sync_repeat, this);
+
+ localIface = new LocalIface(name() + ".int0", txLink, rxLink, multiIface);
+}
+
+MultiEtherLink::~MultiEtherLink()
+{
+ delete txLink;
+ delete rxLink;
+ delete localIface;
+ delete multiIface;
+}
+
+EtherInt*
+MultiEtherLink::getEthPort(const std::string &if_name, int idx)
+{
+ if (if_name != "int0") {
+ return nullptr;
+ } else {
+ panic_if(localIface->getPeer(), "interface already connected to");
+ }
+ return localIface;
+}
+
+void MultiEtherLink::memWriteback()
+{
+ DPRINTF(MultiEthernet,"MultiEtherLink::memWriteback() called\n");
+ multiIface->drainDone();
+}
+
+void
+MultiEtherLink::serialize(CheckpointOut &cp) const
+{
+ multiIface->serialize("multiIface", cp);
+ txLink->serialize("txLink", cp);
+ rxLink->serialize("rxLink", cp);
+}
+
+void
+MultiEtherLink::unserialize(CheckpointIn &cp)
+{
+ multiIface->unserialize("multiIface", cp);
+ txLink->unserialize("txLink", cp);
+ rxLink->unserialize("rxLink", cp);
+}
+
+void
+MultiEtherLink::init()
+{
+ DPRINTF(MultiEthernet,"MultiEtherLink::init() called\n");
+ multiIface->initRandom();
+}
+
+void
+MultiEtherLink::startup()
+{
+ DPRINTF(MultiEthernet,"MultiEtherLink::startup() called\n");
+ multiIface->startPeriodicSync();
+}
+
+void
+MultiEtherLink::RxLink::setMultiInt(MultiIface *m)
+{
+ assert(!multiIface);
+ multiIface = m;
+ // Spawn a new receiver thread that will process messages
+ // coming in from peer gem5 processes.
+ // The receive thread will also schedule a (receive) doneEvent
+ // for each incoming data packet.
+ multiIface->spawnRecvThread(&doneEvent, linkDelay);
+}
+
+void
+MultiEtherLink::RxLink::rxDone()
+{
+ assert(!busy());
+
+ // retrieve the packet that triggered the receive done event
+ packet = multiIface->packetIn();
+
+ if (dump)
+ dump->dump(packet);
+
+ DPRINTF(MultiEthernetPkt, "MultiEtherLink::MultiLink::rxDone() "
+ "packet received: len=%d\n", packet->length);
+ DDUMP(EthernetData, packet->data, packet->length);
+
+ localIface->sendPacket(packet);
+
_______________________________________________
gem5-dev mailing list
[email protected]
http://m5sim.org/mailman/listinfo/gem5-dev
