I wasn't really ready for this---I thought we were still discussing the pd-gem5 vs. multi-gem5 issues (if slowly). Also, Gabor admitted that async checkpoints are unnecessary, so I think the checkpointing code should be simplified based on that observation.
Steve On Wed, Jul 15, 2015 at 5:55 PM Gabor Dozsa <[email protected]> wrote: > changeset 5fe05690d03d in /z/repo/gem5 > details: http://repo.gem5.org/gem5?cmd=changeset;node=5fe05690d03d > description: > dev: add support for multi gem5 runs > > Multi gem5 is an extension to gem5 to enable parallel simulation > of a > distributed system (e.g. simulation of a pool of machines > connected by Ethernet links). A multi gem5 run consists of > seperate gem5 > processes running in parallel (potentially on different > hosts/slots on > a cluster). Each gem5 process executes the simulation of a > component of the > simulated distributed system (e.g. a multi-core board with an > Ethernet NIC). > > The patch implements the "distributed" Ethernet link device > (dev/src/multi_etherlink.[hh.cc]). This device will send/receive > (simulated) Ethernet packets to/from peer gem5 processes. The > interface > to talk to the peer gem5 processes is defined in > dev/src/multi_iface.hh and > in tcp_iface.hh. > > There is also a central message server process > (util/multi/tcp_server.[hh,cc]) > which acts like an Ethernet switch and transfers messages among > the gem5 peers. > > A multi gem5 simulations can be kicked off by the > util/multi/gem5-multi.sh > wrapper script. > > Checkpoints are supported by multi-gem5. The checkpoint must be > initiated by a single gem5 process. E.g., the gem5 process with > rank 0 > can take a checkpoint from the bootscript just before it invokes > 'mpirun' to launch an MPI test. The message server process will > notify > all the other peer gem5 processes and make them take a checkpoint, > too > (after completing a global synchronisation to ensure that there > are no > inflight messages among gem5). > > diffstat: > > src/dev/Ethernet.py | 26 + > src/dev/SConscript | 6 + > src/dev/etherpkt.cc | 18 + > src/dev/etherpkt.hh | 15 + > src/dev/multi_etherlink.cc | 266 +++++++++++++++++++ > src/dev/multi_etherlink.hh | 235 +++++++++++++++++ > src/dev/multi_iface.cc | 622 > +++++++++++++++++++++++++++++++++++++++++++++ > src/dev/multi_iface.hh | 492 +++++++++++++++++++++++++++++++++++ > src/dev/multi_packet.cc | 100 +++++++ > src/dev/multi_packet.hh | 130 +++++++++ > src/dev/tcp_iface.cc | 158 +++++++++++ > src/dev/tcp_iface.hh | 134 +++++++++ > util/multi/Makefile | 63 ++++ > util/multi/bootscript.rcS | 122 ++++++++ > util/multi/gem5-multi.sh | 275 +++++++++++++++++++ > util/multi/tcp_server.cc | 463 +++++++++++++++++++++++++++++++++ > util/multi/tcp_server.hh | 254 ++++++++++++++++++ > 17 files changed, 3379 insertions(+), 0 deletions(-) > > diffs (truncated from 3488 to 300 lines): > > diff -r 5ee72f4b2931 -r 5fe05690d03d src/dev/Ethernet.py > --- a/src/dev/Ethernet.py Mon Jul 13 08:46:28 2015 -0400 > +++ b/src/dev/Ethernet.py Wed Jul 15 19:53:50 2015 -0500 > @@ -1,3 +1,15 @@ > +# Copyright (c) 2015 ARM Limited > +# All rights reserved. > +# > +# The license below extends only to copyright in the software and shall > +# not be construed as granting a license to any other intellectual > +# property including but not limited to intellectual property relating > +# to a hardware implementation of the functionality of the software > +# licensed hereunder. You may use the software subject to the license > +# terms below provided that you ensure that this notice is replicated > +# unmodified and in its entirety in all distributions of the software, > +# modified or unmodified, in source code or in binary form. > +# > # Copyright (c) 2005-2007 The Regents of The University of Michigan > # All rights reserved. > # > @@ -46,6 +58,20 @@ > speed = Param.NetworkBandwidth('1Gbps', "link speed") > dump = Param.EtherDump(NULL, "dump object") > > +class MultiEtherLink(EtherObject): > + type = 'MultiEtherLink' > + cxx_header = "dev/multi_etherlink.hh" > + int0 = SlavePort("interface 0") > + delay = Param.Latency('0us', "packet transmit delay") > + delay_var = Param.Latency('0ns', "packet transmit delay variability") > + speed = Param.NetworkBandwidth('1Gbps', "link speed") > + dump = Param.EtherDump(NULL, "dump object") > + multi_rank = Param.UInt32('0', "Rank of the this gem5 process > (multi run)") > + sync_start = Param.Latency('5200000000000t', "first multi sync > barrier") > + sync_repeat = Param.Latency('10us', "multi sync barrier repeat") > + server_name = Param.String('localhost', "Message server name") > + server_port = Param.UInt32('2200', "Message server port") > + > class EtherBus(EtherObject): > type = 'EtherBus' > cxx_header = "dev/etherbus.hh" > diff -r 5ee72f4b2931 -r 5fe05690d03d src/dev/SConscript > --- a/src/dev/SConscript Mon Jul 13 08:46:28 2015 -0400 > +++ b/src/dev/SConscript Wed Jul 15 19:53:50 2015 -0500 > @@ -60,6 +60,10 @@ > Source('etherdump.cc') > Source('etherint.cc') > Source('etherlink.cc') > +Source('multi_packet.cc') > +Source('multi_iface.cc') > +Source('multi_etherlink.cc') > +Source('tcp_iface.cc') > Source('etherpkt.cc') > Source('ethertap.cc') > Source('i2cbus.cc') > @@ -85,6 +89,8 @@ > DebugFlag('DMA') > DebugFlag('DMACopyEngine') > DebugFlag('Ethernet') > +DebugFlag('MultiEthernet') > +DebugFlag('MultiEthernetPkt') > DebugFlag('EthernetCksum') > DebugFlag('EthernetDMA') > DebugFlag('EthernetData') > diff -r 5ee72f4b2931 -r 5fe05690d03d src/dev/etherpkt.cc > --- a/src/dev/etherpkt.cc Mon Jul 13 08:46:28 2015 -0400 > +++ b/src/dev/etherpkt.cc Wed Jul 15 19:53:50 2015 -0500 > @@ -31,6 +31,7 @@ > #include <iostream> > > #include "base/misc.hh" > +#include "base/inet.hh" > #include "dev/etherpkt.hh" > #include "sim/serialize.hh" > > @@ -50,3 +51,20 @@ > if (length) > arrayParamIn(cp, base + ".data", data, length); > } > + > +void > +EthPacketData::packAddress(uint8_t *src_addr, > + uint8_t *dst_addr, > + unsigned &nbytes) > +{ > + Net::EthHdr *hdr = (Net::EthHdr *)data; > + assert(hdr->src().size() == hdr->dst().size()); > + if (nbytes < hdr->src().size()) > + panic("EthPacketData::packAddress() Buffer overflow"); > + > + memcpy(dst_addr, hdr->dst().bytes(), hdr->dst().size()); > + memcpy(src_addr, hdr->src().bytes(), hdr->src().size()); > + > + nbytes = hdr->src().size(); > +} > + > diff -r 5ee72f4b2931 -r 5fe05690d03d src/dev/etherpkt.hh > --- a/src/dev/etherpkt.hh Mon Jul 13 08:46:28 2015 -0400 > +++ b/src/dev/etherpkt.hh Wed Jul 15 19:53:50 2015 -0500 > @@ -71,8 +71,23 @@ > ~EthPacketData() { if (data) delete [] data; } > > public: > + /** > + * This function pulls out the MAC source and destination addresses > from > + * the packet data and stores them in the caller specified buffers. > + * > + * @param src_addr The buffer to store the source MAC address. > + * @param dst_addr The buffer to store the destination MAC address. > + * @param length This is an inout parameter. The caller stores in this > + * the size of the address buffers. On return, this will contain the > + * actual address size stored in the buffers. (We assume that source > + * address size is equal to that of the destination address.) > + */ > + void packAddress(uint8_t *src_addr, uint8_t *dst_addr, unsigned > &length); > + > void serialize(const std::string &base, CheckpointOut &cp) const; > void unserialize(const std::string &base, CheckpointIn &cp); > + > + unsigned size() const { return length; } > }; > > typedef std::shared_ptr<EthPacketData> EthPacketPtr; > diff -r 5ee72f4b2931 -r 5fe05690d03d src/dev/multi_etherlink.cc > --- /dev/null Thu Jan 01 00:00:00 1970 +0000 > +++ b/src/dev/multi_etherlink.cc Wed Jul 15 19:53:50 2015 -0500 > @@ -0,0 +1,266 @@ > +/* > + * Copyright (c) 2015 ARM Limited > + * All rights reserved > + * > + * The license below extends only to copyright in the software and shall > + * not be construed as granting a license to any other intellectual > + * property including but not limited to intellectual property relating > + * to a hardware implementation of the functionality of the software > + * licensed hereunder. You may use the software subject to the license > + * terms below provided that you ensure that this notice is replicated > + * unmodified and in its entirety in all distributions of the software, > + * modified or unmodified, in source code or in binary form. > + * > + * Redistribution and use in source and binary forms, with or without > + * modification, are permitted provided that the following conditions are > + * met: redistributions of source code must retain the above copyright > + * notice, this list of conditions and the following disclaimer; > + * redistributions in binary form must reproduce the above copyright > + * notice, this list of conditions and the following disclaimer in the > + * documentation and/or other materials provided with the distribution; > + * neither the name of the copyright holders nor the names of its > + * contributors may be used to endorse or promote products derived from > + * this software without specific prior written permission. > + * > + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > + * > + * Authors: Gabor Dozsa > + */ > + > +/* @file > + * Device module for a full duplex ethernet link for multi gem5 > simulations. > + */ > + > +#include "dev/multi_etherlink.hh" > + > +#include <arpa/inet.h> > +#include <sys/socket.h> > +#include <unistd.h> > + > +#include <cmath> > +#include <deque> > +#include <string> > +#include <vector> > + > +#include "base/random.hh" > +#include "base/trace.hh" > +#include "debug/EthernetData.hh" > +#include "debug/MultiEthernet.hh" > +#include "debug/MultiEthernetPkt.hh" > +#include "dev/etherdump.hh" > +#include "dev/etherint.hh" > +#include "dev/etherlink.hh" > +#include "dev/etherobject.hh" > +#include "dev/etherpkt.hh" > +#include "dev/multi_iface.hh" > +#include "dev/tcp_iface.hh" > +#include "params/EtherLink.hh" > +#include "sim/core.hh" > +#include "sim/serialize.hh" > +#include "sim/system.hh" > + > +using namespace std; > + > +MultiEtherLink::MultiEtherLink(const Params *p) > + : EtherObject(p) > +{ > + DPRINTF(MultiEthernet,"MultiEtherLink::MultiEtherLink() " > + "link delay:%llu\n", p->delay); > + > + txLink = new TxLink(name() + ".link0", this, p->speed, p->delay_var, > + p->dump); > + rxLink = new RxLink(name() + ".link1", this, p->delay, p->dump); > + > + // create the multi (TCP) interface to talk to the peer gem5 > processes. > + multiIface = new TCPIface(p->server_name, p->server_port, > p->multi_rank, > + p->sync_start, p->sync_repeat, this); > + > + localIface = new LocalIface(name() + ".int0", txLink, rxLink, > multiIface); > +} > + > +MultiEtherLink::~MultiEtherLink() > +{ > + delete txLink; > + delete rxLink; > + delete localIface; > + delete multiIface; > +} > + > +EtherInt* > +MultiEtherLink::getEthPort(const std::string &if_name, int idx) > +{ > + if (if_name != "int0") { > + return nullptr; > + } else { > + panic_if(localIface->getPeer(), "interface already connected to"); > + } > + return localIface; > +} > + > +void MultiEtherLink::memWriteback() > +{ > + DPRINTF(MultiEthernet,"MultiEtherLink::memWriteback() called\n"); > + multiIface->drainDone(); > +} > + > +void > +MultiEtherLink::serialize(CheckpointOut &cp) const > +{ > + multiIface->serialize("multiIface", cp); > + txLink->serialize("txLink", cp); > + rxLink->serialize("rxLink", cp); > +} > + > +void > +MultiEtherLink::unserialize(CheckpointIn &cp) > +{ > + multiIface->unserialize("multiIface", cp); > + txLink->unserialize("txLink", cp); > + rxLink->unserialize("rxLink", cp); > +} > + > +void > +MultiEtherLink::init() > +{ > + DPRINTF(MultiEthernet,"MultiEtherLink::init() called\n"); > + multiIface->initRandom(); > +} > + > +void > +MultiEtherLink::startup() > +{ > + DPRINTF(MultiEthernet,"MultiEtherLink::startup() called\n"); > + multiIface->startPeriodicSync(); > +} > + > +void > +MultiEtherLink::RxLink::setMultiInt(MultiIface *m) > +{ > + assert(!multiIface); > + multiIface = m; > + // Spawn a new receiver thread that will process messages > + // coming in from peer gem5 processes. > + // The receive thread will also schedule a (receive) doneEvent > + // for each incoming data packet. > + multiIface->spawnRecvThread(&doneEvent, linkDelay); > +} > + > +void > +MultiEtherLink::RxLink::rxDone() > +{ > + assert(!busy()); > + > + // retrieve the packet that triggered the receive done event > + packet = multiIface->packetIn(); > + > + if (dump) > + dump->dump(packet); > + > + DPRINTF(MultiEthernetPkt, "MultiEtherLink::MultiLink::rxDone() " > + "packet received: len=%d\n", packet->length); > + DDUMP(EthernetData, packet->data, packet->length); > + > + localIface->sendPacket(packet); > + > _______________________________________________ > gem5-dev mailing list > [email protected] > http://m5sim.org/mailman/listinfo/gem5-dev > _______________________________________________ gem5-dev mailing list [email protected] http://m5sim.org/mailman/listinfo/gem5-dev
