Yeah, I can't remember what I did, it's been sitting in my tree a while. I noticed the name change after I checked in...must have been a delete and add, sorry about that.
Lisa On Thu, Jun 3, 2010 at 4:16 PM, nathan binkert <[email protected]> wrote: > Did you do a rename or a delete and add? This looks like the latter. > Please use the former in the future. > > Nate > > On Thu, Jun 3, 2010 at 3:27 PM, Lisa Hsu <[email protected]> wrote: > > changeset cbedf338fc44 in /z/repo/m5 > > details: http://repo.m5sim.org/m5?cmd=changeset;node=cbedf338fc44 > > description: > > utils: checkpoint aggregator: some physmem files are too big to > read at once, > > break it up into reading one page at a time. Also, avoid redoing a > aggregating a checkpoint that's > > already done. > > > > diffstat: > > > > util/checkpoint-aggregator.py | 167 > ---------------------------------------- > > util/checkpoint_aggregator.py | 173 > ++++++++++++++++++++++++++++++++++++++++++ > > 2 files changed, 173 insertions(+), 167 deletions(-) > > > > diffs (truncated from 348 to 300 lines): > > > > diff -r 2dc67291f730 -r cbedf338fc44 util/checkpoint-aggregator.py > > --- a/util/checkpoint-aggregator.py Thu Jun 03 12:20:49 2010 -0400 > > +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 > > @@ -1,167 +0,0 @@ > > -# Copyright (c) 2009 The Regents of The University of Michigan > > -# All rights reserved. > > -# > > -# Redistribution and use in source and binary forms, with or without > > -# modification, are permitted provided that the following conditions are > > -# met: redistributions of source code must retain the above copyright > > -# notice, this list of conditions and the following disclaimer; > > -# redistributions in binary form must reproduce the above copyright > > -# notice, this list of conditions and the following disclaimer in the > > -# documentation and/or other materials provided with the distribution; > > -# neither the name of the copyright holders nor the names of its > > -# contributors may be used to endorse or promote products derived from > > -# this software without specific prior written permission. > > -# > > -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > -# > > -# Authors: Lisa Hsu > > - > > -from ConfigParser import ConfigParser > > -import gzip > > - > > -import sys, re, optparse, os > > - > > -class myCP(ConfigParser): > > - def __init__(self): > > - ConfigParser.__init__(self) > > - > > - def optionxform(self, optionstr): > > - return optionstr > > - > > -def aggregate(options, args): > > - merged = myCP() > > - page_ptr = 0 > > - > > - allfiles = os.listdir(os.getcwd()) > > - cpts = [] > > - for arg in args: > > - found = False > > - for f in allfiles: > > - if re.compile("cpt." + arg + ".\d+").search(f): > > - found = True > > - cpts.append(f) > > - break > > - if not found: > > - print "missing checkpoint: ", arg > > - sys.exit(1) > > - > > - dirname = "-".join([options.prefix, "cpt"]) > > - print dirname > > - agg_name = "-".join(args) > > - print agg_name > > - fullpath = os.path.join("..", dirname, "cpt." + agg_name + ".10000") > > - if not os.path.isdir(fullpath): > > - os.system("mkdir -p " + fullpath) > > - > > - myfile = open(fullpath + "/system.physmem.physmem", "wb+") > > - merged_mem = gzip.GzipFile(fileobj=myfile, mode="wb") > > - > > - max_curtick = 0 > > - when = 0 > > - for (i, arg) in enumerate(args): > > - config = myCP() > > - config.readfp(open(cpts[i] + "/m5.cpt")) > > - > > - for sec in config.sections(): > > - if re.compile("cpu").search(sec): > > - newsec = re.sub("cpu", "cpu" + str(i), sec) > > - merged.add_section(newsec) > > - if re.compile("workload$").search(sec): > > - merged.set(newsec, "M5_pid", i) > > - > > - items = config.items(sec) > > - for item in items: > > - if item[0] == "ppn": > > - if config.getint(sec, "tag") != 0: > > - merged.set(newsec, item[0], int(item[1]) + > page_ptr) > > - continue > > - elif item[0] == "asn": > > - tmp = > re.compile("(.*).Entry(\d+)").search(sec).groups() > > - if config.has_option(tmp[0], "nlu"): > > - size = config.getint(tmp[0], "nlu") > > - if int(tmp[1]) < size: > > - merged.set(newsec, item[0], i) > > - continue > > - else: > > - merged.set(newsec, item[0], i) > > - continue > > - merged.set(newsec, item[0], item[1]) > > - elif sec == "system": > > - pass > > - elif sec == "Globals": > > - tick = config.getint(sec, "curTick") > > - if tick > max_curtick: > > - max_curtick = tick > > - when = config.getint("system.cpu.tickEvent", > "_when") > > - else: > > - if i == 0: > > - print sec > > - merged.add_section(sec) > > - for item in config.items(sec): > > - merged.set(sec, item[0], item[1]) > > - if item[0] == "curtick": > > - merged.optionxform(str("curTick")) > > - elif item[0] == "numevents": > > - merged.optionxform(str("numEvents")) > > - > > - page_ptr = page_ptr + int(config.get("system", "page_ptr")) > > - > > - ### memory stuff > > - f = open(cpts[i] + "/system.physmem.physmem", "rb") > > - gf = gzip.GzipFile(fileobj=f, mode="rb") > > - bytes = int(config.get("system", "page_ptr")) << 13 > > - print "bytes to be read: ", bytes > > - > > - bytesRead = gf.read(int(config.get("system", "page_ptr")) << 13) > > - merged_mem.write(bytesRead) > > - > > - gf.close() > > - f.close() > > - > > - merged.add_section("system") > > - merged.set("system", "page_ptr", page_ptr) > > - print "WARNING: " > > - print "Make sure the simulation using this checkpoint has at least " > > - if page_ptr > (1<<20): > > - print "8G ", > > - elif page_ptr > (1<<19): > > - print "4G ", > > - elif page_ptr > (1<<18): > > - print "2G ", > > - elif page_ptr > (1<<17): > > - print "1G ", > > - elif page_ptr > (1<<16): > > - print "512KB ", > > - else: > > - print "this is a small sim, you're probably fine", > > - print "of memory." > > - > > - merged.add_section("Globals") > > - merged.set("Globals", "curTick", max_curtick) > > - > > - for i in xrange(len(args)): > > - merged.set("system.cpu" + str(i) + ".tickEvent", "_when", when) > > - > > - merged.write(file(fullpath + "/m5.cpt", "wb")) > > - merged_mem.close() > > - myfile.close() > > - > > -if __name__ == "__main__": > > - > > - parser = optparse.OptionParser() > > - parser.add_option("--prefix", type="string", default="agg") > > - > > - (options, args) = parser.parse_args() > > - > > - aggregate(options, args) > > - > > diff -r 2dc67291f730 -r cbedf338fc44 util/checkpoint_aggregator.py > > --- /dev/null Thu Jan 01 00:00:00 1970 +0000 > > +++ b/util/checkpoint_aggregator.py Thu Jun 03 10:34:40 2010 -0700 > > @@ -0,0 +1,173 @@ > > +# Copyright (c) 2009 The Regents of The University of Michigan > > +# All rights reserved. > > +# > > +# Redistribution and use in source and binary forms, with or without > > +# modification, are permitted provided that the following conditions are > > +# met: redistributions of source code must retain the above copyright > > +# notice, this list of conditions and the following disclaimer; > > +# redistributions in binary form must reproduce the above copyright > > +# notice, this list of conditions and the following disclaimer in the > > +# documentation and/or other materials provided with the distribution; > > +# neither the name of the copyright holders nor the names of its > > +# contributors may be used to endorse or promote products derived from > > +# this software without specific prior written permission. > > +# > > +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > +# > > +# Authors: Lisa Hsu > > + > > +from ConfigParser import ConfigParser > > +import gzip > > + > > +import sys, re, optparse, os > > + > > +class myCP(ConfigParser): > > + def __init__(self): > > + ConfigParser.__init__(self) > > + > > + def optionxform(self, optionstr): > > + return optionstr > > + > > +def aggregate(options, args): > > + merged = myCP() > > + page_ptr = 0 > > + > > + allfiles = os.listdir(os.getcwd()) > > + cpts = [] > > + for arg in args: > > + found = False > > + for f in allfiles: > > + if re.compile("cpt." + arg + ".\d+").search(f): > > + found = True > > + cpts.append(f) > > + break > > + if not found: > > + print "missing checkpoint: ", arg > > + sys.exit(1) > > + > > + dirname = "-".join([options.prefix, "cpt"]) > > + agg_name = "-".join(args) > > + print agg_name > > + fullpath = os.path.join("..", dirname, "cpt." + agg_name + ".10000") > > + if not os.path.isdir(fullpath): > > + os.system("mkdir -p " + fullpath) > > + elif os.path.isfile(fullpath + "/system.physmem.physmem"): > > + if os.path.isfile(fullpath + "/m5.cpt"): > > + print fullpath, " already done" > > + return > > + > > + myfile = open(fullpath + "/system.physmem.physmem", "wb+") > > + merged_mem = gzip.GzipFile(fileobj=myfile, mode="wb") > > + > > + max_curtick = 0 > > + when = 0 > > + for (i, arg) in enumerate(args): > > + print arg > > + config = myCP() > > + config.readfp(open(cpts[i] + "/m5.cpt")) > > + > > + for sec in config.sections(): > > + if re.compile("cpu").search(sec): > > + newsec = re.sub("cpu", "cpu" + str(i), sec) > > + merged.add_section(newsec) > > + if re.compile("workload$").search(sec): > > + merged.set(newsec, "M5_pid", i) > > + > > + items = config.items(sec) > > + for item in items: > > + if item[0] == "ppn": > > + if config.getint(sec, "tag") != 0: > > + merged.set(newsec, item[0], int(item[1]) + > page_ptr) > > + continue > > + elif item[0] == "asn": > > + tmp = > re.compile("(.*).Entry(\d+)").search(sec).groups() > > + if config.has_option(tmp[0], "nlu"): > > + size = config.getint(tmp[0], "nlu") > > + if int(tmp[1]) < size: > > + merged.set(newsec, item[0], i) > > + continue > > + else: > > + merged.set(newsec, item[0], i) > > + continue > > + merged.set(newsec, item[0], item[1]) > > + elif sec == "system": > > + pass > > + elif sec == "Globals": > > + tick = config.getint(sec, "curTick") > > + if tick > max_curtick: > > + max_curtick = tick > > + when = config.getint("system.cpu.tickEvent", > "_when") > > + else: > > + if i == 0: > > + merged.add_section(sec) > > + for item in config.items(sec): > > + merged.set(sec, item[0], item[1]) > > + if item[0] == "curtick": > > + merged.optionxform(str("curTick")) > > + elif item[0] == "numevents": > > + merged.optionxform(str("numEvents")) > > + > > + page_ptr = page_ptr + int(config.get("system", "page_ptr")) > > + > > + ### memory stuff > > + f = open(cpts[i] + "/system.physmem.physmem", "rb") > > + gf = gzip.GzipFile(fileobj=f, mode="rb") > > + pages = int(config.get("system", "page_ptr")) > > _______________________________________________ > > m5-dev mailing list > > [email protected] > > http://m5sim.org/mailman/listinfo/m5-dev > > > > > _______________________________________________ > m5-dev mailing list > [email protected] > http://m5sim.org/mailman/listinfo/m5-dev > >
_______________________________________________ m5-dev mailing list [email protected] http://m5sim.org/mailman/listinfo/m5-dev
