changeset fae097742b7e in /z/repo/gem5
details: http://repo.gem5.org/gem5?cmd=changeset;node=fae097742b7e
description:
sim: tag-based checkpoint versioning
This commit addresses gem5 checkpoints' linear versioning bottleneck.
Since development is distributed across many private trees, there exists
a sort of 'race' for checkpoint version numbers: internally a checkpoint
version may be used but then resynchronizing with the external tree
causes
a conflict on that version. This change replaces the linear version
number
with a set of unique strings called tags. Now the only conflicts that
can
arise are of tag names, where collisions are much easier to avoid.
The checkpoint upgrader (util/cpt_upgrader.py) upgrades the version
representation, as one would expect. Each tag version implements its
upgrader code in a python file in the util/cpt_upgraders directory
rather than adding a function to the upgrader script itself.
The version tags are stored in the 'Globals' section rather than 'root'
(as the version was previously) because 'Globals' gets unserialized
first and can provide a warning before any other unserialization errors
can occur.
diffstat:
src/SConscript | 5 +
src/sim/SConscript | 1 +
src/sim/root.cc | 24 +-
src/sim/serialize.cc | 57 +
src/sim/serialize.hh | 9 -
util/cpt_upgrader.py | 763 +++-----------------
util/cpt_upgraders/arm-ccregs.py | 28 +
util/cpt_upgraders/arm-contextidr-el2.py | 13 +
util/cpt_upgraders/arm-miscreg-teehbr.py | 15 +
util/cpt_upgraders/armv8.py | 314 ++++++++
util/cpt_upgraders/cpu-pid.py | 12 +
util/cpt_upgraders/dvfs-perflevel.py | 15 +
util/cpt_upgraders/ide-dma-abort.py | 8 +
util/cpt_upgraders/isa-is-simobject.py | 65 +
util/cpt_upgraders/memory-per-range.py | 32 +
util/cpt_upgraders/multiple-event-queues.py | 5 +
util/cpt_upgraders/process-fdmap-rename.py | 32 +
util/cpt_upgraders/remove-arm-cpsr-mode-miscreg.py | 13 +
util/cpt_upgraders/ruby-block-size-bytes.py | 9 +
util/cpt_upgraders/x86-add-tlb.py | 17 +
20 files changed, 773 insertions(+), 664 deletions(-)
diffs (truncated from 1628 to 300 lines):
diff -r 463a4b0f0dda -r fae097742b7e src/SConscript
--- a/src/SConscript Wed Sep 02 15:19:44 2015 -0500
+++ b/src/SConscript Wed Sep 02 15:23:30 2015 -0500
@@ -916,6 +916,11 @@
env.Depends(SWIG, 'debug/flags.cc')
Source('debug/flags.cc')
+# version tags
+env.Command('sim/tags.cc', None,
+ MakeAction('util/cpt_upgrader.py --get-cc-file > $TARGET',
+ Transform("VER TAGS")))
+
# Embed python files. All .py files that have been indicated by a
# PySource() call in a SConscript need to be embedded into the M5
# library. To do that, we compile the file to byte code, marshal the
diff -r 463a4b0f0dda -r fae097742b7e src/sim/SConscript
--- a/src/sim/SConscript Wed Sep 02 15:19:44 2015 -0500
+++ b/src/sim/SConscript Wed Sep 02 15:23:30 2015 -0500
@@ -42,6 +42,7 @@
Source('arguments.cc')
Source('async.cc')
Source('core.cc')
+Source('tags.cc')
Source('cxx_config.cc')
Source('cxx_manager.cc')
Source('cxx_config_ini.cc')
diff -r 463a4b0f0dda -r fae097742b7e src/sim/root.cc
--- a/src/sim/root.cc Wed Sep 02 15:19:44 2015 -0500
+++ b/src/sim/root.cc Wed Sep 02 15:23:30 2015 -0500
@@ -132,8 +132,6 @@
void
Root::serialize(CheckpointOut &cp) const
{
- uint64_t cpt_ver = gem5CheckpointVersion;
- SERIALIZE_SCALAR(cpt_ver);
SERIALIZE_SCALAR(FullSystem);
std::string isa = THE_ISA_STR;
SERIALIZE_SCALAR(isa);
@@ -141,27 +139,7 @@
void
Root::unserialize(CheckpointIn &cp)
-{
- uint64_t cpt_ver = 0;
- UNSERIALIZE_OPT_SCALAR(cpt_ver);
- if (cpt_ver < gem5CheckpointVersion) {
- warn("**********************************************************\n");
- warn("!!!! Checkpoint ver %#x is older than current ver %#x !!!!\n",
- cpt_ver, gem5CheckpointVersion);
- warn("You might experience some issues when restoring and should run "
- "the checkpoint upgrader (util/cpt_upgrader.py) on your "
- "checkpoint\n");
- warn("**********************************************************\n");
- } else if (cpt_ver > gem5CheckpointVersion) {
- warn("**********************************************************\n");
- warn("!!!! Checkpoint ver %#x is newer than current ver %#x !!!!\n",
- cpt_ver, gem5CheckpointVersion);
- warn("Running a new checkpoint with an older version of gem5 is not "
- "supported. While it might work, you may experience incorrect "
- "behavior or crashes.\n");
- warn("**********************************************************\n");
- }
-}
+{}
bool FullSystem;
diff -r 463a4b0f0dda -r fae097742b7e src/sim/serialize.cc
--- a/src/sim/serialize.cc Wed Sep 02 15:19:44 2015 -0500
+++ b/src/sim/serialize.cc Wed Sep 02 15:23:30 2015 -0500
@@ -496,16 +496,73 @@
/// The one and only instance of the Globals class.
Globals globals;
+/// The version tags for this build of the simulator, to be stored in the
+/// Globals section during serialization and compared upon unserialization.
+extern std::set<std::string> version_tags;
+
void
Globals::serialize(CheckpointOut &cp) const
{
paramOut(cp, "curTick", curTick());
+ SERIALIZE_CONTAINER(version_tags);
}
void
Globals::unserialize(CheckpointIn &cp)
{
paramIn(cp, "curTick", unserializedCurTick);
+
+ const std::string §ion(Serializable::currentSection());
+ std::string str;
+ if (!cp.find(section, "version_tags", str)) {
+ warn("**********************************************************\n");
+ warn("!!!! Checkpoint uses an old versioning scheme. !!!!\n");
+ warn("Run the checkpoint upgrader (util/cpt_upgrader.py) on your "
+ "checkpoint\n");
+ warn("**********************************************************\n");
+ return;
+ }
+
+ std::set<std::string> cpt_tags;
+ arrayParamIn(cp, "version_tags", cpt_tags); // UNSERIALIZE_CONTAINER
+
+ bool err = false;
+ for (const auto& t : version_tags) {
+ if (cpt_tags.find(t) == cpt_tags.end()) {
+ // checkpoint is missing tag that this binary has
+ if (!err) {
+
warn("*****************************************************\n");
+ warn("!!!! Checkpoint is missing the following version
tags:\n");
+ err = true;
+ }
+ warn(" %s\n", t);
+ }
+ }
+ if (err) {
+ warn("You might experience some issues when restoring and should run "
+ "the checkpoint upgrader (util/cpt_upgrader.py) on your "
+ "checkpoint\n");
+ warn("**********************************************************\n");
+ }
+
+ err = false;
+ for (const auto& t : cpt_tags) {
+ if (version_tags.find(t) == version_tags.end()) {
+ // gem5 binary is missing tag that this checkpoint has
+ if (!err) {
+
warn("*****************************************************\n");
+ warn("!!!! gem5 is missing the following version tags:\n");
+ err = true;
+ }
+ warn(" %s\n", t);
+ }
+ }
+ if (err) {
+ warn("Running a checkpoint with incompatible version tags is not "
+ "supported. While it might work, you may experience incorrect "
+ "behavior or crashes.\n");
+ warn("**********************************************************\n");
+ }
}
Serializable::Serializable()
diff -r 463a4b0f0dda -r fae097742b7e src/sim/serialize.hh
--- a/src/sim/serialize.hh Wed Sep 02 15:19:44 2015 -0500
+++ b/src/sim/serialize.hh Wed Sep 02 15:23:30 2015 -0500
@@ -71,15 +71,6 @@
typedef std::ostream CheckpointOut;
-/** The current version of the checkpoint format.
- * This should be incremented by 1 and only 1 for every new version, where a
new
- * version is defined as a checkpoint created before this version won't work on
- * the current version until the checkpoint format is updated. Adding a new
- * SimObject shouldn't cause the version number to increase, only changes to
- * existing objects such as serializing/unserializing more state, changing
sizes
- * of serialized arrays, etc. */
-static const uint64_t gem5CheckpointVersion = 0x000000000000000f;
-
template <class T>
void paramOut(CheckpointOut &cp, const std::string &name, const T ¶m);
diff -r 463a4b0f0dda -r fae097742b7e util/cpt_upgrader.py
--- a/util/cpt_upgrader.py Wed Sep 02 15:19:44 2015 -0500
+++ b/util/cpt_upgrader.py Wed Sep 02 15:23:30 2015 -0500
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-# Copyright (c) 2012-2013 ARM Limited
+# Copyright (c) 2012-2013,2015 ARM Limited
# All rights reserved
#
# The license below extends only to copyright in the software and shall
@@ -36,6 +36,7 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Ali Saidi
+# Curtis Dunham
#
# This python code is used to migrate checkpoints that were created in one
@@ -45,624 +46,22 @@
# both time consuming and error-prone.
# This script provides a way to migrate checkpoints to the newer repository in
-# a programatic way. It can be imported into another script or used on the
+# a programmatic way. It can be imported into another script or used on the
# command line. From the command line the script will either migrate every
# checkpoint it finds recursively (-r option) or a single checkpoint. When a
-# change is made to the gem5 repository that breaks previous checkpoints a
-# from_N() method should be implemented here and the gem5CheckpointVersion
-# variable in src/sim/serialize.hh should be incremented. For each version
-# between the checkpoints current version and the new version the from_N()
-# method will be run, passing in a ConfigParser object which contains the open
-# file. As these operations can be isa specific the method can verify the isa
-# and use regexes to find the correct sections that need to be updated.
+# change is made to the gem5 repository that breaks previous checkpoints an
+# upgrade() method should be implemented in its own .py file and placed in
+# src/util/cpt_upgraders/. For each upgrader whose tag is not present in
+# the checkpoint tag list, the upgrade() method will be run, passing in a
+# ConfigParser object which contains the open file. As these operations can
+# be isa specific the method can verify the isa and use regexes to find the
+# correct sections that need to be updated.
import ConfigParser
-import sys, os
+import glob, types, sys, os
import os.path as osp
-# An example of a translator
-def from_0(cpt):
- if cpt.get('root','isa') == 'arm':
- for sec in cpt.sections():
- import re
- # Search for all the execution contexts
- if re.search('.*sys.*\.cpu.*\.x.\..*', sec):
- # Update each one
- mr = cpt.get(sec, 'miscRegs').split()
- #mr.insert(21,0)
- #mr.insert(26,0)
- cpt.set(sec, 'miscRegs', ' '.join(str(x) for x in mr))
-
-# The backing store supporting the memories in the system has changed
-# in that it is now stored globally per address range. As a result the
-# actual storage is separate from the memory controllers themselves.
-def from_1(cpt):
- for sec in cpt.sections():
- import re
- # Search for a physical memory
- if re.search('.*sys.*\.physmem$', sec):
- # Add the number of stores attribute to the global physmem
- cpt.set(sec, 'nbr_of_stores', '1')
-
- # Get the filename and size as this is moving to the
- # specific backing store
- mem_filename = cpt.get(sec, 'filename')
- mem_size = cpt.get(sec, '_size')
- cpt.remove_option(sec, 'filename')
- cpt.remove_option(sec, '_size')
-
- # Get the name so that we can create the new section
- system_name = str(sec).split('.')[0]
- section_name = system_name + '.physmem.store0'
- cpt.add_section(section_name)
- cpt.set(section_name, 'store_id', '0')
- cpt.set(section_name, 'range_size', mem_size)
- cpt.set(section_name, 'filename', mem_filename)
- elif re.search('.*sys.*\.\w*mem$', sec):
- # Due to the lack of information about a start address,
- # this migration only works if there is a single memory in
- # the system, thus starting at 0
- raise ValueError("more than one memory detected (" + sec + ")")
-
-def from_2(cpt):
- for sec in cpt.sections():
- import re
- # Search for a CPUs
- if re.search('.*sys.*cpu', sec):
- try:
- junk = cpt.get(sec, 'instCnt')
- cpt.set(sec, '_pid', '0')
- except ConfigParser.NoOptionError:
- pass
-
-# The ISA is now a separate SimObject, which means that we serialize
-# it in a separate section instead of as a part of the ThreadContext.
-def from_3(cpt):
- isa = cpt.get('root','isa')
- isa_fields = {
- "alpha" : ( "fpcr", "uniq", "lock_flag", "lock_addr", "ipr" ),
- "arm" : ( "miscRegs" ),
- "sparc" : ( "asi", "tick", "fprs", "gsr", "softint", "tick_cmpr",
- "stick", "stick_cmpr", "tpc", "tnpc", "tstate", "tt",
- "tba", "pstate", "tl", "pil", "cwp", "gl", "hpstate",
- "htstate", "hintp", "htba", "hstick_cmpr",
- "strandStatusReg", "fsr", "priContext", "secContext",
- "partId", "lsuCtrlReg", "scratchPad",
- "cpu_mondo_head", "cpu_mondo_tail",
- "dev_mondo_head", "dev_mondo_tail",
- "res_error_head", "res_error_tail",
- "nres_error_head", "nres_error_tail",
- "tick_intr_sched",
- "cpu", "tc_num", "tick_cmp", "stick_cmp", "hstick_cmp"),
- "x86" : ( "regVal" ),
- }
-
- isa_fields = isa_fields.get(isa, [])
- isa_sections = []
- for sec in cpt.sections():
- import re
-
- re_cpu_match = re.match('^(.*sys.*\.cpu[^.]*)\.xc\.(.+)$', sec)
- # Search for all the execution contexts
- if not re_cpu_match:
- continue
-
- if re_cpu_match.group(2) != "0":
- # This shouldn't happen as we didn't support checkpointing
_______________________________________________
gem5-dev mailing list
[email protected]
http://m5sim.org/mailman/listinfo/gem5-dev
