Zhantong Qiu has uploaded this change for review. (
https://gem5-review.googlesource.com/c/public/gem5/+/67195?usp=email )
Change subject: stdlib: Added stdlib LoopPoint classes
......................................................................
stdlib: Added stdlib LoopPoint classes
LoopPoint is a multithreaded workload sampling method that targets
PCs and PC execution counts.
The main idea for LoopPoint is to base the beginning and end of the
simjulation sample on the number of times a particular loop (PC) has
been executed globally across all threads in a region that partitioned
with a set length of instruction counts. This in some senses
generalizes SimPoint which use the instruction count of a single
thread.
The link to the paper: https://ieeexplore.ieee.org/document/9773236
The LoopPointCheckpoint is designed to take in LoopPoint data file
and generate the information needed to take checkpoints for LoopPoint
regions(warmup regions+simulation region)
The LoopPointRestore is designed to take in the LoopPOint data file
and generate information needed to to restore a checkpoint taken by
the LoopPOintCheckpoint.
The LoopPoint is the parent class for LoopPointCheckpoint and
LoopPointRestore.
Change-Id: I595b0ff9d350c7c496639748a9c63ecc61fbaec9
---
M src/python/SConscript
M src/python/gem5/components/processors/abstract_core.py
M src/python/gem5/components/processors/base_cpu_core.py
A src/python/gem5/utils/looppoint.py
4 files changed, 457 insertions(+), 2 deletions(-)
diff --git a/src/python/SConscript b/src/python/SConscript
index aeeb892..68b5e1d 100644
--- a/src/python/SConscript
+++ b/src/python/SConscript
@@ -240,6 +240,7 @@
PySource('gem5.components.processors',
'gem5/components/processors/switchable_processor.py')
PySource('gem5.utils', 'gem5/utils/simpoint.py')
+PySource('gem5.utils', 'gem5/utils/looppoint.py')
PySource('gem5.components.processors',
'gem5/components/processors/traffic_generator_core.py')
PySource('gem5.components.processors',
diff --git a/src/python/gem5/components/processors/abstract_core.py
b/src/python/gem5/components/processors/abstract_core.py
index 58296bc..8259df8 100644
--- a/src/python/gem5/components/processors/abstract_core.py
+++ b/src/python/gem5/components/processors/abstract_core.py
@@ -29,7 +29,8 @@
from ...isas import ISA
-from m5.objects import BaseMMU, Port, SubSystem
+from m5.objects import BaseMMU, Port, SubSystem, PcCountTrackerManager
+from m5.params import PcCountPair
class AbstractCore(SubSystem):
@@ -155,3 +156,9 @@
instruction stop is setup differently dependent on this.
"""
raise NotImplementedError("This core type does not support
MAX_INSTS")
+
+ @abstractmethod
+ def add_pc_tracker_probe(
+ self, target_pair: List[PcCountPair], manager:
PcCountTrackerManager
+ ) -> None:
+ raise NotImplementedError
diff --git a/src/python/gem5/components/processors/base_cpu_core.py
b/src/python/gem5/components/processors/base_cpu_core.py
index 631fd0a..c75c002 100644
--- a/src/python/gem5/components/processors/base_cpu_core.py
+++ b/src/python/gem5/components/processors/base_cpu_core.py
@@ -33,7 +33,15 @@
from ...utils.override import overrides
from ...utils.requires import requires
-from m5.objects import BaseMMU, Port, BaseCPU, Process
+from m5.objects import (
+ BaseMMU,
+ Port,
+ BaseCPU,
+ Process,
+ PcCountTracker,
+ PcCountTrackerManager,
+)
+from m5.params import PcCountPair
class BaseCPUCore(AbstractCore):
@@ -169,3 +177,13 @@
self.core.scheduleInstStopAnyThread(inst)
else:
self.core.max_insts_any_thread = inst
+
+ @overrides(AbstractCore)
+ def add_pc_tracker_probe(
+ self, target_pair: List[PcCountPair], manager:
PcCountTrackerManager
+ ) -> None:
+ pair_tracker = PcCountTracker()
+ pair_tracker.targets = target_pair
+ pair_tracker.core = self.core
+ pair_tracker.ptmanager = manager
+ self.core.probeListener = pair_tracker
diff --git a/src/python/gem5/utils/looppoint.py
b/src/python/gem5/utils/looppoint.py
new file mode 100644
index 0000000..b681e75
--- /dev/null
+++ b/src/python/gem5/utils/looppoint.py
@@ -0,0 +1,401 @@
+# Copyright (c) 2022 The Regents of the University of California
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+from m5.util import fatal
+from m5.params import PcCountPair
+from pathlib import Path
+from typing import List, Dict
+from gem5.components.processors.abstract_processor import AbstractProcessor
+from m5.objects import PcCountTrackerManager
+import csv
+import re
+import json
+
+
+class LoopPoint:
+ """
+ This LoopPoint class is used to manage the information needed for
LoopPoint
+ in workload
+ """
+
+ def __init__(
+ self,
+ targets: List[PcCountPair],
+ regions: Dict[PcCountPair, int],
+ json_file: Dict[int, Dict],
+ ) -> None:
+ """
+ :param targets: a list of PcCountPair that are used to generate
exit
+ event at when the PcCountTrackerManager encounter this PcCountPair
in
+ execution
+ :param regions: a dictionary used to find the corresponding region
id
+ for the significant PcCountPair. This is mainly used to ensure
+ checkpoints are taken in the correct PcCountPair or relative
counts are
+ updated at the correct count
+ :param json_file: all the LoopPoint data including relative counts
and
+ multiplier are stored in this parameter. It can be outputted as a
json
+ file.
+ """
+
+ self._manager = PcCountTrackerManager()
+ self._manager.targets = targets
+ self._targets = targets
+ self._regions = regions
+ self._json_file = json_file
+
+ def setup_processor(
+ self,
+ processor: AbstractProcessor,
+ ) -> None:
+ """
+ This function is used to setup a PC tracker in all the cores and
+ connect all the tracker to the PC tracker manager to perform
+ multithread PC tracking
+ :param processor: the processor used in the simulation
configuration
+ """
+ for core in processor.get_cores():
+ core.add_pc_tracker_probe(self._targets, self._manager)
+
+ def update_relatives_counts(self) -> None:
+ """
+ This function is used to update the relative count for restore
used.
+ The new relative count will be stored in the _json_file and can be
+ outputted into a json file by calling the output_json_file
function.
+ """
+ current_pair = self._manager.getCurrentPcCountPair()
+ if current_pair in self._regions:
+ rid = self._regions[current_pair]
+ region = self._json_file[rid]["simulation"]
+ if "warmup" in self._json_file[rid]:
+ # if this region has a warmup interval,
+ # then update the relative count for the
+ # start of the simulation region
+ start = region["start"]["pc"]
+ temp = region["start"]["global"] -
self._manager.getPcCount(
+ start
+ )
+ self._json_file[rid]["simulation"]["start"]["relative"] =
int(
+ temp
+ )
+ end = region["end"]["pc"]
+ temp = region["end"]["global"] - self._manager.getPcCount(end)
+ self._json_file[rid]["simulation"]["end"]["relative"] =
int(temp)
+
+ def output_json_file(
+ self, input_indent: int = 4, filename: str = "outdir.json"
+ ) -> Dict[int, Dict]:
+ """
+ This function is used to output the _json_file into a json file
+ :param input_indent: the indent value of the json file
+ :param filename: the name of the output file
+ """
+ with open(filename, "w") as file:
+ json.dump(self._json_file, file, indent=input_indent)
+
+ def get_current_region(self) -> int:
+ """
+ This function returns the region id if the current PC Count pair is
+ significant(e.x. beginning of the checkpoint), otherwise, it
returns
+ a '-1' to indicate the current PC Count pair is not significant
+ """
+ current_pair = self._manager.getCurrentPcCountPair()
+ if current_pair in self._regions:
+ return self._regions[current_pair]
+ return -1
+
+ def get_current_pair(self) -> PcCountPair:
+ """
+ This function returns the current PC Count pair
+ """
+ return self._manager.getCurrentPcCountPair()
+
+ def get_regions(self) -> Dict[PcCountPair, int]:
+ """
+ This function returns the complete dictionary of _regions
+ """
+ return self._regions
+
+ def get_targets(self) -> List[PcCountPair]:
+ """
+ This function returns the complete list of _targets
+ """
+ return self._targets
+
+
+class LoopPointCheckpoint(LoopPoint):
+ def __init__(self, looppoint_file: Path, if_csv: bool) -> None:
+ """
+ This class is specifically designed to take in the LoopPoint data
file
+ and generate the information needed to take checkpoints for
LoopPoint
+ regions(warmup region+simulation region)
+ :param looppoint_file: the director of the LoopPoint data file
+ :param if_csv: if the file is a csv file, then it is True. If the
file
+ is a json file, then it is False
+ """
+
+ _json_file = {}
+ _targets = []
+ _region_id = {}
+
+ if if_csv:
+ self.profile_csv(looppoint_file, _targets, _json_file,
_region_id)
+ else:
+ self.profile_json(looppoint_file, _targets, _json_file,
_region_id)
+
+ super().__init__(
+ _targets,
+ _region_id,
+ _json_file,
+ )
+
+ def profile_csv(
+ self,
+ looppoint_file_path: Path,
+ targets: List[PcCountPair],
+ json_file: Dict[int, Dict],
+ region_id: Dict[PcCountPair, int],
+ ) -> None:
+ """
+ This function profiles the csv LoopPoint data file into three
variables
+ to take correct checkpoints for LoopPoint
+ :param looppoint_file_path: the director of the LoopPoint data file
+ :param targets: a list of PcCountPair
+ :param json_file: a dictionary for all the LoopPoint data
+ :param region_id: a dictionary for all the significant PcCountPair
and
+ its corresponding region id
+ """
+
+ # This section is hard-coded to parse the data in the csv file.
+ # The csv file is assumed to have a constant format.
+ with open(looppoint_file_path, newline="") as csvfile:
+ reader = csv.reader(csvfile, delimiter=" ", quotechar="|")
+ for row in reader:
+ if len(row) > 1:
+ if row[0] == "cluster":
+ # if it is a simulation region
+ line = row[4].split(",")
+ start = PcCountPair(int(line[3], 16), int(line[6]))
+ end = PcCountPair(int(line[7], 16), int(line[10]))
+ if int(line[2]) in json_file:
+ # if this region was created in the json_file
+ json_file[int(line[2])]["simulation"] = {
+ "start": {"pc": int(line[3], 16)}
+ }
+ else:
+ json_file[int(line[2])] = {
+ "simulation": {
+ "start": {"pc": int(line[3], 16)}
+ }
+ }
+ json_file[int(line[2])]["simulation"]["start"][
+ "global"
+ ] = int(line[6])
+ json_file[int(line[2])]["simulation"]["end"] = {
+ "pc": int(line[7], 16)
+ }
+ json_file[int(line[2])]["simulation"]["end"][
+ "global"
+ ] = int(line[10])
+ json_file[int(line[2])]["multiplier"] =
float(line[14])
+ targets.append(start)
+ targets.append(end)
+ # store all the PC Count pairs from the file to the
+ # targets list
+ elif row[0] == "Warmup":
+ line = row[3].split(",")
+ start = PcCountPair(int(line[3], 16), int(line[6]))
+ end = PcCountPair(int(line[7], 16), int(line[10]))
+ if int(line[0]) in json_file:
+ json_file[int(line[0])]["warmup"] = {
+ "start": {"pc": int(line[3], 16)}
+ }
+ else:
+ json_file[int(line[0])] = {
+ "warmup": {"start": {"pc": int(line[3],
16)}}
+ }
+ json_file[int(line[0])]["warmup"]["start"][
+ "count"
+ ] = int(line[6])
+ json_file[int(line[0])]["warmup"]["end"] = {
+ "pc": int(line[7], 16)
+ }
+ json_file[int(line[0])]["warmup"]["end"][
+ "count"
+ ] = int(line[10])
+ targets.append(start)
+ targets.append(end)
+ # store all the PC Count pairs from the file to the
+ # targets list
+
+ for rid, region in json_file.items():
+ # this loop iterates all the regions and find the significant
PC
+ # Count pair for the region
+ if "warmup" in region:
+ # if the region has a warmup interval, then the checkpoint
+ # should be taken at the start of the warmup interval
+ start = PcCountPair(
+ region["warmup"]["start"]["pc"],
+ region["warmup"]["start"]["count"],
+ )
+ else:
+ # if the region does not have a warmup interval, then the
+ # checkpoint should be taken at the start of the simulation
+ # region
+ start = PcCountPair(
+ region["simulation"]["start"]["pc"],
+ region["simulation"]["start"]["global"],
+ )
+ region_id[start] = rid
+
+ def profile_json(
+ self,
+ looppoint_file_path: Path,
+ targets: List[PcCountPair],
+ json_file: Dict[int, Dict],
+ region_id: Dict[PcCountPair, int],
+ ) -> None:
+ """
+ This function profiles the json LoopPoint data file into three
+ variables to take correct checkpoints for LoopPoint
+ :param looppoint_file_path: the director of the LoopPoint data file
+ :param targets: a list of PcCountPair
+ :param json_file: a dictionary for all the LoopPoint data
+ :param region_id: a dictionary for all the significant PcCountPair
and
+ its corresponding region id
+ """
+
+ with open(looppoint_file_path) as file:
+ json_file = json.load(file)
+ # load all json information into the json_file variable
+ for rid, region in json_file.items():
+ # iterates all regions
+ sim_start = PcCountPair(
+ region["simulation"]["start"]["pc"],
+ region["simulation"]["start"]["global"],
+ )
+ targets.append(sim_start)
+ # store all PC Count pairs in the file into targets list
+ end = PcCountPair(
+ region["simulation"]["end"]["pc"],
+ region["simulation"]["end"]["global"],
+ )
+ targets.append(end)
+ if "warmup" in region:
+ # if there is a warmup in the region, then the
checkpoint
+ # should be taken at the start of the warmup interval
+ start = PcCountPair(
+ region["warmup"]["start"]["pc"],
+ region["warmup"]["start"]["count"],
+ )
+ targets.append(start)
+ end = PcCountPair(
+ region["warmup"]["end"]["pc"],
+ region["warmup"]["end"]["count"],
+ )
+ targets.append(end)
+ else:
+ # if there is not a warmup interval in the region,
then the
+ # checkpoint should be taken at the start of the
simulation
+ # region
+ start = sim_start
+ region_id[start] = rid
+
+
+class LoopPointRestore(LoopPoint):
+ def __init__(self, looppoint_file: Path, checkpoint_path: Path) ->
None:
+ """
+ This class is specifically designed to take in the LoopPoint data
file and
+ generator information needed to restore a checkpoint taken by the
+ LoopPointCheckPoint.
+ :param looppoint_file: a json file generated by gem5 that has all
the
+ LoopPoint data information
+ :param checkpoint_path: the director of the checkpoint taken by
the gem5
+ standard library looppoint_save_checkpoint_generator
+
+ """
+
+ _json_file = {}
+ _targets = []
+ _region_id = {}
+
+ self.profile_restore(
+ looppoint_file, checkpoint_path, _targets, _json_file,
_region_id
+ )
+
+ super().__init__(
+ _targets,
+ _region_id,
+ _json_file,
+ )
+
+ def profile_restore(
+ self,
+ looppoint_file_path: Path,
+ checkpoint_dir: Path,
+ targets: List[PcCountPair],
+ json_file: Dict[int, Dict],
+ region_id: Dict[PcCountPair, int],
+ ) -> None:
+ """
+ This function is used to profile data from the LoopPoint data file
to
+ information needed to restore the LoopPoint checkpoint
+ :param looppoint_file_path: the director of the LoopPoint data file
+ :param targets: a list of PcCountPair
+ :param json_file: a dictionary for all the LoopPoint data
+ :param region_id: a dictionary for all the significant PcCountPair
and
+ its corresponding region id
+ """
+ regex = re.compile(r"cpt.Region([0-9]+)")
+ rid = regex.findall(checkpoint_dir.as_posix())[0]
+ # finds out the region id from the directory name
+ with open(looppoint_file_path) as file:
+ json_file = json.load(file)
+ if rid not in json_file:
+ # if the region id does not exist in the LoopPoint data
file
+ # raise a fatal message
+ fatal(f"{rid} is not a valid region\n")
+ region = json_file[rid]
+ if "warmup" in region:
+ if "relative" not in region["simulation"]["start"]:
+ # if there are not relative counts for the PC Count
pair
+ # then it means there is not enough information to
restore
+ # this checkpoint
+ fatal(f"region {rid} doesn't have relative count
info\n")
+ start = PcCountPair(
+ region["simulation"]["start"]["pc"],
+ region["simulation"]["start"]["relative"],
+ )
+ region_id[start] = rid
+ targets.append(start)
+ if "relative" not in region["simulation"]["end"]:
+ fatal(f"region {rid} doesn't have relative count info\n")
+ end = PcCountPair(
+ region["simulation"]["end"]["pc"],
+ region["simulation"]["end"]["relative"],
+ )
+ region_id[end] = rid
+ targets.append(end)
--
To view, visit
https://gem5-review.googlesource.com/c/public/gem5/+/67195?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gem5-review.googlesource.com/settings
Gerrit-Project: public/gem5
Gerrit-Branch: develop
Gerrit-Change-Id: I595b0ff9d350c7c496639748a9c63ecc61fbaec9
Gerrit-Change-Number: 67195
Gerrit-PatchSet: 1
Gerrit-Owner: Zhantong Qiu <zt...@ucdavis.edu>
Gerrit-MessageType: newchange
_______________________________________________
gem5-dev mailing list -- gem5-dev@gem5.org
To unsubscribe send an email to gem5-dev-le...@gem5.org
