Attention is currently required from: Gabe Black.
Hello kokoro, Gabe Black,
I'd like you to do a code review.
Please visit
https://gem5-review.googlesource.com/c/public/gem5/+/45822
to review the following change.
Change subject: systemc: Fix verify.py and make it python 3 compatible
......................................................................
systemc: Fix verify.py and make it python 3 compatible
1. The logger behavior change breaks verify.py.
commit 8deb205ea10d6cee0b58c46e097be46c784ed345
Author: Daniel Carvalho <oda...@yahoo.com.br>
Date: Wed Mar 3 16:49:05 2021 -0300
base: Add LOC to Loggers
Printing the line and the file that triggered a log
is useful for debugging.
Change-Id: I74e0637b2943049134bd3e9a4bc6cab3766591a9
Signed-off-by: Daniel R. Carvalho <oda...@yahoo.com.br>
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/42141
Reviewed-by: Giacomo Travaglini <giacomo.travagl...@arm.com>
Maintainer: Giacomo Travaglini <giacomo.travagl...@arm.com>
Tested-by: kokoro <noreply+kok...@google.com>
2. Use bytes diff in LogChecker.
In Python3, string is required to be in a certain encoding, while in
Python2,
it is not. In the testcase, misc/cae_test/general/bitwise/or/datatypes,
it contains some invalid codepoint of utf-8, we need diff the log with
bytes in Pyhton3.
3. Python3 compatible.
* dict.iteritems -> dict.items
* remove object base class
* use `except as` when catching exceptions
* handle map and filter behavior change
Test with
src/systemc/tests/verify.py --update-json build/ARM -j `nproc` \
--filter-file src/systemc/tests/working.filt
src/systemc/tests/verify.py --update-json build/ARM -j `nproc` \
--filter-file src/systemc/tests/working.filt --phase verify --result-file
Change-Id: Ibf5b99d08a948387cf6162c476c294c49a7dac0f
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/44465
Reviewed-by: Gabe Black <gabe.bl...@gmail.com>
Maintainer: Gabe Black <gabe.bl...@gmail.com>
Tested-by: kokoro <noreply+kok...@google.com>
---
M src/systemc/tests/verify.py
1 file changed, 47 insertions(+), 30 deletions(-)
diff --git a/src/systemc/tests/verify.py b/src/systemc/tests/verify.py
index 6b4cf5c..95812f4 100755
--- a/src/systemc/tests/verify.py
+++ b/src/systemc/tests/verify.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
#
# Copyright 2018 Google, Inc.
#
@@ -58,14 +58,14 @@
-class Test(object):
+class Test():
def __init__(self, target, suffix, build_dir, props):
self.target = target
self.suffix = suffix
self.build_dir = build_dir
self.props = {}
- for key, val in props.iteritems():
+ for key, val in props.items():
self.set_prop(key, val)
def set_prop(self, key, val):
@@ -107,7 +107,7 @@
super(TestPhaseMeta, cls).__init__(name, bases, d)
-class TestPhaseBase(object, metaclass=TestPhaseMeta):
+class TestPhaseBase(metaclass=TestPhaseMeta):
abstract = True
def __init__(self, main_args, *args):
@@ -169,7 +169,7 @@
os.makedirs(test.m5out_dir())
try:
subprocess.check_call(cmd, cwd=os.path.dirname(test.dir()))
- except subprocess.CalledProcessError, error:
+ except subprocess.CalledProcessError as error:
returncode = error.returncode
else:
returncode = 0
@@ -180,14 +180,14 @@
runnable = filter(lambda t: not t.compile_only, tests)
if j == 1:
- map(run_test, runnable)
+ list(map(run_test, runnable))
else:
tp = multiprocessing.pool.ThreadPool(j)
- map(lambda t: tp.apply_async(run_test, (t,)), runnable)
+ list(map(lambda t: tp.apply_async(run_test, (t,)), runnable))
tp.close()
tp.join()
-class Checker(object):
+class Checker():
def __init__(self, ref, test, tag):
self.ref = ref
self.test = test
@@ -215,6 +215,13 @@
super(DiffingChecker, self).__init__(ref, test, tag)
self.out_dir = out_dir
+ def is_bytes_mode(self):
+ return False
+
+ def do_diff(self, ref_lines, test_lines, ref_file, test_file):
+ return difflib.unified_diff(ref_lines, test_lines,
+ fromfile=ref_file, tofile=test_file)
+
def diffing_check(self, ref_lines, test_lines):
test_file = os.path.basename(self.test)
ref_file = os.path.basename(self.ref)
@@ -222,11 +229,10 @@
diff_file = '.'.join([ref_file, 'diff'])
diff_path = os.path.join(self.out_dir, diff_file)
if test_lines != ref_lines:
- with open(diff_path, 'w') as diff_f:
- for line in difflib.unified_diff(
- ref_lines, test_lines,
- fromfile=ref_file,
- tofile=test_file):
+ flag = 'wb' if self.is_bytes_mode() else 'w'
+ with open(diff_file, flag) as diff_f:
+ for line in self.do_diff(ref_lines, test_lines,
+ ref_file, test_file):
diff_f.write(line)
return False
else:
@@ -238,7 +244,17 @@
def merge_filts(*filts):
filts = map(lambda f: '(' + f + ')', filts)
filts = '|'.join(filts)
- return re.compile(filts, flags=re.MULTILINE)
+ return re.compile(filts.encode('utf-8'), flags=re.MULTILINE)
+
+ def is_bytes_mode(self):
+ return True
+
+ def do_diff(self, ref_lines, test_lines, ref_file, test_file):
+ return difflib.diff_bytes(
+ difflib.unified_diff,
+ ref_lines, test_lines,
+ fromfile=ref_file.encode('utf-8'),
+ tofile=test_file.encode('utf-8'))
# The reporting mechanism will print the actual filename when running
in
# gem5, and the "golden" output will say "<removed by verify.py>". We
want
@@ -259,8 +275,9 @@
in_file_filt,
)
test_filt = merge_filts(
+ r'^/.*:\d+: ',
r'^Global frequency set at \d* ticks per second\n',
- r'^info: Entering event queue @ \d*\. Starting
simulation\.\.\.\n',
+ r'info: Entering event queue @ \d*\. Starting simulation\.\.\.\n',
r'warn: Ignoring request to set stack size\.\n',
r'^warn: No dot file generated. Please install pydot ' +
r'to generate the dot file and pdf.\n',
@@ -269,12 +286,12 @@
)
def apply_filters(self, data, filts):
- re.sub(filt, '', data)
+ re.sub(filt, b'', data)
def check(self):
- with open(self.test) as test_f, open(self.ref) as ref_f:
- test = re.sub(self.test_filt, '', test_f.read())
- ref = re.sub(self.ref_filt, '', ref_f.read())
+ with open(self.test, 'rb') as test_f, open(self.ref, 'rb') as
ref_f:
+ test = re.sub(self.test_filt, b'', test_f.read())
+ ref = re.sub(self.ref_filt, b'', ref_f.read())
return self.diffing_check(ref.splitlines(True),
test.splitlines(True))
@@ -289,7 +306,7 @@
return self.diffing_check(ref, test)
-class GoldenDir(object):
+class GoldenDir():
def __init__(self, path, platform):
self.path = path
self.platform = platform
@@ -301,7 +318,7 @@
common = filter(lambda t: not t.startswith(tuple(bases)), contents)
self.entries = {}
- class Entry(object):
+ class Entry():
def __init__(self, e_path):
self.used = False
self.path = os.path.join(path, e_path)
@@ -330,7 +347,7 @@
def unused(self):
items = self.entries.items()
- items = filter(lambda i: not i[1].used, items)
+ items = list(filter(lambda i: not i[1].used, items))
items.sort()
sources = []
@@ -367,10 +384,10 @@
def write_result_file(self, path):
results = {
- 'passed': map(lambda t: t.props, self._passed),
+ 'passed': list(map(lambda t: t.props, self._passed)),
'failed': {
- cause: map(lambda t: t.props, tests) for
- cause, tests in self._failed.iteritems()
+ cause: list(map(lambda t: t.props, tests))
+ for cause, tests in self._failed.items()
}
}
with open(path, 'w') as rf:
@@ -472,7 +489,7 @@
self.failed(test, 'missing output', ' '.join(missing))
continue
- failed_diffs = filter(lambda d: not d.check(), diffs)
+ failed_diffs = list(filter(lambda d: not d.check(), diffs))
if failed_diffs:
tags = map(lambda d: d.tag, failed_diffs)
self.failed(test, 'failed diffs', ' '.join(tags))
@@ -568,7 +585,7 @@
filtered_tests = {
target: props for (target, props) in
- test_data.iteritems() if eval(filt, dict(props))
+ test_data.items() if eval(filt, dict(props))
}
if len(filtered_tests) == 0:
@@ -576,15 +593,15 @@
exit()
if main_args.list:
- for target, props in sorted(filtered_tests.iteritems()):
+ for target, props in sorted(filtered_tests.items()):
print('%s.%s' % (target, main_args.flavor))
- for key, val in props.iteritems():
+ for key, val in props.items():
print(' %s: %s' % (key, val))
print('Total tests: %d' % len(filtered_tests))
else:
tests_to_run = list([
Test(target, main_args.flavor, main_args.build_dir, props) for
- target, props in sorted(filtered_tests.iteritems())
+ target, props in sorted(filtered_tests.items())
])
for phase in phases:
--
To view, visit https://gem5-review.googlesource.com/c/public/gem5/+/45822
To unsubscribe, or for help writing mail filters, visit
https://gem5-review.googlesource.com/settings
Gerrit-Project: public/gem5
Gerrit-Branch: minor-release-staging-v21-0-1
Gerrit-Change-Id: Ibf5b99d08a948387cf6162c476c294c49a7dac0f
Gerrit-Change-Number: 45822
Gerrit-PatchSet: 1
Gerrit-Owner: Bobby R. Bruce <bbr...@ucdavis.edu>
Gerrit-Reviewer: Gabe Black <gabe.bl...@gmail.com>
Gerrit-Reviewer: kokoro <noreply+kok...@google.com>
Gerrit-CC: Yu-hsin Wang <yuhsi...@google.com>
Gerrit-Attention: Gabe Black <gabe.bl...@gmail.com>
Gerrit-MessageType: newchange
_______________________________________________
gem5-dev mailing list -- gem5-dev@gem5.org
To unsubscribe send an email to gem5-dev-le...@gem5.org
%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s
