Author: Remi Meier <[email protected]>
Branch: multithread-runner
Changeset: r388:0db2fd012b7e
Date: 2019-07-11 10:40 +0200
http://bitbucket.org/pypy/benchmarks/changeset/0db2fd012b7e/
Log: recent updates to the benchmarks
diff --git a/multithread/common/abstract_threading.py
b/multithread/common/abstract_threading.py
--- a/multithread/common/abstract_threading.py
+++ b/multithread/common/abstract_threading.py
@@ -4,6 +4,8 @@
try:
from pypystm import atomic, getsegmentlimit, hint_commit_soon
+ # don't use atomic-feature
+ atomic = RLock()
except ImportError:
print "NON-STM EXECUTION"
atomic = RLock()
diff --git a/multithread/get_results.py b/multithread/get_results.py
--- a/multithread/get_results.py
+++ b/multithread/get_results.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
import os
import sys
@@ -9,7 +9,7 @@
# group timings by thread (multiple vmstarts)
res_over_run = all_res[run_key]
grouped = {}
- for bench_key, timings in res_over_run.items():
+ for bench_key, timings in list(res_over_run.items()):
for t in timings:
per_bench = grouped.setdefault(bench_key, {})
per_bench.setdefault(t['threads'], []).extend(t['warmiters'])
@@ -17,30 +17,30 @@
return grouped
def retrieve_data(results):
- print "RUNS:", len(results)
+ print("RUNS:", len(results))
all_res = {}
- for run_key, run in results.iteritems():
+ for run_key, run in results.items():
res_over_run = {}
- print "###", "RUN", run_key, "###"
- print "python:", run['python']
- print "python-version:", run['python-version'].strip()
- print "hg-id:", run['hg-id'].strip()
+ print("###", "RUN", run_key, "###")
+ print("python:", run['python'])
+ print("python-version:", run['python-version'].strip())
+ print("hg-id:", run['hg-id'].strip())
run_results = run['results']
- print "RESULTS:", len(run_results)
+ print("RESULTS:", len(run_results))
- print "BENCHMARKS:", run_results.keys()
- for bench_key, bench_res in run_results.items():
- print "BENCHMARK:", bench_key
+ print("BENCHMARKS:", list(run_results.keys()))
+ for bench_key, bench_res in list(run_results.items()):
+ print("BENCHMARK:", bench_key)
if 'fail_reason' in bench_res:
- print "FAILED:", bench_res
+ print("FAILED:", bench_res)
else:
timings = bench_res['timings']
failures = bench_res['failures']
- print "timings:", len(timings), "failures:", len(failures)
+ print("timings:", len(timings), "failures:", len(failures))
res_over_run.setdefault(bench_key, []).extend(timings)
if failures:
- print "############# THERE ARE FAILURES! #############"
+ print("############# THERE ARE FAILURES! #############")
#print "fail reasons:", failures
#import pdb;pdb.set_trace()
# print ""
@@ -52,8 +52,8 @@
# for ts in sorted(per_bench.keys()):
# if per_bench[ts]:
# print "TS:", ts, "times:", per_bench[ts]
- print ""
- print ""
+ print("")
+ print("")
return all_res
@@ -62,12 +62,12 @@
grouped = collect_warmiters(all_res, run_key)
cols = len(grouped) * 2 + 1 # "threads" + avg, stddev for each benchmark
- rows = len(grouped.values()[0]) + 1 # name + threads
+ rows = len(list(grouped.values())[0]) + 1 # name + threads
table = [["" for _ in range(cols)] for _ in range(rows)] # t[row][col]
table[0][0] = "Threads"
for bench_num, (b, per_bench) in enumerate(grouped.items()):
- print "BENCH", b
+ print("BENCH", b)
table[0][1 + bench_num * 2] = b
ts_index = 0
for ts in sorted(per_bench.keys()):
@@ -78,7 +78,7 @@
else:
assert table[row][0] == ts
- print "TS:", ts, "times:", per_bench[ts]
+ print("TS:", ts, "times:", per_bench[ts])
col = 1 + bench_num * 2
table[row][col] = np.mean(per_bench[ts])
table[row][col+1] = np.std(per_bench[ts])
@@ -86,24 +86,24 @@
# print table:
for r in range(rows):
- line = ",\t".join(map(str, table[r]))
- print line
+ line = ";".join(map(str, table[r]))
+ print(line)
def print_latex_table(all_res, gil_run_key, stm_run_key):
- print ""
- print r"\footnotesize"
- print
r"\begin{tabularx}{\textwidth}{l|r@{\hspace{5pt}}r@{\hspace{5pt}}r@{\hspace{5pt}}r|r@{\hspace{5pt}}r@{\hspace{5pt}}r@{\hspace{5pt}}r|r}"
+ print("")
+ print(r"\footnotesize")
+
print(r"\begin{tabularx}{\textwidth}{l|r@{\hspace{5pt}}r@{\hspace{5pt}}r@{\hspace{5pt}}r|r@{\hspace{5pt}}r@{\hspace{5pt}}r@{\hspace{5pt}}r|r}")
#print r"\hline"
- print r"\textbf{Python VM} & \multicolumn{4}{c|}{\textbf{PyPy-GIL}} &
\multicolumn{4}{c}{\textbf{PyPy-STM}} & \multicolumn{1}{|p{2cm}}{\textbf{Max.
speedup}} \\ \hline"
- print r"\textbf{Threads} & \multicolumn{1}{c}{\textbf{1}} &
\multicolumn{1}{c}{\textbf{2}} & \multicolumn{1}{c}{\textbf{4}} &
\multicolumn{1}{c|}{\textbf{8}} & \multicolumn{1}{c}{\textbf{1}} &
\multicolumn{1}{c}{\textbf{2}} & \multicolumn{1}{c}{\textbf{4}} &
\multicolumn{1}{c}{\textbf{8}} & \multicolumn{1}{|c}{*} \\ \hline"
+ print(r"\textbf{Python VM} & \multicolumn{4}{c|}{\textbf{PyPy-GIL}} &
\multicolumn{4}{c}{\textbf{PyPy-STM}} & \multicolumn{1}{|p{2cm}}{\textbf{Max.
speedup}} \\ \hline")
+ print(r"\textbf{Threads} & \multicolumn{1}{c}{\textbf{1}} &
\multicolumn{1}{c}{\textbf{2}} & \multicolumn{1}{c}{\textbf{4}} &
\multicolumn{1}{c|}{\textbf{8}} & \multicolumn{1}{c}{\textbf{1}} &
\multicolumn{1}{c}{\textbf{2}} & \multicolumn{1}{c}{\textbf{4}} &
\multicolumn{1}{c}{\textbf{8}} & \multicolumn{1}{|c}{*} \\ \hline")
gil_grouped = collect_warmiters(all_res, gil_run_key)
stm_grouped = collect_warmiters(all_res, stm_run_key)
- assert stm_grouped.keys() == gil_grouped.keys()
+ assert list(stm_grouped.keys()) == list(gil_grouped.keys())
warnings = ""
lines = 1
- for bench_key in stm_grouped.keys():
+ for bench_key in list(stm_grouped.keys()):
elems = []
gil_bench = gil_grouped[bench_key]
stm_bench = stm_grouped[bench_key]
@@ -134,14 +134,14 @@
#
speedup = min_gil / min_stm
cells.append(r"\multicolumn{1}{c}{$%.2f\times$}" % speedup)
- print r"%s & " % bench_key + " & ".join(cells) + r" \\" + (
- r" \hdashline[0.5pt/5pt]{}" if lines % 3 == 0 else "")
+ print(r"%s & " % bench_key + " & ".join(cells) + r" \\" + (
+ r" \hdashline[0.5pt/5pt]{}" if lines % 3 == 0 else ""))
lines += 1
- print r"\hline"
- print r"\end{tabularx}"
- print r"\normalsize"
- print ""
- print warnings
+ print(r"\hline")
+ print(r"\end{tabularx}")
+ print(r"\normalsize")
+ print("")
+ print(warnings)
def main(argv):
results_file = argv[0]
@@ -152,12 +152,12 @@
all_res = retrieve_data(results)
while True:
- print "select gil and stm run (e.g., 0,1):"
- runs = all_res.keys()
+ print("select gil and stm run (e.g., 0,1):")
+ runs = list(all_res.keys())
choices = ["%s: %s (%s)" % (i, r, results[r]['python'])
for i, r in enumerate(runs)]
- print "\n".join(choices)
- choice = raw_input()
+ print("\n".join(choices))
+ choice = input()
gil_run_key, stm_run_key = [runs[int(c)] for c in choice.split(',')]
print_csv(all_res, gil_run_key)
diff --git a/multithread/k-nucleotide/k-nucleotide.py
b/multithread/k-nucleotide/k-nucleotide.py
--- a/multithread/k-nucleotide/k-nucleotide.py
+++ b/multithread/k-nucleotide/k-nucleotide.py
@@ -80,9 +80,13 @@
sequence = "".join(load()).upper()
plres = []
+ # to increase benchmark time without changing input size, repeat the
nuclei to
+ # look for
+ nucleos = nucleos * 10
+
t = time.time()
# ORIGINAL version only looked for 1, 2 sequences
- for nl in 1, 2, 3, 4, 5, 6, 7, 8:
+ for nl in range(1, 20):
plres.append(Future(sort_seq, sequence, nl))
for se in nucleos.split():
diff --git a/multithread/parsible-bench/parsible-bench.py
b/multithread/parsible-bench/parsible-bench.py
--- a/multithread/parsible-bench/parsible-bench.py
+++ b/multithread/parsible-bench/parsible-bench.py
@@ -17,6 +17,10 @@
pid_file="/tmp/parsible.pid",
debug=False, batch=True, auto_reload=False)
t = time.time()
+ # run several times instead of adding more log-data to this repo
+ p.main(128)
+ p.main(128)
+ p.main(128)
p.main(128)
parallel_time = time.time() - t
diff --git a/multithread/perlin_ex/perlin_noise.py
b/multithread/perlin_ex/perlin_noise.py
new file mode 100644
--- /dev/null
+++ b/multithread/perlin_ex/perlin_noise.py
@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+# from http://rosettacode.org/wiki/Perlin_noise#Python
+
+import sys
+import time, random
+from common.abstract_threading import (atomic, Future, set_thread_pool,
+ ThreadPool, hint_commit_soon,
+ print_abort_info, turn_jitting_off)
+
+import itertools
+from collections import deque
+
+p = [None] * 512
+permutation = [
+ 151, 160, 137, 91, 90, 15, 131, 13, 201, 95, 96, 53, 194, 233, 7, 225, 140,
+ 36, 103, 30, 69, 142, 8, 99, 37, 240, 21, 10, 23, 190, 6, 148, 247, 120,
+ 234, 75, 0, 26, 197, 62, 94, 252, 219, 203, 117, 35, 11, 32, 57, 177, 33,
+ 88, 237, 149, 56, 87, 174, 20, 125, 136, 171, 168, 68, 175, 74, 165, 71,
+ 134, 139, 48, 27, 166, 77, 146, 158, 231, 83, 111, 229, 122, 60, 211, 133,
+ 230, 220, 105, 92, 41, 55, 46, 245, 40, 244, 102, 143, 54, 65, 25, 63, 161,
+ 1, 216, 80, 73, 209, 76, 132, 187, 208, 89, 18, 169, 200, 196, 135, 130,
+ 116, 188, 159, 86, 164, 100, 109, 198, 173, 186, 3, 64, 52, 217, 226, 250,
+ 124, 123, 5, 202, 38, 147, 118, 126, 255, 82, 85, 212, 207, 206, 59, 227,
+ 47, 16, 58, 17, 182, 189, 28, 42, 223, 183, 170, 213, 119, 248, 152, 2, 44,
+ 154, 163, 70, 221, 153, 101, 155, 167, 43, 172, 9, 129, 22, 39, 253, 19,
+ 98, 108, 110, 79, 113, 224, 232, 178, 185, 112, 104, 218, 246, 97, 228,
+ 251, 34, 242, 193, 238, 210, 144, 12, 191, 179, 162, 241, 81, 51, 145, 235,
+ 249, 14, 239, 107, 49, 192, 214, 31, 181, 199, 106, 157, 184, 84, 204, 176,
+ 115, 121, 50, 45, 127, 4, 150, 254, 138, 236, 205, 93, 222, 114, 67, 29,
+ 24, 72, 243, 141, 128, 195, 78, 66, 215, 61, 156, 180
+]
+
+for i in range(256):
+ p[256 + i] = p[i] = permutation[i]
+
+
+def perlin_noise(x, y, z):
+ X = int(x) & 255 # FIND UNIT CUBE THAT
+ Y = int(y) & 255 # CONTAINS POINT.
+ Z = int(z) & 255
+ x -= int(x) # FIND RELATIVE X,Y,Z
+ y -= int(y) # OF POINT IN CUBE.
+ z -= int(z)
+ u = fade(x) # COMPUTE FADE CURVES
+ v = fade(y) # FOR EACH OF X,Y,Z.
+ w = fade(z)
+ A = p[X] + Y
+ AA = p[A] + Z
+ AB = p[A + 1] + Z # HASH COORDINATES OF
+ B = p[X + 1] + Y
+ BA = p[B] + Z
+ BB = p[B + 1] + Z # THE 8 CUBE CORNERS,
+
+ return lerp(
+ w,
+ lerp(
+ v,
+ lerp(
+ u,
+ grad(p[AA], x, y, z), # AND ADD
+ grad(p[BA], x - 1, y, z)), # BLENDED
+ lerp(
+ u,
+ grad(p[AB], x, y - 1, z), # RESULTS
+ grad(p[BB], x - 1, y - 1, z))), # FROM 8
+ lerp(
+ v,
+ lerp(
+ u,
+ grad(p[AA + 1], x, y, z - 1), # CORNERS
+ grad(p[BA + 1], x - 1, y, z - 1)), # OF CUBE
+ lerp(u, grad(p[AB + 1], x, y - 1, z - 1),
+ grad(p[BB + 1], x - 1, y - 1, z - 1))))
+
+
+def fade(t):
+ return t**3 * (t * (t * 6 - 15) + 10)
+
+
+def lerp(t, a, b):
+ return a + t * (b - a)
+
+
+def grad(hash, x, y, z):
+ h = hash & 15 # CONVERT LO 4 BITS OF HASH CODE
+ # u = x if h < 8 else y # INTO 12 GRADIENT DIRECTIONS.
+ hs = h < 8
+ u = x * hs + y * (not hs)
+
+ hs = h < 4
+ hin = h in (12, 14)
+ # v = y if hs else (x if hin else z)
+ v = y * hs + (not hs) * (x * hin + z * (not hin))
+
+ heven = (h & 1) == 0
+ hfour = (h & 2) == 0
+ # return (u if heven else -u) + (v if hfour else -v)
+ return (u * heven + (-u) * (not heven)) + (v * hfour + (-v) * (not hfour))
+
+
+def work(n, x):
+ res = []
+ #hint_commit_soon()
+ for y in xrange(n):
+ for z in xrange(n):
+ res.append(perlin_noise(x / 3., y / 3., z / 3.))
+ #hint_commit_soon()
+ return res
+
+
+def run(threads=2, n=5):
+ threads = int(threads)
+ n = int(n)
+
+ set_thread_pool(ThreadPool(threads))
+
+ res = []
+ for x in range(n):
+ res.append(Future(work, 200, x))
+ res = [f() for f in res]
+ # shutdown current pool
+ set_thread_pool(None)
+
+
+def main(argv):
+ # warmiters threads args...
+ warmiters = int(argv[0])
+ threads = int(argv[1])
+ n = int(argv[2])
+
+ print "params (iters, threads, n):", warmiters, threads, n
+
+ print "do warmup:"
+ # JIT compiles *tons* of bridges for the perlin_noise function
+ # turning it off after /some/ warmup speeds it up by 10x
+ for i in range(1):
+ t = time.time()
+ run(threads, n)
+ print "iter", i, "time:", time.time() - t
+
+ print "turn off jitting"
+ import gc
+ turn_jitting_off()
+ print "do", warmiters, "real iters:"
+ times = []
+ for i in range(warmiters):
+ gc.collect()
+ t = time.time()
+ run(threads, n)
+ times.append(time.time() - t)
+ print "warmiters:", times
+
+
+if __name__ == "__main__":
+ import sys
+ main(sys.argv[1:])
diff --git a/multithread/perlin_noise/perlin_noise.py
b/multithread/perlin_noise/perlin_noise.py
--- a/multithread/perlin_noise/perlin_noise.py
+++ b/multithread/perlin_noise/perlin_noise.py
@@ -83,7 +83,7 @@
res = []
for x in range(n):
- res.append(Future(work, 400, x))
+ res.append(Future(work, 200, x))
res = [f() for f in res]
# shutdown current pool
diff --git a/multithread/regex-dna/regex-dna.py
b/multithread/regex-dna/regex-dna.py
--- a/multithread/regex-dna/regex-dna.py
+++ b/multithread/regex-dna/regex-dna.py
@@ -50,16 +50,17 @@
def var_find(f, seq):
return len(findall(f, seq))
-def main_b():
- with open('regexdna-inputYYY.txt', 'r') as f:
- seq = f.read()
+def main_b(small=False):
+ f = 'regexdna-inputYYY.txt' if not small else 'regexdna-input10000.txt'
+ with open(f, 'r') as f:
+ seq = f.read().lower()
ilen = len(seq)
seq = sub('>.*\n|\n', '', seq)
clen = len(seq)
- variants = (
+ variants = [
'agggtaaa|tttaccct',
'[cgt]gggtaaa|tttaccc[acg]',
'a[act]ggtaaa|tttacc[agt]t',
@@ -68,12 +69,15 @@
'aggg[acg]aaa|ttt[cgt]ccct',
'agggt[cgt]aa|tt[acg]accct',
'agggta[cgt]a|t[acg]taccct',
- 'agggtaa[cgt]|[acg]ttaccct')
+ 'agggtaa[cgt]|[acg]ttaccct']
+ # instead of increasing input data, just search for the variants
+ # multiple times:
+ variants = 40 * variants
t = time.time()
fs = [Future(var_find, v, seq) for v in variants]
for f in zip(variants, fs):
- print(f[0], f[1]())
+ print(f[1]())
t = time.time() - t
subst = {
@@ -90,12 +94,12 @@
return t
-def run(threads=2):
+def run(threads=2, small=False):
threads = int(threads)
set_thread_pool(ThreadPool(threads))
- t = main_b()
+ t = main_b(small)
# shutdown current pool
set_thread_pool(None)
@@ -105,12 +109,14 @@
# warmiters threads args...
warmiters = int(argv[0])
threads = int(argv[1])
+ small = len(argv) == 3
+
print "params (iters, threads):", warmiters, threads
print "do warmup:"
- for i in range(4):
- t = run(threads)
+ for i in range(3):
+ t = run(threads, small)
print "iter", i, "time:", t
print "turn off jitting"
@@ -120,7 +126,7 @@
for i in range(warmiters):
gc.collect()
- t = run(threads)
+ t = run(threads, small)
times.append(t)
print "warmiters:", times
diff --git a/multithread/runner.py b/multithread/runner.py
--- a/multithread/runner.py
+++ b/multithread/runner.py
@@ -5,10 +5,13 @@
import json
import time
import os, sys
+import psutil
import copy
import pprint
from subprocess import Popen, PIPE
+import select
+STM_LOG = False
WITH_NUMACTL = True
MAX_RETRY = 100 # per bench
@@ -20,18 +23,33 @@
return [float(t) for t in times.split(',')]
return None
-def run_benchmark(python_exec, bench_config):
+def read_all_so_far(stream, res=''):
+ while select.select([stream], [], [], 0.0)[0] != []:
+ c = stream.read(1)
+ if c == "": # EOF
+ break
+ res += c
+ return res
+
+def run_benchmark(bench_name, python_exec, bench_config):
vmstarts = bench_config['vmstarts']
threads = bench_config['threads']
print "## run_benchmark", bench_config
+ if bench_config['skipvm'] and bench_config['skipvm'] in python_exec:
+ print "skip benchmark on VM: skipvm =", bench_config['skipvm']
+ return [{'cmd': 'none', 'exception': 'skipvm'}], []
failures = []
timings = []
- retries = 0
for ts in threads:
+ timeout = False
vm = 0
+ retries = 0
while vm < vmstarts:
print "threads: %s, vm: %s" % (ts, vm)
+ if timeout:
+ print "stop", bench_name, "because of timeout"
+ break
bench_file = os.path.abspath(bench_config['file'])
cmd = ([python_exec,
@@ -41,31 +59,57 @@
+ bench_config['args'])
if WITH_NUMACTL:
# run on node 2, allocate preferrably on node 2
- cmd = ["numactl", "-N2", "--preferred=2"] + cmd
+ cmd = ["numactl", "-N1", "--preferred=1"] + cmd
cmd_str = " ".join(cmd)
cwd, _ = os.path.split(bench_file)
cwd = os.path.join(cwd, bench_config['cwd'])
env = os.environ.copy()
env['PYTHONPATH'] = bench_config['PYTHONPATH']
+ env['JYTHONPATH'] = bench_config['PYTHONPATH']
+ env['IRONPYTHONPATH'] = bench_config['PYTHONPATH']
+ if STM_LOG:
+ env['PYPYSTM'] = "log-%s-%s-%s.pypystm" % (
+ bench_name, ts, vm)
print "running:", cmd_str
try:
p = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd)
- # XXX: process could deadlock if stdout pipe is full -> never
terminate -> timeout
start_time = time.time()
+ stdout, stderr = "", ""
+ mems = []
while p.poll() is None:
time.sleep(0.5)
- if time.time() - start_time > 30 * 60:
+ stdout += read_all_so_far(p.stdout)
+ stderr += read_all_so_far(p.stderr)
+
+ try:
+ process = psutil.Process(p.pid)
+ mem = process.memory_info().rss
+ for child in process.children(recursive=True):
+ mem += child.memory_info().rss
+ mems.append(mem)
+ except psutil.NoSuchProcess as e:
+ print "psutil didn't find the process"
+
+ if time.time() - start_time > 60 * 60:
# kill after 30min
+ print "KILLED AFTER 30min!"
+ timeout = True
p.kill()
+ raise Exception("Timeout30min")
+
+ stdout += read_all_so_far(p.stdout)
+ stderr += read_all_so_far(p.stderr)
if p.wait() != 0:
# error
- stdout, stderr = p.stdout.read(), p.stderr.read()
+ print stdout
+ print stderr
failure = {
'cmd': cmd_str,
'exitcode': p.returncode,
+ 'mems': mems,
'stdout': stdout,
'stderr': stderr,
}
@@ -77,13 +121,13 @@
retries += 1
continue # w/o incrementing 'vm'
else:
- stdout, stderr = p.stdout.read(), p.stderr.read()
print stdout
iter_times = extract_iter_times(stdout)
times = {
'cmd': " ".join(cmd),
'threads': ts,
'vmstarts': vm,
+ 'mems': mems,
'stdout': stdout,
'stderr': stderr,
'warmiters': iter_times,
@@ -110,18 +154,22 @@
bench_config.update(temp_config)
try:
- failures, timings = run_benchmark(
+ failures, timings = run_benchmark(bench_key,
results['python'], bench_config)
except Exception as e:
all_results[bench_key] = {
'fail_reason': str(e)}
+ except KeyboardInterrupt as e:
+ all_results[bench_key] = {
+ 'fail_reason': str(e)}
else:
all_results[bench_key] = {
'failures': failures,
'timings': timings}
print bench_key, bench_config
-
+ print "cooldown..."
+ time.sleep(20) # cooldown
@@ -141,7 +189,7 @@
else:
results = {}
- p = Popen([python_exec, "--version"],
+ p = Popen([python_exec, "-V"],
stdout=PIPE, stderr=PIPE)
_, python_version = p.communicate()
assert p.returncode == 0
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
