Author: Antonio Cuni <[email protected]>
Branch:
Changeset: r541:9724de536c4c
Date: 2011-07-29 14:07 +0200
http://bitbucket.org/pypy/buildbot/changeset/9724de536c4c/
Log: use buildbot locks to make sure that benchmarks are run when nothing
else is using the CPU; it needs a bit of care because we use two
different slaves (tannit32 and tannit64) on the same physical
machine, look at the comment at the top of builds.py
diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -2,9 +2,26 @@
from buildbot.steps import source, shell, transfer, master
from buildbot.status.builder import SUCCESS
from buildbot.process.properties import WithProperties
+from buildbot import locks
from pypybuildbot.util import symlink_force
import os
+# buildbot supports SlaveLocks, which can be used to limit the amout of builds
+# to be run on each slave in parallel. However, they assume that each
+# buildslave is on a differen physical machine, which is not the case for
+# tannit32 and tannit64. As a result, we have to use a global lock, and
+# manually tell each builder that uses tannit to acquire it.
+#
+# Look at the various "locks" session in master.py/BuildmasterConfig. For
+# benchmarks, the locks is aquired for the single steps: this way we can run
+# translations in parallel, but then the actual benchmarks are run in
+# sequence.
+
+# there are 8 logical CPUs, but only 4 physical ones. We use a maxCount of 6
+# to allow a bit more parallelism, but not too much
+TannitCPU = locks.MasterLock('tannit_cpu', maxCount=6)
+
+
class ShellCmd(shell.ShellCommand):
# our own version that can distinguish abort cases (rc == -1)
@@ -142,7 +159,7 @@
repourl = 'https://bitbucket.org/pypy/pypy/'
if getpass.getuser() == 'antocuni':
# for debugging
- repourl = '/home/antocuni/pypy/pypy-hg'
+ repourl = '/home/antocuni/pypy/default'
#
if platform == 'win32':
command = "if not exist .hg rmdir /q /s ."
@@ -285,13 +302,23 @@
command=['svn', 'co',
'https://bitbucket.org/pypy/benchmarks/trunk',
'benchmarks'],
workdir='.'))
- self.addStep(Translate(['-Ojit'], []))
+ self.addStep(
+ Translate(
+ translationArgs=['-Ojit'],
+ targetArgs=[],
+ haltOnFailure=False,
+ # this step can be executed in parallel with other builds
+ locks=[TannitCPU.access('counting')],
+ )
+ )
pypy_c_rel = "../build/pypy/translator/goal/pypy-c"
if postfix:
addopts = ['--postfix', postfix]
else:
addopts = []
self.addStep(ShellCmd(
+ # this step needs exclusive access to the CPU
+ locks=[TannitCPU.access('exclusive')],
description="run benchmarks on top of pypy-c",
command=["python", "runner.py", '--output-filename', 'result.json',
'--pypy-c', pypy_c_rel,
diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -105,6 +105,7 @@
pypybuilds = load('pypybuildbot.builds')
+TannitCPU = pypybuilds.TannitCPU
pypyOwnTestFactory = pypybuilds.Own()
pypyOwnTestFactoryWin = pypybuilds.Own(platform="win32")
@@ -250,13 +251,25 @@
"slavenames": ["cobra", "bigdogvm1", "tannit32"],
"builddir": LINUX32,
"factory": pypyOwnTestFactory,
- "category": 'linux32'
+ "category": 'linux32',
+ # this build needs 4 CPUs
+ "locks": [TannitCPU.access('counting'),
+ TannitCPU.access('counting'),
+ TannitCPU.access('counting'),
+ TannitCPU.access('counting'),
+ ],
},
{"name": LINUX64,
"slavenames": ["tannit64"],
"builddir": LINUX64,
"factory": pypyOwnTestFactory,
- "category": 'linux64'
+ "category": 'linux64',
+ # this build needs 4 CPUs
+ "locks": [TannitCPU.access('counting'),
+ TannitCPU.access('counting'),
+ TannitCPU.access('counting'),
+ TannitCPU.access('counting'),
+ ],
},
{"name": MACOSX32,
"slavenames": ["minime"],
@@ -274,25 +287,29 @@
"slavenames": ["bigdogvm1", "tannit32"],
"builddir": APPLVLLINUX32,
"factory": pypyTranslatedAppLevelTestFactory,
- 'category': 'linux32'
+ 'category': 'linux32',
+ "locks": [TannitCPU.access('counting')],
},
{"name": APPLVLLINUX64,
"slavenames": ["tannit64"],
"builddir": APPLVLLINUX64,
"factory": pypyTranslatedAppLevelTestFactory64,
- "category": "linux64"
+ "category": "linux64",
+ "locks": [TannitCPU.access('counting')],
},
{"name": STACKLESSAPPLVLLINUX32,
"slavenames": ["bigdogvm1", "tannit32"],
"builddir": STACKLESSAPPLVLLINUX32,
"factory": pypyStacklessTranslatedAppLevelTestFactory,
- "category": 'linux32-stackless'
+ "category": 'linux32-stackless',
+ "locks": [TannitCPU.access('counting')],
},
{"name": OJITLINUX32,
"slavenames": ["bigdogvm1", "tannit32"],
"builddir": OJITLINUX32,
"factory": pypy_OjitTranslatedTestFactory,
- "category": 'linux32'
+ "category": 'linux32',
+ "locks": [TannitCPU.access('counting')],
},
{"name": APPLVLWIN32,
"slavenames": ["bigboard"],
@@ -311,12 +328,14 @@
'builddir' : JITLINUX32,
'factory' : pypyJITTranslatedTestFactory,
'category' : 'linux32',
+ "locks": [TannitCPU.access('counting')],
},
{'name': JITLINUX64,
'slavenames': ['tannit64'],
'builddir': JITLINUX64,
'factory': pypyJITTranslatedTestFactory64,
'category': 'linux64',
+ "locks": [TannitCPU.access('counting')],
},
{"name" : JITMACOSX64,
"slavenames": ["macmini-mvt", "xerxes"],
@@ -334,19 +353,22 @@
"slavenames": ["tannit32", "bigdogvm1"],
"builddir": JITONLYLINUX32,
"factory": pypyJitOnlyOwnTestFactory,
- "category": 'linux32'
+ "category": 'linux32',
+ "locks": [TannitCPU.access('counting')],
},
{"name": JITBENCH,
"slavenames": ["tannit32"],
"builddir": JITBENCH,
"factory": pypyJITBenchmarkFactory,
"category": 'benchmark-run',
+ # the locks are acquired with fine grain inside the build
},
{"name": JITBENCH64,
"slavenames": ["tannit64"],
"builddir": JITBENCH64,
"factory": pypyJITBenchmarkFactory64,
"category": "benchmark-run",
+ # the locks are acquired with fine grain inside the build
},
],
_______________________________________________
pypy-commit mailing list
[email protected]
http://mail.python.org/mailman/listinfo/pypy-commit
