Author: Maciej Fijalkowski <fij...@gmail.com>
Branch:
Changeset: r548:1b4c4c52f405
Date: 2011-08-03 15:18 +0200
http://bitbucket.org/pypy/buildbot/changeset/1b4c4c52f405/
Log: merge
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -14,18 +14,13 @@
httpd.log*
# ignore all the builders dir (which can be both in master/ and in slave/)
-jit-benchmark-linux-x86-32
-jitonly-own-linux-x86-32
-own-linux-x86-32
-own-linux-x86-64
-own-macosx-x86-32
-pypy-c-Ojit-no-jit-linux-x86-32
-pypy-c-app-level-linux-x86-32
-pypy-c-app-level-linux-x86-64
-pypy-c-app-level-win-x86-32
-pypy-c-jit-linux-x86-32
-pypy-c-jit-linux-x86-64
-pypy-c-jit-macosx-x86-64
-pypy-c-jit-win-x86-32
-pypy-c-stackless-app-level-freebsd-7-x86-64
-pypy-c-stackless-app-level-linux-x86-32
+*-linux-x86-32
+*-linux-x86-32vm
+*-linux-x86-64
+*-macosx-x86-32
+*-macosx-x86-64
+*-win-x86-32
+*-win-32
+*-freebsd-7-x86-64
+*-maemo
+*-maemo-build
diff --git a/bbhook/run.py b/bbhook/run.py
--- a/bbhook/run.py
+++ b/bbhook/run.py
@@ -16,8 +16,11 @@
#HOST_NAME = 'wyvern.cs.uni-duesseldorf.de'
HOST_NAME = ''
PORT_NUMBER = 9237
+ # WARNING:
+ # deply is meant as a argument for running public facing,
+ # its not supposed to be used when running a proxy setup
main.app.run(
- host = HOST_NAME if 'deploy' in sys.argv else 'localhost',
+ host = HOST_NAME if 'deploy' in sys.argv else '127.0.0.1',
debug = 'debug' in sys.argv,
port=PORT_NUMBER)
diff --git a/bbhook/scm.py b/bbhook/scm.py
--- a/bbhook/scm.py
+++ b/bbhook/scm.py
@@ -1,6 +1,7 @@
import sys
from subprocess import Popen, PIPE
+MAX_DIFF_LINES = 10000
def _hgexe(argv):
proc = Popen(['hg'] + list(argv), stdout=PIPE, stderr=PIPE)
@@ -21,8 +22,11 @@
def get_diff(local_repo, hgid):
out = hg('-R', local_repo, 'diff', '--git', '-c', hgid)
- out = out.splitlines(True)
- out_iter = iter(out)
+ lines = out.splitlines(True)
+ return filter_diff(lines)
+
+def filter_diff(lines):
+ out_iter = iter(lines)
lines = []
for line in out_iter:
lines.append(line)
@@ -34,6 +38,9 @@
if item[0]!='z':
break # binary patches end with a empty line
+ if len(lines) > MAX_DIFF_LINES:
+ msg = 'diff too long, truncating to %d out of %d lines\n\n' %
(MAX_DIFF_LINES, len(lines))
+ lines = [msg] + lines[:MAX_DIFF_LINES]
return u''.join(lines)
diff --git a/bbhook/test/test_scm.py b/bbhook/test/test_scm.py
--- a/bbhook/test/test_scm.py
+++ b/bbhook/test/test_scm.py
@@ -32,3 +32,24 @@
with pytest.raises(Exception):
print scm.hg
scm.hg('uhmwrong')
+
+
+def test_huge_diff(monkeypatch):
+ monkeypatch.setattr(scm, 'MAX_DIFF_LINES', 4)
+ lines = """\
+one
+two
+three
+for
+five
+six
+""".splitlines(True)
+ diff = scm.filter_diff(lines)
+ assert diff == """\
+diff too long, truncating to 4 out of 6 lines
+
+one
+two
+three
+for
+"""
diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -2,9 +2,25 @@
from buildbot.steps import source, shell, transfer, master
from buildbot.status.builder import SUCCESS
from buildbot.process.properties import WithProperties
+from buildbot import locks
from pypybuildbot.util import symlink_force
import os
+# buildbot supports SlaveLocks, which can be used to limit the amout of builds
+# to be run on each slave in parallel. However, they assume that each
+# buildslave is on a differen physical machine, which is not the case for
+# tannit32 and tannit64. As a result, we have to use a global lock, and
+# manually tell each builder that uses tannit to acquire it.
+#
+# Look at the various "locks" session in master.py/BuildmasterConfig. For
+# benchmarks, the locks is aquired for the single steps: this way we can run
+# translations in parallel, but then the actual benchmarks are run in
+# sequence.
+
+# there are 8 logical CPUs, but only 4 physical ones
+TannitCPU = locks.MasterLock('tannit_cpu', maxCount=6)
+
+
class ShellCmd(shell.ShellCommand):
# our own version that can distinguish abort cases (rc == -1)
@@ -143,7 +159,7 @@
repourl = 'https://bitbucket.org/pypy/pypy/'
if getpass.getuser() == 'antocuni':
# for debugging
- repourl = '/home/antocuni/pypy/pypy-hg'
+ repourl = '/home/antocuni/pypy/default'
#
if platform == 'win32':
command = "if not exist .hg rmdir /q /s ."
@@ -260,8 +276,12 @@
else:
if '--stackless' in translationArgs:
kind = 'stackless'
+ elif '-Ojit' in translationArgs:
+ kind = 'jitnojit'
+ elif '-O2' in translationArgs:
+ kind = 'nojit'
else:
- kind = 'nojit'
+ kind = 'unknown'
name = 'pypy-c-' + kind + '-%(final_file_name)s-' + platform
self.addStep(ShellCmd(
description="compress pypy-c",
@@ -286,13 +306,23 @@
command=['svn', 'co',
'https://bitbucket.org/pypy/benchmarks/trunk',
'benchmarks'],
workdir='.'))
- self.addStep(Translate(['-Ojit'], []))
+ self.addStep(
+ Translate(
+ translationArgs=['-Ojit'],
+ targetArgs=[],
+ haltOnFailure=True,
+ # this step can be executed in parallel with other builds
+ locks=[TannitCPU.access('counting')],
+ )
+ )
pypy_c_rel = "../build/pypy/translator/goal/pypy-c"
if postfix:
addopts = ['--postfix', postfix]
else:
addopts = []
self.addStep(ShellCmd(
+ # this step needs exclusive access to the CPU
+ locks=[TannitCPU.access('exclusive')],
description="run benchmarks on top of pypy-c",
command=["python", "runner.py", '--output-filename', 'result.json',
'--pypy-c', pypy_c_rel,
@@ -303,9 +333,11 @@
'--branch', WithProperties('%(branch)s'),
] + addopts,
workdir='./benchmarks',
- haltOnFailure=True))
+ haltOnFailure=True,
+ timeout=3600))
# a bit obscure hack to get both os.path.expand and a property
- resfile = os.path.expanduser("~/bench_results/%(got_revision)s.json")
+ filename = '%(got_revision)s' + (postfix or '')
+ resfile = os.path.expanduser("~/bench_results/%s.json" % filename)
self.addStep(transfer.FileUpload(slavesrc="benchmarks/result.json",
masterdest=WithProperties(resfile),
workdir="."))
diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -105,6 +105,7 @@
pypybuilds = load('pypybuildbot.builds')
+TannitCPU = pypybuilds.TannitCPU
pypyOwnTestFactory = pypybuilds.Own()
pypyOwnTestFactoryWin = pypybuilds.Own(platform="win32")
@@ -212,37 +213,67 @@
'slavePortnum': slavePortnum,
'change_source': [],
+ ## 'schedulers': [
+ ## Nightly("nightly-0-00", [
+ ## JITBENCH, # on tannit -- nothing else there during first round!
+ ## MACOSX32, # on minime
+ ## ], hour=0, minute=0),
+ ## Nightly("nighly-2-00", [
+ ## JITBENCH64, # on tannit -- nothing else there during first
round!
+ ## ], hour=2, minute=0),
+ ## Nightly("nightly-4-00", [
+ ## # rule: what we pick here on tannit should take at most 8 cores
+ ## # and be hopefully finished after 2 hours
+ ## LINUX32, # on tannit32, uses 4 cores
+ ## JITLINUX32, # on tannit32, uses 1 core
+ ## JITLINUX64, # on tannit64, uses 1 core
+ ## OJITLINUX32, # on tannit32, uses 1 core
+ ## JITWIN32, # on bigboard
+ ## STACKLESSAPPLVLFREEBSD64, # on headless
+ ## JITMACOSX64, # on mvt's machine
+ ## ], hour=4, minute=0),
+ ## Nightly("nightly-6-00", [
+ ## # there should be only JITLINUX32 that takes a bit longer than
+ ## # that. We can use a few more cores now.
+ ## APPLVLLINUX32, # on tannit32, uses 1 core
+ ## APPLVLLINUX64, # on tannit64, uses 1 core
+ ## STACKLESSAPPLVLLINUX32, # on tannit32, uses 1 core
+ ## ], hour=6, minute=0),
+ ## Nightly("nightly-7-00", [
+ ## # the remaining quickly-run stuff on tannit
+ ## LINUX64, # on tannit64, uses 4 cores
+ ## ], hour=7, minute=0),
+ ## ],
+
'schedulers': [
+ # first of all, we run the benchmarks: the two translations take ~2800
+ # seconds and are executed in parallel. Running benchmarks takes ~3400
+ # seconds and is executed sequentially. In total, 2800 + (3300*2) ~=
+ # 160 minutes
Nightly("nightly-0-00", [
- JITBENCH, # on tannit -- nothing else there during first round!
+ JITBENCH, # on tannit32, uses 1 core (in part
exclusively)
+ JITBENCH64, # on tannit64, uses 1 core (in part
exclusively)
MACOSX32, # on minime
], hour=0, minute=0),
- Nightly("nighly-2-00", [
- JITBENCH64, # on tannit -- nothing else there during first round!
- ], hour=2, minute=0),
- Nightly("nightly-4-00", [
- # rule: what we pick here on tannit should take at most 8 cores
- # and be hopefully finished after 2 hours
+ #
+ # then, we schedule all the rest. The locks will take care not to run
+ # all of them in parallel
+ Nightly("nighly-3-00", [
LINUX32, # on tannit32, uses 4 cores
+ LINUX64, # on tannit64, uses 4 cores
JITLINUX32, # on tannit32, uses 1 core
JITLINUX64, # on tannit64, uses 1 core
OJITLINUX32, # on tannit32, uses 1 core
+ APPLVLLINUX32, # on tannit32, uses 1 core
+ APPLVLLINUX64, # on tannit64, uses 1 core
+ STACKLESSAPPLVLLINUX32, # on tannit32, uses 1 core
+ #
JITWIN32, # on bigboard
STACKLESSAPPLVLFREEBSD64, # on headless
JITMACOSX64, # on mvt's machine
- ], hour=4, minute=0),
- Nightly("nightly-6-00", [
- # there should be only JITLINUX32 that takes a bit longer than
- # that. We can use a few more cores now.
- APPLVLLINUX32, # on tannit32, uses 1 core
- APPLVLLINUX64, # on tannit64, uses 1 core
- STACKLESSAPPLVLLINUX32, # on tannit32, uses 1 core
- ], hour=6, minute=0),
- Nightly("nightly-7-00", [
- # the remaining quickly-run stuff on tannit
- LINUX64, # on tannit64, uses 4 cores
- ], hour=7, minute=0),
+ ], hour=3, minute=0)
],
+
'status': [status],
'slaves': [BuildSlave(name, password)
@@ -254,13 +285,25 @@
"slavenames": ["cobra", "bigdogvm1", "tannit32"],
"builddir": LINUX32,
"factory": pypyOwnTestFactory,
- "category": 'linux32'
+ "category": 'linux32',
+ # this build needs 4 CPUs
+ "locks": [TannitCPU.access('counting'),
+ TannitCPU.access('counting'),
+ TannitCPU.access('counting'),
+ TannitCPU.access('counting'),
+ ],
},
{"name": LINUX64,
"slavenames": ["tannit64"],
"builddir": LINUX64,
"factory": pypyOwnTestFactory,
- "category": 'linux64'
+ "category": 'linux64',
+ # this build needs 4 CPUs
+ "locks": [TannitCPU.access('counting'),
+ TannitCPU.access('counting'),
+ TannitCPU.access('counting'),
+ TannitCPU.access('counting'),
+ ],
},
{"name": MACOSX32,
"slavenames": ["minime"],
@@ -278,25 +321,29 @@
"slavenames": ["bigdogvm1", "tannit32"],
"builddir": APPLVLLINUX32,
"factory": pypyTranslatedAppLevelTestFactory,
- 'category': 'linux32'
+ 'category': 'linux32',
+ "locks": [TannitCPU.access('counting')],
},
{"name": APPLVLLINUX64,
"slavenames": ["tannit64"],
"builddir": APPLVLLINUX64,
"factory": pypyTranslatedAppLevelTestFactory64,
- "category": "linux64"
+ "category": "linux64",
+ "locks": [TannitCPU.access('counting')],
},
{"name": STACKLESSAPPLVLLINUX32,
"slavenames": ["bigdogvm1", "tannit32"],
"builddir": STACKLESSAPPLVLLINUX32,
"factory": pypyStacklessTranslatedAppLevelTestFactory,
- "category": 'linux32-stackless'
+ "category": 'linux32-stackless',
+ "locks": [TannitCPU.access('counting')],
},
{"name": OJITLINUX32,
"slavenames": ["bigdogvm1", "tannit32"],
"builddir": OJITLINUX32,
"factory": pypy_OjitTranslatedTestFactory,
- "category": 'linux32'
+ "category": 'linux32',
+ "locks": [TannitCPU.access('counting')],
},
{"name": APPLVLWIN32,
"slavenames": ["bigboard"],
@@ -315,12 +362,14 @@
'builddir' : JITLINUX32,
'factory' : pypyJITTranslatedTestFactory,
'category' : 'linux32',
+ "locks": [TannitCPU.access('counting')],
},
{'name': JITLINUX64,
'slavenames': ['tannit64'],
'builddir': JITLINUX64,
'factory': pypyJITTranslatedTestFactory64,
'category': 'linux64',
+ "locks": [TannitCPU.access('counting')],
},
{"name" : JITMACOSX64,
"slavenames": ["macmini-mvt", "xerxes"],
@@ -338,22 +387,25 @@
"slavenames": ["tannit32", "bigdogvm1"],
"builddir": JITONLYLINUX32,
"factory": pypyJitOnlyOwnTestFactory,
- "category": 'linux32'
+ "category": 'linux32',
+ "locks": [TannitCPU.access('counting')],
},
{"name": JITBENCH,
"slavenames": ["tannit32"],
"builddir": JITBENCH,
"factory": pypyJITBenchmarkFactory,
"category": 'benchmark-run',
+ # the locks are acquired with fine grain inside the build
},
{"name": JITBENCH64,
"slavenames": ["tannit64"],
"builddir": JITBENCH64,
"factory": pypyJITBenchmarkFactory64,
"category": "benchmark-run",
+ # the locks are acquired with fine grain inside the build
},
],
- 'buildbotURL': 'http://buildbot.pypy.org',
+ 'buildbotURL': 'http://buildbot.pypy.org/', # with a trailing '/'!
'projectURL': 'http://pypy.org/',
'projectName': 'PyPy'}
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
http://mail.python.org/mailman/listinfo/pypy-commit
