The branch, master has been updated via 9d5580e In fix.py, set host_id. via 5664afc Add tree_id and compiler_id to build. via b8f3f62 Fix actually bcc ing dead host mails. via a4cc868 Rename data -> build. via 2be9621 Some formatting fixes. via b053525 Fix typo. via 59fbbb9 Remove some entries from .bzrignore. via 012c724 Rename UploadBuildStore.get_new_builds -> UploadBuildStore.get_all_builds. via 7e3e70b Remove unused utility functions. via 65328b6 parse file line by line rather than loading entire file into memory. via 288e34a Remove unused load_hash function. from 1fc20bd No longer create cache directory.
http://gitweb.samba.org/?p=build-farm.git;a=shortlog;h=master - Log ----------------------------------------------------------------- commit 9d5580e1c7b29848278d0ecab14ab86ada25bf53 Author: Jelmer Vernooij <jel...@samba.org> Date: Mon Nov 22 23:12:56 2010 +0100 In fix.py, set host_id. commit 5664afc4b67cccb16dea594327868935856b2834 Author: Jelmer Vernooij <jel...@samba.org> Date: Mon Nov 22 23:03:49 2010 +0100 Add tree_id and compiler_id to build. commit b8f3f6202a311ea33e884b520da8c1915c7f5d37 Author: Jelmer Vernooij <jel...@samba.org> Date: Mon Nov 22 23:02:53 2010 +0100 Fix actually bcc ing dead host mails. commit a4cc868ee87764fbbbb498e0cd3858467e5b1e89 Author: Jelmer Vernooij <jel...@samba.org> Date: Mon Nov 22 22:50:38 2010 +0100 Rename data -> build. commit 2be9621f0758c0bbfed2643abd55b1cc5f93781f Author: Jelmer Vernooij <jel...@samba.org> Date: Mon Nov 22 22:43:19 2010 +0100 Some formatting fixes. commit b053525bd7aecabb98c92d97e4c1669c68a6decd Author: Jelmer Vernooij <jel...@samba.org> Date: Mon Nov 22 22:33:26 2010 +0100 Fix typo. commit 59fbbb9394ec2cd97924c6bfcac3f5f4d0c6c87d Author: Jelmer Vernooij <jel...@samba.org> Date: Mon Nov 22 22:33:18 2010 +0100 Remove some entries from .bzrignore. commit 012c724e970e8cd180b99c366cce4647dc1d684e Author: Jelmer Vernooij <jel...@samba.org> Date: Mon Nov 22 22:31:29 2010 +0100 Rename UploadBuildStore.get_new_builds -> UploadBuildStore.get_all_builds. commit 7e3e70bace5792ca4a5c232ff4f6a516d7e7badd Author: Jelmer Vernooij <jel...@samba.org> Date: Mon Nov 22 22:26:50 2010 +0100 Remove unused utility functions. commit 65328b644fef4e134abaa2ec80dfd5d2f523e85e Author: Jelmer Vernooij <jel...@samba.org> Date: Mon Nov 22 22:25:19 2010 +0100 parse file line by line rather than loading entire file into memory. commit 288e34ab51c7109da3f9034ff57b039932d4c3d1 Author: Jelmer Vernooij <jel...@samba.org> Date: Mon Nov 22 22:20:52 2010 +0100 Remove unused load_hash function. ----------------------------------------------------------------------- Summary of changes: .bzrignore | 2 - buildfarm/__init__.py | 43 ++++++------ buildfarm/{data.py => build.py} | 2 +- buildfarm/sqldb.py | 30 +++++++- buildfarm/tests/__init__.py | 2 +- buildfarm/tests/{test_data.py => test_build.py} | 83 ++++++++++++---------- buildfarm/tests/test_buildfarm.py | 4 +- buildfarm/tests/test_sqldb.py | 4 +- buildfarm/tests/test_util.py | 42 +----------- buildfarm/util.py | 59 ---------------- buildfarm/web/__init__.py | 73 ++++++++------------ buildfarm/web/tests/test_status.py | 2 +- import-and-analyse.py | 13 ++-- mail-dead-hosts.py | 2 +- tools/fix.py | 10 +++- 15 files changed, 147 insertions(+), 224 deletions(-) rename buildfarm/{data.py => build.py} (99%) rename buildfarm/tests/{test_data.py => test_build.py} (80%) Changeset truncated at 500 lines: diff --git a/.bzrignore b/.bzrignore index 678f66f..23886d3 100644 --- a/.bzrignore +++ b/.bzrignore @@ -1,5 +1,3 @@ -hostdb.sqlite _trial_temp db -cache data diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py index 509b024..bc5f415 100644 --- a/buildfarm/__init__.py +++ b/buildfarm/__init__.py @@ -61,13 +61,13 @@ def read_trees_from_conf(path): return ret -def lcov_extract_percentage(text): +def lcov_extract_percentage(f): """Extract the coverage percentage from the lcov file.""" - m = re.search('\<td class="headerItem".*?\>Code\ \;covered\:\<\/td\>.*?\n.*?\<td class="headerValue".*?\>([0-9.]+) \%', text) - if m: - return m.group(1) - else: - return None + for l in f: + m = re.search('\<td class="headerItem".*?\>Code\ \;covered\:\<\/td\>.*?\n.*?\<td class="headerValue".*?\>([0-9.]+) \%', l) + if m: + return m.group(1) + return None class BuildFarm(object): @@ -94,14 +94,14 @@ class BuildFarm(object): return "%s(%r)" % (self.__class__.__name__, self.path) def _open_build_results(self): - from buildfarm import data + from buildfarm.build import BuildResultStore path = os.path.join(self.path, "data", "oldrevs") - return data.BuildResultStore(path) + return BuildResultStore(path) def _open_upload_build_results(self): - from buildfarm import data + from buildfarm.build import UploadBuildResultStore path = os.path.join(self.path, "data", "upload") - return data.UploadBuildResultStore(path) + return UploadBuildResultStore(path) def _open_hostdb(self): from buildfarm import hostdb @@ -116,20 +116,17 @@ class BuildFarm(object): def lcov_status(self, tree): """get status of build""" - from buildfarm import data, util + from buildfarm.build import NoSuchBuildError file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "index.html") try: - lcov_html = util.FileLoad(file) + lcov_html = open(file, 'r') except (OSError, IOError): # File does not exist - raise data.NoSuchBuildError(tree, self.LCOVHOST, "lcov") - - perc = lcov_extract_percentage(lcov_html) - if perc is None: - ret = "" - else: - ret = perc - return perc + raise NoSuchBuildError(tree, self.LCOVHOST, "lcov") + try: + return lcov_extract_percentage(lcov_html) + finally: + lcov_html.close() def get_build(self, tree, host, compiler, rev=None, checksum=None): if rev is not None: @@ -140,7 +137,7 @@ class BuildFarm(object): def get_new_builds(self): hostnames = set([host.name for host in self.hostdb.hosts()]) - for build in self.upload_builds.get_new_builds(): + for build in self.upload_builds.get_all_builds(): if (build.tree in self.trees and build.compiler in self.compilers and build.host in hostnames): @@ -161,13 +158,13 @@ class BuildFarm(object): return max([build.upload_time for build in self.get_host_builds(host)]) def get_host_builds(self, host): - from buildfarm import data + from buildfarm.build import NoSuchBuildError ret = [] for compiler in self.compilers: for tree in sorted(self.trees.keys()): try: ret.append(self.get_build(tree, host, compiler)) - except data.NoSuchBuildError: + except NoSuchBuildError: pass ret.sort(reverse=True) return ret diff --git a/buildfarm/data.py b/buildfarm/build.py similarity index 99% rename from buildfarm/data.py rename to buildfarm/build.py index bec96f0..7f24a08 100644 --- a/buildfarm/data.py +++ b/buildfarm/build.py @@ -336,7 +336,7 @@ class UploadBuildResultStore(object): """ self.path = path - def get_new_builds(self): + def get_all_builds(self): for name in os.listdir(self.path): try: (build, tree, host, compiler, extension) = name.split(".") diff --git a/buildfarm/sqldb.py b/buildfarm/sqldb.py index 56be24b..8aa2cf8 100644 --- a/buildfarm/sqldb.py +++ b/buildfarm/sqldb.py @@ -21,7 +21,7 @@ from buildfarm import ( BuildFarm, Tree, ) -from buildfarm.data import ( +from buildfarm.build import ( Build, BuildResultStore, BuildStatus, @@ -74,6 +74,8 @@ class StormBuild(Build): status_str = RawStr(name="status") basename = RawStr() host_id = Int() + tree_id = Int() + compiler_id = Int() def status(self): return BuildStatus.__deserialize__(self.status_str) @@ -153,7 +155,8 @@ class StormHostDatabase(HostDatabase): return self.store.find(StormHost).order_by(StormHost.name) def __getitem__(self, name): - ret = self.store.find(StormHost, StormHost.name==name).one() + result = self.store.find(StormHost, Cast(StormHost.name, "TEXT") == Cast(name, "TEXT")) + ret = result.one() if ret is None: raise NoSuchHost(name) return ret @@ -227,6 +230,8 @@ class StormCachingBuildResultStore(BuildResultStore): new_build.upload_time = build.upload_time new_build.status_str = build.status().__serialize__() new_build.basename = new_basename + new_build.host_id = self.store.find( + StormHost, Cast(StormHost.name, "TEXT") == build.host).one().id self.store.add(new_build) return new_build @@ -341,15 +346,19 @@ CREATE TABLE IF NOT EXISTS host ( CREATE TABLE IF NOT EXISTS build ( id integer primary key autoincrement, tree blob not null, + tree_id int, revision blob, host blob not null, host_id integer, compiler blob not null, + compiler_id int, checksum blob, age int, status blob, basename blob, - FOREIGN KEY (host_id) REFERENCES host (id) + FOREIGN KEY (host_id) REFERENCES host (id), + FOREIGN KEY (tree_id) REFERENCES tree (id), + FOREIGN KEY (compiler_id) REFERENCES compiler (id) );""", noresult=True) db.execute("CREATE UNIQUE INDEX IF NOT EXISTS unique_checksum ON build (checksum);", noresult=True) db.execute(""" @@ -360,7 +369,20 @@ CREATE TABLE IF NOT EXISTS tree ( branch blob, subdir blob, repo blob - );""", noresult=True) + ); + """, noresult=True) + db.execute(""" +CREATE UNIQUE INDEX IF NOT EXISTS unique_tree_name ON tree(name); +""", noresult=True) + db.execute(""" +CREATE TABLE IF NOT EXISTS compiler ( + id integer primary key autoincrement, + name blob not null + ); + """, noresult=True) + db.execute(""" +CREATE UNIQUE INDEX IF NOT EXISTS unique_compiler_name ON compiler(name); +""", noresult=True) def memory_store(): diff --git a/buildfarm/tests/__init__.py b/buildfarm/tests/__init__.py index bef98f8..7d9e404 100644 --- a/buildfarm/tests/__init__.py +++ b/buildfarm/tests/__init__.py @@ -15,7 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -from buildfarm.data import Build +from buildfarm.build import Build from buildfarm.sqldb import setup_schema import os from storm import database diff --git a/buildfarm/tests/test_data.py b/buildfarm/tests/test_build.py similarity index 80% rename from buildfarm/tests/test_data.py rename to buildfarm/tests/test_build.py index fd39675..2dce9a3 100755 --- a/buildfarm/tests/test_data.py +++ b/buildfarm/tests/test_build.py @@ -20,7 +20,14 @@ import os import testtools import unittest -from buildfarm import data +from buildfarm.build import ( + Build, + BuildResultStore, + BuildStatus, + NoSuchBuildError, + UploadBuildResultStore, + build_status_from_logs, + ) from buildfarm.tests import BuildFarmTestCase @@ -29,7 +36,7 @@ class NonexistantTests(unittest.TestCase): def test_nonexistant(self): self.assertRaises( - Exception, data.BuildResultStore, "somedirthatdoesn'texist", None) + Exception, BuildResultStore, "somedirthatdoesn'texist", None) class BuildResultStoreTestBase(object): @@ -46,7 +53,7 @@ class BuildResultStoreTestBase(object): logname = build.basename + ".log" build.remove() self.assertFalse(os.path.exists(logname)) - self.assertRaises(data.NoSuchBuildError, self.x.get_build, "tdb", "charis", "cc", "12") + self.assertRaises(NoSuchBuildError, self.x.get_build, "tdb", "charis", "cc", "12") def test_build_repr(self): path = self.upload_mock_logfile(self.x, "tdb", "charis", "cc", @@ -55,7 +62,7 @@ class BuildResultStoreTestBase(object): self.assertEquals("<%s: revision 12 of tdb on charis using cc>" % build.__class__.__name__, repr(build)) def test_get_build_nonexistant(self): - self.assertRaises(data.NoSuchBuildError, self.x.get_build, "tdb", + self.assertRaises(NoSuchBuildError, self.x.get_build, "tdb", "charis", "cc", "12") def test_build_upload_time(self): @@ -120,7 +127,7 @@ error3""") path = self.create_mock_logfile("tdb", "charis", "cc", contents=""" BUILD COMMIT REVISION: myrev """) - build = data.Build(path[:-4], "tdb", "charis", "cc") + build = Build(path[:-4], "tdb", "charis", "cc") self.x.upload_build(build) uploaded_build = self.x.get_build("tdb", "charis", "cc", "myrev") self.assertEquals(uploaded_build.log_checksum(), build.log_checksum()) @@ -128,14 +135,14 @@ BUILD COMMIT REVISION: myrev def test_upload_build_no_rev(self): path = self.create_mock_logfile("tdb", "charis", "cc", contents=""" """) - build = data.Build(path[:-4], "tdb", "charis", "cc") + build = Build(path[:-4], "tdb", "charis", "cc") self.assertRaises(Exception, self.x.upload_build, build) def test_get_previous_revision(self): - self.assertRaises(data.NoSuchBuildError, self.x.get_previous_revision, "tdb", "charis", "cc", "12") + self.assertRaises(NoSuchBuildError, self.x.get_previous_revision, "tdb", "charis", "cc", "12") def test_get_latest_revision_none(self): - self.assertRaises(data.NoSuchBuildError, self.x.get_latest_revision, "tdb", "charis", "cc") + self.assertRaises(NoSuchBuildError, self.x.get_latest_revision, "tdb", "charis", "cc") def test_get_old_builds_none(self): self.assertEquals([], @@ -146,13 +153,13 @@ BUILD COMMIT REVISION: myrev contents=""" BUILD COMMIT REVISION: 12 """) - build = data.Build(path[:-4], "tdb", "charis", "cc") + build = Build(path[:-4], "tdb", "charis", "cc") b1 = self.x.upload_build(build) path = self.create_mock_logfile("tdb", "charis", "cc", contents=""" BUILD COMMIT REVISION: 15 """) - build = data.Build(path[:-4], "tdb", "charis", "cc") + build = Build(path[:-4], "tdb", "charis", "cc") b2 = self.x.upload_build(build) path = self.create_mock_logfile("tdb", "charis", "cc", contents=""" @@ -167,14 +174,14 @@ class BuildResultStoreTests(BuildFarmTestCase,BuildResultStoreTestBase): def setUp(self): super(BuildResultStoreTests, self).setUp() - self.x = data.BuildResultStore( + self.x = BuildResultStore( os.path.join(self.path, "data", "oldrevs")) class BuildStatusFromLogs(testtools.TestCase): def parse_logs(self, log, err): - return data.build_status_from_logs(StringIO(log), StringIO(err)) + return build_status_from_logs(StringIO(log), StringIO(err)) def test_nothing(self): s = self.parse_logs("", "") @@ -248,30 +255,30 @@ CC_CHECKER STATUS: 2 class BuildStatusTest(testtools.TestCase): def test_cmp_equal(self): - a = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)]) - b = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)]) + a = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)]) + b = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)]) self.assertEquals(cmp(a, b), 0) def test_cmp_empty(self): - self.assertEquals(cmp(data.BuildStatus(), data.BuildStatus()), 0) + self.assertEquals(cmp(BuildStatus(), BuildStatus()), 0) def test_cmp_other_failures(self): self.assertEquals(cmp( - data.BuildStatus((), set(["foo"])), data.BuildStatus((), set(["foo"]))), + BuildStatus((), set(["foo"])), BuildStatus((), set(["foo"]))), 0) def test_cmp_intermediate_errors(self): - a = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 3)]) - b = data.BuildStatus([("CONFIGURE", 2), ("TEST", 7), ("CC_CHECKER", 3)]) + a = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 3)]) + b = BuildStatus([("CONFIGURE", 2), ("TEST", 7), ("CC_CHECKER", 3)]) self.assertEquals(cmp(a, b), 1) def test_cmp_bigger(self): - a = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 3)]) - b = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)]) - c = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3)]) - d = data.BuildStatus([], set(["super error"])) - e = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)], set(["super error"])) + a = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 3)]) + b = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)]) + c = BuildStatus([("CONFIGURE", 2), ("TEST", 3)]) + d = BuildStatus([], set(["super error"])) + e = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)], set(["super error"])) # less stage means smaller, more error/higher error code means smaller as well self.assertEquals(cmp(b, a), 1) @@ -283,11 +290,11 @@ class BuildStatusTest(testtools.TestCase): self.assertEquals(cmp(b, e), 1) def test_cmp_smaller(self): - a = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)]) - b = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)]) - c = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3)]) - d = data.BuildStatus([], set(["super error"])) - e = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)], set(["super error"])) + a = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)]) + b = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)]) + c = BuildStatus([("CONFIGURE", 2), ("TEST", 3)]) + d = BuildStatus([], set(["super error"])) + e = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)], set(["super error"])) # less stage means smaller, more error/higher error code means smaller as well self.assertEquals(cmp(a, b), -1) @@ -299,16 +306,16 @@ class BuildStatusTest(testtools.TestCase): self.assertEquals(cmp(e, c), -1) def test_cmp_with_other_failures(self): - d = data.BuildStatus([], set(["super error"])) - e = data.BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)], set(["super error"])) + d = BuildStatus([], set(["super error"])) + e = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)], set(["super error"])) self.assertEquals(cmp(d, e), -1) def test_str(self): - a = data.BuildStatus([("CONFIGURE", 3), ("BUILD", 2)]) + a = BuildStatus([("CONFIGURE", 3), ("BUILD", 2)]) self.assertEquals("3/2", str(a)) def test_str_other_failures(self): - a = data.BuildStatus([("CONFIGURE", 3), ("BUILD", 2)], set(["panic"])) + a = BuildStatus([("CONFIGURE", 3), ("BUILD", 2)], set(["panic"])) self.assertEquals("panic", str(a)) @@ -317,8 +324,8 @@ class BuildStatusRegressedSinceTests(testtools.TestCase): def assertRegressedSince(self, expected, old_status, new_status): (stages1, other_failures1) = old_status (stages2, other_failures2) = new_status - a = data.BuildStatus(stages1, set(other_failures1)) - b = data.BuildStatus(stages2, set(other_failures2)) + a = BuildStatus(stages1, set(other_failures1)) + b = BuildStatus(stages2, set(other_failures2)) self.assertEquals(expected, b.regressed_since(a)) def test_same(self): @@ -365,10 +372,10 @@ class UploadBuildResultStoreTestBase(object): self.x.build_fname("mytree", "myhost", "cc"), "%s/data/upload/build.mytree.myhost.cc" % self.path) - def test_get_new_builds(self): - self.assertEquals([], list(self.x.get_new_builds())) + def test_get_all_builds(self): + self.assertEquals([], list(self.x.get_all_builds())) path = self.create_mock_logfile("tdb", "charis", "cc") - new_builds = list(self.x.get_new_builds()) + new_builds = list(self.x.get_all_builds()) self.assertEquals(1, len(new_builds)) self.assertEquals("tdb", new_builds[0].tree) self.assertEquals("charis", new_builds[0].host) @@ -380,7 +387,7 @@ class UploadBuildResultStoreTests(UploadBuildResultStoreTestBase,BuildFarmTestCa def setUp(self): super(UploadBuildResultStoreTests, self).setUp() - self.x = data.UploadBuildResultStore( + self.x = UploadBuildResultStore( os.path.join(self.path, "data", "upload")) diff --git a/buildfarm/tests/test_buildfarm.py b/buildfarm/tests/test_buildfarm.py index e213f90..70b8cce 100644 --- a/buildfarm/tests/test_buildfarm.py +++ b/buildfarm/tests/test_buildfarm.py @@ -17,9 +17,9 @@ from buildfarm import ( BuildFarm, - data, read_trees_from_conf, ) +from buildfarm.build import NoSuchBuildError from buildfarm.tests import BuildFarmTestCase import os @@ -118,7 +118,7 @@ class BuildFarmTestBase(object): self.assertEquals([], list(self.x.get_host_builds("myhost"))) def test_lcov_status_none(self): - self.assertRaises(data.NoSuchBuildError, self.x.lcov_status, "trivial") + self.assertRaises(NoSuchBuildError, self.x.lcov_status, "trivial") def test_tree(self): self.assertEquals("trivial", self.x.trees["trivial"].name) diff --git a/buildfarm/tests/test_sqldb.py b/buildfarm/tests/test_sqldb.py index d43bb96..0b5dc64 100644 --- a/buildfarm/tests/test_sqldb.py +++ b/buildfarm/tests/test_sqldb.py @@ -15,13 +15,13 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -from buildfarm.data import ( +from buildfarm.build import ( Build, NoSuchBuildError, ) from buildfarm.tests import BuildFarmTestCase from buildfarm.tests.test_buildfarm import BuildFarmTestBase -from buildfarm.tests.test_data import BuildResultStoreTestBase +from buildfarm.tests.test_build import BuildResultStoreTestBase from buildfarm.tests.test_hostdb import HostDatabaseTests from buildfarm.sqldb import ( StormHostDatabase, diff --git a/buildfarm/tests/test_util.py b/buildfarm/tests/test_util.py index 79a311d..17bb161 100755 --- a/buildfarm/tests/test_util.py +++ b/buildfarm/tests/test_util.py @@ -15,22 +15,11 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -import os -import tempfile import testtools -import unittest -- build.samba.org