The branch, master has been updated
via 1bfc40b Add configuration file with tree data.
via 322b0a5 Add convenience testcase class for buildfarm tests.
from bb7fd34 Convert to wsgi application.
http://gitweb.samba.org/?p=build-farm.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 1bfc40bcadef4d572ee3d36680a4dc168cd76178
Author: Jelmer Vernooij <[email protected]>
Date: Tue Nov 2 08:31:12 2010 +0100
Add configuration file with tree data.
commit 322b0a5c68c1cfcd62965bf8076b5673191ea6eb
Author: Jelmer Vernooij <[email protected]>
Date: Tue Nov 2 08:07:17 2010 +0100
Add convenience testcase class for buildfarm tests.
-----------------------------------------------------------------------
Summary of changes:
buildfarm/__init__.py | 1 +
buildfarm/data.py | 169 +++++++++++-------------------------------
buildfarm/history.py | 20 ++---
buildfarm/tests/__init__.py | 64 ++++++++++++++++
buildfarm/tests/test_data.py | 51 ++++++-------
web/build.py | 32 +++++---
web/trees.conf | 93 +++++++++++++++++++++++
7 files changed, 252 insertions(+), 178 deletions(-)
create mode 100644 web/trees.conf
Changeset truncated at 500 lines:
diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py
index e69de29..8b13789 100644
--- a/buildfarm/__init__.py
+++ b/buildfarm/__init__.py
@@ -0,0 +1 @@
+
diff --git a/buildfarm/data.py b/buildfarm/data.py
index ae45e9f..03b206b 100644
--- a/buildfarm/data.py
+++ b/buildfarm/data.py
@@ -22,8 +22,10 @@
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+import ConfigParser
import os
import re
+import time
import util
@@ -57,13 +59,42 @@ def status_info_cmp(self, s1, s2):
return s2["value"] - s1["value"]
+class Tree(object):
+ """A tree to build."""
+
+ def __init__(self, name, scm, repo, branch, subdir="", srcdir=""):
+ self.name = name
+ self.repo = repo
+ self.branch = branch
+ self.subdir = subdir
+ self.srcdir = srcdir
+
+ def __repr__(self):
+ return "<%s %r>" % (self.__class__.__name__, self.name)
+
+
+def read_trees_from_conf(path):
+ ret = {}
+ cfp = ConfigParser.ConfigParser()
+ cfp.readfp(open(path))
+ for s in cfp.sections():
+ ret[s] = Tree(name=s, **dict(cfp.items(s)))
+ return s
+
+
class BuildfarmDatabase(object):
+ """The build farm build result database."""
OLDAGE = 60*60*4,
DEADAGE = 60*60*24*4
LCOVHOST = "magni"
def __init__(self, basedir, readonly=False):
+ """Open the database.
+
+ :param basedir: Build result base directory
+ :param readonly: Whether to avoid saving cache files
+ """
self.basedir = basedir
check_dir_exists("base", self.basedir)
self.readonly = readonly
@@ -83,127 +114,7 @@ class BuildfarmDatabase(object):
self.compilers = util.load_list(os.path.join(self.webdir,
"compilers.list"))
self.hosts = util.load_hash(os.path.join(self.webdir, "hosts.list"))
- self.trees = {
- 'ccache': {
- 'scm': 'git',
- 'repo': 'ccache',
- 'branch': 'master',
- 'subdir': '',
- 'srcdir': ''
- },
- 'ccache-maint': {
- 'scm': 'git',
- 'repo': 'ccache',
- 'branch': 'maint',
- 'subdir': '',
- 'srcdir': ''
- },
- 'ppp': {
- 'scm': 'git',
- 'repo': 'ppp',
- 'branch': 'master',
- 'subdir': '',
- 'srcdir': ''
- },
- 'build_farm': {
- 'scm': 'svn',
- 'repo': 'build-farm',
- 'branch': 'trunk',
- 'subdir': '',
- 'srcdir': ''
- },
- 'samba-web': {
- 'scm': 'svn',
- 'repo': 'samba-web',
- 'branch': 'trunk',
- 'subdir': '',
- 'srcdir': ''
- },
- 'samba-docs': {
- 'scm': 'svn',
- 'repo': 'samba-docs',
- 'branch': 'trunk',
- 'subdir': '',
- 'srcdir': ''
- },
- 'lorikeet': {
- 'scm': 'svn',
- 'repo': 'lorikeeet',
- 'branch': 'trunk',
- 'subdir': '',
- 'srcdir': ''
- },
- 'samba_3_current': {
- 'scm': 'git',
- 'repo': 'samba.git',
- 'branch': 'v3-5-test',
- 'subdir': '',
- 'srcdir': 'source'
- },
- 'samba_3_next': {
- 'scm': 'git',
- 'repo': 'samba.git',
- 'branch': 'v3-6-test',
- 'subdir': '',
- 'srcdir': 'source'
- },
- 'samba_3_master': {
- 'scm': 'git',
- 'repo': 'samba.git',
- 'branch': 'master',
- 'subdir': '',
- 'srcdir': 'source'
- },
- 'samba_4_0_test': {
- 'scm': 'git',
- 'repo': 'samba.git',
- 'branch': 'master',
- 'subdir': '',
- 'srcdir': 'source4'
- },
- 'libreplace': {
- 'scm': 'git',
- 'repo': 'samba.git',
- 'branch': 'master',
- 'subdir': 'lib/replace/',
- 'srcdir': ''
- },
- 'talloc': {
- 'scm': 'git',
- 'repo': 'samba.git',
- 'branch': 'master',
- 'subdir': 'lib/talloc/',
- 'srcdir': ''
- },
- 'tdb': {
- 'scm': 'git',
- 'repo': 'samba.git',
- 'branch': 'master',
- 'subdir': 'lib/tdb/',
- 'srcdir': ''
- },
- 'ldb': {
- 'scm': 'git',
- 'repo': 'samba.git',
- 'branch': 'master',
- 'subdir': 'lib/ldb/',
- 'srcdir': ''
- },
- 'pidl': {
- 'scm': 'git',
- 'repo': 'samba.git',
- 'branch': 'master',
- 'subdir': 'pidl/',
- 'srcdir': ''
- },
- 'rsync': {
- 'scm': 'git',
- 'repo': 'rsync.git',
- 'branch': 'HEAD',
- 'subdir': '',
- 'srcdir': ''
- }
- }
+ self.trees = read_trees_from_conf(os.path.join(self.webdir,
"trees.conf"))
def cache_fname(self, tree, host, compiler, rev=None):
if rev is not None:
@@ -222,7 +133,6 @@ class BuildfarmDatabase(object):
# on a host.
# the ctime age is used to determine when the last real build happened
- ##############################################
def build_age_mtime(self, host, tree, compiler, rev):
"""get the age of build from mtime"""
file = self.build_fname(tree, host, compiler, rev)
@@ -256,11 +166,11 @@ class BuildfarmDatabase(object):
if rev:
if tree not in self.trees:
return rev
- if self.trees[tree]["scm"] != "git":
+ if self.trees[tree].scm != "git":
return rev
try:
- st1 = stat("%s.log" % file)
+ st1 = os.stat("%s.log" % file)
except OSError:
# File does not exist
return "NO SUCH FILE"
@@ -370,7 +280,16 @@ class BuildfarmDatabase(object):
cstatus, bstatus, istatus, tstatus, sstatus, dstatus, tostatus)
def build_status(self, host, tree, compiler, rev):
- """get status of build"""
+ """get status of build
+
+ :param host: Host name
+ :param tree: Tree name
+ :param compiler: Compiler name
+ :param rev: Revision
+ :return: string with build status
+ """
+ # FIXME: This should return a tuple
+
file = self.build_fname(tree, host, compiler, rev)
cachefile = self.cache_fname(tree, host, compiler, rev)+".status"
try:
diff --git a/buildfarm/history.py b/buildfarm/history.py
index 520c716..9a074fe 100644
--- a/buildfarm/history.py
+++ b/buildfarm/history.py
@@ -32,12 +32,6 @@ TIMEZONE = "PST"
TIMEOFFSET = 0
UNPACKED_DIR = "/home/ftp/pub/unpacked"
-CVSWEB_BASE = "http://pserver.samba.org/cgi-bin/cvsweb";
-VIEWCVS_BASE = "http://websvn.samba.org/cgi-bin/viewcvs.cgi";
-UNPACKED_BASE = "http://svn.samba.org/ftp/unpacked";
-GITWEB_BASE = "http://gitweb.samba.org";
-
-
class History(object):
def __init__(self, db):
@@ -51,14 +45,14 @@ class History(object):
# validate the tree
t = self.db.trees[tree]
- if t["scm"] == "cvs":
+ if t.scm == "cvs":
self._cvs_diff(t, author, date, tree)
- elif t["scm"] == "svn":
+ elif t.scm == "svn":
self._svn_diff(t, revision, tree)
- elif t["scm"] == "git":
+ elif t.scm == "git":
self._git_diff(t, revision, tree)
else:
- raise Exception("Unknown VCS %s" % t["scm"])
+ raise Exception("Unknown VCS %s" % t.scm)
def _svn_diff(self, t, revision, tree):
"""show recent svn entries"""
@@ -90,7 +84,7 @@ class History(object):
# get information about the current diff
title = "SVN Diff in %s:%s for revision r%s" % (
- tree, t["branch"], revision)
+ tree, t.branch, revision)
old_revision = revision - 1
cmd = "svn diff -r %s:%s" % (old_revision, revision)
@@ -118,7 +112,7 @@ class History(object):
t1 = time.ctime(date-60+(TIMEOFFSET*60*60)).strip()
t2 = time.ctime(date+60+(TIMEOFFSET*60*60)).strip()
- title = "CVS Diff in %s:%s for %s" % (tree, t["branch"], t1)
+ title = "CVS Diff in %s:%s for %s" % (tree, t.branch, t1)
if entry["TAG"] != "" and entry["REVISIONS"] != "":
raise Exception("sorry, cvs diff on branches not currently
possible due to a limitation in cvs")
@@ -162,7 +156,7 @@ class History(object):
# get information about the current diff
title = "GIT Diff in %s:%s for revision %s" % (
- tree, t["branch"], revision)
+ tree, t.branch, revision)
cmd = "git diff %s^ %s ./" % (revision, revision)
return (title, entry, tree, [(cmd, commands.getoutput("%s 2>
/dev/null" % cmd))])
diff --git a/buildfarm/tests/__init__.py b/buildfarm/tests/__init__.py
index e69de29..b304bcc 100644
--- a/buildfarm/tests/__init__.py
+++ b/buildfarm/tests/__init__.py
@@ -0,0 +1,64 @@
+#!/usr/bin/python
+# Copyright (C) Jelmer Vernooij <[email protected]> 2010
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+import os
+from testtools import TestCase
+import shutil
+import tempfile
+
+
+class BuildFarmTestCase(TestCase):
+ """Test case class that provides a build farm data directory and
convenience methods.
+ """
+
+ def write_hosts(self, hosts):
+ f = open(os.path.join(self.path, "web", "hosts.list"), "w")
+ try:
+ for host in hosts:
+ f.write("%s\n" % host)
+ finally:
+ f.close()
+
+ def write_compilers(self, compilers):
+ f = open(os.path.join(self.path, "web", "compilers.list"), "w")
+ try:
+ for compiler in compilers:
+ f.write("%s\n" % compiler)
+ finally:
+ f.close()
+
+ def write_trees(self, trees):
+ f = open(os.path.join(self.path, "web", "trees.conf"), "w")
+ try:
+ for t in trees:
+ f.write("[%s]\n" % t)
+ for k, v in trees[t].iteritems():
+ f.write("%s = %s\n" % (k, v))
+ f.write("\n")
+ finally:
+ f.close()
+
+ def setUp(self):
+ super(BuildFarmTestCase, self).setUp()
+ self.path = tempfile.mkdtemp()
+
+ for subdir in ["data", "cache", "web", "lcov", "lcov/data"]:
+ os.mkdir(os.path.join(self.path, subdir))
+
+ def tearDown(self):
+ shutil.rmtree(self.path)
+ super(BuildFarmTestCase, self).tearDown()
diff --git a/buildfarm/tests/test_data.py b/buildfarm/tests/test_data.py
index 431063d..c178fa5 100755
--- a/buildfarm/tests/test_data.py
+++ b/buildfarm/tests/test_data.py
@@ -5,58 +5,53 @@
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
-#
+#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
-#
+#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-import os
-import shutil
-import tempfile
import unittest
from buildfarm import data
+from buildfarm.tests import BuildFarmTestCase
+
+
class NonexistantTests(unittest.TestCase):
def test_nonexistant(self):
- self.assertRaises(Exception, data.BuildfarmDatabase,
"somedirthatdoesn'texist", None)
+ self.assertRaises(
+ Exception, data.BuildfarmDatabase, "somedirthatdoesn'texist", None)
-class BuildfarmDatabaseTests(unittest.TestCase):
+class BuildfarmDatabaseTests(BuildFarmTestCase):
def setUp(self):
super(BuildfarmDatabaseTests, self).setUp()
- self.path = tempfile.mkdtemp()
-
- for subdir in ["data", "cache", "web", "lcov", "lcov/data"]:
- os.mkdir(os.path.join(self.path, subdir))
-
- f = open(os.path.join(self.path, "web", "compilers.list"), "w")
- f.write("cc\n")
- f.close()
-
- f = open(os.path.join(self.path, "web", "hosts.list"), "w")
- f.write("gwenhwyvar\n")
- f.write("charis\n")
- f.close()
+ self.write_compilers(["cc"])
+ self.write_hosts(["gwenhwyvar", "charis"])
+ self.write_trees({"tdb": {"scm": "git", "repo": "tdb", "branch":
"master"}})
self.x = data.BuildfarmDatabase(self.path)
def test_build_fname(self):
- self.assertEquals(self.x.build_fname("mytree", "myhost", "cc"),
"%s/data/upload/build.mytree.myhost.cc" % self.path)
- self.assertEquals(self.x.build_fname("mytree", "myhost", "cc", 123),
"%s/data/oldrevs/build.mytree.myhost.cc-123" % self.path)
+ self.assertEquals(
+ self.x.build_fname("mytree", "myhost", "cc"),
+ "%s/data/upload/build.mytree.myhost.cc" % self.path)
+ self.assertEquals(
+ self.x.build_fname("mytree", "myhost", "cc", 123),
+ "%s/data/oldrevs/build.mytree.myhost.cc-123" % self.path)
def test_cache_fname(self):
- self.assertEquals(self.x.cache_fname("mytree", "myhost", "cc", 123),
"%s/cache/build.mytree.myhost.cc-123" % self.path)
- self.assertEquals(self.x.cache_fname("mytree", "myhost", "cc"),
"%s/cache/build.mytree.myhost.cc" % self.path)
-
- def tearDown(self):
- shutil.rmtree(self.path)
- super(BuildfarmDatabaseTests, self).tearDown()
+ self.assertEquals(
+ self.x.cache_fname("mytree", "myhost", "cc", 123),
+ "%s/cache/build.mytree.myhost.cc-123" % self.path)
+ self.assertEquals(
+ self.x.cache_fname("mytree", "myhost", "cc"),
+ "%s/cache/build.mytree.myhost.cc" % self.path)
diff --git a/web/build.py b/web/build.py
index 0ee014e..ec5b318 100755
--- a/web/build.py
+++ b/web/build.py
@@ -24,10 +24,13 @@
# TODO: Allow filtering of the "Recent builds" list to show
# e.g. only broken builds or only builds that you care about.
+import os
+import sys
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
from buildfarm import data, util, history
import cgi
-import os
import re
import time
@@ -44,6 +47,11 @@ hosts = db.hosts
trees = db.trees
OLDAGE = db.OLDAGE
+CVSWEB_BASE = "http://pserver.samba.org/cgi-bin/cvsweb";
+VIEWCVS_BASE = "http://websvn.samba.org/cgi-bin/viewcvs.cgi";
+UNPACKED_BASE = "http://svn.samba.org/ftp/unpacked";
+GITWEB_BASE = "http://gitweb.samba.org";
+
# this is automatically filled in
deadhosts = []
@@ -253,7 +261,7 @@ def view_recent_builds(myself, tree, sort_by):
sorturl = "%s?tree=%s;function=Recent+Builds" % (myself, tree)
yield "<div id='recent-builds' class='build-section'>"
- yield "<h2>Recent builds of %s (%s branch %s)</h2>" % (tree, t["scm"],
t["branch"])
+ yield "<h2>Recent builds of %s (%s branch %s)</h2>" % (tree, t.scm,
t.branch)
yield "<table class='real'>"
yield "<thead>"
yield "<tr>"
@@ -671,7 +679,7 @@ def main_menu():
yield "</select>"
yield "<select name='tree'>"
for tree, t in trees.iteritems():
- yield "<option value='%s'>%s:%s</option>" % (tree, tree, t["branch"])
+ yield "<option value='%s'>%s:%s</option>" % (tree, tree, t.branch)
yield "</select>"
yield "<select name='compiler'>"
for compiler in compilers:
--
build.samba.org
