Hi guys,
I'll be offline from this morning 10.30 CET for a few days (I go to the
countryside without Internet access), I'm attaching uncommitted changes
to Patchy made on Grenouille; this is a total mess, changes should be
split up in several commits, and some code rewrite might not hurt, I'll
take care of this when I'm back online, around Friday.
Best,
John
diff --git a/patches/compile_lilypond_test/__init__.py b/patches/compile_lilypond_test/__init__.py
index 0c8e3e4..0ecb8b2 100644
--- a/patches/compile_lilypond_test/__init__.py
+++ b/patches/compile_lilypond_test/__init__.py
@@ -11,6 +11,7 @@ import email.utils
from ConfigParser import NoOptionError
stderr = sys.stderr
+PID_FILE = "patchy.pid"
def info (s):
stderr.write (s + '\n')
@@ -43,6 +44,18 @@ class NothingToDoException (Exception):
class FailedCommand (Exception):
pass
+class VersionControlError (Exception):
+ pass
+
+class WastedBuildException (Exception):
+ pass
+
+def remote_branch_name (branch):
+ if config.getboolean ("source", "bare_git_repository"):
+ return branch
+ else:
+ return "%s/%s" % (config.get ("source", "git_remote_name"), branch)
+
def run (cmd, **kwargs):
""" runs the command and returns the stdout when successful,
otherwise raises an exception that includes the stderr """
@@ -62,22 +75,22 @@ try:
git_repository_dir = os.path.expanduser (config.get ("source", "git_repository_dir"))
except NoOptionError:
git_repository_dir = ""
- if not (os.path.isdir (git_repository_dir) and run ("git status", cwd=git_repository_dir)):
+ if not (os.path.isdir (git_repository_dir) and run ("git log -1", cwd=git_repository_dir)):
git_repository_dir = os.path.expanduser (os.environ["LILYPOND_GIT"])
- assert os.path.isdir (git_repository_dir) and run ("git status", cwd=git_repository_dir)
+ assert os.path.isdir (git_repository_dir) and run ("git log -1", cwd=git_repository_dir)
except Exception as e:
error ("%s: non-existent directory or directory not containing a valid Git repository: %s" % (e.__class__.__name__, e))
info ("Please set git_repository_dir in [source] section of the configuration file\n or environment variable LILYPOND_GIT.")
sys.exit (1)
-def send_email (email_command, logfile, CC=False):
+def send_email (email_command, logfile, to, cc_replyto, CC=False):
p = os.popen (email_command, 'w')
p.write ("From: %s\n" % config.get ("notification", "from"))
- p.write ("Reply-To: [email protected]\n")
- p.write ("To: [email protected]\n")
+ p.write ("Reply-To: %s\n" % cc_replyto)
+ p.write ("To: %s\n" % to)
if CC:
- p.write ("Cc: [email protected]\n")
+ p.write ("Cc: %s\n" % cc_replyto)
p.write ("Date: %s\n" % email.utils.formatdate (localtime=True))
p.write ("User-Agent: Patchy - LilyPond autobuild\n")
p.write ("Subject: %s\n\n" % config.get ("notification", "subject"))
@@ -90,7 +103,8 @@ def send_email (email_command, logfile, CC=False):
class AutoCompile (object):
### setup
- def __init__ (self):
+ def __init__ (self, branch="staging"):
+ self.branch = branch
self.date = datetime.datetime.now ().strftime ("%Y-%m-%d-%H")
self.git_repository_dir = git_repository_dir
self.auto_compile_dir = os.path.expanduser (
@@ -99,13 +113,19 @@ class AutoCompile (object):
os.mkdir (self.auto_compile_dir)
self.src_build_dir = os.path.expanduser (
config.get ("compiling", "build_dir"))
+ special_build_dir = config.get (branch, "build_dir")
+ if special_build_dir:
+ self.src_build_dir = os.path.expanduser (special_build_dir)
self.build_dir = os.path.join (self.src_build_dir, 'build')
- self.commit = self.get_head ()
- self.prev_good_commit = config.get ("previous good compile", "last_known")
+ self.commit = self.get_head (branch)
self.logfile = build_logfile.BuildLogfile (
os.path.join (self.auto_compile_dir,
str (MAIN_LOG_FILENAME % self.date)),
self.commit)
+ self.prev_good_commit = config.get (branch, "last_known_good_build")
+ self.notification_to = config.get (branch, "notification_to")
+ self.notification_cc_replyto = config.get (branch, "notification_cc")
+ self.web_install_dir = config.get (branch, "web_install_dir")
def debug (self):
""" prints member variables """
@@ -115,16 +135,18 @@ class AutoCompile (object):
def notify (self, CC=False):
email_command = config.get ("notification", "smtp_command")
if len (email_command) > 2:
- send_email (email_command, self.logfile, CC)
+ send_email (email_command, self.logfile, self.notification_to,
+ self.notification_cc_replyto, CC)
else:
info ("Message for you in %s" % self.logfile.filename)
- def get_head (self):
+ def get_head (self, branch="master"):
os.chdir (self.git_repository_dir)
- return run ("git rev-parse %s/master" % config.get ("source", "git_remote_name"))
+ return run ("git rev-parse %s" %
+ remote_branch_name (branch))
- def write_good_commit (self):
- config.set ("previous good compile", "last_known",
+ def write_good_commit (self, branch="staging"):
+ config.set (branch, "last_known_good_build",
self.commit)
config.save ()
@@ -134,14 +156,18 @@ class AutoCompile (object):
shutil.rmtree (self.src_build_dir)
def install_web (self):
- web_dest = config.get ("install", "web_dir").translate (None, ';$`()[]{}|&|')
- if os.path.exists (web_dest):
- shutil.rmtree (web_dest)
- os.makedirs (web_dest)
+ # security measure for use in shell command
+ dest = self.web_install_dir.translate (None, ';$`()[]{}|&|')
+ if not (dest and os.path.isdir (os.path.dirname (dest))):
+ return
+ self.logfile.write ("Installing documentation")
+ if os.path.exists (dest):
+ shutil.rmtree (dest)
+ os.makedirs (dest)
web_root = os.path.join (self.build_dir, "out-www", "offline-root")
os.chdir (web_root)
run ("find -not -type d |xargs %s/scripts/build/out/mass-link hard . %s" %
- (self.build_dir, web_dest), shell=True)
+ (self.build_dir, dest), shell=True)
def update_git (self):
os.chdir (self.git_repository_dir)
@@ -149,7 +175,7 @@ class AutoCompile (object):
def make_directories (self, branch_name):
os.chdir (self.git_repository_dir)
- run ("git branch -f test-%s %s/master" % (branch_name, config.get ("source", "git_remote_name")))
+ run ("git branch -f test-%s %s/master" % (branch_name, remote_branch_name ("master")))
run ("git clone -s -b test-%s -o local %s %s" % (branch_name, self.git_repository_dir, self.src_build_dir))
os.makedirs (self.build_dir)
@@ -252,73 +278,117 @@ class AutoCompile (object):
os.path.join (self.build_dir, "out/test-results/"),
os.path.join (self.build_dir, "show-%i/test-results/" % issue_id))
- def merge_staging_git (self):
+ def merge_git_branch (self, branch):
os.chdir (self.git_repository_dir)
run ("git fetch")
- ### don't force a new branch here; if it already exists,
- ### we want to die. We use the "test-master-lock" branch like
- ### a lockfile
- origin = config.get ("source", "git_remote_name")
- run ("git branch test-master-lock %s/master" % origin)
- run ("git branch -f test-staging %s/staging" % origin)
+ ### Don't force a new branch here; if it already exists, we
+ ### want to fail unless a previous Patchy instance has died.
+ ### We use the "test-master-lock" branch like a lockfile.
+ try:
+ run ("git branch test-master-lock %s" % remote_branch_name ("master"))
+ except FailedCommand as e:
+ pid_file_path = os.path.join (self.build_dir, PID_FILE)
+ if os.path.isfile (pid_file_path):
+ previous_pid = open (pid_file_path).read ()
+ if os.path.isdir (os.path.join ("/proc", previous-pid)):
+ self.logfile.write ("Another instance (PID %s) is already running.")
+ raise
+ else:
+ self.logfile.write (
+ "test-master-lock and PID file exist but previous Patchy\n" +
+ "run (PID %s) died, resetting test-master-lock anyway." %
+ previous_pid)
+ run ("git branch -f test-master-lock %s" % remote_branch_name ("master"))
+ else:
+ raise
+ run ("git branch -f test-%s %s" % (branch, remote_branch_name (branch)))
if os.path.exists (self.src_build_dir):
shutil.rmtree (self.src_build_dir)
run ("git clone -s -b test-master-lock -o local %s %s" % (self.git_repository_dir, self.src_build_dir))
os.chdir (self.src_build_dir)
- run ("git merge --ff-only local/test-staging")
+ run ("git merge --ff-only local/test-%s" % branch)
self.commit = run ("git rev-parse HEAD")
if self.commit == self.prev_good_commit:
raise NothingToDoException ("Nothing to do")
- self.logfile.write ("Merged staging, now at:\t%s\n" % self.commit)
+ self.logfile.write ("Merged %s, now at:\t%s\n" % (branch, self.commit))
run ("git push local test-master-lock")
os.makedirs (self.build_dir)
+ pid_file_path = os.path.join (self.build_dir, PID_FILE)
+ open (pid_file_path, 'w').write (str (os.getpid ()))
def merge_push (self):
os.chdir (self.git_repository_dir)
+ run ("git fetch")
+ if run ("git log -1 %s..test-staging" % remote_branch_name ("staging")):
+ raise VersionControlError (
+ "Branch staging has been reset to some parent commit,\n" +
+ "aborting operation without pushing")
+ origin_head = remote_branch_name ("HEAD")
+ if not run ("git log -1 %s..test-staging" % origin_head):
+ if run ("git log -1 test-staging..%s" % origin_head):
+ self.logfile.write ("origin has a newer revision than test-staging, not pushing")
+ return
+ else:
+ raise WastedBuildException ()
run ("git push %s test-master-lock:master" % config.get ("source", "git_remote_name"))
self.logfile.add_success ("pushed to master\n")
- # TODO: update dev/staging in some way?
def remove_test_master_lock (self):
os.chdir (self.git_repository_dir)
run ("git branch -D test-master-lock")
+ run ("rm -f %s" % os.path.join (self.build_dir, PID_FILE))
-
- def merge_staging (self):
- """ merges the staging branch, then returns whether there
+ def merge_branch (self, branch):
+ """ merges a branch, then returns whether there
is anything to check. """
try:
- self.merge_staging_git ()
+ self.merge_git_branch (branch)
return True
except NothingToDoException:
- self.logfile.add_success ("No new commits in staging")
+ self.logfile.add_success ("No new commits in %s" % branch)
if config.getboolean ("notification", "notify_non_action"):
self.notify ()
except Exception as e:
- self.logfile.failed_step ("merge from staging", str(e))
+ self.logfile.failed_step ("merge from %s" % branch, str(e))
self.notify (CC=True)
return False
def handle_staging (self):
try:
- if self.merge_staging ():
- issue_id = "staging"
+ if self.merge_branch (self.branch):
+ issue_id = self.branch
self.configure (issue_id)
self.build (quick_make=False, issue_id=issue_id)
self.write_good_commit ()
self.merge_push ()
self.write_good_commit ()
- if os.path.isdir (os.path.dirname (config.get ("install", "web_dir"))):
- self.install_web ()
+ self.install_web ()
self.notify ()
+ except WastedBuildException:
+ pass
except Exception as e:
self.logfile.failed_step ("merge from staging", str (e))
self.notify (CC=True)
self.remove_test_master_lock ()
+ def handle_translation (self):
+ try:
+ if self.merge_branch (self.branch):
+ issue_id = self.branch
+ self.configure (issue_id)
+ self.build (quick_make=False, issue_id=issue_id)
+ self.write_good_commit ("translation")
+ self.install_web ()
+ self.notify ()
+ except Exception as e:
+ self.logfile.failed_step ("build translation", str (e))
+ self.notify (CC=True)
+ self.remove_test_master_lock ()
+
+
def main (patches = None):
if not patches:
pass
diff --git a/patches/compile_lilypond_test/patchy_config.py b/patches/compile_lilypond_test/patchy_config.py
index a84cadd..0d7e690 100644
--- a/patches/compile_lilypond_test/patchy_config.py
+++ b/patches/compile_lilypond_test/patchy_config.py
@@ -10,6 +10,7 @@ default_config = {
"source": {
"git_repository_dir": "~/git/lilypond-git",
"git_remote_name": "origin",
+ "bare_git_repository": "no",
},
"configure_environment": {},
"compiling": {
@@ -17,18 +18,26 @@ default_config = {
"build_dir": "/tmp/lilypond-autobuild/",
"auto_compile_results_dir": "~/lilypond-auto-compile-results/",
},
- "previous good compile": {
- "last_known": "",
- },
"notification": {
"notify_non_action": "yes",
"from": "patchy",
"smtp_command": "#msmtp -C ~/.msmtp-patchy -t",
"subject": "Patchy email",
},
- "install": {
- "web_dir": "",
+ "staging": {
+ "last_known_good_build": "",
+ "build_dir": "",
+ "web_install_dir": "",
+ "notification_to": "[email protected]",
+ "notification_cc": "[email protected]",
},
+ "translation": {
+ "last_known_good_build": "",
+ "build_dir": "",
+ "web_install_dir": "",
+ "notification_to": "[email protected]",
+ "notification_cc": "[email protected]",
+ }
}
class PatchyConfig (ConfigParser.RawConfigParser):
diff --git a/patches/lilypond-patchy-translation.py b/patches/lilypond-patchy-translation.py
new file mode 100755
index 0000000..4174de1
--- /dev/null
+++ b/patches/lilypond-patchy-translation.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+import compile_lilypond_test
+
+def main():
+ patchy = compile_lilypond_test.AutoCompile ("translation")
+ patchy.handle_translation ()
+
+if __name__ == "__main__":
+ main()
+
_______________________________________________
lilypond-devel mailing list
[email protected]
https://lists.gnu.org/mailman/listinfo/lilypond-devel
