Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/example.conf URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/example.conf?rev=1413258&r1=1413257&r2=1413258&view=diff ============================================================================== --- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/example.conf (original) +++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/example.conf Sat Nov 24 20:29:11 2012 @@ -3,6 +3,7 @@ [DEFAULT] svnbin: /usr/local/bin/svn streams: http://svn.example.org:2069/commits/xml +hook: /usr/bin/true ## The values below are used by ConfigParser's interpolation syntax. ## See http://docs.python.org/library/configparser
Added: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/irkerbridge.py URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/irkerbridge.py?rev=1413258&view=auto ============================================================================== --- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/irkerbridge.py (added) +++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/irkerbridge.py Sat Nov 24 20:29:11 2012 @@ -0,0 +1,298 @@ +#!/usr/bin/env python +# +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# IrkerBridge - Bridge an SvnPubSub stream to Irker. + +# Example: +# irkerbridge.py --daemon --pidfile pid --logfile log config +# +# For detailed option help use: +# irkerbridge.py --help + +# It expects a config file that has the following parameters: +# streams=url +# Space separated list of URLs to streams. +# This option should only be in the DEFAULT section, is ignored in +# all other sections. +# NOTE: At current svnpubsub.client only accepts hostname and port +# combos so the path is ignored and /commits/xml is used. +# irker=hostname:port +# The hostname/port combination of the irker daemon. If port is +# omitted it defaults to 6659. Irker is connected to over UDP. +# match=What to use to decide if the commit should be sent to irker. +# It consists of the repository UUID followed by a slash and a glob pattern. +# The UUID may be replaced by a * to match all UUIDs. The glob pattern will +# be matched against all of the dirs_changed. Both the UUID and the glob +# pattern must match to send the message to irker. +# to=url +# Space separated list of URLs (any URL that Irker will accept) to +# send the resulting message to. At current Irker only supports IRC. +# template=string +# A string to use to format the output. The string is a Python +# string Template. The following variables are available: +# $author, $rev, $date, $uuid, $log, $log, $log_firstline, +# $log_firstparagraph, $dirs_changed, $dirs_count, $dirs_count_s, +# $subdirs_count, $subdirs_count_s, $dirs_root +# Most of them should be self explanatory. $dirs_count is the number of +# entries in $dirs_changed, $dirs_count_s is a friendly string version, +# $dirs_root is the common root of all the $dirs_changed, $subdirs_count +# is the number of subdirs under the $dirs_root that changed, +# $subdirs_root_s is a friendly string version. $log_firstparagraph cuts +# the log message at the first blank line and replaces newlines with spaces. +# +# Within the config file you have sections. Any configuration option +# missing from a given section is found in the [DEFAULT] section. +# +# Section names are arbitrary names that mean nothing to the bridge. Each +# section other than the [DEFAULT] section consists of a configuration that +# may match and send a message to irker to deliver. All matching sections +# will generate a message. +# +# Interpolation of values within the config file is allowed by including +# %(name)s within a value. For example I can reference the UUID of a repo +# repeatedly by doing: +# [DEFAULT] +# ASF_REPO=13f79535-47bb-0310-9956-ffa450edef68 +# +# [#commits] +# match=%(ASF_REPO)s/ +# +# You can HUP the process to reload the config file without restarting the +# process. However, you cannot change the streams it is listening to without +# restarting the process. +# +# TODO: Logging in a better way. + +# Messages longer than this will be truncated and ... added to the end such +# that the resulting message is no longer than this: +MAX_PRIVMSG = 400 + +import os +import sys +import posixpath +import socket +import json +import urlparse +import optparse +import ConfigParser +import traceback +import signal +import re +import fnmatch +from string import Template + +# Packages that come with svnpubsub +import svnpubsub.client +import daemonize + +class Daemon(daemonize.Daemon): + def __init__(self, logfile, pidfile, bdec): + daemonize.Daemon.__init__(self, logfile, pidfile) + + self.bdec = bdec + + def setup(self): + # There is no setup which the parent needs to wait for. + pass + + def run(self): + print 'irkerbridge started, pid=%d' % (os.getpid()) + + mc = svnpubsub.client.MultiClient(self.bdec.hostports, + self.bdec.commit, + self.bdec.event) + mc.run_forever() + + +class BigDoEverythingClass(object): + def __init__(self, config, options): + self.config = config + self.options = options + self.hostports = [] + for url in config.get_value('streams').split(): + parsed = urlparse.urlparse(url.strip()) + self.hostports.append((parsed.hostname, parsed.port or 80)) + + def locate_matching_configs(self, rev): + result = [ ] + for section in self.config.sections(): + match = self.config.get(section, "match").split('/', 1) + if len(match) < 2: + # No slash so assume all paths + match.append('*') + match_uuid, match_path = match + if rev.uuid == match_uuid or match_uuid == "*": + for path in rev.dirs_changed: + if fnmatch.fnmatch(path, match_path): + result.append(section) + break + return result + + def fill_in_extra_args(self, rev): + # Add entries to the rev object that are useful for + # formatting. + rev.log_firstline = rev.log.split("\n",1)[0] + rev.log_firstparagraph = re.split("\r?\n\r?\n",rev.log,1)[0] + rev.log_firstparagraph = re.sub("\r?\n"," ",rev.log_firstparagraph) + if rev.dirs_changed: + rev.dirs_root = posixpath.commonprefix(rev.dirs_changed) + rev.dirs_count = len(rev.dirs_changed) + if rev.dirs_count > 1: + rev.dirs_count_s = " (%d dirs)" %(rev.dirs_count) + else: + rev.dirs_count_s = "" + + rev.subdirs_count = rev.dirs_count + if rev.dirs_root in rev.dirs_changed: + rev.subdirs_count -= 1 + if rev.subdirs_count > 1: + rev.subdirs_count_s = " + %d subdirs" % (rev.subdirs_count) + else: + rev.subdirs_count_s = "" + + def _send(self, irker, msg): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + irker_list = irker.split(':') + if len(irker_list) < 2: + irker_list.append(6659) + json_msg = json.dumps(msg) + sock.sendto(json_msg, (irker_list[0],int(irker_list[1]))) + if self.options.verbose: + print "SENT: %s to %s" % (json_msg, irker) + + def join_all(self): + # Like self.commit(), but ignores self.config.get(section, "template"). + for section in self.config.sections(): + irker = self.config.get(section, "irker") + to_list = self.config.get(section, "to").split() + if not irker or not to_list: + continue + for to in to_list: + msg = {'to': to, 'privmsg': ''} + self._send(irker, msg) + + def commit(self, host, port, rev): + if self.options.verbose: + print "RECV: from %s:%s" % (host, port) + print json.dumps(vars(rev), indent=2) + + try: + config_sections = self.locate_matching_configs(rev) + if len(config_sections) > 0: + self.fill_in_extra_args(rev) + for section in config_sections: + irker = self.config.get(section, "irker") + to_list = self.config.get(section, "to").split() + template = self.config.get(section, "template") + if not irker or not to_list or not template: + continue + privmsg = Template(template).safe_substitute(vars(rev)) + if len(privmsg) > MAX_PRIVMSG: + privmsg = privmsg[:MAX_PRIVMSG-3] + '...' + for to in to_list: + msg = {'to': to, 'privmsg': privmsg} + self._send(irker, msg) + + except: + print "Unexpected error:" + traceback.print_exc() + sys.stdout.flush() + raise + + def event(self, host, port, event_name): + if self.options.verbose or event_name != "ping": + print 'EVENT: %s from %s:%s' % (event_name, host, port) + sys.stdout.flush() + + + +class ReloadableConfig(ConfigParser.SafeConfigParser): + def __init__(self, fname): + ConfigParser.SafeConfigParser.__init__(self) + + self.fname = fname + self.read(fname) + + signal.signal(signal.SIGHUP, self.hangup) + + def hangup(self, signalnum, frame): + self.reload() + + def reload(self): + print "RELOAD: config file: %s" % self.fname + sys.stdout.flush() + + # Delete everything. Just re-reading would overlay, and would not + # remove sections/options. Note that [DEFAULT] will not be removed. + for section in self.sections(): + self.remove_section(section) + + # Get rid of [DEFAULT] + self.remove_section(ConfigParser.DEFAULTSECT) + + # Now re-read the configuration file. + self.read(self.fname) + + def get_value(self, which): + return self.get(ConfigParser.DEFAULTSECT, which) + + +def main(args): + parser = optparse.OptionParser( + description='An SvnPubSub client that bridges the data to irker.', + usage='Usage: %prog [options] CONFIG_FILE', + ) + parser.add_option('--logfile', + help='filename for logging') + parser.add_option('--verbose', action='store_true', + help="enable verbose logging") + parser.add_option('--pidfile', + help="the process' PID will be written to this file") + parser.add_option('--daemon', action='store_true', + help='run as a background daemon') + + options, extra = parser.parse_args(args) + + if len(extra) != 1: + parser.error('CONFIG_FILE is requried') + config_file = os.path.abspath(extra[0]) + + if options.daemon: + if options.logfile: + logfile = os.path.abspath(options.logfile) + else: + parser.error('LOGFILE is required when running as a daemon') + + if options.pidfile: + pidfile = os.path.abspath(options.pidfile) + else: + parser.error('PIDFILE is required when running as a daemon') + + + config = ReloadableConfig(config_file) + bdec = BigDoEverythingClass(config, options) + + d = Daemon(logfile, pidfile, bdec) + if options.daemon: + d.daemonize_exit() + else: + d.foreground() + +if __name__ == "__main__": + main(sys.argv[1:]) Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub?rev=1413258&r1=1413257&r2=1413258&view=diff ============================================================================== --- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub (original) +++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub Sat Nov 24 20:29:11 2012 @@ -1,35 +1 @@ -#!/bin/sh -# -# PROVIDE: svnpubsub -# REQUIRE: DAEMON -# KEYWORD: shutdown - -. /etc/rc.subr - -name="svnpubsub" -rcvar=`set_rcvar` - -load_rc_config $name - -# -# DO NOT CHANGE THESE DEFAULT VALUES HERE -# SET THEM IN THE /etc/rc.conf FILE -# -svnpubsub_enable=${svnpubsub_enable-"NO"} -svnpubsub_user=${svnpubsub_user-"svn"} -svnpubsub_group=${svnpubsub_group-"svn"} -svnpubsub_reactor=${svnpubsub_reactor-"poll"} -svnpubsub_pidfile=${svnpubsub_pidfile-"/var/run/svnpubsub/svnpubsub.pid"} -pidfile="${svnpubsub_pidfile}" - -export PYTHON_EGG_CACHE="/home/svn/.python-eggs" - -command="/usr/local/bin/twistd" -command_args="-y /usr/local/svnpubsub/svnpubsub.tac \ - --logfile=/var/log/vc/svnpubsub.log \ - --pidfile=${pidfile} \ - --uid=${svnpubsub_user} --gid=${svnpubsub_user} \ - -r${svnpubsub_reactor}" - - -run_rc_command "$1" +link svnpubsub.freebsd \ No newline at end of file Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.debian URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.debian?rev=1413258&r1=1413257&r2=1413258&view=diff ============================================================================== --- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.debian (original) +++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.debian Sat Nov 24 20:29:11 2012 @@ -19,7 +19,7 @@ svnpubsub_pidfile=${svnpubsub_pidfile-"/ pidfile="${svnpubsub_pidfile}" TWSITD_CMD="/usr/bin/twistd -y /opt/svnpubsub/svnpubsub.tac \ - --logfile=/var/bwlog/svnpubsub/svnpubsub.log \ + --logfile=/var/log/svnpubsub/svnpubsub.log \ --pidfile=${pidfile} \ --uid=${svnpubsub_user} --gid=${svnpubsub_user} \ -r${svnpubsub_reactor}" Added: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd?rev=1413258&view=auto ============================================================================== --- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd (added) +++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd Sat Nov 24 20:29:11 2012 @@ -0,0 +1,35 @@ +#!/bin/sh +# +# PROVIDE: svnpubsub +# REQUIRE: DAEMON +# KEYWORD: shutdown + +. /etc/rc.subr + +name="svnpubsub" +rcvar=`set_rcvar` + +load_rc_config $name + +# +# DO NOT CHANGE THESE DEFAULT VALUES HERE +# SET THEM IN THE /etc/rc.conf FILE +# +svnpubsub_enable=${svnpubsub_enable-"NO"} +svnpubsub_user=${svnpubsub_user-"svn"} +svnpubsub_group=${svnpubsub_group-"svn"} +svnpubsub_reactor=${svnpubsub_reactor-"poll"} +svnpubsub_pidfile=${svnpubsub_pidfile-"/var/run/svnpubsub/svnpubsub.pid"} +pidfile="${svnpubsub_pidfile}" + +export PYTHON_EGG_CACHE="/home/svn/.python-eggs" + +command="/usr/local/bin/twistd" +command_args="-y /usr/local/svnpubsub/svnpubsub.tac \ + --logfile=/var/log/vc/svnpubsub.log \ + --pidfile=${pidfile} \ + --uid=${svnpubsub_user} --gid=${svnpubsub_user} \ + -r${svnpubsub_reactor}" + + +run_rc_command "$1" Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub?rev=1413258&r1=1413257&r2=1413258&view=diff ============================================================================== --- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub (original) +++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub Sat Nov 24 20:29:11 2012 @@ -1,38 +1 @@ -#!/bin/sh -# -# PROVIDE: svnwcsub -# REQUIRE: DAEMON -# KEYWORD: shutdown - -. /etc/rc.subr - -name="svnwcsub" -rcvar=`set_rcvar` - -load_rc_config $name - -# -# DO NOT CHANGE THESE DEFAULT VALUES HERE -# SET THEM IN THE /etc/rc.conf FILE -# -svnwcsub_enable=${svnwcsub_enable-"NO"} -svnwcsub_user=${svnwcsub_user-"svnwc"} -svnwcsub_group=${svnwcsub_group-"svnwc"} -svnwcsub_pidfile=${svnwcsub_pidfile-"/var/run/svnwcsub/svnwcsub.pub"} -svnwcsub_env="PYTHON_EGG_CACHE" -svnwcsub_cmd_int=${svnwcsub_cmd_int-"python"} -svnwcsub_config=${svnwcsub_config-"/etc/svnwcsub.conf"} -svnwcsub_logfile=${svnwcsub_logfile-"/var/log/svnwcsub/svnwcsub.log"} -pidfile="${svnwcsub_pidfile}" - -export PYTHON_EGG_CACHE="/var/run/svnwcsub" - -command="/usr/local/svnpubsub/svnwcsub.py" -command_interpreter="/usr/local/bin/${svnwcsub_cmd_int}" -command_args="--daemon \ - --logfile=${svnwcsub_logfile} \ - --pidfile=${pidfile} \ - --uid=${svnwcsub_user} --gid=${svnwcsub_group} \ - --umask=002 ${svnwcsub_config}" - -run_rc_command "$1" +link svnwcsub.freebsd \ No newline at end of file Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.debian URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.debian?rev=1413258&r1=1413257&r2=1413258&view=diff ============================================================================== --- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.debian (original) +++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.debian Sat Nov 24 20:29:11 2012 @@ -16,7 +16,7 @@ svnwcsub_user=${svnwcsub_user-"svnwc"} svnwcsub_group=${svnwcsub_group-"svnwc"} svnwcsub_pidfile=${svnwcsub_pidfile-"/var/run/svnwcsub.pid"} svnwcsub_config=${svnwcsub_config-"/etc/svnwcsub.conf"} -svnwcsub_logfile=${svnwcsub_logfile-"/var/bwlog/svnwcsub/svnwcsub.log"} +svnwcsub_logfile=${svnwcsub_logfile-"/var/log/svnwcsub/svnwcsub.log"} pidfile="${svnwcsub_pidfile}" SVNWCSUB_CMD="/opt/svnpubsub/svnwcsub.py \ @@ -24,6 +24,7 @@ SVNWCSUB_CMD="/opt/svnpubsub/svnwcsub.py --logfile=${svnwcsub_logfile} \ --pidfile=${pidfile} \ --uid=${svnwcsub_user} --gid=${svnwcsub_group} \ + --umask=002 \ ${svnwcsub_config} " RETVAL=0 Added: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.freebsd URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.freebsd?rev=1413258&view=auto ============================================================================== --- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.freebsd (added) +++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.freebsd Sat Nov 24 20:29:11 2012 @@ -0,0 +1,39 @@ +#!/bin/sh +# +# PROVIDE: svnwcsub +# REQUIRE: DAEMON +# KEYWORD: shutdown + +. /etc/rc.subr + +name="svnwcsub" +rcvar=`set_rcvar` + +load_rc_config $name + +# +# DO NOT CHANGE THESE DEFAULT VALUES HERE +# SET THEM IN THE /etc/rc.conf FILE +# +svnwcsub_enable=${svnwcsub_enable-"NO"} +svnwcsub_user=${svnwcsub_user-"svnwc"} +svnwcsub_group=${svnwcsub_group-"svnwc"} +svnwcsub_pidfile=${svnwcsub_pidfile-"/var/run/svnwcsub/svnwcsub.pub"} +svnwcsub_env="PYTHON_EGG_CACHE" +svnwcsub_cmd_int=${svnwcsub_cmd_int-"python"} +svnwcsub_config=${svnwcsub_config-"/etc/svnwcsub.conf"} +svnwcsub_logfile=${svnwcsub_logfile-"/var/log/svnwcsub/svnwcsub.log"} +pidfile="${svnwcsub_pidfile}" + +export PYTHON_EGG_CACHE="/var/run/svnwcsub" + +command="/usr/local/svnpubsub/svnwcsub.py" +command_interpreter="/usr/local/bin/${svnwcsub_cmd_int}" +command_args="--daemon \ + --logfile=${svnwcsub_logfile} \ + --pidfile=${pidfile} \ + --uid=${svnwcsub_user} --gid=${svnwcsub_group} \ + --umask=002 \ + ${svnwcsub_config}" + +run_rc_command "$1" Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.solaris URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.solaris?rev=1413258&r1=1413257&r2=1413258&view=diff ============================================================================== --- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.solaris (original) +++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.solaris Sat Nov 24 20:29:11 2012 @@ -14,8 +14,8 @@ SVNWCSUB_CMD="/usr/local/svnpubsub/svnwc --daemon \ --logfile=${svnwcsub_logfile} \ --pidfile=${pidfile} \ - --umask=002 \ --uid=${svnwcsub_user} --gid=${svnwcsub_group} \ + --umask=002 \ ${svnwcsub_config}" RETVAL=0 Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/client.py URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/client.py?rev=1413258&r1=1413257&r2=1413258&view=diff ============================================================================== --- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/client.py (original) +++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/client.py Sat Nov 24 20:29:11 2012 @@ -137,13 +137,13 @@ class XMLStreamHandler(xml.sax.handler.C elif self.chars and self.rev: value = self.chars.strip() if name == 'path': - self.rev.dirs_changed.append(value) + self.rev.dirs_changed.append(value.decode('unicode_escape')) elif name == 'author': - self.rev.author = value + self.rev.author = value.decode('unicode_escape') elif name == 'date': - self.rev.date = value + self.rev.date = value.decode('unicode_escape') elif name == 'log': - self.rev.log = value + self.rev.log = value.decode('unicode_escape') # Toss out any accumulated characters for this element. self.chars = '' Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/server.py URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/server.py?rev=1413258&r1=1413257&r2=1413258&view=diff ============================================================================== --- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/server.py (original) +++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/server.py Sat Nov 24 20:29:11 2012 @@ -73,12 +73,15 @@ import time class Revision: def __init__(self, r): + # Don't escape the values; json handles binary values fine. + # ET will happily emit literal control characters (eg, NUL), + # thus creating invalid XML, so the XML code paths do escaping. self.rev = r.get('revision') self.repos = r.get('repos') - self.dirs_changed = [x.encode('unicode_escape') for x in r.get('dirs_changed')] - self.author = r.get('author').encode('unicode_escape') - self.log = r.get('log').encode('unicode_escape') - self.date = r.get('date').encode('unicode_escape') + self.dirs_changed = [x for x in r.get('dirs_changed')] + self.author = r.get('author') + self.log = r.get('log') + self.date = r.get('date') def render_commit(self, format): if format == "json": @@ -90,13 +93,13 @@ class Revision: 'date': self.date}}) +"," elif format == "xml": c = ET.Element('commit', {'repository': self.repos, 'revision': "%d" % (self.rev)}) - ET.SubElement(c, 'author').text = self.author - ET.SubElement(c, 'date').text = self.date - ET.SubElement(c, 'log').text = self.log + ET.SubElement(c, 'author').text = self.author.encode('unicode_escape') + ET.SubElement(c, 'date').text = self.date.encode('unicode_escape') + ET.SubElement(c, 'log').text = self.log.encode('unicode_escape') d = ET.SubElement(c, 'dirs_changed') for p in self.dirs_changed: x = ET.SubElement(d, 'path') - x.text = p + x.text = p.encode('unicode_escape') str = ET.tostring(c, 'UTF-8') + "\n" return str[39:] else: @@ -112,7 +115,7 @@ class Revision: d = ET.SubElement(c, 'dirs_changed') for p in self.dirs_changed: x = ET.SubElement(d, 'path') - x.text = p + x.text = p.encode('unicode_escape') str = ET.tostring(c, 'UTF-8') + "\n" return str[39:] else: Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnwcsub.py URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnwcsub.py?rev=1413258&r1=1413257&r2=1413258&view=diff ============================================================================== --- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnwcsub.py (original) +++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnwcsub.py Sat Nov 24 20:29:11 2012 @@ -1,4 +1,5 @@ #!/usr/bin/env python +# encoding: UTF-8 # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -29,6 +30,7 @@ # See svnwcsub.conf for more information on its contents. # +import errno import subprocess import threading import sys @@ -71,6 +73,22 @@ def svn_info(svnbin, env, path): info[line[:idx]] = line[idx+1:].strip() return info +try: + import glob + glob.iglob + def is_emptydir(path): + # ### If the directory contains only dotfile children, this will readdir() + # ### the entire directory. But os.readdir() is not exposed to us... + for x in glob.iglob('%s/*' % path): + return False + for x in glob.iglob('%s/.*' % path): + return False + return True +except (ImportError, AttributeError): + # Python â¤2.4 + def is_emptydir(path): + # This will read the entire directory list to memory. + return not os.listdir(path) class WorkingCopy(object): def __init__(self, bdec, path, url): @@ -106,7 +124,7 @@ class WorkingCopy(object): def _get_match(self, svnbin, env): ### quick little hack to auto-checkout missing working copies - if not os.path.isdir(self.path): + if not os.path.isdir(self.path) or is_emptydir(self.path): logging.info("autopopulate %s from %s" % (self.path, self.url)) subprocess.check_call([svnbin, 'co', '-q', '--non-interactive', @@ -131,7 +149,8 @@ class BigDoEverythingClasss(object): self.svnbin = config.get_value('svnbin') self.env = config.get_env() self.tracking = config.get_track() - self.worker = BackgroundWorker(self.svnbin, self.env) + self.hook = config.get_value('hook') + self.worker = BackgroundWorker(self.svnbin, self.env, self.hook) self.watch = [ ] self.hostports = [ ] @@ -150,7 +169,7 @@ class BigDoEverythingClasss(object): # Add it to our watchers, and trigger an svn update. logging.info("Watching WC at %s <-> %s" % (wc.path, wc.url)) self.watch.append(wc) - self.worker.add_work(OP_UPDATE, wc) + self.worker.add_work(OP_BOOT, wc) def _normalize_path(self, path): if path[0] != '/': @@ -182,11 +201,12 @@ class BigDoEverythingClasss(object): # Start logging warnings if the work backlog reaches this many items BACKLOG_TOO_HIGH = 20 +OP_BOOT = 'boot' OP_UPDATE = 'update' OP_CLEANUP = 'cleanup' class BackgroundWorker(threading.Thread): - def __init__(self, svnbin, env): + def __init__(self, svnbin, env, hook): threading.Thread.__init__(self) # The main thread/process should not wait for this thread to exit. @@ -195,20 +215,28 @@ class BackgroundWorker(threading.Thread) self.svnbin = svnbin self.env = env + self.hook = hook self.q = Queue.Queue() self.has_started = False def run(self): while True: - if self.q.qsize() > BACKLOG_TOO_HIGH: - logging.warn('worker backlog is at %d', self.q.qsize()) - # This will block until something arrives operation, wc = self.q.get() + + # Warn if the queue is too long. + # (Note: the other thread might have added entries to self.q + # after the .get() and before the .qsize().) + qsize = self.q.qsize()+1 + if operation != OP_BOOT and qsize > BACKLOG_TOO_HIGH: + logging.warn('worker backlog is at %d', qsize) + try: if operation == OP_UPDATE: self._update(wc) + elif operation == OP_BOOT: + self._update(wc, boot=True) elif operation == OP_CLEANUP: self._cleanup(wc) else: @@ -228,7 +256,7 @@ class BackgroundWorker(threading.Thread) self.q.put((operation, wc)) - def _update(self, wc): + def _update(self, wc, boot=False): "Update the specified working copy." # For giggles, let's clean up the working copy in case something @@ -239,13 +267,15 @@ class BackgroundWorker(threading.Thread) ### we need to move some of these args into the config. these are ### still specific to the ASF setup. - args = [self.svnbin, 'update', + args = [self.svnbin, 'switch', '--quiet', '--non-interactive', '--trust-server-cert', '--ignore-externals', '--config-option', 'config:miscellany:use-commit-times=on', + '--', + wc.url, wc.path] subprocess.check_call(args, env=self.env) @@ -253,6 +283,15 @@ class BackgroundWorker(threading.Thread) info = svn_info(self.svnbin, self.env, wc.path) logging.info("updated: %s now at r%s", wc.path, info['Revision']) + ## Run the hook + if self.hook: + hook_mode = ['post-update', 'boot'][boot] + logging.info('running hook: %s at revision %s due to %s', + wc.path, info['Revision'], hook_mode) + args = [self.hook, hook_mode, + wc.path, info['Revision'], wc.url] + subprocess.check_call(args, env=self.env) + def _cleanup(self, wc): "Run a cleanup on the specified working copy."
