Modified: subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/client.py URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/client.py?rev=1489765&r1=1489764&r2=1489765&view=diff ============================================================================== --- subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/client.py (original) +++ subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/client.py Wed Jun 5 09:22:43 2013 @@ -62,7 +62,8 @@ class SvnpubsubClientException(Exception class Client(asynchat.async_chat): - def __init__(self, url, commit_callback, event_callback): + def __init__(self, url, commit_callback, event_callback, + metadata_callback = None): asynchat.async_chat.__init__(self) self.last_activity = time.time() @@ -82,7 +83,8 @@ class Client(asynchat.async_chat): self.event_callback = event_callback - self.parser = JSONRecordHandler(commit_callback, event_callback) + self.parser = JSONRecordHandler(commit_callback, event_callback, + metadata_callback) # Wait for the end of headers. Then we start parsing JSON. self.set_terminator(b'\r\n\r\n') @@ -94,7 +96,7 @@ class Client(asynchat.async_chat): except: self.handle_error() return - + self.push(('GET %s HTTP/1.0\r\n\r\n' % resource).encode('ascii')) def handle_connect(self): @@ -123,39 +125,53 @@ class Client(asynchat.async_chat): self.last_activity = time.time() if not self.skipping_headers: - self.ibuffer.append(data) + self.ibuffer.append(data) + + +class Notification(object): + def __init__(self, data): + self.__dict__.update(data) + +class Commit(Notification): + KIND = 'COMMIT' + +class Metadata(Notification): + KIND = 'METADATA' class JSONRecordHandler: - def __init__(self, commit_callback, event_callback): + def __init__(self, commit_callback, event_callback, metadata_callback): self.commit_callback = commit_callback self.event_callback = event_callback + self.metadata_callback = metadata_callback + + EXPECTED_VERSION = 1 def feed(self, record): obj = json.loads(record) if 'svnpubsub' in obj: actual_version = obj['svnpubsub'].get('version') - EXPECTED_VERSION = 1 - if actual_version != EXPECTED_VERSION: - raise SvnpubsubClientException("Unknown svnpubsub format: %r != %d" - % (actual_format, expected_format)) + if actual_version != self.EXPECTED_VERSION: + raise SvnpubsubClientException( + "Unknown svnpubsub format: %r != %d" + % (actual_version, self.EXPECTED_VERSION)) self.event_callback('version', obj['svnpubsub']['version']) elif 'commit' in obj: commit = Commit(obj['commit']) self.commit_callback(commit) elif 'stillalive' in obj: self.event_callback('ping', obj['stillalive']) - - -class Commit(object): - def __init__(self, commit): - self.__dict__.update(commit) + elif 'metadata' in obj and self.metadata_callback: + metadata = Metadata(obj['metadata']) + self.metadata_callback(metadata) class MultiClient(object): - def __init__(self, urls, commit_callback, event_callback): + def __init__(self, urls, commit_callback, event_callback, + metadata_callback = None): self.commit_callback = commit_callback self.event_callback = event_callback + self.metadata_callback = metadata_callback # No target time, as no work to do self.target_time = 0 @@ -185,9 +201,15 @@ class MultiClient(object): def _add_channel(self, url): # Simply instantiating the client will install it into the global map # for processing in the main event loop. - Client(url, - functools.partial(self.commit_callback, url), - functools.partial(self._reconnect, url)) + if self.metadata_callback: + Client(url, + functools.partial(self.commit_callback, url), + functools.partial(self._reconnect, url), + functools.partial(self.metadata_callback, url)) + else: + Client(url, + functools.partial(self.commit_callback, url), + functools.partial(self._reconnect, url)) def _check_stale(self): now = time.time()
Modified: subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/server.py URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/server.py?rev=1489765&r1=1489764&r2=1489765&view=diff ============================================================================== --- subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/server.py (original) +++ subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/server.py Wed Jun 5 09:22:43 2013 @@ -28,17 +28,24 @@ # Currently supports both XML and JSON serialization. # # Example Sub clients: -# curl -sN http://127.0.0.1:2069/commits -# curl -sN http://127.0.0.1:2069/commits/svn/* -# curl -sN http://127.0.0.1:2069/commits/svn -# curl -sN http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68 -# curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68 -# -# URL is built into 2 parts: -# /commits/${optional_type}/${optional_repository} -# -# If the type is included in the URL, you will only get commits of that type. -# The type can be * and then you will receive commits of any type. +# curl -sN http://127.0.0.1:2069/commits +# curl -sN 'http://127.0.0.1:2069/commits/svn/*' +# curl -sN http://127.0.0.1:2069/commits/svn +# curl -sN 'http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68' +# curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68 +# +# curl -sN http://127.0.0.1:2069/metadata +# curl -sN 'http://127.0.0.1:2069/metadata/svn/*' +# curl -sN http://127.0.0.1:2069/metadata/svn +# curl -sN 'http://127.0.0.1:2069/metadata/*/13f79535-47bb-0310-9956-ffa450edef68' +# curl -sN http://127.0.0.1:2069/metadata/svn/13f79535-47bb-0310-9956-ffa450edef68 +# +# URLs are constructed from 3 parts: +# /${notification}/${optional_type}/${optional_repository} +# +# Notifications can be sent for commits or metadata (e.g., revprop) changes. +# If the type is included in the URL, you will only get notifications of that type. +# The type can be * and then you will receive notifications of any type. # # If the repository is included in the URL, you will only receive # messages about that repository. The repository can be * and then you @@ -71,7 +78,7 @@ from twisted.python import log import time -class Commit: +class Notification(object): def __init__(self, r): self.__dict__.update(r) if not self.check_value('repository'): @@ -82,11 +89,20 @@ class Commit: raise ValueError('Invalid Format Value') if not self.check_value('id'): raise ValueError('Invalid ID Value') - + def check_value(self, k): return hasattr(self, k) and self.__dict__[k] - def render_commit(self): + def render(self): + raise NotImplementedError + + def render_log(self): + raise NotImplementedError + +class Commit(Notification): + KIND = 'COMMIT' + + def render(self): obj = {'commit': {}} obj['commit'].update(self.__dict__) return json.dumps(obj) @@ -96,20 +112,32 @@ class Commit: paths_changed = " %d paths changed" % len(self.changed) except: paths_changed = "" - return "%s:%s repo '%s' id '%s'%s" % (self.type, - self.format, - self.repository, - self.id, - paths_changed) + return "commit %s:%s repo '%s' id '%s'%s" % ( + self.type, self.format, self.repository, self.id, + paths_changed) + +class Metadata(Notification): + KIND = 'METADATA' + + def render(self): + obj = {'metadata': {}} + obj['metadata'].update(self.__dict__) + return json.dumps(obj) + + def render_log(self): + return "metadata %s:%s repo '%s' id '%s' revprop '%s'" % ( + self.type, self.format, self.repository, self.id, + self.revprop['name']) HEARTBEAT_TIME = 15 class Client(object): - def __init__(self, pubsub, r, type, repository): + def __init__(self, pubsub, r, kind, type, repository): self.pubsub = pubsub r.notifyFinish().addErrback(self.finished) self.r = r + self.kind = kind self.type = type self.repository = repository self.alive = True @@ -123,14 +151,17 @@ class Client(object): except ValueError: pass - def interested_in(self, commit): - if self.type and self.type != commit.type: + def interested_in(self, notification): + if self.kind != notification.KIND: + return False + + if self.type and self.type != notification.type: return False - if self.repository and self.repository != commit.repository: + if self.repository and self.repository != notification.repository: return False - return True + return True def notify(self, data): self.write(data) @@ -152,7 +183,10 @@ class Client(object): self.r.write(str(input)) def write_start(self): - self.r.setHeader('content-type', 'application/json') + # TODO: use application/x-* or vnd.* - see + # Message-ID: <CADkdwvR=hwwevz+xn2hdild-hbpiz7q5fqafq_f5+m77zg6...@mail.gmail.com> + # on May 2013 + self.r.setHeader('content-type', 'application/octet-stream') self.write('{"svnpubsub": {"version": 1}}\n\0') def write_heartbeat(self): @@ -163,6 +197,13 @@ class SvnPubSub(resource.Resource): isLeaf = True clients = [] + __notification_uri_map = {'commits': Commit.KIND, + 'metadata': Metadata.KIND} + + def __init__(self, notification_class): + resource.Resource.__init__(self) + self.__notification_class = notification_class + def cc(self): return len(self.clients) @@ -173,18 +214,23 @@ class SvnPubSub(resource.Resource): log.msg("REQUEST: %s" % (request.uri)) request.setHeader('content-type', 'text/plain') - repository = None - type = None + repository = None + type = None uri = request.uri.split('/') uri_len = len(uri) - if uri_len < 2 or uri_len > 4: + if uri_len < 2 or uri_len > 4: request.setResponseCode(400) return "Invalid path\n" - if uri_len >= 3: + kind = self.__notification_uri_map.get(uri[1], None) + if kind is None: + request.setResponseCode(400) + return "Invalid path\n" + + if uri_len >= 3: type = uri[2] - + if uri_len == 4: repository = uri[3] @@ -194,17 +240,18 @@ class SvnPubSub(resource.Resource): if repository == '*': repository = None - c = Client(self, request, type, repository) + c = Client(self, request, kind, type, repository) self.clients.append(c) c.start() return twisted.web.server.NOT_DONE_YET - def notifyAll(self, commit): - data = commit.render_commit() + def notifyAll(self, notification): + data = notification.render() - log.msg("COMMIT: %s (%d clients)" % (commit.render_log(), self.cc())) + log.msg("%s: %s (%d clients)" + % (notification.KIND, notification.render_log(), self.cc())) for client in self.clients: - if client.interested_in(commit): + if client.interested_in(notification): client.write_data(data) def render_PUT(self, request): @@ -217,19 +264,23 @@ class SvnPubSub(resource.Resource): #import pdb;pdb.set_trace() #print "input: %s" % (input) try: - c = json.loads(input) - commit = Commit(c) + data = json.loads(input) + notification = self.__notification_class(data) except ValueError as e: request.setResponseCode(400) - log.msg("COMMIT: failed due to: %s" % str(e)) - return str(e) - self.notifyAll(commit) + errstr = str(e) + log.msg("%s: failed due to: %s" % (notification.KIND, errstr)) + return errstr + self.notifyAll(notification) return "Ok" + def svnpubsub_server(): root = resource.Resource() - s = SvnPubSub() - root.putChild("commits", s) + c = SvnPubSub(Commit) + m = SvnPubSub(Metadata) + root.putChild('commits', c) + root.putChild('metadata', m) return server.Site(root) if __name__ == "__main__": Modified: subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svntweet.py URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svntweet.py?rev=1489765&r1=1489764&r2=1489765&view=diff ============================================================================== --- subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svntweet.py (original) +++ subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svntweet.py Wed Jun 5 09:22:43 2013 @@ -93,7 +93,7 @@ class JSONRecordHandler: obj = json.loads(record) if 'svnpubsub' in obj: actual_version = obj['svnpubsub'].get('version') - EXPECTED_VERSION = 1 + EXPECTED_VERSION = 1 if actual_version != EXPECTED_VERSION: raise ValueException("Unknown svnpubsub format: %r != %d" % (actual_format, expected_format)) @@ -207,7 +207,7 @@ class BigDoEverythingClasss(object): if path[0:1] == '/' and len(path) > 1: path = path[1:] - #TODO: allow URL to be configurable. + #TODO: allow URL to be configurable. link = " - http://svn.apache.org/r%d"; % (commit.id) left -= len(link) msg = "r%d in %s by %s: " % (commit.id, path, commit.committer) Modified: subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnwcsub.py URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnwcsub.py?rev=1489765&r1=1489764&r2=1489765&view=diff ============================================================================== --- subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnwcsub.py (original) +++ subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnwcsub.py Wed Jun 5 09:22:43 2013 @@ -69,18 +69,7 @@ except ImportError: import daemonize import svnpubsub.client - -# check_output() is only available in Python 2.7. Allow us to run with -# earlier versions -try: - check_output = subprocess.check_output -except AttributeError: - def check_output(args, env): # note: we only use these two args - pipe = subprocess.Popen(args, stdout=subprocess.PIPE, env=env) - output, _ = pipe.communicate() - if pipe.returncode: - raise subprocess.CalledProcessError(pipe.returncode, args) - return output +import svnpubsub.util assert hasattr(subprocess, 'check_call') def check_call(*args, **kwds): @@ -103,7 +92,7 @@ def check_call(*args, **kwds): def svn_info(svnbin, env, path): "Run 'svn info' on the target path, returning a dict of info data." args = [svnbin, "info", "--non-interactive", "--", path] - output = check_output(args, env=env).strip() + output = svnpubsub.util.check_output(args, env=env).strip() info = { } for line in output.split('\n'): idx = line.index(':') @@ -452,7 +441,7 @@ def prepare_logging(logfile): # Apply the handler to the root logger root = logging.getLogger() root.addHandler(handler) - + ### use logging.INFO for now. switch to cmdline option or a config? root.setLevel(logging.INFO) Modified: subversion/branches/verify-keep-going/tools/server-side/svnpubsub/watcher.py URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/tools/server-side/svnpubsub/watcher.py?rev=1489765&r1=1489764&r2=1489765&view=diff ============================================================================== --- subversion/branches/verify-keep-going/tools/server-side/svnpubsub/watcher.py (original) +++ subversion/branches/verify-keep-going/tools/server-side/svnpubsub/watcher.py Wed Jun 5 09:22:43 2013 @@ -35,6 +35,9 @@ def _commit(url, commit): print('COMMIT: from %s' % url) pprint.pprint(vars(commit), indent=2) +def _metadata(url, metadata): + print('METADATA: from %s' % url) + pprint.pprint(vars(metadata), indent=2) def _event(url, event_name, event_arg): if event_arg: @@ -44,7 +47,7 @@ def _event(url, event_name, event_arg): def main(urls): - mc = svnpubsub.client.MultiClient(urls, _commit, _event) + mc = svnpubsub.client.MultiClient(urls, _commit, _event, _metadata) mc.run_forever() Modified: subversion/branches/verify-keep-going/win-tests.py URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/win-tests.py?rev=1489765&r1=1489764&r2=1489765&view=diff ============================================================================== --- subversion/branches/verify-keep-going/win-tests.py (original) +++ subversion/branches/verify-keep-going/win-tests.py Wed Jun 5 09:22:43 2013 @@ -366,7 +366,7 @@ def locate_libs(): 'mod_authz_svn', 'mod_authz_svn.so') mod_dontdothat_path = os.path.join(abs_objdir, 'tools', 'server-side', 'mod_dontdothat', 'mod_dontdothat.so') - + copy_changed_file(mod_dav_svn_path, abs_objdir) copy_changed_file(mod_authz_svn_path, abs_objdir) copy_changed_file(mod_dontdothat_path, abs_objdir) @@ -396,7 +396,7 @@ class Svnserve: self.path = os.path.join(abs_objdir, 'subversion', 'svnserve', self.name) self.root = os.path.join(abs_builddir, CMDLINE_TEST_SCRIPT_NATIVE_PATH) - self.proc_handle = None + self.proc = None def __del__(self): "Stop svnserve when the object is deleted" @@ -414,26 +414,18 @@ class Svnserve: else: args = [self.name] + self.args print('Starting %s %s' % (self.kind, self.name)) - try: - import win32process - import win32con - args = ' '.join([self._quote(x) for x in args]) - self.proc_handle = ( - win32process.CreateProcess(self._quote(self.path), args, - None, None, 0, - win32con.CREATE_NEW_CONSOLE, - None, None, win32process.STARTUPINFO()))[0] - except ImportError: - os.spawnv(os.P_NOWAIT, self.path, args) + + self.proc = subprocess.Popen([self.path] + args[1:]) def stop(self): - if self.proc_handle is not None: + if self.proc is not None: try: - import win32process print('Stopping %s' % self.name) - win32process.TerminateProcess(self.proc_handle, 0) + self.proc.poll(); + if self.proc.returncode is None: + self.proc.kill(); return - except ImportError: + except AttributeError: pass print('Svnserve.stop not implemented') @@ -456,7 +448,7 @@ class Httpd: self.bulkupdates_option = 'off' self.service = service - self.proc_handle = None + self.proc = None self.path = os.path.join(self.httpd_dir, 'bin', self.name) if short_circuit: @@ -522,7 +514,7 @@ class Httpd: fp.write('PidFile pid\n') fp.write('ErrorLog log\n') fp.write('Listen ' + str(self.httpd_port) + '\n') - + if not no_log: fp.write('LogFormat "%h %l %u %t \\"%r\\" %>s %b" common\n') fp.write('Customlog log common\n') @@ -549,7 +541,7 @@ class Httpd: # Write LoadModule for Subversion modules fp.write(self._svn_module('dav_svn_module', 'mod_dav_svn.so')) fp.write(self._svn_module('authz_svn_module', 'mod_authz_svn.so')) - + # And for mod_dontdothat fp.write(self._svn_module('dontdothat_module', 'mod_dontdothat.so')) @@ -597,7 +589,7 @@ class Httpd: "Create empty mime.types file" fp = open(self.httpd_mime_types, 'w') fp.close() - + def _create_dontdothat_file(self): "Create empty mime.types file" fp = open(self.dontdothat_file, 'w') @@ -644,7 +636,7 @@ class Httpd: ' AuthUserFile ' + self._quote(self.httpd_users) + '\n' \ ' Require valid-user\n' \ ' DontDoThatConfigFile ' + self._quote(self.dontdothat_file) + '\n' \ - '</Location>\n' + '</Location>\n' def start(self): if self.service: @@ -674,27 +666,18 @@ class Httpd: "Start HTTPD as daemon" print('Starting httpd as daemon') print(self.httpd_args) - try: - import win32process - import win32con - args = ' '.join([self._quote(x) for x in self.httpd_args]) - self.proc_handle = ( - win32process.CreateProcess(self._quote(self.path), args, - None, None, 0, - win32con.CREATE_NEW_CONSOLE, - None, None, win32process.STARTUPINFO()))[0] - except ImportError: - os.spawnv(os.P_NOWAIT, self.path, self.httpd_args) + self.proc = subprocess.Popen([self.path] + self.httpd_args[1:]) def _stop_daemon(self): "Stop the HTTPD daemon" - if self.proc_handle is not None: + if self.proc is not None: try: - import win32process print('Stopping %s' % self.name) - win32process.TerminateProcess(self.proc_handle, 0) + self.proc.poll(); + if self.proc.returncode is None: + self.proc.kill(); return - except ImportError: + except AttributeError: pass print('Httpd.stop_daemon not implemented')
