Modified: subversion/branches/move-tracking-1/tools/dev/build-svn-deps-win.pl URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/dev/build-svn-deps-win.pl?rev=1489602&r1=1489601&r2=1489602&view=diff ============================================================================== --- subversion/branches/move-tracking-1/tools/dev/build-svn-deps-win.pl (original) +++ subversion/branches/move-tracking-1/tools/dev/build-svn-deps-win.pl Tue Jun 4 20:56:23 2013 @@ -74,6 +74,8 @@ our $CMAKE = 'cmake'; our $NMAKE = 'nmake'; # Use the .com version so we get output, the .exe doesn't produce any output our $DEVENV = 'devenv.com'; +our $VCUPGRADE = 'vcupgrade'; +our $PYTHON = 'python'; # Versions of the dependencies we will use # Change these if you want but these are known to work with @@ -87,7 +89,8 @@ our $OPENSSL_VER = '1.0.1e'; our $PCRE_VER = '8.32'; our $BDB_VER = '5.3.21'; our $SQLITE_VER = '3071602'; -our $SERF_VER = '1.2.0'; +our $SERF_VER = '1.2.1'; +our $NEON_VER = '0.29.6'; # Sources for files to download our $AWK_URL = 'http://www.cs.princeton.edu/~bwk/btl.mirror/awk95.exe'; @@ -101,6 +104,8 @@ our $PCRE_URL; our $BDB_URL; our $SQLITE_URL; our $SERF_URL; +our $NEON_URL; +our $PROJREF_URL = 'https://downloads.redhoundsoftware.com/blog/ProjRef.py'; # Location of the already downloaded file. # by default these are undefined and set by the downloader. @@ -118,6 +123,8 @@ our $PCRE_FILE; our $BDB_FILE; our $SQLITE_FILE; our $SERF_FILE; +our $NEON_FILE; +our $PROJREF_FILE; # Various directories we use our $TOPDIR = Cwd::cwd(); # top of our tree @@ -125,6 +132,11 @@ our $INSTDIR; # where we install to our $BLDDIR; # directory where we actually build our $SRCDIR; # directory where we store package files +# Some other options +our $VS_VER; +our $NEON; +our $SVN_VER = '1.8.x'; + # Utility function to remove dots from a string sub remove_dots { my $in = shift; @@ -143,6 +155,16 @@ sub set_default { } } +sub set_svn_ver_defaults { + my ($svn_major, $svn_minor, $svn_patch) = $SVN_VER =~ /^(\d+)\.(\d+)\.(.+)$/; + + if ($svn_major > 1 or ($svn_major == 1 and $svn_minor >= 8)) { + $NEON=0 unless defined($NEON); + } else { + $NEON=1 unless defined($NEON); + } +} + # Any variables with defaults that reference other values # should be set here. This defers setting of the default until runtime in these cases. sub set_defaults { @@ -156,9 +178,11 @@ sub set_defaults { set_default(\$BDB_URL, "http://download.oracle.com/berkeley-db/db-5.3.21.zip";); set_default(\$SQLITE_URL, "http://www.sqlite.org/2013/sqlite-amalgamation-$SQLITE_VER.zip";); set_default(\$SERF_URL, "http://serf.googlecode.com/files/serf-$SERF_VER.zip";); + set_default(\$NEON_URL, "http://www.webdav.org/neon/neon-$NEON_VER.tar.gz";); set_default(\$INSTDIR, $TOPDIR); set_default(\$BLDDIR, "$TOPDIR\\build"); set_default(\$SRCDIR, "$TOPDIR\\sources"); + set_svn_ver_defaults(); } ################################# @@ -277,6 +301,27 @@ sub modify_file_in_place { close(OUT); } +sub check_vs_ver { + return if defined($VS_VER); + + # using the vcupgrade command here because it has a consistent name and version + # numbering across versions including express versions. + my $help_output = `"$VCUPGRADE" /?`; + my ($major_version) = $help_output =~ /Version (\d+)\./s; + + if (defined($major_version)) { + if ($major_version eq '11') { + $VS_VER = '2012'; + return; + } elsif ($major_version eq '10') { + $VS_VER = '2010'; + return; + } + } + + die("Visual Studio Version Not Supported"); +} + ################## # TREE STRUCTURE # ################## @@ -363,6 +408,10 @@ sub download_dependencies { unless(-x "$BINDIR\\awk.exe") { # skip the copy if it exists copy_or_die($AWK_FILE, "$BINDIR\\awk.exe"); } + download_file($PROJREF_URL, "$SRCDIR\\ProjRef.py", \$PROJREF_FILE); + unless(-x "$BINDIR\\ProjRef.py") { # skip the copy if it exists + copy_or_die($PROJREF_FILE, $BINDIR); + } download_file($BDB_URL, "$SRCDIR\\db.zip", \$BDB_FILE); download_file($ZLIB_URL, "$SRCDIR\\zlib.zip", \$ZLIB_FILE); download_file($OPENSSL_URL, "$SRCDIR\\openssl.tar.gz", \$OPENSSL_FILE); @@ -373,6 +422,7 @@ sub download_dependencies { download_file($PCRE_URL, "$SRCDIR\\pcre.zip", \$PCRE_FILE); download_file($SQLITE_URL, "$SRCDIR\\sqlite-amalgamation.zip", \$SQLITE_FILE); download_file($SERF_URL, "$SRCDIR\\serf.zip", \$SERF_FILE); + download_file($NEON_URL, "$SRCDIR\\neon.tar.gz", \$NEON_FILE) if defined($NEON); } ############## @@ -437,6 +487,8 @@ sub extract_dependencies { "$INSTDIR\\sqlite-amalgamation"); extract_file($SERF_FILE, $INSTDIR, "$INSTDIR\\serf-$SERF_VER", "$INSTDIR\\serf"); + extract_file($NEON_FILE, $INSTDIR, + "$INSTDIR\\neon-$NEON_VER", "$INSTDIR\\neon") if defined($NEON); } ######### @@ -507,6 +559,8 @@ sub build_openssl { # Visual Studio whining about its backup step. sub upgrade_solution { my $file = shift; + my $interactive = shift; + my $flags = ""; my ($basename, $directories) = fileparse($file, qr/\.[^.]*$/); my $sln = $directories . $basename . '.sln'; @@ -519,7 +573,15 @@ sub upgrade_solution { close(SLN); } print "Upgrading $file (this may take a while)\n"; - system_or_die("Failure upgrading $file", qq("$DEVENV" $file /Upgrade)); + $flags = " /Upgrade" unless $interactive; + system_or_die("Failure upgrading $file", qq("$DEVENV" "$file"$flags)); + if ($interactive) { + print "Can't do automatic upgrade, doing interactive upgrade\n"; + print "IDE will load, choose to convert all projects, exit the IDE and\n"; + print "save the resulting solution file\n\n"; + print "Press Enter to Continue\n"; + <>; + } } # Run the lineends.pl script @@ -629,6 +691,9 @@ sub httpd_enable_bdb { sub build_httpd { chdir_or_die($HTTPD); + my $vs_2012 = $VS_VER eq '2012'; + my $vs_2010 = $VS_VER eq '2010'; + # I don't think cvtdsp.pl is necessary with Visual Studio 2012 # but it shouldn't hurt anything either. Including it allows # for the possibility that this may work for older Visual Studio @@ -636,20 +701,10 @@ sub build_httpd { system_or_die("Failure converting DSP files", qq("$PERL" srclib\\apr\\build\\cvtdsp.pl -2005)); - upgrade_solution('Apache.dsw'); + upgrade_solution('Apache.dsw', $vs_2010); httpd_enable_bdb(); httpd_fix_makefile('Makefile.win'); - # Turn off pre-compiled headers for apr-iconv to avoid: - # LNK2011: http://msdn.microsoft.com/en-us/library/3ay26wa2(v=vs.110).aspx - disable_pch('srclib\apr-iconv\build\modules.mk.win'); - - # ApacheMonitor build fails due a duplicate manifest, turn off - # GenerateManifest - insert_property_group('support\win32\ApacheMonitor.vcxproj', - '<GenerateManifest>false</GenerateManifest>', - '.dupman'); - # Modules and support projects randomly fail due to an error about the # CL.read.1.tlog file already existing. This is really because of the # intermediate dirs being shared between modules, but for the time being @@ -660,25 +715,40 @@ sub build_httpd { } }, 'modules', 'support'); - # The APR libraries have projects named libapr but produce output named libapr-1 - # The problem with this is in newer versions of Visual Studio TargetName defaults - # to the project name and not the basename of the output. Since the PDB file - # is named based on the TargetName the pdb file ends up being named libapr.pdb - # instead of libapr-1.pdb. The below call fixes this by explicitly providing - # a TargetName definition and shuts up some warnings about this problem as well. - # Without this fix the install fails when it tries to copy libapr-1.pdb. - # See this thread for details of the changes: - # http://social.msdn.microsoft.com/Forums/en-US/vcprerelease/thread/3c03e730-6a0e-4ee4-a0d6-6a5c3ce4343c - find(sub { - return unless (/\.vcxproj$/); - my $output_file = get_output_file($_); - return unless (defined($output_file)); - my ($project_name) = fileparse($_, qr/\.[^.]*$/); - my ($old_style_target_name) = fileparse($output_file, qr/\.[^.]*$/); - return if ($old_style_target_name eq $project_name); - insert_property_group($_, - "<TargetName>$old_style_target_name</TargetName>", '.torig'); - }, "$SRCLIB\\apr", "$SRCLIB\\apr-util", "$SRCLIB\\apr-iconv"); + if ($vs_2012) { + # Turn off pre-compiled headers for apr-iconv to avoid: + # LNK2011: http://msdn.microsoft.com/en-us/library/3ay26wa2(v=vs.110).aspx + disable_pch('srclib\apr-iconv\build\modules.mk.win'); + + # ApacheMonitor build fails due a duplicate manifest, turn off + # GenerateManifest + insert_property_group('support\win32\ApacheMonitor.vcxproj', + '<GenerateManifest>false</GenerateManifest>', + '.dupman'); + + # The APR libraries have projects named libapr but produce output named libapr-1 + # The problem with this is in newer versions of Visual Studio TargetName defaults + # to the project name and not the basename of the output. Since the PDB file + # is named based on the TargetName the pdb file ends up being named libapr.pdb + # instead of libapr-1.pdb. The below call fixes this by explicitly providing + # a TargetName definition and shuts up some warnings about this problem as well. + # Without this fix the install fails when it tries to copy libapr-1.pdb. + # See this thread for details of the changes: + # http://social.msdn.microsoft.com/Forums/en-US/vcprerelease/thread/3c03e730-6a0e-4ee4-a0d6-6a5c3ce4343c + find(sub { + return unless (/\.vcxproj$/); + my $output_file = get_output_file($_); + return unless (defined($output_file)); + my ($project_name) = fileparse($_, qr/\.[^.]*$/); + my ($old_style_target_name) = fileparse($output_file, qr/\.[^.]*$/); + return if ($old_style_target_name eq $project_name); + insert_property_group($_, + "<TargetName>$old_style_target_name</TargetName>", '.torig'); + }, "$SRCLIB\\apr", "$SRCLIB\\apr-util", "$SRCLIB\\apr-iconv"); + } elsif ($vs_2010) { + system_or_die("Failed fixing project guid references", + qq("$PYTHON" "$BINDIR\\ProjRef.py" -i Apache.sln")); + } # If you're looking here it's possible that something went # wrong with the httpd build. Debugging it can be a bit of a pain @@ -774,6 +844,9 @@ sub main { Vars::set_defaults(); set_paths(); + # Determine the Visual Studio Version and die if not supported. + check_vs_ver(); + # change directory to our TOPDIR before running any commands # the variable assignment might have changed it. chdir_or_die($TOPDIR);
Modified: subversion/branches/move-tracking-1/tools/dist/backport.pl URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/dist/backport.pl?rev=1489602&r1=1489601&r2=1489602&view=diff ============================================================================== --- subversion/branches/move-tracking-1/tools/dist/backport.pl (original) +++ subversion/branches/move-tracking-1/tools/dist/backport.pl Tue Jun 4 20:56:23 2013 @@ -123,7 +123,9 @@ fi $SVN diff > $backupfile cp STATUS STATUS.$$ $SVNq revert -R . -mv STATUS.$$ STATUS +if $MAY_COMMIT ; then + mv STATUS.$$ STATUS +fi $SVNq up $SVNq merge $mergeargs if [ "`$SVN status -q | wc -l`" -eq 1 ]; then @@ -161,9 +163,9 @@ else fi EOF - open SHELL, '|-', qw#/bin/sh# or die $!; + open SHELL, '|-', qw#/bin/sh# or die "$! (in '$entry{header}')"; print SHELL $script; - close SHELL or warn "$0: sh($?): $!"; + close SHELL or warn "$0: sh($?): $! (in '$entry{header}')"; unlink $backupfile if -z $backupfile; unlink $logmsg_filename unless $? or $!; Modified: subversion/branches/move-tracking-1/tools/dist/make-deps-tarball.sh URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/dist/make-deps-tarball.sh?rev=1489602&r1=1489601&r2=1489602&view=diff ============================================================================== --- subversion/branches/move-tracking-1/tools/dist/make-deps-tarball.sh (original) +++ subversion/branches/move-tracking-1/tools/dist/make-deps-tarball.sh Tue Jun 4 20:56:23 2013 @@ -25,7 +25,7 @@ APR=apr-1.4.6 APR_UTIL=apr-util-1.4.1 NEON=neon-0.29.6 SERF=serf-0.3.1 -ZLIB=zlib-1.2.7 +ZLIB=zlib-1.2.8 SQLITE_VERSION=3071400 SQLITE=sqlite-amalgamation-$SQLITE_VERSION @@ -58,13 +58,13 @@ create_deps() { fi wget -qnc http://webdav.org/neon/$NEON.tar.gz wget -qnc http://serf.googlecode.com/files/$SERF.tar.bz2 - wget -qnc http://www.zlib.net/$ZLIB.tar.bz2 + wget -qnc http://www.zlib.net/$ZLIB.tar.gz wget -qnc http://www.sqlite.org/$SQLITE.zip mkdir $BASEDIR/unix-dependencies cd $BASEDIR/unix-dependencies tar zxf $TEMPDIR/$NEON.tar.gz - tar jxf $TEMPDIR/$ZLIB.tar.bz2 + tar zxf $TEMPDIR/$ZLIB.tar.gz tar jxf $TEMPDIR/$SERF.tar.bz2 unzip -q $TEMPDIR/$SQLITE.zip mv $NEON neon @@ -80,7 +80,7 @@ create_deps() { mkdir $BASEDIR/win32-dependencies cd $BASEDIR/win32-dependencies tar zxf $TEMPDIR/$NEON.tar.gz - tar jxf $TEMPDIR/$ZLIB.tar.bz2 + tar zxf $TEMPDIR/$ZLIB.tar.gz tar jxf $TEMPDIR/$SERF.tar.bz2 unzip -q $TEMPDIR/$SQLITE.zip mv $NEON neon Propchange: subversion/branches/move-tracking-1/tools/dist/make-deps-tarball.sh ------------------------------------------------------------------------------ Merged /subversion/trunk/tools/dist/make-deps-tarball.sh:r1485307-1489600 Modified: subversion/branches/move-tracking-1/tools/dist/release.py URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/dist/release.py?rev=1489602&r1=1489601&r2=1489602&view=diff ============================================================================== --- subversion/branches/move-tracking-1/tools/dist/release.py (original) +++ subversion/branches/move-tracking-1/tools/dist/release.py Tue Jun 4 20:56:23 2013 @@ -66,6 +66,22 @@ except ImportError: import ezt +try: + subprocess.check_output +except AttributeError: + def check_output(cmd): + proc = subprocess.Popen(['svn', 'list', dist_dev_url], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (stdout, stderr) = proc.communicate() + rc = proc.wait() + if rc or stderr: + logging.error('%r failed with stderr %r', cmd, stderr) + raise subprocess.CalledProcessError(rc, cmd) + return stdout + subprocess.check_output = check_output + del check_output + # Our required / recommended release tool versions by release branch tool_versions = { 'trunk' : { @@ -104,7 +120,7 @@ extns = ['zip', 'tar.gz', 'tar.bz2'] # Utility functions class Version(object): - regex = re.compile('(\d+).(\d+).(\d+)(?:-(?:(rc|alpha|beta)(\d+)))?') + regex = re.compile(r'(\d+).(\d+).(\d+)(?:-(?:(rc|alpha|beta)(\d+)))?') def __init__(self, ver_str): # Special case the 'trunk-nightly' version @@ -160,7 +176,7 @@ class Version(object): else: return self.pre_num < that.pre_num - def __str(self): + def __str__(self): if self.pre: if self.pre == 'nightly': return 'nightly' @@ -173,11 +189,7 @@ class Version(object): def __repr__(self): - return "Version('%s')" % self.__str() - - def __str__(self): - return self.__str() - + return "Version(%s)" % repr(str(self)) def get_prefix(base_dir): return os.path.join(base_dir, 'prefix') @@ -188,6 +200,13 @@ def get_tempdir(base_dir): def get_deploydir(base_dir): return os.path.join(base_dir, 'deploy') +def get_target(args): + "Return the location of the artifacts" + if args.target: + return args.target + else: + return get_deploydir(args.base_dir) + def get_tmpldir(): return os.path.join(os.path.abspath(sys.path[0]), 'templates') @@ -199,8 +218,7 @@ def get_tmplfile(filename): return urllib2.urlopen(repos + '/trunk/tools/dist/templates/' + filename) def get_nullfile(): - # This is certainly not cross platform - return open('/dev/null', 'w') + return open(os.path.devnull, 'w') def run_script(verbose, script): if verbose: @@ -376,12 +394,7 @@ def compare_changes(repos, branch, revis mergeinfo_cmd = ['svn', 'mergeinfo', '--show-revs=eligible', repos + '/trunk/CHANGES', repos + '/' + branch + '/' + 'CHANGES'] - proc = subprocess.Popen(mergeinfo_cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (stdout, stderr) = proc.communicate() - rc = proc.wait() - if stderr: - raise RuntimeError('svn mergeinfo failed: %s' % stderr) + stdout = subprocess.check_output(mergeinfo_cmd) if stdout: # Treat this as a warning since we are now putting entries for future # minor releases in CHANGES on trunk. @@ -468,15 +481,11 @@ def sign_candidates(args): def sign_file(filename): asc_file = open(filename + '.asc', 'a') logging.info("Signing %s" % filename) - proc = subprocess.Popen(['gpg', '-ba', '-o', '-', filename], - stdout=asc_file) - proc.wait() + proc = subprocess.check_call(['gpg', '-ba', '-o', '-', filename], + stdout=asc_file) asc_file.close() - if args.target: - target = args.target - else: - target = get_deploydir(args.base_dir) + target = get_target(args) for e in extns: filename = os.path.join(target, 'subversion-%s.%s' % (args.version, e)) @@ -493,17 +502,17 @@ def sign_candidates(args): def post_candidates(args): 'Post candidate artifacts to the dist development directory.' + target = get_target(args) + logging.info('Importing tarballs to %s' % dist_dev_url) svn_cmd = ['svn', 'import', '-m', 'Add %s candidate release artifacts' % args.version.base, '--auto-props', '--config-option', 'config:auto-props:*.asc=svn:eol-style=native;svn:mime-type=text/plain', - get_deploydir(args.base_dir), dist_dev_url] + target, dist_dev_url] if (args.username): svn_cmd += ['--username', args.username] - proc = subprocess.Popen(svn_cmd) - (stdout, stderr) = proc.communicate() - proc.wait() + subprocess.check_call(svn_cmd) #---------------------------------------------------------------------- # Create tag @@ -518,6 +527,7 @@ def create_tag(args): else: branch = secure_repos + '/branches/%d.%d.x' % (args.version.major, args.version.minor) + target = get_target(args) tag = secure_repos + '/tags/' + str(args.version) @@ -526,14 +536,63 @@ def create_tag(args): if (args.username): svnmucc_cmd += ['--username', args.username] svnmucc_cmd += ['cp', str(args.revnum), branch, tag] - svnmucc_cmd += ['put', os.path.join(get_deploydir(args.base_dir), - 'svn_version.h.dist' + '-' + + svnmucc_cmd += ['put', os.path.join(target, 'svn_version.h.dist' + '-' + str(args.version)), tag + '/subversion/include/svn_version.h'] # don't redirect stdout/stderr since svnmucc might ask for a password - proc = subprocess.Popen(svnmucc_cmd) - proc.wait() + subprocess.check_call(svnmucc_cmd) + + if not args.version.is_prerelease(): + logging.info('Bumping revisions on the branch') + def replace_in_place(fd, startofline, flat, spare): + """In file object FD, replace FLAT with SPARE in the first line + starting with STARTOFLINE.""" + + fd.seek(0, os.SEEK_SET) + lines = fd.readlines() + for i, line in enumerate(lines): + if line.startswith(startofline): + lines[i] = line.replace(flat, spare) + break + else: + raise RuntimeError('Definition of %r not found' % startofline) + + fd.seek(0, os.SEEK_SET) + fd.writelines(lines) + fd.truncate() # for current callers, new value is never shorter. + + new_version = Version('%d.%d.%d' % + (args.version.major, args.version.minor, + args.version.patch + 1)) + + def file_object_for(relpath): + fd = tempfile.NamedTemporaryFile() + url = branch + '/' + relpath + fd.url = url + subprocess.check_call(['svn', 'cat', '%s@%d' % (url, args.revnum)], + stdout=fd) + return fd + + svn_version_h = file_object_for('subversion/include/svn_version.h') + replace_in_place(svn_version_h, '#define SVN_VER_PATCH ', + str(args.version.patch), str(new_version.patch)) + + STATUS = file_object_for('STATUS') + replace_in_place(STATUS, 'Status of ', + str(args.version), str(new_version)) + + svn_version_h.seek(0, os.SEEK_SET) + STATUS.seek(0, os.SEEK_SET) + subprocess.check_call(['svnmucc', '-r', str(args.revnum), + '-m', 'Post-release housekeeping: ' + 'bump the %s branch to %s.' + % (branch.split('/')[-1], str(new_version)), + 'put', svn_version_h.name, svn_version_h.url, + 'put', STATUS.name, STATUS.url, + ]) + del svn_version_h + del STATUS #---------------------------------------------------------------------- # Clean dist @@ -541,13 +600,7 @@ def create_tag(args): def clean_dist(args): 'Clean the distribution directory of all but the most recent artifacts.' - proc = subprocess.Popen(['svn', 'list', dist_release_url], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (stdout, stderr) = proc.communicate() - proc.wait() - if stderr: - raise RuntimeError(stderr) + stdout = subprocess.check_output(['svn', 'list', dist_release_url]) filenames = stdout.split('\n') tar_gz_archives = [] @@ -576,8 +629,7 @@ def clean_dist(args): svnmucc_cmd += ['rm', dist_release_url + '/' + filename] # don't redirect stdout/stderr since svnmucc might ask for a password - proc = subprocess.Popen(svnmucc_cmd) - proc.wait() + subprocess.check_call(svnmucc_cmd) #---------------------------------------------------------------------- # Move to dist @@ -585,13 +637,7 @@ def clean_dist(args): def move_to_dist(args): 'Move candidate artifacts to the distribution directory.' - proc = subprocess.Popen(['svn', 'list', dist_dev_url], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (stdout, stderr) = proc.communicate() - proc.wait() - if stderr: - raise RuntimeError(stderr) + stdout = subprocess.check_output(['svn', 'list', dist_dev_url]) filenames = [] for entry in stdout.split('\n'): @@ -609,8 +655,7 @@ def move_to_dist(args): # don't redirect stdout/stderr since svnmucc might ask for a password logging.info('Moving release artifacts to %s' % dist_release_url) - proc = subprocess.Popen(svnmucc_cmd) - proc.wait() + subprocess.check_call(svnmucc_cmd) #---------------------------------------------------------------------- # Write announcements @@ -637,10 +682,7 @@ def write_news(args): def get_sha1info(args, replace=False): 'Return a list of sha1 info for the release' - if args.target: - target = args.target - else: - target = get_deploydir(args.base_dir) + target = get_target(args) sha1s = glob.glob(os.path.join(target, 'subversion*-%s*.sha1' % args.version)) @@ -714,10 +756,7 @@ def get_siginfo(args, quiet=False): import _gnupg as gnupg gpg = gnupg.GPG() - if args.target: - target = args.target - else: - target = get_deploydir(args.base_dir) + target = get_target(args) good_sigs = {} fingerprints = {} @@ -848,6 +887,9 @@ def main(): help='''The release label, such as '1.7.0-alpha1'.''') subparser.add_argument('--username', help='''Username for ''' + dist_repos + '''.''') + subparser.add_argument('--target', + help='''The full path to the directory containing + release artifacts.''') # Setup the parser for the create-tag subcommand subparser = subparsers.add_parser('create-tag', @@ -861,6 +903,9 @@ def main(): help='''The branch to base the release on.''') subparser.add_argument('--username', help='''Username for ''' + secure_repos + '''.''') + subparser.add_argument('--target', + help='''The full path to the directory containing + release artifacts.''') # The clean-dist subcommand subparser = subparsers.add_parser('clean-dist', Modified: subversion/branches/move-tracking-1/tools/hook-scripts/mailer/mailer.py URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/hook-scripts/mailer/mailer.py?rev=1489602&r1=1489601&r2=1489602&view=diff ============================================================================== --- subversion/branches/move-tracking-1/tools/hook-scripts/mailer/mailer.py (original) +++ subversion/branches/move-tracking-1/tools/hook-scripts/mailer/mailer.py Tue Jun 4 20:56:23 2013 @@ -236,16 +236,30 @@ class MailedOutput(OutputBase): and self.reply_to[2] == ']': self.reply_to = self.reply_to[3:] + def _rfc2047_encode(self, hdr): + # Return the result of splitting HDR into tokens (on space + # characters), encoding (per RFC2047) each token as necessary, and + # slapping 'em back to together again. + from email.Header import Header + + def _maybe_encode_header(hdr_token): + try: + hdr_token.encode('ascii') + return hdr_token + except UnicodeError: + return Header(hdr_token, 'utf-8').encode() + + return ' '.join(map(_maybe_encode_header, hdr.split())) + def mail_headers(self, group, params): from email import Utils - subject = self.make_subject(group, params) - try: - subject.encode('ascii') - except UnicodeError: - from email.Header import Header - subject = Header(subject, 'utf-8').encode() - hdrs = 'From: %s\n' \ - 'To: %s\n' \ + + subject = self._rfc2047_encode(self.make_subject(group, params)) + from_hdr = self._rfc2047_encode(self.from_addr) + to_hdr = self._rfc2047_encode(', '.join(self.to_addrs)) + + hdrs = 'From: %s\n' \ + 'To: %s\n' \ 'Subject: %s\n' \ 'Date: %s\n' \ 'Message-ID: %s\n' \ @@ -256,7 +270,7 @@ class MailedOutput(OutputBase): 'X-Svn-Commit-Author: %s\n' \ 'X-Svn-Commit-Revision: %d\n' \ 'X-Svn-Commit-Repository: %s\n' \ - % (self.from_addr, ', '.join(self.to_addrs), subject, + % (from_hdr, to_hdr, subject, Utils.formatdate(), Utils.make_msgid(), group, self.repos.author or 'no_author', self.repos.rev, os.path.basename(self.repos.repos_dir)) Modified: subversion/branches/move-tracking-1/tools/server-side/svnpubsub/commit-hook.py URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/server-side/svnpubsub/commit-hook.py?rev=1489602&r1=1489601&r2=1489602&view=diff ============================================================================== --- subversion/branches/move-tracking-1/tools/server-side/svnpubsub/commit-hook.py (original) +++ subversion/branches/move-tracking-1/tools/server-side/svnpubsub/commit-hook.py Tue Jun 4 20:56:23 2013 @@ -19,8 +19,10 @@ SVNLOOK="/usr/local/svn-install/current/bin/svnlook" #SVNLOOK="/usr/local/bin/svnlook" +HOST="127.0.0.1" +PORT=2069 + import sys -import subprocess try: import simplejson as json except ImportError: @@ -28,35 +30,32 @@ except ImportError: import urllib2 -HOST="127.0.0.1" -PORT=2069 - -def svncmd(cmd): - return subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) +import svnpubsub.util -def svncmd_uuid(repo): - cmd = "%s uuid %s" % (SVNLOOK, repo) - p = svncmd(cmd) - return p.stdout.read().strip() - -def svncmd_info(repo, revision): - cmd = "%s info -r %s %s" % (SVNLOOK, revision, repo) - p = svncmd(cmd) - data = p.stdout.read().split("\n") +def svnlook(cmd, **kwargs): + args = [SVNLOOK] + cmd + return svnpubsub.util.check_output(args, **kwargs) + +def svnlook_uuid(repo): + cmd = ["uuid", "--", repo] + return svnlook(cmd).strip() + +def svnlook_info(repo, revision): + cmd = ["info", "-r", revision, "--", repo] + data = svnlook(cmd, universal_newlines=True).split("\n") #print data return {'author': data[0].strip(), 'date': data[1].strip(), 'log': "\n".join(data[3:]).strip()} -def svncmd_changed(repo, revision): - cmd = "%s changed -r %s %s" % (SVNLOOK, revision, repo) - p = svncmd(cmd) +def svnlook_changed(repo, revision): + cmd = ["changed", "-r", revision, "--", repo] + lines = svnlook(cmd, universal_newlines=True).split("\n") changed = {} - while True: - line = p.stdout.readline() - if not line: - break + for line in lines: line = line.strip() + if not line: + continue (flags, filename) = (line[0:3], line[4:]) changed[filename] = {'flags': flags} return changed @@ -71,23 +70,23 @@ def do_put(body): def main(repo, revision): revision = revision.lstrip('r') - i = svncmd_info(repo, revision) + i = svnlook_info(repo, revision) data = {'type': 'svn', 'format': 1, 'id': int(revision), 'changed': {}, - 'repository': svncmd_uuid(repo), + 'repository': svnlook_uuid(repo), 'committer': i['author'], 'log': i['log'], 'date': i['date'], } - data['changed'].update(svncmd_changed(repo, revision)) + data['changed'].update(svnlook_changed(repo, revision)) body = json.dumps(data) do_put(body) if __name__ == "__main__": - if len(sys.argv) != 3: - print "invalid args" - sys.exit(0) + if len(sys.argv) not in (3, 4): + sys.stderr.write("invalid args\n") + sys.exit(1) - main(sys.argv[1], sys.argv[2]) + main(*sys.argv[1:3]) Modified: subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/client.py URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/client.py?rev=1489602&r1=1489601&r2=1489602&view=diff ============================================================================== --- subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/client.py (original) +++ subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/client.py Tue Jun 4 20:56:23 2013 @@ -62,7 +62,8 @@ class SvnpubsubClientException(Exception class Client(asynchat.async_chat): - def __init__(self, url, commit_callback, event_callback): + def __init__(self, url, commit_callback, event_callback, + metadata_callback = None): asynchat.async_chat.__init__(self) self.last_activity = time.time() @@ -82,7 +83,8 @@ class Client(asynchat.async_chat): self.event_callback = event_callback - self.parser = JSONRecordHandler(commit_callback, event_callback) + self.parser = JSONRecordHandler(commit_callback, event_callback, + metadata_callback) # Wait for the end of headers. Then we start parsing JSON. self.set_terminator(b'\r\n\r\n') @@ -126,36 +128,50 @@ class Client(asynchat.async_chat): self.ibuffer.append(data) +class Notification(object): + def __init__(self, data): + self.__dict__.update(data) + +class Commit(Notification): + KIND = 'COMMIT' + +class Metadata(Notification): + KIND = 'METADATA' + + class JSONRecordHandler: - def __init__(self, commit_callback, event_callback): + def __init__(self, commit_callback, event_callback, metadata_callback): self.commit_callback = commit_callback self.event_callback = event_callback + self.metadata_callback = metadata_callback + + EXPECTED_VERSION = 1 def feed(self, record): obj = json.loads(record) if 'svnpubsub' in obj: actual_version = obj['svnpubsub'].get('version') - EXPECTED_VERSION = 1 - if actual_version != EXPECTED_VERSION: - raise SvnpubsubClientException("Unknown svnpubsub format: %r != %d" - % (actual_format, expected_format)) + if actual_version != self.EXPECTED_VERSION: + raise SvnpubsubClientException( + "Unknown svnpubsub format: %r != %d" + % (actual_version, self.EXPECTED_VERSION)) self.event_callback('version', obj['svnpubsub']['version']) elif 'commit' in obj: commit = Commit(obj['commit']) self.commit_callback(commit) elif 'stillalive' in obj: self.event_callback('ping', obj['stillalive']) - - -class Commit(object): - def __init__(self, commit): - self.__dict__.update(commit) + elif 'metadata' in obj and self.metadata_callback: + metadata = Metadata(obj['metadata']) + self.metadata_callback(metadata) class MultiClient(object): - def __init__(self, urls, commit_callback, event_callback): + def __init__(self, urls, commit_callback, event_callback, + metadata_callback = None): self.commit_callback = commit_callback self.event_callback = event_callback + self.metadata_callback = metadata_callback # No target time, as no work to do self.target_time = 0 @@ -185,9 +201,15 @@ class MultiClient(object): def _add_channel(self, url): # Simply instantiating the client will install it into the global map # for processing in the main event loop. - Client(url, - functools.partial(self.commit_callback, url), - functools.partial(self._reconnect, url)) + if self.metadata_callback: + Client(url, + functools.partial(self.commit_callback, url), + functools.partial(self._reconnect, url), + functools.partial(self.metadata_callback, url)) + else: + Client(url, + functools.partial(self.commit_callback, url), + functools.partial(self._reconnect, url)) def _check_stale(self): now = time.time() Modified: subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/server.py URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/server.py?rev=1489602&r1=1489601&r2=1489602&view=diff ============================================================================== --- subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/server.py (original) +++ subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/server.py Tue Jun 4 20:56:23 2013 @@ -28,17 +28,24 @@ # Currently supports both XML and JSON serialization. # # Example Sub clients: -# curl -sN http://127.0.0.1:2069/commits -# curl -sN http://127.0.0.1:2069/commits/svn/* -# curl -sN http://127.0.0.1:2069/commits/svn -# curl -sN http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68 -# curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68 +# curl -sN http://127.0.0.1:2069/commits +# curl -sN 'http://127.0.0.1:2069/commits/svn/*' +# curl -sN http://127.0.0.1:2069/commits/svn +# curl -sN 'http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68' +# curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68 # -# URL is built into 2 parts: -# /commits/${optional_type}/${optional_repository} +# curl -sN http://127.0.0.1:2069/metadata +# curl -sN 'http://127.0.0.1:2069/metadata/svn/*' +# curl -sN http://127.0.0.1:2069/metadata/svn +# curl -sN 'http://127.0.0.1:2069/metadata/*/13f79535-47bb-0310-9956-ffa450edef68' +# curl -sN http://127.0.0.1:2069/metadata/svn/13f79535-47bb-0310-9956-ffa450edef68 # -# If the type is included in the URL, you will only get commits of that type. -# The type can be * and then you will receive commits of any type. +# URLs are constructed from 3 parts: +# /${notification}/${optional_type}/${optional_repository} +# +# Notifications can be sent for commits or metadata (e.g., revprop) changes. +# If the type is included in the URL, you will only get notifications of that type. +# The type can be * and then you will receive notifications of any type. # # If the repository is included in the URL, you will only receive # messages about that repository. The repository can be * and then you @@ -71,7 +78,7 @@ from twisted.python import log import time -class Commit: +class Notification(object): def __init__(self, r): self.__dict__.update(r) if not self.check_value('repository'): @@ -86,7 +93,16 @@ class Commit: def check_value(self, k): return hasattr(self, k) and self.__dict__[k] - def render_commit(self): + def render(self): + raise NotImplementedError + + def render_log(self): + raise NotImplementedError + +class Commit(Notification): + KIND = 'COMMIT' + + def render(self): obj = {'commit': {}} obj['commit'].update(self.__dict__) return json.dumps(obj) @@ -96,20 +112,32 @@ class Commit: paths_changed = " %d paths changed" % len(self.changed) except: paths_changed = "" - return "%s:%s repo '%s' id '%s'%s" % (self.type, - self.format, - self.repository, - self.id, - paths_changed) + return "commit %s:%s repo '%s' id '%s'%s" % ( + self.type, self.format, self.repository, self.id, + paths_changed) + +class Metadata(Notification): + KIND = 'METADATA' + + def render(self): + obj = {'metadata': {}} + obj['metadata'].update(self.__dict__) + return json.dumps(obj) + + def render_log(self): + return "metadata %s:%s repo '%s' id '%s' revprop '%s'" % ( + self.type, self.format, self.repository, self.id, + self.revprop['name']) HEARTBEAT_TIME = 15 class Client(object): - def __init__(self, pubsub, r, type, repository): + def __init__(self, pubsub, r, kind, type, repository): self.pubsub = pubsub r.notifyFinish().addErrback(self.finished) self.r = r + self.kind = kind self.type = type self.repository = repository self.alive = True @@ -123,11 +151,14 @@ class Client(object): except ValueError: pass - def interested_in(self, commit): - if self.type and self.type != commit.type: + def interested_in(self, notification): + if self.kind != notification.KIND: return False - if self.repository and self.repository != commit.repository: + if self.type and self.type != notification.type: + return False + + if self.repository and self.repository != notification.repository: return False return True @@ -152,7 +183,10 @@ class Client(object): self.r.write(str(input)) def write_start(self): - self.r.setHeader('content-type', 'application/json') + # TODO: use application/x-* or vnd.* - see + # Message-ID: <CADkdwvR=hwwevz+xn2hdild-hbpiz7q5fqafq_f5+m77zg6...@mail.gmail.com> + # on May 2013 + self.r.setHeader('content-type', 'application/octet-stream') self.write('{"svnpubsub": {"version": 1}}\n\0') def write_heartbeat(self): @@ -163,6 +197,13 @@ class SvnPubSub(resource.Resource): isLeaf = True clients = [] + __notification_uri_map = {'commits': Commit.KIND, + 'metadata': Metadata.KIND} + + def __init__(self, notification_class): + resource.Resource.__init__(self) + self.__notification_class = notification_class + def cc(self): return len(self.clients) @@ -182,6 +223,11 @@ class SvnPubSub(resource.Resource): request.setResponseCode(400) return "Invalid path\n" + kind = self.__notification_uri_map.get(uri[1], None) + if kind is None: + request.setResponseCode(400) + return "Invalid path\n" + if uri_len >= 3: type = uri[2] @@ -194,17 +240,18 @@ class SvnPubSub(resource.Resource): if repository == '*': repository = None - c = Client(self, request, type, repository) + c = Client(self, request, kind, type, repository) self.clients.append(c) c.start() return twisted.web.server.NOT_DONE_YET - def notifyAll(self, commit): - data = commit.render_commit() + def notifyAll(self, notification): + data = notification.render() - log.msg("COMMIT: %s (%d clients)" % (commit.render_log(), self.cc())) + log.msg("%s: %s (%d clients)" + % (notification.KIND, notification.render_log(), self.cc())) for client in self.clients: - if client.interested_in(commit): + if client.interested_in(notification): client.write_data(data) def render_PUT(self, request): @@ -217,19 +264,23 @@ class SvnPubSub(resource.Resource): #import pdb;pdb.set_trace() #print "input: %s" % (input) try: - c = json.loads(input) - commit = Commit(c) + data = json.loads(input) + notification = self.__notification_class(data) except ValueError as e: request.setResponseCode(400) - log.msg("COMMIT: failed due to: %s" % str(e)) - return str(e) - self.notifyAll(commit) + errstr = str(e) + log.msg("%s: failed due to: %s" % (notification.KIND, errstr)) + return errstr + self.notifyAll(notification) return "Ok" + def svnpubsub_server(): root = resource.Resource() - s = SvnPubSub() - root.putChild("commits", s) + c = SvnPubSub(Commit) + m = SvnPubSub(Metadata) + root.putChild('commits', c) + root.putChild('metadata', m) return server.Site(root) if __name__ == "__main__": Modified: subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnwcsub.py URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnwcsub.py?rev=1489602&r1=1489601&r2=1489602&view=diff ============================================================================== --- subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnwcsub.py (original) +++ subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnwcsub.py Tue Jun 4 20:56:23 2013 @@ -69,18 +69,7 @@ except ImportError: import daemonize import svnpubsub.client - -# check_output() is only available in Python 2.7. Allow us to run with -# earlier versions -try: - check_output = subprocess.check_output -except AttributeError: - def check_output(args, env): # note: we only use these two args - pipe = subprocess.Popen(args, stdout=subprocess.PIPE, env=env) - output, _ = pipe.communicate() - if pipe.returncode: - raise subprocess.CalledProcessError(pipe.returncode, args) - return output +import svnpubsub.util assert hasattr(subprocess, 'check_call') def check_call(*args, **kwds): @@ -103,7 +92,7 @@ def check_call(*args, **kwds): def svn_info(svnbin, env, path): "Run 'svn info' on the target path, returning a dict of info data." args = [svnbin, "info", "--non-interactive", "--", path] - output = check_output(args, env=env).strip() + output = svnpubsub.util.check_output(args, env=env).strip() info = { } for line in output.split('\n'): idx = line.index(':') Modified: subversion/branches/move-tracking-1/tools/server-side/svnpubsub/watcher.py URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/server-side/svnpubsub/watcher.py?rev=1489602&r1=1489601&r2=1489602&view=diff ============================================================================== --- subversion/branches/move-tracking-1/tools/server-side/svnpubsub/watcher.py (original) +++ subversion/branches/move-tracking-1/tools/server-side/svnpubsub/watcher.py Tue Jun 4 20:56:23 2013 @@ -35,6 +35,9 @@ def _commit(url, commit): print('COMMIT: from %s' % url) pprint.pprint(vars(commit), indent=2) +def _metadata(url, metadata): + print('METADATA: from %s' % url) + pprint.pprint(vars(metadata), indent=2) def _event(url, event_name, event_arg): if event_arg: @@ -44,7 +47,7 @@ def _event(url, event_name, event_arg): def main(urls): - mc = svnpubsub.client.MultiClient(urls, _commit, _event) + mc = svnpubsub.client.MultiClient(urls, _commit, _event, _metadata) mc.run_forever()
