Modified: subversion/branches/addremove/tools/dist/release.py URL: http://svn.apache.org/viewvc/subversion/branches/addremove/tools/dist/release.py?rev=1878061&r1=1878060&r2=1878061&view=diff ============================================================================== --- subversion/branches/addremove/tools/dist/release.py (original) +++ subversion/branches/addremove/tools/dist/release.py Sat May 23 14:16:56 2020 @@ -41,7 +41,10 @@ import sys import glob import fnmatch import shutil -import urllib2 +try: + from urllib.request import urlopen # Python 3 +except: + from urllib2 import urlopen # Python 2 import hashlib import tarfile import logging @@ -51,6 +54,10 @@ import operator import itertools import subprocess import argparse # standard in Python 2.7 +import io +import yaml + +import backport.status # Find ezt, using Subversion's copy, if there isn't one on the system. try: @@ -64,52 +71,33 @@ except ImportError: sys.path.remove(ezt_path) +def get_dist_metadata_file_path(): + return os.path.join(os.path.abspath(sys.path[0]), 'release-lines.yaml') + +# Read the dist metadata (about release lines) +with open(get_dist_metadata_file_path(), 'r') as stream: + dist_metadata = yaml.safe_load(stream) + # Our required / recommended release tool versions by release branch -tool_versions = { - 'trunk' : { - 'autoconf' : ['2.69', - '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'], - 'libtool' : ['2.4.6', - 'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'], - 'swig' : ['3.0.10', - '2939aae39dec06095462f1b95ce1c958ac80d07b926e48871046d17c0094f44c'], - }, - '1.10' : { - 'autoconf' : ['2.69', - '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'], - 'libtool' : ['2.4.6', - 'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'], - 'swig' : ['3.0.10', - '2939aae39dec06095462f1b95ce1c958ac80d07b926e48871046d17c0094f44c'], - }, - '1.9' : { - 'autoconf' : ['2.69', - '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'], - 'libtool' : ['2.4.6', - 'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'], - 'swig' : ['2.0.12', - '65e13f22a60cecd7279c59882ff8ebe1ffe34078e85c602821a541817a4317f7'], - }, - '1.8' : { - 'autoconf' : ['2.69', - '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'], - 'libtool' : ['2.4.3', - '36b4881c1843d7585de9c66c4c3d9a067ed3a3f792bc670beba21f5a4960acdf'], - 'swig' : ['2.0.9', - '586954000d297fafd7e91d1ad31089cc7e249f658889d11a44605d3662569539'], - }, -} +tool_versions = dist_metadata['tool_versions'] # The version that is our current recommended release -# ### TODO: derive this from svn_version.h; see ../../build/getversion.py -recommended_release = '1.9' +recommended_release = dist_metadata['recommended_release'] +# For clean-dist, a whitelist of artifacts to keep, by version. +supported_release_lines = frozenset(dist_metadata['supported_release_lines']) +# Long-Term Support (LTS) versions +lts_release_lines = frozenset(dist_metadata['lts_release_lines']) # Some constants -repos = 'https://svn.apache.org/repos/asf/subversion' -secure_repos = 'https://svn.apache.org/repos/asf/subversion' -dist_repos = 'https://dist.apache.org/repos/dist' +svn_repos = os.getenv('SVN_RELEASE_SVN_REPOS', + 'https://svn.apache.org/repos/asf/subversion') +dist_repos = os.getenv('SVN_RELEASE_DIST_REPOS', + 'https://dist.apache.org/repos/dist') dist_dev_url = dist_repos + '/dev/subversion' dist_release_url = dist_repos + '/release/subversion' +dist_archive_url = 'https://archive.apache.org/dist/subversion' +buildbot_repos = os.getenv('SVN_RELEASE_BUILDBOT_REPOS', + 'https://svn.apache.org/repos/infra/infrastructure/buildbot/aegis/buildmaster') KEYS = 'https://people.apache.org/keys/group/subversion.asc' extns = ['zip', 'tar.gz', 'tar.bz2'] @@ -154,18 +142,6 @@ class Version(object): def is_prerelease(self): return self.pre != None - def is_recommended(self): - return self.branch == recommended_release - - def get_download_anchor(self): - if self.is_prerelease(): - return 'pre-releases' - else: - if self.is_recommended(): - return 'recommended-release' - else: - return 'supported-releases' - def get_ver_tags(self, revnum): # These get substituted into svn_version.h ver_tag = '' @@ -174,7 +150,7 @@ class Version(object): ver_tag = '" (Alpha %d)"' % self.pre_num ver_numtag = '"-alpha%d"' % self.pre_num elif self.pre == 'beta': - ver_tag = '" (Beta %d)"' % args.version.pre_num + ver_tag = '" (Beta %d)"' % self.pre_num ver_numtag = '"-beta%d"' % self.pre_num elif self.pre == 'rc': ver_tag = '" (Release Candidate %d)"' % self.pre_num @@ -183,7 +159,7 @@ class Version(object): ver_tag = '" (Nightly Build r%d)"' % revnum ver_numtag = '"-nightly-r%d"' % revnum else: - ver_tag = '" (r%d)"' % revnum + ver_tag = '" (r%d)"' % revnum ver_numtag = '""' return (ver_tag, ver_numtag) @@ -253,15 +229,21 @@ def get_exportdir(base_dir, version, rev return os.path.join(get_tempdir(base_dir), 'subversion-%s-r%d' % (version, revnum)) -def get_deploydir(base_dir): - return os.path.join(base_dir, 'deploy') - def get_target(args): "Return the location of the artifacts" if args.target: return args.target else: - return get_deploydir(args.base_dir) + return os.path.join(args.base_dir, 'deploy') + +def get_branch_path(args): + if not args.branch: + try: + args.branch = 'branches/%d.%d.x' % (args.version.major, args.version.minor) + except AttributeError: + raise RuntimeError("Please specify the branch using the release version label argument (for certain subcommands) or the '--branch' global option") + + return args.branch.rstrip('/') # canonicalize for later comparisons def get_tmpldir(): return os.path.join(os.path.abspath(sys.path[0]), 'templates') @@ -271,12 +253,14 @@ def get_tmplfile(filename): return open(os.path.join(get_tmpldir(), filename)) except IOError: # Hmm, we had a problem with the local version, let's try the repo - return urllib2.urlopen(repos + '/trunk/tools/dist/templates/' + filename) + return urlopen(svn_repos + '/trunk/tools/dist/templates/' + filename) def get_nullfile(): return open(os.path.devnull, 'w') -def run_script(verbose, script, hide_stderr=False): +def run_command(cmd, verbose=True, hide_stderr=False, dry_run=False): + if verbose: + print("+ " + ' '.join(cmd)) stderr = None if verbose: stdout = None @@ -285,23 +269,62 @@ def run_script(verbose, script, hide_std if hide_stderr: stderr = get_nullfile() + if not dry_run: + subprocess.check_call(cmd, stdout=stdout, stderr=stderr) + else: + print(' ## dry-run; not executed') + +def run_script(verbose, script, hide_stderr=False): for l in script.split('\n'): - subprocess.check_call(l.split(), stdout=stdout, stderr=stderr) + run_command(l.split(), verbose, hide_stderr) def download_file(url, target, checksum): - response = urllib2.urlopen(url) - target_file = open(target, 'w+') + """Download the file at URL to the local path TARGET. + If CHECKSUM is a string, verify the checksum of the downloaded + file and raise RuntimeError if it does not match. If CHECKSUM + is None, do not verify the downloaded file. + """ + assert checksum is None or isinstance(checksum, str) + + response = urlopen(url) + target_file = open(target, 'w+b') target_file.write(response.read()) target_file.seek(0) m = hashlib.sha256() m.update(target_file.read()) target_file.close() checksum2 = m.hexdigest() - if checksum != checksum2: + if checksum is not None and checksum != checksum2: raise RuntimeError("Checksum mismatch for '%s': "\ "downloaded: '%s'; expected: '%s'" % \ (target, checksum, checksum2)) +def run_svn(cmd, verbose=True, dry_run=False, username=None): + if (username): + cmd[:0] = ['--username', username] + run_command(['svn'] + cmd, verbose=verbose, dry_run=dry_run) + +def run_svnmucc(cmd, verbose=True, dry_run=False, username=None): + if (username): + cmd[:0] = ['--username', username] + run_command(['svnmucc'] + cmd, verbose=verbose, dry_run=dry_run) + +#---------------------------------------------------------------------- +def is_lts(version): + return version.branch in lts_release_lines + +def is_recommended(version): + return version.branch == recommended_release + +def get_download_anchor(version): + if version.is_prerelease(): + return 'pre-releases' + else: + if is_recommended(version): + return 'recommended-release' + else: + return 'supported-releases' + #---------------------------------------------------------------------- # ezt helpers @@ -325,7 +348,7 @@ def cleanup(args): shutil.rmtree(get_prefix(args.base_dir), True) shutil.rmtree(get_tempdir(args.base_dir), True) - shutil.rmtree(get_deploydir(args.base_dir), True) + shutil.rmtree(get_target(args), True) #---------------------------------------------------------------------- @@ -340,7 +363,8 @@ class RollDep(object): def _test_version(self, cmd): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + universal_newlines=True) (stdout, stderr) = proc.communicate() rc = proc.wait() if rc: return '' @@ -482,13 +506,236 @@ def build_env(args): #---------------------------------------------------------------------- +# Create a new minor release branch + +def get_trunk_wc_path(base_dir, path=None): + trunk_wc_path = os.path.join(get_tempdir(base_dir), 'svn-trunk') + if path is None: return trunk_wc_path + return os.path.join(trunk_wc_path, path) + +def get_buildbot_wc_path(base_dir, path=None): + buildbot_wc_path = os.path.join(get_tempdir(base_dir), 'svn-buildmaster') + if path is None: return buildbot_wc_path + return os.path.join(buildbot_wc_path, path) + +def get_trunk_url(revnum=None): + return svn_repos + '/trunk' + '@' + (str(revnum) if revnum else '') + +def get_branch_url(ver): + return svn_repos + '/branches/' + ver.branch + '.x' + +def get_tag_url(ver): + return svn_repos + '/tags/' + ver.base + +def edit_file(path, pattern, replacement): + print("Editing '%s'" % (path,)) + print(" pattern='%s'" % (pattern,)) + print(" replace='%s'" % (replacement,)) + old_text = open(path, 'r').read() + new_text = re.sub(pattern, replacement, old_text) + assert new_text != old_text + open(path, 'w').write(new_text) + +def edit_changes_file(path, newtext): + """Insert NEWTEXT in the 'CHANGES' file found at PATH, + just before the first line that starts with 'Version '. + """ + print("Prepending to '%s'" % (path,)) + print(" text='%s'" % (newtext,)) + lines = open(path, 'r').readlines() + for i, line in enumerate(lines): + if line.startswith('Version '): + with open(path, 'w') as newfile: + newfile.writelines(lines[:i]) + newfile.write(newtext) + newfile.writelines(lines[i:]) + break + +#---------------------------------------------------------------------- +def make_release_branch(args): + ver = args.version + run_svn(['copy', + get_trunk_url(args.revnum), + get_branch_url(ver), + '-m', 'Create the ' + ver.branch + '.x release branch.'], + dry_run=args.dry_run) + +#---------------------------------------------------------------------- +def update_minor_ver_in_trunk(args): + """Change the minor version in trunk to the next (future) minor version. + """ + ver = args.version + trunk_wc = get_trunk_wc_path(args.base_dir) + run_svn(['checkout', + get_trunk_url(args.revnum), + trunk_wc]) + + prev_ver = Version('1.%d.0' % (ver.minor - 1,)) + next_ver = Version('1.%d.0' % (ver.minor + 1,)) + relpaths = [] + + relpath = 'subversion/include/svn_version.h' + relpaths.append(relpath) + edit_file(get_trunk_wc_path(args.base_dir, relpath), + r'(#define SVN_VER_MINOR *)%s' % (ver.minor,), + r'\g<1>%s' % (next_ver.minor,)) + + relpath = 'subversion/tests/cmdline/svntest/main.py' + relpaths.append(relpath) + edit_file(get_trunk_wc_path(args.base_dir, relpath), + r'(SVN_VER_MINOR = )%s' % (ver.minor,), + r'\g<1>%s' % (next_ver.minor,)) + + relpath = 'subversion/bindings/javahl/src/org/apache/subversion/javahl/NativeResources.java' + relpaths.append(relpath) + try: + # since r1817921 (just after branching 1.10) + edit_file(get_trunk_wc_path(args.base_dir, relpath), + r'SVN_VER_MINOR = %s;' % (ver.minor,), + r'SVN_VER_MINOR = %s;' % (next_ver.minor,)) + except: + # before r1817921: two separate places + edit_file(get_trunk_wc_path(args.base_dir, relpath), + r'version.isAtLeast\(1, %s, 0\)' % (ver.minor,), + r'version.isAtLeast\(1, %s, 0\)' % (next_ver.minor,)) + edit_file(get_trunk_wc_path(args.base_dir, relpath), + r'1.%s.0, but' % (ver.minor,), + r'1.%s.0, but' % (next_ver.minor,)) + + relpath = 'CHANGES' + relpaths.append(relpath) + # insert at beginning of CHANGES file + edit_changes_file(get_trunk_wc_path(args.base_dir, relpath), + 'Version ' + next_ver.base + '\n' + + '(?? ??? 20XX, from /branches/' + next_ver.branch + '.x)\n' + + get_tag_url(next_ver) + '\n' + + '\n') + + log_msg = '''\ +Increment the trunk version number to %s, and introduce a new CHANGES +section, following the creation of the %s.x release branch. + +* subversion/include/svn_version.h, + subversion/bindings/javahl/src/org/apache/subversion/javahl/NativeResources.java, + subversion/tests/cmdline/svntest/main.py + (SVN_VER_MINOR): Increment to %s. + +* CHANGES: New section for %s.0. +''' % (next_ver.branch, ver.branch, next_ver.minor, next_ver.branch) + commit_paths = [get_trunk_wc_path(args.base_dir, p) for p in relpaths] + run_svn(['commit'] + commit_paths + ['-m', log_msg], + dry_run=args.dry_run) + +#---------------------------------------------------------------------- +def create_status_file_on_branch(args): + ver = args.version + branch_wc = get_workdir(args.base_dir) + branch_url = get_branch_url(ver) + run_svn(['checkout', branch_url, branch_wc, '--depth=immediates']) + + status_local_path = os.path.join(branch_wc, 'STATUS') + template_filename = 'STATUS.ezt' + data = { 'major-minor' : ver.branch, + 'major-minor-patch' : ver.base, + } + + template = ezt.Template(compress_whitespace=False) + template.parse(get_tmplfile(template_filename).read()) + + with open(status_local_path, 'wx') as g: + template.generate(g, data) + run_svn(['add', status_local_path]) + run_svn(['commit', status_local_path, + '-m', '* branches/' + ver.branch + '.x/STATUS: New file.'], + dry_run=args.dry_run) + +#---------------------------------------------------------------------- +def update_backport_bot(args): + ver = args.version + print("""\ + +*** MANUAL STEP REQUIRED *** + + Ask someone with appropriate access to add the %s.x branch + to the backport merge bot. See + http://subversion.apache.org/docs/community-guide/releasing.html#backport-merge-bot + +*** + +""" % (ver.branch,)) + +#---------------------------------------------------------------------- +def update_buildbot_config(args): + """Add the new branch to the list of branches monitored by the buildbot + master. + """ + ver = args.version + buildbot_wc = get_buildbot_wc_path(args.base_dir) + run_svn(['checkout', buildbot_repos, buildbot_wc]) + + prev_ver = Version('1.%d.0' % (ver.minor - 1,)) + next_ver = Version('1.%d.0' % (ver.minor + 1,)) + + relpath = 'master1/projects/subversion.conf' + edit_file(get_buildbot_wc_path(args.base_dir, relpath), + r'(MINOR_LINES=\[.*%s)(\])' % (prev_ver.minor,), + r'\1, %s\2' % (ver.minor,)) + + log_msg = '''\ +Subversion: start monitoring the %s branch. +''' % (ver.branch) + commit_paths = [get_buildbot_wc_path(args.base_dir, relpath)] + run_svn(['commit'] + commit_paths + ['-m', log_msg], + dry_run=args.dry_run) + +#---------------------------------------------------------------------- +def create_release_branch(args): + make_release_branch(args) + update_minor_ver_in_trunk(args) + create_status_file_on_branch(args) + update_backport_bot(args) + update_buildbot_config(args) + + +#---------------------------------------------------------------------- +def write_release_notes(args): + + # Create a skeleton release notes file from template + + template_filename = \ + 'release-notes-lts.ezt' if is_lts(args.version) else 'release-notes.ezt' + + prev_ver = Version('%d.%d.0' % (args.version.major, args.version.minor - 1)) + data = { 'major-minor' : args.version.branch, + 'previous-major-minor' : prev_ver.branch, + } + + template = ezt.Template(compress_whitespace=False) + template.parse(get_tmplfile(template_filename).read()) + + if args.edit_html_file: + with open(args.edit_html_file, 'w') as g: + template.generate(g, data) + else: + template.generate(sys.stdout, data) + + # Add an "in progress" entry in the release notes index + # + index_file = os.path.normpath(args.edit_html_file + '/../index.html') + marker = '<ul id="release-notes-list">\n' + new_item = '<li><a href="%s.html">Subversion %s</a> â <i>in progress</i></li>\n' % (args.version.branch, args.version.branch) + edit_file(index_file, + re.escape(marker), + (marker + new_item).replace('\\', r'\\')) + +#---------------------------------------------------------------------- # Create release artifacts def compare_changes(repos, branch, revision): mergeinfo_cmd = ['svn', 'mergeinfo', '--show-revs=eligible', repos + '/trunk/CHANGES', repos + '/' + branch + '/' + 'CHANGES'] - stdout = subprocess.check_output(mergeinfo_cmd) + stdout = subprocess.check_output(mergeinfo_cmd, universal_newlines=True) if stdout: # Treat this as a warning since we are now putting entries for future # minor releases in CHANGES on trunk. @@ -506,7 +753,7 @@ def check_copyright_year(repos, branch, file_url = (repos + '/' + branch + '/' + branch_relpath + '@' + str(revision)) cat_cmd = ['svn', 'cat', file_url] - stdout = subprocess.check_output(cat_cmd) + stdout = subprocess.check_output(cat_cmd, universal_newlines=True) m = _copyright_re.search(stdout) if m: year = m.group('year') @@ -531,16 +778,12 @@ def replace_lines(path, actions): def roll_tarballs(args): 'Create the release artifacts.' - if not args.branch: - args.branch = 'branches/%d.%d.x' % (args.version.major, args.version.minor) - - branch = args.branch # shorthand - branch = branch.rstrip('/') # canonicalize for later comparisons + branch = get_branch_path(args) logging.info('Rolling release %s from branch %s@%d' % (args.version, branch, args.revnum)) - check_copyright_year(repos, args.branch, args.revnum) + check_copyright_year(svn_repos, branch, args.revnum) # Ensure we've got the appropriate rolling dependencies available autoconf = AutoconfDep(args.base_dir, False, args.verbose, @@ -559,20 +802,21 @@ def roll_tarballs(args): if branch != 'trunk': # Make sure CHANGES is sync'd. - compare_changes(repos, branch, args.revnum) + compare_changes(svn_repos, branch, args.revnum) # Ensure the output directory doesn't already exist - if os.path.exists(get_deploydir(args.base_dir)): + if os.path.exists(get_target(args)): raise RuntimeError('output directory \'%s\' already exists' - % get_deploydir(args.base_dir)) + % get_target(args)) - os.mkdir(get_deploydir(args.base_dir)) + os.mkdir(get_target(args)) logging.info('Preparing working copy source') shutil.rmtree(get_workdir(args.base_dir), True) - run_script(args.verbose, 'svn checkout %s %s' - % (repos + '/' + branch + '@' + str(args.revnum), - get_workdir(args.base_dir))) + run_svn(['checkout', + svn_repos + '/' + branch + '@' + str(args.revnum), + get_workdir(args.base_dir)], + verbose=args.verbose) # Exclude stuff we don't want in the tarball, it will not be present # in the exported tree. @@ -583,8 +827,8 @@ def roll_tarballs(args): exclude += ['packages', 'www'] cwd = os.getcwd() os.chdir(get_workdir(args.base_dir)) - run_script(args.verbose, - 'svn update --set-depth exclude %s' % " ".join(exclude)) + run_svn(['update', '--set-depth=exclude'] + exclude, + verbose=args.verbose) os.chdir(cwd) if args.patches: @@ -594,10 +838,10 @@ def roll_tarballs(args): for name in os.listdir(args.patches): if name.find(majmin) != -1 and name.endswith('patch'): logging.info('Applying patch %s' % name) - run_script(args.verbose, - '''svn patch %s %s''' - % (os.path.join(args.patches, name), - get_workdir(args.base_dir))) + run_svn(['patch', + os.path.join(args.patches, name), + get_workdir(args.base_dir)], + verbose=args.verbose) # Massage the new version number into svn_version.h. ver_tag, ver_numtag = args.version.get_ver_tags(args.revnum) @@ -632,11 +876,12 @@ def roll_tarballs(args): def export(windows): shutil.rmtree(exportdir, True) if windows: - eol_style = "--native-eol CRLF" + eol_style = "--native-eol=CRLF" else: - eol_style = "--native-eol LF" - run_script(args.verbose, "svn export %s %s %s" - % (eol_style, get_workdir(args.base_dir), exportdir)) + eol_style = "--native-eol=LF" + run_svn(['export', + eol_style, get_workdir(args.base_dir), exportdir], + verbose=args.verbose) def transform_sql(): for root, dirs, files in os.walk(exportdir): @@ -687,7 +932,7 @@ def roll_tarballs(args): # Use the gzip -n flag - this prevents it from storing the # original name of the .tar file, and far more importantly, the # mtime of the .tar file, in the produced .tar.gz file. This is - # important, because it makes the gzip encoding reproducable by + # important, because it makes the gzip encoding reproducible by # anyone else who has an similar version of gzip, and also uses # "gzip -9n". This means that committers who want to GPG-sign both # the .tar.gz and the .tar.bz2 can download the .tar.bz2 (which is @@ -710,21 +955,33 @@ def roll_tarballs(args): for e in extns: filename = basename + '.' + e filepath = os.path.join(get_tempdir(args.base_dir), filename) - shutil.move(filepath, get_deploydir(args.base_dir)) - filepath = os.path.join(get_deploydir(args.base_dir), filename) - m = hashlib.sha1() - m.update(open(filepath, 'r').read()) - open(filepath + '.sha1', 'w').write(m.hexdigest()) + shutil.move(filepath, get_target(args)) + filepath = os.path.join(get_target(args), filename) + if args.version < Version("1.11.0-alpha1"): + # 1.10 and earlier generate *.sha1 files for compatibility reasons. + # They are deprecated, however, so we don't publicly link them in + # the announcements any more. + m = hashlib.sha1() + m.update(open(filepath, 'rb').read()) + open(filepath + '.sha1', 'w').write(m.hexdigest()) m = hashlib.sha512() - m.update(open(filepath, 'r').read()) + m.update(open(filepath, 'rb').read()) open(filepath + '.sha512', 'w').write(m.hexdigest()) # Nightlies do not get tagged so do not need the header if args.version.pre != 'nightly': shutil.copy(os.path.join(get_workdir(args.base_dir), 'subversion', 'include', 'svn_version.h'), - os.path.join(get_deploydir(args.base_dir), - 'svn_version.h.dist-%s' % str(args.version))) + os.path.join(get_target(args), + 'svn_version.h.dist-%s' + % (str(args.version),))) + + # Download and "tag" the KEYS file (in case a signing key is removed + # from a committer's LDAP profile down the road) + basename = 'subversion-%s.KEYS' % (str(args.version),) + filepath = os.path.join(get_tempdir(args.base_dir), basename) + download_file(KEYS, filepath, None) + shutil.move(filepath, get_target(args)) # And we're done! @@ -737,8 +994,12 @@ def sign_candidates(args): def sign_file(filename): asc_file = open(filename + '.asc', 'a') logging.info("Signing %s" % filename) - proc = subprocess.check_call(['gpg', '-ba', '-o', '-', filename], - stdout=asc_file) + if args.userid: + proc = subprocess.check_call(['gpg', '-ba', '-u', args.userid, + '-o', '-', filename], stdout=asc_file) + else: + proc = subprocess.check_call(['gpg', '-ba', '-o', '-', filename], + stdout=asc_file) asc_file.close() target = get_target(args) @@ -746,7 +1007,7 @@ def sign_candidates(args): for e in extns: filename = os.path.join(target, 'subversion-%s.%s' % (args.version, e)) sign_file(filename) - if args.version.major >= 1 and args.version.minor <= 6: + if args.version.major == 1 and args.version.minor <= 6: filename = os.path.join(target, 'subversion-deps-%s.%s' % (args.version, e)) sign_file(filename) @@ -762,107 +1023,125 @@ def post_candidates(args): logging.info('Importing tarballs to %s' % dist_dev_url) ver = str(args.version) - svn_cmd = ['svn', 'import', '-m', + svn_cmd = ['import', '-m', 'Add Subversion %s candidate release artifacts' % ver, '--auto-props', '--config-option', 'config:auto-props:*.asc=svn:eol-style=native;svn:mime-type=text/plain', target, dist_dev_url] - if (args.username): - svn_cmd += ['--username', args.username] - subprocess.check_call(svn_cmd) + run_svn(svn_cmd, verbose=args.verbose, username=args.username) #---------------------------------------------------------------------- # Create tag +# Bump versions on branch -def create_tag(args): +def create_tag_only(args): 'Create tag in the repository' target = get_target(args) logging.info('Creating tag for %s' % str(args.version)) - if not args.branch: - args.branch = 'branches/%d.%d.x' % (args.version.major, args.version.minor) - - branch = secure_repos + '/' + args.branch.rstrip('/') + branch_url = svn_repos + '/' + get_branch_path(args) - tag = secure_repos + '/tags/' + str(args.version) + tag = svn_repos + '/tags/' + str(args.version) - svnmucc_cmd = ['svnmucc', '-m', - 'Tagging release ' + str(args.version)] - if (args.username): - svnmucc_cmd += ['--username', args.username] - svnmucc_cmd += ['cp', str(args.revnum), branch, tag] + svnmucc_cmd = ['-m', 'Tagging release ' + str(args.version)] + svnmucc_cmd += ['cp', str(args.revnum), branch_url, tag] svnmucc_cmd += ['put', os.path.join(target, 'svn_version.h.dist' + '-' + str(args.version)), tag + '/subversion/include/svn_version.h'] # don't redirect stdout/stderr since svnmucc might ask for a password try: - subprocess.check_call(svnmucc_cmd) + run_svnmucc(svnmucc_cmd, verbose=args.verbose, username=args.username) except subprocess.CalledProcessError: if args.version.is_prerelease(): logging.error("Do you need to pass --branch=trunk?") raise - if not args.version.is_prerelease(): - logging.info('Bumping revisions on the branch') - def replace_in_place(fd, startofline, flat, spare): - """In file object FD, replace FLAT with SPARE in the first line - starting with STARTOFLINE.""" - - fd.seek(0, os.SEEK_SET) - lines = fd.readlines() - for i, line in enumerate(lines): - if line.startswith(startofline): - lines[i] = line.replace(flat, spare) - break - else: - raise RuntimeError('Definition of %r not found' % startofline) +def bump_versions_on_branch(args): + 'Bump version numbers on branch' + + logging.info('Bumping version numbers on the branch') + + branch_url = svn_repos + '/' + get_branch_path(args) + + def replace_in_place(fd, startofline, flat, spare): + """In file object FD, replace FLAT with SPARE in the first line + starting with regex STARTOFLINE.""" + + pattern = r'^(%s)%s' % (startofline, re.escape(flat)) + repl = r'\g<1>%s' % (spare,) + fd.seek(0, os.SEEK_SET) + lines = fd.readlines() + for i, line in enumerate(lines): + replacement = re.sub(pattern, repl, line) + if replacement != line: + lines[i] = replacement + break + else: + raise RuntimeError("Could not replace r'%s' with r'%s' in '%s'" + % (pattern, repl, fd.url)) + + fd.seek(0, os.SEEK_SET) + fd.writelines(lines) + fd.truncate() # for current callers, new value is never shorter. + + new_version = Version('%d.%d.%d' % + (args.version.major, args.version.minor, + args.version.patch + 1)) + + HEAD = subprocess.check_output(['svn', 'info', '--show-item=revision', + '--', branch_url], + universal_newlines=True).strip() + HEAD = int(HEAD) + def file_object_for(relpath): + fd = tempfile.NamedTemporaryFile(mode='w+', encoding='UTF-8') + url = branch_url + '/' + relpath + fd.url = url + subprocess.check_call(['svn', 'cat', '%s@%d' % (url, HEAD)], + stdout=fd) + return fd + + svn_version_h = file_object_for('subversion/include/svn_version.h') + replace_in_place(svn_version_h, '#define SVN_VER_PATCH *', + str(args.version.patch), str(new_version.patch)) + + STATUS = file_object_for('STATUS') + replace_in_place(STATUS, 'Status of ', + str(args.version), str(new_version)) + + svn_version_h.seek(0, os.SEEK_SET) + STATUS.seek(0, os.SEEK_SET) + run_svnmucc(['-r', str(HEAD), + '-m', 'Post-release housekeeping: ' + 'bump the %s branch to %s.' + % (branch_url.split('/')[-1], str(new_version)), + 'put', svn_version_h.name, svn_version_h.url, + 'put', STATUS.name, STATUS.url, + ], + verbose=args.verbose, username=args.username) + del svn_version_h + del STATUS + +def create_tag_and_bump_versions(args): + '''Create tag in the repository and, if not a prerelease version, + bump version numbers on the branch''' + + create_tag_only(args) - fd.seek(0, os.SEEK_SET) - fd.writelines(lines) - fd.truncate() # for current callers, new value is never shorter. - - new_version = Version('%d.%d.%d' % - (args.version.major, args.version.minor, - args.version.patch + 1)) - - def file_object_for(relpath): - fd = tempfile.NamedTemporaryFile() - url = branch + '/' + relpath - fd.url = url - subprocess.check_call(['svn', 'cat', '%s@%d' % (url, args.revnum)], - stdout=fd) - return fd - - svn_version_h = file_object_for('subversion/include/svn_version.h') - replace_in_place(svn_version_h, '#define SVN_VER_PATCH ', - str(args.version.patch), str(new_version.patch)) - - STATUS = file_object_for('STATUS') - replace_in_place(STATUS, 'Status of ', - str(args.version), str(new_version)) - - svn_version_h.seek(0, os.SEEK_SET) - STATUS.seek(0, os.SEEK_SET) - subprocess.check_call(['svnmucc', '-r', str(args.revnum), - '-m', 'Post-release housekeeping: ' - 'bump the %s branch to %s.' - % (branch.split('/')[-1], str(new_version)), - 'put', svn_version_h.name, svn_version_h.url, - 'put', STATUS.name, STATUS.url, - ]) - del svn_version_h - del STATUS + if not args.version.is_prerelease(): + bump_versions_on_branch(args) #---------------------------------------------------------------------- # Clean dist def clean_dist(args): - 'Clean the distribution directory of all but the most recent artifacts.' + '''Clean the distribution directory of release artifacts of + no-longer-supported minor lines.''' - stdout = subprocess.check_output(['svn', 'list', dist_release_url]) + stdout = subprocess.check_output(['svn', 'list', dist_release_url], + universal_newlines=True) def minor(version): """Return the minor release line of the parameter, which must be @@ -872,23 +1151,20 @@ def clean_dist(args): filenames = stdout.split('\n') filenames = filter(lambda x: x.startswith('subversion-'), filenames) versions = set(map(Version, filenames)) - minor_lines = set(map(minor, versions)) to_keep = set() - # Keep 3 minor lines: 1.10.0-alpha3, 1.9.7, 1.8.19. # TODO: When we release 1.A.0 GA we'll have to manually remove 1.(A-2).* artifacts. - for recent_line in sorted(minor_lines, reverse=True)[:3]: - to_keep.add(max( + for line_to_keep in [minor(Version(x + ".0")) for x in supported_release_lines]: + candidates = list( x for x in versions - if minor(x) == recent_line - )) + if minor(x) == line_to_keep + ) + if candidates: + to_keep.add(max(candidates)) for i in sorted(to_keep): logging.info("Saving release '%s'", i) - svnmucc_cmd = ['svnmucc', '-m', 'Remove old Subversion releases.\n' + - 'They are still available at ' + - 'https://archive.apache.org/dist/subversion/'] - if (args.username): - svnmucc_cmd += ['--username', args.username] + svnmucc_cmd = ['-m', 'Remove old Subversion releases.\n' + + 'They are still available at ' + dist_archive_url] for filename in filenames: if Version(filename) not in to_keep: logging.info("Removing %r", filename) @@ -896,7 +1172,7 @@ def clean_dist(args): # don't redirect stdout/stderr since svnmucc might ask for a password if 'rm' in svnmucc_cmd: - subprocess.check_call(svnmucc_cmd) + run_svnmucc(svnmucc_cmd, verbose=args.verbose, username=args.username) else: logging.info("Nothing to remove") @@ -906,16 +1182,15 @@ def clean_dist(args): def move_to_dist(args): 'Move candidate artifacts to the distribution directory.' - stdout = subprocess.check_output(['svn', 'list', dist_dev_url]) + stdout = subprocess.check_output(['svn', 'list', dist_dev_url], + universal_newlines=True) filenames = [] for entry in stdout.split('\n'): if fnmatch.fnmatch(entry, 'subversion-%s.*' % str(args.version)): filenames.append(entry) - svnmucc_cmd = ['svnmucc', '-m', + svnmucc_cmd = ['-m', 'Publish Subversion-%s.' % str(args.version)] - if (args.username): - svnmucc_cmd += ['--username', args.username] svnmucc_cmd += ['rm', dist_dev_url + '/' + 'svn_version.h.dist' + '-' + str(args.version)] for filename in filenames: @@ -924,20 +1199,25 @@ def move_to_dist(args): # don't redirect stdout/stderr since svnmucc might ask for a password logging.info('Moving release artifacts to %s' % dist_release_url) - subprocess.check_call(svnmucc_cmd) + run_svnmucc(svnmucc_cmd, verbose=args.verbose, username=args.username) #---------------------------------------------------------------------- # Write announcements def write_news(args): 'Write text for the Subversion website.' - data = { 'date' : datetime.date.today().strftime('%Y%m%d'), - 'date_pres' : datetime.date.today().strftime('%Y-%m-%d'), + if args.news_release_date: + release_date = datetime.datetime.strptime(args.news_release_date, '%Y-%m-%d') + else: + release_date = datetime.date.today() + data = { 'date' : release_date.strftime('%Y%m%d'), + 'date_pres' : release_date.strftime('%Y-%m-%d'), 'major-minor' : args.version.branch, 'version' : str(args.version), 'version_base' : args.version.base, - 'anchor': args.version.get_download_anchor(), - 'is_recommended': ezt_bool(args.version.is_recommended()), + 'anchor': get_download_anchor(args.version), + 'is_recommended': ezt_bool(is_recommended(args.version)), + 'announcement_url': args.announcement_url, } if args.version.is_prerelease(): @@ -947,41 +1227,56 @@ def write_news(args): template = ezt.Template() template.parse(get_tmplfile(template_filename).read()) - template.generate(sys.stdout, data) + + # Insert the output into an existing file if requested, else print it + if args.edit_html_file: + tmp_name = args.edit_html_file + '.tmp' + with open(args.edit_html_file, 'r') as f, open(tmp_name, 'w') as g: + inserted = False + for line in f: + if not inserted and line.startswith('<div class="h3" id="news-'): + template.generate(g, data) + g.write('\n') + inserted = True + g.write(line) + os.remove(args.edit_html_file) + os.rename(tmp_name, args.edit_html_file) + else: + template.generate(sys.stdout, data) -def get_sha1info(args): - 'Return a list of sha1 info for the release' +def get_fileinfo(args): + 'Return a list of file info (filenames) for the release tarballs' target = get_target(args) - sha1s = glob.glob(os.path.join(target, 'subversion*-%s*.sha1' % args.version)) + files = glob.glob(os.path.join(target, 'subversion*-%s.*.asc' % args.version)) + files.sort() class info(object): pass - sha1info = [] - for s in sha1s: + fileinfo = [] + for f in files: i = info() - # strip ".sha1" - i.filename = os.path.basename(s)[:-5] - i.sha1 = open(s, 'r').read() - sha1info.append(i) + # strip ".asc" + i.filename = os.path.basename(f)[:-4] + fileinfo.append(i) - return sha1info + return fileinfo def write_announcement(args): 'Write the release announcement.' - sha1info = get_sha1info(args) - siginfo = "\n".join(get_siginfo(args, True)) + "\n" + siginfo = get_siginfo(args, True) + if not siginfo: + raise RuntimeError("No signatures found for %s at %s" % (args.version, args.target)) data = { 'version' : str(args.version), - 'sha1info' : sha1info, - 'siginfo' : siginfo, + 'siginfo' : "\n".join(siginfo) + "\n", 'major-minor' : args.version.branch, 'major-minor-patch' : args.version.base, - 'anchor' : args.version.get_download_anchor(), + 'anchor' : get_download_anchor(args.version), } if args.version.is_prerelease(): @@ -1007,10 +1302,10 @@ def write_announcement(args): def write_downloads(args): 'Output the download section of the website.' - sha1info = get_sha1info(args) + fileinfo = get_fileinfo(args) data = { 'version' : str(args.version), - 'fileinfo' : sha1info, + 'fileinfo' : fileinfo, } template = ezt.Template(compress_whitespace = False) @@ -1046,14 +1341,12 @@ def get_siginfo(args, quiet=False): import security._gnupg as gnupg gpg = gnupg.GPG() - target = get_target(args) - good_sigs = {} fingerprints = {} output = [] - glob_pattern = os.path.join(target, 'subversion*-%s*.asc' % args.version) - for filename in glob.glob(glob_pattern): + for fileinfo in get_fileinfo(args): + filename = os.path.join(get_target(args), fileinfo.filename + '.asc') text = open(filename).read() keys = text.split(key_start) @@ -1078,9 +1371,9 @@ def get_siginfo(args, quiet=False): % (n, filename, key_end)) sys.exit(1) - fd, fn = tempfile.mkstemp() - os.write(fd, key_start + key) - os.close(fd) + fd, fn = tempfile.mkstemp(text=True) + with os.fdopen(fd, 'w') as key_file: + key_file.write(key_start + key) verified = gpg.verify_file(open(fn, 'rb'), filename[:-4]) os.unlink(fn) @@ -1102,6 +1395,7 @@ def get_siginfo(args, quiet=False): gpg_output = subprocess.check_output( ['gpg', '--fixed-list-mode', '--with-colons', '--fingerprint', id], stderr=subprocess.STDOUT, + universal_newlines=True, ) gpg_output = gpg_output.splitlines() @@ -1169,11 +1463,212 @@ def get_keys(args): 'Import the LDAP-based KEYS file to gpg' # We use a tempfile because urlopen() objects don't have a .fileno() with tempfile.SpooledTemporaryFile() as fd: - fd.write(urllib2.urlopen(KEYS).read()) + fd.write(urlopen(KEYS).read()) fd.flush() fd.seek(0) subprocess.check_call(['gpg', '--import'], stdin=fd) +def add_to_changes_dict(changes_dict, audience, section, change, revision): + # Normalize arguments + if audience: + audience = audience.upper() + if section: + section = section.lower() + change = change.strip() + + if not audience in changes_dict: + changes_dict[audience] = dict() + if not section in changes_dict[audience]: + changes_dict[audience][section] = dict() + + changes = changes_dict[audience][section] + if change in changes: + changes[change].add(revision) + else: + changes[change] = set([revision]) + +def print_section(changes_dict, audience, section, title, mandatory=False): + if audience in changes_dict: + audience_changes = changes_dict[audience] + if mandatory or (section in audience_changes): + if title: + print(' - %s:' % title) + if section in audience_changes: + print_changes(audience_changes[section]) + elif mandatory: + print(' (none)') + +def print_changes(changes): + # Print in alphabetical order, so entries with the same prefix are together + for change in sorted(changes): + revs = changes[change] + rev_string = 'r' + str(min(revs)) + (' et al' if len(revs) > 1 else '') + print(' * %s (%s)' % (change, rev_string)) + +def write_changelog(args): + 'Write changelog, parsed from commit messages' + # Changelog lines are lines with the following format: + # '['[audience[:section]]']' <message> + # or: + # <message> '['[audience[:section]]']' + # where audience = U (User-visible) or D (Developer-visible) + # section = general|major|minor|client|server|clientserver|other|api|bindings + # (section is optional and is treated case-insensitively) + # message = the actual text for CHANGES + # + # This means the "changes label" can be used as prefix or suffix, and it + # can also be left empty (which results in an uncategorized changes entry), + # if the committer isn't sure where the changelog entry belongs. + # + # Putting [skip], [ignore], [c:skip] or [c:ignore] somewhere in the + # log message means this commit must be ignored for Changelog processing + # (ignored even with the --include-unlabeled-summaries option). + # + # If there is no changes label anywhere in the commit message, and the + # --include-unlabeled-summaries option is used, we'll consider the summary + # line of the commit message (= first line except if it starts with a *) + # as an uncategorized changes entry, except if it contains "status", + # "changes", "post-release housekeeping" or "follow-up". + # + # Examples: + # [U:major] Better interactive conflict resolution for tree conflicts + # ra_serf: Adjustments for serf versions with HTTP/2 support [U:minor] + # [U] Fix 'svn diff URL@REV WC' wrongly looks up URL@HEAD (issue #4597) + # Fix bug with canonicalizing Window-specific drive-relative URL [] + # New svn_ra_list() API function [D:api] + # [D:bindings] JavaHL: Allow access to constructors of a couple JavaHL classes + + branch_url = svn_repos + '/' + get_branch_path(args) + previous = svn_repos + '/' + args.previous + include_unlabeled = args.include_unlabeled + separator_line = ('-' * 72) + '\n' + + mergeinfo = subprocess.check_output(['svn', 'mergeinfo', '--show-revs', + 'eligible', '--log', branch_url, previous], + universal_newlines=True) + log_messages_dict = { + # This is a dictionary mapping revision numbers to their respective + # log messages. The expression in the "key:" part of the dict + # comprehension extracts the revision number, as integer, from the + # 'svn log' output. + int(log_message.splitlines()[0].split()[0][1:]): log_message + # The [1:-1] ignores the empty first and last element of the split(). + for log_message in mergeinfo.split(separator_line)[1:-1] + } + mergeinfo = mergeinfo.splitlines() + + separator_pattern = re.compile('^-{72}$') + revline_pattern = re.compile('^r(\d+) \| [^\|]+ \| [^\|]+ \| \d+ lines?$') + changes_prefix_pattern = re.compile(r'^\[(U|D)?:?([^\]]+)?\](.+)$') + changes_suffix_pattern = re.compile(r'^(.+)\[(U|D)?:?([^\]]+)?\]$') + # TODO: push this into backport.status as a library function + auto_merge_pattern = \ + re.compile(r'^Merge (r\d+,? |the r\d+ group |the \S+ branch:)') + + changes_dict = dict() # audience -> (section -> (change -> set(revision))) + revision = -1 + got_firstline = False + unlabeled_summary = None + changes_ignore = False + audience = None + section = None + message = None + + for line in mergeinfo: + if separator_pattern.match(line): + # New revision section. Reset variables. + # If there's an unlabeled summary from a previous section, and + # include_unlabeled is True, put it into uncategorized_changes. + if include_unlabeled and unlabeled_summary and not changes_ignore: + if auto_merge_pattern.match(unlabeled_summary): + # 1. Parse revision numbers from the first line + merged_revisions = [ + int(x) for x in + re.compile(r'(?<=\br)\d+\b').findall(unlabeled_summary) + ] + # TODO pass each revnum in MERGED_REVISIONS through this + # logic, in order to extract CHANGES_PREFIX_PATTERN + # and CHANGES_SUFFIX_PATTERN lines from the trunk log + # message. + + # 2. Parse the STATUS entry + this_log_message = log_messages_dict[revision] + status_paragraph = this_log_message.split('\n\n')[2] + logsummary = \ + backport.status.StatusEntry(status_paragraph).logsummary + add_to_changes_dict(changes_dict, None, None, + ' '.join(logsummary), revision) + else: + add_to_changes_dict(changes_dict, None, None, + unlabeled_summary, revision) + revision = -1 + got_firstline = False + unlabeled_summary = None + changes_ignore = False + audience = None + section = None + message = None + continue + + revmatch = revline_pattern.match(line) + if revmatch and (revision == -1): + # A revision line: get the revision number + revision = int(revmatch.group(1)) + logging.debug('Changelog processing revision r%d' % revision) + continue + + if line.strip() == '': + # Skip empty / whitespace lines + continue + + if not got_firstline: + got_firstline = True + if (not re.search(r'status|changes|post-release housekeeping|follow-up|^\*', + line, re.IGNORECASE) + and not changes_prefix_pattern.match(line) + and not changes_suffix_pattern.match(line)): + unlabeled_summary = line + + if re.search(r'\[(c:)?(skip|ignore)\]', line, re.IGNORECASE): + changes_ignore = True + + prefix_match = changes_prefix_pattern.match(line) + if prefix_match: + audience = prefix_match.group(1) + section = prefix_match.group(2) + message = prefix_match.group(3) + add_to_changes_dict(changes_dict, audience, section, message, revision) + + suffix_match = changes_suffix_pattern.match(line) + if suffix_match: + message = suffix_match.group(1) + audience = suffix_match.group(2) + section = suffix_match.group(3) + add_to_changes_dict(changes_dict, audience, section, message, revision) + + # Output the sorted changelog entries + # 1) Uncategorized changes + print_section(changes_dict, None, None, None) + print + # 2) User-visible changes + print(' User-visible changes:') + print_section(changes_dict, 'U', None, None) + print_section(changes_dict, 'U', 'general', 'General') + print_section(changes_dict, 'U', 'major', 'Major new features') + print_section(changes_dict, 'U', 'minor', 'Minor new features and improvements') + print_section(changes_dict, 'U', 'client', 'Client-side bugfixes', mandatory=True) + print_section(changes_dict, 'U', 'server', 'Server-side bugfixes', mandatory=True) + print_section(changes_dict, 'U', 'clientserver', 'Client-side and server-side bugfixes') + print_section(changes_dict, 'U', 'other', 'Other tool improvements and bugfixes') + print_section(changes_dict, 'U', 'bindings', 'Bindings bugfixes', mandatory=True) + print + # 3) Developer-visible changes + print(' Developer-visible changes:') + print_section(changes_dict, 'D', None, None) + print_section(changes_dict, 'D', 'general', 'General', mandatory=True) + print_section(changes_dict, 'D', 'api', 'API changes', mandatory=True) + print_section(changes_dict, 'D', 'bindings', 'Bindings') + #---------------------------------------------------------------------- # Main entry point for argument parsing and handling @@ -1184,13 +1679,25 @@ def main(): parser = argparse.ArgumentParser( description='Create an Apache Subversion release.') parser.add_argument('--clean', action='store_true', default=False, - help='Remove any directories previously created by %(prog)s') + help='''Remove any directories previously created by %(prog)s, + including the 'prefix' dir, the 'temp' dir, and the + default or specified target dir.''') parser.add_argument('--verbose', action='store_true', default=False, help='Increase output verbosity') parser.add_argument('--base-dir', default=os.getcwd(), help='''The directory in which to create needed files and folders. The default is the current working directory.''') + parser.add_argument('--target', + help='''The full path to the directory containing + release artifacts. Default: <BASE_DIR>/deploy''') + parser.add_argument('--branch', + help='''The branch to base the release on, + as a path relative to ^/subversion/. + Default: 'branches/MAJOR.MINOR.x'.''') + parser.add_argument('--username', + help='Username for committing to ' + svn_repos + + ' or ' + dist_repos + '.') subparsers = parser.add_subparsers(title='subcommands') # Setup the parser for the build-env subcommand @@ -1208,6 +1715,40 @@ def main(): help='''Attempt to use existing build dependencies before downloading and building a private set.''') + # Setup the parser for the create-release-branch subcommand + subparser = subparsers.add_parser('create-release-branch', + help='''Create a minor release branch: branch from trunk, + update version numbers on trunk, create status + file on branch, update backport bot, + update buildbot config.''') + subparser.set_defaults(func=create_release_branch) + subparser.add_argument('version', type=Version, + help='''A version number to indicate the branch, such as + '1.7.0' (the '.0' is required).''') + subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')), + nargs='?', default=None, + help='''The trunk revision number to base the branch on. + Default is HEAD.''') + subparser.add_argument('--dry-run', action='store_true', default=False, + help='Avoid committing any changes to repositories.') + + # Setup the parser for the create-release-branch subcommand + subparser = subparsers.add_parser('write-release-notes', + help='''Write a template release-notes file.''') + subparser.set_defaults(func=write_release_notes) + subparser.add_argument('version', type=Version, + help='''A version number to indicate the branch, such as + '1.7.0' (the '.0' is required).''') + subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')), + nargs='?', default=None, + help='''The trunk revision number to base the branch on. + Default is HEAD.''') + subparser.add_argument('--edit-html-file', + help='''Write the template release-notes to this file, + and update 'index.html' in the same directory.''') + subparser.add_argument('--dry-run', action='store_true', default=False, + help='Avoid committing any changes to repositories.') + # Setup the parser for the roll subcommand subparser = subparsers.add_parser('roll', help='''Create the release artifacts.''') @@ -1216,9 +1757,6 @@ def main(): help='''The release label, such as '1.7.0-alpha1'.''') subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')), help='''The revision number to base the release on.''') - subparser.add_argument('--branch', - help='''The branch to base the release on, - relative to ^/subversion/.''') subparser.add_argument('--patches', help='''The path to the directory containing patches.''') @@ -1228,9 +1766,10 @@ def main(): subparser.set_defaults(func=sign_candidates) subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') - subparser.add_argument('--target', - help='''The full path to the directory containing - release artifacts.''') + subparser.add_argument('--userid', + help='''The (optional) USER-ID specifying the key to be + used for signing, such as '110B1C95' (Key-ID). If + omitted, uses the default key.''') # Setup the parser for the post-candidates subcommand subparser = subparsers.add_parser('post-candidates', @@ -1239,55 +1778,55 @@ def main(): subparser.set_defaults(func=post_candidates) subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') - subparser.add_argument('--username', - help='''Username for ''' + dist_repos + '''.''') - subparser.add_argument('--target', - help='''The full path to the directory containing - release artifacts.''') # Setup the parser for the create-tag subcommand subparser = subparsers.add_parser('create-tag', - help='''Create the release tag.''') - subparser.set_defaults(func=create_tag) + help='''Create the release tag and, if not a prerelease + version, bump version numbers on the branch.''') + subparser.set_defaults(func=create_tag_and_bump_versions) + subparser.add_argument('version', type=Version, + help='''The release label, such as '1.7.0-alpha1'.''') + subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')), + help='''The revision number to base the release on.''') + + # Setup the parser for the bump-versions-on-branch subcommand + subparser = subparsers.add_parser('bump-versions-on-branch', + help='''Bump version numbers on branch.''') + subparser.set_defaults(func=bump_versions_on_branch) subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')), help='''The revision number to base the release on.''') - subparser.add_argument('--branch', - help='''The branch to base the release on, - relative to ^/subversion/.''') - subparser.add_argument('--username', - help='''Username for ''' + secure_repos + '''.''') - subparser.add_argument('--target', - help='''The full path to the directory containing - release artifacts.''') # The clean-dist subcommand subparser = subparsers.add_parser('clean-dist', - help='''Clean the distribution directory (and mirrors) of - all but the most recent MAJOR.MINOR release.''') + help=clean_dist.__doc__.split('\n\n')[0]) subparser.set_defaults(func=clean_dist) subparser.add_argument('--dist-dir', help='''The directory to clean.''') - subparser.add_argument('--username', - help='''Username for ''' + dist_repos + '''.''') # The move-to-dist subcommand subparser = subparsers.add_parser('move-to-dist', - help='''Move candiates and signatures from the temporary + help='''Move candidates and signatures from the temporary release dev location to the permanent distribution directory.''') subparser.set_defaults(func=move_to_dist) subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') - subparser.add_argument('--username', - help='''Username for ''' + dist_repos + '''.''') # The write-news subcommand subparser = subparsers.add_parser('write-news', help='''Output to stdout template text for use in the news section of the Subversion website.''') subparser.set_defaults(func=write_news) + subparser.add_argument('--announcement-url', + help='''The URL to the archived announcement email.''') + subparser.add_argument('--news-release-date', + help='''The release date for the news, as YYYY-MM-DD. + Default: today.''') + subparser.add_argument('--edit-html-file', + help='''Insert the text into this file + news.html, index.html).''') subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') @@ -1299,9 +1838,6 @@ def main(): subparser.add_argument('--security', action='store_true', default=False, help='''The release being announced includes security fixes.''') - subparser.add_argument('--target', - help='''The full path to the directory containing - release artifacts.''') subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') @@ -1310,9 +1846,6 @@ def main(): help='''Output to stdout template text for the download table for subversion.apache.org''') subparser.set_defaults(func=write_downloads) - subparser.add_argument('--target', - help='''The full path to the directory containing - release artifacts.''') subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') @@ -1323,9 +1856,6 @@ def main(): subparser.set_defaults(func=check_sigs) subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') - subparser.add_argument('--target', - help='''The full path to the directory containing - release artifacts.''') # get-keys subparser = subparsers.add_parser('get-keys', @@ -1338,6 +1868,25 @@ def main(): separate subcommand.''') subparser.set_defaults(func=cleanup) + # write-changelog + subparser = subparsers.add_parser('write-changelog', + help='''Output to stdout changelog entries parsed from + commit messages, optionally labeled with a category + like [U:client], [D:api], [U], ...''') + subparser.set_defaults(func=write_changelog) + subparser.add_argument('previous', + help='''The "previous" branch or tag, relative to + ^/subversion/, to compare "branch" against.''') + subparser.add_argument('--include-unlabeled-summaries', + dest='include_unlabeled', + action='store_true', default=False, + help='''Include summary lines that do not have a changes + label, unless an explicit [c:skip] or [c:ignore] + is part of the commit message (except if the + summary line contains 'STATUS', 'CHANGES', + 'Post-release housekeeping', 'Follow-up' or starts + with '*').''') + # Parse the arguments args = parser.parse_args()
Modified: subversion/branches/addremove/tools/dist/security/_gnupg.py URL: http://svn.apache.org/viewvc/subversion/branches/addremove/tools/dist/security/_gnupg.py?rev=1878061&r1=1878060&r2=1878061&view=diff ============================================================================== --- subversion/branches/addremove/tools/dist/security/_gnupg.py (original) +++ subversion/branches/addremove/tools/dist/security/_gnupg.py Sat May 23 14:16:56 2020 @@ -1,9 +1,9 @@ # Copyright (c) 2008-2014 by Vinay Sajip. # All rights reserved. -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: -# +# # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, Modified: subversion/branches/addremove/tools/dist/security/parser.py URL: http://svn.apache.org/viewvc/subversion/branches/addremove/tools/dist/security/parser.py?rev=1878061&r1=1878060&r2=1878061&view=diff ============================================================================== --- subversion/branches/addremove/tools/dist/security/parser.py (original) +++ subversion/branches/addremove/tools/dist/security/parser.py Sat May 23 14:16:56 2020 @@ -50,9 +50,16 @@ class Notification(object): CULPRIT_SERVER = 'server' CULPRIT_CLIENT = 'client' - __CULPRITS = ((CULPRIT_SERVER, CULPRIT_CLIENT, - (CULPRIT_SERVER, CULPRIT_CLIENT), - (CULPRIT_CLIENT, CULPRIT_SERVER))) + # For compatibility, 'client' and 'server' may be specified either with + # or without a tuple. + __CULPRITS = ( + CULPRIT_SERVER, + CULPRIT_CLIENT, + (CULPRIT_SERVER,) + (CULPRIT_CLIENT,) + (CULPRIT_SERVER, CULPRIT_CLIENT), + (CULPRIT_CLIENT, CULPRIT_SERVER), + ) def __init__(self, basedir, tracking_id, title, culprit, advisory, patches): Modified: subversion/branches/addremove/tools/dist/templates/download.ezt URL: http://svn.apache.org/viewvc/subversion/branches/addremove/tools/dist/templates/download.ezt?rev=1878061&r1=1878060&r2=1878061&view=diff ============================================================================== --- subversion/branches/addremove/tools/dist/templates/download.ezt (original) +++ subversion/branches/addremove/tools/dist/templates/download.ezt Sat May 23 14:16:56 2020 @@ -2,16 +2,14 @@ <table class="centered"> <tr> <th>File</th> - <th>Checksum (SHA1)</th> <th>Checksum (SHA512)</th> <th>Signatures</th> + <th>PGP Public Keys</th> </tr> [for fileinfo]<tr> <td><a href="[[]preferred]subversion/[fileinfo.filename]">[fileinfo.filename]</a></td> - <td class="checksum">[fileinfo.sha1]</td> - <!-- The sha512 line does not have a class="checksum" since the link needn't - be rendered in monospace. --> - <td>[<a href="http://www.apache.org/dist/subversion/[fileinfo.filename].sha512">SHA-512</a>]</td> - <td>[<a href="http://www.apache.org/dist/subversion/[fileinfo.filename].asc">PGP</a>]</td> + <td>[<a href="https://www.apache.org/dist/subversion/[fileinfo.filename].sha512">SHA-512</a>]</td> + <td>[<a href="https://www.apache.org/dist/subversion/[fileinfo.filename].asc">PGP signatures</a>]</td> + <td>[<a href="https://www.apache.org/dist/subversion/subversion-[version].KEYS">PGP keyring</a>]</td> </tr>[end] </table> Modified: subversion/branches/addremove/tools/dist/templates/rc-news.ezt URL: http://svn.apache.org/viewvc/subversion/branches/addremove/tools/dist/templates/rc-news.ezt?rev=1878061&r1=1878060&r2=1878061&view=diff ============================================================================== --- subversion/branches/addremove/tools/dist/templates/rc-news.ezt (original) +++ subversion/branches/addremove/tools/dist/templates/rc-news.ezt Sat May 23 14:16:56 2020 @@ -8,10 +8,10 @@ release is not intended for production use, but is provided as a milestone to encourage wider testing and feedback from intrepid users and maintainers. Please see the - <a href="">release + <a href="[announcement_url]">release announcement</a> for more information about this release, and the <a href="/docs/release-notes/[major-minor].html">release notes</a> and - <a href="http://svn.apache.org/repos/asf/subversion/tags/[version]/CHANGES"> + <a href="https://svn.apache.org/repos/asf/subversion/tags/[version]/CHANGES"> change log</a> for information about what will eventually be in the [version_base] release.</p> Modified: subversion/branches/addremove/tools/dist/templates/rc-release-ann.ezt URL: http://svn.apache.org/viewvc/subversion/branches/addremove/tools/dist/templates/rc-release-ann.ezt?rev=1878061&r1=1878060&r2=1878061&view=diff ============================================================================== --- subversion/branches/addremove/tools/dist/templates/rc-release-ann.ezt (original) +++ subversion/branches/addremove/tools/dist/templates/rc-release-ann.ezt Sat May 23 14:16:56 2020 @@ -1,16 +1,13 @@ From: ...@apache.org To: annou...@subversion.apache.org, us...@subversion.apache.org, d...@subversion.apache.org, annou...@apache.org +Reply-To: us...@subversion.apache.org Subject: [[]ANNOUNCE] Apache Subversion [version] released I'm happy to announce the release of Apache Subversion [version]. Please choose the mirror closest to you by visiting: - http://subversion.apache.org/download.cgi#[anchor] + https://subversion.apache.org/download.cgi#[anchor] -The SHA1 checksums are: - -[for sha1info] [sha1info.sha1] [sha1info.filename] -[end] SHA-512 checksums are available at: https://www.apache.org/dist/subversion/subversion-[version].tar.bz2.sha512 @@ -19,13 +16,17 @@ SHA-512 checksums are available at: PGP Signatures are available at: - http://www.apache.org/dist/subversion/subversion-[version].tar.bz2.asc - http://www.apache.org/dist/subversion/subversion-[version].tar.gz.asc - http://www.apache.org/dist/subversion/subversion-[version].zip.asc + https://www.apache.org/dist/subversion/subversion-[version].tar.bz2.asc + https://www.apache.org/dist/subversion/subversion-[version].tar.gz.asc + https://www.apache.org/dist/subversion/subversion-[version].zip.asc For this release, the following people have provided PGP signatures: [siginfo] +These public keys are available at: + + https://www.apache.org/dist/subversion/subversion-[version].KEYS + This is a pre-release for what will eventually become version [major-minor-patch] of the Apache Subversion open source version control system. It may contain known issues, a complete list of [major-minor-patch]-blocking issues can be found @@ -57,13 +58,18 @@ end users please. Release notes for the [major-minor].x release series may be found at: - http://subversion.apache.org/docs/release-notes/[major-minor].html + https://subversion.apache.org/docs/release-notes/[major-minor].html You can find the list of changes between [version] and earlier versions at: - http://svn.apache.org/repos/asf/subversion/tags/[version]/CHANGES + https://svn.apache.org/repos/asf/subversion/tags/[version]/CHANGES Questions, comments, and bug reports to us...@subversion.apache.org. Thanks, - The Subversion Team + +-- +To unsubscribe, please see: + + https://subversion.apache.org/mailing-lists.html#unsubscribing Modified: subversion/branches/addremove/tools/dist/templates/stable-news.ezt URL: http://svn.apache.org/viewvc/subversion/branches/addremove/tools/dist/templates/stable-news.ezt?rev=1878061&r1=1878060&r2=1878061&view=diff ============================================================================== --- subversion/branches/addremove/tools/dist/templates/stable-news.ezt (original) +++ subversion/branches/addremove/tools/dist/templates/stable-news.ezt Sat May 23 14:16:56 2020 @@ -10,10 +10,10 @@ [else] This is the most complete release of the [major-minor].x line to date, and we encourage all users to upgrade as soon as reasonable. [end] Please see the - <a href="" + <a href="[announcement_url]" >release announcement</a> and the - <a href="http://svn.apache.org/repos/asf/subversion/tags/[version]/CHANGES" - >change log</a> for more information about this release.</p> + <a href="/docs/release-notes/[major-minor]" + >release notes</a> for more information about this release.</p> <p>To get this release from the nearest mirror, please visit our <a href="/download.cgi#[anchor]">download page</a>.</p> Modified: subversion/branches/addremove/tools/dist/templates/stable-release-ann.ezt URL: http://svn.apache.org/viewvc/subversion/branches/addremove/tools/dist/templates/stable-release-ann.ezt?rev=1878061&r1=1878060&r2=1878061&view=diff ============================================================================== --- subversion/branches/addremove/tools/dist/templates/stable-release-ann.ezt (original) +++ subversion/branches/addremove/tools/dist/templates/stable-release-ann.ezt Sat May 23 14:16:56 2020 @@ -1,5 +1,6 @@ From: ...@apache.org To: annou...@subversion.apache.org, us...@subversion.apache.org, d...@subversion.apache.org, annou...@apache.org +Reply-To: us...@subversion.apache.org [if-any security]Cc: secur...@apache.org, oss-secur...@lists.openwall.com, bugt...@securityfocus.com [end][if-any security]Subject: [[]SECURITY][[]ANNOUNCE] Apache Subversion [version] released [else]Subject: [[]ANNOUNCE] Apache Subversion [version] released @@ -7,7 +8,7 @@ To: annou...@subversion.apache.org, user I'm happy to announce the release of Apache Subversion [version]. Please choose the mirror closest to you by visiting: - http://subversion.apache.org/download.cgi#[anchor] + https://subversion.apache.org/download.cgi#[anchor] [if-any dot-zero] This is a stable feature release of the Apache Subversion open source version control system. @@ -18,10 +19,6 @@ open source version control system. This is a stable bugfix release of the Apache Subversion open source version control system. [end][end] -The SHA1 checksums are: - -[for sha1info] [sha1info.sha1] [sha1info.filename] -[end] SHA-512 checksums are available at: https://www.apache.org/dist/subversion/subversion-[version].tar.bz2.sha512 @@ -30,22 +27,31 @@ SHA-512 checksums are available at: PGP Signatures are available at: - http://www.apache.org/dist/subversion/subversion-[version].tar.bz2.asc - http://www.apache.org/dist/subversion/subversion-[version].tar.gz.asc - http://www.apache.org/dist/subversion/subversion-[version].zip.asc + https://www.apache.org/dist/subversion/subversion-[version].tar.bz2.asc + https://www.apache.org/dist/subversion/subversion-[version].tar.gz.asc + https://www.apache.org/dist/subversion/subversion-[version].zip.asc For this release, the following people have provided PGP signatures: [siginfo] +These public keys are available at: + + https://www.apache.org/dist/subversion/subversion-[version].KEYS + Release notes for the [major-minor].x release series may be found at: - http://subversion.apache.org/docs/release-notes/[major-minor].html + https://subversion.apache.org/docs/release-notes/[major-minor].html You can find the list of changes between [version] and earlier versions at: - http://svn.apache.org/repos/asf/subversion/tags/[version]/CHANGES + https://svn.apache.org/repos/asf/subversion/tags/[version]/CHANGES Questions, comments, and bug reports to us...@subversion.apache.org. Thanks, - The Subversion Team + +-- +To unsubscribe, please see: + + https://subversion.apache.org/mailing-lists.html#unsubscribing Modified: subversion/branches/addremove/tools/examples/ExampleAuthn.java URL: http://svn.apache.org/viewvc/subversion/branches/addremove/tools/examples/ExampleAuthn.java?rev=1878061&r1=1878060&r2=1878061&view=diff ============================================================================== --- subversion/branches/addremove/tools/examples/ExampleAuthn.java (original) +++ subversion/branches/addremove/tools/examples/ExampleAuthn.java Sat May 23 14:16:56 2020 @@ -68,8 +68,11 @@ public class ExampleAuthn { SSLServerCertFailures failures, SSLServerCertInfo info, boolean maySave) { - System.out.println("sslServerTrustPrompt not implemented!"); - return SSLServerTrustResult.acceptTemporarily(); + System.out.println("sslServerTrustPrompt"); + System.out.println("(r)eject or (t)emporary?"); + String s = System.console().readLine(); + return s.equals("t") ? SSLServerTrustResult.acceptTemporarily() + : SSLServerTrustResult.reject(); } public SSLClientCertResult Modified: subversion/branches/addremove/tools/examples/get-location-segments.py URL: http://svn.apache.org/viewvc/subversion/branches/addremove/tools/examples/get-location-segments.py?rev=1878061&r1=1878060&r2=1878061&view=diff ============================================================================== --- subversion/branches/addremove/tools/examples/get-location-segments.py (original) +++ subversion/branches/addremove/tools/examples/get-location-segments.py Sat May 23 14:16:56 2020 @@ -73,7 +73,7 @@ def parse_args(args): def prompt_func_ssl_unknown_cert(realm, failures, cert_info, may_save, pool): - print( "The certficate details are as follows:") + print( "The certificate details are as follows:") print("--------------------------------------") print("Issuer : " + str(cert_info.issuer_dname)) print("Hostname : " + str(cert_info.hostname)) Modified: subversion/branches/addremove/tools/examples/walk-config-auth.py URL: http://svn.apache.org/viewvc/subversion/branches/addremove/tools/examples/walk-config-auth.py?rev=1878061&r1=1878060&r2=1878061&view=diff ============================================================================== --- subversion/branches/addremove/tools/examples/walk-config-auth.py (original) +++ subversion/branches/addremove/tools/examples/walk-config-auth.py Sat May 23 14:16:56 2020 @@ -18,7 +18,7 @@ credentials found. """ % (sys.argv[0])) sys.exit(0) -config_dir = svn.core.svn_config_get_user_config_path(None, '') +config_dir = svn.core.svn_config_get_user_config_path(None, None) if len(sys.argv) > 1: config_dir = sys.argv[1] Modified: subversion/branches/addremove/tools/hook-scripts/mailer/mailer.conf.example URL: http://svn.apache.org/viewvc/subversion/branches/addremove/tools/hook-scripts/mailer/mailer.conf.example?rev=1878061&r1=1878060&r2=1878061&view=diff ============================================================================== --- subversion/branches/addremove/tools/hook-scripts/mailer/mailer.conf.example (original) +++ subversion/branches/addremove/tools/hook-scripts/mailer/mailer.conf.example Sat May 23 14:16:56 2020 @@ -23,6 +23,11 @@ # This option specifies the hostname for delivery via SMTP. #smtp_hostname = localhost +# This option specifies the TCP port number to connect for SMTP. +# If it is not specified, 25 is used for SMTP and 465 is used for +# SMTP-Over-SSL by default. +#smtp_port = 25 + # Username and password for SMTP servers requiring authorisation. #smtp_username = example #smtp_password = example