Repository: kudu Updated Branches: refs/heads/master cc4e3957b -> 3bfcbd8ad
python3 support for some build tools Adds python3 support to the following files while retaining python2 compatibility: - gen_version_info.py - kudu_util.py, - push_to_asf.py Some python build tools are still known not to be python3 compatible, including: - build_source_release.py - dist_test.py - trigger_gerrit.py. These haven't been fixed either because they use urllib, which I can't figure out how to make 2 and 3 compatible, or because I don't have a dist-test compatible system with python 3 available. Change-Id: I547d870a54c7d396d2706d63f65844df01ff7c57 Reviewed-on: http://gerrit.cloudera.org:8080/9492 Reviewed-by: Alexey Serbin <[email protected]> Reviewed-by: Adar Dembo <[email protected]> Tested-by: Dan Burkert <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/249af9a1 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/249af9a1 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/249af9a1 Branch: refs/heads/master Commit: 249af9a101462bb0ca840e20a3dc8b4a76e929fd Parents: cc4e395 Author: Dan Burkert <[email protected]> Authored: Mon Mar 5 09:22:46 2018 -0800 Committer: Dan Burkert <[email protected]> Committed: Mon Mar 5 20:28:37 2018 +0000 ---------------------------------------------------------------------- build-support/build_source_release.py | 57 +++++++++++------------ build-support/gen_version_info.py | 14 +++--- build-support/kudu_util.py | 15 +++++-- build-support/push_to_asf.py | 72 +++++++++++++++--------------- 4 files changed, 84 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/249af9a1/build-support/build_source_release.py ---------------------------------------------------------------------- diff --git a/build-support/build_source_release.py b/build-support/build_source_release.py index bc33019..6d922d4 100755 --- a/build-support/build_source_release.py +++ b/build-support/build_source_release.py @@ -38,9 +38,8 @@ def check_repo_not_dirty(): shell=True) != 0 if not dirty_repo: return - print "The repository does not appear to be clean." - print Colors.RED + "The source release will not include your local changes." + \ - Colors.RESET + print("The repository does not appear to be clean.") + print(Colors.RED + "The source release will not include your local changes." + Colors.RESET) if not confirm_prompt("Continue?"): sys.exit(1) @@ -50,23 +49,22 @@ def check_no_local_commits(): Check that there are no local commits which haven't been pushed to the upstream repo via Jenkins. """ - upstream_commit = check_output(GET_UPSTREAM_COMMIT_SCRIPT).strip() - cur_commit = check_output(["git", "rev-parse", "HEAD"]).strip() + upstream_commit = check_output(GET_UPSTREAM_COMMIT_SCRIPT).strip().decode('utf-8') + cur_commit = check_output(["git", "rev-parse", "HEAD"]).strip().decode('utf-8') if upstream_commit == cur_commit: return - print "The repository appears to have local commits:" + print("The repository appears to have local commits:") subprocess.check_call(["git", "log", "--oneline", "%s..HEAD" % upstream_commit]) - print Colors.RED + "This should not be an official release!" + \ - Colors.RESET + print(Colors.RED + "This should not be an official release!" + Colors.RESET) if not confirm_prompt("Continue?"): sys.exit(1) def get_version_number(): """ Return the current version number of Kudu. """ - return file(os.path.join(ROOT, "version.txt")).read().strip() + return open(os.path.join(ROOT, "version.txt")).read().strip() def create_tarball(): @@ -75,12 +73,12 @@ def create_tarball(): if not os.path.exists(build_dir): os.path.makedirs(build_dir) tarball_path = os.path.join(build_dir, artifact_name + ".tar.gz") - print "Exporting source tarball..." + print("Exporting source tarball...") subprocess.check_output(["git", "archive", "--prefix=%s/" % artifact_name, "--output=%s" % tarball_path, "HEAD"]) - print Colors.GREEN + "Generated tarball:\t" + Colors.RESET, tarball_path + print(Colors.GREEN + "Generated tarball:\t" + Colors.RESET, tarball_path) return tarball_path @@ -91,18 +89,18 @@ def sign_tarball(tarball_path): email = get_my_email() if not email.endswith("@apache.org"): - print Colors.YELLOW, "Your email address for the repository is not an @apache.org address." - print "Release signatures should typically be signed by committers with @apache.org GPG keys." - print Colors.RESET, + print(Colors.YELLOW, "Your email address for the repository is not an @apache.org address.") + print("Release signatures should typically be signed by committers with @apache.org GPG keys.") + print(Colors.RESET) if not confirm_prompt("Continue?"): return try: subprocess.check_call(["gpg", "--detach-sign", "--armor", "-u", email, tarball_path]) except subprocess.CalledProcessError: - print Colors.RED + "GPG signing failed. Artifact will not be signed." + Colors.RESET + print(Colors.RED + "GPG signing failed. Artifact will not be signed." + Colors.RESET) return - print Colors.GREEN + "Generated signature:\t" + Colors.RESET, tarball_path + ".asc" + print(Colors.GREEN + "Generated signature:\t" + Colors.RESET, tarball_path + ".asc") def checksum_file(summer, path): @@ -110,9 +108,12 @@ def checksum_file(summer, path): Calculates the checksum of the file 'path' using the provided hashlib digest implementation. Returns the hex form of the digest. """ - with file(path, "rb") as f: + with open(path, "rb") as f: # Read the file in 4KB chunks until EOF. - for chunk in iter(lambda: f.read(4096), ""): + while True: + chunk = f.read(4096) + if not chunk: + break summer.update(chunk) return summer.hexdigest() @@ -129,9 +130,9 @@ def gen_checksum_files(tarball_path): for hash_func, extension in hashes: digest = checksum_file(hash_func(), tarball_path) path = tarball_path + "." + extension - with file(path, "w") as f: - print >>f, "%s\t%s" % (digest, os.path.basename(tarball_path)) - print Colors.GREEN + ("Generated %s:\t" % extension) + Colors.RESET, path + with open(path, "w") as f: + f.write("%s\t%s\n" % (digest, os.path.basename(tarball_path))) + print(Colors.GREEN + ("Generated %s:\t" % extension) + Colors.RESET, path) def run_rat(tarball_path): @@ -151,25 +152,25 @@ def run_rat(tarball_path): try: rat_jar_dest = "%s/%s" % (tmpdir_path, os.path.basename(rat_url)) - print "> Downloading RAT jar from " + rat_url + print("> Downloading RAT jar from " + rat_url) urllib.urlretrieve(rat_url, rat_jar_dest) - print "> Running RAT..." + print("> Running RAT...") xml = subprocess.check_output(["java", "-jar", rat_jar_dest, "-x", tarball_path]) rat_report_dest = "%s/%s" % (tmpdir_path, "rat_report.xml") with open(rat_report_dest, "w") as f: f.write(xml) - print "> Parsing RAT report..." + print("> Parsing RAT report...") rat_report_result = subprocess.check_output( ["./build-support/release/check-rat-report.py", "./build-support/release/rat_exclude_files.txt", rat_report_dest], stderr=subprocess.STDOUT) - print Colors.GREEN + "RAT: LICENSES APPROVED" + Colors.RESET + print(Colors.GREEN + "RAT: LICENSES APPROVED" + Colors.RESET) except subprocess.CalledProcessError as e: - print Colors.RED + "RAT: LICENSES NOT APPROVED" + Colors.RESET - print e.output + print(Colors.RED + "RAT: LICENSES NOT APPROVED" + Colors.RESET) + print(e.output) raise e finally: shutil.rmtree(tmpdir_path) @@ -185,7 +186,7 @@ def main(): sign_tarball(tarball_path) run_rat(tarball_path) - print Colors.GREEN + "Release successfully generated!" + Colors.RESET + print(Colors.GREEN + "Release successfully generated!" + Colors.RESET) print http://git-wip-us.apache.org/repos/asf/kudu/blob/249af9a1/build-support/gen_version_info.py ---------------------------------------------------------------------- diff --git a/build-support/gen_version_info.py b/build-support/gen_version_info.py index 5c2c181..c0cf6d1 100755 --- a/build-support/gen_version_info.py +++ b/build-support/gen_version_info.py @@ -20,11 +20,11 @@ # This script generates a header file which contains definitions # for the current Kudu build (eg timestamp, git hash, etc) +import hashlib import logging import optparse import os import re -import sha import subprocess import sys import time @@ -39,7 +39,7 @@ def output_up_to_date(path, id_hash): """ if not os.path.exists(path): return False - f = file(path).read() + f = open(path).read() m = re.search("id_hash=(\w+)", f) if not m: return False @@ -99,16 +99,16 @@ def main(): # timestamp. We put this hash in a comment, and use it to check whether to # re-generate the file. If it hasn't changed since a previous run, we don't # re-write the file. This avoids having to rebuild all binaries on every build. - identifying_hash = sha.sha(repr((git_hash, hostname, username, - clean_repo, build_id))).hexdigest() + identifying_hash = hashlib.sha1(repr((git_hash, hostname, username, + clean_repo, build_id)).encode('utf-8')).hexdigest() if output_up_to_date(output_path, identifying_hash): return 0 d = os.path.dirname(output_path) if not os.path.exists(d): os.makedirs(d) - with file(output_path, "w") as f: - print >>f, """ + with open(output_path, "w") as f: + f.write(""" // THIS FILE IS AUTO-GENERATED! DO NOT EDIT! // // id_hash=%(identifying_hash)s @@ -124,7 +124,7 @@ def main(): #define KUDU_BUILD_TYPE "%(build_type)s" #define KUDU_VERSION_STRING "%(version_string)s" #endif -""" % locals() +""" % locals()) return 0 if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/kudu/blob/249af9a1/build-support/kudu_util.py ---------------------------------------------------------------------- diff --git a/build-support/kudu_util.py b/build-support/kudu_util.py index 7945da5..da4dbe6 100644 --- a/build-support/kudu_util.py +++ b/build-support/kudu_util.py @@ -24,6 +24,12 @@ import os import subprocess import sys +# Alias raw_input() to input() in Python 2. +try: + input = raw_input +except NameError: + pass + class Colors(object): """ ANSI color codes. """ @@ -63,14 +69,14 @@ def confirm_prompt(prompt): if the user confirms. """ while True: - print prompt, "[Y/n]:", + print(prompt + "[Y/n]:"), if not os.isatty(sys.stdout.fileno()): - print "Not running interactively. Assuming 'N'." + print("Not running interactively. Assuming 'N'.") return False pass - r = raw_input().strip().lower() + r = input().strip().lower() if r in ['y', 'yes', '']: return True elif r in ['n', 'no']: @@ -79,4 +85,5 @@ def confirm_prompt(prompt): def get_my_email(): """ Return the email address in the user's git config. """ - return check_output(['git', 'config', '--get', 'user.email']).strip() + return check_output(['git', 'config', '--get', + 'user.email']).strip().decode('utf-8') http://git-wip-us.apache.org/repos/asf/kudu/blob/249af9a1/build-support/push_to_asf.py ---------------------------------------------------------------------- diff --git a/build-support/push_to_asf.py b/build-support/push_to_asf.py index f84a261..f92a3b3 100755 --- a/build-support/push_to_asf.py +++ b/build-support/push_to_asf.py @@ -32,6 +32,8 @@ # changes to be pushed, warns them if they are pushing code for # which they weren't the committer, and performs the actual push. +from __future__ import print_function + import logging import optparse import re @@ -59,15 +61,15 @@ def check_apache_remote(): Otherwise, exits with an error message. """ try: - url = check_output(['git', 'config', '--local', '--get', 'remote.apache.url']).strip() + url = check_output(['git', 'config', '--local', '--get', 'remote.apache.url']).strip().decode('utf-8') except subprocess.CalledProcessError: - print >>sys.stderr, "No remote named 'apache'. Please set one up, for example with: " - print >>sys.stderr, " git remote add apache", APACHE_REPO + print("No remote named 'apache'. Please set one up, for example with: ", file=sys.stderr) + print(" git remote add apache", APACHE_REPO, file=sys.stderr) sys.exit(1) if url != APACHE_REPO: - print >>sys.stderr, "Unexpected URL for remote 'apache'." - print >>sys.stderr, " Got: ", url - print >>sys.stderr, " Expected:", APACHE_REPO + print("Unexpected URL for remote 'apache'.", file=sys.stderr) + print(" Got: ", url, file=sys.stderr) + print(" Expected:", APACHE_REPO, file=sys.stderr) sys.exit(1) @@ -77,15 +79,15 @@ def check_gerrit_remote(): Otherwise, exits with an error message. """ try: - url = check_output(['git', 'config', '--local', '--get', 'remote.gerrit.url']).strip() + url = check_output(['git', 'config', '--local', '--get', 'remote.gerrit.url']).strip().decode('utf-8') except subprocess.CalledProcessError: - print >>sys.stderr, "No remote named 'gerrit'. Please set one up following " - print >>sys.stderr, "the contributor guide." + print("No remote named 'gerrit'. Please set one up following ", file=sys.stderr) + print("the contributor guide.", file=sys.stderr) sys.exit(1) if not GERRIT_URL_RE.match(url): - print >>sys.stderr, "Unexpected URL for remote 'gerrit'." - print >>sys.stderr, " Got: ", url - print >>sys.stderr, " Expected to find host '%s' in the URL" % GERRIT_HOST + print("Unexpected URL for remote 'gerrit'.", file=sys.stderr) + print(" Got: ", url, file=sys.stderr) + print(" Expected to find host '%s' in the URL" % GERRIT_HOST, file=sys.stderr) sys.exit(1) @@ -98,7 +100,7 @@ def fetch(remote): def get_branches(remote): """ Fetch a dictionary mapping branch name to SHA1 hash from the given remote. """ - out = check_output(["git", "ls-remote", remote, "refs/heads/*"]) + out = check_output(["git", "ls-remote", remote, "refs/heads/*"]).decode('utf-8') ret = {} for l in out.splitlines(): sha, ref = l.split("\t") @@ -110,7 +112,7 @@ def get_branches(remote): def rev_parse(rev): """Run git rev-parse, returning the sha1, or None if not found""" try: - return check_output(['git', 'rev-parse', rev], stderr=subprocess.STDOUT).strip() + return check_output(['git', 'rev-parse', rev], stderr=subprocess.STDOUT).strip().decode('utf-8') except subprocess.CalledProcessError: return None @@ -123,7 +125,7 @@ def rev_list(arg): def describe_commit(rev): """ Return a one-line description of a commit. """ return check_output( - ['git', 'log', '--color', '-n1', '--oneline', rev]).strip() + ['git', 'log', '--color', '-n1', '--oneline', rev]).strip().decode('utf-8') def is_fast_forward(ancestor, child): @@ -132,7 +134,7 @@ def is_fast_forward(ancestor, child): could be fast-forward merged. """ try: - merge_base = check_output(['git', 'merge-base', ancestor, child]).strip() + merge_base = check_output(['git', 'merge-base', ancestor, child]).strip().decode('utf-8') except: # If either of the commits is unknown, count this as a non-fast-forward. return False @@ -141,7 +143,7 @@ def is_fast_forward(ancestor, child): def get_committer_email(rev): """ Return the email address of the committer of the given revision. """ - return check_output(['git', 'log', '-n1', '--pretty=format:%ce', rev]).strip() + return check_output(['git', 'log', '-n1', '--pretty=format:%ce', rev]).strip().decode('utf-8') def do_update(branch, gerrit_sha, apache_sha): @@ -153,36 +155,36 @@ def do_update(branch, gerrit_sha, apache_sha): # must have gotten committed to Apache outside of gerrit, and we'd need some # manual intervention. if not is_fast_forward(apache_sha, gerrit_sha): - print >>sys.stderr, "Cannot update branch '%s' from gerrit:" % branch - print >>sys.stderr, "Apache revision %s is not an ancestor of gerrit revision %s" % ( - apache_sha[:8], gerrit_sha[:8]) - print >>sys.stderr, "Something must have been committed to Apache and bypassed gerrit." - print >>sys.stderr, "Manual intervention is required." + print("Cannot update branch '%s' from gerrit:" % branch, file=sys.stderr) + print("Apache revision %s is not an ancestor of gerrit revision %s" % ( + apache_sha[:8], gerrit_sha[:8]), file=sys.stderr) + print("Something must have been committed to Apache and bypassed gerrit.", file=sys.stderr) + print("Manual intervention is required.", file=sys.stderr) sys.exit(1) # List the commits that are going to be pushed to the ASF, so that the committer # can verify and "sign off". commits = rev_list("%s..%s" % (apache_sha, gerrit_sha)) commits.reverse() # Display from oldest to newest. - print "-" * 60 - print Colors.GREEN + ("%d commit(s) need to be pushed from Gerrit to ASF:" % len(commits)) + Colors.RESET + print("-" * 60) + print(Colors.GREEN + ("%d commit(s) need to be pushed from Gerrit to ASF:" % len(commits)) + Colors.RESET) push_sha = None for sha in commits: oneline = describe_commit(sha) - print " ", oneline + print(" ", oneline) committer = get_committer_email(sha) if committer != get_my_email(): - print Colors.RED + " !!! Committed by someone else (%s) !!!" % committer, Colors.RESET + print(Colors.RED + " !!! Committed by someone else (%s) !!!" % committer, Colors.RESET) if not confirm_prompt( Colors.RED + " !!! Are you sure you want to push on behalf of another committer?" + Colors.RESET): # Even if they don't want to push this commit, we could still push any # earlier commits that the user _did_ author. if push_sha is not None: - print "... will still update to prior commit %s..." % push_sha + print("... will still update to prior commit %s..." % push_sha) break push_sha = sha if push_sha is None: - print "Nothing to push" + print("Nothing to push") return # Everything has been confirmed. Do the actual push @@ -190,9 +192,9 @@ def do_update(branch, gerrit_sha, apache_sha): if OPTIONS.dry_run: cmd.append('--dry-run') cmd.append('%s:refs/heads/%s' % (push_sha, branch)) - print Colors.GREEN + "Running: " + Colors.RESET + " ".join(cmd) + print(Colors.GREEN + "Running: " + Colors.RESET + " ".join(cmd)) subprocess.check_call(cmd) - print Colors.GREEN + "Successfully updated %s to %s" % (branch, gerrit_sha) + Colors.RESET + print(Colors.GREEN + "Successfully updated %s to %s" % (branch, gerrit_sha) + Colors.RESET) print @@ -218,16 +220,16 @@ def main(): # Check the current state of branches on Apache. # For each branch, we try to update it if the revisions don't match. apache_branches = get_branches('apache') - for branch, apache_sha in sorted(apache_branches.iteritems()): + for branch, apache_sha in sorted(apache_branches.items()): gerrit_sha = rev_parse("remotes/gerrit/" + branch) - print "Branch '%s':\t" % branch, + print("Branch '%s':\t" % branch) if gerrit_sha is None: - print Colors.YELLOW, "found on Apache but not in gerrit", Colors.RESET + print(Colors.YELLOW, "found on Apache but not in gerrit", Colors.RESET) continue if gerrit_sha == apache_sha: - print Colors.GREEN, "up to date", Colors.RESET + print(Colors.GREEN, "up to date", Colors.RESET) continue - print Colors.YELLOW, "needs update", Colors.RESET + print(Colors.YELLOW, "needs update", Colors.RESET) do_update(branch, gerrit_sha, apache_sha)
