Colin Watson has proposed merging ~cjwatson/launchpad:publish-db-tarballs-to-swift into launchpad:master.
Commit message: Support publishing tarballs of db-devel to Swift Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/439659 Unlike our other environments, Launchpad's staging environment is built from the `db-stable` branch, which is fed from `db-devel`. To support deploying this using Juju charms, we need to build a separate stream of deployment artifacts from the `db-devel` branch, old versions of which are pruned on a different schedule. (Discovered while trying to set up auto-upgrades for the new Juju-deployed staging appservers. After this lands, we'll also need a change to lp:ols-jenkaas to actually run the builds.) -- Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:publish-db-tarballs-to-swift into launchpad:master.
diff --git a/Makefile b/Makefile index 7bd08d1..f60189b 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,14 @@ PIP_BIN = \ # Create archives in labelled directories (e.g. # <rev-id>/$(PROJECT_NAME).tar.gz) -TARBALL_BUILD_LABEL ?= $(shell git rev-parse HEAD) +GIT_BRANCH := $(shell git branch --show-current) +TARBALL_REVISION ?= $(shell git rev-parse HEAD) +ifeq ($(GIT_BRANCH),db-devel) +TARBALL_SUFFIX := db +else +TARBALL_SUFFIX := +endif +TARBALL_BUILD_LABEL := $(TARBALL_REVISION)$(if $(TARBALL_SUFFIX),-$(TARBALL_SUFFIX)) TARBALL_FILE_NAME = launchpad.tar.gz TARBALL_BUILD_DIR = dist/$(TARBALL_BUILD_LABEL) TARBALL_BUILD_PATH = $(TARBALL_BUILD_DIR)/$(TARBALL_FILE_NAME) @@ -304,7 +311,7 @@ publish-tarball: build-tarball [ ! -e ~/.config/swift/launchpad ] || . ~/.config/swift/launchpad; \ utilities/publish-to-swift --debug \ $(SWIFT_CONTAINER_NAME) $(SWIFT_OBJECT_PATH) \ - $(TARBALL_BUILD_PATH) + $(TARBALL_BUILD_PATH) $(TARBALL_SUFFIX) # setuptools won't touch files that would have the same contents, but for # Make's sake we need them to get fresh timestamps, so we touch them after diff --git a/utilities/publish-to-swift b/utilities/publish-to-swift index 93280d8..88c30ee 100755 --- a/utilities/publish-to-swift +++ b/utilities/publish-to-swift @@ -8,6 +8,7 @@ import _pythonpath # noqa: F401 import os +import re from argparse import ArgumentParser import iso8601 @@ -76,15 +77,28 @@ def publish_file_to_swift( ) -def prune_old_files_from_swift(options, container_name, object_dir): +def prune_old_files_from_swift(options, container_name, object_dir, suffix): """Prune files from Swift that we no longer need.""" - response = requests.head("https://launchpad.net/") - response.raise_for_status() - production_revision = response.headers["X-VCS-Revision"] + if suffix: + suffix = "-" + suffix + if suffix.endswith("-db"): + response = requests.head("https://staging.launchpad.net/") + try: + response.raise_for_status() + except requests.HTTPError: + # Staging is routinely down in order to restore its database + # from a recent production dump, so don't consider this an + # error; just skip pruning if we can't determine its revision. + print("staging.launchpad.net is down; not pruning.") + return + else: + response = requests.head("https://launchpad.net/") + response.raise_for_status() + deployed_revision = response.headers["X-VCS-Revision"] with SwiftService(options=options) as swift: objs = {} - production_mtime = None + deployed_mtime = None for stats in swift.list( container=container_name, options={"prefix": "{}/".format(object_dir)}, @@ -92,24 +106,35 @@ def prune_old_files_from_swift(options, container_name, object_dir): if not stats["success"]: raise stats["error"] for item in stats["listing"]: + if not item["name"].startswith("{}/".format(object_dir)): + continue + # Only consider pruning builds that have the given suffix. + # (For example, if we're publishing a build with the suffix + # "-db", then we consider pruning builds with the label + # "<commit>-db" but not plain "<commit>".) + build_label = item["name"].split("/")[1] + if not re.match(r"^.{40}%s$" % re.escape(suffix), build_label): + continue if item.get("subdir") is None: mtime = iso8601.parse_date(item["last_modified"]) objs[item["name"]] = mtime - if item["name"].startswith( - "{}/{}/".format(object_dir, production_revision) - ): - production_mtime = mtime + if build_label == deployed_revision + suffix: + deployed_mtime = mtime - if production_mtime is None: + if deployed_mtime is None: print( - "No file in {} corresponding to production revision {}; " - "not pruning.".format(container_name, production_revision) + "No file in {} corresponding to deployed revision {}; " + "not pruning.".format(container_name, deployed_revision) ) return for object_name, mtime in sorted(objs.items()): - if mtime < production_mtime: - print("Pruning {} (older than production)".format(object_name)) + if mtime < deployed_mtime: + print( + "Pruning {} (older than deployed revision)".format( + object_name + ) + ) for r in swift.delete( container=container_name, objects=[object_name] ): @@ -122,6 +147,7 @@ def main(): parser.add_argument("container_name") parser.add_argument("swift_object_path") parser.add_argument("local_path") + parser.add_argument("suffix", nargs="?", default="") add_default_args(parser) args = parser.parse_args() @@ -173,7 +199,10 @@ def main(): overwrite=overwrite, ) prune_old_files_from_swift( - options, args.container_name, args.swift_object_path.split("/")[0] + options, + args.container_name, + args.swift_object_path.split("/")[0], + args.suffix, )
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp