Jean-Baptiste Lallement has proposed merging lp:~jibel/ubuntu-server-iso-testing/download-latest-test-iso.fixes into lp:ubuntu-server-iso-testing.
Requested reviews: Ubuntu Server Iso Testing Developers (ubuntu-server-iso-testing-dev) Related bugs: Bug #740853 in ubuntu-server-iso-testing: "download-latest-test-iso.py fails with KeyError when the image is not listed in SHA256SUMS " https://bugs.launchpad.net/ubuntu-server-iso-testing/+bug/740853 Bug #751292 in ubuntu-server-iso-testing: "testing continues even if ISO is not available" https://bugs.launchpad.net/ubuntu-server-iso-testing/+bug/751292 Bug #752321 in ubuntu-server-iso-testing: "download-latest-test-iso.py fails when launchpad is not available" https://bugs.launchpad.net/ubuntu-server-iso-testing/+bug/752321 For more details, see: https://code.launchpad.net/~jibel/ubuntu-server-iso-testing/download-latest-test-iso.fixes/+merge/69967 Major diff due to code refactoring and some python style cleaning. The logic is exactly the same the main differences being: - Check if both ISO and Checksum files are available on the server before starting sync - Validation of checksum _after_ download to be sure that what's in cache is really what is expected. - More exception handling. -- https://code.launchpad.net/~jibel/ubuntu-server-iso-testing/download-latest-test-iso.fixes/+merge/69967 Your team Ubuntu Server Iso Testing Developers is requested to review the proposed merge of lp:~jibel/ubuntu-server-iso-testing/download-latest-test-iso.fixes into lp:ubuntu-server-iso-testing.
=== modified file 'download-latest-test-iso.py' --- download-latest-test-iso.py 2011-07-19 22:34:19 +0000 +++ download-latest-test-iso.py 2011-08-01 08:22:29 +0000 @@ -1,209 +1,238 @@ #!/usr/bin/python -# -# Copyright (C) 2010, Canonical Ltd (http://www.canonical.com/) +# +# Copyright (C) 2010-2011, Canonical Ltd (http://www.canonical.com/) # # This file is part of ubuntu-server-iso-testing. -# -# ubuntu-server-iso-testing is free software: you can redistribute it -# and/or modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation, either version 3 of +# +# ubuntu-server-iso-testing is free software: you can redistribute it +# and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of # the License, or (at your option) any later version. -# -# ubuntu-server-iso-testing is distributed in the hope that it will -# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +# +# ubuntu-server-iso-testing is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty # of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License -# along with ubuntu-server-iso-testing. If not, see +# along with ubuntu-server-iso-testing. If not, see # <http://www.gnu.org/licenses/>. -# - -from HTMLParser import HTMLParser +# from lockfile import FileLock import os import optparse import sys import logging -import urllib -import urlparse +import urllib2 import subprocess -import datetime import hashlib +from time import sleep # Default options DEFAULT_VARIANT = 'server' -DEFAULT_RELEASE = 'maverick' +DEFAULT_RELEASE = 'oneiric' DEFAULT_ARCH = 'all' -DEFAULT_FLAVOR= 'ubuntu-server' +DEFAULT_FLAVOR = 'ubuntu-server' DEFAULT_ISOROOT = os.path.expanduser('~/isos') +TIMEOUT = 1 # Timeout between retries when the image is not on the server +RETRY_MAX = 4 # Number of retries + # URLS -BASE_URL="http://cdimage.ubuntu.com"; +BASE_URL = "http://cdimage.ubuntu.com"; # List of 'all' architectures -ALL_ARCHS =[ 'i386', 'amd64'] +ALL_ARCHS = [ 'i386', 'amd64'] + +class HeadRequest(urllib2.Request): + """ Used to check to retrieve headers of a remote file """ + def get_method(self): + return "HEAD" # Calculates the sha256 hash for a given file -def sha256_for_file(f, block_size=2**20): +def validate_sha256(fname, sha256_hash, block_size=2**20): + """ Validate checksum of file fname against sha256 hash """ + logging.debug("Calculating hash for file '%s'", fname) sha256 = hashlib.sha256() - while True: - data = f.read(block_size) - if not data: - break - sha256.update(data) - return sha256.hexdigest() - - -# HTML Parser for parsing ISO directory listing -class ISOHTMLParser(HTMLParser): - in_a = False - builds = [] - year = str(datetime.datetime.utcnow().year) - - def handle_starttag(self, tag, attrs): - if tag == 'a': - self.in_a = True - else: - self.in_a = False - - def handle_endtag(self, tag): - if tag == 'a': - self.in_a = False - else: - self.in_a = True - - def handle_data(self, data): - if self.in_a: - l_stripped = data.strip().rstrip('/') - if self.year in l_stripped: - self.builds.append(l_stripped) - - def getbuilds(self): - return self.builds - - def getlatestbuild(self): - self.builds.sort() - return self.builds.pop() - -usage="usage: %prog [options]" -parser = optparse.OptionParser(usage=usage) -parser.add_option("-d", "--debug", dest="debug", action="store_true", default=False, - help="enable debugging") -parser.add_option("-r", "--release", dest="release", default=DEFAULT_RELEASE, - help="release of Ubuntu to download (default=%s)" % DEFAULT_RELEASE) -parser.add_option("-v", "--variant", dest="variant", default=DEFAULT_VARIANT, - help="variant of Ubuntu to download (default=%s)" % DEFAULT_VARIANT) -parser.add_option("-a", "--arch", dest="arch", default=DEFAULT_ARCH, - help="arch of Ubuntu to download (default=%s)" % DEFAULT_ARCH) -parser.add_option("-i", "--isoroot", dest="isoroot", default=DEFAULT_ISOROOT, - help="location to store iso images (default=%s)" % DEFAULT_ISOROOT) -parser.add_option("-f", "--flavor", dest="flavor", default=DEFAULT_FLAVOR, - help="flavor of Ubuntu to download (default=%s)" % DEFAULT_FLAVOR) -parser.add_option('-n', '--no-act', default=False, action='store_true', - dest='no_act', help='compute everything but don\'t actually download') -parser.add_option('-u', '--url', dest="base_url", default=BASE_URL, - help="Base URL for cdimage retrieval website (default=%s)" % BASE_URL) -parser.add_option('-c', '--custom-url', dest="custom_url", default=False, - help="Full URL for cdimage retrieval website") - -(options, args) = parser.parse_args() - -if options.debug: - logging.basicConfig(level=logging.DEBUG) -else: - logging.basicConfig(level=logging.INFO) - -# Setup URLS -ALTERNATE_URL= '%s/daily/' % options.base_url -DESKTOP_URL='%s/daily-live/' % options.base_url -SERVER_URL='%s/ubuntu-server/daily/' % options.base_url - -# Setup Parser to parse CDIMAGE pages. -parser = ISOHTMLParser() -l_url = SERVER_URL -if options.variant == 'server': - l_url= SERVER_URL -elif options.variant == 'desktop': - l_url = DESKTOP_URL -elif options.variant == 'alternate': - l_url = ALTERNATE_URL - -# override url setting if custom url is provided -if options.custom_url: - l_url = options.custom_url - -# Check options for architectures -l_archs = [] -if options.arch == 'all': - l_archs = ALL_ARCHS -else: - l_archs.append(options.arch) - -# Retrieve page and parse list -fh = urllib.urlopen(l_url) -page = fh.read() -fh.close() - -# Parse the page for information -parser.feed(page) -logging.debug(parser.getbuilds()) -l_build = parser.getlatestbuild() - -logging.info("Checking/Downloading ISO build %s for variant %s from %s" % (l_build, options.variant, l_url)) - -# Download and store sha256 digest for required ISO images -l_current_iso_digest = {} -fh = urllib.urlopen(l_url + l_build + '/SHA256SUMS') -# Process the file (will contain a number of entries -for digest in fh: - (id, sep, iso) = digest.partition(' ') - l_current_iso_digest[iso.rstrip('\n').lstrip('*')] = id -fh.close() - -logging.debug(l_current_iso_digest) - -# Check to see if version on disk is already the latest -for arch in l_archs: - l_download = False - l_iso_name = options.release + '-' + options.variant + '-' + arch + '.iso' - lock = FileLock('/tmp/' + l_iso_name) - with lock: - l_iso_dir = options.flavor - l_iso_location = os.path.join(options.isoroot, l_iso_dir, l_iso_name) - # If iso does not exists then mark for retrieval - if os.path.exists(l_iso_location): - # Check to see if latest version - f = open(l_iso_location) - l_sha256 = sha256_for_file(f) - f.close() - # If digests don't match then mark for retrieval - if l_current_iso_digest[l_iso_name] != l_sha256: - logging.debug("ISO is not the latest") - l_download = True - else: - logging.info("Required ISO version (%s,%s) already in cache" % (l_iso_name, l_build)) - else: - # ISO does not exist need to download - logging.info("Downloading ISO (%s,%s) to local cache" % (l_iso_name, l_build)) - l_download = True - - if l_download: - # Call dl-ubunut-test-iso with correct arguments - logging.debug("Downloading ISO as local cache is stale/empty") - cmd = ['dl-ubuntu-test-iso', '--variant=%s' % options.variant, - '--arch=%s' % arch, '--release=%s' % options.release, - '--build=%s' % l_build,'--isoroot=%s' % options.isoroot, '-P'] - if options.flavor: - cmd.append('--flavor=%s' % options.flavor) - - if options.no_act: - cmd.append('-n') - - logging.debug("Cmd: %s" % (cmd)) - subprocess.check_call(cmd) - # Write out build version to file for later use - needs to be refactored - f = open(l_iso_location + '.build-version', 'w') - f.write(l_build) - f.close() - + with open(fname) as f: + while True: + data = f.read(block_size) + if not data: + break + sha256.update(data) + + l_sha256 = sha256.hexdigest() + logging.debug("Expected Checksum: '%s'", sha256_hash) + logging.debug("Local File Checksum: '%s'", l_sha256) + return sha256_hash == l_sha256 + +def download_iso(isoroot, release, variant, arch, flavor=None, no_act=False): + """ Download an iso """ + logging.info("Downloading ISO...") + cmd = ['dl-ubuntu-test-iso', '--variant=%s' % variant, + '--arch=%s' % arch, '--release=%s' % release, + '--build=current' ,'--isoroot=%s' % isoroot, '-P'] + if flavor: + cmd.append('--flavor=%s' % flavor) + + if no_act: + cmd.append('-n') + + logging.debug("Cmd: %s", cmd) + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError, e: + logging.error(e) + return e.returncode + return 0 + +def remote_file_exists(file_url): + """ Check if the remote file exists + + Return file info on success false otherwise """ + logging.debug("Checking if remote file exists: '%s'", file_url) + try: + res = urllib2.urlopen(HeadRequest(file_url)) + logging.debug('URL Found:\n%s', res.info()) + return res.info() + except (urllib2.HTTPError, urllib2.URLError), e: + logging.debug("Check failed: %s", e) + return False + +def main(): + """ Main routine """ + usage = "Usage: %prog [options]" + parser = optparse.OptionParser(usage=usage) + parser.add_option("-d", "--debug", dest="debug", action="store_true", + default=False, help="enable debugging") + parser.add_option("-r", "--release", dest="release", + default=DEFAULT_RELEASE, + help="release of Ubuntu to download (default=%s)" + % DEFAULT_RELEASE) + parser.add_option("-v", "--variant", dest="variant", + default=DEFAULT_VARIANT, + help="variant of Ubuntu to download (default=%s)" + % DEFAULT_VARIANT) + parser.add_option("-a", "--arch", dest="arch", default=DEFAULT_ARCH, + help="arch of Ubuntu to download (default=%s)" + % DEFAULT_ARCH) + parser.add_option("-i", "--isoroot", dest="isoroot", + default=DEFAULT_ISOROOT, + help="location to store iso images (default=%s)" + % DEFAULT_ISOROOT) + parser.add_option("-f", "--flavor", dest="flavor", default=DEFAULT_FLAVOR, + help="flavor of Ubuntu to download (default=%s)" + % DEFAULT_FLAVOR) + parser.add_option('-n', '--no-act', default=False, action='store_true', + dest='no_act', + help='compute everything but don\'t actually download') + parser.add_option('-u', '--url', dest="base_url", default=BASE_URL, + help="Base URL for cdimage retrieval website (default=%s)" + % BASE_URL) + + (options, args) = parser.parse_args() + + if options.debug: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + # Setup URLS + release_part = options.release + if options.release == DEFAULT_RELEASE: + release_part = '' + if options.variant == 'desktop': + l_url = os.path.join(options.base_url, release_part, 'daily-live') + elif options.variant == 'alternate': + l_url = os.path.join(options.base_url, release_part, 'daily') + else: + # Default to server + l_url = os.path.join(options.base_url, 'ubuntu-server', release_part, + 'daily') + + # Check options for architectures + l_archs = [] + if options.arch == 'all': + l_archs = ALL_ARCHS + else: + l_archs.append(options.arch) + + logging.info("Checking/Downloading current build for variant %s from %s", + options.variant, l_url) + + # Download and store sha256 digest for required ISO images + l_current_iso_digest = {} + sha256sum_url = os.path.join(l_url, 'current', 'SHA256SUMS') + try: + fh = urllib2.urlopen(sha256sum_url) + except urllib2.HTTPError, e: + logging.error("Failed to fetch URL '%s': %s . Aborting!", + sha256sum_url, e) + sys.exit(1) + + # Process the file (will contain a number of entries + for digest in fh: + (chksum, sep, iso) = digest.partition(' *') + l_current_iso_digest[iso.rstrip('\n')] = chksum + fh.close() + + # Check to see if version on disk is already the latest + for arch in l_archs: + l_iso_name = '-'.join((options.release, options.variant, arch)) + '.iso' + l_iso_url = os.path.join(l_url, 'current', l_iso_name) + + # Check ISO availability on remote server + for retry in range(1, RETRY_MAX+1): + logging.debug('Try %d', retry) + if remote_file_exists(l_iso_url): + break + + if retry == RETRY_MAX: + logging.info('Max number of retries reached. Aborting!') + sys.exit(1) + logging.info("Remote file not available. Retrying in %d minutes", + l_iso_url, TIMEOUT) + sleep(TIMEOUT * 60) + + try: + iso_sha256 = l_current_iso_digest[l_iso_name] + except KeyError: + logging.error("No digest found for image '%s'", l_iso_name) + + lock = FileLock('/tmp/' + l_iso_name) + with lock: + rc = 0 + l_iso_dir = options.flavor + l_iso_location = os.path.join(options.isoroot, l_iso_dir, + l_iso_name) + logging.debug("Checking state of local cache '%s'", l_iso_location) + # If iso does not exists then mark for retrieval + if os.path.exists(l_iso_location): + # Check to see if latest version + # If digests don't match then mark for retrieval + if not validate_sha256(l_iso_location, iso_sha256): + logging.debug("ISO '%s' is not the latest", l_iso_name) + rc = download_iso(options.isoroot, options.release, + options.variant, arch, options.flavor, + options.no_act) + else: + logging.info("Required ISO version '%s' already in cache", + l_iso_name) + else: + # ISO does not exist need to download + logging.info("Downloading ISO '%s' to local cache", l_iso_name) + rc = download_iso(options.isoroot, options.release, + options.variant, arch, options.flavor, + options.no_act) + + if rc != 0: + logging.error("Failed to download ISO. Aborting!") + sys.exit(1) + else: + # We downloaded something, let see if that is what we expected + if not validate_sha256(l_iso_location, iso_sha256): + logging.info("Local file doesn't match checksum. Aborting!") + sys.exit(1) + +if __name__ == '__main__': + main()
_______________________________________________ Mailing list: https://launchpad.net/~ubuntu-server-iso-testing-dev Post to : [email protected] Unsubscribe : https://launchpad.net/~ubuntu-server-iso-testing-dev More help : https://help.launchpad.net/ListHelp
