This is an automated email from the git hooks/post-receive script. smcv pushed a commit to branch master in repository game-data-packager.
commit 557a065945abab7fa7fda779a0fca4ee4c337e62 Author: Simon McVittie <[email protected]> Date: Tue Nov 28 08:41:23 2017 +0000 build: Break out file downloading to its own module This makes it a little easier to test. Signed-off-by: Simon McVittie <[email protected]> --- game_data_packager/build.py | 146 +++++-------------------------- game_data_packager/download.py | 194 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 218 insertions(+), 122 deletions(-) diff --git a/game_data_packager/build.py b/game_data_packager/build.py index 7080175..f855674 100644 --- a/game_data_packager/build.py +++ b/game_data_packager/build.py @@ -20,12 +20,10 @@ from collections import defaultdict from enum import Enum import logging import os -import random import shutil import stat import subprocess import tempfile -import urllib.request import zipfile import yaml @@ -37,13 +35,13 @@ except ImportError: BACKPORT_SUFFIX = '' from .data import (HashedFile) +from .download import (Downloader, OutOfSpace) from .gog import GOG from .packaging import (get_native_packaging_system) -from .paths import (DATADIR, ETCDIR) +from .paths import (DATADIR) from .unpack import (TarUnpacker, ZipUnpacker) from .unpack.umod import (Umod) -from .util import (AGENT, - TemporaryUmask, +from .util import (TemporaryUmask, check_call, check_output, copy_with_substitutions, @@ -114,60 +112,6 @@ class DownloadNotAllowed(Exception): class CDRipFailed(Exception): pass -def choose_mirror(wanted): - mirrors = [] - mirror = os.environ.get('GDP_MIRROR') - if mirror: - if mirror.startswith('/'): - mirror = 'file://' + mirror - elif mirror.split(':')[0] not in ('http', 'https', 'ftp', 'file'): - mirror = 'http://' + mirror - if not mirror.endswith('/'): - mirror = mirror + '/' - - if type(wanted.download) is str: - if not mirror: - return [wanted.download] - url_basename = os.path.basename(wanted.download) - if '?' not in url_basename: - mirrors.append(mirror + url_basename) - wanted.name = wanted.name.replace(' ','%20') - if wanted.name != url_basename and '?' not in wanted.name: - mirrors.append(mirror + wanted.name) - mirrors.append(wanted.download) - return mirrors - - for mirror_list, details in wanted.download.items(): - try: - f = open(os.path.join(ETCDIR, mirror_list), encoding='utf-8') - for line in f: - url = line.strip() - if not url: - continue - if url.startswith('#'): - continue - if details.get('path', '.') != '.': - if not url.endswith('/'): - url = url + '/' - url = url + details['path'] - if not url.endswith('/'): - url = url + '/' - url = url + details.get('name', wanted.name) - mirrors.append(url) - except: - logger.warning('Could not open mirror list "%s"', mirror_list, - exc_info=True) - random.shuffle(mirrors) - if mirror: - if mirrors and '?' not in mirrors[0]: - mirrors.insert(0, mirror + os.path.basename(mirrors[0])) - elif '?' not in wanted.name: - mirrors.insert(0, mirror + wanted.name) - if not mirrors: - logger.error('Could not select a mirror for "%s"', wanted.name) - return [] - return mirrors - def iter_fat_mounts(folder): with open('/proc/mounts', 'r', encoding='utf8') as mounts: for line in mounts.readlines(): @@ -249,6 +193,7 @@ class PackagingTask(object): # Factory for a progress report (or None). self.progress_factory = lambda info=None: None + self.downloader = None self.game.load_file_data() def __del__(self): @@ -749,71 +694,28 @@ class PackagingTask(object): self.file_status[wanted.name] = FillResult.DOWNLOAD_NEEDED if download: - logger.debug('trying to download %s...', wanted.name) - - tmpdir = self.save_downloads or os.path.dirname(self.get_workdir()) - statvfs = os.statvfs(tmpdir) - if wanted.size > statvfs.f_frsize * statvfs.f_bavail: - logger.error("Out of space on %s, can't download %s.", - tmpdir, wanted.name) - self.download_failed |= set(choose_mirror(wanted)) - return FillResult.IMPOSSIBLE - - urls = choose_mirror(wanted) - for url in urls: - if url in self.download_failed: - logger.debug('... no, it already failed') - continue - - logger.debug('... %s', url) - - tmp = None - try: - rf = urllib.request.urlopen(urllib.request.Request( - url,headers={'User-Agent': AGENT})) - if rf is None: - continue + if self.downloader is None: + self.downloader = Downloader( + progress_factory=self.progress_factory) - try: - size = int(rf.info().get('Content-Length')) - except: - size = None - if size and size != wanted.size: - logger.warning("File doesn't have expected size" - " (%s vs %s), skipping %s", - size, wanted.size, url) - self.download_failed.add(url) - continue + if self.save_downloads is not None: + dest = self.save_downloads + else: + dest = self.get_workdir() - if self.save_downloads is not None: - tmp = os.path.join(self.save_downloads, - wanted.name) - else: - tmp = os.path.join(self.get_workdir(), - 'tmp', wanted.name) - mkdir_p(os.path.dirname(tmp)) - - wf = open(tmp, 'wb') - logger.info('downloading %s', url) - hf = HashedFile.from_file(url, rf, wf, - size=wanted.size, - progress=self.progress_factory()) - wf.close() - - if self.use_file(wanted.name, (wanted,), tmp, hf): - assert self.found[wanted.name] == tmp - assert (self.file_status[wanted.name] == - FillResult.COMPLETE) - return FillResult.COMPLETE - else: - # file corrupted or something - os.remove(tmp) - except Exception as e: - logger.warning('Failed to download "%s": %s', url, - e) - self.download_failed.add(url) - if tmp is not None: - os.remove(tmp) + try: + path, hasher = self.downloader.download(wanted, dest) + except OutOfSpace: + return FillResult.IMPOSSIBLE + else: + if self.use_file(wanted.name, (wanted,), path, hasher): + assert self.found[wanted.name] == path + assert (self.file_status[wanted.name] == + FillResult.COMPLETE) + return FillResult.COMPLETE + else: + # file corrupted or something + os.remove(path) providers = list(self.game.providers.get(wanted.name, ())) diff --git a/game_data_packager/download.py b/game_data_packager/download.py new file mode 100644 index 0000000..d9a9df5 --- /dev/null +++ b/game_data_packager/download.py @@ -0,0 +1,194 @@ +#!/usr/bin/python3 +# encoding=utf-8 +# +# Copyright © 2014-2017 Simon McVittie <[email protected]> +# Copyright © 2015-2016 Alexandre Detiste <[email protected]> +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# You can find the GPL license text on a Debian system under +# /usr/share/common-licenses/GPL-2. + +import logging +import os +import random +import urllib.request + +from .data import (HashedFile) +from .paths import (ETCDIR) +from .util import (AGENT, mkdir_p) + +logging.basicConfig() +logger = logging.getLogger(__name__) + + +class OutOfSpace(Exception): + pass + + +class Downloader: + + def __init__(self, progress_factory=None): + self.download_failed = set() + + if progress_factory is None: + self.progress_factory = lambda info=None: None + else: + self.progress_factory = progress_factory + + @staticmethod + def choose_mirror(wanted): + mirrors = [] + mirror = os.environ.get('GDP_MIRROR') + if mirror: + if mirror.startswith('/'): + mirror = 'file://' + mirror + elif mirror.split(':')[0] not in ('http', 'https', 'ftp', 'file'): + mirror = 'http://' + mirror + if not mirror.endswith('/'): + mirror = mirror + '/' + + if type(wanted.download) is str: + if not mirror: + return [wanted.download] + url_basename = os.path.basename(wanted.download) + if '?' not in url_basename: + mirrors.append(mirror + url_basename) + wanted.name = wanted.name.replace(' ','%20') + if wanted.name != url_basename and '?' not in wanted.name: + mirrors.append(mirror + wanted.name) + mirrors.append(wanted.download) + return mirrors + + for mirror_list, details in wanted.download.items(): + try: + f = open(os.path.join(ETCDIR, mirror_list), encoding='utf-8') + for line in f: + url = line.strip() + if not url: + continue + if url.startswith('#'): + continue + if details.get('path', '.') != '.': + if not url.endswith('/'): + url = url + '/' + url = url + details['path'] + if not url.endswith('/'): + url = url + '/' + url = url + details.get('name', wanted.name) + mirrors.append(url) + except: + logger.warning('Could not open mirror list "%s"', mirror_list, + exc_info=True) + random.shuffle(mirrors) + if mirror: + if mirrors and '?' not in mirrors[0]: + mirrors.insert(0, mirror + os.path.basename(mirrors[0])) + elif '?' not in wanted.name: + mirrors.insert(0, mirror + wanted.name) + if not mirrors: + logger.error('Could not select a mirror for "%s"', wanted.name) + return [] + return mirrors + + def download(self, wanted, dest): + logger.debug('trying to download %s...', wanted.name) + statvfs = os.statvfs(dest) + if wanted.size > statvfs.f_frsize * statvfs.f_bavail: + logger.error("Out of space on %s, can't download %s.", + dest, wanted.name) + self.download_failed |= set(self.choose_mirror(wanted)) + raise OutOfSpace + + urls = self.choose_mirror(wanted) + for url in urls: + if url in self.download_failed: + logger.debug('... no, it already failed') + continue + + logger.debug('... %s', url) + + tmp = None + try: + rf = urllib.request.urlopen(urllib.request.Request( + url,headers={'User-Agent': AGENT})) + if rf is None: + continue + + try: + size = int(rf.info().get('Content-Length')) + except: + size = None + if size and size != wanted.size: + logger.warning("File doesn't have expected size" + " (%s vs %s), skipping %s", + size, wanted.size, url) + self.download_failed.add(url) + continue + + tmp = os.path.join(dest, wanted.name) + mkdir_p(os.path.dirname(tmp)) + + wf = open(tmp, 'wb') + logger.info('downloading %s', url) + hf = HashedFile.from_file(url, rf, wf, + size=wanted.size, + progress=self.progress_factory()) + wf.close() + + return tmp, hf + except Exception as e: + logger.warning('Failed to download "%s": %s', url, + e) + self.download_failed.add(url) + if tmp is not None: + os.remove(tmp) + else: + return None, None + +if __name__ == '__main__': + # Usage: + # GDP_UNINSTALLED=1 \ + # PYTHONPATH=$(pwd) \ + # python3 -m game_data_packager.download \ + # unreal skaarj_logo.jpg . + + import sys + + from . import (load_games) + + game = sys.argv[1] + filename = sys.argv[2] + dest = sys.argv[3] + + games = load_games(game=game) + game = games[game] + game.load_file_data() + wanted = game.files[filename] + + path, hasher = Downloader().download(wanted, dest) + + if path is None: + logger.error('Unable to download "%s"', filename) + else: + logger.info('Downloaded "%s" to "%s"', filename, path) + + if hasher.size != wanted.size: + logger.info('size: %s, expected %s', hasher.size, wanted.size) + + if hasher.md5 != wanted.md5: + logger.info('md5: %s, expected %s', hasher.md5, wanted.md5) + + if hasher.sha1 != wanted.sha1: + logger.info('sha1: %s, expected %s', hasher.sha1, wanted.sha1) + + if hasher.sha256 != wanted.sha256: + logger.info( + 'sha256: %s, expected %s', hasher.sha256, wanted.sha256) -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git _______________________________________________ Pkg-games-commits mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-games-commits
