This is an automated email from the git hooks/post-receive script. detiste-guest pushed a commit to branch master in repository game-data-packager.
commit 2254ab8a114054e9bcb4b938cfadfb7e26c9a93b Author: Alexandre Detiste <[email protected]> Date: Sun Oct 18 10:55:22 2015 +0200 store all game information in a .zip --- Makefile | 8 +++- debian/game-data-packager.install | 8 +--- debian/rules | 4 ++ game_data_packager/__init__.py | 80 +++++++++++++++++++++++++++++---------- game_data_packager/paths.py | 2 + 5 files changed, 74 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index ee64566..531a3f0 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,11 @@ out/%: data/% out/%.json: data/%.yaml python3 tools/yaml2json.py $< $@ +out/vfs.zip: + rm -f out/vfs.zip + find out -regex '.*\.\(json\|files\|size_and_md5\|cksums\|md5sums\|sha1sums\|sha256sums\)' \ + | LC_ALL=C sort | TZ=UTC zip out/vfs.zip -X -j -q -@ + out/bash_completion: $(in_yaml) python3 tools/bash_completion.py > ./out/bash_completion chmod 0644 ./out/bash_completion @@ -82,6 +87,7 @@ clean: rm -f ./out/*.svgz rm -f ./out/*.svg rm -f ./out/*.json + rm -f ./out/vfs.zip rm -f ./out/index.html rm -rf game_data_packager/__pycache__ rm -rf game_data_packager/games/__pycache__ @@ -105,4 +111,4 @@ html: $(DIRS) $(json) LC_ALL=C GDP_UNINSTALLED=1 PYTHONPATH=. python3 -m tools.babel rsync out/index.html alioth.debian.org:/var/lib/gforge/chroot/home/groups/pkg-games/htdocs/game-data/ -e ssh -v -.PHONY: default clean check manual-check html +.PHONY: default clean check manual-check html out/vfs.zip diff --git a/debian/game-data-packager.install b/debian/game-data-packager.install index 9874a25..763195d 100644 --- a/debian/game-data-packager.install +++ b/debian/game-data-packager.install @@ -3,20 +3,14 @@ data/bash-completion/game-data-packager usr/share/bash-completion/completions etc/game-data-packager.conf etc etc/*-mirrors etc/game-data-packager game_data_packager usr/share/games/game-data-packager -out/*.cksums usr/share/games/game-data-packager out/*.control.in usr/share/games/game-data-packager out/*.copyright usr/share/games/game-data-packager out/*.copyright.in usr/share/games/game-data-packager -out/*.files usr/share/games/game-data-packager -out/*.md5sums usr/share/games/game-data-packager out/*.png usr/share/games/game-data-packager -out/*.sha1sums usr/share/games/game-data-packager -out/*.sha256sums usr/share/games/game-data-packager -out/*.size_and_md5 usr/share/games/game-data-packager out/*.svgz usr/share/games/game-data-packager out/*.preinst.in usr/share/games/game-data-packager out/bash_completion usr/share/games/game-data-packager out/changelog.gz usr/share/games/game-data-packager out/copyright usr/share/games/game-data-packager -out/*.json usr/share/games/game-data-packager +out/vfs.zip usr/share/games/game-data-packager runtime/doom2-masterlevels.desktop usr/share/applications diff --git a/debian/rules b/debian/rules index b0a76ad..18fdeda 100755 --- a/debian/rules +++ b/debian/rules @@ -5,12 +5,16 @@ export DH_OPTIONS include /usr/share/dpkg/pkg-info.mk +BUILD_DATE := $(shell dpkg-parsechangelog --show-field=Date) %: dh $@ --parallel --with python3,bash-completion override_dh_auto_build: dh_auto_build --parallel VERSION=$(DEB_VERSION) + find out -regex '.*\.\(json\|files\|size_and_md5\|cksums\|md5sums\|sha1sums\|sha256sums\)' \ + -exec touch --date='$(BUILD_DATE)' {} \; + make out/vfs.zip override_dh_auto_clean: dh_auto_clean --parallel VERSION=$(DEB_VERSION) diff --git a/game_data_packager/__init__.py b/game_data_packager/__init__.py index 47518ce..148eb6a 100644 --- a/game_data_packager/__init__.py +++ b/game_data_packager/__init__.py @@ -25,12 +25,13 @@ import os import random import re import sys +import zipfile from .build import (HashedFile, PackagingTask) from .config import read_config from .gog import run_gog_meta_mode -from .paths import DATADIR +from .paths import (DATADIR,USE_VFS) from .util import ascii_safe from .steam import run_steam_meta_mode from .version import GAME_PACKAGE_VERSION @@ -792,21 +793,41 @@ class GameData(object): logger.debug('loading full data') - filename = os.path.join(DATADIR, '%s.files' % self.shortname) - if os.path.isfile(filename): - logger.debug('... %s', filename) - data = json.load(open(filename, encoding='utf-8')) - self._populate_files(data) - - for alg in ('ck', 'md5', 'sha1', 'sha256', 'size_and_md5'): - filename = os.path.join(DATADIR, '%s.%s%s' % - (self.shortname, alg, - '' if alg == 'size_and_md5' else 'sums')) + if USE_VFS: + zip = os.path.join(DATADIR, 'vfs.zip') + with zipfile.ZipFile(zip, 'r') as zf: + files = zf.namelist() + filename = '%s.files' % self.shortname + if filename in files: + logger.debug('... vfs.zip/%s', filename) + jsondata = zf.open(filename).read().decode('utf-8') + data = json.loads(jsondata) + self._populate_files(data) + + for alg in ('ck', 'md5', 'sha1', 'sha256', 'size_and_md5'): + filename = '%s.%s%s' % (self.shortname, alg, + '' if alg == 'size_and_md5' else 'sums') + if filename in files: + logger.debug('... vfs.zip/%s', filename) + rawdata = zf.open(filename).read().decode('utf-8') + for line in rawdata.splitlines(): + self._add_hash(line.rstrip('\n'), alg) + else: + filename = os.path.join(DATADIR, '%s.files' % self.shortname) if os.path.isfile(filename): logger.debug('... %s', filename) - with open(filename) as f: - for line in f: - self._add_hash(line.rstrip('\n'), alg) + data = json.load(open(filename, encoding='utf-8')) + self._populate_files(data) + + for alg in ('ck', 'md5', 'sha1', 'sha256', 'size_and_md5'): + filename = os.path.join(DATADIR, '%s.%s%s' % + (self.shortname, alg, + '' if alg == 'size_and_md5' else 'sums')) + if os.path.isfile(filename): + logger.debug('... %s', filename) + with open(filename) as f: + for line in f: + self._add_hash(line.rstrip('\n'), alg) self.loaded_file_data = True @@ -911,16 +932,38 @@ class GameData(object): return gog.get('game', gog['url']) def load_games(game='*'): + progress = game == '*' and sys.stderr.isatty() games = {} - for jsonfile in glob.glob(os.path.join(DATADIR, game + '.json')): - if game == '*' and sys.stderr.isatty(): + if USE_VFS: + zip = os.path.join(DATADIR, 'vfs.zip') + with zipfile.ZipFile(zip, 'r') as zf: + if game == '*': + for entry in zf.infolist(): + if entry.filename.split('.')[-1] == 'json': + jsonfile = 'vfs.zip/' + entry.filename + jsondata = zf.open(entry).read().decode('utf-8') + load_json(progress, games, jsonfile, jsondata) + else: + jsonfile = game + '.json' + jsondata = zf.open(jsonfile).read().decode('utf-8') + load_json(progress, games, 'vfs.zip/' + jsonfile, jsondata) + else: + for jsonfile in glob.glob(os.path.join(DATADIR, game + '.json')): + jsondata = open(jsonfile, encoding='utf-8').read() + load_json(progress, games, jsonfile, jsondata) + + print('\r%s\r' % (' ' * len(games)), end='', flush=True, file=sys.stderr) + return games + +def load_json(progress, games, jsonfile, jsondata): + if progress: print('.', end='', flush=True, file=sys.stderr) try: g = os.path.basename(jsonfile) g = g[:len(g) - 5] - data = json.load(open(jsonfile, encoding='utf-8')) + data = json.loads(jsondata) plugin = data.get('plugin', g) try: @@ -937,9 +980,6 @@ def load_games(game='*'): print('Error loading %s:\n' % jsonfile) raise - print('\r%s\r' % (' ' * len(games)), end='', flush=True, file=sys.stderr) - return games - def run_command_line(): logger.debug('Arguments: %r', sys.argv) diff --git a/game_data_packager/paths.py b/game_data_packager/paths.py index 0cba549..417ca31 100644 --- a/game_data_packager/paths.py +++ b/game_data_packager/paths.py @@ -6,7 +6,9 @@ if os.environ.get('GDP_UNINSTALLED'): CONFIG = './etc/game-data-packager.conf' DATADIR = './out' ETCDIR = './etc' + USE_VFS = False else: CONFIG = '/etc/game-data-packager.conf' DATADIR = '/usr/share/games/game-data-packager' ETCDIR = '/etc/game-data-packager' + USE_VFS = True -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git _______________________________________________ Pkg-games-commits mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-games-commits
