This is an automated email from the git hooks/post-receive script. smcv pushed a commit to branch master in repository game-data-packager.
commit 592d963f0cb5d0dcd30fcb52b472635981486e99 Author: Simon McVittie <[email protected]> Date: Sun Nov 1 15:09:35 2015 +0000 Add a test that various forms of the game data are equivalent Also compare with ref.zip if it exists. This provides a way for maintainers to determine whether a "pure refactoring" change to the YAML has had any technical effect. --- Makefile | 1 + debian/changelog | 1 + game_data_packager/__init__.py | 53 +++++++++++++------ tools/check_equivalence.py | 117 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 155 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 54adba7..54f25fb 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,7 @@ clean: check: LC_ALL=C $(PYFLAKES3) game_data_packager/*.py game_data_packager/*/*.py runtime/*.py tools/*.py || : LC_ALL=C GDP_UNINSTALLED=1 PYTHONPATH=. python3 tools/check_syntax.py + LC_ALL=C GDP_UNINSTALLED=1 PYTHONPATH=. python3 tools/check_equivalence.py # Requires additional setup, so not part of "make check" manual-check: diff --git a/debian/changelog b/debian/changelog index ff35637..f4305cd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -17,6 +17,7 @@ game-data-packager (44) UNRELEASED; urgency=medium * quake, quake2: make aliases consistently strings * Always build vfs.zip, and optionally use it even when uninstalled * to_yaml: turn sets into sorted lists, and output more fields + * Add a test that various forms of the game data are equivalent -- Simon McVittie <[email protected]> Sun, 01 Nov 2015 11:10:56 +0100 diff --git a/game_data_packager/__init__.py b/game_data_packager/__init__.py index 609c959..12f0c88 100644 --- a/game_data_packager/__init__.py +++ b/game_data_packager/__init__.py @@ -27,6 +27,8 @@ import re import sys import zipfile +import yaml + from .build import (HashedFile, PackagingTask) from .config import read_config @@ -848,19 +850,22 @@ class GameData(object): if hexdigest is not None: setattr(f, alg, hexdigest) - def load_file_data(self): + def load_file_data(self, use_vfs=USE_VFS): if self.loaded_file_data: return logger.debug('loading full data') - if USE_VFS: - zip = os.path.join(DATADIR, 'vfs.zip') + if use_vfs: + if isinstance(use_vfs, str): + zip = use_vfs + else: + zip = os.path.join(DATADIR, 'vfs.zip') with zipfile.ZipFile(zip, 'r') as zf: files = zf.namelist() filename = '%s.files' % self.shortname if filename in files: - logger.debug('... vfs.zip/%s', filename) + logger.debug('... %s/%s', zip, filename) jsondata = zf.open(filename).read().decode('utf-8') data = json.loads(jsondata) self._populate_files(data) @@ -869,7 +874,7 @@ class GameData(object): filename = '%s.%s%s' % (self.shortname, alg, '' if alg == 'size_and_md5' else 'sums') if filename in files: - logger.debug('... vfs.zip/%s', filename) + logger.debug('... %s/%s', zip, filename) rawdata = zf.open(filename).read().decode('utf-8') for line in rawdata.splitlines(): self._add_hash(line.rstrip('\n'), alg) @@ -1003,40 +1008,54 @@ class GameData(object): return return gog.get('game', gog['url']) -def load_games(game='*'): +def load_games(game='*', use_vfs=USE_VFS, use_yaml=False): progress = (game == '*' and sys.stderr.isatty() and not logging.getLogger().isEnabledFor(logging.DEBUG)) games = {} - if USE_VFS: - zip = os.path.join(DATADIR, 'vfs.zip') + if use_vfs: + if isinstance(use_vfs, str): + zip = use_vfs + else: + zip = os.path.join(DATADIR, 'vfs.zip') + with zipfile.ZipFile(zip, 'r') as zf: if game == '*': for entry in zf.infolist(): if entry.filename.split('.')[-1] == 'json': - jsonfile = 'vfs.zip/' + entry.filename + jsonfile = '%s/%s' % (zip, entry.filename) jsondata = zf.open(entry).read().decode('utf-8') - load_json(progress, games, jsonfile, jsondata) + load_game(progress, games, jsonfile, jsondata) else: jsonfile = game + '.json' jsondata = zf.open(jsonfile).read().decode('utf-8') - load_json(progress, games, 'vfs.zip/' + jsonfile, jsondata) + load_game(progress, games, '%s/%s' % (zip, jsonfile), jsondata) + elif use_yaml: + for yamlfile in glob.glob(os.path.join('data/', game + '.yaml')): + yamldata = open(yamlfile, encoding='utf-8').read() + load_game(progress, games, yamlfile, yamldata) else: for jsonfile in glob.glob(os.path.join(DATADIR, game + '.json')): jsondata = open(jsonfile, encoding='utf-8').read() - load_json(progress, games, jsonfile, jsondata) + load_game(progress, games, jsonfile, jsondata) + + if progress: + print('\r%s\r' % (' ' * len(games)), end='', flush=True, file=sys.stderr) - print('\r%s\r' % (' ' * len(games)), end='', flush=True, file=sys.stderr) return games -def load_json(progress, games, jsonfile, jsondata): +def load_game(progress, games, filename, content): if progress: print('.', end='', flush=True, file=sys.stderr) try: - g = os.path.basename(jsonfile) + g = os.path.basename(filename) g = g[:len(g) - 5] - data = json.loads(jsondata) + if filename.endswith('.yaml'): + data = yaml.load(content, Loader=yaml.CLoader) + else: + data = json.loads(content) + plugin = data.get('plugin', g) plugin = plugin.replace('-', '_') @@ -1051,7 +1070,7 @@ def load_json(progress, games, jsonfile, jsondata): games[g] = game_data_constructor(g, data) except: - print('Error loading %s:\n' % jsonfile) + print('Error loading %s:\n' % filename) raise def run_command_line(): diff --git a/tools/check_equivalence.py b/tools/check_equivalence.py new file mode 100755 index 0000000..2679bab --- /dev/null +++ b/tools/check_equivalence.py @@ -0,0 +1,117 @@ +#!/usr/bin/python3 +# encoding=utf-8 +# +# Copyright © 2015 Simon McVittie <[email protected]> +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# You can find the GPL license text on a Debian system under +# /usr/share/common-licenses/GPL-2. + +import difflib +import json +import os +import sys +import time + +from game_data_packager import load_games +from game_data_packager.util import ascii_safe + +def dump(serialized): + return json.dumps(serialized, sort_keys=True, indent=2) + +if __name__ == '__main__': + games = '*' + + if len(sys.argv) > 1: + assert len(sys.argv) == 2 + games = sys.argv[1] + + if os.path.exists('ref.zip'): + t = time.process_time() + # usage: + # make + # cp out/vfs.zip ref.zip + # make + # make check + from_ref = load_games(games, use_vfs='ref.zip') + dt = time.process_time() - t + print('# loaded game data from ref.zip in %.3f seconds' % dt) + else: + from_ref = None + + t = time.process_time() + from_vfs = load_games(games, use_vfs=True) + dt = time.process_time() - t + print('# loaded game data from vfs.zip in %.3f seconds' % dt) + + t = time.process_time() + from_json = load_games(games, use_vfs=False) + dt = time.process_time() - t + print('# loaded game data from JSON in %.3f seconds' % dt) + + t = time.process_time() + from_yaml = load_games(games, use_vfs=False, use_yaml=True) + dt = time.process_time() - t + print('# loaded game data from YAML in %.3f seconds' % dt) + + assert set(from_vfs.keys()) == set(from_json.keys()) + assert set(from_vfs.keys()) == set(from_yaml.keys()) + + if from_ref is not None: + assert set(from_vfs.keys()) == set(from_ref.keys()) + + fail = False + + for (name, game) in sorted(from_vfs.items()): + print('# %s -----------------------------------------' % name) + + game.load_file_data() + ascii_safe(game.longname, force=True).encode('ascii') + ascii_safe(game.help_text, force=True).encode('ascii') + vfs_to_json = dump(game.to_yaml()) + + json_game = from_json[name] + json_game.load_file_data() + json_to_json = dump(json_game.to_yaml()) + + yaml_game = from_yaml[name] + yaml_game.load_file_data() + yaml_to_json = dump(yaml_game.to_yaml()) + + if yaml_to_json != vfs_to_json: + sys.stdout.writelines(difflib.unified_diff( + yaml_to_json.splitlines(True), + vfs_to_json.splitlines(True), + '%s loaded from YAML' % name, + '%s loaded from vfs.zip' % name, n=50)) + fail = True + + if json_to_json != vfs_to_json: + sys.stdout.writelines(difflib.unified_diff( + json_to_json.splitlines(True), + vfs_to_json.splitlines(True), + '%s loaded from JSON' % name, + '%s loaded from vfs.zip' % name, n=50)) + fail = True + + if from_ref is not None: + ref_game = from_ref[name] + ref_game.load_file_data(use_vfs='ref.zip') + ref_to_json = dump(ref_game.to_yaml()) + + if ref_to_json != vfs_to_json: + sys.stdout.writelines(difflib.unified_diff( + ref_to_json.splitlines(True), + vfs_to_json.splitlines(True), + '%s loaded from ref.zip' % name, + '%s loaded from vfs.zip' % name, n=50)) + + raise SystemExit(fail) -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git _______________________________________________ Pkg-games-commits mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-games-commits
