This is an automated email from the git hooks/post-receive script. smcv pushed a commit to branch master in repository game-data-packager.
commit bdff2960e4c311e72edcfd9d459a80f658c0852a Author: Simon McVittie <[email protected]> Date: Sun Nov 1 20:45:07 2015 +0000 Add support for named groups of files --- Makefile | 3 +- data/quake2.yaml | 54 ++++++--------- game_data_packager/__init__.py | 145 +++++++++++++++++++++++++++++++++++++++-- tools/yaml2json.py | 49 ++++++++++++++ 4 files changed, 209 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index 54f25fb..f4c0043 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ out/vfs.zip: $(json) rm -fr out/vfs mkdir out/vfs cp out/*.json out/*.files out/*.size_and_md5 out/*.cksums out/vfs/ - cp out/*.md5sums out/*.sha1sums out/*.sha256sums out/vfs/ + cp out/*.md5sums out/*.sha1sums out/*.sha256sums out/*.groups out/vfs/ if [ -n "$(BUILD_DATE)" ]; then \ touch --date='$(BUILD_DATE)' out/vfs/*; \ fi @@ -86,6 +86,7 @@ clean: rm -f ./out/*.copyright rm -f ./out/*.copyright.in rm -f ./out/*.files + rm -f ./out/*.groups rm -f ./out/*.md5sums rm -f ./out/*.preinst.in rm -f ./out/*.png diff --git a/data/quake2.yaml b/data/quake2.yaml index dd4e389..bc4400e 100644 --- a/data/quake2.yaml +++ b/data/quake2.yaml @@ -49,19 +49,7 @@ packages: install: - baseq2/pak0.pak optional: - # videos present in Steam and on smcv's Xplosiv-branded CD-ROM, but - # apparently not strictly necessary (#776059) - - baseq2/video/end.cin - - baseq2/video/eou1_.cin - - baseq2/video/eou2_.cin - - baseq2/video/eou3_.cin - - baseq2/video/eou4_.cin - - baseq2/video/eou5_.cin - - baseq2/video/eou6_.cin - - baseq2/video/eou7_.cin - - baseq2/video/eou8_.cin - - baseq2/video/idlog.cin - - baseq2/video/ntro.cin + - baseq2 videos # semi-official bonus content - baseq2/maps/base64.bsp - baseq2/maps/city64.bsp @@ -1562,17 +1550,7 @@ files: - baseq2/players/zumlin/zumlin_i.pcx - baseq2/players/zumlin/zumlinbase.pcx - baseq2/players/zumlin/zumlinstory.txt - - baseq2/video/end.cin - - baseq2/video/eou1_.cin - - baseq2/video/eou2_.cin - - baseq2/video/eou3_.cin - - baseq2/video/eou4_.cin - - baseq2/video/eou5_.cin - - baseq2/video/eou6_.cin - - baseq2/video/eou7_.cin - - baseq2/video/eou8_.cin - - baseq2/video/idlog.cin - - baseq2/video/ntro.cin + - baseq2 videos - ctf/pak0.pak - ctf/readme.txt - ctf/server.cfg @@ -1751,6 +1729,22 @@ files: # + action, arena, capture, chaos, eraser, jail, kick, pb2 & rover: # -> can not supported without source for a native game.so +groups: + baseq2 videos: | + # videos present in Steam and on smcv's Xplosiv-branded CD-ROM, but + # apparently not strictly necessary (#776059) + 19311290 36fdaddd1c1b56ba10472466e4486ff8 baseq2/video/end.cin + 6038210 cbab517cddc03ec676d7153eeb13417b baseq2/video/eou1_.cin + 6586381 37201aa9c798982e7739dfdbad61c004 baseq2/video/eou2_.cin + 6708418 f2b7fac58d5aa24bcf76ec942253dfcc baseq2/video/eou3_.cin + 5464002 b0e360a5c4a55789e00d3f2223e23dc4 baseq2/video/eou4_.cin + 6848133 76651dc4f2d92ed278ee8e2cac7c7600 baseq2/video/eou5_.cin + 7099332 0b14fff288b8d260f812e830ce6bff7c baseq2/video/eou6_.cin + 7766671 a342a79fadfe9efa2ce549842168ec7c baseq2/video/eou7_.cin + 11875656 24a5178220322b3a4fb42f6b0757b821 baseq2/video/eou8_.cin + 3159828 0747670d94cc873f8ce522d7652143a1 baseq2/video/idlog.cin + 82836235 72846e547415856028006aaa4089c9c9 baseq2/video/ntro.cin + cksums: | 1206005048 239924 quake2-rogue-2.02.tar.xz 285620057 1421 quake2-rogue-2.02/CHANGELOG @@ -1856,17 +1850,7 @@ cksums: | size_and_md5: | 183997730 1ec55a724dc3109fd50dde71ab581d70 baseq2/pak0.pak?1dd586a - 19311290 36fdaddd1c1b56ba10472466e4486ff8 baseq2/video/end.cin - 6038210 cbab517cddc03ec676d7153eeb13417b baseq2/video/eou1_.cin - 6586381 37201aa9c798982e7739dfdbad61c004 baseq2/video/eou2_.cin - 6708418 f2b7fac58d5aa24bcf76ec942253dfcc baseq2/video/eou3_.cin - 5464002 b0e360a5c4a55789e00d3f2223e23dc4 baseq2/video/eou4_.cin - 6848133 76651dc4f2d92ed278ee8e2cac7c7600 baseq2/video/eou5_.cin - 7099332 0b14fff288b8d260f812e830ce6bff7c baseq2/video/eou6_.cin - 7766671 a342a79fadfe9efa2ce549842168ec7c baseq2/video/eou7_.cin - 11875656 24a5178220322b3a4fb42f6b0757b821 baseq2/video/eou8_.cin - 3159828 0747670d94cc873f8ce522d7652143a1 baseq2/video/idlog.cin - 82836235 72846e547415856028006aaa4089c9c9 baseq2/video/ntro.cin + # Files from patch to install in game directory 1063 b2c3358d7be61f05651c88d9ef97d6aa baseq2/maps.lst 12992754 42663ea709b7cd3eb9b634b36cfecb1a baseq2/pak1.pak diff --git a/game_data_packager/__init__.py b/game_data_packager/__init__.py index dd7a293..40c8695 100644 --- a/game_data_packager/__init__.py +++ b/game_data_packager/__init__.py @@ -19,6 +19,7 @@ import argparse import glob import importlib +import io import json import logging import os @@ -54,6 +55,7 @@ class WantedFile(HashedFile): def __init__(self, name): super(WantedFile, self).__init__(name) self.alternatives = [] + self.group_members = set() self.distinctive_name = True self.distinctive_size = False self.download = None @@ -68,6 +70,22 @@ class WantedFile(HashedFile): self.unpack = None self.unsuitable = None + def apply_group_attributes(self, attributes): + for k, v in attributes.items(): + if k == 'doc': + if v: + self.install_to = '$docdir' + continue + + if k == 'license' and v: + self.install_to = '$docdir' + self.distinctive_name = False + self.license = v + continue + + assert hasattr(self, k) + setattr(self, k, v) + @property def look_for(self): if self.alternatives: @@ -107,6 +125,7 @@ class WantedFile(HashedFile): 'alternatives', 'distinctive_size', 'executable', + 'group_members', 'license', 'look_for', 'provides', @@ -497,6 +516,42 @@ class GameData(object): self.packages[binary] = package self._populate_package(package, data) + if 'groups' in self.data: + groups = self.data['groups'] + assert isinstance(groups, dict), self.shortname + for group_name, group_data in groups.items(): + attrs = {} + if isinstance(group_data, dict): + members = group_data['group_members'] + for k, v in group_data.items(): + if k != 'group_members': + attrs[k] = v + elif isinstance(group_data, (str, list)): + members = group_data + else: + raise AssertionError('group %r should be dict, str or list' % group_name) + + group = self._ensure_file(group_name) + group.apply_group_attributes(attrs) + + if isinstance(members, str): + for line in members.splitlines(): + f = self._add_hash(line.rstrip('\n'), 'size_and_md5') + if f is not None: + f.apply_group_attributes(attrs) + group.group_members.add(f.name) + elif isinstance(members, list): + for member_name in members: + f = self._ensure_file(member_name) + f.apply_group_attributes(attrs) + group.group_members.add(member_name) + else: + raise AssertionError('group %r members should be str or list' % group_name) + + # an empty group is no use, and would break the assumption + # that we can use f.group_members to detect groups + assert group.group_members + if 'size_and_md5' in self.data: for line in self.data['size_and_md5'].splitlines(): self._add_hash(line, 'size_and_md5') @@ -606,6 +661,7 @@ class GameData(object): def to_yaml(self): files = {} + groups = {} packages = {} def sort_set_values(d): @@ -615,7 +671,10 @@ class GameData(object): ret[k] = sorted(v) for filename, f in self.files.items(): - files[filename] = f.to_yaml() + if f.group_members: + groups[filename] = f.to_yaml() + else: + files[filename] = f.to_yaml() for name, package in self.packages.items(): packages[name] = package.to_yaml() @@ -631,6 +690,7 @@ class GameData(object): 'packages': packages, 'providers': sort_set_values(self.providers), 'files': files, + 'groups': groups, } def size(self, package): @@ -843,12 +903,38 @@ class GameData(object): f = self._ensure_file(filename) - if size is not None: + if size is not None and size != '_': f.size = int(size) - if hexdigest is not None: + if hexdigest is not None and hexdigest != '_': setattr(f, alg, hexdigest) + return f + + def _populate_groups(self, stream): + current_group = None + attributes = {} + + for line in stream: + stripped = line.strip() + if stripped == '' or stripped.startswith('#'): + continue + + if stripped.startswith('['): + assert stripped.endswith(']'), repr(stripped) + current_group = self._ensure_file(stripped[1:-1]) + attributes = {} + elif stripped.startswith('{'): + attributes = {} + attributes = json.loads(stripped) + assert current_group is not None + current_group.apply_group_attributes(attributes) + else: + f = self._add_hash(stripped, 'size_and_md5') + f.apply_group_attributes(attributes) + assert current_group is not None + current_group.group_members.add(f.name) + def load_file_data(self, use_vfs=USE_VFS): if self.loaded_file_data: return @@ -869,6 +955,12 @@ class GameData(object): data = json.loads(jsondata) self._populate_files(data) + filename = '%s.groups' % self.shortname + if filename in files: + logger.debug('... %s/%s', zip, filename) + stream = io.TextIOWrapper(zf.open(filename), encoding='utf-8') + self._populate_groups(stream) + for alg in ('ck', 'md5', 'sha1', 'sha256', 'size_and_md5'): filename = '%s.%s%s' % (self.shortname, alg, '' if alg == 'size_and_md5' else 'sums') @@ -884,6 +976,12 @@ class GameData(object): data = json.load(open(filename, encoding='utf-8')) self._populate_files(data) + filename = os.path.join(DATADIR, '%s.groups' % self.shortname) + if os.path.isfile(filename): + logger.debug('... %s', filename) + stream = open(filename, encoding='utf-8') + self._populate_groups(stream) + for alg in ('ck', 'md5', 'sha1', 'sha256', 'size_and_md5'): filename = os.path.join(DATADIR, '%s.%s%s' % (self.shortname, alg, @@ -902,11 +1000,16 @@ class GameData(object): if filename not in package.optional: package.install.add(filename) + package.install = set(self._iter_expand_groups(package.install)) + package.optional = set(self._iter_expand_groups(package.optional)) + for filename, f in self.files.items(): + f.provides = set(self._iter_expand_groups(f.provides)) + for provided in f.provides: self.providers.setdefault(provided, set()).add(filename) - if f.alternatives: + if f.alternatives or f.group_members: continue if f.distinctive_size and f.size is not None: @@ -982,8 +1085,12 @@ class GameData(object): list), filename if wanted.alternatives: - for alt in wanted.alternatives: - assert alt in self.files, alt + for alt_name in wanted.alternatives: + alt = self.files[alt_name] + # an alternative can't be a placeholder for alternatives + assert not alt.alternatives, alt_name + # an alternative can't be a placeholder for a group + assert not alt.group_members, alt_name # if this is a placeholder for a bunch of alternatives, then # it doesn't make sense for it to have a defined checksum @@ -992,12 +1099,38 @@ class GameData(object): assert wanted.sha1 is None, wanted.name assert wanted.sha256 is None, wanted.name assert wanted.size is None, wanted.name + + # a placeholder for alternatives can't also be a placeholder + # for a group + assert not wanted.group_members, wanted.name + elif wanted.group_members: + for member_name in wanted.group_members: + assert member_name in self.files + + assert wanted.md5 is None, wanted.name + assert wanted.sha1 is None, wanted.name + assert wanted.sha256 is None, wanted.name + assert wanted.size is None, wanted.name + assert not wanted.unpack, wanted.unpack # FIXME: find out file size and add to yaml else: assert (wanted.size is not None or filename in self.data.get('unknown_sizes', ()) ), (self.shortname, wanted.name) + def _iter_expand_groups(self, grouped): + """Given a set of strings that are either filenames or groups, + yield the members of those groups, recursively. + """ + for filename in grouped: + wanted = self.files.get(filename) + assert wanted is not None, filename + if wanted.group_members: + for x in self._iter_expand_groups(wanted.group_members): + yield x + else: + yield filename + def construct_task(self, **kwargs): self.load_file_data() return PackagingTask(self, **kwargs) diff --git a/tools/yaml2json.py b/tools/yaml2json.py index 8c0d1c0..a0efc88 100755 --- a/tools/yaml2json.py +++ b/tools/yaml2json.py @@ -40,6 +40,55 @@ def main(f, out): elif os.path.isfile(offload): os.remove(offload) + groups = data.pop('groups', None) + offload = os.path.splitext(out)[0] + '.groups' + + if groups is not None: + with open(offload + '.tmp', 'w', encoding='utf-8') as writer: + assert isinstance(groups, dict) + for group_name, group_data in groups.items(): + writer.write('[%s]\n' % group_name) + + if isinstance(group_data, dict): + attrs = {} + members = group_data['group_members'] + for k, v in group_data.items(): + if k != 'group_members': + attrs[k] = v + if attrs: + json.dump(attrs, writer, sort_keys=True) + writer.write('\n') + elif isinstance(group_data, (str, list)): + members = group_data + else: + raise AssertionError('group %r should be dict, str or list' % group_name) + + has_members = False + + if isinstance(members, str): + for line in members.splitlines(): + assert not line.startswith('[') + assert not line.startswith('{') + line = line.strip() + if line and not line.startswith('#'): + has_members = True + writer.write(' '.join(line.split())) + writer.write('\n') + elif isinstance(members, list): + for m in members: + has_members = True + writer.write('? ? %s\n' % m) + else: + raise AssertionError('group %r members should be str or list' % group_name) + + # an empty group is no use, and would break the assumption + # that we can use f.group_members to detect groups + assert has_members + + os.rename(offload + '.tmp', offload) + elif os.path.isfile(offload): + os.remove(offload) + for k in ('cksums', 'sha1sums', 'sha256sums', 'md5sums', 'size_and_md5'): v = data.pop(k, None) -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git _______________________________________________ Pkg-games-commits mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-games-commits
