This is an automated email from the git hooks/post-receive script. smcv pushed a commit to branch master in repository game-data-packager.
commit e079ec622104fb20c8bb146db9dbef4eaf6e9b46 Author: Simon McVittie <[email protected]> Date: Mon Jan 12 10:10:08 2015 +0000 Redo logic for files with alternatives to avoid unnecessary warnings (Closes: #775152) --- data/doom.yaml | 10 +- data/doom2.yaml | 14 +- data/final-doom.yaml | 6 - data/heretic.yaml | 4 - data/hexen.yaml | 3 - data/rtcw.yaml | 3 - debian/changelog | 2 + lib/game_data_packager/__init__.py | 272 ++++++++++++++++++++++++--------- lib/game_data_packager/check_syntax.py | 8 +- 9 files changed, 217 insertions(+), 105 deletions(-) diff --git a/data/doom.yaml b/data/doom.yaml index afd45d3..88c584c 100644 --- a/data/doom.yaml +++ b/data/doom.yaml @@ -34,11 +34,13 @@ files: alternatives: - doom.wad_1.9ud - doom.wad_1.9 - # presumably some of the other alternatives would be OK too + - doom.wad_xbox + - doom.wad_psn + - doom.wad_bfg + - doom1.wad_pocketpc # Best available full version: The Ultimate Doom doom.wad_1.9ud: - distinctive_name: false size: 12408292 sha1: 9b07b02ab3c275a6a7570c3f73cc20d63a0e3833 md5: c4fe9fd920207691a9f493668e0a2083 @@ -47,28 +49,24 @@ files: # Other 1.9 variants doom.wad_1.9: - distinctive_name: false size: 11159840 sha1: 7742089b4468a736cadb659a7deca3320fe6dcbd md5: 1cd63c5ddff1bf8ce844237f580e9cf3 look_for: - doom.wad doom.wad_xbox: - distinctive_name: false size: 12538385 md5: 0c8758f102ccafe26a3040bee8ba5021 sha1: 1d1d4f69fe14fa255228d8243470678b1b4efdc5 look_for: - doom.wad doom.wad_psn: - distinctive_name: false size: 12474561 md5: e4f120eab6fb410a5b6e11c947832357 sha1: 117015379c529573510be08cf59810aa10bb934e look_for: - doom.wad doom.wad_bfg: - distinctive_name: false size: 12487824 md5: fb35c4a5a9fd49ec29ab6e900572c524 sha1: e5ec79505530e151ff0e6f517f3ce1fd65969c46 diff --git a/data/doom2.yaml b/data/doom2.yaml index cb05841..df1266c 100644 --- a/data/doom2.yaml +++ b/data/doom2.yaml @@ -25,16 +25,19 @@ files: doom2.wad: alternatives: - doom2.wad_1.9 - # FIXME: do older wads work? + - doom2.wad_bfg + - doom2.wad_xbox360_bfg + - doom2.wad_xbla + - doom2.wad_xbox_roe + - doom2.wad_psn + - doom2.wad_tapwave_zodiac doom2.wad_1.9: size: 14604584 look_for: [doom2.wad] - distinctive_name: false doom2.wad_bfg: size: 14691821 look_for: [doom2.wad] - distinctive_name: false doom2.wad_1.666g: size: 14824716 look_for: [doom2.wad] @@ -58,23 +61,18 @@ files: doom2.wad_xbox360_bfg: size: 14677988 look_for: [doom2.wad] - distinctive_name: false doom2.wad_xbla: size: 14685034 look_for: [doom2.wad] - distinctive_name: false doom2.wad_xbox_roe: size: 14683458 look_for: [doom2.wad] - distinctive_name: false doom2.wad_psn: size: 14599800 look_for: [doom2.wad] - distinctive_name: false doom2.wad_tapwave_zodiac: size: 14639397 look_for: [doom2.wad] - distinctive_name: false md5sums: | 25e1459ca71d321525f84628f45ca8cd doom2.wad_1.9 diff --git a/data/final-doom.yaml b/data/final-doom.yaml index 707f36e..4dd648c 100644 --- a/data/final-doom.yaml +++ b/data/final-doom.yaml @@ -36,7 +36,6 @@ files: - tnt.wad_psn tnt.wad_id_anthology: - distinctive_name: false size: 18654796 sha1: 4a65c8b960225505187c36040b41a40b152f8f3e md5: 1d39e405bf6ee3df69a8d2646c8d5c49 @@ -44,7 +43,6 @@ files: - tnt.wad tnt.wad_1.9: - distinctive_name: false size: 18195736 sha1: 9fbc66aedef7fe3bae0986cdb9323d2b8db4c9d3 md5: 4e158d9953c79ccf97bd0663244cc6b6 @@ -52,7 +50,6 @@ files: - tnt.wad tnt.wad_psn: - distinctive_name: false size: 18222568 sha1: 139e26d801a64b404b8d898defca10227a61867b md5: be626c12b7c9d94b1dfb9c327566b4ff @@ -66,7 +63,6 @@ files: - plutonia.wad_psn plutonia.wad_id_anthology: - distinctive_name: false size: 18240172 sha1: f131cbe1946d7fddb3caec4aa258c83399c21e60 md5: 3493be7e1e2588bc9c8b31eab2587a04 @@ -74,7 +70,6 @@ files: - plutonia.wad plutonia.wad_1.9: - distinctive_name: false size: 17420824 sha1: 90361e2a538d2388506657252ae41aceeb1ba360 md5: 75c8cf89566741fa9d22447604053bd7 @@ -82,7 +77,6 @@ files: - plutonia.wad plutonia.wad_psn: - distinctive_name: false size: 17417800 sha1: 327f8c41ebd4138354e9fca63cebbbd1b9489749 md5: b77ca6a809c4fae086162dad8e7a1335 diff --git a/data/heretic.yaml b/data/heretic.yaml index 15f6cfa..e6e188d 100644 --- a/data/heretic.yaml +++ b/data/heretic.yaml @@ -50,13 +50,10 @@ files: # format: zip heretic.wad: - distinctive_name: false alternatives: - heretic.wad_1.3 - # FIXME: do older wads work? heretic.wad_1.3: - distinctive_name: false size: 14189976 look_for: [heretic.wad] @@ -71,7 +68,6 @@ files: look_for: [heretic.wad] heretic1.wad_1.2: - distinctive_name: false size: 5120920 look_for: [heretic1.wad] diff --git a/data/hexen.yaml b/data/hexen.yaml index 92f0db7..5e65294 100644 --- a/data/hexen.yaml +++ b/data/hexen.yaml @@ -71,17 +71,14 @@ files: look_for: [hexen.wad] hexdd.wad_1.1: - distinctive_name: false size: 4440584 look_for: [hexdd.wad] hexdd.wad_1.0: - distinctive_name: false size: 4429700 look_for: [hexdd.wad] hexen.wad_1.1: - distinctive_name: false size: 20083672 look_for: [hexen.wad] diff --git a/data/rtcw.yaml b/data/rtcw.yaml index ddb2333..a1b4177 100644 --- a/data/rtcw.yaml +++ b/data/rtcw.yaml @@ -48,7 +48,6 @@ packages: install_files: main/scripts/translation.cfg_141_unix: - distinctive_name: false look_for: - main/scripts/translation.cfg install_as: main/scripts/translation.cfg @@ -77,13 +76,11 @@ files: - main/sp_pak1.pk3_fr main/sp_pak1.pk3_en: - distinctive_name: false look_for: - main/sp_pak1.pk3 size: 293887431 main/sp_pak1.pk3_fr: - distinctive_name: false look_for: - main/sp_pak1.pk3 size: 256811934 diff --git a/debian/changelog b/debian/changelog index c7da387..089b740 100644 --- a/debian/changelog +++ b/debian/changelog @@ -34,6 +34,8 @@ game-data-packager (39) UNRELEASED; urgency=medium * Add support for Hexen: Deathkings of the Dark Citadel, loosely based on patches by Johey Shmit (partially addresses #737137) * Switch Doom packages' icons to .png, GNOME Shell doesn't like .xpm + * Redo logic for files with alternatives to avoid unnecessary warnings + (Closes: #775152) -- Simon McVittie <[email protected]> Mon, 05 Jan 2015 19:38:04 +0000 diff --git a/lib/game_data_packager/__init__.py b/lib/game_data_packager/__init__.py index 50ff6f9..3bacd9a 100644 --- a/lib/game_data_packager/__init__.py +++ b/lib/game_data_packager/__init__.py @@ -112,6 +112,12 @@ class HashedFile(object): self.sha256 = sha256.hexdigest() return self + @property + def have_hashes(self): + return ((self.md5 is not None) or + (self.sha1 is not None) or + (self.sha256 is not None)) + def matches(self, other): matched = False @@ -330,19 +336,36 @@ class GameData(object): assert 'install_files_from_cksums' not in self.yaml # Map from WantedFile name to instance. - # { 'baseq3/pak1.pk3' => WantedFile instance } + # { 'baseq3/pak1.pk3': WantedFile instance } self.files = {} # Map from WantedFile name to a set of names of WantedFile instances # from which the file named in the key can be extracted or generated. - # { 'baseq3/pak1.pk3' => set(['linuxq3apoint-1.32b-3.x86.run']) } + # { 'baseq3/pak1.pk3': set(['linuxq3apoint-1.32b-3.x86.run']) } self.providers = {} # Map from WantedFile name to the absolute or relative path of # a matching file on disk. - # { 'baseq3/pak1.pk3' => '/usr/share/games/quake3/baseq3/pak1.pk3' } + # { 'baseq3/pak1.pk3': '/usr/share/games/quake3/baseq3/pak1.pk3' } self.found = {} + # Map from WantedFile look_for name to a set of names of WantedFile + # instances which might be it + # { 'doom2.wad': set(['doom2.wad_1.9', 'doom2.wad_bfg', ...]) } + self.known_filenames = {} + + # Map from WantedFile size to a set of names of WantedFile + # instances which might be it + # { 14604584: set(['doom2.wad_1.9']) } + self.known_sizes = {} + + # Maps from md5, sha1, sha256 to the name of a unique + # WantedFile instance + # { '25e1459...': 'doom2.wad_1.9' } + self.known_md5s = {} + self.known_sha1s = {} + self.known_sha256s = {} + # Failed downloads self.download_failed = set() @@ -381,6 +404,34 @@ class GameData(object): for provided in f.provides: self.providers.setdefault(provided, set()).add(filename) + if f.alternatives: + continue + + if f.distinctive_size and f.size is not None: + self.known_sizes.setdefault(f.size, set()).add(filename) + + if f.distinctive_name: + for lf in f.look_for: + self.known_filenames.setdefault(lf, set()).add(filename) + + if f.md5 is not None: + if self.known_md5s.get(f.md5): + logger.warning('md5 %s matches %s and also %s' % + (f.md5, self.known_md5s[f.md5], filename)) + self.known_md5s[f.md5] = filename + + if f.sha1 is not None: + if self.known_sha1s.get(f.sha1): + logger.warning('sha1 %s matches %s and also %s' % + (f.sha1, self.known_sha1s[f.sha1], filename)) + self.known_sha1s[f.sha1] = filename + + if f.sha256 is not None: + if self.known_sha256s.get(f.sha256): + logger.warning('sha256 %s matches %s and also %s' % + (f.sha256, self.known_sha256s[f.sha256], filename)) + self.known_sha256s[f.sha256] = filename + if 'compress_deb' in self.yaml: self.compress_deb = self.yaml['compress_deb'] @@ -438,6 +489,8 @@ class GameData(object): files = {} providers = {} packages = {} + known_filenames = {} + known_sizes = {} for filename, f in self.files.items(): files[filename] = f.to_yaml() @@ -445,11 +498,22 @@ class GameData(object): for provided, by in self.providers.items(): providers[provided] = list(by) + for size, known in self.known_sizes.items(): + known_sizes[size] = list(known) + + for filename, known in self.known_filenames.items(): + known_filenames[filename] = list(known) + for name, package in self.packages.items(): packages[name] = package.to_yaml() return { 'help_text': self.help_text, + 'known_filenames': known_filenames, + 'known_md5s': self.known_md5s, + 'known_sha1s': self.known_sha1s, + 'known_sha256s': self.known_sha256s, + 'known_sizes': known_sizes, 'packages': packages, 'providers': providers, 'files': files, @@ -539,11 +603,11 @@ class GameData(object): return self.files[name] - def use_file(self, wanted, path, hashes=None): + def use_file(self, wanted, path, hashes=None, log=True): logger.debug('found possible %s at %s', wanted.name, path) size = os.stat(path).st_size if wanted.size is not None and wanted.size != size: - if wanted.distinctive_name: + if log: logger.warning('found possible %s\n' + 'but its size does not match:\n' + ' file: %s\n' + @@ -562,7 +626,8 @@ class GameData(object): progress=(size > QUITE_LARGE)) if not wanted.skip_hash_matching and not hashes.matches(wanted): - logger.warning('found possible %s\n' + + if log: + logger.warning('found possible %s\n' + 'but its checksums do not match:\n' + ' file: %s\n' + ' expected:\n' + @@ -587,45 +652,100 @@ class GameData(object): self.found[wanted.name] = path return True + def _ensure_hashes(self, hashes, path, size): + if hashes is not None: + return hashes + + if size > QUITE_LARGE: + logger.info('identifying %s', path) + return HashedFile.from_file(path, open(path, 'rb'), size=size, + progress=(size > QUITE_LARGE)) + def consider_file(self, path, really_should_match_something): if not os.path.exists(path): # dangling symlink return + tried = set() + match_path = '/' + path.lower() size = os.stat(path).st_size + # if a file (as opposed to a directory) is specified on the + # command-line, try harder to match it to something if really_should_match_something: - if size > QUITE_LARGE: - logger.info('identifying %s', path) - hashes = HashedFile.from_file(path, open(path, 'rb'), size=size, - progress=(size > QUITE_LARGE)) + hashes = self._ensure_hashes(None, path, size) else: hashes = None - for wanted in self.files.values(): - if wanted.alternatives: - continue - - for lf in wanted.look_for: - if match_path.endswith('/' + lf): - self.use_file(wanted, path, hashes) - if wanted.distinctive_name: + for look_for, candidates in self.known_filenames.items(): + if match_path.endswith('/' + look_for): + hashes = self._ensure_hashes(hashes, path, size) + for wanted_name in candidates: + if wanted_name in tried: + continue + tried.add(wanted_name) + if self.use_file(self.files[wanted_name], path, hashes, + log=(len(candidates) == 1)): + return + else: + if len(candidates) > 1: + self._log_not_any_of(path, size, hashes, + 'possible "%s"' % look_for, + [self.files[c] for c in candidates]) + + if size in self.known_sizes: + hashes = self._ensure_hashes(hashes, path, size) + candidates = self.known_sizes[size] + for wanted_name in candidates: + if wanted_name in tried: + continue + tried.add(wanted_name) + if self.use_file(self.files[wanted_name], path, hashes, + log=(len(candidates) == 1)): + return + else: + if len(candidates) > 1: + self._log_not_any_of(path, size, hashes, + 'file of size %d' % size, + [self.files[c] for c in candidates]) + + if hashes is not None: + for wanted_name in (self.known_md5s.get(hashes.md5), + self.known_sha1s.get(hashes.sha1), + self.known_sha256s.get(hashes.sha256)): + if wanted_name is not None and wanted_name not in tried: + tried.add(wanted_name) + if self.use_file(self.files[wanted_name], path, hashes): return - if wanted.distinctive_size: - if wanted.size == size: - logger.debug('... matched by distinctive size %d', size) - self.use_file(wanted, path, hashes) + if really_should_match_something: + logger.warning('file "%s" does not match any known file', path) + + def _log_not_any_of(self, path, size, hashes, why, candidates): + message = ('found %s but it is not one of the expected ' + + 'versions:\n' + + ' file: %s\n' + + ' size: %d bytes\n' + + ' md5: %s\n' + + ' sha1: %s\n' + + ' sha256: %s\n' + + 'expected one of:\n') + args = (why, path, size, hashes.md5, hashes.sha1, hashes.sha256) + + for candidate in candidates: + message = message + (' %s:\n' + + ' size: ' + ( + '%s' if candidate.size is None else '%d bytes') + + '\n' + + ' md5: %s\n' + + ' sha1: %s\n' + + ' sha256: %s\n') + args = args + (candidate.name, candidate.size, candidate.md5, + candidate.sha1, candidate.sha256) + + logger.warning(message, *args) - if hashes is not None: - if not wanted.skip_hash_matching and hashes.matches(wanted): - logger.debug('... matched hashes of %s', wanted.name) - self.use_file(wanted, path, hashes) - return - else: - if really_should_match_something: - logger.warning('file "%s" does not match any known file', path) def consider_file_or_dir(self, path): if os.path.isfile(path): @@ -659,21 +779,13 @@ class GameData(object): for filename in package.install: if filename not in self.found: wanted = self.files[filename] - alt_possible = False for alt in wanted.alternatives: - logger.debug('trying alternative: %s', alt) if alt in self.found: - alt_possible = True break - elif self.fill_gap(self.files[alt], download=download, - log=log): - alt_possible = True - - if alt_possible: - pass - elif not self.fill_gap(wanted, download=download, log=log): - possible = False + else: + if not self.fill_gap(wanted, download=download, log=log): + possible = False return possible @@ -829,6 +941,38 @@ class GameData(object): logger.debug('could not find %s, trying to derive it...', wanted.name) + if wanted.alternatives: + for alt in wanted.alternatives: + if alt in self.found: + return True + elif self.fill_gap(self.files[alt], download=download, + log=False): + return True + + if log: + logger.error('could not find a suitable version of %s:', + wanted.name) + + for alt in wanted.alternatives: + alt = self.files[alt] + logger.error('%s:\n' + + ' expected:\n' + + ' size: ' + ( + '%s' if alt.size is None else '%d bytes') + + '\n' + + ' md5: %s\n' + + ' sha1: %s\n' + + ' sha256: %s', + alt.name, + alt.size, + alt.md5, + alt.sha1, + alt.sha256) + + return False + + # no alternatives: try getting the file itself + possible = False if wanted.download: @@ -932,39 +1076,19 @@ class GameData(object): if not possible: if log: - if wanted.alternatives: - logger.error('could not find any version of %s:', - wanted.name) - - for alt in wanted.alternatives: - alt = self.files[alt] - logger.error('%s:\n' + - ' expected:\n' + - ' size: ' + ( - '%s' if alt.size is None else '%d bytes') + - '\n' + - ' md5: %s\n' + - ' sha1: %s\n' + - ' sha256: %s', - alt.name, - alt.size, - alt.md5, - alt.sha1, - alt.sha256) - else: - logger.error('could not find %s:\n' + - ' expected:\n' + - ' size: ' + ( - '%s' if wanted.size is None else '%d bytes') + - '\n' + - ' md5: %s\n' + - ' sha1: %s\n' + - ' sha256: %s', - wanted.name, - wanted.size, - wanted.md5, - wanted.sha1, - wanted.sha256) + logger.error('could not find %s:\n' + + ' expected:\n' + + ' size: ' + ( + '%s' if wanted.size is None else '%d bytes') + + '\n' + + ' md5: %s\n' + + ' sha1: %s\n' + + ' sha256: %s', + wanted.name, + wanted.size, + wanted.md5, + wanted.sha1, + wanted.sha256) return False diff --git a/lib/game_data_packager/check_syntax.py b/lib/game_data_packager/check_syntax.py index 6c1f86f..969826e 100644 --- a/lib/game_data_packager/check_syntax.py +++ b/lib/game_data_packager/check_syntax.py @@ -15,7 +15,13 @@ # You can find the GPL license text on a Debian system under # /usr/share/common-licenses/GPL-2. +import os +import yaml + from . import load_yaml_games if __name__ == '__main__': - load_yaml_games() + for name, game in load_yaml_games().items(): + if 'DEBUG' in os.environ: + print('# %s -----------------------------------------' % name) + print(yaml.safe_dump(game.to_yaml())) -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git _______________________________________________ Pkg-games-commits mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-games-commits
