At http://people.ubuntu.com/~robertc/baz2.0/plugins/compressbench/trunk
------------------------------------------------------------ revno: 4 revision-id: [email protected] parent: [email protected] committer: Robert Collins <[email protected]> branch nick: trunk timestamp: Thu 2009-01-22 12:08:18 +1100 message: Sort corpus by reverse size and allow a size cap. === modified file 'bench.py' --- a/bench.py 2009-01-20 03:18:51 +0000 +++ b/bench.py 2009-01-22 01:08:18 +0000 @@ -66,15 +66,17 @@ no parent information, None (as opposed to () for an empty list of parents). :ivar _path: The path on disk of the file containing the content. + :ivar _length: The length of the content. """ - def __init__(self, key, parents, sha1, path): + def __init__(self, key, parents, sha1, path, length): """Create a DiskContentFactory.""" self.sha1 = sha1 self.storage_kind = 'fulltext' self.key = key self.parents = parents self._path = path + self._length = length def get_bytes_as(self, storage_kind): # this probably needs to measure the time to read the content to @@ -97,20 +99,28 @@ takes_options = [ ListOption('delta', type=str, help='Compressors to test. (gc, knit)'), + Option('limit', type=int, help='Cap the corpus at limit bytes'), ] - def run(self, delta=None): + def run(self, delta=None, limit=None): if not delta: delta = ['gc', 'knit'] # extract source to full text files on disk with metadata held in memory source_repo = Repository.open('/home/robertc/source/baz') print "Extracting corpus to test with" - total_bytes, stream, cleanup = self.make_corpus(source_repo) + total_bytes, stream, cleanup = self.make_corpus(source_repo, limit) try: print "Corpus size", total_bytes, "bytes in", len(stream), "texts" + # Create an arbitrary compression order: # set arbitrary parents to match the compression order (avoids having # to change knits to handle it better during these benchmarks). - # compress the contents + # --- this sorts by largest->smallest text --- + print "Size sorted corpus" + stream.sort(key=lambda x:x._length, reverse=True) + parents = () + for factory in stream: + factory.parents = parents + parents = (factory.key,) # Now for each compression type benchmark it: factories = {'knit':make_pack_factory, 'gc':make_gc_factory} for label in delta: @@ -182,9 +192,10 @@ sd = math.sqrt(sum((time-mean)**2 for time in per_k_times) / count) print "dev", sd - def make_corpus(self, repo): + def make_corpus(self, repo, limit): """Create a corpus to compress as a record_stream. + :param limit: Byte count to stop adding to the corpus at. :return: a_stream, a_cleanup_callback """ total_bytes = 0 @@ -201,11 +212,14 @@ 'unordered', True): bytes = record.get_bytes_as('fulltext') t.put_bytes("%s" % serial, bytes) - result.append(DiskContentFactory( + factory = DiskContentFactory( record.key, record.parents, record.sha1, - "%s/%s" % (workingdir, serial))) - total_bytes += len(bytes) + "%s/%s" % (workingdir, serial), len(bytes)) + result.append(factory) + total_bytes += factory._length serial += 1 + if limit and total_bytes > limit: + break finally: repo.unlock() except: -- bazaar-commits mailing list [email protected] https://lists.ubuntu.com/mailman/listinfo/bazaar-commits
