[gentoo-commits] proj/catalyst:master commit in: doc/, catalyst/base/, catalyst/

2021-01-23 Thread Matt Turner 
commit: 87b0588ab4f77e413c250a4a3e357624ec41c374
Author: Matt Turner  gentoo  org>
AuthorDate: Mon Jan 18 16:52:20 2021 +
Commit: Matt Turner  gentoo  org>
CommitDate: Sat Jan 23 16:22:22 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=87b0588a

catalyst: Add option to enter the chroot before building

With --enter-chroot, after the mounts and environment are set up,
catalyst will drop you into a shell inside the chroot. Useful for
hacking or debugging.

Signed-off-by: Matt Turner  gentoo.org>

 catalyst/base/stagebase.py | 16 +++-
 catalyst/main.py   |  4 
 doc/catalyst.1.txt |  3 +++
 3 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 40b60af3..676206ff 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -20,7 +20,7 @@ from catalyst import log
 from catalyst.context import namespace
 from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
 from catalyst.support import (CatalystError, file_locate, normpath,
-  cmd, read_makeconf, get_repo_name, ismount,
+  cmd, command, read_makeconf, get_repo_name,
   file_check, sanitize_name)
 from catalyst.base.targetbase import TargetBase
 from catalyst.base.clearbase import ClearBase
@@ -95,6 +95,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
 self.chroot_setup,
 self.setup_environment,
 ]
+if 'enter-chroot' in self.settings['options']:
+self.build_sequence.append(self.enter_chroot)
+
 self.finish_sequence = []
 
 self.set_valid_build_kernel_vars(addlargs)
@@ -1326,6 +1329,17 @@ class StageBase(TargetBase, ClearBase, GenBase):
 
 log.debug('setup_environment(); env = %r', self.env)
 
+def enter_chroot(self):
+chroot = command('chroot')
+bash = command('bash')
+
+log.notice("Entering chroot")
+try:
+cmd([chroot, self.settings['chroot_path'], bash, '-l'],
+env=self.env)
+except CatalystError:
+pass
+
 def run(self):
 self.chroot_lock.write_lock()
 

diff --git a/catalyst/main.py b/catalyst/main.py
index 48daf004..b0d9015f 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -120,6 +120,8 @@ def get_parser():
 parser.add_argument('-V', '--version',
 action='version', version=get_version(),
 help='display version information')
+parser.add_argument('--enter-chroot', default=False, action='store_true',
+help='Enter chroot before starting the build')
 
 group = parser.add_argument_group('Program output options')
 group.add_argument('-d', '--debug',
@@ -293,6 +295,8 @@ def _main(parser, opts):
 options.append('purgetmponly')
 if opts.clear_autoresume:
 options.append('clear-autoresume')
+if opts.enter_chroot:
+options.append('enter-chroot')
 
 # Make sure we have some work before moving further.
 if not myspecfile and not mycmdline:

diff --git a/doc/catalyst.1.txt b/doc/catalyst.1.txt
index 90d5a24b..217fc86a 100644
--- a/doc/catalyst.1.txt
+++ b/doc/catalyst.1.txt
@@ -39,6 +39,9 @@ configuration file is installed at 
'/etc/catalyst/catalyst.conf'.
 *-d*::
 Enable debugging mode
 
+*--enter-chroot*::
+Enter the chroot before starting the build.
+
 *--fetchonly*::
 *-F*::
 This tells *catalyst* to only fetch distfiles for the given packages without

[gentoo-commits] proj/catalyst:master commit in: doc/, catalyst/base/, catalyst/targets/, catalyst/, /

2020-04-30 Thread Matt Turner 
commit: 5b29d4a88f492f6890d0574d0addefb9e6a13271
Author: Matt Turner  gentoo  org>
AuthorDate: Fri Apr 17 23:31:52 2020 +
Commit: Matt Turner  gentoo  org>
CommitDate: Thu Apr 30 23:04:34 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=5b29d4a8

catalyst: Make and use squashfs snapshots

There were a number of problems with catalyst's snapshot system. It was
built around using the build system's portdir and had no control over
what was in that portdir or when it was updated.

As a result, when a stage build failed, it was difficult to tell what
the snapshot consistet of precisely or whether it contained a particular
recent fix.

With snapcache disabled, ebuild repo snapshots were tar'd and compressed
and then unpacked into the stage chroot which is an unnecessarily
expensive process. Moreover, a porttree has more than 100k small files,
which are stored extremely inefficiently on most file systems—a whole
porttree is usually around 700M on disk. Just removing all of those
files during the cleaning stage is an expensive operation.

Instead, we make a compressed squashfs image and mount it in the build
chroot. The porttree has many duplicate files, and squashfs deduplicates
the files and then compresses, so the result is very efficiently packed:
~38M with gzip -9 compression.

The snapshot target has been modified to generate a squashfs image from
a bare ebuild git repo. Piping git-archive to tar2sqfs generates the
squashfs image in less than 10 seconds on a modern system. The git repo
is fetched with --depth=1 to minize bandwidth and disk usage, and git gc
is run after fetch to minimize disk usage. Storage requirements for the
git ebuild repo with metadata are ~70M.

The squashfs snapshot is stored in /var/tmp/catalyst/snapshots/ by
default with a name -.sqfs. With this convention,
we know the exact point in history that the snapshot was taken. The
catalyst-auto script can use the sha1 to get a deterministic timestamp,
so that it is independent on when `catalyst -s` was run, but is instead
the timestamp of the commit date of the repo's git SHA1.

Signed-off-by: Matt Turner  gentoo.org>

 README|   3 +-
 catalyst/base/stagebase.py|  88 
 catalyst/base/targetbase.py   |  16 +++-
 catalyst/defaults.py  |   2 +-
 catalyst/main.py  |  28 +++
 catalyst/targets/embedded.py  |   1 -
 catalyst/targets/livecd_stage1.py |   1 -
 catalyst/targets/livecd_stage2.py |   1 -
 catalyst/targets/netboot.py   |   1 -
 catalyst/targets/snapshot.py  | 165 +++---
 catalyst/targets/stage4.py|   1 -
 doc/catalyst-config.5.txt |  13 +--
 doc/catalyst-spec.5.txt   |   4 +-
 13 files changed, 139 insertions(+), 185 deletions(-)

diff --git a/README b/README
index 1a039fca..1cceb63e 100644
--- a/README
+++ b/README
@@ -18,8 +18,9 @@ Requirements
 ===
 
 - Python 3.6 or greater
-- An ebuild repository snapshot (or an ebuild tree to create one)
 - A generic stage3 tarball for your architecture
+- A squashfs ebuild repository snapshot
+  - Or an ebuild git repo with sys-fs/squashfs-tools-ng and dev-vcs/git
 
 What is catalyst?
 

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 9aecf013..41da97b3 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -35,7 +35,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 self.required_values |= frozenset([
 "profile",
 "rel_type",
-"snapshot",
+"snapshot_treeish",
 "source_subpath",
 "subarch",
 "target",
@@ -149,7 +149,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 self.set_source_subpath()
 
 # Set paths
-self.set_snapshot_path()
+self.set_snapshot()
 self.set_root_path()
 self.set_source_path()
 self.set_chroot_path()
@@ -191,9 +191,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
 # Setup our mount points.
 self.mount = MOUNT_DEFAULTS.copy()
 
-# Always unpack snapshot tarball
-self.mount['portdir']['enable'] = False
-
+self.mount['portdir']['source'] = self.snapshot
+self.mount['portdir']['target'] = self.settings['repo_basedir'] + '/' 
+ self.settings['repo_name']
 self.mount['distdir']['source'] = self.settings['distdir']
 self.mount["distdir"]['target'] = self.settings['target_distdir']
 
@@ -435,21 +434,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
 self.settings["destpath"] = normpath(self.settings["chroot_path"])
 
 def set_cleanables(self):
-self.settings["cleanables"] = ["/etc/resolv.conf", "/var/tmp/*", 
"/tmp/*",
-   self.settings["repo_basedir"] + "/" +
-

[gentoo-commits] proj/catalyst:master commit in: doc/, catalyst/base/

2020-04-30 Thread Matt Turner 
commit: c98bd2b223fab1a2784427d8b09fe8a682647bf4
Author: Matt Turner  gentoo  org>
AuthorDate: Wed Apr 22 00:57:34 2020 +
Commit: Matt Turner  gentoo  org>
CommitDate: Thu Apr 30 23:04:34 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=c98bd2b2

catalyst: Only write out non-default paths to make.conf

Signed-off-by: Matt Turner  gentoo.org>

 catalyst/base/stagebase.py | 11 +++
 doc/catalyst-config.5.txt  | 19 +--
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 1f091829..affdabbe 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -14,7 +14,7 @@ from snakeoil.osutils import pjoin
 from DeComp.compress import CompressMap
 
 from catalyst import log
-from catalyst.defaults import (MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
+from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
 from catalyst.support import (CatalystError, file_locate, normpath,
   cmd, read_makeconf, ismount, file_check)
 from catalyst.base.targetbase import TargetBase
@@ -1055,9 +1055,12 @@ class StageBase(TargetBase, ClearBase, GenBase):
 myf.write(hostuseexpand + '="' +
   ' '.join(myuseexpandvars[hostuseexpand]) + '"\n')
 
-myf.write('PORTDIR="%s"\n' % self.settings['target_portdir'])
-myf.write('DISTDIR="%s"\n' % self.settings['target_distdir'])
-myf.write('PKGDIR="%s"\n' % self.settings['target_pkgdir'])
+# Write non-default PORTDIR/DISTDIR/PKGDIR settings to make.conf
+for x in ['target_portdir', 'target_distdir', 'target_pkgdir']:
+if self.settings[x] != confdefaults[x]:
+varname = x.split('_')[1].upper()
+myf.write(f'{varname}="{self.settings[x]}"\n')
+
 if setup:
 # Setup the portage overlay
 if "portage_overlay" in self.settings:

diff --git a/doc/catalyst-config.5.txt b/doc/catalyst-config.5.txt
index b7d493eb..11b27d90 100644
--- a/doc/catalyst-config.5.txt
+++ b/doc/catalyst-config.5.txt
@@ -114,20 +114,19 @@ The name of the main repository (e.g. gentoo). The git 
repository at
 snapshot.
 
 *target_distdir*::
-This is the target distfiles directory location for the stage being created.
-This is important because this value will be stored in the stage's make.conf
-and will become the default location used if it is not edited by users.
-The default location is `/var/cache/distfiles`.
+Defines the location of the local source file repository in the
+target.  This will be written to the target's make.conf if it is not
+the default value of `/var/cache/distfiles`.
 
 *target_pkgdir*::
-This is the target packages directory for storing binpkgs in the stage being
-built.  This location is stored in the make.conf of the stage being built.
-The default location is `/var/cache/binpkgs`
+Defines the location of binary packages in the target.  This will be
+written to the target's make.conf if it is not the default value of
+`/var/cache/binpkgs`.
 
 *target_portdir*::
-Defines the location of main repository in the target stages.  This
-location is stored in make.conf of the stage being built.
-The default location is `/var/db/repos/gentoo`
+Defines the location of the main ebuild repository in the target.
+This will be written to the target's make.conf if it is not the
+default value of `/var/db/repos/gentoo`.
 
 Other settings
 ~~

[gentoo-commits] proj/catalyst:master commit in: doc/, catalyst/base/

2020-04-30 Thread Matt Turner 
commit: 73d70b4f433147d94d6c1e458fe2341dba1dcff7
Author: Matt Turner  gentoo  org>
AuthorDate: Wed Apr 22 00:57:34 2020 +
Commit: Matt Turner  gentoo  org>
CommitDate: Thu Apr 30 22:56:12 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=73d70b4f

catalyst: Only write out non-default paths to make.conf

Signed-off-by: Matt Turner  gentoo.org>

 catalyst/base/stagebase.py | 11 +++
 doc/catalyst-config.5.txt  | 19 +--
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 1f091829..affdabbe 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -14,7 +14,7 @@ from snakeoil.osutils import pjoin
 from DeComp.compress import CompressMap
 
 from catalyst import log
-from catalyst.defaults import (MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
+from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
 from catalyst.support import (CatalystError, file_locate, normpath,
   cmd, read_makeconf, ismount, file_check)
 from catalyst.base.targetbase import TargetBase
@@ -1055,9 +1055,12 @@ class StageBase(TargetBase, ClearBase, GenBase):
 myf.write(hostuseexpand + '="' +
   ' '.join(myuseexpandvars[hostuseexpand]) + '"\n')
 
-myf.write('PORTDIR="%s"\n' % self.settings['target_portdir'])
-myf.write('DISTDIR="%s"\n' % self.settings['target_distdir'])
-myf.write('PKGDIR="%s"\n' % self.settings['target_pkgdir'])
+# Write non-default PORTDIR/DISTDIR/PKGDIR settings to make.conf
+for x in ['target_portdir', 'target_distdir', 'target_pkgdir']:
+if self.settings[x] != confdefaults[x]:
+varname = x.split('_')[1].upper()
+myf.write(f'{varname}="{self.settings[x]}"\n')
+
 if setup:
 # Setup the portage overlay
 if "portage_overlay" in self.settings:

diff --git a/doc/catalyst-config.5.txt b/doc/catalyst-config.5.txt
index b7d493eb..11b27d90 100644
--- a/doc/catalyst-config.5.txt
+++ b/doc/catalyst-config.5.txt
@@ -114,20 +114,19 @@ The name of the main repository (e.g. gentoo). The git 
repository at
 snapshot.
 
 *target_distdir*::
-This is the target distfiles directory location for the stage being created.
-This is important because this value will be stored in the stage's make.conf
-and will become the default location used if it is not edited by users.
-The default location is `/var/cache/distfiles`.
+Defines the location of the local source file repository in the
+target.  This will be written to the target's make.conf if it is not
+the default value of `/var/cache/distfiles`.
 
 *target_pkgdir*::
-This is the target packages directory for storing binpkgs in the stage being
-built.  This location is stored in the make.conf of the stage being built.
-The default location is `/var/cache/binpkgs`
+Defines the location of binary packages in the target.  This will be
+written to the target's make.conf if it is not the default value of
+`/var/cache/binpkgs`.
 
 *target_portdir*::
-Defines the location of main repository in the target stages.  This
-location is stored in make.conf of the stage being built.
-The default location is `/var/db/repos/gentoo`
+Defines the location of the main ebuild repository in the target.
+This will be written to the target's make.conf if it is not the
+default value of `/var/db/repos/gentoo`.
 
 Other settings
 ~~

[gentoo-commits] proj/catalyst:master commit in: doc/, catalyst/base/, catalyst/, etc/, /

2020-04-17 Thread Matt Turner 
commit: 6565ad2eb90fe0840047d097fa5637d176a7d580
Author: Matt Turner  gentoo  org>
AuthorDate: Fri Apr 17 03:21:34 2020 +
Commit: Matt Turner  gentoo  org>
CommitDate: Fri Apr 17 17:20:18 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=6565ad2e

catalyst: Use hashlib instead of external tools

Signed-off-by: Matt Turner  gentoo.org>

 README |   1 -
 catalyst/base/genbase.py   |  18 ++-
 catalyst/base/stagebase.py |   8 +--
 catalyst/hash_utils.py | 126 -
 catalyst/main.py   |  42 ++-
 doc/catalyst-config.5.txt  |   8 +--
 etc/catalyst.conf  |  18 +++
 7 files changed, 35 insertions(+), 186 deletions(-)

diff --git a/README b/README
index eb75ba67..1a039fca 100644
--- a/README
+++ b/README
@@ -20,7 +20,6 @@ Requirements
 - Python 3.6 or greater
 - An ebuild repository snapshot (or an ebuild tree to create one)
 - A generic stage3 tarball for your architecture
-- shash for digest support
 
 What is catalyst?
 

diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py
index 08076460..632ee0d9 100644
--- a/catalyst/base/genbase.py
+++ b/catalyst/base/genbase.py
@@ -1,4 +1,5 @@
 
+import hashlib
 import io
 import os
 
@@ -11,6 +12,20 @@ class GenBase():
 def __init__(self, myspec):
 self.settings = myspec
 
+@staticmethod
+def generate_hash(filepath, name):
+h = hashlib.new(name)
+
+with open(filepath, 'rb') as f:
+while True:
+data = f.read(8192)
+if not data:
+break
+h.update(data)
+
+filename = os.path.split(filepath)[1]
+return f'# {name.upper()} HASH\n{h.hexdigest()}  {filename}\n'
+
 def gen_contents_file(self, path):
 contents = path + ".CONTENTS"
 if os.path.exists(contents):
@@ -29,11 +44,10 @@ class GenBase():
 if os.path.exists(digests):
 os.remove(digests)
 if "digests" in self.settings:
-hash_map = self.settings["hash_map"]
 if os.path.exists(path):
 with io.open(digests, "w", encoding='utf-8') as myf:
 for f in [path, path + '.CONTENTS']:
 if os.path.exists(f):
 for i in self.settings["digests"].split():
-digest = hash_map.generate_hash(f, hash_=i)
+digest = self.generate_hash(f, name=i)
 myf.write(digest)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 5f3fa1d0..71bf1ef9 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -442,9 +442,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 # XXX: Is this even necessary if the previous check passes?
 if os.path.exists(self.settings["source_path"]):
 self.settings["source_path_hash"] = \
-self.settings["hash_map"].generate_hash(
-self.settings["source_path"],
-hash_="sha1")
+self.generate_hash(self.settings["source_path"], 
"sha1")
 log.notice('Source path set to %s', self.settings['source_path'])
 
 def set_dest_path(self):
@@ -469,9 +467,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 )
 log.info('SNAPSHOT_PATH set to: %s', self.settings['snapshot_path'])
 self.settings["snapshot_path_hash"] = \
-self.settings["hash_map"].generate_hash(
-self.settings["snapshot_path"],
-hash_="sha1")
+self.generate_hash(self.settings["snapshot_path"], "sha1")
 
 def set_snapcache_path(self):
 self.settings["snapshot_cache_path"] = \

diff --git a/catalyst/hash_utils.py b/catalyst/hash_utils.py
deleted file mode 100644
index 3aae890e..
--- a/catalyst/hash_utils.py
+++ /dev/null
@@ -1,126 +0,0 @@
-
-import os
-from collections import namedtuple
-from subprocess import Popen, PIPE
-
-from catalyst import log
-from catalyst.support import CatalystError
-
-
-# Use HashMap.fields for the value legend
-# fields = ["func", "cmd", "args", "id"]
-HASH_DEFINITIONS = {
-"adler32"  :["calc_hash2", "shash", ["-a", "ADLER32"], "ADLER32"],
-"blake2"   :["calc_hash",  "b2sum", [ ], "BLAKE2"],
-"crc32":["calc_hash2", "shash", ["-a", "CRC32"], "CRC32"],
-"crc32b"   :["calc_hash2", "shash", ["-a", "CRC32B"], "CRC32B"],
-"gost" :["calc_hash2", "shash", ["-a", "GOST"], "GOST"],
-"haval128" :["calc_hash2", "shash", ["-a", "HAVAL128"], "HAVAL128"],
-"haval160" :["calc_hash2", "shash", ["-a", "HAVAL160"], "HAVAL160"],
-"haval192" :["calc_hash2", "shash", ["-a", "HAVAL192"], "HAVAL192"],
-"haval224" :["calc_hash2", "shash", ["-a", "HAVAL224"], "HAVAL224"],
-

[gentoo-commits] proj/catalyst:master commit in: doc/, catalyst/base/, catalyst/, etc/

2020-04-17 Thread Matt Turner 
commit: 45edc6510ce9cbf9cbe9ab5d2b7b8cd9f65f84cb
Author: Matt Turner  gentoo  org>
AuthorDate: Fri Apr 17 06:38:14 2020 +
Commit: Matt Turner  gentoo  org>
CommitDate: Fri Apr 17 17:03:47 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=45edc651

catalyst: Remove the 'hash_function' config option

Switch to SHA1, which is plenty fast. The next commit switches from the
external shash/b2sum tools to Python's hashlib, and the removal of this
config option will simplify that since the crc32 hash is in a different
python module (zlib).

Signed-off-by: Matt Turner  gentoo.org>

 catalyst/base/stagebase.py |  4 ++--
 catalyst/defaults.py   |  2 --
 catalyst/main.py   | 16 
 doc/catalyst-config.5.txt  |  6 --
 etc/catalyst.conf  | 10 --
 5 files changed, 2 insertions(+), 36 deletions(-)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index f673382f..5f3fa1d0 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -444,7 +444,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 self.settings["source_path_hash"] = \
 self.settings["hash_map"].generate_hash(
 self.settings["source_path"],
-hash_=self.settings["hash_function"])
+hash_="sha1")
 log.notice('Source path set to %s', self.settings['source_path'])
 
 def set_dest_path(self):
@@ -471,7 +471,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 self.settings["snapshot_path_hash"] = \
 self.settings["hash_map"].generate_hash(
 self.settings["snapshot_path"],
-hash_=self.settings["hash_function"])
+hash_="sha1")
 
 def set_snapcache_path(self):
 self.settings["snapshot_cache_path"] = \

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index e60980d3..f292c211 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -40,7 +40,6 @@ valid_config_file_values.extend([
 "digests",
 "distcc",
 "envscript",
-"hash_function",
 "options",
 "snapshot_cache",
 "VERBOSE",
@@ -63,7 +62,6 @@ confdefaults = {
 "decomp_opt": DECOMPRESSOR_PROGRAM_OPTIONS['linux'],
 "decompressor_search_order": DECOMPRESSOR_SEARCH_ORDER,
 "distdir": DISTDIR[:],
-"hash_function": "crc32",
 "icecream": "/var/cache/icecream",
 'list_xattrs_opt': LIST_XATTRS_OPTIONS['linux'],
 "local_overlay": REPODIR[:] + "/local",

diff --git a/catalyst/main.py b/catalyst/main.py
index ba6e3fcc..cb3cd3f7 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -384,22 +384,6 @@ def _main(parser, opts):
 # Now reload the config with our updated value.
 conf_values['digests'] = ' '.join(digests)
 
-if "hash_function" in conf_values:
-if conf_values["hash_function"] not in HASH_DEFINITIONS:
-log.critical(
-'%s is not a valid hash_function entry\n'
-'Valid hash_function entries:\n'
-'%s', conf_values["hash_function"], HASH_DEFINITIONS.keys())
-try:
-process.find_binary(
-hash_map.hash_map[conf_values["hash_function"]].cmd)
-except process.CommandNotFound:
-log.critical(
-'The "%s" binary needed by hash_function "%s" was not found. '
-'It needs to be in your system path.',
-hash_map.hash_map[conf_values['hash_function']].cmd,
-conf_values['hash_function'])
-
 addlargs = {}
 
 if myspecfile:

diff --git a/doc/catalyst-config.5.txt b/doc/catalyst-config.5.txt
index 220bb528..d5e8c128 100644
--- a/doc/catalyst-config.5.txt
+++ b/doc/catalyst-config.5.txt
@@ -66,12 +66,6 @@ variables using POSIX shell notation:
 export FOO="bar"
 -
 
-*hash_function*::
-Internal hash function catalyst should use for things like autoresume,
-seedcache, etc.  The default and fastest is `crc32`.  You should not
-ever need to change this unless your OS does not support it.  See the
-*SUPPORTED HASHES* section for a list of supported hashes.
-
 **options*::
 Set different build-time options (example: `autoresume bindist
 kerncache pkgcache seedcache snapcache`).  Supported values:

diff --git a/etc/catalyst.conf b/etc/catalyst.conf
index a3c22049..4c4d491e 100644
--- a/etc/catalyst.conf
+++ b/etc/catalyst.conf
@@ -24,16 +24,6 @@ distdir="/var/cache/distfiles"
 # export FOO="bar"
 envscript="/etc/catalyst/catalystrc"
 
-# Internal hash function catalyst should use for things like autoresume,
-# seedcache, etc.  The default and fastest is crc32.  You should not ever need
-# to change this unless your OS does not support it.
-# Supported hashes:
-# adler32, blake2, crc32, crc32b, gost, haval128, haval160, haval192, haval224,
-# haval256, md2, md4, md5, ripemd128, ripemd160, ripemd256,

[gentoo-commits] proj/catalyst:master commit in: doc/, catalyst/base/, catalyst/

2020-04-17 Thread Matt Turner 
commit: bb21b8615e64cb31fa9aa9d533ef328dc1374e45
Author: Matt Turner  gentoo  org>
AuthorDate: Fri Apr 17 18:03:02 2020 +
Commit: Matt Turner  gentoo  org>
CommitDate: Fri Apr 17 19:51:32 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=bb21b861

catalyst: gzip the .CONTENTS file

Other algorithms give better compression ratios, but the difference is
not meaningful for a 2MiB text file. In my testing bzip2 gave a better
compression ratio of 15:1 vs gzip's 11:1, but that ends up being only a
size difference of 50KiB (136 vs 187) which is only an additional 2.5%
savings from the uncompressed input.

Choose gzip because transparent decompression is widely supported by web
servers and clients.

Closes: https://bugs.gentoo.org/630284
Signed-off-by: Matt Turner  gentoo.org>

 catalyst/base/genbase.py | 5 +++--
 catalyst/support.py  | 2 +-
 doc/HOWTO.txt| 6 +++---
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py
index eb09a4e0..3db20f84 100644
--- a/catalyst/base/genbase.py
+++ b/catalyst/base/genbase.py
@@ -2,6 +2,7 @@
 import hashlib
 import io
 import os
+import gzip
 
 class GenBase():
 """
@@ -28,7 +29,7 @@ class GenBase():
 def gen_contents_file(self, path):
 c = self.settings['contents_map']
 
-with io.open(path + '.CONTENTS', 'w', encoding='utf-8') as file:
+with gzip.open(path + '.CONTENTS.gz', 'w', encoding='utf-8') as file:
 file.write(c.contents(path, '', verbose=self.settings['VERBOSE']))
 
 def gen_digest_file(self, path):
@@ -36,6 +37,6 @@ class GenBase():
 return
 
 with io.open(path + '.DIGESTS', 'w', encoding='utf-8') as file:
-for f in [path, path + '.CONTENTS']:
+for f in [path, path + '.CONTENTS.gz']:
 for i in self.settings['digests']:
 file.write(self.generate_hash(f, name=i))

diff --git a/catalyst/support.py b/catalyst/support.py
index 654b23aa..988a81f5 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -65,7 +65,7 @@ def file_check(filepath, extensions=None, strict=True):
 files = glob.glob("%s.*" % filepath)
 # remove any false positive files
 files = [x for x in files if not x.endswith(
-".CONTENTS") and not x.endswith(".DIGESTS")]
+".CONTENTS") and not x.endswith(".CONTENTS.gz") and not 
x.endswith(".DIGESTS")]
 if len(files) == 1:
 return files[0]
 if len(files) > 1 and strict:

diff --git a/doc/HOWTO.txt b/doc/HOWTO.txt
index 960b5761..7b759121 100644
--- a/doc/HOWTO.txt
+++ b/doc/HOWTO.txt
@@ -22,7 +22,7 @@ Create a snapshot of your current Portage tree (you may want 
to
 # catalyst --snapshot 20130131
 # ls /var/tmp/catalyst/snapshots/
 portage-20130131.tar.bz2
-portage-20130131.tar.bz2.CONTENTS
+portage-20130131.tar.bz2.CONTENTS.gz
 portage-20130131.tar.bz2.DIGESTS
 
 where the storage location is relative to the default
@@ -44,7 +44,7 @@ For example,
 Grab the tarball and put it where catalyst will find it:
 
 # wget http://…/stage3-amd64-20121213.tar.bz2
-# wget http://…/stage3-amd64-20121213.tar.bz2.CONTENTS
+# wget http://…/stage3-amd64-20121213.tar.bz2.CONTENTS.gz
 # wget http://…/stage3-amd64-20121213.tar.bz2.DIGESTS.asc
 # sha512sum -c stage3-amd64-20121213.tar.bz2.DIGESTS.asc
 # gpg --verify stage3-amd64-20121213.tar.bz2.DIGESTS.asc
@@ -89,7 +89,7 @@ which will build the target and install something like:
 
 # ls /var/tmp/catalyst/builds/default/stage1-amd64-2013.1.*
 /var/tmp/catalyst/builds/default/stage1-amd64-2013.1.tar.bz2
-/var/tmp/catalyst/builds/default/stage1-amd64-2013.1.tar.bz2.CONTENTS
+/var/tmp/catalyst/builds/default/stage1-amd64-2013.1.tar.bz2.CONTENTS.gz
 /var/tmp/catalyst/builds/default/stage1-amd64-2013.1.tar.bz2.DIGESTS
 
 The name is an expansion of