[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 1314c345cacfce8a5cd6036d66da0b5ab65c6adb
Author: Eli Schwartz gmail com>
AuthorDate: Thu Feb 22 20:40:26 2024 +
Commit: Andreas K. Hüttel gentoo org>
CommitDate: Sun Mar 3 18:50:45 2024 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=1314c345
avoid chrooting and running the $PATH to bash from outside the chroot
The API filename for bash is /bin/bash and that is where it is
guaranteed to be. On merged-usr systems, `shutil.which('bash')` will
search PATH and maybe find /usr/bin/bash first, since it is resolved as
a symlink, but that path will then fail in the chroot if the chroot is
not merged-usr.
Even on merged-usr systems, app-shells/bash always, always, always
installs to /bin/bash and lets portage handle the symlink.
Signed-off-by: Eli Schwartz gmail.com>
Signed-off-by: Andreas K. Hüttel gentoo.org>
catalyst/base/stagebase.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index dd00af45..4bcab30c 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1458,11 +1458,12 @@ class StageBase(TargetBase, ClearBase, GenBase):
def enter_chroot(self):
chroot = command('chroot')
-bash = command('bash')
+# verify existence only
+command(os.path.join(self.settings['chroot_path'], '/bin/bash'))
log.notice("Entering chroot")
try:
-cmd([chroot, self.settings['chroot_path'], bash, '-l'],
+cmd([chroot, self.settings['chroot_path'], '/bin/bash', '-l'],
env=self.env)
except CatalystError:
pass
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: e9d2b62ea8300aae67a483c7e260d3ff7bc80fc5
Author: Andreas K. Hüttel gentoo org>
AuthorDate: Sun Dec 24 21:17:59 2023 +
Commit: Andreas K. Hüttel gentoo org>
CommitDate: Sun Dec 24 21:41:33 2023 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=e9d2b62e
base/stagebase.py: Do not touch subarch if chost is set
Currently, if you set chost in a stage1 spec file, catalyst resets the
subarch to the first part of the triple (i.e. aarch64 for arm64).
With this change, it leaves the subarch alone at the value defined
in the spec file. This makes it much easier to override CHOST for,
say, musl, without having to provide a separate section in arch/* ...
Impact of the commit on the existing releng stage1 specs:
amd64, arm64, x86, ppc, ppc64, ppc64le are all fine (the definitions
are idencal, or subarch = first part of triplet)
power9le is funny because right now the chost (powerpc64le) resets the
subarch and drops the power9le optimizations, so there the commit is
actually an improvement!
Signed-off-by: Andreas K. Hüttel gentoo.org>
catalyst/base/stagebase.py | 5 +
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 3cbdf489..dd00af45 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -105,10 +105,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.makeconf = {}
-if "chost" in self.settings:
-host = self.settings["chost"].split("-")[0]
-else:
-host = self.settings["subarch"]
+host = self.settings["subarch"]
self.settings["hostarch"] = host
if "cbuild" in self.settings:
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 1abd94188e9e9eecd5fe16de3aeaceebac1ec4e0
Author: Andreas K. Hüttel gentoo org>
AuthorDate: Tue Sep 5 01:43:19 2023 +
Commit: Andreas K. Hüttel gentoo org>
CommitDate: Tue Sep 5 01:43:47 2023 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=1abd9418
Turn binrepos.conf into a folder with gentoobinhost.conf inside
Closes: https://bugs.gentoo.org/913660
Signed-off-by: Andreas K. Hüttel gentoo.org>
catalyst/base/stagebase.py | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index fe307eab..3cbdf489 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1200,9 +1200,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
# Write out binrepos.conf (for the chroot)
binrpath = normpath(self.settings["chroot_path"] +
self.settings["binrepos_conf"])
+Path(binrpath).mkdir(mode=0o755, parents=True, exist_ok=True)
-with open(binrpath, "w") as myb:
-log.notice("Writing the stage binrepos.conf to: %s" % binrpath)
+binrfile = binrpath + "/gentoobinhost.conf"
+with open(binrfile, "w") as myb:
+log.notice("Writing the stage binrepo config to: %s" %
binrfile)
myb.write("# These settings were set by the catalyst build
script "
"that automatically\n# built this stage.\n")
myb.write("# Please consider using a local mirror.\n\n")
@@ -1211,8 +1213,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
myb.write("sync-uri = " + self.settings["binhost"] + \
self.settings["binrepo_path"] + "\n")
-return
-
def fsscript(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("fsscript"):
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, targets/support/, targets/livecd-stage2/
commit: 3d4fb24383716958dadea95d100911037bb2b557
Author: Azamat H. Hackimov gmail com>
AuthorDate: Tue Aug 1 08:18:31 2023 +
Commit: Andreas K. Hüttel gentoo org>
CommitDate: Tue Aug 22 20:36:37 2023 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=3d4fb243
Implementing distkernel installation
Signed-off-by: Andreas K. Hüttel gentoo.org>
catalyst/base/stagebase.py | 22 +++---
targets/livecd-stage2/controller.sh | 5 +++
targets/support/bootloader-setup.sh | 19 -
targets/support/kmerge.sh | 85 +++--
targets/support/pre-distkmerge.sh | 7 +++
5 files changed, 116 insertions(+), 22 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index a9147241..fe307eab 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -694,6 +694,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
"boot/kernel/" + x + "/aliases",
"boot/kernel/" + x + "/config",
"boot/kernel/" + x + "/console",
+"boot/kernel/" + x + "/distkernel",
+"boot/kernel/" + x + "/dracut_args",
"boot/kernel/" + x + "/extraversion",
"boot/kernel/" + x + "/gk_action",
"boot/kernel/" + x + "/gk_kernargs",
@@ -716,6 +718,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["gk_mainargs"] = self.settings[gk_mainargs]
del self.settings[gk_mainargs]
+dracut_mainargs = prefix + "/dracut_args"
+if dracut_mainargs in self.settings:
+self.settings["dracut_args"] = self.settings[dracut_mainargs]
+del self.settings[dracut_mainargs]
+
# Ask genkernel to include b2sum if /verify is set
verify = prefix + "/verify"
if verify in self.settings:
@@ -1584,7 +1591,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.resume.enable("build_packages")
def build_kernel(self):
-'''Build all configured kernels'''
+"""Build all configured kernels"""
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("build_kernel"):
log.notice(
@@ -1595,19 +1602,23 @@ class StageBase(TargetBase, ClearBase, GenBase):
mynames = self.settings["boot/kernel"]
if isinstance(mynames, str):
mynames = [mynames]
-# Execute the script that sets up the kernel build environment
-cmd([self.settings['controller_file'], 'pre-kmerge'], env=self.env)
for kname in [sanitize_name(name) for name in mynames]:
+if "boot/kernel/" + kname + "/distkernel" in self.settings:
+cmd([self.settings['controller_file'], 'pre-distkmerge'],
env=self.env)
+else:
+# Execute the script that sets up the kernel build
environment
+cmd([self.settings['controller_file'], 'pre-kmerge'],
env=self.env)
self._build_kernel(kname=kname)
self.resume.enable("build_kernel")
def _build_kernel(self, kname):
-"Build a single configured kernel by name"
+"""Build a single configured kernel by name"""
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("build_kernel_" + kname):
log.notice('Resume point detected, skipping build_kernel '
'for %s operation...', kname)
return
+
self._copy_kernel_config(kname=kname)
key = 'boot/kernel/' + kname + '/extraversion'
@@ -1617,8 +1628,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self._copy_initramfs_overlay(kname=kname)
# Execute the script that builds the kernel
-cmd([self.settings['controller_file'], 'kernel', kname],
-env=self.env)
+cmd([self.settings['controller_file'], 'kernel', kname], env=self.env)
if "boot/kernel/" + kname + "/initramfs_overlay" in self.settings:
log.notice('Cleaning up temporary overlay dir')
diff --git a/targets/livecd-stage2/controller.sh
b/targets/livecd-stage2/controller.sh
index f6522e63..57d947a1 100755
--- a/targets/livecd-stage2/controller.sh
+++ b/targets/livecd-stage2/controller.sh
@@ -20,6 +20,11 @@ case $1 in
extract_modules ${clst_chroot_path} ${kname}
;;
+ pre-distkmerge)
+ # Install dracut
+ exec_in_chroot ${clst_shdir}/support/pre-distkmerge.sh
+ ;;
+
preclean)
# Move over the motd (if applicable)
case ${clst_livecd_type} in
diff --git a/targets/support/bootloader-setup.sh
b/targets/support/bootloader-setup.sh
index d6d5f96e..455a4f07 100755
--- a/targets/support/bootloader-setup.sh
+++
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, etc/, catalyst/
commit: 750947eed34fe51a9502b16585a59f34bd47b1a1
Author: Andreas K. Hüttel gentoo org>
AuthorDate: Wed Aug 9 21:22:55 2023 +
Commit: Andreas K. Hüttel gentoo org>
CommitDate: Fri Aug 11 19:55:12 2023 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=750947ee
Add configuration options and code to write out binrepos.conf
Signed-off-by: Andreas K. Hüttel gentoo.org>
catalyst/base/stagebase.py | 32
catalyst/defaults.py | 3 +++
etc/catalyst.conf | 5 +
3 files changed, 40 insertions(+)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 74d197b5..a9147241 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -62,6 +62,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
])
self.valid_values |= self.required_values | frozenset([
"asflags",
+"binrepo_path",
"catalyst_use",
"cbuild",
"cflags",
@@ -1074,6 +1075,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
if os.path.exists(hosts_file):
os.rename(hosts_file, hosts_file + '.catalyst')
shutil.copy('/etc/hosts', hosts_file)
+
# write out the make.conf
try:
self.write_make_conf(setup=True)
@@ -1081,6 +1083,16 @@ class StageBase(TargetBase, ClearBase, GenBase):
raise CatalystError('Could not write %s: %s' % (
normpath(self.settings["chroot_path"] +
self.settings["make_conf"]), e)) from e
+
+# write out the binrepos.conf
+# we do this here for later user convenience, but normally
+# it should not affect stage builds (which only get --usepkg,
+# but never --getbinpkg as emerge parameters).
+try:
+self.write_binrepos_conf()
+except OSError as e:
+raise CatalystError('Could not write binrepos.conf: %s' % ( e ))
from e
+
self.resume.enable("chroot_setup")
def write_make_conf(self, setup=True):
@@ -1174,6 +1186,26 @@ class StageBase(TargetBase, ClearBase, GenBase):
'# Please keep this setting intact when reporting bugs.\n'
'LC_MESSAGES=C.utf8\n')
+def write_binrepos_conf(self):
+# only if catalyst.conf defines the host and the spec defines the
path...
+if self.settings["binhost"] != '' and "binrepo_path" in self.settings:
+
+# Write out binrepos.conf (for the chroot)
+binrpath = normpath(self.settings["chroot_path"] +
+self.settings["binrepos_conf"])
+
+with open(binrpath, "w") as myb:
+log.notice("Writing the stage binrepos.conf to: %s" % binrpath)
+myb.write("# These settings were set by the catalyst build
script "
+"that automatically\n# built this stage.\n")
+myb.write("# Please consider using a local mirror.\n\n")
+myb.write("[gentoobinhost]\n")
+myb.write("priority = 1\n")
+myb.write("sync-uri = " + self.settings["binhost"] + \
+self.settings["binrepo_path"] + "\n")
+
+return
+
def fsscript(self):
if "autoresume" in self.settings["options"] \
and self.resume.is_enabled("fsscript"):
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 39b71c34..f4d48fef 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -9,6 +9,7 @@ from DeComp.definitions import DECOMPRESSOR_PROGRAM_OPTIONS,
LIST_XATTRS_OPTIONS
valid_config_file_values = frozenset([
+"binhost",
"compression_mode",
"digests",
"digest_format",
@@ -31,6 +32,7 @@ valid_config_file_values = frozenset([
])
confdefaults = {
+"binhost": '',
"comp_prog": COMPRESSOR_PROGRAM_OPTIONS['linux'],
"compression_mode": 'lbzip2',
"compressor_arch": None,
@@ -42,6 +44,7 @@ confdefaults = {
"icecream": "/var/cache/icecream",
'list_xattrs_opt': LIST_XATTRS_OPTIONS['linux'],
"port_conf": "/etc/portage",
+"binrepos_conf": "%(port_conf)s/binrepos.conf",
"make_conf": "%(port_conf)s/make.conf",
"repos_conf": "%(port_conf)s/repos.conf",
"options": set(),
diff --git a/etc/catalyst.conf b/etc/catalyst.conf
index b0b284fa..8745d0b5 100644
--- a/etc/catalyst.conf
+++ b/etc/catalyst.conf
@@ -83,3 +83,8 @@ options = [
# Floating-point value passed to emerge as the parameter to --load-average and
# is used to define MAKEOPTS during the target build.
# load-average = 4.0
+
+# If you want catalyst to drop a binrepos.conf into /etc/portage, then
+# define the binhost here. This value is concatenated with the configuration
+# option binrepo_path in the spec file to obtain the src-uri.
+# binhost = "https://gentoo.osuosl.org/releases/;
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 6272f501c0101a49fcb35f60ba75778ded696b67
Author: Andreas K. Hüttel gentoo org>
AuthorDate: Sat Jul 1 17:39:01 2023 +
Commit: Andreas K. Hüttel gentoo org>
CommitDate: Sat Jul 1 17:45:11 2023 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=6272f501
stagebase: Add blank linkes in make.conf (e.g. before CHOST block)
Signed-off-by: Andreas K. Hüttel gentoo.org>
catalyst/base/stagebase.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 812ac4c8..65f4306e 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1114,13 +1114,13 @@ class StageBase(TargetBase, ClearBase, GenBase):
% (flags, self.settings[flags]))
if "CBUILD" in self.settings:
-myf.write("# This should not be changed unless you know
exactly"
+myf.write("\n# This should not be changed unless you know
exactly"
" what you are doing. You\n# should probably be "
"using a different stage, instead.\n")
myf.write('CBUILD="' + self.settings["CBUILD"] + '"\n')
if "CHOST" in self.settings:
-myf.write("# WARNING: Changing your CHOST is not something "
+myf.write("\n# WARNING: Changing your CHOST is not something "
"that should be done lightly.\n# Please consult "
"https://wiki.gentoo.org/wiki/Changing_the_CHOST_variable "
"before changing.\n")
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/
commit: 9d6e495ca2ee55c21befc183a9a56fc5062a63c7
Author: Andreas K. Hüttel gentoo org>
AuthorDate: Sat Jul 1 17:44:41 2023 +
Commit: Andreas K. Hüttel gentoo org>
CommitDate: Sat Jul 1 19:26:14 2023 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=9d6e495c
stagebase: Fix and improve port_logdir config option
NOTE: The cleaning of log files does not work yet (and probably
never did).
Signed-off-by: Andreas K. Hüttel gentoo.org>
catalyst/base/stagebase.py | 7 +++
catalyst/defaults.py | 2 ++
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 65f4306e..74d197b5 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -271,9 +271,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
if "port_logdir" in self.settings:
self.mount['port_logdir']['enable'] = True
-self.mount['port_logdir']['source'] = self.settings['port_logdir']
-self.env["PORT_LOGDIR"] = self.settings["port_logdir"]
-self.env["PORT_LOGDIR_CLEAN"] = PORT_LOGDIR_CLEAN
+self.mount['port_logdir']['source'] =
normpath(self.settings['port_logdir'] + "/" + self.settings["target_subpath"] +
"/")
+self.env["PORTAGE_LOGDIR"] = self.settings["target_logdir"]
def override_cbuild(self):
if "CBUILD" in self.makeconf:
@@ -1002,7 +1001,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
# We may need to create the source of the bind mount.
E.g., in the
# case of an empty package cache we must create the
directory that
# the binary packages will be stored into.
-source_path.mkdir(mode=0o755, exist_ok=True)
+source_path.mkdir(mode=0o755, parents=True, exist_ok=True)
Path(target).mkdir(mode=0o755, parents=True, exist_ok=True)
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 2cede562..39b71c34 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -25,6 +25,7 @@ valid_config_file_values = frozenset([
"sharedir",
"storedir",
"target_distdir",
+"target_logdir",
"target_pkgdir",
"var_tmpfs_portage",
])
@@ -53,6 +54,7 @@ confdefaults = {
"shdir": "%(sharedir)s/targets",
"storedir": "/var/tmp/catalyst",
"target_distdir": "/var/cache/distfiles",
+"target_logdir": "/var/log/portage",
"target_pkgdir": "/var/cache/binpkgs",
}
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: d7e25003b3657b8fb9dd69787efbdb6dec6cb491 Author: Ben Kohler gentoo org> AuthorDate: Wed May 10 18:10:11 2023 + Commit: Ben Kohler gentoo org> CommitDate: Wed May 10 18:10:11 2023 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=d7e25003 stagebase.py: switch to LC_MESSAGES=C.utf8 in make.conf Signed-off-by: Ben Kohler gentoo.org> catalyst/base/stagebase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 732c5939..812ac4c8 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -1173,7 +1173,7 @@ class StageBase(TargetBase, ClearBase, GenBase): '\n' '# This sets the language of build output to English.\n' '# Please keep this setting intact when reporting bugs.\n' -'LC_MESSAGES=C\n') +'LC_MESSAGES=C.utf8\n') def fsscript(self): if "autoresume" in self.settings["options"] \
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: aa52470e86ebd0578c379813a9144611c97431a8
Author: Matt Turner gentoo org>
AuthorDate: Tue Nov 22 03:26:59 2022 +
Commit: Matt Turner gentoo org>
CommitDate: Tue Nov 22 21:19:03 2022 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=aa52470e
catalyst: Use correct arch setting for setarch testing
Previously we were testing against the 'host' or subarchitecture (e.g.
i486) instead of the architecture (e.g. x86). This manifests on x86
builds on x86_64; on ppc on ppc64 and sparc on sparc64 the arch and
subarch values happen to line up.
Fixes: cd58b37b ("catalyst: Access setarch data correctly")
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 5a3bd375..732c5939 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -129,7 +129,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
# Search for a subarchitecture in each arch in the arch_config
for arch in [x for x in arch_config if x.startswith(name) and
host in arch_config[x]]:
self.settings.update(arch_config[arch][host])
-setarch = arch_config.get('setarch', {}).get(host, {})
+setarch = arch_config.get('setarch', {}).get(arch, {})
break
else:
# Didn't find a matching subarchitecture, keep searching
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: f365eb43a81989ad24f75995df97552e08d3e7ad
Author: Matt Turner gentoo org>
AuthorDate: Thu Nov 17 23:41:33 2022 +
Commit: Matt Turner gentoo org>
CommitDate: Thu Nov 17 23:42:24 2022 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f365eb43
catalyst: Remove stray space
Signed-off-by: Matt Turner gentoo.org>
catalyst/support.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/catalyst/support.py b/catalyst/support.py
index a69566d2..6945d053 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -78,7 +78,7 @@ def file_check(filepath, extensions=None):
files = glob.glob("%s.*" % filepath)
# remove any false positive files
files = [x for x in files if
- not x.endswith( ".CONTENTS") and
+ not x.endswith(".CONTENTS") and
not x.endswith(".CONTENTS.gz") and
not x.endswith(".DIGESTS") and
not x.endswith(".sha256")]
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: f2263466c535d0019be0bcbf1f08128b2399d6ea
Author: Andreas K. Hüttel gentoo org>
AuthorDate: Mon Nov 14 00:05:58 2022 +
Commit: Andreas K. Hüttel gentoo org>
CommitDate: Mon Nov 14 00:09:33 2022 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f2263466
stagebase: Also clean out profile/package.use.force profile/package.use.mask
Signed-off-by: Andreas K. Hüttel gentoo.org>
catalyst/base/stagebase.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index bcf596ae..5a3bd375 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1230,7 +1230,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
"sticky-config" not in self.settings["options"]):
log.debug("clean(), portage_preix = %s, no sticky-config",
self.settings["portage_prefix"])
-for _dir in "package.accept_keywords", "package.keywords",
"package.mask", "package.unmask", "package.use", "package.env", "env":
+for _dir in "package.accept_keywords", "package.keywords",
"package.mask", "package.unmask", "package.use", "package.env", "env",
"profile/package.use.force", "profile/package.use.mask":
target = pjoin(self.settings["stage_path"],
"etc/portage/%s" % _dir,
self.settings["portage_prefix"])
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 64d2993d8ca38cefd76df7a69a44accd60284cd4
Author: Andreas K. Hüttel gentoo org>
AuthorDate: Sat Nov 12 20:42:03 2022 +
Commit: Andreas K. Hüttel gentoo org>
CommitDate: Sun Nov 13 02:17:29 2022 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=64d2993d
Allow "interpreter" parameter as space-separated list of files
I added this feature to the 3.0 branch long ago and forgot about it.
Needed for multilib emulation, e.g., qemu-mips qemu-mipsn32 qemu-mips64
Tested on demeter "in production"
Signed-off-by: Andreas K. Hüttel gentoo.org>
catalyst/base/stagebase.py | 48 +++---
1 file changed, 24 insertions(+), 24 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index eb869c70..bcf596ae 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1036,26 +1036,24 @@ class StageBase(TargetBase, ClearBase, GenBase):
shutil.copy('/etc/resolv.conf',
self.settings['chroot_path'] + '/etc/')
-# Copy over the binary interpreter (qemu), if applicable; note that
it's given
-# as full path and goes to the same place in the chroot
+# Copy over the binary interpreter(s) (qemu), if applicable; note that
they are given
+# as space-separated list of full paths and go to the same place in
the chroot
if "interpreter" in self.settings:
-if not os.path.exists(self.settings["interpreter"]):
-raise CatalystError(
-"Can't find interpreter " +
-self.settings["interpreter"],
-print_traceback=True)
+if isinstance(self.settings["interpreter"], str):
+myints = [self.settings["interpreter"]]
+else:
+myints = self.settings["interpreter"]
-log.notice('Copying binary interpreter %s into chroot',
- self.settings['interpreter'])
+for myi in myints:
+if not os.path.exists(myi):
+raise CatalystError("Can't find interpreter " + myi,
print_traceback=True)
-if os.path.exists(self.settings['chroot_path'] + '/' +
self.settings['interpreter']):
-os.rename(
-self.settings['chroot_path'] +
-'/' + self.settings['interpreter'],
-self.settings['chroot_path'] + '/' +
self.settings['interpreter'] + '.catalyst')
+log.notice('Copying binary interpreter %s into chroot', myi)
-shutil.copy(self.settings['interpreter'],
-self.settings['chroot_path'] + '/' +
self.settings['interpreter'])
+if os.path.exists(self.settings['chroot_path'] + '/' + myi):
+os.rename(self.settings['chroot_path'] + '/' + myi,
self.settings['chroot_path'] + '/' + myi + '.catalyst')
+
+shutil.copy(myi, self.settings['chroot_path'] + '/' + myi)
# Copy over the envscript, if applicable
if "envscript" in self.settings:
@@ -1214,16 +1212,18 @@ class StageBase(TargetBase, ClearBase, GenBase):
if os.path.exists(hosts_file + '.catalyst'):
os.rename(hosts_file + '.catalyst', hosts_file)
-# optionally clean up binary interpreter
+# optionally clean up binary interpreter(s)
if "interpreter" in self.settings:
-if os.path.exists(self.settings['chroot_path'] + '/' +
self.settings['interpreter'] + '.catalyst'):
-os.rename(
-self.settings['chroot_path'] + '/' +
-self.settings['interpreter'] + '.catalyst',
-self.settings['chroot_path'] + '/' +
self.settings['interpreter'])
+if isinstance(self.settings["interpreter"], str):
+myints = [self.settings["interpreter"]]
else:
-os.remove(
-self.settings['chroot_path'] + '/' +
self.settings['interpreter'])
+myints = self.settings["interpreter"]
+
+for myi in myints:
+if os.path.exists(self.settings['chroot_path'] + '/' + myi +
'.catalyst'):
+os.rename(self.settings['chroot_path'] + '/' + myi +
'.catalyst', self.settings['chroot_path'] + '/' + myi)
+else:
+os.remove(self.settings['chroot_path'] + '/' + myi)
# optionally clean up portage configs
if ("portage_prefix" in self.settings and
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 1c46e66ef2c9ab24f383eebd3c6cb1d02492b8fc
Author: Matt Turner gentoo org>
AuthorDate: Wed Apr 20 23:23:07 2022 +
Commit: Matt Turner gentoo org>
CommitDate: Wed Apr 20 23:23:07 2022 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=1c46e66e
catalyst: Replace a tab
Fixes: 5be6069b ("catalyst: support 3 new options")
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 76feb5f0..5c7e9adb 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -597,7 +597,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["groups"] = []
log.info('groups to create: %s' % self.settings["groups"])
- def set_users(self):
+def set_users(self):
users = self.settings["spec_prefix"] + "/users"
if users in self.settings:
if isinstance(self.settings[users], str):
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 5be6069bcbd5a7fa3f114f28366597bc5ddbb891
Author: Patrice Clement gentoo org>
AuthorDate: Tue Mar 8 21:02:55 2022 +
Commit: Matt Turner gentoo org>
CommitDate: Sat Apr 2 23:50:16 2022 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=5be6069b
catalyst: support 3 new options
* stage4/groups: create a a list of groups.
* stage4/users: create a list of users. users can also be added to
groups using the "foo.bar=wheel,audio,baz" format.
* stage4/ssh_public_keys: copy an SSH public key into the stage4 user's home
(.ssh/authorized_keys) and set the file permission to 0644.
Bug: https://bugs.gentoo.org/236905
Signed-off-by: Patrice Clement gentoo.org>
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 70 ++
1 file changed, 70 insertions(+)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index de1e30ef..76feb5f0 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -201,6 +201,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.set_packages()
self.set_rm()
self.set_linuxrc()
+self.set_groups()
+self.set_users()
+self.set_ssh_public_keys()
self.set_busybox_config()
self.set_overlay()
self.set_repos()
@@ -583,6 +586,39 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings[self.settings["spec_prefix"] + "/linuxrc"]
del self.settings[self.settings["spec_prefix"] + "/linuxrc"]
+def set_groups(self):
+groups = self.settings["spec_prefix"] + "/groups"
+if groups in self.settings:
+if isinstance(self.settings[groups], str):
+self.settings["groups"] = self.settings[groups].split(",")
+self.settings["groups"] = self.settings[groups]
+del self.settings[groups]
+else:
+self.settings["groups"] = []
+log.info('groups to create: %s' % self.settings["groups"])
+
+ def set_users(self):
+users = self.settings["spec_prefix"] + "/users"
+if users in self.settings:
+if isinstance(self.settings[users], str):
+self.settings["users"] = self.settings[users].split(",")
+self.settings["users"] = self.settings[users]
+del self.settings[users]
+else:
+self.settings["users"] = []
+log.info('users to create: %s' % self.settings["users"])
+
+def set_ssh_public_keys(self):
+ssh_public_keys = self.settings["spec_prefix"] + "/ssh_public_keys"
+if ssh_public_keys in self.settings:
+if isinstance(self.settings[ssh_public_keys], str):
+self.settings["ssh_public_keys"] =
self.settings[ssh_public_keys].split(",")
+self.settings["ssh_public_keys"] = self.settings[ssh_public_keys]
+del self.settings[ssh_public_keys]
+else:
+self.settings["ssh_public_keys"] = []
+log.info('ssh public keys to copy: %s' %
self.settings["ssh_public_keys"])
+
def set_busybox_config(self):
if self.settings["spec_prefix"] + "/busybox_config" in self.settings:
if isinstance(self.settings[self.settings['spec_prefix'] +
'/busybox_config'], str):
@@ -894,6 +930,40 @@ class StageBase(TargetBase, ClearBase, GenBase):
cmd(['rsync', '-a', x + '/', self.settings['stage_path']],
env=self.env)
+def groups(self):
+for x in self.settings["groups"].split():
+log.notice("Creating group: '%s'", x)
+cmd(["groupadd", "-R", self.settings['chroot_path'], x],
env=self.env)
+
+def users(self):
+for x in self.settings["users"]:
+usr, grp = '', ''
+try:
+usr, grp = x.split("=")
+except ValueError:
+usr = x
+log.debug("users: '=' separator not found on line " + x)
+log.debug("users: missing separator means no groups found")
+uacmd = ["useradd", "-R", self.settings['chroot_path'], "-m", x]
+if grp != '':
+uacmd = ["useradd", "-R", self.settings['chroot_path'], "-m",
"-G", grp, usr]
+log.notice("Creating user: '%s'", f"{usr}={grp}")
+cmd(uacmd, env=self.env)
+
+def ssh_public_keys(self):
+for x in self.settings["ssh_public_keys"]:
+usr, pub_key_src = '', ''
+try:
+usr, pub_key_src = x.split("=")
+except ValueError:
+raise CatalystError(f"ssh_public_keys: '=' separator not found
on line {x}")
+log.notice("Copying SSH public key for user: '%s'", usr)
+pub_key_dest = self.settings['chroot_path'] +
f"/home/{usr}/.ssh/authorized_keys"
+cpcmd = ["cp", "-av", pub_key_src, pub_key_dest]
+cmd(cpcmd,
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/
commit: c2c22f5d07684ed0c6b802254e7b678358111579 Author: Patrice Clement gentoo org> AuthorDate: Tue Mar 8 21:08:09 2022 + Commit: Matt Turner gentoo org> CommitDate: Sat Apr 2 23:50:24 2022 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=c2c22f5d catalyst: add new options to stage4 step list Signed-off-by: Patrice Clement gentoo.org> Signed-off-by: Matt Turner gentoo.org> catalyst/targets/stage4.py | 6 ++ 1 file changed, 6 insertions(+) diff --git a/catalyst/targets/stage4.py b/catalyst/targets/stage4.py index ff1d4dca..35309b45 100644 --- a/catalyst/targets/stage4.py +++ b/catalyst/targets/stage4.py @@ -19,13 +19,16 @@ class stage4(StageBase): "stage4/empty", "stage4/fsscript", "stage4/gk_mainargs", +"stage4/groups", "stage4/linuxrc", "stage4/rcadd", "stage4/rcdel", "stage4/rm", "stage4/root_overlay", +"stage4/ssh_public_keys", "stage4/unmerge", "stage4/use", +"stage4/users", ]) def __init__(self, spec, addlargs): @@ -51,6 +54,9 @@ class stage4(StageBase): ]) self.finish_sequence.extend([ self.remove, +self.groups, +self.users, +self.ssh_public_keys, self.empty, self.clean, ])
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: 1a8d6cd3d31f960755c3237679539d9cab175bc6
Author: Matt Turner gentoo org>
AuthorDate: Wed Feb 16 22:33:17 2022 +
Commit: Matt Turner gentoo org>
CommitDate: Wed Feb 16 22:33:52 2022 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=1a8d6cd3
catalyst: Add .sha256 to list of ignored extensions
Otherwise the presence of a .sha256 file would cause this error:
ERROR:catalyst:CatalystError: Ambiguous Filename:
/home/catalyst/builds/default/stage3-ia64-openrc-20220216T032203Z
ERROR:catalyst:Please specify the correct extension as well
Fixes: 7457cd3b ("catalyst: generate .sha256 file if any digest is enabled")
Signed-off-by: Matt Turner gentoo.org>
catalyst/support.py | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/catalyst/support.py b/catalyst/support.py
index 37d53bc4..a69566d2 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -77,8 +77,11 @@ def file_check(filepath, extensions=None):
# so check if there are files of that name with an extension
files = glob.glob("%s.*" % filepath)
# remove any false positive files
-files = [x for x in files if not x.endswith(
-".CONTENTS") and not x.endswith(".CONTENTS.gz") and not
x.endswith(".DIGESTS")]
+files = [x for x in files if
+ not x.endswith( ".CONTENTS") and
+ not x.endswith(".CONTENTS.gz") and
+ not x.endswith(".DIGESTS") and
+ not x.endswith(".sha256")]
if len(files) == 1:
return files[0]
if len(files) > 1:
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, doc/
commit: 7457cd3b1a5f3ed4a566bbf23e36c939af06967c Author: Georgy Yakovlev gentoo org> AuthorDate: Fri Nov 5 02:14:00 2021 + Commit: Georgy Yakovlev gentoo org> CommitDate: Thu Nov 25 02:43:21 2021 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=7457cd3b catalyst: generate .sha256 file if any digest is enabled checksum format is simple one, identical to one sha256sum from coreutils produces, lines starting with # are ignored. example:[1] # SHA256 HASH . stage3-tar.xz systemd upstream calls it suse-style .sha256 files.[0] infra already supports inline signing of files. Bug: https://bugs.gentoo.org/821568 [0] https://github.com/systemd/systemd/blob/aedec452b9e5dd197881f2164fb205dfe8bfdcec/src/import/pull-common.c#L236 [1] https://mirrors.edge.kernel.org/opensuse/distribution/leap/15.0/iso/openSUSE-Leap-15.0-DVD-x86_64.iso.sha256 Signed-off-by: Georgy Yakovlev gentoo.org> catalyst/base/genbase.py | 3 +++ doc/catalyst-config.5.txt | 7 --- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py index 6d039a8f..1643b1dc 100644 --- a/catalyst/base/genbase.py +++ b/catalyst/base/genbase.py @@ -43,3 +43,6 @@ class GenBase(): for f in [path, path + '.CONTENTS.gz']: for i in self.settings['digests']: file.write(self.generate_hash(f, name=i)) + +with io.open(path + '.sha256', 'w', encoding='utf-8') as sha256file: +sha256file.write(self.generate_hash(path, name='sha256')) diff --git a/doc/catalyst-config.5.txt b/doc/catalyst-config.5.txt index 003d40b6..ca9335d6 100644 --- a/doc/catalyst-config.5.txt +++ b/doc/catalyst-config.5.txt @@ -38,9 +38,10 @@ Basic configuration *digests*:: Create a `.DIGESTS` file containing the hash output from any of the supported options below. Adding them all may take a long time. -(example: `md5 sha1 sha512 whirlpool`). The set of supported hashes -is dependent on the version of Python. To see the set of hashes -supported by the version of Python in use, run +(example: `md5 sha1 sha512 whirlpool`). Enabling ANY digest will +generate `.sha256` file in addition `.DIGESTS` file. The set of +supported hashes is dependent on the version of Python. To see the +set of hashes supported by the version of Python in use, run + - $ python3 -c 'import hashlib; print(hashlib.algorithms_available)'
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, targets/support/
commit: ad04eab48c96350f0866d6c1fe76f289499df21b
Author: Matt Turner gentoo org>
AuthorDate: Thu Jul 29 01:10:58 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Thu Jul 29 01:13:52 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=ad04eab4
catalyst: Remove boot/kernel/*/kernelopts support
This undocumented option was only wired up for hppa, and was used by the
minimal livecd to set panic=30.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 11 ---
targets/support/bootloader-setup.sh | 8 +---
2 files changed, 1 insertion(+), 18 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 10cffd4c..4a1b4eb6 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -662,7 +662,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
"boot/kernel/" + x + "/gk_action",
"boot/kernel/" + x + "/gk_kernargs",
"boot/kernel/" + x + "/initramfs_overlay",
-"boot/kernel/" + x + "/kernelopts",
"boot/kernel/" + x + "/packages",
"boot/kernel/" + x + "/softlevel",
"boot/kernel/" + x + "/sources",
@@ -1508,16 +1507,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
return
self._copy_kernel_config(kname=kname)
-# If we need to pass special options to the bootloader
-# for this kernel put them into the environment
-key = 'boot/kernel/' + kname + '/kernelopts'
-if key in self.settings:
-myopts = self.settings[key]
-
-if not isinstance(myopts, str):
-myopts = ' '.join(myopts)
-self.env[kname + "_kernelopts"] = myopts
-
key = 'boot/kernel/' + kname + '/extraversion'
self.settings.setdefault(key, '')
self.env["clst_kextraversion"] = self.settings[key]
diff --git a/targets/support/bootloader-setup.sh
b/targets/support/bootloader-setup.sh
index a998a420..08d25b59 100755
--- a/targets/support/bootloader-setup.sh
+++ b/targets/support/bootloader-setup.sh
@@ -70,16 +70,10 @@ case ${clst_hostarch} in
boot_kernel_common_name=${first/%32/}
boot_kernel_common_name=${boot_kernel_common_name/%64/}
- for x in ${clst_boot_kernel}
- do
- eval kopt=\$clst_boot_kernel_${x}_kernelopts
- kopts+=(${kopt})
- done
-
# copy the bootloader for the final image
cp /usr/share/palo/iplboot $1/boot/
- echo "--commandline=0/${boot_kernel_common_name}
initrd=${first}.igz ${default_append_line[@]} ${kopts[@]}" >> ${icfg}
+ echo "--commandline=0/${boot_kernel_common_name}
initrd=${first}.igz ${default_append_line[@]}" >> ${icfg}
echo "--bootloader=boot/iplboot" >> ${icfg}
echo "--ramdisk=boot/${first}.igz" >> ${icfg}
for x in ${clst_boot_kernel}
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/, catalyst/
commit: b3c917f7fa73d11c69b7e55dc7a00bc18a18edc7
Author: Matt Turner gentoo org>
AuthorDate: Wed Jun 9 06:17:31 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Thu Jun 10 04:23:54 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b3c917f7
catalyst: Replace snakeoil's locks with fasteners
To no great surprise, the existing locking was broken. For example,
clear_chroot() releases the lock. It is called by unpack(), which is
part of prepare_sequence. The result is that the whole build could be
done without holding the lock.
Just lock around run(). It's not apparent that finer-grained locking
does anything for us.
Bug: https://bugs.gentoo.org/791583
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/clearbase.py | 2 --
catalyst/base/stagebase.py | 10 +++-
catalyst/lock.py | 58
catalyst/targets/snapshot.py | 6 ++---
4 files changed, 7 insertions(+), 69 deletions(-)
diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py
index dcf6c523..6218330e 100644
--- a/catalyst/base/clearbase.py
+++ b/catalyst/base/clearbase.py
@@ -27,12 +27,10 @@ class ClearBase():
self.resume.clear_all(remove=True)
def clear_chroot(self):
-self.chroot_lock.unlock()
log.notice('Clearing the chroot path ...')
clear_dir(self.settings["chroot_path"], mode=0o755)
def remove_chroot(self):
-self.chroot_lock.unlock()
log.notice('Removing the chroot path ...')
clear_dir(self.settings["chroot_path"], mode=0o755, remove=True)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 448d6265..10cffd4c 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -8,6 +8,7 @@ import sys
from pathlib import Path
+import fasteners
import libmount
import toml
@@ -25,7 +26,6 @@ from catalyst.support import (CatalystError, file_locate,
normpath,
from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
from catalyst.base.genbase import GenBase
-from catalyst.lock import LockDir, LockInUse
from catalyst.fileops import ensure_dirs, clear_dir, clear_path
from catalyst.base.resume import AutoResume
@@ -36,9 +36,6 @@ def run_sequence(sequence):
sys.stdout.flush()
try:
func()
-except LockInUse:
-log.error('Unable to aquire the lock...')
-return False
except Exception:
log.error('Exception running action sequence %s',
func.__name__, exc_info=True)
@@ -478,7 +475,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
"""
self.settings["chroot_path"] = normpath(self.settings["storedir"] +
"/tmp/" +
self.settings["target_subpath"].rstrip('/'))
-self.chroot_lock = LockDir(self.settings["chroot_path"])
def set_autoresume_path(self):
self.settings["autoresume_path"] = normpath(pjoin(
@@ -1366,8 +1362,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
pass
def run(self):
-self.chroot_lock.write_lock()
+with fasteners.InterProcessLock(self.settings["chroot_path"] +
'.lock'):
+return self._run()
+def _run(self):
if "clear-autoresume" in self.settings["options"]:
self.clear_autoresume()
diff --git a/catalyst/lock.py b/catalyst/lock.py
deleted file mode 100644
index e31745b2..
--- a/catalyst/lock.py
+++ /dev/null
@@ -1,58 +0,0 @@
-
-import os
-
-from contextlib import contextmanager
-
-from snakeoil import fileutils
-from snakeoil import osutils
-from catalyst.fileops import ensure_dirs
-
-
-LockInUse = osutils.LockException
-
-class Lock:
-"""
-A fnctl-based filesystem lock
-"""
-def __init__(self, lockfile):
-fileutils.touch(lockfile, mode=0o664)
-os.chown(lockfile, uid=-1, gid=250)
-self.lock = osutils.FsLock(lockfile)
-
-def read_lock(self):
-self.lock.acquire_read_lock()
-
-def write_lock(self):
-self.lock.acquire_write_lock()
-
-def unlock(self):
-# Releasing a write lock is the same as a read lock.
-self.lock.release_write_lock()
-
-class LockDir(Lock):
-"""
-A fnctl-based filesystem lock in a directory
-"""
-def __init__(self, lockdir):
-ensure_dirs(lockdir)
-lockfile = os.path.join(lockdir, '.catalyst_lock')
-
-Lock.__init__(self, lockfile)
-
-@contextmanager
-def read_lock(filename):
-lock = Lock(filename)
-lock.read_lock()
-try:
-yield
-finally:
-lock.unlock()
-
-@contextmanager
-def write_lock(filename):
-lock = Lock(filename)
-lock.write_lock()
-try:
-yield
-finally:
-lock.unlock()
diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py
index b494575a..ef68765d
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: dc22f3540c6879e88e5f3bd2ff8999066387d3ea
Author: Matt Turner gentoo org>
AuthorDate: Thu Mar 25 03:02:39 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Thu Mar 25 03:02:39 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=dc22f354
catalyst: Fix indentation
Fixes: 48338534 ("Add option to generate BSD-style tagged hashes ...")
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/genbase.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py
index 4a4c3efa..b89e7c93 100644
--- a/catalyst/base/genbase.py
+++ b/catalyst/base/genbase.py
@@ -26,9 +26,9 @@ class GenBase():
filename = os.path.split(filepath)[1]
if self.settings['digest_format'] == 'bsd':
-return f'# {name.upper()} HASH\n{name.upper()} ({filename}) =
{h.hexdigest()}\n'
+return f'# {name.upper()} HASH\n{name.upper()} ({filename}) =
{h.hexdigest()}\n'
else:
-return f'# {name.upper()} HASH\n{h.hexdigest()} {filename}\n'
+return f'# {name.upper()} HASH\n{h.hexdigest()} {filename}\n'
def gen_contents_file(self, path):
c = self.settings['contents_map']
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 7c4d1c5f3aab4a32795797332f33d5a759f72a9d
Author: Matt Turner gentoo org>
AuthorDate: Thu Mar 25 03:02:53 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Thu Mar 25 03:02:53 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=7c4d1c5f
catalyst: Remove @staticmethod from generate_hash()
Fixes: 48338534 ("Add option to generate BSD-style tagged hashes ...")
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/genbase.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py
index b89e7c93..6d039a8f 100644
--- a/catalyst/base/genbase.py
+++ b/catalyst/base/genbase.py
@@ -12,8 +12,7 @@ class GenBase():
def __init__(self, myspec):
self.settings = myspec
-@staticmethod
-def generate_hash(filepath, name):
+def generate_hash(self, filepath, name):
h = hashlib.new(name)
with open(filepath, 'rb') as f:
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 91dba2e6199b24c7d0b52b8bd6204b4a5f6e5137
Author: Matt Turner gentoo org>
AuthorDate: Tue Mar 9 16:17:33 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Tue Mar 9 16:17:33 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=91dba2e6
catalyst: Remove tabs
Fixes: 48338534 ("Add option to generate BSD-style tagged hashes ...")
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/genbase.py | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py
index 52418877..4a4c3efa 100644
--- a/catalyst/base/genbase.py
+++ b/catalyst/base/genbase.py
@@ -25,10 +25,10 @@ class GenBase():
filename = os.path.split(filepath)[1]
- if self.settings['digest_format'] == 'bsd':
- return f'# {name.upper()} HASH\n{name.upper()} ({filename}) =
{h.hexdigest()}\n'
- else:
- return f'# {name.upper()} HASH\n{h.hexdigest()} {filename}\n'
+if self.settings['digest_format'] == 'bsd':
+return f'# {name.upper()} HASH\n{name.upper()} ({filename}) =
{h.hexdigest()}\n'
+else:
+return f'# {name.upper()} HASH\n{h.hexdigest()} {filename}\n'
def gen_contents_file(self, path):
c = self.settings['contents_map']
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 71037a931ff23141875c40d37605e8614a31077e
Author: Daniel Cordero 0xdc io>
AuthorDate: Thu Feb 25 13:09:12 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Sat Mar 6 16:57:09 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=71037a93
catalyst: Handle specifying target_profiles in repo_name:path format
Fixes: a5004005 ("catalyst: Call config_profile_link for all targets")
Signed-off-by: Daniel Cordero 0xdc.io>
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 9 +++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index a4da7eb3..448d6265 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -814,10 +814,15 @@ class StageBase(TargetBase, ClearBase, GenBase):
make_profile = Path(self.settings['chroot_path'] +
self.settings['port_conf'],
'make.profile')
make_profile.unlink(missing_ok=True)
+try:
+repo_name, target_profile =
self.settings['target_profile'].split(":", 1)
+except ValueError:
+repo_name = self.settings['repo_name']
+target_profile = self.settings['target_profile']
make_profile.symlink_to(Path('../..' + self.settings['repo_basedir'],
- self.settings['repo_name'],
+ repo_name,
'profiles',
- self.settings['target_profile']),
+ target_profile),
target_is_directory=True)
def setup_confdir(self):
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/
commit: 483385348b32fab2230f596816b6e1d0b6489962
Author: Andreas K. Hüttel gentoo org>
AuthorDate: Wed Feb 24 21:26:58 2021 +
Commit: Andreas K. Hüttel gentoo org>
CommitDate: Wed Feb 24 21:26:58 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=48338534
Add option to generate BSD-style tagged hashes (can be verified by modern
coreutils)
Signed-off-by: Andreas K. Hüttel gentoo.org>
catalyst/base/genbase.py | 6 +-
catalyst/defaults.py | 2 ++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py
index c7dd87bc..52418877 100644
--- a/catalyst/base/genbase.py
+++ b/catalyst/base/genbase.py
@@ -24,7 +24,11 @@ class GenBase():
h.update(data)
filename = os.path.split(filepath)[1]
-return f'# {name.upper()} HASH\n{h.hexdigest()} {filename}\n'
+
+ if self.settings['digest_format'] == 'bsd':
+ return f'# {name.upper()} HASH\n{name.upper()} ({filename}) =
{h.hexdigest()}\n'
+ else:
+ return f'# {name.upper()} HASH\n{h.hexdigest()} {filename}\n'
def gen_contents_file(self, path):
c = self.settings['contents_map']
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index ccb0a584..2cede562 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -11,6 +11,7 @@ from DeComp.definitions import DECOMPRESSOR_PROGRAM_OPTIONS,
LIST_XATTRS_OPTIONS
valid_config_file_values = frozenset([
"compression_mode",
"digests",
+"digest_format",
"distcc_hosts",
"distdir",
"envscript",
@@ -35,6 +36,7 @@ confdefaults = {
"compressor_options": XATTRS_OPTIONS['linux'],
"decomp_opt": DECOMPRESSOR_PROGRAM_OPTIONS['linux'],
"decompressor_search_order": DECOMPRESSOR_SEARCH_ORDER,
+"digest_format": 'linux',
"distdir": portage.settings['DISTDIR'],
"icecream": "/var/cache/icecream",
'list_xattrs_opt': LIST_XATTRS_OPTIONS['linux'],
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: c1f5707a9c9f27f730aad908c8f3c86075201fc9 Author: Daniel Cordero 0xdc io> AuthorDate: Thu Feb 4 11:07:19 2021 + Commit: Matt Turner gentoo org> CommitDate: Sun Feb 21 02:05:39 2021 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=c1f5707a catalyst: Remove /etc/machine-id in all stages Machine IDs are unique per installation, but if they are retained in a stage tarball, all installations from that stage share the same machine id. Signed-off-by: Daniel Cordero 0xdc.io> Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index bc0eccd0..a4da7eb3 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -465,6 +465,7 @@ class StageBase(TargetBase, ClearBase, GenBase): def set_cleanables(self): self.settings['cleanables'] = [ +"/etc/machine-id", "/etc/resolv.conf", "/var/tmp/*", "/tmp/*",
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/
commit: 6b78a58c9ac825ecd450d100cd269d29761dbe03
Author: Daniel Cordero 0xdc io>
AuthorDate: Thu Feb 4 11:07:18 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Sun Feb 21 02:05:34 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=6b78a58c
catalyst: Clean up chroot directory on success
The embedded and livecd-stage2 targets override the action sequence
without calling set_completion_action_sequences. This would leave the
temporary build environment unpacked on the filesystem.
Signed-off-by: Daniel Cordero 0xdc.io>
Signed-off-by: Matt Turner gentoo.org>
catalyst/targets/embedded.py | 2 +-
catalyst/targets/livecd_stage2.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py
index b677b226..01ad035d 100644
--- a/catalyst/targets/embedded.py
+++ b/catalyst/targets/embedded.py
@@ -53,8 +53,8 @@ class embedded(StageBase):
self.empty,
self.clean,
self.capture,
-self.clear_autoresume,
])
+self.set_completion_action_sequences()
def set_root_path(self):
self.settings["root_path"] = normpath("/tmp/mergeroot")
diff --git a/catalyst/targets/livecd_stage2.py
b/catalyst/targets/livecd_stage2.py
index 774702f8..832e0998 100644
--- a/catalyst/targets/livecd_stage2.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -108,4 +108,4 @@ class livecd_stage2(StageBase):
self.setup_overlay,
self.create_iso,
])
-self.finish_sequence.append(self.clear_autoresume)
+self.set_completion_action_sequences()
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 5f34226550e221921189ca2184a0afe2de1e8821
Author: Felix Bier rohde-schwarz com>
AuthorDate: Thu Feb 4 00:45:27 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Sat Feb 20 21:27:29 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=5f342265
Drop PORTDIR from make.conf
This commit drops PORTDIR from being included in the generated
make.conf. The variable is no longer needed, since the existing code
will now generate a repos.conf entry for the main repo when needed
(i.e. when the location is non-default, which is the same condition
that was applied for PORTDIR in the code removed by this commit).
The PORTDIR variable is considered deprecated in favor of
repos.conf entries (reference: https://wiki.gentoo.org/wiki/PORTDIR).
Signed-off-by: Felix Bier rohde-schwarz.com>
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 6 --
1 file changed, 6 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 97e2318c..bc0eccd0 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1092,12 +1092,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
myf.write(hostuseexpand + '="' +
' '.join(myuseexpandvars[hostuseexpand]) + '"\n')
-# Write non-default PORTDIR/DISTDIR/PKGDIR settings to make.conf
-if (self.settings['repo_basedir'], self.settings['repo_name']) != \
- (confdefaults['repo_basedir'], confdefaults['repo_name']):
-myf.write('PORTDIR="%s/%s"\n' % (self.settings['repo_basedir'],
- self.settings['repo_name']))
-
for x in ['target_distdir', 'target_pkgdir']:
if self.settings[x] != confdefaults[x]:
varname = x.split('_')[1].upper()
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: 4fd2ac23250ab2ac1f6a506ee433f466a4f9e026 Author: Felix Bier rohde-schwarz com> AuthorDate: Sat Feb 13 23:18:17 2021 + Commit: Matt Turner gentoo org> CommitDate: Sat Feb 20 21:27:29 2021 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4fd2ac23 Enable recursive globbing for clear_path This commit enables recursive globbing in clear_path, allowing the usage of '**' to match an arbitrary number of sub-directories. Before this commit, clear_path used only non-recursive globbing. This allowed to use '*' to expand names within one directory, e.g. '/a/*/c' can expand to '/a/b/c', but not '/a/b/b/c'. With this commit, '/a/**/c' can be used to expand to '/a/b/c', '/a/b/b/c', '/a/b/b/b/c' etc. This is motivated by wanting to recursively delete all occurences of a filename with the 'stage4/rm' entry of a spec file. The '/rm' entries are processed with 'clear_path' in the existing code. Additionally, 'glob.glob' is replaced with 'glob.iglob', which returns the same files as 'glob.glob', but as an iterator instead of as a list (so it no longer necessary to hold all matches in memory at once). Recursive globbing has been added in Python 3.5. References: https://docs.python.org/3/library/glob.html#glob.glob https://docs.python.org/3/library/glob.html#glob.iglob Signed-off-by: Felix Bier rohde-schwarz.com> Signed-off-by: Matt Turner gentoo.org> catalyst/fileops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/fileops.py b/catalyst/fileops.py index 5c6f5cd8..59525420 100644 --- a/catalyst/fileops.py +++ b/catalyst/fileops.py @@ -99,7 +99,7 @@ def clear_dir(target, mode=0o755, remove=False, def clear_path(target_path): """Nuke |target_path| regardless of it being a dir, file or glob.""" -targets = glob.glob(target_path) +targets = glob.iglob(target_path, recursive=True) for path in targets: clear_dir(path, remove=True)
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: caf55a942580e02e66ed846f5c3ab4ad5ab8846f
Author: Felix Bier rohde-schwarz com>
AuthorDate: Thu Feb 4 00:37:06 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Sat Feb 20 21:27:29 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=caf55a94
Extend get_repo_name to handle squashed repos
This commit extends the method get_repo_name to also handle
squashed repos. This is done by mounting the squash file to
a temporary directory and then extracting the repository from
that directory with the already existing code.
This is motivated by wanting to mount each repo
to e.g. /var/db/repos/ in a later commit.
For squashed repos, we don't know without
mounting the repo first. For this reason, it is mounted to
a temporary directory first to extract .
Signed-off-by: Felix Bier rohde-schwarz.com>
Signed-off-by: Matt Turner gentoo.org>
catalyst/support.py | 36 +++-
1 file changed, 35 insertions(+), 1 deletion(-)
diff --git a/catalyst/support.py b/catalyst/support.py
index fc50fa34..37d53bc4 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -10,10 +10,12 @@ from subprocess import Popen
import libmount
from portage.repository.config import RepoConfig
+from tempfile import TemporaryDirectory
from snakeoil.bash import read_bash_dict
from catalyst import log
+from catalyst.context import namespace
BASH_BINARY = "/bin/bash"
@@ -148,7 +150,7 @@ def read_makeconf(mymakeconffile):
return makeconf
-def get_repo_name(repo_path):
+def get_repo_name_from_dir(repo_path):
""" Get the name of the repo at the given repo_path.
References:
@@ -164,6 +166,38 @@ def get_repo_name(repo_path):
return repo_config.name
+def get_repo_name_from_squash(repo_squash_path):
+""" Get the name of the repo at the given repo_squash_path.
+To obtain the name, the squash file is mounted to a temporary
directory.
+"""
+
+repo_name = None
+
+# Mount squash file to temp directory in separate mount namespace
+with TemporaryDirectory() as temp, namespace(mount=True):
+try:
+source = str(repo_squash_path)
+target = str(temp)
+fstype = 'squashfs'
+options = 'ro,loop'
+cxt = libmount.Context(source=source, target=target,
+ fstype=fstype, options=options)
+cxt.mount()
+repo_name = get_repo_name_from_dir(target)
+
+except Exception as e:
+raise CatalystError(f"Couldn't mount: {source}, {e}") from e
+
+return repo_name
+
+
+def get_repo_name(repo_path):
+if not Path(repo_path).is_dir():
+return get_repo_name_from_squash(repo_path)
+
+return get_repo_name_from_dir(repo_path)
+
+
def ismount(path):
"""Like os.path.ismount, but also support bind mounts"""
path = Path(path)
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/
commit: 65d49f1028b49fed6e011526429e553ebb6c903e
Author: Felix Bier rohde-schwarz com>
AuthorDate: Thu Feb 4 00:38:36 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Sat Feb 20 21:27:29 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=65d49f10
Unify handling of main repo and other repos
This commit unifies the handling of the main repo and the other repos.
All are stored in one common list. A mount entry is created for
each entry in the list (previously a mount entry was only created
for the main repo and the other repos were copied into the chroot).
This means each non-main repo can now be either a directory or a
squash files (previously the non-main repos had to be directories).
The existing mount logic will bind-mount the repos that are stored as
directories, removing the need for copying.
A repos.conf entry will be created for each entry in the list.
This means a repos.conf entry for the main repo can now be created
(previously repos.conf entries were only created for the non-main
repos). The repos.conf entry will only be created if the target
location for the main repo is non-default, i.e. unequal to
/var/db/repos/gentoo. This mirrors the behavior of write_make_conf,
which only writes the PORTDIR variable to make.conf if the location
of the main repo is non-default.
As a side effect, the PORTDIR variable is now no longer needed,
and will be removed from write_make_conf in a separate commit.
Signed-off-by: Felix Bier rohde-schwarz.com>
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 88 +++---
catalyst/defaults.py | 5 ---
2 files changed, 60 insertions(+), 33 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index fedc8f87..97e2318c 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -219,8 +219,17 @@ class StageBase(TargetBase, ClearBase, GenBase):
# Setup our mount points.
self.mount = copy.deepcopy(MOUNT_DEFAULTS)
-self.mount['portdir']['source'] = self.snapshot
-self.mount['portdir']['target'] = self.settings['repo_basedir'] + '/'
+ self.settings['repo_name']
+# Create mount entry for each repository
+for path, name, _ in self.repos:
+name = get_repo_name(path)
+mount_id = f'repo_{name}'
+
+self.mount[mount_id] = {
+'enable': True,
+'source': path,
+'target': self.get_repo_location(name)
+}
+
self.mount['distdir']['source'] = self.settings['distdir']
self.mount["distdir"]['target'] = self.settings['target_distdir']
@@ -587,13 +596,41 @@ class StageBase(TargetBase, ClearBase, GenBase):
"/busybox_config"]
def set_repos(self):
+
+# Each entry in this list will be a tuple of the form
+# (source, name, default)
+#
+# source: the location of the repo on the host system,
+# either a directory or a squashfs file.
+#
+# name: the repository name parsed from the repo.
+# This is just a caching mechanism to avoid parsing the name
+# every time the source is processed.
+#
+# default: Default location where the repo is expected in the
+# target system. If this matches the path where we mount the repo to
+# (as per get_repo_location), then we can skip generating a repos.conf
+# entry for that repo. Currently this mechanism is only used for
+# the main repo, which has a default location hard-coded in
+# /usr/share/portage/config/repos.conf. For the other repos,
+# the default is set to None.
+self.repos = []
+
+# Create entry for snapshot
+default_location = Path(confdefaults['repo_basedir'],
confdefaults['repo_name'])
+self.repos.append((self.snapshot, get_repo_name(self.snapshot),
default_location))
+
+# Create entry for every other repo
if 'repos' in self.settings:
if isinstance(self.settings['repos'], str):
self.settings['repos'] = \
self.settings['repos'].split()
-log.info('repos directories are set to: %s',
+log.info('repos are set to: %s',
' '.join(self.settings['repos']))
+get_info = lambda repo: (repo, get_repo_name(repo), None)
+self.repos.extend(map(get_info, self.settings['repos']))
+
def set_overlay(self):
if self.settings["spec_prefix"] + "/overlay" in self.settings:
if isinstance(self.settings[self.settings['spec_prefix'] +
'/overlay'], str):
@@ -832,24 +869,19 @@ class StageBase(TargetBase, ClearBase, GenBase):
raise CatalystError(f'Could not write {repo_conf_chroot}: {e}')
from e
def process_repos(self):
-""" We copy the contents of our repos to
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/targets/
commit: 27d7c7898bb0cccf6962b84058b0e2789bc314c1 Author: Matt Turner gentoo org> AuthorDate: Mon Jan 18 03:20:41 2021 + Commit: Matt Turner gentoo org> CommitDate: Sat Jan 23 16:19:07 2021 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=27d7c789 catalyst: Deduplicate the common build_sequence steps This also provides a single location to add a function to all targets' build_sequence. Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py| 11 ++- catalyst/targets/embedded.py | 3 --- catalyst/targets/livecd_stage1.py | 3 --- catalyst/targets/livecd_stage2.py | 3 --- catalyst/targets/netboot.py | 3 --- catalyst/targets/stage4.py| 3 --- 6 files changed, 6 insertions(+), 20 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 3b8d3a6e..40b60af3 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -90,7 +90,11 @@ class StageBase(TargetBase, ClearBase, GenBase): self.setup_confdir, self.portage_overlay, ] -self.build_sequence = [] +self.build_sequence = [ +self.bind, +self.chroot_setup, +self.setup_environment, +] self.finish_sequence = [] self.set_valid_build_kernel_vars(addlargs) @@ -497,16 +501,13 @@ class StageBase(TargetBase, ClearBase, GenBase): Or it calls the normal set_action_sequence() for the target stage. """ if "purgeonly" in self.settings["options"]: -self.build_sequence.append(self.remove_chroot) +self.build_sequence = [self.remove_chroot] return self.set_action_sequence() def set_action_sequence(self): """Set basic stage1, 2, 3 action sequences""" self.build_sequence.extend([ -self.bind, -self.chroot_setup, -self.setup_environment, self.run_local, self.preclean, ]) diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py index 83b17ad2..b677b226 100644 --- a/catalyst/targets/embedded.py +++ b/catalyst/targets/embedded.py @@ -42,9 +42,6 @@ class embedded(StageBase): def set_action_sequence(self): self.build_sequence.extend([ -self.bind, -self.chroot_setup, -self.setup_environment, self.build_kernel, self.build_packages, self.root_overlay, diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py index 9fe95a00..dbfa54ed 100644 --- a/catalyst/targets/livecd_stage1.py +++ b/catalyst/targets/livecd_stage1.py @@ -24,9 +24,6 @@ class livecd_stage1(StageBase): def set_action_sequence(self): self.build_sequence.extend([ -self.bind, -self.chroot_setup, -self.setup_environment, self.build_packages, ]) self.finish_sequence.extend([ diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py index ab8f4a0e..e90e9f53 100644 --- a/catalyst/targets/livecd_stage2.py +++ b/catalyst/targets/livecd_stage2.py @@ -87,9 +87,6 @@ class livecd_stage2(StageBase): def set_action_sequence(self): self.build_sequence.extend([ -self.bind, -self.chroot_setup, -self.setup_environment, self.run_local, self.build_kernel ]) diff --git a/catalyst/targets/netboot.py b/catalyst/targets/netboot.py index cb5e7ae4..a2a9fcb3 100644 --- a/catalyst/targets/netboot.py +++ b/catalyst/targets/netboot.py @@ -159,9 +159,6 @@ class netboot(StageBase): def set_action_sequence(self): self.build_sequence.extend([ -self.bind, -self.chroot_setup, -self.setup_environment, self.build_packages, self.root_overlay, self.copy_files_to_image, diff --git a/catalyst/targets/stage4.py b/catalyst/targets/stage4.py index 9e53f070..f8383f75 100644 --- a/catalyst/targets/stage4.py +++ b/catalyst/targets/stage4.py @@ -40,9 +40,6 @@ class stage4(StageBase): def set_action_sequence(self): self.build_sequence.extend([ -self.bind, -self.chroot_setup, -self.setup_environment, self.build_packages, self.build_kernel, self.bootloader,
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/targets/
commit: 4f0fa29d5a2499b7178dc4d19f7693e52eb8def9
Author: Matt Turner gentoo org>
AuthorDate: Mon Jan 18 03:15:02 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Sat Jan 23 16:19:07 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4f0fa29d
catalyst: Deduplicate prepare_sequence assignments
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py| 13 ++---
catalyst/targets/embedded.py | 6 --
catalyst/targets/livecd_stage1.py | 6 --
catalyst/targets/livecd_stage2.py | 6 --
catalyst/targets/netboot.py | 6 --
catalyst/targets/stage4.py| 6 --
6 files changed, 6 insertions(+), 37 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 53b0a224..3b8d3a6e 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -84,7 +84,12 @@ class StageBase(TargetBase, ClearBase, GenBase):
"portage_overlay",
"portage_prefix",
])
-self.prepare_sequence = []
+self.prepare_sequence = [
+self.unpack,
+self.config_profile_link,
+self.setup_confdir,
+self.portage_overlay,
+]
self.build_sequence = []
self.finish_sequence = []
@@ -498,12 +503,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
def set_action_sequence(self):
"""Set basic stage1, 2, 3 action sequences"""
-self.prepare_sequence.extend([
-self.unpack,
-self.config_profile_link,
-self.setup_confdir,
-self.portage_overlay,
-])
self.build_sequence.extend([
self.bind,
self.chroot_setup,
diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py
index 7d328808..83b17ad2 100644
--- a/catalyst/targets/embedded.py
+++ b/catalyst/targets/embedded.py
@@ -41,12 +41,6 @@ class embedded(StageBase):
StageBase.__init__(self, spec, addlargs)
def set_action_sequence(self):
-self.prepare_sequence.extend([
-self.unpack,
-self.config_profile_link,
-self.setup_confdir,
-self.portage_overlay,
-])
self.build_sequence.extend([
self.bind,
self.chroot_setup,
diff --git a/catalyst/targets/livecd_stage1.py
b/catalyst/targets/livecd_stage1.py
index eb6bb0f0..9fe95a00 100644
--- a/catalyst/targets/livecd_stage1.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -23,12 +23,6 @@ class livecd_stage1(StageBase):
StageBase.__init__(self, spec, addlargs)
def set_action_sequence(self):
-self.prepare_sequence.extend([
-self.unpack,
-self.config_profile_link,
-self.setup_confdir,
-self.portage_overlay,
-])
self.build_sequence.extend([
self.bind,
self.chroot_setup,
diff --git a/catalyst/targets/livecd_stage2.py
b/catalyst/targets/livecd_stage2.py
index 5fecff85..ab8f4a0e 100644
--- a/catalyst/targets/livecd_stage2.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -86,12 +86,6 @@ class livecd_stage2(StageBase):
print_traceback=True)
def set_action_sequence(self):
-self.prepare_sequence.extend([
-self.unpack,
-self.config_profile_link,
-self.setup_confdir,
-self.portage_overlay,
-])
self.build_sequence.extend([
self.bind,
self.chroot_setup,
diff --git a/catalyst/targets/netboot.py b/catalyst/targets/netboot.py
index 9a7e59e5..cb5e7ae4 100644
--- a/catalyst/targets/netboot.py
+++ b/catalyst/targets/netboot.py
@@ -158,12 +158,6 @@ class netboot(StageBase):
self.resume.enable("empty")
def set_action_sequence(self):
-self.prepare_sequence.extend([
-self.unpack,
-self.config_profile_link,
-self.setup_confdir,
-self.portage_overlay,
-])
self.build_sequence.extend([
self.bind,
self.chroot_setup,
diff --git a/catalyst/targets/stage4.py b/catalyst/targets/stage4.py
index 997139a5..9e53f070 100644
--- a/catalyst/targets/stage4.py
+++ b/catalyst/targets/stage4.py
@@ -39,12 +39,6 @@ class stage4(StageBase):
self.settings["cleanables"].remove('/etc/resolv.conf')
def set_action_sequence(self):
-self.prepare_sequence.extend([
-self.unpack,
-self.config_profile_link,
-self.setup_confdir,
-self.portage_overlay,
-])
self.build_sequence.extend([
self.bind,
self.chroot_setup,
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/
commit: 440a379ae94e6d7df26291bf83657288d1406e98
Author: Matt Turner gentoo org>
AuthorDate: Sun Jan 17 23:53:21 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Sat Jan 23 16:19:07 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=440a379a
catalyst: Store references to functions
... rather than their names. This makes it possible for tooling to
understand the code structure better.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py| 69 ---
catalyst/targets/embedded.py | 34 +--
catalyst/targets/livecd_stage1.py | 18 +-
catalyst/targets/livecd_stage2.py | 46 +-
catalyst/targets/netboot.py | 34 +--
catalyst/targets/stage1.py| 12 +++
catalyst/targets/stage4.py| 36 ++--
7 files changed, 125 insertions(+), 124 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index ed4d1227..447e073d 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -30,6 +30,23 @@ from catalyst.fileops import ensure_dirs, clear_dir,
clear_path
from catalyst.base.resume import AutoResume
+def run_sequence(sequence):
+for func in sequence:
+log.notice('--- Running action sequence: %s', func.__name__)
+sys.stdout.flush()
+try:
+func()
+except LockInUse:
+log.error('Unable to aquire the lock...')
+return False
+except Exception:
+log.error('Exception running action sequence %s',
+ func.__name__, exc_info=True)
+return False
+
+return True
+
+
class StageBase(TargetBase, ClearBase, GenBase):
"""
This class does all of the chroot setup, copying of files, etc. It is
@@ -475,39 +492,39 @@ class StageBase(TargetBase, ClearBase, GenBase):
Or it calls the normal set_action_sequence() for the target stage.
"""
if "purgeonly" in self.settings["options"]:
-self.build_sequence.append("remove_chroot")
+self.build_sequence.append(self.remove_chroot)
return
self.set_action_sequence()
def set_action_sequence(self):
"""Set basic stage1, 2, 3 action sequences"""
self.prepare_sequence.extend([
-"unpack",
-"setup_confdir",
-"portage_overlay",
+self.unpack,
+self.setup_confdir,
+self.portage_overlay,
])
self.build_sequence.extend([
-"bind",
-"chroot_setup",
-"setup_environment",
-"run_local",
-"preclean",
+self.bind,
+self.chroot_setup,
+self.setup_environment,
+self.run_local,
+self.preclean,
])
self.finish_sequence.extend([
-"clean",
+self.clean,
])
self.set_completion_action_sequences()
def set_completion_action_sequences(self):
if "fetch" not in self.settings["options"]:
-self.finish_sequence.append("capture")
+self.finish_sequence.append(self.capture)
if "keepwork" in self.settings["options"]:
-self.finish_sequence.append("clear_autoresume")
+self.finish_sequence.append(self.clear_autoresume)
elif "seedcache" in self.settings["options"]:
-self.finish_sequence.append("remove_autoresume")
+self.finish_sequence.append(self.remove_autoresume)
else:
-self.finish_sequence.append("remove_autoresume")
-self.finish_sequence.append("remove_chroot")
+self.finish_sequence.append(self.remove_autoresume)
+self.finish_sequence.append(self.remove_chroot)
def set_use(self):
use = self.settings["spec_prefix"] + "/use"
@@ -1308,22 +1325,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
log.debug('setup_environment(); env = %r', self.env)
-def run_sequence(self, sequence):
-for func in sequence:
-log.notice('--- Running action sequence: %s', func)
-sys.stdout.flush()
-try:
-getattr(self, func)()
-except LockInUse:
-log.error('Unable to aquire the lock...')
-return False
-except Exception:
-log.error('Exception running action sequence %s',
- func, exc_info=True)
-return False
-
-return True
-
def run(self):
self.chroot_lock.write_lock()
@@ -1342,14 +1343,14 @@ class StageBase(TargetBase, ClearBase, GenBase):
log.info('StageBase: run() purge')
self.purge()
-if not self.run_sequence(self.prepare_sequence):
+if not run_sequence(self.prepare_sequence):
[gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
commit: b556ff31b8d4caaef6c0e7612a70f5f0c397b02c Author: Felix Bier rohde-schwarz com> AuthorDate: Tue Nov 10 00:59:01 2020 + Commit: Matt Turner gentoo org> CommitDate: Sat Nov 14 16:34:57 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b556ff31 Ensure deep copying of config defaults This commit adds deep copying operations when initializing config objects from a default config. This prevents the config from being a shallow copy of the default, ensuring that modifications to the config do not modify the default. In particular, this fixes a check in write_make_conf, where the PORTDIR variable is supposed to be only written to the generated make.conf when a non-default repo_basedir is set in /etc/catalyst/catalyst.conf. This check is never satisfied, because confvalues is a shallow copy of confdefaults, therefore both will always hold the same value for repo_basedir. For self.mounts / MOUNT_DEFAULTS this problem can also be observed, the modifications done to self.mounts are also visible in MOUNT_DEFAULTS. I am not aware of any bugs due to this shallow copy, but I would prefer adding a deep copy to prevent future bugs, in case a comparision against the default mounts is ever needed. Signed-off-by: Felix Bier rohde-schwarz.com> Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py | 3 ++- catalyst/main.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index a75dbdf9..21cf96a0 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -1,4 +1,5 @@ +import copy import os import platform import shutil @@ -187,7 +188,7 @@ class StageBase(TargetBase, ClearBase, GenBase): file_locate(self.settings, ["portage_confdir"], expand=0) # Setup our mount points. -self.mount = MOUNT_DEFAULTS.copy() +self.mount = copy.deepcopy(MOUNT_DEFAULTS) self.mount['portdir']['source'] = self.snapshot self.mount['portdir']['target'] = self.settings['repo_basedir'] + '/' + self.settings['repo_name'] diff --git a/catalyst/main.py b/catalyst/main.py index 5536471a..48daf004 100644 --- a/catalyst/main.py +++ b/catalyst/main.py @@ -1,4 +1,5 @@ import argparse +import copy import datetime import hashlib import os @@ -19,7 +20,7 @@ from catalyst.defaults import (confdefaults, option_messages, from catalyst.support import CatalystError from catalyst.version import get_version -conf_values = confdefaults +conf_values = copy.deepcopy(confdefaults) def version():
[gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
commit: 37a386056f77f7cc8f1c2bdfe680b13bd806b4e6
Author: Felix Bier rohde-schwarz com>
AuthorDate: Tue Nov 10 01:03:03 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Sat Nov 14 16:34:57 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=37a38605
Move from PORTDIR_OVERLAY to repos.conf
This commit fixes the following issues:
* The PORTDIR_OVERLAY variable has been deprecated by Gentoo.
With this commit, the variable is no longer written to the
generated make.conf. Instead, a config file
/etc/portage/repos.conf/.conf
is generated for each overlay. The repo name is read from the
overlay using the portage API. Internally, portage parses
metadata/layout.conf and profiles/repo_name to obtain the name.
References:
https://wiki.gentoo.org/wiki//etc/portage/make.conf
https://wiki.gentoo.org/wiki//etc/portage/repos.conf
* All overlays were copied into the same target directory. If the
same file name occurred in multiple overlays, the last overlay
would overwrite all previous files with this name. In particular,
only the metadata/layout.conf of the last overlay was retained,
so it was not possible to reference the other overlays e.g. via
the masters entry in the layout.conf or the portage-2 syntax
for specifying a parent profile from another overlay. Also,
this created problems when the overlays contained ebuilds
for the same package, but with differing versions, because
after copying, the target directory contained both versions of the
ebuild but only the manifest file of the last overlay.
With this commit, each overlay is copied into a separate
sub-directory, e.g. /var/db/repos/.
This directory is referenced via the location entry in the
generated /etc/portage/repos.conf/.conf.
Signed-off-by: Felix Bier rohde-schwarz.com>
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 84 ++
catalyst/defaults.py | 2 +-
catalyst/support.py| 18 ++
3 files changed, 81 insertions(+), 23 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 21cf96a0..fe79b55a 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1,4 +1,5 @@
+import configparser
import copy
import os
import platform
@@ -19,8 +20,8 @@ from catalyst import log
from catalyst.context import namespace
from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
from catalyst.support import (CatalystError, file_locate, normpath,
- cmd, read_makeconf, ismount, file_check,
- sanitize_name)
+ cmd, read_makeconf, get_repo_name, ismount,
+ file_check, sanitize_name)
from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
from catalyst.base.genbase import GenBase
@@ -786,17 +787,55 @@ class StageBase(TargetBase, ClearBase, GenBase):
env=self.env)
self.resume.enable("setup_confdir")
+def to_chroot(self, path):
+""" Prepend chroot path to the given path. """
+
+chroot = Path(self.settings['chroot_path'])
+return chroot / path.relative_to(path.anchor)
+
+def get_repo_conf_path(self, repo_name):
+""" Construct repo conf path: {repos_conf}/{name}.conf """
+return Path(self.settings['repos_conf'], repo_name + ".conf")
+
+def get_repo_location(self, repo_name):
+""" Construct overlay repo path: {repo_basedir}/{name} """
+return Path(self.settings['repo_basedir'], repo_name)
+
+def write_repo_conf(self, repo_name, config):
+""" Write ConfigParser to {chroot}/{repos_conf}/{name}.conf """
+
+repo_conf = self.get_repo_conf_path(repo_name)
+
+repo_conf_chroot = self.to_chroot(repo_conf)
+repo_conf_chroot.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
+
+log.info(f'Creating repo config {repo_conf_chroot}.')
+
+try:
+with open(repo_conf_chroot, 'w') as f:
+config.write(f)
+except OSError as e:
+raise CatalystError(f'Could not write {repo_conf_chroot}: {e}')
from e
+
def portage_overlay(self):
-""" We copy the contents of our overlays to /usr/local/portage """
+""" We copy the contents of our repos to get_repo_location(repo_name)
"""
if "portage_overlay" in self.settings:
for x in self.settings["portage_overlay"]:
if os.path.exists(x):
-log.info('Copying overlay dir %s', x)
-ensure_dirs(
-self.settings['chroot_path'] +
self.settings['local_overlay'])
-cmd("cp -a " + x + "/* " + self.settings["chroot_path"] +
-self.settings["local_overlay"],
-
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: dccfafea4e8b4622a7527a764d581ac0123b107e
Author: Felix Bier rohde-schwarz com>
AuthorDate: Mon Nov 9 19:36:33 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Sat Nov 14 16:34:57 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=dccfafea
catalyst: Restore root and cwd after exiting mount namespace
This commit saves the file descriptor of /proc/self/{root,cwd}
before entering into the new mount namespace. When restoring the
previous mount namespace, it restores /proc/self/{root,cwd}
based on the saved file descriptors.
Without this change, catalyst cannot be run in a chroot when
using the recent changes regarding mount namespaces: After the
mount namespace has been exited, /proc/self/root points to the "/"
of the host system, not the "/" of the chroot. Therefore, the
cleanup phase of catalyst runs outside of the chroot.
The code is similar to how nsenter(1) sets root and cwd:
https://git.kernel.org/pub/scm/utils/util-linux/util-linux.git/tree/sys-utils/nsenter.c#n452
Tested in a Gentoo chroot and in Gentoo VM (non-chroot).
Signed-off-by: Felix Bier rohde-schwarz.com>
Signed-off-by: Matt Turner gentoo.org>
catalyst/context.py | 22 ++
1 file changed, 22 insertions(+)
diff --git a/catalyst/context.py b/catalyst/context.py
index 8a58f33d..01a6d930 100644
--- a/catalyst/context.py
+++ b/catalyst/context.py
@@ -16,11 +16,21 @@ def namespace(mount=False, uts=False, ipc=False, net=False,
pid=False,
(user, "user"): None,
}
+dirs = {
+"root": None,
+"cwd": None,
+}
+
# Save fds of current namespaces
for ns in [ns for ns in namespaces if ns[0]]:
fp = open(f"/proc/self/ns/{ns[1]}")
namespaces[ns] = fp
+# Save fds of current directories
+if mount:
+for d in dirs:
+dirs[d] = os.open(f"/proc/self/{d}", os.O_RDONLY)
+
simple_unshare(mount=mount, uts=uts, ipc=ipc, net=net, pid=pid, user=user,
hostname=hostname)
try:
@@ -30,3 +40,15 @@ def namespace(mount=False, uts=False, ipc=False, net=False,
pid=False,
fp = namespaces[ns]
setns(fp.fileno(), 0)
fp.close()
+
+if mount:
+# Restore original root and cwd. Since we cannot directly chroot to
+# a fd, first change the current directory to the fd of the
+# original root, then chroot to "."
+
+os.fchdir(dirs["root"])
+os.chroot(".")
+os.fchdir(dirs["cwd"])
+
+for fd in dirs.values():
+os.close(fd)
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/targets/
commit: 2c28062446246e246c992ea7084a3950b1337827
Author: Matt Turner gentoo org>
AuthorDate: Thu Oct 29 00:32:34 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=2c280624
catalyst: Use .extend() and .append() for action_sequence
Ensures that we don't overwrite and lose some settings.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py| 6 +++---
catalyst/targets/embedded.py | 4 ++--
catalyst/targets/livecd_stage1.py | 4 ++--
catalyst/targets/livecd_stage2.py | 8
catalyst/targets/netboot.py | 4 ++--
catalyst/targets/stage4.py| 4 ++--
6 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 801df2fb..46b7c59c 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -478,13 +478,13 @@ class StageBase(TargetBase, ClearBase, GenBase):
Or it calls the normal set_action_sequence() for the target stage.
"""
if "purgeonly" in self.settings["options"]:
-self.action_sequence = ["remove_chroot"]
+self.action_sequence.append("remove_chroot")
return
self.set_action_sequence()
def set_action_sequence(self):
"""Set basic stage1, 2, 3 action sequences"""
-self.action_sequence = [
+self.action_sequence.extend([
"unpack",
"setup_confdir",
"portage_overlay",
@@ -495,7 +495,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
"preclean",
"unbind",
"clean",
-]
+])
self.set_completion_action_sequences()
def set_completion_action_sequences(self):
diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py
index 3899cf1b..75eb68e4 100644
--- a/catalyst/targets/embedded.py
+++ b/catalyst/targets/embedded.py
@@ -41,7 +41,7 @@ class embedded(StageBase):
StageBase.__init__(self, spec, addlargs)
def set_action_sequence(self):
-self.action_sequence = [
+self.action_sequence.extend([
"unpack",
"config_profile_link",
"setup_confdir",
@@ -60,7 +60,7 @@ class embedded(StageBase):
"clean",
"capture",
"clear_autoresume",
-]
+])
def set_stage_path(self):
self.settings["stage_path"] = normpath(
diff --git a/catalyst/targets/livecd_stage1.py
b/catalyst/targets/livecd_stage1.py
index b8c26cb1..9dbfa506 100644
--- a/catalyst/targets/livecd_stage1.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -23,7 +23,7 @@ class livecd_stage1(StageBase):
StageBase.__init__(self, spec, addlargs)
def set_action_sequence(self):
-self.action_sequence = [
+self.action_sequence.extend([
"unpack",
"config_profile_link",
"setup_confdir",
@@ -34,7 +34,7 @@ class livecd_stage1(StageBase):
"build_packages",
"unbind",
"clean",
-]
+])
self.set_completion_action_sequences()
def set_spec_prefix(self):
diff --git a/catalyst/targets/livecd_stage2.py
b/catalyst/targets/livecd_stage2.py
index cac16b6e..c9b5ce08 100644
--- a/catalyst/targets/livecd_stage2.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -87,7 +87,7 @@ class livecd_stage2(StageBase):
print_traceback=True)
def set_action_sequence(self):
-self.action_sequence = [
+self.action_sequence.extend([
"unpack",
"config_profile_link",
"setup_confdir",
@@ -97,9 +97,9 @@ class livecd_stage2(StageBase):
"setup_environment",
"run_local",
"build_kernel"
-]
+])
if "fetch" not in self.settings["options"]:
-self.action_sequence += [
+self.action_sequence.extend([
"bootloader",
"preclean",
"livecd_update",
@@ -114,5 +114,5 @@ class livecd_stage2(StageBase):
"target_setup",
"setup_overlay",
"create_iso",
-]
+])
self.action_sequence.append("clear_autoresume")
diff --git a/catalyst/targets/netboot.py b/catalyst/targets/netboot.py
index 61583f0d..e5c6d43c 100644
--- a/catalyst/targets/netboot.py
+++ b/catalyst/targets/netboot.py
@@ -160,7 +160,7 @@ class netboot(StageBase):
self.resume.enable("empty")
def set_action_sequence(self):
-self.action_sequence = [
+self.action_sequence.extend([
"unpack",
"config_profile_link",
"setup_confdir",
@@ -179,4 +179,4 @@ class netboot(StageBase):
"unbind",
"clean",
"clear_autoresume",
-]
+])
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/
commit: dfc4ed516b839bbc225cc7459e8c3085970801f6
Author: Matt Turner gentoo org>
AuthorDate: Thu Oct 29 00:28:24 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=dfc4ed51
catalyst: Move action_sequence out of self.settings[]
This self.settings[] dictionary is very similar to the god object
anti-pattern. Moving action_sequence out it starts us down the road of
fixing this.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py| 17 +
catalyst/targets/embedded.py | 2 +-
catalyst/targets/livecd_stage1.py | 2 +-
catalyst/targets/livecd_stage2.py | 6 +++---
catalyst/targets/netboot.py | 2 +-
catalyst/targets/stage1.py| 12 ++--
catalyst/targets/stage4.py| 2 +-
7 files changed, 22 insertions(+), 21 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 73eacfbe..801df2fb 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -64,6 +64,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
"portage_overlay",
"portage_prefix",
])
+self.action_sequence = []
self.set_valid_build_kernel_vars(addlargs)
TargetBase.__init__(self, myspec, addlargs)
@@ -477,13 +478,13 @@ class StageBase(TargetBase, ClearBase, GenBase):
Or it calls the normal set_action_sequence() for the target stage.
"""
if "purgeonly" in self.settings["options"]:
-self.settings["action_sequence"] = ["remove_chroot"]
+self.action_sequence = ["remove_chroot"]
return
self.set_action_sequence()
def set_action_sequence(self):
"""Set basic stage1, 2, 3 action sequences"""
-self.settings['action_sequence'] = [
+self.action_sequence = [
"unpack",
"setup_confdir",
"portage_overlay",
@@ -499,14 +500,14 @@ class StageBase(TargetBase, ClearBase, GenBase):
def set_completion_action_sequences(self):
if "fetch" not in self.settings["options"]:
-self.settings["action_sequence"].append("capture")
+self.action_sequence.append("capture")
if "keepwork" in self.settings["options"]:
-self.settings["action_sequence"].append("clear_autoresume")
+self.action_sequence.append("clear_autoresume")
elif "seedcache" in self.settings["options"]:
-self.settings["action_sequence"].append("remove_autoresume")
+self.action_sequence.append("remove_autoresume")
else:
-self.settings["action_sequence"].append("remove_autoresume")
-self.settings["action_sequence"].append("remove_chroot")
+self.action_sequence.append("remove_autoresume")
+self.action_sequence.append("remove_chroot")
def set_use(self):
use = self.settings["spec_prefix"] + "/use"
@@ -1380,7 +1381,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.purge()
failure = False
-for x in self.settings["action_sequence"]:
+for x in self.action_sequence:
log.notice('--- Running action sequence: %s', x)
sys.stdout.flush()
try:
diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py
index 99739512..3899cf1b 100644
--- a/catalyst/targets/embedded.py
+++ b/catalyst/targets/embedded.py
@@ -41,7 +41,7 @@ class embedded(StageBase):
StageBase.__init__(self, spec, addlargs)
def set_action_sequence(self):
-self.settings['action_sequence'] = [
+self.action_sequence = [
"unpack",
"config_profile_link",
"setup_confdir",
diff --git a/catalyst/targets/livecd_stage1.py
b/catalyst/targets/livecd_stage1.py
index f0b6be8b..b8c26cb1 100644
--- a/catalyst/targets/livecd_stage1.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -23,7 +23,7 @@ class livecd_stage1(StageBase):
StageBase.__init__(self, spec, addlargs)
def set_action_sequence(self):
-self.settings['action_sequence'] = [
+self.action_sequence = [
"unpack",
"config_profile_link",
"setup_confdir",
diff --git a/catalyst/targets/livecd_stage2.py
b/catalyst/targets/livecd_stage2.py
index 88c0d95c..cac16b6e 100644
--- a/catalyst/targets/livecd_stage2.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -87,7 +87,7 @@ class livecd_stage2(StageBase):
print_traceback=True)
def set_action_sequence(self):
-self.settings['action_sequence'] = [
+self.action_sequence = [
"unpack",
"config_profile_link",
"setup_confdir",
@@ -99,7 +99,7 @@ class livecd_stage2(StageBase):
"build_kernel"
]
if "fetch" not in
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/targets/
commit: 3d85d537b1a00d05234cea064f863602b83ffaab
Author: Matt Turner gentoo org>
AuthorDate: Thu Oct 29 03:08:04 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=3d85d537
catalyst: Split action_sequence into prepare/build/finish
We want to run the "build" sequence in a different mount namespace from
the "prepare" and "finish" sequences, so this splits action_sequence
into those groupings.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py| 24 +++-
catalyst/targets/embedded.py | 6 +-
catalyst/targets/livecd_stage1.py | 6 +-
catalyst/targets/livecd_stage2.py | 10 +++---
catalyst/targets/netboot.py | 6 +-
catalyst/targets/stage1.py| 12 ++--
catalyst/targets/stage4.py| 6 +-
7 files changed, 48 insertions(+), 22 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 46b7c59c..75c84baa 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -64,7 +64,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
"portage_overlay",
"portage_prefix",
])
-self.action_sequence = []
+self.prepare_sequence = []
+self.build_sequence = []
+self.finish_sequence = []
self.set_valid_build_kernel_vars(addlargs)
TargetBase.__init__(self, myspec, addlargs)
@@ -478,36 +480,40 @@ class StageBase(TargetBase, ClearBase, GenBase):
Or it calls the normal set_action_sequence() for the target stage.
"""
if "purgeonly" in self.settings["options"]:
-self.action_sequence.append("remove_chroot")
+self.build_sequence.append("remove_chroot")
return
self.set_action_sequence()
def set_action_sequence(self):
"""Set basic stage1, 2, 3 action sequences"""
-self.action_sequence.extend([
+self.prepare_sequence.extend([
"unpack",
"setup_confdir",
"portage_overlay",
+])
+self.build_sequence.extend([
"bind",
"chroot_setup",
"setup_environment",
"run_local",
"preclean",
"unbind",
+])
+self.finish_sequence.extend([
"clean",
])
self.set_completion_action_sequences()
def set_completion_action_sequences(self):
if "fetch" not in self.settings["options"]:
-self.action_sequence.append("capture")
+self.finish_sequence.append("capture")
if "keepwork" in self.settings["options"]:
-self.action_sequence.append("clear_autoresume")
+self.finish_sequence.append("clear_autoresume")
elif "seedcache" in self.settings["options"]:
-self.action_sequence.append("remove_autoresume")
+self.finish_sequence.append("remove_autoresume")
else:
-self.action_sequence.append("remove_autoresume")
-self.action_sequence.append("remove_chroot")
+self.finish_sequence.append("remove_autoresume")
+self.finish_sequence.append("remove_chroot")
def set_use(self):
use = self.settings["spec_prefix"] + "/use"
@@ -1381,7 +1387,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.purge()
failure = False
-for x in self.action_sequence:
+for x in self.prepare_sequence + self.build_sequence +
self.finish_sequence:
log.notice('--- Running action sequence: %s', x)
sys.stdout.flush()
try:
diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py
index 75eb68e4..1314ce7c 100644
--- a/catalyst/targets/embedded.py
+++ b/catalyst/targets/embedded.py
@@ -41,11 +41,13 @@ class embedded(StageBase):
StageBase.__init__(self, spec, addlargs)
def set_action_sequence(self):
-self.action_sequence.extend([
+self.prepare_sequence.extend([
"unpack",
"config_profile_link",
"setup_confdir",
"portage_overlay",
+])
+self.build_sequence.extend([
"bind",
"chroot_setup",
"setup_environment",
@@ -55,6 +57,8 @@ class embedded(StageBase):
"fsscript",
"unmerge",
"unbind",
+])
+self.finish_sequence.extend([
"remove",
"empty",
"clean",
diff --git a/catalyst/targets/livecd_stage1.py
b/catalyst/targets/livecd_stage1.py
index 9dbfa506..81367053 100644
--- a/catalyst/targets/livecd_stage1.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -23,16 +23,20 @@ class livecd_stage1(StageBase):
StageBase.__init__(self, spec, addlargs)
def set_action_sequence(self):
-
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 595ab9922f4ed8f88148ca7fbc9fe533db63fb4c
Author: Matt Turner gentoo org>
AuthorDate: Thu Oct 29 14:39:35 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=595ab992
catalyst: Remove mount_safety_check()
mount_safety_check() exists to prevent bad things from happening if a
previous catalyst invocation left bind mounts active in the chroot.
E.g., a previous catalyst invocation is interrupted without unmounting
the bind mounts. A new catalyst invocation runs and cleans the old
chroot, which inadvertently deletes files outside of the chroot via the
bind mounts.
With all the mounts now inside a namespace, it is no longer possible to
have mounts accessible outside the build sequence. In fact, I think this
code has been unnecessary since commit e5a53e42 ("catalyst: create
namespaces for building").
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 37 -
1 file changed, 37 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index bd5ba8d0..b9c220d0 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -638,39 +638,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
assert self.settings[verify] == "blake2"
self.settings.setdefault("gk_mainargs", []).append("--b2sum")
-def mount_safety_check(self):
-"""
-Check and verify that none of our paths in mypath are mounted. We don't
-want to clean up with things still mounted, and this allows us to
check.
-Returns 1 on ok, 0 on "something is still mounted" case.
-"""
-
-if not os.path.exists(self.settings["chroot_path"]):
-return
-
-log.debug('self.mount = %s', self.mount)
-for x in [x for x in self.mount if self.mount[x]['enable']]:
-target = normpath(self.settings['chroot_path'] +
- self.mount[x]['target'])
-log.debug('mount_safety_check() x = %s %s', x, target)
-if not os.path.exists(target):
-continue
-
-if ismount(target):
-# Something is still mounted
-try:
-log.warning(
-'%s is still mounted; performing auto-bind-umount...',
target)
-# Try to umount stuff ourselves
-self.unbind()
-if ismount(target):
-raise CatalystError("Auto-unbind failed for " + target)
-log.notice('Auto-unbind successful...')
-except CatalystError:
-raise CatalystError("Unable to auto-unbind " + target)
-
def unpack(self):
-
clst_unpack_hash = self.resume.get("unpack")
# Set up all unpack info settings
@@ -755,8 +723,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
% self.settings["source_path"])
if _unpack:
-self.mount_safety_check()
-
if invalid_chroot:
if "autoresume" in self.settings["options"]:
log.notice(
@@ -1363,9 +1329,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
def run(self):
self.chroot_lock.write_lock()
-# Check for mounts right away and abort if we cannot unmount them
-self.mount_safety_check()
-
if "clear-autoresume" in self.settings["options"]:
self.clear_autoresume()
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: cf50ce228e766252fe0ea901f86671fed6a99cfa
Author: Matt Turner gentoo org>
AuthorDate: Wed Oct 28 20:50:00 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=cf50ce22
catalyst: Replace pathcompare()
Modern Python allows us to do this in a much cleaner way.
Signed-off-by: Matt Turner gentoo.org>
catalyst/support.py | 24 +++-
1 file changed, 7 insertions(+), 17 deletions(-)
diff --git a/catalyst/support.py b/catalyst/support.py
index f49315a5..4458ed20 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -5,6 +5,7 @@ import os
import re
import shutil
import time
+from pathlib import Path
from subprocess import Popen
from catalyst import log
@@ -179,31 +180,20 @@ def read_makeconf(mymakeconffile):
return makeconf
-def pathcompare(path1, path2):
-# Change double slashes to slash
-path1 = re.sub(r"//", r"/", path1)
-path2 = re.sub(r"//", r"/", path2)
-# Removing ending slash
-path1 = re.sub("/$", "", path1)
-path2 = re.sub("/$", "", path2)
-
-if path1 == path2:
-return 1
-return 0
-
-
def ismount(path):
"""Like os.path.ismount, but also support bind mounts"""
if os.path.ismount(path):
-return 1
+return True
+
a = os.popen("mount")
mylines = a.readlines()
a.close()
for line in mylines:
mysplit = line.split()
-if pathcompare(path, mysplit[2]):
-return 1
-return 0
+if Path(path) == Path(mysplit[2]):
+return True
+
+return False
def addl_arg_parse(myspec, addlargs, requiredspec, validspec):
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/
commit: 488b06bf5dbe1eba68ac11de95f56feeb6cead83 Author: Matt Turner gentoo org> AuthorDate: Thu Oct 29 15:00:42 2020 + Commit: Matt Turner gentoo org> CommitDate: Fri Oct 30 22:40:52 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=488b06bf catalyst: Run the build sequence in new mount namespace Catalyst has a lot of code to unmount the bind mounts it's made, and then more to try harder when something fails. This is important because if bind mounts still exist within the chroot when clean up happens, files outside of the chroot on the host system can inadvertently be deleted. E.g., distfiles, binpkgs, kerncache. Running the build sequence (the steps that need bind mounts) within a mount namespace and exiting the mount namespace when finished ensures that clean up can never accidentally delete files outside the chroot. Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py | 7 --- catalyst/main.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 06ec8727..caec5935 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -15,6 +15,7 @@ from snakeoil.osutils import pjoin from DeComp.compress import CompressMap from catalyst import log +from catalyst.context import namespace from catalyst.defaults import (confdefaults, MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN) from catalyst.support import (CatalystError, file_locate, normpath, cmd, read_makeconf, ismount, file_check, @@ -1405,9 +1406,9 @@ class StageBase(TargetBase, ClearBase, GenBase): if not self.run_sequence(self.prepare_sequence): return False -if not self.run_sequence(self.build_sequence): -self.unbind() -return False +with namespace(mount=True): +if not self.run_sequence(self.build_sequence): +return False if not self.run_sequence(self.finish_sequence): return False diff --git a/catalyst/main.py b/catalyst/main.py index 93a4a0d3..5536471a 100644 --- a/catalyst/main.py +++ b/catalyst/main.py @@ -355,7 +355,7 @@ def _main(parser, opts): # use pid & user namespaces, but snakeoil's namespace module has signal # transfer issues (CTRL+C doesn't propagate), and user namespaces need # more work due to Gentoo build process (uses sudo/root/portage). -with namespace(mount=True, uts=True, ipc=True, hostname='catalyst'): +with namespace(uts=True, ipc=True, hostname='catalyst'): # everything is setup, so the build is a go try: success = build_target(addlargs)
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: d5747d138a0ce706d4b18be60d99ea55bdbc4120
Author: Matt Turner gentoo org>
AuthorDate: Sat May 16 21:44:37 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=d5747d13
catalyst: Use libmount for handling mounts
Handle the mounts/unmounts in all in process rather than shelling out
(pun intended!) to an external program.
While we're here, change some log.notice to log.debug since those cases
are normal and expected.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 57 --
1 file changed, 35 insertions(+), 22 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index e71ce344..73eacfbe 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -6,6 +6,7 @@ import sys
from pathlib import Path
+import libmount
import toml
from snakeoil import fileutils
@@ -853,7 +854,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
source = str(self.mount[x]['source'])
target = self.settings['chroot_path'] +
str(self.mount[x]['target'])
-mount = ['mount']
+fstype = ''
+options = ''
log.debug('bind %s: "%s" -> "%s"', x, source, target)
@@ -861,18 +863,20 @@ class StageBase(TargetBase, ClearBase, GenBase):
if 'var_tmpfs_portage' not in self.settings:
continue
-mount += ['-t', 'tmpfs', '-o',
- f"size={self.settings['var_tmpfs_portage']}G"]
+fstype = 'tmpfs'
+options = f"size={self.settings['var_tmpfs_portage']}G"
elif source == 'tmpfs':
-mount += ['-t', 'tmpfs']
+fstype = 'tmpfs'
elif source == 'shm':
-mount += ['-t', 'tmpfs', '-o', 'noexec,nosuid,nodev']
+fstype = 'tmpfs'
+options = 'noexec,nosuid,nodev'
else:
source_path = Path(self.mount[x]['source'])
if source_path.suffix == '.sqfs':
-mount += ['-o', 'ro']
+fstype = 'squashfs'
+options = 'ro,loop'
else:
-mount.append('--bind')
+options = 'bind'
# We may need to create the source of the bind mount.
E.g., in the
# case of an empty package cache we must create the
directory that
@@ -881,38 +885,47 @@ class StageBase(TargetBase, ClearBase, GenBase):
Path(target).mkdir(mode=0o755, parents=True, exist_ok=True)
-cmd(mount + [source, target], env=self.env, fail_func=self.unbind)
+try:
+cxt = libmount.Context(source=source, target=target,
+ fstype=fstype, options=options)
+cxt.mount()
+except OSError as e:
+self.unbind()
+raise CatalystError(f"Couldn't mount: {source}, {e.strerror}")
def unbind(self):
-ouch = 0
-mypath = self.settings["chroot_path"]
+chroot_path = self.settings["chroot_path"]
+umount_failed = False
# Unmount in reverse order
-for x in [x for x in reversed(self.mount) if self.mount[x]['enable']]:
-target = normpath(mypath + self.mount[x]['target'])
-if not os.path.exists(target):
-log.notice('%s does not exist. Skipping', target)
+for target in [Path(chroot_path + self.mount[x]['target'])
+ for x in reversed(self.mount)
+ if self.mount[x]['enable']]:
+if not target.exists():
+log.debug('%s does not exist. Skipping', target)
continue
if not ismount(target):
-log.notice('%s is not a mount point. Skipping', target)
+log.debug('%s is not a mount point. Skipping', target)
continue
try:
-cmd(['umount', target], env=self.env)
-except CatalystError:
+cxt = libmount.Context(target=str(target))
+cxt.umount()
+except OSError:
log.warning('First attempt to unmount failed: %s', target)
log.warning('Killing any pids still running in the chroot')
self.kill_chroot_pids()
try:
-cmd(['umount', target], env=self.env)
-except CatalystError:
-ouch = 1
-log.warning("Couldn't umount bind mount: %s", target)
+cxt.umount()
+except OSError as e:
+umount_failed = True
+log.warning("Couldn't umount: %s, %s", target,
+
[gentoo-commits] proj/catalyst:master commit in: catalyst/, /
commit: efe2e4fa789550473734269f27a4029df208a2b2
Author: Matt Turner gentoo org>
AuthorDate: Thu Oct 22 18:52:46 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=efe2e4fa
catalyst: Rewrite ismount() to use libmount
libmount is provided by util-linux, so this adds a dependency on
sys-apps/util-linux[python]. A later patch will make more extensive use
of this API.
Signed-off-by: Matt Turner gentoo.org>
README | 2 +-
catalyst/support.py | 14 +++---
setup.py| 2 --
3 files changed, 8 insertions(+), 10 deletions(-)
diff --git a/README b/README
index 1cceb63e..594de9e1 100644
--- a/README
+++ b/README
@@ -17,7 +17,7 @@ simple and reproducable manner. Use at your own risk.
Requirements
===
-- Python 3.6 or greater
+- Python 3.8 or greater
- A generic stage3 tarball for your architecture
- A squashfs ebuild repository snapshot
- Or an ebuild git repo with sys-fs/squashfs-tools-ng and dev-vcs/git
diff --git a/catalyst/support.py b/catalyst/support.py
index 4458ed20..ddbd9ab9 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -8,6 +8,8 @@ import time
from pathlib import Path
from subprocess import Popen
+import libmount
+
from catalyst import log
BASH_BINARY = "/bin/bash"
@@ -182,15 +184,13 @@ def read_makeconf(mymakeconffile):
def ismount(path):
"""Like os.path.ismount, but also support bind mounts"""
-if os.path.ismount(path):
+path = Path(path)
+if path.is_mount():
return True
-a = os.popen("mount")
-mylines = a.readlines()
-a.close()
-for line in mylines:
-mysplit = line.split()
-if Path(path) == Path(mysplit[2]):
+cxt = libmount.Context()
+while (fs := cxt.mtab.next_fs()) is not None:
+if path == Path(fs.target):
return True
return False
diff --git a/setup.py b/setup.py
index 7a97b30c..fc1ac005 100755
--- a/setup.py
+++ b/setup.py
@@ -101,8 +101,6 @@ _setup(
'Topic :: System :: Installation/Setup',
'Topic :: System :: Software Distribution',
'Programming Language :: Python :: 3',
-'Programming Language :: Python :: 3.6',
-'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
],
scripts=['bin/{0}'.format(_package_name)],
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: 4bcc64dcf2b27ab3d0effed0fa2516e3513a9171
Author: Matt Turner gentoo org>
AuthorDate: Wed Oct 28 21:59:17 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4bcc64dc
catalyst: Add and use namespace context manager
Wraps snakeoil's simple_unshare; returns to the previous namespaces on
context exit. Will be used by the next commit.
Signed-off-by: Matt Turner gentoo.org>
catalyst/context.py | 32
catalyst/main.py| 17 +++--
2 files changed, 39 insertions(+), 10 deletions(-)
diff --git a/catalyst/context.py b/catalyst/context.py
new file mode 100644
index ..8a58f33d
--- /dev/null
+++ b/catalyst/context.py
@@ -0,0 +1,32 @@
+
+import contextlib
+import os
+
+from snakeoil.process.namespaces import setns, simple_unshare
+
+@contextlib.contextmanager
+def namespace(mount=False, uts=False, ipc=False, net=False, pid=False,
+ user=False, hostname=None):
+namespaces = {
+(mount, "mnt"): None,
+(uts, "uts"): None,
+(ipc, "ipc"): None,
+(net, "net"): None,
+(pid, "pid"): None,
+(user, "user"): None,
+}
+
+# Save fds of current namespaces
+for ns in [ns for ns in namespaces if ns[0]]:
+fp = open(f"/proc/self/ns/{ns[1]}")
+namespaces[ns] = fp
+
+simple_unshare(mount=mount, uts=uts, ipc=ipc, net=net, pid=pid, user=user,
+ hostname=hostname)
+try:
+yield
+finally:
+for ns in [ns for ns in namespaces if ns[0]]:
+fp = namespaces[ns]
+setns(fp.fileno(), 0)
+fp.close()
diff --git a/catalyst/main.py b/catalyst/main.py
index 543895c6..93a4a0d3 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -7,14 +7,13 @@ import textwrap
import toml
-from snakeoil.process import namespaces
-
from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS,
CONTENTS_DEFINITIONS)
from DeComp.contents import ContentsMap
from catalyst import log
import catalyst.config
+from catalyst.context import namespace
from catalyst.defaults import (confdefaults, option_messages,
DEFAULT_CONFIG_FILE, valid_config_file_values)
from catalyst.support import CatalystError
@@ -356,15 +355,13 @@ def _main(parser, opts):
# use pid & user namespaces, but snakeoil's namespace module has signal
# transfer issues (CTRL+C doesn't propagate), and user namespaces need
# more work due to Gentoo build process (uses sudo/root/portage).
-namespaces.simple_unshare(
-mount=True, uts=True, ipc=True, pid=False, net=False, user=False,
-hostname='catalyst')
+with namespace(mount=True, uts=True, ipc=True, hostname='catalyst'):
+# everything is setup, so the build is a go
+try:
+success = build_target(addlargs)
+except KeyboardInterrupt:
+log.critical('Catalyst build aborted due to user interrupt
(Ctrl-C)')
-# everything is setup, so the build is a go
-try:
-success = build_target(addlargs)
-except KeyboardInterrupt:
-log.critical('Catalyst build aborted due to user interrupt (Ctrl-C)')
if not success:
sys.exit(2)
sys.exit(0)
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: d9010dead74470b2be148412e5b9ed9cca7a8b45
Author: Matt Turner gentoo org>
AuthorDate: Thu Oct 29 13:03:33 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=d9010dea
catalyst: Factor out run_sequence()
This is preparation for the next patch, which will run the build
sequence in a separate mount namespace.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 40 +++-
1 file changed, 23 insertions(+), 17 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 75c84baa..06ec8727 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1362,6 +1362,22 @@ class StageBase(TargetBase, ClearBase, GenBase):
log.debug('setup_environment(); env = %r', self.env)
+def run_sequence(self, sequence):
+for func in sequence:
+log.notice('--- Running action sequence: %s', func)
+sys.stdout.flush()
+try:
+getattr(self, func)()
+except LockInUse:
+log.error('Unable to aquire the lock...')
+return False
+except Exception:
+log.error('Exception running action sequence %s',
+ func, exc_info=True)
+return False
+
+return True
+
def run(self):
self.chroot_lock.write_lock()
@@ -1386,26 +1402,16 @@ class StageBase(TargetBase, ClearBase, GenBase):
log.info('StageBase: run() purge')
self.purge()
-failure = False
-for x in self.prepare_sequence + self.build_sequence +
self.finish_sequence:
-log.notice('--- Running action sequence: %s', x)
-sys.stdout.flush()
-try:
-getattr(self, x)()
-except LockInUse:
-log.error('Unable to aquire the lock...')
-failure = True
-break
-except Exception:
-log.error('Exception running action sequence %s',
- x, exc_info=True)
-failure = True
-break
+if not self.run_sequence(self.prepare_sequence):
+return False
-if failure:
-log.notice('Cleaning up... Running unbind()')
+if not self.run_sequence(self.build_sequence):
self.unbind()
return False
+
+if not self.run_sequence(self.finish_sequence):
+return False
+
return True
def unmerge(self):
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/
commit: 4b5ff905a7ad2ffe1ed8c863b91e9d0ce6981f5f
Author: Matt Turner gentoo org>
AuthorDate: Thu Oct 22 20:37:54 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4b5ff905
catalyst: Drop unbind()
mount_namespaces(7) says
A mount ceases to be a member of a peer group when either the
mount is explicitly unmounted, or when the mount is implicitly
unmounted because a mount namespace is removed (because it has
no more member processes).
As a result, we can rely on exiting the mount namespace to unmount the
bind mounts.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py| 44 +--
catalyst/targets/embedded.py | 1 -
catalyst/targets/livecd_stage1.py | 1 -
catalyst/targets/livecd_stage2.py | 2 --
catalyst/targets/netboot.py | 3 ---
catalyst/targets/stage4.py| 1 -
6 files changed, 1 insertion(+), 51 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index b9c220d0..a75dbdf9 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -498,7 +498,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
"setup_environment",
"run_local",
"preclean",
-"unbind",
])
self.finish_sequence.extend([
"clean",
@@ -853,40 +852,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
fstype=fstype, options=options)
cxt.mount()
except OSError as e:
-self.unbind()
raise CatalystError(f"Couldn't mount: {source}, {e.strerror}")
-def unbind(self):
-chroot_path = self.settings["chroot_path"]
-umount_failed = False
-
-# Unmount in reverse order
-for target in [Path(chroot_path + self.mount[x]['target'])
- for x in reversed(self.mount)
- if self.mount[x]['enable']]:
-if not target.exists():
-log.debug('%s does not exist. Skipping', target)
-continue
-
-if not ismount(target):
-log.debug('%s is not a mount point. Skipping', target)
-continue
-
-try:
-cxt = libmount.Context(target=str(target))
-cxt.umount()
-except OSError as e:
-log.warning("Couldn't umount: %s, %s", target,
-e.strerror)
-umount_failed = True
-
-if umount_failed:
-# if any bind mounts really failed, then we need to raise
-# this to potentially prevent an upcoming bash stage cleanup script
-# from wiping our bind mounts.
-raise CatalystError(
-"Couldn't umount one or more bind-mounts; aborting for
safety.")
-
def chroot_setup(self):
self.makeconf = read_makeconf(normpath(self.settings["chroot_path"] +
self.settings["make_conf"]))
@@ -1190,7 +1157,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
env=self.env)
self.resume.enable("remove")
except:
-self.unbind()
raise
def preclean(self):
@@ -1206,7 +1172,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.resume.enable("preclean")
except:
-self.unbind()
raise CatalystError("Build failed, could not execute preclean")
def capture(self):
@@ -1269,7 +1234,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings['controller_file'])
except CatalystError:
-self.unbind()
raise CatalystError("Stage build aborting due to error.",
print_traceback=False)
@@ -1374,7 +1338,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
env=self.env)
log.info('unmerge shell script')
except CatalystError:
-self.unbind()
raise
self.resume.enable("unmerge")
@@ -1449,7 +1412,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
fileutils.touch(build_packages_resume)
self.resume.enable("build_packages")
except CatalystError:
-self.unbind()
raise CatalystError(
self.settings["spec_prefix"] +
"build aborting due to error.")
@@ -1473,7 +1435,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
self._build_kernel(kname=kname)
self.resume.enable("build_kernel")
except CatalystError:
-self.unbind()
[gentoo-commits] proj/catalyst:master commit in: catalyst/, doc/, catalyst/base/, targets/support/, etc/
commit: 3269cfc50ec2eeccbb1c5aebc89fd4b38ce1b3e8
Author: Matt Turner gentoo org>
AuthorDate: Thu May 14 02:37:11 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Wed Oct 21 01:07:29 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=3269cfc5
catalyst: Set jobs/load-average via catalyst.conf
We currently have two mechanisms of setting MAKEOPTS: in spec files and
in catalystrc.
Setting makeopts in spec files doesn't make sense. The spec should
describe the thing that's being built and not contain options that are
specific to the build system.
Setting makeopts via catalystrc is better, but it only applies to the
actual build system invocations, leaving emerge to run jobs serially or
again requiring configuration specific to the build machine to be put
into the spec file. For example:
update_seed_command: ... --jobs 5 --load-average 5
With jobs and load-average specified in catalyst.conf, catalyst has the
information required to configure both emerge and the build systems
emerge executes.
This removes the undocumented makeopts spec file option and replaces it
with jobs and load-average settings in catalyst.conf.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 12 +---
catalyst/defaults.py| 2 ++
doc/catalyst-config.5.txt | 15 ---
etc/catalyst.conf | 8
etc/catalystrc | 3 ---
targets/support/chroot-functions.sh | 8
6 files changed, 35 insertions(+), 13 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 99d638d8..afea5776 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -59,7 +59,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
"interpreter",
"kerncache_path",
"ldflags",
-"makeopts",
"pkgcache_path",
"portage_confdir",
"portage_overlay",
@@ -1336,12 +1335,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
continue
log.warning("Not making envar for '%s', is a dict", x)
-if "makeopts" in self.settings:
-if isinstance(self.settings["makeopts"], str):
-self.env["MAKEOPTS"] = self.settings["makeopts"]
-else:
-# ensure makeopts is a string
-self.env["MAKEOPTS"] = ' '.join(self.settings["makeopts"])
+makeopts = []
+for flag, setting in {'j': 'jobs', 'l': 'load-average'}.items():
+if setting in self.settings:
+makeopts.append(f'-{flag}{self.settings[setting]}')
+self.env['MAKEOPTS'] = ' '.join(makeopts)
log.debug('setup_environment(); env = %r', self.env)
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index c153fcc4..7a2fe3f3 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -13,6 +13,8 @@ valid_config_file_values = frozenset([
"digests",
"distdir",
"envscript",
+"jobs",
+"load-average",
"options",
"port_logdir",
"repo_basedir",
diff --git a/doc/catalyst-config.5.txt b/doc/catalyst-config.5.txt
index 7ac9a2a3..cbef6092 100644
--- a/doc/catalyst-config.5.txt
+++ b/doc/catalyst-config.5.txt
@@ -48,9 +48,9 @@ $ python3 -c 'import hashlib;
print(hashlib.algorithms_available)'
*envscript*::
Environment script location, which allows users to set options such as
-HTTP proxies, `MAKEOPTS`, `GENTOO_MIRRORS`, or any other environment
-variables needed for building. The envscript file sets environment
-variables using POSIX shell notation:
+HTTP proxies, `GENTOO_MIRRORS`, or any other environment variables
+needed for building. The envscript file sets environment variables
+using POSIX shell notation:
+
-
export FOO="bar"
@@ -136,6 +136,15 @@ written to the target's make.conf if it is not the default
value of
Other settings
~~
+*jobs*::
+Integral value passed to *emerge(1)* as the parameter to --jobs and is
+used to define *MAKEOPTS* during the target build.
+
+*load-average*::
+Floating-point value passed to *emerge(1)* as the parameter to
+--load-average and is used to define *MAKEOPTS* during the target
+build.
+
*sharedir*::
Catalyst runtime script location. `/usr/share/catalyst` should work for
most default installations. If you are running catalyst from a Git
diff --git a/etc/catalyst.conf b/etc/catalyst.conf
index 2272cb86..81693c25 100644
--- a/etc/catalyst.conf
+++ b/etc/catalyst.conf
@@ -76,3 +76,11 @@ options = [
# WARNING: If you use too much RAM everything will fail horribly and it is not
our fault.
# set size of /var/tmp/portage tmpfs in gigabytes
# var_tmpfs_portage = 16
+
+# Integral value passed to emerge as the parameter to --jobs and is used to
+# define MAKEOPTS during the target build.
+# jobs = 4
+
+# Floating-point value passed to
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 5d01bda685236a8543da3a8136a4fe809f6fd9ae
Author: Felix Bier rohde-schwarz com>
AuthorDate: Wed Oct 14 13:38:21 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Wed Oct 14 17:47:48 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=5d01bda6
catalyst: Fix spec file USE flag parsing
In stagebase, the set_use function applies .split() to the use flags
passed from the spec file, if the value is a string. However, the
result is immediately overwritten after the if-statement. Therefore the
.split() is ineffectual.
This results in self.settings["use"] holding a string,
which is then regarded as a list of characters in write_make_conf.
This fix ensures that the result of the split is not overwritten
(matching the similar code in set_catalyst_use).
For example, setting "stage4/use: abc" in a spec file results
in USE="a b c ..." in the generated make.conf.
With this fix, the generated make.conf contains the expected
USE="abc ...".
Fixes: b30dd97d ("Unify all make.conf settings and writing")
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index df1cb844..2e313bd8 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -512,7 +512,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
if use in self.settings:
if isinstance(self.settings[use], str):
self.settings["use"] = self.settings[use].split()
-self.settings["use"] = self.settings[use]
+else:
+self.settings["use"] = self.settings[use]
del self.settings[use]
else:
self.settings["use"] = []
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: b3f0a3535e3550ed7d0dbd3100e43b82e9b34c3b
Author: Matt Turner gentoo org>
AuthorDate: Thu Oct 8 21:15:03 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Thu Oct 8 21:15:03 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b3f0a353
catalyst: Fix typo
Fixes: 3c5ef2e7 ("log: new logging module to standardize catalyst output")
Signed-off-by: Matt Turner gentoo.org>
catalyst/log.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/catalyst/log.py b/catalyst/log.py
index 9d534355..ee124392 100644
--- a/catalyst/log.py
+++ b/catalyst/log.py
@@ -74,7 +74,7 @@ class CatalystFormatter(logging.Formatter):
@staticmethod
def detect_color():
"""Figure out whether the runtime env wants color"""
-if 'NOCOLOR' is os.environ:
+if 'NOCOLOR' in os.environ:
return False
return os.isatty(sys.stdout.fileno())
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: a1efe958a133a49275b359e78cde0f222166cbcd
Author: Matt Turner gentoo org>
AuthorDate: Sat Aug 15 21:11:10 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Sat Aug 15 21:11:10 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=a1efe958
stagebase: Run autopep8 to remove tabs
Fixes: 22f59e6d758b (Add "interpreter" parameter for qemu-user build support)
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 49 +-
1 file changed, 27 insertions(+), 22 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 345fe1db..df1cb844 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -939,23 +939,26 @@ class StageBase(TargetBase, ClearBase, GenBase):
shutil.copy('/etc/resolv.conf',
self.settings['chroot_path'] + '/etc/')
- # Copy over the binary interpreter (qemu), if
applicable; note that it's given
- # as full path and goes to the same place in the chroot
- if "interpreter" in self.settings:
- if not
os.path.exists(self.settings["interpreter"]):
- raise CatalystError(
- "Can't find interpreter " +
self.settings["interpreter"],
- print_traceback=True)
+# Copy over the binary interpreter (qemu), if applicable; note
that it's given
+# as full path and goes to the same place in the chroot
+if "interpreter" in self.settings:
+if not os.path.exists(self.settings["interpreter"]):
+raise CatalystError(
+"Can't find interpreter " +
+self.settings["interpreter"],
+print_traceback=True)
- log.notice('Copying binary interpreter %s into
chroot', self.settings['interpreter'])
+log.notice('Copying binary interpreter %s into chroot',
+ self.settings['interpreter'])
- if os.path.exists(self.settings['chroot_path']
+ '/' + self.settings['interpreter']):
- os.rename(
- self.settings['chroot_path'] +
'/' + self.settings['interpreter'],
- self.settings['chroot_path'] +
'/' + self.settings['interpreter'] + '.catalyst')
+if os.path.exists(self.settings['chroot_path'] + '/' +
self.settings['interpreter']):
+os.rename(
+self.settings['chroot_path'] +
+'/' + self.settings['interpreter'],
+self.settings['chroot_path'] + '/' +
self.settings['interpreter'] + '.catalyst')
- shutil.copy(self.settings['interpreter'],
- self.settings['chroot_path'] + '/' +
self.settings['interpreter'])
+shutil.copy(self.settings['interpreter'],
+self.settings['chroot_path'] + '/' +
self.settings['interpreter'])
# Copy over the envscript, if applicable
if "envscript" in self.settings:
@@ -1124,14 +1127,16 @@ class StageBase(TargetBase, ClearBase, GenBase):
if os.path.exists(hosts_file + '.catalyst'):
os.rename(hosts_file + '.catalyst', hosts_file)
- # optionally clean up binary interpreter
- if "interpreter" in self.settings:
- if os.path.exists(self.settings['chroot_path'] + '/' +
self.settings['interpreter'] + '.catalyst'):
- os.rename(
- self.settings['chroot_path'] + '/' +
self.settings['interpreter'] + '.catalyst',
- self.settings['chroot_path'] + '/' +
self.settings['interpreter'])
- else:
- os.remove(self.settings['chroot_path'] + '/' +
self.settings['interpreter'])
+# optionally clean up binary interpreter
+if "interpreter" in self.settings:
+if os.path.exists(self.settings['chroot_path'] + '/' +
self.settings['interpreter'] + '.catalyst'):
+os.rename(
+self.settings['chroot_path'] + '/' +
+self.settings['interpreter'] + '.catalyst',
+self.settings['chroot_path'] + '/' +
self.settings['interpreter'])
+else:
+os.remove(
+self.settings['chroot_path'] + '/' +
self.settings['interpreter'])
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: b267ba9b28953df3ba6528e8cf189527b037fc1d
Author: Andreas K. Hüttel gentoo org>
AuthorDate: Thu Jul 23 22:25:17 2020 +
Commit: Andreas K. Hüttel gentoo org>
CommitDate: Fri Jul 24 13:34:54 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b267ba9b
stagebase: Extend cleanup logic to more directories
Needed for FEATURES management (switching off pid namespaces in qemu)
and for testing build system hacks (like building python single-threaded
to avoid hangs).
Signed-off-by: Andreas K. Hüttel gentoo.org>
catalyst/base/stagebase.py | 10 --
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 0d3a418b..345fe1db 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1138,13 +1138,19 @@ class StageBase(TargetBase, ClearBase, GenBase):
"sticky-config" not in self.settings["options"]):
log.debug("clean(), portage_preix = %s, no sticky-config",
self.settings["portage_prefix"])
-for _dir in "accept_keywords", "keywords", "mask", "unmask", "use":
+for _dir in "package.accept_keywords", "package.keywords",
"package.mask", "package.unmask", "package.use", "package.env", "env":
target = pjoin(self.settings["destpath"],
- "etc/portage/package.%s" % _dir,
+ "etc/portage/%s" % _dir,
self.settings["portage_prefix"])
log.notice("Clearing portage_prefix target: %s", target)
clear_path(target)
+# Remove hacks that should *never* go into stages
+target = pjoin(self.settings["destpath"], "etc/portage/patches")
+if os.path.exists(target):
+log.warning("You've been hacking. Clearing target patches: %s",
target)
+clear_path(target)
+
# Remove our overlay
overlay = normpath(
self.settings["chroot_path"] + self.settings["local_overlay"])
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 22f59e6d758b3c4801a984ea6a3b4574407143c6
Author: Andreas K. Hüttel gentoo org>
AuthorDate: Mon Jul 20 20:39:53 2020 +
Commit: Andreas K. Hüttel gentoo org>
CommitDate: Mon Jul 20 20:39:53 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=22f59e6d
Add "interpreter" parameter for qemu-user build support
Signed-off-by: Andreas K. Hüttel gentoo.org>
catalyst/base/stagebase.py | 28
1 file changed, 28 insertions(+)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index c2cc2c3e..0d3a418b 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -55,6 +55,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
"fflags",
"hostuse",
"install_mask",
+"interpreter",
"kerncache_path",
"ldflags",
"makeopts",
@@ -938,6 +939,24 @@ class StageBase(TargetBase, ClearBase, GenBase):
shutil.copy('/etc/resolv.conf',
self.settings['chroot_path'] + '/etc/')
+ # Copy over the binary interpreter (qemu), if
applicable; note that it's given
+ # as full path and goes to the same place in the chroot
+ if "interpreter" in self.settings:
+ if not
os.path.exists(self.settings["interpreter"]):
+ raise CatalystError(
+ "Can't find interpreter " +
self.settings["interpreter"],
+ print_traceback=True)
+
+ log.notice('Copying binary interpreter %s into
chroot', self.settings['interpreter'])
+
+ if os.path.exists(self.settings['chroot_path']
+ '/' + self.settings['interpreter']):
+ os.rename(
+ self.settings['chroot_path'] +
'/' + self.settings['interpreter'],
+ self.settings['chroot_path'] +
'/' + self.settings['interpreter'] + '.catalyst')
+
+ shutil.copy(self.settings['interpreter'],
+ self.settings['chroot_path'] + '/' +
self.settings['interpreter'])
+
# Copy over the envscript, if applicable
if "envscript" in self.settings:
if not os.path.exists(self.settings["envscript"]):
@@ -1105,6 +1124,15 @@ class StageBase(TargetBase, ClearBase, GenBase):
if os.path.exists(hosts_file + '.catalyst'):
os.rename(hosts_file + '.catalyst', hosts_file)
+ # optionally clean up binary interpreter
+ if "interpreter" in self.settings:
+ if os.path.exists(self.settings['chroot_path'] + '/' +
self.settings['interpreter'] + '.catalyst'):
+ os.rename(
+ self.settings['chroot_path'] + '/' +
self.settings['interpreter'] + '.catalyst',
+ self.settings['chroot_path'] + '/' +
self.settings['interpreter'])
+ else:
+ os.remove(self.settings['chroot_path'] + '/' +
self.settings['interpreter'])
+
# optionally clean up portage configs
if ("portage_prefix" in self.settings and
"sticky-config" not in self.settings["options"]):
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/
commit: bb83032e1c2c299aca7a82ea765adaa1c44d6707 Author: Daniel Cordero 0xdc io> AuthorDate: Thu May 21 17:32:44 2020 + Commit: Matt Turner gentoo org> CommitDate: Fri Jun 5 21:02:24 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=bb83032e embedded: remove dir_setup action dir_setup() hasn't existed since 2005. Fixes: 1dafb5fa06d2 (Add locking support. ...) Signed-off-by: Matt Turner gentoo.org> catalyst/targets/embedded.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py index aa23f5b3..1b4ad9d6 100644 --- a/catalyst/targets/embedded.py +++ b/catalyst/targets/embedded.py @@ -41,7 +41,6 @@ class embedded(StageBase): def set_action_sequence(self): self.settings['action_sequence'] = [ -"dir_setup", "unpack", "config_profile_link", "setup_confdir", @@ -51,7 +50,6 @@ class embedded(StageBase): "setup_environment", "build_kernel", "build_packages", -"bootloader", "root_overlay", "fsscript", "unmerge",
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 834e1caa4d78ffc7607c6508f54fd192a5a3830c Author: Daniel Cordero 0xdc io> AuthorDate: Thu May 21 17:32:45 2020 + Commit: Matt Turner gentoo org> CommitDate: Fri Jun 5 21:03:26 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=834e1caa stagebase: allow specfiles to define install_mask There is some code that uses this option - set_install_mask() - but it was not added to the list of valid specfile options. The only mention of use is in the embedded target, but it may also be useful for other stages. Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 5a8cd1df..5c59c4d2 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -54,6 +54,7 @@ class StageBase(TargetBase, ClearBase, GenBase): "fcflags", "fflags", "hostuse", +"install_mask", "kerncache_path", "ldflags", "makeopts",
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: fd1903e5cd80a13d397ae07c6472d08ac6675cba
Author: Daniel Cordero 0xdc io>
AuthorDate: Thu May 21 17:32:46 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Jun 5 21:08:36 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=fd1903e5
embedded: use destpath to clean mergeroot
When using remove() with a capture subpath (e.g. the embedded target),
specifying
files in rm removes the files from the seed stage, not the captured stage.
Use destpath as a base for removals. When not using a subpath, destpath
equals chroot_path.
Also unhardcode log prefix to match the target.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 5c59c4d2..ab1d409b 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1168,8 +1168,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
for x in self.settings[self.settings["spec_prefix"] + "/rm"]:
# We're going to shell out for all these cleaning
# operations, so we get easy glob handling.
-log.notice('livecd: removing %s', x)
-clear_path(self.settings["chroot_path"] + x)
+log.notice('%s: removing %s',
self.settings["spec_prefix"], x)
+clear_path(self.settings["destpath"] + x)
try:
if os.path.exists(self.settings["controller_file"]):
cmd([self.settings['controller_file'], 'clean'],
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/
commit: 71d6e0bfeff39e6ffa9a3619060e731446c5b497 Author: Daniel Cordero 0xdc io> AuthorDate: Thu May 21 17:32:48 2020 + Commit: Matt Turner gentoo org> CommitDate: Fri Jun 5 21:08:42 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=71d6e0bf embedded: Allow a root_overlay This was broken until the previous commit. Signed-off-by: Matt Turner gentoo.org> catalyst/targets/embedded.py | 1 + 1 file changed, 1 insertion(+) diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py index 1b4ad9d6..99739512 100644 --- a/catalyst/targets/embedded.py +++ b/catalyst/targets/embedded.py @@ -31,6 +31,7 @@ class embedded(StageBase): "embedded/mergeroot", "embedded/packages", "embedded/rm", +"embedded/root_overlay", "embedded/runscript", "embedded/unmerge", "embedded/use",
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: 2c82d8a8833edd200db1bf6ae36807376bbb61b8 Author: Daniel Cordero 0xdc io> AuthorDate: Fri May 29 19:04:31 2020 + Commit: Matt Turner gentoo org> CommitDate: Fri Jun 5 21:09:29 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=2c82d8a8 catalyst: Allow compression_mode in config file Allow overriding compression mode globally from the catalyst config. Otherwise, if one wanted a different compression method than the default, one would need to add compression_mode to every spec file. Signed-off-by: Matt Turner gentoo.org> catalyst/defaults.py | 1 + 1 file changed, 1 insertion(+) diff --git a/catalyst/defaults.py b/catalyst/defaults.py index 14f671fe..c153fcc4 100644 --- a/catalyst/defaults.py +++ b/catalyst/defaults.py @@ -9,6 +9,7 @@ from DeComp.definitions import DECOMPRESSOR_PROGRAM_OPTIONS, LIST_XATTRS_OPTIONS valid_config_file_values = frozenset([ +"compression_mode", "digests", "distdir", "envscript",
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: d2dd86b8f21337133d9d8c744c5f3fdd96867cd5
Author: Daniel Cordero 0xdc io>
AuthorDate: Thu May 21 17:32:47 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Jun 5 21:08:42 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=d2dd86b8
embedded: copy root overlay into destpath
When using a merge subpath (e.g. the embedded target's /tmp/mergeroot),
root_overlay() copies the overlay into the seed stage, rather than the subpath.
Copy the files into destpath. When not using a subpath, destpath equals
chroot_path (so other stages are not affected).
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index ab1d409b..c2cc2c3e 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -838,7 +838,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
"/root_overlay"]:
if os.path.exists(x):
log.info('Copying root_overlay: %s', x)
-cmd(['rsync', '-a', x + '/', self.settings['chroot_path']],
+cmd(['rsync', '-a', x + '/', self.settings['destpath']],
env=self.env)
def bind(self):
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: 016c8460c88c89e55f85e34664fe7928b0f1c6a6 Author: Matt Turner gentoo org> AuthorDate: Tue May 19 19:14:29 2020 + Commit: Matt Turner gentoo org> CommitDate: Thu May 21 20:26:15 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=016c8460 catalyst: Drop ConfigParser Signed-off-by: Matt Turner gentoo.org> catalyst/config.py | 11 --- 1 file changed, 11 deletions(-) diff --git a/catalyst/config.py b/catalyst/config.py index b527ada0..e1963f71 100644 --- a/catalyst/config.py +++ b/catalyst/config.py @@ -118,14 +118,3 @@ class SpecParser(ParserBase): def __init__(self, filename=""): if filename: self.parse_file(filename) - - -class ConfigParser(ParserBase): - -key_value_separator = '=' -multiple_values = False -empty_values = True - -def __init__(self, filename=""): -if filename: -self.parse_file(filename)
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/
commit: ef304389cf83f6ed8cc3ef1584464140cd58656c Author: Matt Turner gentoo org> AuthorDate: Sun May 17 22:53:44 2020 + Commit: Matt Turner gentoo org> CommitDate: Sun May 17 22:53:44 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=ef304389 catalyst: Remove stdout=, since it conflicts with capture_output Signed-off-by: Matt Turner gentoo.org> catalyst/targets/snapshot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py index b7562f7c..7732312c 100644 --- a/catalyst/targets/snapshot.py +++ b/catalyst/targets/snapshot.py @@ -64,7 +64,6 @@ class snapshot(TargetBase): close_fds=False) sp = subprocess.run([self.git, '-C', self.gitdir, 'rev-parse', 'stable'], -stdout=subprocess.PIPE, capture_output=True, check=True, encoding='utf-8',
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/
commit: bf0244110fea5c612903629194581be42983681d
Author: Matt Turner gentoo org>
AuthorDate: Fri May 15 01:20:59 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri May 15 03:39:56 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=bf024411
catalyst: Add error checking to snapshot target
Signed-off-by: Matt Turner gentoo.org>
catalyst/targets/snapshot.py | 34 ++
1 file changed, 22 insertions(+), 12 deletions(-)
diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py
index b6c72c51..497b2918 100644
--- a/catalyst/targets/snapshot.py
+++ b/catalyst/targets/snapshot.py
@@ -10,7 +10,7 @@ from pathlib import Path
from catalyst import log
from catalyst.base.targetbase import TargetBase
from catalyst.lock import write_lock
-from catalyst.support import command
+from catalyst.support import CatalystError, command
class snapshot(TargetBase):
"""
@@ -52,17 +52,27 @@ class snapshot(TargetBase):
repouri, self.gitdir],
]
-for cmd in git_cmds:
-log.notice('>>> ' + ' '.join(cmd))
-subprocess.run(cmd,
- encoding='utf-8',
- close_fds=False)
-
-sp = subprocess.run([self.git, '-C', self.gitdir, 'rev-parse',
'stable'],
-stdout=subprocess.PIPE,
-encoding='utf-8',
-close_fds=False)
-return sp.stdout.rstrip()
+try:
+for cmd in git_cmds:
+log.notice('>>> ' + ' '.join(cmd))
+subprocess.run(cmd,
+ capture_output=True,
+ check=True,
+ encoding='utf-8',
+ close_fds=False)
+
+sp = subprocess.run([self.git, '-C', self.gitdir, 'rev-parse',
'stable'],
+stdout=subprocess.PIPE,
+capture_output=True,
+check=True,
+encoding='utf-8',
+close_fds=False)
+return sp.stdout.rstrip()
+
+except subprocess.CalledProcessError as e:
+raise CatalystError(f'{e.cmd} failed with return code'
+f'{e.returncode}\n'
+f'{e.output}\n')
def run(self):
if self.settings['snapshot_treeish'] == 'stable':
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 4d2ed2e6ea5bf5e2d79a7a9e9726748752986a35 Author: Matt Turner gentoo org> AuthorDate: Fri May 15 01:22:28 2020 + Commit: Matt Turner gentoo org> CommitDate: Fri May 15 03:39:56 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4d2ed2e6 catalyst: Make parent directories of snapshots/ Otherwise, if /var/tmp/catalyst does not exist when the snapshot target is executed it will fail to make the snapshots/ directory. Signed-off-by: Matt Turner gentoo.org> catalyst/base/targetbase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/base/targetbase.py b/catalyst/base/targetbase.py index 5bcea920..3e338bee 100644 --- a/catalyst/base/targetbase.py +++ b/catalyst/base/targetbase.py @@ -24,7 +24,7 @@ class TargetBase(ABC): def set_snapshot(self, treeish=None): # Make snapshots directory snapshot_dir = Path(self.settings['storedir'], 'snapshots') -snapshot_dir.mkdir(mode=0o755, exist_ok=True) +snapshot_dir.mkdir(mode=0o755, parents=True, exist_ok=True) repo_name = self.settings['repo_name'] if treeish is None:
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/
commit: dbd7ecd637e8a6931e69fd2b20550a9f3a1de8d0 Author: Matt Turner gentoo org> AuthorDate: Fri May 15 01:29:48 2020 + Commit: Matt Turner gentoo org> CommitDate: Fri May 15 03:39:56 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=dbd7ecd6 catalyst: Explain the git gc config options Signed-off-by: Matt Turner gentoo.org> catalyst/targets/snapshot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py index 497b2918..b7562f7c 100644 --- a/catalyst/targets/snapshot.py +++ b/catalyst/targets/snapshot.py @@ -43,6 +43,8 @@ class snapshot(TargetBase): else: git_cmds = [ [self.git, 'clone', '--quiet', '--depth=1', '--bare', + # Set some config options to enable git gc to clean everything + # except what we just fetched. See git-gc(1). '-c', 'gc.reflogExpire=0', '-c', 'gc.reflogExpireUnreachable=0', '-c', 'gc.rerereresolved=0',
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 202556d8a04852f2fa1c39eaca11da3c1421fbfe
Author: Matt Turner gentoo org>
AuthorDate: Thu May 14 03:27:47 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri May 15 03:39:56 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=202556d8
catalyst: Drop unnecessary clear_path()
open(..., 'w') truncates the file.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 964a5129..68945da8 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -974,7 +974,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
# Modify and write out make.conf (for the chroot)
makepath = normpath(self.settings["chroot_path"] +
self.settings["make_conf"])
-clear_path(makepath)
with open(makepath, "w") as myf:
log.notice("Writing the stage make.conf to: %s" % makepath)
myf.write("# These settings were set by the catalyst build script "
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 9f60d7d97a9fb72d10b756801a79ad5878a9bfa2 Author: Etienne Buira free fr> AuthorDate: Wed May 6 16:46:10 2020 + Commit: Matt Turner gentoo org> CommitDate: Wed May 6 17:41:15 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=9f60d7d9 catalyst: Skip only var_tmpfs_portage mount If var_tmpfs_portage is not in settings, we want to skip the tmpfs mount only. Prior to this commit subsequent mounts would have been incorrectly skipped. Fixes: 4c4a82badb5e (catalyst: Clean up bind()) Signed-off-by: Etienne Buira free.fr> Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 951500d8..964a5129 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -857,7 +857,7 @@ class StageBase(TargetBase, ClearBase, GenBase): if source == 'maybe_tmpfs': if 'var_tmpfs_portage' not in self.settings: -return +continue _cmd = ['mount', '-t', 'tmpfs', '-o', 'size=' + self.settings['var_tmpfs_portage'] + 'G', source,
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 7e392bb6998dfb726f72b3c7bddcedbce685261c
Author: Matt Turner gentoo org>
AuthorDate: Tue May 5 05:20:15 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Tue May 5 05:20:15 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=7e392bb6
catalyst: Remove never-used "crosscompile"
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 8 +---
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 80cc16f2..951500d8 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -111,15 +111,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
chroot = 'chroot'
self.settings["CHROOT"] = chroot
-self.settings["crosscompile"] = \
-build != host and not chroot.startswith('setarch')
-
log.notice('Using target: %s', self.settings['target'])
# Print a nice informational message
-if self.settings["crosscompile"]:
-log.info('Cross-compiling on %s for different machine type %s',
- build, host)
-elif chroot.startswith('setarch'):
+if chroot.startswith('setarch'):
log.info('Building on %s for alternate personality type %s',
build, host)
else:
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, examples/, targets/support/, doc/, targets/embedded/
commit: 9ad9eed7833829cc1150d715d48ea32539a5848a
Author: Matt Turner gentoo org>
AuthorDate: Fri May 1 06:28:58 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Mon May 4 02:41:16 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=9ad9eed7
targets: Drop most fstypes
This removes support for various file systems from the embedded target
(a target for producing images for embedded systems, as far as I
understand) and for ISOs. For ISOs, squashfs is great and everyone uses
it -- it provides better performance from a CD than the alternatives
like zisofs, normal, or noloop. See [1] for performance data of squashfs
vs other methods.
For embedded, it's unclear whether the target is used at all. There are
some very old specs in releng.git that use 'rel_type: embedded', but I'm
not sure if the target is used at all these days. To that end, I've
asked in #gentoo-embedded if anyone uses it. I've removed what I believe
to be the file system options that don't provide any value, leaving
jffs2 for now.
[1] https://elinux.org/Squash_Fs_Comparisons
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 4 +-
doc/catalyst-spec.5.txt | 9 +--
examples/livecd-stage2_template.spec| 8 +--
targets/embedded/fs-runscript.sh| 15 -
targets/support/create-iso.sh | 114 +++-
targets/support/filesystem-functions.sh | 65 --
targets/support/functions.sh| 14
targets/support/target_image_setup.sh | 20 --
8 files changed, 58 insertions(+), 191 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 7c82029a..bb008744 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -386,10 +386,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
del self.settings[self.settings["spec_prefix"] + "/fstype"]
if "fstype" not in self.settings:
-self.settings["fstype"] = "normal"
+self.settings["fstype"] = "squashfs"
for x in self.valid_values:
if x == self.settings["spec_prefix"] + "/fstype":
-log.info('%s/fstype is being set to the default of
"normal"',
+log.info('%s/fstype is being set to the default of
"squashfs"',
self.settings['spec_prefix'])
def set_fsops(self):
diff --git a/doc/catalyst-spec.5.txt b/doc/catalyst-spec.5.txt
index e269e16d..40453e0f 100644
--- a/doc/catalyst-spec.5.txt
+++ b/doc/catalyst-spec.5.txt
@@ -170,18 +170,11 @@ Filesystem
*livecd/fstype*::
The fstype is used to determine what sort of CD we should build. This
is used to set the type of loopback filesystem that we will use on our
-CD. Possible values are as follows:
- `squashfs`;; This gives the best compression, but requires a kernel patch.
- `zisofs`;; This uses in-kernel compression and is supported on all platforms.
- `normal`;; This creates a loop without compression.
- `noloop`;; This copies the files to the CD directly, without using a
- loopback.
+CD. The only possible value is `squashfs`.
*livecd/fsops*::
The fsops are a list of optional parameters that can be passed to the
tool which will create the filesystem specified in *livecd/fstype*
-It is valid for the following fstypes: `squashfs`, `jffs`, `jffs2`,
-and `cramfs`.
*livecd/iso*::
This is the full path and filename to the ISO image that the
diff --git a/examples/livecd-stage2_template.spec
b/examples/livecd-stage2_template.spec
index 3b9ca1da..e9758909 100644
--- a/examples/livecd-stage2_template.spec
+++ b/examples/livecd-stage2_template.spec
@@ -83,18 +83,14 @@ kerncache_path:
# The fstype is used to determine what sort of CD we should build. This is
# used to set the type of loopback filesystem that we will use on our CD.
-# Possible options are as follows:
-# squashfs - This gives the best compression, but requires a kernel patch.
-# zisofs - This uses in-kernel compression and is supported on all platforms.
-# normal - This creates a loop without compression.
-# noloop - This copies the files to the CD directly, without using a loopback.
+# Possible options are as follows: squashfs
# example:
# livecd/fstype: squashfs
livecd/fstype:
# The fsops are a list of optional parameters that can be passed to the tool
# which will create the filesystem specified in livecd/fstype. It is valid for
-# the following fstypes: squashfs, jffs, jffs2, cramfs
+# the following fstypes: squashfs
livecd/fsops:
# The cdtar is essentially the bootloader for the CD. It also holds the main
diff --git a/targets/embedded/fs-runscript.sh b/targets/embedded/fs-runscript.sh
index 4746a909..fb17d3ba 100755
--- a/targets/embedded/fs-runscript.sh
+++ b/targets/embedded/fs-runscript.sh
@@ -13,32 +13,17 @@ fs_check() {
}
case ${1} in
- jffs)
- fs_check
[gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/, targets/support/, targets/stage2/, doc/
commit: 0d40bd703e447d660f9e16d27d633700648fff25
Author: Matt Turner gentoo org>
AuthorDate: Sat May 2 01:25:55 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Mon May 4 02:43:00 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=0d40bd70
catalyst: Replace target_portdir with repo_basedir+repo_name
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 12 +---
catalyst/defaults.py | 2 +-
doc/catalyst-config.5.txt | 10 +-
targets/stage2/chroot.sh | 2 +-
targets/support/livecdfs-update.sh | 2 +-
5 files changed, 17 insertions(+), 11 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index bb008744..80cc16f2 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -192,7 +192,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.mount = MOUNT_DEFAULTS.copy()
self.mount['portdir']['source'] = self.snapshot
-self.mount['portdir']['target'] = self.settings['target_portdir']
+self.mount['portdir']['target'] = self.settings['repo_basedir'] + '/'
+ self.settings['repo_name']
self.mount['distdir']['source'] = self.settings['distdir']
self.mount["distdir"]['target'] = self.settings['target_distdir']
@@ -803,7 +803,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
make_profile = Path(self.settings['chroot_path'] +
self.settings['port_conf'],
'make.profile')
make_profile.unlink()
-make_profile.symlink_to(Path('../..' + self.settings['target_portdir'],
+make_profile.symlink_to(Path('../..' + self.settings['repo_basedir'],
+ self.settings['repo_name'],
'profiles',
self.settings['target_profile']),
target_is_directory=True)
@@ -1056,7 +1057,12 @@ class StageBase(TargetBase, ClearBase, GenBase):
' '.join(myuseexpandvars[hostuseexpand]) + '"\n')
# Write non-default PORTDIR/DISTDIR/PKGDIR settings to make.conf
-for x in ['target_portdir', 'target_distdir', 'target_pkgdir']:
+if (self.settings['repo_basedir'], self.settings['repo_name']) != \
+ (confdefaults['repo_basedir'], confdefaults['repo_name']):
+myf.write('PORTDIR="%s/%s"\n' % (self.settings['repo_basedir'],
+ self.settings['repo_name']))
+
+for x in ['target_distdir', 'target_pkgdir']:
if self.settings[x] != confdefaults[x]:
varname = x.split('_')[1].upper()
myf.write(f'{varname}="{self.settings[x]}"\n')
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index bbefa3a8..9d5771d5 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -65,6 +65,7 @@ confdefaults = {
"pkgdir": "/var/cache/binpkgs",
"port_tmpdir": "/var/tmp/portage",
"PythonDir": "./catalyst",
+"repo_basedir": "/var/db/repos",
"repo_name": "gentoo",
"repos": "%(storedir)s/repos",
"sharedir": "/usr/share/catalyst",
@@ -73,7 +74,6 @@ confdefaults = {
"storedir": "/var/tmp/catalyst",
"target_distdir": "/var/cache/distfiles",
"target_pkgdir": "/var/cache/binpkgs",
-"target_portdir": "/var/db/repos/gentoo",
}
DEFAULT_CONFIG_FILE = '/etc/catalyst/catalyst.conf'
diff --git a/doc/catalyst-config.5.txt b/doc/catalyst-config.5.txt
index 11b27d90..d5444b38 100644
--- a/doc/catalyst-config.5.txt
+++ b/doc/catalyst-config.5.txt
@@ -108,6 +108,11 @@ Defaults to the host's DISTDIR.
The directory in which git repositories exist for use by the snapshot target.
Defaults to `${storedir}/repos`.
+*repo_basedir*::
+The target repository directory to contain the primary repo (e.g.,
+gentoo repo) and any overlays. The default location is
+`/var/db/repos`.
+
*repo_name*::
The name of the main repository (e.g. gentoo). The git repository at
`${repos}/${repo_name}.git` will be used to produce the portdir sqfs
@@ -123,11 +128,6 @@ Defines the location of binary packages in the target.
This will be
written to the target's make.conf if it is not the default value of
`/var/cache/binpkgs`.
-*target_portdir*::
-Defines the location of the main ebuild repository in the target.
-This will be written to the target's make.conf if it is not the
-default value of `/var/db/repos/gentoo`.
-
Other settings
~~
diff --git a/targets/stage2/chroot.sh b/targets/stage2/chroot.sh
index aac9a92d..bf98d328 100755
--- a/targets/stage2/chroot.sh
+++ b/targets/stage2/chroot.sh
@@ -9,7 +9,7 @@ export CONFIG_PROTECT="-* /etc/locale.gen"
echo "$locales" > /etc/locale.gen
## START BUILD
-${clst_target_portdir}/scripts/bootstrap.sh ${bootstrap_opts} || exit 1
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: cd58b37bcd07433e93b80b4a93c9e79ec9395e26
Author: Matt Turner gentoo org>
AuthorDate: Fri May 1 05:55:43 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri May 1 05:55:43 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=cd58b37b
catalyst: Access setarch data correctly
This prevented the setarch data in the TOML files from being applied
correctly.
Fixes: b368f6aa5e29 (catalyst: Use arch data from TOML, not python
modules)
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index affdabbe..7c82029a 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -95,7 +95,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
# Search for a subarchitecture in each arch in the arch_config
for arch in [x for x in arch_config if x.startswith(name) and
host in arch_config[x]]:
self.settings.update(arch_config[arch][host])
-setarch = arch_config.get('setarch', {})
+setarch = arch_config.get('setarch', {}).get(host, {})
break
else:
# Didn't find a matching subarchitecture, keep searching
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: ed0de74506f5ce9741fe61c6c1e46701ee7ca3f0
Author: Matt Turner gentoo org>
AuthorDate: Sun Apr 19 18:03:44 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Thu Apr 23 18:20:35 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=ed0de745
catalyst: Add a function to get full path of executable
Just a simple wrapper around shutil.which().
Signed-off-by: Matt Turner gentoo.org>
catalyst/support.py | 6 ++
1 file changed, 6 insertions(+)
diff --git a/catalyst/support.py b/catalyst/support.py
index 988a81f5..0925af47 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -3,6 +3,7 @@ import glob
import sys
import os
import re
+import shutil
import time
from subprocess import Popen
@@ -17,6 +18,11 @@ class CatalystError(Exception):
if message:
log.error('CatalystError: %s', message, exc_info=print_traceback)
+def command(name):
+c = shutil.which(name)
+if not c:
+raise CatalystError(f'"{name}" not found or is not executable')
+return c
def cmd(mycmd, env=None, debug=False, fail_func=None):
"""Run the external |mycmd|.
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 4c4a82badb5ef8bd472dae7b530ef87a4c1f9127
Author: Matt Turner gentoo org>
AuthorDate: Sun Apr 19 01:10:51 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Thu Apr 23 18:20:35 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4c4a82ba
catalyst: Clean up bind()
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 62 +-
1 file changed, 34 insertions(+), 28 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 275c07eb..9aecf013 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -906,35 +906,41 @@ class StageBase(TargetBase, ClearBase, GenBase):
def bind(self):
for x in [x for x in self.mount if self.mount[x]['enable']]:
-log.debug('bind(); x = %s', x)
-target = normpath(self.settings['chroot_path'] +
- self.mount[x]['target'])
-ensure_dirs(target, mode=0o755)
-
-if not os.path.exists(self.mount[x]['source']):
-if self.mount[x]['source'] not in ("maybe_tmpfs", "tmpfs",
"shmfs"):
-ensure_dirs(self.mount[x]['source'], mode=0o755)
-
-src = self.mount[x]['source']
-log.debug('bind(); src = %s', src)
-if src == "maybe_tmpfs":
-if "var_tmpfs_portage" in self.settings:
-_cmd = ['mount', '-t', 'tmpfs',
-'-o', 'size=' +
-self.settings['var_tmpfs_portage'] + 'G',
-src, target]
-elif src == "tmpfs":
-_cmd = ['mount', '-t', 'tmpfs', src, target]
+if str(self.mount[x]['source']) == 'config':
+raise CatalystError(f'"{x}" bind mount source is not
configured')
+if str(self.mount[x]['target']) == 'config':
+raise CatalystError(f'"{x}" bind mount target is not
configured')
+
+source = str(self.mount[x]['source'])
+target = self.settings['chroot_path'] +
str(self.mount[x]['target'])
+
+log.debug('bind %s: "%s" -> "%s"', x, source, target)
+
+if source == 'maybe_tmpfs':
+if 'var_tmpfs_portage' not in self.settings:
+return
+
+_cmd = ['mount', '-t', 'tmpfs', '-o', 'size=' +
+self.settings['var_tmpfs_portage'] + 'G', source,
+target]
+elif source == 'tmpfs':
+_cmd = ['mount', '-t', 'tmpfs', source, target]
+elif source == 'shmfs':
+_cmd = ['mount', '-t', 'tmpfs', '-o', 'noexec,nosuid,nodev',
+'shm', target]
else:
-if src == "shmfs":
-_cmd = ['mount', '-t', 'tmpfs', '-o',
-'noexec,nosuid,nodev', 'shm', target]
-else:
-_cmd = ['mount', '--bind', src, target]
-if _cmd:
-log.debug('bind(); _cmd = %s', _cmd)
-cmd(_cmd, env=self.env, fail_func=self.unbind)
-log.debug('bind(); finished :D')
+_cmd = ['mount', '--bind', source, target]
+
+source = Path(self.mount[x]['source'])
+
+# We may need to create the source of the bind mount. E.g., in
the
+# case of an empty package cache we must create the directory
that
+# the binary packages will be stored into.
+source.mkdir(mode=0o755, exist_ok=True)
+
+Path(target).mkdir(mode=0o755, parents=True, exist_ok=True)
+
+cmd(_cmd, env=self.env, fail_func=self.unbind)
def unbind(self):
ouch = 0
[gentoo-commits] proj/catalyst:master commit in: /, catalyst/targets/, catalyst/base/, catalyst/, doc/
commit: 98cf07d2ca4da1f88e213a520095bfccc9c81ffc Author: Matt Turner gentoo org> AuthorDate: Fri Apr 17 23:31:52 2020 + Commit: Matt Turner gentoo org> CommitDate: Thu Apr 30 22:54:49 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=98cf07d2 catalyst: Make and use squashfs snapshots There were a number of problems with catalyst's snapshot system. It was built around using the build system's portdir and had no control over what was in that portdir or when it was updated. As a result, when a stage build failed, it was difficult to tell what the snapshot consistet of precisely or whether it contained a particular recent fix. With snapcache disabled, ebuild repo snapshots were tar'd and compressed and then unpacked into the stage chroot which is an unnecessarily expensive process. Moreover, a porttree has more than 100k small files, which are stored extremely inefficiently on most file systems—a whole porttree is usually around 700M on disk. Just removing all of those files during the cleaning stage is an expensive operation. Instead, we make a compressed squashfs image and mount it in the build chroot. The porttree has many duplicate files, and squashfs deduplicates the files and then compresses, so the result is very efficiently packed: ~38M with gzip -9 compression. The snapshot target has been modified to generate a squashfs image from a bare ebuild git repo. Piping git-archive to tar2sqfs generates the squashfs image in less than 10 seconds on a modern system. The git repo is fetched with --depth=1 to minize bandwidth and disk usage, and git gc is run after fetch to minimize disk usage. Storage requirements for the git ebuild repo with metadata are ~70M. The squashfs snapshot is stored in /var/tmp/catalyst/snapshots/ by default with a name -.sqfs. With this convention, we know the exact point in history that the snapshot was taken. The catalyst-auto script can use the sha1 to get a deterministic timestamp, so that it is independent on when `catalyst -s` was run, but is instead the timestamp of the commit date of the repo's git SHA1. Signed-off-by: Matt Turner gentoo.org> README| 3 +- catalyst/base/stagebase.py| 88 catalyst/base/targetbase.py | 16 +++- catalyst/defaults.py | 2 +- catalyst/main.py | 24 +++--- catalyst/targets/embedded.py | 1 - catalyst/targets/livecd_stage1.py | 1 - catalyst/targets/livecd_stage2.py | 1 - catalyst/targets/netboot.py | 1 - catalyst/targets/snapshot.py | 165 +++--- catalyst/targets/stage4.py| 1 - doc/catalyst-config.5.txt | 13 +-- doc/catalyst-spec.5.txt | 4 +- 13 files changed, 137 insertions(+), 183 deletions(-) diff --git a/README b/README index 1a039fca..1cceb63e 100644 --- a/README +++ b/README @@ -18,8 +18,9 @@ Requirements === - Python 3.6 or greater -- An ebuild repository snapshot (or an ebuild tree to create one) - A generic stage3 tarball for your architecture +- A squashfs ebuild repository snapshot + - Or an ebuild git repo with sys-fs/squashfs-tools-ng and dev-vcs/git What is catalyst? diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 9aecf013..41da97b3 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -35,7 +35,7 @@ class StageBase(TargetBase, ClearBase, GenBase): self.required_values |= frozenset([ "profile", "rel_type", -"snapshot", +"snapshot_treeish", "source_subpath", "subarch", "target", @@ -149,7 +149,7 @@ class StageBase(TargetBase, ClearBase, GenBase): self.set_source_subpath() # Set paths -self.set_snapshot_path() +self.set_snapshot() self.set_root_path() self.set_source_path() self.set_chroot_path() @@ -191,9 +191,8 @@ class StageBase(TargetBase, ClearBase, GenBase): # Setup our mount points. self.mount = MOUNT_DEFAULTS.copy() -# Always unpack snapshot tarball -self.mount['portdir']['enable'] = False - +self.mount['portdir']['source'] = self.snapshot +self.mount['portdir']['target'] = self.settings['repo_basedir'] + '/' + self.settings['repo_name'] self.mount['distdir']['source'] = self.settings['distdir'] self.mount["distdir"]['target'] = self.settings['target_distdir'] @@ -435,21 +434,11 @@ class StageBase(TargetBase, ClearBase, GenBase): self.settings["destpath"] = normpath(self.settings["chroot_path"]) def set_cleanables(self): -self.settings["cleanables"] = ["/etc/resolv.conf", "/var/tmp/*", "/tmp/*", - self.settings["repo_basedir"] + "/" + -
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: 562a9dddee7038804e6c17d8da79262ad45deaf5 Author: Matt Turner gentoo org> AuthorDate: Sat Apr 18 01:27:31 2020 + Commit: Matt Turner gentoo org> CommitDate: Thu Apr 23 18:20:35 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=562a9ddd catalyst: Add a Lock class and refactor LockDir This allows us to make lock files directly. Signed-off-by: Matt Turner gentoo.org> catalyst/lock.py | 29 ++--- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/catalyst/lock.py b/catalyst/lock.py index 5e039fae..ab005163 100644 --- a/catalyst/lock.py +++ b/catalyst/lock.py @@ -8,17 +8,14 @@ from catalyst.fileops import ensure_dirs LockInUse = osutils.LockException - -class LockDir(): -"""An object that creates locks inside dirs""" - -def __init__(self, lockdir): -self.gid = 250 -self.lockfile = os.path.join(lockdir, '.catalyst_lock') -ensure_dirs(lockdir) -fileutils.touch(self.lockfile, mode=0o664) -os.chown(self.lockfile, -1, self.gid) -self.lock = osutils.FsLock(self.lockfile) +class Lock: +""" +A fnctl-based filesystem lock +""" +def __init__(self, lockfile): +fileutils.touch(lockfile, mode=0o664) +os.chown(lockfile, uid=-1, gid=250) +self.lock = osutils.FsLock(lockfile) def read_lock(self): self.lock.acquire_read_lock() @@ -29,3 +26,13 @@ class LockDir(): def unlock(self): # Releasing a write lock is the same as a read lock. self.lock.release_write_lock() + +class LockDir(Lock): +""" +A fnctl-based filesystem lock in a directory +""" +def __init__(self, lockdir): +ensure_dirs(lockdir) +lockfile = os.path.join(lockdir, '.catalyst_lock') + +Lock.__init__(self, lockfile)
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: be43719867e694a08699aa07c2f5f519df26b59b Author: Matt Turner gentoo org> AuthorDate: Sat Apr 18 04:33:52 2020 + Commit: Matt Turner gentoo org> CommitDate: Thu Apr 23 18:20:35 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=be437198 catalyst: Add read/write_lock contextmanagers Signed-off-by: Matt Turner gentoo.org> catalyst/lock.py | 20 1 file changed, 20 insertions(+) diff --git a/catalyst/lock.py b/catalyst/lock.py index ab005163..e31745b2 100644 --- a/catalyst/lock.py +++ b/catalyst/lock.py @@ -1,6 +1,8 @@ import os +from contextlib import contextmanager + from snakeoil import fileutils from snakeoil import osutils from catalyst.fileops import ensure_dirs @@ -36,3 +38,21 @@ class LockDir(Lock): lockfile = os.path.join(lockdir, '.catalyst_lock') Lock.__init__(self, lockfile) + +@contextmanager +def read_lock(filename): +lock = Lock(filename) +lock.read_lock() +try: +yield +finally: +lock.unlock() + +@contextmanager +def write_lock(filename): +lock = Lock(filename) +lock.write_lock() +try: +yield +finally: +lock.unlock()
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, targets/support/, targets/stage2/, catalyst/, doc/
commit: bec6650faa65d734fbbde63cb5d652e45bff8cac
Author: Matt Turner gentoo org>
AuthorDate: Wed Apr 22 02:09:19 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Thu Apr 30 22:56:12 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=bec6650f
catalyst: Remove repo_basedir setting
Its purpose was very confused.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 7 ++-
catalyst/defaults.py | 1 -
doc/catalyst-config.5.txt | 4
targets/stage2/chroot.sh | 2 +-
targets/support/livecdfs-update.sh | 2 +-
5 files changed, 4 insertions(+), 12 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index e1477bb4..1f091829 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -192,7 +192,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.mount = MOUNT_DEFAULTS.copy()
self.mount['portdir']['source'] = self.snapshot
-self.mount['portdir']['target'] = self.settings['repo_basedir'] + '/'
+ self.settings['repo_name']
+self.mount['portdir']['target'] = self.settings['target_portdir']
self.mount['distdir']['source'] = self.settings['distdir']
self.mount["distdir"]['target'] = self.settings['target_distdir']
@@ -1054,11 +1054,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
for hostuseexpand in myuseexpandvars:
myf.write(hostuseexpand + '="' +
' '.join(myuseexpandvars[hostuseexpand]) + '"\n')
-# write out a shipable version
-target_portdir = normpath(self.settings["repo_basedir"] + "/" +
- self.settings["repo_name"])
-myf.write('PORTDIR="%s"\n' % target_portdir)
+myf.write('PORTDIR="%s"\n' % self.settings['target_portdir'])
myf.write('DISTDIR="%s"\n' % self.settings['target_distdir'])
myf.write('PKGDIR="%s"\n' % self.settings['target_pkgdir'])
if setup:
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 9878b426..bbefa3a8 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -65,7 +65,6 @@ confdefaults = {
"pkgdir": "/var/cache/binpkgs",
"port_tmpdir": "/var/tmp/portage",
"PythonDir": "./catalyst",
-"repo_basedir": "/var/db/repos",
"repo_name": "gentoo",
"repos": "%(storedir)s/repos",
"sharedir": "/usr/share/catalyst",
diff --git a/doc/catalyst-config.5.txt b/doc/catalyst-config.5.txt
index c1ebe9dd..b7d493eb 100644
--- a/doc/catalyst-config.5.txt
+++ b/doc/catalyst-config.5.txt
@@ -108,10 +108,6 @@ Defaults to the host's DISTDIR.
The directory in which git repositories exist for use by the snapshot target.
Defaults to `${storedir}/repos`.
-*repo_basedir*::
-The target repository directory to contain the primary repo (gentoo repo) and
-any overlays. The default location is `/var/db/repos`.
-
*repo_name*::
The name of the main repository (e.g. gentoo). The git repository at
`${repos}/${repo_name}.git` will be used to produce the portdir sqfs
diff --git a/targets/stage2/chroot.sh b/targets/stage2/chroot.sh
index bf98d328..aac9a92d 100755
--- a/targets/stage2/chroot.sh
+++ b/targets/stage2/chroot.sh
@@ -9,7 +9,7 @@ export CONFIG_PROTECT="-* /etc/locale.gen"
echo "$locales" > /etc/locale.gen
## START BUILD
-${clst_repo_basedir}/${clst_repo_name}/scripts/bootstrap.sh ${bootstrap_opts}
|| exit 1
+${clst_target_portdir}/scripts/bootstrap.sh ${bootstrap_opts} || exit 1
# Replace modified /etc/locale.gen with default
etc-update --automode -5
diff --git a/targets/support/livecdfs-update.sh
b/targets/support/livecdfs-update.sh
index b1049671..8297e60d 100755
--- a/targets/support/livecdfs-update.sh
+++ b/targets/support/livecdfs-update.sh
@@ -257,7 +257,7 @@ case ${clst_livecd_type} in
USE="-* $(cat /var/db/pkg/sys-libs/glibc*/USE)" emerge -eqp
@system | grep -e '^\[ebuild' | sed -e 's:^\[ebuild .\+\] ::' -e 's: .\+$::' >
/usr/livecd/systempkgs.txt
# This is my hack to reduce tmpfs usage
- cp -r ${clst_repo_basedir}/${clst_repo_name}/{profiles,eclass}
/usr/livecd
+ cp -r ${clst_target_portdir}/{profiles,eclass} /usr/livecd
rm -rf
/usr/livecd/profiles/{co*,default-{1*,a*,b*,d*,h*,i*,m*,p*,s*,x*},g*,hardened-*,n*,x*}
mv -f /etc/gconf /usr/livecd
ln -sf /usr/livecd/gconf /etc/gconf
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 992cf27d631b064a612c203169f1bc2446e6c6fc
Author: Matt Turner gentoo org>
AuthorDate: Thu Apr 23 02:05:11 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Thu Apr 23 02:05:11 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=992cf27d
catalyst: Fix Path() usage
When an Path argument has a leading /, all previous arguments are
ignored. This is surprising behavior to me...
Fixes: ddc1b50d47ff (catalyst: Simplify config_profile_link())
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 6 ++
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 0531cf61..1c0d98c6 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -906,12 +906,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
def config_profile_link(self):
log.info('Configuring profile link...')
-make_profile = Path(self.settings['chroot_path'],
-self.settings['port_conf'],
+make_profile = Path(self.settings['chroot_path'] +
self.settings['port_conf'],
'make.profile')
make_profile.unlink()
-make_profile.symlink_to(Path('../..',
- self.settings['portdir'],
+make_profile.symlink_to(Path('../..' + self.settings['portdir'],
'profiles',
self.settings['target_profile']),
target_is_directory=True)
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/
commit: 3b0ff52abd04aa213c844b6d716d9edbe809dc65
Author: Matt Turner gentoo org>
AuthorDate: Sat Apr 18 20:20:05 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Wed Apr 22 00:22:24 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=3b0ff52a
catalyst: Remove code to mount /proc in stage1root
Added in commit f75bb07f3cec (Added two patches from Joshua Kinard ...),
mounting /proc in stage1root purports to fix an issue in the preclean
stage. I have no idea what that issue was, but it doesn't occur now.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 5 -
catalyst/targets/stage1.py | 11 +--
2 files changed, 1 insertion(+), 15 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 60cb891a..f8c1611e 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -213,8 +213,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.mounts.append("shm")
self.mounts.append("run")
-self.set_mounts()
-
# Configure any user specified options (either in catalyst.conf or on
# the command line).
if "pkgcache" in self.settings["options"]:
@@ -586,9 +584,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
def set_stage_path(self):
self.settings["stage_path"] = normpath(self.settings["chroot_path"])
-def set_mounts(self):
-pass
-
def set_packages(self):
pass
diff --git a/catalyst/targets/stage1.py b/catalyst/targets/stage1.py
index daa0b854..93c62877 100644
--- a/catalyst/targets/stage1.py
+++ b/catalyst/targets/stage1.py
@@ -5,7 +5,7 @@ stage1 target
from catalyst import log
from catalyst.support import normpath
-from catalyst.fileops import ensure_dirs, move_path
+from catalyst.fileops import move_path
from catalyst.base.stagebase import StageBase
@@ -78,15 +78,6 @@ class stage1(StageBase):
"If you break it, you buy it. Don't complain to us about
it.\n"
"Don't say we did not warn you.")
-def set_mounts(self):
-# stage_path/proc probably doesn't exist yet, so create it
-ensure_dirs(self.settings["stage_path"]+"/proc")
-
-# alter the mount mappings to bind mount proc onto it
-self.mounts.append("stage1root/proc")
-self.target_mounts["stage1root/proc"] = "/tmp/stage1root/proc"
-self.mountmap["stage1root/proc"] = "/proc"
-
def set_completion_action_sequences(self):
'''Override function for stage1
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: 80254c3695752a6732d2d65dfe0f9014eb746ded
Author: Matt Turner gentoo org>
AuthorDate: Wed Apr 22 00:48:18 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Wed Apr 22 01:52:15 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=80254c36
catalyst: Remove and inline some default paths
Signed-off-by: Matt Turner gentoo.org>
catalyst/defaults.py | 25 -
1 file changed, 8 insertions(+), 17 deletions(-)
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index e09d08e8..a0b87f96 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -47,15 +47,6 @@ valid_config_file_values.extend([
"VERBOSE",
])
-# set our base defaults here to keep
-# them in one location.
-BASE_GENTOO_DIR = "/var/gentoo"
-REPODIR = BASE_GENTOO_DIR + "/repos"
-DISTDIR = BASE_GENTOO_DIR + "/distfiles"
-PKGDIR = BASE_GENTOO_DIR + "/packages"
-MAINREPO = "gentoo"
-PORTDIR = REPODIR + "/" + MAINREPO
-
confdefaults = {
"comp_prog": COMPRESSOR_PROGRAM_OPTIONS['linux'],
"compression_mode": 'lbzip2',
@@ -63,27 +54,27 @@ confdefaults = {
"compressor_options": XATTRS_OPTIONS['linux'],
"decomp_opt": DECOMPRESSOR_PROGRAM_OPTIONS['linux'],
"decompressor_search_order": DECOMPRESSOR_SEARCH_ORDER,
-"distdir": DISTDIR[:],
+"distdir": "/var/cache/distfiles",
"icecream": "/var/cache/icecream",
'list_xattrs_opt': LIST_XATTRS_OPTIONS['linux'],
-"local_overlay": REPODIR[:] + "/local",
+"local_overlay": "/var/db/repos/local",
"port_conf": "/etc/portage",
"make_conf": "%(port_conf)s/make.conf",
"options": set(),
-"pkgdir": PKGDIR[:],
-"portdir": PORTDIR[:],
+"pkgdir": "/var/cache/binpkgs",
+"portdir": "/var/db/repos/gentoo",
"port_tmpdir": "/var/tmp/portage",
"PythonDir": "./catalyst",
-"repo_basedir": REPODIR[:],
-"repo_name": MAINREPO[:],
+"repo_basedir": "/var/db/repos",
+"repo_name": "gentoo",
"sharedir": "/usr/share/catalyst",
"shdir": "/usr/share/catalyst/targets/",
"snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
"snapshot_name": "%(repo_name)s-",
"source_matching": "strict",
"storedir": "/var/tmp/catalyst",
-"target_distdir": DISTDIR[:],
-"target_pkgdir": PKGDIR[:],
+"target_distdir": "/var/cache/distfiles",
+"target_pkgdir": "/var/cache/binpkgs",
}
DEFAULT_CONFIG_FILE = '/etc/catalyst/catalyst.conf'
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: d29853f01fbdd87cf9cef0f04ae110b4185b1ecd Author: Matt Turner gentoo org> AuthorDate: Wed Apr 22 00:17:07 2020 + Commit: Matt Turner gentoo org> CommitDate: Wed Apr 22 00:22:24 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=d29853f0 catalyst: Remove unused import Fixes: 6565ad2eb90f (catalyst: Use hashlib instead of external tools) Signed-off-by: Matt Turner gentoo.org> catalyst/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/catalyst/main.py b/catalyst/main.py index 7e2a6c06..8ded4bd1 100644 --- a/catalyst/main.py +++ b/catalyst/main.py @@ -4,7 +4,6 @@ import hashlib import os import sys -from snakeoil import process from snakeoil.process import namespaces from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS,
[gentoo-commits] proj/catalyst:master commit in: catalyst/, catalyst/base/
commit: 773270f3647a0d08d6468eccb141496f26e0a601
Author: Matt Turner gentoo org>
AuthorDate: Sat Apr 18 20:15:18 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Wed Apr 22 00:22:24 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=773270f3
catalyst: Consolidate mount code into an OrderedDict
Keeping source and target together in a single data structure makes
things tidier than having
self.mountmap(map of mount name to source mount)
self.targets_map (map of mount name to target mount)
self.mounts (list of mounts in the order they're mounted)
This also has another nice benefit: the the order that the mounts take
place is stored in one location and isn't affected by the order in which
the code chooses to enable or disable them (dependent on configuration
options).
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 94
catalyst/defaults.py | 97 +-
2 files changed, 112 insertions(+), 79 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index f8c1611e..cc997688 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -11,8 +11,7 @@ from snakeoil.osutils import pjoin
from DeComp.compress import CompressMap
from catalyst import log
-from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS,
- PORT_LOGDIR_CLEAN)
+from catalyst.defaults import (MOUNT_DEFAULTS, PORT_LOGDIR_CLEAN)
from catalyst.support import (CatalystError, file_locate, normpath,
cmd, read_makeconf, ismount, file_check)
from catalyst.base.targetbase import TargetBase
@@ -188,46 +187,39 @@ class StageBase(TargetBase, ClearBase, GenBase):
file_locate(self.settings, ["portage_confdir"], expand=0)
# Setup our mount points.
-# initialize our target mounts.
-self.target_mounts = TARGET_MOUNT_DEFAULTS.copy()
-
-self.mounts = ["proc", "dev", "portdir", "distdir", "port_tmpdir"]
-# initialize our source mounts
-self.mountmap = SOURCE_MOUNT_DEFAULTS.copy()
-# update these from settings
-self.mountmap["portdir"] = self.settings["portdir"]
-self.mountmap["distdir"] = self.settings["distdir"]
-self.target_mounts["portdir"] = normpath(self.settings["repo_basedir"]
+
- "/" +
self.settings["repo_name"])
-self.target_mounts["distdir"] = self.settings["target_distdir"]
-self.target_mounts["packagedir"] = self.settings["target_pkgdir"]
+self.mount = MOUNT_DEFAULTS.copy()
+
+self.mount['portdir']['source'] = self.settings['portdir']
+self.mount['portdir']['target'] =
normpath(self.settings["repo_basedir"]
+ + "/" +
+ self.settings["repo_name"])
+self.mount['distdir']['source'] = self.settings['distdir']
+self.mount["distdir"]['target'] = self.settings["target_distdir"]
+
if "snapcache" not in self.settings["options"]:
-self.mounts.remove("portdir")
-self.mountmap["portdir"] = None
+self.mount['portdir']['enable'] = False
else:
-self.mountmap["portdir"] = normpath("/".join([
-self.settings["snapshot_cache_path"],
-self.settings["repo_name"],
+self.mount['portdir']['source'] = normpath("/".join([
+self.settings['snapshot_cache_path'],
+self.settings['repo_name']
]))
-self.mounts.append("devpts")
-self.mounts.append("shm")
-self.mounts.append("run")
# Configure any user specified options (either in catalyst.conf or on
# the command line).
if "pkgcache" in self.settings["options"]:
self.set_pkgcache_path()
+self.mount['pkgdir']['enable'] = True
+self.mount['pkgdir']['source'] = self.settings['pkgcache_path']
+self.mount['pkgdir']['target'] = self.settings["target_pkgdir"]
log.info('Location of the package cache is %s',
self.settings['pkgcache_path'])
-self.mounts.append("packagedir")
-self.mountmap["packagedir"] = self.settings["pkgcache_path"]
if "kerncache" in self.settings["options"]:
self.set_kerncache_path()
+self.mount['kerncache']['enable'] = True
+self.mount['kerncache']['source'] = self.settings["kerncache_path"]
log.info('Location of the kerncache is %s',
self.settings['kerncache_path'])
-self.mounts.append("kerncache")
-self.mountmap["kerncache"] = self.settings["kerncache_path"]
if "ccache" in
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: ddc1b50d47ffde9b9bbceb83ce0b5a6cb7045e1b
Author: Matt Turner gentoo org>
AuthorDate: Sat Apr 18 23:32:08 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Wed Apr 22 05:51:22 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=ddc1b50d
catalyst: Simplify config_profile_link()
I see no use in making this a resume point since making a symlink is a
cheap operation.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 31 +--
1 file changed, 13 insertions(+), 18 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index cc997688..0531cf61 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -3,6 +3,9 @@ import os
import platform
import shutil
import sys
+
+from pathlib import Path
+
import toml
from snakeoil import fileutils
@@ -902,24 +905,16 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.snapcache_lock.unlock()
def config_profile_link(self):
-if "autoresume" in self.settings["options"] \
-and self.resume.is_enabled("config_profile_link"):
-log.notice(
-'Resume point detected, skipping config_profile_link
operation...')
-else:
-# TODO: zmedico and I discussed making this a directory and pushing
-# in a parent file, as well as other user-specified configuration.
-log.info('Configuring profile link...')
-clear_path(self.settings['chroot_path'] +
- self.settings['port_conf'] + '/make.profile')
-ensure_dirs(self.settings['chroot_path'] +
-self.settings['port_conf'])
-cmd(['ln', '-sf',
- '../..' + self.settings['portdir'] +
- '/profiles/' + self.settings['target_profile'],
- self.settings['chroot_path'] + self.settings['port_conf'] +
'/make.profile'],
-env=self.env)
-self.resume.enable("config_profile_link")
+log.info('Configuring profile link...')
+make_profile = Path(self.settings['chroot_path'],
+self.settings['port_conf'],
+'make.profile')
+make_profile.unlink()
+make_profile.symlink_to(Path('../..',
+ self.settings['portdir'],
+ 'profiles',
+ self.settings['target_profile']),
+target_is_directory=True)
def setup_confdir(self):
if "autoresume" in self.settings["options"] \
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: bc9e8f841689c4b5b778adcc979105dbb4dd4147
Author: Matt Turner gentoo org>
AuthorDate: Wed Apr 22 01:33:49 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Wed Apr 22 05:42:31 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=bc9e8f84
catalyst: Make shdir default match config
Signed-off-by: Matt Turner gentoo.org>
catalyst/defaults.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 0c18f9f5..77133cb1 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -70,9 +70,9 @@ confdefaults = {
"repo_basedir": "/var/db/repos",
"repo_name": "gentoo",
"sharedir": "/usr/share/catalyst",
-"shdir": "/usr/share/catalyst/targets/",
"snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
"snapshot_name": "%(repo_name)s-",
+"shdir": "%(sharedir)s/targets",
"source_matching": "strict",
"storedir": "/var/tmp/catalyst",
"target_distdir": "/var/cache/distfiles",
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/targets/
commit: d3a8f202bf8eb82cc1d61e2b2b9da4eadc550c90 Author: Matt Turner gentoo org> AuthorDate: Sat Apr 18 00:36:46 2020 + Commit: Matt Turner gentoo org> CommitDate: Sat Apr 18 20:29:57 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=d3a8f202 catalyst: Reformat some array definitions Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py| 17 + catalyst/targets/embedded.py | 29 +++-- catalyst/targets/livecd_stage1.py | 17 + catalyst/targets/livecd_stage2.py | 6 +++--- catalyst/targets/netboot.py | 26 +- catalyst/targets/stage4.py| 28 ++-- 6 files changed, 95 insertions(+), 28 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 71bf1ef9..60cb891a 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -531,10 +531,19 @@ class StageBase(TargetBase, ClearBase, GenBase): def set_action_sequence(self): """Set basic stage1, 2, 3 action sequences""" -self.settings["action_sequence"] = ["unpack", "unpack_snapshot", -"setup_confdir", "portage_overlay", -"bind", "chroot_setup", "setup_environment", -"run_local", "preclean", "unbind", "clean"] +self.settings['action_sequence'] = [ +"unpack", +"unpack_snapshot", +"setup_confdir", +"portage_overlay", +"bind", +"chroot_setup", +"setup_environment", +"run_local", +"preclean", +"unbind", +"clean", +] self.set_completion_action_sequences() def set_completion_action_sequences(self): diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py index e288228e..189eb722 100644 --- a/catalyst/targets/embedded.py +++ b/catalyst/targets/embedded.py @@ -40,12 +40,29 @@ class embedded(StageBase): StageBase.__init__(self, spec, addlargs) def set_action_sequence(self): -self.settings["action_sequence"] = ["dir_setup", "unpack", "unpack_snapshot", -"config_profile_link", "setup_confdir", -"portage_overlay", "bind", "chroot_setup", -"setup_environment", "build_kernel", "build_packages", -"bootloader", "root_overlay", "fsscript", "unmerge", -"unbind", "remove", "empty", "clean", "capture", "clear_autoresume"] +self.settings['action_sequence'] = [ +"dir_setup", +"unpack", +"unpack_snapshot", +"config_profile_link", +"setup_confdir", +"portage_overlay", +"bind", +"chroot_setup", +"setup_environment", +"build_kernel", +"build_packages", +"bootloader", +"root_overlay", +"fsscript", +"unmerge", +"unbind", +"remove", +"empty", +"clean", +"capture", +"clear_autoresume", +] def set_stage_path(self): self.settings["stage_path"] = normpath( diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py index f4f6b512..727d95f2 100644 --- a/catalyst/targets/livecd_stage1.py +++ b/catalyst/targets/livecd_stage1.py @@ -23,10 +23,19 @@ class livecd_stage1(StageBase): StageBase.__init__(self, spec, addlargs) def set_action_sequence(self): -self.settings["action_sequence"] = ["unpack", "unpack_snapshot", -"config_profile_link", "setup_confdir", "portage_overlay", -"bind", "chroot_setup", "setup_environment", "build_packages", -"unbind", "clean"] +self.settings['action_sequence'] = [ +"unpack", +"unpack_snapshot", +"config_profile_link", +"setup_confdir", +"portage_overlay", +"bind", +"chroot_setup", +"setup_environment", +"build_packages", +"unbind", +"clean", +] self.set_completion_action_sequences() def set_spec_prefix(self): diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py index d41321e6..09de22fb 100644 --- a/catalyst/targets/livecd_stage2.py +++ b/catalyst/targets/livecd_stage2.py @@ -88,7 +88,7 @@ class livecd_stage2(StageBase): print_traceback=True) def
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 060df92357aed97af955aa36caf47e8518b01e05 Author: Matt Turner gentoo org> AuthorDate: Sat Apr 18 20:25:38 2020 + Commit: Matt Turner gentoo org> CommitDate: Sat Apr 18 20:26:38 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=060df923 catalyst: Open CONTENTS file in text mode Fixes: bb21b8615e64 (catalyst: gzip the .CONTENTS file) Signed-off-by: Matt Turner gentoo.org> catalyst/base/genbase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py index 3db20f84..c7dd87bc 100644 --- a/catalyst/base/genbase.py +++ b/catalyst/base/genbase.py @@ -29,7 +29,7 @@ class GenBase(): def gen_contents_file(self, path): c = self.settings['contents_map'] -with gzip.open(path + '.CONTENTS.gz', 'w', encoding='utf-8') as file: +with gzip.open(path + '.CONTENTS.gz', 'wt', encoding='utf-8') as file: file.write(c.contents(path, '', verbose=self.settings['VERBOSE'])) def gen_digest_file(self, path):
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 907f5a20e28ef9fed45595bdc68daaebd36d5281
Author: Matt Turner gentoo org>
AuthorDate: Fri Apr 17 07:03:14 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Apr 17 07:08:24 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=907f5a20
catalyst: Remove guard rails on duplicate digests
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/genbase.py | 8 ++--
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py
index e75e6e3c..f55b113f 100644
--- a/catalyst/base/genbase.py
+++ b/catalyst/base/genbase.py
@@ -37,17 +37,13 @@ class GenBase():
hash_map = self.settings["hash_map"]
if os.path.exists(path):
with io.open(digests, "w", encoding='utf-8') as myf:
-keys = {}
-for i in self.settings["digests"].split():
-keys[i] = 1
-array = sorted(keys.keys())
for f in [path, path + '.CONTENTS']:
if os.path.exists(f):
-if "all" in array:
+if "all" in self.settings["digests"].split():
for k in list(hash_map.hash_map):
digest = hash_map.generate_hash(f, hash_=k)
myf.write(digest)
else:
-for j in array:
+for j in self.settings["digests"].split():
digest = hash_map.generate_hash(f, hash_=j)
myf.write(digest)
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 46e6f497dcd0c21cd5197e5d9cf1ef6e196af10a
Author: Matt Turner gentoo org>
AuthorDate: Fri Apr 17 05:37:42 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Apr 17 05:51:15 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=46e6f497
catalyst: Generate array outside of loop
I'm going to change all of this code anyway, but I had to make this a
separate commit since I was so flabbergasted by the absurdity of it.
After digging through the history across multiple renames, I found that
this mistake was introduced in 2006.
Fixes: d128d2a554b0 (digests function uses raw output from hash function now)
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/genbase.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py
index 1164fd55..e75e6e3c 100644
--- a/catalyst/base/genbase.py
+++ b/catalyst/base/genbase.py
@@ -22,7 +22,7 @@ class GenBase():
keys = {}
for i in self.settings["contents"].split():
keys[i] = 1
-array = sorted(keys.keys())
+array = sorted(keys.keys())
for j in array:
contents = contents_map.contents(path, j,
verbose=self.settings["VERBOSE"])
@@ -40,7 +40,7 @@ class GenBase():
keys = {}
for i in self.settings["digests"].split():
keys[i] = 1
-array = sorted(keys.keys())
+array = sorted(keys.keys())
for f in [path, path + '.CONTENTS']:
if os.path.exists(f):
if "all" in array:
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: 04aba402175bdcfb63c9ec7127e13b46998904ac
Author: Matt Turner gentoo org>
AuthorDate: Fri Apr 17 17:32:26 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Apr 17 19:51:30 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=04aba402
catalyst: Raise an exception on invalid digest
Signed-off-by: Matt Turner gentoo.org>
catalyst/main.py | 9 -
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/catalyst/main.py b/catalyst/main.py
index c91fc8b6..7e2a6c06 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -347,11 +347,10 @@ def _main(parser, opts):
# First validate all the requested digests are valid keys.
if digests - valid_digests:
-log.critical(
-'These are not valid digest entries:\n%s\n'
-'Valid digest entries:\n%s',
-', '.join(sorted(digests - valid_digests)),
-', '.join(sorted(valid_digests)))
+raise CatalystError('These are not valid digest entries:\n%s\n'
+'Valid digest entries:\n%s' %
+', '.join(sorted(digests - valid_digests)),
+', '.join(sorted(valid_digests)))
addlargs = {}
[gentoo-commits] proj/catalyst:master commit in: catalyst/, targets/stage1/, targets/support/
commit: 80ec0abb4c0f801df3f604aea5fa3b51e9a9db54
Author: Matt Turner gentoo org>
AuthorDate: Wed Apr 15 23:43:42 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Wed Apr 15 23:43:42 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=80ec0abb
catalyst: Remove GNU sed detection
Was added in commit f3d246e24b5b (detect and use a configured sed
command for Gentoo/FreeBSD # 363577) for FreeBSD, but BSD support is now
gone.
Signed-off-by: Matt Turner gentoo.org>
catalyst/defaults.py| 1 -
catalyst/main.py| 6 --
targets/stage1/chroot.sh| 10 +-
targets/support/chroot-functions.sh | 10 +-
targets/support/kmerge.sh | 8
targets/support/livecdfs-update.sh | 32
6 files changed, 30 insertions(+), 37 deletions(-)
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 4f67878c..8e276500 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -52,7 +52,6 @@ confdefaults = {
"PythonDir": "./catalyst",
"repo_basedir": REPODIR[:],
"repo_name": MAINREPO[:],
-"sed": "sed",
"sharedir": "/usr/share/catalyst",
"shdir": "/usr/share/catalyst/targets/",
"snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
diff --git a/catalyst/main.py b/catalyst/main.py
index 8603012d..0fb8a682 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -404,12 +404,6 @@ def _main(parser, opts):
hash_map.hash_map[conf_values['hash_function']].cmd,
conf_values['hash_function'])
-# detect GNU sed
-for sed in ('/usr/bin/gsed', '/bin/sed', '/usr/bin/sed'):
-if os.path.exists(sed):
-conf_values["sed"] = sed
-break
-
addlargs = {}
if myspecfile:
diff --git a/targets/stage1/chroot.sh b/targets/stage1/chroot.sh
index 6e20027a..ac9d68ea 100755
--- a/targets/stage1/chroot.sh
+++ b/targets/stage1/chroot.sh
@@ -48,14 +48,14 @@ else
fi
# Clear USE
-[ -e ${clst_make_conf} ] && ${clst_sed} -i -e "/^USE=\"${BINDIST} ${USE}\"/d"
${clst_make_conf}
+[ -e ${clst_make_conf} ] && sed -i -e "/^USE=\"${BINDIST} ${USE}\"/d"
${clst_make_conf}
make_destpath "${clst_root_path}"
## START BUILD
# First, we drop in a known-good baselayout
[ -e ${clst_make_conf} ] && echo "USE=\"${USE} -build\"" >> ${clst_make_conf}
run_merge "--oneshot --nodeps sys-apps/baselayout"
-${clst_sed} -i "/USE=\"${USE} -build\"/d" ${clst_make_conf}
+sed -i "/USE=\"${USE} -build\"/d" ${clst_make_conf}
echo "$locales" > /etc/locale.gen
for etc in /etc "${clst_root_path}"/etc; do
@@ -87,10 +87,10 @@ locale-gen --destdir "${clst_root_path}"/ || die
"locale-gen failed"
# Why are we removing these? Don't we need them for final make.conf?
for useexpand in ${clst_HOSTUSEEXPAND}; do
x="clst_${useexpand}"
- ${clst_sed} -i "/${useexpand}=\"${!x}\"/d" \
+ sed -i "/${useexpand}=\"${!x}\"/d" \
${clst_make_conf}
done
# Clear USE
-[ -e ${clst_make_conf} ] && ${clst_sed} -i -e "/^CATALYST_USE/d"
${clst_make_conf}
-[ -e ${clst_make_conf} ] && ${clst_sed} -i -e "/^USE=\"/s/\${CATALYST_USE}
${USE} ${BOOTSTRAP_USE}//" ${clst_make_conf}
+[ -e ${clst_make_conf} ] && sed -i -e "/^CATALYST_USE/d" ${clst_make_conf}
+[ -e ${clst_make_conf} ] && sed -i -e "/^USE=\"/s/\${CATALYST_USE} ${USE}
${BOOTSTRAP_USE}//" ${clst_make_conf}
diff --git a/targets/support/chroot-functions.sh
b/targets/support/chroot-functions.sh
index 2c256f06..2fe5ac00 100755
--- a/targets/support/chroot-functions.sh
+++ b/targets/support/chroot-functions.sh
@@ -102,7 +102,7 @@ setup_myfeatures(){
else
clst_root_path=/ run_merge --oneshot --noreplace
sys-devel/distcc || exit 1
fi
- ${clst_sed} -i '/USE="${USE} -avahi -gtk -gnome"/d'
${clst_make_conf}
+ sed -i '/USE="${USE} -avahi -gtk -gnome"/d' ${clst_make_conf}
mkdir -p /etc/distcc
echo "${clst_distcc_hosts}" > /etc/distcc/hosts
@@ -191,7 +191,7 @@ setup_pkgmgr(){
echo "Adding USE=\"\${USE} $1\" to make.conf for portage build"
[ -e "${clst_make_conf}" ] && echo "USE=\"\${USE} $1\"" >>
"${clst_make_conf}"
run_merge --oneshot --update --newuse sys-apps/portage
- ${clst_sed} -i "/USE=\"\${USE} $1\"/d" "${clst_make_conf}"
+ sed -i "/USE=\"\${USE} $1\"/d" "${clst_make_conf}"
else
run_merge --oneshot --update --newuse sys-apps/portage
fi
@@ -245,8 +245,8 @@ cleanup_stages() {
esac
# Remove bindist from use
- ${clst_sed} -i "/USE=\"\${USE} bindist\"/d" "${clst_make_conf}"
- ${clst_sed} -i "/USE=\"bindist\"/d" "${clst_make_conf}"
+ sed -i "/USE=\"\${USE} bindist\"/d" "${clst_make_conf}"
+ sed -i "/USE=\"bindist\"/d" "${clst_make_conf}"
rm -f /var/log/emerge.log
[gentoo-commits] proj/catalyst:master commit in: catalyst/
commit: 0239c60e9a811f46b080fa96074ce9cf82cc55e0 Author: Matt Turner gentoo org> AuthorDate: Fri Apr 17 03:41:24 2020 + Commit: Matt Turner gentoo org> CommitDate: Fri Apr 17 03:42:51 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=0239c60e catalyst: Reformat some array definitions Signed-off-by: Matt Turner gentoo.org> catalyst/defaults.py | 47 --- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/catalyst/defaults.py b/catalyst/defaults.py index 8e276500..570fc4df 100644 --- a/catalyst/defaults.py +++ b/catalyst/defaults.py @@ -5,22 +5,47 @@ from DeComp.definitions import DECOMPRESSOR_PROGRAM_OPTIONS, LIST_XATTRS_OPTIONS # these should never be touched -required_build_targets = ["targetbase", "generic_stage_target"] +required_build_targets = [ +"generic_stage_target", +"targetbase", +] # new build types should be added here -valid_build_targets = ["stage1_target", "stage2_target", "stage3_target", - "stage4_target", "livecd_stage1_target", "livecd_stage2_target", - "embedded_target", "snapshot_target", "netboot_target" - ] +valid_build_targets = [ +"embedded_target", +"livecd_stage1_target", +"livecd_stage2_target", +"netboot_target", +"snapshot_target", +"stage1_target", +"stage2_target", +"stage3_target", +"stage4_target", +] -required_config_file_values = ["storedir", "sharedir", "distdir", "portdir"] +required_config_file_values = [ +"distdir", +"portdir", +"sharedir", +"storedir", +] valid_config_file_values = required_config_file_values[:] -valid_config_file_values.extend(["distcc", "envscript", - "options", "DEBUG", "VERBOSE", - "snapshot_cache", "hash_function", "digests", "contents", "compressor_arch", - "compression_mode", "compressor_options", "decompressor_search_order", - ]) +valid_config_file_values.extend([ +"compression_mode", +"compressor_arch", +"compressor_options", +"contents", +"DEBUG", +"decompressor_search_order", +"digests", +"distcc", +"envscript", +"hash_function", +"options", +"snapshot_cache", +"VERBOSE", +]) # set our base defaults here to keep # them in one location.
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/
commit: bcffd2ef3fc347d6ce346a1737ddbd0ed0b7dff6 Author: Matt Turner gentoo org> AuthorDate: Fri Apr 17 17:27:08 2020 + Commit: Matt Turner gentoo org> CommitDate: Fri Apr 17 18:55:23 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=bcffd2ef catalyst: Make conf_values['digests'] a set This makes it automatically deduplicate values (which I honestly don't really care about) but also makes it so users don't need to .split() themselves. Signed-off-by: Matt Turner gentoo.org> catalyst/base/genbase.py | 2 +- catalyst/main.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py index bc7f9375..eb09a4e0 100644 --- a/catalyst/base/genbase.py +++ b/catalyst/base/genbase.py @@ -37,5 +37,5 @@ class GenBase(): with io.open(path + '.DIGESTS', 'w', encoding='utf-8') as file: for f in [path, path + '.CONTENTS']: -for i in self.settings['digests'].split(): +for i in self.settings['digests']: file.write(self.generate_hash(f, name=i)) diff --git a/catalyst/main.py b/catalyst/main.py index d8e791c4..c91fc8b6 100644 --- a/catalyst/main.py +++ b/catalyst/main.py @@ -341,8 +341,9 @@ def _main(parser, opts): # TODO add capability to config/spec new definitions if "digests" in conf_values: -digests = set(conf_values['digests'].split()) valid_digests = hashlib.algorithms_available +digests = set(conf_values['digests'].split()) +conf_values['digests'] = digests # First validate all the requested digests are valid keys. if digests - valid_digests:
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: f0e78af28a5e1c6275cd5b6b3f8c602e7fd16f7f
Author: Matt Turner gentoo org>
AuthorDate: Fri Apr 17 17:17:49 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Apr 17 18:55:18 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f0e78af2
catalyst: Simplify GenBase
So much was wrong with this code. Deleting a file if it exists only to
open it with 'w' (which would have truncated it anyway!), checking for
presence of files that must exist and doing it in the wrong place, etc.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/genbase.py | 36
1 file changed, 12 insertions(+), 24 deletions(-)
diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py
index 632ee0d9..bc7f9375 100644
--- a/catalyst/base/genbase.py
+++ b/catalyst/base/genbase.py
@@ -3,10 +3,9 @@ import hashlib
import io
import os
-
class GenBase():
"""
-This class does generation of the contents and digests files.
+Generates CONTENTS and DIGESTS files.
"""
def __init__(self, myspec):
@@ -27,27 +26,16 @@ class GenBase():
return f'# {name.upper()} HASH\n{h.hexdigest()} {filename}\n'
def gen_contents_file(self, path):
-contents = path + ".CONTENTS"
-if os.path.exists(contents):
-os.remove(contents)
-
-contents_map = self.settings["contents_map"]
-if os.path.exists(path):
-with io.open(contents, "w", encoding='utf-8') as myf:
-contents = contents_map.contents(path, '',
-
verbose=self.settings["VERBOSE"])
-if contents:
-myf.write(contents)
+c = self.settings['contents_map']
+
+with io.open(path + '.CONTENTS', 'w', encoding='utf-8') as file:
+file.write(c.contents(path, '', verbose=self.settings['VERBOSE']))
def gen_digest_file(self, path):
-digests = path + ".DIGESTS"
-if os.path.exists(digests):
-os.remove(digests)
-if "digests" in self.settings:
-if os.path.exists(path):
-with io.open(digests, "w", encoding='utf-8') as myf:
-for f in [path, path + '.CONTENTS']:
-if os.path.exists(f):
-for i in self.settings["digests"].split():
-digest = self.generate_hash(f, name=i)
-myf.write(digest)
+if 'digests' not in self.settings:
+return
+
+with io.open(path + '.DIGESTS', 'w', encoding='utf-8') as file:
+for f in [path, path + '.CONTENTS']:
+for i in self.settings['digests'].split():
+file.write(self.generate_hash(f, name=i))
[gentoo-commits] proj/catalyst:master commit in: catalyst/, etc/, doc/, catalyst/base/
commit: c6d5b3151d284f62f04fe2999b8c532e9b7323c7
Author: Matt Turner gentoo org>
AuthorDate: Fri Apr 17 03:38:42 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Apr 17 17:03:47 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=c6d5b315
catalyst: Remove 'contents' config option
This option seems completely absurd and on top of that has been
completely broken, since contents() has been passed 'j', which is the
mode, as the positional argument 'destination'. This only worked because
destination is not used in the contents generation and the actual mode
parameter has a default argument of 'auto'.
So the config option has not worked in a long time and no one has
noticed.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/genbase.py | 21 -
catalyst/defaults.py | 1 -
catalyst/main.py | 4
doc/catalyst-config.5.txt | 27 ---
etc/catalyst.conf | 12
5 files changed, 8 insertions(+), 57 deletions(-)
diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py
index 2dc8bad0..08076460 100644
--- a/catalyst/base/genbase.py
+++ b/catalyst/base/genbase.py
@@ -15,19 +15,14 @@ class GenBase():
contents = path + ".CONTENTS"
if os.path.exists(contents):
os.remove(contents)
-if "contents" in self.settings:
-contents_map = self.settings["contents_map"]
-if os.path.exists(path):
-with io.open(contents, "w", encoding='utf-8') as myf:
-keys = {}
-for i in self.settings["contents"].split():
-keys[i] = 1
-array = sorted(keys.keys())
-for j in array:
-contents = contents_map.contents(path, j,
-
verbose=self.settings["VERBOSE"])
-if contents:
-myf.write(contents)
+
+contents_map = self.settings["contents_map"]
+if os.path.exists(path):
+with io.open(contents, "w", encoding='utf-8') as myf:
+contents = contents_map.contents(path, '',
+
verbose=self.settings["VERBOSE"])
+if contents:
+myf.write(contents)
def gen_digest_file(self, path):
digests = path + ".DIGESTS"
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 570fc4df..e60980d3 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -35,7 +35,6 @@ valid_config_file_values.extend([
"compression_mode",
"compressor_arch",
"compressor_options",
-"contents",
"DEBUG",
"decompressor_search_order",
"digests",
diff --git a/catalyst/main.py b/catalyst/main.py
index 0fb8a682..ba6e3fcc 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -68,10 +68,6 @@ def parse_config(config_files):
if key in myconf:
conf_values[key] = myconf[key]
-if "contents" in myconf:
-# replace '-' with '_' (for compatibility with existing configs)
-conf_values["contents"] = myconf["contents"].replace("-", '_')
-
if "envscript" in myconf:
log.info('Envscript support enabled.')
diff --git a/doc/catalyst-config.5.txt b/doc/catalyst-config.5.txt
index 76ac4a29..220bb528 100644
--- a/doc/catalyst-config.5.txt
+++ b/doc/catalyst-config.5.txt
@@ -26,7 +26,6 @@ and empty lines are interpreted as comments. For example:
# /etc/catalyst/catalyst.conf
digests="auto"
-contents="auto"
envscript="/etc/catalyst/catalystrc"
options="autoresume bindist kerncache pkgcache seedcache snapcache"
@@ -57,32 +56,6 @@ supported options below. Adding them all may take a long
time.
(example: `md5 sha1 sha512 whirlpool`). See the *SUPPORTED HASHES*
section for a list of supported hashes.
-*contents*::
-Create a `.CONTENTS` file listing the contents of the file. If this
-variable is empty, no `.CONTENTS` will be generated at all. Supported
-values:
-+
---
-auto::
-Strongly recommended
-
-tar-tv::
-Do `tar tvf FILE`
-
-tar-tvz::
-Do `tar tvzf FILE`
-
-tar-tvy::
-Do `tar tvyf FILE`
-
-isoinfo-l::
-Do `isoinfo -l -i FILE`
-
-isoinfo-f::
-Do `isoinfo -f -i FILE`. 'isoinfo-f' is the only option not chosen
-by the automatic algorithm.
---
-
*envscript*::
Environment script location, which allows users to set options such as
HTTP proxies, `MAKEOPTS`, `GENTOO_MIRRORS`, or any other environment
diff --git a/etc/catalyst.conf b/etc/catalyst.conf
index dd34fae3..a3c22049 100644
--- a/etc/catalyst.conf
+++ b/etc/catalyst.conf
@@ -14,18 +14,6 @@
# tiger160, whirlpool
digests="blake2 sha512"
-# Creates a .CONTENTS file listing the contents of the file. Pick from any of
-# the supported options below:
-# auto - strongly recommended
-# tar_tv - does 'tar tvf FILE'
-# tar_tvz - does 'tar
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: fe133e8058ecfcad654ae80bdfbf7359237d4ec5 Author: Matt Turner gentoo org> AuthorDate: Fri Apr 17 07:11:33 2020 + Commit: Matt Turner gentoo org> CommitDate: Fri Apr 17 07:14:44 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=fe133e80 catalyst: Remove dead 'all' digests case Generating close to 30 digests is not a sensible thing to do in general, but worse this code was not possible: 'all' was not a valid digest option since it is not a key in HASH_DEFINITIONS. Signed-off-by: Matt Turner gentoo.org> catalyst/base/genbase.py | 11 +++ 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py index f55b113f..2dc8bad0 100644 --- a/catalyst/base/genbase.py +++ b/catalyst/base/genbase.py @@ -39,11 +39,6 @@ class GenBase(): with io.open(digests, "w", encoding='utf-8') as myf: for f in [path, path + '.CONTENTS']: if os.path.exists(f): -if "all" in self.settings["digests"].split(): -for k in list(hash_map.hash_map): -digest = hash_map.generate_hash(f, hash_=k) -myf.write(digest) -else: -for j in self.settings["digests"].split(): -digest = hash_map.generate_hash(f, hash_=j) -myf.write(digest) +for i in self.settings["digests"].split(): +digest = hash_map.generate_hash(f, hash_=i) +myf.write(digest)
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, /
commit: 2731f5f4571c91acb0e459ddcec6582bba25f5ce
Author: Matt Turner gentoo org>
AuthorDate: Wed Apr 15 20:03:16 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Wed Apr 15 20:03:16 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=2731f5f4
catalyst: Fix and enable bad-continuation checks
Signed-off-by: Matt Turner gentoo.org>
.pylintrc | 2 --
catalyst/base/stagebase.py | 8
setup.py | 2 +-
3 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/.pylintrc b/.pylintrc
index 8caafef1..5428e349 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -26,7 +26,6 @@ load-plugins=
# We should clean up things so we can enable:
# missing-docstring -- add lots of docstrings everywhere!
# bad-whitespace -- fix spacing everywhere
-# bad-continuation -- might be hard with tab indentation policy
# invalid-name -- need to manage constants better
# line-too-long -- figure out a length and stick to it
# super-init-not-called -- fix the classes __init__ structure
@@ -46,7 +45,6 @@ disable=
fixme,
broad-except,
bad-whitespace,
- bad-continuation,
invalid-name,
line-too-long,
super-init-not-called,
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index d2f16f9b..f673382f 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -443,8 +443,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
if os.path.exists(self.settings["source_path"]):
self.settings["source_path_hash"] = \
self.settings["hash_map"].generate_hash(
-self.settings["source_path"],
-hash_=self.settings["hash_function"])
+self.settings["source_path"],
+hash_=self.settings["hash_function"])
log.notice('Source path set to %s', self.settings['source_path'])
def set_dest_path(self):
@@ -470,8 +470,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
log.info('SNAPSHOT_PATH set to: %s', self.settings['snapshot_path'])
self.settings["snapshot_path_hash"] = \
self.settings["hash_map"].generate_hash(
-self.settings["snapshot_path"],
-hash_=self.settings["hash_function"])
+self.settings["snapshot_path"],
+hash_=self.settings["hash_function"])
def set_snapcache_path(self):
self.settings["snapshot_cache_path"] = \
diff --git a/setup.py b/setup.py
index fef09622..7a97b30c 100755
--- a/setup.py
+++ b/setup.py
@@ -93,7 +93,7 @@ _setup(
long_description=_codecs.open(
_os.path.join(_this_dir, 'README'), 'r', 'utf-8').read(),
classifiers=[
-'Development Status :: 5 - Production/Stable',
+'Development Status :: 5 - Production/Stable',
'License :: OSI Approved :: GNU General Public License v2 or later
(GPLv2+)',
'Intended Audience :: System Administrators',
'Operating System :: POSIX',
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/
commit: 051596bc5da38d0d1b6ccc1f349cec783d9599b3 Author: Matt Turner gentoo org> AuthorDate: Wed Apr 15 19:57:41 2020 + Commit: Matt Turner gentoo org> CommitDate: Wed Apr 15 19:57:41 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=051596bc catalyst: Remove unused import and reorder Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index d1e638be..d2f16f9b 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -1,10 +1,9 @@ import os -import imp -import toml import platform import shutil import sys +import toml from snakeoil import fileutils from snakeoil.osutils import pjoin
[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/targets/
commit: 92af7453a386fb5bd1b03ba5afae6d0165fe4364
Author: Matt Turner gentoo org>
AuthorDate: Wed Apr 15 18:00:15 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Wed Apr 15 18:04:03 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=92af7453
catalyst: Remove dead base_dirs() function
Prior to commit 8ef6e85a57ec (catalyst: Drop BSD remnants) this was used
for some FreeBSD-specific logic.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 5 +
catalyst/targets/stage1.py | 3 ---
2 files changed, 1 insertion(+), 7 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index b01e4825..d993641f 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -528,7 +528,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
"""Set basic stage1, 2, 3 action sequences"""
self.settings["action_sequence"] = ["unpack", "unpack_snapshot",
"setup_confdir", "portage_overlay",
- "base_dirs", "bind", "chroot_setup",
"setup_environment",
+ "bind", "chroot_setup", "setup_environment",
"run_local", "preclean", "unbind", "clean"]
self.set_completion_action_sequences()
@@ -940,9 +940,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
cmd(['rsync', '-a', x + '/',
self.settings['chroot_path']],
env=self.env)
- def base_dirs(self):
- pass
-
def bind(self):
for x in self.mounts:
log.debug('bind(); x = %s', x)
diff --git a/catalyst/targets/stage1.py b/catalyst/targets/stage1.py
index 4ce0d0b4..a1a29cff 100644
--- a/catalyst/targets/stage1.py
+++ b/catalyst/targets/stage1.py
@@ -77,9 +77,6 @@ class stage1(StageBase):
"If you break it, you buy it. Don't complain
to us about it.\n"
"Don't say we did not warn you.")
- def base_dirs(self):
- pass
-
def set_mounts(self):
# stage_path/proc probably doesn't exist yet, so create it
ensure_dirs(self.settings["stage_path"]+"/proc")
[gentoo-commits] proj/catalyst:master commit in: catalyst/arch/
commit: 39e28f75d0308b0cd1f0d41457379510d2c322c4 Author: Matt Turner gentoo org> AuthorDate: Mon Apr 13 04:00:48 2020 + Commit: Matt Turner gentoo org> CommitDate: Tue Apr 14 18:29:14 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=39e28f75 catalyst/arch: Add "r5k" to mips4 configurations I mistakenly thought -march=mips4 was synonymous with -march=r5k, but it actually meant r8k until 2015 when it changed to r10k. Rather than let others be confused (by removing the r10k configuration) just change the mips4 configs to r5k. Bug: https://bugs.gentoo.org/679386 Signed-off-by: Matt Turner gentoo.org> catalyst/arch/mips.py | 48 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/catalyst/arch/mips.py b/catalyst/arch/mips.py index f162a334..b1a4b5dc 100644 --- a/catalyst/arch/mips.py +++ b/catalyst/arch/mips.py @@ -87,29 +87,29 @@ class arch_mips3_multilib(generic_mips64): generic_mips64.__init__(self,myspec) self.settings["COMMON_FLAGS"]="-O2 -march=mips3 -mplt -mfix-r4000 -mfix-r4400 -pipe" -class arch_mips4(generic_mips): +class arch_mips4_r5k(generic_mips): "Builder class for MIPS IV [Big-endian]" def __init__(self,myspec): generic_mips.__init__(self,myspec) - self.settings["COMMON_FLAGS"]="-O2 -march=mips4 -mabi=32 -mplt -pipe" + self.settings["COMMON_FLAGS"]="-O2 -march=r5k -mabi=32 -mplt -pipe" -class arch_mips4_n32(generic_mips64): +class arch_mips4_r5k_n32(generic_mips64): "Builder class for MIPS IV [Big-endian N32]" def __init__(self,myspec): generic_mips64.__init__(self,myspec) - self.settings["COMMON_FLAGS"]="-O2 -march=mips4 -mabi=n32 -mplt -pipe" + self.settings["COMMON_FLAGS"]="-O2 -march=r5k -mabi=n32 -mplt -pipe" -class arch_mips4_n64(generic_mips64): +class arch_mips4_r5k_n64(generic_mips64): "Builder class for MIPS IV [Big-endian N64]" def __init__(self,myspec): generic_mips64.__init__(self,myspec) - self.settings["COMMON_FLAGS"]="-O2 -march=mips4 -mabi=64 -pipe" + self.settings["COMMON_FLAGS"]="-O2 -march=r5k -mabi=64 -pipe" -class arch_mips4_multilib(generic_mips64): +class arch_mips4_r5k_multilib(generic_mips64): "Builder class for MIPS IV [Big-endian multilib]" def __init__(self,myspec): generic_mips64.__init__(self,myspec) - self.settings["COMMON_FLAGS"]="-O2 -march=mips4 -mplt -pipe" + self.settings["COMMON_FLAGS"]="-O2 -march=r5k -mplt -pipe" class arch_mips4_r10k(generic_mips): "Builder class for MIPS IV R10k [Big-endian]" @@ -317,29 +317,29 @@ class arch_loongson2f_multilib(generic_mips64el): generic_mips64el.__init__(self,myspec) self.settings["COMMON_FLAGS"]="-O2 -march=loongson2f -mplt -Wa,-mfix-loongson2f-nop -pipe" -class arch_mipsel4(generic_mipsel): +class arch_mipsel4_r5k(generic_mipsel): "Builder class for MIPS IV [Little-endian]" def __init__(self,myspec): generic_mipsel.__init__(self,myspec) - self.settings["COMMON_FLAGS"]="-O2 -march=mips4 -mabi=32 -mplt -pipe" + self.settings["COMMON_FLAGS"]="-O2 -march=r5k -mabi=32 -mplt -pipe" -class arch_mipsel4_n32(generic_mips64el): +class arch_mipsel4_r5k_n32(generic_mips64el): "Builder class for MIPS IV [Little-endian N32]" def __init__(self,myspec): generic_mips64el.__init__(self,myspec) - self.settings["COMMON_FLAGS"]="-O2 -march=mips4 -mabi=n32 -mplt -pipe" + self.settings["COMMON_FLAGS"]="-O2 -march=r5k -mabi=n32 -mplt -pipe" -class arch_mipsel4_n64(generic_mips64el): +class arch_mipsel4_r5k_n64(generic_mips64el): "Builder class for MIPS IV [Little-endian N64]" def __init__(self,myspec): generic_mips64el.__init__(self,myspec) - self.settings["COMMON_FLAGS"]="-O2 -march=mips4 -mabi=64 -pipe" + self.settings["COMMON_FLAGS"]="-O2 -march=r5k -mabi=64 -pipe" -class arch_mipsel4_multilib(generic_mips64el): +class arch_mipsel4_r5k_multilib(generic_mips64el): "Builder class for MIPS IV [Little-endian multilib]" def __init__(self,myspec): generic_mips64el.__init__(self,myspec) - self.settings["COMMON_FLAGS"]="-O2 -march=mips4 -mplt -pipe" + self.settings["COMMON_FLAGS"]="-O2 -march=r5k -mplt -pipe" class arch_mips64el(generic_mipsel): "Builder class for MIPS 64 [Little-endian]" @@ -440,10 +440,10 @@ def register(): "mips3_n32" : arch_mips3_n32, "mips3_n64" : arch_mips3_n64, "mips3_multilib": arch_mips3_multilib, -
[gentoo-commits] proj/catalyst:master commit in: catalyst/arch/, catalyst/, catalyst/base/
commit: 72b60014663fabea91275f627a83c7f5be2e778b
Author: Matt Turner gentoo org>
AuthorDate: Sun Apr 12 00:50:19 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Mon Apr 13 20:43:10 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=72b60014
catalyst: Set CHROOT in stagebase
Signed-off-by: Matt Turner gentoo.org>
catalyst/arch/powerpc.py | 7 ++-
catalyst/arch/sparc.py | 7 ++-
catalyst/arch/x86.py | 7 ++-
catalyst/base/stagebase.py | 10 --
catalyst/builder.py| 8
5 files changed, 14 insertions(+), 25 deletions(-)
diff --git a/catalyst/arch/powerpc.py b/catalyst/arch/powerpc.py
index 0d88cc59..e2f241d3 100644
--- a/catalyst/arch/powerpc.py
+++ b/catalyst/arch/powerpc.py
@@ -1,5 +1,3 @@
-import platform
-
from catalyst import builder
class generic_ppc(builder.generic):
@@ -7,9 +5,8 @@ class generic_ppc(builder.generic):
def __init__(self,myspec):
builder.generic.__init__(self,myspec)
self.settings["CHOST"]="powerpc-unknown-linux-gnu"
- if platform.machine() == 'ppc64':
- self.setarch('linux32')
- self.settings["crosscompile"] = False
+ self.settings['setarch_build'] = 'ppc64'
+ self.settings['setarch_arch'] = 'linux32'
class generic_ppc64(builder.generic):
"abstract base class for all 64-bit powerpc builders"
diff --git a/catalyst/arch/sparc.py b/catalyst/arch/sparc.py
index 782d8916..38a1479e 100644
--- a/catalyst/arch/sparc.py
+++ b/catalyst/arch/sparc.py
@@ -1,14 +1,11 @@
-import platform
-
from catalyst import builder
class generic_sparc(builder.generic):
"abstract base class for all sparc builders"
def __init__(self,myspec):
builder.generic.__init__(self,myspec)
- if platform.machine() == 'sparc64':
- self.setarch('linux32')
- self.settings["crosscompile"] = False
+ self.settings['setarch_build'] = 'sparc64'
+ self.settings['setarch_arch'] = 'linux32'
class generic_sparc64(builder.generic):
"abstract base class for all sparc64 builders"
diff --git a/catalyst/arch/x86.py b/catalyst/arch/x86.py
index 307365dc..751ae769 100644
--- a/catalyst/arch/x86.py
+++ b/catalyst/arch/x86.py
@@ -1,14 +1,11 @@
-import platform
-
from catalyst import builder
class generic_x86(builder.generic):
"abstract base class for all x86 builders"
def __init__(self,myspec):
builder.generic.__init__(self,myspec)
- if platform.machine() == 'x86_64':
- self.setarch('linux32')
- self.settings["crosscompile"] = False
+ self.settings['setarch_build'] = 'x86_64'
+ self.settings['setarch_arch'] = 'linux32'
class arch_x86(generic_x86):
"builder class for generic x86 (386+)"
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index d18f28af..d4fafb74 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -136,8 +136,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
if buildmachine not in machinemap:
raise CatalystError("Unknown build machine type " +
buildmachine)
self.settings["buildarch"] = machinemap[buildmachine]
- self.settings["crosscompile"] = (self.settings["hostarch"] != \
- self.settings["buildarch"])
# Call arch constructor, pass our settings
try:
@@ -149,6 +147,14 @@ class StageBase(TargetBase, ClearBase, GenBase):
' %s',
self.settings['subarch'], '
'.join(self.subarchmap))
+ if 'setarch_arch' in self.settings and platform.machine() ==
self.settings["setarch_build"]:
+ self.settings["CHROOT"] = f'setarch
{self.settings["setarch_arch"]} chroot'
+ self.settings["crosscompile"] = False
+ else:
+ self.settings["CHROOT"] = 'chroot'
+ self.settings["crosscompile"] = \
+ self.settings["hostarch"] !=
self.settings["buildarch"]
+
log.notice('Using target: %s', self.settings['target'])
# Print a nice informational message
if self.settings["buildarch"] == self.settings["hostarch"]:
diff --git a/catalyst/builder.py b/catalyst/builder.py
index fd8d5eb6..2d139263 100644
--- a/catalyst/builder.py
+++ b/catalyst/builder.py
@@ -1,11 +1,3 @@
class generic():
def __init__(self,myspec):
self.settings=myspec
- self.settings.setdefault('CHROOT', 'chroot')
-
- def setarch(self, arch):
- """Set the chroot wrapper to run through `setarch |arch|`
-
-
[gentoo-commits] proj/catalyst:master commit in: catalyst/arch/
commit: 739c9998475358bacd44dc46fdac885a45289d0f Author: Matt Turner gentoo org> AuthorDate: Sun Apr 12 05:51:57 2020 + Commit: Matt Turner gentoo org> CommitDate: Mon Apr 13 20:43:10 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=739c9998 catalyst/arch: Delete some dead power configs GCC doesn't support either -mcpu=common nor -mcpu=power. Perhaps it did long ago... Signed-off-by: Matt Turner gentoo.org> catalyst/arch/powerpc.py | 14 -- 1 file changed, 14 deletions(-) diff --git a/catalyst/arch/powerpc.py b/catalyst/arch/powerpc.py index e2f241d3..d9938da3 100644 --- a/catalyst/arch/powerpc.py +++ b/catalyst/arch/powerpc.py @@ -64,18 +64,6 @@ class arch_g5(generic_ppc): self.settings["COMMON_FLAGS"]="-O2 -mcpu=G5 -mtune=G5 -maltivec -mabi=altivec -pipe" self.settings["HOSTUSE"]=["altivec"] -class arch_power(generic_ppc): - "builder class for generic power" - def __init__(self,myspec): - generic_ppc.__init__(self,myspec) - self.settings["COMMON_FLAGS"]="-O2 -mcpu=power -mtune=power -pipe" - -class arch_power_ppc(generic_ppc): - "builder class for generic powerpc/power" - def __init__(self,myspec): - generic_ppc.__init__(self,myspec) - self.settings["COMMON_FLAGS"]="-O2 -mcpu=common -mtune=common -pipe" - class arch_power5(arch_ppc64): "builder class for power5 under ppc64" def __init__(self,myspec): @@ -126,8 +114,6 @@ def register(): "g3": arch_g3, "g4": arch_g4, "g5": arch_g5, - "power" : arch_power, - "power-ppc" : arch_power_ppc, "power5": arch_power5, "power6": arch_power6, "power7": arch_power7,
