[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/, catalyst/

2021-06-10 Thread Matt Turner 
commit: b3c917f7fa73d11c69b7e55dc7a00bc18a18edc7
Author: Matt Turner  gentoo  org>
AuthorDate: Wed Jun  9 06:17:31 2021 +
Commit: Matt Turner  gentoo  org>
CommitDate: Thu Jun 10 04:23:54 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b3c917f7

catalyst: Replace snakeoil's locks with fasteners

To no great surprise, the existing locking was broken. For example,
clear_chroot() releases the lock. It is called by unpack(), which is
part of prepare_sequence. The result is that the whole build could be
done without holding the lock.

Just lock around run(). It's not apparent that finer-grained locking
does anything for us.

Bug: https://bugs.gentoo.org/791583
Signed-off-by: Matt Turner  gentoo.org>

 catalyst/base/clearbase.py   |  2 --
 catalyst/base/stagebase.py   | 10 +++-
 catalyst/lock.py | 58 
 catalyst/targets/snapshot.py |  6 ++---
 4 files changed, 7 insertions(+), 69 deletions(-)

diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py
index dcf6c523..6218330e 100644
--- a/catalyst/base/clearbase.py
+++ b/catalyst/base/clearbase.py
@@ -27,12 +27,10 @@ class ClearBase():
 self.resume.clear_all(remove=True)
 
 def clear_chroot(self):
-self.chroot_lock.unlock()
 log.notice('Clearing the chroot path ...')
 clear_dir(self.settings["chroot_path"], mode=0o755)
 
 def remove_chroot(self):
-self.chroot_lock.unlock()
 log.notice('Removing the chroot path ...')
 clear_dir(self.settings["chroot_path"], mode=0o755, remove=True)
 

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 448d6265..10cffd4c 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -8,6 +8,7 @@ import sys
 
 from pathlib import Path
 
+import fasteners
 import libmount
 import toml
 
@@ -25,7 +26,6 @@ from catalyst.support import (CatalystError, file_locate, 
normpath,
 from catalyst.base.targetbase import TargetBase
 from catalyst.base.clearbase import ClearBase
 from catalyst.base.genbase import GenBase
-from catalyst.lock import LockDir, LockInUse
 from catalyst.fileops import ensure_dirs, clear_dir, clear_path
 from catalyst.base.resume import AutoResume
 
@@ -36,9 +36,6 @@ def run_sequence(sequence):
 sys.stdout.flush()
 try:
 func()
-except LockInUse:
-log.error('Unable to aquire the lock...')
-return False
 except Exception:
 log.error('Exception running action sequence %s',
   func.__name__, exc_info=True)
@@ -478,7 +475,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
 """
 self.settings["chroot_path"] = normpath(self.settings["storedir"] +
 "/tmp/" + 
self.settings["target_subpath"].rstrip('/'))
-self.chroot_lock = LockDir(self.settings["chroot_path"])
 
 def set_autoresume_path(self):
 self.settings["autoresume_path"] = normpath(pjoin(
@@ -1366,8 +1362,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
 pass
 
 def run(self):
-self.chroot_lock.write_lock()
+with fasteners.InterProcessLock(self.settings["chroot_path"] + 
'.lock'):
+return self._run()
 
+def _run(self):
 if "clear-autoresume" in self.settings["options"]:
 self.clear_autoresume()
 

diff --git a/catalyst/lock.py b/catalyst/lock.py
deleted file mode 100644
index e31745b2..
--- a/catalyst/lock.py
+++ /dev/null
@@ -1,58 +0,0 @@
-
-import os
-
-from contextlib import contextmanager
-
-from snakeoil import fileutils
-from snakeoil import osutils
-from catalyst.fileops import ensure_dirs
-
-
-LockInUse = osutils.LockException
-
-class Lock:
-"""
-A fnctl-based filesystem lock
-"""
-def __init__(self, lockfile):
-fileutils.touch(lockfile, mode=0o664)
-os.chown(lockfile, uid=-1, gid=250)
-self.lock = osutils.FsLock(lockfile)
-
-def read_lock(self):
-self.lock.acquire_read_lock()
-
-def write_lock(self):
-self.lock.acquire_write_lock()
-
-def unlock(self):
-# Releasing a write lock is the same as a read lock.
-self.lock.release_write_lock()
-
-class LockDir(Lock):
-"""
-A fnctl-based filesystem lock in a directory
-"""
-def __init__(self, lockdir):
-ensure_dirs(lockdir)
-lockfile = os.path.join(lockdir, '.catalyst_lock')
-
-Lock.__init__(self, lockfile)
-
-@contextmanager
-def read_lock(filename):
-lock = Lock(filename)
-lock.read_lock()
-try:
-yield
-finally:
-lock.unlock()
-
-@contextmanager
-def write_lock(filename):
-lock = Lock(filename)
-lock.write_lock()
-try:
-yield
-finally:
-lock.unlock()

diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py
index b494575a..ef68765d

[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/

2021-01-23 Thread Matt Turner 
commit: 440a379ae94e6d7df26291bf83657288d1406e98
Author: Matt Turner  gentoo  org>
AuthorDate: Sun Jan 17 23:53:21 2021 +
Commit: Matt Turner  gentoo  org>
CommitDate: Sat Jan 23 16:19:07 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=440a379a

catalyst: Store references to functions

... rather than their names. This makes it possible for tooling to
understand the code structure better.

Signed-off-by: Matt Turner  gentoo.org>

 catalyst/base/stagebase.py| 69 ---
 catalyst/targets/embedded.py  | 34 +--
 catalyst/targets/livecd_stage1.py | 18 +-
 catalyst/targets/livecd_stage2.py | 46 +-
 catalyst/targets/netboot.py   | 34 +--
 catalyst/targets/stage1.py| 12 +++
 catalyst/targets/stage4.py| 36 ++--
 7 files changed, 125 insertions(+), 124 deletions(-)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index ed4d1227..447e073d 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -30,6 +30,23 @@ from catalyst.fileops import ensure_dirs, clear_dir, 
clear_path
 from catalyst.base.resume import AutoResume
 
 
+def run_sequence(sequence):
+for func in sequence:
+log.notice('--- Running action sequence: %s', func.__name__)
+sys.stdout.flush()
+try:
+func()
+except LockInUse:
+log.error('Unable to aquire the lock...')
+return False
+except Exception:
+log.error('Exception running action sequence %s',
+  func.__name__, exc_info=True)
+return False
+
+return True
+
+
 class StageBase(TargetBase, ClearBase, GenBase):
 """
 This class does all of the chroot setup, copying of files, etc. It is
@@ -475,39 +492,39 @@ class StageBase(TargetBase, ClearBase, GenBase):
 Or it calls the normal set_action_sequence() for the target stage.
 """
 if "purgeonly" in self.settings["options"]:
-self.build_sequence.append("remove_chroot")
+self.build_sequence.append(self.remove_chroot)
 return
 self.set_action_sequence()
 
 def set_action_sequence(self):
 """Set basic stage1, 2, 3 action sequences"""
 self.prepare_sequence.extend([
-"unpack",
-"setup_confdir",
-"portage_overlay",
+self.unpack,
+self.setup_confdir,
+self.portage_overlay,
 ])
 self.build_sequence.extend([
-"bind",
-"chroot_setup",
-"setup_environment",
-"run_local",
-"preclean",
+self.bind,
+self.chroot_setup,
+self.setup_environment,
+self.run_local,
+self.preclean,
 ])
 self.finish_sequence.extend([
-"clean",
+self.clean,
 ])
 self.set_completion_action_sequences()
 
 def set_completion_action_sequences(self):
 if "fetch" not in self.settings["options"]:
-self.finish_sequence.append("capture")
+self.finish_sequence.append(self.capture)
 if "keepwork" in self.settings["options"]:
-self.finish_sequence.append("clear_autoresume")
+self.finish_sequence.append(self.clear_autoresume)
 elif "seedcache" in self.settings["options"]:
-self.finish_sequence.append("remove_autoresume")
+self.finish_sequence.append(self.remove_autoresume)
 else:
-self.finish_sequence.append("remove_autoresume")
-self.finish_sequence.append("remove_chroot")
+self.finish_sequence.append(self.remove_autoresume)
+self.finish_sequence.append(self.remove_chroot)
 
 def set_use(self):
 use = self.settings["spec_prefix"] + "/use"
@@ -1308,22 +1325,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
 
 log.debug('setup_environment(); env = %r', self.env)
 
-def run_sequence(self, sequence):
-for func in sequence:
-log.notice('--- Running action sequence: %s', func)
-sys.stdout.flush()
-try:
-getattr(self, func)()
-except LockInUse:
-log.error('Unable to aquire the lock...')
-return False
-except Exception:
-log.error('Exception running action sequence %s',
-  func, exc_info=True)
-return False
-
-return True
-
 def run(self):
 self.chroot_lock.write_lock()
 
@@ -1342,14 +1343,14 @@ class StageBase(TargetBase, ClearBase, GenBase):
 log.info('StageBase: run() purge')
 self.purge()
 
-if not self.run_sequence(self.prepare_sequence):
+if not run_sequence(self.prepare_sequence):

[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/

2020-10-30 Thread Matt Turner 
commit: dfc4ed516b839bbc225cc7459e8c3085970801f6
Author: Matt Turner  gentoo  org>
AuthorDate: Thu Oct 29 00:28:24 2020 +
Commit: Matt Turner  gentoo  org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=dfc4ed51

catalyst: Move action_sequence out of self.settings[]

This self.settings[] dictionary is very similar to the god object
anti-pattern. Moving action_sequence out it starts us down the road of
fixing this.

Signed-off-by: Matt Turner  gentoo.org>

 catalyst/base/stagebase.py| 17 +
 catalyst/targets/embedded.py  |  2 +-
 catalyst/targets/livecd_stage1.py |  2 +-
 catalyst/targets/livecd_stage2.py |  6 +++---
 catalyst/targets/netboot.py   |  2 +-
 catalyst/targets/stage1.py| 12 ++--
 catalyst/targets/stage4.py|  2 +-
 7 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 73eacfbe..801df2fb 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -64,6 +64,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 "portage_overlay",
 "portage_prefix",
 ])
+self.action_sequence = []
 
 self.set_valid_build_kernel_vars(addlargs)
 TargetBase.__init__(self, myspec, addlargs)
@@ -477,13 +478,13 @@ class StageBase(TargetBase, ClearBase, GenBase):
 Or it calls the normal set_action_sequence() for the target stage.
 """
 if "purgeonly" in self.settings["options"]:
-self.settings["action_sequence"] = ["remove_chroot"]
+self.action_sequence = ["remove_chroot"]
 return
 self.set_action_sequence()
 
 def set_action_sequence(self):
 """Set basic stage1, 2, 3 action sequences"""
-self.settings['action_sequence'] = [
+self.action_sequence = [
 "unpack",
 "setup_confdir",
 "portage_overlay",
@@ -499,14 +500,14 @@ class StageBase(TargetBase, ClearBase, GenBase):
 
 def set_completion_action_sequences(self):
 if "fetch" not in self.settings["options"]:
-self.settings["action_sequence"].append("capture")
+self.action_sequence.append("capture")
 if "keepwork" in self.settings["options"]:
-self.settings["action_sequence"].append("clear_autoresume")
+self.action_sequence.append("clear_autoresume")
 elif "seedcache" in self.settings["options"]:
-self.settings["action_sequence"].append("remove_autoresume")
+self.action_sequence.append("remove_autoresume")
 else:
-self.settings["action_sequence"].append("remove_autoresume")
-self.settings["action_sequence"].append("remove_chroot")
+self.action_sequence.append("remove_autoresume")
+self.action_sequence.append("remove_chroot")
 
 def set_use(self):
 use = self.settings["spec_prefix"] + "/use"
@@ -1380,7 +1381,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 self.purge()
 
 failure = False
-for x in self.settings["action_sequence"]:
+for x in self.action_sequence:
 log.notice('--- Running action sequence: %s', x)
 sys.stdout.flush()
 try:

diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py
index 99739512..3899cf1b 100644
--- a/catalyst/targets/embedded.py
+++ b/catalyst/targets/embedded.py
@@ -41,7 +41,7 @@ class embedded(StageBase):
 StageBase.__init__(self, spec, addlargs)
 
 def set_action_sequence(self):
-self.settings['action_sequence'] = [
+self.action_sequence = [
 "unpack",
 "config_profile_link",
 "setup_confdir",

diff --git a/catalyst/targets/livecd_stage1.py 
b/catalyst/targets/livecd_stage1.py
index f0b6be8b..b8c26cb1 100644
--- a/catalyst/targets/livecd_stage1.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -23,7 +23,7 @@ class livecd_stage1(StageBase):
 StageBase.__init__(self, spec, addlargs)
 
 def set_action_sequence(self):
-self.settings['action_sequence'] = [
+self.action_sequence = [
 "unpack",
 "config_profile_link",
 "setup_confdir",

diff --git a/catalyst/targets/livecd_stage2.py 
b/catalyst/targets/livecd_stage2.py
index 88c0d95c..cac16b6e 100644
--- a/catalyst/targets/livecd_stage2.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -87,7 +87,7 @@ class livecd_stage2(StageBase):
 print_traceback=True)
 
 def set_action_sequence(self):
-self.settings['action_sequence'] = [
+self.action_sequence = [
 "unpack",
 "config_profile_link",
 "setup_confdir",
@@ -99,7 +99,7 @@ class livecd_stage2(StageBase):
 "build_kernel"
 ]
 if "fetch" not in

[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/

2020-10-30 Thread Matt Turner 
commit: 4b5ff905a7ad2ffe1ed8c863b91e9d0ce6981f5f
Author: Matt Turner  gentoo  org>
AuthorDate: Thu Oct 22 20:37:54 2020 +
Commit: Matt Turner  gentoo  org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4b5ff905

catalyst: Drop unbind()

mount_namespaces(7) says

A mount ceases to be a member of a peer group when either the
mount is explicitly unmounted, or when the mount is implicitly
unmounted because a mount namespace is removed (because it has
no more member processes).

As a result, we can rely on exiting the mount namespace to unmount the
bind mounts.

Signed-off-by: Matt Turner  gentoo.org>

 catalyst/base/stagebase.py| 44 +--
 catalyst/targets/embedded.py  |  1 -
 catalyst/targets/livecd_stage1.py |  1 -
 catalyst/targets/livecd_stage2.py |  2 --
 catalyst/targets/netboot.py   |  3 ---
 catalyst/targets/stage4.py|  1 -
 6 files changed, 1 insertion(+), 51 deletions(-)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index b9c220d0..a75dbdf9 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -498,7 +498,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
 "setup_environment",
 "run_local",
 "preclean",
-"unbind",
 ])
 self.finish_sequence.extend([
 "clean",
@@ -853,40 +852,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
fstype=fstype, options=options)
 cxt.mount()
 except OSError as e:
-self.unbind()
 raise CatalystError(f"Couldn't mount: {source}, {e.strerror}")
 
-def unbind(self):
-chroot_path = self.settings["chroot_path"]
-umount_failed = False
-
-# Unmount in reverse order
-for target in [Path(chroot_path + self.mount[x]['target'])
-   for x in reversed(self.mount)
-   if self.mount[x]['enable']]:
-if not target.exists():
-log.debug('%s does not exist. Skipping', target)
-continue
-
-if not ismount(target):
-log.debug('%s is not a mount point. Skipping', target)
-continue
-
-try:
-cxt = libmount.Context(target=str(target))
-cxt.umount()
-except OSError as e:
-log.warning("Couldn't umount: %s, %s", target,
-e.strerror)
-umount_failed = True
-
-if umount_failed:
-# if any bind mounts really failed, then we need to raise
-# this to potentially prevent an upcoming bash stage cleanup script
-# from wiping our bind mounts.
-raise CatalystError(
-"Couldn't umount one or more bind-mounts; aborting for 
safety.")
-
 def chroot_setup(self):
 self.makeconf = read_makeconf(normpath(self.settings["chroot_path"] +
self.settings["make_conf"]))
@@ -1190,7 +1157,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
 env=self.env)
 self.resume.enable("remove")
 except:
-self.unbind()
 raise
 
 def preclean(self):
@@ -1206,7 +1172,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
 self.resume.enable("preclean")
 
 except:
-self.unbind()
 raise CatalystError("Build failed, could not execute preclean")
 
 def capture(self):
@@ -1269,7 +1234,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
  self.settings['controller_file'])
 
 except CatalystError:
-self.unbind()
 raise CatalystError("Stage build aborting due to error.",
 print_traceback=False)
 
@@ -1374,7 +1338,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
 env=self.env)
 log.info('unmerge shell script')
 except CatalystError:
-self.unbind()
 raise
 self.resume.enable("unmerge")
 
@@ -1449,7 +1412,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
 fileutils.touch(build_packages_resume)
 self.resume.enable("build_packages")
 except CatalystError:
-self.unbind()
 raise CatalystError(
 self.settings["spec_prefix"] +
 "build aborting due to error.")
@@ -1473,7 +1435,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
 self._build_kernel(kname=kname)
 self.resume.enable("build_kernel")
 except CatalystError:
-self.unbind()

[gentoo-commits] proj/catalyst:master commit in: /, catalyst/targets/, catalyst/base/, catalyst/, doc/

2020-04-30 Thread Matt Turner 
commit: 98cf07d2ca4da1f88e213a520095bfccc9c81ffc
Author: Matt Turner  gentoo  org>
AuthorDate: Fri Apr 17 23:31:52 2020 +
Commit: Matt Turner  gentoo  org>
CommitDate: Thu Apr 30 22:54:49 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=98cf07d2

catalyst: Make and use squashfs snapshots

There were a number of problems with catalyst's snapshot system. It was
built around using the build system's portdir and had no control over
what was in that portdir or when it was updated.

As a result, when a stage build failed, it was difficult to tell what
the snapshot consistet of precisely or whether it contained a particular
recent fix.

With snapcache disabled, ebuild repo snapshots were tar'd and compressed
and then unpacked into the stage chroot which is an unnecessarily
expensive process. Moreover, a porttree has more than 100k small files,
which are stored extremely inefficiently on most file systems—a whole
porttree is usually around 700M on disk. Just removing all of those
files during the cleaning stage is an expensive operation.

Instead, we make a compressed squashfs image and mount it in the build
chroot. The porttree has many duplicate files, and squashfs deduplicates
the files and then compresses, so the result is very efficiently packed:
~38M with gzip -9 compression.

The snapshot target has been modified to generate a squashfs image from
a bare ebuild git repo. Piping git-archive to tar2sqfs generates the
squashfs image in less than 10 seconds on a modern system. The git repo
is fetched with --depth=1 to minize bandwidth and disk usage, and git gc
is run after fetch to minimize disk usage. Storage requirements for the
git ebuild repo with metadata are ~70M.

The squashfs snapshot is stored in /var/tmp/catalyst/snapshots/ by
default with a name -.sqfs. With this convention,
we know the exact point in history that the snapshot was taken. The
catalyst-auto script can use the sha1 to get a deterministic timestamp,
so that it is independent on when `catalyst -s` was run, but is instead
the timestamp of the commit date of the repo's git SHA1.

Signed-off-by: Matt Turner  gentoo.org>

 README|   3 +-
 catalyst/base/stagebase.py|  88 
 catalyst/base/targetbase.py   |  16 +++-
 catalyst/defaults.py  |   2 +-
 catalyst/main.py  |  24 +++---
 catalyst/targets/embedded.py  |   1 -
 catalyst/targets/livecd_stage1.py |   1 -
 catalyst/targets/livecd_stage2.py |   1 -
 catalyst/targets/netboot.py   |   1 -
 catalyst/targets/snapshot.py  | 165 +++---
 catalyst/targets/stage4.py|   1 -
 doc/catalyst-config.5.txt |  13 +--
 doc/catalyst-spec.5.txt   |   4 +-
 13 files changed, 137 insertions(+), 183 deletions(-)

diff --git a/README b/README
index 1a039fca..1cceb63e 100644
--- a/README
+++ b/README
@@ -18,8 +18,9 @@ Requirements
 ===
 
 - Python 3.6 or greater
-- An ebuild repository snapshot (or an ebuild tree to create one)
 - A generic stage3 tarball for your architecture
+- A squashfs ebuild repository snapshot
+  - Or an ebuild git repo with sys-fs/squashfs-tools-ng and dev-vcs/git
 
 What is catalyst?
 

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 9aecf013..41da97b3 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -35,7 +35,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 self.required_values |= frozenset([
 "profile",
 "rel_type",
-"snapshot",
+"snapshot_treeish",
 "source_subpath",
 "subarch",
 "target",
@@ -149,7 +149,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 self.set_source_subpath()
 
 # Set paths
-self.set_snapshot_path()
+self.set_snapshot()
 self.set_root_path()
 self.set_source_path()
 self.set_chroot_path()
@@ -191,9 +191,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
 # Setup our mount points.
 self.mount = MOUNT_DEFAULTS.copy()
 
-# Always unpack snapshot tarball
-self.mount['portdir']['enable'] = False
-
+self.mount['portdir']['source'] = self.snapshot
+self.mount['portdir']['target'] = self.settings['repo_basedir'] + '/' 
+ self.settings['repo_name']
 self.mount['distdir']['source'] = self.settings['distdir']
 self.mount["distdir"]['target'] = self.settings['target_distdir']
 
@@ -435,21 +434,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
 self.settings["destpath"] = normpath(self.settings["chroot_path"])
 
 def set_cleanables(self):
-self.settings["cleanables"] = ["/etc/resolv.conf", "/var/tmp/*", 
"/tmp/*",
-   self.settings["repo_basedir"] + "/" +
-

[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/

2020-04-21 Thread Matt Turner 
commit: 3b0ff52abd04aa213c844b6d716d9edbe809dc65
Author: Matt Turner  gentoo  org>
AuthorDate: Sat Apr 18 20:20:05 2020 +
Commit: Matt Turner  gentoo  org>
CommitDate: Wed Apr 22 00:22:24 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=3b0ff52a

catalyst: Remove code to mount /proc in stage1root

Added in commit f75bb07f3cec (Added two patches from Joshua Kinard ...),
mounting /proc in stage1root purports to fix an issue in the preclean
stage. I have no idea what that issue was, but it doesn't occur now.

Signed-off-by: Matt Turner  gentoo.org>

 catalyst/base/stagebase.py |  5 -
 catalyst/targets/stage1.py | 11 +--
 2 files changed, 1 insertion(+), 15 deletions(-)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 60cb891a..f8c1611e 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -213,8 +213,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
 self.mounts.append("shm")
 self.mounts.append("run")
 
-self.set_mounts()
-
 # Configure any user specified options (either in catalyst.conf or on
 # the command line).
 if "pkgcache" in self.settings["options"]:
@@ -586,9 +584,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
 def set_stage_path(self):
 self.settings["stage_path"] = normpath(self.settings["chroot_path"])
 
-def set_mounts(self):
-pass
-
 def set_packages(self):
 pass
 

diff --git a/catalyst/targets/stage1.py b/catalyst/targets/stage1.py
index daa0b854..93c62877 100644
--- a/catalyst/targets/stage1.py
+++ b/catalyst/targets/stage1.py
@@ -5,7 +5,7 @@ stage1 target
 
 from catalyst import log
 from catalyst.support import normpath
-from catalyst.fileops import ensure_dirs, move_path
+from catalyst.fileops import move_path
 from catalyst.base.stagebase import StageBase
 
 
@@ -78,15 +78,6 @@ class stage1(StageBase):
 "If you break it, you buy it.  Don't complain to us about 
it.\n"
 "Don't say we did not warn you.")
 
-def set_mounts(self):
-# stage_path/proc probably doesn't exist yet, so create it
-ensure_dirs(self.settings["stage_path"]+"/proc")
-
-# alter the mount mappings to bind mount proc onto it
-self.mounts.append("stage1root/proc")
-self.target_mounts["stage1root/proc"] = "/tmp/stage1root/proc"
-self.mountmap["stage1root/proc"] = "/proc"
-
 def set_completion_action_sequences(self):
 '''Override function for stage1

[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/

2020-04-13 Thread Matt Turner 
commit: 56ee6814b2dade3d0c2b12ede3582a3e479aa532
Author: Matt Turner  gentoo  org>
AuthorDate: Sat Apr 11 22:23:54 2020 +
Commit: Matt Turner  gentoo  org>
CommitDate: Mon Apr 13 20:35:35 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=56ee6814

catalyst: Clean assignments of {valid/required}_values

Making them class variables will allow us to verify that all subclasses
of TargetBase have defined them (in the next commit).

While we're moving them, change them to frozensets, since testing
membership is the thing they're useed for.

Signed-off-by: Matt Turner  gentoo.org>

 catalyst/base/stagebase.py| 68 ++-
 catalyst/targets/embedded.py  | 24 ++
 catalyst/targets/livecd_stage1.py | 11 ---
 catalyst/targets/livecd_stage2.py | 48 +++
 catalyst/targets/netboot.py   | 29 -
 catalyst/targets/snapshot.py  | 11 +--
 catalyst/targets/stage1.py| 10 --
 catalyst/targets/stage2.py|  7 ++--
 catalyst/targets/stage3.py|  5 +--
 catalyst/targets/stage4.py| 26 +++
 10 files changed, 159 insertions(+), 80 deletions(-)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 829f9fb7..a90fc843 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -27,16 +27,36 @@ class StageBase(TargetBase, ClearBase, GenBase):
the driver class for pretty much everything that Catalyst does.
"""
def __init__(self,myspec,addlargs):
-   self.required_values.extend(["version_stamp", "target", 
"subarch",
-   "rel_type", "profile", "snapshot", "source_subpath"])
-
-   self.valid_values.extend(["version_stamp", "target", "subarch",
-   "rel_type", "profile", "snapshot", "source_subpath",
-   "portage_confdir", "portage_prefix", "portage_overlay",
-   "cflags", "cxxflags", "fcflags", "fflags", "ldflags", 
"asflags",
-   "common_flags", "cbuild", "hostuse", "catalyst_use",
-   "distcc_hosts", "makeopts", "pkgcache_path", 
"kerncache_path",
-   "compression_mode", "decompression_mode"])
+   self.required_values |= frozenset([
+   "profile",
+   "rel_type",
+   "snapshot",
+   "source_subpath",
+   "subarch",
+   "target",
+   "version_stamp",
+   ])
+   self.valid_values |= self.required_values | frozenset([
+   "asflags",
+   "catalyst_use",
+   "cbuild",
+   "cflags",
+   "common_flags",
+   "compression_mode",
+   "cxxflags",
+   "decompression_mode",
+   "distcc_hosts",
+   "fcflags",
+   "fflags",
+   "hostuse",
+   "kerncache_path",
+   "ldflags",
+   "makeopts",
+   "pkgcache_path",
+   "portage_confdir",
+   "portage_overlay",
+   "portage_prefix",
+   ])
 
self.set_valid_build_kernel_vars(addlargs)
TargetBase.__init__(self, myspec, addlargs)
@@ -422,7 +442,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 
def set_fsops(self):
if "fstype" in self.settings:
-   self.valid_values.append("fsops")
+   self.valid_values |= {"fsops"}
if self.settings["spec_prefix"] + "/fsops" in 
self.settings:
self.settings["fsops"] = \

self.settings[self.settings["spec_prefix"] + "/fsops"]
@@ -647,18 +667,20 @@ class StageBase(TargetBase, ClearBase, GenBase):
loopy = addlargs["boot/kernel"]
 
for x in loopy:
-   self.valid_values.append("boot/kernel/" + x + 
"/aliases")
-   self.valid_values.append("boot/kernel/" + x + 
"/config")
-   self.valid_values.append("boot/kernel/" + x + 
"/console")
-   self.valid_values.append("boot/kernel/" + x + 
"/extraversion")
-   self.valid_values.append("boot/kernel/" + x + 
"/gk_action")
-   self.valid_values.append("boot/kernel/" + x + 
"/gk_kernargs")
-   self.valid_values.append("boot/kernel/" + x + 
"/initramfs_overlay")
-

[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/, catalyst/

2020-04-10 Thread Matt Turner 
commit: 8ef6e85a57ecefb4eb7b4b50796dbb98ab1a3f2a
Author: Matt Turner  gentoo  org>
AuthorDate: Fri Apr 10 07:40:48 2020 +
Commit: Matt Turner  gentoo  org>
CommitDate: Fri Apr 10 07:42:08 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=8ef6e85a

catalyst: Drop BSD remnants

Signed-off-by: Matt Turner  gentoo.org>

 catalyst/base/clearbase.py |  4 ++--
 catalyst/base/resume.py|  2 +-
 catalyst/base/stagebase.py | 19 ++-
 catalyst/builder.py|  3 +--
 catalyst/defaults.py   | 14 --
 catalyst/fileops.py|  6 +-
 catalyst/main.py   | 17 +++--
 catalyst/targets/stage1.py | 14 +-
 8 files changed, 23 insertions(+), 56 deletions(-)

diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py
index f1d4d1ba..8dfffb06 100644
--- a/catalyst/base/clearbase.py
+++ b/catalyst/base/clearbase.py
@@ -30,13 +30,13 @@ class ClearBase():
def clear_chroot(self):
self.chroot_lock.unlock()
log.notice('Clearing the chroot path ...')
-   clear_dir(self.settings["chroot_path"], mode=0o755, 
chg_flags=True)
+   clear_dir(self.settings["chroot_path"], mode=0o755)
 
 
def remove_chroot(self):
self.chroot_lock.unlock()
log.notice('Removing the chroot path ...')
-   clear_dir(self.settings["chroot_path"], mode=0o755, 
chg_flags=True, remove=True)
+   clear_dir(self.settings["chroot_path"], mode=0o755, remove=True)
 
 
def clear_packages(self, remove=False):

diff --git a/catalyst/base/resume.py b/catalyst/base/resume.py
index 7732dd5d..a0770b9d 100644
--- a/catalyst/base/resume.py
+++ b/catalyst/base/resume.py
@@ -126,7 +126,7 @@ class AutoResume():
@remove: boolean, passed through to clear_dir()
@return boolean
'''
-   if clear_dir(self.basedir, mode=0o755, chg_flags=True, 
remove=remove):
+   if clear_dir(self.basedir, mode=0o755, remove=remove):
self._points = {}
return True
return False

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 320e9d53..021f3beb 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -224,10 +224,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["snapshot_cache_path"],
self.settings["repo_name"],
]))
-   if os.uname()[0] == "Linux":
-   self.mounts.append("devpts")
-   self.mounts.append("shm")
-   self.mounts.append("run")
+   self.mounts.append("devpts")
+   self.mounts.append("shm")
+   self.mounts.append("run")
 
self.set_mounts()
 
@@ -983,16 +982,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
elif src == "tmpfs":
_cmd = ['mount', '-t', 'tmpfs', src, target]
else:
-   if os.uname()[0] == "FreeBSD":
-   if src == "/dev":
-   _cmd = ['mount', '-t', 'devfs', 
'none', target]
-   else:
-   _cmd = ['mount_nullfs', src, 
target]
+   if src == "shmfs":
+   _cmd = ['mount', '-t', 'tmpfs', '-o', 
'noexec,nosuid,nodev', 'shm', target]
else:
-   if src == "shmfs":
-   _cmd = ['mount', '-t', 'tmpfs', 
'-o', 'noexec,nosuid,nodev', 'shm', target]
-   else:
-   _cmd = ['mount', '--bind', src, 
target]
+   _cmd = ['mount', '--bind', src, target]
if _cmd:
log.debug('bind(); _cmd = %s', _cmd)
cmd(_cmd, env=self.env, fail_func=self.unbind)

diff --git a/catalyst/builder.py b/catalyst/builder.py
index 5d7f76c1..e9f468d7 100644
--- a/catalyst/builder.py
+++ b/catalyst/builder.py
@@ -10,8 +10,7 @@ class generic():
 
Useful for building x86-on-amd64 and such.
"""
-   if os.uname()[0] == 'Linux':
-   self.settings['CHROOT'] = 'setarch %s %s' % (arch, 
self.settings['CHROOT'])
+   self.settings['CHROOT'] = 'setarch %s %s' % (arch, 
self.settings['CHROOT'])
 
def mount_safety_check(self):
"""

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 5b4ae83e..bcb59796 100644
---

[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/

2015-12-15 Thread Brian Dolbec 
commit: 159c6b3919f72a41cf60b3ecbb2ce5d07b1b18f2
Author: Brian Dolbec  gentoo  org>
AuthorDate: Tue Dec 15 17:00:48 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Tue Dec 15 17:08:38 2015 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=159c6b39

Split the stagebase's set_action_sequence()

Split off the configurable portions of completion sequences.
Re-use that new function in other stages.

 catalyst/base/stagebase.py| 3 +++
 catalyst/targets/livecd_stage1.py | 3 ++-
 catalyst/targets/stage4.py| 7 +--
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 92f300e..a880249 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -506,6 +506,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
"setup_confdir","portage_overlay",\

"base_dirs","bind","chroot_setup","setup_environment",\
"run_local","preclean","unbind","clean"]
+   self.set_completion_action_sequences()
+
+   def set_completion_action_sequences(self):
if "fetch" not in self.settings["options"]:
self.settings["action_sequence"].append("capture")
if "keepwork" in self.settings["options"]:

diff --git a/catalyst/targets/livecd_stage1.py 
b/catalyst/targets/livecd_stage1.py
index aa234b2..60ef388 100644
--- a/catalyst/targets/livecd_stage1.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -27,7 +27,8 @@ class livecd_stage1(StageBase):
self.settings["action_sequence"]=["unpack","unpack_snapshot",\

"config_profile_link","setup_confdir","portage_overlay",\

"bind","chroot_setup","setup_environment","build_packages",\
-   "unbind", "clean","clear_autoresume"]
+   "unbind", "clean"]
+   self.set_completion_action_sequences()
 
def set_spec_prefix(self):
self.settings["spec_prefix"]="livecd"

diff --git a/catalyst/targets/stage4.py b/catalyst/targets/stage4.py
index 857976b..fdc8465 100644
--- a/catalyst/targets/stage4.py
+++ b/catalyst/targets/stage4.py
@@ -30,9 +30,4 @@ class stage4(StageBase):
"build_kernel","bootloader","root_overlay","fsscript",\

"preclean","rcupdate","unmerge","unbind","remove","empty",\
"clean"]
-
-#  if "TARBALL" in self.settings or \
-#  "fetch" not in self.settings['options']:
-   if "fetch" not in self.settings['options']:
-   self.settings["action_sequence"].append("capture")
-   self.settings["action_sequence"].append("clear_autoresume")
+   self.set_completion_action_sequences()

[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/, catalyst/, /

2015-10-06 Thread Mike Frysinger 
commit: 228211c82aeb92e9aa346e4c5bfa53a2a0cf95b8
Author: Mike Frysinger  gentoo  org>
AuthorDate: Tue Oct  6 04:06:05 2015 +
Commit: Mike Frysinger  gentoo  org>
CommitDate: Tue Oct  6 04:06:05 2015 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=228211c8

lint: fix up unused variables

This variables aren't actually used which the linter likes to warn about,
so add a _ prefix to their name to silence the warnings.

 catalyst/base/resume.py | 2 +-
 catalyst/main.py| 2 +-
 catalyst/support.py | 2 +-
 catalyst/targets/netboot.py | 5 +++--
 setup.py| 2 +-
 5 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/catalyst/base/resume.py b/catalyst/base/resume.py
index 608f574..e3f03e7 100644
--- a/catalyst/base/resume.py
+++ b/catalyst/base/resume.py
@@ -23,7 +23,7 @@ class AutoResumeError(Exception):
def __init__(self, message, print_traceback=False):
if message:
if print_traceback:
-   (type,value)=sys.exc_info()[:2]
+   (_type, value) = sys.exc_info()[:2]
if value!=None:
print
print "Traceback values found.  
listing..."

diff --git a/catalyst/main.py b/catalyst/main.py
index dfa0609..04f689e 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -182,7 +182,7 @@ def main():
 
# parse out the command line arguments
try:
-   opts,args = getopt.getopt(sys.argv[1:], "apPThvdc:C:f:FVs:", 
["purge", "purgeonly", "purgetmponly", "help", "version", "debug",\
+   opts, _args = getopt.getopt(sys.argv[1:], "apPThvdc:C:f:FVs:", 
["purge", "purgeonly", "purgetmponly", "help", "version", "debug",
"clear-autoresume", "config=", "cli=", "file=", 
"fetch", "verbose","snapshot="])
 
except getopt.GetoptError:

diff --git a/catalyst/support.py b/catalyst/support.py
index 5563a15..a879eaf 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -88,7 +88,7 @@ class CatalystError(Exception):
def __init__(self, message, print_traceback=False):
if message:
if print_traceback:
-   (type,value)=sys.exc_info()[:2]
+   (_type, value) = sys.exc_info()[:2]
if value!=None:
print
print "Traceback values found.  
listing..."

diff --git a/catalyst/targets/netboot.py b/catalyst/targets/netboot.py
index c41ed59..b0e322c 100644
--- a/catalyst/targets/netboot.py
+++ b/catalyst/targets/netboot.py
@@ -31,11 +31,12 @@ class netboot(StageBase):
self.required_values=[]
 
try:
+   # XXX: This code does nothing because the for loop 
below is disabled.
if "netboot/packages" in addlargs:
if type(addlargs["netboot/packages"]) == 
types.StringType:
-   loopy=[addlargs["netboot/packages"]]
+   _loopy = [addlargs["netboot/packages"]]
else:
-   loopy=addlargs["netboot/packages"]
+   _loopy = addlargs["netboot/packages"]
 
#   for x in loopy:
#   
self.required_values.append("netboot/packages/"+x+"/files")

diff --git a/setup.py b/setup.py
index feca894..27ed2de 100755
--- a/setup.py
+++ b/setup.py
@@ -34,7 +34,7 @@ def _files(prefix, root):
 
Yielding `(target_dir, (file_source_paths, ...))` tuples.
"""
-   for dirpath, dirnames, filenames in _os.walk(root):
+   for dirpath, _dirnames, filenames in _os.walk(root):
reldir = _os.path.relpath(dirpath, root)
install_directory = _posix_path(
_os.path.join(prefix, reldir))

[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/

2015-10-06 Thread Mike Frysinger 
commit: e10cdc703dddf4caa7990674a59ecdb0e40a8807
Author: Mike Frysinger  gentoo  org>
AuthorDate: Tue Oct  6 04:11:56 2015 +
Commit: Mike Frysinger  gentoo  org>
CommitDate: Tue Oct  6 04:11:56 2015 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=e10cdc70

lint: fix mixed indentation to only use tabs

 catalyst/base/stagebase.py   | 4 ++--
 catalyst/targets/netboot2.py | 8 
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 432ad5c..a9e7848 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -24,9 +24,9 @@ from catalyst.fileops import ensure_dirs, pjoin
 from catalyst.base.resume import AutoResume
 
 if sys.version_info[0] >= 3:
-py_input = input
+   py_input = input
 else:
-py_input = raw_input
+   py_input = raw_input
 
 
 class StageBase(TargetBase, ClearBase, GenBase):

diff --git a/catalyst/targets/netboot2.py b/catalyst/targets/netboot2.py
index 5de1a74..e509cf9 100644
--- a/catalyst/targets/netboot2.py
+++ b/catalyst/targets/netboot2.py
@@ -80,10 +80,10 @@ class netboot2(StageBase):
 
for x in loopy:
if "netboot2/packages/"+x+"/files" in 
self.settings:
-   if 
type(self.settings["netboot2/packages/"+x+"/files"]) == types.ListType:
-   
myfiles.extend(self.settings["netboot2/packages/"+x+"/files"])
-   else:
-   
myfiles.append(self.settings["netboot2/packages/"+x+"/files"])
+   if 
type(self.settings["netboot2/packages/"+x+"/files"]) == types.ListType:
+   
myfiles.extend(self.settings["netboot2/packages/"+x+"/files"])
+   else:
+   
myfiles.append(self.settings["netboot2/packages/"+x+"/files"])
 
if "netboot2/extra_files" in self.settings:
if type(self.settings["netboot2/extra_files"]) 
== types.ListType: