[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/, catalyst/
commit: b3c917f7fa73d11c69b7e55dc7a00bc18a18edc7
Author: Matt Turner gentoo org>
AuthorDate: Wed Jun 9 06:17:31 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Thu Jun 10 04:23:54 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b3c917f7
catalyst: Replace snakeoil's locks with fasteners
To no great surprise, the existing locking was broken. For example,
clear_chroot() releases the lock. It is called by unpack(), which is
part of prepare_sequence. The result is that the whole build could be
done without holding the lock.
Just lock around run(). It's not apparent that finer-grained locking
does anything for us.
Bug: https://bugs.gentoo.org/791583
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/clearbase.py | 2 --
catalyst/base/stagebase.py | 10 +++-
catalyst/lock.py | 58
catalyst/targets/snapshot.py | 6 ++---
4 files changed, 7 insertions(+), 69 deletions(-)
diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py
index dcf6c523..6218330e 100644
--- a/catalyst/base/clearbase.py
+++ b/catalyst/base/clearbase.py
@@ -27,12 +27,10 @@ class ClearBase():
self.resume.clear_all(remove=True)
def clear_chroot(self):
-self.chroot_lock.unlock()
log.notice('Clearing the chroot path ...')
clear_dir(self.settings["chroot_path"], mode=0o755)
def remove_chroot(self):
-self.chroot_lock.unlock()
log.notice('Removing the chroot path ...')
clear_dir(self.settings["chroot_path"], mode=0o755, remove=True)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 448d6265..10cffd4c 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -8,6 +8,7 @@ import sys
from pathlib import Path
+import fasteners
import libmount
import toml
@@ -25,7 +26,6 @@ from catalyst.support import (CatalystError, file_locate,
normpath,
from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
from catalyst.base.genbase import GenBase
-from catalyst.lock import LockDir, LockInUse
from catalyst.fileops import ensure_dirs, clear_dir, clear_path
from catalyst.base.resume import AutoResume
@@ -36,9 +36,6 @@ def run_sequence(sequence):
sys.stdout.flush()
try:
func()
-except LockInUse:
-log.error('Unable to aquire the lock...')
-return False
except Exception:
log.error('Exception running action sequence %s',
func.__name__, exc_info=True)
@@ -478,7 +475,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
"""
self.settings["chroot_path"] = normpath(self.settings["storedir"] +
"/tmp/" +
self.settings["target_subpath"].rstrip('/'))
-self.chroot_lock = LockDir(self.settings["chroot_path"])
def set_autoresume_path(self):
self.settings["autoresume_path"] = normpath(pjoin(
@@ -1366,8 +1362,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
pass
def run(self):
-self.chroot_lock.write_lock()
+with fasteners.InterProcessLock(self.settings["chroot_path"] +
'.lock'):
+return self._run()
+def _run(self):
if "clear-autoresume" in self.settings["options"]:
self.clear_autoresume()
diff --git a/catalyst/lock.py b/catalyst/lock.py
deleted file mode 100644
index e31745b2..
--- a/catalyst/lock.py
+++ /dev/null
@@ -1,58 +0,0 @@
-
-import os
-
-from contextlib import contextmanager
-
-from snakeoil import fileutils
-from snakeoil import osutils
-from catalyst.fileops import ensure_dirs
-
-
-LockInUse = osutils.LockException
-
-class Lock:
-"""
-A fnctl-based filesystem lock
-"""
-def __init__(self, lockfile):
-fileutils.touch(lockfile, mode=0o664)
-os.chown(lockfile, uid=-1, gid=250)
-self.lock = osutils.FsLock(lockfile)
-
-def read_lock(self):
-self.lock.acquire_read_lock()
-
-def write_lock(self):
-self.lock.acquire_write_lock()
-
-def unlock(self):
-# Releasing a write lock is the same as a read lock.
-self.lock.release_write_lock()
-
-class LockDir(Lock):
-"""
-A fnctl-based filesystem lock in a directory
-"""
-def __init__(self, lockdir):
-ensure_dirs(lockdir)
-lockfile = os.path.join(lockdir, '.catalyst_lock')
-
-Lock.__init__(self, lockfile)
-
-@contextmanager
-def read_lock(filename):
-lock = Lock(filename)
-lock.read_lock()
-try:
-yield
-finally:
-lock.unlock()
-
-@contextmanager
-def write_lock(filename):
-lock = Lock(filename)
-lock.write_lock()
-try:
-yield
-finally:
-lock.unlock()
diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py
index b494575a..ef68765d 1006
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/
commit: 440a379ae94e6d7df26291bf83657288d1406e98
Author: Matt Turner gentoo org>
AuthorDate: Sun Jan 17 23:53:21 2021 +
Commit: Matt Turner gentoo org>
CommitDate: Sat Jan 23 16:19:07 2021 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=440a379a
catalyst: Store references to functions
... rather than their names. This makes it possible for tooling to
understand the code structure better.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py| 69 ---
catalyst/targets/embedded.py | 34 +--
catalyst/targets/livecd_stage1.py | 18 +-
catalyst/targets/livecd_stage2.py | 46 +-
catalyst/targets/netboot.py | 34 +--
catalyst/targets/stage1.py| 12 +++
catalyst/targets/stage4.py| 36 ++--
7 files changed, 125 insertions(+), 124 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index ed4d1227..447e073d 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -30,6 +30,23 @@ from catalyst.fileops import ensure_dirs, clear_dir,
clear_path
from catalyst.base.resume import AutoResume
+def run_sequence(sequence):
+for func in sequence:
+log.notice('--- Running action sequence: %s', func.__name__)
+sys.stdout.flush()
+try:
+func()
+except LockInUse:
+log.error('Unable to aquire the lock...')
+return False
+except Exception:
+log.error('Exception running action sequence %s',
+ func.__name__, exc_info=True)
+return False
+
+return True
+
+
class StageBase(TargetBase, ClearBase, GenBase):
"""
This class does all of the chroot setup, copying of files, etc. It is
@@ -475,39 +492,39 @@ class StageBase(TargetBase, ClearBase, GenBase):
Or it calls the normal set_action_sequence() for the target stage.
"""
if "purgeonly" in self.settings["options"]:
-self.build_sequence.append("remove_chroot")
+self.build_sequence.append(self.remove_chroot)
return
self.set_action_sequence()
def set_action_sequence(self):
"""Set basic stage1, 2, 3 action sequences"""
self.prepare_sequence.extend([
-"unpack",
-"setup_confdir",
-"portage_overlay",
+self.unpack,
+self.setup_confdir,
+self.portage_overlay,
])
self.build_sequence.extend([
-"bind",
-"chroot_setup",
-"setup_environment",
-"run_local",
-"preclean",
+self.bind,
+self.chroot_setup,
+self.setup_environment,
+self.run_local,
+self.preclean,
])
self.finish_sequence.extend([
-"clean",
+self.clean,
])
self.set_completion_action_sequences()
def set_completion_action_sequences(self):
if "fetch" not in self.settings["options"]:
-self.finish_sequence.append("capture")
+self.finish_sequence.append(self.capture)
if "keepwork" in self.settings["options"]:
-self.finish_sequence.append("clear_autoresume")
+self.finish_sequence.append(self.clear_autoresume)
elif "seedcache" in self.settings["options"]:
-self.finish_sequence.append("remove_autoresume")
+self.finish_sequence.append(self.remove_autoresume)
else:
-self.finish_sequence.append("remove_autoresume")
-self.finish_sequence.append("remove_chroot")
+self.finish_sequence.append(self.remove_autoresume)
+self.finish_sequence.append(self.remove_chroot)
def set_use(self):
use = self.settings["spec_prefix"] + "/use"
@@ -1308,22 +1325,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
log.debug('setup_environment(); env = %r', self.env)
-def run_sequence(self, sequence):
-for func in sequence:
-log.notice('--- Running action sequence: %s', func)
-sys.stdout.flush()
-try:
-getattr(self, func)()
-except LockInUse:
-log.error('Unable to aquire the lock...')
-return False
-except Exception:
-log.error('Exception running action sequence %s',
- func, exc_info=True)
-return False
-
-return True
-
def run(self):
self.chroot_lock.write_lock()
@@ -1342,14 +1343,14 @@ class StageBase(TargetBase, ClearBase, GenBase):
log.info('StageBase: run() purge')
self.purge()
-if not self.run_sequence(self.prepare_sequence):
+if not run_sequence(self.prepare_sequence):
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/
commit: dfc4ed516b839bbc225cc7459e8c3085970801f6
Author: Matt Turner gentoo org>
AuthorDate: Thu Oct 29 00:28:24 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=dfc4ed51
catalyst: Move action_sequence out of self.settings[]
This self.settings[] dictionary is very similar to the god object
anti-pattern. Moving action_sequence out it starts us down the road of
fixing this.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py| 17 +
catalyst/targets/embedded.py | 2 +-
catalyst/targets/livecd_stage1.py | 2 +-
catalyst/targets/livecd_stage2.py | 6 +++---
catalyst/targets/netboot.py | 2 +-
catalyst/targets/stage1.py| 12 ++--
catalyst/targets/stage4.py| 2 +-
7 files changed, 22 insertions(+), 21 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 73eacfbe..801df2fb 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -64,6 +64,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
"portage_overlay",
"portage_prefix",
])
+self.action_sequence = []
self.set_valid_build_kernel_vars(addlargs)
TargetBase.__init__(self, myspec, addlargs)
@@ -477,13 +478,13 @@ class StageBase(TargetBase, ClearBase, GenBase):
Or it calls the normal set_action_sequence() for the target stage.
"""
if "purgeonly" in self.settings["options"]:
-self.settings["action_sequence"] = ["remove_chroot"]
+self.action_sequence = ["remove_chroot"]
return
self.set_action_sequence()
def set_action_sequence(self):
"""Set basic stage1, 2, 3 action sequences"""
-self.settings['action_sequence'] = [
+self.action_sequence = [
"unpack",
"setup_confdir",
"portage_overlay",
@@ -499,14 +500,14 @@ class StageBase(TargetBase, ClearBase, GenBase):
def set_completion_action_sequences(self):
if "fetch" not in self.settings["options"]:
-self.settings["action_sequence"].append("capture")
+self.action_sequence.append("capture")
if "keepwork" in self.settings["options"]:
-self.settings["action_sequence"].append("clear_autoresume")
+self.action_sequence.append("clear_autoresume")
elif "seedcache" in self.settings["options"]:
-self.settings["action_sequence"].append("remove_autoresume")
+self.action_sequence.append("remove_autoresume")
else:
-self.settings["action_sequence"].append("remove_autoresume")
-self.settings["action_sequence"].append("remove_chroot")
+self.action_sequence.append("remove_autoresume")
+self.action_sequence.append("remove_chroot")
def set_use(self):
use = self.settings["spec_prefix"] + "/use"
@@ -1380,7 +1381,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.purge()
failure = False
-for x in self.settings["action_sequence"]:
+for x in self.action_sequence:
log.notice('--- Running action sequence: %s', x)
sys.stdout.flush()
try:
diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py
index 99739512..3899cf1b 100644
--- a/catalyst/targets/embedded.py
+++ b/catalyst/targets/embedded.py
@@ -41,7 +41,7 @@ class embedded(StageBase):
StageBase.__init__(self, spec, addlargs)
def set_action_sequence(self):
-self.settings['action_sequence'] = [
+self.action_sequence = [
"unpack",
"config_profile_link",
"setup_confdir",
diff --git a/catalyst/targets/livecd_stage1.py
b/catalyst/targets/livecd_stage1.py
index f0b6be8b..b8c26cb1 100644
--- a/catalyst/targets/livecd_stage1.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -23,7 +23,7 @@ class livecd_stage1(StageBase):
StageBase.__init__(self, spec, addlargs)
def set_action_sequence(self):
-self.settings['action_sequence'] = [
+self.action_sequence = [
"unpack",
"config_profile_link",
"setup_confdir",
diff --git a/catalyst/targets/livecd_stage2.py
b/catalyst/targets/livecd_stage2.py
index 88c0d95c..cac16b6e 100644
--- a/catalyst/targets/livecd_stage2.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -87,7 +87,7 @@ class livecd_stage2(StageBase):
print_traceback=True)
def set_action_sequence(self):
-self.settings['action_sequence'] = [
+self.action_sequence = [
"unpack",
"config_profile_link",
"setup_confdir",
@@ -99,7 +99,7 @@ class livecd_stage2(StageBase):
"build_kernel"
]
if "fetch" not in self.settings["
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/
commit: 4b5ff905a7ad2ffe1ed8c863b91e9d0ce6981f5f
Author: Matt Turner gentoo org>
AuthorDate: Thu Oct 22 20:37:54 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Oct 30 22:40:52 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4b5ff905
catalyst: Drop unbind()
mount_namespaces(7) says
A mount ceases to be a member of a peer group when either the
mount is explicitly unmounted, or when the mount is implicitly
unmounted because a mount namespace is removed (because it has
no more member processes).
As a result, we can rely on exiting the mount namespace to unmount the
bind mounts.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py| 44 +--
catalyst/targets/embedded.py | 1 -
catalyst/targets/livecd_stage1.py | 1 -
catalyst/targets/livecd_stage2.py | 2 --
catalyst/targets/netboot.py | 3 ---
catalyst/targets/stage4.py| 1 -
6 files changed, 1 insertion(+), 51 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index b9c220d0..a75dbdf9 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -498,7 +498,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
"setup_environment",
"run_local",
"preclean",
-"unbind",
])
self.finish_sequence.extend([
"clean",
@@ -853,40 +852,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
fstype=fstype, options=options)
cxt.mount()
except OSError as e:
-self.unbind()
raise CatalystError(f"Couldn't mount: {source}, {e.strerror}")
-def unbind(self):
-chroot_path = self.settings["chroot_path"]
-umount_failed = False
-
-# Unmount in reverse order
-for target in [Path(chroot_path + self.mount[x]['target'])
- for x in reversed(self.mount)
- if self.mount[x]['enable']]:
-if not target.exists():
-log.debug('%s does not exist. Skipping', target)
-continue
-
-if not ismount(target):
-log.debug('%s is not a mount point. Skipping', target)
-continue
-
-try:
-cxt = libmount.Context(target=str(target))
-cxt.umount()
-except OSError as e:
-log.warning("Couldn't umount: %s, %s", target,
-e.strerror)
-umount_failed = True
-
-if umount_failed:
-# if any bind mounts really failed, then we need to raise
-# this to potentially prevent an upcoming bash stage cleanup script
-# from wiping our bind mounts.
-raise CatalystError(
-"Couldn't umount one or more bind-mounts; aborting for
safety.")
-
def chroot_setup(self):
self.makeconf = read_makeconf(normpath(self.settings["chroot_path"] +
self.settings["make_conf"]))
@@ -1190,7 +1157,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
env=self.env)
self.resume.enable("remove")
except:
-self.unbind()
raise
def preclean(self):
@@ -1206,7 +1172,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.resume.enable("preclean")
except:
-self.unbind()
raise CatalystError("Build failed, could not execute preclean")
def capture(self):
@@ -1269,7 +1234,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings['controller_file'])
except CatalystError:
-self.unbind()
raise CatalystError("Stage build aborting due to error.",
print_traceback=False)
@@ -1374,7 +1338,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
env=self.env)
log.info('unmerge shell script')
except CatalystError:
-self.unbind()
raise
self.resume.enable("unmerge")
@@ -1449,7 +1412,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
fileutils.touch(build_packages_resume)
self.resume.enable("build_packages")
except CatalystError:
-self.unbind()
raise CatalystError(
self.settings["spec_prefix"] +
"build aborting due to error.")
@@ -1473,7 +1435,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
self._build_kernel(kname=kname)
self.resume.enable("build_kernel")
except CatalystError:
-self.unbind()
[gentoo-commits] proj/catalyst:master commit in: /, catalyst/targets/, catalyst/base/, catalyst/, doc/
commit: 98cf07d2ca4da1f88e213a520095bfccc9c81ffc Author: Matt Turner gentoo org> AuthorDate: Fri Apr 17 23:31:52 2020 + Commit: Matt Turner gentoo org> CommitDate: Thu Apr 30 22:54:49 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=98cf07d2 catalyst: Make and use squashfs snapshots There were a number of problems with catalyst's snapshot system. It was built around using the build system's portdir and had no control over what was in that portdir or when it was updated. As a result, when a stage build failed, it was difficult to tell what the snapshot consistet of precisely or whether it contained a particular recent fix. With snapcache disabled, ebuild repo snapshots were tar'd and compressed and then unpacked into the stage chroot which is an unnecessarily expensive process. Moreover, a porttree has more than 100k small files, which are stored extremely inefficiently on most file systems—a whole porttree is usually around 700M on disk. Just removing all of those files during the cleaning stage is an expensive operation. Instead, we make a compressed squashfs image and mount it in the build chroot. The porttree has many duplicate files, and squashfs deduplicates the files and then compresses, so the result is very efficiently packed: ~38M with gzip -9 compression. The snapshot target has been modified to generate a squashfs image from a bare ebuild git repo. Piping git-archive to tar2sqfs generates the squashfs image in less than 10 seconds on a modern system. The git repo is fetched with --depth=1 to minize bandwidth and disk usage, and git gc is run after fetch to minimize disk usage. Storage requirements for the git ebuild repo with metadata are ~70M. The squashfs snapshot is stored in /var/tmp/catalyst/snapshots/ by default with a name -.sqfs. With this convention, we know the exact point in history that the snapshot was taken. The catalyst-auto script can use the sha1 to get a deterministic timestamp, so that it is independent on when `catalyst -s` was run, but is instead the timestamp of the commit date of the repo's git SHA1. Signed-off-by: Matt Turner gentoo.org> README| 3 +- catalyst/base/stagebase.py| 88 catalyst/base/targetbase.py | 16 +++- catalyst/defaults.py | 2 +- catalyst/main.py | 24 +++--- catalyst/targets/embedded.py | 1 - catalyst/targets/livecd_stage1.py | 1 - catalyst/targets/livecd_stage2.py | 1 - catalyst/targets/netboot.py | 1 - catalyst/targets/snapshot.py | 165 +++--- catalyst/targets/stage4.py| 1 - doc/catalyst-config.5.txt | 13 +-- doc/catalyst-spec.5.txt | 4 +- 13 files changed, 137 insertions(+), 183 deletions(-) diff --git a/README b/README index 1a039fca..1cceb63e 100644 --- a/README +++ b/README @@ -18,8 +18,9 @@ Requirements === - Python 3.6 or greater -- An ebuild repository snapshot (or an ebuild tree to create one) - A generic stage3 tarball for your architecture +- A squashfs ebuild repository snapshot + - Or an ebuild git repo with sys-fs/squashfs-tools-ng and dev-vcs/git What is catalyst? diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 9aecf013..41da97b3 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -35,7 +35,7 @@ class StageBase(TargetBase, ClearBase, GenBase): self.required_values |= frozenset([ "profile", "rel_type", -"snapshot", +"snapshot_treeish", "source_subpath", "subarch", "target", @@ -149,7 +149,7 @@ class StageBase(TargetBase, ClearBase, GenBase): self.set_source_subpath() # Set paths -self.set_snapshot_path() +self.set_snapshot() self.set_root_path() self.set_source_path() self.set_chroot_path() @@ -191,9 +191,8 @@ class StageBase(TargetBase, ClearBase, GenBase): # Setup our mount points. self.mount = MOUNT_DEFAULTS.copy() -# Always unpack snapshot tarball -self.mount['portdir']['enable'] = False - +self.mount['portdir']['source'] = self.snapshot +self.mount['portdir']['target'] = self.settings['repo_basedir'] + '/' + self.settings['repo_name'] self.mount['distdir']['source'] = self.settings['distdir'] self.mount["distdir"]['target'] = self.settings['target_distdir'] @@ -435,21 +434,11 @@ class StageBase(TargetBase, ClearBase, GenBase): self.settings["destpath"] = normpath(self.settings["chroot_path"]) def set_cleanables(self): -self.settings["cleanables"] = ["/etc/resolv.conf", "/var/tmp/*", "/tmp/*", - self.settings["repo_basedir"] + "/" + - self.sett
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/
commit: 3b0ff52abd04aa213c844b6d716d9edbe809dc65
Author: Matt Turner gentoo org>
AuthorDate: Sat Apr 18 20:20:05 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Wed Apr 22 00:22:24 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=3b0ff52a
catalyst: Remove code to mount /proc in stage1root
Added in commit f75bb07f3cec (Added two patches from Joshua Kinard ...),
mounting /proc in stage1root purports to fix an issue in the preclean
stage. I have no idea what that issue was, but it doesn't occur now.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 5 -
catalyst/targets/stage1.py | 11 +--
2 files changed, 1 insertion(+), 15 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 60cb891a..f8c1611e 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -213,8 +213,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.mounts.append("shm")
self.mounts.append("run")
-self.set_mounts()
-
# Configure any user specified options (either in catalyst.conf or on
# the command line).
if "pkgcache" in self.settings["options"]:
@@ -586,9 +584,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
def set_stage_path(self):
self.settings["stage_path"] = normpath(self.settings["chroot_path"])
-def set_mounts(self):
-pass
-
def set_packages(self):
pass
diff --git a/catalyst/targets/stage1.py b/catalyst/targets/stage1.py
index daa0b854..93c62877 100644
--- a/catalyst/targets/stage1.py
+++ b/catalyst/targets/stage1.py
@@ -5,7 +5,7 @@ stage1 target
from catalyst import log
from catalyst.support import normpath
-from catalyst.fileops import ensure_dirs, move_path
+from catalyst.fileops import move_path
from catalyst.base.stagebase import StageBase
@@ -78,15 +78,6 @@ class stage1(StageBase):
"If you break it, you buy it. Don't complain to us about
it.\n"
"Don't say we did not warn you.")
-def set_mounts(self):
-# stage_path/proc probably doesn't exist yet, so create it
-ensure_dirs(self.settings["stage_path"]+"/proc")
-
-# alter the mount mappings to bind mount proc onto it
-self.mounts.append("stage1root/proc")
-self.target_mounts["stage1root/proc"] = "/tmp/stage1root/proc"
-self.mountmap["stage1root/proc"] = "/proc"
-
def set_completion_action_sequences(self):
'''Override function for stage1
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/
commit: 56ee6814b2dade3d0c2b12ede3582a3e479aa532
Author: Matt Turner gentoo org>
AuthorDate: Sat Apr 11 22:23:54 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Mon Apr 13 20:35:35 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=56ee6814
catalyst: Clean assignments of {valid/required}_values
Making them class variables will allow us to verify that all subclasses
of TargetBase have defined them (in the next commit).
While we're moving them, change them to frozensets, since testing
membership is the thing they're useed for.
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py| 68 ++-
catalyst/targets/embedded.py | 24 ++
catalyst/targets/livecd_stage1.py | 11 ---
catalyst/targets/livecd_stage2.py | 48 +++
catalyst/targets/netboot.py | 29 -
catalyst/targets/snapshot.py | 11 +--
catalyst/targets/stage1.py| 10 --
catalyst/targets/stage2.py| 7 ++--
catalyst/targets/stage3.py| 5 +--
catalyst/targets/stage4.py| 26 +++
10 files changed, 159 insertions(+), 80 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 829f9fb7..a90fc843 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -27,16 +27,36 @@ class StageBase(TargetBase, ClearBase, GenBase):
the driver class for pretty much everything that Catalyst does.
"""
def __init__(self,myspec,addlargs):
- self.required_values.extend(["version_stamp", "target",
"subarch",
- "rel_type", "profile", "snapshot", "source_subpath"])
-
- self.valid_values.extend(["version_stamp", "target", "subarch",
- "rel_type", "profile", "snapshot", "source_subpath",
- "portage_confdir", "portage_prefix", "portage_overlay",
- "cflags", "cxxflags", "fcflags", "fflags", "ldflags",
"asflags",
- "common_flags", "cbuild", "hostuse", "catalyst_use",
- "distcc_hosts", "makeopts", "pkgcache_path",
"kerncache_path",
- "compression_mode", "decompression_mode"])
+ self.required_values |= frozenset([
+ "profile",
+ "rel_type",
+ "snapshot",
+ "source_subpath",
+ "subarch",
+ "target",
+ "version_stamp",
+ ])
+ self.valid_values |= self.required_values | frozenset([
+ "asflags",
+ "catalyst_use",
+ "cbuild",
+ "cflags",
+ "common_flags",
+ "compression_mode",
+ "cxxflags",
+ "decompression_mode",
+ "distcc_hosts",
+ "fcflags",
+ "fflags",
+ "hostuse",
+ "kerncache_path",
+ "ldflags",
+ "makeopts",
+ "pkgcache_path",
+ "portage_confdir",
+ "portage_overlay",
+ "portage_prefix",
+ ])
self.set_valid_build_kernel_vars(addlargs)
TargetBase.__init__(self, myspec, addlargs)
@@ -422,7 +442,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
def set_fsops(self):
if "fstype" in self.settings:
- self.valid_values.append("fsops")
+ self.valid_values |= {"fsops"}
if self.settings["spec_prefix"] + "/fsops" in
self.settings:
self.settings["fsops"] = \
self.settings[self.settings["spec_prefix"] + "/fsops"]
@@ -647,18 +667,20 @@ class StageBase(TargetBase, ClearBase, GenBase):
loopy = addlargs["boot/kernel"]
for x in loopy:
- self.valid_values.append("boot/kernel/" + x +
"/aliases")
- self.valid_values.append("boot/kernel/" + x +
"/config")
- self.valid_values.append("boot/kernel/" + x +
"/console")
- self.valid_values.append("boot/kernel/" + x +
"/extraversion")
- self.valid_values.append("boot/kernel/" + x +
"/gk_action")
- self.valid_values.append("boot/kernel/" + x +
"/gk_kernargs")
- self.valid_values.append("boot/kernel/" + x +
"/initramfs_overlay")
- self.valid_values.append("boot/kernel/"
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/
commit: b96f33fad30027609291efd4983396b3f4b29a58 Author: Matt Turner gentoo org> AuthorDate: Sat Apr 11 04:31:49 2020 + Commit: Matt Turner gentoo org> CommitDate: Sat Apr 11 04:31:49 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b96f33fa catalyst: Remove some dead code Signed-off-by: Matt Turner gentoo.org> catalyst/builder.py | 16 catalyst/targets/snapshot.py | 3 --- 2 files changed, 19 deletions(-) diff --git a/catalyst/builder.py b/catalyst/builder.py index 64e14c8d..fd8d5eb6 100644 --- a/catalyst/builder.py +++ b/catalyst/builder.py @@ -9,19 +9,3 @@ class generic(): Useful for building x86-on-amd64 and such. """ self.settings['CHROOT'] = 'setarch %s %s' % (arch, self.settings['CHROOT']) - - def mount_safety_check(self): - """ - Make sure that no bind mounts exist in chrootdir (to use before - cleaning the directory, to make sure we don't wipe the contents of - a bind mount - """ - pass - - def mount_all(self): - """do all bind mounts""" - pass - - def umount_all(self): - """unmount all bind mounts""" - pass diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py index c80d224f..b73135bc 100644 --- a/catalyst/targets/snapshot.py +++ b/catalyst/targets/snapshot.py @@ -33,9 +33,6 @@ class snapshot(TargetBase, GenBase): x=normpath(self.settings["storedir"]+"/snapshots") ensure_dirs(x) - def mount_safety_check(self): - pass - def run(self): if "purgeonly" in self.settings["options"]: self.purge()
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/, catalyst/
commit: 8ef6e85a57ecefb4eb7b4b50796dbb98ab1a3f2a
Author: Matt Turner gentoo org>
AuthorDate: Fri Apr 10 07:40:48 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Apr 10 07:42:08 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=8ef6e85a
catalyst: Drop BSD remnants
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/clearbase.py | 4 ++--
catalyst/base/resume.py| 2 +-
catalyst/base/stagebase.py | 19 ++-
catalyst/builder.py| 3 +--
catalyst/defaults.py | 14 --
catalyst/fileops.py| 6 +-
catalyst/main.py | 17 +++--
catalyst/targets/stage1.py | 14 +-
8 files changed, 23 insertions(+), 56 deletions(-)
diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py
index f1d4d1ba..8dfffb06 100644
--- a/catalyst/base/clearbase.py
+++ b/catalyst/base/clearbase.py
@@ -30,13 +30,13 @@ class ClearBase():
def clear_chroot(self):
self.chroot_lock.unlock()
log.notice('Clearing the chroot path ...')
- clear_dir(self.settings["chroot_path"], mode=0o755,
chg_flags=True)
+ clear_dir(self.settings["chroot_path"], mode=0o755)
def remove_chroot(self):
self.chroot_lock.unlock()
log.notice('Removing the chroot path ...')
- clear_dir(self.settings["chroot_path"], mode=0o755,
chg_flags=True, remove=True)
+ clear_dir(self.settings["chroot_path"], mode=0o755, remove=True)
def clear_packages(self, remove=False):
diff --git a/catalyst/base/resume.py b/catalyst/base/resume.py
index 7732dd5d..a0770b9d 100644
--- a/catalyst/base/resume.py
+++ b/catalyst/base/resume.py
@@ -126,7 +126,7 @@ class AutoResume():
@remove: boolean, passed through to clear_dir()
@return boolean
'''
- if clear_dir(self.basedir, mode=0o755, chg_flags=True,
remove=remove):
+ if clear_dir(self.basedir, mode=0o755, remove=remove):
self._points = {}
return True
return False
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 320e9d53..021f3beb 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -224,10 +224,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["snapshot_cache_path"],
self.settings["repo_name"],
]))
- if os.uname()[0] == "Linux":
- self.mounts.append("devpts")
- self.mounts.append("shm")
- self.mounts.append("run")
+ self.mounts.append("devpts")
+ self.mounts.append("shm")
+ self.mounts.append("run")
self.set_mounts()
@@ -983,16 +982,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
elif src == "tmpfs":
_cmd = ['mount', '-t', 'tmpfs', src, target]
else:
- if os.uname()[0] == "FreeBSD":
- if src == "/dev":
- _cmd = ['mount', '-t', 'devfs',
'none', target]
- else:
- _cmd = ['mount_nullfs', src,
target]
+ if src == "shmfs":
+ _cmd = ['mount', '-t', 'tmpfs', '-o',
'noexec,nosuid,nodev', 'shm', target]
else:
- if src == "shmfs":
- _cmd = ['mount', '-t', 'tmpfs',
'-o', 'noexec,nosuid,nodev', 'shm', target]
- else:
- _cmd = ['mount', '--bind', src,
target]
+ _cmd = ['mount', '--bind', src, target]
if _cmd:
log.debug('bind(); _cmd = %s', _cmd)
cmd(_cmd, env=self.env, fail_func=self.unbind)
diff --git a/catalyst/builder.py b/catalyst/builder.py
index 5d7f76c1..e9f468d7 100644
--- a/catalyst/builder.py
+++ b/catalyst/builder.py
@@ -10,8 +10,7 @@ class generic():
Useful for building x86-on-amd64 and such.
"""
- if os.uname()[0] == 'Linux':
- self.settings['CHROOT'] = 'setarch %s %s' % (arch,
self.settings['CHROOT'])
+ self.settings['CHROOT'] = 'setarch %s %s' % (arch,
self.settings['CHROOT'])
def mount_safety_check(self):
"""
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 5b4ae83e..bcb59796 100644
--- a/catalyst/defaul
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/, catalyst/base/
commit: f6650276525e49b6b1a5bcd9bb6eefb0c0820a2c
Author: Matt Turner gentoo org>
AuthorDate: Fri Apr 10 07:30:36 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Fri Apr 10 07:30:36 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f6650276
catalyst: Drop some dead code and comments
Signed-off-by: Matt Turner gentoo.org>
catalyst/base/stagebase.py | 3 ---
catalyst/config.py | 8
catalyst/defaults.py | 3 ---
catalyst/targets/snapshot.py | 2 +-
4 files changed, 1 insertion(+), 15 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 72ef57d3..53d39536 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1388,7 +1388,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
for x in list(self.settings):
log.debug('setup_environment(); processing: %s', x)
if x == "options":
- #self.env['clst_' + x] = '
'.join(self.settings[x])
for opt in self.settings[x]:
self.env['clst_' + opt.upper()] = "true"
continue
@@ -1398,13 +1397,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
varname = varname.replace(".", "_")
if isinstance(self.settings[x], str):
# Prefix to prevent namespace clashes
- #os.environ[varname] = self.settings[x]
if "path" in x:
self.env[varname] =
self.settings[x].rstrip("/")
else:
self.env[varname] = self.settings[x]
elif isinstance(self.settings[x], list):
- #os.environ[varname] = '
'.join(self.settings[x])
self.env[varname] = ' '.join(self.settings[x])
elif isinstance(self.settings[x], bool):
if self.settings[x]:
diff --git a/catalyst/config.py b/catalyst/config.py
index a3a7200a..9f184ed5 100644
--- a/catalyst/config.py
+++ b/catalyst/config.py
@@ -45,7 +45,6 @@ class ParserBase(object):
cur_array = []
trailing_comment=re.compile(r'\s*#.*$')
- #white_space=re.compile('\s+')
for x, myline in enumerate(self.lines):
myline = myline.strip()
@@ -63,10 +62,6 @@ class ParserBase(object):
mobjs = myline.split(self.key_value_separator,
1)
mobjs[1] = mobjs[1].strip().strip('"')
-# # Check that this key doesn't exist already in
the spec
-# if mobjs[0] in values:
-# raise Exception("You have a duplicate
key (" + mobjs[0] + ") in your spec. Please fix it")
-
# Start a new array using the first element of
mobjs
cur_array = [mobjs[0]]
if mobjs[1]:
@@ -75,7 +70,6 @@ class ParserBase(object):
mobjs[1] = mobjs[1] % values
if self.multiple_values:
# split on white space creating
additional array elements
-# subarray =
white_space.split(mobjs[1])
subarray = mobjs[1].split()
cur_array += subarray
else:
@@ -84,8 +78,6 @@ class ParserBase(object):
# Else add on to the last key we were working on
else:
if self.multiple_values:
-# mobjs = white_space.split(myline)
-# cur_array += mobjs
cur_array += myline.split()
else:
raise CatalystError("Syntax error: %s"
% x, print_traceback=True)
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 72a8f56a..5b4ae83e 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -104,12 +104,10 @@ SOURCE_MOUNT_DEFAULTS = {
"run": "tmpfs",
}
-# legend: key: message
option_messages = {
"autoresume": "Autoresuming support enabled.",
"ccache": "Compiler cache support enabled.",
"clear-autoresume": "Cleaning autoresume flags support enabled.",
- #"compress": "Compression enabled.",
"distcc": "Distcc support enabled.",
"icecream": "Icecream compiler cluster support enable
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/, examples/, targets/netboot2/, targets/netboot/, ...
commit: ccddc3ae36ee4ea01db69176a993234aa56e17af
Author: Matt Turner gentoo org>
AuthorDate: Wed Apr 8 07:25:17 2020 +
Commit: Matt Turner gentoo org>
CommitDate: Thu Apr 9 18:47:23 2020 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=ccddc3ae
targets: Rename netboot2 -> netboot
Signed-off-by: Matt Turner gentoo.org>
catalyst/defaults.py | 3 +-
catalyst/targets/{netboot2.py => netboot.py} | 66 +++---
...etboot2_template.spec => netboot_template.spec} | 50
targets/{netboot2 => netboot}/controller.sh| 2 +-
targets/{netboot2 => netboot}/copyfile.sh | 0
targets/{netboot2 => netboot}/nb-busybox.cf| 0
targets/{netboot2 => netboot}/pkg.sh | 0
targets/support/kmerge.sh | 2 +-
.../{netboot2-final.sh => netboot-final.sh}| 0
9 files changed, 61 insertions(+), 62 deletions(-)
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 11f99070..72a8f56a 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -18,8 +18,7 @@ required_build_targets = ["targetbase",
"generic_stage_target"]
# new build types should be added here
valid_build_targets = ["stage1_target", "stage2_target", "stage3_target",
"stage4_target", "livecd_stage1_target", "livecd_stage2_target",
- "embedded_target", "snapshot_target",
- "netboot2_target"
+ "embedded_target", "snapshot_target", "netboot_target"
]
required_config_file_values = ["storedir", "sharedir", "distdir", "portdir"]
diff --git a/catalyst/targets/netboot2.py b/catalyst/targets/netboot.py
similarity index 72%
rename from catalyst/targets/netboot2.py
rename to catalyst/targets/netboot.py
index 87dada3b..675a6f79 100644
--- a/catalyst/targets/netboot2.py
+++ b/catalyst/targets/netboot.py
@@ -12,7 +12,7 @@ from catalyst.fileops import (ensure_dirs, clear_dir,
clear_path)
from catalyst.base.stagebase import StageBase
-class netboot2(StageBase):
+class netboot(StageBase):
"""
Builder class for a netboot build, version 2
"""
@@ -22,26 +22,26 @@ class netboot2(StageBase):
]
self.valid_values=self.required_values[:]
self.valid_values.extend([
- "netboot2/packages",
- "netboot2/use",
- "netboot2/extra_files",
- "netboot2/overlay",
- "netboot2/busybox_config",
- "netboot2/root_overlay",
- "netboot2/linuxrc"
+ "netboot/packages",
+ "netboot/use",
+ "netboot/extra_files",
+ "netboot/overlay",
+ "netboot/busybox_config",
+ "netboot/root_overlay",
+ "netboot/linuxrc"
])
try:
- if "netboot2/packages" in addlargs:
- if isinstance(addlargs['netboot2/packages'],
str):
- loopy=[addlargs["netboot2/packages"]]
+ if "netboot/packages" in addlargs:
+ if isinstance(addlargs['netboot/packages'],
str):
+ loopy=[addlargs["netboot/packages"]]
else:
- loopy=addlargs["netboot2/packages"]
+ loopy=addlargs["netboot/packages"]
for x in loopy:
-
self.valid_values.append("netboot2/packages/"+x+"/files")
+
self.valid_values.append("netboot/packages/"+x+"/files")
except:
- raise CatalystError("configuration error in
netboot2/packages.")
+ raise CatalystError("configuration error in
netboot/packages.")
StageBase.__init__(self,spec,addlargs)
self.settings["merge_path"]=normpath("/tmp/image/")
@@ -67,24 +67,24 @@ class netboot2(StageBase):
and self.resume.is_enabled("copy_files_to_image"):
log.notice('Resume point detected, skipping target path
setup operation...')
else:
- if "netboot2/packages" in self.settings:
- if
isinstance(self.settings['netboot2/packages'], str):
-
loopy=[self.settings["netboot2/packages"]]
+ if "netboot/packages" in self.settings:
+ if
isinstance(self.settings['netboot/packages'], str):
+
loopy=[self.settings["netboot/packages"]]
else:
-
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/, catalyst/base/
commit: 67af25c7824102b93db0d4016d6a9e101bbf6c09
Author: Mike Frysinger gentoo org>
AuthorDate: Thu May 19 19:18:29 2016 +
Commit: Mike Frysinger gentoo org>
CommitDate: Thu May 19 19:39:22 2016 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=67af25c7
clear_dir: make a bit more robust/flexible
Make it so clear_dir will recover from non-directory paths, and it will
always call ensure_dirs even if the path didn't exist previously. Most
callers want this behavior and call ensure_dirs themselves.
catalyst/base/stagebase.py| 9 ++---
catalyst/fileops.py | 28 +++-
catalyst/targets/livecd_stage2.py | 12 +++-
3 files changed, 24 insertions(+), 25 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index b6dd08d..6aa854b 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -18,7 +18,7 @@ from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
from catalyst.base.genbase import GenBase
from catalyst.lock import LockDir, LockInUse
-from catalyst.fileops import ensure_dirs, pjoin
+from catalyst.fileops import ensure_dirs, pjoin, clear_dir
from catalyst.base.resume import AutoResume
if sys.version_info[0] >= 3:
@@ -821,13 +821,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
cleanup_msg="Cleaning up invalid snapshot cache at
\n\t"+\
self.settings["snapshot_cache_path"]+\
" (this can take a long time)..."
- cleanup_errmsg="Error removing existing snapshot cache
directory."
if
self.settings["snapshot_path_hash"]==snapshot_cache_hash:
log.info('Valid snapshot cache, skipping unpack
of portage tree...')
unpack=False
else:
- cleanup_errmsg="Error removing existing snapshot
directory."
cleanup_msg=\
'Cleaning up existing portage tree (this can
take a long time)...'
unpack_info['destination'] = normpath(
@@ -847,10 +845,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.snapcache_lock.write_lock()
if os.path.exists(target_portdir):
log.info('%s', cleanup_msg)
- cleanup_cmd = "rm -rf " + target_portdir
- log.info('unpack() cleanup_cmd = %s',
cleanup_cmd)
- cmd(cleanup_cmd,cleanup_errmsg,env=self.env)
- ensure_dirs(target_portdir, mode=0o755)
+ clear_dir(target_portdir)
log.notice('Unpacking portage tree (this can take a
long time) ...')
if not self.decompressor.extract(unpack_info):
diff --git a/catalyst/fileops.py b/catalyst/fileops.py
index 4fdc044..5fbca26 100644
--- a/catalyst/fileops.py
+++ b/catalyst/fileops.py
@@ -54,7 +54,8 @@ def ensure_dirs(path, gid=-1, uid=-1, mode=0o755,
minimal=True,
return succeeded
-def clear_dir(target, mode=0o755, chg_flags=False, remove=False):
+def clear_dir(target, mode=0o755, chg_flags=False, remove=False,
+ clear_nondir=True):
'''Universal directory clearing function
@target: string, path to be cleared or removed
@@ -67,27 +68,36 @@ def clear_dir(target, mode=0o755, chg_flags=False,
remove=False):
if not target:
log.debug('no target... returning')
return False
+
+ mystat = None
if os.path.isdir(target):
log.info('Emptying directory: %s', target)
# stat the dir, delete the dir, recreate the dir and set
# the proper perms and ownership
try:
log.debug('os.stat()')
- mystat=os.stat(target)
+ mystat = os.stat(target)
# There's no easy way to change flags recursively in
python
if chg_flags and os.uname()[0] == "FreeBSD":
os.system("chflags -R noschg " + target)
log.debug('shutil.rmtree()')
shutil.rmtree(target)
- if not remove:
- log.debug('ensure_dirs()')
- ensure_dirs(target, mode=mode)
- os.chown(target, mystat[ST_UID], mystat[ST_GID])
- os.chmod(target, mystat[ST_MODE])
except Exception:
log.error('clear_dir failed', exc_info=True)
return False
- else:
- log.info('clear_dir failed: %s: is not a directory', t
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/
commit: 159c6b3919f72a41cf60b3ecbb2ce5d07b1b18f2
Author: Brian Dolbec gentoo org>
AuthorDate: Tue Dec 15 17:00:48 2015 +
Commit: Brian Dolbec gentoo org>
CommitDate: Tue Dec 15 17:08:38 2015 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=159c6b39
Split the stagebase's set_action_sequence()
Split off the configurable portions of completion sequences.
Re-use that new function in other stages.
catalyst/base/stagebase.py| 3 +++
catalyst/targets/livecd_stage1.py | 3 ++-
catalyst/targets/stage4.py| 7 +--
3 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 92f300e..a880249 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -506,6 +506,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
"setup_confdir","portage_overlay",\
"base_dirs","bind","chroot_setup","setup_environment",\
"run_local","preclean","unbind","clean"]
+ self.set_completion_action_sequences()
+
+ def set_completion_action_sequences(self):
if "fetch" not in self.settings["options"]:
self.settings["action_sequence"].append("capture")
if "keepwork" in self.settings["options"]:
diff --git a/catalyst/targets/livecd_stage1.py
b/catalyst/targets/livecd_stage1.py
index aa234b2..60ef388 100644
--- a/catalyst/targets/livecd_stage1.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -27,7 +27,8 @@ class livecd_stage1(StageBase):
self.settings["action_sequence"]=["unpack","unpack_snapshot",\
"config_profile_link","setup_confdir","portage_overlay",\
"bind","chroot_setup","setup_environment","build_packages",\
- "unbind", "clean","clear_autoresume"]
+ "unbind", "clean"]
+ self.set_completion_action_sequences()
def set_spec_prefix(self):
self.settings["spec_prefix"]="livecd"
diff --git a/catalyst/targets/stage4.py b/catalyst/targets/stage4.py
index 857976b..fdc8465 100644
--- a/catalyst/targets/stage4.py
+++ b/catalyst/targets/stage4.py
@@ -30,9 +30,4 @@ class stage4(StageBase):
"build_kernel","bootloader","root_overlay","fsscript",\
"preclean","rcupdate","unmerge","unbind","remove","empty",\
"clean"]
-
-# if "TARBALL" in self.settings or \
-# "fetch" not in self.settings['options']:
- if "fetch" not in self.settings['options']:
- self.settings["action_sequence"].append("capture")
- self.settings["action_sequence"].append("clear_autoresume")
+ self.set_completion_action_sequences()
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/, catalyst/base/
commit: 5cce2772eda7d0885c9d7cfea184a3f5606dddef
Author: Brian Dolbec gentoo org>
AuthorDate: Mon Nov 9 04:46:28 2015 +
Commit: Brian Dolbec gentoo org>
CommitDate: Mon Nov 9 04:46:28 2015 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=5cce2772
catalyst: Convert nearly all use of open() to use the with statement.
There was one use in stagebase that was too large a code block to convert at
this time.
catalyst/base/stagebase.py| 5 ++---
catalyst/config.py| 5 ++---
catalyst/support.py | 5 ++---
catalyst/targets/livecd_stage2.py | 19 ++-
4 files changed, 16 insertions(+), 18 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index ffd84de..edbcaa6 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -848,9 +848,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
log.error('%s', unpack_errmsg % unpack_info)
if "snapcache" in self.settings["options"]:
- myf = open(snapshot_cache_hash_path, 'w')
- myf.write(self.settings["snapshot_path_hash"])
- myf.close()
+ with open(snapshot_cache_hash_path, 'w') as myf:
+
myf.write(self.settings["snapshot_path_hash"])
else:
log.info('Setting snapshot autoresume point')
self.resume.enable("unpack_portage",
diff --git a/catalyst/config.py b/catalyst/config.py
index 5f72e15..ee73abd 100644
--- a/catalyst/config.py
+++ b/catalyst/config.py
@@ -27,12 +27,11 @@ class ParserBase(object):
def parse_file(self, filename):
try:
- myf = open(filename, "r")
+ with open(filename, "r") as myf:
+ self.lines = myf.readlines()
except:
raise CatalystError("Could not open file " + filename,
print_traceback=True)
- self.lines = myf.readlines()
- myf.close()
self.filename = filename
self.parse()
diff --git a/catalyst/support.py b/catalyst/support.py
index 4bf1b22..e132568 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -155,9 +155,8 @@ def read_makeconf(mymakeconffile):
import portage_util
return
portage_util.getconfig(mymakeconffile, tolerant=1, allow_sourcing=True)
except ImportError:
- myf=open(mymakeconffile,"r")
- mylines=myf.readlines()
- myf.close()
+ with open(mymakeconffile, "r")
as myf:
+ mylines=myf.readlines()
return parse_makeconf(mylines)
except Exception:
raise CatalystError("Could not parse make.conf file " +
diff --git a/catalyst/targets/livecd_stage2.py
b/catalyst/targets/livecd_stage2.py
index fa76421..943466e 100644
--- a/catalyst/targets/livecd_stage2.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -68,8 +68,17 @@ class livecd_stage2(StageBase):
def run_local(self):
# what modules do we want to blacklist?
if "livecd/modblacklist" in self.settings:
+ path = normpath(self.settings["chroot_path"],
+
"/etc/modprobe.d/blacklist.conf")
try:
-
myf=open(self.settings["chroot_path"]+"/etc/modprobe.d/blacklist.conf","a")
+ with open(path, "a") as myf:
+ myf.write("\n#Added by Catalyst:")
+ # workaround until config.py is using
configparser
+ if
isinstance(self.settings["livecd/modblacklist"], str):
+
self.settings["livecd/modblacklist"] = self.settings[
+
"livecd/modblacklist"].split()
+ for x in
self.settings["livecd/modblacklist"]:
+ myf.write("\nblacklist "+x)
except:
self.unbind()
raise CatalystError("Couldn't open " +
@@ -77,14 +86,6 @@ class livecd_stage2(StageBase):
"/etc/modprobe.d/b
[gentoo-commits] proj/catalyst:master commit in: /, catalyst/targets/, catalyst/, catalyst/base/
commit: 9b0ba31859207264f5bf7af048d5d89725dd4025 Author: Mike Frysinger gentoo org> AuthorDate: Mon Oct 12 03:54:33 2015 + Commit: Mike Frysinger gentoo org> CommitDate: Wed Oct 28 16:49:56 2015 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=9b0ba318 lint: convert type/types to isinstance pylint complains about using (type(...) == types.xxx) checks as it prefers isinstance(..., xxx) instead. Convert the code base to it. .pylintrc | 3 +-- catalyst/base/stagebase.py| 57 +-- catalyst/support.py | 3 +-- catalyst/targets/grp.py | 5 ++-- catalyst/targets/livecd_stage1.py | 5 ++-- catalyst/targets/netboot.py | 9 +++ catalyst/targets/netboot2.py | 11 7 files changed, 39 insertions(+), 54 deletions(-) diff --git a/.pylintrc b/.pylintrc index b3327cf..2a03f23 100644 --- a/.pylintrc +++ b/.pylintrc @@ -32,11 +32,10 @@ load-plugins= # bad-continuation -- might be hard with tab indentation policy # invalid-name -- need to manage constants better # line-too-long -- figure out a length and stick to it -# unidiomatic-typecheck -- convert to isinstance # redefined-outer-name -- clean up code to not do this # super-init-not-called -- fix the classes __init__ structure # no-init -- update classes w/missing __init__ functions -disable=missing-docstring, too-many-lines, too-many-branches, too-many-statements, too-few-public-methods, too-many-instance-attributes, too-many-public-methods, too-many-locals, too-many-arguments, locally-enabled, locally-disabled, fixme, broad-except, bad-whitespace, bad-continuation, invalid-name, line-too-long, unidiomatic-typecheck, redefined-outer-name, super-init-not-called, no-init +disable=missing-docstring, too-many-lines, too-many-branches, too-many-statements, too-few-public-methods, too-many-instance-attributes, too-many-public-methods, too-many-locals, too-many-arguments, locally-enabled, locally-disabled, fixme, broad-except, bad-whitespace, bad-continuation, invalid-name, line-too-long, redefined-outer-name, super-init-not-called, no-init [REPORTS] diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 6094109..0e4d8c2 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -1,7 +1,6 @@ import os import imp -import types import shutil import sys from stat import ST_UID, ST_GID, ST_MODE @@ -297,7 +296,7 @@ class StageBase(TargetBase, ClearBase, GenBase): def set_install_mask(self): if "install_mask" in self.settings: - if type(self.settings["install_mask"])!=types.StringType: + if not isinstance(self.settings['install_mask'], str): self.settings["install_mask"]=\ ' '.join(self.settings["install_mask"]) @@ -313,14 +312,14 @@ class StageBase(TargetBase, ClearBase, GenBase): self.settings["version_stamp"] +'/' def set_source_subpath(self): - if type(self.settings["source_subpath"])!=types.StringType: + if not isinstance(self.settings['source_subpath'], str): raise CatalystError( "source_subpath should have been a string. Perhaps you have " +\ "something wrong in your spec file?") def set_pkgcache_path(self): if "pkgcache_path" in self.settings: - if type(self.settings["pkgcache_path"])!=types.StringType: + if not isinstance(self.settings['pkgcache_path'], str): self.settings["pkgcache_path"]=\ normpath(self.settings["pkgcache_path"]) else: @@ -330,7 +329,7 @@ class StageBase(TargetBase, ClearBase, GenBase): def set_kerncache_path(self): if "kerncache_path" in self.settings: - if type(self.settings["kerncache_path"])!=types.StringType: + if not isinstance(self.settings['kerncache_path'], str): self.settings["kerncache_path"]=\ normpath(self.settings["kerncache_path"]) else: @@ -536,7 +535,7 @@ class StageBase(TargetBase, ClearBase, GenBase): del self.settings[self.settings["spec_prefix"]+"/use"] if "use" not in self.settings: self.settings["use"]="" - if type(self.settings["use"])==types.StringType: + if isinstance(self.settings['use'], str): self.settings["use"]=self.settings["use"].split() # Force bindist when options ask for it @@ -554,30 +553,27 @@ class StageBase(TargetBase, ClearBase, GenBase):
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/, catalyst/, /
commit: 228211c82aeb92e9aa346e4c5bfa53a2a0cf95b8
Author: Mike Frysinger gentoo org>
AuthorDate: Tue Oct 6 04:06:05 2015 +
Commit: Mike Frysinger gentoo org>
CommitDate: Tue Oct 6 04:06:05 2015 +
URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=228211c8
lint: fix up unused variables
This variables aren't actually used which the linter likes to warn about,
so add a _ prefix to their name to silence the warnings.
catalyst/base/resume.py | 2 +-
catalyst/main.py| 2 +-
catalyst/support.py | 2 +-
catalyst/targets/netboot.py | 5 +++--
setup.py| 2 +-
5 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/catalyst/base/resume.py b/catalyst/base/resume.py
index 608f574..e3f03e7 100644
--- a/catalyst/base/resume.py
+++ b/catalyst/base/resume.py
@@ -23,7 +23,7 @@ class AutoResumeError(Exception):
def __init__(self, message, print_traceback=False):
if message:
if print_traceback:
- (type,value)=sys.exc_info()[:2]
+ (_type, value) = sys.exc_info()[:2]
if value!=None:
print
print "Traceback values found.
listing..."
diff --git a/catalyst/main.py b/catalyst/main.py
index dfa0609..04f689e 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -182,7 +182,7 @@ def main():
# parse out the command line arguments
try:
- opts,args = getopt.getopt(sys.argv[1:], "apPThvdc:C:f:FVs:",
["purge", "purgeonly", "purgetmponly", "help", "version", "debug",\
+ opts, _args = getopt.getopt(sys.argv[1:], "apPThvdc:C:f:FVs:",
["purge", "purgeonly", "purgetmponly", "help", "version", "debug",
"clear-autoresume", "config=", "cli=", "file=",
"fetch", "verbose","snapshot="])
except getopt.GetoptError:
diff --git a/catalyst/support.py b/catalyst/support.py
index 5563a15..a879eaf 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -88,7 +88,7 @@ class CatalystError(Exception):
def __init__(self, message, print_traceback=False):
if message:
if print_traceback:
- (type,value)=sys.exc_info()[:2]
+ (_type, value) = sys.exc_info()[:2]
if value!=None:
print
print "Traceback values found.
listing..."
diff --git a/catalyst/targets/netboot.py b/catalyst/targets/netboot.py
index c41ed59..b0e322c 100644
--- a/catalyst/targets/netboot.py
+++ b/catalyst/targets/netboot.py
@@ -31,11 +31,12 @@ class netboot(StageBase):
self.required_values=[]
try:
+ # XXX: This code does nothing because the for loop
below is disabled.
if "netboot/packages" in addlargs:
if type(addlargs["netboot/packages"]) ==
types.StringType:
- loopy=[addlargs["netboot/packages"]]
+ _loopy = [addlargs["netboot/packages"]]
else:
- loopy=addlargs["netboot/packages"]
+ _loopy = addlargs["netboot/packages"]
# for x in loopy:
#
self.required_values.append("netboot/packages/"+x+"/files")
diff --git a/setup.py b/setup.py
index feca894..27ed2de 100755
--- a/setup.py
+++ b/setup.py
@@ -34,7 +34,7 @@ def _files(prefix, root):
Yielding `(target_dir, (file_source_paths, ...))` tuples.
"""
- for dirpath, dirnames, filenames in _os.walk(root):
+ for dirpath, _dirnames, filenames in _os.walk(root):
reldir = _os.path.relpath(dirpath, root)
install_directory = _posix_path(
_os.path.join(prefix, reldir))
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/base/
commit: e10cdc703dddf4caa7990674a59ecdb0e40a8807 Author: Mike Frysinger gentoo org> AuthorDate: Tue Oct 6 04:11:56 2015 + Commit: Mike Frysinger gentoo org> CommitDate: Tue Oct 6 04:11:56 2015 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=e10cdc70 lint: fix mixed indentation to only use tabs catalyst/base/stagebase.py | 4 ++-- catalyst/targets/netboot2.py | 8 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 432ad5c..a9e7848 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -24,9 +24,9 @@ from catalyst.fileops import ensure_dirs, pjoin from catalyst.base.resume import AutoResume if sys.version_info[0] >= 3: -py_input = input + py_input = input else: -py_input = raw_input + py_input = raw_input class StageBase(TargetBase, ClearBase, GenBase): diff --git a/catalyst/targets/netboot2.py b/catalyst/targets/netboot2.py index 5de1a74..e509cf9 100644 --- a/catalyst/targets/netboot2.py +++ b/catalyst/targets/netboot2.py @@ -80,10 +80,10 @@ class netboot2(StageBase): for x in loopy: if "netboot2/packages/"+x+"/files" in self.settings: - if type(self.settings["netboot2/packages/"+x+"/files"]) == types.ListType: - myfiles.extend(self.settings["netboot2/packages/"+x+"/files"]) - else: - myfiles.append(self.settings["netboot2/packages/"+x+"/files"]) + if type(self.settings["netboot2/packages/"+x+"/files"]) == types.ListType: + myfiles.extend(self.settings["netboot2/packages/"+x+"/files"]) + else: + myfiles.append(self.settings["netboot2/packages/"+x+"/files"]) if "netboot2/extra_files" in self.settings: if type(self.settings["netboot2/extra_files"]) == types.ListType:
[gentoo-commits] proj/catalyst:master commit in: catalyst/targets/, catalyst/
commit: 11423a21603ed6ab4cdf8e1662512b9d275345bc
Author: Brian Dolbec gentoo org>
AuthorDate: Sun Jan 20 21:50:23 2013 +
Commit: Brian Dolbec gentoo org>
CommitDate: Thu Jan 1 05:58:05 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=11423a21
[2 of 3] Update module loading for the new python structure
Rename snapshot_target to snapshot
---
catalyst/main.py | 75 --
catalyst/targets/embedded_target.py| 4 --
catalyst/targets/grp_target.py | 4 --
catalyst/targets/livecd_stage1_target.py | 4 --
catalyst/targets/livecd_stage2_target.py | 4 --
catalyst/targets/netboot2_target.py| 4 --
catalyst/targets/netboot_target.py | 4 --
.../targets/{snapshot_target.py => snapshot.py}| 6 +-
catalyst/targets/stage1_target.py | 4 --
catalyst/targets/stage2_target.py | 4 --
catalyst/targets/stage3_target.py | 4 --
catalyst/targets/stage4_target.py | 5 --
catalyst/targets/tinderbox_target.py | 4 --
13 files changed, 27 insertions(+), 99 deletions(-)
diff --git a/catalyst/main.py b/catalyst/main.py
index 15664de..1446cf9 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -16,21 +16,20 @@ import os.path
__selfpath__ = os.path.abspath(os.path.dirname(__file__))
-sys.path.append(__selfpath__ + "/modules")
from . import __version__
import catalyst.config
import catalyst.util
+from catalyst.contents import ContentsMap, CONTENTS_DEFINITIONS
+from catalyst.defaults import confdefaults, option_messages
+from catalyst.hash_utils import HashMap, HASH_DEFINITIONS
from catalyst.lock import LockInUse
from catalyst.support import CatalystError, find_binary
-from catalyst.defaults import (confdefaults, option_messages,
- required_build_targets, valid_build_targets)
-from hash_utils import HashMap, HASH_DEFINITIONS
-from contents import ContentsMap, CONTENTS_DEFINITIONS
conf_values={}
+
def usage():
print """Usage catalyst [options] [-C variable=value...] [ -s
identifier]
-a --clear-autoresume clear autoresume flags
@@ -135,59 +134,40 @@ def parse_config(myconfig):
print "Envscript support enabled."
-def import_modules():
- # import catalyst's own modules
- # (i.e. stage and the arch modules)
- targetmap={}
-
+def import_module(target):
+ """
+ import catalyst's own modules
+ (i.e. targets and the arch modules)
+ """
try:
- module_dir = __selfpath__ + "/targets/"
- for x in required_build_targets:
- try:
- fh=open(module_dir + x + ".py")
- module=imp.load_module(x, fh,"targets/" + x +
".py",
- (".py", "r", imp.PY_SOURCE))
- fh.close()
-
- except IOError:
- raise CatalystError, "Can't find " + x + ".py
plugin in " + \
- module_dir
- for x in valid_build_targets:
- try:
- fh=open(module_dir + x + ".py")
- module=imp.load_module(x, fh, "targets/" + x +
".py",
- (".py", "r", imp.PY_SOURCE))
- module.register(targetmap)
- fh.close()
-
- except IOError:
- raise CatalystError,"Can't find " + x + ".py
plugin in " + \
- module_dir
-
+ mod_name = "catalyst.targets." + target
+ module = __import__(mod_name, [],[], ["not empty"])
except ImportError as e:
- print "!!! catalyst: Python modules not found in "+\
- module_dir + "; exiting."
- print e
+ print "!!! catalyst: Python module import error: %s " % target
+ \
+ "in catalyst/targets/ ... exiting."
+ print "ERROR was: ", e
sys.exit(1)
+ return module
- return targetmap
-def build_target(addlargs, targetmap):
+def build_target(addlargs):
try:
- if addlargs["target"] not in targetmap:
- raise CatalystError,"Target \""+addlargs["target"]+"\"
not available."
-
- mytarget=targetmap[addlargs["target"]](conf_values, addlargs)
-
- mytarget.run()
+ module = import_module(addlargs["target"])
+ target = getattr(module, addlargs["target"])(conf_values,
addlargs)
+ except AttributeError:
+ raise CatalystError(
+ "Target \"%s\" not available."
