[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: 1256ad85e2bc4f5e1ea4d15b917a6365a8cb5950
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Sun Jan 20 01:50:26 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sun Jun 15 14:02:03 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=1256ad85
Move LockInUse from support.py to lock.py, fix bad execption raising, pyflakes
cleanup
---
catalyst/lock.py| 49 +++--
catalyst/main.py| 3 ++-
catalyst/support.py | 11 ---
3 files changed, 33 insertions(+), 30 deletions(-)
diff --git a/catalyst/lock.py b/catalyst/lock.py
index 2d10d2f..ef00b49 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -3,14 +3,26 @@ import os
import fcntl
import errno
import sys
-import string
import time
-from catalyst.support import *
+from catalyst.support import CatalystError
def writemsg(mystr):
sys.stderr.write(mystr)
sys.stderr.flush()
+
+class LockInUse(Exception):
+ def __init__(self, message):
+ if message:
+ #(type,value)=sys.exc_info()[:2]
+ #if value!=None:
+ #print
+ #kprint traceback.print_exc(file=sys.stdout)
+ print
+ print !!! catalyst lock file in use: +message
+ print
+
+
class LockDir:
locking_method=fcntl.flock
lock_dirs_in_use=[]
@@ -109,7 +121,8 @@ class LockDir:
def fcntl_lock(self,locktype):
if self.myfd==None:
if not os.path.exists(os.path.dirname(self.lockdir)):
- raise DirectoryNotFound,
os.path.dirname(self.lockdir)
+ raise CatalystError(DirectoryNotFound: %s
+ % os.path.dirname(self.lockdir))
if not os.path.exists(self.lockfile):
old_mask=os.umask(000)
self.myfd = os.open(self.lockfile,
os.O_CREAT|os.O_RDWR,0660)
@@ -168,7 +181,7 @@ class LockDir:
print lockfile does not exist '%s' % self.lockfile
if (self.myfd != None):
try:
- os.close(myfd)
+ os.close(self.myfd)
self.myfd=None
except:
pass
@@ -251,12 +264,13 @@ class LockDir:
self.add_hardlock_file_to_cleanup()
if not os.path.exists(self.myhardlock):
- raise FileNotFound, Created lockfile is
missing: %(filename)s % {filename:self.myhardlock}
+ raise CatalystError(FileNotFound: Created
lockfile is missing:
+ %(filename)s %
{filename:self.myhardlock})
try:
- res = os.link(self.myhardlock, self.lockfile)
- except SystemExit, e:
+ os.link(self.myhardlock, self.lockfile)
+ except SystemExit:
raise
- except Exception, e:
+ except Exception:
# if DEBUG in self.settings:
# print lockfile(): Hardlink: Link
failed.
# print Exception: ,e
@@ -286,7 +300,7 @@ class LockDir:
os.unlink(self.myhardlock)
if os.path.exists(self.lockfile):
os.unlink(self.lockfile)
- except SystemExit, e:
+ except SystemExit:
raise
except:
writemsg(Something strange happened to our hardlink
locks.\n)
@@ -314,7 +328,7 @@ class LockDir:
try:
myhls = os.stat(link)
mylfs = os.stat(lock)
- except SystemExit, e:
+ except SystemExit:
raise
except:
myhls = None
@@ -340,7 +354,7 @@ class LockDir:
pass
def hardlock_cleanup(self,path):
- mypid = str(os.getpid())
+ #mypid = str(os.getpid())
myhost = os.uname()[1]
mydl = os.listdir(path)
results = []
@@ -384,26 +398,26 @@ class LockDir:
# We're
sweeping through, unlinking everyone's locks.
os.unlink(filename)
[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/targets/
commit: c643ffa965d22f0dda0beb980ebb29f9fa97d7d2
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Sun Jan 20 01:28:51 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sun Jun 15 13:58:45 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=c643ffa9
Some options cleanup, unifying their use, reducing redundancy.
fix options being reset by a config file
Conflicts:
catalyst/defaults.py
catalyst/main.py
catalyst/targets/generic_stage_target.py
---
catalyst/defaults.py | 22 -
catalyst/main.py | 96 +++---
catalyst/targets/generic_stage_target.py | 133 +--
catalyst/targets/grp_target.py | 2 +-
catalyst/targets/livecd_stage1_target.py | 2 +-
catalyst/targets/livecd_stage2_target.py | 8 +-
catalyst/targets/netboot2_target.py | 10 +--
catalyst/targets/snapshot_target.py | 4 +-
catalyst/targets/stage2_target.py| 2 +-
catalyst/targets/stage4_target.py| 4 +-
10 files changed, 134 insertions(+), 149 deletions(-)
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index b83e4f5..b36eff7 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -13,10 +13,9 @@ valid_build_targets = [stage1_target, stage2_target,
stage3_target,
required_config_file_values = [storedir, sharedir, distdir, portdir]
valid_config_file_values = required_config_file_values[:]
-valid_config_file_values.extend([PKGCACHE, KERNCACHE, CCACHE, DISTCC,
- ICECREAM, ENVSCRIPT, AUTORESUME, FETCH, CLEAR_AUTORESUME,
- options, DEBUG, VERBOSE, PURGE, PURGEONLY, SNAPCACHE,
- snapshot_cache, hash_function, digests, contents, SEEDCACHE
+valid_config_file_values.extend([ distcc, envscript,
+ options, DEBUG, VERBOSE,
+ snapshot_cache, hash_function, digests, contents
])
verbosity = 1
@@ -65,3 +64,18 @@ SOURCE_MOUNT_DEFAULTS = {
shm: shmfs,
}
+# legend: key: message
+option_messages = {
+ autoresume: Autoresuming support enabled.,
+ ccache: Compiler cache support enabled.,
+ clear-autoresume: Cleaning autoresume flags support enabled.,
+ #compress: Compression enabled.,
+ distcc: Distcc support enabled.,
+ icecream: Icecream compiler cluster support enabled.,
+ kerncache: Kernel cache support enabled.,
+ pkgcache: Package cache support enabled.,
+ purge: Purge support enabled.,
+ seedcache: Seed cache support enabled.,
+ snapcache: Snapshot cache support enabled.,
+ #tarball: Tarball creation enabled.,
+ }
diff --git a/catalyst/main.py b/catalyst/main.py
index 5748d31..81d48b5 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -22,13 +22,12 @@ from . import __version__
import catalyst.config
import catalyst.util
from catalyst.support import CatalystError, find_binary, LockInUse
-from catalyst.defaults import (required_build_targets, valid_build_targets,
- confdefaults)
+from catalyst.defaults import (confdefaults, option_messages,
+ required_build_targets, valid_build_targets)
from hash_utils import HashMap, HASH_DEFINITIONS
from contents import ContentsMap, CONTENTS_DEFINITIONS
-
conf_values={}
def usage():
@@ -106,7 +105,10 @@ def parse_config(myconfig):
for x in list(confdefaults):
if x in myconf:
print Setting,x,to config file value
\+myconf[x]+\
- conf_values[x]=myconf[x]
+ if x == 'options':
+ conf_values[x] = set(myconf[x].split())
+ else:
+ conf_values[x]=myconf[x]
else:
print Setting,x,to default value
\+confdefaults[x]+\
conf_values[x]=confdefaults[x]
@@ -114,74 +116,23 @@ def parse_config(myconfig):
# add our python base directory to use for loading target arch's
conf_values[PythonDir] = __selfpath__
- # parse out the rest of the options from the config file
- if autoresume in string.split(conf_values[options]):
- print Autoresuming support enabled.
- conf_values[AUTORESUME]=1
-
- if bindist in string.split(conf_values[options]):
- print Binary redistribution enabled
- conf_values[BINDIST]=1
- else:
- print Bindist is not enabled in catalyst.conf
- print Binary redistribution of generated stages/isos may be
prohibited by law.
- print Please see the use description for bindist on any
package you are including.
-
- if ccache in string.split(conf_values[options]):
- print Compiler cache support enabled.
- conf_values[CCACHE]=1
-
- if clear-autoresume in string.split(conf_values[options]):
-
[gentoo-commits] proj/catalyst:pending commit in: catalyst/targets/
commit: a00a729072e5ca26ba84c99db8e2a851287e7527
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Tue Jan 22 00:10:51 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sun Jun 15 14:05:53 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=a00a7290
Rename all target .py files and classes without _target in them.
This is so they are the named the same as the target .sh files
and work with the now simplified module loading.
---
catalyst/targets/{embedded_target.py = embedded.py} | 2 +-
catalyst/targets/{grp_target.py = grp.py} | 2 +-
catalyst/targets/{livecd_stage1_target.py = livecd_stage1.py} | 2 +-
catalyst/targets/{livecd_stage2_target.py = livecd_stage2.py} | 2 +-
catalyst/targets/{netboot_target.py = netboot.py} | 2 +-
catalyst/targets/{netboot2_target.py = netboot2.py} | 2 +-
catalyst/targets/{stage1_target.py = stage1.py} | 2 +-
catalyst/targets/{stage2_target.py = stage2.py} | 2 +-
catalyst/targets/{stage3_target.py = stage3.py} | 2 +-
catalyst/targets/{stage4_target.py = stage4.py} | 2 +-
catalyst/targets/{tinderbox_target.py = tinderbox.py} | 2 +-
11 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/catalyst/targets/embedded_target.py b/catalyst/targets/embedded.py
similarity index 98%
rename from catalyst/targets/embedded_target.py
rename to catalyst/targets/embedded.py
index aee0f00..3309278 100644
--- a/catalyst/targets/embedded_target.py
+++ b/catalyst/targets/embedded.py
@@ -15,7 +15,7 @@ from catalyst.support import normpath
from catalyst.base.stagebase import StageBase
-class embedded_target(StageBase):
+class embedded(StageBase):
Builder class for embedded target
diff --git a/catalyst/targets/grp_target.py b/catalyst/targets/grp.py
similarity index 99%
rename from catalyst/targets/grp_target.py
rename to catalyst/targets/grp.py
index e3f08a2..0075714 100644
--- a/catalyst/targets/grp_target.py
+++ b/catalyst/targets/grp.py
@@ -14,7 +14,7 @@ from catalyst.support import (CatalystError, normpath,
from catalyst.base.stagebase import StageBase
-class grp_target(StageBase):
+class grp(StageBase):
The builder class for GRP (Gentoo Reference Platform) builds.
diff --git a/catalyst/targets/livecd_stage1_target.py
b/catalyst/targets/livecd_stage1.py
similarity index 98%
rename from catalyst/targets/livecd_stage1_target.py
rename to catalyst/targets/livecd_stage1.py
index 9c74253..661e06e 100644
--- a/catalyst/targets/livecd_stage1_target.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -13,7 +13,7 @@ from catalyst.support import (normpath,
from catalyst.base.stagebase import StageBase
-class livecd_stage1_target(StageBase):
+class livecd_stage1(StageBase):
Builder class for LiveCD stage1.
diff --git a/catalyst/targets/livecd_stage2_target.py
b/catalyst/targets/livecd_stage2.py
similarity index 99%
rename from catalyst/targets/livecd_stage2_target.py
rename to catalyst/targets/livecd_stage2.py
index a4630e6..20f6014 100644
--- a/catalyst/targets/livecd_stage2_target.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -10,7 +10,7 @@ from catalyst.support import (normpath, file_locate,
CatalystError, cmd,
from catalyst.base.stagebase import StageBase
-class livecd_stage2_target(StageBase):
+class livecd_stage2(StageBase):
Builder class for a LiveCD stage2 build.
diff --git a/catalyst/targets/netboot_target.py b/catalyst/targets/netboot.py
similarity index 99%
rename from catalyst/targets/netboot_target.py
rename to catalyst/targets/netboot.py
index 9d92ef2..f753c7f 100644
--- a/catalyst/targets/netboot_target.py
+++ b/catalyst/targets/netboot.py
@@ -12,7 +12,7 @@ from catalyst.support import (CatalystError, normpath,
from catalyst.base.stagebase import StageBase
-class netboot_target(StageBase):
+class netboot(StageBase):
Builder class for a netboot build.
diff --git a/catalyst/targets/netboot2_target.py b/catalyst/targets/netboot2.py
similarity index 99%
rename from catalyst/targets/netboot2_target.py
rename to catalyst/targets/netboot2.py
index 130e2b6..fc8c64c 100644
--- a/catalyst/targets/netboot2_target.py
+++ b/catalyst/targets/netboot2.py
@@ -14,7 +14,7 @@ from catalyst.support import (CatalystError, normpath,
from catalyst.base.stagebase import StageBase
-class netboot2_target(StageBase):
+class netboot2(StageBase):
Builder class for a netboot build, version 2
diff --git a/catalyst/targets/stage1_target.py b/catalyst/targets/stage1.py
similarity index 99%
rename from catalyst/targets/stage1_target.py
rename to catalyst/targets/stage1.py
index 2329b58..e72cbcc 100644
--- a/catalyst/targets/stage1_target.py
+++ b/catalyst/targets/stage1.py
@@ -9,7 +9,7 @@ from catalyst.support import
[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/base/
commit: cd852d68bd0b17e51fce2b498993b4091adb402b
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Tue Feb 12 04:06:01 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sun Jun 15 14:05:53 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=cd852d68
Break out more repeated (path1 + path2)'s...
Just do it once and use the temp variable.
Comment out some debug print's.
Fix options conversion for export to bash.
Conflicts:
catalyst/base/stagebase.py
---
catalyst/base/stagebase.py | 52 ++
catalyst/defaults.py | 1 +
2 files changed, 31 insertions(+), 22 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 5bedbb8..6efa98e 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -631,11 +631,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.setup_environment()
- if os.path.exists(self.settings[sharedir]+\
- /targets/support/kill-chroot-pids.sh):
- cmd(/bin/bash +self.settings[sharedir]+\
- /targets/support/kill-chroot-pids.sh,\
- kill-chroot-pids script failed.,env=self.env)
+ killcmd = normpath(self.settings[sharedir] +
+ self.settings[shdir] + /support/kill-chroot-pids.sh)
+ if os.path.exists(killcmd):
+ cmd(killcmd, kill-chroot-pids script
failed.,env=self.env)
def mount_safety_check(self):
@@ -647,10 +646,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
if not os.path.exists(self.settings[chroot_path]):
return
- print self.mounts =, self.mounts
+ #print self.mounts =, self.mounts
for x in self.mounts:
target = normpath(self.settings[chroot_path] +
self.target_mounts[x])
- print mount_safety_check() x =, x, target
+ #print mount_safety_check() x =, x, target
if not os.path.exists(target):
continue
@@ -1020,8 +1019,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
Couldn't umount one or more bind-mounts;
aborting for safety.
def chroot_setup(self):
- self.makeconf=read_makeconf(self.settings[chroot_path]+\
- /etc/portage/make.conf)
+
self.makeconf=read_makeconf(normpath(self.settings[chroot_path]+
+ self.settings[make.conf]))
self.override_cbuild()
self.override_chost()
self.override_cflags()
@@ -1035,8 +1034,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
else:
print Setting up chroot...
-
#self.makeconf=read_makeconf(self.settings[chroot_path]+/etc/portage/make.conf)
-
cmd(cp /etc/resolv.conf +
self.settings[chroot_path] + /etc/,
Could not copy resolv.conf into
place.,env=self.env)
@@ -1070,10 +1067,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
Could not copy
/etc/hosts,env=self.env)
Modify and write out make.conf (for the chroot)
- cmd(rm -f
+self.settings[chroot_path]+/etc/portage/make.conf,\
- Could not remove
+self.settings[chroot_path]+\
- /etc/portage/make.conf,env=self.env)
-
myf=open(self.settings[chroot_path]+/etc/portage/make.conf,w)
+ makepath = normpath(self.settings[chroot_path] +
+ self.settings[make.conf])
+ cmd(rm -f + makepath,\
+ Could not remove + makepath, env=self.env)
+ myf=open(makepath, w)
myf.write(# These settings were set by the catalyst
build script that automatically\n# built this stage.\n)
myf.write(# Please consult
/usr/share/portage/config/make.conf.example for a more\n# detailed example.\n)
if CFLAGS in self.settings:
@@ -1123,10 +1121,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
myf.write('PORTDIR_OVERLAY=/usr/local/portage\n')
myf.close()
- cmd(cp
+self.settings[chroot_path]+/etc/portage/make.conf +\
-
self.settings[chroot_path]+/etc/portage/make.conf.catalyst,\
- Could not backup
/etc/portage/make.conf,env=self.env)
- touch(chroot_setup_resume)
+
[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: d731435f4adeac15dd59ef9c7a17196468be4379 Author: Brian Dolbec dolsen AT gentoo DOT org AuthorDate: Sun Jan 20 19:22:27 2013 + Commit: Brian Dolbec brian.dolbec AT gmail DOT com CommitDate: Sun Jun 15 14:03:46 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=d731435f some spacing and comment and indent cleanup, etc. Conflicts: catalyst/support.py --- catalyst/support.py | 405 +++- 1 file changed, 213 insertions(+), 192 deletions(-) diff --git a/catalyst/support.py b/catalyst/support.py index feaa645..6f66d3c 100644 --- a/catalyst/support.py +++ b/catalyst/support.py @@ -1,12 +1,19 @@ -import sys,string,os,types,re,signal,traceback,time -#import md5,sha +import sys +import string +import os +import types +import re +import signal +import traceback +import time from catalyst.defaults import verbosity, valid_config_file_values selinux_capable = False #userpriv_capable = (os.getuid() == 0) #fakeroot_capable = False + BASH_BINARY = /bin/bash # set it to 0 for the soft limit, 1 for the hard limit @@ -25,35 +32,35 @@ spawned_pids = [] def cleanup(pids,block_exceptions=True): -function to go through and reap the list of pids passed to it -global spawned_pids -if type(pids) == int: -pids = [pids] -for x in pids: -try: -os.kill(x,signal.SIGTERM) -if os.waitpid(x,os.WNOHANG)[1] == 0: -# feisty bugger, still alive. -os.kill(x,signal.SIGKILL) -os.waitpid(x,0) - -except OSError, oe: -if block_exceptions: -pass -if oe.errno not in (10,3): -raise oe -except SystemExit: -raise -except Exception: -if block_exceptions: -pass -try:spawned_pids.remove(x) -except IndexError: pass - - - -# a function to turn a string of non-printable characters into a string of -# hex characters + function to go through and reap the list of pids passed to it + global spawned_pids + if type(pids) == int: + pids = [pids] + for x in pids: + try: + os.kill(x,signal.SIGTERM) + if os.waitpid(x,os.WNOHANG)[1] == 0: + # feisty bugger, still alive. + os.kill(x,signal.SIGKILL) + os.waitpid(x,0) + except OSError, oe: + if block_exceptions: + pass + if oe.errno not in (10,3): + raise oe + except SystemExit: + raise + except Exception: + if block_exceptions: + pass + try: + spawned_pids.remove(x) + except IndexError: + pass + + +# a function to turn a string of non-printable characters +# into a string of hex characters def hexify(str): hexStr = string.hexdigits r = '' @@ -61,7 +68,6 @@ def hexify(str): i = ord(ch) r = r + hexStr[(i 4) 0xF] + hexStr[i 0xF] return r -# hexify() def read_from_clst(file): @@ -77,7 +83,6 @@ def read_from_clst(file): myline = myline + line myf.close() return myline -# read_from_clst def list_bashify(mylist): @@ -92,6 +97,7 @@ def list_bashify(mylist): mypack=string.join(mypack) return mypack + def list_to_string(mylist): if type(mylist)==types.StringType: mypack=[mylist] @@ -104,6 +110,7 @@ def list_to_string(mylist): mypack=string.join(mypack) return mypack + class CatalystError(Exception): def __init__(self, message): if message: @@ -115,84 +122,83 @@ class CatalystError(Exception): print !!! catalyst: +message print + def die(msg=None): warn(msg) sys.exit(1) + def warn(msg): print !!! catalyst: +msg + def find_binary(myc): look through the environmental path for an executable file named whatever myc is -# this sucks. badly. -p=os.getenv(PATH) -if p == None: -return None -for x in p.split(:): -#if it exists, and is executable -if os.path.exists(%s/%s % (x,myc)) and os.stat(%s/%s % (x,myc))[0] 0x0248: -return %s/%s %
[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: 65ecc7daa279bd2230564494599d2a0608b91720 Author: Brian Dolbec dolsen AT gentoo DOT org AuthorDate: Tue Jan 22 08:34:41 2013 + Commit: Brian Dolbec brian.dolbec AT gmail DOT com CommitDate: Sun Jun 15 14:05:53 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=65ecc7da FIXME! Comment out a small code block causing TypeError. This was also short circuiting another large code block. FIXME This whole class seems overly complicated with TOO MANY nested try:excepts: --- catalyst/lock.py | 23 +-- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/catalyst/lock.py b/catalyst/lock.py index ef00b49..d073861 100644 --- a/catalyst/lock.py +++ b/catalyst/lock.py @@ -27,11 +27,17 @@ class LockDir: locking_method=fcntl.flock lock_dirs_in_use=[] die_on_failed_lock=True + def __del__(self): + #print Lock.__del__() 1 self.clean_my_hardlocks() + #print Lock.__del__() 2 self.delete_lock_from_path_list() + #print Lock.__del__() 3 if self.islocked(): + #print Lock.__del__() 4 self.fcntl_unlock() + #print Lock.__del__() finnished def __init__(self,lockdir): self.locked=False @@ -179,7 +185,9 @@ class LockDir: unlinkfile = 1 if not os.path.exists(self.lockfile): print lockfile does not exist '%s' % self.lockfile + #print fcntl_unlock() , self.myfd:, self.myfd, type(self.myfd) if (self.myfd != None): + #print fcntl_unlock() trying to close it try: os.close(self.myfd) self.myfd=None @@ -193,11 +201,13 @@ class LockDir: unlinkfile = 1 self.locking_method(self.myfd,fcntl.LOCK_UN) except SystemExit, e: - raise + raise e except Exception, e: - os.close(self.myfd) - self.myfd=None - raise IOError, Failed to unlock file '%s'\n % self.lockfile + #if self.myfd is not None: + #print fcntl_unlock() trying to close, self.myfd + #os.close(self.myfd) + #self.myfd=None + #raise IOError, Failed to unlock file '%s'\n%s % (self.lockfile, str(e)) try: # This sleep call was added to allow other processes that are # waiting for a lock to be able to grab it before it is deleted. @@ -224,7 +234,7 @@ class LockDir: # if DEBUG in self.settings: # print Unlinked lockfile... except SystemExit, e: - raise + raise e except Exception, e: # We really don't care... Someone else has the lock. # So it is their problem now. @@ -236,7 +246,7 @@ class LockDir: # open fd closed automatically on them. #if type(lockfilename) == types.StringType: #os.close(myfd) - + #print fcntl_unlock() trying a last ditch close, self.myfd if (self.myfd != None): os.close(self.myfd) self.myfd=None @@ -421,6 +431,7 @@ class LockDir: pass return results + if __name__ == __main__: def lock_work():
[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/base/
commit: 298fd82042b6e1afad7f39a9ceb0f623c1c37e6b
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Tue Jan 22 08:39:18 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sun Jun 15 14:05:53 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=298fd820
rename a make.conf key to make_conf due to bash variable name restrictions
Conflicts:
catalyst/base/stagebase.py
catalyst/defaults.py
---
catalyst/base/stagebase.py | 10 +-
catalyst/defaults.py | 5 -
2 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 6efa98e..99b667a 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1020,7 +1020,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
def chroot_setup(self):
self.makeconf=read_makeconf(normpath(self.settings[chroot_path]+
- self.settings[make.conf]))
+ self.settings[make_conf]))
self.override_cbuild()
self.override_chost()
self.override_cflags()
@@ -1068,7 +1068,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
Modify and write out make.conf (for the chroot)
makepath = normpath(self.settings[chroot_path] +
- self.settings[make.conf])
+ self.settings[make_conf])
cmd(rm -f + makepath,\
Could not remove + makepath, env=self.env)
myf=open(makepath, w)
@@ -1122,9 +1122,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
myf.close()
makepath = normpath(self.settings[chroot_path] +
- self.settings[make.conf])
+ self.settings[make_conf])
cmd(cp + makepath + + makepath + .catalyst,\
- Could not backup +
self.settings[make.conf],env=self.env)
+ Could not backup +
self.settings[make_conf],env=self.env)
touch(self.settings[autoresume_path]+chroot_setup)
def fsscript(self):
@@ -1172,7 +1172,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
cmd(rm -rf + self.settings[chroot_path] +
self.settings[local_overlay],
Could not remove +
self.settings[local_overlay], env=self.env)
cmd(sed -i '/^PORTDIR_OVERLAY/d'
+self.settings[chroot_path]+\
- /etc/portage/make.conf,\
+ self.settings[make_conf],\
Could not remove PORTDIR_OVERLAY from
make.conf,env=self.env)
Clean up old and obsoleted files in /etc
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 2b5eee4..7a9bb23 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -25,7 +25,8 @@ confdefaults={
hash_function: crc32,
icecream: /var/cache/icecream,
local_overlay: /usr/local/portage,
- options: ,
+ make_conf: /etc/portage/make.conf,
+ options: set(),
packagedir: /usr/portage/packages,
portdir: /usr/portage,
port_tmpdir: /var/tmp/portage,
@@ -42,6 +43,8 @@ PORT_LOGDIR_CLEAN = \
TARGET_MOUNT_DEFAULTS = {
ccache: /var/tmp/ccache,
+
+target_mounts = {
dev: /dev,
devpts: /dev/pts,
distdir: /usr/portage/distfiles,
[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/targets/
commit: b55a45005a0077741cef5e00ef47c99e4bd4ce80
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Tue Feb 12 03:43:37 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sat Jun 14 05:00:05 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=b55a4500
remove redundant /bin/bash additions in cmd() calls
Conflicts:
catalyst/support.py
---
catalyst/support.py | 301 +++
catalyst/targets/generic_stage_target.py | 26 +--
catalyst/targets/grp_target.py | 2 +-
catalyst/targets/netboot2_target.py | 4 +-
catalyst/targets/netboot_target.py | 8 +-
catalyst/targets/tinderbox_target.py | 2 +-
6 files changed, 42 insertions(+), 301 deletions(-)
diff --git a/catalyst/support.py b/catalyst/support.py
index 6f66d3c..8fb5da2 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -4,9 +4,10 @@ import string
import os
import types
import re
-import signal
import traceback
import time
+from subprocess import Popen
+
from catalyst.defaults import verbosity, valid_config_file_values
@@ -31,34 +32,6 @@ except:
spawned_pids = []
-def cleanup(pids,block_exceptions=True):
- function to go through and reap the list of pids passed to it
- global spawned_pids
- if type(pids) == int:
- pids = [pids]
- for x in pids:
- try:
- os.kill(x,signal.SIGTERM)
- if os.waitpid(x,os.WNOHANG)[1] == 0:
- # feisty bugger, still alive.
- os.kill(x,signal.SIGKILL)
- os.waitpid(x,0)
- except OSError, oe:
- if block_exceptions:
- pass
- if oe.errno not in (10,3):
- raise oe
- except SystemExit:
- raise
- except Exception:
- if block_exceptions:
- pass
- try:
- spawned_pids.remove(x)
- except IndexError:
- pass
-
-
# a function to turn a string of non-printable characters
# into a string of hex characters
def hexify(str):
@@ -79,8 +52,8 @@ def read_from_clst(file):
return -1
#raise CatalystError, Could not open file +file
for line in myf.readlines():
- #line = string.replace(line, \n, ) # drop newline
- myline = myline + line
+ #line = string.replace(line, \n, ) # drop newline
+ myline = myline + line
myf.close()
return myline
@@ -145,259 +118,27 @@ def find_binary(myc):
return None
-def spawn_bash(mycommand,env={},debug=False,opt_name=None,**keywords):
- spawn mycommand as an arguement to bash
- args=[BASH_BINARY]
- if not opt_name:
- opt_name=mycommand.split()[0]
- if BASH_ENV not in env:
- env[BASH_ENV] = /etc/spork/is/not/valid/profile.env
- if debug:
- args.append(-x)
- args.append(-c)
- args.append(mycommand)
- return spawn(args,env=env,opt_name=opt_name,**keywords)
-
-
-def spawn_get_output(mycommand,raw_exit_code=False,emulate_gso=True, \
- collect_fds=[1],fd_pipes=None,**keywords):
- call spawn, collecting the output to fd's specified in collect_fds
list
- emulate_gso is a compatability hack to emulate
commands.getstatusoutput's return, minus the
- requirement it always be a bash call (spawn_type controls the actual
spawn call), and minus the
- 'lets let log only stdin and let stderr slide by'.
-
- emulate_gso was deprecated from the day it was added, so convert your
code over.
- spawn_type is the passed in function to call- typically spawn_bash,
spawn, spawn_sandbox, or spawn_fakeroot
- global selinux_capable
- pr,pw=os.pipe()
-
- if fd_pipes==None:
- fd_pipes={}
- fd_pipes[0] = 0
-
- for x in collect_fds:
- fd_pipes[x] = pw
- keywords[returnpid]=True
-
- mypid=spawn_bash(mycommand,fd_pipes=fd_pipes,**keywords)
- os.close(pw)
- if type(mypid) != types.ListType:
- os.close(pr)
- return [mypid, %s: No such file or directory %
mycommand.split()[0]]
-
- fd=os.fdopen(pr,r)
- mydata=fd.readlines()
- fd.close()
- if emulate_gso:
- mydata=string.join(mydata)
- if len(mydata) and mydata[-1] == \n:
- mydata=mydata[:-1]
- retval=os.waitpid(mypid[0],0)[1]
- cleanup(mypid)
- if raw_exit_code:
- return [retval,mydata]
-
[gentoo-commits] proj/catalyst:pending commit in: catalyst/targets/
commit: 4eddcf963a9366aea9230c688f3f86e06171b472
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Sun Jan 20 08:10:03 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sat Jun 14 04:49:26 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=4eddcf96
Begin splitting up generic_stage_target into smaller code blocks.
This so snapshot_target does not need to import it since most of it was
not used or initialized properly.
Conflicts:
catalyst/targets/generic_stage_target.py
catalyst/targets/snapshot_target.py
---
catalyst/targets/clearbase.py | 115 ++
catalyst/targets/genbase.py| 58 +++
catalyst/targets/generic_stage_target.py | 169 +++--
catalyst/targets/snapshot_target.py| 16 +-
.../targets/{generic_target.py = targetbase.py} | 6 +-
5 files changed, 208 insertions(+), 156 deletions(-)
diff --git a/catalyst/targets/clearbase.py b/catalyst/targets/clearbase.py
new file mode 100644
index 000..8519acc
--- /dev/null
+++ b/catalyst/targets/clearbase.py
@@ -0,0 +1,115 @@
+
+import os
+import shutil
+from stat import ST_UID, ST_GID, ST_MODE
+
+
+from catalyst.support import cmd, countdown
+
+
+class ClearBase(object):
+
+ This class does all of clearing after task completion
+
+ def __init__(self, myspec):
+ self.settings = myspec
+
+
+
+ def clear_autoresume(self):
+Clean resume points since they are no longer needed
+ if autoresume in self.settings[options]:
+ print Removing AutoResume Points: ...
+ myemp=self.settings[autoresume_path]
+ if os.path.isdir(myemp):
+ if autoresume in self.settings[options]:
+ print Emptying directory,myemp
+
+ stat the dir, delete the dir, recreate the dir
and set
+ the proper perms and ownership
+
+ mystat=os.stat(myemp)
+ if os.uname()[0] == FreeBSD:
+ cmd(chflags -R noschg +myemp,\
+ Could not remove immutable
flag for file \
+ +myemp)
+ #cmd(rm -rf +myemp, Could not remove
existing file: +myemp,env-self.env)
+ shutil.rmtree(myemp)
+ os.makedirs(myemp,0755)
+ os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
+ os.chmod(myemp,mystat[ST_MODE])
+
+
+ def clear_chroot(self):
+ myemp=self.settings[chroot_path]
+ if os.path.isdir(myemp):
+ print Emptying directory,myemp
+
+ stat the dir, delete the dir, recreate the dir and set
+ the proper perms and ownership
+
+ mystat=os.stat(myemp)
+ #cmd(rm -rf +myemp, Could not remove existing file:
+myemp,env=self.env)
+There's no easy way to change flags recursively in
python
+ if os.uname()[0] == FreeBSD:
+ os.system(chflags -R noschg +myemp)
+ shutil.rmtree(myemp)
+ os.makedirs(myemp,0755)
+ os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
+ os.chmod(myemp,mystat[ST_MODE])
+
+
+ def clear_packages(self):
+ if pkgcache in self.settings[options]:
+ print purging the pkgcache ...
+
+ myemp=self.settings[pkgcache_path]
+ if os.path.isdir(myemp):
+ print Emptying directory,myemp
+
+ stat the dir, delete the dir, recreate the dir
and set
+ the proper perms and ownership
+
+ mystat=os.stat(myemp)
+ #cmd(rm -rf +myemp, Could not remove
existing file: +myemp,env=self.env)
+ shutil.rmtree(myemp)
+ os.makedirs(myemp,0755)
+ os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
+ os.chmod(myemp,mystat[ST_MODE])
+
+
+ def clear_kerncache(self):
+ if kerncache in self.settings[options]:
+ print purging the kerncache ...
+
+ myemp=self.settings[kerncache_path]
+
[gentoo-commits] proj/catalyst:pending commit in: catalyst/targets/
commit: 260e6b68530a0adf30494df2cc78c5edd3dd4afd
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Tue Jan 22 00:10:51 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sat Jun 14 05:21:50 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=260e6b68
Rename all target .py files and classes without _target in them.
This is so they are the named the same as the target .sh files
and work with the now simplified module loading.
---
catalyst/targets/{embedded_target.py = embedded.py} | 2 +-
catalyst/targets/{grp_target.py = grp.py} | 2 +-
catalyst/targets/{livecd_stage1_target.py = livecd_stage1.py} | 2 +-
catalyst/targets/{livecd_stage2_target.py = livecd_stage2.py} | 2 +-
catalyst/targets/{netboot_target.py = netboot.py} | 2 +-
catalyst/targets/{netboot2_target.py = netboot2.py} | 2 +-
catalyst/targets/{stage1_target.py = stage1.py} | 2 +-
catalyst/targets/{stage2_target.py = stage2.py} | 2 +-
catalyst/targets/{stage3_target.py = stage3.py} | 2 +-
catalyst/targets/{stage4_target.py = stage4.py} | 2 +-
catalyst/targets/{tinderbox_target.py = tinderbox.py} | 2 +-
11 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/catalyst/targets/embedded_target.py b/catalyst/targets/embedded.py
similarity index 98%
rename from catalyst/targets/embedded_target.py
rename to catalyst/targets/embedded.py
index aee0f00..3309278 100644
--- a/catalyst/targets/embedded_target.py
+++ b/catalyst/targets/embedded.py
@@ -15,7 +15,7 @@ from catalyst.support import normpath
from catalyst.base.stagebase import StageBase
-class embedded_target(StageBase):
+class embedded(StageBase):
Builder class for embedded target
diff --git a/catalyst/targets/grp_target.py b/catalyst/targets/grp.py
similarity index 99%
rename from catalyst/targets/grp_target.py
rename to catalyst/targets/grp.py
index e3f08a2..0075714 100644
--- a/catalyst/targets/grp_target.py
+++ b/catalyst/targets/grp.py
@@ -14,7 +14,7 @@ from catalyst.support import (CatalystError, normpath,
from catalyst.base.stagebase import StageBase
-class grp_target(StageBase):
+class grp(StageBase):
The builder class for GRP (Gentoo Reference Platform) builds.
diff --git a/catalyst/targets/livecd_stage1_target.py
b/catalyst/targets/livecd_stage1.py
similarity index 98%
rename from catalyst/targets/livecd_stage1_target.py
rename to catalyst/targets/livecd_stage1.py
index 9c74253..661e06e 100644
--- a/catalyst/targets/livecd_stage1_target.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -13,7 +13,7 @@ from catalyst.support import (normpath,
from catalyst.base.stagebase import StageBase
-class livecd_stage1_target(StageBase):
+class livecd_stage1(StageBase):
Builder class for LiveCD stage1.
diff --git a/catalyst/targets/livecd_stage2_target.py
b/catalyst/targets/livecd_stage2.py
similarity index 99%
rename from catalyst/targets/livecd_stage2_target.py
rename to catalyst/targets/livecd_stage2.py
index a4630e6..20f6014 100644
--- a/catalyst/targets/livecd_stage2_target.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -10,7 +10,7 @@ from catalyst.support import (normpath, file_locate,
CatalystError, cmd,
from catalyst.base.stagebase import StageBase
-class livecd_stage2_target(StageBase):
+class livecd_stage2(StageBase):
Builder class for a LiveCD stage2 build.
diff --git a/catalyst/targets/netboot_target.py b/catalyst/targets/netboot.py
similarity index 99%
rename from catalyst/targets/netboot_target.py
rename to catalyst/targets/netboot.py
index 9d92ef2..f753c7f 100644
--- a/catalyst/targets/netboot_target.py
+++ b/catalyst/targets/netboot.py
@@ -12,7 +12,7 @@ from catalyst.support import (CatalystError, normpath,
from catalyst.base.stagebase import StageBase
-class netboot_target(StageBase):
+class netboot(StageBase):
Builder class for a netboot build.
diff --git a/catalyst/targets/netboot2_target.py b/catalyst/targets/netboot2.py
similarity index 99%
rename from catalyst/targets/netboot2_target.py
rename to catalyst/targets/netboot2.py
index 130e2b6..fc8c64c 100644
--- a/catalyst/targets/netboot2_target.py
+++ b/catalyst/targets/netboot2.py
@@ -14,7 +14,7 @@ from catalyst.support import (CatalystError, normpath,
from catalyst.base.stagebase import StageBase
-class netboot2_target(StageBase):
+class netboot2(StageBase):
Builder class for a netboot build, version 2
diff --git a/catalyst/targets/stage1_target.py b/catalyst/targets/stage1.py
similarity index 99%
rename from catalyst/targets/stage1_target.py
rename to catalyst/targets/stage1.py
index 2329b58..e72cbcc 100644
--- a/catalyst/targets/stage1_target.py
+++ b/catalyst/targets/stage1.py
@@ -9,7 +9,7 @@ from catalyst.support import
[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/targets/
commit: a74e870514155653753a1ab56fd25a683822b402
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Sun Jan 20 21:50:23 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sat Jun 14 05:17:36 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=a74e8705
Update module loading for the new python structure, rename snapshot_target to
snapshot
Conflicts:
catalyst/defaults.py
catalyst/main.py
catalyst/targets/stage1_target.py
---
catalyst/main.py | 75 --
catalyst/targets/embedded_target.py| 4 --
catalyst/targets/grp_target.py | 4 --
catalyst/targets/livecd_stage1_target.py | 4 --
catalyst/targets/livecd_stage2_target.py | 4 --
catalyst/targets/netboot2_target.py| 4 --
catalyst/targets/netboot_target.py | 4 --
.../targets/{snapshot_target.py = snapshot.py}| 6 +-
catalyst/targets/stage1_target.py | 4 --
catalyst/targets/stage2_target.py | 4 --
catalyst/targets/stage3_target.py | 4 --
catalyst/targets/stage4_target.py | 5 --
catalyst/targets/tinderbox_target.py | 4 --
13 files changed, 27 insertions(+), 99 deletions(-)
diff --git a/catalyst/main.py b/catalyst/main.py
index 30df771..776a678 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -16,21 +16,20 @@ import os.path
__selfpath__ = os.path.abspath(os.path.dirname(__file__))
-sys.path.append(__selfpath__ + /modules)
from . import __version__
import catalyst.config
import catalyst.util
+from catalyst.contents import ContentsMap, CONTENTS_DEFINITIONS
+from catalyst.defaults import confdefaults, option_messages
+from catalyst.hash_utils import HashMap, HASH_DEFINITIONS
from catalyst.lock import LockInUse
from catalyst.support import CatalystError, find_binary
-from catalyst.defaults import (required_build_targets, valid_build_targets,
- confdefaults)
-from hash_utils import HashMap, HASH_DEFINITIONS
-from contents import ContentsMap, CONTENTS_DEFINITIONS
conf_values={}
+
def usage():
print Usage catalyst [options] [-C variable=value...] [ -s
identifier]
-a --clear-autoresume clear autoresume flags
@@ -135,59 +134,40 @@ def parse_config(myconfig):
print Envscript support enabled.
-def import_modules():
- # import catalyst's own modules
- # (i.e. stage and the arch modules)
- targetmap={}
-
+def import_module(target):
+
+ import catalyst's own modules
+ (i.e. targets and the arch modules)
+
try:
- module_dir = __selfpath__ + /targets/
- for x in required_build_targets:
- try:
- fh=open(module_dir + x + .py)
- module=imp.load_module(x, fh,targets/ + x +
.py,
- (.py, r, imp.PY_SOURCE))
- fh.close()
-
- except IOError:
- raise CatalystError, Can't find + x + .py
plugin in + \
- module_dir
- for x in valid_build_targets:
- try:
- fh=open(module_dir + x + .py)
- module=imp.load_module(x, fh, targets/ + x +
.py,
- (.py, r, imp.PY_SOURCE))
- module.register(targetmap)
- fh.close()
-
- except IOError:
- raise CatalystError,Can't find + x + .py
plugin in + \
- module_dir
-
+ mod_name = catalyst.targets. + target
+ module = __import__(mod_name, [],[], [not empty])
except ImportError as e:
- print !!! catalyst: Python modules not found in +\
- module_dir + ; exiting.
- print e
+ print !!! catalyst: Python module import error: %s % target
+ \
+ in catalyst/targets/ ... exiting.
+ print ERROR was: , e
sys.exit(1)
+ return module
- return targetmap
-def build_target(addlargs, targetmap):
+def build_target(addlargs):
try:
- if addlargs[target] not in targetmap:
- raise CatalystError,Target \+addlargs[target]+\
not available.
-
- mytarget=targetmap[addlargs[target]](conf_values, addlargs)
-
- mytarget.run()
+ module = import_module(addlargs[target])
+ target = getattr(module, addlargs[target])(conf_values,
addlargs)
+ except AttributeError:
+ raise CatalystError(
+
[gentoo-commits] proj/catalyst:pending commit in: catalyst/base/, catalyst/targets/
commit: fe2a3542674d2a387bd20055145f3f8af6d720fd
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Tue Feb 12 03:51:52 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sat Jun 14 05:07:40 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=fe2a3542
Move base stage and target files to thier own sub-pkg
Fix an indent error in grp_target.py
Conflicts:
catalyst/targets/embedded_target.py
catalyst/targets/grp_target.py
catalyst/targets/livecd_stage1_target.py
catalyst/targets/livecd_stage2_target.py
catalyst/targets/netboot2_target.py
catalyst/targets/netboot_target.py
catalyst/targets/stage1_target.py
catalyst/targets/stage2_target.py
catalyst/targets/stage3_target.py
catalyst/targets/stage4_target.py
catalyst/targets/tinderbox_target.py
---
catalyst/base/__init__.py | 1 +
catalyst/{targets = base}/clearbase.py| 0
catalyst/{targets = base}/genbase.py | 0
.../generic_stage_target.py = base/stagebase.py} | 8 ++---
catalyst/{targets = base}/targetbase.py | 0
catalyst/targets/embedded_target.py| 12
catalyst/targets/grp_target.py | 34 +-
catalyst/targets/livecd_stage1_target.py | 21 -
catalyst/targets/livecd_stage2_target.py | 13 +
catalyst/targets/netboot2_target.py| 17 +++
catalyst/targets/netboot_target.py | 15 ++
catalyst/targets/snapshot_target.py| 4 +--
catalyst/targets/stage1_target.py | 17 ++-
catalyst/targets/stage2_target.py | 15 ++
catalyst/targets/stage3_target.py | 12
catalyst/targets/stage4_target.py | 8 ++---
catalyst/targets/tinderbox_target.py | 11 ---
17 files changed, 113 insertions(+), 75 deletions(-)
diff --git a/catalyst/base/__init__.py b/catalyst/base/__init__.py
new file mode 100644
index 000..8b13789
--- /dev/null
+++ b/catalyst/base/__init__.py
@@ -0,0 +1 @@
+
diff --git a/catalyst/targets/clearbase.py b/catalyst/base/clearbase.py
similarity index 100%
rename from catalyst/targets/clearbase.py
rename to catalyst/base/clearbase.py
diff --git a/catalyst/targets/genbase.py b/catalyst/base/genbase.py
similarity index 100%
rename from catalyst/targets/genbase.py
rename to catalyst/base/genbase.py
diff --git a/catalyst/targets/generic_stage_target.py
b/catalyst/base/stagebase.py
similarity index 99%
rename from catalyst/targets/generic_stage_target.py
rename to catalyst/base/stagebase.py
index 83c7a2f..173cc68 100644
--- a/catalyst/targets/generic_stage_target.py
+++ b/catalyst/base/stagebase.py
@@ -14,13 +14,13 @@ from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS,
TARGET_MOUNT_DEFAULTS,
PORT_LOGDIR_CLEAN)
from catalyst.support import (CatalystError, msg, file_locate, normpath,
touch, cmd, warn, list_bashify, read_makeconf, read_from_clst, ismount)
-from catalyst.targets.targetbase import TargetBase
-from catalyst.targets.clearbase import ClearBase
-from catalyst.targets.genbase import GenBase
+from catalyst.base.targetbase import TargetBase
+from catalyst.base.clearbase import ClearBase
+from catalyst.base.genbase import GenBase
from catalyst.lock import LockDir
-class generic_stage_target(TargetBase, ClearBase, GenBase):
+class StageBase(TargetBase, ClearBase, GenBase):
This class does all of the chroot setup, copying of files, etc. It is
the driver class for pretty much everything that Catalyst does.
diff --git a/catalyst/targets/targetbase.py b/catalyst/base/targetbase.py
similarity index 100%
rename from catalyst/targets/targetbase.py
rename to catalyst/base/targetbase.py
diff --git a/catalyst/targets/embedded_target.py
b/catalyst/targets/embedded_target.py
index 7cee7a6..528d545 100644
--- a/catalyst/targets/embedded_target.py
+++ b/catalyst/targets/embedded_target.py
@@ -10,12 +10,12 @@ ROOT=/tmp/submerge emerge --something foo bar .
# NOTE: That^^ docstring has influence catalyst-spec(5) man page generation.
-import os,string,imp,types,shutil
-from catalyst.support import *
-from generic_stage_target import *
-from stat import *
-class embedded_target(generic_stage_target):
+from catalyst.support import normpath
+
+from catalyst.base.stagebase import StageBase
+
+class embedded_target(StageBase):
Builder class for embedded target
@@ -27,7 +27,7 @@ class embedded_target(generic_stage_target):
if embedded/fs-type in addlargs:
self.valid_values.append(embedded/fs-ops)
- generic_stage_target.__init__(self,spec,addlargs)
+ StageBase.__init__(self,spec,addlargs)
[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: 00ef5c3294e8b7530aa8881649c5bc21b7def377 Author: Brian Dolbec dolsen AT gentoo DOT org AuthorDate: Mon Jan 21 23:34:32 2013 + Commit: Brian Dolbec brian.dolbec AT gmail DOT com CommitDate: Sat Jun 14 05:18:53 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=00ef5c32 fix options being reset by a config file --- catalyst/main.py | 19 +++ 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/catalyst/main.py b/catalyst/main.py index 776a678..1446cf9 100644 --- a/catalyst/main.py +++ b/catalyst/main.py @@ -204,9 +204,7 @@ def main(): usage() sys.exit(2) - # initialize it if it's not already - if 'options' not in conf_values: - conf_values['options'] = set() + options = set() run = False for o, a in opts: @@ -237,7 +235,7 @@ def main(): myspecfile=a if o in (-F, --fetchonly): - conf_values['options'].add(fetch) + options.add(fetch) if o in (-v, --verbose): conf_values[VERBOSE]=1 @@ -253,16 +251,18 @@ def main(): mycmdline.append(version_stamp=+a) if o in (-p, --purge): - conf_values['options'].add(purge) + options.add(purge) if o in (-P, --purgeonly): - conf_values['options'].add(purgeonly) + options.add(purgeonly) if o in (-T, --purgetmponly): - conf_values['options'].add(purgetmponly) + options.add(purgetmponly) if o in (-a, --clear-autoresume): - conf_values['options'].add(clear-autoresume) + options.add(clear-autoresume) + + #print MAIN: cli options =, options if not run: print !!! catalyst: please specify one of either -f or -C\n @@ -272,6 +272,9 @@ def main(): # import configuration file and import our main module using those settings parse_config(myconfig) + conf_values[options].update(options) + #print MAIN: conf_values['options'] =, conf_values[options] + # initialize our contents generator contents_map = ContentsMap(CONTENTS_DEFINITIONS) conf_values[contents_map] = contents_map
[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: 0ace9c0fde5e10f16441869798a14ab80e4ca90b
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Wed Jan 23 04:57:05 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sat Jun 14 05:45:33 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=0ace9c0f
reduce 2 operations into one simpler one
---
catalyst/config.py | 6 +-
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/catalyst/config.py b/catalyst/config.py
index 460bbd5..8b23342 100644
--- a/catalyst/config.py
+++ b/catalyst/config.py
@@ -54,11 +54,7 @@ class ParserBase:
# Skip any blank lines
if not myline: continue
- # Look for separator
- msearch = myline.find(self.key_value_separator)
-
- # If separator found assume its a new key
- if msearch != -1:
+ if self.key_value_separator in myline:
# Split on the first occurence of the separator
creating two strings in the array mobjs
mobjs = myline.split(self.key_value_separator,
1)
mobjs[1] = mobjs[1].strip().strip('')
[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: 1c8c0c7265ebe13a68399989d34c714fbfdf8f54 Author: Brian Dolbec dolsen AT gentoo DOT org AuthorDate: Sun Jan 20 19:22:27 2013 + Commit: Brian Dolbec brian.dolbec AT gmail DOT com CommitDate: Sat Jun 14 04:55:10 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=1c8c0c72 some spacing and comment and indent cleanup, etc. Conflicts: catalyst/support.py --- catalyst/support.py | 405 +++- 1 file changed, 213 insertions(+), 192 deletions(-) diff --git a/catalyst/support.py b/catalyst/support.py index feaa645..6f66d3c 100644 --- a/catalyst/support.py +++ b/catalyst/support.py @@ -1,12 +1,19 @@ -import sys,string,os,types,re,signal,traceback,time -#import md5,sha +import sys +import string +import os +import types +import re +import signal +import traceback +import time from catalyst.defaults import verbosity, valid_config_file_values selinux_capable = False #userpriv_capable = (os.getuid() == 0) #fakeroot_capable = False + BASH_BINARY = /bin/bash # set it to 0 for the soft limit, 1 for the hard limit @@ -25,35 +32,35 @@ spawned_pids = [] def cleanup(pids,block_exceptions=True): -function to go through and reap the list of pids passed to it -global spawned_pids -if type(pids) == int: -pids = [pids] -for x in pids: -try: -os.kill(x,signal.SIGTERM) -if os.waitpid(x,os.WNOHANG)[1] == 0: -# feisty bugger, still alive. -os.kill(x,signal.SIGKILL) -os.waitpid(x,0) - -except OSError, oe: -if block_exceptions: -pass -if oe.errno not in (10,3): -raise oe -except SystemExit: -raise -except Exception: -if block_exceptions: -pass -try:spawned_pids.remove(x) -except IndexError: pass - - - -# a function to turn a string of non-printable characters into a string of -# hex characters + function to go through and reap the list of pids passed to it + global spawned_pids + if type(pids) == int: + pids = [pids] + for x in pids: + try: + os.kill(x,signal.SIGTERM) + if os.waitpid(x,os.WNOHANG)[1] == 0: + # feisty bugger, still alive. + os.kill(x,signal.SIGKILL) + os.waitpid(x,0) + except OSError, oe: + if block_exceptions: + pass + if oe.errno not in (10,3): + raise oe + except SystemExit: + raise + except Exception: + if block_exceptions: + pass + try: + spawned_pids.remove(x) + except IndexError: + pass + + +# a function to turn a string of non-printable characters +# into a string of hex characters def hexify(str): hexStr = string.hexdigits r = '' @@ -61,7 +68,6 @@ def hexify(str): i = ord(ch) r = r + hexStr[(i 4) 0xF] + hexStr[i 0xF] return r -# hexify() def read_from_clst(file): @@ -77,7 +83,6 @@ def read_from_clst(file): myline = myline + line myf.close() return myline -# read_from_clst def list_bashify(mylist): @@ -92,6 +97,7 @@ def list_bashify(mylist): mypack=string.join(mypack) return mypack + def list_to_string(mylist): if type(mylist)==types.StringType: mypack=[mylist] @@ -104,6 +110,7 @@ def list_to_string(mylist): mypack=string.join(mypack) return mypack + class CatalystError(Exception): def __init__(self, message): if message: @@ -115,84 +122,83 @@ class CatalystError(Exception): print !!! catalyst: +message print + def die(msg=None): warn(msg) sys.exit(1) + def warn(msg): print !!! catalyst: +msg + def find_binary(myc): look through the environmental path for an executable file named whatever myc is -# this sucks. badly. -p=os.getenv(PATH) -if p == None: -return None -for x in p.split(:): -#if it exists, and is executable -if os.path.exists(%s/%s % (x,myc)) and os.stat(%s/%s % (x,myc))[0] 0x0248: -return %s/%s %
[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/base/
commit: 78adef623a95a9adca03be204df15597052264df
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Tue Jan 22 08:39:18 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sat Jun 14 05:42:03 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=78adef62
rename a make.conf key to make_conf due to bash variable name restrictions
Conflicts:
catalyst/base/stagebase.py
catalyst/defaults.py
---
catalyst/base/stagebase.py | 10 +-
catalyst/defaults.py | 5 -
2 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 5cc6244..b67a32a 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1020,7 +1020,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
def chroot_setup(self):
self.makeconf=read_makeconf(normpath(self.settings[chroot_path]+
- self.settings[make.conf]))
+ self.settings[make_conf]))
self.override_cbuild()
self.override_chost()
self.override_cflags()
@@ -1068,7 +1068,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
Modify and write out make.conf (for the chroot)
makepath = normpath(self.settings[chroot_path] +
- self.settings[make.conf])
+ self.settings[make_conf])
cmd(rm -f + makepath,\
Could not remove + makepath, env=self.env)
myf=open(makepath, w)
@@ -1122,9 +1122,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
myf.close()
makepath = normpath(self.settings[chroot_path] +
- self.settings[make.conf])
+ self.settings[make_conf])
cmd(cp + makepath + + makepath + .catalyst,\
- Could not backup +
self.settings[make.conf],env=self.env)
+ Could not backup +
self.settings[make_conf],env=self.env)
touch(self.settings[autoresume_path]+chroot_setup)
def fsscript(self):
@@ -1172,7 +1172,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
cmd(rm -rf + self.settings[chroot_path] +
self.settings[local_overlay],
Could not remove +
self.settings[local_overlay], env=self.env)
cmd(sed -i '/^PORTDIR_OVERLAY/d'
+self.settings[chroot_path]+\
- /etc/portage/make.conf,\
+ self.settings[make_conf],\
Could not remove PORTDIR_OVERLAY from
make.conf,env=self.env)
Clean up old and obsoleted files in /etc
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 2b5eee4..7a9bb23 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -25,7 +25,8 @@ confdefaults={
hash_function: crc32,
icecream: /var/cache/icecream,
local_overlay: /usr/local/portage,
- options: ,
+ make_conf: /etc/portage/make.conf,
+ options: set(),
packagedir: /usr/portage/packages,
portdir: /usr/portage,
port_tmpdir: /var/tmp/portage,
@@ -42,6 +43,8 @@ PORT_LOGDIR_CLEAN = \
TARGET_MOUNT_DEFAULTS = {
ccache: /var/tmp/ccache,
+
+target_mounts = {
dev: /dev,
devpts: /dev/pts,
distdir: /usr/portage/distfiles,
[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: 752882a02214f07b7027b65535e6b4d0eaf6ba50 Author: Brian Dolbec dolsen AT gentoo DOT org AuthorDate: Tue Jan 22 08:34:41 2013 + Commit: Brian Dolbec brian.dolbec AT gmail DOT com CommitDate: Sat Jun 14 05:24:29 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=752882a0 FIXME! Comment out a small code block causing TypeError. This was also short circuiting another large code block. FIXME This whole class seems overly complicated with TOO MANY nested try:excepts: --- catalyst/lock.py | 23 +-- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/catalyst/lock.py b/catalyst/lock.py index ef00b49..d073861 100644 --- a/catalyst/lock.py +++ b/catalyst/lock.py @@ -27,11 +27,17 @@ class LockDir: locking_method=fcntl.flock lock_dirs_in_use=[] die_on_failed_lock=True + def __del__(self): + #print Lock.__del__() 1 self.clean_my_hardlocks() + #print Lock.__del__() 2 self.delete_lock_from_path_list() + #print Lock.__del__() 3 if self.islocked(): + #print Lock.__del__() 4 self.fcntl_unlock() + #print Lock.__del__() finnished def __init__(self,lockdir): self.locked=False @@ -179,7 +185,9 @@ class LockDir: unlinkfile = 1 if not os.path.exists(self.lockfile): print lockfile does not exist '%s' % self.lockfile + #print fcntl_unlock() , self.myfd:, self.myfd, type(self.myfd) if (self.myfd != None): + #print fcntl_unlock() trying to close it try: os.close(self.myfd) self.myfd=None @@ -193,11 +201,13 @@ class LockDir: unlinkfile = 1 self.locking_method(self.myfd,fcntl.LOCK_UN) except SystemExit, e: - raise + raise e except Exception, e: - os.close(self.myfd) - self.myfd=None - raise IOError, Failed to unlock file '%s'\n % self.lockfile + #if self.myfd is not None: + #print fcntl_unlock() trying to close, self.myfd + #os.close(self.myfd) + #self.myfd=None + #raise IOError, Failed to unlock file '%s'\n%s % (self.lockfile, str(e)) try: # This sleep call was added to allow other processes that are # waiting for a lock to be able to grab it before it is deleted. @@ -224,7 +234,7 @@ class LockDir: # if DEBUG in self.settings: # print Unlinked lockfile... except SystemExit, e: - raise + raise e except Exception, e: # We really don't care... Someone else has the lock. # So it is their problem now. @@ -236,7 +246,7 @@ class LockDir: # open fd closed automatically on them. #if type(lockfilename) == types.StringType: #os.close(myfd) - + #print fcntl_unlock() trying a last ditch close, self.myfd if (self.myfd != None): os.close(self.myfd) self.myfd=None @@ -421,6 +431,7 @@ class LockDir: pass return results + if __name__ == __main__: def lock_work():
[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/base/
commit: 4b44a58d52982c92aecd7a223ec3baf27988c4d8
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Tue Feb 12 04:06:01 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sat Jun 14 05:41:58 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=4b44a58d
Break out more repeated (path1 + path2)'s...
Just do it once and use the temp variable.
Comment out some debug print's.
Fix options conversion for export to bash.
Conflicts:
catalyst/base/stagebase.py
---
catalyst/base/stagebase.py | 52 ++
catalyst/defaults.py | 1 +
2 files changed, 31 insertions(+), 22 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 173cc68..5cc6244 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -631,11 +631,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.setup_environment()
- if os.path.exists(self.settings[sharedir]+\
- /targets/support/kill-chroot-pids.sh):
- cmd(/bin/bash +self.settings[sharedir]+\
- /targets/support/kill-chroot-pids.sh,\
- kill-chroot-pids script failed.,env=self.env)
+ killcmd = normpath(self.settings[sharedir] +
+ self.settings[shdir] + /support/kill-chroot-pids.sh)
+ if os.path.exists(killcmd):
+ cmd(killcmd, kill-chroot-pids script
failed.,env=self.env)
def mount_safety_check(self):
@@ -647,10 +646,10 @@ class StageBase(TargetBase, ClearBase, GenBase):
if not os.path.exists(self.settings[chroot_path]):
return
- print self.mounts =, self.mounts
+ #print self.mounts =, self.mounts
for x in self.mounts:
target = normpath(self.settings[chroot_path] +
self.target_mounts[x])
- print mount_safety_check() x =, x, target
+ #print mount_safety_check() x =, x, target
if not os.path.exists(target):
continue
@@ -1020,8 +1019,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
Couldn't umount one or more bind-mounts;
aborting for safety.
def chroot_setup(self):
- self.makeconf=read_makeconf(self.settings[chroot_path]+\
- /etc/portage/make.conf)
+
self.makeconf=read_makeconf(normpath(self.settings[chroot_path]+
+ self.settings[make.conf]))
self.override_cbuild()
self.override_chost()
self.override_cflags()
@@ -1035,8 +1034,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
else:
print Setting up chroot...
-
#self.makeconf=read_makeconf(self.settings[chroot_path]+/etc/portage/make.conf)
-
cmd(cp /etc/resolv.conf +
self.settings[chroot_path] + /etc/,
Could not copy resolv.conf into
place.,env=self.env)
@@ -1070,10 +1067,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
Could not copy
/etc/hosts,env=self.env)
Modify and write out make.conf (for the chroot)
- cmd(rm -f
+self.settings[chroot_path]+/etc/portage/make.conf,\
- Could not remove
+self.settings[chroot_path]+\
- /etc/portage/make.conf,env=self.env)
-
myf=open(self.settings[chroot_path]+/etc/portage/make.conf,w)
+ makepath = normpath(self.settings[chroot_path] +
+ self.settings[make.conf])
+ cmd(rm -f + makepath,\
+ Could not remove + makepath, env=self.env)
+ myf=open(makepath, w)
myf.write(# These settings were set by the catalyst
build script that automatically\n# built this stage.\n)
myf.write(# Please consult
/usr/share/portage/config/make.conf.example for a more\n# detailed example.\n)
if CFLAGS in self.settings:
@@ -1123,10 +1121,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
myf.write('PORTDIR_OVERLAY=/usr/local/portage\n')
myf.close()
- cmd(cp
+self.settings[chroot_path]+/etc/portage/make.conf +\
-
self.settings[chroot_path]+/etc/portage/make.conf.catalyst,\
- Could not backup
/etc/portage/make.conf,env=self.env)
- touch(chroot_setup_resume)
+
[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: 0349b7547b55e6d3810a18691fb38675374ffd7e Author: Brian Dolbec dolsen AT gentoo DOT org AuthorDate: Fri Jan 25 04:00:24 2013 + Commit: Brian Dolbec brian.dolbec AT gmail DOT com CommitDate: Sat Jun 14 05:46:12 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=0349b754 Extend ParserBase to do variable substitution. Also add embedded variable substitiution to default settings. Why are we not using python's built-in ConfigParser? We'll migrate it later. --- catalyst/config.py | 3 +++ catalyst/main.py | 6 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/catalyst/config.py b/catalyst/config.py index 8b23342..bf3c7c6 100644 --- a/catalyst/config.py +++ b/catalyst/config.py @@ -66,6 +66,9 @@ class ParserBase: # Start a new array using the first element of mobjs cur_array = [mobjs[0]] if mobjs[1]: + # do any variable substitiution embeded in it with + # the values already obtained + mobjs[1] = mobjs[1] % values if self.multiple_values: # split on white space creating additional array elements # subarray = white_space.split(mobjs[1]) diff --git a/catalyst/main.py b/catalyst/main.py index 1446cf9..5fa3f1b 100644 --- a/catalyst/main.py +++ b/catalyst/main.py @@ -116,7 +116,6 @@ def parse_config(myconfig): # add our python base directory to use for loading target arch's conf_values[PythonDir] = __selfpath__ - # print out any options messages for opt in conf_values['options']: if opt in option_messages: @@ -133,6 +132,11 @@ def parse_config(myconfig): if envscript in myconf: print Envscript support enabled. + # take care of any variable substitutions that may be left + for x in list(conf_values): + if isinstance(conf_values[x], str): + conf_values[x] = conf_values[x] % conf_values + def import_module(target):
[gentoo-commits] proj/catalyst:pending commit in: catalyst/targets/
commit: 938e55e150d20d8f5ea03541498bd28e49e8f36d Author: Brian Dolbec dolsen AT gentoo DOT org AuthorDate: Sat Apr 5 07:26:58 2014 + Commit: Brian Dolbec brian.dolbec AT gmail DOT com CommitDate: Sat Apr 5 07:41:57 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=938e55e1 generic_stage_target: Add a trailing / to the cp /etc/reolv.conf command --- catalyst/targets/generic_stage_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/targets/generic_stage_target.py b/catalyst/targets/generic_stage_target.py index 05c61e8..1f26e65 100644 --- a/catalyst/targets/generic_stage_target.py +++ b/catalyst/targets/generic_stage_target.py @@ -1025,7 +1025,7 @@ class generic_stage_target(generic_target): #self.makeconf=read_makeconf(self.settings[chroot_path]+/etc/portage/make.conf) - cmd(cp /etc/resolv.conf +self.settings[chroot_path]+/etc,\ + cmd(cp /etc/resolv.conf + self.settings[chroot_path] + /etc/, Could not copy resolv.conf into place.,env=self.env) Copy over the envscript, if applicable
[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/targets/
commit: 6c8aac54653310e32672e7d2d5f4eef963a19c7c
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Fri Jan 3 18:46:55 2014 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Wed Apr 2 20:04:23 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=6c8aac54
Initial separation and creation of contents.py
---
catalyst/contents.py | 87
catalyst/main.py | 8 ++-
catalyst/support.py | 52 ---
catalyst/targets/generic_stage_target.py | 3 +-
4 files changed, 96 insertions(+), 54 deletions(-)
diff --git a/catalyst/contents.py b/catalyst/contents.py
new file mode 100644
index 000..79ef9a6
--- /dev/null
+++ b/catalyst/contents.py
@@ -0,0 +1,87 @@
+
+from collections import namedtuple
+from subprocess import Popen, PIPE
+
+from support import CatalystError, warn
+
+
+# use ContentsMap.fields for the value legend
+# Key:[function, cmd]
+CONTENTS_DEFINITIONS = {
+ # 'find' is disabled because it requires the source path, which is not
+ # always available
+ #find :[calc_contents,find %(path)s],
+ tar_tv:[calc_contents,tar tvf %(file)s],
+ tar_tvz:[calc_contents,tar tvzf %(file)s],
+ tar_tvj:[calc_contents,tar -I lbzip2 -tvf %(file)s],
+ isoinfo_l:[calc_contents,isoinfo -l -i %(file)s],
+ # isoinfo_f should be a last resort only
+ isoinfo_f:[calc_contents,isoinfo -f -i %(file)s],
+}
+
+
+class ContentsMap(object):
+ '''Class to encompass all known commands to list
+ the contents of an archive'''
+
+
+ fields = ['func', 'cmd']
+
+
+ def __init__(self, defs=None):
+ '''Class init
+
+ @param defs: dictionary of Key:[function, cmd]
+ '''
+ if defs is None:
+ defs = {}
+ #self.contents = {}
+ self.contents_map = {}
+
+ # create the archive type namedtuple classes
+ for name in list(defs):
+ #obj = self.contents[name] = namedtuple(name,
self.fields)
+ obj = namedtuple(name, self.fields)
+ obj.__slots__ = ()
+ self.contents_map[name] = obj._make(defs[name])
+ del obj
+
+
+ def generate_contents(self, file_, getter=auto, verbose=False):
+ try:
+ archive = getter
+ if archive == 'auto' and file_.endswith('.iso'):
+ archive = 'isoinfo_l'
+ if (archive in ['tar_tv','auto']):
+ if file_.endswith('.tgz') or
file_.endswith('.tar.gz'):
+ archive = 'tar_tvz'
+ elif file_.endswith('.tbz2') or
file_.endswith('.tar.bz2'):
+ archive = 'tar_tvj'
+ elif file_.endswith('.tar'):
+ archive = 'tar_tv'
+
+ if archive == 'auto':
+ warn('File %r has unknown type for automatic
detection.'
+ % (file_, ))
+ return None
+ else:
+ getter = archive
+ func = getattr(self, '_%s_' %
self.contents_map[getter].func)
+ return func(file_,
self.contents_map[getter].cmd, verbose)
+ except:
+ raise CatalystError,\
+ Error generating contents, is appropriate
utility +\
+ (%s) installed on your system? \
+ % (self.contents_map[getter].cmd)
+
+
+ @staticmethod
+ def _calc_contents_(file_, cmd, verbose):
+ _cmd = (cmd % {'file': file_ }).split()
+ proc = Popen(_cmd, stdout=PIPE, stderr=PIPE)
+ results = proc.communicate()
+ result = \n.join(results)
+ if verbose:
+ print result
+ return result
+
diff --git a/catalyst/main.py b/catalyst/main.py
index 7bcf2cb..4146bca 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -25,6 +25,7 @@ from catalyst.support import (required_build_targets,
valid_build_targets, CatalystError, find_binary, LockInUse)
from hash_utils import HashMap, HASH_DEFINITIONS
+from contents import ContentsMap, CONTENTS_DEFINITIONS
@@ -184,7 +185,8 @@ def parse_config(myconfig):
if digests in myconf:
conf_values[digests]=myconf[digests]
if contents in myconf:
- conf_values[contents]=myconf[contents]
+ # replace '-' with '_' (for compatibility with existing configs)
+
[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/targets/
commit: 264f30494cde1eb44aae89a9872bf12936856252
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Sat Jan 4 01:04:10 2014 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Wed Apr 2 20:04:24 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=264f3049
Move generic_stage_targets.py constants to defaults.py
Rename the source and target mounts defaults to remove
the double plural.
---
catalyst/defaults.py | 27 +
catalyst/targets/generic_stage_target.py | 34
2 files changed, 31 insertions(+), 30 deletions(-)
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index b1dbda4..748d1dd 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -21,3 +21,30 @@ valid_config_file_values.extend([PKGCACHE, KERNCACHE,
CCACHE, DISTCC,
verbosity = 1
+PORT_LOGDIR_CLEAN = \
+ 'find ${PORT_LOGDIR} -type f ! -name summary.log* -mtime +30
-delete'
+
+TARGET_MOUNT_DEFAULTS = {
+ ccache: /var/tmp/ccache,
+ dev: /dev,
+ devpts: /dev/pts,
+ distdir: /usr/portage/distfiles,
+ icecream: /usr/lib/icecc/bin,
+ kerncache: /tmp/kerncache,
+ packagedir: /usr/portage/packages,
+ portdir: /usr/portage,
+ port_tmpdir: /var/tmp/portage,
+ port_logdir: /var/log/portage,
+ proc: /proc,
+ shm: /dev/shm,
+ }
+
+SOURCE_MOUNT_DEFAULTS = {
+ dev: /dev,
+ devpts: /dev/pts,
+ distdir: /usr/portage/distfiles,
+ portdir: /usr/portage,
+ port_tmpdir: tmpfs,
+ proc: /proc,
+ shm: shmfs,
+ }
diff --git a/catalyst/targets/generic_stage_target.py
b/catalyst/targets/generic_stage_target.py
index de4842c..05c61e8 100644
--- a/catalyst/targets/generic_stage_target.py
+++ b/catalyst/targets/generic_stage_target.py
@@ -4,34 +4,8 @@ from generic_target import *
from stat import *
from catalyst.lock import LockDir
-
-PORT_LOGDIR_CLEAN = \
- 'find ${PORT_LOGDIR} -type f ! -name summary.log* -mtime +30
-delete'
-
-TARGET_MOUNTS_DEFAULTS = {
- ccache: /var/tmp/ccache,
- dev: /dev,
- devpts: /dev/pts,
- distdir: /usr/portage/distfiles,
- icecream: /usr/lib/icecc/bin,
- kerncache: /tmp/kerncache,
- packagedir: /usr/portage/packages,
- portdir: /usr/portage,
- port_tmpdir: /var/tmp/portage,
- port_logdir: /var/log/portage,
- proc: /proc,
- shm: /dev/shm,
- }
-
-SOURCE_MOUNTS_DEFAULTS = {
- dev: /dev,
- devpts: /dev/pts,
- distdir: /usr/portage/distfiles,
- portdir: /usr/portage,
- port_tmpdir: tmpfs,
- proc: /proc,
- shm: shmfs,
- }
+from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS,
+ PORT_LOGDIR_CLEAN)
# for convienience
pjoin = os.path.join
@@ -208,11 +182,11 @@ class generic_stage_target(generic_target):
Setup our mount points
# initialize our target mounts.
- self.target_mounts = TARGET_MOUNTS_DEFAULTS.copy()
+ self.target_mounts = TARGET_MOUNT_DEFAULTS.copy()
self.mounts = [proc, dev, portdir, distdir,
port_tmpdir]
# initialize our source mounts
- self.mountmap = SOURCE_MOUNTS_DEFAULTS.copy()
+ self.mountmap = SOURCE_MOUNT_DEFAULTS.copy()
# update them from settings
self.mountmap[distdir] = self.settings[distdir]
if SNAPCACHE not in self.settings:
[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: 380085b1b64a80cd6fc8518309e8ca9e31246ba3 Author: Brian Dolbec dolsen AT gentoo DOT org AuthorDate: Mon Jan 14 03:32:29 2013 + Commit: Brian Dolbec brian.dolbec AT gmail DOT com CommitDate: Wed Apr 2 20:04:23 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=380085b1 Fix undefined variable: RLIMIT_NOFILE It was not imported from resource, it was also not used correctly. --- catalyst/support.py | 12 +++- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/catalyst/support.py b/catalyst/support.py index e25394e..5abf614 100644 --- a/catalyst/support.py +++ b/catalyst/support.py @@ -6,14 +6,16 @@ selinux_capable = False #fakeroot_capable = False BASH_BINARY = /bin/bash +# set it to 0 for the soft limit, 1 for the hard limit +DESIRED_RLIMIT = 0 try: -import resource -max_fd_limit=resource.getrlimit(RLIMIT_NOFILE) + import resource + max_fd_limit=resource.getrlimit(resource.RLIMIT_NOFILE)[DESIRED_RLIMIT] except SystemExit, e: -raise + raise except: -# hokay, no resource module. -max_fd_limit=256 + # hokay, no resource module. + max_fd_limit=256 # pids this process knows of. spawned_pids = []
[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/targets/
commit: e337857ffb575302368d13fc63c5d511aa27c244
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Fri Jan 3 18:42:26 2014 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Wed Apr 2 20:04:23 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=e337857f
Initial separation and creation of a hash_utils.py module
---
catalyst/hash_utils.py | 137 +++
catalyst/main.py | 30 ---
catalyst/support.py | 62 --
catalyst/targets/generic_stage_target.py | 36
catalyst/targets/livecd_stage2_target.py | 4 +-
catalyst/targets/stage2_target.py| 7 +-
6 files changed, 184 insertions(+), 92 deletions(-)
diff --git a/catalyst/hash_utils.py b/catalyst/hash_utils.py
new file mode 100644
index 000..b575ace
--- /dev/null
+++ b/catalyst/hash_utils.py
@@ -0,0 +1,137 @@
+
+import os
+from collections import namedtuple
+from subprocess import Popen, PIPE
+
+from support import CatalystError
+
+
+# Use HashMap.fields for the value legend
+# fields = [func, cmd, args, id]
+HASH_DEFINITIONS = {
+ adler32 :[calc_hash2, shash, [-a, ADLER32], ADLER32],
+ crc32:[calc_hash2, shash, [-a, CRC32], CRC32],
+ crc32b :[calc_hash2, shash, [-a, CRC32B], CRC32B],
+ gost :[calc_hash2, shash, [-a, GOST], GOST],
+ haval128 :[calc_hash2, shash, [-a, HAVAL128], HAVAL128],
+ haval160 :[calc_hash2, shash, [-a, HAVAL160], HAVAL160],
+ haval192 :[calc_hash2, shash, [-a, HAVAL192], HAVAL192],
+ haval224 :[calc_hash2, shash, [-a, HAVAL224], HAVAL224],
+ haval256 :[calc_hash2, shash, [-a, HAVAL256], HAVAL256],
+ md2 :[calc_hash2, shash, [-a, MD2], MD2],
+ md4 :[calc_hash2, shash, [-a, MD4], MD4],
+ md5 :[calc_hash2, shash, [-a, MD5], MD5],
+ ripemd128:[calc_hash2, shash, [-a, RIPEMD128], RIPEMD128],
+ ripemd160:[calc_hash2, shash, [-a, RIPEMD160], RIPEMD160],
+ ripemd256:[calc_hash2, shash, [-a, RIPEMD256], RIPEMD256],
+ ripemd320:[calc_hash2, shash, [-a, RIPEMD320], RIPEMD320],
+ sha1 :[calc_hash2, shash, [-a, SHA1], SHA1],
+ sha224 :[calc_hash2, shash, [-a, SHA224], SHA224],
+ sha256 :[calc_hash2, shash, [-a, SHA256], SHA256],
+ sha384 :[calc_hash2, shash, [-a, SHA384], SHA384],
+ sha512 :[calc_hash2, shash, [-a, SHA512], SHA512],
+ snefru128:[calc_hash2, shash, [-a, SNEFRU128], SNEFRU128],
+ snefru256:[calc_hash2, shash, [-a, SNEFRU256], SNEFRU256],
+ tiger:[calc_hash2, shash, [-a, TIGER], TIGER],
+ tiger128 :[calc_hash2, shash, [-a, TIGER128], TIGER128],
+ tiger160 :[calc_hash2, shash, [-a, TIGER160], TIGER160],
+ whirlpool:[calc_hash2, shash, [-a, WHIRLPOOL], WHIRLPOOL],
+ }
+
+
+class HashMap(object):
+ '''Class for handling
+ Catalyst's hash generation'''
+
+ fields = [func, cmd, args, id]
+
+
+ def __init__(self, hashes=None):
+ '''Class init
+
+ @param hashes: dictionary of Key:[function, cmd, cmd_args,
Print string]
+ @param fields: list of ordered field names for the hashes
+ eg: [func, cmd, args, id]
+ '''
+ if hashes is None:
+ hashes = {}
+ self.hash_map = {}
+
+ # create the hash definition namedtuple classes
+ for name in list(hashes):
+ obj = namedtuple(name, self.fields)
+ obj.__slots__ = ()
+ self.hash_map[name] = obj._make(hashes[name])
+ del obj
+
+
+ def generate_hash(self, file_, hash_=crc32, verbose=False):
+ '''Prefered method of generating a hash for the passed in file_
+
+ @param file_: the file to generate the hash for
+ @param hash_: the hash algorythm to use
+ @param verbose: boolean
+ @returns the hash result
+ '''
+ try:
+ return getattr(self, self.hash_map[hash_].func)(
+ file_,
+ hash_,
+ verbose
+ )
+ except:
+ raise CatalystError,Error generating hash, is
appropriate + \
+ utility installed on your system?
+
+
+ def calc_hash(self, file_, hash_, verbose=False):
+ '''
+ Calculate the hash for file_
+
+ @param file_: the file to generate the hash for
+ @param hash_: the hash algorythm to use
+ @param verbose: boolean
+ @returns the hash result
+ '''
+ _hash = self.hash_map[hash_]
+ args = [_hash.cmd]
+
[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: 69c1839e501cbbf05c1790b267ed1b5b356091c0
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Sat Jan 19 05:51:35 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Wed Apr 2 20:04:24 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=69c1839e
Move confdefaults out of main.py
---
catalyst/defaults.py | 17 +
catalyst/main.py | 20 +++-
2 files changed, 20 insertions(+), 17 deletions(-)
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 748d1dd..b83e4f5 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -21,6 +21,22 @@ valid_config_file_values.extend([PKGCACHE, KERNCACHE,
CCACHE, DISTCC,
verbosity = 1
+confdefaults={
+ distdir: /usr/portage/distfiles,
+ hash_function: crc32,
+ icecream: /var/cache/icecream,
+ local_overlay: /usr/local/portage,
+ options: ,
+ packagedir: /usr/portage/packages,
+ portdir: /usr/portage,
+ port_tmpdir: /var/tmp/portage,
+ repo_name: portage,
+ sharedir: /usr/lib/catalyst,
+ snapshot_cache: /var/tmp/catalyst/snapshot_cache,
+ snapshot_name: portage-,
+ storedir: /var/tmp/catalyst,
+ }
+
PORT_LOGDIR_CLEAN = \
'find ${PORT_LOGDIR} -type f ! -name summary.log* -mtime +30
-delete'
@@ -48,3 +64,4 @@ SOURCE_MOUNT_DEFAULTS = {
proc: /proc,
shm: shmfs,
}
+
diff --git a/catalyst/main.py b/catalyst/main.py
index e2ef976..5748d31 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -22,7 +22,8 @@ from . import __version__
import catalyst.config
import catalyst.util
from catalyst.support import CatalystError, find_binary, LockInUse
-from catalyst.defaults import required_build_targets, valid_build_targets
+from catalyst.defaults import (required_build_targets, valid_build_targets,
+ confdefaults)
from hash_utils import HashMap, HASH_DEFINITIONS
from contents import ContentsMap, CONTENTS_DEFINITIONS
@@ -70,21 +71,6 @@ def parse_config(myconfig):
myconf={}
config_file=
- confdefaults = {
- distdir: /usr/portage/distfiles,
- hash_function: crc32,
- icecream: /var/cache/icecream,
- local_overlay: /usr/local/portage,
- options: ,
- packagedir: /usr/portage/packages,
- portdir: /usr/portage,
- repo_name: portage,
- sharedir: /usr/share/catalyst,
- snapshot_name: portage-,
- snapshot_cache: /var/tmp/catalyst/snapshot_cache,
- storedir: /var/tmp/catalyst,
- }
-
# first, try the one passed (presumably from the cmdline)
if myconfig:
if os.path.exists(myconfig):
@@ -117,7 +103,7 @@ def parse_config(myconfig):
sys.exit(1)
# now, load up the values into conf_values so that we can use them
- for x in confdefaults.keys():
+ for x in list(confdefaults):
if x in myconf:
print Setting,x,to config file value
\+myconf[x]+\
conf_values[x]=myconf[x]
[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: 876edf3306f3d1c739f60199bef6082e4ac96459 Author: Brian Dolbec dolsen AT gentoo DOT org AuthorDate: Fri Jan 3 18:48:31 2014 + Commit: Brian Dolbec brian.dolbec AT gmail DOT com CommitDate: Wed Apr 2 20:04:23 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=876edf33 main.py: print the output of an ImportError to help in debugging. --- catalyst/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/catalyst/main.py b/catalyst/main.py index 4146bca..bba3cba 100644 --- a/catalyst/main.py +++ b/catalyst/main.py @@ -227,9 +227,10 @@ def import_modules(): raise CatalystError,Can't find + x + .py plugin in + \ module_dir - except ImportError: + except ImportError as e: print !!! catalyst: Python modules not found in +\ module_dir + ; exiting. + print e sys.exit(1) return targetmap
[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: a17349d30b9b94d12463be700558f6c5379c312f Author: Brian Dolbec dolsen AT gentoo DOT org AuthorDate: Fri Jan 3 18:48:31 2014 + Commit: Brian Dolbec brian.dolbec AT gmail DOT com CommitDate: Sat Mar 22 18:01:30 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=a17349d3 main.py: print the output of an ImportError to help in debugging. --- catalyst/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/catalyst/main.py b/catalyst/main.py index 4146bca..bba3cba 100644 --- a/catalyst/main.py +++ b/catalyst/main.py @@ -227,9 +227,10 @@ def import_modules(): raise CatalystError,Can't find + x + .py plugin in + \ module_dir - except ImportError: + except ImportError as e: print !!! catalyst: Python modules not found in +\ module_dir + ; exiting. + print e sys.exit(1) return targetmap
[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/targets/
commit: 8eabfb7bef8e2b4ffd805a7cb5914c8ea33b8254
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Sat Jan 4 01:04:10 2014 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sat Mar 22 18:01:31 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=8eabfb7b
Move generic_stage_targets.py constants to defaults.py
Rename the source and target mounts defaults to remove
the double plural.
---
catalyst/defaults.py | 27 +
catalyst/targets/generic_stage_target.py | 34
2 files changed, 31 insertions(+), 30 deletions(-)
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index b1dbda4..748d1dd 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -21,3 +21,30 @@ valid_config_file_values.extend([PKGCACHE, KERNCACHE,
CCACHE, DISTCC,
verbosity = 1
+PORT_LOGDIR_CLEAN = \
+ 'find ${PORT_LOGDIR} -type f ! -name summary.log* -mtime +30
-delete'
+
+TARGET_MOUNT_DEFAULTS = {
+ ccache: /var/tmp/ccache,
+ dev: /dev,
+ devpts: /dev/pts,
+ distdir: /usr/portage/distfiles,
+ icecream: /usr/lib/icecc/bin,
+ kerncache: /tmp/kerncache,
+ packagedir: /usr/portage/packages,
+ portdir: /usr/portage,
+ port_tmpdir: /var/tmp/portage,
+ port_logdir: /var/log/portage,
+ proc: /proc,
+ shm: /dev/shm,
+ }
+
+SOURCE_MOUNT_DEFAULTS = {
+ dev: /dev,
+ devpts: /dev/pts,
+ distdir: /usr/portage/distfiles,
+ portdir: /usr/portage,
+ port_tmpdir: tmpfs,
+ proc: /proc,
+ shm: shmfs,
+ }
diff --git a/catalyst/targets/generic_stage_target.py
b/catalyst/targets/generic_stage_target.py
index de4842c..05c61e8 100644
--- a/catalyst/targets/generic_stage_target.py
+++ b/catalyst/targets/generic_stage_target.py
@@ -4,34 +4,8 @@ from generic_target import *
from stat import *
from catalyst.lock import LockDir
-
-PORT_LOGDIR_CLEAN = \
- 'find ${PORT_LOGDIR} -type f ! -name summary.log* -mtime +30
-delete'
-
-TARGET_MOUNTS_DEFAULTS = {
- ccache: /var/tmp/ccache,
- dev: /dev,
- devpts: /dev/pts,
- distdir: /usr/portage/distfiles,
- icecream: /usr/lib/icecc/bin,
- kerncache: /tmp/kerncache,
- packagedir: /usr/portage/packages,
- portdir: /usr/portage,
- port_tmpdir: /var/tmp/portage,
- port_logdir: /var/log/portage,
- proc: /proc,
- shm: /dev/shm,
- }
-
-SOURCE_MOUNTS_DEFAULTS = {
- dev: /dev,
- devpts: /dev/pts,
- distdir: /usr/portage/distfiles,
- portdir: /usr/portage,
- port_tmpdir: tmpfs,
- proc: /proc,
- shm: shmfs,
- }
+from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS,
+ PORT_LOGDIR_CLEAN)
# for convienience
pjoin = os.path.join
@@ -208,11 +182,11 @@ class generic_stage_target(generic_target):
Setup our mount points
# initialize our target mounts.
- self.target_mounts = TARGET_MOUNTS_DEFAULTS.copy()
+ self.target_mounts = TARGET_MOUNT_DEFAULTS.copy()
self.mounts = [proc, dev, portdir, distdir,
port_tmpdir]
# initialize our source mounts
- self.mountmap = SOURCE_MOUNTS_DEFAULTS.copy()
+ self.mountmap = SOURCE_MOUNT_DEFAULTS.copy()
# update them from settings
self.mountmap[distdir] = self.settings[distdir]
if SNAPCACHE not in self.settings:
[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: 02b002e091994999abd74d9c34e5ad9a34d41e17 Author: Brian Dolbec dolsen AT gentoo DOT org AuthorDate: Sun Mar 2 16:43:16 2014 + Commit: Brian Dolbec brian.dolbec AT gmail DOT com CommitDate: Sun Mar 2 22:54:04 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=02b002e0 catalyst/support.py: Update for snakeoil API change. --- catalyst/support.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/support.py b/catalyst/support.py index 316dfa3..5e7ce92 100644 --- a/catalyst/support.py +++ b/catalyst/support.py @@ -618,8 +618,8 @@ def read_makeconf(mymakeconffile): if os.path.exists(mymakeconffile): try: try: - import snakeoil.fileutils - return snakeoil.fileutils.read_bash_dict(mymakeconffile, sourcing_command=source) + import snakeoil.bash #import snakeoil.fileutils + return snakeoil.bash.read_bash_dict(mymakeconffile, sourcing_command=source) except ImportError: try: import portage.util
[gentoo-commits] proj/catalyst:pending commit in: catalyst/targets/
commit: 509d4ead51fbb861cfa783131b22907c8b8636db Author: Brian Dolbec dolsen AT gentoo DOT org AuthorDate: Sat Jan 11 00:13:06 2014 + Commit: Brian Dolbec brian.dolbec AT gmail DOT com CommitDate: Sat Mar 22 18:01:29 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=509d4ead generic_stage_target.py: Fix an intermittent snapshot_cache_path keyerror --- catalyst/targets/generic_stage_target.py | 11 ++- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/catalyst/targets/generic_stage_target.py b/catalyst/targets/generic_stage_target.py index 9c39d00..eaf2c1f 100644 --- a/catalyst/targets/generic_stage_target.py +++ b/catalyst/targets/generic_stage_target.py @@ -215,13 +215,14 @@ class generic_stage_target(generic_target): self.mountmap = SOURCE_MOUNTS_DEFAULTS.copy() # update them from settings self.mountmap[distdir] = self.settings[distdir] - self.mountmap[portdir] = normpath(/.join([ - self.settings[snapshot_cache_path], - self.settings[repo_name], - ])) if SNAPCACHE not in self.settings: self.mounts.remove(portdir) - #self.mountmap[portdir] = None + self.mountmap[portdir] = None + else: + self.mountmap[portdir] = normpath(/.join([ + self.settings[snapshot_cache_path], + self.settings[repo_name], + ])) if os.uname()[0] == Linux: self.mounts.append(devpts) self.mounts.append(shm)
[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/targets/
commit: ee9831262599e0ee02d8374e677c94462f87b006
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Fri Jan 3 18:42:26 2014 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sat Mar 22 18:01:29 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=ee983126
Initial separation and creation of a hash_utils.py module
---
catalyst/hash_utils.py | 137 +++
catalyst/main.py | 30 ---
catalyst/support.py | 62 --
catalyst/targets/generic_stage_target.py | 36
catalyst/targets/livecd_stage2_target.py | 4 +-
catalyst/targets/stage2_target.py| 7 +-
6 files changed, 184 insertions(+), 92 deletions(-)
diff --git a/catalyst/hash_utils.py b/catalyst/hash_utils.py
new file mode 100644
index 000..b575ace
--- /dev/null
+++ b/catalyst/hash_utils.py
@@ -0,0 +1,137 @@
+
+import os
+from collections import namedtuple
+from subprocess import Popen, PIPE
+
+from support import CatalystError
+
+
+# Use HashMap.fields for the value legend
+# fields = [func, cmd, args, id]
+HASH_DEFINITIONS = {
+ adler32 :[calc_hash2, shash, [-a, ADLER32], ADLER32],
+ crc32:[calc_hash2, shash, [-a, CRC32], CRC32],
+ crc32b :[calc_hash2, shash, [-a, CRC32B], CRC32B],
+ gost :[calc_hash2, shash, [-a, GOST], GOST],
+ haval128 :[calc_hash2, shash, [-a, HAVAL128], HAVAL128],
+ haval160 :[calc_hash2, shash, [-a, HAVAL160], HAVAL160],
+ haval192 :[calc_hash2, shash, [-a, HAVAL192], HAVAL192],
+ haval224 :[calc_hash2, shash, [-a, HAVAL224], HAVAL224],
+ haval256 :[calc_hash2, shash, [-a, HAVAL256], HAVAL256],
+ md2 :[calc_hash2, shash, [-a, MD2], MD2],
+ md4 :[calc_hash2, shash, [-a, MD4], MD4],
+ md5 :[calc_hash2, shash, [-a, MD5], MD5],
+ ripemd128:[calc_hash2, shash, [-a, RIPEMD128], RIPEMD128],
+ ripemd160:[calc_hash2, shash, [-a, RIPEMD160], RIPEMD160],
+ ripemd256:[calc_hash2, shash, [-a, RIPEMD256], RIPEMD256],
+ ripemd320:[calc_hash2, shash, [-a, RIPEMD320], RIPEMD320],
+ sha1 :[calc_hash2, shash, [-a, SHA1], SHA1],
+ sha224 :[calc_hash2, shash, [-a, SHA224], SHA224],
+ sha256 :[calc_hash2, shash, [-a, SHA256], SHA256],
+ sha384 :[calc_hash2, shash, [-a, SHA384], SHA384],
+ sha512 :[calc_hash2, shash, [-a, SHA512], SHA512],
+ snefru128:[calc_hash2, shash, [-a, SNEFRU128], SNEFRU128],
+ snefru256:[calc_hash2, shash, [-a, SNEFRU256], SNEFRU256],
+ tiger:[calc_hash2, shash, [-a, TIGER], TIGER],
+ tiger128 :[calc_hash2, shash, [-a, TIGER128], TIGER128],
+ tiger160 :[calc_hash2, shash, [-a, TIGER160], TIGER160],
+ whirlpool:[calc_hash2, shash, [-a, WHIRLPOOL], WHIRLPOOL],
+ }
+
+
+class HashMap(object):
+ '''Class for handling
+ Catalyst's hash generation'''
+
+ fields = [func, cmd, args, id]
+
+
+ def __init__(self, hashes=None):
+ '''Class init
+
+ @param hashes: dictionary of Key:[function, cmd, cmd_args,
Print string]
+ @param fields: list of ordered field names for the hashes
+ eg: [func, cmd, args, id]
+ '''
+ if hashes is None:
+ hashes = {}
+ self.hash_map = {}
+
+ # create the hash definition namedtuple classes
+ for name in list(hashes):
+ obj = namedtuple(name, self.fields)
+ obj.__slots__ = ()
+ self.hash_map[name] = obj._make(hashes[name])
+ del obj
+
+
+ def generate_hash(self, file_, hash_=crc32, verbose=False):
+ '''Prefered method of generating a hash for the passed in file_
+
+ @param file_: the file to generate the hash for
+ @param hash_: the hash algorythm to use
+ @param verbose: boolean
+ @returns the hash result
+ '''
+ try:
+ return getattr(self, self.hash_map[hash_].func)(
+ file_,
+ hash_,
+ verbose
+ )
+ except:
+ raise CatalystError,Error generating hash, is
appropriate + \
+ utility installed on your system?
+
+
+ def calc_hash(self, file_, hash_, verbose=False):
+ '''
+ Calculate the hash for file_
+
+ @param file_: the file to generate the hash for
+ @param hash_: the hash algorythm to use
+ @param verbose: boolean
+ @returns the hash result
+ '''
+ _hash = self.hash_map[hash_]
+ args = [_hash.cmd]
+
[gentoo-commits] proj/catalyst:pending commit in: catalyst/targets/
commit: f37553c47eef08bf064e7923ff11b4ec154db895
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Mon Dec 30 06:14:48 2013 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sat Mar 22 18:01:21 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=f37553c4
Fix autoresume file paths to only be configured once.
Use: pjoin as a shorter alias to os.path.join()
---
catalyst/targets/generic_stage_target.py | 175 +--
1 file changed, 95 insertions(+), 80 deletions(-)
diff --git a/catalyst/targets/generic_stage_target.py
b/catalyst/targets/generic_stage_target.py
index 2b3d7ce..9c39d00 100644
--- a/catalyst/targets/generic_stage_target.py
+++ b/catalyst/targets/generic_stage_target.py
@@ -33,6 +33,9 @@ SOURCE_MOUNTS_DEFAULTS = {
shm: shmfs,
}
+# for convienience
+pjoin = os.path.join
+
class generic_stage_target(generic_target):
@@ -334,9 +337,10 @@ class generic_stage_target(generic_target):
def set_target_path(self):
self.settings[target_path]=normpath(self.settings[storedir]+\
/builds/+self.settings[target_subpath]+.tar.bz2)
- if AUTORESUME in self.settings\
- and os.path.exists(self.settings[autoresume_path]+\
- setup_target_path):
+ setup_target_path_resume =
pjoin(self.settings[autoresume_path],
+ setup_target_path)
+ if AUTORESUME in self.settings and \
+ os.path.exists(setup_target_path_resume):
print \
Resume point detected, skipping target path
setup operation...
else:
@@ -348,7 +352,7 @@ class generic_stage_target(generic_target):
# cmd(rm -f +self.settings[target_path],\
# Could not remove existing file: \
#
+self.settings[target_path],env=self.env)
-
touch(self.settings[autoresume_path]+setup_target_path)
+ touch(setup_target_path_resume)
if not
os.path.exists(self.settings[storedir]+/builds/):
os.makedirs(self.settings[storedir]+/builds/)
@@ -484,10 +488,12 @@ class generic_stage_target(generic_target):
self.chroot_lock=LockDir(self.settings[chroot_path])
def set_autoresume_path(self):
-
self.settings[autoresume_path]=normpath(self.settings[storedir]+\
- /tmp/+self.settings[rel_type]+/+.autoresume-+\
-
self.settings[target]+-+self.settings[subarch]+-+\
- self.settings[version_stamp]+/)
+ self.settings[autoresume_path] = normpath(pjoin(
+ self.settings[storedir], tmp,
self.settings[rel_type],
+ .autoresume-%s-%s-%s
+ %(self.settings[target], self.settings[subarch],
+ self.settings[version_stamp])
+ ))
if AUTORESUME in self.settings:
print The autoresume path is +
self.settings[autoresume_path]
if not os.path.exists(self.settings[autoresume_path]):
@@ -673,8 +679,8 @@ class generic_stage_target(generic_target):
def unpack(self):
unpack=True
-
clst_unpack_hash=read_from_clst(self.settings[autoresume_path]+\
- unpack)
+ unpack_resume = pjoin(self.settings[autoresume_path],
unpack)
+ clst_unpack_hash=read_from_clst(unpack_resume)
if SEEDCACHE in self.settings:
if os.path.isdir(self.settings[source_path]):
@@ -720,7 +726,7 @@ class generic_stage_target(generic_target):
if AUTORESUME in self.settings:
if os.path.isdir(self.settings[source_path]) \
- and
os.path.exists(self.settings[autoresume_path]+unpack):
+ and os.path.exists(unpack_resume):
Autoresume is valid, SEEDCACHE is valid
unpack=False
invalid_snapshot=False
@@ -732,8 +738,7 @@ class generic_stage_target(generic_target):
invalid_snapshot=True
elif os.path.isdir(self.settings[source_path]) \
- and not
os.path.exists(self.settings[autoresume_path]+\
- unpack):
+ and not os.path.exists(unpack_resume):
Autoresume is invalid, SEEDCACHE
unpack=True
invalid_snapshot=False
@@
[gentoo-commits] proj/catalyst:pending commit in: catalyst/
commit: 612761103527c0cc2801aff7cdb28b9f4a82d9d2 Author: Brian Dolbec dolsen AT gentoo DOT org AuthorDate: Sat Jan 4 01:02:09 2014 + Commit: Brian Dolbec brian.dolbec AT gmail DOT com CommitDate: Sat Mar 22 18:01:30 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=61276110 Creation of a new defaults.py catalyst/support.py: Move some defaults to a new defaults file. --- catalyst/defaults.py | 23 +++ catalyst/main.py | 5 ++--- catalyst/support.py | 35 +++ 3 files changed, 28 insertions(+), 35 deletions(-) diff --git a/catalyst/defaults.py b/catalyst/defaults.py new file mode 100644 index 000..b1dbda4 --- /dev/null +++ b/catalyst/defaults.py @@ -0,0 +1,23 @@ + + +# these should never be touched +required_build_targets = [generic_target, generic_stage_target] + +# new build types should be added here +valid_build_targets = [stage1_target, stage2_target, stage3_target, + stage4_target, grp_target, livecd_stage1_target, livecd_stage2_target, + embedded_target, tinderbox_target, snapshot_target, netboot_target, + netboot2_target + ] + +required_config_file_values = [storedir, sharedir, distdir, portdir] + +valid_config_file_values = required_config_file_values[:] +valid_config_file_values.extend([PKGCACHE, KERNCACHE, CCACHE, DISTCC, + ICECREAM, ENVSCRIPT, AUTORESUME, FETCH, CLEAR_AUTORESUME, + options, DEBUG, VERBOSE, PURGE, PURGEONLY, SNAPCACHE, + snapshot_cache, hash_function, digests, contents, SEEDCACHE + ]) + +verbosity = 1 + diff --git a/catalyst/main.py b/catalyst/main.py index bba3cba..e2ef976 100644 --- a/catalyst/main.py +++ b/catalyst/main.py @@ -21,9 +21,8 @@ sys.path.append(__selfpath__ + /modules) from . import __version__ import catalyst.config import catalyst.util -from catalyst.support import (required_build_targets, - valid_build_targets, CatalystError, find_binary, LockInUse) - +from catalyst.support import CatalystError, find_binary, LockInUse +from catalyst.defaults import required_build_targets, valid_build_targets from hash_utils import HashMap, HASH_DEFINITIONS from contents import ContentsMap, CONTENTS_DEFINITIONS diff --git a/catalyst/support.py b/catalyst/support.py index 5abf614..4fe4603 100644 --- a/catalyst/support.py +++ b/catalyst/support.py @@ -1,6 +1,9 @@ import sys,string,os,types,re,signal,traceback,time #import md5,sha + +from catalyst.defaults import verbosity, valid_config_file_values + selinux_capable = False #userpriv_capable = (os.getuid() == 0) #fakeroot_capable = False @@ -80,38 +83,6 @@ def read_from_clst(file): return myline # read_from_clst -# these should never be touched -required_build_targets=[generic_target,generic_stage_target] - -# new build types should be added here -valid_build_targets=[stage1_target,stage2_target,stage3_target,stage4_target,grp_target, - livecd_stage1_target,livecd_stage2_target,embedded_target, - tinderbox_target,snapshot_target,netboot_target,netboot2_target] - -required_config_file_values=[storedir,sharedir,distdir,portdir] -valid_config_file_values=required_config_file_values[:] -valid_config_file_values.append(PKGCACHE) -valid_config_file_values.append(KERNCACHE) -valid_config_file_values.append(CCACHE) -valid_config_file_values.append(DISTCC) -valid_config_file_values.append(ICECREAM) -valid_config_file_values.append(ENVSCRIPT) -valid_config_file_values.append(AUTORESUME) -valid_config_file_values.append(FETCH) -valid_config_file_values.append(CLEAR_AUTORESUME) -valid_config_file_values.append(options) -valid_config_file_values.append(DEBUG) -valid_config_file_values.append(VERBOSE) -valid_config_file_values.append(PURGE) -valid_config_file_values.append(PURGEONLY) -valid_config_file_values.append(SNAPCACHE) -valid_config_file_values.append(snapshot_cache) -valid_config_file_values.append(hash_function) -valid_config_file_values.append(digests) -valid_config_file_values.append(contents) -valid_config_file_values.append(SEEDCACHE) - -verbosity=1 def list_bashify(mylist): if type(mylist)==types.StringType:
[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/targets/
commit: 6398b227f47e2af2f310ad8af53c6e26680c60b0
Author: Brian Dolbec dolsen AT gentoo DOT org
AuthorDate: Fri Jan 3 18:46:55 2014 +
Commit: Brian Dolbec brian.dolbec AT gmail DOT com
CommitDate: Sat Mar 22 18:01:30 2014 +
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=6398b227
Initial separation and creation of contents.py
---
catalyst/contents.py | 87
catalyst/main.py | 8 ++-
catalyst/support.py | 52 ---
catalyst/targets/generic_stage_target.py | 3 +-
4 files changed, 96 insertions(+), 54 deletions(-)
diff --git a/catalyst/contents.py b/catalyst/contents.py
new file mode 100644
index 000..79ef9a6
--- /dev/null
+++ b/catalyst/contents.py
@@ -0,0 +1,87 @@
+
+from collections import namedtuple
+from subprocess import Popen, PIPE
+
+from support import CatalystError, warn
+
+
+# use ContentsMap.fields for the value legend
+# Key:[function, cmd]
+CONTENTS_DEFINITIONS = {
+ # 'find' is disabled because it requires the source path, which is not
+ # always available
+ #find :[calc_contents,find %(path)s],
+ tar_tv:[calc_contents,tar tvf %(file)s],
+ tar_tvz:[calc_contents,tar tvzf %(file)s],
+ tar_tvj:[calc_contents,tar -I lbzip2 -tvf %(file)s],
+ isoinfo_l:[calc_contents,isoinfo -l -i %(file)s],
+ # isoinfo_f should be a last resort only
+ isoinfo_f:[calc_contents,isoinfo -f -i %(file)s],
+}
+
+
+class ContentsMap(object):
+ '''Class to encompass all known commands to list
+ the contents of an archive'''
+
+
+ fields = ['func', 'cmd']
+
+
+ def __init__(self, defs=None):
+ '''Class init
+
+ @param defs: dictionary of Key:[function, cmd]
+ '''
+ if defs is None:
+ defs = {}
+ #self.contents = {}
+ self.contents_map = {}
+
+ # create the archive type namedtuple classes
+ for name in list(defs):
+ #obj = self.contents[name] = namedtuple(name,
self.fields)
+ obj = namedtuple(name, self.fields)
+ obj.__slots__ = ()
+ self.contents_map[name] = obj._make(defs[name])
+ del obj
+
+
+ def generate_contents(self, file_, getter=auto, verbose=False):
+ try:
+ archive = getter
+ if archive == 'auto' and file_.endswith('.iso'):
+ archive = 'isoinfo_l'
+ if (archive in ['tar_tv','auto']):
+ if file_.endswith('.tgz') or
file_.endswith('.tar.gz'):
+ archive = 'tar_tvz'
+ elif file_.endswith('.tbz2') or
file_.endswith('.tar.bz2'):
+ archive = 'tar_tvj'
+ elif file_.endswith('.tar'):
+ archive = 'tar_tv'
+
+ if archive == 'auto':
+ warn('File %r has unknown type for automatic
detection.'
+ % (file_, ))
+ return None
+ else:
+ getter = archive
+ func = getattr(self, '_%s_' %
self.contents_map[getter].func)
+ return func(file_,
self.contents_map[getter].cmd, verbose)
+ except:
+ raise CatalystError,\
+ Error generating contents, is appropriate
utility +\
+ (%s) installed on your system? \
+ % (self.contents_map[getter].cmd)
+
+
+ @staticmethod
+ def _calc_contents_(file_, cmd, verbose):
+ _cmd = (cmd % {'file': file_ }).split()
+ proc = Popen(_cmd, stdout=PIPE, stderr=PIPE)
+ results = proc.communicate()
+ result = \n.join(results)
+ if verbose:
+ print result
+ return result
+
diff --git a/catalyst/main.py b/catalyst/main.py
index 7bcf2cb..4146bca 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -25,6 +25,7 @@ from catalyst.support import (required_build_targets,
valid_build_targets, CatalystError, find_binary, LockInUse)
from hash_utils import HashMap, HASH_DEFINITIONS
+from contents import ContentsMap, CONTENTS_DEFINITIONS
@@ -184,7 +185,8 @@ def parse_config(myconfig):
if digests in myconf:
conf_values[digests]=myconf[digests]
if contents in myconf:
- conf_values[contents]=myconf[contents]
+ # replace '-' with '_' (for compatibility with existing configs)
+
