Attached is a patch.
It could be broken into two patches for clarity. Please review and apply if
appropriate.
patch#1: add --container-size flag
Using this flag, one can cause the os.img that sits in the squashfs.img, to be a
larger sparse file than the filesystem contained in it. By default, the old
behavior happens. One somewhat unrealistic, but real justification for this,
would be booting a livecd on a system with 16G of ram. Without this flag, you
would be limited to only writing 2G of data (given the existing f7-livecd-i686
choice of a 4G uncompressed-size, and about 2.1G of data). With this flag and a
container of say 1T or 100G, there would be no negative consequences, but the
livecd user would be able to online resize2fs their root filesystem upward if
they liked.
The more realistic usefulness comes if through a persistence implementation, the
overlay file is not a 512M file stored in tmpfs, but an 8G file stored on an
ipod. Everything mentioned above applies.
patch#2: add --cleanup-deleted flag
Using this flag, about 25% is added to the build time, but any files that were
created and deleted in the install_root during installation, will not result in
wasted space on the resulting iso. This happens because of the nature of the
filesystem living on a sparse file. Anecdotally, this could be used to respin
the fedora7-livecd-i686, from its existing 700MB, to 664MB. Or more likely, add
another 100MB or so of uncompressed software onto it.
This is of even greater importance as people spin their own livecds, and do
intensive things during the installation, that might cause even more wasted space.
Also, this somewhat lays the foundation for a 'turboinstaller' that improves
current liveinst efficiency by 10-25% (for cdrom vs usbflash respectively), as
mentioned in bug 248082 (pending efficiency tests with extra device mapper layers)
Please review. Any comments, suggestions, or criticisms are welcome. I still
consider myself a relative novice when it comes to python.
peace...
-dmc
--- livecd-creator.git20070718 2007-07-18 22:20:39.000000000 -0500
+++ livecd-creator.git_dmc 2007-07-18 22:20:39.000000000 -0500
@@ -144,9 +144,10 @@
self.mounted = True
class SparseExt3LoopbackMount(LoopbackMount):
- def __init__(self, lofile, mountdir, size, fslabel, tmpdir):
+ def __init__(self, lofile, mountdir, size, container_size, fslabel, tmpdir):
LoopbackMount.__init__(self, lofile, mountdir, fstype = "ext3")
self.size = size
+ self.container_size = container_size
self.fslabel = fslabel
def _createSparseFile(self):
@@ -156,14 +157,15 @@
# create the sparse file
fd = os.open(self.lofile, os.O_WRONLY | os.O_CREAT)
- off = long(self.size * 1024L * 1024L)
+ off = long(self.container_size * 1024L * 1024L)
os.lseek(fd, off, 0)
os.write(fd, '\x00')
os.close(fd)
def _formatFilesystem(self):
rc = subprocess.call(["/sbin/mkfs.ext3", "-F", "-L", self.fslabel,
- "-m", "1", self.lofile])
+ "-m", "1", "-b", "4096", self.lofile,
+ "%d" % ( self.size * 1024L * 1024L / 4096L )])
if rc != 0:
raise MountError("Error creating ext3 filesystem")
rc = subprocess.call(["/sbin/tune2fs", "-c0", "-i0", "-Odir_index",
@@ -383,7 +385,7 @@
fstab.write("sysfs /sys sysfs defaults 0 0\n")
fstab.close()
- def setup(self, image_size, base_on = None):
+ def setup(self, image_size, container_size, base_on = None):
"""setup target ext3 file system in preparation for an install"""
# setup temporary build dirs
@@ -410,6 +412,7 @@
"%s/install_root"
%(self.build_dir,),
image_size,
+ container_size,
self.fs_label,
self.tmpdir)
@@ -468,6 +471,10 @@
b.umount()
if self.instloop:
+ (xxbsize, xxfrsize, xxblocks, xxbfree,
+ xxbavail, xxfiles, xxffree, xxfavail,
+ xxflag, xxnamemax) = os.statvfs(self.build_dir + "/install_root")
+ print >> sys.stderr, "Installation target uncompressed data size is %d MB" % ( ( xxfrsize * xxblocks - xxbsize * xxbfree ) / ( 1024L * 1024L ) )
self.instloop.cleanup()
self.instloop = None
@@ -916,7 +923,96 @@
shutil.move("%s/data/os.img" %(self.build_dir,),
"%s/out/ext3fs.img" %(self.build_dir,))
+ # cleanupDeleted removes unused data from the sparse ext3 os image file.
+ # The process involves: resize2fs-to-minimal, truncation,
+ # resize2fs-to-uncompressed-size, and finally resparsification back to
+ # container-size
+ #
+ # Anecdotally, on one particular build system, building an iso very
+ # similar to the stock f7-i686-livecd, cleanupDeleted incurred a
+ # 25% build time penalty, i.e. 24m -> 30m.
+ # about 2m was spent in figuring out the minimal resize amount
+ # about 2.5m was spent in the cp --sparse
+ # about 1.5m was spent in the resize2fs minimal
+ # the resize2fs maximal was only a few seconds
+ #
+ # The output changed from 697M to 661M. Which matches what I got
+ # when I manually reconstructed the official f7livecd and got 700->664
+ #
+ # The 2.5m cp --sparse is somewhat silly as it changes the data size
+ # of the sparse file from 2062 to 2036, and because of compression only
+ # resulted in a final savings of a third to half a megabyte. But hey,
+ # when you're trying to squeeze just a little more into a mass shipping
+ # CD, I think it's worth it.
+ def cleanupDeleted(self, uncompressed_size, container_size):
+ # e2fsck -f is required by resize2fs
+ subprocess.call(["/sbin/e2fsck", "-f", "os.img"],
+ cwd="%s/data" %(self.build_dir,),
+ env={"PWD": "%s/data" %(self.build_dir,)})
+
+ # resize2fs doesn't have any kind of minimal setting, so start
+ # with 1MB and increment by 1MB until it works. This happens
+ # quite quickly. Maybe figure out a better method someday.
+ FNULL = os.open('/dev/null', os.O_WRONLY)
+ newsize = 1
+ done = 0
+ while done == 0:
+ # note: might be nice to use a buffer rather than devnull
+ # so that the final successful output can be printed.
+ resize2fs_retval = subprocess.call(["/sbin/resize2fs", "os.img",
+ "%dM" %(newsize,)],
+ cwd="%s/data" %(self.build_dir,),
+ env={"PWD": "%s/data" %(self.build_dir,)},
+ stdout=FNULL,
+ stderr=FNULL)
+
+ if not resize2fs_retval:
+ done = 1
+ print >> sys.stderr, "installation target minimized container size is %d MB" % newsize
+ else:
+ newsize = newsize + 1
+ if newsize > uncompressed_size:
+ raise InstallationError("unexpected error with resize2fs in cleanup-deleted pass")
+
+ os.close(FNULL)
+
+ # truncate the unused excess portion of the container file
+ fd = os.open("%s/data/os.img" %(self.build_dir,), os.O_WRONLY )
+ os.ftruncate(fd, newsize * 1024L * 1024L)
+ os.close(fd)
+
+ # NOTE: cp --sparse=always will shrink os.img here another 1-2%
+ # Not exactly sure why. In compressed size, I got this result
+ # 692828160 vs 692422656 not even half a meg... But hey.
+ try:
+ subprocess.call(["/bin/cp", "-a", "--sparse=always", "os.img",
+ "os.img.sparse"],
+ cwd="%s/data" %(self.build_dir,),
+ env={"PWD": "%s/data" %(self.build_dir,)})
+ os.unlink("%s/data/os.img" %(self.build_dir,))
+ os.rename("%s/data/os.img.sparse" %(self.build_dir,),
+ "%s/data/os.img" %(self.build_dir,))
+
+ except:
+ raise InstallationError("Error while cleaning up deleted files: %s" %(e,))
+
+ # resize back to uncompressed-size
+ resize2fs_retval = subprocess.call(["/sbin/resize2fs", "os.img",
+ "%dM" %(uncompressed_size,)],
+ cwd="%s/data" %(self.build_dir,),
+ env={"PWD": "%s/data" %(self.build_dir,)})
+
+
+ # resparsify back to container size
+ fd = os.open("%s/data/os.img" %(self.build_dir,), os.O_WRONLY )
+ off = long(container_size * 1024L * 1024L)
+ os.lseek(fd, off, 0)
+ os.write(fd, '\x00')
+ os.close(fd)
+
+
def package(self):
+
self.createSquashFS()
self.createIso()
@@ -931,6 +1027,8 @@
[--fslabel=<label>]
[--skip-compression]
[--uncompressed-size=<size-in-MB>]
+ [--container-size=<size-in-MB>]
+ [--cleanup-deleted]
[--shell]
[--tmpdir=<tmpdir>]
@@ -944,6 +1042,10 @@
--skip-compression : Don't compress the image
--prelink : Prelink the image
--uncompressed-size : Size of uncompressed fs in MB (default: 4096)
+ --container-size : Size of uncompressed fs container in MB
+ (default: same as uncompressed-size)
+ --cleanup-deleted : Use a couple resize2fs passes to remove blocks used
+ by files deleted during installation
--shell : Start a shell in the chroot for post-configuration
--tmpdir : Temporary directory to use (default: /var/tmp)
@@ -974,6 +1076,8 @@
self.skip_compression = False
self.skip_prelink = True
self.uncompressed_size = 4096
+ self.container_size = self.uncompressed_size
+ self.cleanup_deleted = False
self.give_shell = False
self.tmpdir = "/var/tmp"
@@ -983,7 +1087,10 @@
["help", "repo=", "base-on=", "package=",
"exclude-package=", "fslabel=", "config=",
"skip-compression", "uncompressed-size=",
- "shell", "no-prelink", "prelink","tmpdir="])
+ "container-size=", "cleanup-deleted",
+ "shell", "no-prelink", "prelink",
+ "tmpdir="])
+
except getopt.GetoptError, msg:
raise Usage(msg)
@@ -1007,6 +1114,12 @@
if o in ("-u", "--uncompressed-size"):
options.uncompressed_size = int(a)
continue
+ if o in ("--container-size"):
+ options.container_size = int(a)
+ continue
+ if o in ("--cleanup-deleted",):
+ options.cleanup_deleted = True
+ continue
if o in ("-c", "--config"):
options.kscfg = a
if not os.path.isfile(options.kscfg):
@@ -1082,7 +1195,9 @@
try:
target.parse(options.kscfg)
- target.setup(options.uncompressed_size, options.base_on)
+ target.setup(options.uncompressed_size,
+ options.container_size,
+ options.base_on)
target.install()
@@ -1091,9 +1206,18 @@
print "----------------------------------"
target.launchShell()
+
target.unmount()
+ # note: it might be more graceful to use something like stat and
+ # dumpe2fs -h to have cleanupDeleted calculate uncompressed_size
+ # and container_size rather than passing them as args here.
+ if options.cleanup_deleted:
+ target.cleanupDeleted(options.uncompressed_size,
+ options.container_size)
+
target.package()
+
except InstallationError, e:
print >> sys.stderr, "Error creating Live CD : %s" % e
target.teardown()
--
Fedora-livecd-list mailing list
[email protected]
https://www.redhat.com/mailman/listinfo/fedora-livecd-list
