The catalyst-auto automation scripts live in a repo separate from catalyst. That increases the difficulty of changing catalyst's interface, and it doesn't seem to offer any advantages otherwise. (Keeping build specs in a separate repo allows them to be updated independent of catalyst and that is valuable). Additionally, since the primary way catalyst is used is via this automation, it makes sense to support this workflow in catalyst directly.
But to get there, there are some changes to catalyst that I think are improvements on their own and simplify the path to integrating automation capabilities directly into catalyst. That's what I'd like to discuss here. I'd like to: 1) Replace the custom .spec file format with TOML 2) Combine .spec file sequences (e.g., stage1 -> stage2 -> stage3 -> livecd-stage1 -> livecd-stage2) into a single file. I suggest naming this a ".build" file. This will also allow us to remove the redundant information that currently has to be specified in stage1.spec, stage2.spec, stage3.spec, like rel_type, version, profile, etc. It also means that we remove the nonsensical ability to change settings from one stage to the next that should not change (e.g., rel_type, version). 3) Add ability to denote which stage builds produce artifacts we care about (and want to save and/or upload) and which are just temporary. If they're temporary (e.g., a stage1 build) we can delete the artifact after the build sequence has no further use of it, and we can skip compressing the result, etc. To that end, I'm starting by figuring out what I would like the new spec file format to look like. Below are some open questions and then a strawman new-style spec file. • The .spec files in releng.git are really templates that are not directly usable without sed'ing @REPO_DIR@ and @TIMESTAMP@. It would be nice if they were directly usable as that would reduce confusion from users. • Can we make them directly usable? • Perhaps we can make catalyst handle the replacements directly? • Calculating @TIMESTAMP@ is trivially doable—we do it today (see below) • We could configure @REPO_DIR@ in catalyst.conf and let catalyst do the replacement, or we could just make the field relative to some path specified in catalyst.conf? • In the current automation scripts, we generate a value for @TIMESTAMP@ from the git HEAD used in creating the snapshot. • Would be nice to remove the dependence on the squashfs snapshot generation—not difficult to do • Can we generate and upload a .build file with replacements done to make stage builds more easily reproducible? Seems easy. Thoughts? Are there advantages to the current system I'm not considering? ppc.build: [build] arch = "ppc" subarch = "ppc" version = "@TIMESTAMP@" rel_type = "default" profile = "default/linux/powerpc/ppc32/17.0" source_subpath = "default/stage3-ppc-latest" portage_confdir = "@REPO_DIR@/releases/portage/stages" [build.snapshot] snapshot = "@TIMESTAMP@" [build.stage1] update_seed = "yes" update_seed_command = "--update --deep --newuse @world" [build.stage2] skip = true # Stage3 will be built directly from stage1 [build.stage3] compression = "xz" [build.iso] portage_confdir = "@REPO_DIR@/releases/portage/isos" [build.iso.stage1] use = [ "compile-locales", "fbcon", "ipv6", "livecd", "modules", "ncurses", "nls", "nptl", "pam", "readline", "socks5", "ssl", "static-libs", "unicode", "xml", ] packages = [ "app-admin/pwgen", "app-admin/syslog-ng", "app-arch/unzip", "app-crypt/gnupg", "app-laptop/pbbuttonsd", "app-misc/livecd-tools", "app-misc/screen", "app-portage/mirrorselect", "app-text/wgetpaste", "net-analyzer/tcptraceroute", "net-analyzer/traceroute", "net-dialup/mingetty", "net-dialup/pptpclient", "net-dialup/rp-pppoe", "net-fs/cifs-utils", "net-fs/nfs-utils", "net-irc/irssi", "net-misc/dhcpcd", "net-misc/iputils", "net-misc/ntp", "net-misc/openssh", "net-misc/rdate", "net-misc/rsync", "net-wireless/wireless-tools", "net-wireless/wpa_supplicant", "sys-apps/busybox", "sys-apps/ethtool", "sys-apps/fxload", "sys-apps/hdparm", "sys-apps/hwsetup", "sys-apps/ibm-powerpc-utils", "sys-apps/iproute2", "sys-apps/lm-sensors", "sys-apps/memtester", "sys-apps/pcmciautils", "sys-apps/powerpc-utils", "sys-apps/sdparm", "sys-block/parted", "sys-boot/grub", "sys-firmware/b43-firmware", "sys-firmware/b43legacy-firmware", "sys-fs/btrfs-progs", "sys-fs/dosfstools", "sys-fs/e2fsprogs", "sys-fs/hfsplusutils", "sys-fs/hfsutils", "sys-fs/iprutils", "sys-fs/jfsutils", "sys-fs/lvm2", "sys-fs/mac-fdisk", "sys-fs/mdadm", "sys-fs/ntfs3g", "sys-fs/reiserfsprogs", "sys-fs/xfsprogs", "sys-libs/gpm", "www-client/links", ] [build.iso.stage2] fstype = "squashfs" iso = "/var/tmp/catalyst/builds/default/install-powerpc-minimal-@TIMESTAMP@.iso" type = "gentoo-release-minimal" rcadd = [ ["pbbuttonsd", "default"], ] unmerge = [ "app-admin/eselect", "app-admin/eselect-ctags", "app-admin/eselect-vi", "app-admin/perl-cleaner", "app-admin/python-updater", "app-arch/cpio", "dev-libs/gmp", "dev-libs/libxml2", "dev-libs/mpfr", "dev-libs/popt", "dev-python/pycrypto", "dev-util/pkgconfig", "perl-core/PodParser", "perl-core/Test-Harness", "sys-apps/debianutils", "sys-apps/diffutils", "sys-apps/groff", "sys-apps/man-db", "sys-apps/man-pages", "sys-apps/miscfiles", "sys-apps/sandbox", "sys-apps/texinfo", "sys-devel/autoconf", "sys-devel/autoconf-wrapper", "sys-devel/automake", "sys-devel/automake-wrapper", "sys-devel/binutils", "sys-devel/binutils-config", "sys-devel/bison", "sys-devel/flex", "sys-devel/gcc", "sys-devel/gcc-config", "sys-devel/gettext", "sys-devel/gnuconfig", "sys-devel/libtool", "sys-devel/m4", "sys-devel/make", "sys-devel/patch", "sys-libs/db", "sys-libs/gdbm", "sys-libs/libkudzu", "sys-kernel/genkernel", "sys-kernel/linux-headers", ] empty = [ "/boot", "/boot/initr*", "/boot/kernel*", "/boot/System*", "/etc/*-", "/etc/cron.daily", "/etc/cron.hourly", "/etc/cron.monthly", "/etc/cron.weekly", "/etc/default/audioctl", "/etc/dispatch-conf.conf", "/etc/env.d/05binutils", "/etc/env.d/05gcc", "/etc/etc-update.conf", "/etc/genkernel.conf", "/etc/hosts.bck", "/etc/issue*", "/etc/logrotate.d", "/etc/make.conf*", "/etc/make.globals", "/etc/make.profile", "/etc/man.conf", "/etc/modules.autoload.d", "/etc/*.old", "/etc/resolv.conf", "/etc/runlevels/single", "/etc/skel", "/lib64/dev-state", "/lib64/udev-state", "/lib*/*.a", "/lib*/cpp", "/lib/dev-state", "/lib*/*.la", "/lib/udev-state", "/root/.bash_history", "/root/.ccache", "/root/.viminfo", "/sbin/fsck.cramfs", "/sbin/fsck.minix", "/sbin/mkfs.bfs", "/sbin/mkfs.cramfs", "/sbin/mkfs.minix", "/sbin/*.static", "/tmp", "/usr/bin/addr2line", "/usr/bin/ar", "/usr/bin/as", "/usr/bin/audioctl", "/usr/bin/c++*", "/usr/bin/cc", "/usr/bin/cjpeg", "/usr/bin/cpp", "/usr/bin/djpeg", "/usr/bin/ebuild", "/usr/bin/egencache", "/usr/bin/elftoaout", "/usr/bin/emerge", "/usr/bin/emerge-webrsync", "/usr/bin/emirrordist", "/usr/bin/f77", "/usr/bin/g++*", "/usr/bin/g77", "/usr/bin/gcc*", "/usr/bin/genkernel", "/usr/bin/gprof", "/usr/bin/jpegtran", "/usr/bin/ld", "/usr/bin/libpng*", "/usr/bin/nm", "/usr/bin/objcopy", "/usr/bin/objdump", "/usr/bin/piggyback*", "/usr/bin/portageq", "/usr/bin/powerpc64-unknown-linux-gnu-*", "/usr/bin/powerpc-unknown-linux-gnu-*", "/usr/bin/ranlib", "/usr/bin/readelf", "/usr/bin/repoman", "/usr/bin/size", "/usr/bin/strings", "/usr/bin/strip", "/usr/bin/tbz2tool", "/usr/bin/xpak", "/usr/bin/yacc", "/usr/diet/include", "/usr/diet/man", "/usr/include", "/usr/lib64/awk", "/usr/lib64/ccache", "/usr/lib64/gcc-config", "/usr/lib64/gconv", "/usr/lib64/nfs", "/usr/lib64/perl5/site_perl", "/usr/lib64/portage", "/usr/lib64/python*/test", "/usr/lib64/X11/config", "/usr/lib64/X11/doc", "/usr/lib64/X11/etc", "/usr/lib*/*.a", "/usr/lib*/gcc-lib/*/*/libgcj*", "/usr/lib*/*.la", "/usr/lib*/perl5/site_perl", "/usr/local", "/usr/portage", "/usr/powerpc64-unknown-linux-gnu", "/usr/powerpc-unknown-linux-gnu", "/usr/sbin/archive-conf", "/usr/sbin/dispatch-conf", "/usr/sbin/emaint", "/usr/sbin/env-update", "/usr/sbin/etc-update", "/usr/sbin/fb*", "/usr/sbin/fixpackages", "/usr/sbin/quickpkg", "/usr/sbin/regenworld", "/usr/share/aclocal", "/usr/share/baselayout", "/usr/share/binutils-data", "/usr/share/consolefonts/1*", "/usr/share/consolefonts/7*", "/usr/share/consolefonts/8*", "/usr/share/consolefonts/9*", "/usr/share/consolefonts/a*", "/usr/share/consolefonts/A*", "/usr/share/consolefonts/c*", "/usr/share/consolefonts/C*", "/usr/share/consolefonts/dr*", "/usr/share/consolefonts/E*", "/usr/share/consolefonts/g*", "/usr/share/consolefonts/G*", "/usr/share/consolefonts/i*", "/usr/share/consolefonts/k*", "/usr/share/consolefonts/l*", "/usr/share/consolefonts/L*", "/usr/share/consolefonts/M*", "/usr/share/consolefonts/partialfonts", "/usr/share/consolefonts/r*", "/usr/share/consolefonts/R*", "/usr/share/consolefonts/s*", "/usr/share/consolefonts/t*", "/usr/share/consolefonts/v*", "/usr/share/consoletrans", "/usr/share/dict", "/usr/share/doc", "/usr/share/emacs", "/usr/share/et", "/usr/share/gcc-data", "/usr/share/genkernel", "/usr/share/gettext", "/usr/share/glib-2.0", "/usr/share/gnuconfig", "/usr/share/gtk-doc", "/usr/share/i18n", "/usr/share/info", "/usr/share/lcms", "/usr/share/libtool", "/usr/share/locale", "/usr/share/man", "/usr/share/misc/*.old", "/usr/share/rfc", "/usr/share/ss", "/usr/share/state", "/usr/share/texinfo", "/usr/share/unimaps", "/usr/share/zoneinfo", "/usr/src", "/var/cache", "/var/empty", "/var/lib/portage", "/var/log", "/var/spool", "/var/state", "/var/tmp", ] # Just for bootloader ordering purposes kernel = ["ppc64", "ppc32", "ibmpower",] [build.iso.stage2.kernel.ibmpower] sources = "sys-kernel/gentoo-sources" config = "../../../kconfig/powerpc/installcd-ibm-4.19.config" console = ["ttyS0,9600", "hvc0", "hvsi0",] gk_kernargs = [ "--kernel-cc='gcc -m64'", "--kernel-ld='ld -m elf64ppc'", "--kernel-as='as -a64'", ] [build.iso.stage2.kernel.ppc64] sources = "sys-kernel/gentoo-sources" config = "../../../kconfig/powerpc/ppc64.config" console = ["ttyS0,57600",] gk_kernargs = [ "--kernel-cc='gcc -m64'", "--kernel-ld='ld -m elf64ppc'", "--kernel-as='as -a64'", ] [build.iso.stage2.kernel.ppc32] sources = "sys-kernel/gentoo-sources" config = "../../../kconfig/powerpc/ppc32.config"