On Tue, 2024-02-13 at 11:00 +0100, Johannes Schneider via
lists.openembedded.org wrote:
> set the package-database of a "lower image" to unpack and build upon when
> installing packages for the current image. This way a lean image will be
> created, which only holds the packages that are not already present in the
> lower
> image, that then could be used with overlayfs or systemd-sysext to extend the
> "lower image" on demand; for development purposes on an RO lower image for
> example.
>
> Signed-off-by: Johannes Schneider <[email protected]>
> ---
> meta/classes-recipe/image.bbclass | 10 +++++++-
> meta/lib/oe/package_manager/deb/rootfs.py | 2 ++
> meta/lib/oe/package_manager/ipk/rootfs.py | 6 +++--
> meta/lib/oe/package_manager/rpm/rootfs.py | 7 ++++--
> meta/lib/oe/rootfs.py | 29 +++++++++++++++++++++++
> 5 files changed, 49 insertions(+), 5 deletions(-)
>
> diff --git a/meta/classes-recipe/image.bbclass
> b/meta/classes-recipe/image.bbclass
> index c688c39f15..b4a2460187 100644
> --- a/meta/classes-recipe/image.bbclass
> +++ b/meta/classes-recipe/image.bbclass
> @@ -42,8 +42,16 @@ IMAGE_FEATURES ?= ""
> IMAGE_FEATURES[type] = "list"
> IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs
> read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password
> allow-empty-password allow-root-login serial-autologin-root
> post-install-logging overlayfs-etc"
>
> +# Image layering:
> +# a "base image" would create a snapshot of the package-database after the
> +# installation of all packages into the rootfs is done. The next/other image
> +# "layered on-top" of the former would then import that database and install
> +# further packages; without reinstalling package dependencies that are
> already
> +# installed in the layer below.
> # Generate snapshot of the package database?
> IMAGE_GEN_PKGDBFS ?= "0"
> +# Package-database of the base image, upon which to build up a new
> image-layer
> +IMAGE_BASE_PKGDB ?= ""
>
> # Generate companion debugfs?
> IMAGE_GEN_DEBUGFS ?= "0"
> @@ -134,7 +142,7 @@ def rootfs_variables(d):
>
> 'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS',
> 'IMAGE_LINGUAS_COMPLEMENTARY', 'IMAGE_LOCALES_ARCHIVE',
>
> 'MULTILIBRE_ALLOW_REP','MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS',
>
> 'PACKAGE_ARCHS','PACKAGE_CLASSES','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS',
> - 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS',
> 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR',
> 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS',
> + 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS', 'IMAGE_BASE_PKGDB',
> 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR',
> 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS',
> 'IMAGE_INSTALL_DEBUGFS']
> variables.extend(rootfs_command_variables(d))
> variables.extend(variable_depends(d))
> diff --git a/meta/lib/oe/package_manager/deb/rootfs.py
> b/meta/lib/oe/package_manager/deb/rootfs.py
> index 43107c8663..71a21df09b 100644
> --- a/meta/lib/oe/package_manager/deb/rootfs.py
> +++ b/meta/lib/oe/package_manager/deb/rootfs.py
> @@ -152,6 +152,8 @@ class PkgRootfs(DpkgOpkgRootfs):
>
> execute_pre_post_process(self.d, deb_pre_process_cmds)
>
> + self._unpack_pkg_db_rootfs(['/var/lib/dpkg'])
> +
> if self.progress_reporter:
> self.progress_reporter.next_stage()
> # Don't support incremental, so skip that
> diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py
> b/meta/lib/oe/package_manager/ipk/rootfs.py
> index 64d9bc7969..408faa8030 100644
> --- a/meta/lib/oe/package_manager/ipk/rootfs.py
> +++ b/meta/lib/oe/package_manager/ipk/rootfs.py
> @@ -276,12 +276,16 @@ class PkgRootfs(DpkgOpkgRootfs):
> pkgs_to_install = self.manifest.parse_initial_manifest()
> opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS')
> opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS')
> + opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
> + opkg_dir = os.path.join(opkg_lib_dir, 'opkg')
>
> # update PM index files
> self.pm.write_index()
>
> execute_pre_post_process(self.d, opkg_pre_process_cmds)
>
> + self._unpack_pkg_db_rootfs([opkg_dir])
> +
> if self.progress_reporter:
> self.progress_reporter.next_stage()
> # Steps are a bit different in order, skip next
> @@ -317,8 +321,6 @@ class PkgRootfs(DpkgOpkgRootfs):
> if self.progress_reporter:
> self.progress_reporter.next_stage()
>
> - opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
> - opkg_dir = os.path.join(opkg_lib_dir, 'opkg')
> self._setup_pkg_db_rootfs([opkg_dir])
> self._setup_dbg_rootfs([opkg_dir])
>
> diff --git a/meta/lib/oe/package_manager/rpm/rootfs.py
> b/meta/lib/oe/package_manager/rpm/rootfs.py
> index 673006c131..9986b13b5f 100644
> --- a/meta/lib/oe/package_manager/rpm/rootfs.py
> +++ b/meta/lib/oe/package_manager/rpm/rootfs.py
> @@ -67,12 +67,15 @@ class PkgRootfs(Rootfs):
> pkgs_to_install = self.manifest.parse_initial_manifest()
> rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS')
> rpm_post_process_cmds = self.d.getVar('RPM_POSTPROCESS_COMMANDS')
> + package_paths = ['/etc/rpm', '/etc/rpmrc', '/etc/dnf',
> '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']
>
> # update PM index files
> self.pm.write_index()
>
> execute_pre_post_process(self.d, rpm_pre_process_cmds)
>
> + self._unpack_pkg_db_rootfs(package_paths)
> +
> if self.progress_reporter:
> self.progress_reporter.next_stage()
>
> @@ -110,8 +113,8 @@ class PkgRootfs(Rootfs):
> if self.progress_reporter:
> self.progress_reporter.next_stage()
>
> - self._setup_pkg_db_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf',
> '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'])
> - self._setup_dbg_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf',
> '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'])
> + self._setup_pkg_db_rootfs(package_paths)
> + self._setup_dbg_rootfs(package_paths)
>
> execute_pre_post_process(self.d, rpm_post_process_cmds)
>
> diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
> index 9268a02531..7ca574e6b1 100644
> --- a/meta/lib/oe/rootfs.py
> +++ b/meta/lib/oe/rootfs.py
> @@ -106,6 +106,35 @@ class Rootfs(object, metaclass=ABCMeta):
> def _cleanup(self):
> pass
>
> + def _unpack_pkg_db_rootfs(self, package_paths):
> + import tarfile
> + gen_pkg_db_fs = self.d.getVar('IMAGE_BASE_PKGDB') or ''
> + if gen_pkg_db_fs == '':
> + return
Can be simplified to:
gen_pkg_db_fs = self.d.getVar('IMAGE_BASE_PKGDB')
if not gen_pkg_db_fs:
return
> +
> + fname = self.d.getVar('DEPLOY_DIR_IMAGE') + '/' +
> self.d.getVar('IMAGE_BASE_PKGDB') + '-pkgdb.tar.gz'
> + if not fname:
> + bb.warn("PKGDB does not exit:", fname)
> + return
Shouldn't that be a hard error?
> +
> + bb.note(" unpacking package database...")
> + bb.utils.mkdirhier(self.image_rootfs + '-pkgdb')
> + if fname.endswith("tar.gz"):
> + tar = tarfile.open(fname, "r:gz")
> + tar.extractall(path=self.image_rootfs + '-pkgdb')
> + tar.close()
Didn't you hardcode it to tar.gz above? FWIW the python tarfile module
is also very slow.
What happens if I set IMAGE_FSTYPES_PKGDBFS to something other than
tar.gz?
> +
> + bb.note(" Copying back package database...")
> + for path in package_paths:
> + try:
> + bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path))
> + except:
> + pass
> + if os.path.isdir(self.image_rootfs + '-pkgdb' + path):
> + shutil.copytree(self.image_rootfs + '-pkgdb' + path,
> self.image_rootfs + path, symlinks=True, dirs_exist_ok=True)
> + elif os.path.isfile(self.image_rootfs + '-pkgdb' + path):
> + shutil.copyfile(self.image_rootfs + '-pkgdb' + path,
> self.image_rootfs + path)
> +
> def _setup_pkg_db_rootfs(self, package_paths):
> gen_pkg_db_fs = self.d.getVar('IMAGE_GEN_PKGDBFS') or '0'
> if gen_pkg_db_fs != '1':
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#195878):
https://lists.openembedded.org/g/openembedded-core/message/195878
Mute This Topic: https://lists.openembedded.org/mt/104329519/21656
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
