On Wed, Jun 11, 2025 at 04:48:53PM +0200, Filip Schauer wrote: > This crate can parse and extract an OCI image bundled as a tar archive. > > Signed-off-by: Filip Schauer <f.scha...@proxmox.com> > --- > Cargo.toml | 1 + > proxmox-oci/Cargo.toml | 22 ++++ > proxmox-oci/debian/changelog | 5 + > proxmox-oci/debian/control | 47 ++++++++ > proxmox-oci/debian/debcargo.toml | 7 ++ > proxmox-oci/src/lib.rs | 196 +++++++++++++++++++++++++++++++ > proxmox-oci/src/oci_tar_image.rs | 167 ++++++++++++++++++++++++++ > 7 files changed, 445 insertions(+) > create mode 100644 proxmox-oci/Cargo.toml > create mode 100644 proxmox-oci/debian/changelog > create mode 100644 proxmox-oci/debian/control > create mode 100644 proxmox-oci/debian/debcargo.toml > create mode 100644 proxmox-oci/src/lib.rs > create mode 100644 proxmox-oci/src/oci_tar_image.rs > > diff --git a/Cargo.toml b/Cargo.toml > index bf9e83d7..8365b18a 100644 > --- a/Cargo.toml > +++ b/Cargo.toml > @@ -26,6 +26,7 @@ members = [ > "proxmox-metrics", > "proxmox-network-api", > "proxmox-notify", > + "proxmox-oci", > "proxmox-openid", > "proxmox-product-config", > "proxmox-rest-server", > diff --git a/proxmox-oci/Cargo.toml b/proxmox-oci/Cargo.toml > new file mode 100644 > index 00000000..4daff6ab > --- /dev/null > +++ b/proxmox-oci/Cargo.toml > @@ -0,0 +1,22 @@ > +[package] > +name = "proxmox-oci" > +description = "OCI image parsing and extraction" > +version = "0.1.0" > + > +authors.workspace = true > +edition.workspace = true > +exclude.workspace = true > +homepage.workspace = true > +license.workspace = true > +repository.workspace = true > +rust-version.workspace = true > + > +[dependencies] > +flate2.workspace = true > +oci-spec = "0.8.1" > +sha2 = "0.10" > +tar.workspace = true > +thiserror = "1" > +zstd.workspace = true > + > +proxmox-io.workspace = true > diff --git a/proxmox-oci/debian/changelog b/proxmox-oci/debian/changelog > new file mode 100644 > index 00000000..754d06c1 > --- /dev/null > +++ b/proxmox-oci/debian/changelog > @@ -0,0 +1,5 @@ > +rust-proxmox-oci (0.1.0-1) bookworm; urgency=medium > + > + * Initial release. > + > + -- Proxmox Support Team <supp...@proxmox.com> Mon, 28 Apr 2025 12:34:56 > +0200 > diff --git a/proxmox-oci/debian/control b/proxmox-oci/debian/control > new file mode 100644 > index 00000000..3974cf48 > --- /dev/null > +++ b/proxmox-oci/debian/control > @@ -0,0 +1,47 @@ > +Source: rust-proxmox-oci > +Section: rust > +Priority: optional > +Build-Depends: debhelper-compat (= 13), > + dh-sequence-cargo > +Build-Depends-Arch: cargo:native <!nocheck>, > + rustc:native (>= 1.82) <!nocheck>, > + libstd-rust-dev <!nocheck>, > + librust-flate2-1+default-dev <!nocheck>, > + librust-oci-spec-0.8+default-dev (>= 0.8.1-~~) <!nocheck>, > + librust-proxmox-io-1+default-dev (>= 1.1.0-~~) <!nocheck>, > + librust-sha2-0.10+default-dev <!nocheck>, > + librust-tar-0.4+default-dev <!nocheck>, > + librust-thiserror-1+default-dev <!nocheck>, > + librust-zstd-0.12+bindgen-dev <!nocheck>, > + librust-zstd-0.12+default-dev <!nocheck> > +Maintainer: Proxmox Support Team <supp...@proxmox.com> > +Standards-Version: 4.7.0 > +Vcs-Git: git://git.proxmox.com/git/proxmox.git > +Vcs-Browser: https://git.proxmox.com/?p=proxmox.git > +Homepage: https://proxmox.com > +X-Cargo-Crate: proxmox-oci > +Rules-Requires-Root: no > + > +Package: librust-proxmox-oci-dev > +Architecture: any > +Multi-Arch: same > +Depends: > + ${misc:Depends}, > + librust-flate2-1+default-dev, > + librust-oci-spec-0.8+default-dev (>= 0.8.1-~~), > + librust-proxmox-io-1+default-dev (>= 1.1.0-~~), > + librust-sha2-0.10+default-dev, > + librust-tar-0.4+default-dev, > + librust-thiserror-1+default-dev, > + librust-zstd-0.12+bindgen-dev, > + librust-zstd-0.12+default-dev > +Provides: > + librust-proxmox-oci+default-dev (= ${binary:Version}), > + librust-proxmox-oci-0-dev (= ${binary:Version}), > + librust-proxmox-oci-0+default-dev (= ${binary:Version}), > + librust-proxmox-oci-0.1-dev (= ${binary:Version}), > + librust-proxmox-oci-0.1+default-dev (= ${binary:Version}), > + librust-proxmox-oci-0.1.0-dev (= ${binary:Version}), > + librust-proxmox-oci-0.1.0+default-dev (= ${binary:Version}) > +Description: OCI image parsing and extraction - Rust source code > + Source code for Debianized Rust crate "proxmox-oci" > diff --git a/proxmox-oci/debian/debcargo.toml > b/proxmox-oci/debian/debcargo.toml > new file mode 100644 > index 00000000..b7864cdb > --- /dev/null > +++ b/proxmox-oci/debian/debcargo.toml > @@ -0,0 +1,7 @@ > +overlay = "." > +crate_src_path = ".." > +maintainer = "Proxmox Support Team <supp...@proxmox.com>" > + > +[source] > +vcs_git = "git://git.proxmox.com/git/proxmox.git" > +vcs_browser = "https://git.proxmox.com/?p=proxmox.git"; > diff --git a/proxmox-oci/src/lib.rs b/proxmox-oci/src/lib.rs > new file mode 100644 > index 00000000..cc5a1d46 > --- /dev/null > +++ b/proxmox-oci/src/lib.rs > @@ -0,0 +1,196 @@ > +use flate2::read::GzDecoder; > +use oci_spec::image::{Arch, Config, ImageConfiguration, ImageManifest, > MediaType}; > +use oci_spec::OciSpecError; > +use oci_tar_image::OciTarImage; > +use sha2::digest::generic_array::GenericArray; > +use sha2::{Digest, Sha256}; > +use std::collections::HashMap; > +use std::fs::File; > +use std::io::{Read, Seek}; > +use std::path::PathBuf; > +use std::str::FromStr; > +use tar::Archive; > +use thiserror::Error; > + > +pub mod oci_tar_image; > + > +fn compute_digest<R: Read, H: Digest>( > + mut reader: R, > + mut hasher: H, > +) -> GenericArray<u8, H::OutputSize> { > + let mut buf = proxmox_io::boxed::zeroed(4096); > + > + loop { > + let bytes_read = reader.read(&mut buf).unwrap(); > + if bytes_read == 0 { > + break hasher.finalize(); > + } > + > + hasher.update(&buf[..bytes_read]); > + } > +} > + > +fn compute_sha256<R: Read>(reader: R) -> oci_spec::image::Sha256Digest { > + let digest = compute_digest(reader, Sha256::new()); > + oci_spec::image::Sha256Digest::from_str(&format!("{:x}", > digest)).unwrap() > +} > + > +/// Build a mapping from uncompressed layer digests (as found in the image > config's `rootfs.diff_ids`) > +/// to their corresponding compressed-layer digests (i.e. the filenames > under `blobs/<algorithm>/<digest>`) > +fn build_layer_map<R: Read + Seek>( > + oci_tar_image: &mut OciTarImage<R>, > + image_manifest: &ImageManifest, > +) -> HashMap<oci_spec::image::Digest, oci_spec::image::Descriptor> { > + let mut layer_mapping = HashMap::new(); > + > + for layer in image_manifest.layers() { > + let digest = match layer.media_type() { > + MediaType::ImageLayer | MediaType::ImageLayerNonDistributable => > { > + Some(layer.digest().clone()) > + } > + MediaType::ImageLayerGzip | > MediaType::ImageLayerNonDistributableGzip => { > + let compressed_blob = > oci_tar_image.open_blob(layer.digest()).unwrap(); > + let decoder = GzDecoder::new(compressed_blob); > + Some(compute_sha256(decoder).into()) > + } > + MediaType::ImageLayerZstd | > MediaType::ImageLayerNonDistributableZstd => { > + let compressed_blob = > oci_tar_image.open_blob(layer.digest()).unwrap(); > + let decoder = zstd::Decoder::new(compressed_blob).unwrap(); > + Some(compute_sha256(decoder).into()) > + } > + _ => None, > + }; > + > + if let Some(digest) = digest { > + layer_mapping.insert(digest, layer.clone()); > + } > + } > + > + layer_mapping > +} > + > +#[derive(Debug, Error)] > +pub enum ProxmoxOciError { > + #[error("Error while parsing OCI image: {0}")] > + ParseError(#[from] ParseError), > + #[error("Error while extracting OCI image: {0}")] > + ExtractError(#[from] ExtractError), > +} > + > +pub fn parse_and_extract_image( > + oci_tar_path: &str, > + rootfs_path: &str, > +) -> Result<Option<Config>, ProxmoxOciError> { > + let (mut oci_tar_image, image_manifest, image_config) = > parse_image(oci_tar_path)?; > + > + extract_image_rootfs( > + &mut oci_tar_image, > + &image_manifest, > + &image_config, > + rootfs_path.into(), > + )?; > + > + Ok(image_config.config().clone()) > +} > + > +#[derive(Debug, Error)] > +pub enum ParseError { > + #[error("Not an OCI image: {0}")] > + NotAnOciImage(OciSpecError), > + #[error("Wrong media type")] > + WrongMediaType, > + #[error("IO error: {0}")] > + IoError(#[from] std::io::Error), > + #[error("Unsupported CPU architecture")] > + UnsupportedArchitecture, > + #[error("Missing image config")] > + MissingImageConfig, > +} > + > +impl From<OciSpecError> for ParseError { > + fn from(oci_spec_err: OciSpecError) -> Self { > + match oci_spec_err { > + OciSpecError::Io(ioerr) => Self::IoError(ioerr), > + ocierr => Self::NotAnOciImage(ocierr), > + } > + } > +} > + > +fn parse_image( > + oci_tar_path: &str, > +) -> Result<(OciTarImage<File>, ImageManifest, ImageConfiguration), > ParseError> { > + let oci_tar_file = File::open(oci_tar_path)?; > + let mut oci_tar_image = OciTarImage::new(oci_tar_file)?; > + > + let image_manifest = oci_tar_image > + .image_manifest(&Arch::Amd64) > + .ok_or(ParseError::UnsupportedArchitecture)??; > + > + let image_config_descriptor = image_manifest.config(); > + > + if image_config_descriptor.media_type() != &MediaType::ImageConfig { > + return Err(ParseError::WrongMediaType); > + } > + > + let image_config_file = oci_tar_image > + .open_blob(image_config_descriptor.digest()) > + .ok_or(ParseError::MissingImageConfig)?; > + let image_config = ImageConfiguration::from_reader(image_config_file)?; > + > + Ok((oci_tar_image, image_manifest, image_config)) > +} > + > +#[derive(Debug, Error)] > +pub enum ExtractError { > + #[error("Rootfs destination path not found")] > + RootfsDestinationNotFound, > + #[error("Unknown layer digest found in rootfs.diff_ids")] > + UnknownLayerDigest, > + #[error("Layer file mentioned in image manifest is missing")] > + MissingLayerFile, > + #[error("IO error: {0}")] > + IoError(#[from] std::io::Error), > + #[error("Layer has wrong media type")] > + WrongMediaType, > +} > + > +fn extract_image_rootfs<R: Read + Seek>( > + oci_tar_image: &mut OciTarImage<R>, > + image_manifest: &ImageManifest, > + image_config: &ImageConfiguration, > + target_path: PathBuf, > +) -> Result<(), ExtractError> { > + if !target_path.exists() { > + return Err(ExtractError::RootfsDestinationNotFound); > + } > + > + let layer_map = build_layer_map(oci_tar_image, image_manifest); > + > + for layer in image_config.rootfs().diff_ids() { > + let layer_digest = oci_spec::image::Digest::from_str(layer) > + .map_err(|_| ExtractError::UnknownLayerDigest)?; > + let layer_descriptor = layer_map > + .get(&layer_digest) > + .ok_or(ExtractError::UnknownLayerDigest)?; > + let layer_file = oci_tar_image > + .open_blob(layer_descriptor.digest()) > + .ok_or(ExtractError::MissingLayerFile)?; > + > + let tar_file: Box<dyn Read> = match layer_descriptor.media_type() { > + MediaType::ImageLayer | MediaType::ImageLayerNonDistributable => > Box::new(layer_file), > + MediaType::ImageLayerGzip | > MediaType::ImageLayerNonDistributableGzip => { > + Box::new(GzDecoder::new(layer_file)) > + } > + MediaType::ImageLayerZstd | > MediaType::ImageLayerNonDistributableZstd => { > + Box::new(zstd::Decoder::new(layer_file)?) > + } > + _ => return Err(ExtractError::WrongMediaType), > + }; > + > + let mut archive = Archive::new(tar_file); > + archive.set_preserve_ownerships(true); > + archive.unpack(&target_path)?;
After some more thought: this is actually insufficient. We need to also handle whiteouts[1] (and later also extracting those whiteouts as overlayfs-compatible whiteout device nodes and opaque dirs via xattrs. You can pull `joomla` as an example image with whiteouts. Eg.: file: blobs/sha256/1017d0351d4278294b94bbede8960513af97265c3fff9a66a179f50df08b13fc (...) ---------- 0/0 0 1970-01-01 01:00 etc/apache2/mods-enabled/.wh.mpm_event.conf (...) [1] https://github.com/opencontainers/image-spec/blob/main/layer.md#whiteouts _______________________________________________ pve-devel mailing list pve-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
