Re: [gentoo-dev] [PATCH] cargo.eclass: Optimize crate unpacking
Michał Górny writes: > Unpack crates in parallel using xargs to utilize multicore systems > better. Perform checksumming via a single sha256sum invocation. > > For dev-python/watchfiles, this speeds up unpacking on my machine > from 2.6 s to 0.75 s (warm cache). > > Signed-off-by: Michał Górny For completeness (acked on PR), lgtm. I assume tested by building all cargo inheritees (nearly said 'heirs' but.. lol)? Give a chance for others to look though. Also, thank you! > --- > eclass/cargo.eclass | 56 ++--- > 1 file changed, 33 insertions(+), 23 deletions(-) > > diff --git a/eclass/cargo.eclass b/eclass/cargo.eclass > index 0f2da982f60c..5a16d3a30528 100644 > --- a/eclass/cargo.eclass > +++ b/eclass/cargo.eclass > @@ -329,40 +329,50 @@ _cargo_gen_git_config() { > cargo_src_unpack() { > debug-print-function ${FUNCNAME} "$@" > > - mkdir -p "${ECARGO_VENDOR}" || die > - mkdir -p "${S}" || die > + mkdir -p "${ECARGO_VENDOR}" "${S}" || die > > local archive shasum pkg > + local crates=() > for archive in ${A}; do > case "${archive}" in > *.crate) > - # when called by pkgdiff-mg, do not unpack > crates > - [[ ${PKGBUMPING} == ${PVR} ]] && continue > - > - ebegin "Loading ${archive} into Cargo registry" > - tar -xf "${DISTDIR}"/${archive} -C > "${ECARGO_VENDOR}/" || die > - # generate sha256sum of the crate itself as > cargo needs this > - shasum=$(sha256sum "${DISTDIR}"/${archive} | > cut -d ' ' -f 1) > - pkg=$(basename ${archive} .crate) > - cat <<- EOF > > ${ECARGO_VENDOR}/${pkg}/.cargo-checksum.json > - { > - "package": "${shasum}", > - "files": {} > - } > - EOF > - # if this is our target package we need it in > ${WORKDIR} too > - # to make ${S} (and handle any revisions too) > - if [[ ${P} == ${pkg}* ]]; then > - tar -xf "${DISTDIR}"/${archive} -C > "${WORKDIR}" || die > - fi > - eend $? > + crates+=( "${archive}" ) > ;; > *) > - unpack ${archive} > + unpack "${archive}" > ;; > esac > done > > + if [[ ${PKGBUMPING} != ${PVR} ]]; then > + pushd "${DISTDIR}" >/dev/null || die > + > + ebegin "Unpacking crates" > + printf '%s\0' "${crates[@]}" | > + xargs -0 -P "$(makeopts_jobs)" -n 1 -- \ > + tar -x -C "${ECARGO_VENDOR}" -f > + assert > + eend $? > + > + while read -d '' -r shasum archive; do > + pkg=${archive%.crate} > + cat <<- EOF > > ${ECARGO_VENDOR}/${pkg}/.cargo-checksum.json || die > + { > + "package": "${shasum}", > + "files": {} > + } > + EOF > + > + # if this is our target package we need it in > ${WORKDIR} too > + # to make ${S} (and handle any revisions too) > + if [[ ${P} == ${pkg}* ]]; then > + tar -xf "${archive}" -C "${WORKDIR}" || die > + fi > + done < <(sha256sum -z "${crates[@]}" || die) > + > + popd >/dev/null || die > + fi > + > cargo_gen_config > }
[gentoo-dev] [PATCH] cargo.eclass: Optimize crate unpacking
Unpack crates in parallel using xargs to utilize multicore systems better. Perform checksumming via a single sha256sum invocation. For dev-python/watchfiles, this speeds up unpacking on my machine from 2.6 s to 0.75 s (warm cache). Signed-off-by: Michał Górny --- eclass/cargo.eclass | 56 ++--- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/eclass/cargo.eclass b/eclass/cargo.eclass index 0f2da982f60c..5a16d3a30528 100644 --- a/eclass/cargo.eclass +++ b/eclass/cargo.eclass @@ -329,40 +329,50 @@ _cargo_gen_git_config() { cargo_src_unpack() { debug-print-function ${FUNCNAME} "$@" - mkdir -p "${ECARGO_VENDOR}" || die - mkdir -p "${S}" || die + mkdir -p "${ECARGO_VENDOR}" "${S}" || die local archive shasum pkg + local crates=() for archive in ${A}; do case "${archive}" in *.crate) - # when called by pkgdiff-mg, do not unpack crates - [[ ${PKGBUMPING} == ${PVR} ]] && continue - - ebegin "Loading ${archive} into Cargo registry" - tar -xf "${DISTDIR}"/${archive} -C "${ECARGO_VENDOR}/" || die - # generate sha256sum of the crate itself as cargo needs this - shasum=$(sha256sum "${DISTDIR}"/${archive} | cut -d ' ' -f 1) - pkg=$(basename ${archive} .crate) - cat <<- EOF > ${ECARGO_VENDOR}/${pkg}/.cargo-checksum.json - { - "package": "${shasum}", - "files": {} - } - EOF - # if this is our target package we need it in ${WORKDIR} too - # to make ${S} (and handle any revisions too) - if [[ ${P} == ${pkg}* ]]; then - tar -xf "${DISTDIR}"/${archive} -C "${WORKDIR}" || die - fi - eend $? + crates+=( "${archive}" ) ;; *) - unpack ${archive} + unpack "${archive}" ;; esac done + if [[ ${PKGBUMPING} != ${PVR} ]]; then + pushd "${DISTDIR}" >/dev/null || die + + ebegin "Unpacking crates" + printf '%s\0' "${crates[@]}" | + xargs -0 -P "$(makeopts_jobs)" -n 1 -- \ + tar -x -C "${ECARGO_VENDOR}" -f + assert + eend $? + + while read -d '' -r shasum archive; do + pkg=${archive%.crate} + cat <<- EOF > ${ECARGO_VENDOR}/${pkg}/.cargo-checksum.json || die + { + "package": "${shasum}", + "files": {} + } + EOF + + # if this is our target package we need it in ${WORKDIR} too + # to make ${S} (and handle any revisions too) + if [[ ${P} == ${pkg}* ]]; then + tar -xf "${archive}" -C "${WORKDIR}" || die + fi + done < <(sha256sum -z "${crates[@]}" || die) + + popd >/dev/null || die + fi + cargo_gen_config } -- 2.45.0
[gentoo-dev] Last rites: app-emulation/docker-machine
# Arthur Zamarin (2024-05-11) # EAPI=6, uses deprecated go eclass, archived upstream. Update to # usage of go-module.eclass isn't simple. # Removal: 2024-06-10. Bugs #931745, #844598. app-emulation/docker-machine OpenPGP_signature.asc Description: OpenPGP digital signature
[gentoo-dev] Last rites: dev-go/fuzzy, dev-go/go-bindata-assetfs, dev-go/godebug-pretty, dev-go/goversion, dev-go/sanitized-anchor-name
# Arthur Zamarin (2024-05-11) # EAPI=6, library only without any reverse dependencies, uses # deprecated go eclasses. # Removal: 2024-06-10. Bug #931725. dev-go/fuzzy dev-go/go-bindata-assetfs dev-go/godebug-pretty dev-go/goversion dev-go/sanitized-anchor-name OpenPGP_signature.asc Description: OpenPGP digital signature