Every access to an object in the sstate cache is recorded with a plain touch, which writes both the modification time and the access time.
Add SSTATE_ATIME_UPDATE_AFTER. When it is set, the modification time is no longer written, and the access time is refreshed only once the existing one is at least that many seconds old. This reduces the number of write cycles a build causes on the cache, which matters most where many build servers share one sstate cache: each of them would otherwise refresh the access time of every object it looks at. With mtime and atime kept apart, a cache can also be measured for reuse and age. When the variable is unset the behaviour is unchanged. Signed-off-by: Michael Haener <[email protected]> Reviewed-by: Adrian Freihofer <[email protected]> Reviewed-by: Peter Marko <[email protected]> --- meta/classes-global/sstate.bbclass | 59 ++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/meta/classes-global/sstate.bbclass b/meta/classes-global/sstate.bbclass index b2fa93650a..2141b50881 100644 --- a/meta/classes-global/sstate.bbclass +++ b/meta/classes-global/sstate.bbclass @@ -699,19 +699,11 @@ def sstate_package(ss, d): if not os.path.exists(siginfo): bb.siggen.dump_this_task(siginfo, d) else: - try: - os.utime(siginfo, None) - except PermissionError: - pass - except OSError as e: - # Handle read-only file systems gracefully - import errno - if e.errno != errno.EROFS: - raise e + sstate_touch(siginfo, d.getVar("SSTATE_ATIME_UPDATE_AFTER")) return -sstate_package[vardepsexclude] += "SSTATE_SIG_KEY SSTATE_PKG" +sstate_package[vardepsexclude] += "SSTATE_SIG_KEY SSTATE_PKG SSTATE_ATIME_UPDATE_AFTER" def pstaging_fetch(sstatefetch, d): import bb.fetch @@ -799,10 +791,15 @@ python sstate_create_and_sign_package () { from pathlib import Path import errno + update_after = d.getVar("SSTATE_ATIME_UPDATE_AFTER") + # Best effort touch def touch(file): try: - file.touch() + if file.exists(): + sstate_touch(file, update_after) + else: + file.touch() except: pass @@ -933,16 +930,44 @@ sstate_unpack_package () { tar -I "$ZSTD" -xvpf ${SSTATE_PKG} + touch_opts="" + if [ -n "${SSTATE_ATIME_UPDATE_AFTER}" ]; then + # the files below are installed together, so decide once for all of them + cutoff=$(date -d "-${SSTATE_ATIME_UPDATE_AFTER} seconds" +%s 2>/dev/null || echo 0) + atime=$(stat -c %X ${SSTATE_PKG} 2>/dev/null || echo 0) + if [ "$cutoff" -gt 0 ] && [ "$atime" -gt "$cutoff" ]; then + return 0 + fi + touch_opts="-a" + fi + # Update both any file and any symlink pointing to the file for sigs as well as the file for file in ${SSTATE_PKG} ${SSTATE_PKG}.sig ${SSTATE_PKG}.siginfo do - [ ! -e $file ] || touch $file 2>/dev/null || true - [ ! -e $file ] || touch --no-dereference $file 2>/dev/null || true + [ ! -e $file ] || touch $touch_opts $file 2>/dev/null || true + [ ! -e $file ] || touch $touch_opts --no-dereference $file 2>/dev/null || true done } BB_HASHCHECK_FUNCTION = "sstate_checkhashes" +def sstate_touch(path, minage): + # Without minage the modification time is reset along with the access time. + # With it, only the access time is refreshed, and only once the existing one + # is at least minage seconds old. + import time + if not minage: + oe.utils.touch(path) + return + try: + stat_info = os.stat(path) + now = time.time() + if now - stat_info.st_atime < float(minage): + return + os.utime(path, (now, stat_info.st_mtime)) + except OSError: + pass + def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs): import itertools @@ -973,15 +998,17 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, spec, extrapath, tname = getpathcomponents(tid, d) return extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d) + update_after = d.getVar("SSTATE_ATIME_UPDATE_AFTER") + for tid in sq_data['hash']: sstatefile = d.expand("${SSTATE_DIR}/" + getsstatefile(tid, siginfo, d)) if os.path.exists(sstatefile): - oe.utils.touch(sstatefile) + sstate_touch(sstatefile, update_after) for ext in ['.sig', '.siginfo']: if os.path.exists(sstatefile + ext): - oe.utils.touch(sstatefile + ext) + sstate_touch(sstatefile + ext, update_after) found.add(tid) bb.debug(2, "SState: Found valid sstate file %s" % sstatefile) else: @@ -1223,7 +1250,7 @@ python sstate_eventhandler() { if not os.path.exists(siginfo): bb.siggen.dump_this_task(siginfo, d) else: - oe.utils.touch(siginfo) + sstate_touch(siginfo, d.getVar("SSTATE_ATIME_UPDATE_AFTER")) } SSTATE_PRUNE_OBSOLETEWORKDIR ?= "1" -- 2.34.1
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#245500): https://lists.openembedded.org/g/openembedded-core/message/245500 Mute This Topic: https://lists.openembedded.org/mt/121170990/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
