We're having speed issues on the autobuilder due to the numbers of files in sstate directories. We previously split these by the first two characters of the hash. This change extends this to split by the next two characters as well, creating more layers of directories.
This should signifiantly speed up eSDK builds on the autobuilder as the current sstate layout simply isn't scaling there but addresses a general complaint. gen-lockedsig-cache needed to be updated for the new split level sstate. Also update tests for new layout. Signed-off-by: Richard Purdie <[email protected]> --- meta/classes/sstate.bbclass | 4 ++-- meta/lib/oeqa/selftest/cases/signing.py | 4 ++-- meta/lib/oeqa/selftest/cases/sstate.py | 4 ++-- scripts/gen-lockedsig-cache | 13 ++++++++++--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass index c0329cd5d18..7d2cb9eb840 100644 --- a/meta/classes/sstate.bbclass +++ b/meta/classes/sstate.bbclass @@ -6,7 +6,7 @@ SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}" def generate_sstatefn(spec, hash, d): if not hash: hash = "INVALID" - return hash[:2] + "/" + spec + hash + return hash[:2] + "/" + hash[2:4] + "/" + spec + hash SSTATE_PKGARCH = "${PACKAGE_ARCH}" SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:" @@ -15,7 +15,7 @@ SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PK SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}" SSTATE_EXTRAPATH = "" SSTATE_EXTRAPATHWILDCARD = "" -SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}" +SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}" # explicitly make PV to depend on evaluated value of PV variable PV[vardepvalue] = "${PV}" diff --git a/meta/lib/oeqa/selftest/cases/signing.py b/meta/lib/oeqa/selftest/cases/signing.py index b65f3de64c4..9ea31328eff 100644 --- a/meta/lib/oeqa/selftest/cases/signing.py +++ b/meta/lib/oeqa/selftest/cases/signing.py @@ -157,8 +157,8 @@ class Signing(OESelftestTestCase): bitbake('-c clean %s' % test_recipe) bitbake('-c populate_lic %s' % test_recipe) - recipe_sig = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz.sig') - recipe_tgz = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz') + recipe_sig = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tgz.sig') + recipe_tgz = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tgz') self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.') self.assertEqual(len(recipe_tgz), 1, 'Failed to find .tgz file.') diff --git a/meta/lib/oeqa/selftest/cases/sstate.py b/meta/lib/oeqa/selftest/cases/sstate.py index 410dec64fcd..80ce9e353c0 100644 --- a/meta/lib/oeqa/selftest/cases/sstate.py +++ b/meta/lib/oeqa/selftest/cases/sstate.py @@ -56,11 +56,11 @@ class SStateBase(OESelftestTestCase): def search_sstate(self, filename_regex, distro_specific=True, distro_nonspecific=True): result = [] for root, dirs, files in os.walk(self.sstate_path): - if distro_specific and re.search("%s/[a-z0-9]{2}$" % self.hostdistro, root): + if distro_specific and re.search(r"%s/%s/[a-z0-9]{2}/[a-z0-9]{2}$" % (self.sstate_path, self.hostdistro), root): for f in files: if re.search(filename_regex, f): result.append(f) - if distro_nonspecific and re.search("%s/[a-z0-9]{2}$" % self.sstate_path, root): + if distro_nonspecific and re.search(r"%s/[a-z0-9]{2}/[a-z0-9]{2}$" % self.sstate_path, root): for f in files: if re.search(filename_regex, f): result.append(f) diff --git a/scripts/gen-lockedsig-cache b/scripts/gen-lockedsig-cache index 9bfae9d8323..6a7d2859104 100755 --- a/scripts/gen-lockedsig-cache +++ b/scripts/gen-lockedsig-cache @@ -78,11 +78,18 @@ files = set() sstate_content_cache = {} for s in sigs: prefix = s[:2] + prefix2 = s[2:4] if prefix not in sstate_content_cache: sstate_content_cache[prefix] = build_sha_cache(prefix) - - for f in sstate_content_cache[prefix][s]: - files.add(f) + if prefix2 not in sstate_content_cache[prefix]: + sstate_content_cache[prefix][prefix2] = build_sha_cache(prefix + "/" + prefix2) + + if s in sstate_content_cache[prefix]: + for f in sstate_content_cache[prefix][s]: + files.add(f) + if s in sstate_content_cache[prefix][prefix2]: + for f in sstate_content_cache[prefix][prefix2][s]: + files.add(f) elapsed = time.perf_counter() - start_time print("Gathering file list took %.1fs" % elapsed) -- 2.20.1 -- _______________________________________________ Openembedded-core mailing list [email protected] http://lists.openembedded.org/mailman/listinfo/openembedded-core
