The rysnc of 20,000 files (650MB) onto the nas is slow taking ~3 minutes at idle and worse at load. This is due to the number of files which is a pain point for NFS. This piece of the build is also a bottleneck since the rest of a build depends on it happening.
If we switch to zstd compressed tar, it takes 2.49s. Other compression methods were much slower but zstd seems 'accptable' and speeds things up too. Signed-off-by: Richard Purdie <[email protected]> --- scripts/prepare-shared-repos | 4 ++-- scripts/send-qa-email | 6 ++++-- scripts/shared-repo-unpack | 5 ++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/prepare-shared-repos b/scripts/prepare-shared-repos index f34ba99..d60ad32 100755 --- a/scripts/prepare-shared-repos +++ b/scripts/prepare-shared-repos @@ -39,5 +39,5 @@ with tempfile.TemporaryDirectory(prefix="shared-repo-temp-", dir="/home/pokybuil if args.publish_dir: utils.publishrepo(tempdir, repo, args.publish_dir) - utils.printheader("Running rsync") - subprocess.check_call("rsync -a " + tempdir + "/* " + args.sharedsrcdir, shell=True) + utils.printheader("Creating shared src tarball") + subprocess.check_call("tar -I zstd -cf " + args.sharedsrcdir.rstrip("/") + ".tar.zst ./*", shell=True, cwd=tempdir) diff --git a/scripts/send-qa-email b/scripts/send-qa-email index 1b69307..fc34f7e 100755 --- a/scripts/send-qa-email +++ b/scripts/send-qa-email @@ -45,9 +45,11 @@ buildtoolsdir = os.path.dirname(args.repojson) + "/build/buildtools" if os.path.exists(buildtoolsdir): utils.enable_buildtools_tarball(buildtoolsdir) +repodir = os.path.dirname(args.repojson) + "/repos" + if 'poky' in repos and os.path.exists(resulttool) and args.results_dir: # Need the finalised revisions (not 'HEAD') - targetrepodir = "%s/poky" % (args.sharedrepodir) + targetrepodir = "%s/poky" % (repodir) revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=targetrepodir).decode('utf-8').strip() branch = repos['poky']['branch'] repo = repos['poky']['url'] @@ -116,7 +118,7 @@ if args.send.lower() != 'true' or not args.publish_dir or not args.release: buildhashes = "" for repo in sorted(repos.keys()): # Need the finalised revisions (not 'HEAD') - targetrepodir = "%s/%s" % (args.sharedrepodir, repo) + targetrepodir = "%s/%s" % (repodir, repo) revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=targetrepodir).decode('utf-8').strip() buildhashes += "%s: %s\n" % (repo, revision) diff --git a/scripts/shared-repo-unpack b/scripts/shared-repo-unpack index f08efa8..3dda5b3 100755 --- a/scripts/shared-repo-unpack +++ b/scripts/shared-repo-unpack @@ -50,11 +50,10 @@ needrepos_baseddirs = [r.split('/')[0] for r in needrepos] for repo in sorted(repos.keys()): if repo not in needrepos_baseddirs: continue - targetrepodir = "%s/%s" % (targetsubdir, repo) if args.cache_dir: utils.printheader("Copying in repo %s" % repo) - utils.mkdir(targetrepodir) - subprocess.check_call(["rsync", "-a", "%s/%s" % (args.cache_dir, repo), targetsubdir]) + utils.mkdir(targetsubdir) + subprocess.check_call(["tar", "-I", "zstd", "-C", targetsubdir, "-xf", "%s.tar.zst" % args.cache_dir, "./" + repo]) else: utils.printheader("Fetching repo %s" % repo) utils.fetchgitrepo(targetsubdir, repo, repos[repo], stashdir) -- 2.32.0
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#55198): https://lists.yoctoproject.org/g/yocto/message/55198 Mute This Topic: https://lists.yoctoproject.org/mt/86716910/21656 Group Owner: [email protected] Unsubscribe: https://lists.yoctoproject.org/g/yocto/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
