[OE-core] [PATCH 1/5] buildstats-plot.sh: fix invoking buildstats.sh
Fix the internal call to buildstats.sh by also providing the buildstats folder location in the system, which might differ from the default location. Upstream-Status: Pending Signed-off-by: Luis Martins --- scripts/contrib/bb-perf/buildstats-plot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/contrib/bb-perf/buildstats-plot.sh b/scripts/contrib/bb-perf/buildstats-plot.sh index 898834e5ac..089621c9f8 100755 --- a/scripts/contrib/bb-perf/buildstats-plot.sh +++ b/scripts/contrib/bb-perf/buildstats-plot.sh @@ -101,7 +101,7 @@ CD=$(dirname $0) # Parse buildstats recipes to produce a single table OUTBUILDSTATS="$PWD/buildstats.log" -$CD/buildstats.sh -H -s "$STATS" -H > $OUTBUILDSTATS +$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -H > $OUTBUILDSTATS # Get headers HEADERS=$(cat $OUTBUILDSTATS | sed -n -e '1s/ /-/g' -e '1s/:/ /gp') -- 2.25.2 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#136632): https://lists.openembedded.org/g/openembedded-core/message/136632 Mute This Topic: https://lists.openembedded.org/mt/72514839/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 4/5] buildstats*.sh: add recipe parameter to scripts
Add option to filter the stats results for a single recipe, allowing to easilly visuality the task resource consumption per recipe. This is specially useful when debugging the performance of the overall bitbake build system. Upstream-Status: Pending Signed-off-by: Luis Martins --- scripts/contrib/bb-perf/buildstats-plot.sh | 9 +++-- scripts/contrib/bb-perf/buildstats.sh | 9 +++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/contrib/bb-perf/buildstats-plot.sh b/scripts/contrib/bb-perf/buildstats-plot.sh index f26312f355..320c4910a1 100755 --- a/scripts/contrib/bb-perf/buildstats-plot.sh +++ b/scripts/contrib/bb-perf/buildstats-plot.sh @@ -39,6 +39,7 @@ set -o errexit BS_DIR="tmp/buildstats" N=10 +RECIPE="" TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack" STATS="utime" ACCUMULATE="" @@ -53,6 +54,7 @@ Usage: $CMD [-b buildstats_dir] [-t do_task] (default: "$BS_DIR") -n N Top N recipes to display. Ignored if -S is present (default: "$N") + -r recipe The recipe mask to be searched -t tasks The tasks to be computed (default: "$TASKS") -s stats The stats to be matched. If more that one stat, units @@ -69,7 +71,7 @@ EOM } # Parse and validate arguments -while getopts "b:n:t:s:o:aSh" OPT; do +while getopts "b:n:r:t:s:o:aSh" OPT; do case $OPT in b) BS_DIR="$OPTARG" @@ -77,6 +79,9 @@ while getopts "b:n:t:s:o:aSh" OPT; do n) N="$OPTARG" ;; +r) +RECIPE="-r $OPTARG" +;; t) TASKS="$OPTARG" ;; @@ -112,7 +117,7 @@ CD=$(dirname $0) # Parse buildstats recipes to produce a single table OUTBUILDSTATS="$PWD/buildstats.log" -$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -t "$TASKS" $ACCUMULATE -H > $OUTBUILDSTATS +$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -t "$TASKS" $RECIPE $ACCUMULATE -H > $OUTBUILDSTATS # Get headers HEADERS=$(cat $OUTBUILDSTATS | sed -n -e '1s/ /-/g' -e '1s/:/ /gp') diff --git a/scripts/contrib/bb-perf/buildstats.sh b/scripts/contrib/bb-perf/buildstats.sh index 26bfaca86b..2f017efeb8 100755 --- a/scripts/contrib/bb-perf/buildstats.sh +++ b/scripts/contrib/bb-perf/buildstats.sh @@ -36,6 +36,7 @@ Child rusage ru_majflt:Child rusage ru_inblock:Child rusage ru_oublock:Child rus Child rusage ru_nivcsw" BS_DIR="tmp/buildstats" +RECIPE="" TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack" STATS="$TIME" ACCUMULATE="" @@ -47,6 +48,7 @@ cat <-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#136635): https://lists.openembedded.org/g/openembedded-core/message/136635 Mute This Topic: https://lists.openembedded.org/mt/72514842/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 5/5] buildstats*.sh: fix spacing indentation in the files
Fix indentation in buildstats.sh and buildstats-plot.sh
to convert every line to 4 spaces indentation.
Upstream-Status: Pending
Signed-off-by: Luis Martins
---
scripts/contrib/bb-perf/buildstats-plot.sh | 56 -
scripts/contrib/bb-perf/buildstats.sh | 131 +++--
2 files changed, 94 insertions(+), 93 deletions(-)
diff --git a/scripts/contrib/bb-perf/buildstats-plot.sh
b/scripts/contrib/bb-perf/buildstats-plot.sh
index 320c4910a1..80fd30960c 100755
--- a/scripts/contrib/bb-perf/buildstats-plot.sh
+++ b/scripts/contrib/bb-perf/buildstats-plot.sh
@@ -72,40 +72,40 @@ EOM
# Parse and validate arguments
while getopts "b:n:r:t:s:o:aSh" OPT; do
- case $OPT in
- b)
- BS_DIR="$OPTARG"
- ;;
- n)
- N="$OPTARG"
- ;;
+case $OPT in
+b)
+BS_DIR="$OPTARG"
+;;
+n)
+N="$OPTARG"
+;;
r)
RECIPE="-r $OPTARG"
;;
t)
TASKS="$OPTARG"
;;
- s)
- STATS="$OPTARG"
- ;;
- a)
+s)
+STATS="$OPTARG"
+;;
+a)
ACCUMULATE="-a"
;;
- S)
- SUM="y"
- ;;
- o)
- OUTDATA_FILE="$OPTARG"
- ;;
- h)
- usage
- exit 0
- ;;
- *)
- usage
- exit 1
- ;;
- esac
+S)
+SUM="y"
+;;
+o)
+OUTDATA_FILE="$OPTARG"
+;;
+h)
+usage
+exit 0
+;;
+*)
+usage
+exit 1
+;;
+esac
done
# Get number of stats
@@ -141,8 +141,8 @@ else
declare -a sumargs
j=0
for i in `seq $nstats`; do
- sumargs[j]=sum; j=$(( $j + 1 ))
- sumargs[j]=`expr 3 + $i - 1`; j=$(( $j + 1 ))
+sumargs[j]=sum; j=$(( $j + 1 ))
+sumargs[j]=`expr 3 + $i - 1`; j=$(( $j + 1 ))
done
# Do the processing with datamash
diff --git a/scripts/contrib/bb-perf/buildstats.sh
b/scripts/contrib/bb-perf/buildstats.sh
index 2f017efeb8..e45cfc146d 100755
--- a/scripts/contrib/bb-perf/buildstats.sh
+++ b/scripts/contrib/bb-perf/buildstats.sh
@@ -66,62 +66,63 @@ EOM
# Parse and validate arguments
while getopts "b:r:t:s:aHh" OPT; do
- case $OPT in
- b)
- BS_DIR="$OPTARG"
- ;;
- r)
- RECIPE="$OPTARG"
- ;;
- t)
- TASKS="$OPTARG"
- ;;
- s)
- STATS="$OPTARG"
- ;;
- a)
+case $OPT in
+b)
+BS_DIR="$OPTARG"
+;;
+r)
+RECIPE="$OPTARG"
+;;
+t)
+TASKS="$OPTARG"
+;;
+s)
+STATS="$OPTARG"
+;;
+a)
ACCUMULATE="y"
;;
- H)
- HEADER="y"
- ;;
- h)
- usage
- exit 0
- ;;
- *)
- usage
- exit 1
- ;;
- esac
+H)
+HEADER="y"
+;;
+h)
+usage
+exit 0
+;;
+*)
+usage
+exit 1
+;;
+esac
done
# Ensure the buildstats folder exists
if [ ! -d "$BS_DIR" ]; then
- echo "ERROR: $BS_DIR does not exist"
- usage
- exit 1
+echo "ERROR: $BS_DIR does not exist"
+usage
+exit 1
fi
stats=""
IFS=":"
for stat in ${STATS}; do
- case $stat in
- TIME)
- stats="${stats}:${TIME}"
- ;;
- IO)
- stats="${stats}:${IO}"
- ;;
- RUSAGE)
- stats="${stats}:${RUSAGE}"
- ;;
- CHILD_RUSAGE)
- stats="${stats}:${CHILD_RUSAGE}"
- ;;
- *)
- stats="${STATS}"
- esac
+case $stat in
+TIME)
+stats="${stats}:${TIME}"
+;;
+IO)
+stats="${stats}:${IO}"
+;;
+RUSAGE)
+stats="${stats}:${RUSAGE}"
+;;
+CHILD_RUSAGE)
+stats="${stats}:${CHILD_RUSAGE}"
+;;
+*)
+stats="${STATS}"
+;;
+esac
done
# remove possible colon at the beginning
@@ -140,27 +141,27 @@ for task in ${TASKS}; do
task="do_${task}"
for file in $(find ${BS_DIR} -type f -path *${RECIPE}*/${task} | awk
'BEGIN{ ORS=""; OFS=":" } { print $0,"" }'); d
[OE-core] [PATCH 3/5] buildstats*.sh: add accumulate parameter to scripts
Add option to accumulate stats values per recipe, allowing
to sum related values such as memory (main process + childs).
This is specially useful when debugging the performance of
the overall bitbake build system.
Upstream-Status: Pending
Signed-off-by: Luis Martins
---
scripts/contrib/bb-perf/buildstats-plot.sh | 9 +++--
scripts/contrib/bb-perf/buildstats.sh | 23 --
2 files changed, 28 insertions(+), 4 deletions(-)
diff --git a/scripts/contrib/bb-perf/buildstats-plot.sh
b/scripts/contrib/bb-perf/buildstats-plot.sh
index 1d22e60d73..f26312f355 100755
--- a/scripts/contrib/bb-perf/buildstats-plot.sh
+++ b/scripts/contrib/bb-perf/buildstats-plot.sh
@@ -41,6 +41,7 @@ BS_DIR="tmp/buildstats"
N=10
TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack"
STATS="utime"
+ACCUMULATE=""
SUM=""
OUTDATA_FILE="$PWD/buildstats-plot.out"
@@ -59,6 +60,7 @@ Usage: $CMD [-b buildstats_dir] [-t do_task]
(see buildstats.sh -h for all options) or any other defined
(build)stat separated by colons, i.e. stime:utime
(default: "$STATS")
+ -aAccumulate all stats values for found recipes
-SSum values for a particular stat for found recipes
-oOutput data file.
(default: "$OUTDATA_FILE")
@@ -67,7 +69,7 @@ EOM
}
# Parse and validate arguments
-while getopts "b:n:t:s:o:Sh" OPT; do
+while getopts "b:n:t:s:o:aSh" OPT; do
case $OPT in
b)
BS_DIR="$OPTARG"
@@ -81,6 +83,9 @@ while getopts "b:n:t:s:o:Sh" OPT; do
s)
STATS="$OPTARG"
;;
+ a)
+ACCUMULATE="-a"
+;;
S)
SUM="y"
;;
@@ -107,7 +112,7 @@ CD=$(dirname $0)
# Parse buildstats recipes to produce a single table
OUTBUILDSTATS="$PWD/buildstats.log"
-$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -t "$TASKS" -H > $OUTBUILDSTATS
+$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -t "$TASKS" $ACCUMULATE -H >
$OUTBUILDSTATS
# Get headers
HEADERS=$(cat $OUTBUILDSTATS | sed -n -e '1s/ /-/g' -e '1s/:/ /gp')
diff --git a/scripts/contrib/bb-perf/buildstats.sh
b/scripts/contrib/bb-perf/buildstats.sh
index e9ec2d476a..26bfaca86b 100755
--- a/scripts/contrib/bb-perf/buildstats.sh
+++ b/scripts/contrib/bb-perf/buildstats.sh
@@ -38,6 +38,7 @@ Child rusage ru_nivcsw"
BS_DIR="tmp/buildstats"
TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack"
STATS="$TIME"
+ACCUMULATE=""
HEADER="" # No header by default
function usage {
@@ -56,12 +57,13 @@ Usage: $CMD [-b buildstats_dir] [-t do_task]
IO=$IO
RUSAGE=$RUSAGE
CHILD_RUSAGE=$CHILD_RUSAGE
+ -aAccumulate all stats values for found recipes
-hDisplay this help message
EOM
}
# Parse and validate arguments
-while getopts "b:t:s:Hh" OPT; do
+while getopts "b:t:s:aHh" OPT; do
case $OPT in
b)
BS_DIR="$OPTARG"
@@ -72,6 +74,9 @@ while getopts "b:t:s:Hh" OPT; do
s)
STATS="$OPTARG"
;;
+ a)
+ACCUMULATE="y"
+;;
H)
HEADER="y"
;;
@@ -118,7 +123,13 @@ done
stats="$(echo "$stats" | sed -e 's/^://1')"
# Provide a header if required by the user
-[ -n "$HEADER" ] && { echo "task:recipe:$stats"; }
+if [ -n "$HEADER" ] ; then
+if [ -n "$ACCUMULATE" ]; then
+echo "task:recipe:accumulated(${stats//:/;})"
+else
+echo "task:recipe:$stats"
+fi
+fi
for task in ${TASKS}; do
task="do_${task}"
@@ -137,6 +148,14 @@ for task in ${TASKS}; do
times="${times} ${time}"
fi
done
+if [ -n "$ACCUMULATE" ]; then
+IFS=' '; valuesarray=(${times}); IFS=':'
+times=0
+for value in "${valuesarray[@]}"; do
+[ "$value" == "NA" ] && { echo "ERROR: stat is not present.";
usage; exit 1; }
+times=$(( $times + $value ))
+done
+fi
echo "${task} ${recipe} ${times}"
done
done
--
2.25.2
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#136634):
https://lists.openembedded.org/g/openembedded-core/message/136634
Mute This Topic: https://lists.openembedded.org/mt/72514841/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 2/5] buildstats-plot.sh: filter by task
Extend buildstats-plot.sh script to also accept the name of the tasks as parameter. This value will be passed directly to buildstats.sh is already provides this option. Upstream-Status: Pending Signed-off-by: Luis Martins --- scripts/contrib/bb-perf/buildstats-plot.sh | 10 -- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/contrib/bb-perf/buildstats-plot.sh b/scripts/contrib/bb-perf/buildstats-plot.sh index 089621c9f8..1d22e60d73 100755 --- a/scripts/contrib/bb-perf/buildstats-plot.sh +++ b/scripts/contrib/bb-perf/buildstats-plot.sh @@ -39,6 +39,7 @@ set -o errexit BS_DIR="tmp/buildstats" N=10 +TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack" STATS="utime" SUM="" OUTDATA_FILE="$PWD/buildstats-plot.out" @@ -51,6 +52,8 @@ Usage: $CMD [-b buildstats_dir] [-t do_task] (default: "$BS_DIR") -n N Top N recipes to display. Ignored if -S is present (default: "$N") + -t tasks The tasks to be computed +(default: "$TASKS") -s stats The stats to be matched. If more that one stat, units should be the same because data is plot as histogram. (see buildstats.sh -h for all options) or any other defined @@ -64,7 +67,7 @@ EOM } # Parse and validate arguments -while getopts "b:n:s:o:Sh" OPT; do +while getopts "b:n:t:s:o:Sh" OPT; do case $OPT in b) BS_DIR="$OPTARG" @@ -72,6 +75,9 @@ while getopts "b:n:s:o:Sh" OPT; do n) N="$OPTARG" ;; +t) +TASKS="$OPTARG" +;; s) STATS="$OPTARG" ;; @@ -101,7 +107,7 @@ CD=$(dirname $0) # Parse buildstats recipes to produce a single table OUTBUILDSTATS="$PWD/buildstats.log" -$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -H > $OUTBUILDSTATS +$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -t "$TASKS" -H > $OUTBUILDSTATS # Get headers HEADERS=$(cat $OUTBUILDSTATS | sed -n -e '1s/ /-/g' -e '1s/:/ /gp') -- 2.25.2 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#136633): https://lists.openembedded.org/g/openembedded-core/message/136633 Mute This Topic: https://lists.openembedded.org/mt/72514840/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 0/5] improvements to buildstats scripts
This patches add some new functionalities to the buildstats scripts to allow easier debugging of the bitbake performance. Some new features like accumulating the stats values allow to compare the total amount of memory used by a single task (main process + child processes) and other features allow convenient analysis of task performance per recipe. Luis Martins (5): buildstats-plot.sh: fix invoking buildstats.sh buildstats-plot.sh: filter by task buildstats*.sh: add accumulate parameter to scripts buildstats*.sh: add recipe parameter to scripts buildstats*.sh: fix spacing indentation in the files scripts/contrib/bb-perf/buildstats-plot.sh | 74 ++- scripts/contrib/bb-perf/buildstats.sh | 139 - 2 files changed, 127 insertions(+), 86 deletions(-) -- 2.25.2 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#136631): https://lists.openembedded.org/g/openembedded-core/message/136631 Mute This Topic: https://lists.openembedded.org/mt/72514838/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 2/5] buildstats-plot.sh: filter by task
Extend buildstats-plot.sh script to also accept the name of the tasks as parameter. This value will be passed directly to buildstats.sh is already provides this option. Signed-off-by: Luis Martins --- scripts/contrib/bb-perf/buildstats-plot.sh | 10 -- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/contrib/bb-perf/buildstats-plot.sh b/scripts/contrib/bb-perf/buildstats-plot.sh index 089621c9f8..1d22e60d73 100755 --- a/scripts/contrib/bb-perf/buildstats-plot.sh +++ b/scripts/contrib/bb-perf/buildstats-plot.sh @@ -39,6 +39,7 @@ set -o errexit BS_DIR="tmp/buildstats" N=10 +TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack" STATS="utime" SUM="" OUTDATA_FILE="$PWD/buildstats-plot.out" @@ -51,6 +52,8 @@ Usage: $CMD [-b buildstats_dir] [-t do_task] (default: "$BS_DIR") -n N Top N recipes to display. Ignored if -S is present (default: "$N") + -t tasks The tasks to be computed +(default: "$TASKS") -s stats The stats to be matched. If more that one stat, units should be the same because data is plot as histogram. (see buildstats.sh -h for all options) or any other defined @@ -64,7 +67,7 @@ EOM } # Parse and validate arguments -while getopts "b:n:s:o:Sh" OPT; do +while getopts "b:n:t:s:o:Sh" OPT; do case $OPT in b) BS_DIR="$OPTARG" @@ -72,6 +75,9 @@ while getopts "b:n:s:o:Sh" OPT; do n) N="$OPTARG" ;; +t) +TASKS="$OPTARG" +;; s) STATS="$OPTARG" ;; @@ -101,7 +107,7 @@ CD=$(dirname $0) # Parse buildstats recipes to produce a single table OUTBUILDSTATS="$PWD/buildstats.log" -$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -H > $OUTBUILDSTATS +$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -t "$TASKS" -H > $OUTBUILDSTATS # Get headers HEADERS=$(cat $OUTBUILDSTATS | sed -n -e '1s/ /-/g' -e '1s/:/ /gp') -- 2.25.2 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#136651): https://lists.openembedded.org/g/openembedded-core/message/136651 Mute This Topic: https://lists.openembedded.org/mt/72518490/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 5/5] buildstats*.sh: fix spacing indentation in the files
Fix indentation in buildstats.sh and buildstats-plot.sh
to convert every line to 4 spaces indentation.
Signed-off-by: Luis Martins
---
scripts/contrib/bb-perf/buildstats-plot.sh | 56 -
scripts/contrib/bb-perf/buildstats.sh | 131 +++--
2 files changed, 94 insertions(+), 93 deletions(-)
diff --git a/scripts/contrib/bb-perf/buildstats-plot.sh
b/scripts/contrib/bb-perf/buildstats-plot.sh
index 320c4910a1..80fd30960c 100755
--- a/scripts/contrib/bb-perf/buildstats-plot.sh
+++ b/scripts/contrib/bb-perf/buildstats-plot.sh
@@ -72,40 +72,40 @@ EOM
# Parse and validate arguments
while getopts "b:n:r:t:s:o:aSh" OPT; do
- case $OPT in
- b)
- BS_DIR="$OPTARG"
- ;;
- n)
- N="$OPTARG"
- ;;
+case $OPT in
+b)
+BS_DIR="$OPTARG"
+;;
+n)
+N="$OPTARG"
+;;
r)
RECIPE="-r $OPTARG"
;;
t)
TASKS="$OPTARG"
;;
- s)
- STATS="$OPTARG"
- ;;
- a)
+s)
+STATS="$OPTARG"
+;;
+a)
ACCUMULATE="-a"
;;
- S)
- SUM="y"
- ;;
- o)
- OUTDATA_FILE="$OPTARG"
- ;;
- h)
- usage
- exit 0
- ;;
- *)
- usage
- exit 1
- ;;
- esac
+S)
+SUM="y"
+;;
+o)
+OUTDATA_FILE="$OPTARG"
+;;
+h)
+usage
+exit 0
+;;
+*)
+usage
+exit 1
+;;
+esac
done
# Get number of stats
@@ -141,8 +141,8 @@ else
declare -a sumargs
j=0
for i in `seq $nstats`; do
- sumargs[j]=sum; j=$(( $j + 1 ))
- sumargs[j]=`expr 3 + $i - 1`; j=$(( $j + 1 ))
+sumargs[j]=sum; j=$(( $j + 1 ))
+sumargs[j]=`expr 3 + $i - 1`; j=$(( $j + 1 ))
done
# Do the processing with datamash
diff --git a/scripts/contrib/bb-perf/buildstats.sh
b/scripts/contrib/bb-perf/buildstats.sh
index 2f017efeb8..e45cfc146d 100755
--- a/scripts/contrib/bb-perf/buildstats.sh
+++ b/scripts/contrib/bb-perf/buildstats.sh
@@ -66,62 +66,63 @@ EOM
# Parse and validate arguments
while getopts "b:r:t:s:aHh" OPT; do
- case $OPT in
- b)
- BS_DIR="$OPTARG"
- ;;
- r)
- RECIPE="$OPTARG"
- ;;
- t)
- TASKS="$OPTARG"
- ;;
- s)
- STATS="$OPTARG"
- ;;
- a)
+case $OPT in
+b)
+BS_DIR="$OPTARG"
+;;
+r)
+RECIPE="$OPTARG"
+;;
+t)
+TASKS="$OPTARG"
+;;
+s)
+STATS="$OPTARG"
+;;
+a)
ACCUMULATE="y"
;;
- H)
- HEADER="y"
- ;;
- h)
- usage
- exit 0
- ;;
- *)
- usage
- exit 1
- ;;
- esac
+H)
+HEADER="y"
+;;
+h)
+usage
+exit 0
+;;
+*)
+usage
+exit 1
+;;
+esac
done
# Ensure the buildstats folder exists
if [ ! -d "$BS_DIR" ]; then
- echo "ERROR: $BS_DIR does not exist"
- usage
- exit 1
+echo "ERROR: $BS_DIR does not exist"
+usage
+exit 1
fi
stats=""
IFS=":"
for stat in ${STATS}; do
- case $stat in
- TIME)
- stats="${stats}:${TIME}"
- ;;
- IO)
- stats="${stats}:${IO}"
- ;;
- RUSAGE)
- stats="${stats}:${RUSAGE}"
- ;;
- CHILD_RUSAGE)
- stats="${stats}:${CHILD_RUSAGE}"
- ;;
- *)
- stats="${STATS}"
- esac
+case $stat in
+TIME)
+stats="${stats}:${TIME}"
+;;
+IO)
+stats="${stats}:${IO}"
+;;
+RUSAGE)
+stats="${stats}:${RUSAGE}"
+;;
+CHILD_RUSAGE)
+stats="${stats}:${CHILD_RUSAGE}"
+;;
+*)
+stats="${STATS}"
+;;
+esac
done
# remove possible colon at the beginning
@@ -140,27 +141,27 @@ for task in ${TASKS}; do
task="do_${task}"
for file in $(find ${BS_DIR} -type f -path *${RECIPE}*/${task} | awk
'BEGIN{ ORS=""; OFS=":" } { print $0,"" }'); do
recipe=&quo
[OE-core] [PATCH v2 3/5] buildstats*.sh: add accumulate parameter to scripts
Add option to accumulate stats values per recipe, allowing
to sum related values such as memory (main process + childs).
This is specially useful when debugging the performance of
the overall bitbake build system.
Signed-off-by: Luis Martins
---
scripts/contrib/bb-perf/buildstats-plot.sh | 9 +++--
scripts/contrib/bb-perf/buildstats.sh | 23 --
2 files changed, 28 insertions(+), 4 deletions(-)
diff --git a/scripts/contrib/bb-perf/buildstats-plot.sh
b/scripts/contrib/bb-perf/buildstats-plot.sh
index 1d22e60d73..f26312f355 100755
--- a/scripts/contrib/bb-perf/buildstats-plot.sh
+++ b/scripts/contrib/bb-perf/buildstats-plot.sh
@@ -41,6 +41,7 @@ BS_DIR="tmp/buildstats"
N=10
TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack"
STATS="utime"
+ACCUMULATE=""
SUM=""
OUTDATA_FILE="$PWD/buildstats-plot.out"
@@ -59,6 +60,7 @@ Usage: $CMD [-b buildstats_dir] [-t do_task]
(see buildstats.sh -h for all options) or any other defined
(build)stat separated by colons, i.e. stime:utime
(default: "$STATS")
+ -aAccumulate all stats values for found recipes
-SSum values for a particular stat for found recipes
-oOutput data file.
(default: "$OUTDATA_FILE")
@@ -67,7 +69,7 @@ EOM
}
# Parse and validate arguments
-while getopts "b:n:t:s:o:Sh" OPT; do
+while getopts "b:n:t:s:o:aSh" OPT; do
case $OPT in
b)
BS_DIR="$OPTARG"
@@ -81,6 +83,9 @@ while getopts "b:n:t:s:o:Sh" OPT; do
s)
STATS="$OPTARG"
;;
+ a)
+ACCUMULATE="-a"
+;;
S)
SUM="y"
;;
@@ -107,7 +112,7 @@ CD=$(dirname $0)
# Parse buildstats recipes to produce a single table
OUTBUILDSTATS="$PWD/buildstats.log"
-$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -t "$TASKS" -H > $OUTBUILDSTATS
+$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -t "$TASKS" $ACCUMULATE -H >
$OUTBUILDSTATS
# Get headers
HEADERS=$(cat $OUTBUILDSTATS | sed -n -e '1s/ /-/g' -e '1s/:/ /gp')
diff --git a/scripts/contrib/bb-perf/buildstats.sh
b/scripts/contrib/bb-perf/buildstats.sh
index e9ec2d476a..26bfaca86b 100755
--- a/scripts/contrib/bb-perf/buildstats.sh
+++ b/scripts/contrib/bb-perf/buildstats.sh
@@ -38,6 +38,7 @@ Child rusage ru_nivcsw"
BS_DIR="tmp/buildstats"
TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack"
STATS="$TIME"
+ACCUMULATE=""
HEADER="" # No header by default
function usage {
@@ -56,12 +57,13 @@ Usage: $CMD [-b buildstats_dir] [-t do_task]
IO=$IO
RUSAGE=$RUSAGE
CHILD_RUSAGE=$CHILD_RUSAGE
+ -aAccumulate all stats values for found recipes
-hDisplay this help message
EOM
}
# Parse and validate arguments
-while getopts "b:t:s:Hh" OPT; do
+while getopts "b:t:s:aHh" OPT; do
case $OPT in
b)
BS_DIR="$OPTARG"
@@ -72,6 +74,9 @@ while getopts "b:t:s:Hh" OPT; do
s)
STATS="$OPTARG"
;;
+ a)
+ACCUMULATE="y"
+;;
H)
HEADER="y"
;;
@@ -118,7 +123,13 @@ done
stats="$(echo "$stats" | sed -e 's/^://1')"
# Provide a header if required by the user
-[ -n "$HEADER" ] && { echo "task:recipe:$stats"; }
+if [ -n "$HEADER" ] ; then
+if [ -n "$ACCUMULATE" ]; then
+echo "task:recipe:accumulated(${stats//:/;})"
+else
+echo "task:recipe:$stats"
+fi
+fi
for task in ${TASKS}; do
task="do_${task}"
@@ -137,6 +148,14 @@ for task in ${TASKS}; do
times="${times} ${time}"
fi
done
+if [ -n "$ACCUMULATE" ]; then
+IFS=' '; valuesarray=(${times}); IFS=':'
+times=0
+for value in "${valuesarray[@]}"; do
+[ "$value" == "NA" ] && { echo "ERROR: stat is not present.";
usage; exit 1; }
+times=$(( $times + $value ))
+done
+fi
echo "${task} ${recipe} ${times}"
done
done
--
2.25.2
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#136652):
https://lists.openembedded.org/g/openembedded-core/message/136652
Mute This Topic: https://lists.openembedded.org/mt/72518492/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 4/5] buildstats*.sh: add recipe parameter to scripts
Add option to filter the stats results for a single recipe, allowing to easilly visuality the task resource consumption per recipe. This is specially useful when debugging the performance of the overall bitbake build system. Signed-off-by: Luis Martins --- scripts/contrib/bb-perf/buildstats-plot.sh | 9 +++-- scripts/contrib/bb-perf/buildstats.sh | 9 +++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/contrib/bb-perf/buildstats-plot.sh b/scripts/contrib/bb-perf/buildstats-plot.sh index f26312f355..320c4910a1 100755 --- a/scripts/contrib/bb-perf/buildstats-plot.sh +++ b/scripts/contrib/bb-perf/buildstats-plot.sh @@ -39,6 +39,7 @@ set -o errexit BS_DIR="tmp/buildstats" N=10 +RECIPE="" TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack" STATS="utime" ACCUMULATE="" @@ -53,6 +54,7 @@ Usage: $CMD [-b buildstats_dir] [-t do_task] (default: "$BS_DIR") -n N Top N recipes to display. Ignored if -S is present (default: "$N") + -r recipe The recipe mask to be searched -t tasks The tasks to be computed (default: "$TASKS") -s stats The stats to be matched. If more that one stat, units @@ -69,7 +71,7 @@ EOM } # Parse and validate arguments -while getopts "b:n:t:s:o:aSh" OPT; do +while getopts "b:n:r:t:s:o:aSh" OPT; do case $OPT in b) BS_DIR="$OPTARG" @@ -77,6 +79,9 @@ while getopts "b:n:t:s:o:aSh" OPT; do n) N="$OPTARG" ;; +r) +RECIPE="-r $OPTARG" +;; t) TASKS="$OPTARG" ;; @@ -112,7 +117,7 @@ CD=$(dirname $0) # Parse buildstats recipes to produce a single table OUTBUILDSTATS="$PWD/buildstats.log" -$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -t "$TASKS" $ACCUMULATE -H > $OUTBUILDSTATS +$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -t "$TASKS" $RECIPE $ACCUMULATE -H > $OUTBUILDSTATS # Get headers HEADERS=$(cat $OUTBUILDSTATS | sed -n -e '1s/ /-/g' -e '1s/:/ /gp') diff --git a/scripts/contrib/bb-perf/buildstats.sh b/scripts/contrib/bb-perf/buildstats.sh index 26bfaca86b..2f017efeb8 100755 --- a/scripts/contrib/bb-perf/buildstats.sh +++ b/scripts/contrib/bb-perf/buildstats.sh @@ -36,6 +36,7 @@ Child rusage ru_majflt:Child rusage ru_inblock:Child rusage ru_oublock:Child rus Child rusage ru_nivcsw" BS_DIR="tmp/buildstats" +RECIPE="" TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack" STATS="$TIME" ACCUMULATE="" @@ -47,6 +48,7 @@ cat <-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#136653): https://lists.openembedded.org/g/openembedded-core/message/136653 Mute This Topic: https://lists.openembedded.org/mt/72518493/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 1/5] buildstats-plot.sh: fix invoking buildstats.sh
Fix the internal call to buildstats.sh by also providing the buildstats folder location in the system, which might differ from the default location. Signed-off-by: Luis Martins --- scripts/contrib/bb-perf/buildstats-plot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/contrib/bb-perf/buildstats-plot.sh b/scripts/contrib/bb-perf/buildstats-plot.sh index 898834e5ac..089621c9f8 100755 --- a/scripts/contrib/bb-perf/buildstats-plot.sh +++ b/scripts/contrib/bb-perf/buildstats-plot.sh @@ -101,7 +101,7 @@ CD=$(dirname $0) # Parse buildstats recipes to produce a single table OUTBUILDSTATS="$PWD/buildstats.log" -$CD/buildstats.sh -H -s "$STATS" -H > $OUTBUILDSTATS +$CD/buildstats.sh -b "$BS_DIR" -s "$STATS" -H > $OUTBUILDSTATS # Get headers HEADERS=$(cat $OUTBUILDSTATS | sed -n -e '1s/ /-/g' -e '1s/:/ /gp') -- 2.25.2 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#136650): https://lists.openembedded.org/g/openembedded-core/message/136650 Mute This Topic: https://lists.openembedded.org/mt/72518489/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 0/5] improvements to buildstats scripts
This patches add some new functionalities to the buildstats scripts to allow easier debugging of the bitbake performance. Some new features like accumulating the stats values allow to compare the total amount of memory used by a single task (main process + child processes) and other features allow convenient analysis of task performance per recipe. changes from v1 * removed unecessary "Upstream-Status: Pending" line Luis Martins (5): buildstats-plot.sh: fix invoking buildstats.sh buildstats-plot.sh: filter by task buildstats*.sh: add accumulate parameter to scripts buildstats*.sh: add recipe parameter to scripts buildstats*.sh: fix spacing indentation in the files scripts/contrib/bb-perf/buildstats-plot.sh | 74 ++- scripts/contrib/bb-perf/buildstats.sh | 139 - 2 files changed, 127 insertions(+), 86 deletions(-) -- 2.25.2 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#136649): https://lists.openembedded.org/g/openembedded-core/message/136649 Mute This Topic: https://lists.openembedded.org/mt/72518487/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH] rm_work.bbclass: use HOSTTOOLS 'rm' binary exclusively
Hi, I see this was accepted for master, is it okay to port this to dunfell
and kirkstone as well ?
Best regards,
Luís Martins
On Sat, 24 Dec 2022 at 01:49, Luis Martins
wrote:
> The do_rm_work() task is using the first available 'rm' binary
> available in PATH to remove files and folders.
> However, depending on the PATH setup and RECIPE_SYSROOT_NATIVE
> contents, the function can be using the 'rm' binary available
> in RECIPE_SYSROOT_NATIVE, a folder that will get removed.
> This causes a sporadic race-condition when trying to access the
> 'rm' binary of a folder already deleted.
> Solve this by exclusively using the HOSTTOOLS 'rm' binary, as
> this folder will not get removed.
>
> Signed-off-by: Luis Martins
> ---
> meta/classes/rm_work.bbclass | 15 +++
> 1 file changed, 11 insertions(+), 4 deletions(-)
>
> diff --git a/meta/classes/rm_work.bbclass b/meta/classes/rm_work.bbclass
> index 1f28bc7187..8b5fe1b808 100644
> --- a/meta/classes/rm_work.bbclass
> +++ b/meta/classes/rm_work.bbclass
> @@ -33,6 +33,13 @@ BB_SCHEDULER ?= "completion"
> BB_TASK_IONICE_LEVEL:task-rm_work = "3.0"
>
> do_rm_work () {
> +# Force using the HOSTTOOLS 'rm' - otherwise the SYSROOT_NATIVE 'rm'
> can be selected depending on PATH
> +# Avoids race-condition accessing 'rm' when deleting WORKDIR folders
> at the end of this function
> +RM_BIN="$(PATH=${HOSTTOOLS_DIR} command -v rm)"
> +if [ -z "${RM_BIN}" ]; then
> +bbfatal "Binary 'rm' not found in HOSTTOOLS_DIR, cannot remove
> WORKDIR data."
> +fi
> +
> # If the recipe name is in the RM_WORK_EXCLUDE, skip the recipe.
> for p in ${RM_WORK_EXCLUDE}; do
> if [ "$p" = "${PN}" ]; then
> @@ -79,7 +86,7 @@ do_rm_work () {
> # sstate version since otherwise we'd need to leave
> 'plaindirs' around
> # such as 'packages' and 'packages-split' and these can be
> large. No end
> # of chain tasks depend directly on do_package anymore.
> -rm -f -- $i;
> +"${RM_BIN}" -f -- $i;
> ;;
> *_setscene*)
> # Skip stamps which are already setscene versions
> @@ -96,7 +103,7 @@ do_rm_work () {
> ;;
> esac
> done
> -rm -f -- $i
> +"${RM_BIN}" -f -- $i
> esac
> done
>
> @@ -106,9 +113,9 @@ do_rm_work () {
> # Retain only logs and other files in temp, safely ignore
> # failures of removing pseudo folers on NFS2/3 server.
> if [ $dir = 'pseudo' ]; then
> -rm -rf -- $dir 2> /dev/null || true
> +"${RM_BIN}" -rf -- $dir 2> /dev/null || true
> elif ! echo "$excludes" | grep -q -w "$dir"; then
> -rm -rf -- $dir
> +"${RM_BIN}" -rf -- $dir
> fi
> done
> }
> --
> 2.25.1
>
>
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#175179):
https://lists.openembedded.org/g/openembedded-core/message/175179
Mute This Topic: https://lists.openembedded.org/mt/95856273/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
