The wrapper script; odp_run_app_isolated.sh can be used to run ODP
testcases in a isolated environment. Background noise can also be
generated.
Signed-off-by: Ravineet Singh <ravineet.si...@linaro.org>
v2: moved task-isolation dir to odp/scripts, requested my Maxim Uvarov
v3: fixed checkpatch.pl warnings
---
scripts/task-isolation/README | 22 ++
scripts/task-isolation/isolate-cpu.sh | 287 +++++++++++++++++++++
scripts/task-isolation/isolate-task.sh | 160 ++++++++++++
.../performance/odp_run_app_isolated.sh | 108 ++++++++
4 files changed, 577 insertions(+)
create mode 100644 scripts/task-isolation/README
create mode 100755 scripts/task-isolation/isolate-cpu.sh
create mode 100755 scripts/task-isolation/isolate-task.sh
create mode 100755 test/linux-generic/performance/odp_run_app_isolated.sh
diff --git a/scripts/task-isolation/README b/scripts/task-isolation/README
new file mode 100644
index 0000000..cb02056
--- /dev/null
+++ b/scripts/task-isolation/README
@@ -0,0 +1,22 @@
+Helper scripts to check and set CPU isolation and execution of application in
+isolated CPU(s)
+
+Files:
+isolate-cpu.sh isolates desired CPUs
+isolate-task.sh uses isolate-cpu.sh to isolate CPUs before running
+ the desired task. It also provides the possibility
+ to trace kernel disturbance on the isolated CPUs.
+
+isolate-cpu.sh checks the kernel configuration and the kernel cmdline to
+ determine if one or several CPUs are isolated, i.e. it checks for;
+ CONFIG_NO_HZ_FULL_ALL=y", and CONFIG_RCU_NOCB_CPU_ALL=y" in the kernel config
+ and rcu_nocbs,nohz_full in the kernel cmdline.
+ If the desired CPU(s) are not inte the above configuration, it warns but
continues
+ to isolate the CPU(s) as much as possibe. The isolation is accomplished by
+ - Redirecting all IRQ's away from desired isolated cores
+ - Using cset (cpuset) to move all running processes and kernel threads
+ away from desired isolated cores
+
+isolate-task.sh uses isolate-cpu.sh to isolate desired CPU(s). In addition
+ it starts the supplied application on isolated CPU(s) and optionally traces
+ the isolated CPU(s) for kernel interaction.
diff --git a/scripts/task-isolation/isolate-cpu.sh
b/scripts/task-isolation/isolate-cpu.sh
new file mode 100755
index 0000000..3eaf98a
--- /dev/null
+++ b/scripts/task-isolation/isolate-cpu.sh
@@ -0,0 +1,287 @@
+#!/bin/bash
+#
+# Copyright (c) 2017, Linaro Limited
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Script that passes command line arguments to odp_scheduling after,
+# optionally, isolating CPU
+#
+# This script isolates desired CPUS, i.e.
+# - Checks kernel cmdline and kernel config to determine
+# if the environment is optimised for isolated task execution;
+# - Moves CPU interrupts, kernel threads, tasks etc. away from the
+# targeted CPU.
+# *Note* CPU 0 cannot be isolated, i.e minimum 2 CPU's are required.
+
+
+print_usage() {
+ echo "$0 [-h] [-a] [-c <cpu list>] [-l] [-r] [-d]"
+ echo
+ echo " Isolate CPU(s) from other tasks, kernel threads and IRQs"
+ echo " Args:"
+ echo " -h Print this message"
+ echo " -a Isolate all CPUs (except CPU 0)"
+ echo " -c List of CPUs to be isolated."
+ echo " -l Show isolation proporties"
+ echo " -r Reset isolation"
+ echo " -d Show debug printouts"
+ echo ""
+ echo " Examples:"
+ echo " Isolate all CPU(s) (except 0) "
+ echo " $0 -a"
+ echo
+ echo " Isolate CPUs 1-3 "
+ echo " $0 -c 1-3"
+ echo
+ echo " Isolate CPUs 1 and 4 "
+ echo " $0 -c 1,4 "
+}
+
+dlog() {
+ [ $DEBUG ] && echo "$*"
+}
+
+warn() {
+ printf "Warning: $*\n" >&2
+}
+
+die() {
+ printf "Error: $*\n" >&2
+ exit 1
+}
+
+get_cpu_array() {
+ [ $1 ] || die "$FUNCNAME internal error!"
+
+ local cpus=""
+ IFS=. a=$1; IFS=, a=$a;
+ for str in $a; do
+ if [[ $str == *[\-]* ]]; then
+ str=$(echo $str| sed 's/-/../g')
+ str=$(eval echo {$str})
+ fi
+
+ if [ "$cpus" != "" ]; then
+ cpus="$cpus $str"
+ else
+ cpus=$str
+ fi
+ done
+
+ echo $cpus
+}
+
+##
+# Check kernel config and kernel cmdline for rcu callbacs and no_hz
+# *Note* isolcpu= kernel cmdline option isolates CPUs from SMP balancing
+# If needed, this can be done via
+# cpusets/user/cpuset.sched_load_balance
+##
+check_kernel_config() {
+
+ eval $(cat /proc/cmdline | grep -o 'nohz_full=[^ ]*')
+
+ local configs="/proc/config.gz /boot/config-$(uname -r) /boot/config "
+ dlog "Looking for Kernel configs; $configs "
+ for config in $configs; do
+ if [ -e $config ]; then
+ dlog "Kenel configuration found:$config"
+ break
+ fi
+ done
+
+ local all_except_0="1-$(($(getconf _NPROCESSORS_ONLN) - 1))"
+ if [ $config ]; then
+ nohz_full=$(zgrep "CONFIG_NO_HZ_FULL_ALL=y" $config 2>/dev/null) \
+ && nohz_full=$all_except_0
+ else
+ warn "Kernel config not found, only checking /proc/cmdline for"\
+ " isolation features."
+ fi
+
+ if ! [ "$nohz_full" ]; then
+ eval $(cat /proc/cmdline | grep -o 'nohz_full=[^ ]*')
+ fi
+
+ eval $(cat /proc/cmdline | grep -o 'isolcpus=[^ ]*')
+ if [ -z "$isolcpus" ]; then
+ warn "No CPU is isolated from kernel/user threads, isolcpus= is "\
+ "not set in kernel cmdline."
+ else
+ gbl_isolated_cpus=$isolcpus
+ export gbl_isolated_cpus
+ fi
+
+ if [ -z "$nohz_full" ]; then
+ warn "No CPU is isolated from kernel ticks, CONFIG_NO_HZ_FULL_ALL=y" \
+ " not set in kernel, nor nohz_full= set in kernel cmdline."
+ fi
+
+ for i in `pgrep rcu` ; do taskset -pc 0 $i >/dev/null; done
+
+ dlog "isolcpus:$isolcpus"
+ dlog "nohz_full:$nohz_full"
+ #dlog "rcu_nocbs:$rcu_nocbs"
+
+ return 0
+}
+
+cpus_valid() {
+ local cpus="$1"
+ local isolated=$2
+ local iarray=$(get_cpu_array $isolated)
+ local carray=$(get_cpu_array $cpus)
+
+ for c in $carray; do
+ for i in $iarray; do
+ if [ $i = $c ]; then
+ yah=$i
+ fi
+ done
+ [ -z "$yah" ] && return 1
+ done
+
+ return 0
+}
+
+check_prequesties() {
+ dlog "Checking prequesties; user is root, kernel has cpuset support,"\
+ " and commads; set, zgrep, getconf are available"
+ [ $UID -eq 0 ] || die "You need to be root!"
+ grep -q -s cpuset /proc/filesystems || die "Kernel does not support
cpuset!"
+ which getconf > /dev/null 2>&1 || die "getconf command not found, please "\
+ "install getconf"
+ which cset > /dev/null 2>&1 || die "cset command not found, please "\
+ "install cpuset"
+ which zgrep > /dev/null 2>&1 || die "zgrep command not found, please "\
+ "install gzip"
+}
+
+shield_reset() {
+ cset shield -r >/dev/null 2>&1
+ sleep 0.1
+}
+
+shield_list() {
+ sets="/cpusets/*/"
+ for i in $sets ; do
+ if ! [ -e $i ]; then
+ continue
+ fi
+ printf "Domain %s cpus %s, running %d tasks\n" \
+ $(basename $i) $(cat $i/cpuset.cpus) $(cat $i/tasks | wc -l)
+ done
+}
+
+shield_cpus() {
+ local cpus="$1"
+
+ dlog "shielding CPU:s $cpus"
+
+ #Reset and create new shield
+ shield_reset
+ out=$(cset shield -c $cpus -k on 2>&1) || die "cset failed; $out"
+ # Delay the annoying vmstat timer far away
+ sysctl vm.stat_interval=120 >/dev/null
+
+ # Shutdown nmi watchdog as it uses perf events
+ sysctl -w kernel.watchdog=0 >/dev/null
+
+ # Pin the writeback workqueue to CPU0
+ #Fixme, check that /sys/bus is mounted?
+ echo 1 > /sys/bus/workqueue/devices/writeback/cpumask
+
+ # Disable load balanser.
+ echo 0 > /cpusets/user/cpuset.sched_load_balance
+
+ #Fixme, for now just send all irqs to core 0
+ for affinity in /proc/irq/*/smp_affinity; do
+ dlog "redirecting $affinity"
+ echo 1 > $affinity 2>/dev/null || dlog "$affinity redirection failed."
+ done
+
+
+ #Fixme, not implemented.
+ if [ $false ]; then
+ for affinity in /proc/irq/*/smp_affinity; do
+ local old_mask=$(cat $affinity)
+ local new_mask=$((oldmask ^ cpus ))
+ echo $new_mask > $affinity
+ done
+ fi
+}
+
+isolate_cpus() {
+
+ local cpus="$1"
+
+ check_kernel_config
+
+ if [ "$gbl_isolated_cpus" ]; then
+ cpus_valid $cpus $gbl_isolated_cpus ||
+ warn "Selected CPU '$cpus' is not inside isolated cpus "\
+ "array:$gbl_isolated_cpus"
+ fi
+
+ dlog "Isolating CPUs $cpus"
+
+ shield_cpus $cpus
+
+ # Verfiy cores empty
+ for c in $(get_cpu_array $cpus); do
+ running=$(ps ax -o pid,psr,comm | \
+ awk -v cpu="$c" '{if($2==cpu){print $3}}')
+ if [ "$running" != "" ]; then
+ warn "Core $c not empty!"
+ dlog "; running tasks:\n$running\n"
+ fi
+ done
+
+ return 0
+}
+
+##
+# Script entry point
+##
+while getopts hdarlc: arguments
+do
+ case $arguments in
+ h)
+ print_usage
+ exit 0
+ ;;
+ d)
+ DEBUG=1
+ ;;
+ a)
+ ISOL_CPUS="1-$(($(getconf _NPROCESSORS_ONLN) - 1))"
+ ;;
+ r)
+ shield_reset
+ exit 0
+ ;;
+ l)
+ shield_list
+ exit 0
+ ;;
+ c)
+ [ "$ISOL_CPUS" ] || ISOL_CPUS=$OPTARG
+ ;;
+ *)
+ print_usage
+ exit 1
+ ;;
+ esac
+done
+#Remove all flags
+shift $((OPTIND-1))
+
+if ! [ $ISOL_CPUS ]; then
+ print_usage
+ exit 1
+fi
+
+check_prequesties
+isolate_cpus $ISOL_CPUS || die "isolate_cpus failed."
diff --git a/scripts/task-isolation/isolate-task.sh
b/scripts/task-isolation/isolate-task.sh
new file mode 100755
index 0000000..3e588bc
--- /dev/null
+++ b/scripts/task-isolation/isolate-task.sh
@@ -0,0 +1,160 @@
+#!/bin/bash
+#
+# Copyright (c) 2017, Linaro Limited
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Script that passes command line arguments to odp_scheduling after,
+# optionally, isolating CPU
+#
+# This script isolates a task on desired CPUs and
+# optionally creates background noise.
+
+print_usage() {
+ echo "$0 [-c <cpu list>] [-d] [-h] [-n] <application arg1, arg2, ...>"
+ echo
+ echo " Isolate CPU(s) from other tasks, kernel threads and IRQs"
+ echo " and run an application on isolated CPUs"
+ echo " Args:"
+ echo " -c List of CPUs to be isolated"
+ echo " -d Show debug printouts"
+ echo " -h Print this message"
+ echo " -n Create background noise (stress)"
+ echo ""
+ echo "All CPU's, except CPU 0, are isolated unless '-c' specified"
+ echo " Examples:"
+ echo " Isolate CPU 1,2 and run application in the same."
+ echo " $0 -n -c 1,2 /some/path/application"
+ echo
+ echo " Isolate all possible CPUs and run applicatipon"
+ echo " $0 /path/application"
+}
+
+dlog() {
+ [ $DEBUG ] && echo "$*"
+}
+
+die() {
+ printf "Error: $*\n" >&2
+ exit 1
+}
+
+trap cleanup INT EXIT
+
+cleanup(){
+ local pids=$(pgrep $MY_STRESS 2>/dev/null)
+ local base=$(dirname $0)
+
+ $base/isolate-cpu.sh -r
+ [ "$pids" != "" ] && kill -9 $pids >/dev/null 2>&1
+ kill -9 $CHILD >/dev/null 2>&1
+ rm -f $MY_STRESS_PATH
+}
+
+wait_app_started () {
+ local child=$1
+ local ltasks=0
+
+ while true; do
+ sleep 0.01
+ kill -0 $child 2>/dev/null || break
+ tasks=$(ls /proc/$child/task | wc -l)
+ [ $tasks -eq $ltasks ] && break
+ ltasks=$tasks
+ done
+ dlog "app started, # threads:$ltasks"
+}
+
+create_noise() {
+ local mpath=$1
+ local nr=$(grep processor /proc/cpuinfo | wc -l)
+
+ ln -sf $(which stress) $mpath || die "ln failed"
+ $mpath -c $nr 2>&1 >/dev/null &
+ disown $!
+}
+
+isolate_cpu(){
+ local cpus=$1
+ local base=$(dirname $0)
+ $base/isolate-cpu.sh -c $cpus || die "$0 failed"
+}
+
+run_application() {
+ local app="$1"
+
+ dlog "Starting application: $app"
+ $app&
+ child=$!
+ CHILD=$child
+
+ echo $child >> /cpusets/user/tasks
+ if [ $? -ne 0 ]; then
+ kill -9 $child
+ die "Failed to isolate task..."
+ fi
+
+ wait_app_started $child
+ wait $child
+}
+
+check_prequesties() {
+ local base=$(dirname $0)
+
+ [ -e $base/isolate-cpu.sh ] || die "$base/isolate-cpu.sh not found!"
+ [ $UID -eq 0 ] || die "You need to be root!"
+ which stress > /dev/null 2>&1 || die "stress command not found, "\
+ "please install stress"
+}
+
+##
+# Script entry point
+##
+
+ISOL_CPUS="1-$(($(getconf _NPROCESSORS_ONLN) - 1))"
+
+while getopts hdnc: arguments
+do
+ case $arguments in
+ h)
+ print_usage
+ exit 0
+ ;;
+ d)
+ DEBUG=1
+ ;;
+ n)
+ NOISE=1
+ ;;
+ c)
+ ISOL_CPUS=$OPTARG
+ ;;
+ *)
+ print_usage
+ exit 1
+ ;;
+ esac
+done
+# Remove all flags
+shift $((OPTIND-1))
+
+if ! [ "$1" ]; then
+ print_usage
+ exit 1
+fi
+
+#Isolate and optionally create noise
+command="$*"
+set -- $command
+
+check_prequesties
+
+MY_STRESS=stress-by-$$
+MY_STRESS_PATH=/tmp/$MY_STRESS
+
+isolate_cpu $ISOL_CPUS || die "isolate cpu failed!"
+[ -z $NOISE ] || create_noise $MY_STRESS_PATH
+run_application "$command"
+
+exit $?
diff --git a/test/linux-generic/performance/odp_run_app_isolated.sh
b/test/linux-generic/performance/odp_run_app_isolated.sh
new file mode 100755
index 0000000..3bed0a7
--- /dev/null
+++ b/test/linux-generic/performance/odp_run_app_isolated.sh
@@ -0,0 +1,108 @@
+#!/bin/sh
+#
+# Copyright (c) 2017, Linaro Limited
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+
+TEST_DIR="${TEST_DIR:-$(dirname $0)}"
+PERFORMANCE="$TEST_DIR/../../common_plat/performance"
+ISOL_DIR="${TEST_DIR}/../../../scripts/task-isolation"
+APPLICATION="${PERFORMANCE}/odp_pktio_perf${EXEEXT}"
+APPLICATION_ARGS=""
+APPLICATION_BASE="$(basename ${APPLICATION})"
+
+cleanup(){
+ pids=$(pgrep stress 2>/dev/null)
+ [ "$pids" != "" ] && kill -9 $pids
+}
+
+print_usage() {
+ echo "$0 [-i] [-n] [-h] [<application> <application args>]"
+ echo
+ echo " Run an application with or without isolation and background noise"
+ echo " Flags:"
+ echo " -h Print this message"
+ echo " -i Isolate CPU prior to running application."
+ echo " -n Create background noise (stress)"
+ echo ""
+ echo " <application> targeted application"
+ echo " <args> targeted application arguments"
+ echo " *Note* Default application is ${APPLICATION_BASE}"
+ echo ""
+ echo " Example:"
+ echo " Isolate CPU, create background noise and run ${APPLICATION_BASE}:"
+ echo " $0 -i -n"
+ echo
+ echo " Run ${APPLICATION_BASE}, w/o isolation but with background noise:"
+ echo " $0 -n"
+ echo
+ echo " Run Myapp, without isolation but with background noise:"
+ echo " $0 -n Myapp -s ome args"
+}
+
+run() {
+ local isolate=$1
+ local noise=$2
+ if [ ${isolate} -eq 1 ]; then
+ [ ${noise} -eq 1 ] && noise_par="-n"
+ echo Running ${APPLICATION_BASE} with isolation and background noise
+ echo =====================================================
+ $ISOL_DIR/isolate-task.sh ${noise_par} ${APPLICATION} \
+ ${APPLICATION_ARGS} || exit 1
+ #reset isolation
+ $ISOL_DIR/isolate-cpu.sh -r
+ else
+ echo Running ${APPLICATION_BASE} without isolation
+ echo =====================================================
+ if [ ${noise} -eq 1 ]; then
+ local nr=$(grep processor /proc/cpuinfo | wc -l)
+ echo " Creating background noise..."
+ stress -c $nr 2>&1 >/dev/null &
+ fi
+ ${APPLICATION} ${APPLICATION_ARGS} || exit 2
+ fi
+}
+
+trap cleanup INT EXIT
+ISOLATE=0
+NOISE=0
+while getopts hni arguments
+do
+ case $arguments in
+ h)
+ print_usage
+ exit 0
+ ;;
+ n)
+ NOISE=1
+ $(which stress > /dev/null 2>&1)
+ ret=$?
+ if [ ${ret} -ne 0 ]; then
+ echo "'stress' not found, bailing" >&2
+ exit 3
+ fi
+ ;;
+ i)
+ ISOLATE=1
+ ;;
+ *)
+ print_usage
+ exit 1
+ ;;
+ esac
+done
+#Remove flags
+shift $((OPTIND-1))
+
+if [ $# -gt 0 ]; then
+ APPLICATION="$1"
+ shift
+fi
+[ $# -gt 0 ] && APPLICATION_ARGS=$*
+
+run ${ISOLATE} ${NOISE}
+
+exit $?
--
2.7.4
