On Thu, Feb 16, 2017 at 09:52:37AM +0100, Ravineet Singh wrote:
> The wrapper script; odp_run_app_isolated.sh can be used to run ODP
> testcases in a isolated environment. Background noise can also be
> generated.
>
> Signed-off-by: Ravineet Singh <ravineet.si...@linaro.org>
>
> v2: moved task-isolation dir to odp/scripts, requested my Maxim Uvarov
> v3: fixed checkpatch.pl warnings
> ---
> scripts/task-isolation/README | 22 ++
> scripts/task-isolation/isolate-cpu.sh | 287
> +++++++++++++++++++++
> scripts/task-isolation/isolate-task.sh | 160 ++++++++++++
> .../performance/odp_run_app_isolated.sh | 108 ++++++++
> 4 files changed, 577 insertions(+)
> create mode 100644 scripts/task-isolation/README
> create mode 100755 scripts/task-isolation/isolate-cpu.sh
> create mode 100755 scripts/task-isolation/isolate-task.sh
> create mode 100755 test/linux-generic/performance/odp_run_app_isolated.sh
>
> diff --git a/scripts/task-isolation/README b/scripts/task-isolation/README
> new file mode 100644
> index 0000000..cb02056
> --- /dev/null
> +++ b/scripts/task-isolation/README
> @@ -0,0 +1,22 @@
> +Helper scripts to check and set CPU isolation and execution of application in
> +isolated CPU(s)
> +
> +Files:
> +isolate-cpu.sh isolates desired CPUs
> +isolate-task.sh uses isolate-cpu.sh to isolate CPUs before running
> + the desired task. It also provides the possibility
> + to trace kernel disturbance on the isolated CPUs.
> +
> +isolate-cpu.sh checks the kernel configuration and the kernel cmdline to
> + determine if one or several CPUs are isolated, i.e. it checks for;
> + CONFIG_NO_HZ_FULL_ALL=y", and CONFIG_RCU_NOCB_CPU_ALL=y" in the kernel
> config
> + and rcu_nocbs,nohz_full in the kernel cmdline.
> + If the desired CPU(s) are not inte the above configuration, it warns but
> continues
> + to isolate the CPU(s) as much as possibe. The isolation is accomplished by
> + - Redirecting all IRQ's away from desired isolated cores
> + - Using cset (cpuset) to move all running processes and kernel threads
> + away from desired isolated cores
> +
> +isolate-task.sh uses isolate-cpu.sh to isolate desired CPU(s). In addition
> + it starts the supplied application on isolated CPU(s) and optionally traces
> + the isolated CPU(s) for kernel interaction.
> diff --git a/scripts/task-isolation/isolate-cpu.sh
> b/scripts/task-isolation/isolate-cpu.sh
> new file mode 100755
> index 0000000..3eaf98a
> --- /dev/null
> +++ b/scripts/task-isolation/isolate-cpu.sh
> @@ -0,0 +1,287 @@
> +#!/bin/bash
> +#
> +# Copyright (c) 2017, Linaro Limited
> +# All rights reserved.
> +#
> +# SPDX-License-Identifier: BSD-3-Clause
> +#
> +# Script that passes command line arguments to odp_scheduling after,
> +# optionally, isolating CPU
> +#
> +# This script isolates desired CPUS, i.e.
> +# - Checks kernel cmdline and kernel config to determine
> +# if the environment is optimised for isolated task execution;
> +# - Moves CPU interrupts, kernel threads, tasks etc. away from the
> +# targeted CPU.
> +# *Note* CPU 0 cannot be isolated, i.e minimum 2 CPU's are required.
> +
> +
> +print_usage() {
> + echo "$0 [-h] [-a] [-c <cpu list>] [-l] [-r] [-d]"
> + echo
> + echo " Isolate CPU(s) from other tasks, kernel threads and IRQs"
> + echo " Args:"
> + echo " -h Print this message"
> + echo " -a Isolate all CPUs (except CPU 0)"
> + echo " -c List of CPUs to be isolated."
> + echo " -l Show isolation proporties"
> + echo " -r Reset isolation"
> + echo " -d Show debug printouts"
> + echo ""
> + echo " Examples:"
> + echo " Isolate all CPU(s) (except 0) "
> + echo " $0 -a"
> + echo
> + echo " Isolate CPUs 1-3 "
> + echo " $0 -c 1-3"
> + echo
> + echo " Isolate CPUs 1 and 4 "
> + echo " $0 -c 1,4 "
maybe use a here document?
> +}
> +
> +dlog() {
> + [ $DEBUG ] && echo "$*"
> +}
> +
> +warn() {
> + printf "Warning: $*\n" >&2
> +}
> +
> +die() {
> + printf "Error: $*\n" >&2
> + exit 1
> +}
> +
> +get_cpu_array() {
> + [ $1 ] || die "$FUNCNAME internal error!"
> +
> + local cpus=""
> + IFS=. a=$1; IFS=, a=$a;
I don't understand why IFS=. is needed...
Also, shouldn't IFS be restored here?
> + for str in $a; do
> + if [[ $str == *[\-]* ]]; then
> + str=$(echo $str| sed 's/-/../g')
> + str=$(eval echo {$str})
> + fi
nice!
> +
> + if [ "$cpus" != "" ]; then
> + cpus="$cpus $str"
> + else
> + cpus=$str
> + fi
you can probably get away with cpus="$cpus $str"
> + done
> +
> + echo $cpus
> +}
> +
> +##
> +# Check kernel config and kernel cmdline for rcu callbacs and no_hz
> +# *Note* isolcpu= kernel cmdline option isolates CPUs from SMP balancing
> +# If needed, this can be done via
> +# cpusets/user/cpuset.sched_load_balance
> +##
> +check_kernel_config() {
> +
> + eval $(cat /proc/cmdline | grep -o 'nohz_full=[^ ]*')
> +
> + local configs="/proc/config.gz /boot/config-$(uname -r) /boot/config "
> + dlog "Looking for Kernel configs; $configs "
> + for config in $configs; do
> + if [ -e $config ]; then
> + dlog "Kenel configuration found:$config"
> + break
> + fi
> + done
> +
> + local all_except_0="1-$(($(getconf _NPROCESSORS_ONLN) - 1))"
> + if [ $config ]; then
I think this will always be evaluated to true, shouldn't it be
if [ -e $config ] ? or even -r $config?
> + nohz_full=$(zgrep "CONFIG_NO_HZ_FULL_ALL=y" $config 2>/dev/null) \
> + && nohz_full=$all_except_0
> + else
> + warn "Kernel config not found, only checking /proc/cmdline for"\
> + " isolation features."
> + fi
> +
> + if ! [ "$nohz_full" ]; then
> + eval $(cat /proc/cmdline | grep -o 'nohz_full=[^ ]*')
no need to pipe: grep ... /proc/cmdline
> + fi
> +
> + eval $(cat /proc/cmdline | grep -o 'isolcpus=[^ ]*')
ditto
> + if [ -z "$isolcpus" ]; then
> + warn "No CPU is isolated from kernel/user threads, isolcpus= is "\
> + "not set in kernel cmdline."
> + else
> + gbl_isolated_cpus=$isolcpus
> + export gbl_isolated_cpus
> + fi
> +
> + if [ -z "$nohz_full" ]; then
> + warn "No CPU is isolated from kernel ticks, CONFIG_NO_HZ_FULL_ALL=y"
> \
> + " not set in kernel, nor nohz_full= set in kernel cmdline."
> + fi
> +
> + for i in `pgrep rcu` ; do taskset -pc 0 $i >/dev/null; done
can you do this?
> +
> + dlog "isolcpus:$isolcpus"
> + dlog "nohz_full:$nohz_full"
> + #dlog "rcu_nocbs:$rcu_nocbs"
> +
> + return 0
> +}
> +
> +cpus_valid() {
> + local cpus="$1"
> + local isolated=$2
> + local iarray=$(get_cpu_array $isolated)
> + local carray=$(get_cpu_array $cpus)
> +
> + for c in $carray; do
> + for i in $iarray; do
maybe check that the user didn't provide 0 as a CPU to isolate?
> + if [ $i = $c ]; then
> + yah=$i
> + fi
> + done
> + [ -z "$yah" ] && return 1
> + done
> +
> + return 0
> +}
> +
> +check_prequesties() {
> + dlog "Checking prequesties; user is root, kernel has cpuset support,"\
> + " and commads; set, zgrep, getconf are available"
> + [ $UID -eq 0 ] || die "You need to be root!"
> + grep -q -s cpuset /proc/filesystems || die "Kernel does not support
> cpuset!"
> + which getconf > /dev/null 2>&1 || die "getconf command not found, please
> "\
> + "install getconf"
> + which cset > /dev/null 2>&1 || die "cset command not found, please "\
> + "install cpuset"
> + which zgrep > /dev/null 2>&1 || die "zgrep command not found, please "\
> + "install gzip"
> +}
> +
> +shield_reset() {
> + cset shield -r >/dev/null 2>&1
> + sleep 0.1
> +}
> +
> +shield_list() {
> + sets="/cpusets/*/"
> + for i in $sets ; do
> + if ! [ -e $i ]; then
> + continue
> + fi
> + printf "Domain %s cpus %s, running %d tasks\n" \
> + $(basename $i) $(cat $i/cpuset.cpus) $(cat $i/tasks | wc -l)
> + done
> +}
> +
> +shield_cpus() {
> + local cpus="$1"
> +
> + dlog "shielding CPU:s $cpus"
> +
> + #Reset and create new shield
> + shield_reset
> + out=$(cset shield -c $cpus -k on 2>&1) || die "cset failed; $out"
> + # Delay the annoying vmstat timer far away
> + sysctl vm.stat_interval=120 >/dev/null
> +
> + # Shutdown nmi watchdog as it uses perf events
> + sysctl -w kernel.watchdog=0 >/dev/null
> +
> + # Pin the writeback workqueue to CPU0
> + #Fixme, check that /sys/bus is mounted?
> + echo 1 > /sys/bus/workqueue/devices/writeback/cpumask
> +
> + # Disable load balanser.
balancer
> + echo 0 > /cpusets/user/cpuset.sched_load_balance
> +
> + #Fixme, for now just send all irqs to core 0
> + for affinity in /proc/irq/*/smp_affinity; do
> + dlog "redirecting $affinity"
> + echo 1 > $affinity 2>/dev/null || dlog "$affinity redirection
> failed."
> + done
> +
> +
> + #Fixme, not implemented.
> + if [ $false ]; then
> + for affinity in /proc/irq/*/smp_affinity; do
> + local old_mask=$(cat $affinity)
> + local new_mask=$((oldmask ^ cpus ))
> + echo $new_mask > $affinity
> + done
> + fi
> +}
> +
> +isolate_cpus() {
> +
> + local cpus="$1"
> +
> + check_kernel_config
> +
> + if [ "$gbl_isolated_cpus" ]; then
> + cpus_valid $cpus $gbl_isolated_cpus ||
> + warn "Selected CPU '$cpus' is not inside isolated cpus "\
> + "array:$gbl_isolated_cpus"
> + fi
> +
> + dlog "Isolating CPUs $cpus"
> +
> + shield_cpus $cpus
> +
> + # Verfiy cores empty
> + for c in $(get_cpu_array $cpus); do
> + running=$(ps ax -o pid,psr,comm | \
> + awk -v cpu="$c" '{if($2==cpu){print $3}}')
> + if [ "$running" != "" ]; then
> + warn "Core $c not empty!"
> + dlog "; running tasks:\n$running\n"
> + fi
> + done
indentation (tabs used here but spaces above)
> +
> + return 0
> +}
> +
> +##
> +# Script entry point
> +##
> +while getopts hdarlc: arguments
> +do
> + case $arguments in
> + h)
> + print_usage
> + exit 0
> + ;;
> + d)
> + DEBUG=1
> + ;;
> + a)
> + ISOL_CPUS="1-$(($(getconf _NPROCESSORS_ONLN) - 1))"
> + ;;
> + r)
> + shield_reset
> + exit 0
> + ;;
> + l)
> + shield_list
> + exit 0
> + ;;
> + c)
> + [ "$ISOL_CPUS" ] || ISOL_CPUS=$OPTARG
> + ;;
> + *)
> + print_usage
> + exit 1
> + ;;
> + esac
> +done
> +#Remove all flags
> +shift $((OPTIND-1))
> +
> +if ! [ $ISOL_CPUS ]; then
> + print_usage
> + exit 1
> +fi
> +
> +check_prequesties
> +isolate_cpus $ISOL_CPUS || die "isolate_cpus failed."
isolate_cpus() always retuns success...
> diff --git a/scripts/task-isolation/isolate-task.sh
> b/scripts/task-isolation/isolate-task.sh
> new file mode 100755
> index 0000000..3e588bc
> --- /dev/null
> +++ b/scripts/task-isolation/isolate-task.sh
> @@ -0,0 +1,160 @@
> +#!/bin/bash
> +#
> +# Copyright (c) 2017, Linaro Limited
> +# All rights reserved.
> +#
> +# SPDX-License-Identifier: BSD-3-Clause
> +#
> +# Script that passes command line arguments to odp_scheduling after,
> +# optionally, isolating CPU
> +#
> +# This script isolates a task on desired CPUs and
> +# optionally creates background noise.
> +
> +print_usage() {
> + echo "$0 [-c <cpu list>] [-d] [-h] [-n] <application arg1, arg2, ...>"
> + echo
> + echo " Isolate CPU(s) from other tasks, kernel threads and IRQs"
> + echo " and run an application on isolated CPUs"
> + echo " Args:"
> + echo " -c List of CPUs to be isolated"
> + echo " -d Show debug printouts"
> + echo " -h Print this message"
> + echo " -n Create background noise (stress)"
> + echo ""
> + echo "All CPU's, except CPU 0, are isolated unless '-c' specified"
> + echo " Examples:"
> + echo " Isolate CPU 1,2 and run application in the same."
> + echo " $0 -n -c 1,2 /some/path/application"
> + echo
> + echo " Isolate all possible CPUs and run applicatipon"
> + echo " $0 /path/application"
> +}
> +
> +dlog() {
> + [ $DEBUG ] && echo "$*"
> +}
> +
> +die() {
> + printf "Error: $*\n" >&2
> + exit 1
> +}
> +
> +trap cleanup INT EXIT
> +
> +cleanup(){
> + local pids=$(pgrep $MY_STRESS 2>/dev/null)
> + local base=$(dirname $0)
> +
> + $base/isolate-cpu.sh -r
> + [ "$pids" != "" ] && kill -9 $pids >/dev/null 2>&1
> + kill -9 $CHILD >/dev/null 2>&1
> + rm -f $MY_STRESS_PATH
> +}
> +
> +wait_app_started () {
> + local child=$1
> + local ltasks=0
> +
> + while true; do
> + sleep 0.01
> + kill -0 $child 2>/dev/null || break
> + tasks=$(ls /proc/$child/task | wc -l)
> + [ $tasks -eq $ltasks ] && break
> + ltasks=$tasks
> + done
> + dlog "app started, # threads:$ltasks"
> +}
> +
> +create_noise() {
> + local mpath=$1
> + local nr=$(grep processor /proc/cpuinfo | wc -l)
> +
> + ln -sf $(which stress) $mpath || die "ln failed"
> + $mpath -c $nr 2>&1 >/dev/null &
> + disown $!
> +}
> +
> +isolate_cpu(){
> + local cpus=$1
> + local base=$(dirname $0)
> + $base/isolate-cpu.sh -c $cpus || die "$0 failed"
> +}
> +
> +run_application() {
> + local app="$1"
> +
> + dlog "Starting application: $app"
> + $app&
> + child=$!
> + CHILD=$child
> +
> + echo $child >> /cpusets/user/tasks
> + if [ $? -ne 0 ]; then
> + kill -9 $child
> + die "Failed to isolate task..."
> + fi
> +
> + wait_app_started $child
> + wait $child
> +}
> +
> +check_prequesties() {
> + local base=$(dirname $0)
> +
> + [ -e $base/isolate-cpu.sh ] || die "$base/isolate-cpu.sh not found!"
> + [ $UID -eq 0 ] || die "You need to be root!"
> + which stress > /dev/null 2>&1 || die "stress command not found, "\
> + "please install stress"
> +}
> +
> +##
> +# Script entry point
> +##
> +
> +ISOL_CPUS="1-$(($(getconf _NPROCESSORS_ONLN) - 1))"
> +
> +while getopts hdnc: arguments
> +do
> + case $arguments in
> + h)
> + print_usage
> + exit 0
> + ;;
> + d)
> + DEBUG=1
> + ;;
> + n)
> + NOISE=1
> + ;;
> + c)
> + ISOL_CPUS=$OPTARG
> + ;;
> + *)
> + print_usage
> + exit 1
> + ;;
> + esac
> +done
> +# Remove all flags
> +shift $((OPTIND-1))
> +
> +if ! [ "$1" ]; then
> + print_usage
> + exit 1
> +fi
> +
> +#Isolate and optionally create noise
> +command="$*"
> +set -- $command
> +
> +check_prequesties
> +
> +MY_STRESS=stress-by-$$
> +MY_STRESS_PATH=/tmp/$MY_STRESS
> +
> +isolate_cpu $ISOL_CPUS || die "isolate cpu failed!"
> +[ -z $NOISE ] || create_noise $MY_STRESS_PATH
> +run_application "$command"
> +
> +exit $?
> diff --git a/test/linux-generic/performance/odp_run_app_isolated.sh
> b/test/linux-generic/performance/odp_run_app_isolated.sh
> new file mode 100755
> index 0000000..3bed0a7
> --- /dev/null
> +++ b/test/linux-generic/performance/odp_run_app_isolated.sh
there is a mix of spaces and tabs for indentation in this script
> @@ -0,0 +1,108 @@
> +#!/bin/sh
> +#
> +# Copyright (c) 2017, Linaro Limited
> +# All rights reserved.
> +#
> +# SPDX-License-Identifier: BSD-3-Clause
> +#
> +
> +
> +TEST_DIR="${TEST_DIR:-$(dirname $0)}"
> +PERFORMANCE="$TEST_DIR/../../common_plat/performance"
> +ISOL_DIR="${TEST_DIR}/../../../scripts/task-isolation"
> +APPLICATION="${PERFORMANCE}/odp_pktio_perf${EXEEXT}"
> +APPLICATION_ARGS=""
> +APPLICATION_BASE="$(basename ${APPLICATION})"
> +
> +cleanup(){
> + pids=$(pgrep stress 2>/dev/null)
> + [ "$pids" != "" ] && kill -9 $pids
> +}
> +
> +print_usage() {
> + echo "$0 [-i] [-n] [-h] [<application> <application args>]"
> + echo
> + echo " Run an application with or without isolation and background noise"
> + echo " Flags:"
> + echo " -h Print this message"
> + echo " -i Isolate CPU prior to running application."
> + echo " -n Create background noise (stress)"
> + echo ""
> + echo " <application> targeted application"
> + echo " <args> targeted application arguments"
> + echo " *Note* Default application is ${APPLICATION_BASE}"
> + echo ""
> + echo " Example:"
> + echo " Isolate CPU, create background noise and run
> ${APPLICATION_BASE}:"
> + echo " $0 -i -n"
> + echo
> + echo " Run ${APPLICATION_BASE}, w/o isolation but with background
> noise:"
> + echo " $0 -n"
> + echo
> + echo " Run Myapp, without isolation but with background noise:"
> + echo " $0 -n Myapp -s ome args"
> +}
> +
> +run() {
> + local isolate=$1
> + local noise=$2
> + if [ ${isolate} -eq 1 ]; then
> + [ ${noise} -eq 1 ] && noise_par="-n"
> + echo Running ${APPLICATION_BASE} with isolation and background noise
> + echo =====================================================
> + $ISOL_DIR/isolate-task.sh ${noise_par} ${APPLICATION} \
> + ${APPLICATION_ARGS} || exit 1
> + #reset isolation
> + $ISOL_DIR/isolate-cpu.sh -r
> + else
> + echo Running ${APPLICATION_BASE} without isolation
> + echo =====================================================
> + if [ ${noise} -eq 1 ]; then
> + local nr=$(grep processor /proc/cpuinfo | wc -l)
> + echo " Creating background noise..."
> + stress -c $nr 2>&1 >/dev/null &
> + fi
> + ${APPLICATION} ${APPLICATION_ARGS} || exit 2
> + fi
> +}
> +
> +trap cleanup INT EXIT
> +ISOLATE=0
> +NOISE=0
> +while getopts hni arguments
> +do
> + case $arguments in
> + h)
> + print_usage
> + exit 0
> + ;;
> + n)
> + NOISE=1
> + $(which stress > /dev/null 2>&1)
> + ret=$?
> + if [ ${ret} -ne 0 ]; then
or:
if ! $(which stress > /dev/null 2>&1); then
> + echo "'stress' not found, bailing" >&2
> + exit 3
> + fi
> + ;;
> + i)
> + ISOLATE=1
> + ;;
> + *)
> + print_usage
> + exit 1
> + ;;
> + esac
> +done
> +#Remove flags
> +shift $((OPTIND-1))
> +
> +if [ $# -gt 0 ]; then
> + APPLICATION="$1"
> + shift
> +fi
> +[ $# -gt 0 ] && APPLICATION_ARGS=$*
> +
> +run ${ISOLATE} ${NOISE}
> +
> +exit $?
not needed, the script will exit with the exit value of the last
command executed.
> --
> 2.7.4
>
