On 09/20/2015 03:50 PM, Pádraig Brady wrote:
> On 20/09/15 12:58, Bernhard Voelker wrote:
> https://github.com/pixelb/coreutils/commit/b429f5d8
Thanks!
That's the best we can do with the sort-month test case
for now.
>> Would an adaptive 'ulimit -v'-based approach work, i.e. first
>> getting the basic memory needed by e.g. "echo hello | sort -c -M",
>> and then use that limit (plus a little more) for a run with much
>> more input?
>
> Also an adaptive `ulimit -v` is awkward anywhere the tools are adaptive
> to memory limits themselves. There is the setlocale() gotcha for
> all tools, but sort(1) for example will adjust its heap allocations
> down to that available (see initbuf()).
Currently, we have `ulimit -v` tests for cp, csplit, cut, dd, head,
printf, split and rm. It seems an adaptive approach would work for
all of these programs - in regard to all of the
--enable-single-binary={no,symlinks,shebangs}
cases.
The attached follows this approach - does it work for you?
Have a nice day,
Berny
>From 1335ab2713aab020564275c49fdb3e92bb9a207b Mon Sep 17 00:00:00 2001
From: Bernhard Voelker <[email protected]>
Date: Mon, 21 Sep 2015 01:40:33 +0200
Subject: [PATCH] maint: use adaptive approach for `ulimit -v` based tests
When configured with either 'symlinks' or 'shebangs' as value for
the --enable-single-binary option, tests based on `ulimit -v` are
skipped. The reason is that the multicall 'coreutils' binary requires
much more memory due to shared libraries being loaded, and the size of
the 'date' binary (~290KiB) compared to the multicall binary (~5MiB),
of course. Finally, in the case of 'shebangs', the starting shell
requires more memory, too
Instead of using hard-coded values for the memory limit, use an
adaptive approach: first determine the amount of memory for a similar,
yet more trivial command, and then do the real test run using that
limit (plus some buffer in some cases).
* init.cfg (require_ulimit_v_): Remove function.
(get_min_ulimit_v_): Add function to determine the minimum memory limit
required for a given command in an adaptive way.
* cfg.mk (sc_prohibit_test_ulimit_without_require_): Change the name
of the above function in the syntax-check rule.
* tests/cp/link-heap.sh: Use the above function to determine the
minimum memory required to run a command simpler than in the real test
run. Use that limit plus a buffer there. While at it, change to list
of commands in the subshell to fail also if the beginning `ulimit -v`
fails.
* tests/dd/no-allocate.sh: Likewise.
* tests/misc/csplit-heap.sh: Likewise.
* tests/misc/cut-huge-range.sh: Likewise.
* tests/misc/head-c.sh: Likewise.
* tests/misc/printf-surprise.sh: Likewise.
* tests/split/line-bytes.sh: Likewise.
* tests/rm/many-dir-entries-vs-OOM.sh: Likewise - doing it separately
for each program under test.
---
cfg.mk | 6 +++---
init.cfg | 36 +++++++++++++++++++-----------------
tests/cp/link-heap.sh | 11 ++++++++---
tests/dd/no-allocate.sh | 16 +++++++++++-----
tests/misc/csplit-heap.sh | 9 ++++++---
tests/misc/cut-huge-range.sh | 12 +++++++-----
tests/misc/head-c.sh | 6 ++++--
tests/misc/printf-surprise.sh | 5 +++--
tests/rm/many-dir-entries-vs-OOM.sh | 30 +++++++++++++++++++-----------
tests/split/line-bytes.sh | 5 +++--
10 files changed, 83 insertions(+), 53 deletions(-)
diff --git a/cfg.mk b/cfg.mk
index ace3f5a..d38a8dd 100644
--- a/cfg.mk
+++ b/cfg.mk
@@ -503,12 +503,12 @@ sc_some_programs_must_avoid_exit_failure:
&& { echo '$(ME): do not use EXIT_FAILURE in the above' \
1>&2; exit 1; } || :
-# Ensure that tests call the require_ulimit_v_ function if using ulimit -v
+# Ensure that tests call the get_min_ulimit_v_ function if using ulimit -v
sc_prohibit_test_ulimit_without_require_:
- @(git grep -l require_ulimit_v_ $(srcdir)/tests; \
+ @(git grep -l get_min_ulimit_v_ $(srcdir)/tests; \
git grep -l 'ulimit -v' $(srcdir)/tests) \
| sort | uniq -u | grep . && { echo "$(ME): the above test(s)"\
- " should match require_ulimit_v_ with ulimit -v" 1>&2; exit 1; } || :
+ " should match get_min_ulimit_v_ with ulimit -v" 1>&2; exit 1; } || :
# Ensure that tests call the cleanup_ function if using background processes
sc_prohibit_test_background_without_cleanup_:
diff --git a/init.cfg b/init.cfg
index 3beba5a..f71f94c 100644
--- a/init.cfg
+++ b/init.cfg
@@ -144,24 +144,26 @@ require_openat_support_()
fi
}
-require_ulimit_v_()
-{
- local ulimit_works=yes
- # Expect to be able to exec a program in 10MiB of virtual memory,
- # (10MiB is usually plenty, but valgrind-wrapped date requires 19000KiB,
- # so allow more in that case)
- # but not in 20KiB. I chose "date". It must not be a shell built-in
- # function, so you can't use echo, printf, true, etc.
- # Of course, in coreutils, I could use $top_builddir/src/true,
- # but this should be able to work for other projects, too.
+# Determine the minimum required VM limit to run the given command.
+# Output that value to stdout ... to be used by the caller.
+# Return 0 in case of success, and a non-Zero value otherwise.
+get_min_ulimit_v_()
+{
local vm
- case $(printenv LD_PRELOAD) in */valgrind/*) vm=22000;; *) vm=10000;; esac
-
- ( ulimit -v $vm; date ) > /dev/null 2>&1 || ulimit_works=no
- ( ulimit -v 20; date ) > /dev/null 2>&1 && ulimit_works=no
-
- test $ulimit_works = no \
- && skip_ "this shell lacks ulimit support"
+ for v in $( seq 5000 5000 50000 ); do
+ if ( ulimit -v $v && "$@" ) >/dev/null; then
+ local vm_prev
+ prev_v=$v
+ for v in $( seq $(($prev_v-1000)) -1000 1000 ); do
+ ( ulimit -v $v && "$@" ) >/dev/null \
+ || { echo $prev_v; return 0; }
+ prev_v=$v
+ done
+ fi
+ done
+ # The above did not find a working limit. Echo a very small number - just
+ # in case the caller does not handle the non-Zero return value.
+ echo 1; return 1
}
require_readable_root_()
diff --git a/tests/cp/link-heap.sh b/tests/cp/link-heap.sh
index 901a92f..b65223a 100755
--- a/tests/cp/link-heap.sh
+++ b/tests/cp/link-heap.sh
@@ -19,7 +19,12 @@
. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
print_ver_ cp
expensive_
-require_ulimit_v_
+
+# Determine basic amount of memory needed for 'cp -al'.
+touch f || framework_failure_
+vm=$(get_min_ulimit_v_ cp -al f f2) \
+ || skip_ "this shell lacks ulimit support"
+rm f f2 || framework_failure_
a=$(printf %031d 0)
b=$(printf %031d 1)
@@ -31,7 +36,7 @@ cp -al $a $b || framework_failure_
mkdir e || framework_failure_
mv $a $b e || framework_failure_
-# Increased from 20000 to 22000 in 2012, for pre-F18 rawhide.
-(ulimit -v 22000; cp -al e f) || fail=1
+# Allow cp(1) to use 4MiB more virtual memory than for the above trivial case.
+(ulimit -v $(($vm+4000)) && cp -al e f) || fail=1
Exit $fail
diff --git a/tests/dd/no-allocate.sh b/tests/dd/no-allocate.sh
index 99e0542..d122e35 100755
--- a/tests/dd/no-allocate.sh
+++ b/tests/dd/no-allocate.sh
@@ -18,12 +18,17 @@
. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
print_ver_ dd
-require_ulimit_v_
+
+# Determine basic amount of memory needed.
+echo . > f || framework_failure_
+vm=$(get_min_ulimit_v_ dd if=f of=f2 status=none) \
+ || skip_ "this shell lacks ulimit support"
+rm -f f || framework_failure_
# count and skip are zero, we don't need to allocate memory
-(ulimit -v 20000; dd bs=30M count=0) || fail=1
-(ulimit -v 20000; dd ibs=30M count=0) || fail=1
-(ulimit -v 20000; dd obs=30M count=0) || fail=1
+(ulimit -v $vm && dd bs=30M count=0) || fail=1
+(ulimit -v $vm && dd ibs=30M count=0) || fail=1
+(ulimit -v $vm && dd obs=30M count=0) || fail=1
check_dd_seek_alloc() {
local file="$1"
@@ -38,7 +43,8 @@ check_dd_seek_alloc() {
timeout 10 dd count=1 if=/dev/zero of=tape&
# Allocate buffer and read from the "tape"
- (ulimit -v 20000; timeout 10 dd $dd_buf=30M $dd_op=1 count=0 $dd_file=tape)
+ (ulimit -v $(($vm+4000)) \
+ && timeout 10 dd $dd_buf=30M $dd_op=1 count=0 $dd_file=tape)
local ret=$?
# Be defensive in case the tape reader is blocked for some reason
diff --git a/tests/misc/csplit-heap.sh b/tests/misc/csplit-heap.sh
index 9e51cb1..0ffe70a 100755
--- a/tests/misc/csplit-heap.sh
+++ b/tests/misc/csplit-heap.sh
@@ -19,11 +19,14 @@
. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
print_ver_ csplit
-require_ulimit_v_
+# Determine basic amount of memory needed.
+{ echo y; echo n; } > f || framework_failure_
+vm=$(get_min_ulimit_v_ csplit -z f %n%1) \
+ || skip_ "this shell lacks ulimit support"
(
- ulimit -v 20000
- { yes | head -n2500000; echo n; } | csplit -z - %n%1
+ ulimit -v $vm \
+ && { yes | head -n2500000; echo n; } | csplit -z - %n%1
) || fail=1
Exit $fail
diff --git a/tests/misc/cut-huge-range.sh b/tests/misc/cut-huge-range.sh
index 035d8d0..4df2fc0 100755
--- a/tests/misc/cut-huge-range.sh
+++ b/tests/misc/cut-huge-range.sh
@@ -18,9 +18,11 @@
. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
print_ver_ cut
-require_ulimit_v_
getlimits_
+vm=$(get_min_ulimit_v_ cut -b1 /dev/null) \
+ || skip_ "this shell lacks ulimit support"
+
# sed script to subtract one from the input.
# Each input line should consist of a positive decimal number.
# Each output line's number is one less than the input's.
@@ -49,15 +51,15 @@ CUT_MAX=$(echo $SIZE_MAX | sed "$subtract_one")
# From coreutils-8.10 through 8.20, this would make cut try to allocate
# a 256MiB bit vector. With a 20MB limit on VM, the following would fail.
-(ulimit -v 20000; : | cut -b$CUT_MAX- > err 2>&1) || fail=1
+(ulimit -v $vm && : | cut -b$CUT_MAX- > err 2>&1) || fail=1
# Up to and including coreutils-8.21, cut would allocate possibly needed
# memory upfront. Subsequently extra memory is no longer needed.
-(ulimit -v 20000; : | cut -b1-$CUT_MAX >> err 2>&1) || fail=1
+(ulimit -v $vm && : | cut -b1-$CUT_MAX >> err 2>&1) || fail=1
# Explicitly disallow values above CUT_MAX
-(ulimit -v 20000; : | returns_ 1 cut -b$SIZE_MAX 2>/dev/null) || fail=1
-(ulimit -v 20000; : | returns_ 1 cut -b$SIZE_OFLOW 2>/dev/null) || fail=1
+(ulimit -v $vm && : | returns_ 1 cut -b$SIZE_MAX 2>/dev/null) || fail=1
+(ulimit -v $vm && : | returns_ 1 cut -b$SIZE_OFLOW 2>/dev/null) || fail=1
compare /dev/null err || fail=1
diff --git a/tests/misc/head-c.sh b/tests/misc/head-c.sh
index a553a55..ab821ac 100755
--- a/tests/misc/head-c.sh
+++ b/tests/misc/head-c.sh
@@ -18,9 +18,11 @@
. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
print_ver_ head
-require_ulimit_v_
getlimits_
+vm=$(get_min_ulimit_v_ head -c1 /dev/null) \
+ || skip_ "this shell lacks ulimit support"
+
# exercise the fix of 2001-08-18, based on test case from Ian Bruce
echo abc > in || framework_failure_
(head -c1; head -c1) < in > out || fail=1
@@ -40,7 +42,7 @@ esac
# Only allocate memory as needed.
# Coreutils <= 8.21 would allocate memory up front
# based on the value passed to -c
-(ulimit -v 20000; head --bytes=-$SSIZE_MAX < /dev/null) || fail=1
+(ulimit -v $vm && head --bytes=-$SSIZE_MAX < /dev/null) || fail=1
# Make sure it works on funny files in /proc and /sys.
diff --git a/tests/misc/printf-surprise.sh b/tests/misc/printf-surprise.sh
index 20e6b09..8480693 100755
--- a/tests/misc/printf-surprise.sh
+++ b/tests/misc/printf-surprise.sh
@@ -20,8 +20,9 @@ prog=printf
. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
print_ver_ printf
-require_ulimit_v_
+vm=$(get_min_ulimit_v_ env $prog %20f 0) \
+ || skip_ "this shell lacks ulimit support"
# Up to coreutils-6.9, "printf %.Nf 0" would encounter an ENOMEM internal
# error from glibc's printf(3) function whenever N was large relative to
@@ -60,7 +61,7 @@ head -c 10 fifo > out & pid=$!
# Choosing the virtual memory limit, 11000 is enough, but 10000 is too
# little and provokes a "memory exhausted" diagnostic on FreeBSD 9.0-p3.
-( ulimit -v 15000; env $prog %20000000f 0 2>err-msg > fifo )
+( ulimit -v $vm && env $prog %20000000f 0 2>err-msg > fifo )
exit=$?
# Map this longer, and rarer, diagnostic to the common one.
diff --git a/tests/rm/many-dir-entries-vs-OOM.sh b/tests/rm/many-dir-entries-vs-OOM.sh
index e9c95d5..8cc839b 100755
--- a/tests/rm/many-dir-entries-vs-OOM.sh
+++ b/tests/rm/many-dir-entries-vs-OOM.sh
@@ -19,21 +19,29 @@
. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
print_ver_ rm du chmod
-require_ulimit_v_
-
expensive_
+mkdir d2 \
+ && touch d2/f || framework_failure_
+
+# Restrict memory. Each of these coreutils-8.12 programs would fail
+# with a diagnostic like "rm: fts_read failed: Cannot allocate memory".
+vm=$(get_min_ulimit_v_ du -sh d2) \
+ || skip_ "this shell lacks ulimit support"
+
# With many files in a single directory...
-mkdir d && cd d || framework_failure_
-seq 200000|xargs touch || framework_failure_
+mkdir d || framework_failure_
+seq --format="d/%06g" 200000 | xargs touch || framework_failure_
-cd ..
+# Allow 35MiB more memory as for the trivial case above.
+(ulimit -v $(($vm + 35000)) && du -sh d) || fail=1
-# Restricted to 40MB, each of these coreutils-8.12 programs would fail
-# with a diagnostic like "rm: fts_read failed: Cannot allocate memory".
-ulimit -v 40000
-du -sh d || fail=1
-chmod -R 700 d || fail=1
-rm -rf d || fail=1
+vm=$(get_min_ulimit_v_ chmod -R 700 d2) \
+ || skip_ "this shell lacks ulimit support"
+(ulimit -v $(($vm + 35000)) && chmod -R 700 d) || fail=1
+
+vm=$(get_min_ulimit_v_ rm -rf d2) \
+ || skip_ "this shell lacks ulimit support"
+(ulimit -v $(($vm + 35000)) && rm -rf d) || fail=1
Exit $fail
diff --git a/tests/split/line-bytes.sh b/tests/split/line-bytes.sh
index 6e2f137..f2f0f8f 100755
--- a/tests/split/line-bytes.sh
+++ b/tests/split/line-bytes.sh
@@ -18,11 +18,12 @@
. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
print_ver_ split
-require_ulimit_v_
+vm=$(get_min_ulimit_v_ split -C 'K' /dev/null) \
+ || skip_ "this shell lacks ulimit support"
# Ensure memory is not allocated up front
-(ulimit -v 20000; split -C 'G' /dev/null) || fail=1
+(ulimit -v $vm && split -C 'G' /dev/null) || fail=1
# Ensure correct operation with various split and buffer size combinations
--
2.1.4
