tests: make tests requiring a delay to pass, more robust

* tests/init.cfg: Introduce a retry_delay_() function to
repeatedly call a test function that requires a delay.
This delay can now be shorter for the common case on fast
systems, but will double until a configurable limit it reached
before failing on slower systems.
* tests/dd/reblock: Use retry_delay_.
* tests/misc/cat-buf: Likewise.
* tests/misc/stdbuf: Likewise.
* tests/tail-2/F-vs-rename: Likewise.
* tests/tail-2/flush-initial: Likewise.
* tests/tail-2/tail-n0f: Likewise.
* tests/tail-2/wait: Likewise.
* test/dd/misc: Comment that delay is needed to trigger failure.

tests/dd/misc

@@ -87,6 +87,8 @@ fi
 outbytes=`echo x | dd bs=3 ibs=10 obs=10 conv=sync 2>/dev/null | wc -c`
 test "$outbytes" -eq 3 || fail=1
 
+# A delay is required to trigger a failure.
+# There might be some missed failures but it's unlikely.
 (echo a; sleep .1; echo b) \
   | env LC_ALL=C dd bs=4 status=noxfer iflag=fullblock >out 2>err || fail=1
 printf 'a\nb\n' > out_ok || framework_failure

tests/dd/reblock

@@ -45,18 +45,32 @@ EOF
 # from each printf separately.
 mkfifo dd.fifo || framework_failure
 
-# ensure that dd reblocks when bs= is not specified
-dd ibs=3 obs=3 if=dd.fifo > out 2> err&
-(printf 'ab'; sleep .2; printf 'cd') > dd.fifo
-wait #for dd to complete
-sed 's/,.*//' err > k && mv k err
-compare err exp-reblock || fail=1
+dd_reblock_1()
+{
+  local delay="$1"
 
-# Demonstrate that bs=N supersedes even following ibs= and obs= settings.
-dd bs=3 ibs=1 obs=1 if=dd.fifo > out 2> err&
-(printf 'ab'; sleep .2; printf 'cd') > dd.fifo
-wait #for dd to complete
-sed 's/,.*//' err > k && mv k err
-compare err exp-no-reblock || fail=1
+  # ensure that dd reblocks when bs= is not specified
+  dd ibs=3 obs=3 if=dd.fifo > out 2> err&
+  (printf 'ab'; sleep $delay; printf 'cd') > dd.fifo
+  wait #for dd to complete
+  sed 's/,.*//' err > k && mv k err
+  compare err exp-reblock
+}
+
+retry_delay_ dd_reblock_1 .1 6 || fail=1
+
+dd_reblock_2()
+{
+  local delay="$1"
+
+  # Demonstrate that bs=N supersedes even following ibs= and obs= settings.
+  dd bs=3 ibs=1 obs=1 if=dd.fifo > out 2> err&
+  (printf 'ab'; sleep $delay; printf 'cd') > dd.fifo
+  wait #for dd to complete
+  sed 's/,.*//' err > k && mv k err
+  compare err exp-no-reblock
+}
+
+retry_delay_ dd_reblock_2 .1 6 || fail=1
 
 Exit $fail

tests/init.cfg

@@ -357,4 +357,31 @@ working_umask_or_skip_()
   esac
 }
 
+# Continually retry a function requiring a sufficient delay to _pass_
+#     Example: retry_delay dd_reblock_1 .1 6
+# This example will call the dd_reblock_1 function with
+# an initial delay of .1 second and call it at most 6 times
+# with a max delay of 3.2s (doubled each time), or a total of 6.3s
+# Note ensure you do _not_ quote the parameter to GNU sleep in
+# your function, as it may contain separate values that `sleep`
+# needs to accumulate.
+retry_delay_()
+{
+  local test_func=$1
+  local init_delay=$2
+  local max_n_tries=$3
+
+  local attempt=1
+  local num_sleeps=$attempt
+  local time_fail
+  while test $attempt -le $max_n_tries; do
+    local delay=$($AWK -v n=$num_sleeps -v s="$init_delay" \
+                  'BEGIN { for (i=0;i<n;i++) t = s" "t; print t }')
+    "$test_func" "$delay" && { time_fail=0; break; } || time_fail=1
+    attempt=$(expr $attempt + 1)
+    num_sleeps=$(expr $num_sleeps '*' 2)
+  done
+  test "$time_fail" = 0
+}
+
 sanitize_path_

tests/misc/cat-buf

@@ -33,13 +33,16 @@ mkfifo fifo || framework_failure
 
 echo 1 > exp
 
-dd count=1 if=fifo > out &
-(echo 1; sleep .5; echo 2) | cat -v > fifo
-wait # for dd to complete
+cat_buf_1()
+{
+  local delay="$1"
 
-# Though unlikely, this test may fail because dd was starved
-# between opening the fifo and reading from it until after the
-# second echo.  So ask to double check rather than failing.
-compare out exp || skip_test_ "possible test failure. Please verify."
+  dd count=1 if=fifo > out &
+  (echo 1; sleep $delay; echo 2) | cat -v > fifo
+  wait # for dd to complete
+  compare out exp
+}
+
+retry_delay_ cat_buf_1 .1 6 || fail=1
 
 Exit $fail

tests/misc/stdbuf

@@ -59,18 +59,32 @@ stdbuf -o1 no_such # no such command
 test $? = 127 || fail=1
 
 # Ensure line buffering stdout takes effect
-printf '1\n' > exp
-dd count=1 if=fifo > out 2> err &
-(printf '1\n'; sleep .2; printf '2\n') | stdbuf -oL uniq > fifo
-wait # for dd to complete
-compare out exp || fail=1
+stdbuf_linebuffer()
+{
+  local delay="$1"
 
-# Ensure un buffering stdout takes effect
-printf '1\n' > exp
-dd count=1 if=fifo > out 2> err &
-(printf '1\n'; sleep .2; printf '2\n') | stdbuf -o0 uniq > fifo
-wait # for dd to complete
-compare out exp || fail=1
+  printf '1\n' > exp
+  dd count=1 if=fifo > out 2> err &
+  (printf '1\n'; sleep $delay; printf '2\n') | stdbuf -oL uniq > fifo
+  wait # for dd to complete
+  compare out exp
+}
+
+retry_delay_ stdbuf_linebuffer .1 6 || fail=1
+
+stdbuf_unbuffer()
+{
+  local delay="$1"
+
+  # Ensure un buffering stdout takes effect
+  printf '1\n' > exp
+  dd count=1 if=fifo > out 2> err &
+  (printf '1\n'; sleep $delay; printf '2\n') | stdbuf -o0 uniq > fifo
+  wait # for dd to complete
+  compare out exp
+}
+
+retry_delay_ stdbuf_unbuffer .1 6 || fail=1
 
 # Ensure un buffering stdin takes effect
 #  The following works for me, but is racy. I.E. we're depending
@@ -93,10 +107,17 @@ compare out exp || fail=1
 # Ensure block buffering stdout takes effect
 # We don't currently test block buffering failures as
 # this doesn't work on on GLIBC-2.7 or GLIBC-2.9 at least.
-   # printf '1\n2\n' > exp
-   # dd count=1 if=fifo > out 2> err &
-   # (printf '1\n'; sleep .2; printf '2\n') | stdbuf -o4 uniq > fifo
-   # wait # for dd to complete
-   # compare out exp || fail=1
+   # stdbuf_blockbuffer()
+   # {
+   #   local delay="$1"
+   #
+   #   printf '1\n2\n' > exp
+   #   dd count=1 if=fifo > out 2> err &
+   #   (printf '1\n'; sleep $delay; printf '2\n') | stdbuf -o4 uniq > fifo
+   #   wait # for dd to complete
+   #   compare out exp
+   # }
+   #
+   # retry_delay_ stdbuf_blockbuffer .1 6 || fail=1
 
 Exit $fail

tests/tail-2/F-vs-rename

@@ -42,36 +42,36 @@ mv a b || fail=1
 until grep inaccessible out >/dev/null 2>&1; do :; done
 
 echo x > a
-# Wait up to 4s for this to appear in the output:
+# Wait up to 6.3s for this to appear in the output:
 # "tail: `...' has appeared;  following end of new file"
-found=false
-for i in $(seq 20); do
-  grep 'has appeared;' out > /dev/null && { found=true; break; }
-  sleep .2
-done
-$found || { echo "$0: a: unexpected delay?"; cat out; fail=1; }
+tail_f_vs_rename_1()
+{
+  local delay="$1"
+  grep 'has appeared;' out > /dev/null ||
+    { sleep $delay; return 1; }
+}
+retry_delay_ tail_f_vs_rename_1 .1 7 ||
+  { echo "$0: a: unexpected delay?"; cat out; fail=1; }
 
 echo y >> b
-# Wait up to 4s for "y" to appear in the output:
-found=false
-for i in $(seq 20); do
-  case $(tr '\n' @ < out) in
-    *'@@==> b <==@y@') found=true; break 2;;
-  esac
-  sleep .2
-done
-$found || { echo "$0: b: unexpected delay?"; cat out; fail=1; }
+# Wait up to 6.3s for "y" to appear in the output:
+tail_f_vs_rename_2() {
+  local delay="$1"
+  tr '\n' @ < out | grep '@@==> b <==@y@$' > /dev/null ||
+    { sleep $delay; return 1; }
+}
+retry_delay_ tail_f_vs_rename_2 .1 7 ||
+  { echo "$0: b: unexpected delay?"; cat out; fail=1; }
 
 echo z >> a
-# Wait up to 4s for "z" to appear in the output:
-found=false
-for i in $(seq 20); do
-  case $(tr '\n' @ < out) in
-    *'@@==> a <==@z@') found=true; break 2;;
-  esac
-  sleep .2
-done
-$found || { echo "$0: b: unexpected delay?"; cat out; fail=1; }
+# Wait up to 6.3s for "z" to appear in the output:
+tail_f_vs_rename_3() {
+  local delay="$1"
+  tr '\n' @ < out | grep '@@==> a <==@z@$' > /dev/null ||
+    { sleep $delay; return 1; }
+}
+retry_delay_ tail_f_vs_rename_3 .1 7 ||
+  { echo "$0: a: unexpected delay?"; cat out; fail=1; }
 
 kill -HUP $pid
 

tests/tail-2/flush-initial

@@ -29,14 +29,15 @@ echo line > in || fail=1
 tail -f in > out &
 tail_pid=$!
 
-# Wait for 1 second for the file to be flushed.
-for i in $(seq 10); do
-  test -s out && break
-  echo sleep .1s
-  sleep .1
-done
+# Wait for 1.5s for the file to be flushed.
+tail_flush()
+{
+  local delay="$1"
 
-test -s out || fail=1
+  test -s out ||
+    { sleep "$delay"; return 1; }
+}
+retry_delay_ tail_flush .1 5 || fail=1
 
 kill $tail_pid
 

tests/tail-2/tail-n0f

@@ -39,12 +39,18 @@ for inotify in ---disable-inotify ''; do
     for c_or_n in c n; do
       tail --sleep=4 -${c_or_n} 0 -f $inotify $file &
       pid=$!
-      sleep .5
-      state=$(get_process_status_ $pid)
-      case $state in
-        S*) ;;
-        *) echo $0: process in unexpected state: $state 1>&2; fail=1 ;;
-      esac
+      tail_sleeping()
+      {
+        local delay="$1"; sleep $delay
+        state=$(get_process_status_ $pid)
+        case $state in
+          S*) ;;
+          *) return 1;;
+        esac
+      }
+      # Wait up to 1.5s for tail to sleep
+      retry_delay_ tail_sleeping .1 4 ||
+        { echo $0: process in unexpected state: $state >&2; fail=1; }
       kill $pid
     done
   done

tests/tail-2/wait

@@ -25,7 +25,6 @@ fi
 . $srcdir/test-lib.sh
 
 touch here || framework_failure
-touch k || framework_failure
 { touch unreadable && chmod a-r unreadable; } || framework_failure
 
 
@@ -59,19 +58,27 @@ for inotify in ---disable-inotify ''; do
   test -s tail.err && fail=1
   :>tail.err
 
+  tail_F()
+  {
+    local delay="$1"
 
-  tail -s.1 --max-unchanged-stats=2 -F $inotify k > tail.out &
-  pid=$!
-  sleep .5
-  mv k l
-  sleep .5
-  touch k
-  mv k l
-  sleep .5
-  echo NO >> l
-  sleep .5
-  kill $pid
-  test -s tail.out && fail=1
+    touch k || framework_failure
+    tail -s.1 --max-unchanged-stats=2 -F $inotify k > tail.out &
+    pid=$!
+    sleep $delay
+    mv k l
+    sleep $delay
+    touch k
+    mv k l
+    sleep $delay
+    echo NO >> l
+    sleep $delay
+    kill $pid
+    rm -f k l
+
+    test ! -s tail.out
+  }
+  retry_delay_ tail_F .1 4 || fail=1
 done
 
 Exit $fail