tests: Add pane_status command to check for success of issued commands

When we use pane_wait to wait for a command issued to a tmux pane to finish
we have no idea whether the command succeeded or not.  This means that the
test scripts can keep running long after the point something vital has
failed, making it difficult to work out what went wrong.

Add a new pane_status command that checks for success of the issued command
and use it in most places instead of pane_wait.  We still need explicit
pane_wait where we're gathering explicit output with pane_parse, because
the way we check the status with 'echo $?' means we lose track of that
output.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
[sbrivio:
 - instead of quitting the script, make a test fail if a command
    issued in a pane fails during a test, and loop until the status code is
    numeric in pane_status() as a hack to make it a bit more robust
 - retain usage of pane_wait() in iperf3 and teardown functions as we
   interrupt iperf3, passt, and pasta, so a non-zero exit code is expected
 - drop bogus ns_{1,2}_wait() calls in teardown_two_guests(), those
   functions were never implemented
 - use pane_status() for "guest" test directives too
]
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>

3 files changed, 59 insertions, 50 deletions

diff --git a/test/lib/setup b/test/lib/setup
index b954596..6139e8f 100755
--- a/test/lib/setup
+++ b/test/lib/setup
@@ -32,10 +32,10 @@ setup_passt() {
 
 	__mbuto_dir="$(mktemp -d)"
 	pane_run GUEST "git -C ${__mbuto_dir} clone git://mbuto.sh/mbuto"
-	pane_wait GUEST
+	pane_status GUEST
 
 	pane_run GUEST "${__mbuto_dir}/mbuto/mbuto -p passt -c lz4 -f mbuto.img"
-	pane_wait GUEST
+	pane_status GUEST
 
 	rm -rf "${__mbuto_dir}"
 
@@ -52,9 +52,9 @@ setup_passt() {
 	[ ${TRACE} -eq 1 ] && __opts="${__opts} --trace"
 
 	pane_run PASST "make clean"
-	pane_wait PASST
+	pane_status PASST
 	pane_run PASST "make valgrind"
-	pane_wait PASST
+	pane_status PASST
 	pane_run PASST "valgrind --max-stackframe=$((4 * 1024 * 1024)) --trace-children=yes --vgdb=no --error-exitcode=1 --suppressions=test/valgrind.supp ./passt ${__opts} -f -t 10001 -u 10001 -P passt.pid"
 	sleep 5
 
@@ -66,7 +66,7 @@ setup_passt() {
 		'virtio-net.napi_tx=1"'					   \
 		" -device virtio-net-pci,netdev=hostnet0,x-txburst=16384"  \
 		" -netdev socket,fd=5,id=hostnet0"
-	pane_wait GUEST
+	pane_status GUEST
 }
 
 # setup_pasta() - Create a network and user namespace, connect pasta to it
@@ -80,7 +80,7 @@ setup_pasta() {
 	__tty_pid="$(pane_parse NS)"
 
 	pane_run NS "unshare -rUnpf /bin/sh"
-	pane_wait NS
+	pane_status NS
 
 	pane_run PASST "pstree -p ${__tty_pid} | sed -n 's/.*(\([0-9].*\))$/\1/p'"
 	pane_wait PASST
@@ -131,24 +131,24 @@ setup_passt_in_ns() {
 	pane_run PASST "./pasta ${__opts} -t 10001,10002,10011,10012 -T 10003,10013 -u 10001,10002,10011,10012 -U 10003,10013 -P ${__pid_file}"
 	sleep 1
 	pane_run PASST ''
-	pane_wait PASST
+	pane_status PASST
 	__pasta_pid="$(cat "${__pid_file}")"
 	__ns_pid="$(cat /proc/${__pasta_pid}/task/${__pasta_pid}/children | cut -f1 -d' ')"
 	rm "${__pid_file}"
 
 	pane_run GUEST "nsenter -t ${__ns_pid} -U -n --preserve-credentials"
 	pane_run NS "nsenter -t ${__ns_pid} -U -n -p --preserve-credentials"
-	pane_wait GUEST
-	pane_wait NS
+	pane_status GUEST
+	pane_status NS
 
 	pane_run NS "ip -j li sh | jq -rM '.[] | select(.link_type == \"ether\").ifname'"
 	pane_wait NS
 	__ifname="$(pane_parse NS)"
 	pane_run NS "/sbin/udhcpc -i ${__ifname}"
-	pane_wait NS
+	pane_status NS
 	sleep 2
 	pane_run NS "/sbin/dhclient -6 ${__ifname}"
-	pane_wait NS
+	pane_status NS
 
 	__opts=
 	[ ${PCAP} -eq 1 ] && __opts="${__opts} -p /tmp/passt_in_pasta.pcap"
@@ -157,15 +157,15 @@ setup_passt_in_ns() {
 
 	if [ ${VALGRIND} -eq 1 ]; then
 		pane_run PASST "make clean"
-		pane_wait PASST
+		pane_status PASST
 		pane_run PASST "make valgrind"
-		pane_wait PASST
+		pane_status PASST
 		pane_run PASST "valgrind --max-stackframe=$((4 * 1024 * 1024)) --trace-children=yes --vgdb=no --error-exitcode=1 --suppressions=test/valgrind.supp ./passt -f ${__opts} -t 10001,10011,10021,10031 -u 10001,10011,10021,10031 -P passt.pid"
 	else
 		pane_run PASST "make clean"
-		pane_wait PASST
+		pane_status PASST
 		pane_run PASST "make"
-		pane_wait PASST
+		pane_status PASST
 		pane_run PASST "./passt -f ${__opts} -t 10001,10011,10021,10031 -u 10001,10011,10021,10031 -P passt.pid"
 	fi
 	sleep 5
@@ -178,7 +178,7 @@ setup_passt_in_ns() {
 		'virtio-net.napi_tx=1"'					   \
 		" -device virtio-net-pci,netdev=hostnet0,x-txburst=524288" \
 		" -netdev socket,fd=5,id=hostnet0"
-	pane_wait GUEST
+	pane_status GUEST
 }
 
 # setup_two_guests() - Set up two namespace, run qemu and passt in both of them
@@ -216,8 +216,8 @@ setup_two_guests() {
 	pane_run PASST_1 ''
 	pane_run PASST_2 ''
 
-	pane_wait PASST_1
-	pane_wait PASST_2
+	pane_status PASST_1
+	pane_status PASST_2
 	__pasta1_pid="$(cat "${__pid1_file}")"
 	__ns1_pid="$(cat /proc/${__pasta1_pid}/task/${__pasta1_pid}/children | cut -f1 -d' ')"
 	rm "${__pid1_file}"
@@ -234,13 +234,13 @@ setup_two_guests() {
 
 	pane_run GUEST_1 "/sbin/udhcpc -i ${__ifname}"
 	pane_run GUEST_2 "/sbin/udhcpc -i ${__ifname}"
-	pane_wait GUEST_1
-	pane_wait GUEST_2
+	pane_status GUEST_1
+	pane_status GUEST_2
 	sleep 2
 	pane_run GUEST_1 "/sbin/dhclient -6 ${__ifname}"
 	pane_run GUEST_2 "/sbin/dhclient -6 ${__ifname}"
-	pane_wait GUEST_1
-	pane_wait GUEST_2
+	pane_status GUEST_1
+	pane_status GUEST_2
 
 	__opts=
 	[ ${PCAP} -eq 1 ] && __opts="${__opts} -p /tmp/passt_1.pcap"
@@ -257,7 +257,7 @@ setup_two_guests() {
 	pane_run PASST_2 "./passt -f ${__opts} -t 10004 -u 10004"
 
 	pane_run GUEST_2 'cp mbuto.img mbuto_2.img'
-	pane_wait GUEST_2
+	pane_status GUEST_2
 
 	pane_run GUEST_1 './qrap 5 kvm -m '${VMEM}' -cpu host -smp '${VCPUS} \
 		' -kernel ' "/boot/vmlinuz-$(uname -r)"			     \
@@ -275,8 +275,8 @@ setup_two_guests() {
 		'virtio-net.napi_tx=1"'					     \
 		" -device virtio-net-pci,netdev=hostnet0,x-txburst=16384"    \
 		" -netdev socket,fd=5,id=hostnet0"
-	pane_wait GUEST_1
-	pane_wait GUEST_2
+	pane_status GUEST_1
+	pane_status GUEST_2
 }
 
 # teardown_passt() - Kill qemu, remove passt PID file
@@ -301,7 +301,7 @@ teardown_passt_in_ns() {
 	tmux send-keys -t ${PANE_GUEST} "C-d"
 
 	[ ${VALGRIND} -eq 0 ] && tmux send-keys -t ${PANE_PASST} "C-c"
-	[ ${VALGRIND} -eq 0 ] && pane_wait GUEST
+	[ ${VALGRIND} -eq 0 ] && pane_status GUEST
 	tmux send-keys -t ${PANE_PASST} "C-d"
 
 	tmux send-keys -t ${PANE_NS} "C-d"
@@ -331,13 +331,8 @@ teardown_two_guests() {
 	pane_wait PASST_2
 	tmux send-keys -t ${PANE_PASST_2} "C-d"
 
-	tmux send-keys -t ${PANE_NS_1} "C-d"
-	tmux send-keys -t ${PANE_NS_2} "C-d"
-
 	pane_wait GUEST_1
 	pane_wait GUEST_2
-	ns_1_wait
-	ns_2_wait
 	pane_wait PASST_1
 	pane_wait PASST_2
 }
diff --git a/test/lib/term b/test/lib/term
index 0093f90..004c2a2 100755
--- a/test/lib/term
+++ b/test/lib/term
@@ -228,6 +228,20 @@ pane_parse() {
 		printf '%s' "${__buf}" || printf '@EMPTY@'
 }
 
+# pane_status() - Wait for command to complete and return its exit status
+# $1:	Pane name
+pane_status() {
+	pane_wait "${1}"
+	__status="$(pane_parse "${1}")"
+	while ! [ "${__status}" -eq "${__status}" ]; do
+		sleep 1
+		pane_run "${1}" 'echo $?'
+		pane_wait "${1}"
+		__status="$(pane_parse "${1}")"
+	done
+	return ${__status}
+}
+
 # status_file_end() - Display and log messages when tests from one file are done
 status_file_end() {
 	[ -z "${STATUS_FILE}" ] && return
diff --git a/test/lib/test b/test/lib/test
index 2cfec9a..4262343 100755
--- a/test/lib/test
+++ b/test/lib/test
@@ -52,7 +52,7 @@ test_iperf3() {
 
 	pane_run "${__pane}" 'for i in $(seq 0 '${__procs}'); do'	\
 		':> s${i}.bw; done'
-	pane_wait "${__pane}"
+	pane_status "${__pane}"
 
 	if [ ${UDP_CLIENT} -eq 0 ]; then
 		pane_run "${__pane}" 'for i in $(seq 0 '${__procs}');'	\
@@ -70,7 +70,7 @@ test_iperf3() {
 			'done'
 	fi
 
-	pane_wait "${__pane}"
+	pane_status "${__pane}"
 	sleep 45
 	pane_run "${__pane}" 'for i in $(seq 0 '${__procs}'); do'	\
 		 'kill -INT $(cat s${i}.pid) 2>/dev/null; done'
@@ -83,7 +83,7 @@ test_iperf3() {
 	pane_parse "${__pane}"
 	pane_run "${__pane}" 'for i in $(seq 0 '${__procs}'); do'	\
 		'rm -f s${i}.bw; done'
-	pane_wait "${__pane}"
+	pane_status "${__pane}"
 }
 
 test_one_line() {
@@ -139,13 +139,13 @@ test_one_line() {
 		;;
 	"host")
 		pane_run HOST "${__arg}"
-		pane_wait HOST
+		pane_status HOST || TEST_ONE_nok=1
 		;;
 	"hostb")
 		pane_run HOST "${__arg}"
 		;;
 	"hostw")
-		pane_wait HOST
+		pane_status HOST || TEST_ONE_nok=1
 		;;
 	"hint")
 		tmux send-keys -t ${PANE_HOST} "C-c"
@@ -157,13 +157,13 @@ test_one_line() {
 		;;
 	"passt")
 		pane_run PASST "${__arg}"
-		pane_wait PASST
+		pane_status PASST || TEST_ONE_nok=1
 		;;
 	"passtb")
 		pane_run PASST "${__arg}"
 		;;
 	"passtw")
-		pane_wait PASST
+		pane_status PASST || TEST_ONE_nok=1
 		;;
 	"pout")
 		__varname="${__arg%% *}"
@@ -173,23 +173,23 @@ test_one_line() {
 		;;
 	"guest")
 		pane_run GUEST "${__arg}"
-		pane_wait GUEST
+		pane_status GUEST || TEST_ONE_nok=1
 		;;
 	"guestb")
 		pane_run GUEST "${__arg}"
 		;;
 	"guestw")
-		pane_wait GUEST
+		pane_status GUEST || TEST_ONE_nok=1
 		;;
 	"guest1")
 		pane_run GUEST_1 "${__arg}"
-		pane_wait GUEST_1
+		pane_status GUEST_1 || TEST_ONE_nok=1
 		;;
 	"guest1b")
 		pane_run GUEST_1 "${__arg}"
 		;;
 	"guest1w")
-		pane_wait GUEST_1
+		pane_status GUEST_1 || TEST_ONE_nok=1
 		;;
 	"gtools")
 		pane_run GUEST 'which '"${__arg}"' >/dev/null || echo skip'
@@ -208,25 +208,25 @@ test_one_line() {
 		;;
 	"guest2")
 		pane_run GUEST_2 "${__arg}"
-		pane_wait GUEST_2
+		pane_status GUEST_2 || TEST_ONE_nok=1
 		;;
 	"guest2b")
 		pane_run GUEST_2 "${__arg}"
 		;;
 	"guest2w")
-		pane_wait GUEST_2
+		pane_status GUEST_2 || TEST_ONE_nok=1
 		;;
 	"ns")
 		pane_run NS "${__arg}"
-		pane_wait NS
+		pane_status NS || TEST_ONE_nok=1
 		;;
 	"ns1")
 		pane_run NS1 "${__arg}"
-		pane_wait NS1
+		pane_status NS1 || TEST_ONE_nok=1
 		;;
 	"ns2")
 		pane_run NS2 "${__arg}"
-		pane_wait NS2
+		pane_status NS2 || TEST_ONE_nok=1
 		;;
 	"nsb")
 		pane_run NS "${__arg}"
@@ -238,13 +238,13 @@ test_one_line() {
 		pane_run NS2 "${__arg}"
 		;;
 	"nsw")
-		pane_wait NS
+		pane_status NS || TEST_ONE_nok=1
 		;;
 	"ns1w")
-		pane_wait NS1
+		pane_status NS1 || TEST_ONE_nok=1
 		;;
 	"ns2w")
-		pane_wait NS2
+		pane_status NS2 || TEST_ONE_nok=1
 		;;
 	"nstools")
 		pane_run NS 'which '"${__arg}"' >/dev/null || echo skip'