From 061519b5620f594b5e5711ae6f3372ff152bc14c Mon Sep 17 00:00:00 2001 From: Stefano Brivio Date: Mon, 27 Sep 2021 15:10:35 +0200 Subject: test: Add CI/demo scripts Not really quick, definitely dirty. Signed-off-by: Stefano Brivio --- test/lib/layout | 276 +++++++++++++++++++++++ test/lib/perf_report | 262 ++++++++++++++++++++++ test/lib/setup | 322 +++++++++++++++++++++++++++ test/lib/term | 610 +++++++++++++++++++++++++++++++++++++++++++++++++++ test/lib/test | 378 +++++++++++++++++++++++++++++++ test/lib/util | 142 ++++++++++++ test/lib/video | 119 ++++++++++ 7 files changed, 2109 insertions(+) create mode 100644 test/lib/layout create mode 100755 test/lib/perf_report create mode 100755 test/lib/setup create mode 100755 test/lib/term create mode 100755 test/lib/test create mode 100755 test/lib/util create mode 100755 test/lib/video (limited to 'test/lib') diff --git a/test/lib/layout b/test/lib/layout new file mode 100644 index 0000000..a07f9c9 --- /dev/null +++ b/test/lib/layout @@ -0,0 +1,276 @@ +#!/bin/sh +# +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# PASST - Plug A Simple Socket Transport +# for qemu/UNIX domain socket mode +# +# PASTA - Pack A Subtle Tap Abstraction +# for network namespace/tap device mode +# +# test/lib/layout - tmux pane layouts +# +# Copyright (c) 2021 Red Hat GmbH +# Author: Stefano Brivio + +# layout_host() - Simple host commands layout with info and host panes +layout_host() { + sleep 3 + + tmux kill-pane -a -t 0 + cmd_write 0 clear + + tmux split-window -h -l '35%' -t passt_test:1.0 + + PANE_HOST=0 + PANE_INFO=1 + + get_info_cols + + tmux send-keys -l -t ${PANE_INFO} 'while cat /tmp/.passt_test_log_pipe; do :; done' + tmux send-keys -t ${PANE_INFO} -N 100 C-m + tmux select-pane -t ${PANE_INFO} -T "test log" + + tmux pipe-pane -O -t ${PANE_HOST} "cat >> ${LOGDIR}/pane_host.log" + tmux select-pane -t ${PANE_HOST} -T "host" + + info_layout "host commands only" + + sleep 1 +} + +# layout_pasta() - Panes for host, pasta, and separate one for namespace +layout_pasta() { + sleep 3 + + tmux kill-pane -a -t 0 + cmd_write 0 clear + + tmux split-window -v -t passt_test + tmux split-window -h -t passt_test + tmux split-window -h -l '42%' -t passt_test:1.0 + + PANE_NS=0 + PANE_INFO=1 + PANE_HOST=2 + PANE_PASST=3 + + get_info_cols + + tmux pipe-pane -O -t ${PANE_NS} "cat >> ${LOGDIR}/pane_ns.log" + tmux select-pane -t ${PANE_NS} -T "namespace" + + tmux send-keys -l -t ${PANE_INFO} 'while cat /tmp/.passt_test_log_pipe; do :; done' + tmux send-keys -t ${PANE_INFO} -N 100 C-m + tmux select-pane -t ${PANE_INFO} -T "test log" + + tmux pipe-pane -O -t ${PANE_HOST} "cat >> ${LOGDIR}/pane_host.log" + tmux select-pane -t ${PANE_HOST} -T "host" + + tmux pipe-pane -O -t ${PANE_PASST} "cat >> ${LOGDIR}/pane_passt.log" + tmux select-pane -t ${PANE_PASST} -T "pasta" + + info_layout "single pasta instance with namespace" + + sleep 1 +} + +# layout_passt() - Panes for host, passt, and guest +layout_passt() { + sleep 3 + + tmux kill-pane -a -t 0 + cmd_write 0 clear + + tmux split-window -v -t passt_test + tmux split-window -h -t passt_test + tmux split-window -h -l '42%' -t passt_test:1.0 + + PANE_GUEST=0 + PANE_INFO=1 + PANE_HOST=2 + PANE_PASST=3 + + get_info_cols + + tmux pipe-pane -O -t ${PANE_GUEST} "cat >> ${LOGDIR}/pane_guest.log" + tmux select-pane -t ${PANE_GUEST} -T "guest" + + tmux send-keys -l -t ${PANE_INFO} 'while cat /tmp/.passt_test_log_pipe; do :; done' + tmux send-keys -t ${PANE_INFO} -N 100 C-m + tmux select-pane -t ${PANE_INFO} -T "test log" + + tmux pipe-pane -O -t ${PANE_HOST} "cat >> ${LOGDIR}/pane_host.log" + tmux select-pane -t ${PANE_HOST} -T "host" + + tmux pipe-pane -O -t ${PANE_PASST} "cat >> ${LOGDIR}/pane_passt.log" + tmux select-pane -t ${PANE_PASST} -T "passt" + + info_layout "single passt instance with guest" + + sleep 1 +} + +# layout_passt_in_pasta() - Host, passt within pasta, namespace and guest +layout_passt_in_pasta() { + sleep 3 + + tmux kill-pane -a -t 0 + cmd_write 0 clear + + tmux split-window -v -l '45%' -t passt_test + tmux split-window -h -t passt_test + tmux split-window -h -l '42%' -t passt_test:1.0 + tmux split-window -v -t passt_test:1.0 + + PANE_GUEST=0 + PANE_NS=1 + PANE_INFO=2 + PANE_HOST=3 + PANE_PASST=4 + + get_info_cols + + tmux pipe-pane -O -t ${PANE_GUEST} "cat >> ${LOGDIR}/pane_guest.log" + tmux select-pane -t ${PANE_GUEST} -T "guest" + + tmux pipe-pane -O -t ${PANE_NS} "cat >> ${LOGDIR}/pane_ns.log" + tmux select-pane -t ${PANE_NS} -T "namespace" + + tmux send-keys -l -t ${PANE_INFO} 'while cat /tmp/.passt_test_log_pipe; do :; done' + tmux send-keys -t ${PANE_INFO} -N 100 C-m + tmux select-pane -t ${PANE_INFO} -T "test log" + + tmux pipe-pane -O -t ${PANE_HOST} "cat >> ${LOGDIR}/pane_host.log" + tmux select-pane -t ${PANE_HOST} -T "host" + + tmux pipe-pane -O -t ${PANE_PASST} "cat >> ${LOGDIR}/pane_passt.log" + tmux select-pane -t ${PANE_PASST} -T "passt in pasta (namespace)" + + info_layout "passt and guest in namespace, connected by pasta" + + sleep 1 +} + +# layout_two_guests() - Two guest panes, two passt panes, plus host and log +layout_two_guests() { + sleep 3 + + tmux kill-pane -a -t 0 + cmd_write 0 clear + + tmux split-window -v -t passt_test + tmux split-window -h -l '33%' + tmux split-window -h -t passt_test:1.1 + + tmux split-window -h -l '35%' -t passt_test:1.0 + tmux split-window -v -t passt_test:1.0 + + for i in `seq 0 5`; do + tmux select-pane -t $i -T "${i}" + done + + PANE_GUEST_1=0 + PANE_GUEST_2=1 + PANE_INFO=2 + PANE_HOST=3 + PANE_PASST_1=4 + PANE_PASST_2=5 + + get_info_cols + + tmux pipe-pane -O -t ${PANE_GUEST_1} "cat >> ${LOGDIR}/pane_guest_1.log" + tmux select-pane -t ${PANE_GUEST_1} -T "guest #1 in namespace #1" + + tmux pipe-pane -O -t ${PANE_GUEST_2} "cat >> ${LOGDIR}/pane_guest_2.log" + tmux select-pane -t ${PANE_GUEST_2} -T "guest #2 in namespace #2" + + tmux send-keys -l -t ${PANE_INFO} 'while cat /tmp/.passt_test_log_pipe; do :; done' + tmux send-keys -t ${PANE_INFO} -N 100 C-m + tmux select-pane -t ${PANE_INFO} -T "test log" + + tmux pipe-pane -O -t ${PANE_HOST} "cat >> ${LOGDIR}/pane_host.log" + tmux select-pane -t ${PANE_HOST} -T "host" + + tmux pipe-pane -O -t ${PANE_PASST_1} "cat >> ${LOGDIR}/pane_passt_1.log" + tmux select-pane -t ${PANE_PASST_1} -T "passt #1 in namespace #1" + + tmux pipe-pane -O -t ${PANE_PASST_2} "cat >> ${LOGDIR}/pane_passt_2.log" + tmux select-pane -t ${PANE_PASST_2} -T "passt #2 in namespace #2" + + info_layout "two guests, two passt instances, in namespaces" + + sleep 1 +} + +# layout_demo_pasta() - Four panes for pasta demo +layout_demo_pasta() { + sleep 3 + + tmux kill-pane -a -t 0 + cmd_write 0 clear + sleep 1 + cmd_write 0 clear + + tmux split-window -v -t passt_test + tmux split-window -h -t passt_test + tmux split-window -h -l '42%' -t passt_test:1.0 + + PANE_NS=0 + PANE_INFO=1 + PANE_HOST=2 + PANE_PASST=3 + + get_info_cols + + tmux pipe-pane -O -t ${PANE_NS} "cat >> ${LOGDIR}/pane_ns.log" + tmux select-pane -t ${PANE_NS} -T "namespace" + + tmux send-keys -l -t ${PANE_INFO} 'while cat /tmp/.passt_test_log_pipe; do :; done' + tmux send-keys -t ${PANE_INFO} -N 100 C-m + tmux select-pane -t ${PANE_INFO} -T "" + + tmux pipe-pane -O -t ${PANE_HOST} "cat >> ${LOGDIR}/pane_host.log" + tmux select-pane -t ${PANE_HOST} -T "host" + + tmux pipe-pane -O -t ${PANE_PASST} "cat >> ${LOGDIR}/pane_passt.log" + tmux select-pane -t ${PANE_PASST} -T "pasta" + + sleep 1 +} + +# layout_demo_passt() - Four panes for passt demo +layout_demo_passt() { + sleep 3 + + tmux kill-pane -a -t 0 + cmd_write 0 clear + sleep 1 + cmd_write 0 clear + + tmux split-window -v -t passt_test + tmux split-window -h -t passt_test + tmux split-window -h -l '42%' -t passt_test:1.0 + + PANE_GUEST=0 + PANE_INFO=1 + PANE_HOST=2 + PANE_PASST=3 + + get_info_cols + + tmux pipe-pane -O -t ${PANE_GUEST} "cat >> ${LOGDIR}/pane_guest.log" + tmux select-pane -t ${PANE_GUEST} -T "guest" + + tmux send-keys -l -t ${PANE_INFO} 'while cat /tmp/.passt_test_log_pipe; do :; done' + tmux send-keys -t ${PANE_INFO} -N 100 C-m + tmux select-pane -t ${PANE_INFO} -T "" + + tmux pipe-pane -O -t ${PANE_HOST} "cat >> ${LOGDIR}/pane_host.log" + tmux select-pane -t ${PANE_HOST} -T "host" + + tmux pipe-pane -O -t ${PANE_PASST} "cat >> ${LOGDIR}/pane_passt.log" + tmux select-pane -t ${PANE_PASST} -T "passt in pasta (namespace)" + + sleep 1 +} diff --git a/test/lib/perf_report b/test/lib/perf_report new file mode 100755 index 0000000..111c154 --- /dev/null +++ b/test/lib/perf_report @@ -0,0 +1,262 @@ +#!/bin/sh +# +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# PASST - Plug A Simple Socket Transport +# for qemu/UNIX domain socket mode +# +# PASTA - Pack A Subtle Tap Abstraction +# for network namespace/tap device mode +# +# test/lib/perf_report - Prepare JavaScript report for performance tests +# +# Copyright (c) 2021 Red Hat GmbH +# Author: Stefano Brivio + +PERF_LINK_COUNT=0 +PERF_JS="${BASEPATH}/perf.js" + +PERF_TEMPLATE_HTML="document.write('"' +Throughput in Gbps, latency in µs. Threads are iperf3 processes, passt and pasta are currently single-threaded.
+Click on numbers to show test execution. Measured at head, commit __commit__. + + + +
    +

  • passt

    + + + + + + + + __passt_tcp_header__ + __passt_udp_header__ + + __passt_tcp_LINE__ __passt_udp_LINE__ +
    + TCP, __passt_tcp_threads__ at __passt_tcp_freq__ GHzUDP, __passt_udp_threads__ at __passt_udp_freq__ GHz
    MTU:
    + + + +

  • pasta: local connections/traffic

    + + + + + + + + __pasta_lo_tcp_header__ + __pasta_lo_udp_header__ + + __pasta_lo_tcp_LINE__ __pasta_lo_udp_LINE__ +
    + TCP, __pasta_lo_tcp_threads__ at __pasta_lo_tcp_freq__ GHzUDP, __pasta_lo_udp_threads__ at __pasta_lo_udp_freq__ GHz
    MTU:
    + +

  • pasta: connections/traffic via tap

    + + + + + + + + __pasta_tap_tcp_header__ + __pasta_tap_udp_header__ + + __pasta_tap_tcp_LINE__ __pasta_tap_udp_LINE__ +
    + TCP, __pasta_tap_tcp_threads__ at __pasta_tap_tcp_freq__ GHzUDP, __pasta_tap_udp_threads__ at __pasta_tap_udp_freq__ GHz
    MTU:
    + +
' + +PERF_TEMPLATE_JS="'); + +var perf_links = [ +" + +PERF_TEMPLATE_POST=']; + +for (var i = 0; i < perf_links.length; i++) { + var obj = document.getElementById(perf_links[i][0]); + + obj.addEventListener("click", function(event) { + var ci_video = document.getElementById("ci_video"); + var top = ci_video.offsetTop - 5; + + event.preventDefault(); + ci_video.play(); + ci_video.pause(); + for (var i = 0; i < perf_links.length; i++) { + if (this.id == perf_links[i][0]) { + ci_video.currentTime = perf_links[i][1] - 10; + } + } + window.scrollTo({ top: top, behavior: "smooth" }) + ci_video.play(); + }, false); +} +' + +# perf_init() - Process first part of template +perf_init() { + echo "${PERF_TEMPLATE_HTML}" > "${PERF_JS}" + perf_report_sub commit "$(echo ${COMMIT} | sed "s/'/\\\'/g")" +} + +# perf_fill_lines() - Fill multiple "LINE" directives in template, matching rows +perf_fill_lines() { + while true; do + __file_line="$(sed -n '/__.*_LINE__/{=;q}' "${PERF_JS}")" + [ -z "${__file_line}" ] && break + + __line_no=0 + __done=0 + __line_buf="" + while true; do + __match_first_td=0 + for __t in $(sed -n '/__.*_LINE__/{p;q}' "${PERF_JS}"); do + if [ ${__match_first_td} -eq 1 ]; then + __matching_line_no=0 + while true; do + __line_part= + __var_name="$(echo $__t | sed -n 's/__\(.*\)__/\1_'"${__matching_line_no}"'/p')" + [ -z "$(eval echo \$${__var_name})" ] && break + __line_part="$(eval echo \$${__var_name})" + __td_check="$(echo "${__line_part}" | sed -n 's/^\([^>]*\)<\/td>.*$/\1/p')" + if [ "${__td_check}" = "${__td_match}" ]; then + __line_part="$(echo "${__line_part}" | sed -n 's/^[^>]*<\/td>\(.*\)$/\1/p')" + break + fi + __matching_line_no=$((__matching_line_no + 1)) + done + else + __var_name="$(echo $__t | sed -n 's/__\(.*\)__/\1_'"${__line_no}"'/p')" + [ -z "$(eval echo \$${__var_name})" ] && __done=1 && break + __line_part="$(eval echo \$${__var_name})" + __td_match="$(echo "${__line_part}" | sed -n 's/^\([^>]*\)<\/td>.*$/\1/p')" + fi + __line_buf="${__line_buf}${__line_part}" + __match_first_td=1 + done + [ ${__done} -eq 1 ] && break + __line_no=$((__line_no + 1)) + __line_buf="${__line_buf}" + done + __line_buf="${__line_buf}" + __line_buf="$(printf '%s\n' "${__line_buf}" | sed -e 's/[]\/$*.^[]/\\&/g')" + sed -i "${__file_line}s/.*/${__line_buf}/" "${PERF_JS}" + done +} + +# perf_finish() - Add trailing backslashes and process ending templates +perf_finish() { + perf_fill_lines + sed -i 's/^.*$/&\\/g' "${PERF_JS}" + echo "${PERF_TEMPLATE_JS}" >> "${PERF_JS}" + echo "${PERF_TEMPLATE_POST}" >> "${PERF_JS}" +} + +# perf_report_sub() - Apply simple substitutions in template +perf_report_sub() { + __et="$(printf '%s\n' "${1}" | sed -e 's/[\/&]/\\&/g')" + __es="$(printf '%s\n' "${2}" | sed -e 's/[]\/$*.^[]/\\&/g')" + + sed -i 's/__'"${__et}"'__/'"${__es}"'/g' "${PERF_JS}" +} + +# perf_report_append() - Append generic string to current JavaScript report +perf_report_append() { + echo "${@}" >> "${PERF_JS}" +} + +# perf_report_append() - Append generic string to current template buffer +perf_report_append_js() { + PERF_TEMPLATE_JS="${PERF_TEMPLATE_JS}${@}" +} + +# perf_report() - Start of single test report +perf_report() { + __mode="${1}" + __proto="${2}" + __threads="${3}" + __freq="${4}" + + REPORT_IN="${__mode}_${__proto}" + + [ ${__threads} -eq 1 ] && __threads="one thread" || __threads="${__threads} threads" + perf_report_sub "${__mode}_${__proto}_threads" "${__threads}" + perf_report_sub "${__mode}_${__proto}_freq" "${__freq}" + + perf_report_append_js "[ 'perf_${__mode}_${__proto}', $(video_time_now) ]," +} + +# perf_th() - Table header for a set of tests +perf_th() { + shift + + __th_buf= + __cols_count=0 + for __arg; do + __th_buf="${__th_buf}${__arg}" + __cols_count=$((__cols_count + 1)) + done + perf_report_sub "${REPORT_IN}_header" "${__th_buf}" + perf_report_sub "${REPORT_IN}_cols" ${__cols_count} +} + +# perf_tr() - Main table row +perf_tr() { + __line_no=0 + shift + while true; do + [ -z "$(eval echo \$${REPORT_IN}_LINE_${__line_no})" ] && break + __line_no=$((__line_no + 1)) + done + eval ${REPORT_IN}_LINE_${__line_no}="\"${@}\"" +} + +# perf_td() - Single cell with test result +perf_td() { + __rewind="${1}" + shift + + __line_no=0 + while true; do + [ -z "$(eval echo \$${REPORT_IN}_LINE_${__line_no})" ] && break + __line_no=$((__line_no + 1)) + done + __line_no=$((__line_no - 1)) + [ -z "${1}" ] && __id=0 || __id="perf_${PERF_LINK_COUNT}" + eval ${REPORT_IN}_LINE_${__line_no}=\""\${${REPORT_IN}_LINE_${__line_no}}${1}"\" + [ -z "${1}" ] && return + + perf_report_append_js "[ '${__id}', $(($(video_time_now) - ${__rewind})) ]," + PERF_LINK_COUNT=$((PERF_LINK_COUNT + 1)) +} + +# perf_te() - End of a table, currently unused +pert_te() { + : +} diff --git a/test/lib/setup b/test/lib/setup new file mode 100755 index 0000000..fd68f05 --- /dev/null +++ b/test/lib/setup @@ -0,0 +1,322 @@ +#!/bin/sh +# +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# PASST - Plug A Simple Socket Transport +# for qemu/UNIX domain socket mode +# +# PASTA - Pack A Subtle Tap Abstraction +# for network namespace/tap device mode +# +# test/lib/setup - Set up and tear down passt and pasta environments +# +# Copyright (c) 2021 Red Hat GmbH +# Author: Stefano Brivio + +VCPUS="$( [ $(nproc) -ge 8 ] && echo 4 || echo $(( $(nproc) / 2 + 1 )) )" +__mem_kib="$(sed -n 's/MemTotal:[ ]*\([0-9]*\) kB/\1/p' /proc/meminfo)" +VMEM="$((${__mem_kib} / 1024 / 8))" + +# setup_build() - Set up pane layout for build tests +setup_build() { + MODE=build + + layout_host +} + +# setup_passt() - Build guest initrd with mbuto, start qemu and passt +setup_passt() { + MODE=passt + + layout_passt + + __mbuto_dir="$(mktemp -d)" + pane_run GUEST "git -C ${__mbuto_dir} clone https://mbuto.lameexcu.se/mbuto/" + pane_wait GUEST + + pane_run GUEST "${__mbuto_dir}/mbuto/mbuto -p passt -c lz4 -f mbuto.img" + pane_wait GUEST + + rm -rf "${__mbuto_dir}" + + # Ports: + # + # guest | host + # --------------|--------------------- + # 10001 as server | forwarded to guest + # 10003 | as server + + __opts= + [ ${PCAP} -eq 1 ] && __opts="${__opts} -p /tmp/passt.pcap" + [ ${DEBUG} -eq 1 ] && __opts="${__opts} -d" + pane_run PASST "./passt ${__opts} -f -t 10001 -u 10001" + sleep 1 + + pane_run GUEST './qrap 5 kvm -m '${VMEM}' -cpu host -smp '${VCPUS} \ + '-kernel' "/boot/vmlinuz-$(uname -r)" \ + '-initrd mbuto.img -nographic -serial stdio' \ + '-nodefaults ' \ + '-append "console=ttyS0 mitigations=off apparmor=0 ' \ + 'virtio-net.napi_tx=1"' \ + "-device virtio-net-pci,netdev=hostnet0,x-txburst=16384"\ + "-netdev socket,fd=5,id=hostnet0" + pane_wait GUEST +} + +# setup_pasta() - Create a network and user namespace, connect pasta to it +setup_pasta() { + MODE=pasta + + layout_pasta + + pane_run NS "unshare -rUn /bin/sh " + pane_wait NS + + pane_run NS 'echo $$' + pane_wait NS + __pasta_pid="$(pane_parse NS)" + + # Ports: + # + # ns | host + # ------------------|--------------------- + # 10002 as server | spliced to ns + # 10003 spliced to init | as server + + __opts= + [ ${PCAP} -eq 1 ] && __opts="${__opts} -p /tmp/pasta.pcap" + [ ${DEBUG} -eq 1 ] && __opts="${__opts} -d" + + pane_run PASST "./pasta ${__opts} -f -t 10002 -T 10003 -u 10002 -U 10003 ${__pasta_pid}" + sleep 1 +} + +# setup_passt_in_ns() - Set up namespace (with pasta), run qemu and passt into it +setup_passt_in_ns() { + MODE=passt_in_ns + + layout_passt_in_pasta + + # Ports: + # + # guest | ns | host + # -------------|--------------------|----------------- + # 10001 as server | forwarded to guest | spliced to ns + # 10002 | as server | spliced to ns + # 10003 | spliced to init | as server + # 10011 as server | forwarded to guest | spliced to ns + # 10012 | as server | spliced to ns + # 10013 | spliced to init | as server + + __opts= + [ ${PCAP} -eq 1 ] && __opts="${__opts} -p /tmp/pasta_with_passt.pcap" + [ ${DEBUG} -eq 1 ] && __opts="${__opts} -d" + + pane_run PASST "./pasta ${__opts} -t 10001,10002,10011,10012 -T 10003,10013 -u 10001,10002,10011,10012 -U 10003,10013" + sleep 1 + pane_run PASST '' + pane_wait PASST + pane_run PASST 'echo $$' + pane_wait PASST + __ns_pid="$(pane_parse PASST)" + + pane_run GUEST "nsenter -t ${__ns_pid} -U -n --preserve-credentials" + pane_run NS "nsenter -t ${__ns_pid} -U -n --preserve-credentials" + pane_wait GUEST + pane_wait NS + + pane_run NS "ip -j li sh | jq -rM '.[] | select(.link_type == \"ether\").ifname'" + pane_wait NS + __ifname="$(pane_parse NS)" + pane_run NS "/sbin/udhcpc -i ${__ifname}" + pane_wait NS + sleep 2 + pane_run NS "/sbin/dhclient -6 ${__ifname}" + pane_wait NS + + __opts= + [ ${PCAP} -eq 1 ] && __opts="${__opts} -p /tmp/passt_in_pasta.pcap" + [ ${DEBUG} -eq 1 ] && __opts="${__opts} -d" + + #pane_run PASST "valgrind --max-stackframe=2097208 ./passt -f ${__opts} -t 10001,10011 -u 10001,10011" + pane_run PASST "./passt -f ${__opts} -t 10001,10011 -u 10001,10011" + sleep 1 + + pane_run GUEST './qrap 5 kvm -m '${VMEM}' -cpu host -smp '${VCPUS} \ + '-kernel' "/boot/vmlinuz-$(uname -r)" \ + '-initrd mbuto.img -nographic -serial stdio' \ + '-nodefaults ' \ + '-append "console=ttyS0 mitigations=off apparmor=0 ' \ + 'virtio-net.napi_tx=1"' \ + "-device virtio-net-pci,netdev=hostnet0,x-txburst=131072"\ + "-netdev socket,fd=5,id=hostnet0" + pane_wait GUEST +} + +# setup_two_guests() - Set up two namespace, run qemu and passt in both of them +setup_two_guests() { + MODE=passt_in_ns + + layout_two_guests + + # Ports: + # + # guest #1 | guest #2 | ns #1 | ns #2 | host + # --------- |-----------|-----------|------------|------------ + # 10001 as server | | to guest | to init | to ns #1 + # 10002 | | as server | | to ns #1 + # 10003 | | to init | to init | as server + # 10004 | as server | to init | to guest | to ns #2 + # 10005 | | | as server | to ns #2 + + __opts= + [ ${PCAP} -eq 1 ] && __opts="${__opts} -p /tmp/pasta_1.pcap" + [ ${DEBUG} -eq 1 ] && __opts="${__opts} -d" + pane_run PASST_1 "./pasta ${__opts} -t 10001,10002 -T 10003,10004 -u 10001,10002 -U 10003,10004" + + __opts= + [ ${PCAP} -eq 1 ] && __opts="${__opts} -p /tmp/pasta_2.pcap" + [ ${DEBUG} -eq 1 ] && __opts="${__opts} -d" + pane_run PASST_2 "./pasta ${__opts} -t 10004,10005 -T 10003,10001 -u 10004,10005 -U 10003,10001" + + sleep 1 + pane_run PASST_1 '' + pane_run PASST_2 '' + + pane_wait PASST_1 + pane_wait PASST_2 + pane_run PASST_1 'echo $$' + pane_run PASST_2 'echo $$' + pane_wait PASST_1 + pane_wait PASST_2 + __ns1_pid="$(pane_parse PASST_1)" + __ns2_pid="$(pane_parse PASST_2)" + + pane_run GUEST_1 "nsenter -t ${__ns1_pid} -U -n --preserve-credentials" + pane_run GUEST_2 "nsenter -t ${__ns2_pid} -U -n --preserve-credentials" + + pane_run PASST_1 "ip -j li sh | jq -rM '.[] | select(.link_type == \"ether\").ifname'" + pane_wait PASST_1 + __ifname="$(pane_parse PASST_1)" + + pane_run GUEST_1 "/sbin/udhcpc -i ${__ifname}" + pane_run GUEST_2 "/sbin/udhcpc -i ${__ifname}" + pane_wait GUEST_1 + pane_wait GUEST_2 + sleep 2 + pane_run GUEST_1 "/sbin/dhclient -6 ${__ifname}" + pane_run GUEST_2 "/sbin/dhclient -6 ${__ifname}" + pane_wait GUEST_1 + pane_wait GUEST_2 + + __opts= + [ ${PCAP} -eq 1 ] && __opts="${__opts} -p /tmp/passt_1.pcap" + [ ${DEBUG} -eq 1 ] && __opts="${__opts} -d" + pane_run PASST_1 "./passt -f ${__opts} -t 10001 -u 10001" + sleep 1 + + __opts= + [ ${PCAP} -eq 1 ] && __opts="${__opts} -p /tmp/passt_2.pcap" + [ ${DEBUG} -eq 1 ] && __opts="${__opts} -d" + pane_run PASST_2 "./passt -f ${__opts} -t 10004 -u 10004" + + pane_run GUEST_2 'cp mbuto.img mbuto_2.img' + pane_wait GUEST_2 + + pane_run GUEST_1 './qrap 5 kvm -m '${VMEM}' -cpu host -smp '${VCPUS} \ + '-kernel' "/boot/vmlinuz-$(uname -r)" \ + '-initrd mbuto.img -nographic -serial stdio' \ + '-nodefaults ' \ + '-append "console=ttyS0 mitigations=off apparmor=0 ' \ + 'virtio-net.napi_tx=1"' \ + "-device virtio-net-pci,netdev=hostnet0,x-txburst=16384"\ + "-netdev socket,fd=5,id=hostnet0" + pane_run GUEST_2 './qrap 5 kvm -m '${VMEM}' -cpu host -smp '${VCPUS} \ + '-kernel' "/boot/vmlinuz-$(uname -r)" \ + '-initrd mbuto_2.img -nographic -serial stdio' \ + '-nodefaults ' \ + '-append "console=ttyS0 mitigations=off apparmor=0 ' \ + 'virtio-net.napi_tx=1"' \ + "-device virtio-net-pci,netdev=hostnet0,x-txburst=16384"\ + "-netdev socket,fd=5,id=hostnet0" + pane_wait GUEST_1 + pane_wait GUEST_2 +} + +# teardown_passt() - Kill qemu and passt +teardown_passt() { + tmux send-keys -t ${PANE_PASST} "C-c" + pane_wait PASST + tmux send-keys -t ${PANE_GUEST} "C-c" + pane_wait GUEST +} + +# teardown_passt() - Exit namespace, kill pasta process +teardown_pasta() { + tmux send-keys -t ${PANE_PASST} "C-c" + pane_wait PASST + tmux send-keys -t ${PANE_NS} "C-d" + pane_wait NS +} + +# teardown_passt_in_ns() - Exit namespace, kill qemu, passt and pasta +teardown_passt_in_ns() { + tmux send-keys -t ${PANE_GUEST} "C-c" + pane_wait GUEST + tmux send-keys -t ${PANE_GUEST} "C-d" + + tmux send-keys -t ${PANE_PASST} "C-c" + pane_wait PASST + tmux send-keys -t ${PANE_PASST} "C-d" + + tmux send-keys -t ${PANE_NS} "C-d" + + pane_wait GUEST + pane_wait NS + pane_wait PASST +} + +# teardown_two_guests() - Exit namespaces, kill qemu processes, passt and pasta +teardown_two_guests() { + tmux send-keys -t ${PANE_GUEST_1} "C-c" + pane_wait GUEST_1 + tmux send-keys -t ${PANE_GUEST_1} "C-d" + + tmux send-keys -t ${PANE_GUEST_2} "C-c" + pane_wait GUEST_2 + tmux send-keys -t ${PANE_GUEST_2} "C-d" + + tmux send-keys -t ${PANE_PASST_1} "C-c" + pane_wait PASST_1 + tmux send-keys -t ${PANE_PASST_1} "C-d" + + tmux send-keys -t ${PANE_PASST_2} "C-c" + pane_wait PASST_2 + tmux send-keys -t ${PANE_PASST_2} "C-d" + + tmux send-keys -t ${PANE_NS_1} "C-d" + tmux send-keys -t ${PANE_NS_2} "C-d" + + pane_wait GUEST_1 + pane_wait GUEST_2 + ns_1_wait + ns_2_wait + pane_wait PASST_1 + pane_wait PASST_2 +} + +# setup() - Run setup_*() functions +# $*: Suffix list of setup_*() functions to be called +setup() { + for arg do + eval setup_${arg} + done +} + +# teardown() - Run teardown_*() functions +# $*: Suffix list of teardown_*() functions to be called +teardown() { + for arg do + eval teardown_${arg} + done +} diff --git a/test/lib/term b/test/lib/term new file mode 100755 index 0000000..48f0f6a --- /dev/null +++ b/test/lib/term @@ -0,0 +1,610 @@ +#!/bin/sh +# +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# PASST - Plug A Simple Socket Transport +# for qemu/UNIX domain socket mode +# +# PASTA - Pack A Subtle Tap Abstraction +# for network namespace/tap device mode +# +# test/lib/term - Set up tmux sessions and panes, handle terminals and logs +# +# Copyright (c) 2021 Red Hat GmbH +# Author: Stefano Brivio + +# Commands of X terminals for "CI" and "demo" runs +DEMO_XTERM="cool-retro-term --verbose --workdir" +CI_XTERM="mate-terminal --hide-menubar --profile=passt_ci --working-directory" + +STATUS_FILE= +STATUS_FILE_NTESTS= +STATUS_FILE_INDEX=0 +STATUS_COLS= +STATUS_PASS=0 +STATUS_FAIL=0 + +PR_RED='\033[1;31m' +PR_GREEN='\033[1;32m' +PR_YELLOW='\033[1;33m' +PR_BLUE='\033[1;34m' +PR_NC='\033[0m' +PR_DELAY_INIT=100 # ms + +# info() - Highlight test log pane, print message to it and to log file +# $@: Message to print +info() { + tmux select-pane -t ${PANE_INFO} + echo "${@}" >> /tmp/.passt_test_log_pipe + echo "${@}" >> "${LOGFILE}" +} + +# info_n() - Highlight, print message to pane and to log file without newline +# $@: Message to print +info_n() { + tmux select-pane -t ${PANE_INFO} + printf "${@}" >> /tmp/.passt_test_log_pipe + printf "${@}" >> "${LOGFILE}" +} + +# info_nolog() - Highlight test log pane, print message to it +# $@: Message to print +info_nolog() { + tmux select-pane -t ${PANE_INFO} + echo "${@}" >> /tmp/.passt_test_log_pipe +} + +# info_nolog() - Print message to log file +# $@: Message to print +log() { + echo "${@}" >> "${LOGFILE}" +} + +# info_nolog_n() - Send message to pane without highlighting it, without newline +# $@: Message to print +info_nolog_n() { + tmux send-keys -l -t ${PANE_INFO} "${@}" +} + +# info_sep() - Print given separator, horizontally filling test log pane +# $1: Separator character +info_sep() { + tmux send-keys -l -N ${STATUS_COLS} -t ${PANE_INFO} "${1}" + tmux send-keys -t ${PANE_INFO} C-m +} + +# sleep_char() - Sleep for typed characted resembling interactive input +# $1: Character typed to pane +sleep_char() { + [ ${FAST} -eq 1 ] && return + + if [ "${1}" = " " ]; then + PR_DELAY=$((PR_DELAY + 40)) + elif [ -n "$(printf '%s' "${1}" | tr -d [:alnum:])" ]; then + PR_DELAY=$((PR_DELAY + 30)) + elif [ ${PR_DELAY} -ge 30 ]; then + PR_DELAY=$((PR_DELAY / 3 * 2)) + fi + + sleep "$(printf 0.%03i ${PR_DELAY})" || sleep 1 +} + +# display_delay() - Simple delay, omitted if $FAST is set +display_delay() { + [ ${FAST} -eq 1 ] && return + + sleep "${1}" || sleep 1 +} + +# switch_pane() - Highlight given pane and reset character delay +# $1: Pane number +switch_pane() { + tmux select-pane -t ${1} + PR_DELAY=${PR_DELAY_INIT} + display_delay "0.2" +} + +# cmd_write() - Write a command to a pane, letter by letter, and execute it +# $1: Pane number +# $@: Command to issue +cmd_write() { + __pane_no=${1} + shift + + switch_pane ${__pane_no} + + __str="${@}" + while [ -n "${__str}" ]; do + __rem="${__str#?}" + __first="${__str%"$__rem"}" + if [ "${__first}" = ";" ]; then + tmux send-keys -t ${__pane_no} -l '\;' + else + tmux send-keys -t ${__pane_no} -l "${__first}" + fi + sleep_char "${__first}" + __str="${__rem}" + done + tmux send-keys -t ${__pane_no} "C-m" +} + +# text_write() - Write text to info pane, letter by letter +# $1: Pane number +# $@: Command to issue +text_write() { + __str="${@}" + while [ -n "${__str}" ]; do + __rem="${__str#?}" + __first="${__str%"$__rem"}" + if [ "${__first}" = ";" ]; then + tmux send-keys -t ${PANE_INFO} -l '\;' + else + tmux send-keys -t ${PANE_INFO} -l "${__first}" + fi + sleep_char "${__first}" + __str="${__rem}" + done +} + +# text_backspace() - Slow backspace motion for demo +# $1: Number of backspace characters +text_backspace() { + for __count in $(seq 0 ${1}); do + tmux send-keys -t ${PANE_INFO} Bspace + sleep 0.1 + done +} + +# em_write() - Write to log pane in red, for demo +# $@: Text +em_write() { + info_n "${PR_RED}${@}${PR_NC}" +} + +# pane_kill() - Kill a single pane given its name +# $1: Pane name +pane_kill() { + __pane_number=$(eval echo \$PANE_${1}) + tmux kill-pane -t ${__pane_number} +} + +# pane_highlight() - Highlight a single pane given its name +# $1: Pane name +pane_highlight() { + __pane_number=$(eval echo \$PANE_${1}) + switch_pane ${__pane_number} + sleep 3 +} + +# pane_run() - Issue a command in given pane name +# $1: Pane name +# $@: Command to issue +pane_run() { + __pane_name="${1}" + shift + + __pane_number=$(eval echo \$PANE_${__pane_name}) + + eval ${__pane_name}_LAST_CMD=\"\${@}\" + + cmd_write ${__pane_number} "${@}" +} + +# pane_wait() - Wait for command to be done in given pane name +# $1: Pane name +pane_wait() { + __pane_lc="$(echo "${1}" | tr [A-Z] [a-z])" + + while [ "$(tail -n1 ${LOGDIR}/pane_${__pane_lc}.log)" != '$ ' ] && \ + [ "$(tail -n1 ${LOGDIR}/pane_${__pane_lc}.log)" != '# ' ] && \ + [ "$(tail -n1 ${LOGDIR}/pane_${__pane_lc}.log)" != '# # ' ]; do + sleep 0.1 || sleep 1 + done +} + +# pane_parse() - Print last line, @EMPTY@ if command had no output +# $1: Pane name +pane_parse() { + __pane_lc="$(echo "${1}" | tr [A-Z] [a-z])" + + __buf="$(tail -n2 ${LOGDIR}/pane_${__pane_lc}.log | head -n1 | tr -d -c [:print:])" + + [ "# $(eval printf '%s' \"\$${1}_LAST_CMD\")" != "${__buf}" ] && \ + [ "$ $(eval printf '%s' \"\$${1}_LAST_CMD\")" != "${__buf}" ] && + printf '%s' "${__buf}" || printf '@EMPTY@' +} + +# status_file_end() - Display and log messages when tests from one file are done +status_file_end() { + [ -z "${STATUS_FILE}" ] && return + + info_sep "=" + log + tmux select-pane -t ${PANE_INFO} -T "" + STATUS_FILE= +} + +# status_file_start() - Display and log messages when tests from one file start +status_file_start() { + switch_pane ${PANE_INFO} + + status_file_end + + info_nolog "Starting tests in file: ${1}\n" + log "=== ${1}" + tmux select-pane -t ${PANE_INFO} -T "${1}" + + STATUS_FILE="${1}" + STATUS_FILE_NTESTS="${2}" + STATUS_FILE_INDEX=0 +} + +# status_file_start() - Display and log messages when a single test starts +status_test_start() { + switch_pane ${PANE_INFO} + + info_nolog "Starting test: ${1}" + log "> ${1}" + + STATUS_FILE_INDEX=$((STATUS_FILE_INDEX + 1)) + tmux select-pane -t ${PANE_INFO} -T "${STATUS_FILE} [${STATUS_FILE_INDEX}/${STATUS_FILE_NTESTS}] - ${1}" +} + +# info_check() - Display and log messages for a single test condition check +info_check() { + switch_pane ${PANE_INFO} + + printf "${PR_YELLOW}?${PR_NC} ${@}" >> /tmp/.passt_test_log_pipe + printf "? ${@}" >> "${LOGFILE}" +} + +# info_check_passed() - Display and log a new line when a check passes +info_check_passed() { + switch_pane ${PANE_INFO} + + printf "\n" >> /tmp/.passt_test_log_pipe + printf "\n" >> ${LOGFILE} +} + +# info_check_failed() - Display and log messages when a check fails +info_check_failed() { + switch_pane ${PANE_INFO} + + printf " ${PR_RED}!${PR_NC}\n" >> /tmp/.passt_test_log_pipe + printf " < failed.\n" >> "${LOGFILE}" +} + +# info_passed() - Display, log, and make status bar blink when a test passes +info_passed() { + switch_pane ${PANE_INFO} + + info_nolog "...${PR_GREEN}passed${PR_NC}.\n" + log "...passed." + log + + for i in `seq 1 3`; do + tmux set status-right-style 'bg=colour1 fg=colour2 bold' + sleep "0.1" + tmux set status-right-style 'bg=colour1 fg=colour233 bold' + sleep "0.1" + done +} + +# info_failed() - Display, log, and make status bar blink when a test passes +info_failed() { + switch_pane ${PANE_INFO} + + info_nolog "...${PR_RED}failed${PR_NC}.\n" + log "...failed." + log + + for i in `seq 1 3`; do + tmux set status-right-style 'bg=colour1 fg=colour196 bold' + sleep "0.1" + tmux set status-right-style 'bg=colour1 fg=colour233 bold' + sleep "0.1" + done + + pause_continue \ + "Press any key to pause test session" \ + "Resuming in " \ + "Paused, press any key to continue" \ + 5 +} + +# info_skipped() - Display and log skipped test +info_skipped() { + switch_pane ${PANE_INFO} + + info_nolog "...${PR_YELLOW}skipped${PR_NC}.\n" + log "...skipped." + log +} + +# info_layout() - Display string for new test layout +info_layout() { + switch_pane ${PANE_INFO} + + info_nolog "Test layout: ${PR_BLUE}${@}${PR_NC}.\n" +} + +# status_test_ok() - Update counter of passed tests, log and display message +status_test_ok() { + STATUS_PASS=$((STATUS_PASS + 1)) + tmux set status-right "PASS: ${STATUS_PASS} | FAIL: ${STATUS_FAIL} | #(TZ="UTC" date -Iseconds)" + info_passed +} + +# status_test_fail() - Update counter of failed tests, log and display message +status_test_fail() { + STATUS_FAIL=$((STATUS_FAIL + 1)) + tmux set status-right "PASS: ${STATUS_PASS} | FAIL: ${STATUS_FAIL} | #(TZ="UTC" date -Iseconds)" + info_failed +} + +# status_test_fail() - Update counter of failed tests, log and display message +status_test_skip() { + info_skipped +} + +# table_header() - Print table header to log pane +# $1: Header description +# $@: Column headers +table_header() { + perf_th ${@} + + __ifs="${IFS}" + IFS=" " + + __desc="${1}" + shift + + __max_len=4 + __count=0 + for __h in ${@}; do + [ ${#__h} -gt ${__max_len} ] && __max_len=${#__h} + __count=$((__count + 1)) + done + + # > xxxx |< + __outer_len=$((__max_len + 3)) + __width_fields=$((__outer_len * __count + 1)) + + TABLE_HEADER_LEFT=$((STATUS_COLS - __width_fields)) + TABLE_CELL_SIZE=$((__max_len + 2)) + TABLE_COLS=${__count} + + __pad_left=$((TABLE_HEADER_LEFT - ${#__desc} - 2)) + __buf="$(printf %-${__pad_left}s%s "" "${__desc}: ")" + for __h in ${@}; do + __pad_left=$(( (TABLE_CELL_SIZE - ${#__h} + 1) / 2)) + __pad_right=$(( (TABLE_CELL_SIZE - ${#__h}) / 2)) + __buf="${__buf}$(printf "|%-${__pad_left}s%s%-${__pad_right}s" "" ${__h} "")" + done + + info_n "${__buf}|" + + IFS="${__ifs}" +} + +# table_row() - Print main table row to log pane +# $@: Column headers +table_row() { + perf_tr ${@} + + __line="${@}" + __buf="$(printf %-${TABLE_HEADER_LEFT}s "")" + for __i in $(seq 1 ${TABLE_COLS}); do + __buf="${__buf}|" + for __j in $(seq 1 ${TABLE_CELL_SIZE}); do + __buf="${__buf}-" + done + done + info_n "\n${__buf}|\n" + + __pad_left=$(( (TABLE_HEADER_LEFT - ${#__line} + 1) / 2)) + __pad_right=$(( (TABLE_HEADER_LEFT - ${#__line}) / 2)) + info_n "$(printf "%-${__pad_left}s%s%-${__pad_right}s|" "" "${__line}" "")" +} + +# table_line() - Print simple line to log pane +# $@: Column headers +table_line() { + perf_tr ${@} + + __line="${@}" + info_n "\n" + + __pad_left=$(( (TABLE_HEADER_LEFT - ${#__line} + 1) / 2)) + __pad_right=$(( (TABLE_HEADER_LEFT - ${#__line}) / 2)) + info_n "$(printf "%-${__pad_left}s%s%-${__pad_right}s|" "" "${__line}" "")" +} + +table_cell() { + __len="${1}" + shift + + __content="${@}" + + __pad_left=$((TABLE_CELL_SIZE - __len - 1)) + info_n "$(printf "%-${__pad_left}s%s |" "" "${__content}")" +} + +table_end() { + __buf="$(printf %-${TABLE_HEADER_LEFT}s "")" + for __i in $(seq 1 ${TABLE_COLS}); do + __buf="${__buf}'" + for __j in $(seq 1 ${TABLE_CELL_SIZE}); do + __buf="${__buf}-" + done + done + info_n "\n${__buf}'\n" +} + +table_value_throughput() { + [ "${1}" = "-" ] && table_cell 1 "-" && perf_td 0 "" && return 0 + __v="$(echo "scale=1; x=( ${1} + 10^8 / 2 ) / 10^9; if ( x < 1 && x > 0 ) print 0; x" | bc -l)" + perf_td 31 "${__v}" + + __red="${2}" + __yellow="${3}" + if [ "$(echo "${__v} < ${__red}" | bc -l)" = "1" ]; then + table_cell ${#__v} "${PR_RED}${__v}${PR_NC}" + return 1 + elif [ "$(echo "${__v} < ${__yellow}" | bc -l)" = "1" ]; then + table_cell ${#__v} "${PR_YELLOW}${__v}${PR_NC}" + return 1 + else + table_cell ${#__v} "${PR_GREEN}${__v}${PR_NC}" + return 0 + fi +} + +table_value_latency() { + [ "${1}" = "-" ] && table_cell 1 "-" && perf_td 0 "" && return 0 + + __v="$(echo "scale=6; 1 / ${1} * 10^6" | bc -l)" + __v="${__v%.*}" + + perf_td 11 "${__v}" + + __red="${2}" + __yellow="${3}" + if [ "$(echo "${__v} > ${__red}" | bc -l)" = "1" ]; then + table_cell ${#__v} "${PR_RED}${__v}${PR_NC}" + return 1 + elif [ "$(echo "${__v} > ${__yellow}" | bc -l)" = "1" ]; then + table_cell ${#__v} "${PR_YELLOW}${__v}${PR_NC}" + return 1 + else + table_cell ${#__v} "${PR_GREEN}${__v}${PR_NC}" + return 0 + fi +} + +# pause_continue() - Pause for a while, wait for keystroke, resume on second one +pause_continue() { + tmux select-pane -t ${PANE_INFO} + info_nolog "${1}" + info_nolog_n "${2}" + + __pause_tmp="$(mktemp)" + echo >> "${__pause_tmp}" + tmux pipe-pane -O -t ${PANE_INFO} "cat >> ${__pause_tmp}" + __pane_buf= + __wait=0 + sleep 1 + for __i in $(seq ${4} -1 0); do + if [ "$(tail -n1 ${__pause_tmp} | tr -d -c [:print:])" != "${__pane_buf}" ]; then + __wait=1 + break + fi + + if [ ${__i} -ne ${4} ]; then + tmux send-keys -t ${PANE_INFO} Bspace + tmux send-keys -t ${PANE_INFO} Bspace + __pane_buf="${__pane_buf} " + fi + info_nolog_n "${__i} " + __pane_buf="${__pane_buf}${__i} " + sleep 1 + done + + if [ ${__wait} -eq 1 ]; then + tmux send-keys -t ${PANE_INFO} Bspace + tmux send-keys -t ${PANE_INFO} Bspace + info_nolog "" + info_nolog "${3}" + __pane_buf="$(tail -n1 ${__pause_tmp})" + while true; do + [ "$(tail -n1 ${__pause_tmp})" != "${__pane_buf}" ] && break + sleep 1 + done + fi + tmux pipe-pane -O -t ${PANE_INFO} "" + rm "${__pause_tmp}" + info_nolog "" +} + +# run_term() - Start tmux session, X terminal if requested, running entry point +run_term() { + export SHELL="/bin/sh" + + if [ ${CI} -eq 1 ]; then + __xterm_done="$(mktemp)" + ${CI_XTERM} "$(pwd)" -e "sh -c \"printf '\e[8;50;240t'; tmux new-session -s passt_test ./ci from_term; echo >${__xterm_done}\"" + while ! [ -s "${__xterm_done}" ]; do sleep 1; done + rm "${__xterm_done}" + elif [ ${DEMO} -eq 1 ]; then + while true; do + ${DEMO_XTERM} "$(pwd)" -e sh -c 'tmux new-session -s passt_test ./run_demo from_term' + [ $? -ne 0 ] && { tmux kill-session -t passt_test; continue; } + break + done + else + tmux new-session -s passt_test ./run from_term + fi +} + +# term() - Set up terminal window and panes for regular tests or CI +term() { + tmux set status-interval 1 + tmux rename-window '' + + tmux set window-status-format '#W' + tmux set window-status-current-format '#W' + tmux set status-left '' + tmux set window-status-separator '' + + tmux set window-status-style 'bg=colour1 fg=colour233 bold' + tmux set status-style 'bg=colour1 fg=colour233 bold' + tmux set status-right-style 'bg=colour1 fg=colour233 bold' + + tmux new-window -n "Testing commit: ${COMMIT}" + + tmux set window-status-format '#W' + tmux set window-status-current-format '#W' + tmux set status-left '' + tmux set window-status-separator '' + + tmux set window-status-current-style 'bg=colour1 fg=colour233 bold' + tmux set status-right '#(TZ="UTC" date -Iseconds)' + tmux set status-right-length 50 + tmux set status-right-style 'bg=colour1 fg=colour233 bold' + + tmux set history-limit 500000 + tmux select-pane -t 0 -T '' + tmux set pane-border-format '#T' + tmux set pane-border-style 'fg=colour2 bg=colour233' + tmux set pane-active-border-style 'fg=colour233 bg=colour4 bold' + tmux set pane-border-status bottom +} + +# term_demo() - Set up terminal window and panes for demo +term_demo() { + tmux set status-interval 1 + tmux rename-window '' + + tmux set window-status-format '#W' + tmux set window-status-current-format '#W' + tmux set status-left '' + tmux set window-status-separator '' + + tmux set window-status-style 'bg=colour1 fg=colour15 bold' + tmux set status-right '' + tmux set status-style 'bg=colour1 fg=colour15 bold' + tmux set status-right-style 'bg=colour1 fg=colour15 bold' + + tmux new-window -n "Demo at commit: ${COMMIT}" + + tmux set window-status-format '#W' + tmux set window-status-current-format '#W' + tmux set status-left '' + tmux set window-status-separator '' + + tmux select-pane -t 0 -T '' + tmux set pane-border-format '#T' + tmux set pane-border-style 'fg=colour2 bg=colour233' + tmux set pane-active-border-style 'fg=colour15 bg=colour4 bold' + tmux set pane-border-status bottom +} diff --git a/test/lib/test b/test/lib/test new file mode 100755 index 0000000..2e3f4ba --- /dev/null +++ b/test/lib/test @@ -0,0 +1,378 @@ +#!/bin/sh +# +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# PASST - Plug A Simple Socket Transport +# for qemu/UNIX domain socket mode +# +# PASTA - Pack A Subtle Tap Abstraction +# for network namespace/tap device mode +# +# test/lib/test - List tests and run them, evaluating directives from files +# +# Copyright (c) 2021 Red Hat GmbH +# Author: Stefano Brivio + +# Empty, 'passt' or 'pasta', to match against 'onlyfor' directive +MODE= + +# test_iperf3() - Ugly helper for iperf3c/iperf3s directives +# $1: Role: client or server +# $2: Pane name, can be lowercase +# $3: Destination name or address for client +# $4: Port number, ${i} is translated to process index +# $5: Number of processes to run in parallel +# $@: Options +test_iperf3() { + __role="${1}"; shift + __pane="$(echo "${1}" | tr [a-z] [A-Z])"; shift + [ "${__role}" = "client" ] && __dest="${1}" && shift || __dest="" + __port="${1}"; shift + __procs="$((${1} - 1))"; shift + + [ "${__role}" = "server" ] && __role_opt="-c" || __role_opt="-s1J" + + if [ ${__role} = "client" ]; then + UDP_CLIENT=0 + for __opt in ${@}; do + [ "${__opt}" = "-u" ] && UDP_CLIENT=1 + done + + ( + sleep 2 + pane_run "${__pane}" 'for i in $(seq 0 '${__procs}');' \ + 'do ( iperf3 -c '"${__dest}"' -p '"${__port}" \ + "${@}" ' -T s${i} & echo $! > c${i}.pid & ); done' + sleep 36 + pane_run "${__pane}" 'for i in $(seq 0 '${__procs}'); do'\ + 'kill -INT $(cat c${i}.pid) 2>/dev/null; done' + ) & + return + fi + + pane_run "${__pane}" 'for i in $(seq 0 '${__procs}'); do' \ + ':> s${i}.bw; done' + pane_wait "${__pane}" + + if [ ${UDP_CLIENT} -eq 0 ]; then + pane_run "${__pane}" 'for i in $(seq 0 '${__procs}');' \ + 'do ( ( iperf3 -s1J -p '"${__port} ${@}" \ + '& echo $! > s${i}.pid ) 2>/dev/null' \ + '| jq -rM ".end.sum_received.bits_per_second"' \ + '> s${i}.bw & );' \ + 'done' + else + pane_run "${__pane}" 'for i in $(seq 0 '${__procs}');' \ + 'do ( ( iperf3 -s1J -i 30 -p '"${__port} ${@}" \ + '& echo $! > s${i}.pid ) 2>/dev/null' \ + '| jq -rM ".intervals[0].sum.bits_per_second"' \ + '> s${i}.bw & );' \ + 'done' + fi + + pane_wait "${__pane}" + sleep 38 + pane_run "${__pane}" 'for i in $(seq 0 '${__procs}'); do' \ + 'kill -INT $(cat s${i}.pid) 2>/dev/null; done' + sleep 1 + pane_wait "${__pane}" + pane_run "${__pane}" '(cat s*.bw |' \ + 'sed '"'"'s/\(.*\)/\1\+/g'"'"' |' \ + 'tr -d "\n"; echo 0) | bc -l' + pane_wait "${__pane}" + pane_parse "${__pane}" + pane_run "${__pane}" 'for i in $(seq 0 '${__procs}'); do' \ + 'rm -f s${i}.bw; done' + pane_wait "${__pane}" +} + +# test_one() - Run a single test file evaluating directives +# $1: Name of test file, relative to test/ directory +test_one() { + __dirclean= + __test_file="test/${1}" + + __type="$(file -b --mime-type ${__test_file})" + if [ "${__type}" = "text/x-shellscript" ]; then + status_file_start "${1}" 1 + "${__test_file}" && status_test_ok || status_test_fail + return + fi + + __ntests="$(grep -c "^test$(printf '\t')" "${__test_file}")" + [ ${DEMO} -eq 0 ] && status_file_start "${1}" "${__ntests}" + + [ ${CI} -eq 1 ] && video_link "${1}" + + __subs= + __nok=-1 + __perf_nok=0 + __skip=0 + while IFS= read -r __line; do + # Strip comments + __line="${__line%%#*}" + + # tab-split command and arguments, apply variable substitutions + __cmd="${__line%%$(printf '\t')*}" + __arg="${__line#*$(printf '\t')*}" + __arg="$(subs_apply "${__subs}" "${__arg}")" + + [ ${__nok} -eq 1 ] && [ "${__cmd}" != "test" ] && continue + case ${__cmd} in + "tempdir") + __tmpdir="$(mktemp -d)" + __subs="$(list_add_pair "${__subs}" "__${__arg}__" "${__tmpdir}")" + __dirclean="$(list_add "${__dirclean}" "${__tmpdir}")" + ;; + "temp") + __tmpfile="$(mktemp)" + __subs="$(list_add_pair "${__subs}" "__${__arg}__" "${__tmpfile}")" + __dirclean="$(list_add "${__dirclean}" "${__tmpfile}")" + ;; + "test") + [ ${__perf_nok} -eq 0 ] || __nok=1 + [ ${__nok} -eq 1 ] && status_test_fail + [ ${__nok} -eq 0 ] && status_test_ok + + status_test_start "${__arg}" + __nok=0 + __perf_nok=0 + ;; + "host") + pane_run HOST "${__arg}" + pane_wait HOST + ;; + "hostb") + pane_run HOST "${__arg}" + ;; + "hostw") + pane_wait HOST + ;; + "htools") + pane_run HOST 'which '"${__arg}"' >/dev/null || echo skip' + pane_wait HOST + [ "$(pane_parse HOST)" = "skip" ] && { __skip=1; break; } + ;; + "passt") + pane_run PASST "${__arg}" + pane_wait PASST + ;; + "passtb") + pane_run PASST "${__arg}" + ;; + "passtw") + pane_wait PASST + ;; + "pout") + __varname="${__arg%% *}" + pane_run PASST "${__arg#* }" + pane_wait PASST + __subs="$(list_add_pair "${__subs}" "__${__varname}__" "$(pane_parse PASST)")" + ;; + "guest") + pane_run GUEST "${__arg}" + pane_wait GUEST + ;; + "guestb") + pane_run GUEST "${__arg}" + ;; + "guestw") + pane_wait GUEST + ;; + "guest1") + pane_run GUEST_1 "${__arg}" + pane_wait GUEST_1 + ;; + "guest1b") + pane_run GUEST_1 "${__arg}" + ;; + "guest1w") + pane_wait GUEST_1 + ;; + "gtools") + pane_run GUEST 'which '"${__arg}"' >/dev/null || echo skip' + pane_wait GUEST + [ "$(pane_parse GUEST)" = "skip" ] && { __skip=1; break; } + ;; + "g1tools") + pane_run GUEST_1 'which '"${__arg}"' >/dev/null || echo skip' + pane_wait GUEST_1 + [ "$(pane_parse GUEST_1)" = "skip" ] && { __skip=1; break; } + ;; + "g2tools") + pane_run GUEST_2 'which '"${__arg}"' >/dev/null || echo skip' + pane_wait GUEST_2 + [ "$(pane_parse GUEST_2)" = "skip" ] && { __skip=1; break; } + ;; + "guest2") + pane_run GUEST_2 "${__arg}" + pane_wait GUEST_2 + ;; + "guest2b") + pane_run GUEST_2 "${__arg}" + ;; + "guest2w") + pane_wait GUEST_2 + ;; + "ns") + pane_run NS "${__arg}" + pane_wait NS + ;; + "nsb") + pane_run NS "${__arg}" + ;; + "nsw") + pane_wait NS + ;; + "nstools") + pane_run NS 'which '"${__arg}"' >/dev/null || echo skip' + pane_wait NS + [ "$(pane_parse NS)" = "skip" ] && { __skip=1; break; } + ;; + "gout") + __varname="${__arg%% *}" + pane_run GUEST "${__arg#* }" + pane_wait GUEST + __subs="$(list_add_pair "${__subs}" "__${__varname}__" "$(pane_parse GUEST)")" + ;; + "g1out") + __varname="${__arg%% *}" + pane_run GUEST_1 "${__arg#* }" + pane_wait GUEST_1 + __subs="$(list_add_pair "${__subs}" "__${__varname}__" "$(pane_parse GUEST_1)")" + ;; + "g2out") + __varname="${__arg%% *}" + pane_run GUEST_2 "${__arg#* }" + pane_wait GUEST_2 + __subs="$(list_add_pair "${__subs}" "__${__varname}__" "$(pane_parse GUEST_2)")" + ;; + "hout") + __varname="${__arg%% *}" + pane_run HOST "${__arg#* }" + pane_wait HOST + __subs="$(list_add_pair "${__subs}" "__${__varname}__" "$(pane_parse HOST)")" + ;; + "nsout") + __varname="${__arg%% *}" + pane_run NS "${__arg#* }" + pane_wait NS + __subs="$(list_add_pair "${__subs}" "__${__varname}__" "$(pane_parse NS)")" + ;; + "check") + info_check "${__arg}" + eval "${__arg} || __nok=1" + if [ ${__nok} -eq 1 ]; then + info_check_failed + else + info_check_passed + fi + ;; + "sleep") + sleep "${__arg}" + ;; + "info") + info "${__arg}" + ;; + "report") + perf_report ${__arg} + ;; + "th") + table_header ${__arg} + ;; + "tr") + table_row "${__arg}" + ;; + "tl") + table_line "${__arg}" + ;; + "te") + table_end + ;; + "bw") + table_value_throughput ${__arg} || __perf_nok=1 + ;; + "lat") + table_value_latency ${__arg} || __perf_nok=1 + ;; + "iperf3c") + set -x + test_iperf3 client ${__arg} + set +x + ;; + "iperf3s") + set -x + __subs="$(list_add_pair "${__subs}" "__${__arg%% *}__" "$(test_iperf3 server ${__arg#* })" )" + set +x + ;; + "set") + __subs="$(list_add_pair "${__subs}" "__${__arg%% *}__" "${__arg#* }")" + ;; + + # Demo commands + "say") + text_write "${__arg}" + ;; + "em") + em_write "${__arg}" + ;; + "nl") + info_nolog "" + ;; + "hl") + pane_highlight "${__arg}" + ;; + "bsp") + text_backspace "${__arg}" + ;; + "killp") + pane_kill "${__arg}" + ;; + esac + done < "${__test_file}" + + for __d in ${__dirclean}; do + rm -rf ${__d} + done + + [ ${DEMO} -eq 1 ] && return + + [ ${__skip} -eq 1 ] && status_test_skip && return + [ ${__perf_nok} -eq 0 ] || __nok=1 + [ ${__nok} -eq 0 ] && status_test_ok || status_test_fail +} + +# test() - Build list of tests to run, in order, then issue test_one() +# $1: Name of directory containing set of test files, relative to test/ +test() { + __list= + __rem=1 + + cd test + while [ ${__rem} -eq 1 ]; do + __rem=0 + for __f in "${1}"/*; do + __type="$(file -b --mime-type ${__f})" + if [ "${__type}" = "text/x-shellscript" ]; then + __list="$(list_add "${__list}" "${__f}")" + continue + fi + + if [ -n "$(file_def "${__f}" onlyfor)" ] && \ + ! list_has "$(file_def "${__f}" onlyfor)" "${MODE}"; then + continue + fi + + if list_has_all "${__list}" "$(file_def "${__f}" req)"; then + __list="$(list_add "${__list}" "${__f}")" + else + __rem=1 + fi + done + done + cd .. + + for __f in ${__list}; do + test_one "${__f}" + done +} diff --git a/test/lib/util b/test/lib/util new file mode 100755 index 0000000..52ae171 --- /dev/null +++ b/test/lib/util @@ -0,0 +1,142 @@ +#!/bin/sh +# +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# PASST - Plug A Simple Socket Transport +# for qemu/UNIX domain socket mode +# +# PASTA - Pack A Subtle Tap Abstraction +# for network namespace/tap device mode +# +# test/lib/util - Convenience functions +# +# Copyright (c) 2021 Red Hat GmbH +# Author: Stefano Brivio + +# list_has() - Check whether a tab-separated list contains a given token +# $1: List +# $2: Token +# Return: 0 if token was found or is empty, 1 otherwise +list_has() { + [ -z "${2}" ] && return 0 + + __ifs="${IFS}" + IFS=' ' + for __t in ${1}; do + [ "${__t}" = "${2}" ] && IFS="${__ifs}" && return 0 + done + + IFS="${__ifs}" + return 1 +} + +# list_add() - Add token to tab-separated list, unless it's already present +# $1: List +# $2: Token +list_add() { + list_has "${1}" "${2}" && return + [ -n "${1}" ] && printf '%s\t%s\n' "${1}" "${2}" || printf '%s\n' "${2}" +} + +# list_remove_pair() - Drop pair with given key if present +# $1: List +# $2: Key +list_remove_pair() +{ + __ifs="${IFS}" + IFS=' ' + __skip_next=0 + for __t in ${1}; do + [ ${__skip_next} -eq 1 ] && __skip_next=0 && continue + [ "${__t}" = "${2}" ] && __skip_next=1 && continue + printf '%s\t' "${__t}" + done + printf "\n" + IFS="${__ifs}" +} + +# list_add_pair() - Add token pair to list, replace if the first one is present +# $1: List +# $2: First token +# $3: Second token +list_add_pair() { + [ -z "${3}" ] && return + + + if [ -n "${1}" ]; then + __new_list="$(list_remove_pair "${1}" "${2}")" + printf '%s\t%s\t%s' "${__new_list}" "${2}" "${3}" + else + printf '%s\t%s' "${2}" "${3}" + fi + printf "\n" +} + +# list_has_all() - Check whether a list contains all given IFS-separated tokens +# $1: List +# $2: List of tokens +# Return: 0 if list of tokens was found or is empty, 1 otherwise +list_has_all() { + [ -z "${2}" ] && return 0 + + for __i in ${2}; do + list_has "${1}" "${__i}" || return 1 + done + return 0 +} + +# file_def() - List of tokens tab-separated line from file, starting with key +# $1: Filename +# $2: Token +file_def() { + sed -n 's/^'"${2}"'\t\(.*\)/\1/p' "${1}" | tr ' ' '\t' +} + +# subs_apply() - Apply substitutions using a list of token pairs +# $1: List of substitutions +# $2: String where substitutions have to be applied +subs_apply() { + echo "in subs_apply" >> /tmp/subs_apply + + __ifs="${IFS}" + IFS=' ' + __newarg="${2}" + __s= + for __t in ${1}; do + [ -z "${__s}" ] && __s="${__t}" && continue + + echo "t: --${__t}--, --${__s}--" >> /tmp/subs_apply + + __et="$(printf '%s\n' "$__t" | sed -e 's/[\/&]/\\&/g')" + __es="$(printf '%s\n' "$__s" | sed -e 's/[]\/$*.^[]/\\&/g')" + + __newarg="$(printf '%s' "${__newarg}" | sed "s/${__es}/${__et}/g")" + __s= + done + + printf '%s' "${__newarg}" + IFS="${__ifs}" +} + +# set_mode() - Set 'passt' or 'pasta' mode for terminal control, renaming panes +# $1: Mode to be set +set_mode() { + MODE="${1}" + if [ "${1}" = "pasta" ]; then + tmux select-pane -t ${PANE_GUEST} -T "namespace" + tmux select-pane -t ${PANE_PASST} -T "pasta" + else + tmux select-pane -t ${PANE_GUEST} -T "guest" + tmux select-pane -t ${PANE_PASST} -T "passt" + fi +} + +# get_info_cols() - Get number of columns for info pane +get_info_cols() { + __log_pane_cols= + __j=0 + for __i in $(tmux list-panes -t passt_test:1.0 -F "#{pane_width}"); do + [ ${__j} -eq ${PANE_INFO} ] && STATUS_COLS=${__i} && break + __j=$((__j + 1)) + done +} diff --git a/test/lib/video b/test/lib/video new file mode 100755 index 0000000..fd5849b --- /dev/null +++ b/test/lib/video @@ -0,0 +1,119 @@ +#!/bin/sh +# +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# PASST - Plug A Simple Socket Transport +# for qemu/UNIX domain socket mode +# +# PASTA - Pack A Subtle Tap Abstraction +# for network namespace/tap device mode +# +# test/lib/video - Video grabbing, JavaScript fragments with links +# +# Copyright (c) 2021 Red Hat GmbH +# Author: Stefano Brivio + +FFMPEG_PID_FILE="$(mktemp)" +VIDEO_START_SECONDS= +VIDEO_NAME= + +VIDEO_LINKS_TEMPLATE="document.write('"' + Skip to: +' + +VIDEO_LINKS_TEMPLATE_JS=" +'); + +var video___VIDEO_NAME__links = [ +" + +VIDEO_LINKS_TEMPLATE_POST=']; + +for (var i = 0; i < video___VIDEO_NAME__links.length; i++) { + var obj = document.getElementById(video___VIDEO_NAME__links[i][0]); + + obj.addEventListener("click", function(event) { + var __VIDEO_NAME___video = document.getElementById("__VIDEO_NAME___video"); + var top = __VIDEO_NAME___video.offsetTop - 5; + + event.preventDefault(); + __VIDEO_NAME___video.play(); + __VIDEO_NAME___video.pause(); + for (var i = 0; i < video___VIDEO_NAME__links.length; i++) { + if (this.id == video___VIDEO_NAME__links[i][0]) { + __VIDEO_NAME___video.currentTime = video___VIDEO_NAME__links[i][1]; + } + } + window.scrollTo({ top: top, behavior: "smooth" }) + __VIDEO_NAME___video.play(); + }, false); +} +' + +VIDEO_LINKS_BUF= +VIDEO_LINKS_COUNT=0 + +# video_append_links() - Append generic string to JavaScript links file +video_append_links() +{ + printf "${@}" >> "${BASEPATH}/${VIDEO_NAME}.js" +} + +# video_append_links() - Append generic string to buffer for links +video_append_links_js() +{ + VIDEO_LINKS_BUF="${VIDEO_LINKS_BUF}${@}" +} + +# video_grab() - Fetch window geometry, start grabbing video +video_grab() { + VIDEO_NAME="${1}" + + rm -f "${BASEPATH}/${VIDEO_NAME}.mp4" "${BASEPATH}/${VIDEO_NAME}.webm" "${BASEPATH}/${VIDEO_NAME}.js" + + echo "${VIDEO_LINKS_TEMPLATE}" > "${BASEPATH}/${VIDEO_NAME}.js" + + __x=$(xwininfo -id $(xdotool getactivewindow) | sed -n 's/[ ]*Absolute upper-left X:[ ]*\([0-9]*\)$/\1/p') + __y=$(xwininfo -id $(xdotool getactivewindow) | sed -n 's/[ ]*Absolute upper-left Y:[ ]*\([0-9]*\)$/\1/p') + __width=$(xwininfo -id $(xdotool getactivewindow) | sed -n 's/[ ]*Width:[ ]*\([0-9]*\)$/\1/p') + __height=$(xwininfo -id $(xdotool getactivewindow) | sed -n 's/[ ]*Height:[ ]*\([0-9]*\)$/\1/p') + + [ $((__width % 2)) ] && __width=$((__width - 1)) + [ $((__height % 2)) ] && __height=$((__height - 1)) + + sleep 3 + VIDEO_START_SECONDS=$(sed -n 's/\([0-9]*\).[0-9]* [0-9]*.[0-9]*/\1/p' /proc/uptime) + ffmpeg -f x11grab -framerate 15 -video_size "${__width}x${__height}" -i "+${__x},${__y}" -vcodec libx264 -preset ultrafast -qp 0 -pix_fmt yuv444p -draw_mouse 0 "${BASEPATH}/${VIDEO_NAME}.mp4" & echo $! > "${FFMPEG_PID_FILE}" +} + +# video_time_now() - Print current video timestamp, in seconds +video_time_now() { + __now=$(sed -n 's/\([0-9]*\).[0-9]* [0-9]*.[0-9]*/\1/p' /proc/uptime) + echo $((__now - VIDEO_START_SECONDS)) +} + +# video_stop() - Stop grabbing, finalise JavaScript templates, convert to webm +video_stop() { + sed -i 's/^.*$/&\\/g' "${BASEPATH}/${VIDEO_NAME}.js" + echo "${VIDEO_LINKS_TEMPLATE_JS}" | sed "s/__VIDEO_NAME__/${VIDEO_NAME}/g" >> "${BASEPATH}/${VIDEO_NAME}.js" + echo "${VIDEO_LINKS_BUF}" >> "${BASEPATH}/${VIDEO_NAME}.js" + echo "${VIDEO_LINKS_TEMPLATE_POST}" | sed "s/__VIDEO_NAME__/${VIDEO_NAME}/g" >> "${BASEPATH}/${VIDEO_NAME}.js" + + kill -INT $(cat "${FFMPEG_PID_FILE}") + while ps -p $(cat "${FFMPEG_PID_FILE}") >/dev/null; do sleep 1; done + rm "${FFMPEG_PID_FILE}" + + [ ${1} -ne 0 ] && return + + ffmpeg -an -fflags +genpts -i "${BASEPATH}/${VIDEO_NAME}.mp4" -c:v libvpx-vp9 -row-mt 1 -minrate 10k -maxrate 200k -b:v 200k "${BASEPATH}/${VIDEO_NAME}.webm" +} + +# video_link() - Append single link to given video chapter +video_link() { + [ ${VIDEO_LINKS_COUNT} -eq 0 ] && __sep="" || __sep=" |" + __id="video_link_${VIDEO_LINKS_COUNT}" + video_append_links "${__sep} ${1}" + video_append_links_js "[ '${__id}', $(($(video_time_now) - 1)) ]," + + VIDEO_LINKS_COUNT=$((VIDEO_LINKS_COUNT + 1)) +} -- cgit v1.2.3