diff options
129 files changed, 11049 insertions, 6286 deletions
diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..78f177a --- /dev/null +++ b/.clang-format @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# clang-format configuration file. Intended for clang-format >= 11. +# +# For more information, see: +# +# Documentation/dev-tools/clang-format.rst +# https://clang.llvm.org/docs/ClangFormat.html +# https://clang.llvm.org/docs/ClangFormatStyleOptions.html +# +--- +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: false +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: true + AfterNamespace: true + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakBeforeTernaryOperators: false +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeComma +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: false +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 8 +ContinuationIndentWidth: 8 +Cpp11BracedListStyle: false +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: false + +# Taken from: +# git grep -h '^#define [^[:space:]]*for_each[^[:space:]]*(' include/ tools/ \ +# | sed "s,^#define \([^[:space:]]*for_each[^[:space:]]*\)(.*$, - '\1'," \ +# | LC_ALL=C sort -u +ForEachMacros: + - 'for_each_nst' + +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '.*' + Priority: 1 +IncludeIsMainRegex: '(Test)?$' +IndentCaseLabels: false +IndentGotoLabels: false +IndentPPDirectives: None +IndentWidth: 8 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 8 +ObjCSpaceAfterProperty: true +ObjCSpaceBeforeProtocolList: true + +# Taken from git's rules +PenaltyBreakAssignment: 10 +PenaltyBreakBeforeFirstCallParameter: 30 +PenaltyBreakComment: 10 +PenaltyBreakFirstLessLess: 0 +PenaltyBreakString: 10 +PenaltyExcessCharacter: 100 +PenaltyReturnTypeOnItsOwnLine: 60 + +PointerAlignment: Right +ReflowComments: false +SortIncludes: false +SortUsingDeclarations: false +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatementsExceptForEachMacros +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: false +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp03 +TabWidth: 8 +UseTab: Always +... diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000..9d346ec --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,93 @@ +--- +Checks: + - "clang-diagnostic-*,clang-analyzer-*,*,-modernize-*" + + # TODO: enable once https://bugs.llvm.org/show_bug.cgi?id=41311 is fixed + - "-clang-analyzer-valist.Uninitialized" + + # Dubious value, would kill readability + - "-cppcoreguidelines-init-variables" + + # Dubious value over the compiler's built-in warning. Would + # increase verbosity. + - "-bugprone-assignment-in-if-condition" + + # Debatable whether these improve readability, right now it would look + # like a mess + - "-google-readability-braces-around-statements" + - "-hicpp-braces-around-statements" + - "-readability-braces-around-statements" + + # TODO: in most cases they are justified, but probably not everywhere + # + - "-readability-magic-numbers" + - "-cppcoreguidelines-avoid-magic-numbers" + + # TODO: this is Linux-only for the moment, nice to fix eventually + - "-llvmlibc-restrict-system-libc-headers" + + # Those are needed for syscalls, epoll_wait flags, etc. + - "-hicpp-signed-bitwise" + + # Probably not doable to impement this without plain memcpy(), memset() + - "-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling" + + # TODO: not really important, but nice to fix eventually + - "-llvm-include-order" + + # Dubious value, would kill readability + - "-readability-isolate-declaration" + + # TODO: nice to fix eventually + - "-bugprone-narrowing-conversions" + - "-cppcoreguidelines-narrowing-conversions" + + # TODO: check, fix, and more in general constify wherever possible + - "-cppcoreguidelines-avoid-non-const-global-variables" + + # TODO: check paths where it might make sense to improve performance + - "-altera-unroll-loops" + - "-altera-id-dependent-backward-branch" + + # Not much can be done about them other than being careful + - "-bugprone-easily-swappable-parameters" + + # TODO: split reported functions + - "-readability-function-cognitive-complexity" + + # "Poor" alignment needed for structs reflecting message formats/headers + - "-altera-struct-pack-align" + + # TODO: check again if multithreading is implemented + - "-concurrency-mt-unsafe" + + # Complains about any identifier <3 characters, reasonable for + # globals, pointlessly verbose for locals and parameters. + - "-readability-identifier-length" + + # Wants to include headers which *directly* provide the things + # we use. That sounds nice, but means it will often want a OS + # specific header instead of a mostly standard one, such as + # <linux/limits.h> instead of <limits.h>. + - "-misc-include-cleaner" + + # Want to replace all #defines of integers with enums. Kind of + # makes sense when those defines form an enum-like set, but + # weird for cases like standalone constants, and causes other + # awkwardness for a bunch of cases we use + - "-cppcoreguidelines-macro-to-enum" + + # It's been a couple of centuries since multiplication has been granted + # precedence over addition in modern mathematical notation. Adding + # parentheses to reinforce that certainly won't improve readability. + - "-readability-math-missing-parentheses" +WarningsAsErrors: "*" +HeaderFileExtensions: + - h +ImplementationFileExtensions: + - c +HeaderFilterRegex: "" +FormatStyle: none +CheckOptions: + bugprone-suspicious-string-compare.WarnOnImplicitComparison: "false" +SystemHeaders: false @@ -0,0 +1,3 @@ +CompileFlags: + # Don't try to interpret our headers as C++' + Add: [-xc, -Wall] @@ -15,66 +15,43 @@ VERSION ?= $(shell git describe --tags HEAD 2>/dev/null || echo "unknown\ versio # the IPv6 socket API? (Linux does) DUAL_STACK_SOCKETS := 1 -RLIMIT_STACK_VAL := $(shell /bin/sh -c 'ulimit -s') -ifeq ($(RLIMIT_STACK_VAL),unlimited) -RLIMIT_STACK_VAL := 1024 -endif - TARGET ?= $(shell $(CC) -dumpmachine) # Get 'uname -m'-like architecture description for target TARGET_ARCH := $(shell echo $(TARGET) | cut -f1 -d- | tr [A-Z] [a-z]) TARGET_ARCH := $(shell echo $(TARGET_ARCH) | sed 's/powerpc/ppc/') -AUDIT_ARCH := $(shell echo $(TARGET_ARCH) | tr [a-z] [A-Z] | sed 's/^ARM.*/ARM/') -AUDIT_ARCH := $(shell echo $(AUDIT_ARCH) | sed 's/I[456]86/I386/') -AUDIT_ARCH := $(shell echo $(AUDIT_ARCH) | sed 's/PPC64/PPC/') -AUDIT_ARCH := $(shell echo $(AUDIT_ARCH) | sed 's/PPCLE/PPC64LE/') -AUDIT_ARCH := $(shell echo $(AUDIT_ARCH) | sed 's/MIPS64EL/MIPSEL64/') -AUDIT_ARCH := $(shell echo $(AUDIT_ARCH) | sed 's/HPPA/PARISC/') -AUDIT_ARCH := $(shell echo $(AUDIT_ARCH) | sed 's/SH4/SH/') +# On some systems enabling optimization also enables source fortification, +# automagically. Do not override it. +FORTIFY_FLAG := +ifeq ($(shell $(CC) -O2 -dM -E - < /dev/null 2>&1 | grep ' _FORTIFY_SOURCE ' > /dev/null; echo $$?),1) +FORTIFY_FLAG := -D_FORTIFY_SOURCE=2 +endif FLAGS := -Wall -Wextra -Wno-format-zero-length FLAGS += -pedantic -std=c11 -D_XOPEN_SOURCE=700 -D_GNU_SOURCE -FLAGS += -D_FORTIFY_SOURCE=2 -O2 -pie -fPIE +FLAGS += $(FORTIFY_FLAG) -O2 -pie -fPIE FLAGS += -DPAGE_SIZE=$(shell getconf PAGE_SIZE) -FLAGS += -DNETNS_RUN_DIR=\"/run/netns\" -FLAGS += -DPASST_AUDIT_ARCH=AUDIT_ARCH_$(AUDIT_ARCH) -FLAGS += -DRLIMIT_STACK_VAL=$(RLIMIT_STACK_VAL) -FLAGS += -DARCH=\"$(TARGET_ARCH)\" FLAGS += -DVERSION=\"$(VERSION)\" FLAGS += -DDUAL_STACK_SOCKETS=$(DUAL_STACK_SOCKETS) PASST_SRCS = arch.c arp.c checksum.c conf.c dhcp.c dhcpv6.c flow.c fwd.c \ icmp.c igmp.c inany.c iov.c ip.c isolation.c lineread.c log.c mld.c \ ndp.c netlink.c packet.c passt.c pasta.c pcap.c pif.c tap.c tcp.c \ - tcp_buf.c tcp_splice.c tcp_vu.c udp.c udp_vu.c util.c vhost_user.c virtio.c + tcp_buf.c tcp_splice.c tcp_vu.c udp.c udp_flow.c udp_vu.c util.c \ + vhost_user.c virtio.c vu_common.c QRAP_SRCS = qrap.c SRCS = $(PASST_SRCS) $(QRAP_SRCS) MANPAGES = passt.1 pasta.1 qrap.1 PASST_HEADERS = arch.h arp.h checksum.h conf.h dhcp.h dhcpv6.h flow.h fwd.h \ - flow_table.h icmp.h inany.h iov.h ip.h isolation.h lineread.h log.h \ - ndp.h netlink.h packet.h passt.h pasta.h pcap.h pif.h siphash.h tap.h \ - tcp.h tcp_buf.h tcp_conn.h tcp_splice.h tcp_vu.h udp.h udp_internal.h \ - udp_vu.h util.h vhost_user.h virtio.h + flow_table.h icmp.h icmp_flow.h inany.h iov.h ip.h isolation.h \ + lineread.h log.h ndp.h netlink.h packet.h passt.h pasta.h pcap.h pif.h \ + siphash.h tap.h tcp.h tcp_buf.h tcp_conn.h tcp_internal.h tcp_splice.h \ + tcp_vu.h udp.h udp_flow.h udp_internal.h udp_vu.h util.h vhost_user.h \ + virtio.h vu_common.h HEADERS = $(PASST_HEADERS) seccomp.h -C := \#include <linux/tcp.h>\nstruct tcp_info x = { .tcpi_snd_wnd = 0 }; -ifeq ($(shell printf "$(C)" | $(CC) -S -xc - -o - >/dev/null 2>&1; echo $$?),0) - FLAGS += -DHAS_SND_WND -endif - -C := \#include <linux/tcp.h>\nstruct tcp_info x = { .tcpi_bytes_acked = 0 }; -ifeq ($(shell printf "$(C)" | $(CC) -S -xc - -o - >/dev/null 2>&1; echo $$?),0) - FLAGS += -DHAS_BYTES_ACKED -endif - -C := \#include <linux/tcp.h>\nstruct tcp_info x = { .tcpi_min_rtt = 0 }; -ifeq ($(shell printf "$(C)" | $(CC) -S -xc - -o - >/dev/null 2>&1; echo $$?),0) - FLAGS += -DHAS_MIN_RTT -endif - C := \#include <sys/random.h>\nint main(){int a=getrandom(0, 0, 0);} ifeq ($(shell printf "$(C)" | $(CC) -S -xc - -o - >/dev/null 2>&1; echo $$?),0) FLAGS += -DHAS_GETRANDOM @@ -84,11 +61,6 @@ ifeq ($(shell :|$(CC) -fstack-protector-strong -S -xc - -o - >/dev/null 2>&1; ec FLAGS += -fstack-protector-strong endif -C := \#define _GNU_SOURCE\n\#include <fcntl.h>\nint x = FALLOC_FL_COLLAPSE_RANGE; -ifeq ($(shell printf "$(C)" | $(CC) -S -xc - -o - >/dev/null 2>&1; echo $$?),0) - EXTRA_SYSCALLS += fallocate -endif - prefix ?= /usr/local exec_prefix ?= $(prefix) bindir ?= $(exec_prefix)/bin @@ -125,11 +97,12 @@ pasta.avx2 pasta.1 pasta: pasta%: passt% ln -sf $< $@ qrap: $(QRAP_SRCS) passt.h - $(CC) $(FLAGS) $(CFLAGS) $(CPPFLAGS) $(QRAP_SRCS) -o qrap $(LDFLAGS) + $(CC) $(FLAGS) $(CFLAGS) $(CPPFLAGS) -DARCH=\"$(TARGET_ARCH)\" $(QRAP_SRCS) -o qrap $(LDFLAGS) valgrind: EXTRA_SYSCALLS += rt_sigprocmask rt_sigtimedwait rt_sigaction \ - getpid gettid kill clock_gettime mmap \ - munmap open unlink gettimeofday futex + rt_sigreturn getpid gettid kill clock_gettime mmap \ + mmap2 munmap open unlink gettimeofday futex statx \ + readlink valgrind: FLAGS += -g -DVALGRIND valgrind: all @@ -189,111 +162,11 @@ docs: README.md done < README.md; \ ) > README.plain.md -# Checkers currently disabled for clang-tidy: -# - llvmlibc-restrict-system-libc-headers -# TODO: this is Linux-only for the moment, nice to fix eventually -# -# - bugprone-macro-parentheses -# - google-readability-braces-around-statements -# - hicpp-braces-around-statements -# - readability-braces-around-statements -# Debatable whether that improves readability, right now it would look -# like a mess -# -# - readability-magic-numbers -# - cppcoreguidelines-avoid-magic-numbers -# TODO: in most cases they are justified, but probably not everywhere -# -# - clang-analyzer-valist.Uninitialized -# TODO: enable once https://bugs.llvm.org/show_bug.cgi?id=41311 is fixed -# -# - clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling -# Probably not doable to impement this without plain memcpy(), memset() -# -# - cppcoreguidelines-init-variables -# Dubious value, would kill readability -# -# - hicpp-signed-bitwise -# Those are needed for syscalls, epoll_wait flags, etc. -# -# - llvm-include-order -# TODO: not really important, but nice to fix eventually -# -# - readability-isolate-declaration -# Dubious value, would kill readability -# -# - bugprone-narrowing-conversions -# - cppcoreguidelines-narrowing-conversions -# TODO: nice to fix eventually -# -# - cppcoreguidelines-avoid-non-const-global-variables -# TODO: check, fix, and more in general constify wherever possible -# -# - altera-unroll-loops -# - altera-id-dependent-backward-branch -# TODO: check paths where it might make sense to improve performance -# -# - bugprone-easily-swappable-parameters -# Not much can be done about them other than being careful -# -# - readability-function-cognitive-complexity -# TODO: split reported functions -# -# - altera-struct-pack-align -# "Poor" alignment needed for structs reflecting message formats/headers -# -# - concurrency-mt-unsafe -# TODO: check again if multithreading is implemented -# -# - readability-identifier-length -# Complains about any identifier <3 characters, reasonable for -# globals, pointlessly verbose for locals and parameters. -# -# - bugprone-assignment-in-if-condition -# Dubious value over the compiler's built-in warning. Would -# increase verbosity. -# -# - misc-include-cleaner -# Wants to include headers which *directly* provide the things -# we use. That sounds nice, but means it will often want a OS -# specific header instead of a mostly standard one, such as -# <linux/limits.h> instead of <limits.h>. +clang-tidy: $(PASST_SRCS) $(HEADERS) + clang-tidy $(PASST_SRCS) -- $(filter-out -pie,$(FLAGS) $(CFLAGS) $(CPPFLAGS)) \ + -DCLANG_TIDY_58992 -clang-tidy: $(SRCS) $(HEADERS) - clang-tidy -checks=*,-modernize-*,\ - -clang-analyzer-valist.Uninitialized,\ - -cppcoreguidelines-init-variables,\ - -bugprone-assignment-in-if-condition,\ - -bugprone-macro-parentheses,\ - -google-readability-braces-around-statements,\ - -hicpp-braces-around-statements,\ - -readability-braces-around-statements,\ - -readability-magic-numbers,\ - -llvmlibc-restrict-system-libc-headers,\ - -hicpp-signed-bitwise,\ - -clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling,\ - -llvm-include-order,\ - -cppcoreguidelines-avoid-magic-numbers,\ - -readability-isolate-declaration,\ - -bugprone-narrowing-conversions,\ - -cppcoreguidelines-narrowing-conversions,\ - -cppcoreguidelines-avoid-non-const-global-variables,\ - -altera-unroll-loops,-altera-id-dependent-backward-branch,\ - -bugprone-easily-swappable-parameters,\ - -readability-function-cognitive-complexity,\ - -altera-struct-pack-align,\ - -concurrency-mt-unsafe,\ - -readability-identifier-length,\ - -misc-include-cleaner \ - -config='{CheckOptions: [{key: bugprone-suspicious-string-compare.WarnOnImplicitComparison, value: "false"}]}' \ - --warnings-as-errors=* $(SRCS) -- $(filter-out -pie,$(FLAGS) $(CFLAGS) $(CPPFLAGS)) -DCLANG_TIDY_58992 - -SYSTEM_INCLUDES := /usr/include $(wildcard /usr/include/$(TARGET)) -ifeq ($(shell $(CC) -v 2>&1 | grep -c "gcc version"),1) -VER := $(shell $(CC) -dumpversion) -SYSTEM_INCLUDES += /usr/lib/gcc/$(TARGET)/$(VER)/include -endif -cppcheck: $(SRCS) $(HEADERS) +cppcheck: $(PASST_SRCS) $(HEADERS) if cppcheck --check-level=exhaustive /dev/null > /dev/null 2>&1; then \ CPPCHECK_EXHAUSTIVE="--check-level=exhaustive"; \ else \ @@ -302,11 +175,8 @@ cppcheck: $(SRCS) $(HEADERS) cppcheck --std=c11 --error-exitcode=1 --enable=all --force \ --inconclusive --library=posix --quiet \ $${CPPCHECK_EXHAUSTIVE} \ - $(SYSTEM_INCLUDES:%=-I%) \ - $(SYSTEM_INCLUDES:%=--config-exclude=%) \ - $(SYSTEM_INCLUDES:%=--suppress=*:%/*) \ - $(SYSTEM_INCLUDES:%=--suppress=unmatchedSuppression:%/*) \ --inline-suppr \ + --suppress=missingIncludeSystem \ --suppress=unusedStructMember \ - $(filter -D%,$(FLAGS) $(CFLAGS) $(CPPFLAGS)) \ - . + $(filter -D%,$(FLAGS) $(CFLAGS) $(CPPFLAGS)) -D CPPCHECK_6936 \ + $(PASST_SRCS) $(HEADERS) @@ -338,20 +338,24 @@ speeding up local connections, and usually requiring NAT. _pasta_: [_slirp4netns_ replacement](/passt/tree/slirp4netns.sh) * ✅ out-of-tree patch for [Kata Containers](/passt/tree/contrib/kata-containers) available -* ⌚ drop-in replacement for VPNKit (rootless Docker) +* ✅ rootless Docker + [network back-end](https://docs.docker.com/engine/security/rootless/#networking-errors) + via moby/rootlesskit ### Availability * official packages for: + * ✅ [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=passt) * ✅ [Arch Linux](https://archlinux.org/packages/extra/x86_64/passt/) ([aarch64](https://archlinuxarm.org/packages/aarch64/passt), [i486](https://www.archlinux32.org/packages/?q=passt)) * ✅ [CentOS Stream](https://gitlab.com/redhat/centos-stream/rpms/passt) * ✅ [Debian](https://tracker.debian.org/pkg/passt) * ✅ [Fedora](https://src.fedoraproject.org/rpms/passt) * ✅ [Gentoo](https://packages.gentoo.org/packages/net-misc/passt) + * ✅ [GNU Guix](https://packages.guix.gnu.org/packages/passt/) + * ✅ [OpenSUSE](https://build.opensuse.org/package/requests/Virtualization:containers/passt) * ✅ [Ubuntu](https://launchpad.net/ubuntu/+source/passt) * ✅ [Void Linux](https://voidlinux.org/packages/?q=passt) * unofficial packages for: * ✅ [EPEL, Mageia](https://copr.fedorainfracloud.org/coprs/sbrivio/passt/) - * 🛠[openSUSE](https://build.opensuse.org/package/show/Virtualization:containers/passt) * ✅ unofficial [packages](https://passt.top/builds/latest/x86_64/) from x86_64 static builds for other RPM-based distributions * ✅ unofficial [packages](https://passt.top/builds/latest/x86_64/) from x86_64 @@ -396,7 +400,7 @@ services: and nameserver using SLAAC * [DHCPv6 server](/passt/tree/dhcpv6.c): a simple implementation handing out one single IPv6 address to the guest or namespace, - namely, the the same address as the first one configured for the upstream host + namely, the same address as the first one configured for the upstream host interface, and passing the nameservers configured on the host ## Addresses @@ -18,6 +18,9 @@ #include <string.h> #include <unistd.h> +#include "log.h" +#include "util.h" + /** * arch_avx2_exec() - Switch to AVX2 build if supported * @argv: Arguments from command line @@ -28,10 +31,8 @@ void arch_avx2_exec(char **argv) char exe[PATH_MAX] = { 0 }; const char *p; - if (readlink("/proc/self/exe", exe, PATH_MAX - 1) < 0) { - perror("readlink /proc/self/exe"); - exit(EXIT_FAILURE); - } + if (readlink("/proc/self/exe", exe, PATH_MAX - 1) < 0) + die_perror("Failed to read own /proc/self/exe link"); p = strstr(exe, ".avx2"); if (p && strlen(p) == strlen(".avx2")) @@ -40,9 +41,12 @@ void arch_avx2_exec(char **argv) if (__builtin_cpu_supports("avx2")) { char new_path[PATH_MAX + sizeof(".avx2")]; - snprintf(new_path, PATH_MAX + sizeof(".avx2"), "%s.avx2", exe); - execve(new_path, argv, environ); - perror("Can't run AVX2 build, using non-AVX2 version"); + if (snprintf_check(new_path, PATH_MAX + sizeof(".avx2"), + "%s.avx2", exe)) + die_perror("Can't build AVX2 executable path"); + + execv(new_path, argv); + warn_perror("Can't run AVX2 build, using non-AVX2 version"); } } #else @@ -43,8 +43,7 @@ int arp(const struct ctx *c, const struct pool *p) struct ethhdr *eh; struct arphdr *ah; struct arpmsg *am; - size_t len; - int ret; + size_t l2len; eh = packet_get(p, 0, 0, sizeof(*eh), NULL); ah = packet_get(p, 0, sizeof(*eh), sizeof(*ah), NULL); @@ -60,31 +59,28 @@ int arp(const struct ctx *c, const struct pool *p) ah->ar_op != htons(ARPOP_REQUEST)) return 1; - /* Discard announcements (but not 0.0.0.0 "probes"): we might have the - * same IP address, hide that. - */ - if (memcmp(am->sip, (unsigned char[4]){ 0 }, sizeof(am->tip)) && + /* Discard announcements, but not 0.0.0.0 "probes" */ + if (memcmp(am->sip, &in4addr_any, sizeof(am->sip)) && !memcmp(am->sip, am->tip, sizeof(am->sip))) return 1; - /* Don't resolve our own address, either. */ + /* Don't resolve the guest's assigned address, either. */ if (!memcmp(am->tip, &c->ip4.addr, sizeof(am->tip))) return 1; ah->ar_op = htons(ARPOP_REPLY); memcpy(am->tha, am->sha, sizeof(am->tha)); - memcpy(am->sha, c->mac, sizeof(am->sha)); + memcpy(am->sha, c->our_tap_mac, sizeof(am->sha)); memcpy(swap, am->tip, sizeof(am->tip)); memcpy(am->tip, am->sip, sizeof(am->tip)); memcpy(am->sip, swap, sizeof(am->sip)); - len = sizeof(*eh) + sizeof(*ah) + sizeof(*am); + l2len = sizeof(*eh) + sizeof(*ah) + sizeof(*am); memcpy(eh->h_dest, eh->h_source, sizeof(eh->h_dest)); - memcpy(eh->h_source, c->mac, sizeof(eh->h_source)); + memcpy(eh->h_source, c->our_tap_mac, sizeof(eh->h_source)); - if ((ret = tap_send(c, eh, len)) < 0) - warn("ARP: send: %s", strerror(ret)); + tap_send_single(c, eh, l2len); return 1; } @@ -59,6 +59,7 @@ #include "util.h" #include "ip.h" #include "checksum.h" +#include "iov.h" /* Checksums are optional for UDP over IPv4, so we usually just set * them to 0. Change this to 1 to calculate real UDP over IPv4 @@ -116,19 +117,19 @@ uint16_t csum_fold(uint32_t sum) /** * csum_ip4_header() - Calculate IPv4 header checksum - * @tot_len: IPv4 payload length (data + IP header, network order) - * @protocol: Protocol number (network order) - * @saddr: IPv4 source address (network order) - * @daddr: IPv4 destination address (network order) + * @l3len: IPv4 packet length (host order) + * @protocol: Protocol number + * @saddr: IPv4 source address + * @daddr: IPv4 destination address * * Return: 16-bit folded sum of the IPv4 header */ -uint16_t csum_ip4_header(uint16_t tot_len, uint8_t protocol, +uint16_t csum_ip4_header(uint16_t l3len, uint8_t protocol, struct in_addr saddr, struct in_addr daddr) { uint32_t sum = L2_BUF_IP4_PSUM(protocol); - sum += tot_len; + sum += htons(l3len); sum += (saddr.s_addr >> 16) & 0xffff; sum += saddr.s_addr & 0xffff; sum += (daddr.s_addr >> 16) & 0xffff; @@ -140,13 +141,13 @@ uint16_t csum_ip4_header(uint16_t tot_len, uint8_t protocol, /** * proto_ipv4_header_psum() - Calculates the partial checksum of an * IPv4 header for UDP or TCP - * @tot_len: IPv4 Payload length (host order) - * @proto: Protocol number (host order) - * @saddr: Source address (network order) - * @daddr: Destination address (network order) + * @l4len: IPv4 Payload length (host order) + * @proto: Protocol number + * @saddr: Source address + * @daddr: Destination address * Returns: Partial checksum of the IPv4 header */ -uint32_t proto_ipv4_header_psum(uint16_t tot_len, uint8_t protocol, +uint32_t proto_ipv4_header_psum(uint16_t l4len, uint8_t protocol, struct in_addr saddr, struct in_addr daddr) { uint32_t psum = htons(protocol); @@ -155,7 +156,7 @@ uint32_t proto_ipv4_header_psum(uint16_t tot_len, uint8_t protocol, psum += saddr.s_addr & 0xffff; psum += (daddr.s_addr >> 16) & 0xffff; psum += daddr.s_addr & 0xffff; - psum += htons(tot_len); + psum += htons(l4len); return psum; } @@ -165,22 +166,24 @@ uint32_t proto_ipv4_header_psum(uint16_t tot_len, uint8_t protocol, * @udp4hr: UDP header, initialised apart from checksum * @saddr: IPv4 source address * @daddr: IPv4 destination address - * @payload: ICMPv4 packet payload - * @len: Length of @payload (not including UDP) + * @iov: Pointer to the array of IO vectors + * @iov_cnt: Length of the array + * @offset: UDP payload offset in the iovec array */ void csum_udp4(struct udphdr *udp4hr, struct in_addr saddr, struct in_addr daddr, - const void *payload, size_t len) + const struct iovec *iov, int iov_cnt, size_t offset) { /* UDP checksums are optional, so don't bother */ udp4hr->check = 0; if (UDP4_REAL_CHECKSUMS) { - uint16_t tot_len = len + sizeof(struct udphdr); - uint32_t psum = proto_ipv4_header_psum(tot_len, IPPROTO_UDP, + uint16_t l4len = iov_size(iov, iov_cnt) - offset + + sizeof(struct udphdr); + uint32_t psum = proto_ipv4_header_psum(l4len, IPPROTO_UDP, saddr, daddr); psum = csum_unfolded(udp4hr, sizeof(struct udphdr), psum); - udp4hr->check = csum(payload, len, psum); + udp4hr->check = csum_iov(iov, iov_cnt, offset, psum); } } @@ -188,9 +191,9 @@ void csum_udp4(struct udphdr *udp4hr, * csum_icmp4() - Calculate and set checksum for an ICMP packet * @icmp4hr: ICMP header, initialised apart from checksum * @payload: ICMP packet payload - * @len: Length of @payload (not including ICMP header) + * @dlen: Length of @payload (not including ICMP header) */ -void csum_icmp4(struct icmphdr *icmp4hr, const void *payload, size_t len) +void csum_icmp4(struct icmphdr *icmp4hr, const void *payload, size_t dlen) { uint32_t psum; @@ -199,16 +202,16 @@ void csum_icmp4(struct icmphdr *icmp4hr, const void *payload, size_t len) /* Partial checksum for ICMP header alone */ psum = sum_16b(icmp4hr, sizeof(*icmp4hr)); - icmp4hr->checksum = csum(payload, len, psum); + icmp4hr->checksum = csum(payload, dlen, psum); } /** * proto_ipv6_header_psum() - Calculates the partial checksum of an * IPv6 header for UDP or TCP * @payload_len: IPv6 payload length (host order) - * @proto: Protocol number (host order) - * @saddr: Source address (network order) - * @daddr: Destination address (network order) + * @proto: Protocol number + * @saddr: Source address + * @daddr: Destination address * Returns: Partial checksum of the IPv6 header */ uint32_t proto_ipv6_header_psum(uint16_t payload_len, uint8_t protocol, @@ -226,19 +229,24 @@ uint32_t proto_ipv6_header_psum(uint16_t payload_len, uint8_t protocol, /** * csum_udp6() - Calculate and set checksum for a UDP over IPv6 packet * @udp6hr: UDP header, initialised apart from checksum - * @payload: UDP packet payload - * @len: Length of @payload (not including UDP header) + * @saddr: Source address + * @daddr: Destination address + * @iov: Pointer to the array of IO vectors + * @iov_cnt: Length of the array + * @offset: UDP payload offset in the iovec array */ void csum_udp6(struct udphdr *udp6hr, const struct in6_addr *saddr, const struct in6_addr *daddr, - const void *payload, size_t len) + const struct iovec *iov, int iov_cnt, size_t offset) { - uint32_t psum = proto_ipv6_header_psum(len + sizeof(struct udphdr), - IPPROTO_UDP, saddr, daddr); + uint16_t l4len = iov_size(iov, iov_cnt) - offset + + sizeof(struct udphdr); + uint32_t psum = proto_ipv6_header_psum(l4len, IPPROTO_UDP, + saddr, daddr); udp6hr->check = 0; psum = csum_unfolded(udp6hr, sizeof(struct udphdr), psum); - udp6hr->check = csum(payload, len, psum); + udp6hr->check = csum_iov(iov, iov_cnt, offset, psum); } /** @@ -247,21 +255,19 @@ void csum_udp6(struct udphdr *udp6hr, * @saddr: IPv6 source address * @daddr: IPv6 destination address * @payload: ICMP packet payload - * @len: Length of @payload (not including ICMPv6 header) + * @dlen: Length of @payload (not including ICMPv6 header) */ void csum_icmp6(struct icmp6hdr *icmp6hr, const struct in6_addr *saddr, const struct in6_addr *daddr, - const void *payload, size_t len) + const void *payload, size_t dlen) { - /* Partial checksum for the pseudo-IPv6 header */ - uint32_t psum = sum_16b(saddr, sizeof(*saddr)) + - sum_16b(daddr, sizeof(*daddr)) + - htons(len + sizeof(*icmp6hr)) + htons(IPPROTO_ICMPV6); + uint32_t psum = proto_ipv6_header_psum(dlen + sizeof(*icmp6hr), + IPPROTO_ICMPV6, saddr, daddr); icmp6hr->icmp6_cksum = 0; /* Add in partial checksum for the ICMPv6 header alone */ psum += sum_16b(icmp6hr, sizeof(*icmp6hr)); - icmp6hr->icmp6_cksum = csum(payload, len, psum); + icmp6hr->icmp6_cksum = csum(payload, dlen, psum); } #ifdef __AVX2__ @@ -499,16 +505,26 @@ uint16_t csum(const void *buf, size_t len, uint32_t init) * * @iov Pointer to the array of IO vectors * @n Length of the array + * @offset: Offset of the data to checksum within the full data length * @init Initial 32-bit checksum, 0 for no pre-computed checksum * * Return: 16-bit folded, complemented checksum */ -/* cppcheck-suppress unusedFunction */ -uint16_t csum_iov(const struct iovec *iov, size_t n, uint32_t init) +uint16_t csum_iov(const struct iovec *iov, size_t n, size_t offset, + uint32_t init) { unsigned int i; + size_t first; + + i = iov_skip_bytes(iov, n, offset, &first); + if (i >= n) + return (uint16_t)~csum_fold(init); + + init = csum_unfolded((char *)iov[i].iov_base + first, + iov[i].iov_len - first, init); + i++; - for (i = 0; i < n; i++) + for (; i < n; i++) init = csum_unfolded(iov[i].iov_base, iov[i].iov_len, init); return (uint16_t)~csum_fold(init); @@ -13,25 +13,26 @@ struct icmp6hdr; uint32_t sum_16b(const void *buf, size_t len); uint16_t csum_fold(uint32_t sum); uint16_t csum_unaligned(const void *buf, size_t len, uint32_t init); -uint16_t csum_ip4_header(uint16_t tot_len, uint8_t protocol, +uint16_t csum_ip4_header(uint16_t l3len, uint8_t protocol, struct in_addr saddr, struct in_addr daddr); -uint32_t proto_ipv4_header_psum(uint16_t tot_len, uint8_t protocol, +uint32_t proto_ipv4_header_psum(uint16_t l4len, uint8_t protocol, struct in_addr saddr, struct in_addr daddr); void csum_udp4(struct udphdr *udp4hr, struct in_addr saddr, struct in_addr daddr, - const void *payload, size_t len); -void csum_icmp4(struct icmphdr *icmp4hr, const void *payload, size_t len); + const struct iovec *iov, int iov_cnt, size_t offset); +void csum_icmp4(struct icmphdr *icmp4hr, const void *payload, size_t dlen); uint32_t proto_ipv6_header_psum(uint16_t payload_len, uint8_t protocol, const struct in6_addr *saddr, const struct in6_addr *daddr); void csum_udp6(struct udphdr *udp6hr, const struct in6_addr *saddr, const struct in6_addr *daddr, - const void *payload, size_t len); + const struct iovec *iov, int iov_cnt, size_t offset); void csum_icmp6(struct icmp6hdr *icmp6hr, const struct in6_addr *saddr, const struct in6_addr *daddr, - const void *payload, size_t len); + const void *payload, size_t dlen); uint32_t csum_unfolded(const void *buf, size_t len, uint32_t init); uint16_t csum(const void *buf, size_t len, uint32_t init); -uint16_t csum_iov(const struct iovec *iov, size_t n, uint32_t init); +uint16_t csum_iov(const struct iovec *iov, size_t n, size_t offset, + uint32_t init); #endif /* CHECKSUM_H */ @@ -38,6 +38,7 @@ #include "ip.h" #include "passt.h" #include "netlink.h" +#include "tap.h" #include "udp.h" #include "tcp.h" #include "pasta.h" @@ -46,6 +47,8 @@ #include "log.h" #include "vhost_user.h" +#define NETNS_RUN_DIR "/run/netns" + /** * next_chunk - Return the next piece of a string delimited by a character * @s: String to search @@ -116,11 +119,10 @@ static int parse_port_range(const char *s, char **endptr, static void conf_ports(const struct ctx *c, char optname, const char *optarg, struct fwd_ports *fwd) { - char addr_buf[sizeof(struct in6_addr)] = { 0 }, *addr = addr_buf; + union inany_addr addr_buf = inany_any6, *addr = &addr_buf; char buf[BUFSIZ], *spec, *ifname = NULL, *p; bool exclude_only = true, bound_one = false; uint8_t exclude[PORT_BITMAP_SIZE] = { 0 }; - sa_family_t af = AF_UNSPEC; unsigned i; int ret; @@ -132,6 +134,11 @@ static void conf_ports(const struct ctx *c, char optname, const char *optarg, return; } + if ((optname == 't' || optname == 'T') && c->no_tcp) + die("TCP port forwarding requested but TCP is disabled"); + if ((optname == 'u' || optname == 'U') && c->no_udp) + die("UDP port forwarding requested but UDP is disabled"); + if (!strcmp(optarg, "auto")) { if (fwd->mode) goto mode_conflict; @@ -151,19 +158,23 @@ static void conf_ports(const struct ctx *c, char optname, const char *optarg, die("'all' port forwarding is only allowed for passt"); fwd->mode = FWD_ALL; - memset(fwd->map, 0xff, PORT_EPHEMERAL_MIN / 8); - for (i = 0; i < PORT_EPHEMERAL_MIN; i++) { + /* Skip port 0. It has special meaning for many socket APIs, so + * trying to bind it is not really safe. + */ + for (i = 1; i < NUM_PORTS; i++) { + if (fwd_port_is_ephemeral(i)) + continue; + + bitmap_set(fwd->map, i); if (optname == 't') { - ret = tcp_sock_init(c, AF_UNSPEC, NULL, NULL, - i); + ret = tcp_sock_init(c, NULL, NULL, i); if (ret == -ENFILE || ret == -EMFILE) goto enfile; if (!ret) bound_one = true; } else if (optname == 'u') { - ret = udp_sock_init(c, 0, AF_UNSPEC, NULL, NULL, - i); + ret = udp_sock_init(c, 0, NULL, NULL, i); if (ret == -ENFILE || ret == -EMFILE) goto enfile; if (!ret) @@ -204,14 +215,20 @@ static void conf_ports(const struct ctx *c, char optname, const char *optarg, } - if (ifname == buf + 1) /* Interface without address */ + if (ifname == buf + 1) { /* Interface without address */ addr = NULL; - else if (inet_pton(AF_INET, buf, addr)) - af = AF_INET; - else if (inet_pton(AF_INET6, buf, addr)) - af = AF_INET6; - else - goto bad; + } else { + p = buf; + + /* Allow square brackets for IPv4 too for convenience */ + if (*p == '[' && p[strlen(p) - 1] == ']') { + p[strlen(p) - 1] = '\0'; + p++; + } + + if (!inany_pton(p, addr)) + goto bad; + } } else { spec = buf; @@ -244,20 +261,24 @@ static void conf_ports(const struct ctx *c, char optname, const char *optarg, } while ((p = next_chunk(p, ','))); if (exclude_only) { - for (i = 0; i < PORT_EPHEMERAL_MIN; i++) { - if (bitmap_isset(exclude, i)) + /* Skip port 0. It has special meaning for many socket APIs, so + * trying to bind it is not really safe. + */ + for (i = 1; i < NUM_PORTS; i++) { + if (fwd_port_is_ephemeral(i) || + bitmap_isset(exclude, i)) continue; bitmap_set(fwd->map, i); if (optname == 't') { - ret = tcp_sock_init(c, af, addr, ifname, i); + ret = tcp_sock_init(c, addr, ifname, i); if (ret == -ENFILE || ret == -EMFILE) goto enfile; if (!ret) bound_one = true; } else if (optname == 'u') { - ret = udp_sock_init(c, 0, af, addr, ifname, i); + ret = udp_sock_init(c, 0, addr, ifname, i); if (ret == -ENFILE || ret == -EMFILE) goto enfile; if (!ret) @@ -313,9 +334,9 @@ static void conf_ports(const struct ctx *c, char optname, const char *optarg, ret = 0; if (optname == 't') - ret = tcp_sock_init(c, af, addr, ifname, i); + ret = tcp_sock_init(c, addr, ifname, i); else if (optname == 'u') - ret = udp_sock_init(c, 0, af, addr, ifname, i); + ret = udp_sock_init(c, 0, addr, ifname, i); if (ret) goto bind_fail; } @@ -338,55 +359,93 @@ bind_all_fail: /** * add_dns4() - Possibly add the IPv4 address of a DNS resolver to configuration * @c: Execution context - * @addr: Address found in /etc/resolv.conf - * @conf: Pointer to reference of current entry in array of IPv4 resolvers + * @addr: Guest nameserver IPv4 address + * @idx: Index of free entry in array of IPv4 resolvers + * + * Return: Number of entries added (0 or 1) */ -static void add_dns4(struct ctx *c, const struct in_addr *addr, - struct in_addr **conf) +static unsigned add_dns4(struct ctx *c, const struct in_addr *addr, + unsigned idx) { - /* Guest or container can only access local addresses via redirect */ - if (IN4_IS_ADDR_LOOPBACK(addr)) { - if (!c->no_map_gw) { - **conf = c->ip4.gw; - (*conf)++; - - if (IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns_match)) - c->ip4.dns_match = c->ip4.gw; - } - } else { - **conf = *addr; - (*conf)++; - } + if (idx >= ARRAY_SIZE(c->ip4.dns)) + return 0; - if (IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns_host)) - c->ip4.dns_host = *addr; + c->ip4.dns[idx] = *addr; + return 1; } /** * add_dns6() - Possibly add the IPv6 address of a DNS resolver to configuration * @c: Execution context - * @addr: Address found in /etc/resolv.conf - * @conf: Pointer to reference of current entry in array of IPv6 resolvers + * @addr: Guest nameserver IPv6 address + * @idx: Index of free entry in array of IPv6 resolvers + * + * Return: Number of entries added (0 or 1) */ -static void add_dns6(struct ctx *c, - struct in6_addr *addr, struct in6_addr **conf) +static unsigned add_dns6(struct ctx *c, const struct in6_addr *addr, + unsigned idx) { - /* Guest or container can only access local addresses via redirect */ - if (IN6_IS_ADDR_LOOPBACK(addr)) { - if (!c->no_map_gw) { - memcpy(*conf, &c->ip6.gw, sizeof(**conf)); - (*conf)++; + if (idx >= ARRAY_SIZE(c->ip6.dns)) + return 0; - if (IN6_IS_ADDR_UNSPECIFIED(&c->ip6.dns_match)) - memcpy(&c->ip6.dns_match, addr, sizeof(*addr)); + c->ip6.dns[idx] = *addr; + return 1; +} + +/** + * add_dns_resolv() - Possibly add ns from host resolv.conf to configuration + * @c: Execution context + * @nameserver: Nameserver address string from /etc/resolv.conf + * @idx4: Pointer to index of current entry in array of IPv4 resolvers + * @idx6: Pointer to index of current entry in array of IPv6 resolvers + * + * @idx4 or @idx6 may be NULL, in which case resolvers of the corresponding type + * are ignored. + */ +static void add_dns_resolv(struct ctx *c, const char *nameserver, + unsigned *idx4, unsigned *idx6) +{ + struct in6_addr ns6; + struct in_addr ns4; + + if (idx4 && inet_pton(AF_INET, nameserver, &ns4)) { + if (IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns_host)) + c->ip4.dns_host = ns4; + + /* Guest or container can only access local addresses via + * redirect + */ + if (IN4_IS_ADDR_LOOPBACK(&ns4)) { + if (IN4_IS_ADDR_UNSPECIFIED(&c->ip4.map_host_loopback)) + return; + + ns4 = c->ip4.map_host_loopback; + if (IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns_match)) + c->ip4.dns_match = c->ip4.map_host_loopback; } - } else { - memcpy(*conf, addr, sizeof(**conf)); - (*conf)++; + + *idx4 += add_dns4(c, &ns4, *idx4); } - if (IN6_IS_ADDR_UNSPECIFIED(&c->ip6.dns_host)) - c->ip6.dns_host = *addr; + if (idx6 && inet_pton(AF_INET6, nameserver, &ns6)) { + if (IN6_IS_ADDR_UNSPECIFIED(&c->ip6.dns_host)) + c->ip6.dns_host = ns6; + + /* Guest or container can only access local addresses via + * redirect + */ + if (IN6_IS_ADDR_LOOPBACK(&ns6)) { + if (IN6_IS_ADDR_UNSPECIFIED(&c->ip6.map_host_loopback)) + return; + + ns6 = c->ip6.map_host_loopback; + + if (IN6_IS_ADDR_UNSPECIFIED(&c->ip6.dns_match)) + c->ip6.dns_match = c->ip6.map_host_loopback; + } + + *idx6 += add_dns6(c, &ns6, *idx6); + } } /** @@ -395,17 +454,16 @@ static void add_dns6(struct ctx *c, */ static void get_dns(struct ctx *c) { - struct in6_addr *dns6 = &c->ip6.dns[0], dns6_tmp; - struct in_addr *dns4 = &c->ip4.dns[0], dns4_tmp; int dns4_set, dns6_set, dnss_set, dns_set, fd; + unsigned dns4_idx = 0, dns6_idx = 0; struct fqdn *s = c->dns_search; struct lineread resolvconf; + ssize_t line_len; char *line, *end; const char *p; - int line_len; - dns4_set = !c->ifi4 || !IN4_IS_ADDR_UNSPECIFIED(dns4); - dns6_set = !c->ifi6 || !IN6_IS_ADDR_UNSPECIFIED(dns6); + dns4_set = !c->ifi4 || !IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns[0]); + dns6_set = !c->ifi6 || !IN6_IS_ADDR_UNSPECIFIED(&c->ip6.dns[0]); dnss_set = !!*s->n || c->no_dns_search; dns_set = (dns4_set && dns6_set) || c->no_dns; @@ -426,15 +484,9 @@ static void get_dns(struct ctx *c) if (end) *end = 0; - if (!dns4_set && - dns4 - &c->ip4.dns[0] < ARRAY_SIZE(c->ip4.dns) - 1 - && inet_pton(AF_INET, p + 1, &dns4_tmp)) - add_dns4(c, &dns4_tmp, &dns4); - - if (!dns6_set && - dns6 - &c->ip6.dns[0] < ARRAY_SIZE(c->ip6.dns) - 1 - && inet_pton(AF_INET6, p + 1, &dns6_tmp)) - add_dns6(c, &dns6_tmp, &dns6); + add_dns_resolv(c, p + 1, + dns4_set ? NULL : &dns4_idx, + dns6_set ? NULL : &dns6_idx); } else if (!dnss_set && strstr(line, "search ") == line && s == c->dns_search) { end = strpbrk(line, "\n"); @@ -448,7 +500,7 @@ static void get_dns(struct ctx *c) while (s - c->dns_search < ARRAY_SIZE(c->dns_search) - 1 /* cppcheck-suppress strtokCalled */ && (p = strtok(NULL, " \t"))) { - strncpy(s->n, p, sizeof(c->dns_search[0])); + strncpy(s->n, p, sizeof(c->dns_search[0]) - 1); s++; *s->n = 0; } @@ -456,12 +508,25 @@ static void get_dns(struct ctx *c) } if (line_len < 0) - warn("Error reading /etc/resolv.conf: %s", strerror(errno)); + warn_perror("Error reading /etc/resolv.conf"); close(fd); out: - if (!dns_set && dns4 == c->ip4.dns && dns6 == c->ip6.dns) - warn("Couldn't get any nameserver address"); + if (!dns_set) { + if (!(dns4_idx + dns6_idx)) + warn("Couldn't get any nameserver address"); + + if (c->no_dhcp_dns) + return; + + if (c->ifi4 && !c->no_dhcp && + IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns[0])) + warn("No IPv4 nameserver available for DHCP"); + + if (c->ifi6 && ((!c->no_ndp && !c->no_ra) || !c->no_dhcpv6) && + IN6_IS_ADDR_UNSPECIFIED(&c->ip6.dns[0])) + warn("No IPv6 nameserver available for NDP/DHCPv6"); + } } /** @@ -499,9 +564,6 @@ static void conf_netns_opt(char *netns, const char *arg) static void conf_pasta_ns(int *netns_only, char *userns, char *netns, int optind, int argc, char *argv[]) { - if (*netns_only && *userns) - die("Both --userns and --netns-only given"); - if (*netns && optind != argc) die("Both --netns and PID or command given"); @@ -515,10 +577,15 @@ static void conf_pasta_ns(int *netns_only, char *userns, char *netns, if (pidval < 0 || pidval > INT_MAX) die("Invalid PID %s", argv[optind]); - snprintf(netns, PATH_MAX, "/proc/%ld/ns/net", pidval); - if (!*userns) - snprintf(userns, PATH_MAX, "/proc/%ld/ns/user", - pidval); + if (snprintf_check(netns, PATH_MAX, + "/proc/%ld/ns/net", pidval)) + die_perror("Can't build netns path"); + + if (!*userns) { + if (snprintf_check(userns, PATH_MAX, + "/proc/%ld/ns/user", pidval)) + die_perror("Can't build userns path"); + } } } @@ -556,23 +623,22 @@ static int conf_ip4_prefix(const char *arg) * conf_ip4() - Verify or detect IPv4 support, get relevant addresses * @ifi: Host interface to attempt (0 to determine one) * @ip4: IPv4 context (will be written) - * @mac: MAC address to use (written if unset) * * Return: Interface index for IPv4, or 0 on failure. */ -static unsigned int conf_ip4(unsigned int ifi, - struct ip4_ctx *ip4, unsigned char *mac) +static unsigned int conf_ip4(unsigned int ifi, struct ip4_ctx *ip4) { if (!ifi) ifi = nl_get_ext_if(nl_sock, AF_INET); if (!ifi) { - info("No interface with a default route for IPv4: disabling IPv4"); + info("Couldn't pick external interface: disabling IPv4"); return 0; } - if (IN4_IS_ADDR_UNSPECIFIED(&ip4->gw)) { - int rc = nl_route_get_def(nl_sock, ifi, AF_INET, &ip4->gw); + if (IN4_IS_ADDR_UNSPECIFIED(&ip4->guest_gw)) { + int rc = nl_route_get_def(nl_sock, ifi, AF_INET, + &ip4->guest_gw); if (rc < 0) { err("Couldn't discover IPv4 gateway address: %s", strerror(-rc)); @@ -602,20 +668,11 @@ static unsigned int conf_ip4(unsigned int ifi, ip4->prefix_len = 32; } - memcpy(&ip4->addr_seen, &ip4->addr, sizeof(ip4->addr_seen)); + ip4->addr_seen = ip4->addr; - if (MAC_IS_ZERO(mac)) { - int rc = nl_link_get_mac(nl_sock, ifi, mac); - if (rc < 0) { - char ifname[IFNAMSIZ]; - err("Couldn't discover MAC for %s: %s", - if_indextoname(ifi, ifname), strerror(-rc)); - return 0; - } - } + ip4->our_tap_addr = ip4->guest_gw; - if (IN4_IS_ADDR_UNSPECIFIED(&ip4->addr) || - MAC_IS_ZERO(mac)) + if (IN4_IS_ADDR_UNSPECIFIED(&ip4->addr)) return 0; return ifi; @@ -625,12 +682,10 @@ static unsigned int conf_ip4(unsigned int ifi, * conf_ip6() - Verify or detect IPv6 support, get relevant addresses * @ifi: Host interface to attempt (0 to determine one) * @ip6: IPv6 context (will be written) - * @mac: MAC address to use (written if unset) * * Return: Interface index for IPv6, or 0 on failure. */ -static unsigned int conf_ip6(unsigned int ifi, - struct ip6_ctx *ip6, unsigned char *mac) +static unsigned int conf_ip6(unsigned int ifi, struct ip6_ctx *ip6) { int prefix_len = 0; int rc; @@ -639,12 +694,12 @@ static unsigned int conf_ip6(unsigned int ifi, ifi = nl_get_ext_if(nl_sock, AF_INET6); if (!ifi) { - info("No interface with a default route for IPv6: disabling IPv6"); + info("Couldn't pick external interface: disabling IPv6"); return 0; } - if (IN6_IS_ADDR_UNSPECIFIED(&ip6->gw)) { - rc = nl_route_get_def(nl_sock, ifi, AF_INET6, &ip6->gw); + if (IN6_IS_ADDR_UNSPECIFIED(&ip6->guest_gw)) { + rc = nl_route_get_def(nl_sock, ifi, AF_INET6, &ip6->guest_gw); if (rc < 0) { err("Couldn't discover IPv6 gateway address: %s", strerror(-rc)); @@ -654,246 +709,249 @@ static unsigned int conf_ip6(unsigned int ifi, rc = nl_addr_get(nl_sock, ifi, AF_INET6, IN6_IS_ADDR_UNSPECIFIED(&ip6->addr) ? &ip6->addr : NULL, - &prefix_len, &ip6->addr_ll); + &prefix_len, &ip6->our_tap_ll); if (rc < 0) { err("Couldn't discover IPv6 address: %s", strerror(-rc)); return 0; } - memcpy(&ip6->addr_seen, &ip6->addr, sizeof(ip6->addr)); - memcpy(&ip6->addr_ll_seen, &ip6->addr_ll, sizeof(ip6->addr_ll)); + ip6->addr_seen = ip6->addr; - if (MAC_IS_ZERO(mac)) { - rc = nl_link_get_mac(nl_sock, ifi, mac); - if (rc < 0) { - char ifname[IFNAMSIZ]; - err("Couldn't discover MAC for %s: %s", - if_indextoname(ifi, ifname), strerror(-rc)); - return 0; - } - } + if (IN6_IS_ADDR_LINKLOCAL(&ip6->guest_gw)) + ip6->our_tap_ll = ip6->guest_gw; if (IN6_IS_ADDR_UNSPECIFIED(&ip6->addr) || - IN6_IS_ADDR_UNSPECIFIED(&ip6->addr_ll) || - MAC_IS_ZERO(mac)) + IN6_IS_ADDR_UNSPECIFIED(&ip6->our_tap_ll)) return 0; return ifi; } /** - * print_usage() - Print usage, exit with given status code + * usage() - Print usage, exit with given status code * @name: Executable name + * @f: Stream to print usage info to * @status: Status code for exit() */ -static void print_usage(const char *name, int status) +static void usage(const char *name, FILE *f, int status) { if (strstr(name, "pasta")) { - info("Usage: %s [OPTION]... [COMMAND] [ARGS]...", name); - info(" %s [OPTION]... PID", name); - info(" %s [OPTION]... --netns [PATH|NAME]", name); - info(""); - info("Without PID or --netns, run the given command or a"); - info("default shell in a new network and user namespace, and"); - info("connect it via pasta."); + FPRINTF(f, "Usage: %s [OPTION]... [COMMAND] [ARGS]...\n", name); + FPRINTF(f, " %s [OPTION]... PID\n", name); + FPRINTF(f, " %s [OPTION]... --netns [PATH|NAME]\n", name); + FPRINTF(f, + "\n" + "Without PID or --netns, run the given command or a\n" + "default shell in a new network and user namespace, and\n" + "connect it via pasta.\n"); } else { - info("Usage: %s [OPTION]...", name); + FPRINTF(f, "Usage: %s [OPTION]...\n", name); } - info(""); - - - info( " -d, --debug Be verbose"); - info( " --trace Be extra verbose, implies --debug"); - info( " -q, --quiet Don't print informational messages"); - info( " -f, --foreground Don't run in background"); - info( " default: run in background if started from a TTY"); - info( " -e, --stderr Log to stderr too"); - info( " default: log to system logger only if started from a TTY"); - info( " -l, --log-file PATH Log (only) to given file"); - info( " --log-size BYTES Maximum size of log file"); - info( " default: 1 MiB"); - info( " --runas UID|UID:GID Run as given UID, GID, which can be"); - info( " numeric, or login and group names"); - info( " default: drop to user \"nobody\""); - info( " -h, --help Display this help message and exit"); - info( " --version Show version and exit"); + + FPRINTF(f, + "\n" + " -d, --debug Be verbose\n" + " --trace Be extra verbose, implies --debug\n" + " -q, --quiet Don't print informational messages\n" + " -f, --foreground Don't run in background\n" + " default: run in background\n" + " -l, --log-file PATH Log (only) to given file\n" + " --log-size BYTES Maximum size of log file\n" + " default: 1 MiB\n" + " --runas UID|UID:GID Run as given UID, GID, which can be\n" + " numeric, or login and group names\n" + " default: drop to user \"nobody\"\n" + " -h, --help Display this help message and exit\n" + " --version Show version and exit\n"); if (strstr(name, "pasta")) { - info( " -I, --ns-ifname NAME namespace interface name"); - info( " default: same interface name as external one"); + FPRINTF(f, + " -I, --ns-ifname NAME namespace interface name\n" + " default: same interface name as external one\n"); } else { - info( " -s, --socket, --socket-path PATH UNIX domain socket path"); - info( " default: probe free path starting from " - UNIX_SOCK_PATH, 1); - info( " --vhost-user Enable vhost-user mode"); - info( " UNIX domain socket is provided by -s option"); - info( " --print-capabilities print back-end capabilities in JSON format"); + FPRINTF(f, + " -s, --socket, --socket-path PATH UNIX domain socket path\n" + " default: probe free path starting from " + UNIX_SOCK_PATH "\n", 1); + FPRINTF(f, + " --vhost-user Enable vhost-user mode\n" + " UNIX domain socket is provided by -s option\n" + " --print-capabilities print back-end capabilities in JSON format,\n" + " only meaningful for vhost-user mode\n"); } - info( " -F, --fd FD Use FD as pre-opened connected socket"); - info( " -p, --pcap FILE Log tap-facing traffic to pcap file"); - info( " -P, --pid FILE Write own PID to the given file"); - info( " -m, --mtu MTU Assign MTU via DHCP/NDP"); - info( " a zero value disables assignment"); - info( " default: 65520: maximum 802.3 MTU minus 802.3 header"); - info( " length, rounded to 32 bits (IPv4 words)"); - info( " -a, --address ADDR Assign IPv4 or IPv6 address ADDR"); - info( " can be specified zero to two times (for IPv4 and IPv6)"); - info( " default: use addresses from interface with default route"); - info( " -n, --netmask MASK Assign IPv4 MASK, dot-decimal or bits"); - info( " default: netmask from matching address on the host"); - info( " -M, --mac-addr ADDR Use source MAC address ADDR"); - info( " default: MAC address from interface with default route"); - info( " -g, --gateway ADDR Pass IPv4 or IPv6 address as gateway"); - info( " default: gateway from interface with default route"); - info( " -i, --interface NAME Interface for addresses and routes"); - info( " default: from --outbound-if4 and --outbound-if6, if any"); - info( " otherwise interface with first default route"); - info( " -o, --outbound ADDR Bind to address as outbound source"); - info( " can be specified zero to two times (for IPv4 and IPv6)"); - info( " default: use source address from routing tables"); - info( " --outbound-if4 NAME Bind to outbound interface for IPv4"); - info( " default: use interface from default route"); - info( " --outbound-if6 NAME Bind to outbound interface for IPv6"); - info( " default: use interface from default route"); - info( " -D, --dns ADDR Use IPv4 or IPv6 address as DNS"); - info( " can be specified multiple times"); - info( " a single, empty option disables DNS information"); + FPRINTF(f, + " -F, --fd FD Use FD as pre-opened connected socket\n" + " -p, --pcap FILE Log tap-facing traffic to pcap file\n" + " -P, --pid FILE Write own PID to the given file\n" + " -m, --mtu MTU Assign MTU via DHCP/NDP\n" + " a zero value disables assignment\n" + " default: 65520: maximum 802.3 MTU minus 802.3 header\n" + " length, rounded to 32 bits (IPv4 words)\n" + " -a, --address ADDR Assign IPv4 or IPv6 address ADDR\n" + " can be specified zero to two times (for IPv4 and IPv6)\n" + " default: use addresses from interface with default route\n" + " -n, --netmask MASK Assign IPv4 MASK, dot-decimal or bits\n" + " default: netmask from matching address on the host\n" + " -M, --mac-addr ADDR Use source MAC address ADDR\n" + " default: MAC address from interface with default route\n" + " -g, --gateway ADDR Pass IPv4 or IPv6 address as gateway\n" + " default: gateway from interface with default route\n" + " -i, --interface NAME Interface for addresses and routes\n" + " default: from --outbound-if4 and --outbound-if6, if any\n" + " otherwise interface with first default route\n" + " -o, --outbound ADDR Bind to address as outbound source\n" + " can be specified zero to two times (for IPv4 and IPv6)\n" + " default: use source address from routing tables\n" + " --outbound-if4 NAME Bind to outbound interface for IPv4\n" + " default: use interface from default route\n" + " --outbound-if6 NAME Bind to outbound interface for IPv6\n" + " default: use interface from default route\n" + " -D, --dns ADDR Use IPv4 or IPv6 address as DNS\n" + " can be specified multiple times\n" + " a single, empty option disables DNS information\n"); if (strstr(name, "pasta")) - info( " default: don't use any addresses"); + FPRINTF(f, " default: don't use any addresses\n"); else - info( " default: use addresses from /etc/resolv.conf"); - - info( " -S, --search LIST Space-separated list, search domains"); - info( " a single, empty option disables the DNS search list"); + FPRINTF(f, " default: use addresses from /etc/resolv.conf\n"); + FPRINTF(f, + " -S, --search LIST Space-separated list, search domains\n" + " a single, empty option disables the DNS search list\n"); if (strstr(name, "pasta")) - info( " default: don't use any search list"); + FPRINTF(f, " default: don't use any search list\n"); else - info( " default: use search list from /etc/resolv.conf"); + FPRINTF(f, " default: use search list from /etc/resolv.conf\n"); if (strstr(name, "pasta")) - info(" --dhcp-dns \tPass DNS list via DHCP/DHCPv6/NDP"); + FPRINTF(f, " --dhcp-dns \tPass DNS list via DHCP/DHCPv6/NDP\n"); else - info(" --no-dhcp-dns No DNS list in DHCP/DHCPv6/NDP"); + FPRINTF(f, " --no-dhcp-dns No DNS list in DHCP/DHCPv6/NDP\n"); if (strstr(name, "pasta")) - info(" --dhcp-search Pass list via DHCP/DHCPv6/NDP"); + FPRINTF(f, " --dhcp-search Pass list via DHCP/DHCPv6/NDP\n"); else - info(" --no-dhcp-search No list in DHCP/DHCPv6/NDP"); - - info( " --dns-forward ADDR Forward DNS queries sent to ADDR"); - info( " can be specified zero to two times (for IPv4 and IPv6)"); - info( " default: don't forward DNS queries"); - - info( " --no-tcp Disable TCP protocol handler"); - info( " --no-udp Disable UDP protocol handler"); - info( " --no-icmp Disable ICMP/ICMPv6 protocol handler"); - info( " --no-dhcp Disable DHCP server"); - info( " --no-ndp Disable NDP responses"); - info( " --no-dhcpv6 Disable DHCPv6 server"); - info( " --no-ra Disable router advertisements"); - info( " --no-map-gw Don't map gateway address to host"); - info( " -4, --ipv4-only Enable IPv4 operation only"); - info( " -6, --ipv6-only Enable IPv6 operation only"); + FPRINTF(f, " --no-dhcp-search No list in DHCP/DHCPv6/NDP\n"); + + FPRINTF(f, + " --map-host-loopback ADDR Translate ADDR to refer to host\n" + " can be specified zero to two times (for IPv4 and IPv6)\n" + " default: gateway address\n" + " --map-guest-addr ADDR Translate ADDR to guest's address\n" + " can be specified zero to two times (for IPv4 and IPv6)\n" + " default: none\n" + " --dns-forward ADDR Forward DNS queries sent to ADDR\n" + " can be specified zero to two times (for IPv4 and IPv6)\n" + " default: don't forward DNS queries\n" + " --dns-host ADDR Host nameserver to direct queries to\n" + " can be specified zero to two times (for IPv4 and IPv6)\n" + " default: first nameserver from host's /etc/resolv.conf\n" + " --no-tcp Disable TCP protocol handler\n" + " --no-udp Disable UDP protocol handler\n" + " --no-icmp Disable ICMP/ICMPv6 protocol handler\n" + " --no-dhcp Disable DHCP server\n" + " --no-ndp Disable NDP responses\n" + " --no-dhcpv6 Disable DHCPv6 server\n" + " --no-ra Disable router advertisements\n" + " --freebind Bind to any address for forwarding\n" + " --no-map-gw Don't map gateway address to host\n" + " -4, --ipv4-only Enable IPv4 operation only\n" + " -6, --ipv6-only Enable IPv6 operation only\n"); if (strstr(name, "pasta")) goto pasta_opts; - info( " -1, --one-off Quit after handling one single client"); - info( " -t, --tcp-ports SPEC TCP port forwarding to guest"); - info( " can be specified multiple times"); - info( " SPEC can be:"); - info( " 'none': don't forward any ports"); - info( " 'all': forward all unbound, non-ephemeral ports"); - info( " a comma-separated list, optionally ranged with '-'"); - info( " and optional target ports after ':', with optional"); - info( " address specification suffixed by '/' and optional"); - info( " interface prefixed by '%%'. Ranges can be reduced by"); - info( " excluding ports or ranges prefixed by '~'"); - info( " Examples:"); - info( " -t 22 Forward local port 22 to 22 on guest"); - info( " -t 22:23 Forward local port 22 to 23 on guest"); - info( " -t 22,25 Forward ports 22, 25 to ports 22, 25"); - info( " -t 22-80 Forward ports 22 to 80"); - info( " -t 22-80:32-90 Forward ports 22 to 80 to"); - info( " corresponding port numbers plus 10"); - info( " -t 192.0.2.1/5 Bind port 5 of 192.0.2.1 to guest"); - info( " -t 5-25,~10-20 Forward ports 5 to 9, and 21 to 25"); - info( " -t ~25 Forward all ports except for 25"); - info( " default: none"); - info( " -u, --udp-ports SPEC UDP port forwarding to guest"); - info( " SPEC is as described for TCP above"); - info( " default: none"); + FPRINTF(f, + " -1, --one-off Quit after handling one single client\n" + " -t, --tcp-ports SPEC TCP port forwarding to guest\n" + " can be specified multiple times\n" + " SPEC can be:\n" + " 'none': don't forward any ports\n" + " 'all': forward all unbound, non-ephemeral ports\n" + " a comma-separated list, optionally ranged with '-'\n" + " and optional target ports after ':', with optional\n" + " address specification suffixed by '/' and optional\n" + " interface prefixed by '%%'. Ranges can be reduced by\n" + " excluding ports or ranges prefixed by '~'\n" + " Examples:\n" + " -t 22 Forward local port 22 to 22 on guest\n" + " -t 22:23 Forward local port 22 to 23 on guest\n" + " -t 22,25 Forward ports 22, 25 to ports 22, 25\n" + " -t 22-80 Forward ports 22 to 80\n" + " -t 22-80:32-90 Forward ports 22 to 80 to\n" + " corresponding port numbers plus 10\n" + " -t 192.0.2.1/5 Bind port 5 of 192.0.2.1 to guest\n" + " -t 5-25,~10-20 Forward ports 5 to 9, and 21 to 25\n" + " -t ~25 Forward all ports except for 25\n" + " default: none\n" + " -u, --udp-ports SPEC UDP port forwarding to guest\n" + " SPEC is as described for TCP above\n" + " default: none\n"); exit(status); pasta_opts: - info( " -t, --tcp-ports SPEC TCP port forwarding to namespace"); - info( " can be specified multiple times"); - info( " SPEC can be:"); - info( " 'none': don't forward any ports"); - info( " 'auto': forward all ports currently bound in namespace"); - info( " a comma-separated list, optionally ranged with '-'"); - info( " and optional target ports after ':', with optional"); - info( " address specification suffixed by '/' and optional"); - info( " interface prefixed by '%%'. Examples:"); - info( " -t 22 Forward local port 22 to port 22 in netns"); - info( " -t 22:23 Forward local port 22 to port 23"); - info( " -t 22,25 Forward ports 22, 25 to ports 22, 25"); - info( " -t 22-80 Forward ports 22 to 80"); - info( " -t 22-80:32-90 Forward ports 22 to 80 to"); - info( " corresponding port numbers plus 10"); - info( " -t 192.0.2.1/5 Bind port 5 of 192.0.2.1 to namespace"); - info( " -t 5-25,~10-20 Forward ports 5 to 9, and 21 to 25"); - info( " -t ~25 Forward all bound ports except for 25"); - info( " default: auto"); - info( " IPv6 bound ports are also forwarded for IPv4"); - info( " -u, --udp-ports SPEC UDP port forwarding to namespace"); - info( " SPEC is as described for TCP above"); - info( " default: auto"); - info( " IPv6 bound ports are also forwarded for IPv4"); - info( " unless specified, with '-t auto', UDP ports with numbers"); - info( " corresponding to forwarded TCP port numbers are"); - info( " forwarded too"); - info( " -T, --tcp-ns SPEC TCP port forwarding to init namespace"); - info( " SPEC is as described above"); - info( " default: auto"); - info( " -U, --udp-ns SPEC UDP port forwarding to init namespace"); - info( " SPEC is as described above"); - info( " default: auto"); - info( " --userns NSPATH Target user namespace to join"); - info( " --netns PATH|NAME Target network namespace to join"); - info( " --netns-only Don't join existing user namespace"); - info( " implied if PATH or NAME are given without --userns"); - info( " --no-netns-quit Don't quit if filesystem-bound target"); - info( " network namespace is deleted"); - info( " --config-net Configure tap interface in namespace"); - info( " --no-copy-routes DEPRECATED:"); - info( " Don't copy all routes to namespace"); - info( " --no-copy-addrs DEPRECATED:"); - info( " Don't copy all addresses to namespace"); - info( " --ns-mac-addr ADDR Set MAC address on tap interface"); + FPRINTF(f, + " -t, --tcp-ports SPEC TCP port forwarding to namespace\n" + " can be specified multiple times\n" + " SPEC can be:\n" + " 'none': don't forward any ports\n" + " 'auto': forward all ports currently bound in namespace\n" + " a comma-separated list, optionally ranged with '-'\n" + " and optional target ports after ':', with optional\n" + " address specification suffixed by '/' and optional\n" + " interface prefixed by '%%'. Examples:\n" + " -t 22 Forward local port 22 to port 22 in netns\n" + " -t 22:23 Forward local port 22 to port 23\n" + " -t 22,25 Forward ports 22, 25 to ports 22, 25\n" + " -t 22-80 Forward ports 22 to 80\n" + " -t 22-80:32-90 Forward ports 22 to 80 to\n" + " corresponding port numbers plus 10\n" + " -t 192.0.2.1/5 Bind port 5 of 192.0.2.1 to namespace\n" + " -t 5-25,~10-20 Forward ports 5 to 9, and 21 to 25\n" + " -t ~25 Forward all bound ports except for 25\n" + " default: auto\n" + " IPv6 bound ports are also forwarded for IPv4\n" + " -u, --udp-ports SPEC UDP port forwarding to namespace\n" + " SPEC is as described for TCP above\n" + " default: auto\n" + " IPv6 bound ports are also forwarded for IPv4\n" + " unless specified, with '-t auto', UDP ports with numbers\n" + " corresponding to forwarded TCP port numbers are\n" + " forwarded too\n" + " -T, --tcp-ns SPEC TCP port forwarding to init namespace\n" + " SPEC is as described above\n" + " default: auto\n" + " -U, --udp-ns SPEC UDP port forwarding to init namespace\n" + " SPEC is as described above\n" + " default: auto\n" + " --host-lo-to-ns-lo DEPRECATED:\n" + " Translate host-loopback forwards to\n" + " namespace loopback\n" + " --userns NSPATH Target user namespace to join\n" + " --netns PATH|NAME Target network namespace to join\n" + " --netns-only Don't join existing user namespace\n" + " implied if PATH or NAME are given without --userns\n" + " --no-netns-quit Don't quit if filesystem-bound target\n" + " network namespace is deleted\n" + " --config-net Configure tap interface in namespace\n" + " --no-copy-routes DEPRECATED:\n" + " Don't copy all routes to namespace\n" + " --no-copy-addrs DEPRECATED:\n" + " Don't copy all addresses to namespace\n" + " --ns-mac-addr ADDR Set MAC address on tap interface\n"); exit(status); } /** - * usage() - Print usage and exit with failure - * @name: Executable name - */ -static void usage(const char *name) -{ - print_usage(name, EXIT_FAILURE); -} - -/** * conf_print() - Print fundamental configuration parameters * @c: Execution context */ static void conf_print(const struct ctx *c) { - char buf4[INET_ADDRSTRLEN], buf6[INET6_ADDRSTRLEN], ifn[IFNAMSIZ]; + char buf4[INET_ADDRSTRLEN], buf6[INET6_ADDRSTRLEN]; + char bufmac[ETH_ADDRSTRLEN], ifn[IFNAMSIZ]; int i; info("Template interface: %s%s%s%s%s", @@ -927,11 +985,14 @@ static void conf_print(const struct ctx *c) info("Namespace interface: %s", c->pasta_ifn); info("MAC:"); - info(" host: %02x:%02x:%02x:%02x:%02x:%02x", - c->mac[0], c->mac[1], c->mac[2], - c->mac[3], c->mac[4], c->mac[5]); + info(" host: %s", eth_ntop(c->our_tap_mac, bufmac, sizeof(bufmac))); if (c->ifi4) { + if (!IN4_IS_ADDR_UNSPECIFIED(&c->ip4.map_host_loopback)) + info(" NAT to host 127.0.0.1: %s", + inet_ntop(AF_INET, &c->ip4.map_host_loopback, + buf4, sizeof(buf4))); + if (!c->no_dhcp) { uint32_t mask; @@ -943,7 +1004,8 @@ static void conf_print(const struct ctx *c) info(" mask: %s", inet_ntop(AF_INET, &mask, buf4, sizeof(buf4))); info(" router: %s", - inet_ntop(AF_INET, &c->ip4.gw, buf4, sizeof(buf4))); + inet_ntop(AF_INET, &c->ip4.guest_gw, + buf4, sizeof(buf4))); } for (i = 0; !IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns[i]); i++) { @@ -961,6 +1023,11 @@ static void conf_print(const struct ctx *c) } if (c->ifi6) { + if (!IN6_IS_ADDR_UNSPECIFIED(&c->ip6.map_host_loopback)) + info(" NAT to host ::1: %s", + inet_ntop(AF_INET6, &c->ip6.map_host_loopback, + buf6, sizeof(buf6))); + if (!c->no_ndp && !c->no_dhcpv6) info("NDP/DHCPv6:"); else if (!c->no_ndp) @@ -973,9 +1040,10 @@ static void conf_print(const struct ctx *c) info(" assign: %s", inet_ntop(AF_INET6, &c->ip6.addr, buf6, sizeof(buf6))); info(" router: %s", - inet_ntop(AF_INET6, &c->ip6.gw, buf6, sizeof(buf6))); + inet_ntop(AF_INET6, &c->ip6.guest_gw, buf6, sizeof(buf6))); info(" our link-local: %s", - inet_ntop(AF_INET6, &c->ip6.addr_ll, buf6, sizeof(buf6))); + inet_ntop(AF_INET6, &c->ip6.our_tap_ll, + buf6, sizeof(buf6))); dns6: for (i = 0; !IN6_IS_ADDR_UNSPECIFIED(&c->ip6.dns[i]); i++) { @@ -1075,16 +1143,14 @@ static void conf_ugid(char *runas, uid_t *uid, gid_t *gid) return; /* ...otherwise use nobody:nobody */ - warn("Don't run as root. Changing to nobody..."); + warn("Started as root, will change to nobody."); { #ifndef GLIBC_NO_STATIC_NSS const struct passwd *pw; /* cppcheck-suppress getpwnamCalled */ pw = getpwnam("nobody"); - if (!pw) { - perror("getpwnam"); - exit(EXIT_FAILURE); - } + if (!pw) + die_perror("Can't get password file entry for nobody"); *uid = pw->pw_uid; *gid = pw->pw_gid; @@ -1096,6 +1162,87 @@ static void conf_ugid(char *runas, uid_t *uid, gid_t *gid) } /** + * conf_nat() - Parse --map-host-loopback or --map-guest-addr option + * @arg: String argument to option + * @addr4: IPv4 to update with parsed address + * @addr6: IPv6 to update with parsed address + * @no_map_gw: --no-map-gw flag, or NULL, updated for "none" argument + */ +static void conf_nat(const char *arg, struct in_addr *addr4, + struct in6_addr *addr6, int *no_map_gw) +{ + if (strcmp(arg, "none") == 0) { + *addr4 = in4addr_any; + *addr6 = in6addr_any; + if (no_map_gw) + *no_map_gw = 1; + } + + if (inet_pton(AF_INET6, arg, addr6) && + !IN6_IS_ADDR_UNSPECIFIED(addr6) && + !IN6_IS_ADDR_LOOPBACK(addr6) && + !IN6_IS_ADDR_MULTICAST(addr6)) + return; + + if (inet_pton(AF_INET, arg, addr4) && + !IN4_IS_ADDR_UNSPECIFIED(addr4) && + !IN4_IS_ADDR_LOOPBACK(addr4) && + !IN4_IS_ADDR_MULTICAST(addr4)) + return; + + die("Invalid address to remap to host: %s", optarg); +} + +/** + * conf_open_files() - Open files as requested by configuration + * @c: Execution context + */ +static void conf_open_files(struct ctx *c) +{ + if (c->mode != MODE_PASTA && c->fd_tap == -1) + c->fd_tap_listen = tap_sock_unix_open(c->sock_path); + + if (*c->pidfile) { + c->pidfile_fd = output_file_open(c->pidfile, O_WRONLY); + if (c->pidfile_fd < 0) + die_perror("Couldn't open PID file %s", c->pidfile); + } +} + +/** + * parse_mac - Parse a MAC address from a string + * @mac: Binary MAC address, initialised on success + * @str: String to parse + * + * Parses @str as an Ethernet MAC address stored in @mac on success. Exits on + * failure. + */ +static void parse_mac(unsigned char mac[ETH_ALEN], const char *str) +{ + size_t i; + + if (strlen(str) != (ETH_ALEN * 3 - 1)) + goto fail; + + for (i = 0; i < ETH_ALEN; i++) { + const char *octet = str + 3 * i; + unsigned long b; + char *end; + + errno = 0; + b = strtoul(octet, &end, 16); + if (b > UCHAR_MAX || errno || end != octet + 2 || + *end != ((i == ETH_ALEN - 1) ? '\0' : ':')) + goto fail; + mac[i] = b; + } + return; + +fail: + die("Invalid MAC address: %s", str); +} + +/** * conf() - Process command-line arguments and set configuration * @c: Execution context * @argc: Argument count @@ -1103,7 +1250,7 @@ static void conf_ugid(char *runas, uid_t *uid, gid_t *gid) */ void conf(struct ctx *c, int argc, char **argv) { - int netns_only = 0; + int netns_only = 0, no_map_gw = 0; const struct option options[] = { {"debug", no_argument, NULL, 'd' }, {"quiet", no_argument, NULL, 'q' }, @@ -1113,7 +1260,6 @@ void conf(struct ctx *c, int argc, char **argv) {"help", no_argument, NULL, 'h' }, {"socket", required_argument, NULL, 's' }, {"fd", required_argument, NULL, 'F' }, - {"socket-path", required_argument, NULL, 's' }, /* vhost-user mandatory */ {"ns-ifname", required_argument, NULL, 'I' }, {"pcap", required_argument, NULL, 'p' }, {"pid", required_argument, NULL, 'P' }, @@ -1133,7 +1279,8 @@ void conf(struct ctx *c, int argc, char **argv) {"no-dhcpv6", no_argument, &c->no_dhcpv6, 1 }, {"no-ndp", no_argument, &c->no_ndp, 1 }, {"no-ra", no_argument, &c->no_ra, 1 }, - {"no-map-gw", no_argument, &c->no_map_gw, 1 }, + {"freebind", no_argument, &c->freebind, 1 }, + {"no-map-gw", no_argument, &no_map_gw, 1 }, {"ipv4-only", no_argument, NULL, '4' }, {"ipv6-only", no_argument, NULL, '6' }, {"one-off", no_argument, NULL, '1' }, @@ -1143,7 +1290,6 @@ void conf(struct ctx *c, int argc, char **argv) {"udp-ns", required_argument, NULL, 'U' }, {"userns", required_argument, NULL, 2 }, {"netns", required_argument, NULL, 3 }, - {"netns-only", no_argument, &netns_only, 1 }, {"ns-mac-addr", required_argument, NULL, 4 }, {"dhcp-dns", no_argument, NULL, 5 }, {"no-dhcp-dns", no_argument, NULL, 6 }, @@ -1160,37 +1306,47 @@ void conf(struct ctx *c, int argc, char **argv) {"config-net", no_argument, NULL, 17 }, {"no-copy-routes", no_argument, NULL, 18 }, {"no-copy-addrs", no_argument, NULL, 19 }, - {"vhost-user", no_argument, NULL, 20 }, - {"print-capabilities", no_argument, NULL, 21 }, /* vhost-user mandatory */ + {"netns-only", no_argument, NULL, 20 }, + {"map-host-loopback", required_argument, NULL, 21 }, + {"map-guest-addr", required_argument, NULL, 22 }, + {"host-lo-to-ns-lo", no_argument, NULL, 23 }, + {"dns-host", required_argument, NULL, 24 }, + {"vhost-user", no_argument, NULL, 25 }, + /* vhost-user backend program convention */ + {"print-capabilities", no_argument, NULL, 26 }, + {"socket-path", required_argument, NULL, 's' }, { 0 }, }; + const char *logname = (c->mode == MODE_PASTA) ? "pasta" : "passt"; char userns[PATH_MAX] = { 0 }, netns[PATH_MAX] = { 0 }; bool copy_addrs_opt = false, copy_routes_opt = false; enum fwd_ports_mode fwd_default = FWD_NONE; bool v4_only = false, v6_only = false; - struct in6_addr *dns6 = c->ip6.dns; + unsigned dns4_idx = 0, dns6_idx = 0; struct fqdn *dnss = c->dns_search; - struct in_addr *dns4 = c->ip4.dns; unsigned int ifi4 = 0, ifi6 = 0; const char *logfile = NULL; const char *optstring; - int name, ret, b, i; size_t logsize = 0; char *runas = NULL; + long fd_tap_opt; + int name, ret; uid_t uid; gid_t gid; if (c->mode == MODE_PASTA) { c->no_dhcp_dns = c->no_dhcp_dns_search = 1; fwd_default = FWD_AUTO; - optstring = "dqfel:hF:I:p:P:m:a:n:M:g:i:o:D:S:46t:u:T:U:"; + optstring = "+dqfel:hF:I:p:P:m:a:n:M:g:i:o:D:S:46t:u:T:U:"; } else { - optstring = "dqfel:hs:F:p:P:m:a:n:M:g:i:o:D:S:461t:u:"; + optstring = "+dqfel:hs:F:p:P:m:a:n:M:g:i:o:D:S:461t:u:"; } - c->tcp.fwd_in.mode = c->tcp.fwd_out.mode = 0; - c->udp.fwd_in.f.mode = c->udp.fwd_out.f.mode = 0; + c->tcp.fwd_in.mode = c->tcp.fwd_out.mode = FWD_UNSET; + c->udp.fwd_in.mode = c->udp.fwd_out.mode = FWD_UNSET; + memcpy(c->our_tap_mac, MAC_OUR_LAA, ETH_ALEN); + optind = 0; do { name = getopt_long(argc, argv, optstring, options, NULL); @@ -1206,6 +1362,8 @@ void conf(struct ctx *c, int argc, char **argv) if (ret <= 0 || ret >= (int)sizeof(userns)) die("Invalid userns: %s", optarg); + netns_only = 0; + break; case 3: if (c->mode != MODE_PASTA) @@ -1217,14 +1375,7 @@ void conf(struct ctx *c, int argc, char **argv) if (c->mode != MODE_PASTA) die("--ns-mac-addr is for pasta mode only"); - for (i = 0; i < ETH_ALEN; i++) { - errno = 0; - b = strtol(optarg + (intptr_t)i * 3, NULL, 16); - if (b < 0 || b > UCHAR_MAX || errno) - die("Invalid MAC address: %s", optarg); - - c->mac_guest[i] = b; - } + parse_mac(c->guest_mac, optarg); break; case 5: if (c->mode != MODE_PASTA) @@ -1251,14 +1402,12 @@ void conf(struct ctx *c, int argc, char **argv) c->no_dhcp_dns_search = 1; break; case 9: - if (IN6_IS_ADDR_UNSPECIFIED(&c->ip6.dns_match) && - inet_pton(AF_INET6, optarg, &c->ip6.dns_match) && + if (inet_pton(AF_INET6, optarg, &c->ip6.dns_match) && !IN6_IS_ADDR_UNSPECIFIED(&c->ip6.dns_match) && !IN6_IS_ADDR_LOOPBACK(&c->ip6.dns_match)) break; - if (IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns_match) && - inet_pton(AF_INET, optarg, &c->ip4.dns_match) && + if (inet_pton(AF_INET, optarg, &c->ip4.dns_match) && !IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns_match) && !IN4_IS_ADDR_BROADCAST(&c->ip4.dns_match) && !IN4_IS_ADDR_LOOPBACK(&c->ip4.dns_match)) @@ -1273,24 +1422,13 @@ void conf(struct ctx *c, int argc, char **argv) c->no_netns_quit = 1; break; case 11: - if (c->trace) - die("Multiple --trace options given"); - - if (c->quiet) - die("Either --trace or --quiet"); - c->trace = c->debug = 1; + c->quiet = 0; break; case 12: - if (runas) - die("Multiple --runas options given"); - runas = optarg; break; case 13: - if (logsize) - die("Multiple --log-size options given"); - errno = 0; logsize = strtol(optarg, NULL, 0); @@ -1299,14 +1437,11 @@ void conf(struct ctx *c, int argc, char **argv) break; case 14: - fprintf(stdout, + FPRINTF(stdout, c->mode == MODE_PASTA ? "pasta " : "passt "); - fprintf(stdout, VERSION_BLOB); + FPRINTF(stdout, VERSION_BLOB); exit(EXIT_SUCCESS); case 15: - if (*c->ip4.ifname_out) - die("Redundant outbound interface: %s", optarg); - ret = snprintf(c->ip4.ifname_out, sizeof(c->ip4.ifname_out), "%s", optarg); if (ret <= 0 || ret >= (int)sizeof(c->ip4.ifname_out)) @@ -1314,13 +1449,11 @@ void conf(struct ctx *c, int argc, char **argv) break; case 16: - if (*c->ip6.ifname_out) - die("Redundant outbound interface: %s", optarg); - ret = snprintf(c->ip6.ifname_out, sizeof(c->ip6.ifname_out), "%s", optarg); if (ret <= 0 || ret >= (int)sizeof(c->ip6.ifname_out)) die("Invalid interface name: %s", optarg); + break; case 17: if (c->mode != MODE_PASTA) @@ -1333,94 +1466,96 @@ void conf(struct ctx *c, int argc, char **argv) die("--no-copy-routes is for pasta mode only"); warn("--no-copy-routes will be dropped soon"); - c->no_copy_routes = copy_routes_opt = true; + c->ip4.no_copy_routes = c->ip6.no_copy_routes = true; + copy_routes_opt = true; break; case 19: if (c->mode != MODE_PASTA) die("--no-copy-addrs is for pasta mode only"); warn("--no-copy-addrs will be dropped soon"); - c->no_copy_addrs = copy_addrs_opt = true; + c->ip4.no_copy_addrs = c->ip6.no_copy_addrs = true; + copy_addrs_opt = true; break; case 20: + if (c->mode != MODE_PASTA) + die("--netns-only is for pasta mode only"); + + netns_only = 1; + *userns = 0; + break; + case 21: + conf_nat(optarg, &c->ip4.map_host_loopback, + &c->ip6.map_host_loopback, &no_map_gw); + break; + case 22: + conf_nat(optarg, &c->ip4.map_guest_addr, + &c->ip6.map_guest_addr, NULL); + break; + case 23: + if (c->mode != MODE_PASTA) + die("--host-lo-to-ns-lo is for pasta mode only"); + c->host_lo_to_ns_lo = 1; + break; + case 24: + if (inet_pton(AF_INET6, optarg, &c->ip6.dns_host) && + !IN6_IS_ADDR_UNSPECIFIED(&c->ip6.dns_host)) + break; + + if (inet_pton(AF_INET, optarg, &c->ip4.dns_host) && + !IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns_host) && + !IN4_IS_ADDR_BROADCAST(&c->ip4.dns_host)) + break; + + die("Invalid host nameserver address: %s", optarg); + case 25: if (c->mode == MODE_PASTA) { err("--vhost-user is for passt mode only"); - usage(argv[0]); + usage(argv[0], stdout, EXIT_SUCCESS); } c->mode = MODE_VU; break; - case 21: + case 26: vu_print_capabilities(); break; case 'd': - if (c->debug) - die("Multiple --debug options given"); - - if (c->quiet) - die("Either --debug or --quiet"); - c->debug = 1; + c->quiet = 0; break; case 'e': - if (logfile) - die("Can't log to both file and stderr"); - - if (c->force_stderr) - die("Multiple --stderr options given"); - - c->force_stderr = 1; + warn("--stderr will be dropped soon"); break; case 'l': - if (c->force_stderr) - die("Can't log to both stderr and file"); - - if (logfile) - die("Multiple --log-file options given"); - logfile = optarg; break; case 'q': - if (c->quiet) - die("Multiple --quiet options given"); - - if (c->debug) - die("Either --debug or --quiet"); - c->quiet = 1; + c->debug = c->trace = 0; break; case 'f': - if (c->foreground) - die("Multiple --foreground options given"); - c->foreground = 1; break; case 's': - if (*c->sock_path) - die("Multiple --socket options given"); - - ret = snprintf(c->sock_path, UNIX_SOCK_MAX - 1, "%s", + ret = snprintf(c->sock_path, sizeof(c->sock_path), "%s", optarg); if (ret <= 0 || ret >= (int)sizeof(c->sock_path)) die("Invalid socket path: %s", optarg); + c->fd_tap = -1; break; case 'F': - if (c->fd_tap >= 0) - die("Multiple --fd options given"); - errno = 0; - c->fd_tap = strtol(optarg, NULL, 0); + fd_tap_opt = strtol(optarg, NULL, 0); - if (c->fd_tap < 0 || errno) + if (errno || + fd_tap_opt <= STDERR_FILENO || fd_tap_opt > INT_MAX) die("Invalid --fd: %s", optarg); + c->fd_tap = fd_tap_opt; c->one_off = true; - + *c->sock_path = 0; break; case 'I': - if (*c->pasta_ifn) - die("Multiple --ns-ifname options given"); - ret = snprintf(c->pasta_ifn, IFNAMSIZ, "%s", optarg); if (ret <= 0 || ret >= IFNAMSIZ) @@ -1428,28 +1563,19 @@ void conf(struct ctx *c, int argc, char **argv) break; case 'p': - if (*c->pcap) - die("Multiple --pcap options given"); - ret = snprintf(c->pcap, sizeof(c->pcap), "%s", optarg); if (ret <= 0 || ret >= (int)sizeof(c->pcap)) die("Invalid pcap path: %s", optarg); break; case 'P': - if (*c->pid_file) - die("Multiple --pid options given"); - - ret = snprintf(c->pid_file, sizeof(c->pid_file), "%s", + ret = snprintf(c->pidfile, sizeof(c->pidfile), "%s", optarg); - if (ret <= 0 || ret >= (int)sizeof(c->pid_file)) + if (ret <= 0 || ret >= (int)sizeof(c->pidfile)) die("Invalid PID file: %s", optarg); break; case 'm': - if (c->mtu) - die("Multiple --mtu options given"); - errno = 0; c->mtu = strtol(optarg, NULL, 0); @@ -1464,25 +1590,26 @@ void conf(struct ctx *c, int argc, char **argv) break; case 'a': - if (c->mode == MODE_PASTA) - c->no_copy_addrs = 1; - - if (IN6_IS_ADDR_UNSPECIFIED(&c->ip6.addr) && - inet_pton(AF_INET6, optarg, &c->ip6.addr) && + if (inet_pton(AF_INET6, optarg, &c->ip6.addr) && !IN6_IS_ADDR_UNSPECIFIED(&c->ip6.addr) && !IN6_IS_ADDR_LOOPBACK(&c->ip6.addr) && !IN6_IS_ADDR_V4MAPPED(&c->ip6.addr) && !IN6_IS_ADDR_V4COMPAT(&c->ip6.addr) && - !IN6_IS_ADDR_MULTICAST(&c->ip6.addr)) + !IN6_IS_ADDR_MULTICAST(&c->ip6.addr)) { + if (c->mode == MODE_PASTA) + c->ip6.no_copy_addrs = true; break; + } - if (IN4_IS_ADDR_UNSPECIFIED(&c->ip4.addr) && - inet_pton(AF_INET, optarg, &c->ip4.addr) && + if (inet_pton(AF_INET, optarg, &c->ip4.addr) && !IN4_IS_ADDR_UNSPECIFIED(&c->ip4.addr) && !IN4_IS_ADDR_BROADCAST(&c->ip4.addr) && !IN4_IS_ADDR_LOOPBACK(&c->ip4.addr) && - !IN4_IS_ADDR_MULTICAST(&c->ip4.addr)) + !IN4_IS_ADDR_MULTICAST(&c->ip4.addr)) { + if (c->mode == MODE_PASTA) + c->ip4.no_copy_addrs = true; break; + } die("Invalid address: %s", optarg); break; @@ -1493,45 +1620,34 @@ void conf(struct ctx *c, int argc, char **argv) break; case 'M': - for (i = 0; i < ETH_ALEN; i++) { - errno = 0; - b = strtol(optarg + (intptr_t)i * 3, NULL, 16); - if (b < 0 || b > UCHAR_MAX || errno) - die("Invalid MAC address: %s", optarg); - - c->mac[i] = b; - } + parse_mac(c->our_tap_mac, optarg); break; case 'g': - if (c->mode == MODE_PASTA) - c->no_copy_routes = 1; - - if (IN6_IS_ADDR_UNSPECIFIED(&c->ip6.gw) && - inet_pton(AF_INET6, optarg, &c->ip6.gw) && - !IN6_IS_ADDR_UNSPECIFIED(&c->ip6.gw) && - !IN6_IS_ADDR_LOOPBACK(&c->ip6.gw)) + if (inet_pton(AF_INET6, optarg, &c->ip6.guest_gw) && + !IN6_IS_ADDR_UNSPECIFIED(&c->ip6.guest_gw) && + !IN6_IS_ADDR_LOOPBACK(&c->ip6.guest_gw)) { + if (c->mode == MODE_PASTA) + c->ip6.no_copy_routes = true; break; + } - if (IN4_IS_ADDR_UNSPECIFIED(&c->ip4.gw) && - inet_pton(AF_INET, optarg, &c->ip4.gw) && - !IN4_IS_ADDR_UNSPECIFIED(&c->ip4.gw) && - !IN4_IS_ADDR_BROADCAST(&c->ip4.gw) && - !IN4_IS_ADDR_LOOPBACK(&c->ip4.gw)) + if (inet_pton(AF_INET, optarg, &c->ip4.guest_gw) && + !IN4_IS_ADDR_UNSPECIFIED(&c->ip4.guest_gw) && + !IN4_IS_ADDR_BROADCAST(&c->ip4.guest_gw) && + !IN4_IS_ADDR_LOOPBACK(&c->ip4.guest_gw)) { + if (c->mode == MODE_PASTA) + c->ip4.no_copy_routes = true; break; + } die("Invalid gateway address: %s", optarg); break; case 'i': - if (ifi4 || ifi6) - die("Redundant interface: %s", optarg); - if (!(ifi4 = ifi6 = if_nametoindex(optarg))) - die("Invalid interface name %s: %s", optarg, - strerror(errno)); + die_perror("Invalid interface name %s", optarg); break; case 'o': - if (IN6_IS_ADDR_UNSPECIFIED(&c->ip6.addr_out) && - inet_pton(AF_INET6, optarg, &c->ip6.addr_out) && + if (inet_pton(AF_INET6, optarg, &c->ip6.addr_out) && !IN6_IS_ADDR_UNSPECIFIED(&c->ip6.addr_out) && !IN6_IS_ADDR_LOOPBACK(&c->ip6.addr_out) && !IN6_IS_ADDR_V4MAPPED(&c->ip6.addr_out) && @@ -1539,8 +1655,7 @@ void conf(struct ctx *c, int argc, char **argv) !IN6_IS_ADDR_MULTICAST(&c->ip6.addr_out)) break; - if (IN4_IS_ADDR_UNSPECIFIED(&c->ip4.addr_out) && - inet_pton(AF_INET, optarg, &c->ip4.addr_out) && + if (inet_pton(AF_INET, optarg, &c->ip4.addr_out) && !IN4_IS_ADDR_UNSPECIFIED(&c->ip4.addr_out) && !IN4_IS_ADDR_BROADCAST(&c->ip4.addr_out) && !IN4_IS_ADDR_MULTICAST(&c->ip4.addr_out)) @@ -1549,49 +1664,16 @@ void conf(struct ctx *c, int argc, char **argv) die("Invalid or redundant outbound address: %s", optarg); break; - case 'D': - if (!strcmp(optarg, "none")) { - if (c->no_dns) - die("Redundant DNS options"); - - if (dns4 - c->ip4.dns || dns6 - c->ip6.dns) - die("Conflicting DNS options"); - - c->no_dns = 1; - break; - } - - if (c->no_dns) - die("Conflicting DNS options"); - - if (dns4 - &c->ip4.dns[0] < ARRAY_SIZE(c->ip4.dns) && - inet_pton(AF_INET, optarg, dns4)) { - dns4++; - break; - } - - if (dns6 - &c->ip6.dns[0] < ARRAY_SIZE(c->ip6.dns) && - inet_pton(AF_INET6, optarg, dns6)) { - dns6++; - break; - } - - die("Cannot use DNS address %s", optarg); - break; case 'S': if (!strcmp(optarg, "none")) { - if (c->no_dns_search) - die("Redundant DNS search options"); + c->no_dns_search = 1; - if (dnss != c->dns_search) - die("Conflicting DNS search options"); + memset(c->dns_search, 0, sizeof(c->dns_search)); - c->no_dns_search = 1; break; } - if (c->no_dns_search) - die("Conflicting DNS search options"); + c->no_dns_search = 0; if (dnss - c->dns_search < ARRAY_SIZE(c->dns_search)) { ret = snprintf(dnss->n, sizeof(*c->dns_search), @@ -1607,42 +1689,35 @@ void conf(struct ctx *c, int argc, char **argv) break; case '4': v4_only = true; + v6_only = false; break; case '6': v6_only = true; + v4_only = false; break; case '1': if (c->mode == MODE_PASTA) die("--one-off is for passt mode only"); - if (c->one_off) - die("Redundant --one-off option"); - c->one_off = true; break; case 't': case 'u': case 'T': case 'U': + case 'D': /* Handle these later, once addresses are configured */ break; case 'h': - log_to_stdout = 1; - print_usage(argv[0], EXIT_SUCCESS); + usage(argv[0], stdout, EXIT_SUCCESS); break; case '?': default: - usage(argv[0]); + usage(argv[0], stderr, EXIT_FAILURE); break; } } while (name != -1); - if (v4_only && v6_only) - die("Options ipv4-only and ipv6-only are mutually exclusive"); - - if (*c->sock_path && c->fd_tap >= 0) - die("Options --socket and --fd are mutually exclusive"); - if (c->mode == MODE_PASTA && !c->pasta_conf_ns) { if (copy_routes_opt) die("--no-copy-routes needs --config-net"); @@ -1658,14 +1733,11 @@ void conf(struct ctx *c, int argc, char **argv) conf_ugid(runas, &uid, &gid); - if (logfile) { - logfile_init(c->mode == MODE_PASTA ? "pasta" : "passt", - logfile, logsize); - } + if (logfile) + logfile_init(logname, logfile, logsize); + else + __openlog(logname, 0, LOG_DAEMON); - /* Once the log mask is not LOG_EARLY, we will no longer log to stderr - * if there was a log file specified. - */ if (c->debug) __setlogmask(LOG_UPTO(LOG_DEBUG)); else if (c->quiet) @@ -1673,32 +1745,77 @@ void conf(struct ctx *c, int argc, char **argv) else __setlogmask(LOG_UPTO(LOG_INFO)); + log_conf_parsed = true; /* Stop printing everything */ + nl_sock_init(c, false); if (!v6_only) - c->ifi4 = conf_ip4(ifi4, &c->ip4, c->mac); + c->ifi4 = conf_ip4(ifi4, &c->ip4); if (!v4_only) - c->ifi6 = conf_ip6(ifi6, &c->ip6, c->mac); + c->ifi6 = conf_ip6(ifi6, &c->ip6); if ((!c->ifi4 && !c->ifi6) || (*c->ip4.ifname_out && !c->ifi4) || (*c->ip6.ifname_out && !c->ifi6)) die("External interface not usable"); - if (c->ifi4 && IN4_IS_ADDR_UNSPECIFIED(&c->ip4.gw)) - c->no_map_gw = c->no_dhcp = 1; + if (c->ifi4 && !no_map_gw && + IN4_IS_ADDR_UNSPECIFIED(&c->ip4.map_host_loopback)) + c->ip4.map_host_loopback = c->ip4.guest_gw; - if (c->ifi6 && IN6_IS_ADDR_UNSPECIFIED(&c->ip6.gw)) - c->no_map_gw = 1; + if (c->ifi6 && !no_map_gw && + IN6_IS_ADDR_UNSPECIFIED(&c->ip6.map_host_loopback)) + c->ip6.map_host_loopback = c->ip6.guest_gw; - /* Inbound port options can be parsed now (after IPv4/IPv6 settings) */ + if (c->ifi4 && IN4_IS_ADDR_UNSPECIFIED(&c->ip4.guest_gw)) + c->no_dhcp = 1; + + /* Inbound port options & DNS can be parsed now (after IPv4/IPv6 + * settings) + */ + fwd_probe_ephemeral(); udp_portmap_clear(); - optind = 1; + optind = 0; do { name = getopt_long(argc, argv, optstring, options, NULL); - if (name == 't') + if (name == 't') { conf_ports(c, name, optarg, &c->tcp.fwd_in); - else if (name == 'u') - conf_ports(c, name, optarg, &c->udp.fwd_in.f); + } else if (name == 'u') { + conf_ports(c, name, optarg, &c->udp.fwd_in); + } else if (name == 'D') { + struct in6_addr dns6_tmp; + struct in_addr dns4_tmp; + + if (!strcmp(optarg, "none")) { + c->no_dns = 1; + + dns4_idx = 0; + memset(c->ip4.dns, 0, sizeof(c->ip4.dns)); + c->ip4.dns[0] = (struct in_addr){ 0 }; + c->ip4.dns_match = (struct in_addr){ 0 }; + c->ip4.dns_host = (struct in_addr){ 0 }; + + dns6_idx = 0; + memset(c->ip6.dns, 0, sizeof(c->ip6.dns)); + c->ip6.dns_match = (struct in6_addr){ 0 }; + c->ip6.dns_host = (struct in6_addr){ 0 }; + + continue; + } + + c->no_dns = 0; + + if (inet_pton(AF_INET, optarg, &dns4_tmp)) { + dns4_idx += add_dns4(c, &dns4_tmp, dns4_idx); + continue; + } + + if (inet_pton(AF_INET6, optarg, &dns6_tmp)) { + dns6_idx += add_dns6(c, &dns6_tmp, dns6_idx); + continue; + } + + die("Cannot use DNS address %s", optarg); + } } while (name != -1); if (c->mode == MODE_PASTA) @@ -1706,6 +1823,8 @@ void conf(struct ctx *c, int argc, char **argv) else if (optind != argc) die("Extra non-option argument: %s", argv[optind]); + conf_open_files(c); /* Before any possible setuid() / setgid() */ + isolate_user(uid, gid, !netns_only, userns, c->mode); if (c->pasta_conf_ns) @@ -1724,14 +1843,14 @@ void conf(struct ctx *c, int argc, char **argv) nl_sock_init(c, true); /* ...and outbound port options now that namespaces are set up. */ - optind = 1; + optind = 0; do { name = getopt_long(argc, argv, optstring, options, NULL); if (name == 'T') conf_ports(c, name, optarg, &c->tcp.fwd_out); else if (name == 'U') - conf_ports(c, name, optarg, &c->udp.fwd_out.f); + conf_ports(c, name, optarg, &c->udp.fwd_out); } while (name != -1); if (!c->ifi4) @@ -1758,10 +1877,10 @@ void conf(struct ctx *c, int argc, char **argv) c->tcp.fwd_in.mode = fwd_default; if (!c->tcp.fwd_out.mode) c->tcp.fwd_out.mode = fwd_default; - if (!c->udp.fwd_in.f.mode) - c->udp.fwd_in.f.mode = fwd_default; - if (!c->udp.fwd_out.f.mode) - c->udp.fwd_out.f.mode = fwd_default; + if (!c->udp.fwd_in.mode) + c->udp.fwd_in.mode = fwd_default; + if (!c->udp.fwd_out.mode) + c->udp.fwd_out.mode = fwd_default; fwd_scan_ports_init(c); diff --git a/contrib/apparmor/abstractions/passt b/contrib/apparmor/abstractions/passt index 6bb25e0..43fd63f 100644 --- a/contrib/apparmor/abstractions/passt +++ b/contrib/apparmor/abstractions/passt @@ -26,13 +26,16 @@ capability sys_ptrace, / r, # isolate_prefork(), isolation.c - mount options=(rw, runbindable) /, + mount options=(rw, runbindable) -> /, + mount "" -> "/", mount "" -> "/tmp/", pivot_root "/tmp/" -> "/tmp/", umount "/", owner @{PROC}/@{pid}/uid_map r, # conf_ugid() + @{PROC}/sys/net/ipv4/ip_local_port_range r, # fwd_probe_ephemeral() + network netlink raw, # nl_sock_init_do(), netlink.c network inet stream, # tcp.c diff --git a/contrib/apparmor/abstractions/pasta b/contrib/apparmor/abstractions/pasta index a890391..9f73bee 100644 --- a/contrib/apparmor/abstractions/pasta +++ b/contrib/apparmor/abstractions/pasta @@ -27,8 +27,9 @@ @{PROC}/@{pid}/net/udp r, @{PROC}/@{pid}/net/udp6 r, - @{run}/user/@{uid}/netns/* r, # pasta_open_ns(), pasta.c + @{run}/user/@{uid}/** rw, # pasta_open_ns() + @{PROC}/[0-9]*/ns/ r, # pasta_netns_quit_init(), @{PROC}/[0-9]*/ns/net r, # pasta_wait_for_ns(), @{PROC}/[0-9]*/ns/user r, # conf_pasta_ns() @@ -42,3 +43,5 @@ /{usr/,}bin/** Ux, /usr/bin/pasta.avx2 ix, # arch_avx2_exec(), arch.c + + ptrace r, # pasta_open_ns() diff --git a/contrib/apparmor/usr.bin.passt b/contrib/apparmor/usr.bin.passt index 564f82f..9568189 100644 --- a/contrib/apparmor/usr.bin.passt +++ b/contrib/apparmor/usr.bin.passt @@ -19,9 +19,12 @@ profile passt /usr/bin/passt{,.avx2} { include <abstractions/passt> # Alternatively: include <abstractions/user-tmp> - owner /tmp/** w, # tap_sock_unix_init(), pcap(), - # write_pidfile(), + owner /tmp/** w, # tap_sock_unix_open(), + # tap_sock_unix_init(), pcap(), + # pidfile_open(), + # pidfile_write(), # logfile_init() - owner @{HOME}/** w, # pcap(), write_pidfile() + owner @{HOME}/** w, # pcap(), pidfile_open(), + # pidfile_write() } diff --git a/contrib/apparmor/usr.bin.pasta b/contrib/apparmor/usr.bin.pasta index e5ee4df..2483968 100644 --- a/contrib/apparmor/usr.bin.pasta +++ b/contrib/apparmor/usr.bin.pasta @@ -19,9 +19,13 @@ profile pasta /usr/bin/pasta{,.avx2} flags=(attach_disconnected) { include <abstractions/pasta> # Alternatively: include <abstractions/user-tmp> - owner /tmp/** w, # tap_sock_unix_init(), pcap(), - # write_pidfile(), - # logfile_init() + /tmp/** rw, # tap_sock_unix_open(), + # tap_sock_unix_init(), pcap(), + # pidfile_open(), + # pidfile_write(), + # logfile_init(), + # pasta_open_ns() - owner @{HOME}/** w, # pcap(), write_pidfile() + owner @{HOME}/** w, # pcap(), pidfile_open(), + # pidfile_write() } diff --git a/contrib/fedora/passt.spec b/contrib/fedora/passt.spec index 825cd76..7950fb9 100644 --- a/contrib/fedora/passt.spec +++ b/contrib/fedora/passt.spec @@ -14,7 +14,7 @@ Name: passt Version: {{{ git_version }}} Release: 1%{?dist} Summary: User-mode networking daemons for virtual machines and namespaces -License: GPLv2+ and BSD +License: GPL-2.0-or-later AND BSD-3-Clause Group: System Environment/Daemons URL: https://passt.top/ Source: https://passt.top/passt/snapshot/passt-%{git_hash}.tar.xz diff --git a/contrib/fedora/rpkg.macros b/contrib/fedora/rpkg.macros index c226c84..c98b791 100644 --- a/contrib/fedora/rpkg.macros +++ b/contrib/fedora/rpkg.macros @@ -29,7 +29,11 @@ function passt_git_changelog_entry { [ -z "${__from}" ] && __from="$(git rev-list --max-parents=0 HEAD)" __date="$(git log --pretty="format:%cI" "${__to}" -1)" - __author="$(git log -1 --pretty="format:%an <%ae>" ${__to} -- contrib/fedora)" + __author="Stefano Brivio <sbrivio@redhat.com>" + # Use: + # __author="$(git log -1 --pretty="format:%an <%ae>" ${__to} -- contrib/fedora)" + # if you want the author of changelog entries to match the latest + # author for contrib/fedora printf "* %s %s - %s\n" "$(date "+%a %b %e %Y" -d "${__date}")" "${__author}" "$(git_version "${__to}")-1" diff --git a/contrib/selinux/passt.te b/contrib/selinux/passt.te index bbb0917..c6cea34 100644 --- a/contrib/selinux/passt.te +++ b/contrib/selinux/passt.te @@ -47,9 +47,8 @@ require { type port_t; type http_port_t; - type passwd_file_t; - class netlink_route_socket { bind create nlmsg_read }; + type sysctl_net_t; class capability { sys_tty_config setuid setgid }; class cap_userns { setpcap sys_admin sys_ptrace }; @@ -95,8 +94,7 @@ allow passt_t self:capability { sys_tty_config setpcap net_bind_service setuid s allow passt_t self:cap_userns { setpcap sys_admin sys_ptrace }; allow passt_t self:user_namespace create; -allow passt_t passwd_file_t:file read_file_perms; -sssd_search_lib(passt_t) +auth_read_passwd(passt_t) allow passt_t proc_net_t:file read; allow passt_t net_conf_t:file { open read }; @@ -104,6 +102,8 @@ allow passt_t net_conf_t:lnk_file read; allow passt_t tmp_t:sock_file { create unlink write }; allow passt_t self:netlink_route_socket { bind create nlmsg_read read write setopt }; kernel_search_network_sysctl(passt_t) +allow passt_t sysctl_net_t:dir search; +allow passt_t sysctl_net_t:file { open read }; corenet_tcp_bind_all_nodes(passt_t) corenet_udp_bind_all_nodes(passt_t) diff --git a/contrib/selinux/pasta.te b/contrib/selinux/pasta.te index 0ceda06..69be081 100644 --- a/contrib/selinux/pasta.te +++ b/contrib/selinux/pasta.te @@ -68,9 +68,6 @@ require { type system_dbusd_t; type systemd_hostnamed_t; type systemd_systemctl_exec_t; - type passwd_file_t; - type sssd_public_t; - type sssd_var_lib_t; class dbus send_msg; class system module_request; class system status; @@ -115,8 +112,7 @@ allow pasta_t self:capability { setpcap net_bind_service sys_tty_config dac_read allow pasta_t self:cap_userns { setpcap sys_admin sys_ptrace net_admin net_bind_service }; allow pasta_t self:user_namespace create; -allow pasta_t passwd_file_t:file read_file_perms; -sssd_search_lib(pasta_t) +auth_read_passwd(pasta_t) domain_auto_trans(pasta_t, bin_t, unconfined_t); domain_auto_trans(pasta_t, shell_exec_t, unconfined_t); @@ -178,12 +174,9 @@ allow pasta_t init_t:system status; allow pasta_t unconfined_t:dir search; allow pasta_t unconfined_t:file read; allow pasta_t unconfined_t:lnk_file read; -allow pasta_t passwd_file_t:file { getattr open read }; allow pasta_t self:process { setpgid setcap }; allow pasta_t shell_exec_t:file { execute execute_no_trans map }; -allow pasta_t sssd_var_lib_t:dir search; -allow pasta_t sssd_public_t:dir search; allow pasta_t hostname_exec_t:file { execute execute_no_trans getattr open read map }; allow pasta_t system_dbusd_t:unix_stream_socket connectto; allow pasta_t system_dbusd_t:dbus send_msg; @@ -196,7 +189,7 @@ allow pasta_t ifconfig_var_run_t:dir { read search watch }; allow pasta_t self:tun_socket create; allow pasta_t tun_tap_device_t:chr_file { ioctl open read write }; allow pasta_t sysctl_net_t:dir search; -allow pasta_t sysctl_net_t:file { open write }; +allow pasta_t sysctl_net_t:file { open read write }; allow pasta_t kernel_t:system module_request; allow pasta_t nsfs_t:file read; @@ -211,3 +204,4 @@ allow pasta_t ifconfig_t:process { noatsecure rlimitinh siginh }; allow pasta_t netutils_t:process { noatsecure rlimitinh siginh }; allow pasta_t ping_t:process { noatsecure rlimitinh siginh }; allow pasta_t user_tty_device_t:chr_file { append read write }; +allow pasta_t user_devpts_t:chr_file { append read write }; @@ -275,7 +275,8 @@ static void opt_set_dns_search(const struct ctx *c, size_t max_len) */ int dhcp(const struct ctx *c, const struct pool *p) { - size_t mlen, len, offset = 0, opt_len, opt_off = 0; + size_t mlen, dlen, offset = 0, opt_len, opt_off = 0; + char macstr[ETH_ADDRSTRLEN]; const struct ethhdr *eh; const struct iphdr *iph; const struct udphdr *uh; @@ -340,26 +341,26 @@ int dhcp(const struct ctx *c, const struct pool *p) return -1; } - info(" from %02x:%02x:%02x:%02x:%02x:%02x", - m->chaddr[0], m->chaddr[1], m->chaddr[2], - m->chaddr[3], m->chaddr[4], m->chaddr[5]); + info(" from %s", eth_ntop(m->chaddr, macstr, sizeof(macstr))); m->yiaddr = c->ip4.addr; mask.s_addr = htonl(0xffffffff << (32 - c->ip4.prefix_len)); - memcpy(opts[1].s, &mask, sizeof(mask)); - memcpy(opts[3].s, &c->ip4.gw, sizeof(c->ip4.gw)); - memcpy(opts[54].s, &c->ip4.gw, sizeof(c->ip4.gw)); + memcpy(opts[1].s, &mask, sizeof(mask)); + memcpy(opts[3].s, &c->ip4.guest_gw, sizeof(c->ip4.guest_gw)); + memcpy(opts[54].s, &c->ip4.our_tap_addr, sizeof(c->ip4.our_tap_addr)); /* If the gateway is not on the assigned subnet, send an option 121 * (Classless Static Routing) adding a dummy route to it. */ if ((c->ip4.addr.s_addr & mask.s_addr) - != (c->ip4.gw.s_addr & mask.s_addr)) { + != (c->ip4.guest_gw.s_addr & mask.s_addr)) { /* a.b.c.d/32:0.0.0.0, 0:a.b.c.d */ opts[121].slen = 14; opts[121].s[0] = 32; - memcpy(opts[121].s + 1, &c->ip4.gw, sizeof(c->ip4.gw)); - memcpy(opts[121].s + 10, &c->ip4.gw, sizeof(c->ip4.gw)); + memcpy(opts[121].s + 1, + &c->ip4.guest_gw, sizeof(c->ip4.guest_gw)); + memcpy(opts[121].s + 10, + &c->ip4.guest_gw, sizeof(c->ip4.guest_gw)); } if (c->mtu != -1) { @@ -377,8 +378,8 @@ int dhcp(const struct ctx *c, const struct pool *p) if (!c->no_dhcp_dns_search) opt_set_dns_search(c, sizeof(m->o)); - len = offsetof(struct msg, o) + fill(m); - tap_udp4_send(c, c->ip4.gw, 67, c->ip4.addr, 68, m, len); + dlen = offsetof(struct msg, o) + fill(m); + tap_udp4_send(c, c->ip4.our_tap_addr, 67, c->ip4.addr, 68, m, dlen); return 1; } @@ -296,45 +296,42 @@ static struct opt_hdr *dhcpv6_opt(const struct pool *p, size_t *offset, static struct opt_hdr *dhcpv6_ia_notonlink(const struct pool *p, struct in6_addr *la) { + int ia_types[2] = { OPT_IA_NA, OPT_IA_TA }, *ia_type; + const struct opt_ia_addr *opt_addr; char buf[INET6_ADDRSTRLEN]; struct in6_addr req_addr; - struct opt_hdr *ia, *h; + const struct opt_hdr *h; + struct opt_hdr *ia; size_t offset; - int ia_type; - ia_type = OPT_IA_NA; -ia_ta: - offset = 0; - while ((ia = dhcpv6_opt(p, &offset, ia_type))) { - if (ntohs(ia->l) < OPT_VSIZE(ia_na)) - return NULL; + foreach(ia_type, ia_types) { + offset = 0; + while ((ia = dhcpv6_opt(p, &offset, *ia_type))) { + if (ntohs(ia->l) < OPT_VSIZE(ia_na)) + return NULL; - offset += sizeof(struct opt_ia_na); + offset += sizeof(struct opt_ia_na); - while ((h = dhcpv6_opt(p, &offset, OPT_IAAADR))) { - struct opt_ia_addr *opt_addr = (struct opt_ia_addr *)h; + while ((h = dhcpv6_opt(p, &offset, OPT_IAAADR))) { + if (ntohs(h->l) != OPT_VSIZE(ia_addr)) + return NULL; - if (ntohs(h->l) != OPT_VSIZE(ia_addr)) - return NULL; + opt_addr = (const struct opt_ia_addr *)h; + req_addr = opt_addr->addr; + if (!IN6_ARE_ADDR_EQUAL(la, &req_addr)) + goto err; - memcpy(&req_addr, &opt_addr->addr, sizeof(req_addr)); - if (!IN6_ARE_ADDR_EQUAL(la, &req_addr)) { - info("DHCPv6: requested address %s not on link", - inet_ntop(AF_INET6, &req_addr, - buf, sizeof(buf))); - return ia; + offset += sizeof(struct opt_ia_addr); } - - offset += sizeof(struct opt_ia_addr); } } - if (ia_type == OPT_IA_NA) { - ia_type = OPT_IA_TA; - goto ia_ta; - } - return NULL; + +err: + info("DHCPv6: requested address %s not on link", + inet_ntop(AF_INET6, &req_addr, buf, sizeof(buf))); + return ia; } /** @@ -363,7 +360,7 @@ static size_t dhcpv6_dns_fill(const struct ctx *c, char *buf, int offset) srv->hdr.l = 0; } - memcpy(&srv->addr[i], &c->ip6.dns[i], sizeof(srv->addr[i])); + srv->addr[i] = c->ip6.dns[i]; srv->hdr.l += sizeof(srv->addr[i]); offset += sizeof(srv->addr[i]); } @@ -426,11 +423,11 @@ search: int dhcpv6(struct ctx *c, const struct pool *p, const struct in6_addr *saddr, const struct in6_addr *daddr) { - struct opt_hdr *ia, *bad_ia, *client_id; - const struct opt_hdr *server_id; + const struct opt_hdr *client_id, *server_id, *ia; const struct in6_addr *src; const struct msg_hdr *mh; const struct udphdr *uh; + struct opt_hdr *bad_ia; size_t mlen, n; uh = packet_get(p, 0, 0, sizeof(*uh), &mlen); @@ -451,10 +448,7 @@ int dhcpv6(struct ctx *c, const struct pool *p, c->ip6.addr_ll_seen = *saddr; - if (IN6_IS_ADDR_LINKLOCAL(&c->ip6.gw)) - src = &c->ip6.gw; - else - src = &c->ip6.addr_ll; + src = &c->ip6.our_tap_ll; mh = packet_get(p, 0, sizeof(*uh), sizeof(*mh), NULL); if (!mh) @@ -574,8 +568,10 @@ void dhcpv6_init(const struct ctx *c) resp.server_id.duid_time = duid_time; resp_not_on_link.server_id.duid_time = duid_time; - memcpy(resp.server_id.duid_lladdr, c->mac, sizeof(c->mac)); - memcpy(resp_not_on_link.server_id.duid_lladdr, c->mac, sizeof(c->mac)); + memcpy(resp.server_id.duid_lladdr, + c->our_tap_mac, sizeof(c->our_tap_mac)); + memcpy(resp_not_on_link.server_id.duid_lladdr, + c->our_tap_mac, sizeof(c->our_tap_mac)); resp.ia_addr.addr = c->ip6.addr; } diff --git a/doc/platform-requirements/.gitignore b/doc/platform-requirements/.gitignore new file mode 100644 index 0000000..3b5a10a --- /dev/null +++ b/doc/platform-requirements/.gitignore @@ -0,0 +1,3 @@ +/reuseaddr-priority +/recv-zero +/udp-close-dup diff --git a/doc/platform-requirements/Makefile b/doc/platform-requirements/Makefile new file mode 100644 index 0000000..6a7d374 --- /dev/null +++ b/doc/platform-requirements/Makefile @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Copyright Red Hat +# Author: David Gibson <david@gibson.dropbear.id.au> + +TARGETS = reuseaddr-priority recv-zero udp-close-dup +SRCS = reuseaddr-priority.c recv-zero.c udp-close-dup.c +CFLAGS = -Wall + +all: cppcheck clang-tidy $(TARGETS:%=check-%) + +$(TARGETS): %: %.c common.c common.h + +check-%: % + ./$< + +cppcheck: + cppcheck --std=c11 --error-exitcode=1 --enable=all --force \ + --check-level=exhaustive --inline-suppr \ + --inconclusive --library=posix --quiet \ + --suppress=missingIncludeSystem \ + $(SRCS) + +clang-tidy: + clang-tidy --checks=*,\ + -altera-id-dependent-backward-branch,\ + -altera-unroll-loops,\ + -bugprone-easily-swappable-parameters,\ + -clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling,\ + -concurrency-mt-unsafe,\ + -cppcoreguidelines-avoid-non-const-global-variables,\ + -cppcoreguidelines-init-variables,\ + -cppcoreguidelines-macro-to-enum,\ + -google-readability-braces-around-statements,\ + -hicpp-braces-around-statements,\ + -llvmlibc-restrict-system-libc-headers,\ + -misc-include-cleaner,\ + -modernize-macro-to-enum,\ + -readability-braces-around-statements,\ + -readability-identifier-length,\ + -readability-isolate-declaration \ + $(SRCS) + +clean: + rm -f $(TARGETS) *.o *~ diff --git a/doc/platform-requirements/README b/doc/platform-requirements/README new file mode 100644 index 0000000..3914d22 --- /dev/null +++ b/doc/platform-requirements/README @@ -0,0 +1,18 @@ +Platform Requirements +===================== + +TODO: document the various Linux specific features we currently require + + +Test Programs +------------- + +In some places we rely on quite specific behaviour of sockets. +Although Linux, at least, seems to behave as required, It's not always +clear from the available documentation if this is required by POSIX or +some other specification. + +To specifically document those expectations this directory has some +test programs which explicitly check for the behaviour we need. +When/if we attempt a port to a new platform, running these to check +behaviour would be a good place to start. diff --git a/doc/platform-requirements/common.c b/doc/platform-requirements/common.c new file mode 100644 index 0000000..d687377 --- /dev/null +++ b/doc/platform-requirements/common.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +/* common.c + * + * Common helper functions for testing SO_REUSEADDR behaviour + * + * Copyright Red Hat + * Author: David Gibson <david@gibson.dropbear.id.au> + */ + +#include <errno.h> +#include <netinet/in.h> +#include <string.h> +#include <sys/socket.h> + +#include "common.h" + +int sock_reuseaddr(void) +{ + int y = 1; + int s; + + + s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (s < 0) + die("socket(): %s\n", strerror(errno)); + + if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &y, sizeof(y)) , 0) + die("SO_REUSEADDR: %s\n", strerror(errno)); + + return s; +} + +/* Send a token via the given connected socket */ +void send_token(int s, long token) +{ + ssize_t rc; + + rc = send(s, &token, sizeof(token), 0); + if (rc < 0) + die("send(): %s\n", strerror(errno)); + if (rc < sizeof(token)) + die("short send()\n"); +} + +/* Attempt to receive a token via the given socket. + * + * Returns true if we received the token, false if we got an EAGAIN, dies in any + * other case */ +bool recv_token(int s, long token) +{ + ssize_t rc; + long buf; + + rc = recv(s, &buf, sizeof(buf), MSG_DONTWAIT); + if (rc < 0) { + if (errno == EWOULDBLOCK) + return false; + die("recv(): %s\n", strerror(errno)); + } + if (rc < sizeof(buf)) + die("short recv()\n"); + if (buf != token) + die("data mismatch\n"); + return true; +} diff --git a/doc/platform-requirements/common.h b/doc/platform-requirements/common.h new file mode 100644 index 0000000..8844b1e --- /dev/null +++ b/doc/platform-requirements/common.h @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +/* common.h + * + * Useful shared functions + * + * Copyright Red Hat + * Author: David Gibson <david@gibson.dropbear.id.au> + */ +#ifndef REUSEADDR_COMMON_H +#define REUSEADDR_COMMON_H + +#include <stdarg.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> + +static inline void die(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + (void)vfprintf(stderr, fmt, ap); + va_end(ap); + exit(EXIT_FAILURE); +} + +#if __BYTE_ORDER == __BIG_ENDIAN +#define htons_constant(x) (x) +#define htonl_constant(x) (x) +#else +#define htons_constant(x) (__bswap_constant_16(x)) +#define htonl_constant(x) (__bswap_constant_32(x)) +#endif + +#define SOCKADDR_INIT(addr, port) \ + { \ + .sin_family = AF_INET, \ + .sin_addr = { .s_addr = htonl_constant(addr) }, \ + .sin_port = htons_constant(port), \ + } + +int sock_reuseaddr(void); +void send_token(int s, long token); +bool recv_token(int s, long token); + +#endif /* REUSEADDR_COMMON_H */ diff --git a/doc/platform-requirements/recv-zero.c b/doc/platform-requirements/recv-zero.c new file mode 100644 index 0000000..2a2a561 --- /dev/null +++ b/doc/platform-requirements/recv-zero.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +/* recv-zero.c + * + * Verify that we're able to discard datagrams by recv()ing into a zero-length + * buffer. + * + * Copyright Red Hat + * Author: David Gibson <david@gibson.dropbear.id.au> + */ + +#include <arpa/inet.h> +#include <errno.h> +#include <net/if.h> +#include <netinet/in.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include "common.h" + +#define DSTPORT 13257U + +enum discard_method { + DISCARD_NULL_BUF, + DISCARD_ZERO_IOV, + DISCARD_NULL_IOV, + NUM_METHODS, +}; + +/* 127.0.0.1:DSTPORT */ +static const struct sockaddr_in lo_dst = SOCKADDR_INIT(INADDR_LOOPBACK, DSTPORT); + +static void test_discard(enum discard_method method) +{ + struct iovec zero_iov = { .iov_base = NULL, .iov_len = 0, }; + struct msghdr mh_zero = { + .msg_iov = &zero_iov, + .msg_iovlen = 1, + }; + struct msghdr mh_null = { + .msg_iov = NULL, + .msg_iovlen = 0, + }; + long token1, token2; + int recv_s, send_s; + ssize_t rc; + + token1 = random(); + token2 = random(); + + recv_s = sock_reuseaddr(); + if (bind(recv_s, (struct sockaddr *)&lo_dst, sizeof(lo_dst)) < 0) + die("bind(): %s\n", strerror(errno)); + + send_s = sock_reuseaddr(); + if (connect(send_s, (struct sockaddr *)&lo_dst, sizeof(lo_dst)) < 0) + die("connect(): %s\n", strerror(errno)); + + send_token(send_s, token1); + send_token(send_s, token2); + + switch (method) { + case DISCARD_NULL_BUF: + /* cppcheck-suppress nullPointer */ + rc = recv(recv_s, NULL, 0, MSG_DONTWAIT); + if (rc < 0) + die("discarding recv(): %s\n", strerror(errno)); + break; + + case DISCARD_ZERO_IOV: + rc = recvmsg(recv_s, &mh_zero, MSG_DONTWAIT); + if (rc < 0) + die("recvmsg() with zero-length buffer: %s\n", + strerror(errno)); + if (!((unsigned)mh_zero.msg_flags & MSG_TRUNC)) + die("Missing MSG_TRUNC flag\n"); + break; + + case DISCARD_NULL_IOV: + rc = recvmsg(recv_s, &mh_null, MSG_DONTWAIT); + if (rc < 0) + die("recvmsg() with zero-length iov: %s\n", + strerror(errno)); + if (!((unsigned)mh_null.msg_flags & MSG_TRUNC)) + die("Missing MSG_TRUNC flag\n"); + break; + + default: + die("Bad method\n"); + } + + recv_token(recv_s, token2); + + /* cppcheck-suppress nullPointer */ + rc = recv(recv_s, NULL, 0, MSG_DONTWAIT); + if (rc < 0 && errno != EAGAIN) + die("redundant discarding recv(): %s\n", strerror(errno)); + if (rc >= 0) + die("Unexpected receive: rc=%zd\n", rc); +} + +int main(int argc, char *argv[]) +{ + enum discard_method method; + + (void)argc; + (void)argv; + + for (method = 0; method < NUM_METHODS; method++) + test_discard(method); + + printf("Discarding datagrams with 0-length receives seems to work\n"); + + exit(0); +} diff --git a/doc/platform-requirements/reuseaddr-priority.c b/doc/platform-requirements/reuseaddr-priority.c new file mode 100644 index 0000000..701b6ff --- /dev/null +++ b/doc/platform-requirements/reuseaddr-priority.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +/* reuseaddr-priority.c + * + * Verify which SO_REUSEADDR UDP sockets get priority to receive + * ============================================================= + * + * SO_REUSEADDR allows multiple sockets to bind to overlapping addresses, so + * there can be multiple sockets eligible to receive the same packet. The exact + * semantics of which socket will receive in this circumstance isn't very well + * documented. + * + * This program verifies that things behave the way we expect. Specifically we + * expect: + * + * - If both a connected and an unconnected socket could receive a datagram, the + * connected one will receive it in preference to the unconnected one. + * + * - If an unconnected socket bound to a specific address and an unconnected + * socket bound to the "any" address (0.0.0.0 or ::) could receive a datagram, + * then the one with a specific address will receive it in preference to the + * other. + * + * These should be true regardless of the order the sockets are created in, or + * the order they're polled in. + * + * Copyright Red Hat + * Author: David Gibson <david@gibson.dropbear.id.au> + */ + +#include <arpa/inet.h> +#include <errno.h> +#include <net/if.h> +#include <netinet/in.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include "common.h" + +#define SRCPORT 13246U +#define DSTPORT 13247U + +/* Different cases for receiving socket configuration */ +enum sock_type { + /* Socket is bound to 0.0.0.0:DSTPORT and not connected */ + SOCK_BOUND_ANY = 0, + + /* Socket is bound to 127.0.0.1:DSTPORT and not connected */ + SOCK_BOUND_LO = 1, + + /* Socket is bound to 0.0.0.0:DSTPORT and connected to 127.0.0.1:SRCPORT */ + SOCK_CONNECTED = 2, + + NUM_SOCK_TYPES, +}; + +typedef enum sock_type order_t[NUM_SOCK_TYPES]; + +static order_t orders[] = { + {0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}, +}; + +/* 127.0.0.2 */ +#define INADDR_LOOPBACK2 ((in_addr_t)(0x7f000002)) + +/* 0.0.0.0:DSTPORT */ +static const struct sockaddr_in any_dst = SOCKADDR_INIT(INADDR_ANY, DSTPORT); +/* 127.0.0.1:DSTPORT */ +static const struct sockaddr_in lo_dst = SOCKADDR_INIT(INADDR_LOOPBACK, DSTPORT); + +/* 127.0.0.2:DSTPORT */ +static const struct sockaddr_in lo2_dst = SOCKADDR_INIT(INADDR_LOOPBACK2, DSTPORT); + +/* 127.0.0.1:SRCPORT */ +static const struct sockaddr_in lo_src = SOCKADDR_INIT(INADDR_LOOPBACK, SRCPORT); + +/* Random token to send in datagram */ +static long token; + +/* Get a socket of the specified type for receiving */ +static int sock_recv(enum sock_type type) +{ + const struct sockaddr *connect_sa = NULL; + const struct sockaddr *bind_sa = NULL; + int s; + + s = sock_reuseaddr(); + + switch (type) { + case SOCK_CONNECTED: + connect_sa = (struct sockaddr *)&lo_src; + /* fallthrough */ + case SOCK_BOUND_ANY: + bind_sa = (struct sockaddr *)&any_dst; + break; + + case SOCK_BOUND_LO: + bind_sa = (struct sockaddr *)&lo_dst; + break; + + default: + die("bug"); + } + + if (bind_sa) + if (bind(s, bind_sa, sizeof(struct sockaddr_in)) < 0) + die("bind(): %s\n", strerror(errno)); + if (connect_sa) + if (connect(s, connect_sa, sizeof(struct sockaddr_in)) < 0) + die("connect(): %s\n", strerror(errno)); + + return s; +} + +/* Get a socket suitable for sending to the given type of receiving socket */ +static int sock_send(enum sock_type type) +{ + const struct sockaddr *connect_sa = NULL; + const struct sockaddr *bind_sa = NULL; + int s; + + s = sock_reuseaddr(); + + switch (type) { + case SOCK_BOUND_ANY: + connect_sa = (struct sockaddr *)&lo2_dst; + break; + + case SOCK_CONNECTED: + bind_sa = (struct sockaddr *)&lo_src; + /* fallthrough */ + case SOCK_BOUND_LO: + connect_sa = (struct sockaddr *)&lo_dst; + break; + + default: + die("bug"); + } + + if (bind_sa) + if (bind(s, bind_sa, sizeof(struct sockaddr_in)) < 0) + die("bind(): %s\n", strerror(errno)); + if (connect_sa) + if (connect(s, connect_sa, sizeof(struct sockaddr_in)) < 0) + die("connect(): %s\n", strerror(errno)); + + return s; +} + +/* Check for expected behaviour with one specific ordering for various operations: + * + * @recv_create_order: Order to create receiving sockets in + * @send_create_order: Order to create sending sockets in + * @test_order: Order to test the behaviour of different types + * @recv_order: Order to check the receiving sockets + */ +static void check_one_order(const order_t recv_create_order, + const order_t send_create_order, + const order_t test_order, + const order_t recv_order) +{ + int rs[NUM_SOCK_TYPES]; + int ss[NUM_SOCK_TYPES]; + int nfds = 0; + int i, j; + + for (i = 0; i < NUM_SOCK_TYPES; i++) { + enum sock_type t = recv_create_order[i]; + int s; + + s = sock_recv(t); + if (s >= nfds) + nfds = s + 1; + + rs[t] = s; + } + + for (i = 0; i < NUM_SOCK_TYPES; i++) { + enum sock_type t = send_create_order[i]; + + ss[t] = sock_send(t); + } + + for (i = 0; i < NUM_SOCK_TYPES; i++) { + enum sock_type ti = test_order[i]; + int recv_via = -1; + + send_token(ss[ti], token); + + for (j = 0; j < NUM_SOCK_TYPES; j++) { + enum sock_type tj = recv_order[j]; + + if (recv_token(rs[tj], token)) { + if (recv_via != -1) + die("Received token more than once\n"); + recv_via = tj; + } + } + + if (recv_via == -1) + die("Didn't receive token at all\n"); + if (recv_via != ti) + die("Received token via unexpected socket\n"); + } + + for (i = 0; i < NUM_SOCK_TYPES; i++) { + close(rs[i]); + close(ss[i]); + } +} + +static void check_all_orders(void) +{ + int norders = sizeof(orders) / sizeof(orders[0]); + int i, j, k, l; + + for (i = 0; i < norders; i++) + for (j = 0; j < norders; j++) + for (k = 0; k < norders; k++) + for (l = 0; l < norders; l++) + check_one_order(orders[i], orders[j], + orders[k], orders[l]); +} + +int main(int argc, char *argv[]) +{ + (void)argc; + (void)argv; + + token = random(); + + check_all_orders(); + + printf("SO_REUSEADDR receive priorities seem to work as expected\n"); + + exit(0); +} diff --git a/doc/platform-requirements/udp-close-dup.c b/doc/platform-requirements/udp-close-dup.c new file mode 100644 index 0000000..99060fc --- /dev/null +++ b/doc/platform-requirements/udp-close-dup.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +/* udp-close-dup.c + * + * Verify that closing one dup() of a UDP socket won't stop other dups from + * receiving packets. + * + * Copyright Red Hat + * Author: David Gibson <david@gibson.dropbear.id.au> + */ + +#include <arpa/inet.h> +#include <errno.h> +#include <fcntl.h> +#include <net/if.h> +#include <netinet/in.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include "common.h" + +#define DSTPORT 13257U + +/* 127.0.0.1:DSTPORT */ +static const struct sockaddr_in lo_dst = SOCKADDR_INIT(INADDR_LOOPBACK, DSTPORT); + +enum dup_method { + DUP_DUP, + DUP_FCNTL, + NUM_METHODS, +}; + +static void test_close_dup(enum dup_method method) +{ + long token; + int s1, s2, send_s; + ssize_t rc; + + s1 = sock_reuseaddr(); + if (bind(s1, (struct sockaddr *)&lo_dst, sizeof(lo_dst)) < 0) + die("bind(): %s\n", strerror(errno)); + + send_s = sock_reuseaddr(); + if (connect(send_s, (struct sockaddr *)&lo_dst, sizeof(lo_dst)) < 0) + die("connect(): %s\n", strerror(errno)); + + /* Receive before duplicating */ + token = random(); + send_token(send_s, token); + recv_token(s1, token); + + switch (method) { + case DUP_DUP: + /* NOLINTNEXTLINE(android-cloexec-dup) */ + s2 = dup(s1); + if (s2 < 0) + die("dup(): %s\n", strerror(errno)); + break; + case DUP_FCNTL: + s2 = fcntl(s1, F_DUPFD_CLOEXEC, 0); + if (s2 < 0) + die("F_DUPFD_CLOEXEC: %s\n", strerror(errno)); + break; + default: + die("Bad method\n"); + } + + /* Receive via original handle */ + token = random(); + send_token(send_s, token); + recv_token(s1, token); + + /* Receive via duplicated handle */ + token = random(); + send_token(send_s, token); + recv_token(s2, token); + + /* Close duplicate */ + rc = close(s2); + if (rc < 0) + die("close() dup: %s\n", strerror(errno)); + + /* Receive after closing duplicate */ + token = random(); + send_token(send_s, token); + recv_token(s1, token); +} + +int main(int argc, char *argv[]) +{ + enum dup_method method; + + (void)argc; + (void)argv; + + for (method = 0; method < NUM_METHODS; method++) + test_close_dup(method); + + printf("Closing dup()ed UDP sockets seems to work as expected\n"); + + exit(0); +} diff --git a/epoll_type.h b/epoll_type.h new file mode 100644 index 0000000..f3ef415 --- /dev/null +++ b/epoll_type.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * Copyright Red Hat + * Author: David Gibson <david@gibson.dropbear.id.au> + */ + +#ifndef EPOLL_TYPE_H +#define EPOLL_TYPE_H + +/** + * enum epoll_type - Different types of fds we poll over + */ +enum epoll_type { + /* Special value to indicate an invalid type */ + EPOLL_TYPE_NONE = 0, + /* Connected TCP sockets */ + EPOLL_TYPE_TCP, + /* Connected TCP sockets (spliced) */ + EPOLL_TYPE_TCP_SPLICE, + /* Listening TCP sockets */ + EPOLL_TYPE_TCP_LISTEN, + /* timerfds used for TCP timers */ + EPOLL_TYPE_TCP_TIMER, + /* UDP "listening" sockets */ + EPOLL_TYPE_UDP_LISTEN, + /* UDP socket for replies on a specific flow */ + EPOLL_TYPE_UDP_REPLY, + /* ICMP/ICMPv6 ping sockets */ + EPOLL_TYPE_PING, + /* inotify fd watching for end of netns (pasta) */ + EPOLL_TYPE_NSQUIT_INOTIFY, + /* timer fd watching for end of netns, fallback for inotify (pasta) */ + EPOLL_TYPE_NSQUIT_TIMER, + /* tuntap character device */ + EPOLL_TYPE_TAP_PASTA, + /* socket connected to qemu */ + EPOLL_TYPE_TAP_PASST, + /* socket listening for qemu socket connections */ + EPOLL_TYPE_TAP_LISTEN, + /* vhost-user command socket */ + EPOLL_TYPE_VHOST_CMD, + /* vhost-user kick event socket */ + EPOLL_TYPE_VHOST_KICK, + + EPOLL_NUM_TYPES, +}; + +#endif /* EPOLL_TYPE_H */ @@ -5,9 +5,11 @@ * Tracking for logical "flows" of packets. */ +#include <errno.h> #include <stdint.h> #include <stdio.h> #include <unistd.h> +#include <sched.h> #include <string.h> #include "util.h" @@ -18,10 +20,24 @@ #include "flow.h" #include "flow_table.h" +const char *flow_state_str[] = { + [FLOW_STATE_FREE] = "FREE", + [FLOW_STATE_NEW] = "NEW", + [FLOW_STATE_INI] = "INI", + [FLOW_STATE_TGT] = "TGT", + [FLOW_STATE_TYPED] = "TYPED", + [FLOW_STATE_ACTIVE] = "ACTIVE", +}; +static_assert(ARRAY_SIZE(flow_state_str) == FLOW_NUM_STATES, + "flow_state_str[] doesn't match enum flow_state"); + const char *flow_type_str[] = { [FLOW_TYPE_NONE] = "<none>", [FLOW_TCP] = "TCP connection", [FLOW_TCP_SPLICE] = "TCP connection (spliced)", + [FLOW_PING4] = "ICMP ping sequence", + [FLOW_PING6] = "ICMPv6 ping sequence", + [FLOW_UDP] = "UDP flow", }; static_assert(ARRAY_SIZE(flow_type_str) == FLOW_NUM_TYPES, "flow_type_str[] doesn't match enum flow_type"); @@ -29,6 +45,9 @@ static_assert(ARRAY_SIZE(flow_type_str) == FLOW_NUM_TYPES, const uint8_t flow_proto[] = { [FLOW_TCP] = IPPROTO_TCP, [FLOW_TCP_SPLICE] = IPPROTO_TCP, + [FLOW_PING4] = IPPROTO_ICMP, + [FLOW_PING6] = IPPROTO_ICMPV6, + [FLOW_UDP] = IPPROTO_UDP, }; static_assert(ARRAY_SIZE(flow_proto) == FLOW_NUM_TYPES, "flow_proto[] doesn't match enum flow_type"); @@ -36,46 +55,6 @@ static_assert(ARRAY_SIZE(flow_proto) == FLOW_NUM_TYPES, /* Global Flow Table */ /** - * DOC: Theory of Operation - flow entry life cycle - * - * An individual flow table entry moves through these logical states, usually in - * this order. - * - * FREE - Part of the general pool of free flow table entries - * Operations: - * - flow_alloc() finds an entry and moves it to ALLOC state - * - * ALLOC - A tentatively allocated entry - * Operations: - * - flow_alloc_cancel() returns the entry to FREE state - * - FLOW_START() set the entry's type and moves to START state - * Caveats: - * - It's not safe to write fields in the flow entry - * - It's not safe to allocate further entries with flow_alloc() - * - It's not safe to return to the main epoll loop (use FLOW_START() - * to move to START state before doing so) - * - It's not safe to use flow_*() logging functions - * - * START - An entry being prepared by flow type specific code - * Operations: - * - Flow type specific fields may be accessed - * - flow_*() logging functions - * - flow_alloc_cancel() returns the entry to FREE state - * Caveats: - * - Returning to the main epoll loop or allocating another entry - * with flow_alloc() implicitly moves the entry to ACTIVE state. - * - * ACTIVE - An active flow entry managed by flow type specific code - * Operations: - * - Flow type specific fields may be accessed - * - flow_*() logging functions - * - Flow may be expired by returning 'true' from flow type specific - * deferred or timer handler. This will return it to FREE state. - * Caveats: - * - It's not safe to call flow_alloc_cancel() - */ - -/** * DOC: Theory of Operation - allocating and freeing flow entries * * Flows are entries in flowtab[]. We need to routinely scan the whole table to @@ -128,10 +107,156 @@ static_assert(ARRAY_SIZE(flow_proto) == FLOW_NUM_TYPES, unsigned flow_first_free; union flow flowtab[FLOW_MAX]; +static const union flow *flow_new_entry; /* = NULL */ + +/* Hash table to index it */ +#define FLOW_HASH_LOAD 70 /* % */ +#define FLOW_HASH_SIZE ((2 * FLOW_MAX * 100 / FLOW_HASH_LOAD)) + +/* Table for lookup from flowside information */ +static flow_sidx_t flow_hashtab[FLOW_HASH_SIZE]; + +static_assert(ARRAY_SIZE(flow_hashtab) >= 2 * FLOW_MAX, +"Safe linear probing requires hash table with more entries than the number of sides in the flow table"); /* Last time the flow timers ran */ static struct timespec flow_timer_run; +/** flowside_from_af() - Initialise flowside from addresses + * @side: flowside to initialise + * @af: Address family (AF_INET or AF_INET6) + * @eaddr: Endpoint address (pointer to in_addr or in6_addr) + * @eport: Endpoint port + * @oaddr: Our address (pointer to in_addr or in6_addr) + * @oport: Our port + */ +static void flowside_from_af(struct flowside *side, sa_family_t af, + const void *eaddr, in_port_t eport, + const void *oaddr, in_port_t oport) +{ + if (oaddr) + inany_from_af(&side->oaddr, af, oaddr); + else + side->oaddr = inany_any6; + side->oport = oport; + + if (eaddr) + inany_from_af(&side->eaddr, af, eaddr); + else + side->eaddr = inany_any6; + side->eport = eport; +} + +/** + * struct flowside_sock_args - Parameters for flowside_sock_splice() + * @c: Execution context + * @fd: Filled in with new socket fd + * @err: Filled in with errno if something failed + * @type: Socket epoll type + * @sa: Socket address + * @sl: Length of @sa + * @data: epoll reference data + */ +struct flowside_sock_args { + const struct ctx *c; + int fd; + int err; + enum epoll_type type; + const struct sockaddr *sa; + socklen_t sl; + const char *path; + uint32_t data; +}; + +/** flowside_sock_splice() - Create and bind socket for PIF_SPLICE based on flowside + * @arg: Argument as a struct flowside_sock_args + * + * Return: 0 + */ +static int flowside_sock_splice(void *arg) +{ + struct flowside_sock_args *a = arg; + + ns_enter(a->c); + + a->fd = sock_l4_sa(a->c, a->type, a->sa, a->sl, NULL, + a->sa->sa_family == AF_INET6, a->data); + a->err = errno; + + return 0; +} + +/** flowside_sock_l4() - Create and bind socket based on flowside + * @c: Execution context + * @type: Socket epoll type + * @pif: Interface for this socket + * @tgt: Target flowside + * @data: epoll reference portion for protocol handlers + * + * Return: socket fd of protocol @proto bound to our address and port from @tgt + * (if specified). + */ +int flowside_sock_l4(const struct ctx *c, enum epoll_type type, uint8_t pif, + const struct flowside *tgt, uint32_t data) +{ + const char *ifname = NULL; + union sockaddr_inany sa; + socklen_t sl; + + ASSERT(pif_is_socket(pif)); + + pif_sockaddr(c, &sa, &sl, pif, &tgt->oaddr, tgt->oport); + + switch (pif) { + case PIF_HOST: + if (inany_is_loopback(&tgt->oaddr)) + ifname = NULL; + else if (sa.sa_family == AF_INET) + ifname = c->ip4.ifname_out; + else if (sa.sa_family == AF_INET6) + ifname = c->ip6.ifname_out; + + return sock_l4_sa(c, type, &sa, sl, ifname, + sa.sa_family == AF_INET6, data); + + case PIF_SPLICE: { + struct flowside_sock_args args = { + .c = c, .type = type, + .sa = &sa.sa, .sl = sl, .data = data, + }; + NS_CALL(flowside_sock_splice, &args); + errno = args.err; + return args.fd; + } + + default: + /* If we add new socket pifs, they'll need to be implemented + * here + */ + ASSERT(0); + } +} + +/** flowside_connect() - Connect a socket based on flowside + * @c: Execution context + * @s: Socket to connect + * @pif: Target pif + * @tgt: Target flowside + * + * Connect @s to the endpoint address and port from @tgt. + * + * Return: 0 on success, negative on error + */ +int flowside_connect(const struct ctx *c, int s, + uint8_t pif, const struct flowside *tgt) +{ + union sockaddr_inany sa; + socklen_t sl; + + pif_sockaddr(c, &sa, &sl, pif, &tgt->eaddr, tgt->eport); + return connect(s, &sa.sa, sl); +} + /** flow_log_ - Log flow-related message * @f: flow the message is related to * @pri: Log priority @@ -140,6 +265,7 @@ static struct timespec flow_timer_run; */ void flow_log_(const struct flow_common *f, int pri, const char *fmt, ...) { + const char *type_or_state; char msg[BUFSIZ]; va_list args; @@ -147,40 +273,221 @@ void flow_log_(const struct flow_common *f, int pri, const char *fmt, ...) (void)vsnprintf(msg, sizeof(msg), fmt, args); va_end(args); - logmsg(pri, "Flow %u (%s): %s", flow_idx(f), FLOW_TYPE(f), msg); + /* Show type if it's set, otherwise the state */ + if (f->state < FLOW_STATE_TYPED) + type_or_state = FLOW_STATE(f); + else + type_or_state = FLOW_TYPE(f); + + logmsg(true, false, pri, + "Flow %u (%s): %s", flow_idx(f), type_or_state, msg); +} + +/** flow_log_details_() - Log the details of a flow + * @f: flow to log + * @pri: Log priority + * @state: State to log details according to + * + * Logs the details of the flow: endpoints, interfaces, type etc. + */ +void flow_log_details_(const struct flow_common *f, int pri, + enum flow_state state) +{ + char estr0[INANY_ADDRSTRLEN], fstr0[INANY_ADDRSTRLEN]; + char estr1[INANY_ADDRSTRLEN], fstr1[INANY_ADDRSTRLEN]; + const struct flowside *ini = &f->side[INISIDE]; + const struct flowside *tgt = &f->side[TGTSIDE]; + + if (state >= FLOW_STATE_TGT) + flow_log_(f, pri, + "%s [%s]:%hu -> [%s]:%hu => %s [%s]:%hu -> [%s]:%hu", + pif_name(f->pif[INISIDE]), + inany_ntop(&ini->eaddr, estr0, sizeof(estr0)), + ini->eport, + inany_ntop(&ini->oaddr, fstr0, sizeof(fstr0)), + ini->oport, + pif_name(f->pif[TGTSIDE]), + inany_ntop(&tgt->oaddr, fstr1, sizeof(fstr1)), + tgt->oport, + inany_ntop(&tgt->eaddr, estr1, sizeof(estr1)), + tgt->eport); + else if (state >= FLOW_STATE_INI) + flow_log_(f, pri, "%s [%s]:%hu -> [%s]:%hu => ?", + pif_name(f->pif[INISIDE]), + inany_ntop(&ini->eaddr, estr0, sizeof(estr0)), + ini->eport, + inany_ntop(&ini->oaddr, fstr0, sizeof(fstr0)), + ini->oport); +} + +/** + * flow_set_state() - Change flow's state + * @f: Flow changing state + * @state: New state + */ +static void flow_set_state(struct flow_common *f, enum flow_state state) +{ + uint8_t oldstate = f->state; + + ASSERT(state < FLOW_NUM_STATES); + ASSERT(oldstate < FLOW_NUM_STATES); + + f->state = state; + flow_log_(f, LOG_DEBUG, "%s -> %s", flow_state_str[oldstate], + FLOW_STATE(f)); + + flow_log_details_(f, LOG_DEBUG, MAX(state, oldstate)); +} + +/** + * flow_initiate_() - Move flow to INI, setting pif[INISIDE] + * @flow: Flow to change state + * @pif: pif of the initiating side + */ +static void flow_initiate_(union flow *flow, uint8_t pif) +{ + struct flow_common *f = &flow->f; + + ASSERT(pif != PIF_NONE); + ASSERT(flow_new_entry == flow && f->state == FLOW_STATE_NEW); + ASSERT(f->type == FLOW_TYPE_NONE); + ASSERT(f->pif[INISIDE] == PIF_NONE && f->pif[TGTSIDE] == PIF_NONE); + + f->pif[INISIDE] = pif; + flow_set_state(f, FLOW_STATE_INI); +} + +/** + * flow_initiate_af() - Move flow to INI, setting INISIDE details + * @flow: Flow to change state + * @pif: pif of the initiating side + * @af: Address family of @saddr and @daddr + * @saddr: Source address (pointer to in_addr or in6_addr) + * @sport: Endpoint port + * @daddr: Destination address (pointer to in_addr or in6_addr) + * @dport: Destination port + * + * Return: pointer to the initiating flowside information + */ +const struct flowside *flow_initiate_af(union flow *flow, uint8_t pif, + sa_family_t af, + const void *saddr, in_port_t sport, + const void *daddr, in_port_t dport) +{ + struct flowside *ini = &flow->f.side[INISIDE]; + + flowside_from_af(ini, af, saddr, sport, daddr, dport); + flow_initiate_(flow, pif); + return ini; } /** - * flow_start() - Set flow type for new flow and log - * @flow: Flow to set type for - * @type: Type for new flow - * @iniside: Which side initiated the new flow + * flow_initiate_sa() - Move flow to INI, setting INISIDE details + * @flow: Flow to change state + * @pif: pif of the initiating side + * @ssa: Source socket address + * @dport: Destination port * - * Return: @flow + * Return: pointer to the initiating flowside information + */ +const struct flowside *flow_initiate_sa(union flow *flow, uint8_t pif, + const union sockaddr_inany *ssa, + in_port_t dport) +{ + struct flowside *ini = &flow->f.side[INISIDE]; + + inany_from_sockaddr(&ini->eaddr, &ini->eport, ssa); + if (inany_v4(&ini->eaddr)) + ini->oaddr = inany_any4; + else + ini->oaddr = inany_any6; + ini->oport = dport; + flow_initiate_(flow, pif); + return ini; +} + +/** + * flow_target() - Determine where flow should forward to, and move to TGT + * @c: Execution context + * @flow: Flow to forward + * @proto: Protocol * - * Should be called before setting any flow type specific fields in the flow - * table entry. + * Return: pointer to the target flowside information + */ +const struct flowside *flow_target(const struct ctx *c, union flow *flow, + uint8_t proto) +{ + char estr[INANY_ADDRSTRLEN], fstr[INANY_ADDRSTRLEN]; + struct flow_common *f = &flow->f; + const struct flowside *ini = &f->side[INISIDE]; + struct flowside *tgt = &f->side[TGTSIDE]; + uint8_t tgtpif = PIF_NONE; + + ASSERT(flow_new_entry == flow && f->state == FLOW_STATE_INI); + ASSERT(f->type == FLOW_TYPE_NONE); + ASSERT(f->pif[INISIDE] != PIF_NONE && f->pif[TGTSIDE] == PIF_NONE); + ASSERT(flow->f.state == FLOW_STATE_INI); + + switch (f->pif[INISIDE]) { + case PIF_TAP: + tgtpif = fwd_nat_from_tap(c, proto, ini, tgt); + break; + + case PIF_SPLICE: + tgtpif = fwd_nat_from_splice(c, proto, ini, tgt); + break; + + case PIF_HOST: + tgtpif = fwd_nat_from_host(c, proto, ini, tgt); + break; + + default: + flow_err(flow, "No rules to forward %s [%s]:%hu -> [%s]:%hu", + pif_name(f->pif[INISIDE]), + inany_ntop(&ini->eaddr, estr, sizeof(estr)), + ini->eport, + inany_ntop(&ini->oaddr, fstr, sizeof(fstr)), + ini->oport); + } + + if (tgtpif == PIF_NONE) + return NULL; + + f->pif[TGTSIDE] = tgtpif; + flow_set_state(f, FLOW_STATE_TGT); + return tgt; +} + +/** + * flow_set_type() - Set type and move to TYPED + * @flow: Flow to change state + * @pif: pif of the initiating side */ -union flow *flow_start(union flow *flow, enum flow_type type, - unsigned iniside) +union flow *flow_set_type(union flow *flow, enum flow_type type) { - (void)iniside; - flow->f.type = type; - flow_dbg(flow, "START %s", flow_type_str[flow->f.type]); + struct flow_common *f = &flow->f; + + ASSERT(type != FLOW_TYPE_NONE); + ASSERT(flow_new_entry == flow && f->state == FLOW_STATE_TGT); + ASSERT(f->type == FLOW_TYPE_NONE); + ASSERT(f->pif[INISIDE] != PIF_NONE && f->pif[TGTSIDE] != PIF_NONE); + + f->type = type; + flow_set_state(f, FLOW_STATE_TYPED); return flow; } /** - * flow_end() - Clear flow type for finished flow and log - * @flow: Flow to clear + * flow_activate() - Move flow to ACTIVE + * @f: Flow to change state */ -static void flow_end(union flow *flow) +void flow_activate(struct flow_common *f) { - if (flow->f.type == FLOW_TYPE_NONE) - return; /* Nothing to do */ + ASSERT(&flow_new_entry->f == f && f->state == FLOW_STATE_TYPED); + ASSERT(f->pif[INISIDE] != PIF_NONE && f->pif[TGTSIDE] != PIF_NONE); - flow_dbg(flow, "END %s", flow_type_str[flow->f.type]); - flow->f.type = FLOW_TYPE_NONE; + flow_set_state(f, FLOW_STATE_ACTIVE); + flow_new_entry = NULL; } /** @@ -192,9 +499,12 @@ union flow *flow_alloc(void) { union flow *flow = &flowtab[flow_first_free]; + ASSERT(!flow_new_entry); + if (flow_first_free >= FLOW_MAX) return NULL; + ASSERT(flow->f.state == FLOW_STATE_FREE); ASSERT(flow->f.type == FLOW_TYPE_NONE); ASSERT(flow->free.n >= 1); ASSERT(flow_first_free + flow->free.n <= FLOW_MAX); @@ -217,7 +527,10 @@ union flow *flow_alloc(void) flow_first_free = flow->free.next; } + flow_new_entry = flow; memset(flow, 0, sizeof(*flow)); + flow_set_state(&flow->f, FLOW_STATE_NEW); + return flow; } @@ -229,15 +542,228 @@ union flow *flow_alloc(void) */ void flow_alloc_cancel(union flow *flow) { + ASSERT(flow_new_entry == flow); + ASSERT(flow->f.state == FLOW_STATE_NEW || + flow->f.state == FLOW_STATE_INI || + flow->f.state == FLOW_STATE_TGT || + flow->f.state == FLOW_STATE_TYPED); ASSERT(flow_first_free > FLOW_IDX(flow)); - flow_end(flow); + flow_set_state(&flow->f, FLOW_STATE_FREE); + memset(flow, 0, sizeof(*flow)); + /* Put it back in a length 1 free cluster, don't attempt to fully * reverse flow_alloc()s steps. This will get folded together the next * time flow_defer_handler runs anyway() */ flow->free.n = 1; flow->free.next = flow_first_free; flow_first_free = FLOW_IDX(flow); + flow_new_entry = NULL; +} + +/** + * flow_hash() - Calculate hash value for one side of a flow + * @c: Execution context + * @proto: Protocol of this flow (IP L4 protocol number) + * @pif: pif of the side to hash + * @side: Flowside (must not have unspecified parts) + * + * Return: hash value + */ +static uint64_t flow_hash(const struct ctx *c, uint8_t proto, uint8_t pif, + const struct flowside *side) +{ + struct siphash_state state = SIPHASH_INIT(c->hash_secret); + + inany_siphash_feed(&state, &side->oaddr); + inany_siphash_feed(&state, &side->eaddr); + + return siphash_final(&state, 38, (uint64_t)proto << 40 | + (uint64_t)pif << 32 | + (uint64_t)side->oport << 16 | + (uint64_t)side->eport); +} + +/** + * flow_sidx_hash() - Calculate hash value for given side of a given flow + * @c: Execution context + * @sidx: Flow & side index to get hash for + * + * Return: hash value, of the flow & side represented by @sidx + */ +static uint64_t flow_sidx_hash(const struct ctx *c, flow_sidx_t sidx) +{ + const struct flow_common *f = &flow_at_sidx(sidx)->f; + const struct flowside *side = &f->side[sidx.sidei]; + uint8_t pif = f->pif[sidx.sidei]; + + /* For the hash table to work, entries must have complete endpoint + * information, and at least a forwarding port. + */ + ASSERT(pif != PIF_NONE && !inany_is_unspecified(&side->eaddr) && + side->eport != 0 && side->oport != 0); + + return flow_hash(c, FLOW_PROTO(f), pif, side); +} + +/** + * flow_hash_probe_() - Find hash bucket for a flow, given hash + * @hash: Raw hash value for flow & side + * @sidx: Flow and side to find bucket for + * + * Return: If @sidx is in the hash table, its current bucket, otherwise a + * suitable free bucket for it. + */ +static inline unsigned flow_hash_probe_(uint64_t hash, flow_sidx_t sidx) +{ + unsigned b = hash % FLOW_HASH_SIZE; + + /* Linear probing */ + while (flow_sidx_valid(flow_hashtab[b]) && + !flow_sidx_eq(flow_hashtab[b], sidx)) + b = mod_sub(b, 1, FLOW_HASH_SIZE); + + return b; +} + +/** + * flow_hash_probe() - Find hash bucket for a flow + * @c: Execution context + * @sidx: Flow and side to find bucket for + * + * Return: If @sidx is in the hash table, its current bucket, otherwise a + * suitable free bucket for it. + */ +static inline unsigned flow_hash_probe(const struct ctx *c, flow_sidx_t sidx) +{ + return flow_hash_probe_(flow_sidx_hash(c, sidx), sidx); +} + +/** + * flow_hash_insert() - Insert side of a flow into into hash table + * @c: Execution context + * @sidx: Flow & side index + * + * Return: raw (un-modded) hash value of side of flow + */ +uint64_t flow_hash_insert(const struct ctx *c, flow_sidx_t sidx) +{ + uint64_t hash = flow_sidx_hash(c, sidx); + unsigned b = flow_hash_probe_(hash, sidx); + + flow_hashtab[b] = sidx; + flow_dbg(flow_at_sidx(sidx), "Side %u hash table insert: bucket: %u", + sidx.sidei, b); + + return hash; +} + +/** + * flow_hash_remove() - Drop side of a flow from the hash table + * @c: Execution context + * @sidx: Side of flow to remove + */ +void flow_hash_remove(const struct ctx *c, flow_sidx_t sidx) +{ + unsigned b = flow_hash_probe(c, sidx), s; + + if (!flow_sidx_valid(flow_hashtab[b])) + return; /* Redundant remove */ + + flow_dbg(flow_at_sidx(sidx), "Side %u hash table remove: bucket: %u", + sidx.sidei, b); + + /* Scan the remainder of the cluster */ + for (s = mod_sub(b, 1, FLOW_HASH_SIZE); + flow_sidx_valid(flow_hashtab[s]); + s = mod_sub(s, 1, FLOW_HASH_SIZE)) { + unsigned h = flow_sidx_hash(c, flow_hashtab[s]) % FLOW_HASH_SIZE; + + if (!mod_between(h, s, b, FLOW_HASH_SIZE)) { + /* flow_hashtab[s] can live in flow_hashtab[b]'s slot */ + debug("hash table remove: shuffle %u -> %u", s, b); + flow_hashtab[b] = flow_hashtab[s]; + b = s; + } + } + + flow_hashtab[b] = FLOW_SIDX_NONE; +} + +/** + * flowside_lookup() - Look for a matching flowside in the flow table + * @c: Execution context + * @proto: Protocol of the flow (IP L4 protocol number) + * @pif: pif to look for in the table + * @side: Flowside to look for in the table + * + * Return: sidx of the matching flow & side, FLOW_SIDX_NONE if not found + */ +static flow_sidx_t flowside_lookup(const struct ctx *c, uint8_t proto, + uint8_t pif, const struct flowside *side) +{ + flow_sidx_t sidx; + union flow *flow; + unsigned b; + + b = flow_hash(c, proto, pif, side) % FLOW_HASH_SIZE; + while ((sidx = flow_hashtab[b], flow = flow_at_sidx(sidx)) && + !(FLOW_PROTO(&flow->f) == proto && + flow->f.pif[sidx.sidei] == pif && + flowside_eq(&flow->f.side[sidx.sidei], side))) + b = mod_sub(b, 1, FLOW_HASH_SIZE); + + return flow_hashtab[b]; +} + +/** + * flow_lookup_af() - Look up a flow given addressing information + * @c: Execution context + * @proto: Protocol of the flow (IP L4 protocol number) + * @pif: Interface of the flow + * @af: Address family, AF_INET or AF_INET6 + * @eaddr: Guest side endpoint address (guest local address) + * @oaddr: Our guest side address (guest remote address) + * @eport: Guest side endpoint port (guest local port) + * @oport: Our guest side port (guest remote port) + * + * Return: sidx of the matching flow & side, FLOW_SIDX_NONE if not found + */ +flow_sidx_t flow_lookup_af(const struct ctx *c, + uint8_t proto, uint8_t pif, sa_family_t af, + const void *eaddr, const void *oaddr, + in_port_t eport, in_port_t oport) +{ + struct flowside side; + + flowside_from_af(&side, af, eaddr, eport, oaddr, oport); + return flowside_lookup(c, proto, pif, &side); +} + +/** + * flow_lookup_sa() - Look up a flow given an endpoint socket address + * @c: Execution context + * @proto: Protocol of the flow (IP L4 protocol number) + * @pif: Interface of the flow + * @esa: Socket address of the endpoint + * @oport: Our port number + * + * Return: sidx of the matching flow & side, FLOW_SIDX_NONE if not found + */ +flow_sidx_t flow_lookup_sa(const struct ctx *c, uint8_t proto, uint8_t pif, + const void *esa, in_port_t oport) +{ + struct flowside side = { + .oport = oport, + }; + + inany_from_sockaddr(&side.eaddr, &side.eport, esa); + if (inany_v4(&side.eaddr)) + side.oaddr = inany_any4; + else + side.oaddr = inany_any6; + + return flowside_lookup(c, proto, pif, &side); } /** @@ -257,11 +783,14 @@ void flow_defer_handler(const struct ctx *c, const struct timespec *now) flow_timer_run = *now; } + ASSERT(!flow_new_entry); /* Incomplete flow at end of cycle */ + for (idx = 0; idx < FLOW_MAX; idx++) { union flow *flow = &flowtab[idx]; bool closed = false; - if (flow->f.type == FLOW_TYPE_NONE) { + switch (flow->f.state) { + case FLOW_STATE_FREE: { unsigned skip = flow->free.n; /* First entry of a free cluster must have n >= 1 */ @@ -283,17 +812,43 @@ void flow_defer_handler(const struct ctx *c, const struct timespec *now) continue; } + case FLOW_STATE_NEW: + case FLOW_STATE_INI: + case FLOW_STATE_TGT: + case FLOW_STATE_TYPED: + /* Incomplete flow at end of cycle */ + ASSERT(false); + break; + + case FLOW_STATE_ACTIVE: + /* Nothing to do */ + break; + + default: + ASSERT(false); + } + switch (flow->f.type) { case FLOW_TYPE_NONE: ASSERT(false); break; case FLOW_TCP: - closed = tcp_flow_defer(flow); + closed = tcp_flow_defer(&flow->tcp); break; case FLOW_TCP_SPLICE: - closed = tcp_splice_flow_defer(flow); + closed = tcp_splice_flow_defer(&flow->tcp_splice); if (!closed && timer) - tcp_splice_timer(c, flow); + tcp_splice_timer(c, &flow->tcp_splice); + break; + case FLOW_PING4: + case FLOW_PING6: + if (timer) + closed = icmp_ping_timer(c, &flow->ping, now); + break; + case FLOW_UDP: + closed = udp_flow_defer(&flow->udp); + if (!closed && timer) + closed = udp_flow_timer(c, &flow->udp, now); break; default: /* Assume other flow types don't need any handling */ @@ -301,7 +856,8 @@ void flow_defer_handler(const struct ctx *c, const struct timespec *now) } if (closed) { - flow_end(flow); + flow_set_state(&flow->f, FLOW_STATE_FREE); + memset(flow, 0, sizeof(*flow)); if (free_head) { /* Add slot to current free cluster */ @@ -328,7 +884,12 @@ void flow_defer_handler(const struct ctx *c, const struct timespec *now) */ void flow_init(void) { + unsigned b; + /* Initial state is a single free cluster containing the whole table */ flowtab[0].free.n = FLOW_MAX; flowtab[0].free.next = FLOW_MAX; + + for (b = 0; b < FLOW_HASH_SIZE; b++) + flow_hashtab[b] = FLOW_SIDX_NONE; } @@ -10,6 +10,98 @@ #define FLOW_TIMER_INTERVAL 1000 /* ms */ /** + * enum flow_state - States of a flow table entry + * + * An individual flow table entry moves through these states, usually in this + * order. + * General rules: + * - Code outside flow.c should never write common fields of union flow. + * - The state field may always be read. + * + * FREE - Part of the general pool of free flow table entries + * Operations: + * - flow_alloc() finds an entry and moves it to NEW + * + * NEW - Freshly allocated, uninitialised entry + * Operations: + * - flow_alloc_cancel() returns the entry to FREE + * - flow_initiate() sets the entry's INISIDE details and moves to + * INI + * - FLOW_SET_TYPE() sets the entry's type and moves to TYPED + * Caveats: + * - No fields other than state may be accessed + * - At most one entry may be NEW, INI, TGT or TYPED at a time, so + * it's unsafe to use flow_alloc() again until this entry moves to + * ACTIVE or FREE + * - You may not return to the main epoll loop while any flow is NEW + * + * INI - An entry with INISIDE common information completed + * Operations: + * - Common fields related to INISIDE may be read + * - flow_alloc_cancel() returns the entry to FREE + * - flow_target() sets the entry's TGTSIDE details and moves to TGT + * Caveats: + * - Other common fields may not be read + * - Type specific fields may not be read or written + * - At most one entry may be NEW, INI, TGT or TYPED at a time, so + * it's unsafe to use flow_alloc() again until this entry moves to + * ACTIVE or FREE + * - You may not return to the main epoll loop while any flow is INI + * + * TGT - An entry with only INISIDE and TGTSIDE common information completed + * Operations: + * - Common fields related to INISIDE & TGTSIDE may be read + * - flow_alloc_cancel() returns the entry to FREE + * - FLOW_SET_TYPE() sets the entry's type and moves to TYPED + * Caveats: + * - Other common fields may not be read + * - Type specific fields may not be read or written + * - At most one entry may be NEW, INI, TGT or TYPED at a time, so + * it's unsafe to use flow_alloc() again until this entry moves to + * ACTIVE or FREE + * - You may not return to the main epoll loop while any flow is TGT + * + * TYPED - Generic info initialised, type specific initialisation underway + * Operations: + * - All common fields may be read + * - Type specific fields may be read and written + * - flow_alloc_cancel() returns the entry to FREE + * - FLOW_ACTIVATE() moves the entry to ACTIVE + * Caveats: + * - At most one entry may be NEW, INI, TGT or TYPED at a time, so + * it's unsafe to use flow_alloc() again until this entry moves to + * ACTIVE or FREE + * - You may not return to the main epoll loop while any flow is + * TYPED + * + * ACTIVE - An active, fully-initialised flow entry + * Operations: + * - All common fields may be read + * - Type specific fields may be read and written + * - Flow returns to FREE when it expires, signalled by returning + * 'true' from flow type specific deferred or timer handler + * Caveats: + * - flow_alloc_cancel() may not be called on it + */ +enum flow_state { + FLOW_STATE_FREE, + FLOW_STATE_NEW, + FLOW_STATE_INI, + FLOW_STATE_TGT, + FLOW_STATE_TYPED, + FLOW_STATE_ACTIVE, + + FLOW_NUM_STATES, +}; +#define FLOW_STATE_BITS 8 +static_assert(FLOW_NUM_STATES <= (1 << FLOW_STATE_BITS), + "Too many flow states for FLOW_STATE_BITS"); + +extern const char *flow_state_str[]; +#define FLOW_STATE(f) \ + ((f)->state < FLOW_NUM_STATES ? flow_state_str[(f)->state] : "?") + +/** * enum flow_type - Different types of packet flows we track */ enum flow_type { @@ -19,9 +111,18 @@ enum flow_type { FLOW_TCP, /* A TCP connection between a host socket and ns socket */ FLOW_TCP_SPLICE, + /* ICMP echo requests from guest to host and matching replies back */ + FLOW_PING4, + /* ICMPv6 echo requests from guest to host and matching replies back */ + FLOW_PING6, + /* UDP pseudo-connection */ + FLOW_UDP, FLOW_NUM_TYPES, }; +#define FLOW_TYPE_BITS 8 +static_assert(FLOW_NUM_TYPES <= (1 << FLOW_TYPE_BITS), + "Too many flow types for FLOW_TYPE_BITS"); extern const char *flow_type_str[]; #define FLOW_TYPE(f) \ @@ -31,12 +132,66 @@ extern const uint8_t flow_proto[]; #define FLOW_PROTO(f) \ ((f)->type < FLOW_NUM_TYPES ? flow_proto[(f)->type] : 0) +#define SIDES 2 + +#define INISIDE 0 /* Initiating side index */ +#define TGTSIDE 1 /* Target side index */ + +/** + * struct flowside - Address information for one side of a flow + * @eaddr: Endpoint address (remote address from passt's PoV) + * @oaddr: Our address (local address from passt's PoV) + * @eport: Endpoint port + * @oport: Our port + */ +struct flowside { + union inany_addr oaddr; + union inany_addr eaddr; + in_port_t oport; + in_port_t eport; +}; + +/** + * flowside_eq() - Check if two flowsides are equal + * @left, @right: Flowsides to compare + * + * Return: true if equal, false otherwise + */ +static inline bool flowside_eq(const struct flowside *left, + const struct flowside *right) +{ + return inany_equals(&left->eaddr, &right->eaddr) && + left->eport == right->eport && + inany_equals(&left->oaddr, &right->oaddr) && + left->oport == right->oport; +} + +int flowside_sock_l4(const struct ctx *c, enum epoll_type type, uint8_t pif, + const struct flowside *tgt, uint32_t data); +int flowside_connect(const struct ctx *c, int s, + uint8_t pif, const struct flowside *tgt); + /** * struct flow_common - Common fields for packet flows + * @state: State of the flow table entry * @type: Type of packet flow + * @pif[]: Interface for each side of the flow + * @side[]: Information for each side of the flow */ struct flow_common { +#ifdef __GNUC__ + enum flow_state state:FLOW_STATE_BITS; + enum flow_type type:FLOW_TYPE_BITS; +#else + uint8_t state; + static_assert(sizeof(uint8_t) * 8 >= FLOW_STATE_BITS, + "Not enough bits for state field"); uint8_t type; + static_assert(sizeof(uint8_t) * 8 >= FLOW_TYPE_BITS, + "Not enough bits for type field"); +#endif + uint8_t pif[SIDES]; + struct flowside side[SIDES]; }; #define FLOW_INDEX_BITS 17 /* 128k - 1 */ @@ -45,24 +200,30 @@ struct flow_common { #define FLOW_TABLE_PRESSURE 30 /* % of FLOW_MAX */ #define FLOW_FILE_PRESSURE 30 /* % of c->nofile */ -union flow *flow_start(union flow *flow, enum flow_type type, - unsigned iniside); -#define FLOW_START(flow_, t_, var_, i_) \ - (&flow_start((flow_), (t_), (i_))->var_) - /** * struct flow_sidx - ID for one side of a specific flow - * @side: Side referenced (0 or 1) - * @flow: Index of flow referenced + * @sidei: Index of side referenced (0 or 1) + * @flowi: Index of flow referenced */ typedef struct flow_sidx { - unsigned side :1; - unsigned flow :FLOW_INDEX_BITS; + unsigned sidei :1; + unsigned flowi :FLOW_INDEX_BITS; } flow_sidx_t; static_assert(sizeof(flow_sidx_t) <= sizeof(uint32_t), "flow_sidx_t must fit within 32 bits"); -#define FLOW_SIDX_NONE ((flow_sidx_t){ .flow = FLOW_MAX }) +#define FLOW_SIDX_NONE ((flow_sidx_t){ .flowi = FLOW_MAX }) + +/** + * flow_sidx_valid() - Test if a sidx is valid + * @sidx: sidx value + * + * Return: true if @sidx refers to a valid flow & side + */ +static inline bool flow_sidx_valid(flow_sidx_t sidx) +{ + return sidx.flowi < FLOW_MAX; +} /** * flow_sidx_eq() - Test if two sidx values are equal @@ -72,9 +233,18 @@ static_assert(sizeof(flow_sidx_t) <= sizeof(uint32_t), */ static inline bool flow_sidx_eq(flow_sidx_t a, flow_sidx_t b) { - return (a.flow == b.flow) && (a.side == b.side); + return (a.flowi == b.flowi) && (a.sidei == b.sidei); } +uint64_t flow_hash_insert(const struct ctx *c, flow_sidx_t sidx); +void flow_hash_remove(const struct ctx *c, flow_sidx_t sidx); +flow_sidx_t flow_lookup_af(const struct ctx *c, + uint8_t proto, uint8_t pif, sa_family_t af, + const void *eaddr, const void *oaddr, + in_port_t eport, in_port_t oport); +flow_sidx_t flow_lookup_sa(const struct ctx *c, uint8_t proto, uint8_t pif, + const void *esa, in_port_t oport); + union flow; void flow_init(void); @@ -94,4 +264,11 @@ void flow_log_(const struct flow_common *f, int pri, const char *fmt, ...) flow_dbg((f), __VA_ARGS__); \ } while (0) +void flow_log_details_(const struct flow_common *f, int pri, + enum flow_state state); +#define flow_log_details(f_, pri) \ + flow_log_details_(&((f_)->f), (pri), (f_)->f.state) +#define flow_dbg_details(f_) flow_log_details((f_), LOG_DEBUG) +#define flow_err_details(f_) flow_log_details((f_), LOG_ERR) + #endif /* FLOW_H */ diff --git a/flow_table.h b/flow_table.h index eecf884..f15db53 100644 --- a/flow_table.h +++ b/flow_table.h @@ -8,6 +8,8 @@ #define FLOW_TABLE_H #include "tcp_conn.h" +#include "icmp_flow.h" +#include "udp_flow.h" /** * struct flow_free_cluster - Information about a cluster of free entries @@ -33,14 +35,22 @@ union flow { struct flow_free_cluster free; struct tcp_tap_conn tcp; struct tcp_splice_conn tcp_splice; + struct icmp_ping_flow ping; + struct udp_flow udp; }; /* Global Flow Table */ extern unsigned flow_first_free; extern union flow flowtab[]; +/** + * flow_foreach_sidei() - 'for' type macro to step through each side of flow + * @sidei_: Takes value INISIDE, then TGTSIDE + */ +#define flow_foreach_sidei(sidei_) \ + for ((sidei_) = INISIDE; (sidei_) < SIDES; (sidei_)++) -/** flow_idx - Index of flow from common structure +/** flow_idx() - Index of flow from common structure * @f: Common flow fields pointer * * Return: index of @f in the flow table @@ -50,59 +60,122 @@ static inline unsigned flow_idx(const struct flow_common *f) return (union flow *)f - flowtab; } -/** FLOW_IDX - Find the index of a flow +/** FLOW_IDX() - Find the index of a flow * @f_: Flow pointer, either union flow * or protocol specific * * Return: index of @f in the flow table */ #define FLOW_IDX(f_) (flow_idx(&(f_)->f)) -/** FLOW - Flow entry at a given index +/** FLOW() - Flow entry at a given index * @idx: Flow index * * Return: pointer to entry @idx in the flow table */ #define FLOW(idx) (&flowtab[(idx)]) -/** flow_at_sidx - Flow entry for a given sidx +/** flow_at_sidx() - Flow entry for a given sidx * @sidx: Flow & side index * * Return: pointer to the corresponding flow entry, or NULL */ static inline union flow *flow_at_sidx(flow_sidx_t sidx) { - if (sidx.flow >= FLOW_MAX) + if (!flow_sidx_valid(sidx)) + return NULL; + return FLOW(sidx.flowi); +} + +/** pif_at_sidx() - Interface for a given flow and side + * @sidx: Flow & side index + * + * Return: pif for the flow & side given by @sidx + */ +static inline uint8_t pif_at_sidx(flow_sidx_t sidx) +{ + const union flow *flow = flow_at_sidx(sidx); + + if (!flow) + return PIF_NONE; + return flow->f.pif[sidx.sidei]; +} + +/** flowside_at_sidx() - Retrieve a specific flowside + * @sidx: Flow & side index + * + * Return: Flowside for the flow & side given by @sidx + */ +static inline const struct flowside *flowside_at_sidx(flow_sidx_t sidx) +{ + const union flow *flow = flow_at_sidx(sidx); + + if (!flow) return NULL; - return FLOW(sidx.flow); + + return &flow->f.side[sidx.sidei]; } -/** flow_sidx_t - Index of one side of a flow from common structure +/** flow_sidx_opposite() - Get the other side of the same flow + * @sidx: Flow & side index + * + * Return: sidx for the other side of the same flow as @sidx + */ +static inline flow_sidx_t flow_sidx_opposite(flow_sidx_t sidx) +{ + if (!flow_sidx_valid(sidx)) + return FLOW_SIDX_NONE; + + return (flow_sidx_t){.flowi = sidx.flowi, .sidei = !sidx.sidei}; +} + +/** flow_sidx() - Index of one side of a flow from common structure * @f: Common flow fields pointer - * @side: Which side to refer to (0 or 1) + * @sidei: Which side to refer to (0 or 1) * * Return: index of @f and @side in the flow table */ static inline flow_sidx_t flow_sidx(const struct flow_common *f, - int side) + unsigned sidei) { /* cppcheck-suppress [knownConditionTrueFalse, unmatchedSuppression] */ - ASSERT(side == !!side); + ASSERT(sidei == !!sidei); return (flow_sidx_t){ - .side = side, - .flow = flow_idx(f), + .sidei = sidei, + .flowi = flow_idx(f), }; } -/** FLOW_SIDX - Find the index of one side of a flow +/** FLOW_SIDX() - Find the index of one side of a flow * @f_: Flow pointer, either union flow * or protocol specific - * @side: Which side to index (0 or 1) + * @sidei: Which side to index (0 or 1) * * Return: index of @f and @side in the flow table */ -#define FLOW_SIDX(f_, side) (flow_sidx(&(f_)->f, (side))) +#define FLOW_SIDX(f_, sidei) (flow_sidx(&(f_)->f, (sidei))) union flow *flow_alloc(void); void flow_alloc_cancel(union flow *flow); +const struct flowside *flow_initiate_af(union flow *flow, uint8_t pif, + sa_family_t af, + const void *saddr, in_port_t sport, + const void *daddr, in_port_t dport); +const struct flowside *flow_initiate_sa(union flow *flow, uint8_t pif, + const union sockaddr_inany *ssa, + in_port_t dport); +const struct flowside *flow_target_af(union flow *flow, uint8_t pif, + sa_family_t af, + const void *saddr, in_port_t sport, + const void *daddr, in_port_t dport); +const struct flowside *flow_target(const struct ctx *c, union flow *flow, + uint8_t proto); + +union flow *flow_set_type(union flow *flow, enum flow_type type); +#define FLOW_SET_TYPE(flow_, t_, var_) (&flow_set_type((flow_), (t_))->var_) + +void flow_activate(struct flow_common *f); +#define FLOW_ACTIVATE(flow_) \ + (flow_activate(&(flow_)->f)) + #endif /* FLOW_TABLE_H */ @@ -25,6 +25,81 @@ #include "fwd.h" #include "passt.h" #include "lineread.h" +#include "flow_table.h" + +/* Empheral port range: values from RFC 6335 */ +static in_port_t fwd_ephemeral_min = (1 << 15) + (1 << 14); +static in_port_t fwd_ephemeral_max = NUM_PORTS - 1; + +#define PORT_RANGE_SYSCTL "/proc/sys/net/ipv4/ip_local_port_range" + +/** fwd_probe_ephemeral() - Determine what ports this host considers ephemeral + * + * Work out what ports the host thinks are emphemeral and record it for later + * use by fwd_port_is_ephemeral(). If we're unable to probe, assume the range + * recommended by RFC 6335. + */ +void fwd_probe_ephemeral(void) +{ + char *line, *tab, *end; + struct lineread lr; + long min, max; + ssize_t len; + int fd; + + fd = open(PORT_RANGE_SYSCTL, O_RDONLY | O_CLOEXEC); + if (fd < 0) { + warn_perror("Unable to open %s", PORT_RANGE_SYSCTL); + return; + } + + lineread_init(&lr, fd); + len = lineread_get(&lr, &line); + close(fd); + + if (len < 0) + goto parse_err; + + tab = strchr(line, '\t'); + if (!tab) + goto parse_err; + *tab = '\0'; + + errno = 0; + min = strtol(line, &end, 10); + if (*end || errno) + goto parse_err; + + errno = 0; + max = strtol(tab + 1, &end, 10); + if (*end || errno) + goto parse_err; + + if (min < 0 || min >= (long)NUM_PORTS || + max < 0 || max >= (long)NUM_PORTS) + goto parse_err; + + fwd_ephemeral_min = min; + fwd_ephemeral_max = max; + + return; + +parse_err: + warn("Unable to parse %s", PORT_RANGE_SYSCTL); +} + +/** + * fwd_port_is_ephemeral() - Is port number ephemeral? + * @port: Port number + * + * Return: true if @port is ephemeral, that is may be allocated by the kernel as + * a local port for outgoing connections or datagrams, but should not be + * used for binding services to. + */ +bool fwd_port_is_ephemeral(in_port_t port) +{ + return (port >= fwd_ephemeral_min) && (port <= fwd_ephemeral_max); +} /* See enum in kernel's include/net/tcp_states.h */ #define UDP_LISTEN 0x07 @@ -38,7 +113,7 @@ * @exclude: Bitmap of ports to exclude from setting (and clear) * * #syscalls:pasta lseek - * #syscalls:pasta ppc64le:_llseek ppc64:_llseek armv6l:_llseek armv7l:_llseek + * #syscalls:pasta ppc64le:_llseek ppc64:_llseek arm:_llseek */ static void procfs_scan_listen(int fd, unsigned int lstate, uint8_t *map, const uint8_t *exclude) @@ -52,7 +127,7 @@ static void procfs_scan_listen(int fd, unsigned int lstate, return; if (lseek(fd, 0, SEEK_SET)) { - warn("lseek() failed on /proc/net file: %s", strerror(errno)); + warn_perror("lseek() failed on /proc/net file"); return; } @@ -128,18 +203,18 @@ void fwd_scan_ports_init(struct ctx *c) c->tcp.fwd_in.scan4 = c->tcp.fwd_in.scan6 = -1; c->tcp.fwd_out.scan4 = c->tcp.fwd_out.scan6 = -1; - c->udp.fwd_in.f.scan4 = c->udp.fwd_in.f.scan6 = -1; - c->udp.fwd_out.f.scan4 = c->udp.fwd_out.f.scan6 = -1; + c->udp.fwd_in.scan4 = c->udp.fwd_in.scan6 = -1; + c->udp.fwd_out.scan4 = c->udp.fwd_out.scan6 = -1; if (c->tcp.fwd_in.mode == FWD_AUTO) { c->tcp.fwd_in.scan4 = open_in_ns(c, "/proc/net/tcp", flags); c->tcp.fwd_in.scan6 = open_in_ns(c, "/proc/net/tcp6", flags); fwd_scan_ports_tcp(&c->tcp.fwd_in, &c->tcp.fwd_out); } - if (c->udp.fwd_in.f.mode == FWD_AUTO) { - c->udp.fwd_in.f.scan4 = open_in_ns(c, "/proc/net/udp", flags); - c->udp.fwd_in.f.scan6 = open_in_ns(c, "/proc/net/udp6", flags); - fwd_scan_ports_udp(&c->udp.fwd_in.f, &c->udp.fwd_out.f, + if (c->udp.fwd_in.mode == FWD_AUTO) { + c->udp.fwd_in.scan4 = open_in_ns(c, "/proc/net/udp", flags); + c->udp.fwd_in.scan6 = open_in_ns(c, "/proc/net/udp6", flags); + fwd_scan_ports_udp(&c->udp.fwd_in, &c->udp.fwd_out, &c->tcp.fwd_in, &c->tcp.fwd_out); } if (c->tcp.fwd_out.mode == FWD_AUTO) { @@ -147,10 +222,298 @@ void fwd_scan_ports_init(struct ctx *c) c->tcp.fwd_out.scan6 = open("/proc/net/tcp6", flags); fwd_scan_ports_tcp(&c->tcp.fwd_out, &c->tcp.fwd_in); } - if (c->udp.fwd_out.f.mode == FWD_AUTO) { - c->udp.fwd_out.f.scan4 = open("/proc/net/udp", flags); - c->udp.fwd_out.f.scan6 = open("/proc/net/udp6", flags); - fwd_scan_ports_udp(&c->udp.fwd_out.f, &c->udp.fwd_in.f, + if (c->udp.fwd_out.mode == FWD_AUTO) { + c->udp.fwd_out.scan4 = open("/proc/net/udp", flags); + c->udp.fwd_out.scan6 = open("/proc/net/udp6", flags); + fwd_scan_ports_udp(&c->udp.fwd_out, &c->udp.fwd_in, &c->tcp.fwd_out, &c->tcp.fwd_in); } } + +/** + * is_dns_flow() - Determine if flow appears to be a DNS request + * @proto: Protocol (IP L4 protocol number) + * @ini: Flow address information of the initiating side + * + * Return: true if the flow appears to be directed at a dns server, that is a + * TCP or UDP flow to port 53 (domain) or port 853 (domain-s) + */ +static bool is_dns_flow(uint8_t proto, const struct flowside *ini) +{ + return ((proto == IPPROTO_UDP) || (proto == IPPROTO_TCP)) && + ((ini->oport == 53) || (ini->oport == 853)); +} + +/** + * fwd_guest_accessible4() - Is IPv4 address guest-accessible + * @c: Execution context + * @addr: Host visible IPv4 address + * + * Return: true if @addr on the host is accessible to the guest without + * translation, false otherwise + */ +static bool fwd_guest_accessible4(const struct ctx *c, + const struct in_addr *addr) +{ + if (IN4_IS_ADDR_LOOPBACK(addr)) + return false; + + /* In socket interfaces 0.0.0.0 generally means "any" or unspecified, + * however on the wire it can mean "this host on this network". Since + * that has a different meaning for host and guest, we can't let it + * through untranslated. + */ + if (IN4_IS_ADDR_UNSPECIFIED(addr)) + return false; + + /* For IPv4, addr_seen is initialised to addr, so is always a valid + * address + */ + if (IN4_ARE_ADDR_EQUAL(addr, &c->ip4.addr) || + IN4_ARE_ADDR_EQUAL(addr, &c->ip4.addr_seen)) + return false; + + return true; +} + +/** + * fwd_guest_accessible6() - Is IPv6 address guest-accessible + * @c: Execution context + * @addr: Host visible IPv6 address + * + * Return: true if @addr on the host is accessible to the guest without + * translation, false otherwise + */ +static bool fwd_guest_accessible6(const struct ctx *c, + const struct in6_addr *addr) +{ + if (IN6_IS_ADDR_LOOPBACK(addr)) + return false; + + if (IN6_ARE_ADDR_EQUAL(addr, &c->ip6.addr)) + return false; + + /* For IPv6, addr_seen starts unspecified, because we don't know what LL + * address the guest will take until we see it. Only check against it + * if it has been set to a real address. + */ + if (!IN6_IS_ADDR_UNSPECIFIED(&c->ip6.addr_seen) && + IN6_ARE_ADDR_EQUAL(addr, &c->ip6.addr_seen)) + return false; + + return true; +} + +/** + * fwd_guest_accessible() - Is IPv[46] address guest-accessible + * @c: Execution context + * @addr: Host visible IPv[46] address + * + * Return: true if @addr on the host is accessible to the guest without + * translation, false otherwise + */ +static bool fwd_guest_accessible(const struct ctx *c, + const union inany_addr *addr) +{ + const struct in_addr *a4 = inany_v4(addr); + + if (a4) + return fwd_guest_accessible4(c, a4); + + return fwd_guest_accessible6(c, &addr->a6); +} + +/** + * fwd_nat_from_tap() - Determine to forward a flow from the tap interface + * @c: Execution context + * @proto: Protocol (IP L4 protocol number) + * @ini: Flow address information of the initiating side + * @tgt: Flow address information on the target side (updated) + * + * Return: pif of the target interface to forward the flow to, PIF_NONE if the + * flow cannot or should not be forwarded at all. + */ +uint8_t fwd_nat_from_tap(const struct ctx *c, uint8_t proto, + const struct flowside *ini, struct flowside *tgt) +{ + if (is_dns_flow(proto, ini) && + inany_equals4(&ini->oaddr, &c->ip4.dns_match)) + tgt->eaddr = inany_from_v4(c->ip4.dns_host); + else if (is_dns_flow(proto, ini) && + inany_equals6(&ini->oaddr, &c->ip6.dns_match)) + tgt->eaddr.a6 = c->ip6.dns_host; + else if (inany_equals4(&ini->oaddr, &c->ip4.map_host_loopback)) + tgt->eaddr = inany_loopback4; + else if (inany_equals6(&ini->oaddr, &c->ip6.map_host_loopback)) + tgt->eaddr = inany_loopback6; + else if (inany_equals4(&ini->oaddr, &c->ip4.map_guest_addr)) + tgt->eaddr = inany_from_v4(c->ip4.addr); + else if (inany_equals6(&ini->oaddr, &c->ip6.map_guest_addr)) + tgt->eaddr.a6 = c->ip6.addr; + else + tgt->eaddr = ini->oaddr; + + tgt->eport = ini->oport; + + /* The relevant addr_out controls the host side source address. This + * may be unspecified, which allows the kernel to pick an address. + */ + if (inany_v4(&tgt->eaddr)) + tgt->oaddr = inany_from_v4(c->ip4.addr_out); + else + tgt->oaddr.a6 = c->ip6.addr_out; + + /* Let the kernel pick a host side source port */ + tgt->oport = 0; + if (proto == IPPROTO_UDP) { + /* But for UDP we preserve the source port */ + tgt->oport = ini->eport; + } + + return PIF_HOST; +} + +/** + * fwd_nat_from_splice() - Determine to forward a flow from the splice interface + * @c: Execution context + * @proto: Protocol (IP L4 protocol number) + * @ini: Flow address information of the initiating side + * @tgt: Flow address information on the target side (updated) + * + * Return: pif of the target interface to forward the flow to, PIF_NONE if the + * flow cannot or should not be forwarded at all. + */ +uint8_t fwd_nat_from_splice(const struct ctx *c, uint8_t proto, + const struct flowside *ini, struct flowside *tgt) +{ + if (!inany_is_loopback(&ini->eaddr) || + (!inany_is_loopback(&ini->oaddr) && !inany_is_unspecified(&ini->oaddr))) { + char estr[INANY_ADDRSTRLEN], fstr[INANY_ADDRSTRLEN]; + + debug("Non loopback address on %s: [%s]:%hu -> [%s]:%hu", + pif_name(PIF_SPLICE), + inany_ntop(&ini->eaddr, estr, sizeof(estr)), ini->eport, + inany_ntop(&ini->oaddr, fstr, sizeof(fstr)), ini->oport); + return PIF_NONE; + } + + if (inany_v4(&ini->eaddr)) + tgt->eaddr = inany_loopback4; + else + tgt->eaddr = inany_loopback6; + + /* Preserve the specific loopback adddress used, but let the kernel pick + * a source port on the target side + */ + tgt->oaddr = ini->eaddr; + tgt->oport = 0; + + tgt->eport = ini->oport; + if (proto == IPPROTO_TCP) + tgt->eport += c->tcp.fwd_out.delta[tgt->eport]; + else if (proto == IPPROTO_UDP) + tgt->eport += c->udp.fwd_out.delta[tgt->eport]; + + /* Let the kernel pick a host side source port */ + tgt->oport = 0; + if (proto == IPPROTO_UDP) + /* But for UDP preserve the source port */ + tgt->oport = ini->eport; + + return PIF_HOST; +} + +/** + * fwd_nat_from_host() - Determine to forward a flow from the host interface + * @c: Execution context + * @proto: Protocol (IP L4 protocol number) + * @ini: Flow address information of the initiating side + * @tgt: Flow address information on the target side (updated) + * + * Return: pif of the target interface to forward the flow to, PIF_NONE if the + * flow cannot or should not be forwarded at all. + */ +uint8_t fwd_nat_from_host(const struct ctx *c, uint8_t proto, + const struct flowside *ini, struct flowside *tgt) +{ + /* Common for spliced and non-spliced cases */ + tgt->eport = ini->oport; + if (proto == IPPROTO_TCP) + tgt->eport += c->tcp.fwd_in.delta[tgt->eport]; + else if (proto == IPPROTO_UDP) + tgt->eport += c->udp.fwd_in.delta[tgt->eport]; + + if (c->mode == MODE_PASTA && inany_is_loopback(&ini->eaddr) && + (proto == IPPROTO_TCP || proto == IPPROTO_UDP)) { + /* spliceable */ + + /* The traffic will go over the guest's 'lo' interface, but by + * default use its external address, so we don't inadvertently + * expose services that listen only on the guest's loopback + * address. That can be overridden by --host-lo-to-ns-lo which + * will instead forward to the loopback address in the guest. + * + * In either case, let the kernel pick the source address to + * match. + */ + if (inany_v4(&ini->eaddr)) { + if (c->host_lo_to_ns_lo) + tgt->eaddr = inany_loopback4; + else + tgt->eaddr = inany_from_v4(c->ip4.addr_seen); + tgt->oaddr = inany_any4; + } else { + if (c->host_lo_to_ns_lo) + tgt->eaddr = inany_loopback6; + else + tgt->eaddr.a6 = c->ip6.addr_seen; + tgt->oaddr = inany_any6; + } + + /* Let the kernel pick source port */ + tgt->oport = 0; + if (proto == IPPROTO_UDP) + /* But for UDP preserve the source port */ + tgt->oport = ini->eport; + + return PIF_SPLICE; + } + + if (!IN4_IS_ADDR_UNSPECIFIED(&c->ip4.map_host_loopback) && + inany_equals4(&ini->eaddr, &in4addr_loopback)) { + /* Specifically 127.0.0.1, not 127.0.0.0/8 */ + tgt->oaddr = inany_from_v4(c->ip4.map_host_loopback); + } else if (!IN6_IS_ADDR_UNSPECIFIED(&c->ip6.map_host_loopback) && + inany_equals6(&ini->eaddr, &in6addr_loopback)) { + tgt->oaddr.a6 = c->ip6.map_host_loopback; + } else if (!IN4_IS_ADDR_UNSPECIFIED(&c->ip4.map_guest_addr) && + inany_equals4(&ini->eaddr, &c->ip4.addr)) { + tgt->oaddr = inany_from_v4(c->ip4.map_guest_addr); + } else if (!IN6_IS_ADDR_UNSPECIFIED(&c->ip6.map_guest_addr) && + inany_equals6(&ini->eaddr, &c->ip6.addr)) { + tgt->oaddr.a6 = c->ip6.map_guest_addr; + } else if (!fwd_guest_accessible(c, &ini->eaddr)) { + if (inany_v4(&ini->eaddr)) { + if (IN4_IS_ADDR_UNSPECIFIED(&c->ip4.our_tap_addr)) + /* No source address we can use */ + return PIF_NONE; + tgt->oaddr = inany_from_v4(c->ip4.our_tap_addr); + } else { + tgt->oaddr.a6 = c->ip6.our_tap_ll; + } + } else { + tgt->oaddr = ini->eaddr; + } + tgt->oport = ini->eport; + + if (inany_v4(&tgt->oaddr)) { + tgt->eaddr = inany_from_v4(c->ip4.addr_seen); + } else { + if (inany_is_linklocal6(&tgt->oaddr)) + tgt->eaddr.a6 = c->ip6.addr_ll_seen; + else + tgt->eaddr.a6 = c->ip6.addr_seen; + } + + return PIF_TAP; +} @@ -7,10 +7,16 @@ #ifndef FWD_H #define FWD_H +struct flowside; + /* Number of ports for both TCP and UDP */ #define NUM_PORTS (1U << 16) +void fwd_probe_ephemeral(void); +bool fwd_port_is_ephemeral(in_port_t port); + enum fwd_ports_mode { + FWD_UNSET = 0, FWD_SPEC = 1, FWD_NONE, FWD_AUTO, @@ -41,4 +47,11 @@ void fwd_scan_ports_udp(struct fwd_ports *fwd, const struct fwd_ports *rev, const struct fwd_ports *tcp_rev); void fwd_scan_ports_init(struct ctx *c); +uint8_t fwd_nat_from_tap(const struct ctx *c, uint8_t proto, + const struct flowside *ini, struct flowside *tgt); +uint8_t fwd_nat_from_splice(const struct ctx *c, uint8_t proto, + const struct flowside *ini, struct flowside *tgt); +uint8_t fwd_nat_from_host(const struct ctx *c, uint8_t proto, + const struct flowside *ini, struct flowside *tgt); + #endif /* FWD_H */ @@ -40,36 +40,38 @@ #include "siphash.h" #include "inany.h" #include "icmp.h" +#include "flow_table.h" #define ICMP_ECHO_TIMEOUT 60 /* s, timeout for ICMP socket activity */ #define ICMP_NUM_IDS (1U << 16) /** - * struct icmp_id_sock - Tracking information for single ICMP echo identifier - * @sock: Bound socket for identifier - * @seq: Last sequence number sent to tap, host order, -1: not sent yet - * @ts: Last associated activity from tap, seconds + * ping_at_sidx() - Get ping specific flow at given sidx + * @sidx: Flow and side to retrieve + * + * Return: ping specific flow at @sidx, or NULL of @sidx is invalid. Asserts if + * the flow at @sidx is not FLOW_PING4 or FLOW_PING6 */ -struct icmp_id_sock { - int sock; - int seq; - time_t ts; -}; +static struct icmp_ping_flow *ping_at_sidx(flow_sidx_t sidx) +{ + union flow *flow = flow_at_sidx(sidx); + + if (!flow) + return NULL; -/* Indexed by ICMP echo identifier */ -static struct icmp_id_sock icmp_id_map[IP_VERSIONS][ICMP_NUM_IDS]; + ASSERT(flow->f.type == FLOW_PING4 || flow->f.type == FLOW_PING6); + return &flow->ping; +} /** * icmp_sock_handler() - Handle new data from ICMP or ICMPv6 socket * @c: Execution context - * @af: Address family (AF_INET or AF_INET6) * @ref: epoll reference */ -void icmp_sock_handler(const struct ctx *c, sa_family_t af, union epoll_ref ref) +void icmp_sock_handler(const struct ctx *c, union epoll_ref ref) { - struct icmp_id_sock *const id_sock = af == AF_INET - ? &icmp_id_map[V4][ref.icmp.id] : &icmp_id_map[V6][ref.icmp.id]; - const char *const pname = af == AF_INET ? "ICMP" : "ICMPv6"; + struct icmp_ping_flow *pingf = ping_at_sidx(ref.flowside); + const struct flowside *ini = &pingf->f.side[INISIDE]; union sockaddr_inany sr; socklen_t sl = sizeof(sr); char buf[USHRT_MAX]; @@ -79,33 +81,33 @@ void icmp_sock_handler(const struct ctx *c, sa_family_t af, union epoll_ref ref) if (c->no_icmp) return; + ASSERT(pingf); + n = recvfrom(ref.fd, buf, sizeof(buf), 0, &sr.sa, &sl); if (n < 0) { - warn("%s: recvfrom() error on ping socket: %s", - pname, strerror(errno)); + flow_err(pingf, "recvfrom() error: %s", strerror(errno)); return; } - if (sr.sa_family != af) - goto unexpected; - if (af == AF_INET) { + if (pingf->f.type == FLOW_PING4) { struct icmphdr *ih4 = (struct icmphdr *)buf; - if ((size_t)n < sizeof(*ih4) || ih4->type != ICMP_ECHOREPLY) + if (sr.sa_family != AF_INET || (size_t)n < sizeof(*ih4) || + ih4->type != ICMP_ECHOREPLY) goto unexpected; /* Adjust packet back to guest-side ID */ - ih4->un.echo.id = htons(ref.icmp.id); + ih4->un.echo.id = htons(ini->eport); seq = ntohs(ih4->un.echo.sequence); - } else if (af == AF_INET6) { + } else if (pingf->f.type == FLOW_PING6) { struct icmp6hdr *ih6 = (struct icmp6hdr *)buf; - if ((size_t)n < sizeof(*ih6) || + if (sr.sa_family != AF_INET6 || (size_t)n < sizeof(*ih6) || ih6->icmp6_type != ICMPV6_ECHO_REPLY) goto unexpected; /* Adjust packet back to guest-side ID */ - ih6->icmp6_identifier = htons(ref.icmp.id); + ih6->icmp6_identifier = htons(ini->eport); seq = ntohs(ih6->icmp6_sequence); } else { ASSERT(0); @@ -113,87 +115,111 @@ void icmp_sock_handler(const struct ctx *c, sa_family_t af, union epoll_ref ref) /* In PASTA mode, we'll get any reply we send, discard them. */ if (c->mode == MODE_PASTA) { - if (id_sock->seq == seq) + if (pingf->seq == seq) return; - id_sock->seq = seq; + pingf->seq = seq; } - debug("%s: echo reply to tap, ID: %"PRIu16", seq: %"PRIu16, pname, - ref.icmp.id, seq); - if (af == AF_INET) - tap_icmp4_send(c, sr.sa4.sin_addr, tap_ip4_daddr(c), buf, n); - else if (af == AF_INET6) - tap_icmp6_send(c, &sr.sa6.sin6_addr, - tap_ip6_daddr(c, &sr.sa6.sin6_addr), buf, n); + flow_dbg(pingf, "echo reply to tap, ID: %"PRIu16", seq: %"PRIu16, + ini->eport, seq); + + if (pingf->f.type == FLOW_PING4) { + const struct in_addr *saddr = inany_v4(&ini->oaddr); + const struct in_addr *daddr = inany_v4(&ini->eaddr); + + ASSERT(saddr && daddr); /* Must have IPv4 addresses */ + tap_icmp4_send(c, *saddr, *daddr, buf, n); + } else if (pingf->f.type == FLOW_PING6) { + const struct in6_addr *saddr = &ini->oaddr.a6; + const struct in6_addr *daddr = &ini->eaddr.a6; + + tap_icmp6_send(c, saddr, daddr, buf, n); + } return; unexpected: - warn("%s: Unexpected packet on ping socket", pname); + flow_err(pingf, "Unexpected packet on ping socket"); } /** - * icmp_ping_close() - Close and clean up a ping socket + * icmp_ping_close() - Close and clean up a ping flow * @c: Execution context - * @id_sock: Socket number and other info + * @pingf: ping flow entry to close */ -static void icmp_ping_close(const struct ctx *c, struct icmp_id_sock *id_sock) +static void icmp_ping_close(const struct ctx *c, + const struct icmp_ping_flow *pingf) { - epoll_ctl(c->epollfd, EPOLL_CTL_DEL, id_sock->sock, NULL); - close(id_sock->sock); - id_sock->sock = -1; - id_sock->seq = -1; + epoll_ctl(c->epollfd, EPOLL_CTL_DEL, pingf->sock, NULL); + close(pingf->sock); + flow_hash_remove(c, FLOW_SIDX(pingf, INISIDE)); } /** * icmp_ping_new() - Prepare a new ping socket for a new id * @c: Execution context - * @id_sock: Socket fd and other information * @af: Address family, AF_INET or AF_INET6 * @id: ICMP id for the new socket + * @saddr: Source address + * @daddr: Destination address * - * Return: Newly opened ping socket fd, or -1 on failure + * Return: Newly opened ping flow, or NULL on failure */ -static int icmp_ping_new(const struct ctx *c, struct icmp_id_sock *id_sock, - sa_family_t af, uint16_t id) +static struct icmp_ping_flow *icmp_ping_new(const struct ctx *c, + sa_family_t af, uint16_t id, + const void *saddr, const void *daddr) { uint8_t proto = af == AF_INET ? IPPROTO_ICMP : IPPROTO_ICMPV6; - const char *const pname = af == AF_INET ? "ICMP" : "ICMPv6"; - union icmp_epoll_ref iref = { .id = id }; - const void *bind_addr; - const char *bind_if; - int s; + uint8_t flowtype = af == AF_INET ? FLOW_PING4 : FLOW_PING6; + union epoll_ref ref = { .type = EPOLL_TYPE_PING }; + union flow *flow = flow_alloc(); + struct icmp_ping_flow *pingf; + const struct flowside *tgt; - if (af == AF_INET) { - bind_addr = &c->ip4.addr_out; - bind_if = c->ip4.ifname_out; - } else { - bind_addr = &c->ip6.addr_out; - bind_if = c->ip6.ifname_out; + if (!flow) + return NULL; + + flow_initiate_af(flow, PIF_TAP, af, saddr, id, daddr, id); + if (!(tgt = flow_target(c, flow, proto))) + goto cancel; + + if (flow->f.pif[TGTSIDE] != PIF_HOST) { + flow_err(flow, "No support for forwarding %s from %s to %s", + proto == IPPROTO_ICMP ? "ICMP" : "ICMPv6", + pif_name(flow->f.pif[INISIDE]), + pif_name(flow->f.pif[TGTSIDE])); + goto cancel; } - s = sock_l4(c, af, proto, bind_addr, bind_if, 0, iref.u32); + pingf = FLOW_SET_TYPE(flow, flowtype, ping); + + pingf->seq = -1; - if (s < 0) { + ref.flowside = FLOW_SIDX(flow, TGTSIDE); + pingf->sock = flowside_sock_l4(c, EPOLL_TYPE_PING, PIF_HOST, + tgt, ref.data); + + if (pingf->sock < 0) { warn("Cannot open \"ping\" socket. You might need to:"); warn(" sysctl -w net.ipv4.ping_group_range=\"0 2147483647\""); warn("...echo requests/replies will fail."); goto cancel; } - if (s > FD_REF_MAX) + if (pingf->sock > FD_REF_MAX) goto cancel; - id_sock->sock = s; + flow_dbg(pingf, "new socket %i for echo ID %"PRIu16, pingf->sock, id); + + flow_hash_insert(c, FLOW_SIDX(pingf, INISIDE)); - debug("%s: new socket %i for echo ID %"PRIu16, pname, s, id); + FLOW_ACTIVATE(pingf); - return s; + return pingf; cancel: - if (s >= 0) - close(s); - return -1; + flow_alloc_cancel(flow); + return NULL; } /** @@ -212,111 +238,93 @@ int icmp_tap_handler(const struct ctx *c, uint8_t pif, sa_family_t af, const void *saddr, const void *daddr, const struct pool *p, const struct timespec *now) { - const char *const pname = af == AF_INET ? "ICMP" : "ICMPv6"; - union sockaddr_inany sa = { .sa_family = af }; - const socklen_t sl = af == AF_INET ? sizeof(sa.sa4) : sizeof(sa.sa6); - struct icmp_id_sock *id_sock; + struct icmp_ping_flow *pingf; + const struct flowside *tgt; + union sockaddr_inany sa; + size_t dlen, l4len; uint16_t id, seq; - size_t plen; + union flow *flow; + uint8_t proto; + socklen_t sl; void *pkt; - int s; (void)saddr; - (void)pif; + ASSERT(pif == PIF_TAP); if (af == AF_INET) { const struct icmphdr *ih; - if (!(pkt = packet_get(p, 0, 0, sizeof(*ih), &plen))) + if (!(pkt = packet_get(p, 0, 0, sizeof(*ih), &dlen))) return 1; ih = (struct icmphdr *)pkt; - plen += sizeof(*ih); + l4len = dlen + sizeof(*ih); if (ih->type != ICMP_ECHO) return 1; + proto = IPPROTO_ICMP; id = ntohs(ih->un.echo.id); - id_sock = &icmp_id_map[V4][id]; seq = ntohs(ih->un.echo.sequence); - sa.sa4.sin_addr = *(struct in_addr *)daddr; } else if (af == AF_INET6) { const struct icmp6hdr *ih; - if (!(pkt = packet_get(p, 0, 0, sizeof(*ih), &plen))) + if (!(pkt = packet_get(p, 0, 0, sizeof(*ih), &dlen))) return 1; ih = (struct icmp6hdr *)pkt; - plen += sizeof(*ih); + l4len = dlen + sizeof(*ih); if (ih->icmp6_type != ICMPV6_ECHO_REQUEST) return 1; + proto = IPPROTO_ICMPV6; id = ntohs(ih->icmp6_identifier); - id_sock = &icmp_id_map[V6][id]; seq = ntohs(ih->icmp6_sequence); - sa.sa6.sin6_addr = *(struct in6_addr *)daddr; - sa.sa6.sin6_scope_id = c->ifi6; } else { ASSERT(0); } - if ((s = id_sock->sock) < 0) - if ((s = icmp_ping_new(c, id_sock, af, id)) < 0) - return 1; + flow = flow_at_sidx(flow_lookup_af(c, proto, PIF_TAP, + af, saddr, daddr, id, id)); + + if (flow) + pingf = &flow->ping; + else if (!(pingf = icmp_ping_new(c, af, id, saddr, daddr))) + return 1; + + tgt = &pingf->f.side[TGTSIDE]; - id_sock->ts = now->tv_sec; + ASSERT(flow_proto[pingf->f.type] == proto); + pingf->ts = now->tv_sec; - if (sendto(s, pkt, plen, MSG_NOSIGNAL, &sa.sa, sl) < 0) { - debug("%s: failed to relay request to socket: %s", - pname, strerror(errno)); + pif_sockaddr(c, &sa, &sl, PIF_HOST, &tgt->eaddr, 0); + if (sendto(pingf->sock, pkt, l4len, MSG_NOSIGNAL, &sa.sa, sl) < 0) { + flow_dbg(pingf, "failed to relay request to socket: %s", + strerror(errno)); } else { - debug("%s: echo request to socket, ID: %"PRIu16", seq: %"PRIu16, - pname, id, seq); + flow_dbg(pingf, + "echo request to socket, ID: %"PRIu16", seq: %"PRIu16, + id, seq); } return 1; } /** - * icmp_timer_one() - Handler for timed events related to a given identifier + * icmp_ping_timer() - Handler for timed events related to a given flow * @c: Execution context - * @id_sock: Socket fd and activity timestamp + * @pingf: Ping flow to check for timeout * @now: Current timestamp + * + * Return: true if the flow is ready to free, false otherwise */ -static void icmp_timer_one(const struct ctx *c, struct icmp_id_sock *id_sock, - const struct timespec *now) -{ - if (id_sock->sock < 0 || now->tv_sec - id_sock->ts <= ICMP_ECHO_TIMEOUT) - return; - - icmp_ping_close(c, id_sock); -} - -/** - * icmp_timer() - Scan activity bitmap for identifiers with timed events - * @c: Execution context - * @now: Current timestamp - */ -void icmp_timer(const struct ctx *c, const struct timespec *now) -{ - unsigned int i; - - for (i = 0; i < ICMP_NUM_IDS; i++) { - icmp_timer_one(c, &icmp_id_map[V4][i], now); - icmp_timer_one(c, &icmp_id_map[V6][i], now); - } -} - -/** - * icmp_init() - Initialise sequences in ID map to -1 (no sequence sent yet) - */ -void icmp_init(void) +bool icmp_ping_timer(const struct ctx *c, const struct icmp_ping_flow *pingf, + const struct timespec *now) { - unsigned i; + if (now->tv_sec - pingf->ts <= ICMP_ECHO_TIMEOUT) + return false; - for (i = 0; i < ICMP_NUM_IDS; i++) { - icmp_id_map[V4][i].seq = icmp_id_map[V6][i].seq = -1; - icmp_id_map[V4][i].sock = icmp_id_map[V6][i].sock = -1; - } + icmp_ping_close(c, pingf); + return true; } @@ -9,26 +9,15 @@ #define ICMP_TIMER_INTERVAL 10000 /* ms */ struct ctx; +struct icmp_ping_flow; -void icmp_sock_handler(const struct ctx *c, sa_family_t af, union epoll_ref ref); +void icmp_sock_handler(const struct ctx *c, union epoll_ref ref); int icmp_tap_handler(const struct ctx *c, uint8_t pif, sa_family_t af, const void *saddr, const void *daddr, const struct pool *p, const struct timespec *now); -void icmp_timer(const struct ctx *c, const struct timespec *now); void icmp_init(void); /** - * union icmp_epoll_ref - epoll reference portion for ICMP tracking - * @v6: Set for IPv6 sockets or connections - * @u32: Opaque u32 value of reference - * @id: Associated echo identifier, needed if bind() fails - */ -union icmp_epoll_ref { - uint16_t id; - uint32_t u32; -}; - -/** * struct icmp_ctx - Execution context for ICMP routines * @timer_run: Timestamp of most recent timer run */ diff --git a/icmp_flow.h b/icmp_flow.h new file mode 100644 index 0000000..fb93801 --- /dev/null +++ b/icmp_flow.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * Copyright Red Hat + * Author: David Gibson <david@gibson.dropbear.id.au> + * + * ICMP flow tracking data structures + */ +#ifndef ICMP_FLOW_H +#define ICMP_FLOW_H + +/** + * struct icmp_ping_flow - Descriptor for a flow of ping requests/replies + * @f: Generic flow information + * @seq: Last sequence number sent to tap, host order, -1: not sent yet + * @sock: "ping" socket + * @ts: Last associated activity from tap, seconds + */ +struct icmp_ping_flow { + /* Must be first element */ + struct flow_common f; + + int seq; + int sock; + time_t ts; +}; + +bool icmp_ping_timer(const struct ctx *c, const struct icmp_ping_flow *pingf, + const struct timespec *now); + +#endif /* ICMP_FLOW_H */ @@ -17,21 +17,8 @@ #include "siphash.h" #include "inany.h" -const union inany_addr inany_loopback4 = { - .v4mapped = { - .zero = { 0 }, - .one = { 0xff, 0xff, }, - .a4 = IN4ADDR_LOOPBACK_INIT, - }, -}; - -const union inany_addr inany_any4 = { - .v4mapped = { - .zero = { 0 }, - .one = { 0xff, 0xff, }, - .a4 = IN4ADDR_ANY_INIT, - }, -}; +const union inany_addr inany_loopback4 = INANY_INIT4(IN4ADDR_LOOPBACK_INIT); +const union inany_addr inany_any4 = INANY_INIT4(IN4ADDR_ANY_INIT); /** inany_ntop - Convert an IPv[46] address to text format * @src: IPv[46] address @@ -49,3 +36,23 @@ const char *inany_ntop(const union inany_addr *src, char *dst, socklen_t size) return inet_ntop(AF_INET6, &src->a6, dst, size); } + +/** inany_pton - Parse an IPv[46] address from text format + * @src: IPv[46] address + * @dst: output buffer, filled with parsed address + * + * Return: On success, 1, if no parseable address is found, 0 + */ +int inany_pton(const char *src, union inany_addr *dst) +{ + if (inet_pton(AF_INET, src, &dst->v4mapped.a4)) { + memset(&dst->v4mapped.zero, 0, sizeof(dst->v4mapped.zero)); + memset(&dst->v4mapped.one, 0xff, sizeof(dst->v4mapped.one)); + return 1; + } + + if (inet_pton(AF_INET6, src, &dst->a6)) + return 1; + + return 0; +} @@ -43,6 +43,17 @@ extern const union inany_addr inany_any4; #define in4addr_loopback (inany_loopback4.v4mapped.a4) #define in4addr_any (inany_any4.v4mapped.a4) +#define INANY_INIT4(a4init) { \ + .v4mapped = { \ + .zero = { 0 }, \ + .one = { 0xff, 0xff }, \ + .a4 = a4init, \ + }, \ + } + +#define inany_from_v4(a4) \ + ((union inany_addr)INANY_INIT4((a4))) + /** union sockaddr_inany - Either a sockaddr_in or a sockaddr_in6 * @sa_family: Address family, AF_INET or AF_INET6 * @sa: Plain struct sockaddr (useful to avoid casts) @@ -79,6 +90,54 @@ static inline bool inany_equals(const union inany_addr *a, return IN6_ARE_ADDR_EQUAL(&a->a6, &b->a6); } +/** inany_equals4 - Compare an IPv[46] address to an IPv4 address + * @a: IPv[46] addresses + * @b: IPv4 address + * + * Return: true if @a and @b are the same address + */ +static inline bool inany_equals4(const union inany_addr *a, + const struct in_addr *b) +{ + const struct in_addr *a4 = inany_v4(a); + + return a4 && IN4_ARE_ADDR_EQUAL(a4, b); +} + +/** inany_equals6 - Compare an IPv[46] address to an IPv6 address + * @a: IPv[46] addresses + * @b: IPv6 address + * + * Return: true if @a and @b are the same address + */ +static inline bool inany_equals6(const union inany_addr *a, + const struct in6_addr *b) +{ + return IN6_ARE_ADDR_EQUAL(&a->a6, b); +} + +/** inany_is_loopback4() - Check if address is IPv4 loopback + * @a: IPv[46] address + * + * Return: true if @a is in 127.0.0.1/8 + */ +static inline bool inany_is_loopback4(const union inany_addr *a) +{ + const struct in_addr *v4 = inany_v4(a); + + return v4 && IN4_IS_ADDR_LOOPBACK(v4); +} + +/** inany_is_loopback6() - Check if address is IPv6 loopback + * @a: IPv[46] address + * + * Return: true if @a is in ::1 + */ +static inline bool inany_is_loopback6(const union inany_addr *a) +{ + return IN6_IS_ADDR_LOOPBACK(&a->a6); +} + /** inany_is_loopback() - Check if address is loopback * @a: IPv[46] address * @@ -86,9 +145,29 @@ static inline bool inany_equals(const union inany_addr *a, */ static inline bool inany_is_loopback(const union inany_addr *a) { + return inany_is_loopback4(a) || inany_is_loopback6(a); +} + +/** inany_is_unspecified4() - Check if address is unspecified IPv4 + * @a: IPv[46] address + * + * Return: true if @a is 0.0.0.0 + */ +static inline bool inany_is_unspecified4(const union inany_addr *a) +{ const struct in_addr *v4 = inany_v4(a); - return IN6_IS_ADDR_LOOPBACK(&a->a6) || (v4 && IN4_IS_ADDR_LOOPBACK(v4)); + return v4 && IN4_IS_ADDR_UNSPECIFIED(v4); +} + +/** inany_is_unspecified6() - Check if address is unspecified IPv6 + * @a: IPv[46] address + * + * Return: true if @a is :: + */ +static inline bool inany_is_unspecified6(const union inany_addr *a) +{ + return IN6_IS_ADDR_UNSPECIFIED(&a->a6); } /** inany_is_unspecified() - Check if address is unspecified @@ -98,10 +177,19 @@ static inline bool inany_is_loopback(const union inany_addr *a) */ static inline bool inany_is_unspecified(const union inany_addr *a) { - const struct in_addr *v4 = inany_v4(a); + return inany_is_unspecified4(a) || inany_is_unspecified6(a); +} + +/* FIXME: consider handling of IPv4 link-local addresses */ - return IN6_IS_ADDR_UNSPECIFIED(&a->a6) || - (v4 && IN4_IS_ADDR_UNSPECIFIED(v4)); +/** inany_is_linklocal6() - Check if address is link-local IPv6 + * @a: IPv[46] address + * + * Return: true if @a is in fe80::/10 (IPv6 link local unicast) + */ +static inline bool inany_is_linklocal6(const union inany_addr *a) +{ + return IN6_IS_ADDR_LINKLOCAL(&a->a6); } /** inany_is_multicast() - Check if address is multicast or broadcast @@ -123,7 +211,6 @@ static inline bool inany_is_multicast(const union inany_addr *a) * * Return: true if @a is specified and a unicast address */ -/* cppcheck-suppress unusedFunction */ static inline bool inany_is_unicast(const union inany_addr *a) { return !inany_is_unspecified(a) && !inany_is_multicast(a); @@ -183,5 +270,6 @@ static inline void inany_siphash_feed(struct siphash_state *state, #define INANY_ADDRSTRLEN MAX(INET_ADDRSTRLEN, INET6_ADDRSTRLEN) const char *inany_ntop(const union inany_addr *src, char *dst, socklen_t size); +int inany_pton(const char *src, union inany_addr *dst); #endif /* INANY_H */ @@ -68,7 +68,6 @@ size_t iov_skip_bytes(const struct iovec *iov, size_t n, * * Returns: The number of bytes successfully copied. */ -/* cppcheck-suppress unusedFunction */ size_t iov_from_buf(const struct iovec *iov, size_t iov_cnt, size_t offset, const void *buf, size_t bytes) { @@ -18,6 +18,9 @@ #include <unistd.h> #include <string.h> +#define IOV_OF_LVALUE(lval) \ + (struct iovec){ .iov_base = &(lval), .iov_len = sizeof(lval) } + size_t iov_skip_bytes(const struct iovec *iov, size_t n, size_t skip, size_t *offset); size_t iov_from_buf(const struct iovec *iov, size_t iov_cnt, @@ -24,6 +24,11 @@ #define IN4ADDR_ANY_INIT \ { .s_addr = htonl_constant(INADDR_ANY) } +#define IN4_IS_ADDR_LINKLOCAL(a) \ + ((ntohl(((struct in_addr *)(a))->s_addr) >> 16) == 0xa9fe) +#define IN4_IS_PREFIX_LINKLOCAL(a, len) \ + ((len) >= 16 && IN4_IS_ADDR_LINKLOCAL(a)) + #define L2_BUF_IP4_INIT(proto) \ { \ .version = 4, \ @@ -38,7 +43,11 @@ .daddr = 0, \ } #define L2_BUF_IP4_PSUM(proto) ((uint32_t)htons_constant(0x4500) + \ - (uint32_t)htons_constant(0xff00 | (proto))) + (uint32_t)htons(0xff00 | (proto))) + + +#define IN6_IS_PREFIX_LINKLOCAL(a, len) \ + ((len) >= 10 && IN6_IS_ADDR_LINKLOCAL(a)) #define L2_BUF_IP6_INIT(proto) \ { \ @@ -83,4 +92,13 @@ struct ipv6_opt_hdr { char *ipv6_l4hdr(const struct pool *p, int idx, size_t offset, uint8_t *proto, size_t *dlen); + +/* IPv6 link-local all-nodes multicast adddress, ff02::1 */ +static const struct in6_addr in6addr_ll_all_nodes = { + .s6_addr = { + 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + }, +}; + #endif /* IP_H */ diff --git a/isolation.c b/isolation.c index ca2c68b..c944fb3 100644 --- a/isolation.c +++ b/isolation.c @@ -29,7 +29,8 @@ * * Executed immediately after startup, drops capabilities we don't * need at any point during execution (or which we gain back when we - * need by joining other namespaces). + * need by joining other namespaces), and closes any leaked file we + * might have inherited from the parent process. * * 2. isolate_user() * ================= @@ -105,7 +106,7 @@ static void drop_caps_ep_except(uint64_t keep) int i; if (syscall(SYS_capget, &hdr, data)) - die("Couldn't get current capabilities: %s", strerror(errno)); + die_perror("Couldn't get current capabilities"); for (i = 0; i < CAP_WORDS; i++) { uint32_t mask = keep >> (32 * i); @@ -115,7 +116,7 @@ static void drop_caps_ep_except(uint64_t keep) } if (syscall(SYS_capset, &hdr, data)) - die("Couldn't drop capabilities: %s", strerror(errno)); + die_perror("Couldn't drop capabilities"); } /** @@ -152,30 +153,31 @@ static void clamp_caps(void) */ if (prctl(PR_CAPBSET_DROP, i, 0, 0, 0) && errno != EINVAL && errno != EPERM) - die("Couldn't drop cap %i from bounding set: %s", - i, strerror(errno)); + die_perror("Couldn't drop cap %i from bounding set", i); } if (syscall(SYS_capget, &hdr, data)) - die("Couldn't get current capabilities: %s", strerror(errno)); + die_perror("Couldn't get current capabilities"); for (i = 0; i < CAP_WORDS; i++) data[i].inheritable = 0; if (syscall(SYS_capset, &hdr, data)) - die("Couldn't drop inheritable capabilities: %s", - strerror(errno)); + die_perror("Couldn't drop inheritable capabilities"); } /** - * isolate_initial() - Early, config independent self isolation + * isolate_initial() - Early, mostly config independent self isolation + * @argc: Argument count + * @argv: Command line options: only --fd (if present) is relevant here * * Should: * - drop unneeded capabilities + * - close all open files except for standard streams and the one from --fd * Musn't: * - remove filesytem access (we need to access files during setup) */ -void isolate_initial(void) +void isolate_initial(int argc, char **argv) { uint64_t keep; @@ -209,6 +211,8 @@ void isolate_initial(void) keep |= BIT(CAP_SETFCAP) | BIT(CAP_SYS_PTRACE); drop_caps_ep_except(keep); + + close_open_files(argc, argv); } /** @@ -234,34 +238,30 @@ void isolate_user(uid_t uid, gid_t gid, bool use_userns, const char *userns, if (setgroups(0, NULL)) { /* If we don't have CAP_SETGID, this will EPERM */ if (errno != EPERM) - die("Can't drop supplementary groups: %s", - strerror(errno)); + die_perror("Can't drop supplementary groups"); } if (setgid(gid) != 0) - die("Can't set GID to %u: %s", gid, strerror(errno)); + die_perror("Can't set GID to %u", gid); if (setuid(uid) != 0) - die("Can't set UID to %u: %s", uid, strerror(errno)); + die_perror("Can't set UID to %u", uid); if (*userns) { /* If given a userns, join it */ int ufd; ufd = open(userns, O_RDONLY | O_CLOEXEC); if (ufd < 0) - die("Couldn't open user namespace %s: %s", - userns, strerror(errno)); + die_perror("Couldn't open user namespace %s", userns); if (setns(ufd, CLONE_NEWUSER) != 0) - die("Couldn't enter user namespace %s: %s", - userns, strerror(errno)); + die_perror("Couldn't enter user namespace %s", userns); close(ufd); } else if (use_userns) { /* Create and join a new userns */ if (unshare(CLONE_NEWUSER) != 0) - die("Couldn't create user namespace: %s", - strerror(errno)); + die_perror("Couldn't create user namespace"); } /* Joining a new userns gives us full capabilities; drop the @@ -316,34 +316,34 @@ int isolate_prefork(const struct ctx *c) flags |= CLONE_NEWPID; if (unshare(flags)) { - perror("unshare"); + err_perror("Failed to detach isolating namespaces"); return -errno; } if (mount("", "/", "", MS_UNBINDABLE | MS_REC, NULL)) { - perror("mount /"); + err_perror("Failed to remount /"); return -errno; } if (mount("", TMPDIR, "tmpfs", MS_NODEV | MS_NOEXEC | MS_NOSUID | MS_RDONLY, "nr_inodes=2,nr_blocks=0")) { - perror("mount tmpfs"); + err_perror("Failed to mount empty tmpfs for pivot_root()"); return -errno; } if (chdir(TMPDIR)) { - perror("chdir"); + err_perror("Failed to change directory into empty tmpfs"); return -errno; } if (syscall(SYS_pivot_root, ".", ".")) { - perror("pivot_root"); + err_perror("Failed to pivot_root() into empty tmpfs"); return -errno; } if (umount2(".", MNT_DETACH | UMOUNT_NOFOLLOW)) { - perror("umount2"); + err_perror("Failed to unmount original root filesystem"); return -errno; } @@ -379,17 +379,24 @@ void isolate_postfork(const struct ctx *c) prctl(PR_SET_DUMPABLE, 0); - if (c->mode == MODE_PASTA) { - prog.len = (unsigned short)ARRAY_SIZE(filter_pasta); - prog.filter = filter_pasta; - } else { + switch (c->mode) { + case MODE_PASST: prog.len = (unsigned short)ARRAY_SIZE(filter_passt); prog.filter = filter_passt; + break; + case MODE_PASTA: + prog.len = (unsigned short)ARRAY_SIZE(filter_pasta); + prog.filter = filter_pasta; + break; + case MODE_VU: + prog.len = (unsigned short)ARRAY_SIZE(filter_vu); + prog.filter = filter_vu; + break; + default: + ASSERT(0); } if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) || - prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog)) { - perror("prctl"); - exit(EXIT_FAILURE); - } + prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog)) + die_perror("Failed to apply seccomp filter"); } diff --git a/isolation.h b/isolation.h index 846b2af..80bb68d 100644 --- a/isolation.h +++ b/isolation.h @@ -7,7 +7,7 @@ #ifndef ISOLATION_H #define ISOLATION_H -void isolate_initial(void); +void isolate_initial(int argc, char **argv); void isolate_user(uid_t uid, gid_t gid, bool use_userns, const char *userns, enum passt_modes mode); int isolate_prefork(const struct ctx *c); @@ -39,13 +39,11 @@ void lineread_init(struct lineread *lr, int fd) * * Return: length of line in bytes, -1 if no line was found */ -static int peek_line(struct lineread *lr, bool eof) +static ssize_t peek_line(struct lineread *lr, bool eof) { char *nl; /* Sanity checks (which also document invariants) */ - ASSERT(lr->count >= 0); - ASSERT(lr->next_line >= 0); ASSERT(lr->next_line + lr->count >= lr->next_line); ASSERT(lr->next_line + lr->count <= LINEREAD_BUFFER_SIZE); @@ -74,13 +72,13 @@ static int peek_line(struct lineread *lr, bool eof) * * Return: Length of line read on success, 0 on EOF, negative on error */ -int lineread_get(struct lineread *lr, char **line) +ssize_t lineread_get(struct lineread *lr, char **line) { bool eof = false; - int line_len; + ssize_t line_len; while ((line_len = peek_line(lr, eof)) < 0) { - int rc; + ssize_t rc; if ((lr->next_line + lr->count) == LINEREAD_BUFFER_SIZE) { /* No space at end */ @@ -18,14 +18,15 @@ * @buf: Buffer storing data read from file. */ struct lineread { - int fd; int next_line; - int count; + int fd; + ssize_t next_line; + ssize_t count; /* One extra byte for possible trailing \0 */ char buf[LINEREAD_BUFFER_SIZE+1]; }; void lineread_init(struct lineread *lr, int fd); -int lineread_get(struct lineread *lr, char **line); +ssize_t lineread_get(struct lineread *lr, char **line); #endif /* _LINEREAD_H */ diff --git a/linux_dep.h b/linux_dep.h new file mode 100644 index 0000000..240f50a --- /dev/null +++ b/linux_dep.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * Copyright Red Hat + * + * Declarations for Linux specific dependencies + */ + +#ifndef LINUX_DEP_H +#define LINUX_DEP_H + +/* struct tcp_info_linux - Information from Linux TCP_INFO getsockopt() + * + * Largely derived from include/linux/tcp.h in the Linux kernel + * + * Some fields returned by TCP_INFO have been there for ages and are shared with + * BSD. struct tcp_info from netinet/tcp.h has only those fields. There are + * also a many Linux specific extensions to the structure, which are only found + * in the linux/tcp.h version of struct tcp_info. + * + * We want to use some of those extension fields, when available. We can test + * for availability in the runtime kernel using the length returned from + * getsockopt(). However, we won't necessarily be compiled against the same + * kernel headers as we'll run with, so compiling directly against linux/tcp.h + * means wrapping every field access in an #ifdef whose #else does the same + * thing as when the field is missing at runtime. This rapidly gets messy. + * + * Instead we define here struct tcp_info_linux which includes all the Linux + * extensions that we want to use. This is taken from v6.11 of the kernel. + */ +struct tcp_info_linux { + uint8_t tcpi_state; + uint8_t tcpi_ca_state; + uint8_t tcpi_retransmits; + uint8_t tcpi_probes; + uint8_t tcpi_backoff; + uint8_t tcpi_options; + uint8_t tcpi_snd_wscale : 4, tcpi_rcv_wscale : 4; + uint8_t tcpi_delivery_rate_app_limited:1, tcpi_fastopen_client_fail:2; + + uint32_t tcpi_rto; + uint32_t tcpi_ato; + uint32_t tcpi_snd_mss; + uint32_t tcpi_rcv_mss; + + uint32_t tcpi_unacked; + uint32_t tcpi_sacked; + uint32_t tcpi_lost; + uint32_t tcpi_retrans; + uint32_t tcpi_fackets; + + /* Times. */ + uint32_t tcpi_last_data_sent; + uint32_t tcpi_last_ack_sent; + uint32_t tcpi_last_data_recv; + uint32_t tcpi_last_ack_recv; + + /* Metrics. */ + uint32_t tcpi_pmtu; + uint32_t tcpi_rcv_ssthresh; + uint32_t tcpi_rtt; + uint32_t tcpi_rttvar; + uint32_t tcpi_snd_ssthresh; + uint32_t tcpi_snd_cwnd; + uint32_t tcpi_advmss; + uint32_t tcpi_reordering; + + uint32_t tcpi_rcv_rtt; + uint32_t tcpi_rcv_space; + + uint32_t tcpi_total_retrans; + + /* Linux extensions */ + uint64_t tcpi_pacing_rate; + uint64_t tcpi_max_pacing_rate; + uint64_t tcpi_bytes_acked; /* RFC4898 tcpEStatsAppHCThruOctetsAcked */ + uint64_t tcpi_bytes_received; /* RFC4898 tcpEStatsAppHCThruOctetsReceived */ + uint32_t tcpi_segs_out; /* RFC4898 tcpEStatsPerfSegsOut */ + uint32_t tcpi_segs_in; /* RFC4898 tcpEStatsPerfSegsIn */ + + uint32_t tcpi_notsent_bytes; + uint32_t tcpi_min_rtt; + uint32_t tcpi_data_segs_in; /* RFC4898 tcpEStatsDataSegsIn */ + uint32_t tcpi_data_segs_out; /* RFC4898 tcpEStatsDataSegsOut */ + + uint64_t tcpi_delivery_rate; + + uint64_t tcpi_busy_time; /* Time (usec) busy sending data */ + uint64_t tcpi_rwnd_limited; /* Time (usec) limited by receive window */ + uint64_t tcpi_sndbuf_limited; /* Time (usec) limited by send buffer */ + + uint32_t tcpi_delivered; + uint32_t tcpi_delivered_ce; + + uint64_t tcpi_bytes_sent; /* RFC4898 tcpEStatsPerfHCDataOctetsOut */ + uint64_t tcpi_bytes_retrans; /* RFC4898 tcpEStatsPerfOctetsRetrans */ + uint32_t tcpi_dsack_dups; /* RFC4898 tcpEStatsStackDSACKDups */ + uint32_t tcpi_reord_seen; /* reordering events seen */ + + uint32_t tcpi_rcv_ooopack; /* Out-of-order packets received */ + + uint32_t tcpi_snd_wnd; /* peer's advertised receive window after + * scaling (bytes) + */ + uint32_t tcpi_rcv_wnd; /* local advertised receive window after + * scaling (bytes) + */ + + uint32_t tcpi_rehash; /* PLB or timeout triggered rehash attempts */ + + uint16_t tcpi_total_rto; /* Total number of RTO timeouts, including + * SYN/SYN-ACK and recurring timeouts. + */ + uint16_t tcpi_total_rto_recoveries; /* Total number of RTO + * recoveries, including any + * unfinished recovery. + */ + uint32_t tcpi_total_rto_time; /* Total time spent in RTO recoveries + * in milliseconds, including any + * unfinished recovery. + */ +}; + +#include <linux/falloc.h> + +#ifndef FALLOC_FL_COLLAPSE_RANGE +#define FALLOC_FL_COLLAPSE_RANGE 0x08 +#endif + +#include <linux/close_range.h> + +/* glibc < 2.34 and musl as of 1.2.5 need these */ +#ifndef SYS_close_range +#define SYS_close_range 436 +#endif +#ifndef CLOSE_RANGE_UNSHARE /* Linux kernel < 5.9 */ +#define CLOSE_RANGE_UNSHARE (1U << 1) +#endif + +__attribute__ ((weak)) +/* cppcheck-suppress funcArgNamesDifferent */ +int close_range(unsigned int first, unsigned int last, int flags) { + return syscall(SYS_close_range, first, last, flags); +} + +#endif /* LINUX_DEP_H */ @@ -26,17 +26,14 @@ #include <stdarg.h> #include <sys/socket.h> +#include "linux_dep.h" #include "log.h" #include "util.h" #include "passt.h" -/* LOG_EARLY means we don't know yet: log everything. LOG_EMERG is unused */ -#define LOG_EARLY LOG_MASK(LOG_EMERG) - static int log_sock = -1; /* Optional socket to system logger */ static char log_ident[BUFSIZ]; /* Identifier string for openlog() */ -static int log_mask = LOG_EARLY; /* Current log priority mask */ -static int log_opt; /* Options for openlog() */ +static int log_mask; /* Current log priority mask */ static int log_file = -1; /* Optional log file descriptor */ static size_t log_size; /* Maximum log file size in bytes */ @@ -44,50 +41,46 @@ static size_t log_written; /* Currently used bytes in log file */ static size_t log_cut_size; /* Bytes to cut at start on rotation */ static char log_header[BUFSIZ]; /* File header, written back on cuts */ -static time_t log_start; /* Start timestamp */ -int log_trace; /* --trace mode enabled */ -int log_to_stdout; /* Print to stdout instead of stderr */ +struct timespec log_start; /* Start timestamp */ -void vlogmsg(int pri, const char *format, va_list ap) -{ - bool debug_print = (log_mask & LOG_MASK(LOG_DEBUG)) && log_file == -1; - bool early_print = LOG_PRI(log_mask) == LOG_EARLY; - FILE *out = log_to_stdout ? stdout : stderr; - struct timespec tp; - - if (debug_print) { - clock_gettime(CLOCK_REALTIME, &tp); - fprintf(out, "%lli.%04lli: ", - (long long int)tp.tv_sec - log_start, - (long long int)tp.tv_nsec / (100L * 1000)); - } - - if ((log_mask & LOG_MASK(LOG_PRI(pri))) || early_print) { - va_list ap2; - - va_copy(ap2, ap); /* Don't clobber ap, we need it again */ - if (log_file != -1) - logfile_write(pri, format, ap2); - else if (!(log_mask & LOG_MASK(LOG_DEBUG))) - passt_vsyslog(pri, format, ap2); +int log_trace; /* --trace mode enabled */ +bool log_conf_parsed; /* Logging options already parsed */ +bool log_stderr = true; /* Not daemonised, no shell spawned */ - va_end(ap2); - } +#define LL_STRLEN (sizeof("-9223372036854775808")) +#define LOGTIME_STRLEN (LL_STRLEN + 5) - if (debug_print || (early_print && !(log_opt & LOG_PERROR))) { - (void)vfprintf(out, format, ap); - if (format[strlen(format)] != '\n') - fprintf(out, "\n"); - } +/** + * logtime() - Get the current time for logging purposes + * @ts: Buffer into which to store the timestamp + * + * Return: pointer to @now, or NULL if there was an error retrieving the time + */ +const struct timespec *logtime(struct timespec *ts) +{ + if (clock_gettime(CLOCK_MONOTONIC, ts)) + return NULL; + return ts; } -void logmsg(int pri, const char *format, ...) +/** + * logtime_fmt() - Format timestamp into a string for the log + * @buf: Buffer into which to format the time + * @size: Size of @buf + * @ts: Time to format (or NULL on error) + * + * Return: number of characters written to @buf (excluding \0) + */ +static int logtime_fmt(char *buf, size_t size, const struct timespec *ts) { - va_list ap; + if (ts) { + int64_t delta = timespec_diff_us(ts, &log_start); - va_start(ap, format); - vlogmsg(pri, format, ap); - va_end(ap); + return snprintf(buf, size, "%lli.%04lli", delta / 1000000LL, + (delta / 100LL) % 10000); + } + + return snprintf(buf, size, "<error>"); } /* Prefixes for log file messages, indexed by priority */ @@ -101,126 +94,11 @@ const char *logfile_prefix[] = { }; /** - * trace_init() - Set log_trace depending on trace (debug) mode - * @enable: Tracing debug mode enabled if non-zero - */ -void trace_init(int enable) -{ - log_trace = enable; -} - -/** - * __openlog() - Non-optional openlog() implementation, for custom vsyslog() - * @ident: openlog() identity (program name) - * @option: openlog() options - * @facility: openlog() facility (LOG_DAEMON) - */ -void __openlog(const char *ident, int option, int facility) -{ - struct timespec tp; - - clock_gettime(CLOCK_REALTIME, &tp); - log_start = tp.tv_sec; - - if (log_sock < 0) { - struct sockaddr_un a = { .sun_family = AF_UNIX, }; - - log_sock = socket(AF_UNIX, SOCK_DGRAM | SOCK_CLOEXEC, 0); - if (log_sock < 0) - return; - - strncpy(a.sun_path, _PATH_LOG, sizeof(a.sun_path)); - if (connect(log_sock, (const struct sockaddr *)&a, sizeof(a))) { - close(log_sock); - log_sock = -1; - return; - } - } - - log_mask |= facility; - strncpy(log_ident, ident, sizeof(log_ident) - 1); - log_opt = option; -} - -/** - * __setlogmask() - setlogmask() wrapper, to allow custom vsyslog() - * @mask: Same as setlogmask() mask - */ -void __setlogmask(int mask) -{ - log_mask = mask; - setlogmask(mask); -} - -/** - * passt_vsyslog() - vsyslog() implementation not using heap memory - * @pri: Facility and level map, same as priority for vsyslog() - * @format: Same as vsyslog() format - * @ap: Same as vsyslog() ap - */ -void passt_vsyslog(int pri, const char *format, va_list ap) -{ - int prefix_len, n; - char buf[BUFSIZ]; - - /* Send without timestamp, the system logger should add it */ - n = prefix_len = snprintf(buf, BUFSIZ, "<%i> %s: ", pri, log_ident); - - n += vsnprintf(buf + n, BUFSIZ - n, format, ap); - - if (format[strlen(format)] != '\n') - n += snprintf(buf + n, BUFSIZ - n, "\n"); - - if (log_opt & LOG_PERROR) - fprintf(stderr, "%s", buf + prefix_len); - - if (send(log_sock, buf, n, 0) != n) - fprintf(stderr, "Failed to send %i bytes to syslog\n", n); -} - -/** - * logfile_init() - Open log file and write header with PID, version, path - * @name: Identifier for header: passt or pasta - * @path: Path to log file - * @size: Maximum size of log file: log_cut_size is calculatd here - */ -void logfile_init(const char *name, const char *path, size_t size) -{ - char nl = '\n', exe[PATH_MAX] = { 0 }; - int n; - - if (readlink("/proc/self/exe", exe, PATH_MAX - 1) < 0) { - perror("readlink /proc/self/exe"); - exit(EXIT_FAILURE); - } - - log_file = open(path, O_CREAT | O_TRUNC | O_APPEND | O_RDWR | O_CLOEXEC, - S_IRUSR | S_IWUSR); - if (log_file == -1) - die("Couldn't open log file %s: %s", path, strerror(errno)); - - log_size = size ? size : LOGFILE_SIZE_DEFAULT; - - n = snprintf(log_header, sizeof(log_header), "%s " VERSION ": %s (%i)", - name, exe, getpid()); - - if (write(log_file, log_header, n) <= 0 || - write(log_file, &nl, 1) <= 0) { - perror("Couldn't write to log file\n"); - exit(EXIT_FAILURE); - } - - /* For FALLOC_FL_COLLAPSE_RANGE: VFS block size can be up to one page */ - log_cut_size = ROUND_UP(log_size * LOGFILE_CUT_RATIO / 100, PAGE_SIZE); -} - -#ifdef FALLOC_FL_COLLAPSE_RANGE -/** * logfile_rotate_fallocate() - Write header, set log_written after fallocate() * @fd: Log file descriptor * @now: Current timestamp * - * #syscalls lseek ppc64le:_llseek ppc64:_llseek armv6l:_llseek armv7l:_llseek + * #syscalls lseek ppc64le:_llseek ppc64:_llseek arm:_llseek i686:_llseek */ static void logfile_rotate_fallocate(int fd, const struct timespec *now) { @@ -233,10 +111,8 @@ static void logfile_rotate_fallocate(int fd, const struct timespec *now) if (read(fd, buf, BUFSIZ) == -1) return; - n = snprintf(buf, BUFSIZ, - "%s - log truncated at %lli.%04lli", log_header, - (long long int)(now->tv_sec - log_start), - (long long int)(now->tv_nsec / (100L * 1000))); + n = snprintf(buf, BUFSIZ, "%s - log truncated at ", log_header); + n += logtime_fmt(buf + n, BUFSIZ - n, now); /* Avoid partial lines by padding the header with spaces */ nl = memchr(buf + n + 1, '\n', BUFSIZ - n - 1); @@ -250,14 +126,13 @@ static void logfile_rotate_fallocate(int fd, const struct timespec *now) log_written -= log_cut_size; } -#endif /* FALLOC_FL_COLLAPSE_RANGE */ /** * logfile_rotate_move() - Fallback: move recent entries toward start, then cut * @fd: Log file descriptor * @now: Current timestamp * - * #syscalls lseek ppc64le:_llseek ppc64:_llseek armv6l:_llseek armv7l:_llseek + * #syscalls lseek ppc64le:_llseek ppc64:_llseek arm:_llseek * #syscalls ftruncate */ static void logfile_rotate_move(int fd, const struct timespec *now) @@ -266,10 +141,10 @@ static void logfile_rotate_move(int fd, const struct timespec *now) char buf[BUFSIZ]; const char *nl; - header_len = snprintf(buf, BUFSIZ, - "%s - log truncated at %lli.%04lli\n", log_header, - (long long int)(now->tv_sec - log_start), - (long long int)(now->tv_nsec / (100L * 1000))); + header_len = snprintf(buf, BUFSIZ, "%s - log truncated at ", + log_header); + header_len += logtime_fmt(buf + header_len, BUFSIZ - header_len, now); + if (lseek(fd, 0, SEEK_SET) == -1) return; if (write(fd, buf, header_len) == -1) @@ -322,21 +197,17 @@ out: * * Return: 0 on success, negative error code on failure * - * #syscalls fcntl - * - * fallocate() passed as EXTRA_SYSCALL only if FALLOC_FL_COLLAPSE_RANGE is there + * #syscalls fcntl fallocate */ static int logfile_rotate(int fd, const struct timespec *now) { if (fcntl(fd, F_SETFL, O_RDWR /* Drop O_APPEND: explicit lseek() */)) return -errno; -#ifdef FALLOC_FL_COLLAPSE_RANGE /* Only for Linux >= 3.15, extent-based ext4 or XFS, glibc >= 2.18 */ if (!fallocate(fd, FALLOC_FL_COLLAPSE_RANGE, 0, log_cut_size)) logfile_rotate_fallocate(fd, now); else -#endif logfile_rotate_move(fd, now); if (fcntl(fd, F_SETFL, O_RDWR | O_APPEND)) @@ -347,32 +218,212 @@ static int logfile_rotate(int fd, const struct timespec *now) /** * logfile_write() - Write entry to log file, trigger rotation if full + * @newline: Append newline at the end of the message, if missing + * @cont: Continuation of a previous message, on the same line * @pri: Facility and level map, same as priority for vsyslog() + * @now: Timestamp * @format: Same as vsyslog() format * @ap: Same as vsyslog() ap */ -void logfile_write(int pri, const char *format, va_list ap) +static void logfile_write(bool newline, bool cont, int pri, + const struct timespec *now, + const char *format, va_list ap) { - struct timespec now; char buf[BUFSIZ]; - int n; - - if (clock_gettime(CLOCK_REALTIME, &now)) - return; + int n = 0; - n = snprintf(buf, BUFSIZ, "%lli.%04lli: %s", - (long long int)(now.tv_sec - log_start), - (long long int)(now.tv_nsec / (100L * 1000)), - logfile_prefix[pri]); + if (!cont) { + n += logtime_fmt(buf, BUFSIZ, now); + n += snprintf(buf + n, BUFSIZ - n, ": %s", logfile_prefix[pri]); + } n += vsnprintf(buf + n, BUFSIZ - n, format, ap); - if (format[strlen(format)] != '\n') + if (newline && format[strlen(format)] != '\n') n += snprintf(buf + n, BUFSIZ - n, "\n"); - if ((log_written + n >= log_size) && logfile_rotate(log_file, &now)) + if ((log_written + n >= log_size) && logfile_rotate(log_file, now)) return; if ((n = write(log_file, buf, n)) >= 0) log_written += n; } + +/** + * vlogmsg() - Print or send messages to log or output files as configured + * @newline: Append newline at the end of the message, if missing + * @cont: Continuation of a previous message, on the same line + * @pri: Facility and level map, same as priority for vsyslog() + * @format: Message + * @ap: Variable argument list + */ +void vlogmsg(bool newline, bool cont, int pri, const char *format, va_list ap) +{ + bool debug_print = (log_mask & LOG_MASK(LOG_DEBUG)) && log_file == -1; + const struct timespec *now; + struct timespec ts; + + now = logtime(&ts); + + if (debug_print && !cont) { + char timestr[LOGTIME_STRLEN]; + + logtime_fmt(timestr, sizeof(timestr), now); + FPRINTF(stderr, "%s: ", timestr); + } + + if ((log_mask & LOG_MASK(LOG_PRI(pri))) || !log_conf_parsed) { + va_list ap2; + + va_copy(ap2, ap); /* Don't clobber ap, we need it again */ + if (log_file != -1) + logfile_write(newline, cont, pri, now, format, ap2); + else if (!(log_mask & LOG_MASK(LOG_DEBUG))) + passt_vsyslog(newline, pri, format, ap2); + + va_end(ap2); + } + + if (debug_print || !log_conf_parsed || + (log_stderr && (log_mask & LOG_MASK(LOG_PRI(pri))))) { + (void)vfprintf(stderr, format, ap); + if (newline && format[strlen(format)] != '\n') + FPRINTF(stderr, "\n"); + } +} + +/** + * logmsg() - vlogmsg() wrapper for variable argument lists + * @newline: Append newline at the end of the message, if missing + * @cont: Continuation of a previous message, on the same line + * @pri: Facility and level map, same as priority for vsyslog() + * @format: Message + */ +void logmsg(bool newline, bool cont, int pri, const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + vlogmsg(newline, cont, pri, format, ap); + va_end(ap); +} + +/** + * logmsg_perror() - vlogmsg() wrapper with perror()-like functionality + * @pri: Facility and level map, same as priority for vsyslog() + * @format: Message + */ +void logmsg_perror(int pri, const char *format, ...) +{ + int errno_copy = errno; + va_list ap; + + va_start(ap, format); + vlogmsg(false, false, pri, format, ap); + va_end(ap); + + logmsg(true, true, pri, ": %s", strerror(errno_copy)); +} + +/** + * trace_init() - Set log_trace depending on trace (debug) mode + * @enable: Tracing debug mode enabled if non-zero + */ +void trace_init(int enable) +{ + log_trace = enable; +} + +/** + * __openlog() - Non-optional openlog() implementation, for custom vsyslog() + * @ident: openlog() identity (program name) + * @option: openlog() options, unused + * @facility: openlog() facility (LOG_DAEMON) + */ +void __openlog(const char *ident, int option, int facility) +{ + (void)option; + + if (log_sock < 0) { + struct sockaddr_un a = { .sun_family = AF_UNIX, }; + + log_sock = socket(AF_UNIX, SOCK_DGRAM | SOCK_CLOEXEC, 0); + if (log_sock < 0) + return; + + strncpy(a.sun_path, _PATH_LOG, sizeof(a.sun_path)); + if (connect(log_sock, (const struct sockaddr *)&a, sizeof(a))) { + close(log_sock); + log_sock = -1; + return; + } + } + + log_mask |= facility; + strncpy(log_ident, ident, sizeof(log_ident) - 1); +} + +/** + * __setlogmask() - setlogmask() wrapper, to allow custom vsyslog() + * @mask: Same as setlogmask() mask + */ +void __setlogmask(int mask) +{ + log_mask = mask; + setlogmask(mask); +} + +/** + * passt_vsyslog() - vsyslog() implementation not using heap memory + * @newline: Append newline at the end of the message, if missing + * @pri: Facility and level map, same as priority for vsyslog() + * @format: Same as vsyslog() format + * @ap: Same as vsyslog() ap + */ +void passt_vsyslog(bool newline, int pri, const char *format, va_list ap) +{ + char buf[BUFSIZ]; + int n; + + /* Send without timestamp, the system logger should add it */ + n = snprintf(buf, BUFSIZ, "<%i> %s: ", pri, log_ident); + + n += vsnprintf(buf + n, BUFSIZ - n, format, ap); + + if (newline && format[strlen(format)] != '\n') + n += snprintf(buf + n, BUFSIZ - n, "\n"); + + if (log_sock >= 0 && send(log_sock, buf, n, 0) != n && log_stderr) + FPRINTF(stderr, "Failed to send %i bytes to syslog\n", n); +} + +/** + * logfile_init() - Open log file and write header with PID, version, path + * @name: Identifier for header: passt or pasta + * @path: Path to log file + * @size: Maximum size of log file: log_cut_size is calculatd here + */ +void logfile_init(const char *name, const char *path, size_t size) +{ + char nl = '\n', exe[PATH_MAX] = { 0 }; + int n; + + if (readlink("/proc/self/exe", exe, PATH_MAX - 1) < 0) + die_perror("Failed to read own /proc/self/exe link"); + + log_file = output_file_open(path, O_APPEND | O_RDWR); + if (log_file == -1) + die_perror("Couldn't open log file %s", path); + + log_size = size ? size : LOGFILE_SIZE_DEFAULT; + + n = snprintf(log_header, sizeof(log_header), "%s " VERSION ": %s (%i)", + name, exe, getpid()); + + if (write(log_file, log_header, n) <= 0 || + write(log_file, &nl, 1) <= 0) + die_perror("Couldn't write to log file"); + + /* For FALLOC_FL_COLLAPSE_RANGE: VFS block size can be up to one page */ + log_cut_size = ROUND_UP(log_size * LOGFILE_CUT_RATIO / 100, PAGE_SIZE); +} @@ -6,20 +6,28 @@ #ifndef LOG_H #define LOG_H +#include <stdbool.h> #include <syslog.h> #define LOGFILE_SIZE_DEFAULT (1024 * 1024UL) #define LOGFILE_CUT_RATIO 30 /* When full, cut ~30% size */ #define LOGFILE_SIZE_MIN (5UL * MAX(BUFSIZ, PAGE_SIZE)) -void vlogmsg(int pri, const char *format, va_list ap); -void logmsg(int pri, const char *format, ...) +void vlogmsg(bool newline, bool cont, int pri, const char *format, va_list ap); +void logmsg(bool newline, bool cont, int pri, const char *format, ...) + __attribute__((format(printf, 4, 5))); +void logmsg_perror(int pri, const char *format, ...) __attribute__((format(printf, 2, 3))); -#define err(...) logmsg(LOG_ERR, __VA_ARGS__) -#define warn(...) logmsg(LOG_WARNING, __VA_ARGS__) -#define info(...) logmsg(LOG_INFO, __VA_ARGS__) -#define debug(...) logmsg(LOG_DEBUG, __VA_ARGS__) +#define err(...) logmsg(true, false, LOG_ERR, __VA_ARGS__) +#define warn(...) logmsg(true, false, LOG_WARNING, __VA_ARGS__) +#define info(...) logmsg(true, false, LOG_INFO, __VA_ARGS__) +#define debug(...) logmsg(true, false, LOG_DEBUG, __VA_ARGS__) + +#define err_perror(...) logmsg_perror( LOG_ERR, __VA_ARGS__) +#define warn_perror(...) logmsg_perror( LOG_WARNING, __VA_ARGS__) +#define info_perror(...) logmsg_perror( LOG_INFO, __VA_ARGS__) +#define debug_perror(...) logmsg_perror( LOG_DEBUG, __VA_ARGS__) #define die(...) \ do { \ @@ -27,8 +35,17 @@ void logmsg(int pri, const char *format, ...) exit(EXIT_FAILURE); \ } while (0) +#define die_perror(...) \ + do { \ + err_perror(__VA_ARGS__); \ + exit(EXIT_FAILURE); \ + } while (0) + extern int log_trace; -extern int log_to_stdout; +extern bool log_conf_parsed; +extern bool log_stderr; +extern struct timespec log_start; + void trace_init(int enable); #define trace(...) \ do { \ @@ -38,8 +55,7 @@ void trace_init(int enable); void __openlog(const char *ident, int option, int facility); void logfile_init(const char *name, const char *path, size_t size); -void passt_vsyslog(int pri, const char *format, va_list ap); -void logfile_write(int pri, const char *format, va_list ap); +void passt_vsyslog(bool newline, int pri, const char *format, va_list ap); void __setlogmask(int mask); #endif /* LOG_H */ @@ -33,161 +33,396 @@ #include "tap.h" #include "log.h" +#define RT_LIFETIME 65535 + #define RS 133 #define RA 134 #define NS 135 #define NA 136 +enum ndp_option_types { + OPT_SRC_L2_ADDR = 1, + OPT_TARGET_L2_ADDR = 2, + OPT_PREFIX_INFO = 3, + OPT_MTU = 5, + OPT_RDNSS_TYPE = 25, + OPT_DNSSL_TYPE = 31, +}; + /** - * ndp() - Check for NDP solicitations, reply as needed + * struct opt_header - Option header + * @type: Option type + * @len: Option length, in units of 8 bytes +*/ +struct opt_header { + uint8_t type; + uint8_t len; +} __attribute__((packed)); + +/** + * struct opt_l2_addr - Link-layer address + * @header: Option header + * @mac: MAC address + */ +struct opt_l2_addr { + struct opt_header header; + unsigned char mac[ETH_ALEN]; +} __attribute__((packed)); + +/** + * struct ndp_na - NDP Neighbor Advertisement (NA) message + * @ih: ICMPv6 header + * @target_addr: Target IPv6 address + * @target_l2_addr: Target link-layer address + */ +struct ndp_na { + struct icmp6hdr ih; + struct in6_addr target_addr; + struct opt_l2_addr target_l2_addr; +} __attribute__((packed)); + +/** + * struct opt_prefix_info - Prefix Information option + * @header: Option header + * @prefix_len: The number of leading bits in the Prefix that are valid + * @prefix_flags: Flags associated with the prefix + * @valid_lifetime: Valid lifetime (ms) + * @pref_lifetime: Preferred lifetime (ms) + * @reserved: Unused + */ +struct opt_prefix_info { + struct opt_header header; + uint8_t prefix_len; + uint8_t prefix_flags; + uint32_t valid_lifetime; + uint32_t pref_lifetime; + uint32_t reserved; +} __attribute__((packed)); + +/** + * struct opt_mtu - Maximum transmission unit (MTU) option + * @header: Option header + * @reserved: Unused + * @value: MTU value, network order + */ +struct opt_mtu { + struct opt_header header; + uint16_t reserved; + uint32_t value; +} __attribute__((packed)); + +/** + * struct rdnss - Recursive DNS Server (RDNSS) option + * @header: Option header + * @reserved: Unused + * @lifetime: Validity time (s) + * @dns: List of DNS server addresses + */ +struct opt_rdnss { + struct opt_header header; + uint16_t reserved; + uint32_t lifetime; + struct in6_addr dns[MAXNS + 1]; +} __attribute__((packed)); + +/** + * struct dnssl - DNS Search List (DNSSL) option + * @header: Option header + * @reserved: Unused + * @lifetime: Validity time (s) + * @domains: List of NULL-seperated search domains + */ +struct opt_dnssl { + struct opt_header header; + uint16_t reserved; + uint32_t lifetime; + unsigned char domains[MAXDNSRCH * NS_MAXDNAME]; +} __attribute__((packed)); + +/** + * struct ndp_ra - NDP Router Advertisement (RA) message + * @ih: ICMPv6 header + * @reachable: Reachability time, after confirmation (ms) + * @retrans: Time between retransmitted NS messages (ms) + * @prefix_info: Prefix Information option + * @prefix: IPv6 prefix + * @mtu: MTU option + * @source_ll: Target link-layer address + * @var: Variable fields + */ +struct ndp_ra { + struct icmp6hdr ih; + uint32_t reachable; + uint32_t retrans; + struct opt_prefix_info prefix_info; + struct in6_addr prefix; + struct opt_l2_addr source_ll; + + unsigned char var[sizeof(struct opt_mtu) + sizeof(struct opt_rdnss) + + sizeof(struct opt_dnssl)]; +} __attribute__((packed, aligned(__alignof__(struct in6_addr)))); + +/** + * struct ndp_ns - NDP Neighbor Solicitation (NS) message + * @ih: ICMPv6 header + * @target_addr: Target IPv6 address + */ +struct ndp_ns { + struct icmp6hdr ih; + struct in6_addr target_addr; +} __attribute__((packed, aligned(__alignof__(struct in6_addr)))); + +/** + * ndp_send() - Send an NDP message * @c: Execution context - * @ih: ICMPv6 header - * @saddr Source IPv6 address - * - * Return: 0 if not handled here, 1 if handled, -1 on failure + * @dst: IPv6 address to send the message to + * @buf: ICMPv6 header + message payload + * @l4len: Length of message, including ICMPv6 header */ -int ndp(struct ctx *c, const struct icmp6hdr *ih, const struct in6_addr *saddr) +static void ndp_send(const struct ctx *c, const struct in6_addr *dst, + const void *buf, size_t l4len) { - const struct in6_addr *rsaddr; /* src addr for reply */ - char buf[BUFSIZ] = { 0 }; - struct ipv6hdr *ip6hr; - struct icmp6hdr *ihr; - struct ethhdr *ehr; - unsigned char *p; - size_t len; + const struct in6_addr *src = &c->ip6.our_tap_ll; - if (ih->icmp6_type < RS || ih->icmp6_type > NA) - return 0; + tap_icmp6_send(c, src, dst, buf, l4len); +} - if (c->no_ndp) - return 1; +/** + * ndp_na() - Send an NDP Neighbour Advertisement (NA) message + * @c: Execution context + * @dst: IPv6 address to send the NA to + * @addr: IPv6 address to advertise + */ +static void ndp_na(const struct ctx *c, const struct in6_addr *dst, + const struct in6_addr *addr) +{ + struct ndp_na na = { + .ih = { + .icmp6_type = NA, + .icmp6_code = 0, + .icmp6_router = 1, + .icmp6_solicited = 1, + .icmp6_override = 1, + }, + .target_addr = *addr, + .target_l2_addr = { + .header = { + .type = OPT_TARGET_L2_ADDR, + .len = 1, + }, + } + }; - ehr = (struct ethhdr *)buf; - ip6hr = (struct ipv6hdr *)(ehr + 1); - ihr = (struct icmp6hdr *)(ip6hr + 1); + memcpy(na.target_l2_addr.mac, c->our_tap_mac, ETH_ALEN); - if (ih->icmp6_type == NS) { - if (IN6_IS_ADDR_UNSPECIFIED(saddr)) - return 1; + ndp_send(c, dst, &na, sizeof(na)); +} - info("NDP: received NS, sending NA"); - ihr->icmp6_type = NA; - ihr->icmp6_code = 0; - ihr->icmp6_router = 1; - ihr->icmp6_solicited = 1; - ihr->icmp6_override = 1; - - p = (unsigned char *)(ihr + 1); - memcpy(p, ih + 1, sizeof(struct in6_addr)); /* target address */ - p += 16; - *p++ = 2; /* target ll */ - *p++ = 1; /* length */ - memcpy(p, c->mac, ETH_ALEN); - p += 6; - } else if (ih->icmp6_type == RS) { - size_t dns_s_len = 0; - int i, n; +/** + * ndp_ra() - Send an NDP Router Advertisement (RA) message + * @c: Execution context + * @dst: IPv6 address to send the RA to + */ +static void ndp_ra(const struct ctx *c, const struct in6_addr *dst) +{ + struct ndp_ra ra = { + .ih = { + .icmp6_type = RA, + .icmp6_code = 0, + .icmp6_hop_limit = 255, + /* RFC 8319 */ + .icmp6_rt_lifetime = htons_constant(RT_LIFETIME), + .icmp6_addrconf_managed = 1, + }, + .prefix_info = { + .header = { + .type = OPT_PREFIX_INFO, + .len = 4, + }, + .prefix_len = 64, + .prefix_flags = 0xc0, /* prefix flags: L, A */ + .valid_lifetime = ~0U, + .pref_lifetime = ~0U, + }, + .prefix = c->ip6.addr, + .source_ll = { + .header = { + .type = OPT_SRC_L2_ADDR, + .len = 1, + }, + }, + }; + unsigned char *ptr = NULL; - if (c->no_ra) - return 1; + ptr = &ra.var[0]; - info("NDP: received RS, sending RA"); - ihr->icmp6_type = RA; - ihr->icmp6_code = 0; - ihr->icmp6_hop_limit = 255; - ihr->icmp6_rt_lifetime = htons(65535); /* RFC 8319 */ - ihr->icmp6_addrconf_managed = 1; - - p = (unsigned char *)(ihr + 1); - p += 8; /* reachable, retrans time */ - *p++ = 3; /* prefix */ - *p++ = 4; /* length */ - *p++ = 64; /* prefix length */ - *p++ = 0xc0; /* prefix flags: L, A */ - *(uint32_t *)p = (uint32_t)~0U; /* lifetime */ - p += 4; - *(uint32_t *)p = (uint32_t)~0U; /* preferred lifetime */ - p += 8; - memcpy(p, &c->ip6.addr, 8); /* prefix */ - p += 16; - - if (c->mtu != -1) { - *p++ = 5; /* type */ - *p++ = 1; /* length */ - p += 2; /* reserved */ - *(uint32_t *)p = htonl(c->mtu); /* MTU */ - p += 4; - } + if (c->mtu != -1) { + struct opt_mtu *mtu = (struct opt_mtu *)ptr; + *mtu = (struct opt_mtu) { + .header = { + .type = OPT_MTU, + .len = 1, + }, + .value = htonl(c->mtu), + }; + ptr += sizeof(struct opt_mtu); + } - if (c->no_dhcp_dns) - goto dns_done; + if (!c->no_dhcp_dns) { + size_t dns_s_len = 0; + int i, n; for (n = 0; !IN6_IS_ADDR_UNSPECIFIED(&c->ip6.dns[n]); n++); if (n) { - *p++ = 25; /* RDNSS */ - *p++ = 1 + 2 * n; /* length */ - p += 2; /* reserved */ - *(uint32_t *)p = (uint32_t)~0U; /* lifetime */ - p += 4; - + struct opt_rdnss *rdnss = (struct opt_rdnss *)ptr; + *rdnss = (struct opt_rdnss) { + .header = { + .type = OPT_RDNSS_TYPE, + .len = 1 + 2 * n, + }, + .lifetime = ~0U, + }; for (i = 0; i < n; i++) { - memcpy(p, &c->ip6.dns[i], 16); /* address */ - p += 16; + rdnss->dns[i] = c->ip6.dns[i]; } + ptr += offsetof(struct opt_rdnss, dns) + + i * sizeof(rdnss->dns[0]); for (n = 0; *c->dns_search[n].n; n++) dns_s_len += strlen(c->dns_search[n].n) + 2; } if (!c->no_dhcp_dns_search && dns_s_len) { - *p++ = 31; /* DNSSL */ - *p++ = (dns_s_len + 8 - 1) / 8 + 1; /* length */ - p += 2; /* reserved */ - *(uint32_t *)p = (uint32_t)~0U; /* lifetime */ - p += 4; + struct opt_dnssl *dnssl = (struct opt_dnssl *)ptr; + *dnssl = (struct opt_dnssl) { + .header = { + .type = OPT_DNSSL_TYPE, + .len = DIV_ROUND_UP(dns_s_len, 8) + 1, + }, + .lifetime = ~0U, + }; + ptr = dnssl->domains; for (i = 0; i < n; i++) { + size_t len; char *dot; - *(p++) = '.'; + *(ptr++) = '.'; + + len = sizeof(dnssl->domains) - + (ptr - dnssl->domains); - strncpy((char *)p, c->dns_search[i].n, - sizeof(buf) - - ((intptr_t)p - (intptr_t)buf)); - for (dot = (char *)p - 1; *dot; dot++) { + strncpy((char *)ptr, c->dns_search[i].n, len); + for (dot = (char *)ptr - 1; *dot; dot++) { if (*dot == '.') *dot = strcspn(dot + 1, "."); } - p += strlen(c->dns_search[i].n); - *(p++) = 0; + ptr += strlen(c->dns_search[i].n); + *(ptr++) = 0; } - memset(p, 0, 8 - dns_s_len % 8); /* padding */ - p += 8 - dns_s_len % 8; + memset(ptr, 0, 8 - dns_s_len % 8); /* padding */ + ptr += 8 - dns_s_len % 8; } + } + + memcpy(&ra.source_ll.mac, c->our_tap_mac, ETH_ALEN); -dns_done: - *p++ = 1; /* source ll */ - *p++ = 1; /* length */ - memcpy(p, c->mac, ETH_ALEN); - p += 6; - } else { + ndp_send(c, dst, &ra, ptr - (unsigned char *)&ra); +} + +/** + * ndp() - Check for NDP solicitations, reply as needed + * @c: Execution context + * @ih: ICMPv6 header + * @saddr: Source IPv6 address + * @p: Packet pool + * + * Return: 0 if not handled here, 1 if handled, -1 on failure + */ +int ndp(const struct ctx *c, const struct icmp6hdr *ih, + const struct in6_addr *saddr, const struct pool *p) +{ + if (ih->icmp6_type < RS || ih->icmp6_type > NA) + return 0; + + if (c->no_ndp) return 1; - } - len = (uintptr_t)p - (uintptr_t)ihr - sizeof(*ihr); + if (ih->icmp6_type == NS) { + const struct ndp_ns *ns; + + ns = packet_get(p, 0, 0, sizeof(struct ndp_ns), NULL); + if (!ns) + return -1; - if (IN6_IS_ADDR_LINKLOCAL(saddr)) - c->ip6.addr_ll_seen = *saddr; - else - c->ip6.addr_seen = *saddr; + if (IN6_IS_ADDR_UNSPECIFIED(saddr)) + return 1; - if (IN6_IS_ADDR_LINKLOCAL(&c->ip6.gw)) - rsaddr = &c->ip6.gw; - else - rsaddr = &c->ip6.addr_ll; + info("NDP: received NS, sending NA"); + + ndp_na(c, saddr, &ns->target_addr); + } else if (ih->icmp6_type == RS) { + if (c->no_ra) + return 1; - tap_icmp6_send(c, rsaddr, saddr, ihr, len + sizeof(*ihr)); + info("NDP: received RS, sending RA"); + ndp_ra(c, saddr); + } return 1; } + +/* Default interval between unsolicited RAs (seconds) */ +#define DEFAULT_MAX_RTR_ADV_INTERVAL 600 /* RFC 4861, 6.2.1 */ + +/* Minimum required interval between RAs (seconds) */ +#define MIN_DELAY_BETWEEN_RAS 3 /* RFC 4861, 10 */ + +static time_t next_ra; + +/** + * ndp_timer() - Send unsolicited NDP messages if necessary + * @c: Execution context + * @now: Current (monotonic) time + */ +void ndp_timer(const struct ctx *c, const struct timespec *now) +{ + time_t max_rtr_adv_interval = DEFAULT_MAX_RTR_ADV_INTERVAL; + time_t min_rtr_adv_interval, interval; + + if (c->no_ra || now->tv_sec < next_ra) + return; + + /* We must advertise before the route's lifetime expires */ + max_rtr_adv_interval = MIN(max_rtr_adv_interval, RT_LIFETIME - 1); + + /* But we must not go smaller than the minimum delay */ + max_rtr_adv_interval = MAX(max_rtr_adv_interval, MIN_DELAY_BETWEEN_RAS); + + /* RFC 4861, 6.2.1 */ + min_rtr_adv_interval = MAX(max_rtr_adv_interval / 3, + MIN_DELAY_BETWEEN_RAS); + + /* As required by RFC 4861, we randomise the interval between + * unsolicited RAs. This is to prevent multiple routers on a link + * getting synchronised (e.g. after booting a bunch of routers at once) + * and causing flurries of RAs at the same time. + * + * This random doesn't need to be cryptographically strong, so random(3) + * is fine. Other routers on the link also want to avoid + * synchronisation, and anything malicious has much easier ways to cause + * trouble. + * + * The modulus also makes this not strictly a uniform distribution, but, + * again, it's close enough for our purposes. + */ + interval = min_rtr_adv_interval + + random() % (max_rtr_adv_interval - min_rtr_adv_interval); + + info("NDP: sending unsolicited RA, next in %llds", (long long)interval); + + ndp_ra(c, &in6addr_ll_all_nodes); + + next_ra = now->tv_sec + interval; +} @@ -6,6 +6,10 @@ #ifndef NDP_H #define NDP_H -int ndp(struct ctx *c, const struct icmp6hdr *ih, const struct in6_addr *saddr); +struct icmp6hdr; + +int ndp(const struct ctx *c, const struct icmp6hdr *ih, + const struct in6_addr *saddr, const struct pool *p); +void ndp_timer(const struct ctx *c, const struct timespec *now); #endif /* NDP_H */ @@ -33,8 +33,13 @@ #include "util.h" #include "passt.h" #include "log.h" +#include "ip.h" #include "netlink.h" +/* Same as RTA_NEXT() but for nexthops: RTNH_NEXT() doesn't take 'attrlen' */ +#define RTNH_NEXT_AND_DEC(rtnh, attrlen) \ + ((attrlen) -= RTNH_ALIGN((rtnh)->rtnh_len), RTNH_NEXT(rtnh)) + /* Netlink expects a buffer of at least 8kiB or the system page size, * whichever is larger. 32kiB is recommended for more efficient. * Since the largest page size on any remotely common Linux setup is @@ -128,7 +133,7 @@ static uint32_t nl_send(int s, void *req, uint16_t type, n = send(s, req, len, 0); if (n < 0) - die("netlink: Failed to send(): %s", strerror(errno)); + die_perror("netlink: Failed to send()"); else if (n < len) die("netlink: Short send (%zd of %zd bytes)", n, len); @@ -184,7 +189,7 @@ static struct nlmsghdr *nl_next(int s, char *buf, struct nlmsghdr *nh, ssize_t * *n = recv(s, buf, NLBUFSIZ, 0); if (*n < 0) - die("netlink: Failed to recv(): %s", strerror(errno)); + die_perror("netlink: Failed to recv()"); nh = (struct nlmsghdr *)buf; if (!NLMSG_OK(nh, *n)) @@ -254,7 +259,8 @@ unsigned int nl_get_ext_if(int s, sa_family_t af) .rtm.rtm_type = RTN_UNICAST, .rtm.rtm_family = af, }; - unsigned int ifi = 0; + unsigned defifi = 0, anyifi = 0; + unsigned ndef = 0, nany = 0; struct nlmsghdr *nh; struct rtattr *rta; char buf[NLBUFSIZ]; @@ -262,30 +268,80 @@ unsigned int nl_get_ext_if(int s, sa_family_t af) uint32_t seq; size_t na; + /* Look for an interface with a default route first, failing that, look + * for any interface with a route, and pick the first one, if any. + */ seq = nl_send(s, &req, RTM_GETROUTE, NLM_F_DUMP, sizeof(req)); nl_foreach_oftype(nh, status, s, buf, seq, RTM_NEWROUTE) { struct rtmsg *rtm = (struct rtmsg *)NLMSG_DATA(nh); + const void *dst = NULL; + unsigned thisifi = 0; - if (ifi || rtm->rtm_dst_len || rtm->rtm_family != af) + if (rtm->rtm_family != af) continue; for (rta = RTM_RTA(rtm), na = RTM_PAYLOAD(nh); RTA_OK(rta, na); rta = RTA_NEXT(rta, na)) { if (rta->rta_type == RTA_OIF) { - ifi = *(unsigned int *)RTA_DATA(rta); + thisifi = *(unsigned int *)RTA_DATA(rta); } else if (rta->rta_type == RTA_MULTIPATH) { const struct rtnexthop *rtnh; rtnh = (struct rtnexthop *)RTA_DATA(rta); - ifi = rtnh->rtnh_ifindex; + thisifi = rtnh->rtnh_ifindex; + } else if (rta->rta_type == RTA_DST) { + dst = RTA_DATA(rta); } } + + if (!thisifi) + continue; /* No interface for this route */ + + /* Skip routes to link-local addresses */ + if (af == AF_INET && dst && + IN4_IS_PREFIX_LINKLOCAL(dst, rtm->rtm_dst_len)) + continue; + + if (af == AF_INET6 && dst && + IN6_IS_PREFIX_LINKLOCAL(dst, rtm->rtm_dst_len)) + continue; + + if (rtm->rtm_dst_len == 0) { + /* Default route */ + ndef++; + if (!defifi) + defifi = thisifi; + } else { + /* Non-default route */ + nany++; + if (!anyifi) + anyifi = thisifi; + } } if (status < 0) warn("netlink: RTM_GETROUTE failed: %s", strerror(-status)); - return ifi; + if (defifi) { + if (ndef > 1) { + info("Multiple default %s routes, picked first", + af_name(af)); + } + return defifi; + } + + if (anyifi) { + if (nany > 1) { + info("Multiple interfaces with %s routes, picked first", + af_name(af)); + } + return anyifi; + } + + if (!nany) + info("No interfaces with usable %s routes", af_name(af)); + + return 0; } /** @@ -297,12 +353,13 @@ unsigned int nl_get_ext_if(int s, sa_family_t af) */ bool nl_route_get_def_multipath(struct rtattr *rta, void *gw) { + int nh_len = RTA_PAYLOAD(rta); struct rtnexthop *rtnh; bool found = false; int hops = -1; for (rtnh = (struct rtnexthop *)RTA_DATA(rta); - RTNH_OK(rtnh, RTA_PAYLOAD(rta)); rtnh = RTNH_NEXT(rtnh)) { + RTNH_OK(rtnh, nh_len); rtnh = RTNH_NEXT_AND_DEC(rtnh, nh_len)) { size_t len = rtnh->rtnh_len - sizeof(*rtnh); struct rtattr *rta_inner; @@ -332,7 +389,7 @@ bool nl_route_get_def_multipath(struct rtattr *rta, void *gw) * @af: Address family * @gw: Default gateway to fill on NL_GET * - * Return: 0 on success, negative error code on failure + * Return: error on netlink failure, or 0 (gw unset if default route not found) */ int nl_route_get_def(int s, unsigned int ifi, sa_family_t af, void *gw) { @@ -479,7 +536,7 @@ int nl_route_dup(int s_src, unsigned int ifi_src, .rta.rta_len = RTA_LENGTH(sizeof(unsigned int)), .ifi = ifi_src, }; - ssize_t nlmsgs_size, status; + ssize_t nlmsgs_size, left, status; unsigned dup_routes = 0; struct nlmsghdr *nh; char buf[NLBUFSIZ]; @@ -493,39 +550,83 @@ int nl_route_dup(int s_src, unsigned int ifi_src, * routes in the buffer at once. */ nh = nl_next(s_src, buf, NULL, &nlmsgs_size); - for (status = nlmsgs_size; - NLMSG_OK(nh, status) && (status = nl_status(nh, status, seq)) > 0; - nh = NLMSG_NEXT(nh, status)) { + for (left = nlmsgs_size; + NLMSG_OK(nh, left) && (status = nl_status(nh, left, seq)) > 0; + nh = NLMSG_NEXT(nh, left)) { struct rtmsg *rtm = (struct rtmsg *)NLMSG_DATA(nh); + bool discard = false; struct rtattr *rta; size_t na; if (nh->nlmsg_type != RTM_NEWROUTE) continue; - dup_routes++; - for (rta = RTM_RTA(rtm), na = RTM_PAYLOAD(nh); RTA_OK(rta, na); rta = RTA_NEXT(rta, na)) { + /* RTA_OIF and RTA_MULTIPATH attributes carry the + * identifier of a host interface. If they match the + * host interface we're copying from, change them to + * match the corresponding identifier in the target + * namespace. + * + * If RTA_OIF doesn't match (NETLINK_GET_STRICT_CHK not + * available), or if any interface index in nexthop + * objects differ from the host interface, discard the + * route altogether. + */ if (rta->rta_type == RTA_OIF) { - /* The host obviously list's the host interface - * id here, we need to change it to the - * namespace's interface id - */ + if (*(unsigned int *)RTA_DATA(rta) != ifi_src) { + discard = true; + break; + } + *(unsigned int *)RTA_DATA(rta) = ifi_dst; - } else if (rta->rta_type == RTA_PREFSRC) { - /* Host routes might include a preferred source - * address, which must be one of the host's - * addresses. However, with -a pasta will use a - * different namespace address, making such a - * route invalid in the namespace. Strip off - * RTA_PREFSRC attributes to avoid that. */ + } else if (rta->rta_type == RTA_MULTIPATH) { + int nh_len = RTA_PAYLOAD(rta); + struct rtnexthop *rtnh; + + for (rtnh = (struct rtnexthop *)RTA_DATA(rta); + RTNH_OK(rtnh, nh_len); + rtnh = RTNH_NEXT_AND_DEC(rtnh, nh_len)) { + int src = (int)ifi_src; + + if (rtnh->rtnh_ifindex != src) { + discard = true; + break; + } + + rtnh->rtnh_ifindex = ifi_dst; + } + + if (discard) + break; + } else if (rta->rta_type == RTA_PREFSRC || + rta->rta_type == RTA_NH_ID) { + /* Strip RTA_PREFSRC attributes: host routes + * might include a preferred source address, + * which must be one of the host's addresses. + * However, with -a, pasta will use a different + * namespace address, making such a route + * invalid in the namespace. + * + * Strip RTA_NH_ID attributes: host routes set + * up via routing protocols (e.g. OSPF) might + * contain a nexthop ID (and not nexthop + * objects, which are taken care of in the + * RTA_MULTIPATH case above) that's not valid + * in the target namespace. + */ rta->rta_type = RTA_UNSPEC; } } + + if (discard) + nh->nlmsg_type = NLMSG_NOOP; + else + dup_routes++; } - if (!NLMSG_OK(nh, status) || status > 0) { + if (!NLMSG_OK(nh, left)) { /* Process any remaining datagrams in a different * buffer so we don't overwrite the first one. */ @@ -551,9 +652,9 @@ int nl_route_dup(int s_src, unsigned int ifi_src, * to calculate dependencies: let the kernel do that. */ for (i = 0; i < dup_routes; i++) { - for (nh = (struct nlmsghdr *)buf, status = nlmsgs_size; - NLMSG_OK(nh, status); - nh = NLMSG_NEXT(nh, status)) { + for (nh = (struct nlmsghdr *)buf, left = nlmsgs_size; + NLMSG_OK(nh, left); + nh = NLMSG_NEXT(nh, left)) { uint16_t flags = nh->nlmsg_flags; int rc; @@ -563,7 +664,8 @@ int nl_route_dup(int s_src, unsigned int ifi_src, rc = nl_do(s_dst, nh, RTM_NEWROUTE, (flags & ~NLM_F_DUMP_FILTERED) | NLM_F_CREATE, nh->nlmsg_len); - if (rc < 0 && rc != -ENETUNREACH && rc != -EEXIST) + if (rc < 0 && rc != -EEXIST && + rc != -ENETUNREACH && rc != -EHOSTUNREACH) return rc; } } @@ -572,6 +674,63 @@ int nl_route_dup(int s_src, unsigned int ifi_src, } /** + * nl_addr_set_ll_nodad() - Set IFA_F_NODAD on IPv6 link-local addresses + * @s: Netlink socket + * @ifi: Interface index in target namespace + * + * Return: 0 on success, negative error code on failure + */ +int nl_addr_set_ll_nodad(int s, unsigned int ifi) +{ + struct req_t { + struct nlmsghdr nlh; + struct ifaddrmsg ifa; + } req = { + .ifa.ifa_family = AF_INET6, + .ifa.ifa_index = ifi, + }; + uint32_t seq, last_seq = 0; + ssize_t status, ret = 0; + struct nlmsghdr *nh; + char buf[NLBUFSIZ]; + + seq = nl_send(s, &req, RTM_GETADDR, NLM_F_DUMP, sizeof(req)); + nl_foreach_oftype(nh, status, s, buf, seq, RTM_NEWADDR) { + struct ifaddrmsg *ifa = (struct ifaddrmsg *)NLMSG_DATA(nh); + struct rtattr *rta; + size_t na; + + if (ifa->ifa_index != ifi || ifa->ifa_scope != RT_SCOPE_LINK) + continue; + + ifa->ifa_flags |= IFA_F_NODAD; + + for (rta = IFA_RTA(ifa), na = IFA_PAYLOAD(nh); RTA_OK(rta, na); + rta = RTA_NEXT(rta, na)) { + /* If 32-bit flags are used, add IFA_F_NODAD there */ + if (rta->rta_type == IFA_FLAGS) + *(uint32_t *)RTA_DATA(rta) |= IFA_F_NODAD; + } + + last_seq = nl_send(s, nh, RTM_NEWADDR, NLM_F_REPLACE, + nh->nlmsg_len); + } + + if (status < 0) + ret = status; + + for (seq = seq + 1; seq <= last_seq; seq++) { + nl_foreach(nh, status, s, buf, seq) + warn("netlink: Unexpected response message"); + + if (!ret && status < 0) + ret = status; + } + + return ret; +} + +/** * nl_addr_get() - Get most specific global address, given interface and family * @s: Netlink socket * @ifi: Interface index in outer network namespace @@ -580,7 +739,7 @@ int nl_route_dup(int s_src, unsigned int ifi_src, * @prefix_len: Mask or prefix length, to fill (for IPv4) * @addr_l: Link-scoped address to fill (for IPv6) * - * Return: 9 on success, negative error code on failure + * Return: 0 on success, negative error code on failure */ int nl_addr_get(int s, unsigned int ifi, sa_family_t af, void *addr, int *prefix_len, void *addr_l) @@ -604,12 +763,13 @@ int nl_addr_get(int s, unsigned int ifi, sa_family_t af, struct rtattr *rta; size_t na; - if (ifa->ifa_index != ifi) + if (ifa->ifa_index != ifi || ifa->ifa_flags & IFA_F_DEPRECATED) continue; for (rta = IFA_RTA(ifa), na = IFA_PAYLOAD(nh); RTA_OK(rta, na); rta = RTA_NEXT(rta, na)) { - if (rta->rta_type != IFA_ADDRESS) + if ((af == AF_INET && rta->rta_type != IFA_LOCAL) || + (af == AF_INET6 && rta->rta_type != IFA_ADDRESS)) continue; if (af == AF_INET && ifa->ifa_prefixlen > prefix_max) { @@ -637,7 +797,54 @@ int nl_addr_get(int s, unsigned int ifi, sa_family_t af, } /** - * nl_add_set() - Set IP addresses for given interface and address family + * nl_addr_get_ll() - Get first IPv6 link-local address for a given interface + * @s: Netlink socket + * @ifi: Interface index in outer network namespace + * @addr: Link-local address to fill + * + * Return: 0 on success, negative error code on failure + */ +int nl_addr_get_ll(int s, unsigned int ifi, struct in6_addr *addr) +{ + struct req_t { + struct nlmsghdr nlh; + struct ifaddrmsg ifa; + } req = { + .ifa.ifa_family = AF_INET6, + .ifa.ifa_index = ifi, + }; + struct nlmsghdr *nh; + bool found = false; + char buf[NLBUFSIZ]; + ssize_t status; + uint32_t seq; + + seq = nl_send(s, &req, RTM_GETADDR, NLM_F_DUMP, sizeof(req)); + nl_foreach_oftype(nh, status, s, buf, seq, RTM_NEWADDR) { + struct ifaddrmsg *ifa = (struct ifaddrmsg *)NLMSG_DATA(nh); + struct rtattr *rta; + size_t na; + + if (ifa->ifa_index != ifi || ifa->ifa_scope != RT_SCOPE_LINK || + found) + continue; + + for (rta = IFA_RTA(ifa), na = IFA_PAYLOAD(nh); RTA_OK(rta, na); + rta = RTA_NEXT(rta, na)) { + if (rta->rta_type != IFA_ADDRESS) + continue; + + if (!found) { + memcpy(addr, RTA_DATA(rta), RTA_PAYLOAD(rta)); + found = true; + } + } + } + return status; +} + +/** + * nl_addr_set() - Set IP addresses for given interface and address family * @s: Netlink socket * @ifi: Interface index * @af: Address family @@ -740,10 +947,13 @@ int nl_addr_dup(int s_src, unsigned int ifi_src, ifa = (struct ifaddrmsg *)NLMSG_DATA(nh); if (rc < 0 || ifa->ifa_scope == RT_SCOPE_LINK || - ifa->ifa_index != ifi_src) + ifa->ifa_index != ifi_src || + ifa->ifa_flags & IFA_F_DEPRECATED) continue; ifa->ifa_index = ifi_dst; + /* Same as nl_addr_set(), but here it's more than a default */ + ifa->ifa_flags |= IFA_F_NODAD; for (rta = IFA_RTA(ifa), na = IFA_PAYLOAD(nh); RTA_OK(rta, na); rta = RTA_NEXT(rta, na)) { @@ -751,6 +961,10 @@ int nl_addr_dup(int s_src, unsigned int ifi_src, if (rta->rta_type == IFA_LABEL || rta->rta_type == IFA_CACHEINFO) rta->rta_type = IFA_UNSPEC; + + /* If 32-bit flags are used, add IFA_F_NODAD there */ + if (rta->rta_type == IFA_FLAGS) + *(uint32_t *)RTA_DATA(rta) |= IFA_F_NODAD; } rc = nl_do(s_dst, nh, RTM_NEWADDR, @@ -832,14 +1046,14 @@ int nl_link_set_mac(int s, unsigned int ifi, const void *mac) } /** - * nl_link_up() - Bring link up + * nl_link_set_mtu() - Set link MTU * @s: Netlink socket * @ifi: Interface index - * @mtu: If non-zero, set interface MTU + * @mtu: Interface MTU * * Return: 0 on success, negative error code on failure */ -int nl_link_up(int s, unsigned int ifi, int mtu) +int nl_link_set_mtu(int s, unsigned int ifi, int mtu) { struct req_t { struct nlmsghdr nlh; @@ -849,17 +1063,35 @@ int nl_link_up(int s, unsigned int ifi, int mtu) } req = { .ifm.ifi_family = AF_UNSPEC, .ifm.ifi_index = ifi, - .ifm.ifi_flags = IFF_UP, - .ifm.ifi_change = IFF_UP, .rta.rta_type = IFLA_MTU, .rta.rta_len = RTA_LENGTH(sizeof(unsigned int)), .mtu = mtu, }; - ssize_t len = sizeof(req); - if (!mtu) - /* Shorten request to drop MTU attribute */ - len = offsetof(struct req_t, rta); + return nl_do(s, &req, RTM_NEWLINK, 0, sizeof(req)); +} + +/** + * nl_link_set_flags() - Set link flags + * @s: Netlink socket + * @ifi: Interface index + * @set: Device flags to set + * @change: Mask of device flag changes + * + * Return: 0 on success, negative error code on failure + */ +int nl_link_set_flags(int s, unsigned int ifi, + unsigned int set, unsigned int change) +{ + struct req_t { + struct nlmsghdr nlh; + struct ifinfomsg ifm; + } req = { + .ifm.ifi_family = AF_UNSPEC, + .ifm.ifi_index = ifi, + .ifm.ifi_flags = set, + .ifm.ifi_change = change, + }; - return nl_do(s, &req, RTM_NEWLINK, 0, len); + return nl_do(s, &req, RTM_NEWLINK, 0, sizeof(req)); } @@ -19,10 +19,14 @@ int nl_addr_get(int s, unsigned int ifi, sa_family_t af, void *addr, int *prefix_len, void *addr_l); int nl_addr_set(int s, unsigned int ifi, sa_family_t af, const void *addr, int prefix_len); +int nl_addr_get_ll(int s, unsigned int ifi, struct in6_addr *addr); +int nl_addr_set_ll_nodad(int s, unsigned int ifi); int nl_addr_dup(int s_src, unsigned int ifi_src, int s_dst, unsigned int ifi_dst, sa_family_t af); int nl_link_get_mac(int s, unsigned int ifi, void *mac); int nl_link_set_mac(int s, unsigned int ifi, const void *mac); -int nl_link_up(int s, unsigned int ifi, int mtu); +int nl_link_set_mtu(int s, unsigned int ifi, int mtu); +int nl_link_set_flags(int s, unsigned int ifi, + unsigned int set, unsigned int change); #endif /* NETLINK_H */ @@ -22,39 +22,43 @@ #include "util.h" #include "log.h" +/** + * packet_check_range() - Check if a packet memory range is valid + * @p: Packet pool + * @offset: Offset of data range in packet descriptor + * @len: Length of desired data range + * @start: Start of the packet descriptor + * @func: For tracing: name of calling function + * @line: For tracing: caller line of function call + * + * Return: 0 if the range is valid, -1 otherwise + */ static int packet_check_range(const struct pool *p, size_t offset, size_t len, const char *start, const char *func, int line) { - ASSERT(p->buf); + if (p->buf_size == 0) { + int ret; - if (p->buf_size == 0) - return vu_packet_check_range((void *)p->buf, offset, len, start, - func, line); + ret = vu_packet_check_range((void *)p->buf, offset, len, start); - if (start < p->buf) { - if (func) { - trace("add packet start %p before buffer start %p, " - "%s:%i", (void *)start, (void *)p->buf, func, line); - } - return -1; + if (ret == -1) + trace("cannot find region, %s:%i", func, line); + + return ret; } - if (start + len + offset > p->buf + p->buf_size) { - if (func) { - trace("packet offset plus length %lu from size %lu, " - "%s:%i", start - p->buf + len + offset, - p->buf_size, func, line); - } + if (start < p->buf) { + trace("packet start %p before buffer start %p, " + "%s:%i", (void *)start, (void *)p->buf, func, line); return -1; } -#if UINTPTR_MAX == UINT64_MAX - if ((uintptr_t)start - (uintptr_t)p->buf > UINT32_MAX) { - trace("add packet start %p, buffer start %p, %s:%i", - (void *)start, (void *)p->buf, func, line); + if (start + len + offset > p->buf + p->buf_size) { + trace("packet offset plus length %lu from size %lu, " + "%s:%i", start - p->buf + len + offset, + p->buf_size, func, line); return -1; } -#endif return 0; } @@ -114,10 +118,10 @@ void *packet_get_do(const struct pool *p, size_t idx, size_t offset, return NULL; } - if (len > UINT16_MAX || len + offset > UINT32_MAX) { + if (len > UINT16_MAX) { if (func) { - trace("packet data length %zu, offset %zu, %s:%i", - len, offset, func, line); + trace("packet data length %zu, %s:%i", + len, func, line); } return NULL; } @@ -8,8 +8,10 @@ /** * struct pool - Generic pool of packets stored in a buffer - * @buf: Buffer storing packet descriptors - * @buf_size: Total size of buffer + * @buf: Buffer storing packet descriptors, + * a struct vu_dev_region array for passt vhost-user mode + * @buf_size: Total size of buffer, + * 0 for passt vhost-user mode * @size: Number of usable descriptors for the pool * @count: Number of used descriptors for the pool * @pkt: Descriptors: see macros below @@ -23,7 +25,7 @@ struct pool { }; int vu_packet_check_range(void *buf, size_t offset, size_t len, - const char *start, const char *func, int line); + const char *start); void packet_add_do(struct pool *p, size_t len, const char *start, const char *func, int line); void *packet_get_do(const struct pool *p, const size_t idx, @@ -73,6 +73,9 @@ for performance reasons. .SH OPTIONS +Unless otherwise noted below, \fBif conflicting or multiple options are given, +the last one takes effect.\fR + .TP .BR \-d ", " \-\-debug Be verbose, don't log to the system logger. @@ -92,14 +95,18 @@ detached PID namespace after starting, because the PID itself cannot change. Default is to fork into background. .TP -.BR \-e ", " \-\-stderr -Log to standard error too. -Default is to log to the system logger only, if started from an interactive -terminal, and to both system logger and standard error otherwise. +.BR \-e ", " \-\-stderr " " (DEPRECATED) +This option has no effect, and is maintained for compatibility purposes only. + +Note that this configuration option is \fBdeprecated\fR and will be removed in a +future version. .TP .BR \-l ", " \-\-log-file " " \fIPATH\fR -Log to file \fIPATH\fR, not to standard error, and not to the system logger. +Log to file \fIPATH\fR, and not to the system logger. + +Specifying this option multiple times does \fInot\fR lead to multiple log files: +the last given option takes effect. .TP .BR \-\-log-size " " \fISIZE\fR @@ -128,6 +135,9 @@ Show version and exit. Capture tap-facing (that is, guest-side or namespace-side) network packets to \fIfile\fR in \fBpcap\fR format. +Specifying this option multiple times does \fInot\fR lead to multiple capture +files: the last given option takes effect. + .TP .BR \-P ", " \-\-pid " " \fIfile Write own PID to \fIfile\fR once initialisation is done, before forking to @@ -148,7 +158,9 @@ for an IPv6 \fIaddr\fR. This option can be specified zero (for defaults) to two times (once for IPv4, once for IPv6). By default, assigned IPv4 and IPv6 addresses are taken from the host interfaces -with the first default route for the corresponding IP version. +with the first default route, if any, for the corresponding IP version. If no +default routes are available and there is any interface with any route for a +given IP version, the first of these interfaces will be chosen instead. .TP .BR \-n ", " \-\-netmask " " \fImask @@ -172,9 +184,11 @@ Assign IPv4 \fIaddr\fR as default gateway via DHCP (option 3), or IPv6 This option can be specified zero (for defaults) to two times (once for IPv4, once for IPv6). By default, IPv4 and IPv6 gateways are taken from the host interface with the -first default route for the corresponding IP version. If the default route is a -multipath one, the gateway is the first nexthop router returned by the kernel -which has the highest weight in the set of paths. +first default route, if any, for the corresponding IP version. If the default +route is a multipath one, the gateway is the first nexthop router returned by +the kernel which has the highest weight in the set of paths. If no default +routes are available and there is just one interface with any route, that +interface will be chosen instead. Note: these addresses are also used as source address for packets directed to the guest or to the target namespace having a loopback or local source address, @@ -185,9 +199,11 @@ to allow mapping of local traffic to guest and target namespace. See the .BR \-i ", " \-\-interface " " \fIname Use host interface \fIname\fR to derive addresses and routes. Default is to use the interfaces specified by \fB--outbound-if4\fR and -\fB--outbound-if6\fR, for IPv4 and IPv6 addresses and routes, respectively. If -no interfaces are given, the interface with the first default routes for each IP -version is selected. +\fB--outbound-if6\fR, for IPv4 and IPv6 addresses and routes, respectively. + +If no interfaces are given, the interface with the first default routes for each +IP version is selected. If no default routes are available and there is just one +interface with any route, that interface will be chosen instead. .TP .BR \-o ", " \-\-outbound " " \fIaddr @@ -203,30 +219,49 @@ By default, the source address is selected by the routing tables. Bind IPv4 outbound sockets to host interface \fIname\fR, and, unless another interface is specified via \fB-i\fR, \fB--interface\fR, use this interface to derive IPv4 addresses and routes. -By default, the interface given by the default route is selected. + +By default, the interface given by the default route is selected. If no default +routes are available and there is just one interface with any route, that +interface will be chosen instead. .TP .BR \-\-outbound-if6 " " \fIname Bind IPv6 outbound sockets to host interface \fIname\fR, and, unless another interface is specified via \fB-i\fR, \fB--interface\fR, use this interface to derive IPv6 addresses and routes. -By default, the interface given by the default route is selected. + +By default, the interface given by the default route is selected. If no default +routes are available and there is just one interface with any route, that +interface will be chosen instead. .TP .BR \-D ", " \-\-dns " " \fIaddr -Use \fIaddr\fR (IPv4 or IPv6) for DHCP, DHCPv6, NDP or DNS forwarding, as -configured (see options \fB--no-dhcp-dns\fR, \fB--dhcp-dns\fR, -\fB--dns-forward\fR) instead of reading addresses from \fI/etc/resolv.conf\fR. -This option can be specified multiple times. Specifying \fB-D none\fR disables -usage of DNS addresses altogether. +Instruct the guest (via DHCP, DHVPv6 or NDP) to use \fIaddr\fR (IPv4 +or IPv6) as a nameserver, as configured (see options +\fB--no-dhcp-dns\fR, \fB--dhcp-dns\fR) instead of reading addresses +from \fI/etc/resolv.conf\fR. This option can be specified multiple +times. Specifying \fB-D none\fR disables usage of DNS addresses +altogether. Unlike addresses from \fI/etc/resolv.conf\fR, \fIaddr\fR +is given to the guest without remapping. For example \fB--dns +127.0.0.1\fR will instruct the guest to use itself as nameserver, not +the host. .TP .BR \-\-dns-forward " " \fIaddr -Map \fIaddr\fR (IPv4 or IPv6) as seen from guest or namespace to the first -configured DNS resolver (with corresponding IP version). Mapping is limited to -UDP traffic directed to port 53, and DNS answers are translated back with a -reverse mapping. -This option can be specified zero to two times (once for IPv4, once for IPv6). +Map \fIaddr\fR (IPv4 or IPv6) as seen from guest or namespace to the +nameserver (with corresponding IP version) specified by the +\fB\-\-dns-host\fR option. Maps only UDP and TCP traffic to port 53 or +port 853. Replies are translated back with a reverse mapping. This +option can be specified zero to two times (once for IPv4, once for +IPv6). + +.TP +.BR \-\-dns-host " " \fIaddr +Configure the host nameserver which guest or namespace queries to the +\fB\-\-dns-forward\fR address will be redirected to. This option can +be specified zero to two times (once for IPv4, once for IPv6). +By default, the first nameserver from the host's +\fI/etc/resolv.conf\fR. .TP .BR \-S ", " \-\-search " " \fIlist @@ -237,28 +272,28 @@ list altogether (if you need to search a domain called "none" you can use \fB--search none.\fR). .TP -.BR \-\-no-dhcp-dns " " \fIaddr +.BR \-\-no-dhcp-dns In \fIpasst\fR mode, do not assign IPv4 addresses via DHCP (option 23) or IPv6 addresses via NDP Router Advertisement (option type 25) and DHCPv6 (option 23) as DNS resolvers. By default, all the configured addresses are passed. .TP -.BR \-\-dhcp-dns " " \fIaddr +.BR \-\-dhcp-dns In \fIpasta\fR mode, assign IPv4 addresses via DHCP (option 23) or IPv6 addresses via NDP Router Advertisement (option type 25) and DHCPv6 (option 23) as DNS resolvers. By default, configured addresses, if any, are not passed. .TP -.BR \-\-no-dhcp-search " " \fIaddr +.BR \-\-no-dhcp-search In \fIpasst\fR mode, do not send the DNS domain search list addresses via DHCP (option 119), via NDP Router Advertisement (option type 31) and DHCPv6 (option 24). By default, the DNS domain search list resulting from configuration is passed. .TP -.BR \-\-dhcp-search " " \fIaddr +.BR \-\-dhcp-search In \fIpasta\fR mode, send the DNS domain search list addresses via DHCP (option 119), via NDP Router Advertisement (option type 31) and DHCPv6 (option 24). By default, the DNS domain search list resulting from configuration is not @@ -302,33 +337,81 @@ Disable Router Advertisements. Router Solicitations coming from guest or target namespace will be ignored. .TP +.BR \-\-freebind +Allow any binding address to be specified for \fB-t\fR and \fB-u\fR +options. Usually binding addresses must be addresses currently +configured on the host. With \fB\-\-freebind\fR, the +\fBIP_FREEBIND\fR or \fBIPV6_FREEBIND\fR socket option is enabled +allowing any address to be used. This is typically used to bind +addresses which might be configured on the host in future, at which +point the forwarding will immediately start operating. + +.TP +.BR \-\-map-host-loopback " " \fIaddr +Translate \fIaddr\fR to refer to the host. Packets from the guest to +\fIaddr\fR will be redirected to the host. On the host such packets +will appear to have both source and destination of 127.0.0.1 or ::1. + +If \fIaddr\fR is 'none', no address is mapped (this implies +\fB--no-map-gw\fR). Only one IPv4 and one IPv6 address can be +translated, if the option is specified multiple times, the last one +takes effect. + +Default is to translate the guest's default gateway address, unless +\fB--no-map-gw\fR is given, in which case no address is mapped. + +.TP .BR \-\-no-map-gw Don't remap TCP connections and untracked UDP traffic, with the gateway address as destination, to the host. Implied if there is no gateway on the selected -default route for any of the enabled address families. +default route, or if there is no default route, for any of the enabled address +families. + +.TP +.BR \-\-map-guest-addr " " \fIaddr +Translate \fIaddr\fR in the guest to be equal to the guest's assigned +address on the host. That is, packets from the guest to \fIaddr\fR +will be redirected to the address assigned to the guest with \fB-a\fR, +or by default the host's global address. This allows the guest to +access services availble on the host's global address, even though its +own address shadows that of the host. + +If \fIaddr\fR is 'none', no address is mapped. Only one IPv4 and one +IPv6 address can be translated, and if the option is specified +multiple times, the last one for each address type takes effect. + +Default is no mapping. .TP .BR \-4 ", " \-\-ipv4-only Enable IPv4-only operation. IPv6 traffic will be ignored. -By default, IPv6 operation is enabled as long as at least an IPv6 default route -and an interface address are configured on a given host interface. +By default, IPv6 operation is enabled as long as at least an IPv6 route and an +interface address are configured on a given host interface. .TP .BR \-6 ", " \-\-ipv6-only Enable IPv6-only operation. IPv4 traffic will be ignored. -By default, IPv4 operation is enabled as long as at least an IPv4 default route -and an interface address are configured on a given host interface. +By default, IPv4 operation is enabled as long as at least an IPv4 route and an +interface address are configured on a given host interface. .SS \fBpasst\fR-only options .TP -.BR \-s ", " \-\-socket " " \fIpath +.BR \-s ", " \-\-socket-path ", " \-\-socket " " \fIpath Path for UNIX domain socket used by \fBqemu\fR(1) or \fBqrap\fR(1) to connect to \fBpasst\fR. Default is to probe a free socket, not accepting connections, starting from \fI/tmp/passt_1.socket\fR to \fI/tmp/passt_64.socket\fR. .TP +.BR \-\-vhost-user +Enable vhost-user. The vhost-user command socket is provided by \fB--socket\fR. + +.TP +.BR \-\-print-capabilities +Print back-end capabilities in JSON format, only meaningful for vhost-user mode. + +.TP .BR \-F ", " \-\-fd " " \fIFD Pass a pre-opened, connected socket to \fBpasst\fR. Usually the socket is opened in the parent process and \fBpasst\fR inherits it when run as a child. This @@ -531,6 +614,13 @@ Configure UDP port forwarding from target namespace to init namespace. Default is \fBauto\fR. .TP +.BR \-\-host-lo-to-ns-lo " " (DEPRECATED) +If specified, connections forwarded with \fB\-t\fR and \fB\-u\fR from +the host's loopback address will appear on the loopback address in the +guest as well. Without this option such forwarded packets will appear +to come from the guest's public address. + +.TP .BR \-\-userns " " \fIspec Target user namespace to join, as a path. If PID is given, without this option, the user namespace will be the one of the corresponding process. @@ -566,7 +656,7 @@ or sourced from the host, and bring up the tap interface. .BR \-\-no-copy-routes " " (DEPRECATED) With \-\-config-net, do not copy all the routes associated to the interface we derive addresses and routes from: set up only the default gateway. Implied by --g, \-\-gateway. +-g, \-\-gateway, for the corresponding IP version only. Default is to copy all the routing entries from the interface in the outer namespace to the target namespace, translating the output interface attribute to @@ -581,7 +671,7 @@ below. .BR \-\-no-copy-addrs " " (DEPRECATED) With \-\-config-net, do not copy all the addresses associated to the interface we derive addresses and routes from: set up a single one. Implied by \-a, -\-\-address. +\-\-address, for the corresponding IP version only. Default is to copy all the addresses, except for link-local ones, from the interface from the outer namespace to the target namespace. @@ -807,38 +897,41 @@ root@localhost's password: .SH NOTES -.SS Handling of traffic with local destination and source addresses - -Both \fBpasst\fR and \fBpasta\fR can bind on ports with a local address, -depending on the configuration. Local destination or source addresses need to be -changed before packets are delivered to the guest or target namespace: most -operating systems would drop packets received from non-loopback interfaces with -local addresses, and it would also be impossible for guest or target namespace -to route answers back. - -For convenience, and somewhat arbitrarily, the source address on these packets -is translated to the address of the default IPv4 or IPv6 gateway -- this is -known to be an existing, valid address on the same subnet. - -Loopback destination addresses are instead translated to the observed external -address of the guest or target namespace. For IPv6 packets, if usage of a -link-local address by guest or namespace has ever been observed, and the -original destination address is also a link-local address, the observed -link-local address is used. Otherwise, the observed global address is used. For -both IPv4 and IPv6, if no addresses have been seen yet, the configured addresses -will be used instead. +.SS Handling of traffic with loopback destination and source addresses + +Both \fBpasst\fR and \fBpasta\fR can bind on ports with a loopback +address (127.0.0.0/8 or ::1), depending on the configuration. Loopback +destination or source addresses need to be changed before packets are +delivered to the guest or target namespace: most operating systems +would drop packets received with loopback addresses on non-loopback +interfaces, and it would also be impossible for guest or target +namespace to route answers back. + +For convenience, the source address on these packets is translated to +the address specified by the \fB\-\-map-host-loopback\fR option (with +some exceptions in pasta mode, see next section below). If not +specified this defaults, somewhat arbitrarily, to the address of +default IPv4 or IPv6 gateway (if any) -- this is known to be an +existing, valid address on the same subnet. If \fB\-\-no-map-gw\fR or +\fB\-\-map-host-loopback none\fR are specified this translation is +disabled and packets with loopback addresses are simply dropped. + +Loopback destination addresses are translated to the observed external +address of the guest or target namespace. For IPv6, the observed +link-local address is used if the translated source address is +link-local, otherwise the observed global address is used. For both +IPv4 and IPv6, if no addresses have been seen yet, the configured +addresses will be used instead. For example, if \fBpasst\fR or \fBpasta\fR receive a connection from 127.0.0.1, with destination 127.0.0.10, and the default IPv4 gateway is 192.0.2.1, while the last observed source address from guest or namespace is 192.0.2.2, this will be translated to a connection from 192.0.2.1 to 192.0.2.2. -Similarly, for traffic coming from guest or namespace, packets with destination -address corresponding to the default gateway will have their destination address -translated to a loopback address, if and only if a packet, in the opposite -direction, with a loopback destination or source address, port-wise matching for -UDP, or connection-wise for TCP, has been recently forwarded to guest or -namespace. This behaviour can be disabled with \-\-no\-map\-gw. +Similarly, for traffic coming from guest or namespace, packets with +destination address corresponding to the \fB\-\-map-host-loopback\fR +address will have their destination address translated to a loopback +address. .SS Handling of local traffic in pasta @@ -854,8 +947,15 @@ and the new socket using the \fBsplice\fR(2) system call, and for UDP, a pair of \fBrecvmmsg\fR(2) and \fBsendmmsg\fR(2) system calls deals with packet transfers. -This bypass only applies to local connections and traffic, because it's not -possible to bind sockets to foreign addresses. +Because it's not possible to bind sockets to foreign addresses, this +bypass only applies to local connections and traffic. It also means +that the address translation differs slightly from passt mode. +Connections from loopback to loopback on the host will appear to come +from the target namespace's public address within the guest, unless +\fB\-\-host-lo-to-ns-lo\fR is specified, in which case they will +appear to come from loopback in the namespace as well. The latter +behaviour used to be the default, but is usually undesirable, since it +can unintentionally expose namespace local services to the host. .SS Binding to low numbered ports (well-known or system ports, up to 1023) @@ -964,8 +1064,8 @@ https://passt.top/passt/lists. Copyright (c) 2020-2022 Red Hat GmbH. \fBpasst\fR and \fBpasta\fR are free software: you can redistribute them and/or -modify them under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the License, or +modify them under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 2 of the License, or (at your option) any later version. .SH SEE ALSO @@ -35,9 +35,7 @@ #include <syslog.h> #include <sys/prctl.h> #include <netinet/if_ether.h> -#ifdef HAS_GETRANDOM -#include <sys/random.h> -#endif +#include <libgen.h> #include "util.h" #include "passt.h" @@ -51,6 +49,8 @@ #include "arch.h" #include "log.h" #include "tcp_splice.h" +#include "ndp.h" +#include "vu_common.h" #define EPOLL_EVENTS 8 @@ -65,9 +65,9 @@ char *epoll_type_str[] = { [EPOLL_TYPE_TCP_SPLICE] = "connected spliced TCP socket", [EPOLL_TYPE_TCP_LISTEN] = "listening TCP socket", [EPOLL_TYPE_TCP_TIMER] = "TCP timer", - [EPOLL_TYPE_UDP] = "UDP socket", - [EPOLL_TYPE_ICMP] = "ICMP socket", - [EPOLL_TYPE_ICMPV6] = "ICMPv6 socket", + [EPOLL_TYPE_UDP_LISTEN] = "listening UDP socket", + [EPOLL_TYPE_UDP_REPLY] = "UDP reply socket", + [EPOLL_TYPE_PING] = "ICMP/ICMPv6 ping socket", [EPOLL_TYPE_NSQUIT_INOTIFY] = "namespace inotify watch", [EPOLL_TYPE_NSQUIT_TIMER] = "namespace timer watch", [EPOLL_TYPE_TAP_PASTA] = "/dev/net/tun device", @@ -86,7 +86,7 @@ static_assert(ARRAY_SIZE(epoll_type_str) == EPOLL_NUM_TYPES, */ static void post_handler(struct ctx *c, const struct timespec *now) { -#define CALL_PROTO_HANDLER(c, now, lc, uc) \ +#define CALL_PROTO_HANDLER(lc, uc) \ do { \ extern void \ lc ## _defer_handler (struct ctx *c) \ @@ -105,49 +105,30 @@ static void post_handler(struct ctx *c, const struct timespec *now) } while (0) /* NOLINTNEXTLINE(bugprone-branch-clone): intervals can be the same */ - CALL_PROTO_HANDLER(c, now, tcp, TCP); + CALL_PROTO_HANDLER(tcp, TCP); /* NOLINTNEXTLINE(bugprone-branch-clone): intervals can be the same */ - CALL_PROTO_HANDLER(c, now, udp, UDP); - /* NOLINTNEXTLINE(bugprone-branch-clone): intervals can be the same */ - CALL_PROTO_HANDLER(c, now, icmp, ICMP); + CALL_PROTO_HANDLER(udp, UDP); flow_defer_handler(c, now); #undef CALL_PROTO_HANDLER + + ndp_timer(c, now); } /** - * secret_init() - Create secret value for SipHash calculations + * random_init() - Initialise things based on random data * @c: Execution context */ -static void secret_init(struct ctx *c) +static void random_init(struct ctx *c) { -#ifndef HAS_GETRANDOM - int dev_random = open("/dev/random", O_RDONLY); - unsigned int random_read = 0; - - while (dev_random && random_read < sizeof(c->hash_secret)) { - int ret = read(dev_random, - (uint8_t *)&c->hash_secret + random_read, - sizeof(c->hash_secret) - random_read); + unsigned int seed; - if (ret == -1 && errno == EINTR) - continue; - - if (ret <= 0) - break; + /* Create secret value for SipHash calculations */ + raw_random(&c->hash_secret, sizeof(c->hash_secret)); - random_read += ret; - } - if (dev_random >= 0) - close(dev_random); - if (random_read < sizeof(c->hash_secret)) { -#else - if (getrandom(&c->hash_secret, sizeof(c->hash_secret), - GRND_RANDOM) < 0) { -#endif /* !HAS_GETRANDOM */ - perror("TCP initial sequence getrandom"); - exit(EXIT_FAILURE); - } + /* Seed pseudo-RNG for things that need non-cryptographic random */ + raw_random(&seed, sizeof(seed)); + srandom(seed); } /** @@ -167,7 +148,7 @@ static void timer_init(struct ctx *c, const struct timespec *now) */ void proto_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s) { - tcp_buf_update_l2(eth_d, eth_s); + tcp_update_l2_buf(eth_d, eth_s); udp_update_l2_buf(eth_d, eth_s); } @@ -195,28 +176,30 @@ void exit_handler(int signal) * Return: non-zero on failure * * #syscalls read write writev - * #syscalls socket bind connect getsockopt setsockopt s390x:socketcall close - * #syscalls recvfrom sendto shutdown - * #syscalls armv6l:recv armv7l:recv ppc64le:recv - * #syscalls armv6l:send armv7l:send ppc64le:send + * #syscalls socket getsockopt setsockopt s390x:socketcall i686:socketcall close + * #syscalls bind connect recvfrom sendto shutdown + * #syscalls arm:recv ppc64le:recv arm:send ppc64le:send * #syscalls accept4|accept listen epoll_ctl epoll_wait|epoll_pwait epoll_pwait - * #syscalls clock_gettime armv6l:clock_gettime64 armv7l:clock_gettime64 + * #syscalls clock_gettime arm:clock_gettime64 i686:clock_gettime64 */ int main(int argc, char **argv) { - int nfds, i, devnull_fd = -1, pidfile_fd = -1; struct epoll_event events[EPOLL_EVENTS]; - char *log_name, argv0[PATH_MAX], *name; + int nfds, i, devnull_fd = -1; + char argv0[PATH_MAX], *name; struct ctx c = { 0 }; struct rlimit limit; struct timespec now; struct sigaction sa; + if (clock_gettime(CLOCK_MONOTONIC, &log_start)) + die_perror("Failed to get CLOCK_MONOTONIC time"); + arch_avx2_exec(argv); - isolate_initial(); + isolate_initial(argc, argv); - c.pasta_netns_fd = c.fd_tap = c.fd_tap_listen = -1; + c.pasta_netns_fd = c.fd_tap = c.pidfile_fd = -1; sigemptyset(&sa.sa_mask); sa.sa_flags = 0; @@ -231,70 +214,52 @@ int main(int argc, char **argv) name = basename(argv0); if (strstr(name, "pasta")) { sa.sa_handler = pasta_child_handler; - if (sigaction(SIGCHLD, &sa, NULL)) { - die("Couldn't install signal handlers: %s", - strerror(errno)); - } + if (sigaction(SIGCHLD, &sa, NULL)) + die_perror("Couldn't install signal handlers"); - if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) { - die("Couldn't set disposition for SIGPIPE: %s", - strerror(errno)); - } + if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) + die_perror("Couldn't set disposition for SIGPIPE"); c.mode = MODE_PASTA; - log_name = "pasta"; } else if (strstr(name, "passt")) { c.mode = MODE_PASST; - log_name = "passt"; } else { exit(EXIT_FAILURE); } madvise(pkt_buf, TAP_BUF_BYTES, MADV_HUGEPAGE); - __openlog(log_name, 0, LOG_DAEMON); - c.epollfd = epoll_create1(EPOLL_CLOEXEC); - if (c.epollfd == -1) { - perror("epoll_create1"); - exit(EXIT_FAILURE); - } + if (c.epollfd == -1) + die_perror("Failed to create epoll file descriptor"); + + if (getrlimit(RLIMIT_NOFILE, &limit)) + die_perror("Failed to get maximum value of open files limit"); - if (getrlimit(RLIMIT_NOFILE, &limit)) { - perror("getrlimit"); - exit(EXIT_FAILURE); - } c.nofile = limit.rlim_cur = limit.rlim_max; - if (setrlimit(RLIMIT_NOFILE, &limit)) { - perror("setrlimit"); - exit(EXIT_FAILURE); - } + if (setrlimit(RLIMIT_NOFILE, &limit)) + die_perror("Failed to set current limit for open files"); + sock_probe_mem(&c); conf(&c, argc, argv); trace_init(c.trace); - if (c.force_stderr || isatty(fileno(stdout))) - __openlog(log_name, LOG_PERROR, LOG_DAEMON); - pasta_netns_quit_init(&c); - tap_sock_init(&c); - vu_init(&c); + tap_backend_init(&c); - secret_init(&c); + random_init(&c); - clock_gettime(CLOCK_MONOTONIC, &now); + if (clock_gettime(CLOCK_MONOTONIC, &now)) + die_perror("Failed to get CLOCK_MONOTONIC time"); flow_init(); if ((!c.no_udp && udp_init(&c)) || (!c.no_tcp && tcp_init(&c))) exit(EXIT_FAILURE); - if (!c.no_icmp) - icmp_init(); - - proto_update_l2_buf(c.mac_guest, c.mac); + proto_update_l2_buf(c.guest_mac, c.our_tap_mac); if (c.ifi4 && !c.no_dhcp) dhcp_init(); @@ -305,46 +270,39 @@ int main(int argc, char **argv) pcap_init(&c); if (!c.foreground) { - if ((devnull_fd = open("/dev/null", O_RDWR | O_CLOEXEC)) < 0) { - perror("/dev/null open"); - exit(EXIT_FAILURE); - } - } - - if (*c.pid_file) { - if ((pidfile_fd = open(c.pid_file, - O_CREAT | O_TRUNC | O_WRONLY | O_CLOEXEC, - S_IRUSR | S_IWUSR)) < 0) { - perror("PID file open"); - exit(EXIT_FAILURE); - } + if ((devnull_fd = open("/dev/null", O_RDWR | O_CLOEXEC)) < 0) + die_perror("Failed to open /dev/null"); } if (isolate_prefork(&c)) die("Failed to sandbox process, exiting"); - if (!c.foreground) - __daemon(pidfile_fd, devnull_fd); - else - write_pidfile(pidfile_fd, getpid()); + if (!c.foreground) { + __daemon(c.pidfile_fd, devnull_fd); + log_stderr = false; + } else { + pidfile_write(c.pidfile_fd, getpid()); + } - if (pasta_child_pid) + if (pasta_child_pid) { kill(pasta_child_pid, SIGUSR1); + log_stderr = false; + } isolate_postfork(&c); timer_init(&c, &now); loop: - /* NOLINTNEXTLINE(bugprone-branch-clone): intervals can be the same */ + /* NOLINTBEGIN(bugprone-branch-clone): intervals can be the same */ /* cppcheck-suppress [duplicateValueTernary, unmatchedSuppression] */ nfds = epoll_wait(c.epollfd, events, EPOLL_EVENTS, TIMER_INTERVAL); - if (nfds == -1 && errno != EINTR) { - perror("epoll_wait"); - exit(EXIT_FAILURE); - } + /* NOLINTEND(bugprone-branch-clone) */ + if (nfds == -1 && errno != EINTR) + die_perror("epoll_wait() failed in main loop"); - clock_gettime(CLOCK_MONOTONIC, &now); + if (clock_gettime(CLOCK_MONOTONIC, &now)) + err_perror("Failed to get CLOCK_MONOTONIC time"); for (i = 0; i < nfds; i++) { union epoll_ref ref = *((union epoll_ref *)&events[i].data.u64); @@ -382,23 +340,20 @@ loop: case EPOLL_TYPE_TCP_TIMER: tcp_timer_handler(&c, ref); break; - case EPOLL_TYPE_UDP: - if (c.mode == MODE_VU) - udp_vu_sock_handler(&c, ref, eventmask, &now); - else - udp_buf_sock_handler(&c, ref, eventmask, &now); + case EPOLL_TYPE_UDP_LISTEN: + udp_listen_sock_handler(&c, ref, eventmask, &now); break; - case EPOLL_TYPE_ICMP: - icmp_sock_handler(&c, AF_INET, ref); + case EPOLL_TYPE_UDP_REPLY: + udp_reply_sock_handler(&c, ref, eventmask, &now); break; - case EPOLL_TYPE_ICMPV6: - icmp_sock_handler(&c, AF_INET6, ref); + case EPOLL_TYPE_PING: + icmp_sock_handler(&c, ref); break; case EPOLL_TYPE_VHOST_CMD: - tap_handler_vu(&c, eventmask); + vu_control_handler(c.vdev, c.fd_tap, eventmask); break; case EPOLL_TYPE_VHOST_KICK: - vu_kick_cb(&c, ref); + vu_kick_cb(c.vdev, ref, &now); break; default: /* Can't happen */ @@ -9,26 +9,6 @@ #define UNIX_SOCK_MAX 100 #define UNIX_SOCK_PATH "/tmp/passt_%i.socket" -/** - * struct tap_msg - Generic message descriptor for arrays of messages - * @pkt_buf_offset: Offset from @pkt_buf - * @len: Message length, with L2 headers - */ -struct tap_msg { - uint32_t pkt_buf_offset; - uint16_t len; -}; - -/** - * struct tap_l4_msg - Layer-4 message descriptor for protocol handlers - * @pkt_buf_offset: Offset of message from @pkt_buf - * @l4_len: Length of Layer-4 payload, host order - */ -struct tap_l4_msg { - uint32_t pkt_buf_offset; - uint16_t l4_len; -}; - union epoll_ref; #include <stdbool.h> @@ -37,51 +17,22 @@ union epoll_ref; #include "pif.h" #include "packet.h" +#include "siphash.h" +#include "ip.h" +#include "inany.h" #include "flow.h" #include "icmp.h" #include "fwd.h" #include "tcp.h" #include "udp.h" -#include "udp_vu.h" #include "vhost_user.h" -/** - * enum epoll_type - Different types of fds we poll over +/* Default address for our end on the tap interface. Bit 0 of byte 0 must be 0 + * (unicast) and bit 1 of byte 1 must be 1 (locally administered). Otherwise + * it's arbitrary. */ -enum epoll_type { - /* Special value to indicate an invalid type */ - EPOLL_TYPE_NONE = 0, - /* Connected TCP sockets */ - EPOLL_TYPE_TCP, - /* Connected TCP sockets (spliced) */ - EPOLL_TYPE_TCP_SPLICE, - /* Listening TCP sockets */ - EPOLL_TYPE_TCP_LISTEN, - /* timerfds used for TCP timers */ - EPOLL_TYPE_TCP_TIMER, - /* UDP sockets */ - EPOLL_TYPE_UDP, - /* IPv4 ICMP sockets */ - EPOLL_TYPE_ICMP, - /* ICMPv6 sockets */ - EPOLL_TYPE_ICMPV6, - /* inotify fd watching for end of netns (pasta) */ - EPOLL_TYPE_NSQUIT_INOTIFY, - /* timer fd watching for end of netns, fallback for inotify (pasta) */ - EPOLL_TYPE_NSQUIT_TIMER, - /* tuntap character device */ - EPOLL_TYPE_TAP_PASTA, - /* socket connected to qemu */ - EPOLL_TYPE_TAP_PASST, - /* socket listening for qemu socket connections */ - EPOLL_TYPE_TAP_LISTEN, - /* vhost-user command socket */ - EPOLL_TYPE_VHOST_CMD, - /* vhost-user kick event socket */ - EPOLL_TYPE_VHOST_KICK, - - EPOLL_NUM_TYPES, -}; +#define MAC_OUR_LAA \ + ((uint8_t [ETH_ALEN]){0x9a, 0x55, 0x9a, 0x55, 0x9a, 0x55}) /** * union epoll_ref - Breakdown of reference for epoll fd bookkeeping @@ -93,6 +44,7 @@ enum epoll_type { * @icmp: ICMP-specific reference part * @data: Data handled by protocol handlers * @nsdir_fd: netns dirfd for fallback timer checking if namespace is gone + * @queue: vhost-user queue index for this fd * @u64: Opaque reference for epoll_ctl() and epoll_wait() */ union epoll_ref { @@ -105,10 +57,10 @@ union epoll_ref { uint32_t flow; flow_sidx_t flowside; union tcp_listen_epoll_ref tcp_listen; - union udp_epoll_ref udp; - union icmp_epoll_ref icmp; + union udp_listen_epoll_ref udp; uint32_t data; int nsdir_fd; + int queue; }; }; uint64_t u64; @@ -118,7 +70,6 @@ static_assert(sizeof(union epoll_ref) <= sizeof(union epoll_data), #define TAP_BUF_BYTES \ ROUND_DOWN(((ETH_MAX_MTU + sizeof(uint32_t)) * 128), PAGE_SIZE) -#define TAP_BUF_FILL (TAP_BUF_BYTES - ETH_MAX_MTU - sizeof(uint32_t)) #define TAP_MSGS \ DIV_ROUND_UP(TAP_BUF_BYTES, ETH_ZLEN - 2 * ETH_ALEN + sizeof(uint32_t)) @@ -151,54 +102,84 @@ enum passt_modes { /** * struct ip4_ctx - IPv4 execution context - * @addr: IPv4 address for external, routable interface + * @addr: IPv4 address assigned to guest * @addr_seen: Latest IPv4 address seen as source from tap * @prefixlen: IPv4 prefix length (netmask) - * @gw: Default IPv4 gateway, network order - * @dns: DNS addresses for DHCP, zero-terminated, network order - * @dns_match: Forward DNS query if sent to this address, network order - * @dns_host: Use this DNS on the host for forwarding, network order + * @guest_gw: IPv4 gateway as seen by the guest + * @map_host_loopback: Outbound connections to this address are NATted to the + * host's 127.0.0.1 + * @map_guest_addr: Outbound connections to this address are NATted to the + * guest's assigned address + * @dns: DNS addresses for DHCP, zero-terminated + * @dns_match: Forward DNS query if sent to this address + * @our_tap_addr: IPv4 address for passt's use on tap + * @dns_host: Use this DNS on the host for forwarding * @addr_out: Optional source address for outbound traffic * @ifname_out: Optional interface name to bind outbound sockets to + * @no_copy_routes: Don't copy all routes when configuring target namespace + * @no_copy_addrs: Don't copy all addresses when configuring namespace */ struct ip4_ctx { + /* PIF_TAP addresses */ struct in_addr addr; struct in_addr addr_seen; int prefix_len; - struct in_addr gw; + struct in_addr guest_gw; + struct in_addr map_host_loopback; + struct in_addr map_guest_addr; struct in_addr dns[MAXNS + 1]; struct in_addr dns_match; - struct in_addr dns_host; + struct in_addr our_tap_addr; + /* PIF_HOST addresses */ + struct in_addr dns_host; struct in_addr addr_out; + char ifname_out[IFNAMSIZ]; + + bool no_copy_routes; + bool no_copy_addrs; }; /** * struct ip6_ctx - IPv6 execution context - * @addr: IPv6 address for external, routable interface - * @addr_ll: Link-local IPv6 address on external, routable interface + * @addr: IPv6 address assigned to guest * @addr_seen: Latest IPv6 global/site address seen as source from tap * @addr_ll_seen: Latest IPv6 link-local address seen as source from tap - * @gw: Default IPv6 gateway + * @guest_gw: IPv6 gateway as seen by the guest + * @map_host_loopback: Outbound connections to this address are NATted to the + * host's [::1] + * @map_guest_addr: Outbound connections to this address are NATted to the + * guest's assigned address * @dns: DNS addresses for DHCPv6 and NDP, zero-terminated * @dns_match: Forward DNS query if sent to this address + * @our_tap_ll: Link-local IPv6 address for passt's use on tap * @dns_host: Use this DNS on the host for forwarding * @addr_out: Optional source address for outbound traffic * @ifname_out: Optional interface name to bind outbound sockets to + * @no_copy_routes: Don't copy all routes when configuring target namespace + * @no_copy_addrs: Don't copy all addresses when configuring namespace */ struct ip6_ctx { + /* PIF_TAP addresses */ struct in6_addr addr; - struct in6_addr addr_ll; struct in6_addr addr_seen; struct in6_addr addr_ll_seen; - struct in6_addr gw; + struct in6_addr guest_gw; + struct in6_addr map_host_loopback; + struct in6_addr map_guest_addr; struct in6_addr dns[MAXNS + 1]; struct in6_addr dns_match; - struct in6_addr dns_host; + struct in6_addr our_tap_ll; + /* PIF_HOST addresses */ + struct in6_addr dns_host; struct in6_addr addr_out; + char ifname_out[IFNAMSIZ]; + + bool no_copy_routes; + bool no_copy_addrs; }; #include <netinet/if_ether.h> @@ -210,11 +191,11 @@ struct ip6_ctx { * @trace: Enable tracing (extra debug) mode * @quiet: Don't print informational messages * @foreground: Run in foreground, don't log to stderr by default - * @force_stderr: Force logging to stderr * @nofile: Maximum number of open files (ulimit -n) * @sock_path: Path for UNIX domain socket * @pcap: Path for packet capture file - * @pid_file: Path to PID file, empty string if not configured + * @pidfile: Path to PID file, empty string if not configured + * @pidfile_fd: File descriptor for PID file, -1 if none * @pasta_netns_fd: File descriptor for network namespace in pasta mode * @no_netns_quit: In pasta mode, don't exit if fs-bound namespace is gone * @netns_base: Base name for fs-bound namespace, if any, in pasta mode @@ -222,8 +203,8 @@ struct ip6_ctx { * @epollfd: File descriptor for epoll instance * @fd_tap_listen: File descriptor for listening AF_UNIX socket, if any * @fd_tap: AF_UNIX socket, tuntap device, or pre-opened socket - * @mac: Host MAC address - * @mac_guest: MAC address of guest or namespace, seen or configured + * @our_tap_mac: Pasta/passt's MAC on the tap link + * @guest_mac: MAC address of guest or namespace, seen or configured * @hash_secret: 128-bit secret for siphash functions * @ifi4: Index of template interface for IPv4, 0 if IPv4 disabled * @ip: IPv4 configuration @@ -233,8 +214,6 @@ struct ip6_ctx { * @pasta_ifn: Name of namespace interface for pasta * @pasta_ifi: Index of namespace interface for pasta * @pasta_conf_ns: Configure namespace after creating it - * @no_copy_routes: Don't copy all routes when configuring target namespace - * @no_copy_addrs: Don't copy all addresses when configuring namespace * @no_tcp: Disable TCP operation * @tcp: Context for TCP protocol handler * @no_tcp: Disable UDP operation @@ -250,9 +229,11 @@ struct ip6_ctx { * @no_dhcpv6: Disable DHCPv6 server * @no_ndp: Disable NDP handler altogether * @no_ra: Disable router advertisements - * @no_map_gw: Don't map connections, untracked UDP to gateway to host + * @host_lo_to_ns_lo: Map host loopback addresses to ns loopback addresses + * @freebind: Allow binding of non-local addresses for forwarding * @low_wmem: Low probed net.core.wmem_max * @low_rmem: Low probed net.core.rmem_max + * @vdev: vhost-user device */ struct ctx { enum passt_modes mode; @@ -260,11 +241,13 @@ struct ctx { int trace; int quiet; int foreground; - int force_stderr; int nofile; char sock_path[UNIX_PATH_MAX]; char pcap[PATH_MAX]; - char pid_file[PATH_MAX]; + + char pidfile[PATH_MAX]; + int pidfile_fd; + int one_off; int pasta_netns_fd; @@ -276,8 +259,8 @@ struct ctx { int epollfd; int fd_tap_listen; int fd_tap; - unsigned char mac[ETH_ALEN]; - unsigned char mac_guest[ETH_ALEN]; + unsigned char our_tap_mac[ETH_ALEN]; + unsigned char guest_mac[ETH_ALEN]; uint64_t hash_secret[2]; unsigned int ifi4; @@ -291,8 +274,6 @@ struct ctx { char pasta_ifn[IF_NAMESIZE]; unsigned int pasta_ifi; int pasta_conf_ns; - int no_copy_routes; - int no_copy_addrs; int no_tcp; struct tcp_ctx tcp; @@ -310,13 +291,13 @@ struct ctx { int no_dhcpv6; int no_ndp; int no_ra; - int no_map_gw; + int host_lo_to_ns_lo; + int freebind; int low_wmem; int low_rmem; - /* vhost-user */ - struct VuDev vdev; + struct vu_dev *vdev; }; void proto_update_l2_buf(const unsigned char *eth_d, @@ -12,8 +12,8 @@ * Author: Stefano Brivio <sbrivio@redhat.com> * * #syscalls:pasta clone waitid exit exit_group rt_sigprocmask - * #syscalls:pasta rt_sigreturn|sigreturn armv6l:sigreturn armv7l:sigreturn - * #syscalls:pasta ppc64:sigreturn s390x:sigreturn + * #syscalls:pasta rt_sigreturn|sigreturn + * #syscalls:pasta arm:sigreturn ppc64:sigreturn s390x:sigreturn i686:sigreturn */ #include <sched.h> @@ -50,6 +50,8 @@ #include "netlink.h" #include "log.h" +#define HOSTNAME_PREFIX "pasta-" + /* PID of child, in case we created a namespace */ int pasta_child_pid; @@ -59,6 +61,7 @@ int pasta_child_pid; */ void pasta_child_handler(int signal) { + int errno_save = errno; siginfo_t infop; (void)signal; @@ -83,6 +86,8 @@ void pasta_child_handler(int signal) waitid(P_ALL, 0, NULL, WEXITED | WNOHANG); waitid(P_ALL, 0, NULL, WEXITED | WNOHANG); + + errno = errno_save; } /** @@ -97,7 +102,9 @@ static int pasta_wait_for_ns(void *arg) int flags = O_RDONLY | O_CLOEXEC; char ns[PATH_MAX]; - snprintf(ns, PATH_MAX, "/proc/%i/ns/net", pasta_child_pid); + if (snprintf_check(ns, PATH_MAX, "/proc/%i/ns/net", pasta_child_pid)) + die_perror("Can't build netns path"); + do { while ((c->pasta_netns_fd = open(ns, flags)) < 0) { if (errno != ENOENT) @@ -138,17 +145,15 @@ void pasta_open_ns(struct ctx *c, const char *netns) int nfd = -1; nfd = open(netns, O_RDONLY | O_CLOEXEC); - if (nfd < 0) { - die("Couldn't open network namespace %s: %s", - netns, strerror(errno)); - } + if (nfd < 0) + die_perror("Couldn't open network namespace %s", netns); c->pasta_netns_fd = nfd; NS_CALL(ns_check, c); if (c->pasta_netns_fd < 0) - die("Couldn't switch to pasta namespaces: %s", strerror(errno)); + die_perror("Couldn't switch to pasta namespaces"); if (!c->no_netns_quit) { char buf[PATH_MAX] = { 0 }; @@ -176,18 +181,28 @@ struct pasta_spawn_cmd_arg { * * Return: this function never returns */ +/* cppcheck-suppress [constParameterCallback, unmatchedSuppression] */ static int pasta_spawn_cmd(void *arg) { + char hostname[HOST_NAME_MAX + 1] = HOSTNAME_PREFIX; const struct pasta_spawn_cmd_arg *a; sigset_t set; /* We run in a detached PID and mount namespace: mount /proc over */ if (mount("", "/proc", "proc", 0, NULL)) - warn("Couldn't mount /proc: %s", strerror(errno)); + warn_perror("Couldn't mount /proc"); if (write_file("/proc/sys/net/ipv4/ping_group_range", "0 0")) warn("Cannot set ping_group_range, ICMP requests might fail"); + if (!gethostname(hostname + sizeof(HOSTNAME_PREFIX) - 1, + HOST_NAME_MAX + 1 - sizeof(HOSTNAME_PREFIX)) || + errno == ENAMETOOLONG) { + hostname[HOST_NAME_MAX] = '\0'; + if (sethostname(hostname, strlen(hostname))) + warn("Unable to set pasta-prefixed hostname"); + } + /* Wait for the parent to be ready: see main() */ sigemptyset(&set); sigaddset(&set, SIGUSR1); @@ -196,8 +211,7 @@ static int pasta_spawn_cmd(void *arg) a = (const struct pasta_spawn_cmd_arg *)arg; execvp(a->exe, a->argv); - perror("execvp"); - exit(EXIT_FAILURE); + die_perror("Failed to start command or shell"); } /** @@ -211,12 +225,13 @@ static int pasta_spawn_cmd(void *arg) void pasta_start_ns(struct ctx *c, uid_t uid, gid_t gid, int argc, char *argv[]) { + char ns_fn_stack[NS_FN_STACK_SIZE] + __attribute__ ((aligned(__alignof__(max_align_t)))); struct pasta_spawn_cmd_arg arg = { .exe = argv[0], .argv = argv, }; char uidmap[BUFSIZ], gidmap[BUFSIZ]; - char ns_fn_stack[NS_FN_STACK_SIZE]; char *sh_argv[] = { NULL, NULL }; char sh_arg0[PATH_MAX + 1]; sigset_t set; @@ -226,8 +241,11 @@ void pasta_start_ns(struct ctx *c, uid_t uid, gid_t gid, c->quiet = 1; /* Configure user and group mappings */ - snprintf(uidmap, BUFSIZ, "0 %u 1", uid); - snprintf(gidmap, BUFSIZ, "0 %u 1", gid); + if (snprintf_check(uidmap, BUFSIZ, "0 %u 1", uid)) + die_perror("Can't build uidmap"); + + if (snprintf_check(gidmap, BUFSIZ, "0 %u 1", gid)) + die_perror("Can't build gidmap"); if (write_file("/proc/self/uid_map", uidmap) || write_file("/proc/self/setgroups", "deny") || @@ -259,14 +277,12 @@ void pasta_start_ns(struct ctx *c, uid_t uid, gid_t gid, CLONE_NEWUTS | CLONE_NEWNS | SIGCHLD, (void *)&arg); - if (pasta_child_pid == -1) { - perror("clone"); - exit(EXIT_FAILURE); - } + if (pasta_child_pid == -1) + die_perror("Failed to clone process with detached namespaces"); NS_CALL(pasta_wait_for_ns, c); if (c->pasta_netns_fd < 0) - die("Failed to join network namespace: %s", strerror(errno)); + die_perror("Failed to join network namespace"); } /** @@ -277,25 +293,33 @@ void pasta_ns_conf(struct ctx *c) { int rc = 0; - rc = nl_link_up(nl_sock_ns, 1 /* lo */, 0); + rc = nl_link_set_flags(nl_sock_ns, 1 /* lo */, IFF_UP, IFF_UP); if (rc < 0) die("Couldn't bring up loopback interface in namespace: %s", strerror(-rc)); /* Get or set MAC in target namespace */ - if (MAC_IS_ZERO(c->mac_guest)) - nl_link_get_mac(nl_sock_ns, c->pasta_ifi, c->mac_guest); + if (MAC_IS_ZERO(c->guest_mac)) + nl_link_get_mac(nl_sock_ns, c->pasta_ifi, c->guest_mac); else - rc = nl_link_set_mac(nl_sock_ns, c->pasta_ifi, c->mac_guest); + rc = nl_link_set_mac(nl_sock_ns, c->pasta_ifi, c->guest_mac); if (rc < 0) die("Couldn't set MAC address in namespace: %s", strerror(-rc)); if (c->pasta_conf_ns) { - nl_link_up(nl_sock_ns, c->pasta_ifi, c->mtu); + unsigned int flags = IFF_UP; + + if (c->mtu != -1) + nl_link_set_mtu(nl_sock_ns, c->pasta_ifi, c->mtu); + + if (c->ifi6) /* Avoid duplicate address detection on link up */ + flags |= IFF_NOARP; + + nl_link_set_flags(nl_sock_ns, c->pasta_ifi, flags, flags); if (c->ifi4) { - if (c->no_copy_addrs) { + if (c->ip4.no_copy_addrs) { rc = nl_addr_set(nl_sock_ns, c->pasta_ifi, AF_INET, &c->ip4.addr, @@ -311,9 +335,10 @@ void pasta_ns_conf(struct ctx *c) strerror(-rc)); } - if (c->no_copy_routes) { + if (c->ip4.no_copy_routes) { rc = nl_route_set_def(nl_sock_ns, c->pasta_ifi, - AF_INET, &c->ip4.gw); + AF_INET, + &c->ip4.guest_gw); } else { rc = nl_route_dup(nl_sock, c->ifi4, nl_sock_ns, c->pasta_ifi, AF_INET); @@ -326,7 +351,24 @@ void pasta_ns_conf(struct ctx *c) } if (c->ifi6) { - if (c->no_copy_addrs) { + rc = nl_addr_get_ll(nl_sock_ns, c->pasta_ifi, + &c->ip6.addr_ll_seen); + if (rc < 0) { + warn("Can't get LL address from namespace: %s", + strerror(-rc)); + } + + rc = nl_addr_set_ll_nodad(nl_sock_ns, c->pasta_ifi); + if (rc < 0) { + warn("Can't set nodad for LL in namespace: %s", + strerror(-rc)); + } + + /* We dodged DAD: re-enable neighbour solicitations */ + nl_link_set_flags(nl_sock_ns, c->pasta_ifi, + 0, IFF_NOARP); + + if (c->ip6.no_copy_addrs) { rc = nl_addr_set(nl_sock_ns, c->pasta_ifi, AF_INET6, &c->ip6.addr, 64); } else { @@ -340,9 +382,10 @@ void pasta_ns_conf(struct ctx *c) strerror(-rc)); } - if (c->no_copy_routes) { + if (c->ip6.no_copy_routes) { rc = nl_route_set_def(nl_sock_ns, c->pasta_ifi, - AF_INET6, &c->ip6.gw); + AF_INET6, + &c->ip6.guest_gw); } else { rc = nl_route_dup(nl_sock, c->ifi6, nl_sock_ns, c->pasta_ifi, @@ -356,7 +399,7 @@ void pasta_ns_conf(struct ctx *c) } } - proto_update_l2_buf(c->mac_guest, NULL); + proto_update_l2_buf(c->guest_mac, NULL); } /** @@ -370,12 +413,12 @@ static int pasta_netns_quit_timer(void) struct itimerspec it = { { 1, 0 }, { 1, 0 } }; /* one-second interval */ if (fd == -1) { - err("timerfd_create(): %s", strerror(errno)); + err_perror("Failed to create timerfd for quit timer"); return -errno; } if (timerfd_settime(fd, 0, &it, NULL) < 0) { - err("timerfd_settime(): %s", strerror(errno)); + err_perror("Failed to set interval for quit timer"); close(fd); return -errno; } @@ -389,12 +432,12 @@ static int pasta_netns_quit_timer(void) */ void pasta_netns_quit_init(const struct ctx *c) { - union epoll_ref ref = { .type = EPOLL_TYPE_NSQUIT_INOTIFY }; struct epoll_event ev = { .events = EPOLLIN }; int flags = O_NONBLOCK | O_CLOEXEC; struct statfs s = { 0 }; bool try_inotify = true; int fd = -1, dir_fd; + union epoll_ref ref; if (c->mode != MODE_PASTA || c->no_netns_quit || !*c->netns_base) return; @@ -425,6 +468,7 @@ void pasta_netns_quit_init(const struct ctx *c) ref.type = EPOLL_TYPE_NSQUIT_TIMER; } else { close(dir_fd); + ref.type = EPOLL_TYPE_NSQUIT_INOTIFY; } if (fd > FD_REF_MAX) @@ -468,7 +512,7 @@ void pasta_netns_quit_timer_handler(struct ctx *c, union epoll_ref ref) n = read(ref.fd, &expirations, sizeof(expirations)); if (n < 0) - die("Namespace watch timer read() error: %s", strerror(errno)); + die_perror("Namespace watch timer read() error"); if ((size_t)n < sizeof(expirations)) warn("Namespace watch timer: short read(): %zi", n); @@ -72,44 +72,43 @@ struct pcap_pkthdr { * @iov: IO vector containing frame (with L2 headers and tap headers) * @iovcnt: Number of buffers (@iov entries) in frame * @offset: Byte offset of the L2 headers within @iov - * @tv: Timestamp + * @now: Timestamp * * Returns: 0 on success, -errno on error writing to the file */ static void pcap_frame(const struct iovec *iov, size_t iovcnt, - size_t offset, const struct timeval *tv) + size_t offset, const struct timespec *now) { - size_t len = iov_size(iov, iovcnt) - offset; + size_t l2len = iov_size(iov, iovcnt) - offset; struct pcap_pkthdr h = { - .tv_sec = tv->tv_sec, - .tv_usec = tv->tv_usec, - .caplen = len, - .len = len + .tv_sec = now->tv_sec, + .tv_usec = DIV_ROUND_CLOSEST(now->tv_nsec, 1000), + .caplen = l2len, + .len = l2len }; - struct iovec hiov = { &h, sizeof(h) }; - if (write_remainder(pcap_fd, &hiov, 1, 0) < 0 || - write_remainder(pcap_fd, iov, iovcnt, offset) < 0) { - debug("Cannot log packet, length %zu: %s", - len, strerror(errno)); - } + if (write_all_buf(pcap_fd, &h, sizeof(h)) < 0 || + write_remainder(pcap_fd, iov, iovcnt, offset) < 0) + debug_perror("Cannot log packet, length %zu", l2len); } /** * pcap() - Capture a single frame to pcap file * @pkt: Pointer to data buffer, including L2 headers - * @len: L2 packet length + * @l2len: L2 frame length */ -void pcap(const char *pkt, size_t len) +void pcap(const char *pkt, size_t l2len) { - struct iovec iov = { (char *)pkt, len }; - struct timeval tv; + struct iovec iov = { (char *)pkt, l2len }; + struct timespec now = { 0 }; if (pcap_fd == -1) return; - gettimeofday(&tv, NULL); - pcap_frame(&iov, 1, 0, &tv); + if (clock_gettime(CLOCK_REALTIME, &now)) + err_perror("Failed to get CLOCK_REALTIME time"); + + pcap_frame(&iov, 1, 0, &now); } /** @@ -122,16 +121,17 @@ void pcap(const char *pkt, size_t len) void pcap_multiple(const struct iovec *iov, size_t frame_parts, unsigned int n, size_t offset) { - struct timeval tv; + struct timespec now = { 0 }; unsigned int i; if (pcap_fd == -1) return; - gettimeofday(&tv, NULL); + if (clock_gettime(CLOCK_REALTIME, &now)) + err_perror("Failed to get CLOCK_REALTIME time"); for (i = 0; i < n; i++) - pcap_frame(iov + i * frame_parts, frame_parts, offset, &tv); + pcap_frame(iov + i * frame_parts, frame_parts, offset, &now); } /* @@ -141,17 +141,19 @@ void pcap_multiple(const struct iovec *iov, size_t frame_parts, unsigned int n, * @iov: Pointer to the array of struct iovec describing the I/O vector * containing packet data to write, including L2 header * @iovcnt: Number of buffers (@iov entries) + * @offset: Offset of the L2 frame within the full data length */ -/* cppcheck-suppress unusedFunction */ -void pcap_iov(const struct iovec *iov, size_t iovcnt) +void pcap_iov(const struct iovec *iov, size_t iovcnt, size_t offset) { - struct timeval tv; + struct timespec now = { 0 }; if (pcap_fd == -1) return; - gettimeofday(&tv, NULL); - pcap_frame(iov, iovcnt, 0, &tv); + if (clock_gettime(CLOCK_REALTIME, &now)) + err_perror("Failed to get CLOCK_REALTIME time"); + + pcap_frame(iov, iovcnt, offset, &now); } /** @@ -160,23 +162,20 @@ void pcap_iov(const struct iovec *iov, size_t iovcnt) */ void pcap_init(struct ctx *c) { - int flags = O_WRONLY | O_CREAT | O_TRUNC; - if (pcap_fd != -1) return; if (!*c->pcap) return; - flags |= c->foreground ? O_CLOEXEC : 0; - pcap_fd = open(c->pcap, flags, S_IRUSR | S_IWUSR); + pcap_fd = output_file_open(c->pcap, O_WRONLY); if (pcap_fd == -1) { - perror("open"); + err_perror("Couldn't open pcap file %s", c->pcap); return; } info("Saving packet capture to %s", c->pcap); if (write(pcap_fd, &pcap_hdr, sizeof(pcap_hdr)) < 0) - warn("Cannot write PCAP header: %s", strerror(errno)); + warn_perror("Cannot write PCAP header"); } @@ -6,10 +6,10 @@ #ifndef PCAP_H #define PCAP_H -void pcap(const char *pkt, size_t len); +void pcap(const char *pkt, size_t l2len); void pcap_multiple(const struct iovec *iov, size_t frame_parts, unsigned int n, size_t offset); -void pcap_iov(const struct iovec *iov, size_t iovcnt); +void pcap_iov(const struct iovec *iov, size_t iovcnt, size_t offset); void pcap_init(struct ctx *c); #endif /* PCAP_H */ @@ -7,9 +7,14 @@ #include <stdint.h> #include <assert.h> +#include <netinet/in.h> #include "util.h" #include "pif.h" +#include "siphash.h" +#include "ip.h" +#include "inany.h" +#include "passt.h" const char *pif_type_str[] = { [PIF_NONE] = "<none>", @@ -19,3 +24,80 @@ const char *pif_type_str[] = { }; static_assert(ARRAY_SIZE(pif_type_str) == PIF_NUM_TYPES, "pif_type_str[] doesn't match enum pif_type"); + + +/** pif_sockaddr() - Construct a socket address suitable for an interface + * @c: Execution context + * @sa: Pointer to sockaddr to fill in + * @sl: Updated to relevant length of initialised @sa + * @pif: Interface to create the socket address + * @addr: IPv[46] address + * @port: Port (host byte order) + */ +void pif_sockaddr(const struct ctx *c, union sockaddr_inany *sa, socklen_t *sl, + uint8_t pif, const union inany_addr *addr, in_port_t port) +{ + const struct in_addr *v4 = inany_v4(addr); + + ASSERT(pif_is_socket(pif)); + + if (v4) { + sa->sa_family = AF_INET; + sa->sa4.sin_addr = *v4; + sa->sa4.sin_port = htons(port); + memset(&sa->sa4.sin_zero, 0, sizeof(sa->sa4.sin_zero)); + *sl = sizeof(sa->sa4); + } else { + sa->sa_family = AF_INET6; + sa->sa6.sin6_addr = addr->a6; + sa->sa6.sin6_port = htons(port); + if (pif == PIF_HOST && IN6_IS_ADDR_LINKLOCAL(&addr->a6)) + sa->sa6.sin6_scope_id = c->ifi6; + else + sa->sa6.sin6_scope_id = 0; + sa->sa6.sin6_flowinfo = 0; + *sl = sizeof(sa->sa6); + } +} + +/** pif_sock_l4() - Open a socket bound to an address on a specified interface + * @c: Execution context + * @type: Socket epoll type + * @pif: Interface for this socket + * @addr: Address to bind to, or NULL for dual-stack any + * @ifname: Interface for binding, NULL for any + * @port: Port number to bind to (host byte order) + * @data: epoll reference portion for protocol handlers + * + * NOTE: For namespace pifs, this must be called having already entered the + * relevant namespace. + * + * Return: newly created socket, negative error code on failure + */ +int pif_sock_l4(const struct ctx *c, enum epoll_type type, uint8_t pif, + const union inany_addr *addr, const char *ifname, + in_port_t port, uint32_t data) +{ + union sockaddr_inany sa = { + .sa6.sin6_family = AF_INET6, + .sa6.sin6_addr = in6addr_any, + .sa6.sin6_port = htons(port), + }; + socklen_t sl; + + ASSERT(pif_is_socket(pif)); + + if (pif == PIF_SPLICE) { + /* Sanity checks */ + ASSERT(!ifname); + ASSERT(addr && inany_is_loopback(addr)); + } + + if (!addr) + return sock_l4_sa(c, type, &sa, sizeof(sa.sa6), + ifname, false, data); + + pif_sockaddr(c, &sa, &sl, pif, addr, port); + return sock_l4_sa(c, type, &sa, sl, + ifname, sa.sa_family == AF_INET6, data); +} @@ -7,6 +7,9 @@ #ifndef PIF_H #define PIF_H +union inany_addr; +union sockaddr_inany; + /** * enum pif_type - Type of passt/pasta interface ("pif") * @@ -38,10 +41,26 @@ static inline const char *pif_type(enum pif_type pt) return "?"; } -/* cppcheck-suppress unusedFunction */ static inline const char *pif_name(uint8_t pif) { return pif_type(pif); } +/** + * pif_is_socket() - Is interface implemented via L4 sockets? + * @pif: pif to check + * + * Return: true of @pif is an L4 socket based interface, otherwise false + */ +static inline bool pif_is_socket(uint8_t pif) +{ + return pif == PIF_HOST || pif == PIF_SPLICE; +} + +void pif_sockaddr(const struct ctx *c, union sockaddr_inany *sa, socklen_t *sl, + uint8_t pif, const union inany_addr *addr, in_port_t port); +int pif_sock_l4(const struct ctx *c, enum epoll_type type, uint8_t pif, + const union inany_addr *addr, const char *ifname, + in_port_t port, uint32_t data); + #endif /* PIF_H */ @@ -66,8 +66,8 @@ issues to Stefano Brivio <sbrivio@redhat.com>. Copyright (c) 2020-2021 Red Hat GmbH. \fBqrap\fR is free software: you can redistribute is and/or modify it under the -terms of the GNU Affero General Public License as published by the Free Software -Foundation, either version 3 of the License, or (at your option) any later +terms of the GNU General Public License as published by the Free Software +Foundation, either version 2 of the License, or (at your option) any later version. .SH SEE ALSO @@ -20,6 +20,15 @@ OUT="$(mktemp)" [ -z "${ARCH}" ] && ARCH="$(uname -m)" [ -z "${CC}" ] && CC="cc" +AUDIT_ARCH="AUDIT_ARCH_$(echo ${ARCH} | tr [a-z] [A-Z] \ + | sed 's/^ARM.*/ARM/' \ + | sed 's/I[456]86/I386/' \ + | sed 's/PPC64/PPC/' \ + | sed 's/PPCLE/PPC64LE/' \ + | sed 's/MIPS64EL/MIPSEL64/' \ + | sed 's/HPPA/PARISC/' \ + | sed 's/SH4/SH/')" + HEADER="/* This file was automatically generated by $(basename ${0}) */ #ifndef AUDIT_ARCH_PPC64LE @@ -29,11 +38,11 @@ HEADER="/* This file was automatically generated by $(basename ${0}) */ # Prefix for each profile: check that 'arch' in seccomp_data is matching PRE=' struct sock_filter filter_@PROFILE@[] = { - /* cppcheck-suppress badBitmaskCheck */ + /* cppcheck-suppress [badBitmaskCheck, unmatchedSuppression] */ BPF_STMT(BPF_LD | BPF_W | BPF_ABS, (offsetof(struct seccomp_data, arch))), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, PASST_AUDIT_ARCH, 0, @KILL@), - /* cppcheck-suppress badBitmaskCheck */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, @AUDIT_ARCH@, 0, @KILL@), + /* cppcheck-suppress [badBitmaskCheck, unmatchedSuppression] */ BPF_STMT(BPF_LD | BPF_W | BPF_ABS, (offsetof(struct seccomp_data, nr))), @@ -233,7 +242,8 @@ gen_profile() { sub ${__i} CALL "NR:${__nr}" "NAME:${__name}" "ALLOW:${__allow}" done - finish PRE "PROFILE:${__profile}" "KILL:$(( __statements + 1))" + finish PRE "PROFILE:${__profile}" "KILL:$(( __statements + 1))" \ + "AUDIT_ARCH:${AUDIT_ARCH}" } printf '%s\n' "${HEADER}" > "${OUT}" @@ -242,7 +252,10 @@ for __p in ${__profiles}; do __calls="$(sed -n 's/[\t ]*\*[\t ]*#syscalls\(:'"${__p}"'\|\)[\t ]\{1,\}\(.*\)/\2/p' ${IN})" __calls="${__calls} ${EXTRA_SYSCALLS:-}" __calls="$(filter ${__calls})" - echo "seccomp profile ${__p} allows: ${__calls}" | tr '\n' ' ' | fmt -t + + cols="$(stty -a | sed -n 's/.*columns \([0-9]*\).*/\1/p' || :)" 2>/dev/null + case $cols in [0-9]*) col_args="-w ${cols}";; *) col_args="";; esac + echo "seccomp profile ${__p} allows: ${__calls}" | tr '\n' ' ' | fmt -t ${col_args} # Pad here to keep gen_profile() "simple" __count=0 @@ -115,10 +115,4 @@ static inline uint64_t siphash_final(struct siphash_state *state, return state->v[0] ^ state->v[1] ^ state->v[2] ^ state->v[3]; } -uint64_t siphash_8b(const uint8_t *in, const uint64_t *k); -uint64_t siphash_12b(const uint8_t *in, const uint64_t *k); -uint64_t siphash_20b(const uint8_t *in, const uint64_t *k); -uint64_t siphash_32b(const uint8_t *in, const uint64_t *k); -uint64_t siphash_36b(const uint8_t *in, const uint64_t *k); - #endif /* SIPHASH_H */ @@ -59,6 +59,7 @@ #include "tap.h" #include "log.h" #include "vhost_user.h" +#include "vu_common.h" /* IPv4 (plus ARP) and IPv6 message batches from tap/guest to IP handlers */ static PACKET_POOL_NOINIT(pool_tap4, TAP_MSGS, pkt_buf); @@ -68,42 +69,33 @@ static PACKET_POOL_NOINIT(pool_tap6, TAP_MSGS, pkt_buf); #define FRAGMENT_MSG_RATE 10 /* # seconds between fragment warnings */ /** - * tap_send() - Send frame, with qemu socket header if needed + * tap_send_single() - Send a single frame * @c: Execution context * @data: Packet buffer - * @len: Total L2 packet length - * - * Return: return code from send() or write() + * @l2len: Total L2 packet length */ -int tap_send(const struct ctx *c, const void *data, size_t len) +void tap_send_single(const struct ctx *c, const void *data, size_t l2len) { - int flags = MSG_NOSIGNAL | MSG_DONTWAIT; - uint32_t vnet_len = htonl(len); - - pcap(data, len); + uint32_t vnet_len = htonl(l2len); + struct iovec iov[2]; + size_t iovcnt = 0; switch (c->mode) { case MODE_PASST: - if (send(c->fd_tap, &vnet_len, 4, flags) < 0) - return -1; - return send(c->fd_tap, data, len, flags); + iov[iovcnt] = IOV_OF_LVALUE(vnet_len); + iovcnt++; + /* fall through */ case MODE_PASTA: - return write(c->fd_tap, (char *)data, len); + iov[iovcnt].iov_base = (void *)data; + iov[iovcnt].iov_len = l2len; + iovcnt++; + + tap_send_frames(c, iov, iovcnt, 1); + break; case MODE_VU: - return vu_send(c, data, len); + vu_send_single(c, data, l2len); + break; } - return 0; -} - -/** - * tap_ip4_daddr() - Normal IPv4 destination address for inbound packets - * @c: Execution context - * - * Return: IPv4 address, network order - */ -struct in_addr tap_ip4_daddr(const struct ctx *c) -{ - return c->ip4.addr_seen; } /** @@ -134,8 +126,8 @@ static void *tap_push_l2h(const struct ctx *c, void *buf, uint16_t proto) struct ethhdr *eh = (struct ethhdr *)buf; /* TODO: ARP table lookup */ - memcpy(eh->h_dest, c->mac_guest, ETH_ALEN); - memcpy(eh->h_source, c->mac, ETH_ALEN); + memcpy(eh->h_dest, c->guest_mac, ETH_ALEN); + memcpy(eh->h_source, c->our_tap_mac, ETH_ALEN); eh->h_proto = ntohs(proto); return eh + 1; } @@ -143,29 +135,29 @@ static void *tap_push_l2h(const struct ctx *c, void *buf, uint16_t proto) /** * tap_push_ip4h() - Build IPv4 header for inbound packet, with checksum * @c: Execution context - * @src: IPv4 source address, network order - * @dst: IPv4 destination address, network order - * @len: L4 payload length + * @src: IPv4 source address + * @dst: IPv4 destination address + * @l4len: IPv4 payload length * @proto: L4 protocol number * * Return: pointer at which to write the packet's payload */ -static void *tap_push_ip4h(char *buf, struct in_addr src, struct in_addr dst, - size_t len, uint8_t proto) +static void *tap_push_ip4h(struct iphdr *ip4h, struct in_addr src, + struct in_addr dst, size_t l4len, uint8_t proto) { - struct iphdr *ip4h = (struct iphdr *)buf; + uint16_t l3len = l4len + sizeof(*ip4h); ip4h->version = 4; ip4h->ihl = sizeof(struct iphdr) / 4; ip4h->tos = 0; - ip4h->tot_len = htons(len + sizeof(*ip4h)); + ip4h->tot_len = htons(l3len); ip4h->id = 0; ip4h->frag_off = 0; ip4h->ttl = 255; ip4h->protocol = proto; ip4h->saddr = src.s_addr; ip4h->daddr = dst.s_addr; - ip4h->check = csum_ip4_header(ip4h->tot_len, proto, src, dst); + ip4h->check = csum_ip4_header(l3len, proto, src, dst); return ip4h + 1; } @@ -177,27 +169,29 @@ static void *tap_push_ip4h(char *buf, struct in_addr src, struct in_addr dst, * @dst: IPv4 destination address * @dport: UDP destination port * @in: UDP payload contents (not including UDP header) - * @len: UDP payload length (not including UDP header) + * @dlen: UDP payload length (not including UDP header) */ void tap_udp4_send(const struct ctx *c, struct in_addr src, in_port_t sport, struct in_addr dst, in_port_t dport, - const void *in, size_t len) + const void *in, size_t dlen) { - size_t udplen = len + sizeof(struct udphdr); + size_t l4len = dlen + sizeof(struct udphdr); char buf[USHRT_MAX]; - void *ip4h = tap_push_l2h(c, buf, ETH_P_IP); - void *uhp = tap_push_ip4h(ip4h, src, dst, udplen, IPPROTO_UDP); - struct udphdr *uh = (struct udphdr *)uhp; + struct iphdr *ip4h = tap_push_l2h(c, buf, ETH_P_IP); + struct udphdr *uh = tap_push_ip4h(ip4h, src, dst, l4len, IPPROTO_UDP); char *data = (char *)(uh + 1); + const struct iovec iov = { + .iov_base = (void *)in, + .iov_len = dlen + }; uh->source = htons(sport); uh->dest = htons(dport); - uh->len = htons(udplen); - csum_udp4(uh, src, dst, in, len); - memcpy(data, in, len); + uh->len = htons(l4len); + csum_udp4(uh, src, dst, &iov, 1, 0); + memcpy(data, in, dlen); - if (tap_send(c, buf, len + (data - buf)) < 0) - debug("tap: failed to send %zu bytes (IPv4)", len); + tap_send_single(c, buf, dlen + (data - buf)); } /** @@ -206,21 +200,20 @@ void tap_udp4_send(const struct ctx *c, struct in_addr src, in_port_t sport, * @src: IPv4 source address * @dst: IPv4 destination address * @in: ICMP packet, including ICMP header - * @len: ICMP packet length, including ICMP header + * @l4len: ICMP packet length, including ICMP header */ void tap_icmp4_send(const struct ctx *c, struct in_addr src, struct in_addr dst, - const void *in, size_t len) + const void *in, size_t l4len) { char buf[USHRT_MAX]; - void *ip4h = tap_push_l2h(c, buf, ETH_P_IP); - char *data = tap_push_ip4h(ip4h, src, dst, len, IPPROTO_ICMP); - struct icmphdr *icmp4h = (struct icmphdr *)data; + struct iphdr *ip4h = tap_push_l2h(c, buf, ETH_P_IP); + struct icmphdr *icmp4h = tap_push_ip4h(ip4h, src, dst, + l4len, IPPROTO_ICMP); - memcpy(data, in, len); - csum_icmp4(icmp4h, icmp4h + 1, len - sizeof(*icmp4h)); + memcpy(icmp4h, in, l4len); + csum_icmp4(icmp4h, icmp4h + 1, l4len - sizeof(*icmp4h)); - if (tap_send(c, buf, len + (data - buf)) < 0) - debug("tap: failed to send %zu bytes (IPv4)", len); + tap_send_single(c, buf, l4len + ((char *)icmp4h - buf)); } /** @@ -228,20 +221,18 @@ void tap_icmp4_send(const struct ctx *c, struct in_addr src, struct in_addr dst, * @c: Execution context * @src: IPv6 source address * @dst: IPv6 destination address - * @len: L4 payload length + * @l4len: L4 payload length * @proto: L4 protocol number * @flow: IPv6 flow identifier * * Return: pointer at which to write the packet's payload */ -static void *tap_push_ip6h(char *buf, +static void *tap_push_ip6h(struct ipv6hdr *ip6h, const struct in6_addr *src, const struct in6_addr *dst, - size_t len, uint8_t proto, uint32_t flow) + size_t l4len, uint8_t proto, uint32_t flow) { - struct ipv6hdr *ip6h = (struct ipv6hdr *)buf; - - ip6h->payload_len = htons(len); + ip6h->payload_len = htons(l4len); ip6h->priority = 0; ip6h->version = 6; ip6h->nexthdr = proto; @@ -263,28 +254,31 @@ static void *tap_push_ip6h(char *buf, * @dport: UDP destination port * @flow: Flow label * @in: UDP payload contents (not including UDP header) - * @len: UDP payload length (not including UDP header) + * @dlen: UDP payload length (not including UDP header) */ void tap_udp6_send(const struct ctx *c, const struct in6_addr *src, in_port_t sport, const struct in6_addr *dst, in_port_t dport, - uint32_t flow, const void *in, size_t len) + uint32_t flow, void *in, size_t dlen) { - size_t udplen = len + sizeof(struct udphdr); + size_t l4len = dlen + sizeof(struct udphdr); char buf[USHRT_MAX]; - void *ip6h = tap_push_l2h(c, buf, ETH_P_IPV6); - void *uhp = tap_push_ip6h(ip6h, src, dst, udplen, IPPROTO_UDP, flow); - struct udphdr *uh = (struct udphdr *)uhp; + struct ipv6hdr *ip6h = tap_push_l2h(c, buf, ETH_P_IPV6); + struct udphdr *uh = tap_push_ip6h(ip6h, src, dst, + l4len, IPPROTO_UDP, flow); char *data = (char *)(uh + 1); + const struct iovec iov = { + .iov_base = in, + .iov_len = dlen + }; uh->source = htons(sport); uh->dest = htons(dport); - uh->len = htons(udplen); - csum_udp6(uh, src, dst, in, len); - memcpy(data, in, len); + uh->len = htons(l4len); + csum_udp6(uh, src, dst, &iov, 1, 0); + memcpy(data, in, dlen); - if (tap_send(c, buf, len + (data - buf)) < 1) - debug("tap: failed to send %zu bytes (IPv6)", len); + tap_send_single(c, buf, dlen + (data - buf)); } /** @@ -293,44 +287,50 @@ void tap_udp6_send(const struct ctx *c, * @src: IPv6 source address * @dst: IPv6 destination address * @in: ICMP packet, including ICMP header - * @len: ICMP packet length, including ICMP header + * @l4len: ICMP packet length, including ICMP header */ void tap_icmp6_send(const struct ctx *c, const struct in6_addr *src, const struct in6_addr *dst, - const void *in, size_t len) + const void *in, size_t l4len) { char buf[USHRT_MAX]; - void *ip6h = tap_push_l2h(c, buf, ETH_P_IPV6); - char *data = tap_push_ip6h(ip6h, src, dst, len, IPPROTO_ICMPV6, 0); - struct icmp6hdr *icmp6h = (struct icmp6hdr *)data; + struct ipv6hdr *ip6h = tap_push_l2h(c, buf, ETH_P_IPV6); + struct icmp6hdr *icmp6h = tap_push_ip6h(ip6h, src, dst, l4len, + IPPROTO_ICMPV6, 0); - memcpy(data, in, len); - csum_icmp6(icmp6h, src, dst, icmp6h + 1, len - sizeof(*icmp6h)); + memcpy(icmp6h, in, l4len); + csum_icmp6(icmp6h, src, dst, icmp6h + 1, l4len - sizeof(*icmp6h)); - if (tap_send(c, buf, len + (data - buf)) < 1) - debug("tap: failed to send %zu bytes (IPv6)", len); + tap_send_single(c, buf, l4len + ((char *)icmp6h - buf)); } /** * tap_send_frames_pasta() - Send multiple frames to the pasta tap - * @c: Execution context - * @iov: Array of buffers, each containing one frame - * @n: Number of buffers/frames in @iov + * @c: Execution context + * @iov: Array of buffers + * @bufs_per_frame: Number of buffers (iovec entries) per frame + * @nframes: Number of frames to send + * + * @iov must have total length @bufs_per_frame * @nframes, with each set of + * @bufs_per_frame contiguous buffers representing a single frame. * * Return: number of frames successfully sent * * #syscalls:pasta write */ static size_t tap_send_frames_pasta(const struct ctx *c, - const struct iovec *iov, size_t n) + const struct iovec *iov, + size_t bufs_per_frame, size_t nframes) { + size_t nbufs = bufs_per_frame * nframes; size_t i; - for (i = 0; i < n; i++) { - ssize_t rc = write(c->fd_tap, iov[i].iov_base, iov[i].iov_len); + for (i = 0; i < nbufs; i += bufs_per_frame) { + ssize_t rc = writev(c->fd_tap, iov + i, bufs_per_frame); + size_t framelen = iov_size(iov + i, bufs_per_frame); if (rc < 0) { - debug("tap write: %s", strerror(errno)); + debug_perror("tap write"); switch (errno) { case EAGAIN: @@ -340,60 +340,42 @@ static size_t tap_send_frames_pasta(const struct ctx *c, case EINTR: case ENOBUFS: case ENOSPC: + case EIO: /* interface down? */ break; default: die("Write error on tap device, exiting"); } - } else if ((size_t)rc < iov[i].iov_len) { - debug("short write on tuntap: %zd/%zu", - rc, iov[i].iov_len); + } else if ((size_t)rc < framelen) { + debug("short write on tuntap: %zd/%zu", rc, framelen); break; } } - return i; -} - -/** - * tap_send_iov_pasta() - Send out multiple prepared frames - * @c: Execution context - * @iov: Array of frames, each frames is divided in an array of iovecs. - * The first entry of the iovec is ignored - * @n: Number of frames in @iov - * - * Return: number of frames actually sent - */ -static size_t tap_send_iov_pasta(const struct ctx *c, - struct iovec iov[][TCP_IOV_NUM], size_t n) -{ - unsigned int i; - - for (i = 0; i < n; i++) { - if (!tap_send_frames_pasta(c, &iov[i][TCP_IOV_ETH], - TCP_IOV_NUM - TCP_IOV_ETH)) - break; - } - - return i; - + return i / bufs_per_frame; } /** * tap_send_frames_passt() - Send multiple frames to the passt tap - * @c: Execution context - * @iov: Array of buffers, each containing one frame - * @n: Number of buffers/frames in @iov + * @c: Execution context + * @iov: Array of buffers, each containing one frame + * @bufs_per_frame: Number of buffers (iovec entries) per frame + * @nframes: Number of frames to send + * + * @iov must have total length @bufs_per_frame * @nframes, with each set of + * @bufs_per_frame contiguous buffers representing a single frame. * * Return: number of frames successfully sent * * #syscalls:passt sendmsg */ static size_t tap_send_frames_passt(const struct ctx *c, - const struct iovec *iov, size_t n) + const struct iovec *iov, + size_t bufs_per_frame, size_t nframes) { + size_t nbufs = bufs_per_frame * nframes; struct msghdr mh = { .msg_iov = (void *)iov, - .msg_iovlen = n, + .msg_iovlen = nbufs, }; size_t buf_offset; unsigned int i; @@ -404,134 +386,61 @@ static size_t tap_send_frames_passt(const struct ctx *c, return 0; /* Check for any partial frames due to short send */ - i = iov_skip_bytes(iov, n, sent, &buf_offset); + i = iov_skip_bytes(iov, nbufs, sent, &buf_offset); - if (i < n && buf_offset) { - /* A partial frame was sent */ - if (write_remainder(c->fd_tap, &iov[i], 1, buf_offset) < 0) { - err("tap: partial frame send: %s", strerror(errno)); + if (i < nbufs && (buf_offset || (i % bufs_per_frame))) { + /* Number of unsent or partially sent buffers for the frame */ + size_t rembufs = bufs_per_frame - (i % bufs_per_frame); + + if (write_remainder(c->fd_tap, &iov[i], rembufs, buf_offset) < 0) { + err_perror("tap: partial frame send"); return i; } - i++; - } - - return i; -} - -/** - * tap_send_iov_passt() - Send out multiple prepared frames - * @c: Execution context - * @iov: Array of frames, each frames is divided in an array of iovecs. - * The first entry of the iovec is updated to point to an - * uint32_t storing the frame length. - * @n: Number of frames in @iov - * - * Return: number of frames actually sent - */ -static size_t tap_send_iov_passt(const struct ctx *c, - struct iovec iov[][TCP_IOV_NUM], - size_t n) -{ - unsigned int i; - - for (i = 0; i < n; i++) { - uint32_t vnet_len; - int j; - - vnet_len = 0; - for (j = TCP_IOV_ETH; j < TCP_IOV_NUM; j++) - vnet_len += iov[i][j].iov_len; - - vnet_len = htonl(vnet_len); - iov[i][TCP_IOV_VNET].iov_base = &vnet_len; - iov[i][TCP_IOV_VNET].iov_len = sizeof(vnet_len); - - if (!tap_send_frames_passt(c, iov[i], TCP_IOV_NUM)) - break; + i += rembufs; } - return i; - + return i / bufs_per_frame; } /** * tap_send_frames() - Send out multiple prepared frames - * @c: Execution context - * @iov: Array of buffers, each containing one frame (with L2 headers) - * @n: Number of buffers/frames in @iov + * @c: Execution context + * @iov: Array of buffers, each containing one frame (with L2 headers) + * @bufs_per_frame: Number of buffers (iovec entries) per frame + * @nframes: Number of frames to send + * + * @iov must have total length @bufs_per_frame * @nframes, with each set of + * @bufs_per_frame contiguous buffers representing a single frame. * * Return: number of frames actually sent */ -size_t tap_send_frames(const struct ctx *c, const struct iovec *iov, size_t n) +size_t tap_send_frames(const struct ctx *c, const struct iovec *iov, + size_t bufs_per_frame, size_t nframes) { size_t m; - if (!n) + if (!nframes) return 0; switch (c->mode) { case MODE_PASTA: - m = tap_send_frames_pasta(c, iov, n); + m = tap_send_frames_pasta(c, iov, bufs_per_frame, nframes); break; case MODE_PASST: - m = tap_send_frames_passt(c, iov, n); + m = tap_send_frames_passt(c, iov, bufs_per_frame, nframes); break; case MODE_VU: - m = tap_send_frames_vu(c, iov, n); - break; - default: - m = 0; - break; - } - - if (m < n) - debug("tap: failed to send %zu frames of %zu", n - m, n); - - pcap_multiple(iov, 1, n, c->mode == MODE_PASST ? sizeof(uint32_t) : 0); - - return m; -} - -/** - * tap_send_iov() - Send out multiple prepared frames - * @c: Execution context - * @iov: Array of frames, each frames is divided in an array of iovecs. - * iovec array is: - * TCP_IOV_VNET (0) vnet length - * TCP_IOV_ETH (1) ethernet header - * TCP_IOV_IP (2) IP (v4/v6) header - * TCP_IOV_PAYLOAD (3) IP payload (TCP header + data) - * TCP_IOV_NUM (4) is the number of entries in the iovec array - * TCP_IOV_VNET entry is updated with passt, ignored with pasta. - * @n: Number of frames in @iov - * - * Return: number of frames actually sent - */ -size_t tap_send_iov(const struct ctx *c, struct iovec iov[][TCP_IOV_NUM], - size_t n) -{ - size_t m; - unsigned int i; - - if (!n) - return 0; - - switch (c->mode) { - case MODE_PASST: - m = tap_send_iov_passt(c, iov, n); - break; - case MODE_PASTA: - m = tap_send_iov_pasta(c, iov, n); - break; + /* fall through */ default: ASSERT(0); } - if (m < n) - debug("tap: failed to send %zu frames of %zu", n - m, n); + if (m < nframes) + debug("tap: failed to send %zu frames of %zu", + nframes - m, nframes); - for (i = 0; i < m; i++) - pcap_iov(&iov[i][TCP_IOV_ETH], TCP_IOV_NUM - TCP_IOV_ETH); + pcap_multiple(iov, bufs_per_frame, m, + c->mode == MODE_PASST ? sizeof(uint32_t) : 0); return m; } @@ -695,21 +604,21 @@ static int tap4_handler(struct ctx *c, const struct pool *in, i = 0; resume: for (seq_count = 0, seq = NULL; i < in->count; i++) { - size_t l2_len, l3_len, hlen, l4_len; + size_t l2len, l3len, hlen, l4len; const struct ethhdr *eh; const struct udphdr *uh; struct iphdr *iph; const char *l4h; - packet_get(in, i, 0, 0, &l2_len); + packet_get(in, i, 0, 0, &l2len); - eh = packet_get(in, i, 0, sizeof(*eh), &l3_len); + eh = packet_get(in, i, 0, sizeof(*eh), &l3len); if (!eh) continue; if (ntohs(eh->h_proto) == ETH_P_ARP) { PACKET_POOL_P(pkt, 1, in->buf, in->buf_size); - packet_add(pkt, l2_len, (char *)eh); + packet_add(pkt, l2len, (char *)eh); arp(c, pkt); continue; } @@ -719,15 +628,15 @@ resume: continue; hlen = iph->ihl * 4UL; - if (hlen < sizeof(*iph) || htons(iph->tot_len) > l3_len || - hlen > l3_len) + if (hlen < sizeof(*iph) || htons(iph->tot_len) > l3len || + hlen > l3len) continue; /* We don't handle IP fragments, drop them */ if (tap4_is_fragment(iph, now)) continue; - l4_len = htons(iph->tot_len) - hlen; + l4len = htons(iph->tot_len) - hlen; if (IN4_IS_ADDR_LOOPBACK(&iph->saddr) || IN4_IS_ADDR_LOOPBACK(&iph->daddr)) { @@ -742,7 +651,7 @@ resume: if (iph->saddr && c->ip4.addr_seen.s_addr != iph->saddr) c->ip4.addr_seen.s_addr = iph->saddr; - l4h = packet_get(in, i, sizeof(*eh) + hlen, l4_len, NULL); + l4h = packet_get(in, i, sizeof(*eh) + hlen, l4len, NULL); if (!l4h) continue; @@ -754,7 +663,7 @@ resume: tap_packet_debug(iph, NULL, NULL, 0, NULL, 1); - packet_add(pkt, l4_len, l4h); + packet_add(pkt, l4len, l4h); icmp_tap_handler(c, PIF_TAP, AF_INET, &iph->saddr, &iph->daddr, pkt, now); @@ -768,7 +677,7 @@ resume: if (iph->protocol == IPPROTO_UDP) { PACKET_POOL_P(pkt, 1, in->buf, in->buf_size); - packet_add(pkt, l2_len, (char *)eh); + packet_add(pkt, l2len, (char *)eh); if (dhcp(c, pkt)) continue; } @@ -779,18 +688,18 @@ resume: continue; } -#define L4_MATCH(iph, uh, seq) \ - (seq->protocol == iph->protocol && \ - seq->source == uh->source && seq->dest == uh->dest && \ - seq->saddr.s_addr == iph->saddr && seq->daddr.s_addr == iph->daddr) +#define L4_MATCH(iph, uh, seq) \ + ((seq)->protocol == (iph)->protocol && \ + (seq)->source == (uh)->source && (seq)->dest == (uh)->dest && \ + (seq)->saddr.s_addr == (iph)->saddr && (seq)->daddr.s_addr == (iph)->daddr) #define L4_SET(iph, uh, seq) \ do { \ - seq->protocol = iph->protocol; \ - seq->source = uh->source; \ - seq->dest = uh->dest; \ - seq->saddr.s_addr = iph->saddr; \ - seq->daddr.s_addr = iph->daddr; \ + (seq)->protocol = (iph)->protocol; \ + (seq)->source = (uh)->source; \ + (seq)->dest = (uh)->dest; \ + (seq)->saddr.s_addr = (iph)->saddr; \ + (seq)->daddr.s_addr = (iph)->daddr; \ } while (0) if (seq && L4_MATCH(iph, uh, seq) && seq->p.count < UIO_MAXIOV) @@ -817,7 +726,7 @@ resume: #undef L4_SET append: - packet_add((struct pool *)&seq->p, l4_len, l4h); + packet_add((struct pool *)&seq->p, l4len, l4h); } for (j = 0, seq = tap4_l4; j < seq_count; j++, seq++) { @@ -869,7 +778,7 @@ static int tap6_handler(struct ctx *c, const struct pool *in, i = 0; resume: for (seq_count = 0, seq = NULL; i < in->count; i++) { - size_t l4_len, plen, check; + size_t l4len, plen, check; struct in6_addr *saddr, *daddr; const struct ethhdr *eh; const struct udphdr *uh; @@ -892,7 +801,7 @@ resume: if (plen != check) continue; - if (!(l4h = ipv6_l4hdr(in, i, sizeof(*eh), &proto, &l4_len))) + if (!(l4h = ipv6_l4hdr(in, i, sizeof(*eh), &proto, &l4len))) continue; if (IN6_IS_ADDR_LOOPBACK(saddr) || IN6_IS_ADDR_LOOPBACK(daddr)) { @@ -920,28 +829,29 @@ resume: if (c->no_icmp) continue; - if (l4_len < sizeof(struct icmp6hdr)) + if (l4len < sizeof(struct icmp6hdr)) continue; - if (ndp(c, (struct icmp6hdr *)l4h, saddr)) + packet_add(pkt, l4len, l4h); + + if (ndp(c, (struct icmp6hdr *)l4h, saddr, pkt)) continue; tap_packet_debug(NULL, ip6h, NULL, proto, NULL, 1); - packet_add(pkt, l4_len, l4h); icmp_tap_handler(c, PIF_TAP, AF_INET6, saddr, daddr, pkt, now); continue; } - if (l4_len < sizeof(*uh)) + if (l4len < sizeof(*uh)) continue; uh = (struct udphdr *)l4h; if (proto == IPPROTO_UDP) { PACKET_POOL_P(pkt, 1, in->buf, in->buf_size); - packet_add(pkt, l4_len, l4h); + packet_add(pkt, l4len, l4h); if (dhcpv6(c, pkt, saddr, daddr)) continue; @@ -953,18 +863,19 @@ resume: } #define L4_MATCH(ip6h, proto, uh, seq) \ - (seq->protocol == proto && \ - seq->source == uh->source && seq->dest == uh->dest && \ - IN6_ARE_ADDR_EQUAL(&seq->saddr, saddr) && \ - IN6_ARE_ADDR_EQUAL(&seq->daddr, daddr)) + ((seq)->protocol == (proto) && \ + (seq)->source == (uh)->source && \ + (seq)->dest == (uh)->dest && \ + IN6_ARE_ADDR_EQUAL(&(seq)->saddr, saddr) && \ + IN6_ARE_ADDR_EQUAL(&(seq)->daddr, daddr)) #define L4_SET(ip6h, proto, uh, seq) \ do { \ - seq->protocol = proto; \ - seq->source = uh->source; \ - seq->dest = uh->dest; \ - seq->saddr = *saddr; \ - seq->daddr = *daddr; \ + (seq)->protocol = (proto); \ + (seq)->source = (uh)->source; \ + (seq)->dest = (uh)->dest; \ + (seq)->saddr = *saddr; \ + (seq)->daddr = *daddr; \ } while (0) if (seq && L4_MATCH(ip6h, proto, uh, seq) && @@ -992,7 +903,7 @@ resume: #undef L4_SET append: - packet_add((struct pool *)&seq->p, l4_len, l4h); + packet_add((struct pool *)&seq->p, l4len, l4h); } for (j = 0, seq = tap6_l4; j < seq_count; j++, seq++) { @@ -1025,39 +936,52 @@ append: return in->count; } -void pool_flush_all(void) +/** + * tap_flush_pools() - Flush both IPv4 and IPv6 packet pools + */ +void tap_flush_pools(void) { pool_flush(pool_tap4); pool_flush(pool_tap6); } -void tap_handler_all(struct ctx *c, const struct timespec *now) +/** + * tap_handler() - IPv4/IPv6 and ARP packet handler for tap file descriptor + * @c: Execution context + * @now: Current timestamp + */ +void tap_handler(struct ctx *c, const struct timespec *now) { tap4_handler(c, pool_tap4, now); tap6_handler(c, pool_tap6, now); } -void packet_add_all_do(struct ctx *c, ssize_t len, char *p, - const char *func, int line) +/** + * tap_add_packet() - Queue/capture packet, update notion of guest MAC address + * @c: Execution context + * @l2len: Total L2 packet length + * @p: Packet buffer + */ +void tap_add_packet(struct ctx *c, ssize_t l2len, char *p) { const struct ethhdr *eh; - pcap(p, len); + pcap(p, l2len); eh = (struct ethhdr *)p; - if (memcmp(c->mac_guest, eh->h_source, ETH_ALEN)) { - memcpy(c->mac_guest, eh->h_source, ETH_ALEN); - proto_update_l2_buf(c->mac_guest, NULL); + if (memcmp(c->guest_mac, eh->h_source, ETH_ALEN)) { + memcpy(c->guest_mac, eh->h_source, ETH_ALEN); + proto_update_l2_buf(c->guest_mac, NULL); } switch (ntohs(eh->h_proto)) { case ETH_P_ARP: case ETH_P_IP: - packet_add_do(pool_tap4, len, p, func, line); + packet_add(pool_tap4, l2len, p); break; case ETH_P_IPV6: - packet_add_do(pool_tap6, len, p, func, line); + packet_add(pool_tap6, l2len, p); break; default: break; @@ -1070,174 +994,194 @@ void packet_add_all_do(struct ctx *c, ssize_t len, char *p, */ void tap_sock_reset(struct ctx *c) { - if (c->one_off) { - info("Client closed connection, exiting"); + info("Client connection closed%s", c->one_off ? ", exiting" : ""); + + if (c->one_off) exit(EXIT_SUCCESS); - } /* Close the connected socket, wait for a new connection */ epoll_ctl(c->epollfd, EPOLL_CTL_DEL, c->fd_tap, NULL); close(c->fd_tap); c->fd_tap = -1; + if (c->mode == MODE_VU) + vu_cleanup(c->vdev); } /** - * tap_handler_passt() - Packet handler for AF_UNIX file descriptor + * tap_passt_input() - Handler for new data on the socket to qemu * @c: Execution context - * @events: epoll events * @now: Current timestamp */ -void tap_handler_passt(struct ctx *c, uint32_t events, - const struct timespec *now) +static void tap_passt_input(struct ctx *c, const struct timespec *now) { - ssize_t n, rem; + static const char *partial_frame; + static ssize_t partial_len = 0; + ssize_t n; char *p; - if (events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)) { - tap_sock_reset(c); - return; - } + tap_flush_pools(); -redo: - p = pkt_buf; - rem = 0; + if (partial_len) { + /* We have a partial frame from an earlier pass. Move it to the + * start of the buffer, top up with new data, then process all + * of it. + */ + memmove(pkt_buf, partial_frame, partial_len); + } - pool_flush_all(); + do { + n = recv(c->fd_tap, pkt_buf + partial_len, + TAP_BUF_BYTES - partial_len, MSG_DONTWAIT); + } while ((n < 0) && errno == EINTR); - n = recv(c->fd_tap, p, TAP_BUF_FILL, MSG_DONTWAIT); if (n < 0) { - if (errno != EINTR && errno != EAGAIN && errno != EWOULDBLOCK) + if (errno != EAGAIN && errno != EWOULDBLOCK) { + err_perror("Receive error on guest connection, reset"); tap_sock_reset(c); + } return; } - while (n > (ssize_t)sizeof(uint32_t)) { - ssize_t len = ntohl(*(uint32_t *)p); + p = pkt_buf; + n += partial_len; - p += sizeof(uint32_t); - n -= sizeof(uint32_t); + while (n >= (ssize_t)sizeof(uint32_t)) { + uint32_t l2len = ntohl_unaligned(p); - /* At most one packet might not fit in a single read, and this - * needs to be blocking. - */ - if (len > n) { - rem = recv(c->fd_tap, p + n, len - n, 0); - if ((n += rem) != len) - return; + if (l2len < sizeof(struct ethhdr) || l2len > ETH_MAX_MTU) { + err("Bad frame size from guest, resetting connection"); + tap_sock_reset(c); + return; } - /* Complete the partial read above before discarding a malformed - * frame, otherwise the stream will be inconsistent. - */ - if (len < (ssize_t)sizeof(struct ethhdr) || - len > (ssize_t)ETH_MAX_MTU) - goto next; + if (l2len + sizeof(uint32_t) > (size_t)n) + /* Leave this incomplete frame for later */ + break; - packet_add_all(c, len, p); + p += sizeof(uint32_t); + n -= sizeof(uint32_t); -next: - p += len; - n -= len; + tap_add_packet(c, l2len, p); + + p += l2len; + n -= l2len; } - tap_handler_all(c, now); + partial_len = n; + partial_frame = p; - /* We can't use EPOLLET otherwise. */ - if (rem) - goto redo; + tap_handler(c, now); } /** - * tap_handler_pasta() - Packet handler for /dev/net/tun file descriptor + * tap_handler_passt() - Event handler for AF_UNIX file descriptor * @c: Execution context * @events: epoll events * @now: Current timestamp */ -void tap_handler_pasta(struct ctx *c, uint32_t events, +void tap_handler_passt(struct ctx *c, uint32_t events, const struct timespec *now) { - ssize_t n, len; - int ret; - - if (events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)) - die("Disconnect event on /dev/net/tun device, exiting"); + if (events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)) { + tap_sock_reset(c); + return; + } -redo: - n = 0; + if (events & EPOLLIN) + tap_passt_input(c, now); +} - pool_flush_all(); -restart: - while ((len = read(c->fd_tap, pkt_buf + n, TAP_BUF_BYTES - n)) > 0) { +/** + * tap_pasta_input() - Handler for new data on the socket to hypervisor + * @c: Execution context + * @now: Current timestamp + */ +static void tap_pasta_input(struct ctx *c, const struct timespec *now) +{ + ssize_t n, len; - if (len < (ssize_t)sizeof(struct ethhdr) || - len > (ssize_t)ETH_MAX_MTU) { - n += len; - continue; - } + tap_flush_pools(); + for (n = 0; n <= (ssize_t)(TAP_BUF_BYTES - ETH_MAX_MTU); n += len) { + len = read(c->fd_tap, pkt_buf + n, ETH_MAX_MTU); - packet_add_all(c, len, pkt_buf + n); + if (len == 0) { + die("EOF on tap device, exiting"); + } else if (len < 0) { + if (errno == EINTR) { + len = 0; + continue; + } - if ((n += len) == TAP_BUF_BYTES) - break; - } + if (errno == EAGAIN && errno == EWOULDBLOCK) + break; /* all done for now */ - if (len < 0 && errno == EINTR) - goto restart; + die("Error on tap device, exiting"); + } - ret = errno; + /* Ignore frames of bad length */ + if (len < (ssize_t)sizeof(struct ethhdr) || + len > (ssize_t)ETH_MAX_MTU) + continue; - tap_handler_all(c, now); + tap_add_packet(c, len, pkt_buf + n); + } - if (len > 0 || ret == EAGAIN) - return; + tap_handler(c, now); +} - if (n == TAP_BUF_BYTES) - goto redo; +/** + * tap_handler_pasta() - Packet handler for /dev/net/tun file descriptor + * @c: Execution context + * @events: epoll events + * @now: Current timestamp + */ +void tap_handler_pasta(struct ctx *c, uint32_t events, + const struct timespec *now) +{ + if (events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)) + die("Disconnect event on /dev/net/tun device, exiting"); - die("Error on tap device, exiting"); + if (events & EPOLLIN) + tap_pasta_input(c, now); } /** - * tap_sock_unix_init() - Create and bind AF_UNIX socket, listen for connection - * @c: Execution context + * tap_sock_unix_open() - Create and bind AF_UNIX socket + * @sock_path: Socket path. If empty, set on return (UNIX_SOCK_PATH as prefix) + * + * Return: socket descriptor on success, won't return on failure */ -static void tap_sock_unix_init(struct ctx *c) +int tap_sock_unix_open(char *sock_path) { - int fd = socket(AF_UNIX, SOCK_STREAM, 0); - union epoll_ref ref = { .type = EPOLL_TYPE_TAP_LISTEN }; - struct epoll_event ev = { 0 }; + int fd = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); struct sockaddr_un addr = { .sun_family = AF_UNIX, }; int i; if (fd < 0) - die("UNIX socket: %s", strerror(errno)); - - /* In passt mode, we don't know the guest's MAC until it sends - * us packets. Use the broadcast address so our first packets - * will reach it. - */ - memset(&c->mac_guest, 0xff, sizeof(c->mac_guest)); + die_perror("Failed to open UNIX domain socket"); for (i = 1; i < UNIX_SOCK_MAX; i++) { char *path = addr.sun_path; int ex, ret; - if (*c->sock_path) - memcpy(path, c->sock_path, UNIX_PATH_MAX); - else - snprintf(path, UNIX_PATH_MAX - 1, UNIX_SOCK_PATH, i); + if (*sock_path) + memcpy(path, sock_path, UNIX_PATH_MAX); + else if (snprintf_check(path, UNIX_PATH_MAX - 1, + UNIX_SOCK_PATH, i)) + die_perror("Can't build UNIX domain socket path"); - ex = socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0); + ex = socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, + 0); if (ex < 0) - die("UNIX domain socket check: %s", strerror(errno)); + die_perror("Failed to check for UNIX domain conflicts"); ret = connect(ex, (const struct sockaddr *)&addr, sizeof(addr)); if (!ret || (errno != ENOENT && errno != ECONNREFUSED && errno != EACCES)) { - if (*c->sock_path) + if (*sock_path) die("Socket path %s already in use", path); close(ex); @@ -1246,45 +1190,75 @@ static void tap_sock_unix_init(struct ctx *c) close(ex); unlink(path); - if (!bind(fd, (const struct sockaddr *)&addr, sizeof(addr)) || - *c->sock_path) + ret = bind(fd, (const struct sockaddr *)&addr, sizeof(addr)); + if (*sock_path && ret) + die_perror("Failed to bind UNIX domain socket"); + + if (!ret) break; } if (i == UNIX_SOCK_MAX) - die("UNIX socket bind: %s", strerror(errno)); - - info("UNIX domain socket bound at %s\n", addr.sun_path); + die_perror("Failed to bind UNIX domain socket"); - listen(fd, 0); + info("UNIX domain socket bound at %s", addr.sun_path); + if (!*sock_path) + memcpy(sock_path, addr.sun_path, UNIX_PATH_MAX); - ref.fd = c->fd_tap_listen = fd; - ev.events = EPOLLIN | EPOLLET; - ev.data.u64 = ref.u64; - epoll_ctl(c->epollfd, EPOLL_CTL_ADD, c->fd_tap_listen, &ev); + return fd; +} - if (c->mode == MODE_VU) { - info("You can start qemu with:"); - info(" kvm ... -chardev socket,id=chr0,path=%s -netdev vhost-user,id=netdev0,chardev=chr0 -device virtio-net,netdev=netdev0 -object memory-backend-memfd,id=memfd0,share=on,size=$RAMSIZE -numa node,memdev=memfd0\n", - addr.sun_path); - } else { - info("You can now start qemu (>= 7.2, with commit 13c6be96618c):"); +/** + * tap_backend_show_hints() - Give help information to start QEMU + * @c: Execution context + */ +static void tap_backend_show_hints(struct ctx *c) +{ + switch (c->mode) { + case MODE_PASTA: + /* No hints */ + break; + case MODE_PASST: + info("\nYou can now start qemu (>= 7.2, with commit 13c6be96618c):"); info(" kvm ... -device virtio-net-pci,netdev=s -netdev stream,id=s,server=off,addr.type=unix,addr.path=%s", - addr.sun_path); + c->sock_path); info("or qrap, for earlier qemu versions:"); info(" ./qrap 5 kvm ... -net socket,fd=5 -net nic,model=virtio"); + break; + case MODE_VU: + info("You can start qemu with:"); + info(" kvm ... -chardev socket,id=chr0,path=%s -netdev vhost-user,id=netdev0,chardev=chr0 -device virtio-net,netdev=netdev0 -object memory-backend-memfd,id=memfd0,share=on,size=$RAMSIZE -numa node,memdev=memfd0\n", + c->sock_path); + break; } } /** + * tap_sock_unix_init() - Start listening for connections on AF_UNIX socket + * @c: Execution context + */ +static void tap_sock_unix_init(const struct ctx *c) +{ + union epoll_ref ref = { .type = EPOLL_TYPE_TAP_LISTEN }; + struct epoll_event ev = { 0 }; + + listen(c->fd_tap_listen, 0); + + ref.fd = c->fd_tap_listen; + ev.events = EPOLLIN | EPOLLET; + ev.data.u64 = ref.u64; + epoll_ctl(c->epollfd, EPOLL_CTL_ADD, c->fd_tap_listen, &ev); +} + +/** * tap_listen_handler() - Handle new connection on listening socket * @c: Execution context * @events: epoll events */ void tap_listen_handler(struct ctx *c, uint32_t events) { - union epoll_ref ref; struct epoll_event ev = { 0 }; + union epoll_ref ref = { 0 }; int v = INT_MAX / 2; struct ucred ucred; socklen_t len; @@ -1324,13 +1298,11 @@ void tap_listen_handler(struct ctx *c, uint32_t events) trace("tap: failed to set SO_SNDBUF to %i", v); ref.fd = c->fd_tap; - if (c->mode == MODE_VU) { + if (c->mode == MODE_VU) ref.type = EPOLL_TYPE_VHOST_CMD; - ev.events = EPOLLIN | EPOLLRDHUP; - } else { + else ref.type = EPOLL_TYPE_TAP_PASST; - ev.events = EPOLLIN | EPOLLRDHUP | EPOLLET; - } + ev.events = EPOLLIN | EPOLLRDHUP; ev.data.u64 = ref.u64; epoll_ctl(c->epollfd, EPOLL_CTL_ADD, c->fd_tap, &ev); } @@ -1356,11 +1328,11 @@ static int tap_ns_tun(void *arg) fd = open("/dev/net/tun", flags); if (fd < 0) - die("Failed to open() /dev/net/tun: %s", strerror(errno)); + die_perror("Failed to open() /dev/net/tun"); - rc = ioctl(fd, TUNSETIFF, &ifr); + rc = ioctl(fd, (int)TUNSETIFF, &ifr); if (rc < 0) - die("TUNSETIFF failed: %s", strerror(errno)); + die_perror("TUNSETIFF ioctl on /dev/net/tun failed"); if (!(c->pasta_ifi = if_nametoindex(c->pasta_ifn))) die("Tap device opened but no network interface found"); @@ -1391,51 +1363,35 @@ static void tap_sock_tun_init(struct ctx *c) epoll_ctl(c->epollfd, EPOLL_CTL_ADD, c->fd_tap, &ev); } -void tap_sock_update_buf(void *base, size_t size) +/** + * tap_sock_update_pool() - Set the buffer base and size for the pool of packets + * @base: Buffer base + * @size Buffer size + */ +void tap_sock_update_pool(void *base, size_t size) { int i; - pool_tap4_storage.buf = base; - pool_tap4_storage.buf_size = size; - pool_tap6_storage.buf = base; - pool_tap6_storage.buf_size = size; + pool_tap4_storage = PACKET_INIT(pool_tap4, TAP_MSGS, base, size); + pool_tap6_storage = PACKET_INIT(pool_tap6, TAP_MSGS, base, size); for (i = 0; i < TAP_SEQS; i++) { - tap4_l4[i].p.buf = base; - tap4_l4[i].p.buf_size = size; - tap6_l4[i].p.buf = base; - tap6_l4[i].p.buf_size = size; + tap4_l4[i].p = PACKET_INIT(pool_l4, UIO_MAXIOV, base, size); + tap6_l4[i].p = PACKET_INIT(pool_l4, UIO_MAXIOV, base, size); } } /** - * tap_sock_init() - Create and set up AF_UNIX socket or tuntap file descriptor + * tap_backend_init() - Create and set up AF_UNIX socket or + * tuntap file descriptor * @c: Execution context */ -void tap_sock_init(struct ctx *c) +void tap_backend_init(struct ctx *c) { - size_t sz = sizeof(pkt_buf); - int i; - - pool_tap4_storage = PACKET_INIT(pool_tap4, TAP_MSGS, pkt_buf, sz); - pool_tap6_storage = PACKET_INIT(pool_tap6, TAP_MSGS, pkt_buf, sz); - if (c->mode == MODE_VU) { - pool_tap4_storage.buf = NULL; - pool_tap4_storage.buf_size = 0; - pool_tap6_storage.buf = NULL; - pool_tap6_storage.buf_size = 0; - } - - for (i = 0; i < TAP_SEQS; i++) { - tap4_l4[i].p = PACKET_INIT(pool_l4, UIO_MAXIOV, pkt_buf, sz); - tap6_l4[i].p = PACKET_INIT(pool_l4, UIO_MAXIOV, pkt_buf, sz); - if (c->mode == MODE_VU) { - tap4_l4[i].p.buf = NULL; - tap4_l4[i].p.buf_size = 0; - tap6_l4[i].p.buf = NULL; - tap6_l4[i].p.buf_size = 0; - } - } + if (c->mode == MODE_VU) + tap_sock_update_pool(NULL, 0); + else + tap_sock_update_pool(pkt_buf, sizeof(pkt_buf)); if (c->fd_tap != -1) { /* Passed as --fd */ struct epoll_event ev = { 0 }; @@ -1446,27 +1402,38 @@ void tap_sock_init(struct ctx *c) switch (c->mode) { case MODE_PASST: ref.type = EPOLL_TYPE_TAP_PASST; - ev.events = EPOLLIN | EPOLLET | EPOLLRDHUP; break; case MODE_PASTA: ref.type = EPOLL_TYPE_TAP_PASTA; - ev.events = EPOLLIN | EPOLLET | EPOLLRDHUP; break; case MODE_VU: ref.type = EPOLL_TYPE_VHOST_CMD; - ev.events = EPOLLIN | EPOLLRDHUP; break; } + ev.events = EPOLLIN | EPOLLRDHUP; ev.data.u64 = ref.u64; epoll_ctl(c->epollfd, EPOLL_CTL_ADD, c->fd_tap, &ev); return; } - if (c->mode == MODE_PASTA) { + switch (c->mode) { + case MODE_PASTA: tap_sock_tun_init(c); - } else { - if (c->fd_tap_listen == -1) - tap_sock_unix_init(c); + break; + case MODE_VU: + vu_init(c); + /* fall through */ + case MODE_PASST: + tap_sock_unix_init(c); + + /* In passt mode, we don't know the guest's MAC address until it + * sends us packets. Use the broadcast address so that our + * first packets will reach it. + */ + memset(&c->guest_mac, 0xff, sizeof(c->guest_mac)); + break; } + + tap_backend_show_hints(c); } @@ -6,90 +6,61 @@ #ifndef TAP_H #define TAP_H -/* - * TCP frame iovec array: - * TCP_IOV_VNET vnet length - * TCP_IOV_ETH ethernet header - * TCP_IOV_IP IP (v4/v6) header - * TCP_IOV_PAYLOAD IP payload (TCP header + data) - * TCP_IOV_NUM is the number of entries in the iovec array - */ -#define TCP_IOV_VNET 0 -#define TCP_IOV_ETH 1 -#define TCP_IOV_IP 2 -#define TCP_IOV_PAYLOAD 3 -#define TCP_IOV_NUM 4 +#define ETH_HDR_INIT(proto) { .h_proto = htons_constant(proto) } /** - * struct tap_hdr - L2 and tap specific headers + * struct tap_hdr - tap backend specific headers * @vnet_len: Frame length (for qemu socket transport) - * @eh: Ethernet header */ struct tap_hdr { uint32_t vnet_len; - struct ethhdr eh; } __attribute__((packed)); -#define TAP_HDR_INIT(proto) { .eh.h_proto = htons_constant(proto) } - -static inline size_t tap_hdr_len_(const struct ctx *c) -{ - if (c->mode == MODE_PASST) - return sizeof(struct tap_hdr); - else - return sizeof(struct ethhdr); -} - /** - * tap_iov_base() - Find start of tap frame + * tap_hdr_iov() - struct iovec for a tap header * @c: Execution context - * @taph: Pointer to L2 header buffer + * @taph: Pointer to tap specific header buffer * - * Returns: pointer to the start of tap frame - suitable for an - * iov_base to be passed to tap_send_frames()) + * Returns: A struct iovec covering the correct portion of @taph to use as the + * tap specific header in the current configuration. */ -static inline void *tap_iov_base(const struct ctx *c, struct tap_hdr *taph) +static inline struct iovec tap_hdr_iov(const struct ctx *c, + struct tap_hdr *thdr) { - return (char *)(taph + 1) - tap_hdr_len_(c); + return (struct iovec){ + .iov_base = thdr, + .iov_len = c->mode == MODE_PASST ? sizeof(*thdr) : 0, + }; } /** - * tap_iov_len() - Finalize tap frame and return total length - * @c: Execution context - * @taph: Tap header to finalize - * @plen: L2 payload length (excludes L2 and tap specific headers) - * - * Returns: length of the tap frame including L2 and tap specific - * headers - suitable for an iov_len to be passed to - * tap_send_frames() + * tap_hdr_update() - Update the tap specific header for a frame + * @taph: Tap specific header buffer to update + * @l2len: Frame length (including L2 headers) */ -static inline size_t tap_iov_len(const struct ctx *c, struct tap_hdr *taph, - size_t plen) +static inline void tap_hdr_update(struct tap_hdr *thdr, size_t l2len) { - if (c->mode == MODE_PASST) - taph->vnet_len = htonl(plen + sizeof(taph->eh)); - return plen + tap_hdr_len_(c); + if (thdr) + thdr->vnet_len = htonl(l2len); } -struct in_addr tap_ip4_daddr(const struct ctx *c); void tap_udp4_send(const struct ctx *c, struct in_addr src, in_port_t sport, struct in_addr dst, in_port_t dport, - const void *in, size_t len); + const void *in, size_t dlen); void tap_icmp4_send(const struct ctx *c, struct in_addr src, struct in_addr dst, - const void *in, size_t len); + const void *in, size_t l4len); const struct in6_addr *tap_ip6_daddr(const struct ctx *c, const struct in6_addr *src); void tap_udp6_send(const struct ctx *c, const struct in6_addr *src, in_port_t sport, const struct in6_addr *dst, in_port_t dport, - uint32_t flow, const void *in, size_t len); + uint32_t flow, void *in, size_t dlen); void tap_icmp6_send(const struct ctx *c, const struct in6_addr *src, const struct in6_addr *dst, - const void *in, size_t len); -int tap_send(const struct ctx *c, const void *data, size_t len); -size_t tap_send_frames(const struct ctx *c, const struct iovec *iov, size_t n); -size_t tap_send_iov(const struct ctx *c, struct iovec iov[][TCP_IOV_NUM], - size_t n); + const void *in, size_t l4len); +void tap_send_single(const struct ctx *c, const void *data, size_t l2len); +size_t tap_send_frames(const struct ctx *c, const struct iovec *iov, + size_t bufs_per_frame, size_t nframes); void eth_update_mac(struct ethhdr *eh, const unsigned char *eth_d, const unsigned char *eth_s); void tap_listen_handler(struct ctx *c, uint32_t events); @@ -97,17 +68,12 @@ void tap_handler_pasta(struct ctx *c, uint32_t events, const struct timespec *now); void tap_handler_passt(struct ctx *c, uint32_t events, const struct timespec *now); +int tap_sock_unix_open(char *sock_path); void tap_sock_reset(struct ctx *c); -void tap_sock_update_buf(void *base, size_t size); -void tap_sock_init(struct ctx *c); -void pool_flush_all(void); -void tap_handler_all(struct ctx *c, const struct timespec *now); - -void packet_add_do(struct pool *p, size_t len, const char *start, - const char *func, int line); -void packet_add_all_do(struct ctx *c, ssize_t len, char *p, - const char *func, int line); -#define packet_add_all(p, len, start) \ - packet_add_all_do(p, len, start, __func__, __LINE__) +void tap_sock_update_pool(void *base, size_t size); +void tap_backend_init(struct ctx *c); +void tap_flush_pools(void); +void tap_handler(struct ctx *c, const struct timespec *now); +void tap_add_packet(struct ctx *c, ssize_t l2len, char *p); #endif /* TAP_H */ @@ -274,6 +274,7 @@ #include <net/if.h> #include <netinet/in.h> #include <netinet/ip.h> +#include <netinet/tcp.h> #include <stdint.h> #include <stdbool.h> #include <stddef.h> @@ -286,10 +287,9 @@ #include <time.h> #include <arpa/inet.h> -#include <linux/tcp.h> /* For struct tcp_info */ - #include "checksum.h" #include "util.h" +#include "iov.h" #include "ip.h" #include "passt.h" #include "tap.h" @@ -299,28 +299,16 @@ #include "log.h" #include "inany.h" #include "flow.h" +#include "linux_dep.h" #include "flow_table.h" #include "tcp_internal.h" #include "tcp_buf.h" #include "tcp_vu.h" -/* Sides of a flow as we use them in "tap" connections */ -#define SOCKSIDE 0 -#define TAPSIDE 1 - -#define TCP_HASH_TABLE_LOAD 70 /* % */ -#define TCP_HASH_TABLE_SIZE (FLOW_MAX * 100 / TCP_HASH_TABLE_LOAD) - /* MSS rounding: see SET_MSS() */ #define MSS_DEFAULT 536 - #define WINDOW_DEFAULT 14600 /* RFC 6928 */ -#ifdef HAS_SND_WND -# define KERNEL_REPORTS_SND_WND(c) (c->tcp.kernel_snd_wnd) -#else -# define KERNEL_REPORTS_SND_WND(c) (0 && (c)) -#endif #define ACK_INTERVAL 10 /* ms */ #define SYN_TIMEOUT 10 /* s */ @@ -331,18 +319,12 @@ #define LOW_RTT_TABLE_SIZE 8 #define LOW_RTT_THRESHOLD 10 /* us */ -/* We need to include <linux/tcp.h> for tcpi_bytes_acked, instead of - * <netinet/tcp.h>, but that doesn't include a definition for SOL_TCP - */ -#define SOL_TCP IPPROTO_TCP - -#define ACK_IF_NEEDED 0 /* See tcp_buf_send_flag() */ - +#define ACK_IF_NEEDED 0 /* See tcp_send_flag() */ #define CONN_IS_CLOSING(conn) \ - ((conn->events & ESTABLISHED) && \ - (conn->events & (SOCK_FIN_RCVD | TAP_FIN_RCVD))) -#define CONN_HAS(conn, set) ((conn->events & (set)) == (set)) + (((conn)->events & ESTABLISHED) && \ + ((conn)->events & (SOCK_FIN_RCVD | TAP_FIN_RCVD))) +#define CONN_HAS(conn, set) (((conn)->events & (set)) == (set)) static const char *tcp_event_str[] __attribute((__unused__)) = { "SOCK_ACCEPTED", "TAP_SYN_RCVD", "ESTABLISHED", "TAP_SYN_ACK_SENT", @@ -370,29 +352,75 @@ static const char *tcp_flag_str[] __attribute((__unused__)) = { static int tcp_sock_init_ext [NUM_PORTS][IP_VERSIONS]; static int tcp_sock_ns [NUM_PORTS][IP_VERSIONS]; -/* Table of guest side forwarding addresses with very low RTT (assumed - * to be local to the host), LRU +/* Table of our guest side addresses with very low RTT (assumed to be local to + * the host), LRU */ static union inany_addr low_rtt_dst[LOW_RTT_TABLE_SIZE]; char tcp_buf_discard [MAX_WINDOW]; -/* sendmsg() to socket */ -static struct iovec tcp_iov [UIO_MAXIOV]; +/* Does the kernel support TCP_PEEK_OFF? */ +bool peek_offset_cap; -#define CONN(idx) (&(FLOW(idx)->tcp)) +/* Size of data returned by TCP_INFO getsockopt() */ +socklen_t tcp_info_size; -/* Table for lookup from remote address, local port, remote port */ -static flow_sidx_t tc_hash[TCP_HASH_TABLE_SIZE]; +#define tcp_info_cap(f_) \ + ((offsetof(struct tcp_info_linux, tcpi_##f_) + \ + sizeof(((struct tcp_info_linux *)NULL)->tcpi_##f_)) <= tcp_info_size) -static_assert(ARRAY_SIZE(tc_hash) >= FLOW_MAX, - "Safe linear probing requires hash table larger than connection table"); +/* Kernel reports sending window in TCP_INFO (kernel commit 8f7baad7f035) */ +#define snd_wnd_cap tcp_info_cap(snd_wnd) +/* Kernel reports bytes acked in TCP_INFO (kernel commit 0df48c26d84) */ +#define bytes_acked_cap tcp_info_cap(bytes_acked) +/* Kernel reports minimum RTT in TCP_INFO (kernel commit cd9b266095f4) */ +#define min_rtt_cap tcp_info_cap(min_rtt) + +/* sendmsg() to socket */ +static struct iovec tcp_iov [UIO_MAXIOV]; /* Pools for pre-opened sockets (in init) */ int init_sock_pool4 [TCP_SOCK_POOL_SIZE]; int init_sock_pool6 [TCP_SOCK_POOL_SIZE]; /** + * conn_at_sidx() - Get TCP connection specific flow at given sidx + * @sidx: Flow and side to retrieve + * + * Return: TCP connection at @sidx, or NULL of @sidx is invalid. Asserts if the + * flow at @sidx is not FLOW_TCP. + */ +static struct tcp_tap_conn *conn_at_sidx(flow_sidx_t sidx) +{ + union flow *flow = flow_at_sidx(sidx); + + if (!flow) + return NULL; + + ASSERT(flow->f.type == FLOW_TCP); + return &flow->tcp; +} + +/** + * tcp_set_peek_offset() - Set SO_PEEK_OFF offset on a socket if supported + * @s: Socket to update + * @offset: Offset in bytes + * + * Return: -1 when it fails, 0 otherwise. + */ +int tcp_set_peek_offset(int s, int offset) +{ + if (!peek_offset_cap) + return 0; + + if (setsockopt(s, SOL_SOCKET, SO_PEEK_OFF, &offset, sizeof(offset))) { + err("Failed to set SO_PEEK_OFF to %i in socket %i", offset, s); + return -1; + } + return 0; +} + +/** * tcp_conn_epoll_events() - epoll events mask for given connection state * @events: Current connection events * @conn_flags Connection flags @@ -417,7 +445,7 @@ static uint32_t tcp_conn_epoll_events(uint8_t events, uint8_t conn_flags) if (events == TAP_SYN_RCVD) return EPOLLOUT | EPOLLET | EPOLLRDHUP; - return EPOLLRDHUP; + return EPOLLET | EPOLLRDHUP; } /** @@ -431,7 +459,7 @@ static int tcp_epoll_ctl(const struct ctx *c, struct tcp_tap_conn *conn) { int m = conn->in_epoll ? EPOLL_CTL_MOD : EPOLL_CTL_ADD; union epoll_ref ref = { .type = EPOLL_TYPE_TCP, .fd = conn->sock, - .flowside = FLOW_SIDX(conn, SOCKSIDE) }; + .flowside = FLOW_SIDX(conn, !TAPSIDE(conn)), }; struct epoll_event ev = { .data.u64 = ref.u64 }; if (conn->events == CLOSED) { @@ -522,7 +550,8 @@ static void tcp_timer_ctl(const struct ctx *c, struct tcp_tap_conn *conn) (unsigned long long)it.it_value.tv_sec, (unsigned long long)it.it_value.tv_nsec / 1000 / 1000); - timerfd_settime(conn->timer, 0, &it, NULL); + if (timerfd_settime(conn->timer, 0, &it, NULL)) + flow_err(conn, "failed to set timer: %s", strerror(errno)); } /** @@ -573,9 +602,6 @@ void conn_flag_do(const struct ctx *c, struct tcp_tap_conn *conn, tcp_timer_ctl(c, conn); } -static void tcp_hash_remove(const struct ctx *c, - const struct tcp_tap_conn *conn); - /** * conn_event_do() - Set and log connection events, update epoll state * @c: Execution context @@ -621,7 +647,7 @@ void conn_event_do(const struct ctx *c, struct tcp_tap_conn *conn, num == -1 ? "CLOSED" : tcp_event_str[num]); if (event == CLOSED) - tcp_hash_remove(c, conn); + flow_hash_remove(c, TAP_SIDX(conn)); else if ((event == TAP_FIN_RCVD) && !(conn->events & SOCK_FIN_RCVD)) conn_flag(c, conn, ACTIVE_CLOSE); else @@ -639,10 +665,11 @@ void conn_event_do(const struct ctx *c, struct tcp_tap_conn *conn, */ static int tcp_rtt_dst_low(const struct tcp_tap_conn *conn) { + const struct flowside *tapside = TAPFLOW(conn); int i; for (i = 0; i < LOW_RTT_TABLE_SIZE; i++) - if (inany_equals(&conn->faddr, low_rtt_dst + i)) + if (inany_equals(&tapside->oaddr, low_rtt_dst + i)) return 1; return 0; @@ -654,17 +681,17 @@ static int tcp_rtt_dst_low(const struct tcp_tap_conn *conn) * @tinfo: Pointer to struct tcp_info for socket */ static void tcp_rtt_dst_check(const struct tcp_tap_conn *conn, - const struct tcp_info *tinfo) + const struct tcp_info_linux *tinfo) { -#ifdef HAS_MIN_RTT + const struct flowside *tapside = TAPFLOW(conn); int i, hole = -1; - if (!tinfo->tcpi_min_rtt || + if (!min_rtt_cap || (int)tinfo->tcpi_min_rtt > LOW_RTT_THRESHOLD) return; for (i = 0; i < LOW_RTT_TABLE_SIZE; i++) { - if (inany_equals(&conn->faddr, low_rtt_dst + i)) + if (inany_equals(&tapside->oaddr, low_rtt_dst + i)) return; if (hole == -1 && IN6_IS_ADDR_UNSPECIFIED(low_rtt_dst + i)) hole = i; @@ -676,14 +703,10 @@ static void tcp_rtt_dst_check(const struct tcp_tap_conn *conn, if (hole == -1) return; - low_rtt_dst[hole++] = conn->faddr; + low_rtt_dst[hole++] = tapside->oaddr; if (hole == LOW_RTT_TABLE_SIZE) hole = 0; inany_from_af(low_rtt_dst + hole, AF_INET6, &in6addr_any); -#else - (void)conn; - (void)tinfo; -#endif /* HAS_MIN_RTT */ } /** @@ -730,34 +753,106 @@ static void tcp_sock_set_bufsize(const struct ctx *c, int s) } /** - * tcp_update_check_tcp4() - Update TCP checksum from stored one + * tcp_update_check_tcp4() - Calculate TCP checksum for IPv4 * @iph: IPv4 header - * @th: TCP header followed by TCP payload + * @iov: Pointer to the array of IO vectors + * @iov_cnt: Length of the array + * @l4offset: IPv4 payload offset in the iovec array */ -static void tcp_update_check_tcp4(const struct iphdr *iph, struct tcphdr *th) +void tcp_update_check_tcp4(const struct iphdr *iph, + const struct iovec *iov, int iov_cnt, + size_t l4offset) { - uint16_t tlen = ntohs(iph->tot_len) - sizeof(struct iphdr); + uint16_t l4len = ntohs(iph->tot_len) - sizeof(struct iphdr); struct in_addr saddr = { .s_addr = iph->saddr }; struct in_addr daddr = { .s_addr = iph->daddr }; - uint32_t sum = proto_ipv4_header_psum(tlen, IPPROTO_TCP, saddr, daddr); + size_t check_ofs; + uint16_t *check; + int check_idx; + uint32_t sum; + char *ptr; + + sum = proto_ipv4_header_psum(l4len, IPPROTO_TCP, saddr, daddr); + + check_idx = iov_skip_bytes(iov, iov_cnt, + l4offset + offsetof(struct tcphdr, check), + &check_ofs); + + if (check_idx >= iov_cnt) { + err("TCP4 buffer is too small, iov size %zd, check offset %zd", + iov_size(iov, iov_cnt), + l4offset + offsetof(struct tcphdr, check)); + return; + } + + if (check_ofs + sizeof(*check) > iov[check_idx].iov_len) { + err("TCP4 checksum field memory is not contiguous " + "check_ofs %zd check_idx %d iov_len %zd", + check_ofs, check_idx, iov[check_idx].iov_len); + return; + } + + ptr = (char *)iov[check_idx].iov_base + check_ofs; + if ((uintptr_t)ptr & (__alignof__(*check) - 1)) { + err("TCP4 checksum field is not correctly aligned in memory"); + return; + } + + check = (uint16_t *)ptr; - th->check = 0; - th->check = csum(th, tlen, sum); + *check = 0; + *check = csum_iov(iov, iov_cnt, l4offset, sum); } /** * tcp_update_check_tcp6() - Calculate TCP checksum for IPv6 * @ip6h: IPv6 header - * @th: TCP header followed by TCP payload + * @iov: Pointer to the array of IO vectors + * @iov_cnt: Length of the array + * @l4offset: IPv6 payload offset in the iovec array */ -static void tcp_update_check_tcp6(struct ipv6hdr *ip6h, struct tcphdr *th) +void tcp_update_check_tcp6(const struct ipv6hdr *ip6h, + const struct iovec *iov, int iov_cnt, + size_t l4offset) { - uint16_t payload_len = ntohs(ip6h->payload_len); - uint32_t sum = proto_ipv6_header_psum(payload_len, IPPROTO_TCP, - &ip6h->saddr, &ip6h->daddr); + uint16_t l4len = ntohs(ip6h->payload_len); + size_t check_ofs; + uint16_t *check; + int check_idx; + uint32_t sum; + char *ptr; + + sum = proto_ipv6_header_psum(l4len, IPPROTO_TCP, &ip6h->saddr, + &ip6h->daddr); + + check_idx = iov_skip_bytes(iov, iov_cnt, + l4offset + offsetof(struct tcphdr, check), + &check_ofs); + + if (check_idx >= iov_cnt) { + err("TCP6 buffer is too small, iov size %zd, check offset %zd", + iov_size(iov, iov_cnt), + l4offset + offsetof(struct tcphdr, check)); + return; + } + + if (check_ofs + sizeof(*check) > iov[check_idx].iov_len) { + err("TCP6 checksum field memory is not contiguous " + "check_ofs %zd check_idx %d iov_len %zd", + check_ofs, check_idx, iov[check_idx].iov_len); + return; + } + + ptr = (char *)iov[check_idx].iov_base + check_ofs; + if ((uintptr_t)ptr & (__alignof__(*check) - 1)) { + err("TCP6 checksum field is not correctly aligned in memory"); + return; + } + + check = (uint16_t *)ptr; - th->check = 0; - th->check = csum(th, payload_len, sum); + *check = 0; + *check = csum_iov(iov, iov_cnt, l4offset, sum); } /** @@ -819,163 +914,14 @@ static int tcp_opt_get(const char *opts, size_t len, uint8_t type_find, } /** - * tcp_hash_match() - Check if a connection entry matches address and ports - * @conn: Connection entry to match against - * @faddr: Guest side forwarding address - * @eport: Guest side endpoint port - * @fport: Guest side forwarding port - * - * Return: 1 on match, 0 otherwise - */ -static int tcp_hash_match(const struct tcp_tap_conn *conn, - const union inany_addr *faddr, - in_port_t eport, in_port_t fport) -{ - if (inany_equals(&conn->faddr, faddr) && - conn->eport == eport && conn->fport == fport) - return 1; - - return 0; -} - -/** - * tcp_hash() - Calculate hash value for connection given address and ports - * @c: Execution context - * @faddr: Guest side forwarding address - * @eport: Guest side endpoint port - * @fport: Guest side forwarding port - * - * Return: hash value, needs to be adjusted for table size - */ -static uint64_t tcp_hash(const struct ctx *c, const union inany_addr *faddr, - in_port_t eport, in_port_t fport) -{ - struct siphash_state state = SIPHASH_INIT(c->hash_secret); - - inany_siphash_feed(&state, faddr); - return siphash_final(&state, 20, (uint64_t)eport << 16 | fport); -} - -/** - * tcp_conn_hash() - Calculate hash bucket of an existing connection - * @c: Execution context - * @conn: Connection - * - * Return: hash value, needs to be adjusted for table size - */ -static uint64_t tcp_conn_hash(const struct ctx *c, - const struct tcp_tap_conn *conn) -{ - return tcp_hash(c, &conn->faddr, conn->eport, conn->fport); -} - -/** - * tcp_hash_probe() - Find hash bucket for a connection - * @c: Execution context - * @conn: Connection to find bucket for - * - * Return: If @conn is in the table, its current bucket, otherwise a suitable - * free bucket for it. - */ -static inline unsigned tcp_hash_probe(const struct ctx *c, - const struct tcp_tap_conn *conn) -{ - flow_sidx_t sidx = FLOW_SIDX(conn, TAPSIDE); - unsigned b = tcp_conn_hash(c, conn) % TCP_HASH_TABLE_SIZE; - - /* Linear probing */ - while (!flow_sidx_eq(tc_hash[b], FLOW_SIDX_NONE) && - !flow_sidx_eq(tc_hash[b], sidx)) - b = mod_sub(b, 1, TCP_HASH_TABLE_SIZE); - - return b; -} - -/** - * tcp_hash_insert() - Insert connection into hash table, chain link - * @c: Execution context - * @conn: Connection pointer - */ -static void tcp_hash_insert(const struct ctx *c, struct tcp_tap_conn *conn) -{ - unsigned b = tcp_hash_probe(c, conn); - - tc_hash[b] = FLOW_SIDX(conn, TAPSIDE); - flow_dbg(conn, "hash table insert: sock %i, bucket: %u", conn->sock, b); -} - -/** - * tcp_hash_remove() - Drop connection from hash table, chain unlink - * @c: Execution context - * @conn: Connection pointer - */ -static void tcp_hash_remove(const struct ctx *c, - const struct tcp_tap_conn *conn) -{ - unsigned b = tcp_hash_probe(c, conn), s; - union flow *flow = flow_at_sidx(tc_hash[b]); - - if (!flow) - return; /* Redundant remove */ - - flow_dbg(conn, "hash table remove: sock %i, bucket: %u", conn->sock, b); - - /* Scan the remainder of the cluster */ - for (s = mod_sub(b, 1, TCP_HASH_TABLE_SIZE); - (flow = flow_at_sidx(tc_hash[s])); - s = mod_sub(s, 1, TCP_HASH_TABLE_SIZE)) { - unsigned h = tcp_conn_hash(c, &flow->tcp) % TCP_HASH_TABLE_SIZE; - - if (!mod_between(h, s, b, TCP_HASH_TABLE_SIZE)) { - /* tc_hash[s] can live in tc_hash[b]'s slot */ - debug("hash table remove: shuffle %u -> %u", s, b); - tc_hash[b] = tc_hash[s]; - b = s; - } - } - - tc_hash[b] = FLOW_SIDX_NONE; -} - -/** - * tcp_hash_lookup() - Look up connection given remote address and ports - * @c: Execution context - * @af: Address family, AF_INET or AF_INET6 - * @faddr: Guest side forwarding address (guest remote address) - * @eport: Guest side endpoint port (guest local port) - * @fport: Guest side forwarding port (guest remote port) - * - * Return: connection pointer, if found, -ENOENT otherwise - */ -static struct tcp_tap_conn *tcp_hash_lookup(const struct ctx *c, - sa_family_t af, const void *faddr, - in_port_t eport, in_port_t fport) -{ - union inany_addr aany; - union flow *flow; - unsigned b; - - inany_from_af(&aany, af, faddr); - - b = tcp_hash(c, &aany, eport, fport) % TCP_HASH_TABLE_SIZE; - while ((flow = flow_at_sidx(tc_hash[b])) && - !tcp_hash_match(&flow->tcp, &aany, eport, fport)) - b = mod_sub(b, 1, TCP_HASH_TABLE_SIZE); - - return &flow->tcp; -} - -/** * tcp_flow_defer() - Deferred per-flow handling (clean up closed connections) - * @flow: Flow table entry for this connection + * @conn: Connection to handle * - * Return: true if the flow is ready to free, false otherwise + * Return: true if the connection is ready to free, false otherwise */ -bool tcp_flow_defer(union flow *flow) +bool tcp_flow_defer(const struct tcp_tap_conn *conn) { - const struct tcp_tap_conn *conn = &flow->tcp; - - if (flow->tcp.events != CLOSED) + if (conn->events != CLOSED) return false; close(conn->sock); @@ -992,8 +938,7 @@ bool tcp_flow_defer(union flow *flow) /* cppcheck-suppress [constParameterPointer, unmatchedSuppression] */ void tcp_defer_handler(struct ctx *c) { - tcp_buf_l2_flags_flush(c); - tcp_buf_l2_data_flush(c); + tcp_payload_flush(c); } /** @@ -1004,10 +949,12 @@ void tcp_defer_handler(struct ctx *c) * @seq: Sequence number */ static void tcp_fill_header(struct tcphdr *th, - const struct tcp_tap_conn *conn, uint32_t seq) + const struct tcp_tap_conn *conn, uint32_t seq) { - th->source = htons(conn->fport); - th->dest = htons(conn->eport); + const struct flowside *tapside = TAPFLOW(conn); + + th->source = htons(tapside->oport); + th->dest = htons(tapside->eport); th->seq = htonl(seq); th->ack_seq = htonl(conn->seq_ack_to_tap); if (conn->events & ESTABLISHED) { @@ -1021,68 +968,73 @@ static void tcp_fill_header(struct tcphdr *th, /** * tcp_fill_headers4() - Fill 802.3, IPv4, TCP headers in pre-cooked buffers - * @c: Execution context - * @conn: Connection pointer - * @iph: Pointer to IPv4 header - * @th: Pointer to TCP header - * @plen: Payload length (including TCP header options) - * @check: Checksum, if already known - * @seq: Sequence number for this segment - * - * Return: The total length of the IPv4 packet, host order - */ -size_t tcp_fill_headers4(const struct ctx *c, - const struct tcp_tap_conn *conn, - struct iphdr *iph, struct tcphdr *th, - size_t plen, const uint16_t *check, - uint32_t seq) + * @conn: Connection pointer + * @taph: tap backend specific header + * @iph: Pointer to IPv4 header + * @bp: Pointer to TCP header followed by TCP payload + * @dlen: TCP payload length + * @check: Checksum, if already known + * @seq: Sequence number for this segment + * @no_tcp_csum: Do not set TCP checksum + */ +void tcp_fill_headers4(const struct tcp_tap_conn *conn, + struct tap_hdr *taph, struct iphdr *iph, + struct tcp_payload_t *bp, size_t dlen, + const uint16_t *check, uint32_t seq, bool no_tcp_csum) { - size_t ip_len = plen + sizeof(struct iphdr) + sizeof(struct tcphdr); - const struct in_addr *a4 = inany_v4(&conn->faddr); + const struct flowside *tapside = TAPFLOW(conn); + const struct in_addr *src4 = inany_v4(&tapside->oaddr); + const struct in_addr *dst4 = inany_v4(&tapside->eaddr); + size_t l4len = dlen + sizeof(bp->th); + size_t l3len = l4len + sizeof(*iph); - ASSERT(a4); + ASSERT(src4 && dst4); - iph->tot_len = htons(ip_len); - iph->saddr = a4->s_addr; - iph->daddr = c->ip4.addr_seen.s_addr; + iph->tot_len = htons(l3len); + iph->saddr = src4->s_addr; + iph->daddr = dst4->s_addr; iph->check = check ? *check : - csum_ip4_header(iph->tot_len, IPPROTO_TCP, - *a4, c->ip4.addr_seen); + csum_ip4_header(l3len, IPPROTO_TCP, *src4, *dst4); + + tcp_fill_header(&bp->th, conn, seq); - tcp_fill_header(th, conn, seq); + if (no_tcp_csum) { + bp->th.check = 0; + } else { + const struct iovec iov = { + .iov_base = bp, + .iov_len = ntohs(iph->tot_len) - sizeof(struct iphdr), + }; - if (c->mode != MODE_VU) - tcp_update_check_tcp4(iph, th); + tcp_update_check_tcp4(iph, &iov, 1, 0); + } - return ip_len; + tap_hdr_update(taph, l3len + sizeof(struct ethhdr)); } /** * tcp_fill_headers6() - Fill 802.3, IPv6, TCP headers in pre-cooked buffers - * @c: Execution context - * @conn: Connection pointer - * @ip6h: Pointer to IPv6 header - * @th: Pointer to TCP header - * @plen: Payload length (including TCP header options) - * @check: Checksum, if already known - * @seq: Sequence number for this segment - * - * Return: The total length of the IPv6 packet, host order - */ -size_t tcp_fill_headers6(const struct ctx *c, - const struct tcp_tap_conn *conn, - struct ipv6hdr *ip6h, struct tcphdr *th, - size_t plen, uint32_t seq) + * @conn: Connection pointer + * @taph: tap backend specific header + * @ip6h: Pointer to IPv6 header + * @bp: Pointer to TCP header followed by TCP payload + * @dlen: TCP payload length + * @check: Checksum, if already known + * @seq: Sequence number for this segment + * @no_tcp_csum: Do not set TCP checksum + */ +void tcp_fill_headers6(const struct tcp_tap_conn *conn, + struct tap_hdr *taph, struct ipv6hdr *ip6h, + struct tcp_payload_t *bp, size_t dlen, + uint32_t seq, bool no_tcp_csum) { - size_t ip_len = plen + sizeof(struct ipv6hdr) + sizeof(struct tcphdr); + const struct flowside *tapside = TAPFLOW(conn); + size_t l4len = dlen + sizeof(bp->th); - ip6h->payload_len = htons(plen + sizeof(struct tcphdr)); - ip6h->saddr = conn->faddr.a6; - if (IN6_IS_ADDR_LINKLOCAL(&ip6h->saddr)) - ip6h->daddr = c->ip6.addr_ll_seen; - else - ip6h->daddr = c->ip6.addr_seen; + ip6h->payload_len = htons(l4len); + ip6h->saddr = tapside->oaddr.a6; + ip6h->daddr = tapside->eaddr.a6; ip6h->hop_limit = 255; ip6h->version = 6; @@ -1092,12 +1044,20 @@ size_t tcp_fill_headers6(const struct ctx *c, ip6h->flow_lbl[1] = (conn->sock >> 8) & 0xff; ip6h->flow_lbl[2] = (conn->sock >> 0) & 0xff; - tcp_fill_header(th, conn, seq); + tcp_fill_header(&bp->th, conn, seq); + + if (no_tcp_csum) { + bp->th.check = 0; + } else { + const struct iovec iov = { + .iov_base = bp, + .iov_len = ntohs(ip6h->payload_len) + }; - if (c->mode != MODE_VU) - tcp_update_check_tcp6(ip6h, th); + tcp_update_check_tcp6(ip6h, &iov, 1, 0); + } - return ip_len; + tap_hdr_update(taph, l4len + sizeof(*ip6h) + sizeof(struct ethhdr)); } /** @@ -1110,42 +1070,41 @@ size_t tcp_fill_headers6(const struct ctx *c, * Return: 1 if sequence or window were updated, 0 otherwise */ int tcp_update_seqack_wnd(const struct ctx *c, struct tcp_tap_conn *conn, - int force_seq, struct tcp_info *tinfo) + bool force_seq, struct tcp_info_linux *tinfo) { uint32_t prev_wnd_to_tap = conn->wnd_to_tap << conn->ws_to_tap; uint32_t prev_ack_to_tap = conn->seq_ack_to_tap; /* cppcheck-suppress [ctunullpointer, unmatchedSuppression] */ socklen_t sl = sizeof(*tinfo); - struct tcp_info tinfo_new; + struct tcp_info_linux tinfo_new; uint32_t new_wnd_to_tap = prev_wnd_to_tap; int s = conn->sock; -#ifndef HAS_BYTES_ACKED - (void)force_seq; - - conn->seq_ack_to_tap = conn->seq_from_tap; - if (SEQ_LT(conn->seq_ack_to_tap, prev_ack_to_tap)) - conn->seq_ack_to_tap = prev_ack_to_tap; -#else - if ((unsigned)SNDBUF_GET(conn) < SNDBUF_SMALL || tcp_rtt_dst_low(conn) - || CONN_IS_CLOSING(conn) || (conn->flags & LOCAL) || force_seq) { + if (!bytes_acked_cap) { conn->seq_ack_to_tap = conn->seq_from_tap; - } else if (conn->seq_ack_to_tap != conn->seq_from_tap) { - if (!tinfo) { - tinfo = &tinfo_new; - if (getsockopt(s, SOL_TCP, TCP_INFO, tinfo, &sl)) - return 0; - } - - conn->seq_ack_to_tap = tinfo->tcpi_bytes_acked + - conn->seq_init_from_tap; - if (SEQ_LT(conn->seq_ack_to_tap, prev_ack_to_tap)) conn->seq_ack_to_tap = prev_ack_to_tap; + } else { + if ((unsigned)SNDBUF_GET(conn) < SNDBUF_SMALL || + tcp_rtt_dst_low(conn) || CONN_IS_CLOSING(conn) || + (conn->flags & LOCAL) || force_seq) { + conn->seq_ack_to_tap = conn->seq_from_tap; + } else if (conn->seq_ack_to_tap != conn->seq_from_tap) { + if (!tinfo) { + tinfo = &tinfo_new; + if (getsockopt(s, SOL_TCP, TCP_INFO, tinfo, &sl)) + return 0; + } + + conn->seq_ack_to_tap = tinfo->tcpi_bytes_acked + + conn->seq_init_from_tap; + + if (SEQ_LT(conn->seq_ack_to_tap, prev_ack_to_tap)) + conn->seq_ack_to_tap = prev_ack_to_tap; + } } -#endif /* !HAS_BYTES_ACKED */ - if (!KERNEL_REPORTS_SND_WND(c)) { + if (!snd_wnd_cap) { tcp_get_sndbuf(conn); new_wnd_to_tap = MIN(SNDBUF_GET(conn), MAX_WINDOW); conn->wnd_to_tap = MIN(new_wnd_to_tap >> conn->ws_to_tap, @@ -1156,14 +1115,13 @@ int tcp_update_seqack_wnd(const struct ctx *c, struct tcp_tap_conn *conn, if (!tinfo) { if (prev_wnd_to_tap > WINDOW_DEFAULT) { goto out; -} + } tinfo = &tinfo_new; if (getsockopt(s, SOL_TCP, TCP_INFO, tinfo, &sl)) { goto out; -} + } } -#ifdef HAS_SND_WND if ((conn->flags & LOCAL) || tcp_rtt_dst_low(conn)) { new_wnd_to_tap = tinfo->tcpi_snd_wnd; } else { @@ -1171,7 +1129,6 @@ int tcp_update_seqack_wnd(const struct ctx *c, struct tcp_tap_conn *conn, new_wnd_to_tap = MIN((int)tinfo->tcpi_snd_wnd, SNDBUF_GET(conn)); } -#endif new_wnd_to_tap = MIN(new_wnd_to_tap, MAX_WINDOW); if (!(conn->events & ESTABLISHED)) @@ -1217,25 +1174,23 @@ static void tcp_update_seqack_from_tap(const struct ctx *c, } /** - * tcp_fill_flag_header() - Prepare header for flags-only segment (no payload) + * tcp_prepare_flags() - Prepare header for flags-only segment (no payload) * @c: Execution context * @conn: Connection pointer * @flags: TCP flags: if not set, send segment only if ACK is due * @th: TCP header to update - * @opts: buffer to store TCP option - * @optlen: size of the TCP option buffer + * @data: buffer to store TCP option + * @optlen: size of the TCP option buffer (output parameter) * * Return: < 0 error code on connection reset, - * 0 if there is no flag to send - * 1 otherwise + * 0 if there is no flag to send + * 1 otherwise */ -int tcp_fill_flag_header(struct ctx *c, struct tcp_tap_conn *conn, - int flags, struct tcphdr *th, char *opts, - size_t *optlen) +int tcp_prepare_flags(const struct ctx *c, struct tcp_tap_conn *conn, + int flags, struct tcphdr *th, struct tcp_syn_opts *opts, + size_t *optlen) { - uint32_t prev_ack_to_tap = conn->seq_ack_to_tap; - uint32_t prev_wnd_to_tap = conn->wnd_to_tap; - struct tcp_info tinfo = { 0 }; + struct tcp_info_linux tinfo = { 0 }; socklen_t sl = sizeof(tinfo); int s = conn->sock; @@ -1248,30 +1203,24 @@ int tcp_fill_flag_header(struct ctx *c, struct tcp_tap_conn *conn, return -ECONNRESET; } -#ifdef HAS_SND_WND - if (!c->tcp.kernel_snd_wnd && tinfo.tcpi_snd_wnd) - c->tcp.kernel_snd_wnd = 1; -#endif - if (!(conn->flags & LOCAL)) tcp_rtt_dst_check(conn, &tinfo); - if (!tcp_update_seqack_wnd(c, conn, flags, &tinfo) && !flags) + if (!tcp_update_seqack_wnd(c, conn, !!flags, &tinfo) && !flags) return 0; + *optlen = 0; if (flags & SYN) { int mss; - /* Options: MSS, NOP and window scale (8 bytes) */ - *optlen = OPT_MSS_LEN + 1 + OPT_WS_LEN; - - *opts++ = OPT_MSS; - *opts++ = OPT_MSS_LEN; - if (c->mtu == -1) { mss = tinfo.tcpi_snd_mss; } else { mss = c->mtu - sizeof(struct tcphdr); + if (CONN_V4(conn)) + mss -= sizeof(struct iphdr); + else + mss -= sizeof(struct ipv6hdr); if (c->low_wmem && !(conn->flags & LOCAL) && !tcp_rtt_dst_low(conn)) @@ -1279,26 +1228,18 @@ int tcp_fill_flag_header(struct ctx *c, struct tcp_tap_conn *conn, else if (mss > PAGE_SIZE) mss = ROUND_DOWN(mss, PAGE_SIZE); } - *(uint16_t *)opts = htons(MIN(USHRT_MAX, mss)); - - opts += OPT_MSS_LEN - 2; conn->ws_to_tap = MIN(MAX_WS, tinfo.tcpi_snd_wscale); - *opts++ = OPT_NOP; - *opts++ = OPT_WS; - *opts++ = OPT_WS_LEN; - *opts++ = conn->ws_to_tap; - - th->ack = !!(flags & ACK); - } else { - th->ack = !!(flags & (ACK | DUP_ACK)) || - conn->seq_ack_to_tap != prev_ack_to_tap || - !prev_wnd_to_tap; + *opts = TCP_SYN_OPTS(mss, conn->ws_to_tap); + *optlen = sizeof(*opts); + } else if (!(flags & RST)) { + flags |= ACK; } th->doff = (sizeof(*th) + *optlen) / 4; + th->ack = !!(flags & ACK); th->rst = !!(flags & RST); th->syn = !!(flags & SYN); th->fin = !!(flags & FIN); @@ -1320,10 +1261,20 @@ int tcp_fill_flag_header(struct ctx *c, struct tcp_tap_conn *conn, return 1; } -int tcp_send_flag(struct ctx *c, struct tcp_tap_conn *conn, int flags) +/** + * tcp_send_flag() - Send segment with flags to tap (no payload) + * @c: Execution context + * @conn: Connection pointer + * @flags: TCP flags: if not set, send segment only if ACK is due + * + * Return: negative error code on connection reset, 0 otherwise + */ +static int tcp_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, + int flags) { if (c->mode == MODE_VU) return tcp_vu_send_flag(c, conn, flags); + return tcp_buf_send_flag(c, conn, flags); } @@ -1332,7 +1283,7 @@ int tcp_send_flag(struct ctx *c, struct tcp_tap_conn *conn, int flags) * @c: Execution context * @conn: Connection pointer */ -void tcp_rst_do(struct ctx *c, struct tcp_tap_conn *conn) +void tcp_rst_do(const struct ctx *c, struct tcp_tap_conn *conn) { if (conn->events == CLOSED) return; @@ -1366,6 +1317,14 @@ static void tcp_get_tap_ws(struct tcp_tap_conn *conn, static void tcp_tap_window_update(struct tcp_tap_conn *conn, unsigned wnd) { wnd = MIN(MAX_WINDOW, wnd << conn->ws_from_tap); + + /* Work-around for bug introduced in peer kernel code, commit + * e2142825c120 ("net: tcp: send zero-window ACK when no memory"). + * We don't update if window shrank to zero. + */ + if (!wnd && SEQ_LT(conn->seq_ack_from_tap, conn->seq_to_tap)) + return; + conn->wnd_from_tap = MIN(wnd >> conn->ws_from_tap, USHRT_MAX); /* FIXME: reflect the tap-side receiver's window back to the sock-side @@ -1373,33 +1332,16 @@ static void tcp_tap_window_update(struct tcp_tap_conn *conn, unsigned wnd) } /** - * tcp_seq_init() - Calculate initial sequence number according to RFC 6528 - * @c: Execution context - * @conn: TCP connection, with faddr, fport and eport populated + * tcp_init_seq() - Calculate initial sequence number according to RFC 6528 + * @hash: Hash of connection details * @now: Current timestamp */ -static void tcp_seq_init(const struct ctx *c, struct tcp_tap_conn *conn, - const struct timespec *now) +static uint32_t tcp_init_seq(uint64_t hash, const struct timespec *now) { - struct siphash_state state = SIPHASH_INIT(c->hash_secret); - union inany_addr aany; - uint64_t hash; - uint32_t ns; - - if (CONN_V4(conn)) - inany_from_af(&aany, AF_INET, &c->ip4.addr); - else - inany_from_af(&aany, AF_INET6, &c->ip6.addr); - - inany_siphash_feed(&state, &conn->faddr); - inany_siphash_feed(&state, &aany); - hash = siphash_final(&state, 36, - (uint64_t)conn->fport << 16 | conn->eport); - /* 32ns ticks, overflows 32 bits every 137s */ - ns = (now->tv_sec * 1000000000 + now->tv_nsec) >> 5; + uint32_t ns = (now->tv_sec * 1000000000 + now->tv_nsec) >> 5; - conn->seq_to_tap = ((uint32_t)(hash >> 32) ^ (uint32_t)hash) + ns; + return ((uint32_t)(hash >> 32) ^ (uint32_t)hash) + ns; } /** @@ -1431,7 +1373,7 @@ static int tcp_conn_new_sock(const struct ctx *c, sa_family_t af) { int s; - s = socket(af, SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP); + s = socket(af, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, IPPROTO_TCP); if (s > FD_REF_MAX) { close(s); @@ -1480,22 +1422,21 @@ int tcp_conn_sock(const struct ctx *c, sa_family_t af) * * Return: clamped MSS value */ -static uint16_t tcp_conn_tap_mss(const struct ctx *c, - const struct tcp_tap_conn *conn, +static uint16_t tcp_conn_tap_mss(const struct tcp_tap_conn *conn, const char *opts, size_t optlen) { unsigned int mss; int ret; - (void)c; /* unused */ - (void)conn; /* unused */ - if ((ret = tcp_opt_get(opts, optlen, OPT_MSS, NULL, NULL)) < 0) mss = MSS_DEFAULT; else mss = ret; - mss = MIN(MSS, mss); + if (CONN_V4(conn)) + mss = MIN(MSS4, mss); + else + mss = MIN(MSS6, mss); return MIN(mss, USHRT_MAX); } @@ -1503,53 +1444,47 @@ static uint16_t tcp_conn_tap_mss(const struct ctx *c, /** * tcp_bind_outbound() - Bind socket to outbound address and interface if given * @c: Execution context + * @conn: Connection entry for socket to bind * @s: Outbound TCP socket - * @af: Address family */ -static void tcp_bind_outbound(const struct ctx *c, int s, sa_family_t af) +static void tcp_bind_outbound(const struct ctx *c, + const struct tcp_tap_conn *conn, int s) { - if (af == AF_INET) { - if (!IN4_IS_ADDR_UNSPECIFIED(&c->ip4.addr_out)) { - struct sockaddr_in addr4 = { - .sin_family = AF_INET, - .sin_port = 0, - .sin_addr = c->ip4.addr_out, - }; - - if (bind(s, (struct sockaddr *)&addr4, sizeof(addr4))) { - debug("Can't bind IPv4 TCP socket address: %s", - strerror(errno)); - } + const struct flowside *tgt = &conn->f.side[TGTSIDE]; + union sockaddr_inany bind_sa; + socklen_t sl; + + + pif_sockaddr(c, &bind_sa, &sl, PIF_HOST, &tgt->oaddr, tgt->oport); + if (!inany_is_unspecified(&tgt->oaddr) || tgt->oport) { + if (bind(s, &bind_sa.sa, sl)) { + char sstr[INANY_ADDRSTRLEN]; + + flow_dbg(conn, + "Can't bind TCP outbound socket to %s:%hu: %s", + inany_ntop(&tgt->oaddr, sstr, sizeof(sstr)), + tgt->oport, strerror(errno)); } + } + if (bind_sa.sa_family == AF_INET) { if (*c->ip4.ifname_out) { if (setsockopt(s, SOL_SOCKET, SO_BINDTODEVICE, c->ip4.ifname_out, strlen(c->ip4.ifname_out))) { - debug("Can't bind IPv4 TCP socket to interface:" - " %s", strerror(errno)); - } - } - } else if (af == AF_INET6) { - if (!IN6_IS_ADDR_UNSPECIFIED(&c->ip6.addr_out)) { - struct sockaddr_in6 addr6 = { - .sin6_family = AF_INET6, - .sin6_port = 0, - .sin6_addr = c->ip6.addr_out, - }; - - if (bind(s, (struct sockaddr *)&addr6, sizeof(addr6))) { - debug("Can't bind IPv6 TCP socket address: %s", - strerror(errno)); + flow_dbg(conn, "Can't bind IPv4 TCP socket to" + " interface %s: %s", c->ip4.ifname_out, + strerror(errno)); } } - + } else if (bind_sa.sa_family == AF_INET6) { if (*c->ip6.ifname_out) { if (setsockopt(s, SOL_SOCKET, SO_BINDTODEVICE, c->ip6.ifname_out, strlen(c->ip6.ifname_out))) { - debug("Can't bind IPv6 TCP socket to interface:" - " %s", strerror(errno)); + flow_dbg(conn, "Can't bind IPv6 TCP socket to" + " interface %s: %s", c->ip6.ifname_out, + strerror(errno)); } } } @@ -1566,92 +1501,81 @@ static void tcp_bind_outbound(const struct ctx *c, int s, sa_family_t af) * @optlen: Bytes in options: caller MUST ensure available length * @now: Current timestamp */ -static void tcp_conn_from_tap(struct ctx *c, sa_family_t af, +static void tcp_conn_from_tap(const struct ctx *c, sa_family_t af, const void *saddr, const void *daddr, const struct tcphdr *th, const char *opts, size_t optlen, const struct timespec *now) { in_port_t srcport = ntohs(th->source); in_port_t dstport = ntohs(th->dest); - struct sockaddr_in addr4 = { - .sin_family = AF_INET, - .sin_port = htons(dstport), - .sin_addr = *(struct in_addr *)daddr, - }; - struct sockaddr_in6 addr6 = { - .sin6_family = AF_INET6, - .sin6_port = htons(dstport), - .sin6_addr = *(struct in6_addr *)daddr, - }; - const struct sockaddr *sa; + const struct flowside *ini, *tgt; struct tcp_tap_conn *conn; + union sockaddr_inany sa; union flow *flow; int s = -1, mss; + uint64_t hash; socklen_t sl; if (!(flow = flow_alloc())) return; - if (af == AF_INET) { - if (IN4_IS_ADDR_UNSPECIFIED(saddr) || - IN4_IS_ADDR_BROADCAST(saddr) || - IN4_IS_ADDR_MULTICAST(saddr) || srcport == 0 || - IN4_IS_ADDR_UNSPECIFIED(daddr) || - IN4_IS_ADDR_BROADCAST(daddr) || - IN4_IS_ADDR_MULTICAST(daddr) || dstport == 0) { - char sstr[INET_ADDRSTRLEN], dstr[INET_ADDRSTRLEN]; - - debug("Invalid endpoint in TCP SYN: %s:%hu -> %s:%hu", - inet_ntop(AF_INET, saddr, sstr, sizeof(sstr)), - srcport, - inet_ntop(AF_INET, daddr, dstr, sizeof(dstr)), - dstport); - goto cancel; - } - } else if (af == AF_INET6) { - if (IN6_IS_ADDR_UNSPECIFIED(saddr) || - IN6_IS_ADDR_MULTICAST(saddr) || srcport == 0 || - IN6_IS_ADDR_UNSPECIFIED(daddr) || - IN6_IS_ADDR_MULTICAST(daddr) || dstport == 0) { - char sstr[INET6_ADDRSTRLEN], dstr[INET6_ADDRSTRLEN]; - - debug("Invalid endpoint in TCP SYN: %s:%hu -> %s:%hu", - inet_ntop(AF_INET6, saddr, sstr, sizeof(sstr)), - srcport, - inet_ntop(AF_INET6, daddr, dstr, sizeof(dstr)), - dstport); - goto cancel; - } - } + ini = flow_initiate_af(flow, PIF_TAP, + af, saddr, srcport, daddr, dstport); - if ((s = tcp_conn_sock(c, af)) < 0) + if (!(tgt = flow_target(c, flow, IPPROTO_TCP))) goto cancel; - if (!c->no_map_gw) { - if (af == AF_INET && IN4_ARE_ADDR_EQUAL(daddr, &c->ip4.gw)) - addr4.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - if (af == AF_INET6 && IN6_ARE_ADDR_EQUAL(daddr, &c->ip6.gw)) - addr6.sin6_addr = in6addr_loopback; + if (flow->f.pif[TGTSIDE] != PIF_HOST) { + flow_err(flow, "No support for forwarding TCP from %s to %s", + pif_name(flow->f.pif[INISIDE]), + pif_name(flow->f.pif[TGTSIDE])); + goto cancel; } - if (af == AF_INET6 && IN6_IS_ADDR_LINKLOCAL(&addr6.sin6_addr)) { - struct sockaddr_in6 addr6_ll = { - .sin6_family = AF_INET6, - .sin6_addr = c->ip6.addr_ll, - .sin6_scope_id = c->ifi6, - }; - if (bind(s, (struct sockaddr *)&addr6_ll, sizeof(addr6_ll))) + conn = FLOW_SET_TYPE(flow, FLOW_TCP, tcp); + + if (!inany_is_unicast(&ini->eaddr) || ini->eport == 0 || + !inany_is_unicast(&ini->oaddr) || ini->oport == 0) { + char sstr[INANY_ADDRSTRLEN], dstr[INANY_ADDRSTRLEN]; + + debug("Invalid endpoint in TCP SYN: %s:%hu -> %s:%hu", + inany_ntop(&ini->eaddr, sstr, sizeof(sstr)), ini->eport, + inany_ntop(&ini->oaddr, dstr, sizeof(dstr)), ini->oport); + goto cancel; + } + + if ((s = tcp_conn_sock(c, af)) < 0) + goto cancel; + + pif_sockaddr(c, &sa, &sl, PIF_HOST, &tgt->eaddr, tgt->eport); + + /* Use bind() to check if the target address is local (EADDRINUSE or + * similar) and already bound, and set the LOCAL flag in that case. + * + * If bind() succeeds, in general, we could infer that nobody (else) is + * listening on that address and port and reset the connection attempt + * early, but we can't rely on that if non-local binds are enabled, + * because bind() would succeed for any non-local address we can reach. + * + * So, if bind() succeeds, close the socket, get a new one, and proceed. + */ + if (bind(s, &sa.sa, sl)) { + if (errno != EADDRNOTAVAIL && errno != EACCES) + conn_flag(c, conn, LOCAL); + } else { + /* Not a local, bound destination, inconclusive test */ + close(s); + if ((s = tcp_conn_sock(c, af)) < 0) goto cancel; } - conn = FLOW_START(flow, FLOW_TCP, tcp, TAPSIDE); conn->sock = s; conn->timer = -1; conn_event(c, conn, TAP_SYN_RCVD); conn->wnd_to_tap = WINDOW_DEFAULT; - mss = tcp_conn_tap_mss(c, conn, opts, optlen); + mss = tcp_conn_tap_mss(conn, opts, optlen); if (setsockopt(s, SOL_TCP, TCP_MAXSEG, &mss, sizeof(mss))) flow_trace(conn, "failed to set TCP_MAXSEG on socket %i", s); MSS_SET(conn, mss); @@ -1664,44 +1588,20 @@ static void tcp_conn_from_tap(struct ctx *c, sa_family_t af, if (!(conn->wnd_from_tap = (htons(th->window) >> conn->ws_from_tap))) conn->wnd_from_tap = 1; - inany_from_af(&conn->faddr, af, daddr); - - if (af == AF_INET) { - sa = (struct sockaddr *)&addr4; - sl = sizeof(addr4); - } else { - sa = (struct sockaddr *)&addr6; - sl = sizeof(addr6); - } - - conn->fport = dstport; - conn->eport = srcport; - conn->seq_init_from_tap = ntohl(th->seq); conn->seq_from_tap = conn->seq_init_from_tap + 1; conn->seq_ack_to_tap = conn->seq_from_tap; - tcp_seq_init(c, conn, now); + hash = flow_hash_insert(c, TAP_SIDX(conn)); + conn->seq_to_tap = tcp_init_seq(hash, now); conn->seq_ack_from_tap = conn->seq_to_tap; - tcp_hash_insert(c, conn); - - if (!bind(s, sa, sl)) { - tcp_rst(c, conn); /* Nobody is listening then */ - return; - } - if (errno != EADDRNOTAVAIL && errno != EACCES) - conn_flag(c, conn, LOCAL); - - if ((af == AF_INET && !IN4_IS_ADDR_LOOPBACK(&addr4.sin_addr)) || - (af == AF_INET6 && !IN6_IS_ADDR_LOOPBACK(&addr6.sin6_addr) && - !IN6_IS_ADDR_LINKLOCAL(&addr6.sin6_addr))) - tcp_bind_outbound(c, s, af); + tcp_bind_outbound(c, conn, s); - if (connect(s, sa, sl)) { + if (connect(s, &sa.sa, sl)) { if (errno != EINPROGRESS) { tcp_rst(c, conn); - return; + goto cancel; } tcp_get_sndbuf(conn); @@ -1709,12 +1609,13 @@ static void tcp_conn_from_tap(struct ctx *c, sa_family_t af, tcp_get_sndbuf(conn); if (tcp_send_flag(c, conn, SYN | ACK)) - return; + goto cancel; conn_event(c, conn, TAP_SYN_ACK_SENT); } tcp_epoll_ctl(c, conn); + FLOW_ACTIVATE(conn); return; cancel: @@ -1756,7 +1657,16 @@ static int tcp_sock_consume(const struct tcp_tap_conn *conn, uint32_t ack_seq) return 0; } -static int tcp_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn) +/** + * tcp_data_from_sock() - Handle new data from socket, queue to tap, in window + * @c: Execution context + * @conn: Connection pointer + * + * Return: negative on connection reset, 0 otherwise + * + * #syscalls recvmsg + */ +static int tcp_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn) { if (c->mode == MODE_VU) return tcp_vu_data_from_sock(c, conn); @@ -1775,8 +1685,8 @@ static int tcp_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn) * * Return: count of consumed packets */ -static int tcp_data_from_tap(struct ctx *c, struct tcp_tap_conn *conn, - const struct pool *p, int idx) +static int tcp_data_from_tap(const struct ctx *c, struct tcp_tap_conn *conn, + const struct pool *p, int idx) { int i, iov_i, ack = 0, fin = 0, retr = 0, keep = -1, partial_send = 0; uint16_t max_ack_seq_wnd = conn->wnd_from_tap; @@ -1895,6 +1805,10 @@ static int tcp_data_from_tap(struct ctx *c, struct tcp_tap_conn *conn, "fast re-transmit, ACK: %u, previous sequence: %u", max_ack_seq, conn->seq_to_tap); conn->seq_to_tap = max_ack_seq; + if (tcp_set_peek_offset(conn->sock, 0)) { + tcp_rst(c, conn); + return -1; + } tcp_data_from_sock(c, conn); } @@ -1941,7 +1855,7 @@ out: */ if (conn->seq_dup_ack_approx != (conn->seq_from_tap & 0xff)) { conn->seq_dup_ack_approx = conn->seq_from_tap & 0xff; - tcp_send_flag(c, conn, DUP_ACK); + tcp_send_flag(c, conn, ACK | DUP_ACK); } return p->count - idx; } @@ -1969,7 +1883,8 @@ out: * @opts: Pointer to start of options * @optlen: Bytes in options: caller MUST ensure available length */ -static void tcp_conn_from_sock_finish(struct ctx *c, struct tcp_tap_conn *conn, +static void tcp_conn_from_sock_finish(const struct ctx *c, + struct tcp_tap_conn *conn, const struct tcphdr *th, const char *opts, size_t optlen) { @@ -1980,19 +1895,24 @@ static void tcp_conn_from_sock_finish(struct ctx *c, struct tcp_tap_conn *conn, if (!(conn->wnd_from_tap >>= conn->ws_from_tap)) conn->wnd_from_tap = 1; - MSS_SET(conn, tcp_conn_tap_mss(c, conn, opts, optlen)); + MSS_SET(conn, tcp_conn_tap_mss(conn, opts, optlen)); conn->seq_init_from_tap = ntohl(th->seq) + 1; conn->seq_from_tap = conn->seq_init_from_tap; conn->seq_ack_to_tap = conn->seq_from_tap; conn_event(c, conn, ESTABLISHED); + if (tcp_set_peek_offset(conn->sock, 0)) { + tcp_rst(c, conn); + return; + } + + tcp_send_flag(c, conn, ACK); /* The client might have sent data already, which we didn't * dequeue waiting for SYN,ACK from tap -- check now. */ tcp_data_from_sock(c, conn); - tcp_send_flag(c, conn, ACK); } /** @@ -2008,7 +1928,7 @@ static void tcp_conn_from_sock_finish(struct ctx *c, struct tcp_tap_conn *conn, * * Return: count of consumed packets */ -int tcp_tap_handler(struct ctx *c, uint8_t pif, sa_family_t af, +int tcp_tap_handler(const struct ctx *c, uint8_t pif, sa_family_t af, const void *saddr, const void *daddr, const struct pool *p, int idx, const struct timespec *now) { @@ -2016,6 +1936,8 @@ int tcp_tap_handler(struct ctx *c, uint8_t pif, sa_family_t af, const struct tcphdr *th; size_t optlen, len; const char *opts; + union flow *flow; + flow_sidx_t sidx; int ack_due = 0; int count; @@ -2031,16 +1953,22 @@ int tcp_tap_handler(struct ctx *c, uint8_t pif, sa_family_t af, optlen = MIN(optlen, ((1UL << 4) /* from doff width */ - 6) * 4UL); opts = packet_get(p, idx, sizeof(*th), optlen, NULL); - conn = tcp_hash_lookup(c, af, daddr, ntohs(th->source), ntohs(th->dest)); + sidx = flow_lookup_af(c, IPPROTO_TCP, PIF_TAP, af, saddr, daddr, + ntohs(th->source), ntohs(th->dest)); + flow = flow_at_sidx(sidx); /* New connection from tap */ - if (!conn) { + if (!flow) { if (opts && th->syn && !th->ack) tcp_conn_from_tap(c, af, saddr, daddr, th, opts, optlen, now); return 1; } + ASSERT(flow->f.type == FLOW_TCP); + ASSERT(pif_at_sidx(sidx) == PIF_TAP); + conn = &flow->tcp; + flow_trace(conn, "packet length %zu from tap", len); if (th->rst) { @@ -2067,6 +1995,8 @@ int tcp_tap_handler(struct ctx *c, uint8_t pif, sa_family_t af, goto reset; conn_event(c, conn, ESTABLISHED); + if (tcp_set_peek_offset(conn->sock, 0)) + goto reset; if (th->fin) { conn->seq_from_tap++; @@ -2136,7 +2066,7 @@ reset: * @c: Execution context * @conn: Connection pointer */ -static void tcp_connect_finish(struct ctx *c, struct tcp_tap_conn *conn) +static void tcp_connect_finish(const struct ctx *c, struct tcp_tap_conn *conn) { socklen_t sl; int so; @@ -2155,61 +2085,26 @@ static void tcp_connect_finish(struct ctx *c, struct tcp_tap_conn *conn) } /** - * tcp_snat_inbound() - Translate source address for inbound data if needed - * @c: Execution context - * @addr: Source address of inbound packet/connection - */ -static void tcp_snat_inbound(const struct ctx *c, union inany_addr *addr) -{ - struct in_addr *addr4 = inany_v4(addr); - - if (addr4) { - if (IN4_IS_ADDR_LOOPBACK(addr4) || - IN4_IS_ADDR_UNSPECIFIED(addr4) || - IN4_ARE_ADDR_EQUAL(addr4, &c->ip4.addr_seen)) - *addr4 = c->ip4.gw; - } else { - struct in6_addr *addr6 = &addr->a6; - - if (IN6_IS_ADDR_LOOPBACK(addr6) || - IN6_ARE_ADDR_EQUAL(addr6, &c->ip6.addr_seen) || - IN6_ARE_ADDR_EQUAL(addr6, &c->ip6.addr)) { - if (IN6_IS_ADDR_LINKLOCAL(&c->ip6.gw)) - *addr6 = c->ip6.gw; - else - *addr6 = c->ip6.addr_ll; - } - } -} - -/** * tcp_tap_conn_from_sock() - Initialize state for non-spliced connection * @c: Execution context - * @dstport: Destination port for connection (host side) * @flow: flow to initialise * @s: Accepted socket * @sa: Peer socket address (from accept()) * @now: Current timestamp */ -static void tcp_tap_conn_from_sock(struct ctx *c, in_port_t dstport, - union flow *flow, int s, - const union sockaddr_inany *sa, - const struct timespec *now) +static void tcp_tap_conn_from_sock(const struct ctx *c, union flow *flow, + int s, const struct timespec *now) { - struct tcp_tap_conn *conn = FLOW_START(flow, FLOW_TCP, tcp, SOCKSIDE); + struct tcp_tap_conn *conn = FLOW_SET_TYPE(flow, FLOW_TCP, tcp); + uint64_t hash; conn->sock = s; conn->timer = -1; conn->ws_to_tap = conn->ws_from_tap = 0; conn_event(c, conn, SOCK_ACCEPTED); - inany_from_sockaddr(&conn->faddr, &conn->fport, sa); - conn->eport = dstport + c->tcp.fwd_in.delta[dstport]; - - tcp_snat_inbound(c, &conn->faddr); - - tcp_seq_init(c, conn, now); - tcp_hash_insert(c, conn); + hash = flow_hash_insert(c, TAP_SIDX(conn)); + conn->seq_to_tap = tcp_init_seq(hash, now); conn->seq_ack_from_tap = conn->seq_to_tap; @@ -2219,6 +2114,8 @@ static void tcp_tap_conn_from_sock(struct ctx *c, in_port_t dstport, conn_flag(c, conn, ACK_FROM_TAP_DUE); tcp_get_sndbuf(conn); + + FLOW_ACTIVATE(conn); } /** @@ -2227,53 +2124,58 @@ static void tcp_tap_conn_from_sock(struct ctx *c, in_port_t dstport, * @ref: epoll reference of listening socket * @now: Current timestamp */ -void tcp_listen_handler(struct ctx *c, union epoll_ref ref, +void tcp_listen_handler(const struct ctx *c, union epoll_ref ref, const struct timespec *now) { + const struct flowside *ini; union sockaddr_inany sa; socklen_t sl = sizeof(sa); union flow *flow; int s; - if (c->no_tcp || !(flow = flow_alloc())) + ASSERT(!c->no_tcp); + + if (!(flow = flow_alloc())) return; s = accept4(ref.fd, &sa.sa, &sl, SOCK_NONBLOCK); if (s < 0) goto cancel; - if (sa.sa_family == AF_INET) { - const struct in_addr *addr = &sa.sa4.sin_addr; - in_port_t port = sa.sa4.sin_port; + /* FIXME: When listening port has a specific bound address, record that + * as our address + */ + ini = flow_initiate_sa(flow, ref.tcp_listen.pif, &sa, + ref.tcp_listen.port); - if (IN4_IS_ADDR_UNSPECIFIED(addr) || - IN4_IS_ADDR_BROADCAST(addr) || - IN4_IS_ADDR_MULTICAST(addr) || port == 0) { - char str[INET_ADDRSTRLEN]; + if (!inany_is_unicast(&ini->eaddr) || ini->eport == 0) { + char sastr[SOCKADDR_STRLEN]; - err("Invalid endpoint from TCP accept(): %s:%hu", - inet_ntop(AF_INET, addr, str, sizeof(str)), port); - goto cancel; - } - } else if (sa.sa_family == AF_INET6) { - const struct in6_addr *addr = &sa.sa6.sin6_addr; - in_port_t port = sa.sa6.sin6_port; + err("Invalid endpoint from TCP accept(): %s", + sockaddr_ntop(&sa, sastr, sizeof(sastr))); + goto cancel; + } - if (IN6_IS_ADDR_UNSPECIFIED(addr) || - IN6_IS_ADDR_MULTICAST(addr) || port == 0) { - char str[INET6_ADDRSTRLEN]; + if (!flow_target(c, flow, IPPROTO_TCP)) + goto cancel; - err("Invalid endpoint from TCP accept(): %s:%hu", - inet_ntop(AF_INET6, addr, str, sizeof(str)), port); - goto cancel; - } - } + switch (flow->f.pif[TGTSIDE]) { + case PIF_SPLICE: + case PIF_HOST: + tcp_splice_conn_from_sock(c, flow, s); + break; - if (tcp_splice_conn_from_sock(c, ref.tcp_listen.pif, - ref.tcp_listen.port, flow, s, &sa)) - return; + case PIF_TAP: + tcp_tap_conn_from_sock(c, flow, s, now); + break; + + default: + flow_err(flow, "No support for forwarding TCP from %s to %s", + pif_name(flow->f.pif[INISIDE]), + pif_name(flow->f.pif[TGTSIDE])); + goto cancel; + } - tcp_tap_conn_from_sock(c, ref.tcp_listen.port, flow, s, &sa, now); return; cancel: @@ -2285,21 +2187,23 @@ cancel: * @c: Execution context * @ref: epoll reference of timer (not connection) * - * #syscalls timerfd_gettime + * #syscalls timerfd_gettime arm:timerfd_gettime64 i686:timerfd_gettime64 */ -void tcp_timer_handler(struct ctx *c, union epoll_ref ref) +void tcp_timer_handler(const struct ctx *c, union epoll_ref ref) { struct itimerspec check_armed = { { 0 }, { 0 } }; - struct tcp_tap_conn *conn = CONN(ref.flow); + struct tcp_tap_conn *conn = &FLOW(ref.flow)->tcp; - if (c->no_tcp) - return; + ASSERT(!c->no_tcp); + ASSERT(conn->f.type == FLOW_TCP); /* We don't reset timers on ~ACK_FROM_TAP_DUE, ~ACK_TO_TAP_DUE. If the * timer is currently armed, this event came from a previous setting, * and we just set the timer to a new point in the future: discard it. */ - timerfd_gettime(conn->timer, &check_armed); + if (timerfd_gettime(conn->timer, &check_armed)) + flow_err(conn, "failed to read timer: %s", strerror(errno)); + if (check_armed.it_value.tv_sec || check_armed.it_value.tv_nsec) return; @@ -2320,8 +2224,12 @@ void tcp_timer_handler(struct ctx *c, union epoll_ref ref) flow_dbg(conn, "ACK timeout, retry"); conn->retrans++; conn->seq_to_tap = conn->seq_ack_from_tap; - tcp_data_from_sock(c, conn); - tcp_timer_ctl(c, conn); + if (tcp_set_peek_offset(conn->sock, 0)) { + tcp_rst(c, conn); + } else { + tcp_data_from_sock(c, conn); + tcp_timer_ctl(c, conn); + } } } else { struct itimerspec new = { { 0 }, { ACT_TIMEOUT, 0 } }; @@ -2333,7 +2241,10 @@ void tcp_timer_handler(struct ctx *c, union epoll_ref ref) * case. This avoids having to preemptively reset the timer on * ~ACK_TO_TAP_DUE or ~ACK_FROM_TAP_DUE. */ - timerfd_settime(conn->timer, 0, &new, &old); + if (timerfd_settime(conn->timer, 0, &new, &old)) + flow_err(conn, "failed to set timer: %s", + strerror(errno)); + if (old.it_value.tv_sec == ACT_TIMEOUT) { flow_dbg(conn, "activity timeout"); tcp_rst(c, conn); @@ -2347,12 +2258,13 @@ void tcp_timer_handler(struct ctx *c, union epoll_ref ref) * @ref: epoll reference * @events: epoll events bitmap */ -void tcp_sock_handler(struct ctx *c, union epoll_ref ref, uint32_t events) +void tcp_sock_handler(const struct ctx *c, union epoll_ref ref, + uint32_t events) { - struct tcp_tap_conn *conn = CONN(ref.flowside.flow); + struct tcp_tap_conn *conn = conn_at_sidx(ref.flowside); - ASSERT(conn->f.type == FLOW_TCP); - ASSERT(ref.flowside.side == SOCKSIDE); + ASSERT(!c->no_tcp); + ASSERT(pif_at_sidx(ref.flowside) != PIF_TAP); if (conn->events == CLOSED) return; @@ -2378,7 +2290,7 @@ void tcp_sock_handler(struct ctx *c, union epoll_ref ref, uint32_t events) tcp_data_from_sock(c, conn); if (events & EPOLLOUT) - tcp_update_seqack_wnd(c, conn, 0, NULL); + tcp_update_seqack_wnd(c, conn, false, NULL); return; } @@ -2401,17 +2313,16 @@ void tcp_sock_handler(struct ctx *c, union epoll_ref ref, uint32_t events) } /** - * tcp_sock_init_af() - Initialise listening socket for a given af and port + * tcp_sock_init_one() - Initialise listening socket for address and port * @c: Execution context - * @af: Address family to listen on - * @port: Port, host order - * @addr: Pointer to address for binding, NULL if not configured + * @addr: Pointer to address for binding, NULL for dual stack any * @ifname: Name of interface to bind to, NULL if not configured + * @port: Port, host order * * Return: fd for the new listening socket, negative error code on failure */ -static int tcp_sock_init_af(const struct ctx *c, sa_family_t af, in_port_t port, - const void *addr, const char *ifname) +static int tcp_sock_init_one(const struct ctx *c, const union inany_addr *addr, + const char *ifname, in_port_t port) { union tcp_listen_epoll_ref tref = { .port = port, @@ -2419,12 +2330,13 @@ static int tcp_sock_init_af(const struct ctx *c, sa_family_t af, in_port_t port, }; int s; - s = sock_l4(c, af, IPPROTO_TCP, addr, ifname, port, tref.u32); + s = pif_sock_l4(c, EPOLL_TYPE_TCP_LISTEN, PIF_HOST, addr, + ifname, port, tref.u32); if (c->tcp.fwd_in.mode == FWD_AUTO) { - if (af == AF_INET || af == AF_UNSPEC) + if (!addr || inany_v4(addr)) tcp_sock_init_ext[port][V4] = s < 0 ? -1 : s; - if (af == AF_INET6 || af == AF_UNSPEC) + if (!addr || !inany_v4(addr)) tcp_sock_init_ext[port][V6] = s < 0 ? -1 : s; } @@ -2438,29 +2350,32 @@ static int tcp_sock_init_af(const struct ctx *c, sa_family_t af, in_port_t port, /** * tcp_sock_init() - Create listening sockets for a given host ("inbound") port * @c: Execution context - * @af: Address family to select a specific IP version, or AF_UNSPEC * @addr: Pointer to address for binding, NULL if not configured * @ifname: Name of interface to bind to, NULL if not configured * @port: Port, host order * * Return: 0 on (partial) success, negative error code on (complete) failure */ -int tcp_sock_init(const struct ctx *c, sa_family_t af, const void *addr, +int tcp_sock_init(const struct ctx *c, const union inany_addr *addr, const char *ifname, in_port_t port) { int r4 = FD_REF_MAX + 1, r6 = FD_REF_MAX + 1; - if (af == AF_UNSPEC && c->ifi4 && c->ifi6) + ASSERT(!c->no_tcp); + + if (!addr && c->ifi4 && c->ifi6) /* Attempt to get a dual stack socket */ - if (tcp_sock_init_af(c, AF_UNSPEC, port, addr, ifname) >= 0) + if (tcp_sock_init_one(c, NULL, ifname, port) >= 0) return 0; /* Otherwise create a socket per IP version */ - if ((af == AF_INET || af == AF_UNSPEC) && c->ifi4) - r4 = tcp_sock_init_af(c, AF_INET, port, addr, ifname); + if ((!addr || inany_v4(addr)) && c->ifi4) + r4 = tcp_sock_init_one(c, addr ? addr : &inany_any4, + ifname, port); - if ((af == AF_INET6 || af == AF_UNSPEC) && c->ifi6) - r6 = tcp_sock_init_af(c, AF_INET6, port, addr, ifname); + if ((!addr || !inany_v4(addr)) && c->ifi6) + r6 = tcp_sock_init_one(c, addr ? addr : &inany_any6, + ifname, port); if (IN_INTERVAL(0, FD_REF_MAX, r4) || IN_INTERVAL(0, FD_REF_MAX, r6)) return 0; @@ -2483,8 +2398,8 @@ static void tcp_ns_sock_init4(const struct ctx *c, in_port_t port) ASSERT(c->mode == MODE_PASTA); - s = sock_l4(c, AF_INET, IPPROTO_TCP, &in4addr_loopback, NULL, port, - tref.u32); + s = pif_sock_l4(c, EPOLL_TYPE_TCP_LISTEN, PIF_SPLICE, &inany_loopback4, + NULL, port, tref.u32); if (s >= 0) tcp_sock_set_bufsize(c, s); else @@ -2509,8 +2424,8 @@ static void tcp_ns_sock_init6(const struct ctx *c, in_port_t port) ASSERT(c->mode == MODE_PASTA); - s = sock_l4(c, AF_INET6, IPPROTO_TCP, &in6addr_loopback, NULL, port, - tref.u32); + s = pif_sock_l4(c, EPOLL_TYPE_TCP_LISTEN, PIF_SPLICE, &inany_loopback6, + NULL, port, tref.u32); if (s >= 0) tcp_sock_set_bufsize(c, s); else @@ -2527,6 +2442,8 @@ static void tcp_ns_sock_init6(const struct ctx *c, in_port_t port) */ void tcp_ns_sock_init(const struct ctx *c, in_port_t port) { + ASSERT(!c->no_tcp); + if (c->ifi4) tcp_ns_sock_init4(c, port); if (c->ifi6) @@ -2539,6 +2456,7 @@ void tcp_ns_sock_init(const struct ctx *c, in_port_t port) * * Return: 0 */ +/* cppcheck-suppress [constParameterCallback, unmatchedSuppression] */ static int tcp_ns_socks_init(void *arg) { const struct ctx *c = (const struct ctx *)arg; @@ -2604,6 +2522,57 @@ static void tcp_sock_refill_init(const struct ctx *c) } /** + * tcp_probe_peek_offset_cap() - Check if SO_PEEK_OFF is supported by kernel + * @af: Address family, IPv4 or IPv6 + * + * Return: true if supported, false otherwise + */ +static bool tcp_probe_peek_offset_cap(sa_family_t af) +{ + bool ret = false; + int s, optv = 0; + + s = socket(af, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + if (s < 0) { + warn_perror("Temporary TCP socket creation failed"); + } else { + if (!setsockopt(s, SOL_SOCKET, SO_PEEK_OFF, &optv, sizeof(int))) + ret = true; + close(s); + } + + return ret; +} + +/** + * tcp_probe_tcp_info() - Check what data TCP_INFO reports + * + * Return: Number of bytes returned by TCP_INFO getsockopt() + */ +static socklen_t tcp_probe_tcp_info(void) +{ + struct tcp_info_linux tinfo; + socklen_t sl = sizeof(tinfo); + int s; + + s = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + if (s < 0) { + warn_perror("Temporary TCP socket creation failed"); + return false; + } + + if (getsockopt(s, SOL_TCP, TCP_INFO, &tinfo, &sl)) { + warn_perror("Failed to get TCP_INFO on temporary socket"); + close(s); + return false; + } + + close(s); + + return sl; +} + +/** * tcp_init() - Get initial sequence, hash secret, initialise per-socket data * @c: Execution context * @@ -2611,16 +2580,9 @@ static void tcp_sock_refill_init(const struct ctx *c) */ int tcp_init(struct ctx *c) { - unsigned b; - - for (b = 0; b < TCP_HASH_TABLE_SIZE; b++) - tc_hash[b] = FLOW_SIDX_NONE; - - if (c->ifi4) - tcp_buf_sock4_iov_init(c); + ASSERT(!c->no_tcp); - if (c->ifi6) - tcp_buf_sock6_iov_init(c); + tcp_sock_iov_init(c); memset(init_sock_pool4, 0xff, sizeof(init_sock_pool4)); memset(init_sock_pool6, 0xff, sizeof(init_sock_pool6)); @@ -2635,6 +2597,19 @@ int tcp_init(struct ctx *c) NS_CALL(tcp_ns_socks_init, c); } + peek_offset_cap = (!c->ifi4 || tcp_probe_peek_offset_cap(AF_INET)) && + (!c->ifi6 || tcp_probe_peek_offset_cap(AF_INET6)); + debug("SO_PEEK_OFF%ssupported", peek_offset_cap ? " " : " not "); + + tcp_info_size = tcp_probe_tcp_info(); + +#define dbg_tcpi(f_) debug("TCP_INFO tcpi_%s field%s supported", \ + STRINGIFY(f_), tcp_info_cap(f_) ? " " : " not ") + dbg_tcpi(snd_wnd); + dbg_tcpi(bytes_acked); + dbg_tcpi(min_rtt); +#undef dbg_tcpi + return 0; } @@ -2676,7 +2651,7 @@ static void tcp_port_rebind(struct ctx *c, bool outbound) if (outbound) tcp_ns_sock_init(c, port); else - tcp_sock_init(c, AF_UNSPEC, NULL, NULL, port); + tcp_sock_init(c, NULL, NULL, port); } } } @@ -10,20 +10,24 @@ struct ctx; -void tcp_timer_handler(struct ctx *c, union epoll_ref ref); -void tcp_listen_handler(struct ctx *c, union epoll_ref ref, +void tcp_timer_handler(const struct ctx *c, union epoll_ref ref); +void tcp_listen_handler(const struct ctx *c, union epoll_ref ref, const struct timespec *now); -void tcp_sock_handler(struct ctx *c, union epoll_ref ref, uint32_t events); -int tcp_tap_handler(struct ctx *c, uint8_t pif, sa_family_t af, +void tcp_sock_handler(const struct ctx *c, union epoll_ref ref, + uint32_t events); +int tcp_tap_handler(const struct ctx *c, uint8_t pif, sa_family_t af, const void *saddr, const void *daddr, const struct pool *p, int idx, const struct timespec *now); -int tcp_sock_init(const struct ctx *c, sa_family_t af, const void *addr, +int tcp_sock_init(const struct ctx *c, const union inany_addr *addr, const char *ifname, in_port_t port); int tcp_init(struct ctx *c); void tcp_timer(struct ctx *c, const struct timespec *now); void tcp_defer_handler(struct ctx *c); -void tcp_buf_update_l2(const unsigned char *eth_d, const unsigned char *eth_s); +void tcp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s); +int tcp_set_peek_offset(int s, int offset); + +extern bool peek_offset_cap; /** * union tcp_epoll_ref - epoll reference portion for TCP connections @@ -55,16 +59,12 @@ union tcp_listen_epoll_ref { * @fwd_in: Port forwarding configuration for inbound packets * @fwd_out: Port forwarding configuration for outbound packets * @timer_run: Timestamp of most recent timer run - * @kernel_snd_wnd: Kernel reports sending window (with commit 8f7baad7f035) * @pipe_size: Size of pipes for spliced connections */ struct tcp_ctx { struct fwd_ports fwd_in; struct fwd_ports fwd_out; struct timespec timer_run; -#ifdef HAS_SND_WND - int kernel_snd_wnd; -#endif size_t pipe_size; }; @@ -6,9 +6,9 @@ * PASTA - Pack A Subtle Tap Abstraction * for network namespace/tap device mode * - * tcp_buf.c - TCP L2-L4 translation state machine + * tcp_buf.c - TCP L2 buffer management functions * - * Copyright (c) 2020-2022 Red Hat GmbH + * Copyright Red Hat * Author: Stefano Brivio <sbrivio@redhat.com> */ @@ -20,10 +20,11 @@ #include <netinet/ip.h> -#include <linux/tcp.h> +#include <netinet/tcp.h> #include "util.h" #include "ip.h" +#include "iov.h" #include "passt.h" #include "tap.h" #include "siphash.h" @@ -33,283 +34,201 @@ #include "tcp_buf.h" #define TCP_FRAMES_MEM 128 -#define TCP_FRAMES \ +#define TCP_FRAMES \ (c->mode == MODE_PASTA ? 1 : TCP_FRAMES_MEM) -/** - * tcp_buf_seq_update - Sequences to update with length of frames once sent - * @seq: Pointer to sequence number sent to tap-side, to be updated - * @len: TCP payload length - */ -struct tcp_buf_seq_update { - uint32_t *seq; - uint16_t len; -}; - /* Static buffers */ -/** - * tcp_l2_flags_t - TCP header and data to send option flags - * @th: TCP header - * @opts TCP option flags - */ -struct tcp_l2_flags_t { - struct tcphdr th; - char opts[OPT_MSS_LEN + OPT_WS_LEN + 1]; -}; -/** - * tcp_l2_payload_t - TCP header and data to send data - * 32 bytes aligned to be able to use AVX2 checksum - * @th: TCP header - * @data: TCP data - */ -struct tcp_l2_payload_t { - struct tcphdr th; /* 20 bytes */ - uint8_t data[MSS]; /* 65516 bytes */ -#ifdef __AVX2__ -} __attribute__ ((packed, aligned(32))); -#else -} __attribute__ ((packed, aligned(__alignof__(unsigned int)))); -#endif - -/* Ethernet header for IPv4 frames */ -static struct ethhdr tcp4_eth_src; - -/* IPv4 headers */ -static struct iphdr tcp4_l2_ip[TCP_FRAMES_MEM]; -/* TCP headers and data for IPv4 frames */ -static struct tcp_l2_payload_t tcp4_l2_payload[TCP_FRAMES_MEM]; - -static struct tcp_buf_seq_update tcp4_l2_buf_seq_update[TCP_FRAMES_MEM]; -static unsigned int tcp4_l2_buf_used; - -/* IPv4 headers for TCP option flags frames */ -static struct iphdr tcp4_l2_flags_ip[TCP_FRAMES_MEM]; -/* TCP headers and option flags for IPv4 frames */ -static struct tcp_l2_flags_t tcp4_l2_flags[TCP_FRAMES_MEM]; -static unsigned int tcp4_l2_flags_buf_used; - -/* Ethernet header for IPv6 frames */ +/* Ethernet header for IPv4 and IPv6 frames */ +static struct ethhdr tcp4_eth_src; static struct ethhdr tcp6_eth_src; -/* IPv6 headers */ -static struct ipv6hdr tcp6_l2_ip[TCP_FRAMES_MEM]; -/* TCP headers and data for IPv6 frames */ -static struct tcp_l2_payload_t tcp6_l2_payload[TCP_FRAMES_MEM]; +static struct tap_hdr tcp_payload_tap_hdr[TCP_FRAMES_MEM]; + +/* IP headers for IPv4 and IPv6 */ +struct iphdr tcp4_payload_ip[TCP_FRAMES_MEM]; +struct ipv6hdr tcp6_payload_ip[TCP_FRAMES_MEM]; -static struct tcp_buf_seq_update tcp6_l2_buf_seq_update[TCP_FRAMES_MEM]; -static unsigned int tcp6_l2_buf_used; +/* TCP segments with payload for IPv4 and IPv6 frames */ +static struct tcp_payload_t tcp_payload[TCP_FRAMES_MEM]; -/* IPv6 headers for TCP option flags frames */ -static struct ipv6hdr tcp6_l2_flags_ip[TCP_FRAMES_MEM]; -/* TCP headers and option flags for IPv6 frames */ -static struct tcp_l2_flags_t tcp6_l2_flags[TCP_FRAMES_MEM]; +static_assert(MSS4 <= sizeof(tcp_payload[0].data), "MSS4 is greater than 65516"); +static_assert(MSS6 <= sizeof(tcp_payload[0].data), "MSS6 is greater than 65516"); -static unsigned int tcp6_l2_flags_buf_used; +/* References tracking the owner connection of frames in the tap outqueue */ +static struct tcp_tap_conn *tcp_frame_conns[TCP_FRAMES_MEM]; +static unsigned int tcp_payload_used; /* recvmsg()/sendmsg() data for tap */ static struct iovec iov_sock [TCP_FRAMES_MEM + 1]; -static struct iovec tcp4_l2_iov [TCP_FRAMES_MEM][TCP_IOV_NUM]; -static struct iovec tcp6_l2_iov [TCP_FRAMES_MEM][TCP_IOV_NUM]; -static struct iovec tcp4_l2_flags_iov [TCP_FRAMES_MEM][TCP_IOV_NUM]; -static struct iovec tcp6_l2_flags_iov [TCP_FRAMES_MEM][TCP_IOV_NUM]; +static struct iovec tcp_l2_iov[TCP_FRAMES_MEM][TCP_NUM_IOVS]; /** - * tcp_buf_update_l2() - Update L2 buffers with Ethernet and IPv4 addresses + * tcp_update_l2_buf() - Update Ethernet header buffers with addresses * @eth_d: Ethernet destination address, NULL if unchanged * @eth_s: Ethernet source address, NULL if unchanged */ -void tcp_buf_update_l2(const unsigned char *eth_d, const unsigned char *eth_s) +void tcp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s) { eth_update_mac(&tcp4_eth_src, eth_d, eth_s); eth_update_mac(&tcp6_eth_src, eth_d, eth_s); } /** - * tcp_buf_sock4_iov_init() - Initialise scatter-gather L2 buffers for IPv4 sockets + * tcp_sock_iov_init() - Initialise scatter-gather L2 buffers for IPv4 sockets * @c: Execution context */ -void tcp_buf_sock4_iov_init(const struct ctx *c) +void tcp_sock_iov_init(const struct ctx *c) { + struct ipv6hdr ip6 = L2_BUF_IP6_INIT(IPPROTO_TCP); struct iphdr iph = L2_BUF_IP4_INIT(IPPROTO_TCP); int i; - (void)c; - + tcp6_eth_src.h_proto = htons_constant(ETH_P_IPV6); tcp4_eth_src.h_proto = htons_constant(ETH_P_IP); + + for (i = 0; i < ARRAY_SIZE(tcp_payload); i++) { + tcp6_payload_ip[i] = ip6; + tcp4_payload_ip[i] = iph; + } + for (i = 0; i < TCP_FRAMES_MEM; i++) { - struct iovec *iov; - - /* headers */ - tcp4_l2_ip[i] = iph; - tcp4_l2_payload[i].th = (struct tcphdr){ - .doff = sizeof(struct tcphdr) / 4, - .ack = 1 - }; - - tcp4_l2_flags_ip[i] = iph; - tcp4_l2_flags[i].th = (struct tcphdr){ - .doff = sizeof(struct tcphdr) / 4, - .ack = 1 - }; - - /* iovecs */ - iov = tcp4_l2_iov[i]; - iov[TCP_IOV_ETH].iov_base = &tcp4_eth_src; - iov[TCP_IOV_ETH].iov_len = sizeof(struct ethhdr); - iov[TCP_IOV_IP].iov_base = &tcp4_l2_ip[i]; - iov[TCP_IOV_IP].iov_len = sizeof(struct iphdr); - iov[TCP_IOV_PAYLOAD].iov_base = &tcp4_l2_payload[i]; + struct iovec *iov = tcp_l2_iov[i]; - iov = tcp4_l2_flags_iov[i]; - iov[TCP_IOV_ETH].iov_base = &tcp4_eth_src; + iov[TCP_IOV_TAP] = tap_hdr_iov(c, &tcp_payload_tap_hdr[i]); iov[TCP_IOV_ETH].iov_len = sizeof(struct ethhdr); - iov[TCP_IOV_IP].iov_base = &tcp4_l2_flags_ip[i]; - iov[TCP_IOV_IP].iov_len = sizeof(struct iphdr); - iov[TCP_IOV_PAYLOAD].iov_base = &tcp4_l2_flags[i]; + iov[TCP_IOV_PAYLOAD].iov_base = &tcp_payload[i]; } } /** - * tcp_buf_sock6_iov_init() - Initialise scatter-gather L2 buffers for IPv6 sockets - * @c: Execution context + * tcp_revert_seq() - Revert affected conn->seq_to_tap after failed transmission + * @ctx: Execution context + * @conns: Array of connection pointers corresponding to queued frames + * @frames: Two-dimensional array containing queued frames with sub-iovs + * @num_frames: Number of entries in the two arrays to be compared */ -void tcp_buf_sock6_iov_init(const struct ctx *c) +static void tcp_revert_seq(const struct ctx *c, struct tcp_tap_conn **conns, + struct iovec (*frames)[TCP_NUM_IOVS], int num_frames) { - struct ipv6hdr ip6 = L2_BUF_IP6_INIT(IPPROTO_TCP); int i; - (void)c; + for (i = 0; i < num_frames; i++) { + const struct tcphdr *th = frames[i][TCP_IOV_PAYLOAD].iov_base; + struct tcp_tap_conn *conn = conns[i]; + uint32_t seq = ntohl(th->seq); + uint32_t peek_offset; - tcp6_eth_src.h_proto = htons_constant(ETH_P_IPV6); - for (i = 0; i < TCP_FRAMES_MEM; i++) { - struct iovec *iov; - - /* headers */ - tcp6_l2_ip[i] = ip6; - tcp6_l2_payload[i].th = (struct tcphdr){ - .doff = sizeof(struct tcphdr) / 4, - .ack = 1 - }; - - tcp6_l2_flags_ip[i] = ip6; - tcp6_l2_flags[i].th = (struct tcphdr){ - .doff = sizeof(struct tcphdr) / 4, - .ack = 1 - }; - - /* iovecs */ - iov = tcp6_l2_iov[i]; - iov[TCP_IOV_ETH].iov_base = &tcp6_eth_src; - iov[TCP_IOV_ETH].iov_len = sizeof(struct ethhdr); - iov[TCP_IOV_IP].iov_base = &tcp6_l2_ip[i]; - iov[TCP_IOV_IP].iov_len = sizeof(struct ipv6hdr); - iov[TCP_IOV_PAYLOAD].iov_base = &tcp6_l2_payload[i]; + if (SEQ_LE(conn->seq_to_tap, seq)) + continue; - iov = tcp6_l2_flags_iov[i]; - iov[TCP_IOV_ETH].iov_base = &tcp6_eth_src; - iov[TCP_IOV_ETH].iov_len = sizeof(struct ethhdr); - iov[TCP_IOV_IP].iov_base = &tcp6_l2_flags_ip[i]; - iov[TCP_IOV_IP].iov_len = sizeof(struct ipv6hdr); - iov[TCP_IOV_PAYLOAD].iov_base = &tcp6_l2_flags[i]; + conn->seq_to_tap = seq; + peek_offset = conn->seq_to_tap - conn->seq_ack_from_tap; + if (tcp_set_peek_offset(conn->sock, peek_offset)) + tcp_rst(c, conn); } } /** - * tcp_buf_l2_flags_flush() - Send out buffers for segments with no data (flags) + * tcp_payload_flush() - Send out buffers for segments with data or flags * @c: Execution context */ -void tcp_buf_l2_flags_flush(const struct ctx *c) +void tcp_payload_flush(const struct ctx *c) { - tap_send_iov(c, tcp6_l2_flags_iov, tcp6_l2_flags_buf_used); - tcp6_l2_flags_buf_used = 0; + size_t m; - tap_send_iov(c, tcp4_l2_flags_iov, tcp4_l2_flags_buf_used); - tcp4_l2_flags_buf_used = 0; + m = tap_send_frames(c, &tcp_l2_iov[0][0], TCP_NUM_IOVS, + tcp_payload_used); + if (m != tcp_payload_used) { + tcp_revert_seq(c, &tcp_frame_conns[m], &tcp_l2_iov[m], + tcp_payload_used - m); + } + tcp_payload_used = 0; } /** - * tcp_buf_l2_data_flush() - Send out buffers for segments with data - * @c: Execution context + * tcp_buf_fill_headers() - Fill 802.3, IP, TCP headers in pre-cooked buffers + * @conn: Connection pointer + * @iov: Pointer to an array of iovec of TCP pre-cooked buffers + * @dlen: TCP payload length + * @check: Checksum, if already known + * @seq: Sequence number for this segment + * @no_tcp_csum: Do not set TCP checksum */ -void tcp_buf_l2_data_flush(const struct ctx *c) +static void tcp_l2_buf_fill_headers(const struct tcp_tap_conn *conn, + struct iovec *iov, size_t dlen, + const uint16_t *check, uint32_t seq, + bool no_tcp_csum) { - unsigned i; - size_t m; - - m = tap_send_iov(c, tcp6_l2_iov, tcp6_l2_buf_used); - for (i = 0; i < m; i++) - *tcp6_l2_buf_seq_update[i].seq += tcp6_l2_buf_seq_update[i].len; - tcp6_l2_buf_used = 0; - - m = tap_send_iov(c, tcp4_l2_iov, tcp4_l2_buf_used); - for (i = 0; i < m; i++) - *tcp4_l2_buf_seq_update[i].seq += tcp4_l2_buf_seq_update[i].len; - tcp4_l2_buf_used = 0; + const struct flowside *tapside = TAPFLOW(conn); + const struct in_addr *a4 = inany_v4(&tapside->oaddr); + + if (a4) { + tcp_fill_headers4(conn, iov[TCP_IOV_TAP].iov_base, + iov[TCP_IOV_IP].iov_base, + iov[TCP_IOV_PAYLOAD].iov_base, dlen, + check, seq, no_tcp_csum); + } else { + tcp_fill_headers6(conn, iov[TCP_IOV_TAP].iov_base, + iov[TCP_IOV_IP].iov_base, + iov[TCP_IOV_PAYLOAD].iov_base, dlen, + seq, no_tcp_csum); + } } -int tcp_buf_send_flag(struct ctx *c, struct tcp_tap_conn *conn, int flags) +/** + * tcp_buf_send_flag() - Send segment with flags to tap (no payload) + * @c: Execution context + * @conn: Connection pointer + * @flags: TCP flags: if not set, send segment only if ACK is due + * + * Return: negative error code on connection reset, 0 otherwise + */ +int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags) { - struct tcp_l2_flags_t *payload; - struct iovec *dup_iov; + struct tcp_payload_t *payload; struct iovec *iov; - struct tcphdr *th; - size_t optlen = 0; - size_t ip_len; - char *data; + size_t optlen; + size_t l4len; + uint32_t seq; int ret; + iov = tcp_l2_iov[tcp_payload_used]; if (CONN_V4(conn)) { - iov = tcp4_l2_flags_iov[tcp4_l2_flags_buf_used++]; - dup_iov = tcp4_l2_flags_iov[tcp4_l2_flags_buf_used]; + iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp4_payload_ip[tcp_payload_used]); + iov[TCP_IOV_ETH].iov_base = &tcp4_eth_src; } else { - iov = tcp6_l2_flags_iov[tcp6_l2_flags_buf_used++]; - dup_iov = tcp6_l2_flags_iov[tcp6_l2_flags_buf_used]; + iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp6_payload_ip[tcp_payload_used]); + iov[TCP_IOV_ETH].iov_base = &tcp6_eth_src; } - payload = iov[TCP_IOV_PAYLOAD].iov_base; - th = &payload->th; - data = payload->opts; - ret = tcp_fill_flag_header(c, conn, flags, th, data, &optlen); + payload = iov[TCP_IOV_PAYLOAD].iov_base; + seq = conn->seq_to_tap; + ret = tcp_prepare_flags(c, conn, flags, &payload->th, + (struct tcp_syn_opts *)&payload->data, &optlen); if (ret <= 0) return ret; - if (CONN_V4(conn)) { - struct iphdr *iph = iov[TCP_IOV_IP].iov_base; - - ip_len = tcp_fill_headers4(c, conn, iph, th, optlen, NULL, - conn->seq_to_tap); - } else { - struct ipv6hdr *ip6h = iov[TCP_IOV_IP].iov_base; - - ip_len = tcp_fill_headers6(c, conn, ip6h, th, optlen, - conn->seq_to_tap); - } - iov[TCP_IOV_PAYLOAD].iov_len = ip_len; + tcp_payload_used++; + l4len = optlen + sizeof(struct tcphdr); + iov[TCP_IOV_PAYLOAD].iov_len = l4len; + tcp_l2_buf_fill_headers(conn, iov, optlen, NULL, seq, false); if (flags & DUP_ACK) { - int i; - for (i = 0; i < TCP_IOV_NUM; i++) { - memcpy(dup_iov[i].iov_base, iov[i].iov_base, - iov[i].iov_len); - dup_iov[i].iov_len = iov[i].iov_len; - } + struct iovec *dup_iov = tcp_l2_iov[tcp_payload_used++]; + + memcpy(dup_iov[TCP_IOV_TAP].iov_base, iov[TCP_IOV_TAP].iov_base, + iov[TCP_IOV_TAP].iov_len); + dup_iov[TCP_IOV_ETH].iov_base = iov[TCP_IOV_ETH].iov_base; + dup_iov[TCP_IOV_IP] = iov[TCP_IOV_IP]; + memcpy(dup_iov[TCP_IOV_PAYLOAD].iov_base, + iov[TCP_IOV_PAYLOAD].iov_base, l4len); + dup_iov[TCP_IOV_PAYLOAD].iov_len = l4len; } - if (CONN_V4(conn)) { - if (flags & DUP_ACK) - tcp4_l2_flags_buf_used++; - - if (tcp4_l2_flags_buf_used > TCP_FRAMES_MEM - 2) - tcp_buf_l2_flags_flush(c); - } else { - if (flags & DUP_ACK) - tcp6_l2_flags_buf_used++; - - if (tcp6_l2_flags_buf_used > TCP_FRAMES_MEM - 2) - tcp_buf_l2_flags_flush(c); - } + if (tcp_payload_used > TCP_FRAMES_MEM - 2) + tcp_payload_flush(c); return 0; } @@ -318,49 +237,42 @@ int tcp_buf_send_flag(struct ctx *c, struct tcp_tap_conn *conn, int flags) * tcp_data_to_tap() - Finalise (queue) highest-numbered scatter-gather buffer * @c: Execution context * @conn: Connection pointer - * @plen: Payload length at L4 + * @dlen: TCP payload length * @no_csum: Don't compute IPv4 checksum, use the one from previous buffer * @seq: Sequence number to be sent */ static void tcp_data_to_tap(const struct ctx *c, struct tcp_tap_conn *conn, - ssize_t plen, int no_csum, uint32_t seq) + ssize_t dlen, int no_csum, uint32_t seq) { - uint32_t *seq_update = &conn->seq_to_tap; + struct tcp_payload_t *payload; + const uint16_t *check = NULL; struct iovec *iov; + conn->seq_to_tap = seq + dlen; + tcp_frame_conns[tcp_payload_used] = conn; + iov = tcp_l2_iov[tcp_payload_used]; if (CONN_V4(conn)) { - struct iovec *iov_prev = tcp4_l2_iov[tcp4_l2_buf_used - 1]; - const uint16_t *check = NULL; - if (no_csum) { + struct iovec *iov_prev = tcp_l2_iov[tcp_payload_used - 1]; struct iphdr *iph = iov_prev[TCP_IOV_IP].iov_base; + check = &iph->check; } - - tcp4_l2_buf_seq_update[tcp4_l2_buf_used].seq = seq_update; - tcp4_l2_buf_seq_update[tcp4_l2_buf_used].len = plen; - - iov = tcp4_l2_iov[tcp4_l2_buf_used++]; - iov[TCP_IOV_PAYLOAD].iov_len = tcp_fill_headers4(c, conn, - iov[TCP_IOV_IP].iov_base, - iov[TCP_IOV_PAYLOAD].iov_base, - plen, check, seq); - - if (tcp4_l2_buf_used > TCP_FRAMES_MEM - 1) - tcp_buf_l2_data_flush(c); + iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp4_payload_ip[tcp_payload_used]); + iov[TCP_IOV_ETH].iov_base = &tcp4_eth_src; } else if (CONN_V6(conn)) { - tcp6_l2_buf_seq_update[tcp6_l2_buf_used].seq = seq_update; - tcp6_l2_buf_seq_update[tcp6_l2_buf_used].len = plen; - - iov = tcp6_l2_iov[tcp6_l2_buf_used++]; - iov[TCP_IOV_PAYLOAD].iov_len = tcp_fill_headers6(c, conn, - iov[TCP_IOV_IP].iov_base, - iov[TCP_IOV_PAYLOAD].iov_base, - plen, seq); - - if (tcp6_l2_buf_used > TCP_FRAMES_MEM - 1) - tcp_buf_l2_data_flush(c); + iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp6_payload_ip[tcp_payload_used]); + iov[TCP_IOV_ETH].iov_base = &tcp6_eth_src; } + payload = iov[TCP_IOV_PAYLOAD].iov_base; + payload->th.th_off = sizeof(struct tcphdr) / 4; + payload->th.th_x2 = 0; + payload->th.th_flags = 0; + payload->th.ack = 1; + iov[TCP_IOV_PAYLOAD].iov_len = dlen + sizeof(struct tcphdr); + tcp_l2_buf_fill_headers(conn, iov, dlen, check, seq, false); + if (++tcp_payload_used > TCP_FRAMES_MEM - 1) + tcp_payload_flush(c); } /** @@ -372,17 +284,17 @@ static void tcp_data_to_tap(const struct ctx *c, struct tcp_tap_conn *conn, * * #syscalls recvmsg */ -int tcp_buf_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn) +int tcp_buf_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn) { uint32_t wnd_scaled = conn->wnd_from_tap << conn->ws_from_tap; int fill_bufs, send_bufs = 0, last_len, iov_rem = 0; - int sendlen, len, plen, v4 = CONN_V4(conn); - int s = conn->sock, i, ret = 0; + int len, dlen, i, s = conn->sock; struct msghdr mh_sock = { 0 }; uint16_t mss = MSS_GET(conn); uint32_t already_sent, seq; struct iovec *iov; + /* How much have we read/sent since last received ack ? */ already_sent = conn->seq_to_tap - conn->seq_ack_from_tap; if (SEQ_LT(already_sent, 0)) { @@ -391,6 +303,10 @@ int tcp_buf_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn) conn->seq_ack_from_tap, conn->seq_to_tap); conn->seq_to_tap = conn->seq_ack_from_tap; already_sent = 0; + if (tcp_set_peek_offset(s, 0)) { + tcp_rst(c, conn); + return -1; + } } if (!wnd_scaled || already_sent >= wnd_scaled) { @@ -408,25 +324,26 @@ int tcp_buf_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn) iov_rem = (wnd_scaled - already_sent) % mss; } - mh_sock.msg_iov = iov_sock; - mh_sock.msg_iovlen = fill_bufs + 1; - - iov_sock[0].iov_base = tcp_buf_discard; - iov_sock[0].iov_len = already_sent; + /* Prepare iov according to kernel capability */ + if (!peek_offset_cap) { + mh_sock.msg_iov = iov_sock; + iov_sock[0].iov_base = tcp_buf_discard; + iov_sock[0].iov_len = already_sent; + mh_sock.msg_iovlen = fill_bufs + 1; + } else { + mh_sock.msg_iov = &iov_sock[1]; + mh_sock.msg_iovlen = fill_bufs; + } - if (( v4 && tcp4_l2_buf_used + fill_bufs > TCP_FRAMES_MEM) || - (!v4 && tcp6_l2_buf_used + fill_bufs > TCP_FRAMES_MEM)) { - tcp_buf_l2_data_flush(c); + if (tcp_payload_used + fill_bufs > TCP_FRAMES_MEM) { + tcp_payload_flush(c); /* Silence Coverity CWE-125 false positive */ - tcp4_l2_buf_used = tcp6_l2_buf_used = 0; + tcp_payload_used = 0; } for (i = 0, iov = iov_sock + 1; i < fill_bufs; i++, iov++) { - if (v4) - iov->iov_base = &tcp4_l2_payload[tcp4_l2_buf_used + i].data; - else - iov->iov_base = &tcp6_l2_payload[tcp6_l2_buf_used + i].data; + iov->iov_base = &tcp_payload[tcp_payload_used + i].data; iov->iov_len = mss; } if (iov_rem) @@ -437,12 +354,19 @@ int tcp_buf_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn) len = recvmsg(s, &mh_sock, MSG_PEEK); while (len < 0 && errno == EINTR); - if (len < 0) - goto err; + if (len < 0) { + if (errno != EAGAIN && errno != EWOULDBLOCK) { + tcp_rst(c, conn); + return -errno; + } + + return 0; + } if (!len) { if ((conn->events & (SOCK_FIN_RCVD | TAP_FIN_SENT)) == SOCK_FIN_RCVD) { - if ((ret = tcp_buf_send_flag(c, conn, FIN | ACK))) { + int ret = tcp_buf_send_flag(c, conn, FIN | ACK); + if (ret) { tcp_rst(c, conn); return ret; } @@ -453,42 +377,36 @@ int tcp_buf_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn) return 0; } - sendlen = len - already_sent; - if (sendlen <= 0) { + if (!peek_offset_cap) + len -= already_sent; + + if (len <= 0) { conn_flag(c, conn, STALLED); return 0; } conn_flag(c, conn, ~STALLED); - send_bufs = DIV_ROUND_UP(sendlen, mss); - last_len = sendlen - (send_bufs - 1) * mss; + send_bufs = DIV_ROUND_UP(len, mss); + last_len = len - (send_bufs - 1) * mss; /* Likely, some new data was acked too. */ - tcp_update_seqack_wnd(c, conn, 0, NULL); + tcp_update_seqack_wnd(c, conn, false, NULL); /* Finally, queue to tap */ - plen = mss; + dlen = mss; seq = conn->seq_to_tap; for (i = 0; i < send_bufs; i++) { - int no_csum = i && i != send_bufs - 1 && tcp4_l2_buf_used; + int no_csum = i && i != send_bufs - 1 && tcp_payload_used; if (i == send_bufs - 1) - plen = last_len; + dlen = last_len; - tcp_data_to_tap(c, conn, plen, no_csum, seq); - seq += plen; + tcp_data_to_tap(c, conn, dlen, no_csum, seq); + seq += dlen; } conn_flag(c, conn, ACK_FROM_TAP_DUE); return 0; - -err: - if (errno != EAGAIN && errno != EWOULDBLOCK) { - ret = -errno; - tcp_rst(c, conn); - } - - return ret; } @@ -6,12 +6,9 @@ #ifndef TCP_BUF_H #define TCP_BUF_H -void tcp_buf_sock4_iov_init(const struct ctx *c); -void tcp_buf_sock6_iov_init(const struct ctx *c); -void tcp_buf_l2_flags_flush(const struct ctx *c); -void tcp_buf_l2_data_flush(const struct ctx *c); -uint16_t tcp_buf_conn_tap_mss(const struct tcp_tap_conn *conn); -int tcp_buf_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn); -int tcp_buf_send_flag(struct ctx *c, struct tcp_tap_conn *conn, int flags); +void tcp_sock_iov_init(const struct ctx *c); +void tcp_payload_flush(const struct ctx *c); +int tcp_buf_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn); +int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags); #endif /*TCP_BUF_H */ @@ -13,19 +13,16 @@ * struct tcp_tap_conn - Descriptor for a TCP connection (not spliced) * @f: Generic flow information * @in_epoll: Is the connection in the epoll set? + * @retrans: Number of retransmissions occurred due to ACK_TIMEOUT + * @ws_from_tap: Window scaling factor advertised from tap/guest + * @ws_to_tap: Window scaling factor advertised to tap/guest * @tap_mss: MSS advertised by tap/guest, rounded to 2 ^ TCP_MSS_BITS * @sock: Socket descriptor number * @events: Connection events, implying connection states * @timer: timerfd descriptor for timeout events * @flags: Connection flags representing internal attributes - * @retrans: Number of retransmissions occurred due to ACK_TIMEOUT - * @ws_from_tap: Window scaling factor advertised from tap/guest - * @ws_to_tap: Window scaling factor advertised to tap/guest * @sndbuf: Sending buffer in kernel, rounded to 2 ^ SNDBUF_BITS * @seq_dup_ack_approx: Last duplicate ACK number sent to tap - * @faddr: Guest side forwarding address (guest's remote address) - * @eport: Guest side endpoint port (guest's local port) - * @fport: Guest side forwarding port (guest's remote port) * @wnd_from_tap: Last window size from tap, unscaled (as received) * @wnd_to_tap: Sending window advertised to tap, unscaled (as sent) * @seq_to_tap: Next sequence for packets to tap @@ -49,6 +46,10 @@ struct tcp_tap_conn { unsigned int ws_from_tap :TCP_WS_BITS; unsigned int ws_to_tap :TCP_WS_BITS; +#define TCP_MSS_BITS 14 + unsigned int tap_mss :TCP_MSS_BITS; +#define MSS_SET(conn, mss) (conn->tap_mss = (mss >> (16 - TCP_MSS_BITS))) +#define MSS_GET(conn) (conn->tap_mss << (16 - TCP_MSS_BITS)) int sock :FD_REF_BITS; @@ -77,13 +78,6 @@ struct tcp_tap_conn { #define ACK_TO_TAP_DUE BIT(3) #define ACK_FROM_TAP_DUE BIT(4) - -#define TCP_MSS_BITS 14 - unsigned int tap_mss :TCP_MSS_BITS; -#define MSS_SET(conn, mss) (conn->tap_mss = (mss >> (16 - TCP_MSS_BITS))) -#define MSS_GET(conn) (conn->tap_mss << (16 - TCP_MSS_BITS)) - - #define SNDBUF_BITS 24 unsigned int sndbuf :SNDBUF_BITS; #define SNDBUF_SET(conn, bytes) (conn->sndbuf = ((bytes) >> (32 - SNDBUF_BITS))) @@ -91,11 +85,6 @@ struct tcp_tap_conn { uint8_t seq_dup_ack_approx; - - union inany_addr faddr; - in_port_t eport; - in_port_t fport; - uint16_t wnd_from_tap; uint16_t wnd_to_tap; @@ -106,47 +95,41 @@ struct tcp_tap_conn { uint32_t seq_init_from_tap; }; -#define SIDES 2 /** * struct tcp_splice_conn - Descriptor for a spliced TCP connection * @f: Generic flow information - * @in_epoll: Is the connection in the epoll set? * @s: File descriptor for sockets * @pipe: File descriptors for pipes - * @events: Events observed/actions performed on connection - * @flags: Connection flags (attributes, not events) * @read: Bytes read (not fully written to other side in one shot) * @written: Bytes written (not fully written from one other side read) -*/ + * @events: Events observed/actions performed on connection + * @flags: Connection flags (attributes, not events) + * @in_epoll: Is the connection in the epoll set? + */ struct tcp_splice_conn { /* Must be first element */ struct flow_common f; - bool in_epoll :1; int s[SIDES]; int pipe[SIDES][2]; + uint32_t read[SIDES]; + uint32_t written[SIDES]; + uint8_t events; #define SPLICE_CLOSED 0 #define SPLICE_CONNECT BIT(0) #define SPLICE_ESTABLISHED BIT(1) -#define OUT_WAIT_0 BIT(2) -#define OUT_WAIT_1 BIT(3) -#define FIN_RCVD_0 BIT(4) -#define FIN_RCVD_1 BIT(5) -#define FIN_SENT_0 BIT(6) -#define FIN_SENT_1 BIT(7) +#define OUT_WAIT(sidei_) ((sidei_) ? BIT(3) : BIT(2)) +#define FIN_RCVD(sidei_) ((sidei_) ? BIT(5) : BIT(4)) +#define FIN_SENT(sidei_) ((sidei_) ? BIT(7) : BIT(6)) uint8_t flags; -#define SPLICE_V6 BIT(0) -#define RCVLOWAT_SET_0 BIT(1) -#define RCVLOWAT_SET_1 BIT(2) -#define RCVLOWAT_ACT_0 BIT(3) -#define RCVLOWAT_ACT_1 BIT(4) -#define CLOSING BIT(5) +#define RCVLOWAT_SET(sidei_) ((sidei_) ? BIT(1) : BIT(0)) +#define RCVLOWAT_ACT(sidei_) ((sidei_) ? BIT(3) : BIT(2)) +#define CLOSING BIT(4) - uint32_t read[SIDES]; - uint32_t written[SIDES]; + bool in_epoll :1; }; /* Socket pools */ @@ -155,9 +138,9 @@ struct tcp_splice_conn { extern int init_sock_pool4 [TCP_SOCK_POOL_SIZE]; extern int init_sock_pool6 [TCP_SOCK_POOL_SIZE]; -bool tcp_flow_defer(union flow *flow); -bool tcp_splice_flow_defer(union flow *flow); -void tcp_splice_timer(const struct ctx *c, union flow *flow); +bool tcp_flow_defer(const struct tcp_tap_conn *conn); +bool tcp_splice_flow_defer(struct tcp_splice_conn *conn); +void tcp_splice_timer(const struct ctx *c, struct tcp_splice_conn *conn); int tcp_conn_pool_sock(int pool[]); int tcp_conn_sock(const struct ctx *c, sa_family_t af); int tcp_sock_refill_pool(const struct ctx *c, int pool[], sa_family_t af); diff --git a/tcp_internal.h b/tcp_internal.h index 1b5dbed..d7b125f 100644 --- a/tcp_internal.h +++ b/tcp_internal.h @@ -8,7 +8,15 @@ #define MAX_WS 8 #define MAX_WINDOW (1 << (16 + (MAX_WS))) -#define MSS (USHRT_MAX - sizeof(struct tcphdr)) + +#define MSS4 ROUND_DOWN(IP_MAX_MTU - \ + sizeof(struct tcphdr) - \ + sizeof(struct iphdr), \ + sizeof(uint32_t)) +#define MSS6 ROUND_DOWN(IP_MAX_MTU - \ + sizeof(struct tcphdr) - \ + sizeof(struct ipv6hdr), \ + sizeof(uint32_t)) #define SEQ_LE(a, b) ((b) - (a) < MAX_WINDOW) #define SEQ_LT(a, b) ((b) - (a) - 1 < MAX_WINDOW) @@ -25,17 +33,108 @@ #define OPT_EOL 0 #define OPT_NOP 1 #define OPT_MSS 2 -#define OPT_MSS_LEN 4 #define OPT_WS 3 -#define OPT_WS_LEN 3 #define OPT_SACKP 4 #define OPT_SACK 5 #define OPT_TS 8 -#define CONN_V4(conn) (!!inany_v4(&(conn)->faddr)) +#define TAPSIDE(conn_) ((conn_)->f.pif[1] == PIF_TAP) +#define TAPFLOW(conn_) (&((conn_)->f.side[TAPSIDE(conn_)])) +#define TAP_SIDX(conn_) (FLOW_SIDX((conn_), TAPSIDE(conn_))) + +#define CONN_V4(conn) (!!inany_v4(&TAPFLOW(conn)->oaddr)) #define CONN_V6(conn) (!CONN_V4(conn)) -extern char tcp_buf_discard[MAX_WINDOW]; +/* + * enum tcp_iov_parts - I/O vector parts for one TCP frame + * @TCP_IOV_TAP tap backend specific header + * @TCP_IOV_ETH Ethernet header + * @TCP_IOV_IP IP (v4/v6) header + * @TCP_IOV_PAYLOAD IP payload (TCP header + data) + * @TCP_NUM_IOVS the number of entries in the iovec array + */ +enum tcp_iov_parts { + TCP_IOV_TAP = 0, + TCP_IOV_ETH = 1, + TCP_IOV_IP = 2, + TCP_IOV_PAYLOAD = 3, + TCP_NUM_IOVS +}; + +/** + * struct tcp_payload_t - TCP header and data to send segments with payload + * @th: TCP header + * @data: TCP data + */ +struct tcp_payload_t { + struct tcphdr th; + uint8_t data[IP_MAX_MTU - sizeof(struct tcphdr)]; +#ifdef __AVX2__ +} __attribute__ ((packed, aligned(32))); /* For AVX2 checksum routines */ +#else +} __attribute__ ((packed, aligned(__alignof__(unsigned int)))); +#endif + +/** struct tcp_opt_nop - TCP NOP option + * @kind: Option kind (OPT_NOP = 1) + */ +struct tcp_opt_nop { + uint8_t kind; +} __attribute__ ((packed)); +#define TCP_OPT_NOP ((struct tcp_opt_nop){ .kind = OPT_NOP, }) + +/** struct tcp_opt_mss - TCP MSS option + * @kind: Option kind (OPT_MSS == 2) + * @len: Option length (4) + * @mss: Maximum Segment Size + */ +struct tcp_opt_mss { + uint8_t kind; + uint8_t len; + uint16_t mss; +} __attribute__ ((packed)); +#define TCP_OPT_MSS(mss_) \ + ((struct tcp_opt_mss) { \ + .kind = OPT_MSS, \ + .len = sizeof(struct tcp_opt_mss), \ + .mss = htons(mss_), \ + }) + +/** struct tcp_opt_ws - TCP Window Scaling option + * @kind: Option kind (OPT_WS == 3) + * @len: Option length (3) + * @shift: Window scaling shift + */ +struct tcp_opt_ws { + uint8_t kind; + uint8_t len; + uint8_t shift; +} __attribute__ ((packed)); +#define TCP_OPT_WS(shift_) \ + ((struct tcp_opt_ws) { \ + .kind = OPT_WS, \ + .len = sizeof(struct tcp_opt_ws), \ + .shift = (shift_), \ + }) + +/** struct tcp_syn_opts - TCP options we apply to SYN packets + * @mss: Maximum Segment Size (MSS) option + * @nop: NOP opt (for alignment) + * @ws: Window Scaling (WS) option + */ +struct tcp_syn_opts { + struct tcp_opt_mss mss; + struct tcp_opt_nop nop; + struct tcp_opt_ws ws; +} __attribute__ ((packed)); +#define TCP_SYN_OPTS(mss_, ws_) \ + ((struct tcp_syn_opts){ \ + .mss = TCP_OPT_MSS(mss_), \ + .nop = TCP_OPT_NOP, \ + .ws = TCP_OPT_WS(ws_), \ + }) + +extern char tcp_buf_discard [MAX_WINDOW]; void conn_flag_do(const struct ctx *c, struct tcp_tap_conn *conn, unsigned long flag); @@ -54,28 +153,34 @@ void conn_event_do(const struct ctx *c, struct tcp_tap_conn *conn, conn_event_do(c, conn, event); \ } while (0) -void tcp_rst_do(struct ctx *c, struct tcp_tap_conn *conn); +void tcp_rst_do(const struct ctx *c, struct tcp_tap_conn *conn); #define tcp_rst(c, conn) \ do { \ flow_dbg((conn), "TCP reset at %s:%i", __func__, __LINE__); \ tcp_rst_do(c, conn); \ } while (0) - - -size_t tcp_fill_headers4(const struct ctx *c, - const struct tcp_tap_conn *conn, - struct iphdr *iph, struct tcphdr *th, - size_t plen, const uint16_t *check, - uint32_t seq); -size_t tcp_fill_headers6(const struct ctx *c, - const struct tcp_tap_conn *conn, - struct ipv6hdr *ip6h, struct tcphdr *th, - size_t plen, uint32_t seq); +struct tcp_info_linux; + +void tcp_update_check_tcp4(const struct iphdr *iph, + const struct iovec *iov, int iov_cnt, + size_t l4offset); +void tcp_update_check_tcp6(const struct ipv6hdr *ip6h, + const struct iovec *iov, int iov_cnt, + size_t l4offset); +void tcp_fill_headers4(const struct tcp_tap_conn *conn, + struct tap_hdr *taph, struct iphdr *iph, + struct tcp_payload_t *bp, size_t dlen, + const uint16_t *check, uint32_t seq, bool no_tcp_csum); +void tcp_fill_headers6(const struct tcp_tap_conn *conn, + struct tap_hdr *taph, struct ipv6hdr *ip6h, + struct tcp_payload_t *bp, size_t dlen, + uint32_t seq, bool no_tcp_csum); int tcp_update_seqack_wnd(const struct ctx *c, struct tcp_tap_conn *conn, - int force_seq, struct tcp_info *tinfo); -int tcp_fill_flag_header(struct ctx *c, struct tcp_tap_conn *conn, int flags, - struct tcphdr *th, char *opts, size_t *optlen); + bool force_seq, struct tcp_info_linux *tinfo); +int tcp_prepare_flags(const struct ctx *c, struct tcp_tap_conn *conn, + int flags, struct tcphdr *th, struct tcp_syn_opts *opts, + size_t *optlen); #endif /* TCP_INTERNAL_H */ diff --git a/tcp_splice.c b/tcp_splice.c index d066112..93f8bce 100644 --- a/tcp_splice.c +++ b/tcp_splice.c @@ -28,7 +28,7 @@ * - FIN_SENT_0: FIN (write shutdown) sent to accepted socket * - FIN_SENT_1: FIN (write shutdown) sent to target socket * - * #syscalls:pasta pipe2|pipe fcntl armv6l:fcntl64 armv7l:fcntl64 ppc64:fcntl64 + * #syscalls:pasta pipe2|pipe fcntl arm:fcntl64 ppc64:fcntl64 i686:fcntl64 */ #include <sched.h> @@ -73,10 +73,7 @@ static int ns_sock_pool6 [TCP_SOCK_POOL_SIZE]; /* Pool of pre-opened pipes */ static int splice_pipe_pool [TCP_SPLICE_PIPE_POOL_SIZE][2]; -#define CONN_V6(x) (x->flags & SPLICE_V6) -#define CONN_V4(x) (!CONN_V6(x)) -#define CONN_HAS(conn, set) ((conn->events & (set)) == (set)) -#define CONN(idx) (&FLOW(idx)->tcp_splice) +#define CONN_HAS(conn, set) (((conn)->events & (set)) == (set)) /* Display strings for connection events */ static const char *tcp_splice_event_str[] __attribute((__unused__)) = { @@ -95,6 +92,24 @@ static int tcp_sock_refill_ns(void *arg); static int tcp_conn_sock_ns(const struct ctx *c, sa_family_t af); /** + * conn_at_sidx() - Get spliced TCP connection specific flow at given sidx + * @sidx: Flow and side to retrieve + * + * Return: Spliced TCP connection at @sidx, or NULL of @sidx is invalid. + * Asserts if the flow at @sidx is not FLOW_TCP_SPLICE. + */ +static struct tcp_splice_conn *conn_at_sidx(flow_sidx_t sidx) +{ + union flow *flow = flow_at_sidx(sidx); + + if (!flow) + return NULL; + + ASSERT(flow->f.type == FLOW_TCP_SPLICE); + return &flow->tcp_splice; +} + +/** * tcp_splice_conn_epoll_events() - epoll events masks for given state * @events: Connection event flags * @ev: Events to fill in, 0 is accepted socket, 1 is connecting socket @@ -102,19 +117,22 @@ static int tcp_conn_sock_ns(const struct ctx *c, sa_family_t af); static void tcp_splice_conn_epoll_events(uint16_t events, struct epoll_event ev[]) { - ev[0].events = ev[1].events = 0; + unsigned sidei; + + flow_foreach_sidei(sidei) + ev[sidei].events = 0; if (events & SPLICE_ESTABLISHED) { - if (!(events & FIN_SENT_1)) - ev[0].events = EPOLLIN | EPOLLRDHUP; - if (!(events & FIN_SENT_0)) - ev[1].events = EPOLLIN | EPOLLRDHUP; + flow_foreach_sidei(sidei) { + if (!(events & FIN_SENT(!sidei))) + ev[sidei].events = EPOLLIN | EPOLLRDHUP; + } } else if (events & SPLICE_CONNECT) { ev[1].events = EPOLLOUT; } - ev[0].events |= (events & OUT_WAIT_0) ? EPOLLOUT : 0; - ev[1].events |= (events & OUT_WAIT_1) ? EPOLLOUT : 0; + flow_foreach_sidei(sidei) + ev[sidei].events |= (events & OUT_WAIT(sidei)) ? EPOLLOUT : 0; } /** @@ -235,32 +253,31 @@ static void conn_event_do(const struct ctx *c, struct tcp_splice_conn *conn, /** * tcp_splice_flow_defer() - Deferred per-flow handling (clean up closed) - * @flow: Flow table entry for this connection + * @conn: Connection entry to handle * * Return: true if the flow is ready to free, false otherwise */ -bool tcp_splice_flow_defer(union flow *flow) +bool tcp_splice_flow_defer(struct tcp_splice_conn *conn) { - struct tcp_splice_conn *conn = &flow->tcp_splice; - unsigned side; + unsigned sidei; - if (!(flow->tcp_splice.flags & CLOSING)) + if (!(conn->flags & CLOSING)) return false; - for (side = 0; side < SIDES; side++) { + flow_foreach_sidei(sidei) { /* Flushing might need to block: don't recycle them. */ - if (conn->pipe[side][0] >= 0) { - close(conn->pipe[side][0]); - close(conn->pipe[side][1]); - conn->pipe[side][0] = conn->pipe[side][1] = -1; + if (conn->pipe[sidei][0] >= 0) { + close(conn->pipe[sidei][0]); + close(conn->pipe[sidei][1]); + conn->pipe[sidei][0] = conn->pipe[sidei][1] = -1; } - if (conn->s[side] >= 0) { - close(conn->s[side]); - conn->s[side] = -1; + if (conn->s[sidei] >= 0) { + close(conn->s[sidei]); + conn->s[sidei] = -1; } - conn->read[side] = conn->written[side] = 0; + conn->read[sidei] = conn->written[sidei] = 0; } conn->events = SPLICE_CLOSED; @@ -280,33 +297,33 @@ bool tcp_splice_flow_defer(union flow *flow) static int tcp_splice_connect_finish(const struct ctx *c, struct tcp_splice_conn *conn) { - unsigned side; + unsigned sidei; int i = 0; - for (side = 0; side < SIDES; side++) { + flow_foreach_sidei(sidei) { for (; i < TCP_SPLICE_PIPE_POOL_SIZE; i++) { if (splice_pipe_pool[i][0] >= 0) { - SWAP(conn->pipe[side][0], + SWAP(conn->pipe[sidei][0], splice_pipe_pool[i][0]); - SWAP(conn->pipe[side][1], + SWAP(conn->pipe[sidei][1], splice_pipe_pool[i][1]); break; } } - if (conn->pipe[side][0] < 0) { - if (pipe2(conn->pipe[side], O_NONBLOCK | O_CLOEXEC)) { + if (conn->pipe[sidei][0] < 0) { + if (pipe2(conn->pipe[sidei], O_NONBLOCK | O_CLOEXEC)) { flow_err(conn, "cannot create %d->%d pipe: %s", - side, !side, strerror(errno)); + sidei, !sidei, strerror(errno)); conn_flag(c, conn, CLOSING); return -EIO; } - if (fcntl(conn->pipe[side][0], F_SETPIPE_SZ, - c->tcp.pipe_size)) { + if (fcntl(conn->pipe[sidei][0], F_SETPIPE_SZ, + c->tcp.pipe_size) != (int)c->tcp.pipe_size) { flow_trace(conn, "cannot set %d->%d pipe size to %zu", - side, !side, c->tcp.pipe_size); + sidei, !sidei, c->tcp.pipe_size); } } } @@ -321,31 +338,20 @@ static int tcp_splice_connect_finish(const struct ctx *c, * tcp_splice_connect() - Create and connect socket for new spliced connection * @c: Execution context * @conn: Connection pointer - * @af: Address family - * @pif: pif on which to create socket - * @port: Destination port, host order * * Return: 0 for connect() succeeded or in progress, negative value on error */ -static int tcp_splice_connect(const struct ctx *c, struct tcp_splice_conn *conn, - sa_family_t af, uint8_t pif, in_port_t port) +static int tcp_splice_connect(const struct ctx *c, struct tcp_splice_conn *conn) { - struct sockaddr_in6 addr6 = { - .sin6_family = AF_INET6, - .sin6_port = htons(port), - .sin6_addr = IN6ADDR_LOOPBACK_INIT, - }; - struct sockaddr_in addr4 = { - .sin_family = AF_INET, - .sin_port = htons(port), - .sin_addr = IN4ADDR_LOOPBACK_INIT, - }; - const struct sockaddr *sa; + const struct flowside *tgt = &conn->f.side[TGTSIDE]; + sa_family_t af = inany_v4(&tgt->eaddr) ? AF_INET : AF_INET6; + uint8_t tgtpif = conn->f.pif[TGTSIDE]; + union sockaddr_inany sa; socklen_t sl; - if (pif == PIF_HOST) + if (tgtpif == PIF_HOST) conn->s[1] = tcp_conn_sock(c, af); - else if (pif == PIF_SPLICE) + else if (tgtpif == PIF_SPLICE) conn->s[1] = tcp_conn_sock_ns(c, af); else ASSERT(0); @@ -359,15 +365,9 @@ static int tcp_splice_connect(const struct ctx *c, struct tcp_splice_conn *conn, conn->s[1]); } - if (CONN_V6(conn)) { - sa = (struct sockaddr *)&addr6; - sl = sizeof(addr6); - } else { - sa = (struct sockaddr *)&addr4; - sl = sizeof(addr4); - } + pif_sockaddr(c, &sa, &sl, tgtpif, &tgt->eaddr, tgt->eport); - if (connect(conn->s[1], sa, sl)) { + if (connect(conn->s[1], &sa.sa, sl)) { if (errno != EINPROGRESS) { flow_trace(conn, "Couldn't connect socket for splice: %s", strerror(errno)); @@ -414,67 +414,19 @@ static int tcp_conn_sock_ns(const struct ctx *c, sa_family_t af) /** * tcp_splice_conn_from_sock() - Attempt to init state for a spliced connection * @c: Execution context - * @pif0: pif id of side 0 - * @dstport: Side 0 destination port of connection * @flow: flow to initialise * @s0: Accepted (side 0) socket * @sa: Peer address of connection * - * Return: true if able to create a spliced connection, false otherwise * #syscalls:pasta setsockopt */ -bool tcp_splice_conn_from_sock(const struct ctx *c, - uint8_t pif0, in_port_t dstport, - union flow *flow, int s0, - const union sockaddr_inany *sa) +void tcp_splice_conn_from_sock(const struct ctx *c, union flow *flow, int s0) { - struct tcp_splice_conn *conn; - union inany_addr src; - in_port_t srcport; - sa_family_t af; - uint8_t pif1; + struct tcp_splice_conn *conn = FLOW_SET_TYPE(flow, FLOW_TCP_SPLICE, + tcp_splice); - if (c->mode != MODE_PASTA) - return false; + ASSERT(c->mode == MODE_PASTA); - inany_from_sockaddr(&src, &srcport, sa); - af = inany_v4(&src) ? AF_INET : AF_INET6; - - switch (pif0) { - case PIF_SPLICE: - if (!inany_is_loopback(&src)) { - char str[INANY_ADDRSTRLEN]; - - /* We can't use flow_err() etc. because we haven't set - * the flow type yet - */ - warn("Bad source address %s for splice, closing", - inany_ntop(&src, str, sizeof(str))); - - /* We *don't* want to fall back to tap */ - flow_alloc_cancel(flow); - return true; - } - - pif1 = PIF_HOST; - dstport += c->tcp.fwd_out.delta[dstport]; - break; - - case PIF_HOST: - if (!inany_is_loopback(&src)) - return false; - - pif1 = PIF_SPLICE; - dstport += c->tcp.fwd_in.delta[dstport]; - break; - - default: - return false; - } - - conn = FLOW_START(flow, FLOW_TCP_SPLICE, tcp_splice, 0); - - conn->flags = af == AF_INET ? 0 : SPLICE_V6; conn->s[0] = s0; conn->s[1] = -1; conn->pipe[0][0] = conn->pipe[0][1] = -1; @@ -483,10 +435,10 @@ bool tcp_splice_conn_from_sock(const struct ctx *c, if (setsockopt(s0, SOL_TCP, TCP_QUICKACK, &((int){ 1 }), sizeof(int))) flow_trace(conn, "failed to set TCP_QUICKACK on %i", s0); - if (tcp_splice_connect(c, conn, af, pif1, dstport)) + if (tcp_splice_connect(c, conn)) conn_flag(c, conn, CLOSING); - return true; + FLOW_ACTIVATE(conn); } /** @@ -500,8 +452,8 @@ bool tcp_splice_conn_from_sock(const struct ctx *c, void tcp_splice_sock_handler(struct ctx *c, union epoll_ref ref, uint32_t events) { - struct tcp_splice_conn *conn = CONN(ref.flowside.flow); - unsigned side = ref.flowside.side, fromside; + struct tcp_splice_conn *conn = conn_at_sidx(ref.flowside); + unsigned evsidei = ref.flowside.sidei, fromsidei; uint8_t lowat_set_flag, lowat_act_flag; int eof, never_read; @@ -533,30 +485,31 @@ void tcp_splice_sock_handler(struct ctx *c, union epoll_ref ref, } if (events & EPOLLOUT) { - fromside = !side; - conn_event(c, conn, side == 0 ? ~OUT_WAIT_0 : ~OUT_WAIT_1); + fromsidei = !evsidei; + conn_event(c, conn, ~OUT_WAIT(evsidei)); } else { - fromside = side; + fromsidei = evsidei; } if (events & EPOLLRDHUP) /* For side 0 this is fake, but implied */ - conn_event(c, conn, side == 0 ? FIN_RCVD_0 : FIN_RCVD_1); + conn_event(c, conn, FIN_RCVD(evsidei)); swap: eof = 0; never_read = 1; - lowat_set_flag = fromside == 0 ? RCVLOWAT_SET_0 : RCVLOWAT_SET_1; - lowat_act_flag = fromside == 0 ? RCVLOWAT_ACT_0 : RCVLOWAT_ACT_1; + lowat_set_flag = RCVLOWAT_SET(fromsidei); + lowat_act_flag = RCVLOWAT_ACT(fromsidei); while (1) { - ssize_t readlen, to_write = 0, written; + ssize_t readlen, written, pending; int more = 0; retry: - readlen = splice(conn->s[fromside], NULL, - conn->pipe[fromside][1], NULL, c->tcp.pipe_size, + readlen = splice(conn->s[fromsidei], NULL, + conn->pipe[fromsidei][1], NULL, + c->tcp.pipe_size, SPLICE_F_MOVE | SPLICE_F_NONBLOCK); flow_trace(conn, "%zi from read-side call", readlen); if (readlen < 0) { @@ -565,14 +518,11 @@ retry: if (errno != EAGAIN) goto close; - - to_write = c->tcp.pipe_size; } else if (!readlen) { eof = 1; - to_write = c->tcp.pipe_size; } else { never_read = 0; - to_write += readlen; + if (readlen >= (long)c->tcp.pipe_size * 90 / 100) more = SPLICE_F_MORE; @@ -581,11 +531,11 @@ retry: } eintr: - written = splice(conn->pipe[fromside][0], NULL, - conn->s[!fromside], NULL, to_write, + written = splice(conn->pipe[fromsidei][0], NULL, + conn->s[!fromsidei], NULL, c->tcp.pipe_size, SPLICE_F_MOVE | more | SPLICE_F_NONBLOCK); flow_trace(conn, "%zi from write-side call (passed %zi)", - written, to_write); + written, c->tcp.pipe_size); /* Most common case: skip updating counters. */ if (readlen > 0 && readlen == written) { @@ -596,18 +546,23 @@ eintr: readlen > (long)c->tcp.pipe_size / 10) { int lowat = c->tcp.pipe_size / 4; - setsockopt(conn->s[fromside], SOL_SOCKET, - SO_RCVLOWAT, &lowat, sizeof(lowat)); - - conn_flag(c, conn, lowat_set_flag); - conn_flag(c, conn, lowat_act_flag); + if (setsockopt(conn->s[fromsidei], SOL_SOCKET, + SO_RCVLOWAT, + &lowat, sizeof(lowat))) { + flow_trace(conn, + "Setting SO_RCVLOWAT %i: %s", + lowat, strerror(errno)); + } else { + conn_flag(c, conn, lowat_set_flag); + conn_flag(c, conn, lowat_act_flag); + } } break; } - conn->read[fromside] += readlen > 0 ? readlen : 0; - conn->written[fromside] += written > 0 ? written : 0; + conn->read[fromsidei] += readlen > 0 ? readlen : 0; + conn->written[fromsidei] += written > 0 ? written : 0; if (written < 0) { if (errno == EINTR) @@ -616,47 +571,43 @@ eintr: if (errno != EAGAIN) goto close; - if (never_read) + if (conn->read[fromsidei] == conn->written[fromsidei]) break; - conn_event(c, conn, - fromside == 0 ? OUT_WAIT_1 : OUT_WAIT_0); + conn_event(c, conn, OUT_WAIT(!fromsidei)); break; } if (never_read && written == (long)(c->tcp.pipe_size)) goto retry; - if (!never_read && written < to_write) { - to_write -= written; + pending = conn->read[fromsidei] - conn->written[fromsidei]; + if (!never_read && written > 0 && written < pending) goto retry; - } if (eof) break; } - if ((conn->events & FIN_RCVD_0) && !(conn->events & FIN_SENT_1)) { - if (conn->read[fromside] == conn->written[fromside] && eof) { - shutdown(conn->s[1], SHUT_WR); - conn_event(c, conn, FIN_SENT_1); - } - } + if (conn->read[fromsidei] == conn->written[fromsidei] && eof) { + unsigned sidei; - if ((conn->events & FIN_RCVD_1) && !(conn->events & FIN_SENT_0)) { - if (conn->read[fromside] == conn->written[fromside] && eof) { - shutdown(conn->s[0], SHUT_WR); - conn_event(c, conn, FIN_SENT_0); + flow_foreach_sidei(sidei) { + if ((conn->events & FIN_RCVD(sidei)) && + !(conn->events & FIN_SENT(!sidei))) { + shutdown(conn->s[!sidei], SHUT_WR); + conn_event(c, conn, FIN_SENT(!sidei)); + } } } - if (CONN_HAS(conn, FIN_SENT_0 | FIN_SENT_1)) + if (CONN_HAS(conn, FIN_SENT(0) | FIN_SENT(1))) goto close; if ((events & (EPOLLIN | EPOLLOUT)) == (EPOLLIN | EPOLLOUT)) { events = EPOLLIN; - fromside = !fromside; + fromsidei = !fromsidei; goto swap; } @@ -721,7 +672,7 @@ static void tcp_splice_pipe_refill(const struct ctx *c) continue; if (fcntl(splice_pipe_pool[i][0], F_SETPIPE_SZ, - c->tcp.pipe_size)) { + c->tcp.pipe_size) != (int)c->tcp.pipe_size) { trace("TCP (spliced): cannot set pool pipe size to %zu", c->tcp.pipe_size); } @@ -734,6 +685,7 @@ static void tcp_splice_pipe_refill(const struct ctx *c) * * Return: 0 */ +/* cppcheck-suppress [constParameterCallback, unmatchedSuppression] */ static int tcp_sock_refill_ns(void *arg) { const struct ctx *c = (const struct ctx *)arg; @@ -786,29 +738,26 @@ void tcp_splice_init(struct ctx *c) /** * tcp_splice_timer() - Timer for spliced connections * @c: Execution context - * @flow: Flow table entry + * @conn: Connection to handle */ -void tcp_splice_timer(const struct ctx *c, union flow *flow) +void tcp_splice_timer(const struct ctx *c, struct tcp_splice_conn *conn) { - struct tcp_splice_conn *conn = &flow->tcp_splice; - int side; + unsigned sidei; ASSERT(!(conn->flags & CLOSING)); - for (side = 0; side < SIDES; side++) { - uint8_t set = side == 0 ? RCVLOWAT_SET_0 : RCVLOWAT_SET_1; - uint8_t act = side == 0 ? RCVLOWAT_ACT_0 : RCVLOWAT_ACT_1; - - if ((conn->flags & set) && !(conn->flags & act)) { - if (setsockopt(conn->s[side], SOL_SOCKET, SO_RCVLOWAT, + flow_foreach_sidei(sidei) { + if ((conn->flags & RCVLOWAT_SET(sidei)) && + !(conn->flags & RCVLOWAT_ACT(sidei))) { + if (setsockopt(conn->s[sidei], SOL_SOCKET, SO_RCVLOWAT, &((int){ 1 }), sizeof(int))) { flow_trace(conn, "can't set SO_RCVLOWAT on %d", - conn->s[side]); + conn->s[sidei]); } - conn_flag(c, conn, ~set); + conn_flag(c, conn, ~RCVLOWAT_SET(sidei)); } } - conn_flag(c, conn, ~RCVLOWAT_ACT_0); - conn_flag(c, conn, ~RCVLOWAT_ACT_1); + flow_foreach_sidei(sidei) + conn_flag(c, conn, ~RCVLOWAT_ACT(sidei)); } diff --git a/tcp_splice.h b/tcp_splice.h index ed8f0c5..a20f3e2 100644 --- a/tcp_splice.h +++ b/tcp_splice.h @@ -11,10 +11,7 @@ union sockaddr_inany; void tcp_splice_sock_handler(struct ctx *c, union epoll_ref ref, uint32_t events); -bool tcp_splice_conn_from_sock(const struct ctx *c, - uint8_t pif0, in_port_t dstport, - union flow *flow, int s0, - const union sockaddr_inany *sa); +void tcp_splice_conn_from_sock(const struct ctx *c, union flow *flow, int s0); void tcp_splice_init(struct ctx *c); #endif /* TCP_SPLICE_H */ @@ -1,14 +1,19 @@ // SPDX-License-Identifier: GPL-2.0-or-later +/* tcp_vu.c - TCP L2 vhost-user management functions + * + * Copyright Red Hat + * Author: Laurent Vivier <lvivier@redhat.com> + */ #include <errno.h> #include <stddef.h> #include <stdint.h> #include <netinet/ip.h> +#include <netinet/tcp.h> #include <sys/socket.h> -#include <linux/tcp.h> #include <linux/virtio_net.h> #include "util.h" @@ -23,177 +28,340 @@ #include "tcp_conn.h" #include "flow_table.h" #include "tcp_vu.h" +#include "tap.h" #include "tcp_internal.h" #include "checksum.h" +#include "vu_common.h" +#include <time.h> + +static struct iovec iov_vu[VIRTQUEUE_MAX_SIZE + 1]; +static struct vu_virtq_element elem[VIRTQUEUE_MAX_SIZE]; + +/** + * tcp_vu_hdrlen() - return the size of the header in level 2 frame (TCP) + * @v6: Set for IPv6 packet + * + * Return: Return the size of the header + */ +static size_t tcp_vu_hdrlen(bool v6) +{ + size_t hdrlen; -#define CONN_V4(conn) (!!inany_v4(&(conn)->faddr)) -#define CONN_V6(conn) (!CONN_V4(conn)) + hdrlen = sizeof(struct virtio_net_hdr_mrg_rxbuf) + + sizeof(struct ethhdr) + sizeof(struct tcphdr); -/* vhost-user */ -static const struct virtio_net_hdr vu_header = { - .flags = VIRTIO_NET_HDR_F_DATA_VALID, - .gso_type = VIRTIO_NET_HDR_GSO_NONE, -}; + if (v6) + hdrlen += sizeof(struct ipv6hdr); + else + hdrlen += sizeof(struct iphdr); -static unsigned char buffer[65536]; -static struct iovec iov_vu [VIRTQUEUE_MAX_SIZE]; -static unsigned int indexes [VIRTQUEUE_MAX_SIZE]; + return hdrlen; +} -uint16_t tcp_vu_conn_tap_mss(const struct tcp_tap_conn *conn) +/** + * tcp_vu_update_check() - Calculate TCP checksum + * @tapside: Address information for one side of the flow + * @iov: Pointer to the array of IO vectors + * @iov_used: Length of the array + */ +static void tcp_vu_update_check(const struct flowside *tapside, + struct iovec *iov, int iov_used) { - (void)conn; - return USHRT_MAX; + char *base = iov[0].iov_base; + + if (inany_v4(&tapside->oaddr)) { + const struct iphdr *iph = vu_ip(base); + + tcp_update_check_tcp4(iph, iov, iov_used, + (char *)vu_payloadv4(base) - base); + } else { + const struct ipv6hdr *ip6h = vu_ip(base); + + tcp_update_check_tcp6(ip6h, iov, iov_used, + (char *)vu_payloadv6(base) - base); + } } -int tcp_vu_send_flag(struct ctx *c, struct tcp_tap_conn *conn, int flags) +/** + * tcp_vu_send_flag() - Send segment with flags to vhost-user (no payload) + * @c: Execution context + * @conn: Connection pointer + * @flags: TCP flags: if not set, send segment only if ACK is due + * + * Return: negative error code on connection reset, 0 otherwise + */ +int tcp_vu_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags) { - VuDev *vdev = (VuDev *)&c->vdev; - VuVirtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE]; - size_t tlen, vnet_hdrlen, ip_len, optlen = 0; - struct virtio_net_hdr_mrg_rxbuf *vh; - VuVirtqElement *elem; + struct vu_dev *vdev = c->vdev; + struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE]; + const struct flowside *tapside = TAPFLOW(conn); + size_t optlen, hdrlen; + struct vu_virtq_element flags_elem[2]; + struct tcp_payload_t *payload; + struct ipv6hdr *ip6h = NULL; + struct iovec flags_iov[2]; + struct iphdr *iph = NULL; struct ethhdr *eh; + uint32_t seq; + int elem_cnt; int nb_ack; int ret; - elem = vu_queue_pop(vdev, vq, sizeof(VuVirtqElement), buffer); - if (!elem) - return 0; + hdrlen = tcp_vu_hdrlen(CONN_V6(conn)); - if (elem->in_num < 1) { - err("virtio-net receive queue contains no in buffers"); - vu_queue_rewind(vdev, vq, 1); - return 0; - } + vu_set_element(&flags_elem[0], NULL, &flags_iov[0]); + + elem_cnt = vu_collect(vdev, vq, &flags_elem[0], 1, + hdrlen + sizeof(struct tcp_syn_opts), NULL); + if (elem_cnt != 1) + return -1; + + vu_set_vnethdr(vdev, flags_elem[0].in_sg[0].iov_base, 1); + + eh = vu_eth(flags_elem[0].in_sg[0].iov_base); + + memcpy(eh->h_dest, c->guest_mac, sizeof(eh->h_dest)); + memcpy(eh->h_source, c->our_tap_mac, sizeof(eh->h_source)); - vh = elem->in_sg[0].iov_base; + if (CONN_V4(conn)) { + eh->h_proto = htons(ETH_P_IP); + + iph = vu_ip(flags_elem[0].in_sg[0].iov_base); + *iph = (struct iphdr)L2_BUF_IP4_INIT(IPPROTO_TCP); - vh->hdr = vu_header; - if (vu_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) { - vnet_hdrlen = sizeof(struct virtio_net_hdr_mrg_rxbuf); - vh->num_buffers = htole16(1); + payload = vu_payloadv4(flags_elem[0].in_sg[0].iov_base); } else { - vnet_hdrlen = sizeof(struct virtio_net_hdr); + eh->h_proto = htons(ETH_P_IPV6); + + ip6h = vu_ip(flags_elem[0].in_sg[0].iov_base); + *ip6h = (struct ipv6hdr)L2_BUF_IP6_INIT(IPPROTO_TCP); + payload = vu_payloadv6(flags_elem[0].in_sg[0].iov_base); + } + + memset(&payload->th, 0, sizeof(payload->th)); + payload->th.doff = offsetof(struct tcp_payload_t, data) / 4; + payload->th.ack = 1; + + seq = conn->seq_to_tap; + ret = tcp_prepare_flags(c, conn, flags, &payload->th, + (struct tcp_syn_opts *)payload->data, + &optlen); + if (ret <= 0) { + vu_queue_rewind(vq, 1); + return ret; } - eh = (struct ethhdr *)((char *)elem->in_sg[0].iov_base + vnet_hdrlen); - memcpy(eh->h_dest, c->mac_guest, sizeof(eh->h_dest)); - memcpy(eh->h_source, c->mac, sizeof(eh->h_source)); + flags_elem[0].in_sg[0].iov_len = hdrlen + optlen; if (CONN_V4(conn)) { - struct iphdr *iph = (struct iphdr *)(eh + 1); - struct tcphdr *th = (struct tcphdr *)(iph + 1); - char *data = (char *)(th + 1); + tcp_fill_headers4(conn, NULL, iph, payload, optlen, NULL, seq, + true); + } else { + tcp_fill_headers6(conn, NULL, ip6h, payload, optlen, seq, true); + } - eh->h_proto = htons(ETH_P_IP); + if (*c->pcap) { + tcp_vu_update_check(tapside, &flags_elem[0].in_sg[0], 1); + pcap_iov(&flags_elem[0].in_sg[0], 1, + sizeof(struct virtio_net_hdr_mrg_rxbuf)); + } + nb_ack = 1; - *th = (struct tcphdr){ - .doff = sizeof(struct tcphdr) / 4, - .ack = 1 - }; + if (flags & DUP_ACK) { + vu_set_element(&flags_elem[1], NULL, &flags_iov[1]); + + elem_cnt = vu_collect(vdev, vq, &flags_elem[1], 1, + flags_elem[0].in_sg[0].iov_len, NULL); + if (elem_cnt == 1) { + memcpy(flags_elem[1].in_sg[0].iov_base, + flags_elem[0].in_sg[0].iov_base, + flags_elem[0].in_sg[0].iov_len); + nb_ack++; + + if (*c->pcap) + pcap_iov(&flags_elem[1].in_sg[0], 1, 0); + } + } - *iph = (struct iphdr)L2_BUF_IP4_INIT(IPPROTO_TCP); + vu_flush(vdev, vq, flags_elem, nb_ack); - ret = tcp_fill_flag_header(c, conn, flags, th, data, &optlen); - if (ret <= 0) { - vu_queue_rewind(vdev, vq, 1); - return ret; - } + return 0; +} - ip_len = tcp_fill_headers4(c, conn, iph, - (struct tcphdr *)(iph + 1), optlen, - NULL, conn->seq_to_tap); +/** tcp_vu_sock_recv() - Receive datastream from socket into vhost-user buffers + * @c: Execution context + * @conn: Connection pointer + * @v6: Set for IPv6 connections + * @already_sent: Number of bytes already sent + * @fillsize: Number of bytes we can receive + * @iov_cnt: number of iov (output) + * + * Return: Number of iov entries used to store the data + */ +static ssize_t tcp_vu_sock_recv(const struct ctx *c, + const struct tcp_tap_conn *conn, bool v6, + uint32_t already_sent, size_t fillsize, + int *iov_cnt) +{ + struct vu_dev *vdev = c->vdev; + struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE]; + struct msghdr mh_sock = { 0 }; + uint16_t mss = MSS_GET(conn); + int s = conn->sock; + size_t hdrlen; + int elem_cnt; + ssize_t ret; - tlen = ip_len + sizeof(struct ethhdr); + *iov_cnt = 0; - if (*c->pcap) { - uint32_t sum = proto_ipv4_header_psum(iph->tot_len, - IPPROTO_TCP, - (struct in_addr){ .s_addr = iph->saddr }, - (struct in_addr){ .s_addr = iph->daddr }); + hdrlen = tcp_vu_hdrlen(v6); - th->check = csum(th, optlen + sizeof(struct tcphdr), sum); - } - } else { - struct ipv6hdr *ip6h = (struct ipv6hdr *)(eh + 1); - struct tcphdr *th = (struct tcphdr *)(ip6h + 1); - char *data = (char *)(th + 1); + vu_init_elem(elem, &iov_vu[1], VIRTQUEUE_MAX_SIZE); - eh->h_proto = htons(ETH_P_IPV6); + elem_cnt = 0; - *th = (struct tcphdr){ - .doff = sizeof(struct tcphdr) / 4, - .ack = 1 - }; + while (fillsize > 0 && elem_cnt < VIRTQUEUE_MAX_SIZE) { + struct iovec *iov; + size_t frame_size; + int cnt; - *ip6h = (struct ipv6hdr)L2_BUF_IP6_INIT(IPPROTO_TCP); + if (mss > fillsize) + mss = fillsize; - ret = tcp_fill_flag_header(c, conn, flags, th, data, &optlen); - if (ret <= 0) { - vu_queue_rewind(vdev, vq, 1); - return ret; - } + cnt = vu_collect(vdev, vq, &elem[elem_cnt], + VIRTQUEUE_MAX_SIZE - elem_cnt, + mss + hdrlen, &frame_size); + if (cnt == 0) + break; - ip_len = tcp_fill_headers6(c, conn, ip6h, - (struct tcphdr *)(ip6h + 1), - optlen, conn->seq_to_tap); + frame_size -= hdrlen; + iov = &elem[elem_cnt].in_sg[0]; + iov->iov_base = (char *)iov->iov_base + hdrlen; + iov->iov_len -= hdrlen; - tlen = ip_len + sizeof(struct ethhdr); + fillsize -= frame_size; + elem_cnt += cnt; - if (*c->pcap) { - uint32_t sum = proto_ipv6_header_psum(ip6h->payload_len, - IPPROTO_TCP, - &ip6h->saddr, - &ip6h->daddr); + /* All the frames must have the same size (except the last one), + * otherwise we will no able to scan the iov array + * to find iov entries with headers + * (headers are spread every frame_size in the the array + */ + if (frame_size < mss) + break; + } - th->check = csum(th, optlen + sizeof(struct tcphdr), sum); - } + if (peek_offset_cap) { + mh_sock.msg_iov = iov_vu + 1; + mh_sock.msg_iovlen = elem_cnt; + } else { + iov_vu[0].iov_base = tcp_buf_discard; + iov_vu[0].iov_len = already_sent; + + mh_sock.msg_iov = iov_vu; + mh_sock.msg_iovlen = elem_cnt + 1; } - pcap((void *)eh, tlen); + do + ret = recvmsg(s, &mh_sock, MSG_PEEK); + while (ret < 0 && errno == EINTR); - tlen += vnet_hdrlen; - vu_queue_fill(vdev, vq, elem, tlen, 0); - nb_ack = 1; + *iov_cnt = elem_cnt; - if (flags & DUP_ACK) { - elem = vu_queue_pop(vdev, vq, sizeof(VuVirtqElement), buffer); - if (elem) { - if (elem->in_num < 1 || elem->in_sg[0].iov_len < tlen) { - vu_queue_rewind(vdev, vq, 1); - } else { - memcpy(elem->in_sg[0].iov_base, vh, tlen); - nb_ack++; - } - } + return ret; +} + +/** + * tcp_vu_prepare() - Prepare the frame header + * @c: Execution context + * @conn: Connection pointer + * @first: Pointer to the array of IO vectors + * @dlen: Packet data length + * @check: Checksum, if already known + */ +static void tcp_vu_prepare(const struct ctx *c, + struct tcp_tap_conn *conn, struct iovec *first, + size_t dlen, const uint16_t **check) +{ + const struct flowside *toside = TAPFLOW(conn); + struct tcp_payload_t *payload; + char *base = first->iov_base; + struct ipv6hdr *ip6h = NULL; + struct iphdr *iph = NULL; + struct ethhdr *eh; + + /* we guess the first iovec provided by the guest can embed + * all the headers needed by L2 frame + */ + + eh = vu_eth(base); + + memcpy(eh->h_dest, c->guest_mac, sizeof(eh->h_dest)); + memcpy(eh->h_source, c->our_tap_mac, sizeof(eh->h_source)); + + /* initialize header */ + + if (inany_v4(&toside->eaddr) && inany_v4(&toside->oaddr)) { + ASSERT(first[0].iov_len >= tcp_vu_hdrlen(false)); + + eh->h_proto = htons(ETH_P_IP); + + iph = vu_ip(base); + *iph = (struct iphdr)L2_BUF_IP4_INIT(IPPROTO_TCP); + payload = vu_payloadv4(base); + } else { + ASSERT(first[0].iov_len >= tcp_vu_hdrlen(true)); + + eh->h_proto = htons(ETH_P_IPV6); + + ip6h = vu_ip(base); + *ip6h = (struct ipv6hdr)L2_BUF_IP6_INIT(IPPROTO_TCP); + + payload = vu_payloadv6(base); } - vu_queue_flush(vdev, vq, nb_ack); - vu_queue_notify(vdev, vq); + memset(&payload->th, 0, sizeof(payload->th)); + payload->th.doff = offsetof(struct tcp_payload_t, data) / 4; + payload->th.ack = 1; - return 0; + if (inany_v4(&toside->eaddr) && inany_v4(&toside->oaddr)) { + tcp_fill_headers4(conn, NULL, iph, payload, dlen, + *check, conn->seq_to_tap, true); + *check = &iph->check; + } else { + tcp_fill_headers6(conn, NULL, ip6h, payload, dlen, + conn->seq_to_tap, true); + } } -int tcp_vu_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn) +/** + * tcp_vu_data_from_sock() - Handle new data from socket, queue to vhost-user, + * in window + * @c: Execution context + * @conn: Connection pointer + * + * Return: Negative on connection reset, 0 otherwise + */ +int tcp_vu_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn) { uint32_t wnd_scaled = conn->wnd_from_tap << conn->ws_from_tap; - uint32_t already_sent; - VuDev *vdev = (VuDev *)&c->vdev; - VuVirtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE]; - int s = conn->sock, v4 = CONN_V4(conn); - int i, ret = 0, iov_count, iov_used; - struct msghdr mh_sock = { 0 }; - size_t l2_hdrlen, vnet_hdrlen, fillsize; - ssize_t len; - uint16_t *check; + struct vu_dev *vdev = c->vdev; + struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE]; + const struct flowside *tapside = TAPFLOW(conn); uint16_t mss = MSS_GET(conn); - int num_buffers; - int segment_size; + size_t hdrlen, fillsize; + int i, iov_cnt, iov_used; + int v6 = CONN_V6(conn); + uint32_t already_sent = 0; + const uint16_t *check; struct iovec *first; - bool has_mrg_rxbuf; + int frame_size; + int num_buffers; + ssize_t len; if (!vu_queue_enabled(vq) || !vu_queue_started(vq)) { - err("Got packet, but no available descriptors on RX virtq."); + flow_err(conn, + "Got packet, but RX virtqueue not usable yet"); return 0; } @@ -205,6 +373,10 @@ int tcp_vu_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn) conn->seq_ack_from_tap, conn->seq_to_tap); conn->seq_to_tap = conn->seq_ack_from_tap; already_sent = 0; + if (tcp_set_peek_offset(conn->sock, 0)) { + tcp_rst(c, conn); + return -1; + } } if (!wnd_scaled || already_sent >= wnd_scaled) { @@ -215,85 +387,26 @@ int tcp_vu_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn) /* Set up buffer descriptors we'll fill completely and partially. */ - fillsize = wnd_scaled; - - iov_vu[0].iov_base = tcp_buf_discard; - iov_vu[0].iov_len = already_sent; - fillsize -= already_sent; - - has_mrg_rxbuf = vu_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF); - if (has_mrg_rxbuf) { - vnet_hdrlen = sizeof(struct virtio_net_hdr_mrg_rxbuf); - } else { - vnet_hdrlen = sizeof(struct virtio_net_hdr); - } - l2_hdrlen = vnet_hdrlen + sizeof(struct ethhdr) + sizeof(struct tcphdr); - if (v4) { - l2_hdrlen += sizeof(struct iphdr); - } else { - l2_hdrlen += sizeof(struct ipv6hdr); - } - - iov_count = 0; - segment_size = 0; - while (fillsize > 0 && iov_count < VIRTQUEUE_MAX_SIZE - 1) { - VuVirtqElement *elem; - - elem = vu_queue_pop(vdev, vq, sizeof(VuVirtqElement), buffer); - if (!elem) - break; - - if (elem->in_num < 1) { - err("virtio-net receive queue contains no in buffers"); - goto err; - } - - ASSERT(elem->in_num == 1); - ASSERT(elem->in_sg[0].iov_len >= l2_hdrlen); - - indexes[iov_count] = elem->index; - - if (segment_size == 0) { - iov_vu[iov_count + 1].iov_base = - (char *)elem->in_sg[0].iov_base + l2_hdrlen; - iov_vu[iov_count + 1].iov_len = - elem->in_sg[0].iov_len - l2_hdrlen; - } else { - iov_vu[iov_count + 1].iov_base = elem->in_sg[0].iov_base; - iov_vu[iov_count + 1].iov_len = elem->in_sg[0].iov_len; + fillsize = wnd_scaled - already_sent; + + /* collect the buffers from vhost-user and fill them with the + * data from the socket + */ + len = tcp_vu_sock_recv(c, conn, v6, already_sent, fillsize, &iov_cnt); + if (len < 0) { + vu_queue_rewind(vq, iov_cnt); + if (errno != EAGAIN && errno != EWOULDBLOCK) { + tcp_rst(c, conn); + return -errno; } - - if (iov_vu[iov_count + 1].iov_len > fillsize) - iov_vu[iov_count + 1].iov_len = fillsize; - - segment_size += iov_vu[iov_count + 1].iov_len; - if (!has_mrg_rxbuf) { - segment_size = 0; - } else if (segment_size >= mss) { - iov_vu[iov_count + 1].iov_len -= segment_size - mss; - segment_size = 0; - } - fillsize -= iov_vu[iov_count + 1].iov_len; - - iov_count++; - } - if (iov_count == 0) return 0; - - mh_sock.msg_iov = iov_vu; - mh_sock.msg_iovlen = iov_count + 1; - - do - len = recvmsg(s, &mh_sock, MSG_PEEK); - while (len < 0 && errno == EINTR); - - if (len < 0) - goto err; + } if (!len) { - vu_queue_rewind(vdev, vq, iov_count); + vu_queue_rewind(vq, iov_cnt); if ((conn->events & (SOCK_FIN_RCVD | TAP_FIN_SENT)) == SOCK_FIN_RCVD) { - if ((ret = tcp_vu_send_flag(c, conn, FIN | ACK))) { + int ret = tcp_vu_send_flag(c, conn, FIN | ACK); + if (ret) { tcp_rst(c, conn); return ret; } @@ -304,26 +417,36 @@ int tcp_vu_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn) return 0; } - len -= already_sent; + if (!peek_offset_cap) + len -= already_sent; + if (len <= 0) { + vu_queue_rewind(vq, iov_cnt); conn_flag(c, conn, STALLED); - vu_queue_rewind(vdev, vq, iov_count); return 0; } conn_flag(c, conn, ~STALLED); /* Likely, some new data was acked too. */ - tcp_update_seqack_wnd(c, conn, 0, NULL); + tcp_update_seqack_wnd(c, conn, false, NULL); /* initialize headers */ + hdrlen = tcp_vu_hdrlen(v6); iov_used = 0; num_buffers = 0; check = NULL; - segment_size = 0; - for (i = 0; i < iov_count && len; i++) { + frame_size = 0; - if (segment_size == 0) + /* iov_vu is an array of buffers and the buffer size can be + * smaller than the frame size we want to use but with + * num_buffer we can merge several virtio iov buffers in one packet + * we need only to set the packet headers in the first iov and + * num_buffer to the number of iov entries + */ + for (i = 0; i < iov_cnt && len; i++) { + + if (frame_size == 0) first = &iov_vu[i + 1]; if (iov_vu[i + 1].iov_len > (size_t)len) @@ -332,129 +455,40 @@ int tcp_vu_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn) len -= iov_vu[i + 1].iov_len; iov_used++; - segment_size += iov_vu[i + 1].iov_len; + frame_size += iov_vu[i + 1].iov_len; num_buffers++; - if (segment_size >= mss || len == 0 || - i + 1 == iov_count || !has_mrg_rxbuf) { - - struct ethhdr *eh; - struct virtio_net_hdr_mrg_rxbuf *vh; - char *base = (char *)first->iov_base - l2_hdrlen; - size_t size = first->iov_len + l2_hdrlen; - - vh = (struct virtio_net_hdr_mrg_rxbuf *)base; - - vh->hdr = vu_header; - if (has_mrg_rxbuf) - vh->num_buffers = htole16(num_buffers); - - eh = (struct ethhdr *)((char *)base + vnet_hdrlen); - - memcpy(eh->h_dest, c->mac_guest, sizeof(eh->h_dest)); - memcpy(eh->h_source, c->mac, sizeof(eh->h_source)); - - /* initialize header */ - if (v4) { - struct iphdr *iph = (struct iphdr *)(eh + 1); - struct tcphdr *th = (struct tcphdr *)(iph + 1); - - eh->h_proto = htons(ETH_P_IP); - - *th = (struct tcphdr){ - .doff = sizeof(struct tcphdr) / 4, - .ack = 1 - }; - - *iph = (struct iphdr)L2_BUF_IP4_INIT(IPPROTO_TCP); - - tcp_fill_headers4(c, conn, iph, - (struct tcphdr *)(iph + 1), - segment_size, len ? check : NULL, - conn->seq_to_tap); - - if (*c->pcap) { - uint32_t sum = proto_ipv4_header_psum(iph->tot_len, - IPPROTO_TCP, - (struct in_addr){ .s_addr = iph->saddr }, - (struct in_addr){ .s_addr = iph->daddr }); - - first->iov_base = th; - first->iov_len = size - l2_hdrlen + sizeof(*th); - - th->check = csum_iov(first, num_buffers, sum); - } - - check = &iph->check; - } else { - struct ipv6hdr *ip6h = (struct ipv6hdr *)(eh + 1); - struct tcphdr *th = (struct tcphdr *)(ip6h + 1); - - eh->h_proto = htons(ETH_P_IPV6); - - *th = (struct tcphdr){ - .doff = sizeof(struct tcphdr) / 4, - .ack = 1 - }; - - *ip6h = (struct ipv6hdr)L2_BUF_IP6_INIT(IPPROTO_TCP); - - tcp_fill_headers6(c, conn, ip6h, - (struct tcphdr *)(ip6h + 1), - segment_size, conn->seq_to_tap); - if (*c->pcap) { - uint32_t sum = proto_ipv6_header_psum(ip6h->payload_len, - IPPROTO_TCP, - &ip6h->saddr, - &ip6h->daddr); - - first->iov_base = th; - first->iov_len = size - l2_hdrlen + sizeof(*th); - - th->check = csum_iov(first, num_buffers, sum); - } + if (frame_size >= mss || len == 0 || + i + 1 == iov_cnt || !vu_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) { + if (i + 1 == iov_cnt) + check = NULL; + + /* restore first iovec base: point to vnet header */ + first->iov_base = (char *)first->iov_base - hdrlen; + first->iov_len += hdrlen; + vu_set_vnethdr(vdev, first->iov_base, num_buffers); + + tcp_vu_prepare(c, conn, first, frame_size, &check); + if (*c->pcap) { + tcp_vu_update_check(tapside, first, num_buffers); + pcap_iov(first, num_buffers, + sizeof(struct virtio_net_hdr_mrg_rxbuf)); } - /* set iov for pcap logging */ - first->iov_base = eh; - first->iov_len = size - vnet_hdrlen; - - pcap_iov(first, num_buffers); - - /* set iov_len for vu_queue_fill_by_index(); */ + conn->seq_to_tap += frame_size; - first->iov_base = base; - first->iov_len = size; - - conn->seq_to_tap += segment_size; - - segment_size = 0; + frame_size = 0; num_buffers = 0; } } /* release unused buffers */ - vu_queue_rewind(vdev, vq, iov_count - iov_used); + vu_queue_rewind(vq, iov_cnt - iov_used); /* send packets */ - for (i = 0; i < iov_used; i++) { - vu_queue_fill_by_index(vdev, vq, indexes[i], - iov_vu[i + 1].iov_len, i); - } - - vu_queue_flush(vdev, vq, iov_used); - vu_queue_notify(vdev, vq); + vu_flush(vdev, vq, elem, iov_used); conn_flag(c, conn, ACK_FROM_TAP_DUE); return 0; -err: - vu_queue_rewind(vdev, vq, iov_count); - - if (errno != EAGAIN && errno != EWOULDBLOCK) { - ret = -errno; - tcp_rst(c, conn); - } - - return ret; } @@ -1,9 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-or-later +/* Copyright Red Hat + * Author: Laurent Vivier <lvivier@redhat.com> + */ #ifndef TCP_VU_H #define TCP_VU_H -int tcp_vu_send_flag(struct ctx *c, struct tcp_tap_conn *conn, int flags); -int tcp_vu_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn); +int tcp_vu_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags); +int tcp_vu_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn); #endif /*TCP_VU_H */ diff --git a/test/.gitignore b/test/.gitignore index 4837402..6dd4790 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1,5 +1,6 @@ test_logs/ mbuto/ +podman/ *.img QEMU_EFI.fd *.qcow2 diff --git a/test/Makefile b/test/Makefile index 7b00bef..5e49047 100644 --- a/test/Makefile +++ b/test/Makefile @@ -8,7 +8,6 @@ WGET = wget -c DEBIAN_IMGS = debian-8.11.0-openstack-amd64.qcow2 \ - debian-9-nocloud-amd64-daily-20200210-166.qcow2 \ debian-10-nocloud-amd64.qcow2 \ debian-10-generic-arm64.qcow2 \ debian-10-generic-ppc64el-20220911-1135.qcow2 \ @@ -42,8 +41,7 @@ OPENSUSE_IMGS = openSUSE-Leap-15.1-JeOS.x86_64-kvm-and-xen.qcow2 \ openSUSE-Leap-15.2-JeOS.x86_64-kvm-and-xen.qcow2 \ openSUSE-Leap-15.3-JeOS.x86_64-kvm-and-xen.qcow2 \ openSUSE-Tumbleweed-ARM-JeOS-efi.aarch64.raw.xz \ - openSUSE-Tumbleweed-ARM-JeOS-efi.armv7l.raw.xz \ - openSUSE-Tumbleweed-JeOS.x86_64-kvm-and-xen.qcow2 + openSUSE-Tumbleweed-ARM-JeOS-efi.armv7l.raw.xz UBUNTU_OLD_IMGS = trusty-server-cloudimg-amd64-disk1.img \ trusty-server-cloudimg-i386-disk1.img \ @@ -52,10 +50,10 @@ UBUNTU_NEW_IMGS = xenial-server-cloudimg-powerpc-disk1.img \ jammy-server-cloudimg-s390x.img UBUNTU_IMGS = $(UBUNTU_OLD_IMGS) $(UBUNTU_NEW_IMGS) -DOWNLOAD_ASSETS = mbuto \ +DOWNLOAD_ASSETS = mbuto podman \ $(DEBIAN_IMGS) $(FEDORA_IMGS) $(OPENSUSE_IMGS) $(UBUNTU_IMGS) TESTDATA_ASSETS = small.bin big.bin medium.bin -LOCAL_ASSETS = mbuto.img mbuto.mem.img QEMU_EFI.fd \ +LOCAL_ASSETS = mbuto.img mbuto.mem.img podman/bin/podman QEMU_EFI.fd \ $(DEBIAN_IMGS:%=prepared-%) $(FEDORA_IMGS:%=prepared-%) \ $(UBUNTU_NEW_IMGS:%=prepared-%) \ nstool guest-key guest-key.pub \ @@ -67,13 +65,27 @@ CFLAGS = -Wall -Werror -Wextra -pedantic -std=c99 assets: $(ASSETS) +.PHONY: pull-% +pull-%: % + git -C $* pull + mbuto: git clone git://mbuto.sh/mbuto +mbuto/mbuto: pull-mbuto + +podman: + git clone https://github.com/containers/podman.git + +# To succesfully build podman, you will need gpgme and systemd +# development packages +podman/bin/podman: pull-podman + $(MAKE) -C podman + guest-key guest-key.pub: ssh-keygen -f guest-key -N '' -mbuto.img: passt.mbuto mbuto guest-key.pub $(TESTDATA_ASSETS) +mbuto.img: passt.mbuto mbuto/mbuto guest-key.pub $(TESTDATA_ASSETS) ./mbuto/mbuto -p ./$< -c lz4 -f $@ mbuto.mem.img: passt.mem.mbuto mbuto ../passt.avx2 @@ -121,9 +133,6 @@ realclean: clean debian-8.11.0-openstack-%.qcow2: $(WGET) -O $@ https://cloud.debian.org/images/cloud/OpenStack/archive/8.11.0/debian-8.11.0-openstack-$*.qcow2 -debian-9-nocloud-%-daily-20200210-166.qcow2: - $(WGET) -O $@ https://cloud.debian.org/images/cloud/stretch/daily/20200210-166/debian-9-nocloud-$*-daily-20200210-166.qcow2 - debian-10-nocloud-%.qcow2: $(WGET) -O $@ https://cloud.debian.org/images/cloud/buster/latest/debian-10-nocloud-$*.qcow2 @@ -189,9 +198,6 @@ openSUSE-Tumbleweed-ARM-JeOS-efi.aarch64.raw.xz: openSUSE-Tumbleweed-ARM-JeOS-efi.armv7l.raw.xz: $(WGET) -O $@ http://download.opensuse.org/ports/armv7hl/tumbleweed/appliances/openSUSE-Tumbleweed-ARM-JeOS-efi.armv7l.raw.xz -openSUSE-Tumbleweed-JeOS.x86_64-kvm-and-xen.qcow2: - $(WGET) -O $@ https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-kvm-and-xen.qcow2 - # Ubuntu downloads trusty-server-cloudimg-%-disk1.img: $(WGET) -O $@ https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-$*-disk1.img diff --git a/test/README.md b/test/README.md index 0936b04..91ca603 100644 --- a/test/README.md +++ b/test/README.md @@ -28,10 +28,11 @@ on a system, i.e. common utilities such as a shell are not included here. Example for Debian, and possibly most Debian-based distributions: - build-essential git jq strace iperf3 qemu-system-x86 tmux sipcalc bats bc - catatonit clang-tidy cppcheck go isc-dhcp-common psmisc linux-cpupower socat - netcat-openbsd fakeroot lz4 lm-sensors qemu-system-arm qemu-system-ppc - qemu-system-misc qemu-system-x86 valgrind + bats bc build-essential catatonit clang-tidy conmon cppcheck crun fakeroot + git go iperf3 isc-dhcp-common jq libgpgme-dev libseccomp-dev linux-cpupower + lm-sensors lz4 netavark netcat-openbsd psmisc qemu-efi-aarch64 + qemu-system-arm qemu-system-misc qemu-system-ppc qemu-system-x86 + qemu-system-x86 sipcalc socat strace tmux uidmap valgrind NOTE: the tests need a qemu version >= 7.2, or one that contains commit 13c6be96618c ("net: stream: add unix socket"): this change introduces support diff --git a/test/lib/layout b/test/lib/layout index f9a1cf1..4d03572 100644 --- a/test/lib/layout +++ b/test/lib/layout @@ -15,7 +15,7 @@ # layout_pasta() - Panes for host, pasta, and separate one for namespace layout_pasta() { - sleep 3 + sleep 1 tmux kill-pane -a -t 0 cmd_write 0 clear @@ -46,7 +46,7 @@ layout_pasta() { # layout_passt() - Panes for host, passt, and guest layout_passt() { - sleep 3 + sleep 1 tmux kill-pane -a -t 0 cmd_write 0 clear @@ -77,7 +77,7 @@ layout_passt() { # layout_passt_in_pasta() - Host, passt within pasta, namespace and guest layout_passt_in_pasta() { - sleep 3 + sleep 1 tmux kill-pane -a -t 0 cmd_write 0 clear @@ -113,7 +113,7 @@ layout_passt_in_pasta() { # layout_two_guests() - Two guest panes, two passt panes, plus host and log layout_two_guests() { - sleep 3 + sleep 1 tmux kill-pane -a -t 0 cmd_write 0 clear @@ -152,7 +152,7 @@ layout_two_guests() { # layout_demo_pasta() - Four panes for pasta demo layout_demo_pasta() { - sleep 3 + sleep 1 cmd_write 0 cd ${BASEPATH} cmd_write 0 clear @@ -188,7 +188,7 @@ layout_demo_pasta() { # layout_demo_passt() - Four panes for passt demo layout_demo_passt() { - sleep 3 + sleep 1 cmd_write 0 cd ${BASEPATH} cmd_write 0 clear @@ -224,7 +224,7 @@ layout_demo_passt() { # layout_demo_podman() - Four panes for pasta demo with Podman layout_demo_podman() { - sleep 3 + sleep 1 cmd_write 0 cd ${BASEPATH} cmd_write 0 clear diff --git a/test/lib/perf_report b/test/lib/perf_report index 67f9f4e..c4ec817 100755 --- a/test/lib/perf_report +++ b/test/lib/perf_report @@ -18,7 +18,7 @@ PERF_LINK_COUNT=0 PERF_JS="${LOGDIR}/web/perf.js" PERF_TEMPLATE_HTML="document.write('"' -Throughput in Gbps, latency in µs. Threads are <span style="font-family: monospace;">iperf3</span> processes, <i>passt</i> and <i>pasta</i> are currently single-threaded.<br/> +Throughput in Gbps, latency in µs. Threads are <span style="font-family: monospace;">iperf3</span> threads, <i>passt</i> and <i>pasta</i> are currently single-threaded.<br/> Click on numbers to show test execution. Measured at head, commit <span style="font-family: monospace;">__commit__</span>. <style type="text/CSS"> @@ -49,6 +49,21 @@ td:empty { visibility: hidden; } __passt_tcp_LINE__ __passt_udp_LINE__ </table> +</li><li><p>passt with vhost-user support</p> +<table class="passt" width="70%"> + <tr> + <th/> + <th id="perf_passt_vu_tcp" colspan="__passt_vu_tcp_cols__">TCP, __passt_vu_tcp_threads__ at __passt_vu_tcp_freq__ GHz</th> + <th id="perf_passt_vu_udp" colspan="__passt_vu_udp_cols__">UDP, __passt_vu_udp_threads__ at __passt_vu_udp_freq__ GHz</th> + </tr> + <tr> + <td align="right">MTU:</td> + __passt_vu_tcp_header__ + __passt_vu_udp_header__ + </tr> + __passt_vu_tcp_LINE__ __passt_vu_udp_LINE__ +</table> + <style type="text/CSS"> table.pasta_local td { border: 0px solid; padding: 6px; line-height: 1; } table.pasta_local td { text-align: right; } @@ -56,7 +71,7 @@ table.pasta_local th { text-align: center; font-weight: bold; } table.pasta_local tr:not(:first-of-type) td:not(:first-of-type) { font-family: monospace; font-weight: bolder; } table.pasta_local tr:nth-child(3n+0) { background-color: #112315; } table.pasta_local tr:not(:nth-child(3n+0)) td { background-color: #101010; } -table.pasta_local td:nth-child(3n+2) { background-color: #603302; } +table.pasta_local td:nth-child(4n+2) { background-color: #603302; } table.pasta_local tr:nth-child(1) { background-color: #363e61; } table.pasta td { border: 0px solid; padding: 6px; line-height: 1; } table.pasta td { text-align: right; } diff --git a/test/lib/setup b/test/lib/setup index 9b39b9f..580825f 100755 --- a/test/lib/setup +++ b/test/lib/setup @@ -15,8 +15,9 @@ INITRAMFS="${BASEPATH}/mbuto.img" VCPUS="$( [ $(nproc) -ge 8 ] && echo 6 || echo $(( $(nproc) / 2 + 1 )) )" -__mem_kib="$(sed -n 's/MemTotal:[ ]*\([0-9]*\) kB/\1/p' /proc/meminfo)" -VMEM="$((${__mem_kib} / 1024 / 4))" +MEM_KIB="$(sed -n 's/MemTotal:[ ]*\([0-9]*\) kB/\1/p' /proc/meminfo)" +QEMU_ARCH="$(uname -m)" +[ "${QEMU_ARCH}" = "i686" ] && QEMU_ARCH=i386 # setup_build() - Set up pane layout for build tests setup_build() { @@ -44,6 +45,7 @@ setup_passt() { [ ${PCAP} -eq 1 ] && __opts="${__opts} -p ${LOGDIR}/passt.pcap" [ ${DEBUG} -eq 1 ] && __opts="${__opts} -d" [ ${TRACE} -eq 1 ] && __opts="${__opts} --trace" + [ ${VHOST_USER} -eq 1 ] && __opts="${__opts} --vhost-user" context_run passt "make clean" context_run passt "make valgrind" @@ -52,16 +54,29 @@ setup_passt() { # pidfile isn't created until passt is listening wait_for [ -f "${STATESETUP}/passt.pid" ] + __vmem="$((${MEM_KIB} / 1024 / 4))" + if [ ${VHOST_USER} -eq 1 ]; then + __vmem="$(((${__vmem} + 500) / 1000))G" + __qemu_netdev=" \ + -chardev socket,id=c,path=${STATESETUP}/passt.socket \ + -netdev vhost-user,id=v,chardev=c \ + -device virtio-net,netdev=v \ + -object memory-backend-memfd,id=m,share=on,size=${__vmem} \ + -numa node,memdev=m" + else + __qemu_netdev="-device virtio-net-pci,netdev=s \ + -netdev stream,id=s,server=off,addr.type=unix,addr.path=${STATESETUP}/passt.socket" + fi + GUEST_CID=94557 - context_run_bg qemu 'qemu-system-$(uname -m)' \ + context_run_bg qemu 'qemu-system-'"${QEMU_ARCH}" \ ' -machine accel=kvm' \ - ' -m '${VMEM}' -cpu host -smp '${VCPUS} \ - ' -kernel ' "/boot/vmlinuz-$(uname -r)" \ + ' -m '${__vmem}' -cpu host -smp '${VCPUS} \ + ' -kernel '"${KERNEL}" \ ' -initrd '${INITRAMFS}' -nographic -serial stdio' \ ' -nodefaults' \ ' -append "console=ttyS0 mitigations=off apparmor=0" ' \ - ' -device virtio-net-pci,netdev=s0 ' \ - " -netdev stream,id=s0,server=off,addr.type=unix,addr.path=${STATESETUP}/passt.socket " \ + " ${__qemu_netdev}" \ " -pidfile ${STATESETUP}/qemu.pid" \ " -device vhost-vsock-pci,guest-cid=$GUEST_CID" @@ -124,7 +139,12 @@ setup_passt_in_ns() { [ ${DEBUG} -eq 1 ] && __opts="${__opts} -d" [ ${TRACE} -eq 1 ] && __opts="${__opts} --trace" - context_run_bg pasta "./pasta ${__opts} -t 10001,10002,10011,10012 -T 10003,10013 -u 10001,10002,10011,10012 -U 10003,10013 -P ${STATESETUP}/pasta.pid --config-net ${NSTOOL} hold ${STATESETUP}/ns.hold" + __map_host4=192.0.2.1 + __map_host6=2001:db8:9a55::1 + __map_ns4=192.0.2.2 + __map_ns6=2001:db8:9a55::2 + + context_run_bg pasta "./pasta ${__opts} -t 10001,10002,10011,10012 -T 10003,10013 -u 10001,10002,10011,10012 -U 10003,10013 -P ${STATESETUP}/pasta.pid --map-host-loopback ${__map_host4} --map-host-loopback ${__map_host6} --config-net ${NSTOOL} hold ${STATESETUP}/ns.hold" wait_for [ -f "${STATESETUP}/pasta.pid" ] context_setup_nstool qemu ${STATESETUP}/ns.hold @@ -135,29 +155,43 @@ setup_passt_in_ns() { [ ${PCAP} -eq 1 ] && __opts="${__opts} -p ${LOGDIR}/passt_in_pasta.pcap" [ ${DEBUG} -eq 1 ] && __opts="${__opts} -d" [ ${TRACE} -eq 1 ] && __opts="${__opts} --trace" + [ ${VHOST_USER} -eq 1 ] && __opts="${__opts} --vhost-user" if [ ${VALGRIND} -eq 1 ]; then context_run passt "make clean" context_run passt "make valgrind" - context_run_bg passt "valgrind --max-stackframe=$((4 * 1024 * 1024)) --trace-children=yes --vgdb=no --error-exitcode=1 --suppressions=test/valgrind.supp ./passt -f ${__opts} -s ${STATESETUP}/passt.socket -t 10001,10011,10021,10031 -u 10001,10011,10021,10031 -P ${STATESETUP}/passt.pid" + context_run_bg passt "valgrind --max-stackframe=$((4 * 1024 * 1024)) --trace-children=yes --vgdb=no --error-exitcode=1 --suppressions=test/valgrind.supp ./passt -f ${__opts} -s ${STATESETUP}/passt.socket -t 10001,10011,10021,10031 -u 10001,10011,10021,10031 -P ${STATESETUP}/passt.pid --map-host-loopback ${__map_ns4} --map-host-loopback ${__map_ns6}" else context_run passt "make clean" context_run passt "make" - context_run_bg passt "./passt -f ${__opts} -s ${STATESETUP}/passt.socket -t 10001,10011,10021,10031 -u 10001,10011,10021,10031 -P ${STATESETUP}/passt.pid" + context_run_bg passt "./passt -f ${__opts} -s ${STATESETUP}/passt.socket -t 10001,10011,10021,10031 -u 10001,10011,10021,10031 -P ${STATESETUP}/passt.pid --map-host-loopback ${__map_ns4} --map-host-loopback ${__map_ns6}" fi wait_for [ -f "${STATESETUP}/passt.pid" ] + __vmem="$((${MEM_KIB} / 1024 / 4))" + if [ ${VHOST_USER} -eq 1 ]; then + __vmem="$(((${__vmem} + 500) / 1000))G" + __qemu_netdev=" \ + -chardev socket,id=c,path=${STATESETUP}/passt.socket \ + -netdev vhost-user,id=v,chardev=c \ + -device virtio-net,netdev=v \ + -object memory-backend-memfd,id=m,share=on,size=${__vmem} \ + -numa node,memdev=m" + else + __qemu_netdev="-device virtio-net-pci,netdev=s \ + -netdev stream,id=s,server=off,addr.type=unix,addr.path=${STATESETUP}/passt.socket" + fi + GUEST_CID=94557 - context_run_bg qemu 'qemu-system-$(uname -m)' \ + context_run_bg qemu 'qemu-system-'"${QEMU_ARCH}" \ ' -machine accel=kvm' \ ' -M accel=kvm:tcg' \ - ' -m '${VMEM}' -cpu host -smp '${VCPUS} \ - ' -kernel ' "/boot/vmlinuz-$(uname -r)" \ + ' -m '${__vmem}' -cpu host -smp '${VCPUS} \ + ' -kernel '"${KERNEL}" \ ' -initrd '${INITRAMFS}' -nographic -serial stdio' \ ' -nodefaults' \ ' -append "console=ttyS0 mitigations=off apparmor=0" ' \ - ' -device virtio-net-pci,netdev=s0 ' \ - " -netdev stream,id=s0,server=off,addr.type=unix,addr.path=${STATESETUP}/passt.socket " \ + " ${__qemu_netdev}" \ " -pidfile ${STATESETUP}/qemu.pid" \ " -device vhost-vsock-pci,guest-cid=$GUEST_CID" @@ -207,6 +241,7 @@ setup_two_guests() { [ ${PCAP} -eq 1 ] && __opts="${__opts} -p ${LOGDIR}/passt_1.pcap" [ ${DEBUG} -eq 1 ] && __opts="${__opts} -d" [ ${TRACE} -eq 1 ] && __opts="${__opts} --trace" + [ ${VHOST_USER} -eq 1 ] && __opts="${__opts} --vhost-user" context_run_bg passt_1 "./passt -s ${STATESETUP}/passt_1.socket -P ${STATESETUP}/passt_1.pid -f ${__opts} -t 10001 -u 10001" wait_for [ -f "${STATESETUP}/passt_1.pid" ] @@ -215,33 +250,54 @@ setup_two_guests() { [ ${PCAP} -eq 1 ] && __opts="${__opts} -p ${LOGDIR}/passt_2.pcap" [ ${DEBUG} -eq 1 ] && __opts="${__opts} -d" [ ${TRACE} -eq 1 ] && __opts="${__opts} --trace" + [ ${VHOST_USER} -eq 1 ] && __opts="${__opts} --vhost-user" context_run_bg passt_2 "./passt -s ${STATESETUP}/passt_2.socket -P ${STATESETUP}/passt_2.pid -f ${__opts} -t 10004 -u 10004" wait_for [ -f "${STATESETUP}/passt_2.pid" ] + __vmem="$((${MEM_KIB} / 1024 / 4))" + if [ ${VHOST_USER} -eq 1 ]; then + __vmem="$(((${__vmem} + 500) / 1000))G" + __qemu_netdev1=" \ + -chardev socket,id=c,path=${STATESETUP}/passt_1.socket \ + -netdev vhost-user,id=v,chardev=c \ + -device virtio-net,netdev=v \ + -object memory-backend-memfd,id=m,share=on,size=${__vmem} \ + -numa node,memdev=m" + __qemu_netdev2=" \ + -chardev socket,id=c,path=${STATESETUP}/passt_2.socket \ + -netdev vhost-user,id=v,chardev=c \ + -device virtio-net,netdev=v \ + -object memory-backend-memfd,id=m,share=on,size=${__vmem} \ + -numa node,memdev=m" + else + __qemu_netdev1="-device virtio-net-pci,netdev=s \ + -netdev stream,id=s,server=off,addr.type=unix,addr.path=${STATESETUP}/passt_1.socket" + __qemu_netdev2="-device virtio-net-pci,netdev=s \ + -netdev stream,id=s,server=off,addr.type=unix,addr.path=${STATESETUP}/passt_2.socket" + fi + GUEST_1_CID=94557 - context_run_bg qemu_1 'qemu-system-$(uname -m)' \ + context_run_bg qemu_1 'qemu-system-'"${QEMU_ARCH}" \ ' -M accel=kvm:tcg' \ - ' -m '${VMEM}' -cpu host -smp '${VCPUS} \ - ' -kernel ' "/boot/vmlinuz-$(uname -r)" \ + ' -m '${__vmem}' -cpu host -smp '${VCPUS} \ + ' -kernel '"${KERNEL}" \ ' -initrd '${INITRAMFS}' -nographic -serial stdio' \ ' -nodefaults' \ ' -append "console=ttyS0 mitigations=off apparmor=0" ' \ - ' -device virtio-net-pci,netdev=s0 ' \ - " -netdev stream,id=s0,server=off,addr.type=unix,addr.path=${STATESETUP}/passt_1.socket " \ + " ${__qemu_netdev1}" \ " -pidfile ${STATESETUP}/qemu_1.pid" \ " -device vhost-vsock-pci,guest-cid=$GUEST_1_CID" GUEST_2_CID=94558 - context_run_bg qemu_2 'qemu-system-$(uname -m)' \ + context_run_bg qemu_2 'qemu-system-'"${QEMU_ARCH}" \ ' -M accel=kvm:tcg' \ - ' -m '${VMEM}' -cpu host -smp '${VCPUS} \ - ' -kernel ' "/boot/vmlinuz-$(uname -r)" \ + ' -m '${__vmem}' -cpu host -smp '${VCPUS} \ + ' -kernel '"${KERNEL}" \ ' -initrd '${INITRAMFS}' -nographic -serial stdio' \ ' -nodefaults' \ ' -append "console=ttyS0 mitigations=off apparmor=0" ' \ - ' -device virtio-net-pci,netdev=s0 ' \ - " -netdev stream,id=s0,server=off,addr.type=unix,addr.path=${STATESETUP}/passt_2.socket " \ + " ${__qemu_netdev2}" \ " -pidfile ${STATESETUP}/qemu_2.pid" \ " -device vhost-vsock-pci,guest-cid=$GUEST_2_CID" diff --git a/test/lib/setup_ugly b/test/lib/setup_ugly index 4b2a077..2802cc3 100755 --- a/test/lib/setup_ugly +++ b/test/lib/setup_ugly @@ -33,7 +33,7 @@ setup_memory() { pane_or_context_run guest 'qemu-system-$(uname -m)' \ ' -machine accel=kvm' \ - ' -m '${VMEM}' -cpu host -smp '${VCPUS} \ + ' -m '$((${MEM_KIB} / 1024 / 4))' -cpu host -smp '${VCPUS} \ ' -kernel ' "/boot/vmlinuz-$(uname -r)" \ ' -initrd '${INITRAMFS_MEM}' -nographic -serial stdio' \ ' -nodefaults' \ diff --git a/test/lib/term b/test/lib/term index 262937e..ed690de 100755 --- a/test/lib/term +++ b/test/lib/term @@ -31,8 +31,8 @@ PR_DELAY_INIT=100 # ms # $@: Message to print info() { tmux select-pane -t ${PANE_INFO} - echo "${@}" >> $STATEBASE/log_pipe - echo "${@}" >> "${LOGFILE}" + printf "${@}\n" >> $STATEBASE/log_pipe + printf "${@}\n" >> "${LOGFILE}" } # info_n() - Highlight, print message to pane and to log file without newline @@ -47,13 +47,13 @@ info_n() { # $@: Message to print info_nolog() { tmux select-pane -t ${PANE_INFO} - echo "${@}" >> $STATEBASE/log_pipe + printf "${@}\n" >> $STATEBASE/log_pipe } # info_nolog() - Print message to log file # $@: Message to print log() { - echo "${@}" >> "${LOGFILE}" + printf "${@}\n" >> "${LOGFILE}" } # info_nolog_n() - Send message to pane without highlighting it, without newline @@ -97,7 +97,6 @@ display_delay() { switch_pane() { tmux select-pane -t ${1} PR_DELAY=${PR_DELAY_INIT} - display_delay "0.2" } # cmd_write() - Write a command to a pane, letter by letter, and execute it @@ -199,7 +198,7 @@ pane_run() { # $1: Pane name pane_wait() { __lc="$(echo "${1}" | tr [A-Z] [a-z])" - sleep 0.1 || sleep 1 + sleep 0.01 || sleep 1 __done=0 while @@ -207,7 +206,7 @@ pane_wait() { case ${__l} in *"$ " | *"# ") return ;; esac - do sleep 0.1 || sleep 1; done + do sleep 0.01 || sleep 1; done } # pane_parse() - Print last line, @EMPTY@ if command had no output @@ -231,7 +230,7 @@ pane_status() { __status="$(pane_parse "${1}")" while ! [ "${__status}" -eq "${__status}" ] 2>/dev/null; do - sleep 1 + sleep 0.01 || sleep 1 pane_run "${1}" 'echo $?' pane_wait "${1}" __status="$(pane_parse "${1}")" @@ -383,6 +382,16 @@ info_check_failed() { printf " < failed.\n" >> "${LOGFILE}" } +# status_bar_blink() - Make status bar blink +status_bar_blink() { + for i in `seq 1 3`; do + tmux set status-right-style 'bg=colour1 fg=colour196 bold' + sleep 0.1 || sleep 1 + tmux set status-right-style 'bg=colour1 fg=colour233 bold' + sleep 0.1 || sleep 1 + done +} + # info_passed() - Display, log, and make status bar blink when a test passes info_passed() { switch_pane ${PANE_INFO} @@ -391,12 +400,7 @@ info_passed() { log "...passed." log - for i in `seq 1 3`; do - tmux set status-right-style 'bg=colour1 fg=colour2 bold' - sleep "0.1" - tmux set status-right-style 'bg=colour1 fg=colour233 bold' - sleep "0.1" - done + [ ${FAST} -eq 1 ] || status_bar_blink } # info_failed() - Display, log, and make status bar blink when a test passes @@ -407,12 +411,7 @@ info_failed() { log "...failed." log - for i in `seq 1 3`; do - tmux set status-right-style 'bg=colour1 fg=colour196 bold' - sleep "0.1" - tmux set status-right-style 'bg=colour1 fg=colour233 bold' - sleep "0.1" - done + [ ${FAST} -eq 1 ] || status_bar_blink pause_continue \ "Press any key to pause test session" \ @@ -665,7 +664,7 @@ pause_continue() { # run_term() - Start tmux session, running entry point, with recording if needed run_term() { - TMUX="tmux new-session -s passt_test -eSTATEBASE=$STATEBASE -ePCAP=$PCAP -eDEBUG=$DEBUG" + TMUX="tmux new-session -s passt_test -eSTATEBASE=$STATEBASE -ePCAP=$PCAP -eDEBUG=$DEBUG -eTRACE=$TRACE -eKERNEL=$KERNEL" if [ ${CI} -eq 1 ]; then printf '\e[8;50;240t' diff --git a/test/lib/test b/test/lib/test index 1d571c3..e6726be 100755 --- a/test/lib/test +++ b/test/lib/test @@ -15,18 +15,13 @@ # test_iperf3s() - Start iperf3 server # $1: Destination/server context -# $2: Port number, ${i} is translated to process index -# $3: Number of processes to run in parallel +# $2: Port number test_iperf3s() { __sctx="${1}" __port="${2}" - __procs="$((${3} - 1))" pane_or_context_run_bg "${__sctx}" \ - 'for i in $(seq 0 '${__procs}'); do' \ - ' iperf3 -s -p'${__port}' &' \ - ' echo $! > s${i}.pid; ' \ - 'done' \ + 'iperf3 -s -p'${__port}' & echo $! > s.pid' \ sleep 1 # Wait for server to be ready } @@ -36,9 +31,9 @@ test_iperf3s() { test_iperf3k() { __sctx="${1}" - pane_or_context_run "${__sctx}" 'kill -INT $(cat s*.pid); rm s*.pid' + pane_or_context_run "${__sctx}" 'kill -INT $(cat s.pid); rm s.pid' - sleep 3 # Wait for kernel to free up ports + sleep 1 # Wait for kernel to free up ports } # test_iperf3() - Ugly helper for iperf3 directive @@ -46,37 +41,29 @@ test_iperf3k() { # $2: Source/client context # $3: Destination name or address for client # $4: Port number, ${i} is translated to process index -# $5: Number of processes to run in parallel -# $6: Run time, in seconds +# $5: Run time, in seconds # $@: Client options test_iperf3() { __var="${1}"; shift __cctx="${1}"; shift __dest="${1}"; shift __port="${1}"; shift - __procs="$((${1} - 1))"; shift __time="${1}"; shift - pane_or_context_run "${__cctx}" 'rm -f c*.json' + pane_or_context_run "${__cctx}" 'rm -f c.json' # A 1s wait for connection on what's basically a local link # indicates something is pretty wrong __timeout=1000 pane_or_context_run "${__cctx}" \ - '(' \ - ' for i in $(seq 0 '${__procs}'); do' \ - ' iperf3 -J -c '${__dest}' -p '${__port} \ - ' --connect-timeout '${__timeout} \ - ' -t'${__time}' -i0 -T c${i} '"${@}" \ - ' > c${i}.json &' \ - ' done;' \ - ' wait' \ - ')' + 'iperf3 -J -c '${__dest}' -p '${__port} \ + ' --connect-timeout '${__timeout} \ + ' -t'${__time}' -i0 '"${@}"' > c.json' \ __jval=".end.sum_received.bits_per_second" __bw=$(pane_or_context_output "${__cctx}" \ - 'cat c*.json | jq -rMs "map('${__jval}') | add"') + 'cat c.json | jq -rMs "map('${__jval}') | add"') TEST_ONE_subs="$(list_add_pair "${TEST_ONE_subs}" "__${__var}__" "${__bw}" )" } diff --git a/test/memory/passt b/test/memory/passt index 1193af8..7e45724 100644 --- a/test/memory/passt +++ b/test/memory/passt @@ -44,7 +44,7 @@ endef def start_stop_diff guest sed /proc/slabinfo -ne 's/^\([^ ]* *[^ ]* *[^ ]* *[^ ]*\).*/\\\1/p' > /tmp/slabinfo.before guest cat /proc/meminfo > /tmp/meminfo.before -guest /bin/passt.avx2 -l /tmp/log -s /tmp/sock -P /tmp/pid __OPTS__ --netns-only +guest /bin/passt.avx2 -l /tmp/log -s /tmp/sock -P /tmp/pid __OPTS__ sleep 2 guest cat /proc/meminfo > /tmp/meminfo.after guest sed /proc/slabinfo -ne 's/^\([^ ]* *[^ ]* *[^ ]* *[^ ]*\).*/\\\1/p' > /tmp/slabinfo.after @@ -78,9 +78,16 @@ guest mount -o bind /proc /test/proc guest mount -o bind /dev /test/dev guest cp -Lr /bin /lib /lib64 /usr /sbin /test/ +guest exec switch_root /test /bin/sh + guest ulimit -Hn 300000 -guest unshare -rUm -R /test -guest chroot . +guest unshare -rUn +guest ip link add eth0 type dummy +guest ip link set eth0 up +guest ip address add 192.0.2.2/24 dev eth0 +guest ip address add 2001:db8::2/64 dev eth0 +guest ip route add default via 192.0.2.1 +guest ip -6 route add default via 2001:db8::1 dev eth0 guest meminfo_size() { grep "^$2:" $1 | tr -s ' ' | cut -f2 -d ' '; } guest meminfo_diff() { echo $(( $(meminfo_size $2 $3) - $(meminfo_size $1 $3) )); } @@ -103,27 +110,17 @@ info th symbol MiB set WHAT tcp_buf_discard nm_row -set WHAT tcp6_l2_buf +set WHAT flowtab nm_row -set WHAT tcp4_l2_buf +set WHAT tcp6_payload nm_row -set WHAT tc +set WHAT tcp4_payload nm_row set WHAT pkt_buf nm_row -set WHAT udp_splice_map -nm_row -set WHAT udp6_l2_buf -nm_row -set WHAT udp4_l2_buf -nm_row -set WHAT udp_tap_map +set WHAT udp_payload nm_row -set WHAT icmp_id_map -nm_row -set WHAT udp_splice_buf -nm_row -set WHAT tc_hash +set WHAT flow_hashtab nm_row set WHAT pool_tap6_storage nm_row @@ -142,8 +139,6 @@ set WHAT pid slab_row set WHAT dentry slab_row -set WHAT Acpi-Parse -slab_row set WHAT kmalloc-64 slab_row set WHAT kmalloc-32 diff --git a/test/nstool.c b/test/nstool.c index 1bdf44e..7ab5d2a 100644 --- a/test/nstool.c +++ b/test/nstool.c @@ -31,10 +31,15 @@ #define ARRAY_SIZE(a) ((int)(sizeof(a) / sizeof((a)[0]))) -#define die(...) \ - do { \ - fprintf(stderr, __VA_ARGS__); \ - exit(1); \ +#define die(...) \ + do { \ + fprintf(stderr, "nstool: " __VA_ARGS__); \ + exit(1); \ + } while (0) + +#define err(...) \ + do { \ + fprintf(stderr, "nstool: " __VA_ARGS__); \ } while (0) struct ns_type { @@ -156,6 +161,9 @@ static int connect_ctl(const char *sockpath, bool wait, static void cmd_hold(int argc, char *argv[]) { + struct sigaction sa = { + .sa_handler = SIG_IGN, + }; int fd = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, PF_UNIX); struct sockaddr_un addr; const char *sockpath = argv[1]; @@ -185,6 +193,10 @@ static void cmd_hold(int argc, char *argv[]) if (!getcwd(info.cwd, sizeof(info.cwd))) die("getcwd(): %s\n", strerror(errno)); + rc = sigaction(SIGPIPE, &sa, NULL); + if (rc) + die("sigaction(SIGPIPE): %s\n", strerror(errno)); + do { int afd = accept(fd, NULL, NULL); char buf; @@ -193,17 +205,21 @@ static void cmd_hold(int argc, char *argv[]) die("accept(): %s\n", strerror(errno)); rc = write(afd, &info, sizeof(info)); - if (rc < 0) - die("write(): %s\n", strerror(errno)); + if (rc < 0) { + err("holder write() to control socket: %s\n", + strerror(errno)); + } if ((size_t)rc < sizeof(info)) - die("short write() on control socket\n"); + err("holder short write() on control socket\n"); rc = read(afd, &buf, sizeof(buf)); - if (rc < 0) - die("read(): %s\n", strerror(errno)); + if (rc < 0) { + err("holder read() on control socket: %s\n", + strerror(errno)); + } close(afd); - } while (rc == 0); + } while (rc <= 0); unlink(sockpath); } @@ -345,21 +361,43 @@ static int openns(const char *fmt, ...) return fd; } +static pid_t sig_pid; +static void sig_propagate(int signum) +{ + int err; + + err = kill(sig_pid, signum); + if (err) + die("Propagating %s: %s\n", strsignal(signum), strerror(errno)); +} + static void wait_for_child(pid_t pid) { - int status; + struct sigaction sa = { + .sa_handler = sig_propagate, + .sa_flags = SA_RESETHAND, + }; + int status, err; + + sig_pid = pid; + err = sigaction(SIGTERM, &sa, NULL); + if (err) + die("sigaction(SIGTERM): %s\n", strerror(errno)); /* Match the child's exit status, if possible */ for (;;) { pid_t rc; rc = waitpid(pid, &status, WUNTRACED); - if (rc < 0) + if (rc < 0) { + if (errno == EINTR) + continue; die("waitpid() on %d: %s\n", pid, strerror(errno)); + } if (rc != pid) die("waitpid() on %d returned %d", pid, rc); if (WIFSTOPPED(status)) { - /* Stop the parent to patch */ + /* Stop the parent to match */ kill(getpid(), SIGSTOP); /* We must have resumed, resume the child */ kill(pid, SIGCONT); @@ -508,7 +546,7 @@ static void cmd_exec(int argc, char *argv[]) /* CHILD */ if (argc > optind + 1) { exe = argv[optind + 1]; - xargs = (const char * const*)(argv + optind + 1); + xargs = (const char *const *)(argv + optind + 1); } else { exe = getenv("SHELL"); if (!exe) diff --git a/test/passt.mbuto b/test/passt.mbuto index 6240d5c..138d365 100755 --- a/test/passt.mbuto +++ b/test/passt.mbuto @@ -15,6 +15,14 @@ PROGS="${PROGS:-ash,dash,bash ip mount ls insmod mkdir ln cat chmod lsmod sed tr chown sipcalc cut socat dd strace ping tail killall sleep sysctl nproc tcp_rr tcp_crr udp_rr which tee seq bc sshd ssh-keygen cmp}" +# OpenSSH 9.8 introduced split binaries, with sshd being the daemon, and +# sshd-session the per-session program. We need the latter as well, and the path +# depends on the distribution. It doesn't exist on older versions. +for bin in /usr/lib/openssh/sshd-session /usr/lib/ssh/sshd-session \ + /usr/libexec/openssh/sshd-session; do + command -v "${bin}" >/dev/null && PROGS="${PROGS} ${bin}" +done + KMODS="${KMODS:- virtio_net virtio_pci vmw_vsock_virtio_transport}" LINKS="${LINKS:- @@ -54,7 +62,7 @@ EOF ln -s /run /var/run :> /etc/fstab - # sshd(dropbear) via vsock + # sshd via vsock cat > /etc/passwd << EOF root:x:0:0:root:/root:/bin/sh sshd:x:100:100:Privilege-separated SSH:/var/empty/sshd:/sbin/nologin @@ -64,7 +72,9 @@ root:::0:99999:7::: EOF chmod 000 /etc/shadow - :> /etc/ssh/sshd_config + cat > /etc/ssh/sshd_config << EOF +Subsystem sftp internal-sftp +EOF ssh-keygen -A chmod 700 /root/.ssh chmod 700 /run/sshd @@ -76,7 +86,7 @@ EOF EOF chmod 600 /root/.ssh/authorized_keys chmod 700 /root - socat VSOCK-LISTEN:22,fork EXEC:"sshd -i -e" 2> /var/log/vsock-ssh.log & + socat VSOCK-LISTEN:22,fork EXEC:"/sbin/sshd -i -e" 2> /var/log/vsock-ssh.log & sh +m ' diff --git a/test/passt.mem.mbuto b/test/passt.mem.mbuto index 56f5139..532eae0 100755 --- a/test/passt.mem.mbuto +++ b/test/passt.mem.mbuto @@ -12,7 +12,7 @@ PROGS="${PROGS:-ash,dash,bash chmod ip mount insmod mkdir ln cat chmod modprobe grep mknod sed chown sleep bc ls ps mount unshare chroot cp kill diff - head tail sort tr tee cut nm which}" + head tail sort tr tee cut nm which switch_root}" KMODS="${KMODS:- dummy}" @@ -29,13 +29,6 @@ COPIES="${COPIES} ../passt.avx2,/bin/passt.avx2" FIXUP="${FIXUP}"' ln -s /bin /usr/bin chmod 777 /tmp -ip link add eth0 type dummy -ip link set eth0 up -ip address add 192.0.2.2/24 dev eth0 -ip address add 2001:db8::2/64 dev eth0 -ip route add default via 192.0.2.1 -ip -6 route add default via 2001:db8::1 dev eth0 -sleep 2 sh +m ' diff --git a/test/passt/dhcp b/test/passt/dhcp index 53ee641..9925ab9 100644 --- a/test/passt/dhcp +++ b/test/passt/dhcp @@ -38,7 +38,7 @@ check [ __MTU__ = 65520 ] test DHCP: DNS gout DNS sed -n 's/^nameserver \([0-9]*\.\)\(.*\)/\1\2/p' /etc/resolv.conf | tr '\n' ',' | sed 's/,$//;s/$/\n/' hout HOST_DNS sed -n 's/^nameserver \([0-9]*\.\)\(.*\)/\1\2/p' /etc/resolv.conf | head -n3 | tr '\n' ',' | sed 's/,$//;s/$/\n/' -check [ "__DNS__" = "__HOST_DNS__" ] || [ "__DNS__" = "__HOST_GW__" -a "__HOST_DNS__" = "127.0.0.1" ] +check [ "__DNS__" = "__HOST_DNS__" ] || ( [ "__DNS__" = "__HOST_GW__" ] && expr "__HOST_DNS__" : "127[.]" ) # FQDNs should be terminated by dots, but the guest DHCP client might omit them: # strip them first @@ -49,8 +49,10 @@ check [ "__SEARCH__" = "__HOST_SEARCH__" ] test DHCPv6: address guest /sbin/dhclient -6 __IFNAME__ +# Wait for DAD to complete +guest while ip -j -6 addr show tentative | jq -e '.[].addr_info'; do sleep 0.1; done gout ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__IFNAME__").addr_info[] | select(.prefixlen == 128).local] | .[0]' -hout HOST_ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__HOST_IFNAME6__").addr_info[] | select(.scope == "global").local] | .[0]' +hout HOST_ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__HOST_IFNAME6__").addr_info[] | select(.scope == "global" and .deprecated != true).local] | .[0]' check [ "__ADDR6__" = "__HOST_ADDR6__" ] test DHCPv6: route diff --git a/test/passt/ndp b/test/passt/ndp index 7b2dbfe..56b385b 100644 --- a/test/passt/ndp +++ b/test/passt/ndp @@ -16,14 +16,16 @@ htools ip jq sipcalc grep cut test Interface name gout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' -guest ip link set dev __IFNAME__ up && sleep 2 +guest ip link set dev __IFNAME__ up +# Wait for DAD to complete +guest while ip -j -6 addr show tentative | jq -e '.[].addr_info'; do sleep 0.1; done hout HOST_IFNAME6 ip -j -6 route show|jq -rM '[.[] | select(.dst == "default").dev] | .[0]' check [ -n "__IFNAME__" ] test SLAAC: prefix -gout ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__IFNAME__").addr_info[] | select(.scope == "global" and .prefixlen == 64).local] | .[0]' -gout PREFIX6 sipcalc __ADDR6__/64 | grep prefix | cut -d' ' -f4 -hout HOST_ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__HOST_IFNAME6__").addr_info[] | select(.scope == "global").local] | .[0]' +gout ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__IFNAME__").addr_info[] | select(.scope == "global" and .protocol == "kernel_ra") | .local + "/" + (.prefixlen | tostring)] | .[0]' +gout PREFIX6 sipcalc __ADDR6__ | grep prefix | cut -d' ' -f4 +hout HOST_ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__HOST_IFNAME6__").addr_info[] | select(.scope == "global" and .deprecated != true).local] | .[0]' hout HOST_PREFIX6 sipcalc __HOST_ADDR6__/64 | grep prefix | cut -d' ' -f4 check [ "__PREFIX6__" = "__HOST_PREFIX6__" ] diff --git a/test/passt_in_ns/dhcp b/test/passt_in_ns/dhcp new file mode 100644 index 0000000..a38a690 --- /dev/null +++ b/test/passt_in_ns/dhcp @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# PASST - Plug A Simple Socket Transport +# for qemu/UNIX domain socket mode +# +# PASTA - Pack A Subtle Tap Abstraction +# for network namespace/tap device mode +# +# test/passt/dhcp - Check DHCP and DHCPv6 functionality in passt mode +# +# Copyright (c) 2021 Red Hat GmbH +# Author: Stefano Brivio <sbrivio@redhat.com> + +gtools ip jq dhclient sed tr +htools ip jq sed tr head + +set MAP_NS4 192.0.2.2 +set MAP_NS6 2001:db8:9a55::2 + +test Interface name +gout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' +hout HOST_IFNAME ip -j -4 route show|jq -rM '[.[] | select(.dst == "default").dev] | .[0]' +hout HOST_IFNAME6 ip -j -6 route show|jq -rM '[.[] | select(.dst == "default").dev] | .[0]' +check [ -n "__IFNAME__" ] + +test DHCP: address +guest /sbin/dhclient -4 __IFNAME__ +gout ADDR ip -j -4 addr show|jq -rM '.[] | select(.ifname == "__IFNAME__").addr_info[0].local' +hout HOST_ADDR ip -j -4 addr show|jq -rM '.[] | select(.ifname == "__HOST_IFNAME__").addr_info[0].local' +check [ "__ADDR__" = "__HOST_ADDR__" ] + +test DHCP: route +gout GW ip -j -4 route show|jq -rM '.[] | select(.dst == "default").gateway' +hout HOST_GW ip -j -4 route show|jq -rM '[.[] | select(.dst == "default").gateway] | .[0]' +check [ "__GW__" = "__HOST_GW__" ] + +test DHCP: MTU +gout MTU ip -j link show | jq -rM '.[] | select(.ifname == "__IFNAME__").mtu' +check [ __MTU__ = 65520 ] + +test DHCP: DNS +gout DNS sed -n 's/^nameserver \([0-9]*\.\)\(.*\)/\1\2/p' /etc/resolv.conf | tr '\n' ',' | sed 's/,$//;s/$/\n/' +hout HOST_DNS sed -n 's/^nameserver \([0-9]*\.\)\(.*\)/\1\2/p' /etc/resolv.conf | head -n3 | tr '\n' ',' | sed 's/,$//;s/$/\n/' +check [ "__DNS__" = "__HOST_DNS__" ] || ( [ "__DNS__" = "__MAP_NS4__" ] && expr "__HOST_DNS__" : "127[.]" ) + +# FQDNs should be terminated by dots, but the guest DHCP client might omit them: +# strip them first +test DHCP: search list +gout SEARCH sed 's/\. / /g' /etc/resolv.conf | sed 's/\.$//g' | sed -n 's/^search \(.*\)/\1/p' | tr ' \n' ',' | sed 's/,$//;s/$/\n/' +hout HOST_SEARCH sed 's/\. / /g' /etc/resolv.conf | sed 's/\.$//g' | sed -n 's/^search \(.*\)/\1/p' | tr ' \n' ',' | sed 's/,$//;s/$/\n/' +check [ "__SEARCH__" = "__HOST_SEARCH__" ] + +test DHCPv6: address +guest /sbin/dhclient -6 __IFNAME__ +# Wait for DAD to complete +guest while ip -j -6 addr show tentative | jq -e '.[].addr_info'; do sleep 0.1; done +gout ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__IFNAME__").addr_info[] | select(.prefixlen == 128).local] | .[0]' +hout HOST_ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__HOST_IFNAME6__").addr_info[] | select(.scope == "global" and .deprecated != true).local] | .[0]' +check [ "__ADDR6__" = "__HOST_ADDR6__" ] + +test DHCPv6: route +gout GW6 ip -j -6 route show|jq -rM '.[] | select(.dst == "default").gateway' +hout HOST_GW6 ip -j -6 route show|jq -rM '[.[] | select(.dst == "default").gateway] | .[0]' +check [ "__GW6__" = "__HOST_GW6__" ] + +# Strip interface specifier: interface names might differ between host and guest +test DHCPv6: DNS +gout DNS6 sed -n 's/^nameserver \([^:]*:\)\([^%]*\).*/\1\2/p' /etc/resolv.conf | tr '\n' ',' | sed 's/,$//;s/$/\n/' +hout HOST_DNS6 sed -n 's/^nameserver \([^:]*:\)\([^%]*\).*/\1\2/p' /etc/resolv.conf | tr '\n' ',' | sed 's/,$//;s/$/\n/' +check [ "__DNS6__" = "__HOST_DNS6__" ] || [ "__DNS6__" = "__MAP_NS6__" -a "__HOST_DNS6__" = "::1" ] + +test DHCPv6: search list +gout SEARCH6 sed 's/\. / /g' /etc/resolv.conf | sed 's/\.$//g' | sed -n 's/^search \(.*\)/\1/p' | tr ' \n' ',' | sed 's/,$//;s/$/\n/' +hout HOST_SEARCH6 sed 's/\. / /g' /etc/resolv.conf | sed 's/\.$//g' | sed -n 's/^search \(.*\)/\1/p' | tr ' \n' ',' | sed 's/,$//;s/$/\n/' +check [ "__SEARCH6__" = "__HOST_SEARCH6__" ] diff --git a/test/passt_in_ns/tcp b/test/passt_in_ns/tcp index cdb7060..319880b 100644 --- a/test/passt_in_ns/tcp +++ b/test/passt_in_ns/tcp @@ -15,6 +15,11 @@ gtools socat ip jq htools socat ip jq nstools socat ip jq +set MAP_HOST4 192.0.2.1 +set MAP_HOST6 2001:db8:9a55::1 +set MAP_NS4 192.0.2.2 +set MAP_NS6 2001:db8:9a55::2 + set TEMP_BIG __STATEDIR__/test_big.bin set TEMP_SMALL __STATEDIR__/test_small.bin set TEMP_NS_BIG __STATEDIR__/test_ns_big.bin @@ -27,7 +32,7 @@ host socat -u OPEN:__BASEPATH__/big.bin TCP4:127.0.0.1:10001 guestw guest cmp test_big.bin /root/big.bin -test TCP/IPv4: host to ns: big transfer +test TCP/IPv4: host to ns (spliced): big transfer nsb socat -u TCP4-LISTEN:10002 OPEN:__TEMP_NS_BIG__,create,trunc sleep 1 host socat -u OPEN:__BASEPATH__/big.bin TCP4:127.0.0.1:10002 @@ -36,16 +41,15 @@ check cmp __TEMP_NS_BIG__ __BASEPATH__/big.bin test TCP/IPv4: guest to host: big transfer hostb socat -u TCP4-LISTEN:10003 OPEN:__TEMP_BIG__,create,trunc -gout GW ip -j -4 route show|jq -rM '.[] | select(.dst == "default").gateway' sleep 1 -guest socat -u OPEN:/root/big.bin TCP4:__GW__:10003 +guest socat -u OPEN:/root/big.bin TCP4:__MAP_HOST4__:10003 hostw check cmp __TEMP_BIG__ __BASEPATH__/big.bin test TCP/IPv4: guest to ns: big transfer nsb socat -u TCP4-LISTEN:10002 OPEN:__TEMP_NS_BIG__,create,trunc sleep 1 -guest socat -u OPEN:/root/big.bin TCP4:__GW__:10002 +guest socat -u OPEN:/root/big.bin TCP4:__MAP_NS4__:10002 nsw check cmp __TEMP_NS_BIG__ __BASEPATH__/big.bin @@ -59,7 +63,7 @@ check cmp __TEMP_BIG__ __BASEPATH__/big.bin test TCP/IPv4: ns to host (via tap): big transfer hostb socat -u TCP4-LISTEN:10003 OPEN:__TEMP_BIG__,create,trunc sleep 1 -ns socat -u OPEN:__BASEPATH__/big.bin TCP4:__GW__:10003 +ns socat -u OPEN:__BASEPATH__/big.bin TCP4:__MAP_HOST4__:10003 hostw check cmp __TEMP_BIG__ __BASEPATH__/big.bin @@ -86,7 +90,7 @@ host socat -u OPEN:__BASEPATH__/small.bin TCP4:127.0.0.1:10001 guestw guest cmp test_small.bin /root/small.bin -test TCP/IPv4: host to ns: small transfer +test TCP/IPv4: host to ns (spliced): small transfer nsb socat -u TCP4-LISTEN:10002 OPEN:__TEMP_NS_SMALL__,create,trunc sleep 1 host socat -u OPEN:__BASEPATH__/small.bin TCP4:127.0.0.1:10002 @@ -95,16 +99,15 @@ check cmp __TEMP_NS_SMALL__ __BASEPATH__/small.bin test TCP/IPv4: guest to host: small transfer hostb socat -u TCP4-LISTEN:10003 OPEN:__TEMP_SMALL__,create,trunc -gout GW ip -j -4 route show|jq -rM '.[] | select(.dst == "default").gateway' sleep 1 -guest socat -u OPEN:/root/small.bin TCP4:__GW__:10003 +guest socat -u OPEN:/root/small.bin TCP4:__MAP_HOST4__:10003 hostw check cmp __TEMP_SMALL__ __BASEPATH__/small.bin test TCP/IPv4: guest to ns: small transfer nsb socat -u TCP4-LISTEN:10002 OPEN:__TEMP_NS_SMALL__,create,trunc sleep 1 -guest socat -u OPEN:/root/small.bin TCP4:__GW__:10002 +guest socat -u OPEN:/root/small.bin TCP4:__MAP_NS4__:10002 nsw check cmp __TEMP_NS_SMALL__ __BASEPATH__/small.bin @@ -118,7 +121,7 @@ check cmp __TEMP_SMALL__ __BASEPATH__/small.bin test TCP/IPv4: ns to host (via tap): small transfer hostb socat -u TCP4-LISTEN:10003 OPEN:__TEMP_SMALL__,create,trunc sleep 1 -ns socat -u OPEN:__BASEPATH__/small.bin TCP4:__GW__:10003 +ns socat -u OPEN:__BASEPATH__/small.bin TCP4:__MAP_HOST4__:10003 hostw check cmp __TEMP_SMALL__ __BASEPATH__/small.bin @@ -143,7 +146,7 @@ host socat -u OPEN:__BASEPATH__/big.bin TCP6:[::1]:10001 guestw guest cmp test_big.bin /root/big.bin -test TCP/IPv6: host to ns: big transfer +test TCP/IPv6: host to ns (spliced): big transfer nsb socat -u TCP6-LISTEN:10002 OPEN:__TEMP_NS_BIG__,create,trunc sleep 1 host socat -u OPEN:__BASEPATH__/big.bin TCP6:[::1]:10002 @@ -152,17 +155,15 @@ check cmp __TEMP_NS_BIG__ __BASEPATH__/big.bin test TCP/IPv6: guest to host: big transfer hostb socat -u TCP6-LISTEN:10003 OPEN:__TEMP_BIG__,create,trunc -gout GW6 ip -j -6 route show|jq -rM '.[] | select(.dst == "default").gateway' -gout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' sleep 1 -guest socat -u OPEN:/root/big.bin TCP6:[__GW6__%__IFNAME__]:10003 +guest socat -u OPEN:/root/big.bin TCP6:[__MAP_HOST6__]:10003 hostw check cmp __TEMP_BIG__ __BASEPATH__/big.bin test TCP/IPv6: guest to ns: big transfer nsb socat -u TCP6-LISTEN:10002 OPEN:__TEMP_NS_BIG__,create,trunc sleep 1 -guest socat -u OPEN:/root/big.bin TCP6:[__GW6__%__IFNAME__]:10002 +guest socat -u OPEN:/root/big.bin TCP6:[__MAP_NS6__]:10002 nsw check cmp __TEMP_NS_BIG__ __BASEPATH__/big.bin @@ -175,9 +176,8 @@ check cmp __TEMP_BIG__ __BASEPATH__/big.bin test TCP/IPv6: ns to host (via tap): big transfer hostb socat -u TCP6-LISTEN:10003 OPEN:__TEMP_BIG__,create,trunc -nsout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' sleep 1 -ns socat -u OPEN:__BASEPATH__/big.bin TCP6:[__GW6__%__IFNAME__]:10003 +ns socat -u OPEN:__BASEPATH__/big.bin TCP6:[__MAP_HOST6__]:10003 hostw check cmp __TEMP_BIG__ __BASEPATH__/big.bin @@ -190,6 +190,7 @@ guest cmp test_big.bin /root/big.bin test TCP/IPv6: ns to guest (using namespace address): big transfer guestb socat -u TCP6-LISTEN:10001 OPEN:test_big.bin,create,trunc +nsout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' nsout ADDR6 ip -j -6 addr show|jq -rM '.[] | select(.ifname == "__IFNAME__").addr_info[0].local' sleep 1 ns socat -u OPEN:__BASEPATH__/big.bin TCP6:[__ADDR6__]:10001 @@ -203,7 +204,7 @@ host socat -u OPEN:__BASEPATH__/small.bin TCP6:[::1]:10001 guestw guest cmp test_small.bin /root/small.bin -test TCP/IPv6: host to ns: small transfer +test TCP/IPv6: host to ns (spliced): small transfer nsb socat -u TCP6-LISTEN:10002 OPEN:__TEMP_NS_SMALL__,create,trunc sleep 1 host socat -u OPEN:__BASEPATH__/small.bin TCP6:[::1]:10002 @@ -212,17 +213,15 @@ check cmp __TEMP_NS_SMALL__ __BASEPATH__/small.bin test TCP/IPv6: guest to host: small transfer hostb socat -u TCP6-LISTEN:10003 OPEN:__TEMP_SMALL__,create,trunc -gout GW6 ip -j -6 route show|jq -rM '.[] | select(.dst == "default").gateway' -gout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' sleep 1 -guest socat -u OPEN:/root/small.bin TCP6:[__GW6__%__IFNAME__]:10003 +guest socat -u OPEN:/root/small.bin TCP6:[__MAP_HOST6__]:10003 hostw check cmp __TEMP_SMALL__ __BASEPATH__/small.bin test TCP/IPv6: guest to ns: small transfer nsb socat -u TCP6-LISTEN:10002 OPEN:__TEMP_NS_SMALL__ sleep 1 -guest socat -u OPEN:/root/small.bin TCP6:[__GW6__%__IFNAME__]:10002 +guest socat -u OPEN:/root/small.bin TCP6:[__MAP_NS6__]:10002 nsw check cmp __TEMP_NS_SMALL__ __BASEPATH__/small.bin @@ -235,9 +234,8 @@ check cmp __TEMP_SMALL__ __BASEPATH__/small.bin test TCP/IPv6: ns to host (via tap): small transfer hostb socat -u TCP6-LISTEN:10003 OPEN:__TEMP_SMALL__,create,trunc -nsout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' sleep 1 -ns socat -u OPEN:__BASEPATH__/small.bin TCP6:[__GW6__%__IFNAME__]:10003 +ns socat -u OPEN:__BASEPATH__/small.bin TCP6:[__MAP_HOST6__]:10003 hostw check cmp __TEMP_SMALL__ __BASEPATH__/small.bin diff --git a/test/passt_in_ns/udp b/test/passt_in_ns/udp index 8a02513..791511c 100644 --- a/test/passt_in_ns/udp +++ b/test/passt_in_ns/udp @@ -15,6 +15,11 @@ gtools socat ip jq nstools socat ip jq htools socat ip jq +set MAP_HOST4 192.0.2.1 +set MAP_HOST6 2001:db8:9a55::1 +set MAP_NS4 192.0.2.2 +set MAP_NS6 2001:db8:9a55::2 + set TEMP __STATEDIR__/test.bin set TEMP_NS __STATEDIR__/test_ns.bin @@ -25,7 +30,7 @@ host socat -u OPEN:__BASEPATH__/medium.bin UDP4:127.0.0.1:10001,shut-null guestw guest cmp test.bin /root/medium.bin -test UDP/IPv4: host to ns +test UDP/IPv4: host to ns (recvmmsg/sendmmsg) nsb socat -u UDP4-LISTEN:10002,null-eof OPEN:__TEMP_NS__,create,trunc sleep 1 host socat -u OPEN:__BASEPATH__/medium.bin UDP4:127.0.0.1:10002,shut-null @@ -34,16 +39,15 @@ check cmp __TEMP_NS__ __BASEPATH__/medium.bin test UDP/IPv4: guest to host hostb socat -u UDP4-LISTEN:10003,null-eof OPEN:__TEMP__,create,trunc -gout GW ip -j -4 route show|jq -rM '.[] | select(.dst == "default").gateway' sleep 1 -guest socat -u OPEN:/root/medium.bin UDP4:__GW__:10003,shut-null +guest socat -u OPEN:/root/medium.bin UDP4:__MAP_HOST4__:10003,shut-null hostw check cmp __TEMP__ __BASEPATH__/medium.bin test UDP/IPv4: guest to ns nsb socat -u UDP4-LISTEN:10002,null-eof OPEN:__TEMP_NS__,create,trunc sleep 1 -guest socat -u OPEN:/root/medium.bin UDP4:__GW__:10002,shut-null +guest socat -u OPEN:/root/medium.bin UDP4:__MAP_NS4__:10002,shut-null nsw check cmp __TEMP_NS__ __BASEPATH__/medium.bin @@ -57,7 +61,7 @@ check cmp __TEMP__ __BASEPATH__/medium.bin test UDP/IPv4: ns to host (via tap) hostb socat -u UDP4-LISTEN:10003,null-eof OPEN:__TEMP__,create,trunc sleep 1 -ns socat -u OPEN:__BASEPATH__/medium.bin UDP4:__GW__:10003,shut-null +ns socat -u OPEN:__BASEPATH__/medium.bin UDP4:__MAP_HOST4__:10003,shut-null hostw check cmp __TEMP__ __BASEPATH__/medium.bin @@ -84,7 +88,7 @@ host socat -u OPEN:__BASEPATH__/medium.bin UDP6:[::1]:10001,shut-null guestw guest cmp test.bin /root/medium.bin -test UDP/IPv6: host to ns +test UDP/IPv6: host to ns (recvmmsg/sendmmsg) nsb socat -u UDP6-LISTEN:10002,null-eof OPEN:__TEMP_NS__,create,trunc sleep 1 host socat -u OPEN:__BASEPATH__/medium.bin UDP6:[::1]:10002,shut-null @@ -93,17 +97,15 @@ check cmp __TEMP_NS__ __BASEPATH__/medium.bin test UDP/IPv6: guest to host hostb socat -u UDP6-LISTEN:10003,null-eof OPEN:__TEMP__,create,trunc -gout GW6 ip -j -6 route show|jq -rM '.[] | select(.dst == "default").gateway' -gout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' sleep 1 -guest socat -u OPEN:/root/medium.bin UDP6:[__GW6__%__IFNAME__]:10003,shut-null +guest socat -u OPEN:/root/medium.bin UDP6:[__MAP_HOST6__]:10003,shut-null hostw check cmp __TEMP__ __BASEPATH__/medium.bin test UDP/IPv6: guest to ns nsb socat -u UDP6-LISTEN:10002,null-eof OPEN:__TEMP_NS__,create,trunc sleep 1 -guest socat -u OPEN:/root/medium.bin UDP6:[__GW6__%__IFNAME__]:10002,shut-null +guest socat -u OPEN:/root/medium.bin UDP6:[__MAP_NS6__]:10002,shut-null nsw check cmp __TEMP_NS__ __BASEPATH__/medium.bin @@ -116,9 +118,8 @@ check cmp __TEMP__ __BASEPATH__/medium.bin test UDP/IPv6: ns to host (via tap) hostb socat -u UDP6-LISTEN:10003,null-eof OPEN:__TEMP__,create,trunc -nsout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' sleep 1 -ns socat -u OPEN:__BASEPATH__/medium.bin UDP6:[__GW6__%__IFNAME__]:10003,shut-null +ns socat -u OPEN:__BASEPATH__/medium.bin UDP6:[__MAP_HOST6__]:10003,shut-null hostw check cmp __TEMP__ __BASEPATH__/medium.bin @@ -131,6 +132,7 @@ guest cmp test.bin /root/medium.bin test UDP/IPv6: ns to guest (using namespace address) guestb socat -u UDP6-LISTEN:10001,null-eof OPEN:test.bin,create,trunc +nsout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' nsout ADDR6 ip -j -6 addr show|jq -rM '.[] | select(.ifname == "__IFNAME__").addr_info[0].local' sleep 1 ns socat -u OPEN:__BASEPATH__/medium.bin UDP6:[__ADDR6__]:10001,shut-null diff --git a/test/passt_vu b/test/passt_vu new file mode 120000 index 0000000..22f1840 --- /dev/null +++ b/test/passt_vu @@ -0,0 +1 @@ +passt
\ No newline at end of file diff --git a/test/passt_vu_in_ns b/test/passt_vu_in_ns new file mode 120000 index 0000000..3ff479e --- /dev/null +++ b/test/passt_vu_in_ns @@ -0,0 +1 @@ +passt_in_ns
\ No newline at end of file diff --git a/test/pasta/dhcp b/test/pasta/dhcp index 112633a..d4f3ad5 100644 --- a/test/pasta/dhcp +++ b/test/pasta/dhcp @@ -35,9 +35,11 @@ check [ __MTU__ = 65520 ] test DHCPv6: address ns /sbin/dhclient -6 --no-pid __IFNAME__ +# Wait for DAD to complete +ns while ip -j -6 addr show tentative | jq -e '.[].addr_info'; do sleep 0.1; done hout HOST_IFNAME6 ip -j -6 route show|jq -rM '[.[] | select(.dst == "default").dev] | .[0]' nsout ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__IFNAME__").addr_info[] | select(.prefixlen == 128).local] | .[0]' -hout HOST_ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__HOST_IFNAME6__").addr_info[] | select(.scope == "global").local] | .[0]' +hout HOST_ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__HOST_IFNAME6__").addr_info[] | select(.scope == "global" and .deprecated != true).local] | .[0]' check [ __ADDR6__ = __HOST_ADDR6__ ] test DHCPv6: route diff --git a/test/pasta/ndp b/test/pasta/ndp index 2a8afe6..2442ab5 100644 --- a/test/pasta/ndp +++ b/test/pasta/ndp @@ -18,12 +18,13 @@ test Interface name nsout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' check [ -n "__IFNAME__" ] ns ip link set dev __IFNAME__ up -sleep 2 +# Wait for DAD to complete +ns while ip -j -6 addr show tentative | jq -e '.[].addr_info'; do sleep 0.1; done test SLAAC: prefix -nsout ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__IFNAME__").addr_info[] | select(.scope == "global" and .prefixlen == 64).local] | .[0]' -nsout PREFIX6 sipcalc __ADDR6__/64 | grep prefix | cut -d' ' -f4 -hout HOST_ADDR6 ip -j -6 addr show|jq -rM ['.[] | select(.ifname == "__IFNAME__").addr_info[] | select(.scope == "global").local] | .[0]' +nsout ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__IFNAME__").addr_info[] | select(.scope == "global" and .protocol == "kernel_ra") | .local + "/" + (.prefixlen | tostring)] | .[0]' +nsout PREFIX6 sipcalc __ADDR6__ | grep prefix | cut -d' ' -f4 +hout HOST_ADDR6 ip -j -6 addr show|jq -rM ['.[] | select(.ifname == "__IFNAME__").addr_info[] | select(.scope == "global" and .deprecated != true).local] | .[0]' hout HOST_PREFIX6 sipcalc __HOST_ADDR6__/64 | grep prefix | cut -d' ' -f4 check [ "__PREFIX6__" = "__HOST_PREFIX6__" ] diff --git a/test/pasta/tcp b/test/pasta/tcp index 6ab18c5..53b6f25 100644 --- a/test/pasta/tcp +++ b/test/pasta/tcp @@ -19,8 +19,8 @@ set TEMP_NS_BIG __STATEDIR__/test_ns_big.bin set TEMP_SMALL __STATEDIR__/test_small.bin set TEMP_NS_SMALL __STATEDIR__/test_ns_small.bin -test TCP/IPv4: host to ns: big transfer -nsb socat -u TCP4-LISTEN:10002,bind=127.0.0.1 OPEN:__TEMP_NS_BIG__,create,trunc +test TCP/IPv4: host to ns (spliced): big transfer +nsb socat -u TCP4-LISTEN:10002 OPEN:__TEMP_NS_BIG__,create,trunc host socat -u OPEN:__BASEPATH__/big.bin TCP4:127.0.0.1:10002 nsw check cmp __BASEPATH__/big.bin __TEMP_NS_BIG__ @@ -38,8 +38,8 @@ ns socat -u OPEN:__BASEPATH__/big.bin TCP4:__GW__:10003 hostw check cmp __BASEPATH__/big.bin __TEMP_BIG__ -test TCP/IPv4: host to ns: small transfer -nsb socat -u TCP4-LISTEN:10002,bind=127.0.0.1 OPEN:__TEMP_NS_SMALL__,create,trunc +test TCP/IPv4: host to ns (spliced): small transfer +nsb socat -u TCP4-LISTEN:10002 OPEN:__TEMP_NS_SMALL__,create,trunc host socat OPEN:__BASEPATH__/small.bin TCP4:127.0.0.1:10002 nsw check cmp __BASEPATH__/small.bin __TEMP_NS_SMALL__ @@ -57,8 +57,8 @@ ns socat -u OPEN:__BASEPATH__/small.bin TCP4:__GW__:10003 hostw check cmp __BASEPATH__/small.bin __TEMP_SMALL__ -test TCP/IPv6: host to ns: big transfer -nsb socat -u TCP6-LISTEN:10002,bind=[::1] OPEN:__TEMP_NS_BIG__,create,trunc +test TCP/IPv6: host to ns (spliced): big transfer +nsb socat -u TCP6-LISTEN:10002 OPEN:__TEMP_NS_BIG__,create,trunc host socat -u OPEN:__BASEPATH__/big.bin TCP6:[::1]:10002 nsw check cmp __BASEPATH__/big.bin __TEMP_NS_BIG__ @@ -77,8 +77,8 @@ ns socat -u OPEN:__BASEPATH__/big.bin TCP6:[__GW6__%__IFNAME__]:10003 hostw check cmp __BASEPATH__/big.bin __TEMP_BIG__ -test TCP/IPv6: host to ns: small transfer -nsb socat -u TCP6-LISTEN:10002,bind=[::1] OPEN:__TEMP_NS_SMALL__,create,trunc +test TCP/IPv6: host to ns (spliced): small transfer +nsb socat -u TCP6-LISTEN:10002 OPEN:__TEMP_NS_SMALL__,create,trunc host socat -u OPEN:__BASEPATH__/small.bin TCP6:[::1]:10002 nsw check cmp __BASEPATH__/small.bin __TEMP_NS_SMALL__ diff --git a/test/pasta/udp b/test/pasta/udp index 30e3a85..7734d02 100644 --- a/test/pasta/udp +++ b/test/pasta/udp @@ -17,8 +17,8 @@ htools dd socat ip jq set TEMP __STATEDIR__/test.bin set TEMP_NS __STATEDIR__/test_ns.bin -test UDP/IPv4: host to ns -nsb socat -u UDP4-LISTEN:10002,bind=127.0.0.1,null-eof OPEN:__TEMP_NS__,create,trunc +test UDP/IPv4: host to ns (recvmmsg/sendmmsg) +nsb socat -u UDP4-LISTEN:10002,null-eof OPEN:__TEMP_NS__,create,trunc host socat OPEN:__BASEPATH__/medium.bin UDP4:127.0.0.1:10002,shut-null nsw check cmp __BASEPATH__/medium.bin __TEMP_NS__ @@ -37,8 +37,8 @@ ns socat -u OPEN:__BASEPATH__/medium.bin UDP4:__GW__:10003,shut-null hostw check cmp __BASEPATH__/medium.bin __TEMP__ -test UDP/IPv6: host to ns -nsb socat -u UDP6-LISTEN:10002,bind=[::1],null-eof OPEN:__TEMP_NS__,create,trunc +test UDP/IPv6: host to ns (recvmmsg/sendmmsg) +nsb socat -u UDP6-LISTEN:10002,null-eof OPEN:__TEMP_NS__,create,trunc host socat -u OPEN:__BASEPATH__/medium.bin UDP6:[::1]:10002,shut-null nsw check cmp __BASEPATH__/medium.bin __TEMP_NS__ diff --git a/test/pasta_options/log_to_file b/test/pasta_options/log_to_file index fcdd553..3ead06c 100644 --- a/test/pasta_options/log_to_file +++ b/test/pasta_options/log_to_file @@ -19,7 +19,7 @@ sleep 1 endef def flood_log_client -host tcp_crr --nolog -P 10001 -C 10002 -6 -c -H ::1 +host tcp_crr --nolog -l1 -P 10001 -C 10002 -6 -c -H ::1 endef def check_log_size_mountns @@ -33,19 +33,16 @@ test Log creation set PORTS -t 10001,10002 -u 10001,10002 set LOG_FILE __STATEDIR__/pasta.log -passt ./pasta -l __LOG_FILE__ -passtb exit -sleep 1 +passt ./pasta -l __LOG_FILE__ -- /bin/true check [ -s __LOG_FILE__ ] test Log truncated on creation -passt ./pasta -l __LOG_FILE__ -passtb exit -sleep 1 -check [ $(cat __LOG_FILE__ | wc -l) -eq 1 ] +passt ./pasta -l __LOG_FILE__ -- /bin/true & wait +pout PID2 echo $! +check head -1 __LOG_FILE__ | grep '^pasta .* [(]__PID2__[)]$' test Maximum log size -passtb ./pasta --config-net -d -f -l __LOG_FILE__ --log-size $((100 * 1024)) -- sh -c 'while true; do tcp_crr --nolog -P 10001 -C 10002 -6; done' +passtb ./pasta --config-net -d -f -l __LOG_FILE__ --log-size $((100 * 1024)) -- sh -c 'while true; do tcp_crr --nolog -l1 -P 10001 -C 10002 -6; done' sleep 1 flood_log_client diff --git a/test/pasta_podman/bats b/test/pasta_podman/bats index 21446f0..6b1c575 100644 --- a/test/pasta_podman/bats +++ b/test/pasta_podman/bats @@ -11,11 +11,16 @@ # Copyright (c) 2022 Red Hat GmbH # Author: Stefano Brivio <sbrivio@redhat.com> -htools git make go bats catatonit ip jq socat +htools git make go bats ip jq socat ./test/podman/bin/podman + +set PODMAN test/podman/bin/podman +hout WD pwd + +test Podman pasta path + +hout PASTA_BIN CONTAINERS_HELPER_BINARY_DIR="__WD__" __PODMAN__ info --format "{{.Host.Pasta.Executable}}" +check [ "__PASTA_BIN__" = "__WD__/pasta" ] test Podman system test with bats -host git -C __STATEDIR__ clone https://github.com/containers/podman.git -host make -C __STATEDIR__/podman -hout WD pwd -host PODMAN="__STATEDIR__/podman/bin/podman" CONTAINERS_HELPER_BINARY_DIR="__WD__" bats __STATEDIR__/podman/test/system/505-networking-pasta.bats +host PODMAN="__PODMAN__" CONTAINERS_HELPER_BINARY_DIR="__WD__" bats test/podman/test/system/505-networking-pasta.bats diff --git a/test/perf/passt_tcp b/test/perf/passt_tcp index 631a407..5978c49 100644 --- a/test/perf/passt_tcp +++ b/test/perf/passt_tcp @@ -15,6 +15,9 @@ gtools /sbin/sysctl ip jq nproc seq sleep iperf3 tcp_rr tcp_crr # From neper nstools /sbin/sysctl ip jq nproc seq sleep iperf3 tcp_rr tcp_crr htools bc head sed seq +set MAP_NS4 192.0.2.2 +set MAP_NS6 2001:db8:9a55::2 + test passt: throughput and latency guest /sbin/sysctl -w net.core.rmem_max=536870912 @@ -29,42 +32,39 @@ ns /sbin/sysctl -w net.ipv4.tcp_rmem="4096 524288 134217728" ns /sbin/sysctl -w net.ipv4.tcp_wmem="4096 524288 134217728" ns /sbin/sysctl -w net.ipv4.tcp_timestamps=0 -gout GW ip -j -4 route show|jq -rM '.[] | select(.dst == "default").gateway' -gout GW6 ip -j -6 route show|jq -rM '.[] | select(.dst == "default").gateway' gout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' hout FREQ_PROCFS (echo "scale=1"; sed -n 's/cpu MHz.*: \([0-9]*\)\..*$/(\1+10^2\/2)\/10^3/p' /proc/cpuinfo) | bc -l | head -n1 hout FREQ_CPUFREQ (echo "scale=1"; printf '( %i + 10^5 / 2 ) / 10^6\n' $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq) ) | bc -l hout FREQ [ -n "__FREQ_CPUFREQ__" ] && echo __FREQ_CPUFREQ__ || echo __FREQ_PROCFS__ -set THREADS 1 -set STREAMS 8 -set TIME 10 +set THREADS 4 +set TIME 1 set OMIT 0.1 -set OPTS -Z -P __STREAMS__ -l 1M -O__OMIT__ +set OPTS -Z -P __THREADS__ -l 1M -O__OMIT__ -info Throughput in Gbps, latency in µs, one thread at __FREQ__ GHz, __STREAMS__ streams +info Throughput in Gbps, latency in µs, __THREADS__ threads at __FREQ__ GHz report passt tcp __THREADS__ __FREQ__ th MTU 256B 576B 1280B 1500B 9000B 65520B tr TCP throughput over IPv6: guest to host -iperf3s ns 100${i}2 __THREADS__ +iperf3s ns 10002 bw - bw - guest ip link set dev __IFNAME__ mtu 1280 -iperf3 BW guest __GW6__%__IFNAME__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -w 4M +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -w 4M bw __BW__ 1.2 1.5 guest ip link set dev __IFNAME__ mtu 1500 -iperf3 BW guest __GW6__%__IFNAME__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -w 4M +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -w 4M bw __BW__ 1.6 1.8 guest ip link set dev __IFNAME__ mtu 9000 -iperf3 BW guest __GW6__%__IFNAME__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -w 8M +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -w 8M bw __BW__ 4.0 5.0 guest ip link set dev __IFNAME__ mtu 65520 -iperf3 BW guest __GW6__%__IFNAME__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -w 16M +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -w 16M bw __BW__ 7.0 8.0 iperf3k ns @@ -76,7 +76,7 @@ lat - lat - lat - nsb tcp_rr --nolog -6 -gout LAT tcp_rr --nolog -6 -c -H __GW6__%__IFNAME__ | sed -n 's/^throughput=\(.*\)/\1/p' +gout LAT tcp_rr --nolog -l1 -6 -c -H __MAP_NS6__ | sed -n 's/^throughput=\(.*\)/\1/p' lat __LAT__ 200 150 tl TCP CRR latency over IPv6: guest to host @@ -86,33 +86,39 @@ lat - lat - lat - nsb tcp_crr --nolog -6 -gout LAT tcp_crr --nolog -6 -c -H __GW6__%__IFNAME__ | sed -n 's/^throughput=\(.*\)/\1/p' +gout LAT tcp_crr --nolog -l1 -6 -c -H __MAP_NS6__ | sed -n 's/^throughput=\(.*\)/\1/p' lat __LAT__ 500 400 tr TCP throughput over IPv4: guest to host -iperf3s ns 100${i}2 __THREADS__ +iperf3s ns 10002 guest ip link set dev __IFNAME__ mtu 256 -iperf3 BW guest __GW__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -w 1M +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -w 1M bw __BW__ 0.2 0.3 guest ip link set dev __IFNAME__ mtu 576 -iperf3 BW guest __GW__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -w 1M +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -w 1M bw __BW__ 0.5 0.8 guest ip link set dev __IFNAME__ mtu 1280 -iperf3 BW guest __GW__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -w 4M +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -w 4M bw __BW__ 1.2 1.5 guest ip link set dev __IFNAME__ mtu 1500 -iperf3 BW guest __GW__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -w 4M +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -w 4M bw __BW__ 1.6 1.8 guest ip link set dev __IFNAME__ mtu 9000 -iperf3 BW guest __GW__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -w 8M +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -w 8M bw __BW__ 4.0 5.0 guest ip link set dev __IFNAME__ mtu 65520 -iperf3 BW guest __GW__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -w 16M +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -w 16M bw __BW__ 7.0 8.0 iperf3k ns +# Reducing MTU below 1280 deconfigures IPv6, get our address back +guest dhclient -6 -x +guest dhclient -6 __IFNAME__ +# Wait for DAD to complete +guest while ip -j -6 addr show tentative | jq -e '.[].addr_info'; do sleep 0.1; done + tl TCP RR latency over IPv4: guest to host lat - lat - @@ -120,7 +126,7 @@ lat - lat - lat - nsb tcp_rr --nolog -4 -gout LAT tcp_rr --nolog -4 -c -H __GW__ | sed -n 's/^throughput=\(.*\)/\1/p' +gout LAT tcp_rr --nolog -l1 -4 -c -H __MAP_NS4__ | sed -n 's/^throughput=\(.*\)/\1/p' lat __LAT__ 200 150 tl TCP CRR latency over IPv4: guest to host @@ -130,18 +136,18 @@ lat - lat - lat - nsb tcp_crr --nolog -4 -gout LAT tcp_crr --nolog -4 -c -H __GW__ | sed -n 's/^throughput=\(.*\)/\1/p' +gout LAT tcp_crr --nolog -l1 -4 -c -H __MAP_NS4__ | sed -n 's/^throughput=\(.*\)/\1/p' lat __LAT__ 500 400 tr TCP throughput over IPv6: host to guest -iperf3s guest 100${i}1 __THREADS__ +iperf3s guest 10001 bw - bw - bw - bw - bw - -iperf3 BW ns ::1 100${i}1 __THREADS__ __TIME__ __OPTS__ +iperf3 BW ns ::1 10001 __TIME__ __OPTS__ bw __BW__ 6.0 6.8 iperf3k guest @@ -154,7 +160,7 @@ lat - lat - guestb tcp_rr --nolog -P 10001 -C 10011 -6 sleep 1 -nsout LAT tcp_rr --nolog -P 10001 -C 10011 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT tcp_rr --nolog -l1 -P 10001 -C 10011 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' lat __LAT__ 200 150 tl TCP CRR latency over IPv6: host to guest @@ -165,19 +171,19 @@ lat - lat - guestb tcp_crr --nolog -P 10001 -C 10011 -6 sleep 1 -nsout LAT tcp_crr --nolog -P 10001 -C 10011 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT tcp_crr --nolog -l1 -P 10001 -C 10011 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' lat __LAT__ 500 350 tr TCP throughput over IPv4: host to guest -iperf3s guest 100${i}1 __THREADS__ +iperf3s guest 10001 bw - bw - bw - bw - bw - -iperf3 BW ns 127.0.0.1 100${i}1 __THREADS__ __TIME__ __OPTS__ +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ bw __BW__ 6.0 6.8 iperf3k guest @@ -190,7 +196,7 @@ lat - lat - guestb tcp_rr --nolog -P 10001 -C 10011 -4 sleep 1 -nsout LAT tcp_rr --nolog -P 10001 -C 10011 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT tcp_rr --nolog -l1 -P 10001 -C 10011 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' lat __LAT__ 200 150 tl TCP CRR latency over IPv6: host to guest @@ -201,7 +207,7 @@ lat - lat - guestb tcp_crr --nolog -P 10001 -C 10011 -4 sleep 1 -nsout LAT tcp_crr --nolog -P 10001 -C 10011 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT tcp_crr --nolog -l1 -P 10001 -C 10011 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' lat __LAT__ 500 300 te diff --git a/test/perf/passt_udp b/test/perf/passt_udp index 10f638f..4c66c41 100644 --- a/test/perf/passt_udp +++ b/test/perf/passt_udp @@ -15,6 +15,9 @@ gtools /sbin/sysctl ip jq nproc sleep iperf3 udp_rr # From neper nstools ip jq sleep iperf3 udp_rr htools bc head sed +set MAP_NS4 192.0.2.2 +set MAP_NS6 2001:db8:9a55::2 + test passt: throughput and latency guest /sbin/sysctl -w net.core.rmem_max=16777216 @@ -22,38 +25,33 @@ guest /sbin/sysctl -w net.core.wmem_max=16777216 guest /sbin/sysctl -w net.core.rmem_default=16777216 guest /sbin/sysctl -w net.core.wmem_default=16777216 -gout GW ip -j -4 route show|jq -rM '.[] | select(.dst == "default").gateway' -gout GW6 ip -j -6 route show|jq -rM '.[] | select(.dst == "default").gateway' -gout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' - hout FREQ_PROCFS (echo "scale=1"; sed -n 's/cpu MHz.*: \([0-9]*\)\..*$/(\1+10^2\/2)\/10^3/p' /proc/cpuinfo) | bc -l | head -n1 hout FREQ_CPUFREQ (echo "scale=1"; printf '( %i + 10^5 / 2 ) / 10^6\n' $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq) ) | bc -l hout FREQ [ -n "__FREQ_CPUFREQ__" ] && echo __FREQ_CPUFREQ__ || echo __FREQ_PROCFS__ -set THREADS 4 -set STREAMS 1 -set TIME 10 -set OPTS -u -P __STREAMS__ --pacing-timer 1000 +set THREADS 2 +set TIME 1 +set OPTS -u -P __THREADS__ --pacing-timer 1000 -info Throughput in Gbps, latency in µs, __THREADS__ threads at __FREQ__ GHz, one stream each +info Throughput in Gbps, latency in µs, __THREADS__ threads at __FREQ__ GHz report passt udp __THREADS__ __FREQ__ th pktlen 256B 576B 1280B 1500B 9000B 65520B tr UDP throughput over IPv6: guest to host -iperf3s ns 100${i}2 __THREADS__ +iperf3s ns 10002 # (datagram size) = (packet size) - 48: 40 bytes of IPv6 header, 8 of UDP header bw - bw - -iperf3 BW guest __GW6__%__IFNAME__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 2G -l 1232 +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -b 3G -l 1232 bw __BW__ 0.8 1.2 -iperf3 BW guest __GW6__%__IFNAME__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 3G -l 1452 +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -b 4G -l 1452 bw __BW__ 1.0 1.5 -iperf3 BW guest __GW6__%__IFNAME__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 5G -l 8952 +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -b 8G -l 8952 bw __BW__ 4.0 5.0 -iperf3 BW guest __GW6__%__IFNAME__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 7G -l 64372 +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -b 15G -l 64372 bw __BW__ 4.0 5.0 iperf3k ns @@ -65,25 +63,25 @@ lat - lat - lat - nsb udp_rr --nolog -6 -gout LAT udp_rr --nolog -6 -c -H __GW6__%__IFNAME__ | sed -n 's/^throughput=\(.*\)/\1/p' +gout LAT udp_rr --nolog -6 -c -H __MAP_NS6__ | sed -n 's/^throughput=\(.*\)/\1/p' lat __LAT__ 200 150 tr UDP throughput over IPv4: guest to host -iperf3s ns 100${i}2 __THREADS__ +iperf3s ns 10002 # (datagram size) = (packet size) - 28: 20 bytes of IPv4 header, 8 of UDP header -iperf3 BW guest __GW__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 500M -l 228 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -b 1G -l 228 bw __BW__ 0.0 0.0 -iperf3 BW guest __GW__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 1G -l 548 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -b 2G -l 548 bw __BW__ 0.4 0.6 -iperf3 BW guest __GW__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 2G -l 1252 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -b 3G -l 1252 bw __BW__ 0.8 1.2 -iperf3 BW guest __GW__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 3G -l 1472 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -b 4G -l 1472 bw __BW__ 1.0 1.5 -iperf3 BW guest __GW__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 6G -l 8972 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -b 8G -l 8972 bw __BW__ 4.0 5.0 -iperf3 BW guest __GW__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 7G -l 65492 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -b 15G -l 65492 bw __BW__ 4.0 5.0 iperf3k ns @@ -95,23 +93,23 @@ lat - lat - lat - nsb udp_rr --nolog -4 -gout LAT udp_rr --nolog -4 -c -H __GW__ | sed -n 's/^throughput=\(.*\)/\1/p' +gout LAT udp_rr --nolog -4 -c -H __MAP_NS4__ | sed -n 's/^throughput=\(.*\)/\1/p' lat __LAT__ 200 150 tr UDP throughput over IPv6: host to guest -iperf3s guest 100${i}1 __THREADS__ +iperf3s guest 10001 # (datagram size) = (packet size) - 48: 40 bytes of IPv6 header, 8 of UDP header bw - bw - -iperf3 BW ns ::1 100${i}1 __THREADS__ __TIME__ __OPTS__ -b 2G -l 1232 +iperf3 BW ns ::1 10001 __TIME__ __OPTS__ -b 3G -l 1232 bw __BW__ 0.8 1.2 -iperf3 BW ns ::1 100${i}1 __THREADS__ __TIME__ __OPTS__ -b 2G -l 1452 +iperf3 BW ns ::1 10001 __TIME__ __OPTS__ -b 4G -l 1452 bw __BW__ 1.0 1.5 -iperf3 BW ns ::1 100${i}1 __THREADS__ __TIME__ __OPTS__ -b 3G -l 8952 +iperf3 BW ns ::1 10001 __TIME__ __OPTS__ -b 8G -l 8952 bw __BW__ 3.0 4.0 -iperf3 BW ns ::1 100${i}1 __THREADS__ __TIME__ __OPTS__ -b 3G -l 64372 +iperf3 BW ns ::1 10001 __TIME__ __OPTS__ -b 15G -l 64372 bw __BW__ 3.0 4.0 iperf3k guest @@ -129,20 +127,20 @@ lat __LAT__ 200 150 tr UDP throughput over IPv4: host to guest -iperf3s guest 100${i}1 __THREADS__ +iperf3s guest 10001 # (datagram size) = (packet size) - 28: 20 bytes of IPv4 header, 8 of UDP header -iperf3 BW ns 127.0.0.1 100${i}1 __THREADS__ __TIME__ __OPTS__ -b 1G -l 228 +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ -b 1G -l 228 bw __BW__ 0.0 0.0 -iperf3 BW ns 127.0.0.1 100${i}1 __THREADS__ __TIME__ __OPTS__ -b 1G -l 548 +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ -b 2G -l 548 bw __BW__ 0.4 0.6 -iperf3 BW ns 127.0.0.1 100${i}1 __THREADS__ __TIME__ __OPTS__ -b 3G -l 1252 +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ -b 3G -l 1252 bw __BW__ 0.8 1.2 -iperf3 BW ns 127.0.0.1 100${i}1 __THREADS__ __TIME__ __OPTS__ -b 3G -l 1472 +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ -b 4G -l 1472 bw __BW__ 1.0 1.5 -iperf3 BW ns 127.0.0.1 100${i}1 __THREADS__ __TIME__ __OPTS__ -b 3G -l 8972 +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ -b 8G -l 8972 bw __BW__ 3.0 4.0 -iperf3 BW ns 127.0.0.1 100${i}1 __THREADS__ __TIME__ __OPTS__ -b 3G -l 65492 +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ -b 15G -l 65492 bw __BW__ 3.0 4.0 iperf3k guest diff --git a/test/perf/passt_vu_tcp b/test/perf/passt_vu_tcp new file mode 100644 index 0000000..b434008 --- /dev/null +++ b/test/perf/passt_vu_tcp @@ -0,0 +1,211 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# PASST - Plug A Simple Socket Transport +# for qemu/UNIX domain socket mode +# +# PASTA - Pack A Subtle Tap Abstraction +# for network namespace/tap device mode +# +# test/perf/passt_vu_tcp - Check TCP performance in passt vhost-user mode +# +# Copyright (c) 2021 Red Hat GmbH +# Author: Stefano Brivio <sbrivio@redhat.com> + +gtools /sbin/sysctl ip jq nproc seq sleep iperf3 tcp_rr tcp_crr # From neper +nstools /sbin/sysctl ip jq nproc seq sleep iperf3 tcp_rr tcp_crr +htools bc head sed seq + +set MAP_NS4 192.0.2.2 +set MAP_NS6 2001:db8:9a55::2 + +test passt: throughput and latency + +guest /sbin/sysctl -w net.core.rmem_max=536870912 +guest /sbin/sysctl -w net.core.wmem_max=536870912 +guest /sbin/sysctl -w net.core.rmem_default=33554432 +guest /sbin/sysctl -w net.core.wmem_default=33554432 +guest /sbin/sysctl -w net.ipv4.tcp_rmem="4096 131072 268435456" +guest /sbin/sysctl -w net.ipv4.tcp_wmem="4096 131072 268435456" +guest /sbin/sysctl -w net.ipv4.tcp_timestamps=0 + +ns /sbin/sysctl -w net.ipv4.tcp_rmem="4096 524288 134217728" +ns /sbin/sysctl -w net.ipv4.tcp_wmem="4096 524288 134217728" +ns /sbin/sysctl -w net.ipv4.tcp_timestamps=0 + +gout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' + +hout FREQ_PROCFS (echo "scale=1"; sed -n 's/cpu MHz.*: \([0-9]*\)\..*$/(\1+10^2\/2)\/10^3/p' /proc/cpuinfo) | bc -l | head -n1 +hout FREQ_CPUFREQ (echo "scale=1"; printf '( %i + 10^5 / 2 ) / 10^6\n' $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq) ) | bc -l +hout FREQ [ -n "__FREQ_CPUFREQ__" ] && echo __FREQ_CPUFREQ__ || echo __FREQ_PROCFS__ + +set THREADS 4 +set TIME 5 +set OMIT 0.1 +set OPTS -Z -P __THREADS__ -l 1M -O__OMIT__ -N + +info Throughput in Gbps, latency in µs, __THREADS__ threads at __FREQ__ GHz +report passt_vu tcp __THREADS__ __FREQ__ + +th MTU 256B 576B 1280B 1500B 9000B 65520B + + +tr TCP throughput over IPv6: guest to host +iperf3s ns 10002 + +bw - +bw - +guest ip link set dev __IFNAME__ mtu 1280 +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -w 16M +bw __BW__ 1.2 1.5 +guest ip link set dev __IFNAME__ mtu 1500 +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -w 32M +bw __BW__ 1.6 1.8 +guest ip link set dev __IFNAME__ mtu 9000 +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -w 64M +bw __BW__ 4.0 5.0 +guest ip link set dev __IFNAME__ mtu 65520 +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -w 64M +bw __BW__ 7.0 8.0 + +iperf3k ns + +tl TCP RR latency over IPv6: guest to host +lat - +lat - +lat - +lat - +lat - +nsb tcp_rr --nolog -6 +gout LAT tcp_rr --nolog -l1 -6 -c -H __MAP_NS6__ | sed -n 's/^throughput=\(.*\)/\1/p' +lat __LAT__ 200 150 + +tl TCP CRR latency over IPv6: guest to host +lat - +lat - +lat - +lat - +lat - +nsb tcp_crr --nolog -6 +gout LAT tcp_crr --nolog -l1 -6 -c -H __MAP_NS6__ | sed -n 's/^throughput=\(.*\)/\1/p' +lat __LAT__ 500 400 + +tr TCP throughput over IPv4: guest to host +iperf3s ns 10002 + +guest ip link set dev __IFNAME__ mtu 256 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -w 2M +bw __BW__ 0.2 0.3 +guest ip link set dev __IFNAME__ mtu 576 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -w 4M +bw __BW__ 0.5 0.8 +guest ip link set dev __IFNAME__ mtu 1280 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -w 8M +bw __BW__ 1.2 1.5 +guest ip link set dev __IFNAME__ mtu 1500 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -w 16M +bw __BW__ 1.6 1.8 +guest ip link set dev __IFNAME__ mtu 9000 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -w 64M +bw __BW__ 4.0 5.0 +guest ip link set dev __IFNAME__ mtu 65520 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -w 64M +bw __BW__ 7.0 8.0 + +iperf3k ns + +# Reducing MTU below 1280 deconfigures IPv6, get our address back +guest dhclient -6 -x +guest dhclient -6 __IFNAME__ + +tl TCP RR latency over IPv4: guest to host +lat - +lat - +lat - +lat - +lat - +nsb tcp_rr --nolog -4 +gout LAT tcp_rr --nolog -l1 -4 -c -H __MAP_NS4__ | sed -n 's/^throughput=\(.*\)/\1/p' +lat __LAT__ 200 150 + +tl TCP CRR latency over IPv4: guest to host +lat - +lat - +lat - +lat - +lat - +nsb tcp_crr --nolog -4 +gout LAT tcp_crr --nolog -l1 -4 -c -H __MAP_NS4__ | sed -n 's/^throughput=\(.*\)/\1/p' +lat __LAT__ 500 400 + +tr TCP throughput over IPv6: host to guest +iperf3s guest 10001 + +bw - +bw - +bw - +bw - +bw - +iperf3 BW ns ::1 10001 __TIME__ __OPTS__ -w 32M +bw __BW__ 6.0 6.8 + +iperf3k guest + +tl TCP RR latency over IPv6: host to guest +lat - +lat - +lat - +lat - +lat - +guestb tcp_rr --nolog -P 10001 -C 10011 -6 +sleep 1 +nsout LAT tcp_rr --nolog -l1 -P 10001 -C 10011 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' +lat __LAT__ 200 150 + +tl TCP CRR latency over IPv6: host to guest +lat - +lat - +lat - +lat - +lat - +guestb tcp_crr --nolog -P 10001 -C 10011 -6 +sleep 1 +nsout LAT tcp_crr --nolog -l1 -P 10001 -C 10011 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' +lat __LAT__ 500 350 + + +tr TCP throughput over IPv4: host to guest +iperf3s guest 10001 + +bw - +bw - +bw - +bw - +bw - +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ -w 32M +bw __BW__ 6.0 6.8 + +iperf3k guest + +tl TCP RR latency over IPv4: host to guest +lat - +lat - +lat - +lat - +lat - +guestb tcp_rr --nolog -P 10001 -C 10011 -4 +sleep 1 +nsout LAT tcp_rr --nolog -l1 -P 10001 -C 10011 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' +lat __LAT__ 200 150 + +tl TCP CRR latency over IPv6: host to guest +lat - +lat - +lat - +lat - +lat - +guestb tcp_crr --nolog -P 10001 -C 10011 -4 +sleep 1 +nsout LAT tcp_crr --nolog -l1 -P 10001 -C 10011 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' +lat __LAT__ 500 300 + +te diff --git a/test/perf/passt_vu_udp b/test/perf/passt_vu_udp new file mode 100644 index 0000000..943ac11 --- /dev/null +++ b/test/perf/passt_vu_udp @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# PASST - Plug A Simple Socket Transport +# for qemu/UNIX domain socket mode +# +# PASTA - Pack A Subtle Tap Abstraction +# for network namespace/tap device mode +# +# test/perf/passt_vu_udp - Check UDP performance in passt vhost-user mode +# +# Copyright (c) 2021 Red Hat GmbH +# Author: Stefano Brivio <sbrivio@redhat.com> + +gtools /sbin/sysctl ip jq nproc sleep iperf3 udp_rr # From neper +nstools ip jq sleep iperf3 udp_rr +htools bc head sed + +set MAP_NS4 192.0.2.2 +set MAP_NS6 2001:db8:9a55::2 + +test passt: throughput and latency + +guest /sbin/sysctl -w net.core.rmem_max=16777216 +guest /sbin/sysctl -w net.core.wmem_max=16777216 +guest /sbin/sysctl -w net.core.rmem_default=16777216 +guest /sbin/sysctl -w net.core.wmem_default=16777216 + +hout FREQ_PROCFS (echo "scale=1"; sed -n 's/cpu MHz.*: \([0-9]*\)\..*$/(\1+10^2\/2)\/10^3/p' /proc/cpuinfo) | bc -l | head -n1 +hout FREQ_CPUFREQ (echo "scale=1"; printf '( %i + 10^5 / 2 ) / 10^6\n' $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq) ) | bc -l +hout FREQ [ -n "__FREQ_CPUFREQ__" ] && echo __FREQ_CPUFREQ__ || echo __FREQ_PROCFS__ + +set THREADS 2 +set TIME 1 +set OPTS -u -P __THREADS__ --pacing-timer 1000 + +info Throughput in Gbps, latency in µs, __THREADS__ threads at __FREQ__ GHz + +report passt_vu udp __THREADS__ __FREQ__ + +th pktlen 256B 576B 1280B 1500B 9000B 65520B + +tr UDP throughput over IPv6: guest to host +iperf3s ns 10002 +# (datagram size) = (packet size) - 48: 40 bytes of IPv6 header, 8 of UDP header + +bw - +bw - +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -b 3G -l 1232 +bw __BW__ 0.8 1.2 +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -b 4G -l 1452 +bw __BW__ 1.0 1.5 +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -b 10G -l 8952 +bw __BW__ 4.0 5.0 +iperf3 BW guest __MAP_NS6__ 10002 __TIME__ __OPTS__ -b 20G -l 64372 +bw __BW__ 4.0 5.0 + +iperf3k ns + +tl UDP RR latency over IPv6: guest to host +lat - +lat - +lat - +lat - +lat - +nsb udp_rr --nolog -6 +gout LAT udp_rr --nolog -6 -c -H __MAP_NS6__ | sed -n 's/^throughput=\(.*\)/\1/p' +lat __LAT__ 200 150 + + +tr UDP throughput over IPv4: guest to host +iperf3s ns 10002 +# (datagram size) = (packet size) - 28: 20 bytes of IPv4 header, 8 of UDP header + +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -b 1G -l 228 +bw __BW__ 0.0 0.0 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -b 2G -l 548 +bw __BW__ 0.4 0.6 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -b 3G -l 1252 +bw __BW__ 0.8 1.2 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -b 4G -l 1472 +bw __BW__ 1.0 1.5 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -b 10G -l 8972 +bw __BW__ 4.0 5.0 +iperf3 BW guest __MAP_NS4__ 10002 __TIME__ __OPTS__ -b 20G -l 65492 +bw __BW__ 4.0 5.0 + +iperf3k ns + +tl UDP RR latency over IPv4: guest to host +lat - +lat - +lat - +lat - +lat - +nsb udp_rr --nolog -4 +gout LAT udp_rr --nolog -4 -c -H __MAP_NS4__ | sed -n 's/^throughput=\(.*\)/\1/p' +lat __LAT__ 200 150 + + +tr UDP throughput over IPv6: host to guest +iperf3s guest 10001 +# (datagram size) = (packet size) - 48: 40 bytes of IPv6 header, 8 of UDP header + +bw - +bw - +iperf3 BW ns ::1 10001 __TIME__ __OPTS__ -b 3G -l 1232 +bw __BW__ 0.8 1.2 +iperf3 BW ns ::1 10001 __TIME__ __OPTS__ -b 4G -l 1452 +bw __BW__ 1.0 1.5 +iperf3 BW ns ::1 10001 __TIME__ __OPTS__ -b 10G -l 8952 +bw __BW__ 3.0 4.0 +iperf3 BW ns ::1 10001 __TIME__ __OPTS__ -b 20G -l 64372 +bw __BW__ 3.0 4.0 + +iperf3k guest + +tl UDP RR latency over IPv6: host to guest +lat - +lat - +lat - +lat - +lat - +guestb udp_rr --nolog -P 10001 -C 10011 -6 +sleep 1 +nsout LAT udp_rr --nolog -P 10001 -C 10011 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' +lat __LAT__ 200 150 + + +tr UDP throughput over IPv4: host to guest +iperf3s guest 10001 +# (datagram size) = (packet size) - 28: 20 bytes of IPv4 header, 8 of UDP header + +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ -b 1G -l 228 +bw __BW__ 0.0 0.0 +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ -b 2G -l 548 +bw __BW__ 0.4 0.6 +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ -b 3G -l 1252 +bw __BW__ 0.8 1.2 +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ -b 4G -l 1472 +bw __BW__ 1.0 1.5 +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ -b 10G -l 8972 +bw __BW__ 3.0 4.0 +iperf3 BW ns 127.0.0.1 10001 __TIME__ __OPTS__ -b 20G -l 65492 +bw __BW__ 3.0 4.0 + +iperf3k guest + +tl UDP RR latency over IPv4: host to guest +lat - +lat - +lat - +lat - +lat - +guestb udp_rr --nolog -P 10001 -C 10011 -4 +sleep 1 +nsout LAT udp_rr --nolog -P 10001 -C 10011 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' +lat __LAT__ 200 150 + +te diff --git a/test/perf/pasta_tcp b/test/perf/pasta_tcp index 7777532..88284b2 100644 --- a/test/perf/pasta_tcp +++ b/test/perf/pasta_tcp @@ -14,6 +14,9 @@ htools head ip seq bc sleep iperf3 tcp_rr tcp_crr jq sed nstools /sbin/sysctl nproc ip seq sleep iperf3 tcp_rr tcp_crr jq sed +set MAP_HOST4 192.0.2.1 +set MAP_HOST6 2001:db8:9a55::1 + test pasta: throughput and latency (local connections) ns /sbin/sysctl -w net.ipv4.tcp_rmem="131072 524288 134217728" @@ -21,101 +24,100 @@ ns /sbin/sysctl -w net.ipv4.tcp_wmem="131072 524288 134217728" ns /sbin/sysctl -w net.ipv4.tcp_timestamps=0 -set THREADS 2 -set STREAMS 2 -set TIME 10 +set THREADS 4 +set TIME 1 set OMIT 0.1 -set OPTS -Z -w 4M -l 1M -P __STREAMS__ -O__OMIT__ +set OPTS -Z -w 4M -l 1M -P __THREADS__ -O__OMIT__ hout FREQ_PROCFS (echo "scale=1"; sed -n 's/cpu MHz.*: \([0-9]*\)\..*$/(\1+10^2\/2)\/10^3/p' /proc/cpuinfo) | bc -l | head -n1 hout FREQ_CPUFREQ (echo "scale=1"; printf '( %i + 10^5 / 2 ) / 10^6\n' $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq) ) | bc -l hout FREQ [ -n "__FREQ_CPUFREQ__" ] && echo __FREQ_CPUFREQ__ || echo __FREQ_PROCFS__ -info Throughput in Gbps, latency in µs, __THREADS__ threads at __FREQ__ GHz, __STREAMS__ streams each +info Throughput in Gbps, latency in µs, __THREADS__ threads at __FREQ__ GHz report pasta lo_tcp __THREADS__ __FREQ__ th MTU 65535B tr TCP throughput over IPv6: ns to host -iperf3s host 100${i}3 __THREADS__ +iperf3s host 10003 -iperf3 BW ns ::1 100${i}3 __THREADS__ __TIME__ __OPTS__ +iperf3 BW ns ::1 10003 __THREADS__ __TIME__ __OPTS__ bw __BW__ 15.0 20.0 iperf3k host tl TCP RR latency over IPv6: ns to host hostb tcp_rr --nolog -P 10003 -C 10013 -6 -nsout LAT tcp_rr --nolog -P 10003 -C 10013 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT tcp_rr --nolog -l1 -P 10003 -C 10013 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' hostw lat __LAT__ 150 100 tl TCP CRR latency over IPv6: ns to host hostb tcp_crr --nolog -P 10003 -C 10013 -6 -nsout LAT tcp_crr --nolog -P 10003 -C 10013 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT tcp_crr --nolog -l1 -P 10003 -C 10013 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' hostw lat __LAT__ 500 350 tr TCP throughput over IPv4: ns to host -iperf3s host 100${i}3 __THREADS__ +iperf3s host 10003 -iperf3 BW ns 127.0.0.1 100${i}3 __THREADS__ __TIME__ __OPTS__ +iperf3 BW ns 127.0.0.1 10003 __THREADS__ __TIME__ __OPTS__ bw __BW__ 15.0 20.0 iperf3k host tl TCP RR latency over IPv4: ns to host hostb tcp_rr --nolog -P 10003 -C 10013 -4 -nsout LAT tcp_rr --nolog -P 10003 -C 10013 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT tcp_rr --nolog -l1 -P 10003 -C 10013 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' hostw lat __LAT__ 150 100 tl TCP CRR latency over IPv4: ns to host hostb tcp_crr --nolog -P 10003 -C 10013 -4 -nsout LAT tcp_crr --nolog -P 10003 -C 10013 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT tcp_crr --nolog -l1 -P 10003 -C 10013 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' hostw lat __LAT__ 500 350 tr TCP throughput over IPv6: host to ns -iperf3s ns 100${i}2 __THREADS__ +iperf3s ns 10002 -iperf3 BW host ::1 100${i}2 __THREADS__ __TIME__ __OPTS__ +iperf3 BW host ::1 10002 __TIME__ __OPTS__ bw __BW__ 15.0 20.0 iperf3k ns tl TCP RR latency over IPv6: host to ns nsb tcp_rr --nolog -P 10002 -C 10012 -6 -hout LAT tcp_rr --nolog -P 10002 -C 10012 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' +hout LAT tcp_rr --nolog -l1 -P 10002 -C 10012 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' nsw lat __LAT__ 150 100 tl TCP CRR latency over IPv6: host to ns nsb tcp_crr --nolog -P 10002 -C 10012 -6 -hout LAT tcp_crr --nolog -P 10002 -C 10012 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' +hout LAT tcp_crr --nolog -l1 -P 10002 -C 10012 -6 -c -H ::1 | sed -n 's/^throughput=\(.*\)/\1/p' nsw lat __LAT__ 1000 700 tr TCP throughput over IPv4: host to ns -iperf3s ns 100${i}2 __THREADS__ +iperf3s ns 10002 -iperf3 BW host 127.0.0.1 100${i}2 __THREADS__ __TIME__ __OPTS__ +iperf3 BW host 127.0.0.1 10002 __TIME__ __OPTS__ bw __BW__ 15.0 20.0 iperf3k ns tl TCP RR latency over IPv4: host to ns nsb tcp_rr --nolog -P 10002 -C 10012 -4 -hout LAT tcp_rr --nolog -P 10002 -C 10012 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' +hout LAT tcp_rr --nolog -l1 -P 10002 -C 10012 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' nsw lat __LAT__ 150 100 tl TCP CRR latency over IPv4: host to ns nsb tcp_crr --nolog -P 10002 -C 10012 -4 -hout LAT tcp_crr --nolog -P 10002 -C 10012 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' +hout LAT tcp_crr --nolog -l1 -P 10002 -C 10012 -4 -c -H 127.0.0.1 | sed -n 's/^throughput=\(.*\)/\1/p' nsw lat __LAT__ 1000 700 @@ -123,32 +125,29 @@ te test pasta: throughput and latency (connections via tap) -nsout GW ip -j -4 route show|jq -rM '.[] | select(.dst == "default").gateway' -nsout GW6 ip -j -6 route show|jq -rM '.[] | select(.dst == "default").gateway' nsout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' -set THREADS 1 -set STREAMS 2 -set OPTS -Z -P __STREAMS__ -i1 -O__OMIT__ +set THREADS 2 +set OPTS -Z -P __THREADS__ -i1 -O__OMIT__ -info Throughput in Gbps, latency in µs, one thread at __FREQ__ GHz, __STREAMS__ streams +info Throughput in Gbps, latency in µs, __THREADS__ threads at __FREQ__ GHz report pasta tap_tcp __THREADS__ __FREQ__ th MTU 1500B 4000B 16384B 65520B tr TCP throughput over IPv6: ns to host -iperf3s host 100${i}3 __THREADS__ +iperf3s host 10003 ns ip link set dev __IFNAME__ mtu 1500 -iperf3 BW ns __GW6__%__IFNAME__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -w 512k +iperf3 BW ns __MAP_HOST6__ 10003 __TIME__ __OPTS__ -w 512k bw __BW__ 0.2 0.4 ns ip link set dev __IFNAME__ mtu 4000 -iperf3 BW ns __GW6__%__IFNAME__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -w 1M +iperf3 BW ns __MAP_HOST6__ 10003 __TIME__ __OPTS__ -w 1M bw __BW__ 0.3 0.5 ns ip link set dev __IFNAME__ mtu 16384 -iperf3 BW ns __GW6__%__IFNAME__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -w 8M +iperf3 BW ns __MAP_HOST6__ 10003 __TIME__ __OPTS__ -w 8M bw __BW__ 1.5 2.0 ns ip link set dev __IFNAME__ mtu 65520 -iperf3 BW ns __GW6__%__IFNAME__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -w 8M +iperf3 BW ns __MAP_HOST6__ 10003 __TIME__ __OPTS__ -w 8M bw __BW__ 2.0 2.5 iperf3k host @@ -158,7 +157,7 @@ lat - lat - lat - hostb tcp_rr --nolog -P 10003 -C 10013 -6 -nsout LAT tcp_rr --nolog -P 10003 -C 10013 -6 -c -H __GW6__%__IFNAME__ | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT tcp_rr --nolog -l1 -P 10003 -C 10013 -6 -c -H __MAP_HOST6__ | sed -n 's/^throughput=\(.*\)/\1/p' hostw lat __LAT__ 150 100 @@ -167,25 +166,25 @@ lat - lat - lat - hostb tcp_crr --nolog -P 10003 -C 10013 -6 -nsout LAT tcp_crr --nolog -P 10003 -C 10013 -6 -c -H __GW6__%__IFNAME__ | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT tcp_crr --nolog -l1 -P 10003 -C 10013 -6 -c -H __MAP_HOST6__ | sed -n 's/^throughput=\(.*\)/\1/p' hostw lat __LAT__ 1500 500 tr TCP throughput over IPv4: ns to host -iperf3s host 100${i}3 __THREADS__ +iperf3s host 10003 ns ip link set dev __IFNAME__ mtu 1500 -iperf3 BW ns __GW__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -w 512k +iperf3 BW ns __MAP_HOST4__ 10003 __TIME__ __OPTS__ -w 512k bw __BW__ 0.2 0.4 ns ip link set dev __IFNAME__ mtu 4000 -iperf3 BW ns __GW__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -w 1M +iperf3 BW ns __MAP_HOST4__ 10003 __TIME__ __OPTS__ -w 1M bw __BW__ 0.3 0.5 ns ip link set dev __IFNAME__ mtu 16384 -iperf3 BW ns __GW__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -w 8M +iperf3 BW ns __MAP_HOST4__ 10003 __TIME__ __OPTS__ -w 8M bw __BW__ 1.5 2.0 ns ip link set dev __IFNAME__ mtu 65520 -iperf3 BW ns __GW__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -w 8M +iperf3 BW ns __MAP_HOST4__ 10003 __TIME__ __OPTS__ -w 8M bw __BW__ 2.0 2.5 iperf3k host @@ -195,7 +194,7 @@ lat - lat - lat - hostb tcp_rr --nolog -P 10003 -C 10013 -4 -nsout LAT tcp_rr --nolog -P 10003 -C 10013 -4 -c -H __GW__ | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT tcp_rr --nolog -l1 -P 10003 -C 10013 -4 -c -H __MAP_HOST4__ | sed -n 's/^throughput=\(.*\)/\1/p' hostw lat __LAT__ 150 100 @@ -204,19 +203,19 @@ lat - lat - lat - hostb tcp_crr --nolog -P 10003 -C 10013 -4 -nsout LAT tcp_crr --nolog -P 10003 -C 10013 -4 -c -H __GW__ | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT tcp_crr --nolog -l1 -P 10003 -C 10013 -4 -c -H __MAP_HOST4__ | sed -n 's/^throughput=\(.*\)/\1/p' hostw lat __LAT__ 1500 500 tr TCP throughput over IPv6: host to ns -iperf3s ns 100${i}2 __THREADS__ +iperf3s ns 10002 nsout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' -nsout ADDR6 ip -j -6 addr show|jq -rM '.[] | select(.ifname == "__IFNAME__").addr_info[] | select(.scope == "global" and .prefixlen == 64).local' +nsout ADDR6 ip -j -6 addr show|jq -rM '.[] | select(.ifname == "__IFNAME__").addr_info[] | select(.scope == "global").local' bw - bw - bw - -iperf3 BW host __ADDR6__ 100${i}2 __THREADS__ __TIME__ __OPTS__ +iperf3 BW host __ADDR6__ 10002 __TIME__ __OPTS__ bw __BW__ 8.0 10.0 iperf3k ns @@ -226,7 +225,7 @@ lat - lat - lat - nsb tcp_rr --nolog -P 10002 -C 10012 -6 -hout LAT tcp_rr --nolog -P 10002 -C 10012 -6 -c -H __ADDR6__ | sed -n 's/^throughput=\(.*\)/\1/p' +hout LAT tcp_rr --nolog -l1 -P 10002 -C 10012 -6 -c -H __ADDR6__ | sed -n 's/^throughput=\(.*\)/\1/p' nsw lat __LAT__ 150 100 @@ -236,19 +235,19 @@ lat - lat - sleep 1 nsb tcp_crr --nolog -P 10002 -C 10012 -6 -hout LAT tcp_crr --nolog -P 10002 -C 10012 -6 -c -H __ADDR6__ | sed -n 's/^throughput=\(.*\)/\1/p' +hout LAT tcp_crr --nolog -l1 -P 10002 -C 10012 -6 -c -H __ADDR6__ | sed -n 's/^throughput=\(.*\)/\1/p' nsw lat __LAT__ 5000 10000 tr TCP throughput over IPv4: host to ns -iperf3s ns 100${i}2 __THREADS__ +iperf3s ns 10002 nsout ADDR ip -j -4 addr show|jq -rM '.[] | select(.ifname == "__IFNAME__").addr_info[0].local' bw - bw - bw - -iperf3 BW host __ADDR__ 100${i}2 __THREADS__ __TIME__ __OPTS__ +iperf3 BW host __ADDR__ 10002 __TIME__ __OPTS__ bw __BW__ 8.0 10.0 iperf3k ns @@ -258,7 +257,7 @@ lat - lat - lat - nsb tcp_rr --nolog -P 10002 -C 10012 -4 -hout LAT tcp_rr --nolog -P 10002 -C 10012 -4 -c -H __ADDR__ | sed -n 's/^throughput=\(.*\)/\1/p' +hout LAT tcp_rr --nolog -l1 -P 10002 -C 10012 -4 -c -H __ADDR__ | sed -n 's/^throughput=\(.*\)/\1/p' nsw lat __LAT__ 150 100 @@ -268,7 +267,7 @@ lat - lat - sleep 1 nsb tcp_crr --nolog -P 10002 -C 10012 -4 -hout LAT tcp_crr --nolog -P 10002 -C 10012 -4 -c -H __ADDR__ | sed -n 's/^throughput=\(.*\)/\1/p' +hout LAT tcp_crr --nolog -l1 -P 10002 -C 10012 -4 -c -H __ADDR__ | sed -n 's/^throughput=\(.*\)/\1/p' nsw lat __LAT__ 5000 10000 diff --git a/test/perf/pasta_udp b/test/perf/pasta_udp index 5e3db1e..3d07091 100644 --- a/test/perf/pasta_udp +++ b/test/perf/pasta_udp @@ -14,6 +14,9 @@ htools bc head ip sleep iperf3 udp_rr jq sed nstools ip sleep iperf3 udp_rr jq sed +set MAP_HOST4 192.0.2.1 +set MAP_HOST6 2001:db8:9a55::1 + test pasta: throughput and latency (local traffic) hout FREQ_PROCFS (echo "scale=1"; sed -n 's/cpu MHz.*: \([0-9]*\)\..*$/(\1+10^2\/2)\/10^3/p' /proc/cpuinfo) | bc -l | head -n1 @@ -21,11 +24,10 @@ hout FREQ_CPUFREQ (echo "scale=1"; printf '( %i + 10^5 / 2 ) / 10^6\n' $(cat /sy hout FREQ [ -n "__FREQ_CPUFREQ__" ] && echo __FREQ_CPUFREQ__ || echo __FREQ_PROCFS__ set THREADS 1 -set STREAMS 4 -set TIME 10 -set OPTS -u -P __STREAMS__ +set TIME 1 +set OPTS -u -P __THREADS__ -info Throughput in Gbps, latency in µs, one thread at __FREQ__ GHz, __STREAMS__ streams +info Throughput in Gbps, latency in µs, one thread at __FREQ__ GHz report pasta lo_udp 1 __FREQ__ @@ -33,16 +35,16 @@ th pktlen 1500B 4000B 16384B 65535B tr UDP throughput over IPv6: ns to host -iperf3s host 100${i}3 __THREADS__ +iperf3s host 10003 # (datagram size) = (packet size) - 48: 40 bytes of IPv6 header, 8 of UDP header -iperf3 BW ns ::1 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 3G -l 1452 +iperf3 BW ns ::1 10003 __TIME__ __OPTS__ -b 5G -l 1452 bw __BW__ 1.0 1.5 -iperf3 BW ns ::1 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 3G -l 3972 +iperf3 BW ns ::1 10003 __TIME__ __OPTS__ -b 10G -l 3972 bw __BW__ 1.2 1.8 -iperf3 BW ns ::1 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 10G -l 16336 +iperf3 BW ns ::1 10003 __TIME__ __OPTS__ -b 30G -l 16336 bw __BW__ 5.0 6.0 -iperf3 BW ns ::1 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 15G -l 65487 +iperf3 BW ns ::1 10003 __TIME__ __OPTS__ -b 40G -l 65487 bw __BW__ 7.0 9.0 iperf3k host @@ -58,16 +60,16 @@ lat __LAT__ 200 150 tr UDP throughput over IPv4: ns to host -iperf3s host 100${i}3 __THREADS__ +iperf3s host 10003 # (datagram size) = (packet size) - 28: 20 bytes of IPv4 header, 8 of UDP header -iperf3 BW ns 127.0.0.1 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 3G -l 1372 +iperf3 BW ns 127.0.0.1 10003 __TIME__ __OPTS__ -b 5G -l 1372 bw __BW__ 1.0 1.5 -iperf3 BW ns 127.0.0.1 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 3G -l 3972 +iperf3 BW ns 127.0.0.1 10003 __TIME__ __OPTS__ -b 10G -l 3972 bw __BW__ 1.2 1.8 -iperf3 BW ns 127.0.0.1 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 10G -l 16356 +iperf3 BW ns 127.0.0.1 10003 __TIME__ __OPTS__ -b 30G -l 16356 bw __BW__ 5.0 6.0 -iperf3 BW ns 127.0.0.1 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 15G -l 65507 +iperf3 BW ns 127.0.0.1 10003 __TIME__ __OPTS__ -b 40G -l 65507 bw __BW__ 7.0 9.0 iperf3k host @@ -83,15 +85,15 @@ lat __LAT__ 200 150 tr UDP throughput over IPv6: host to ns -iperf3s ns 100${i}2 __THREADS__ +iperf3s ns 10002 -iperf3 BW host ::1 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 3G -l 1452 +iperf3 BW host ::1 10002 __TIME__ __OPTS__ -b 5G -l 1452 bw __BW__ 1.0 1.5 -iperf3 BW host ::1 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 3G -l 3972 +iperf3 BW host ::1 10002 __TIME__ __OPTS__ -b 10G -l 3972 bw __BW__ 1.2 1.8 -iperf3 BW host ::1 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 10G -l 16336 +iperf3 BW host ::1 10002 __TIME__ __OPTS__ -b 30G -l 16336 bw __BW__ 5.0 6.0 -iperf3 BW host ::1 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 15G -l 16336 +iperf3 BW host ::1 10002 __TIME__ __OPTS__ -b 40G -l 65487 bw __BW__ 7.0 9.0 iperf3k ns @@ -107,14 +109,14 @@ lat __LAT__ 200 150 tr UDP throughput over IPv4: host to ns -iperf3s ns 100${i}2 __THREADS__ -iperf3 BW host 127.0.0.1 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 3G -l 1372 +iperf3s ns 10002 +iperf3 BW host 127.0.0.1 10002 __TIME__ __OPTS__ -b 5G -l 1372 bw __BW__ 1.0 1.5 -iperf3 BW host 127.0.0.1 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 3G -l 3972 +iperf3 BW host 127.0.0.1 10002 __TIME__ __OPTS__ -b 10G -l 3972 bw __BW__ 1.2 1.8 -iperf3 BW host 127.0.0.1 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 10G -l 16356 +iperf3 BW host 127.0.0.1 10002 __TIME__ __OPTS__ -b 30G -l 16356 bw __BW__ 5.0 6.0 -iperf3 BW host 127.0.0.1 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 15G -l 65507 +iperf3 BW host 127.0.0.1 10002 __TIME__ __OPTS__ -b 40G -l 65507 bw __BW__ 7.0 9.0 iperf3k ns @@ -134,26 +136,24 @@ te test pasta: throughput and latency (traffic via tap) -nsout GW ip -j -4 route show|jq -rM '.[] | select(.dst == "default").gateway' -nsout GW6 ip -j -6 route show|jq -rM '.[] | select(.dst == "default").gateway' nsout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' -info Throughput in Gbps, latency in µs, one thread at __FREQ__ GHz, __STREAMS__ streams +info Throughput in Gbps, latency in µs, one thread at __FREQ__ GHz report pasta tap_udp 1 __FREQ__ th pktlen 1500B 4000B 16384B 65520B tr UDP throughput over IPv6: ns to host -iperf3s host 100${i}3 __THREADS__ +iperf3s host 10003 # (datagram size) = (packet size) - 48: 40 bytes of IPv6 header, 8 of UDP header -iperf3 BW ns __GW6__%__IFNAME__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 2G -l 1472 +iperf3 BW ns __MAP_HOST6__ 10003 __TIME__ __OPTS__ -b 8G -l 1472 bw __BW__ 0.3 0.5 -iperf3 BW ns __GW6__%__IFNAME__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 3G -l 3972 +iperf3 BW ns __MAP_HOST6__ 10003 __TIME__ __OPTS__ -b 12G -l 3972 bw __BW__ 0.5 0.8 -iperf3 BW ns __GW6__%__IFNAME__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 4G -l 16356 +iperf3 BW ns __MAP_HOST6__ 10003 __TIME__ __OPTS__ -b 20G -l 16356 bw __BW__ 3.0 4.0 -iperf3 BW ns __GW6__%__IFNAME__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 6G -l 65472 +iperf3 BW ns __MAP_HOST6__ 10003 __TIME__ __OPTS__ -b 30G -l 65472 bw __BW__ 6.0 7.0 iperf3k host @@ -163,22 +163,22 @@ lat - lat - lat - hostb udp_rr --nolog -P 10003 -C 10013 -6 -nsout LAT udp_rr --nolog -P 10003 -C 10013 -6 -c -H __GW6__%__IFNAME__ | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT udp_rr --nolog -P 10003 -C 10013 -6 -c -H __MAP_HOST6__ | sed -n 's/^throughput=\(.*\)/\1/p' hostw lat __LAT__ 200 150 tr UDP throughput over IPv4: ns to host -iperf3s host 100${i}3 __THREADS__ +iperf3s host 10003 # (datagram size) = (packet size) - 28: 20 bytes of IPv4 header, 8 of UDP header -iperf3 BW ns __GW__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 2G -l 1472 +iperf3 BW ns __MAP_HOST4__ 10003 __TIME__ __OPTS__ -b 8G -l 1472 bw __BW__ 0.3 0.5 -iperf3 BW ns __GW__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 3G -l 3972 +iperf3 BW ns __MAP_HOST4__ 10003 __TIME__ __OPTS__ -b 12G -l 3972 bw __BW__ 0.5 0.8 -iperf3 BW ns __GW__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 4G -l 16356 +iperf3 BW ns __MAP_HOST4__ 10003 __TIME__ __OPTS__ -b 20G -l 16356 bw __BW__ 3.0 4.0 -iperf3 BW ns __GW__ 100${i}3 __THREADS__ __TIME__ __OPTS__ -b 6G -l 65492 +iperf3 BW ns __MAP_HOST4__ 10003 __TIME__ __OPTS__ -b 30G -l 65492 bw __BW__ 6.0 7.0 iperf3k host @@ -188,22 +188,22 @@ lat - lat - lat - hostb udp_rr --nolog -P 10003 -C 10013 -4 -nsout LAT udp_rr --nolog -P 10003 -C 10013 -4 -c -H __GW__ | sed -n 's/^throughput=\(.*\)/\1/p' +nsout LAT udp_rr --nolog -P 10003 -C 10013 -4 -c -H __MAP_HOST4__ | sed -n 's/^throughput=\(.*\)/\1/p' hostw lat __LAT__ 200 150 tr UDP throughput over IPv6: host to ns -iperf3s ns 100${i}2 __THREADS__ +iperf3s ns 10002 nsout IFNAME ip -j link show | jq -rM '.[] | select(.link_type == "ether").ifname' -nsout ADDR6 ip -j -6 addr show|jq -rM '.[] | select(.ifname == "__IFNAME__").addr_info[] | select(.scope == "global" and .prefixlen == 64).local' -iperf3 BW host __ADDR6__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 2G -l 1472 +nsout ADDR6 ip -j -6 addr show|jq -rM '.[] | select(.ifname == "__IFNAME__").addr_info[] | select(.scope == "global").local' +iperf3 BW host __ADDR6__ 10002 __TIME__ __OPTS__ -b 8G -l 1472 bw __BW__ 0.3 0.5 -iperf3 BW host __ADDR6__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 3G -l 3972 +iperf3 BW host __ADDR6__ 10002 __TIME__ __OPTS__ -b 12G -l 3972 bw __BW__ 0.5 0.8 -iperf3 BW host __ADDR6__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 4G -l 16356 +iperf3 BW host __ADDR6__ 10002 __TIME__ __OPTS__ -b 20G -l 16356 bw __BW__ 3.0 4.0 -iperf3 BW host __ADDR6__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 15G -l 65472 +iperf3 BW host __ADDR6__ 10002 __TIME__ __OPTS__ -b 30G -l 65472 bw __BW__ 7.0 9.0 iperf3k ns @@ -219,16 +219,16 @@ lat __LAT__ 200 150 tr UDP throughput over IPv4: host to ns -iperf3s ns 100${i}2 __THREADS__ +iperf3s ns 10002 nsout ADDR ip -j -4 addr show|jq -rM '.[] | select(.ifname == "__IFNAME__").addr_info[0].local' -iperf3 BW host __ADDR__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 2G -l 1472 +iperf3 BW host __ADDR__ 10002 __TIME__ __OPTS__ -b 8G -l 1472 bw __BW__ 0.3 0.5 -iperf3 BW host __ADDR__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 3G -l 3972 +iperf3 BW host __ADDR__ 10002 __TIME__ __OPTS__ -b 12G -l 3972 bw __BW__ 0.5 0.8 -iperf3 BW host __ADDR__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 4G -l 16356 +iperf3 BW host __ADDR__ 10002 __TIME__ __OPTS__ -b 20G -l 16356 bw __BW__ 3.0 4.0 -iperf3 BW host __ADDR__ 100${i}2 __THREADS__ __TIME__ __OPTS__ -b 15G -l 65492 +iperf3 BW host __ADDR__ 10002 __TIME__ __OPTS__ -b 30G -l 65492 bw __BW__ 7.0 9.0 iperf3k ns @@ -38,6 +38,9 @@ TRACE=${TRACE:-0} # If set, tell passt and pasta to take packet captures PCAP=${PCAP:-0} +# Custom kernel to boot guests with, if given +KERNEL=${KERNEL:-"/boot/vmlinuz-$(uname -r)"} + COMMIT="$(git log --oneline --no-decorate -1)" . lib/util @@ -90,6 +93,7 @@ run() { test memory/passt teardown memory + VHOST_USER=0 setup passt test passt/ndp test passt/dhcp @@ -101,7 +105,7 @@ run() { VALGRIND=1 setup passt_in_ns test passt/ndp - test passt/dhcp + test passt_in_ns/dhcp test passt_in_ns/icmp test passt_in_ns/tcp test passt_in_ns/udp @@ -112,10 +116,25 @@ run() { test two_guests/basic teardown two_guests + VHOST_USER=1 + setup passt_in_ns + test passt_vu/ndp + test passt_vu_in_ns/dhcp + test passt_vu_in_ns/icmp + test passt_vu_in_ns/tcp + test passt_vu_in_ns/udp + test passt_vu_in_ns/shutdown + teardown passt_in_ns + + setup two_guests + test two_guests_vu/basic + teardown two_guests + VALGRIND=0 + VHOST_USER=0 setup passt_in_ns test passt/ndp - test passt/dhcp + test passt_in_ns/dhcp test perf/passt_tcp test perf/passt_udp test perf/pasta_tcp @@ -123,6 +142,15 @@ run() { test passt_in_ns/shutdown teardown passt_in_ns + VHOST_USER=1 + setup passt_in_ns + test passt_vu/ndp + test passt_vu_in_ns/dhcp + test perf/passt_vu_tcp + test perf/passt_vu_udp + test passt_vu_in_ns/shutdown + teardown passt_in_ns + # TODO: Make those faster by at least pre-installing gcc and make on # non-x86 images, then re-enable. skip_distro() { diff --git a/test/two_guests/basic b/test/two_guests/basic index fa0608b..e2338ff 100644 --- a/test/two_guests/basic +++ b/test/two_guests/basic @@ -36,45 +36,49 @@ check [ "__ADDR2__" = "__HOST_ADDR__" ] test DHCPv6: addresses # Link is up now, wait for DAD to complete -sleep 2 +guest1 while ip -j -6 addr show tentative | jq -e '.[].addr_info'; do sleep 0.1; done +guest2 while ip -j -6 addr show tentative | jq -e '.[].addr_info'; do sleep 0.1; done guest1 /sbin/dhclient -6 __IFNAME1__ guest2 /sbin/dhclient -6 __IFNAME2__ +# Wait for DAD to complete on the DHCP address +guest1 while ip -j -6 addr show tentative | jq -e '.[].addr_info'; do sleep 0.1; done +guest2 while ip -j -6 addr show tentative | jq -e '.[].addr_info'; do sleep 0.1; done g1out ADDR1_6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__IFNAME1__").addr_info[] | select(.prefixlen == 128).local] | .[0]' g2out ADDR2_6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__IFNAME2__").addr_info[] | select(.prefixlen == 128).local] | .[0]' -hout HOST_ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__HOST_IFNAME6__").addr_info[] | select(.scope == "global").local] | .[0]' +hout HOST_ADDR6 ip -j -6 addr show|jq -rM '[.[] | select(.ifname == "__HOST_IFNAME6__").addr_info[] | select(.scope == "global" and .deprecated != true).local] | .[0]' check [ "__ADDR1_6__" = "__HOST_ADDR6__" ] check [ "__ADDR2_6__" = "__HOST_ADDR6__" ] test TCP/IPv4: guest 1 > guest 2 g1out GW1 ip -j -4 route show|jq -rM '.[] | select(.dst == "default").gateway' guest2b socat -u TCP4-LISTEN:10004 OPEN:msg,create,trunc +sleep 1 guest1 echo "Hello_from_guest_1" | socat -u STDIN TCP4:__GW1__:10004 guest2w -sleep 1 g2out MSG2 cat msg check [ "__MSG2__" = "Hello_from_guest_1" ] test TCP/IPv6: guest 2 > guest 1 g2out GW2_6 ip -j -6 route show|jq -rM '.[] | select(.dst == "default").gateway' guest1b socat -u TCP6-LISTEN:10001 OPEN:msg,create,trunc +sleep 1 guest2 echo "Hello_from_guest_2" | socat -u STDIN TCP6:[__GW2_6__%__IFNAME2__]:10001 guest1w -sleep 1 g1out MSG1 cat msg check [ "__MSG1__" = "Hello_from_guest_2" ] test UDP/IPv4: guest 1 > guest 2 guest2b socat -u TCP4-LISTEN:10004 OPEN:msg,create,trunc +sleep 1 guest1 echo "Hello_from_guest_1" | socat -u STDIN TCP4:__GW1__:10004 guest2w -sleep 1 g2out MSG2 cat msg check [ "__MSG2__" = "Hello_from_guest_1" ] test UDP/IPv6: guest 2 > guest 1 guest1b socat -u TCP6-LISTEN:10001 OPEN:msg,create,trunc +sleep 1 guest2 echo "Hello_from_guest_2" | socat -u STDIN TCP6:[__GW2_6__%__IFNAME2__]:10001 guest1w -sleep 1 g1out MSG1 cat msg check [ "__MSG1__" = "Hello_from_guest_2" ] diff --git a/test/two_guests_vu b/test/two_guests_vu new file mode 120000 index 0000000..a8648fc --- /dev/null +++ b/test/two_guests_vu @@ -0,0 +1 @@ +two_guests
\ No newline at end of file diff --git a/test/valgrind.supp b/test/valgrind.supp index a158394..735b5f6 100644 --- a/test/valgrind.supp +++ b/test/valgrind.supp @@ -6,3 +6,12 @@ ... fun:tcp_sock_consume } + +# same as above, for architectures with the recv() system call (at least i686): +{ + passt_recv_MSG_TRUNC_into_NULL_buffer + Memcheck:Param + socketcall.recv(buf) + ... + fun:tcp_sock_consume +} @@ -15,79 +15,64 @@ /** * DOC: Theory of Operation * + * UDP Flows + * ========= * - * For UDP, a reduced version of port-based connection tracking is implemented - * with two purposes: - * - binding ephemeral ports when they're used as source port by the guest, so - * that replies on those ports can be forwarded back to the guest, with a - * fixed timeout for this binding - * - packets received from the local host get their source changed to a local - * address (gateway address) so that they can be forwarded to the guest, and - * packets sent as replies by the guest need their destination address to - * be changed back to the address of the local host. This is dynamic to allow - * connections from the gateway as well, and uses the same fixed 180s timeout - * - * Sockets for bound ports are created at initialisation time, one set for IPv4 - * and one for IPv6. + * UDP doesn't have true connections, but many protocols use a connection-like + * format. The flow is initiated by a client sending a datagram from a port of + * its choosing (usually ephemeral) to a specific port (usually well known) on a + * server. Both client and server address must be unicast. The server sends + * replies using the same addresses & ports with src/dest swapped. * - * Packets are forwarded back and forth, by prepending and stripping UDP headers - * in the obvious way, with no port translation. + * We track pseudo-connections of this type as flow table entries of type + * FLOW_UDP. We store the time of the last traffic on the flow in uflow->ts, + * and let the flow expire if there is no traffic for UDP_CONN_TIMEOUT seconds. * - * In PASTA mode, the L2-L4 translation is skipped for connections to ports - * bound between namespaces using the loopback interface, messages are directly - * transferred between L4 sockets instead. These are called spliced connections - * for consistency with the TCP implementation, but the splice() syscall isn't - * actually used as it wouldn't make sense for datagram-based connections: a - * pair of recvmmsg() and sendmmsg() deals with this case. + * NOTE: This won't handle multicast protocols, or some protocols with different + * port usage. We'll need specific logic if we want to handle those. + * + * "Listening" sockets + * =================== * - * The connection tracking for PASTA mode is slightly complicated by the absence - * of actual connections, see struct udp_splice_port, and these examples: + * UDP doesn't use listen(), but we consider long term sockets which are allowed + * to create new flows "listening" by analogy with TCP. This listening socket + * could receive packets from multiple flows, so we use a hash table match to + * find the specific flow for a datagram. * - * - from init to namespace: + * When a UDP flow is initiated from a listening socket we take a duplicate of + * the socket and store it in uflow->s[INISIDE]. This will last for the + * lifetime of the flow, even if the original listening socket is closed due to + * port auto-probing. The duplicate is used to deliver replies back to the + * originating side. * - * - forward direction: 127.0.0.1:5000 -> 127.0.0.1:80 in init from socket s, - * with epoll reference: index = 80, splice = 1, orig = 1, ns = 0 - * - if udp_splice_ns[V4][5000].sock: - * - send packet to udp_splice_ns[V4][5000].sock, with destination port - * 80 - * - otherwise: - * - create new socket udp_splice_ns[V4][5000].sock - * - bind in namespace to 127.0.0.1:5000 - * - add to epoll with reference: index = 5000, splice = 1, orig = 0, - * ns = 1 - * - update udp_splice_init[V4][80].ts and udp_splice_ns[V4][5000].ts with - * current time + * Reply sockets + * ============= * - * - reverse direction: 127.0.0.1:80 -> 127.0.0.1:5000 in namespace socket s, - * having epoll reference: index = 5000, splice = 1, orig = 0, ns = 1 - * - if udp_splice_init[V4][80].sock: - * - send to udp_splice_init[V4][80].sock, with destination port 5000 - * - update udp_splice_init[V4][80].ts and udp_splice_ns[V4][5000].ts with - * current time - * - otherwise, discard + * When a UDP flow targets a socket, we create a "reply" socket in + * uflow->s[TGTSIDE] both to deliver datagrams to the target side and receive + * replies on the target side. This socket is both bound and connected and has + * EPOLL_TYPE_UDP_REPLY. The connect() means it will only receive datagrams + * associated with this flow, so the epoll reference directly points to the flow + * and we don't need a hash lookup. * - * - from namespace to init: + * NOTE: it's possible that the reply socket could have a bound address + * overlapping with an unrelated listening socket. We assume datagrams for the + * flow will come to the reply socket in preference to a listening socket. The + * sample program doc/platform-requirements/reuseaddr-priority.c documents and + * tests that assumption. * - * - forward direction: 127.0.0.1:2000 -> 127.0.0.1:22 in namespace from - * socket s, with epoll reference: index = 22, splice = 1, orig = 1, ns = 1 - * - if udp4_splice_init[V4][2000].sock: - * - send packet to udp_splice_init[V4][2000].sock, with destination - * port 22 - * - otherwise: - * - create new socket udp_splice_init[V4][2000].sock - * - bind in init to 127.0.0.1:2000 - * - add to epoll with reference: index = 2000, splice = 1, orig = 0, - * ns = 0 - * - update udp_splice_ns[V4][22].ts and udp_splice_init[V4][2000].ts with - * current time + * "Spliced" flows + * =============== + * + * In PASTA mode, L2-L4 translation is skipped for connections to ports bound + * between namespaces using the loopback interface, messages are directly + * transferred between L4 sockets instead. These are called spliced connections + * in analogy with the TCP implementation. The the splice() syscall isn't + * actually used; it doesn't make sense for datagrams and instead a pair of + * recvmmsg() and sendmmsg() is used to forward the datagrams. * - * - reverse direction: 127.0.0.1:22 -> 127.0.0.1:2000 in init from socket s, - * having epoll reference: index = 2000, splice = 1, orig = 0, ns = 0 - * - if udp_splice_ns[V4][22].sock: - * - send to udp_splice_ns[V4][22].sock, with destination port 2000 - * - update udp_splice_ns[V4][22].ts and udp_splice_init[V4][2000].ts with - * current time - * - otherwise, discard + * Note that a spliced flow will have *both* a duplicated listening socket and a + * reply socket (see above). */ #include <sched.h> @@ -110,9 +95,12 @@ #include <sys/socket.h> #include <sys/uio.h> #include <time.h> +#include <arpa/inet.h> +#include <linux/errqueue.h> #include "checksum.h" #include "util.h" +#include "iov.h" #include "ip.h" #include "siphash.h" #include "inany.h" @@ -120,125 +108,74 @@ #include "tap.h" #include "pcap.h" #include "log.h" +#include "flow_table.h" #include "udp_internal.h" +#include "udp_vu.h" -/** - * struct udp_tap_port - Port tracking based on tap-facing source port - * @sock: Socket bound to source port used as index - * @flags: Flags for local bind, loopback address/unicast address as source - * @ts: Activity timestamp from tap, used for socket aging - */ -struct udp_tap_port { - int sock; - uint8_t flags; -#define PORT_LOCAL BIT(0) -#define PORT_LOOPBACK BIT(1) -#define PORT_GUA BIT(2) - - time_t ts; -}; - -/** - * struct udp_splice_port - Bound socket for spliced communication - * @sock: Socket bound to index port - * @ts: Activity timestamp - */ -struct udp_splice_port { - int sock; - time_t ts; -}; +/* "Spliced" sockets indexed by bound port (host order) */ +static int udp_splice_ns [IP_VERSIONS][NUM_PORTS]; +static int udp_splice_init[IP_VERSIONS][NUM_PORTS]; -/* Port tracking, arrays indexed by packet source port (host order) */ -static struct udp_tap_port udp_tap_map [IP_VERSIONS][NUM_PORTS]; +/* Static buffers */ -/* "Spliced" sockets indexed by bound port (host order) */ -static struct udp_splice_port udp_splice_ns [IP_VERSIONS][NUM_PORTS]; -static struct udp_splice_port udp_splice_init[IP_VERSIONS][NUM_PORTS]; - -enum udp_act_type { - UDP_ACT_TAP, - UDP_ACT_SPLICE_NS, - UDP_ACT_SPLICE_INIT, - UDP_ACT_TYPE_MAX, -}; +/* UDP header and data for inbound messages */ +static struct udp_payload_t udp_payload[UDP_MAX_FRAMES]; -/* Activity-based aging for bindings */ -static uint8_t udp_act[IP_VERSIONS][UDP_ACT_TYPE_MAX][DIV_ROUND_UP(NUM_PORTS, 8)]; +/* Ethernet header for IPv4 frames */ +static struct ethhdr udp4_eth_hdr; -/* Static buffers */ +/* Ethernet header for IPv6 frames */ +static struct ethhdr udp6_eth_hdr; /** - * udp4_l2_buf_t - Pre-cooked IPv4 packet buffers for tap connections + * struct udp_meta_t - Pre-cooked headers and metadata for UDP packets + * @ip6h: Pre-filled IPv6 header (except for payload_len and addresses) + * @ip4h: Pre-filled IPv4 header (except for tot_len and saddr) + * @taph: Tap backend specific header * @s_in: Source socket address, filled in by recvmmsg() - * @taph: Tap-level headers (partially pre-filled) - * @iph: Pre-filled IP header (except for tot_len and saddr) - * @uh: Headroom for UDP header - * @data: Storage for UDP payload + * @tosidx: sidx for the destination side of this datagram's flow */ -static struct udp4_l2_buf_t { - struct sockaddr_in s_in; - +static struct udp_meta_t { + struct ipv6hdr ip6h; + struct iphdr ip4h; struct tap_hdr taph; - struct iphdr iph; - struct udphdr uh; - uint8_t data[USHRT_MAX - - (sizeof(struct iphdr) + sizeof(struct udphdr))]; -} __attribute__ ((packed, aligned(__alignof__(unsigned int)))) -udp4_l2_buf[UDP_MAX_FRAMES]; - -/** - * udp6_l2_buf_t - Pre-cooked IPv6 packet buffers for tap connections - * @s_in6: Source socket address, filled in by recvmmsg() - * @taph: Tap-level headers (partially pre-filled) - * @ip6h: Pre-filled IP header (except for payload_len and addresses) - * @uh: Headroom for UDP header - * @data: Storage for UDP payload - */ -struct udp6_l2_buf_t { - struct sockaddr_in6 s_in6; -#ifdef __AVX2__ - /* Align ip6h to 32-byte boundary. */ - uint8_t pad[64 - (sizeof(struct sockaddr_in6) + sizeof(struct ethhdr) + - sizeof(uint32_t))]; -#endif - struct tap_hdr taph; - struct ipv6hdr ip6h; - struct udphdr uh; - uint8_t data[USHRT_MAX - - (sizeof(struct ipv6hdr) + sizeof(struct udphdr))]; + union sockaddr_inany s_in; + flow_sidx_t tosidx; +} #ifdef __AVX2__ -} __attribute__ ((packed, aligned(32))) -#else -} __attribute__ ((packed, aligned(__alignof__(unsigned int)))) +__attribute__ ((aligned(32))) #endif -udp6_l2_buf[UDP_MAX_FRAMES]; - -/* recvmmsg()/sendmmsg() data for tap */ -static struct iovec udp4_l2_iov_sock [UDP_MAX_FRAMES]; -static struct iovec udp6_l2_iov_sock [UDP_MAX_FRAMES]; +udp_meta[UDP_MAX_FRAMES]; -static struct iovec udp4_l2_iov_tap [UDP_MAX_FRAMES]; -static struct iovec udp6_l2_iov_tap [UDP_MAX_FRAMES]; +/** + * enum udp_iov_idx - Indices for the buffers making up a single UDP frame + * @UDP_IOV_TAP tap specific header + * @UDP_IOV_ETH Ethernet header + * @UDP_IOV_IP IP (v4/v6) header + * @UDP_IOV_PAYLOAD IP payload (UDP header + data) + * @UDP_NUM_IOVS the number of entries in the iovec array + */ +enum udp_iov_idx { + UDP_IOV_TAP, + UDP_IOV_ETH, + UDP_IOV_IP, + UDP_IOV_PAYLOAD, + UDP_NUM_IOVS, +}; -static struct mmsghdr udp4_l2_mh_sock [UDP_MAX_FRAMES]; -static struct mmsghdr udp6_l2_mh_sock [UDP_MAX_FRAMES]; +/* IOVs and msghdr arrays for receiving datagrams from sockets */ +static struct iovec udp_iov_recv [UDP_MAX_FRAMES]; +static struct mmsghdr udp_mh_recv [UDP_MAX_FRAMES]; -/* recvmmsg()/sendmmsg() data for "spliced" connections */ -static struct iovec udp4_iov_splice [UDP_MAX_FRAMES]; -static struct iovec udp6_iov_splice [UDP_MAX_FRAMES]; +/* IOVs and msghdr arrays for sending "spliced" datagrams to sockets */ +static union sockaddr_inany udp_splice_to; -struct sockaddr_in udp4_localname = { - .sin_family = AF_INET, - .sin_addr = IN4ADDR_LOOPBACK_INIT, -}; -struct sockaddr_in6 udp6_localname = { - .sin6_family = AF_INET6, - .sin6_addr = IN6ADDR_LOOPBACK_INIT, -}; +static struct iovec udp_iov_splice [UDP_MAX_FRAMES]; +static struct mmsghdr udp_mh_splice [UDP_MAX_FRAMES]; -static struct mmsghdr udp4_mh_splice [UDP_MAX_FRAMES]; -static struct mmsghdr udp6_mh_splice [UDP_MAX_FRAMES]; +/* IOVs for L2 frames */ +static struct iovec udp_l2_iov [UDP_MAX_FRAMES][UDP_NUM_IOVS]; /** * udp_portmap_clear() - Clear UDP port map before configuration @@ -248,28 +185,8 @@ void udp_portmap_clear(void) unsigned i; for (i = 0; i < NUM_PORTS; i++) { - udp_tap_map[V4][i].sock = udp_tap_map[V6][i].sock = -1; - udp_splice_ns[V4][i].sock = udp_splice_ns[V6][i].sock = -1; - udp_splice_init[V4][i].sock = udp_splice_init[V6][i].sock = -1; - } -} - -/** - * udp_invert_portmap() - Compute reverse port translations for return packets - * @fwd: Port forwarding configuration to compute reverse map for - */ -static void udp_invert_portmap(struct udp_fwd_ports *fwd) -{ - unsigned int i; - - static_assert(ARRAY_SIZE(fwd->f.delta) == ARRAY_SIZE(fwd->rdelta), - "Forward and reverse delta arrays must have same size"); - for (i = 0; i < ARRAY_SIZE(fwd->f.delta); i++) { - in_port_t delta = fwd->f.delta[i]; - in_port_t rport = i + delta; - - if (delta) - fwd->rdelta[rport] = NUM_PORTS - delta; + udp_splice_ns[V4][i] = udp_splice_ns[V6][i] = -1; + udp_splice_init[V4][i] = udp_splice_init[V6][i] = -1; } } @@ -280,473 +197,448 @@ static void udp_invert_portmap(struct udp_fwd_ports *fwd) */ void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s) { - int i; - - for (i = 0; i < UDP_MAX_FRAMES; i++) { - struct udp4_l2_buf_t *b4 = &udp4_l2_buf[i]; - struct udp6_l2_buf_t *b6 = &udp6_l2_buf[i]; - - eth_update_mac(&b4->taph.eh, eth_d, eth_s); - eth_update_mac(&b6->taph.eh, eth_d, eth_s); - } + eth_update_mac(&udp4_eth_hdr, eth_d, eth_s); + eth_update_mac(&udp6_eth_hdr, eth_d, eth_s); } /** - * udp_sock4_iov_init() - Initialise scatter-gather L2 buffers for IPv4 sockets + * udp_iov_init_one() - Initialise scatter-gather lists for one buffer * @c: Execution context + * @i: Index of buffer to initialize */ -static void udp_sock4_iov_init(const struct ctx *c) +static void udp_iov_init_one(const struct ctx *c, size_t i) { - struct mmsghdr *h; - int i; - - for (i = 0; i < ARRAY_SIZE(udp4_l2_buf); i++) { - udp4_l2_buf[i] = (struct udp4_l2_buf_t) { - .taph = TAP_HDR_INIT(ETH_P_IP), - .iph = L2_BUF_IP4_INIT(IPPROTO_UDP) - }; - } - - for (i = 0, h = udp4_l2_mh_sock; i < UDP_MAX_FRAMES; i++, h++) { - struct msghdr *mh = &h->msg_hdr; - - mh->msg_name = &udp4_l2_buf[i].s_in; - mh->msg_namelen = sizeof(udp4_l2_buf[i].s_in); - - udp4_l2_iov_sock[i].iov_base = udp4_l2_buf[i].data; - udp4_l2_iov_sock[i].iov_len = sizeof(udp4_l2_buf[i].data); - mh->msg_iov = &udp4_l2_iov_sock[i]; - mh->msg_iovlen = 1; - } - - for (i = 0; i < UDP_MAX_FRAMES; i++) { - struct iovec *iov = &udp4_l2_iov_tap[i]; - - iov->iov_base = tap_iov_base(c, &udp4_l2_buf[i].taph); - } + struct udp_payload_t *payload = &udp_payload[i]; + struct msghdr *mh = &udp_mh_recv[i].msg_hdr; + struct udp_meta_t *meta = &udp_meta[i]; + struct iovec *siov = &udp_iov_recv[i]; + struct iovec *tiov = udp_l2_iov[i]; + + *meta = (struct udp_meta_t) { + .ip4h = L2_BUF_IP4_INIT(IPPROTO_UDP), + .ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP), + }; + + *siov = IOV_OF_LVALUE(payload->data); + + tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &meta->taph); + tiov[UDP_IOV_PAYLOAD].iov_base = payload; + + mh->msg_name = &meta->s_in; + mh->msg_namelen = sizeof(meta->s_in); + mh->msg_iov = siov; + mh->msg_iovlen = 1; } /** - * udp_sock6_iov_init() - Initialise scatter-gather L2 buffers for IPv6 sockets + * udp_iov_init() - Initialise scatter-gather L2 buffers * @c: Execution context */ -static void udp_sock6_iov_init(const struct ctx *c) +static void udp_iov_init(const struct ctx *c) { - struct mmsghdr *h; - int i; + size_t i; - for (i = 0; i < ARRAY_SIZE(udp6_l2_buf); i++) { - udp6_l2_buf[i] = (struct udp6_l2_buf_t) { - .taph = TAP_HDR_INIT(ETH_P_IPV6), - .ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP) - }; - } + udp4_eth_hdr.h_proto = htons_constant(ETH_P_IP); + udp6_eth_hdr.h_proto = htons_constant(ETH_P_IPV6); - for (i = 0, h = udp6_l2_mh_sock; i < UDP_MAX_FRAMES; i++, h++) { - struct msghdr *mh = &h->msg_hdr; + for (i = 0; i < UDP_MAX_FRAMES; i++) + udp_iov_init_one(c, i); +} - mh->msg_name = &udp6_l2_buf[i].s_in6; - mh->msg_namelen = sizeof(struct sockaddr_in6); +/** + * udp_splice_prepare() - Prepare one datagram for splicing + * @mmh: Receiving mmsghdr array + * @idx: Index of the datagram to prepare + */ +static void udp_splice_prepare(struct mmsghdr *mmh, unsigned idx) +{ + udp_mh_splice[idx].msg_hdr.msg_iov->iov_len = mmh[idx].msg_len; +} - udp6_l2_iov_sock[i].iov_base = udp6_l2_buf[i].data; - udp6_l2_iov_sock[i].iov_len = sizeof(udp6_l2_buf[i].data); - mh->msg_iov = &udp6_l2_iov_sock[i]; - mh->msg_iovlen = 1; - } +/** + * udp_splice_send() - Send a batch of datagrams from socket to socket + * @c: Execution context + * @start: Index of batch's first datagram in udp[46]_l2_buf + * @n: Number of datagrams in batch + * @src: Source port for datagram (target side) + * @dst: Destination port for datagrams (target side) + * @ref: epoll reference for origin socket + * @now: Timestamp + */ +static void udp_splice_send(const struct ctx *c, size_t start, size_t n, + flow_sidx_t tosidx) +{ + const struct flowside *toside = flowside_at_sidx(tosidx); + const struct udp_flow *uflow = udp_at_sidx(tosidx); + uint8_t topif = pif_at_sidx(tosidx); + int s = uflow->s[tosidx.sidei]; + socklen_t sl; - for (i = 0; i < UDP_MAX_FRAMES; i++) { - struct iovec *iov = &udp6_l2_iov_tap[i]; + pif_sockaddr(c, &udp_splice_to, &sl, topif, + &toside->eaddr, toside->eport); - iov->iov_base = tap_iov_base(c, &udp6_l2_buf[i].taph); - } + sendmmsg(s, udp_mh_splice + start, n, MSG_NOSIGNAL); } /** - * udp_splice_new() - Create and prepare socket for "spliced" binding - * @c: Execution context - * @v6: Set for IPv6 sockets - * @src: Source port of original connection, host order - * @ns: Does the splice originate in the ns or not - * - * Return: prepared socket, negative error code on failure + * udp_update_hdr4() - Update headers for one IPv4 datagram + * @ip4h: Pre-filled IPv4 header (except for tot_len and saddr) + * @bp: Pointer to udp_payload_t to update + * @toside: Flowside for destination side + * @dlen: Length of UDP payload + * @no_udp_csum: Do not set UDP checksum * - * #syscalls:pasta getsockname + * Return: size of IPv4 payload (UDP header + data) */ -int udp_splice_new(const struct ctx *c, int v6, in_port_t src, bool ns) +size_t udp_update_hdr4(struct iphdr *ip4h, struct udp_payload_t *bp, + const struct flowside *toside, size_t dlen, + bool no_udp_csum) { - struct epoll_event ev = { .events = EPOLLIN | EPOLLRDHUP | EPOLLHUP }; - union epoll_ref ref = { .type = EPOLL_TYPE_UDP, - .udp = { .splice = true, .v6 = v6, .port = src } - }; - struct udp_splice_port *sp; - int act, s; - - if (ns) { - ref.udp.pif = PIF_SPLICE; - sp = &udp_splice_ns[v6 ? V6 : V4][src]; - act = UDP_ACT_SPLICE_NS; + const struct in_addr *src = inany_v4(&toside->oaddr); + const struct in_addr *dst = inany_v4(&toside->eaddr); + size_t l4len = dlen + sizeof(bp->uh); + size_t l3len = l4len + sizeof(*ip4h); + + ASSERT(src && dst); + + ip4h->tot_len = htons(l3len); + ip4h->daddr = dst->s_addr; + ip4h->saddr = src->s_addr; + ip4h->check = csum_ip4_header(l3len, IPPROTO_UDP, *src, *dst); + + bp->uh.source = htons(toside->oport); + bp->uh.dest = htons(toside->eport); + bp->uh.len = htons(l4len); + if (no_udp_csum) { + bp->uh.check = 0; } else { - ref.udp.pif = PIF_HOST; - sp = &udp_splice_init[v6 ? V6 : V4][src]; - act = UDP_ACT_SPLICE_INIT; + const struct iovec iov = { + .iov_base = bp->data, + .iov_len = dlen + }; + csum_udp4(&bp->uh, *src, *dst, &iov, 1, 0); } - s = socket(v6 ? AF_INET6 : AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, - IPPROTO_UDP); - - if (s > FD_REF_MAX) { - close(s); - return -EIO; - } + return l4len; +} - if (s < 0) - return s; +/** + * udp_update_hdr6() - Update headers for one IPv6 datagram + * @ip6h: Pre-filled IPv6 header (except for payload_len and + * addresses) + * @bp: Pointer to udp_payload_t to update + * @toside: Flowside for destination side + * @dlen: Length of UDP payload + * @no_udp_csum: Do not set UDP checksum + * + * Return: size of IPv6 payload (UDP header + data) + */ +size_t udp_update_hdr6(struct ipv6hdr *ip6h, struct udp_payload_t *bp, + const struct flowside *toside, size_t dlen, + bool no_udp_csum) +{ + uint16_t l4len = dlen + sizeof(bp->uh); - ref.fd = s; + ip6h->payload_len = htons(l4len); + ip6h->daddr = toside->eaddr.a6; + ip6h->saddr = toside->oaddr.a6; + ip6h->version = 6; + ip6h->nexthdr = IPPROTO_UDP; + ip6h->hop_limit = 255; - if (v6) { - struct sockaddr_in6 addr6 = { - .sin6_family = AF_INET6, - .sin6_port = htons(src), - .sin6_addr = IN6ADDR_LOOPBACK_INIT, - }; - if (bind(s, (struct sockaddr *)&addr6, sizeof(addr6))) - goto fail; + bp->uh.source = htons(toside->oport); + bp->uh.dest = htons(toside->eport); + bp->uh.len = ip6h->payload_len; + if (no_udp_csum) { + /* 0 is an invalid checksum for UDP IPv6 and dropped by + * the kernel stack, even if the checksum is disabled by virtio + * flags. We need to put any non-zero value here. + */ + bp->uh.check = 0xffff; } else { - struct sockaddr_in addr4 = { - .sin_family = AF_INET, - .sin_port = htons(src), - .sin_addr = IN4ADDR_LOOPBACK_INIT, + const struct iovec iov = { + .iov_base = bp->data, + .iov_len = dlen }; - if (bind(s, (struct sockaddr *)&addr4, sizeof(addr4))) - goto fail; + csum_udp6(&bp->uh, &toside->oaddr.a6, &toside->eaddr.a6, + &iov, 1, 0); } - sp->sock = s; - bitmap_set(udp_act[v6 ? V6 : V4][act], src); - - ev.data.u64 = ref.u64; - epoll_ctl(c->epollfd, EPOLL_CTL_ADD, s, &ev); - return s; - -fail: - close(s); - return -1; + return l4len; } /** - * struct udp_splice_new_ns_arg - Arguments for udp_splice_new_ns() - * @c: Execution context - * @v6: Set for IPv6 - * @src: Source port of originating datagram, host order - * @dst: Destination port of originating datagram, host order - * @s: Newly created socket or negative error code + * udp_tap_prepare() - Convert one datagram into a tap frame + * @mmh: Receiving mmsghdr array + * @idx: Index of the datagram to prepare + * @toside: Flowside for destination side + * @no_udp_csum: Do not set UDP checksum */ -struct udp_splice_new_ns_arg { - const struct ctx *c; - int v6; - in_port_t src; - int s; -}; +static void udp_tap_prepare(const struct mmsghdr *mmh, + unsigned idx, const struct flowside *toside, + bool no_udp_csum) +{ + struct iovec (*tap_iov)[UDP_NUM_IOVS] = &udp_l2_iov[idx]; + struct udp_payload_t *bp = &udp_payload[idx]; + struct udp_meta_t *bm = &udp_meta[idx]; + size_t l4len; + + if (!inany_v4(&toside->eaddr) || !inany_v4(&toside->oaddr)) { + l4len = udp_update_hdr6(&bm->ip6h, bp, toside, + mmh[idx].msg_len, no_udp_csum); + tap_hdr_update(&bm->taph, l4len + sizeof(bm->ip6h) + + sizeof(udp6_eth_hdr)); + (*tap_iov)[UDP_IOV_ETH] = IOV_OF_LVALUE(udp6_eth_hdr); + (*tap_iov)[UDP_IOV_IP] = IOV_OF_LVALUE(bm->ip6h); + } else { + l4len = udp_update_hdr4(&bm->ip4h, bp, toside, + mmh[idx].msg_len, no_udp_csum); + tap_hdr_update(&bm->taph, l4len + sizeof(bm->ip4h) + + sizeof(udp4_eth_hdr)); + (*tap_iov)[UDP_IOV_ETH] = IOV_OF_LVALUE(udp4_eth_hdr); + (*tap_iov)[UDP_IOV_IP] = IOV_OF_LVALUE(bm->ip4h); + } + (*tap_iov)[UDP_IOV_PAYLOAD].iov_len = l4len; +} /** - * udp_splice_new_ns() - Enter namespace and call udp_splice_new() - * @arg: See struct udp_splice_new_ns_arg + * udp_sock_recverr() - Receive and clear an error from a socket + * @s: Socket to receive from * - * Return: 0 + * Return: 1 if error received and processed, 0 if no more errors in queue, < 0 + * if there was an error reading the queue + * + * #syscalls recvmsg */ -static int udp_splice_new_ns(void *arg) +static int udp_sock_recverr(int s) { - struct udp_splice_new_ns_arg *a; + const struct sock_extended_err *ee; + const struct cmsghdr *hdr; + char buf[CMSG_SPACE(sizeof(*ee))]; + struct msghdr mh = { + .msg_name = NULL, + .msg_namelen = 0, + .msg_iov = NULL, + .msg_iovlen = 0, + .msg_control = buf, + .msg_controllen = sizeof(buf), + }; + ssize_t rc; + + rc = recvmsg(s, &mh, MSG_ERRQUEUE); + if (rc < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) + return 0; + + err_perror("UDP: Failed to read error queue"); + return -1; + } - a = (struct udp_splice_new_ns_arg *)arg; + if (!(mh.msg_flags & MSG_ERRQUEUE)) { + err("Missing MSG_ERRQUEUE flag reading error queue"); + return -1; + } - ns_enter(a->c); + hdr = CMSG_FIRSTHDR(&mh); + if (!((hdr->cmsg_level == IPPROTO_IP && + hdr->cmsg_type == IP_RECVERR) || + (hdr->cmsg_level == IPPROTO_IPV6 && + hdr->cmsg_type == IPV6_RECVERR))) { + err("Unexpected cmsg reading error queue"); + return -1; + } - a->s = udp_splice_new(a->c, a->v6, a->src, true); + ee = (const struct sock_extended_err *)CMSG_DATA(hdr); - return 0; + /* TODO: When possible propagate and otherwise handle errors */ + debug("%s error on UDP socket %i: %s", + str_ee_origin(ee), s, strerror(ee->ee_errno)); + + return 1; } /** - * udp_mmh_splice_port() - Is source address of message suitable for splicing? - * @v6: Is @sa a sockaddr_in6 (otherwise sockaddr_in)? - * @mmh: mmsghdr of incoming message + * udp_sock_errs() - Process errors on a socket + * @c: Execution context + * @s: Socket to receive from + * @events: epoll events bitmap * - * Return: if @sa refers to localhost (127.0.0.1 or ::1) the port from - * @sa in host order, otherwise -1. + * Return: Number of errors handled, or < 0 if we have an unrecoverable error */ -static int udp_mmh_splice_port(bool v6, const struct mmsghdr *mmh) +int udp_sock_errs(const struct ctx *c, int s, uint32_t events) { - const struct sockaddr_in6 *sa6 = mmh->msg_hdr.msg_name; - const struct sockaddr_in *sa4 = mmh->msg_hdr.msg_name; + unsigned n_err = 0; + socklen_t errlen; + int rc, err; - if (v6 && IN6_IS_ADDR_LOOPBACK(&sa6->sin6_addr)) - return ntohs(sa6->sin6_port); + ASSERT(!c->no_udp); - if (!v6 && IN4_IS_ADDR_LOOPBACK(&sa4->sin_addr)) - return ntohs(sa4->sin_port); + if (!(events & EPOLLERR)) + return 0; /* Nothing to do */ - return -1; -} + /* Empty the error queue */ + while ((rc = udp_sock_recverr(s)) > 0) + n_err += rc; -/** - * udp_splice_sendfrom() - Send datagrams from given port to given port - * @c: Execution context - * @start: Index of first datagram in udp[46]_l2_buf - * @n: Number of datagrams to send - * @src: Datagrams will be sent from this port (on origin side) - * @dst: Datagrams will be send to this port (on destination side) - * @from_pif: pif from which the packet originated - * @v6: Send as IPv6? - * @allow_new: If true create sending socket if needed, if false discard - * if no sending socket is available - * @now: Timestamp - */ -static void udp_splice_sendfrom(const struct ctx *c, unsigned start, unsigned n, - in_port_t src, in_port_t dst, uint8_t from_pif, - bool v6, bool allow_new, - const struct timespec *now) -{ - struct mmsghdr *mmh_recv, *mmh_send; - unsigned int i; - int s; + if (rc < 0) + return -1; /* error reading error, unrecoverable */ - if (v6) { - mmh_recv = udp6_l2_mh_sock; - mmh_send = udp6_mh_splice; - } else { - mmh_recv = udp4_l2_mh_sock; - mmh_send = udp4_mh_splice; + errlen = sizeof(err); + if (getsockopt(s, SOL_SOCKET, SO_ERROR, &err, &errlen) < 0 || + errlen != sizeof(err)) { + err_perror("Error reading SO_ERROR"); + return -1; /* error reading error, unrecoverable */ } - if (from_pif == PIF_SPLICE) { - src += c->udp.fwd_in.rdelta[src]; - s = udp_splice_init[v6][src].sock; - if (s < 0 && allow_new) - s = udp_splice_new(c, v6, src, false); - - if (s < 0) - return; - - udp_splice_ns[v6][dst].ts = now->tv_sec; - udp_splice_init[v6][src].ts = now->tv_sec; - } else { - ASSERT(from_pif == PIF_HOST); - src += c->udp.fwd_out.rdelta[src]; - s = udp_splice_ns[v6][src].sock; - if (s < 0 && allow_new) { - struct udp_splice_new_ns_arg arg = { - c, v6, src, -1, - }; - - NS_CALL(udp_splice_new_ns, &arg); - s = arg.s; - } - if (s < 0) - return; - - udp_splice_init[v6][dst].ts = now->tv_sec; - udp_splice_ns[v6][src].ts = now->tv_sec; + if (err) { + debug("Unqueued error on UDP socket %i: %s", s, strerror(err)); + n_err++; } - for (i = start; i < start + n; i++) - mmh_send[i].msg_hdr.msg_iov->iov_len = mmh_recv[i].msg_len; + if (!n_err) { + /* EPOLLERR, but no errors to clear !? */ + err("EPOLLERR event without reported errors on socket %i", s); + return -1; /* no way to clear, unrecoverable */ + } - sendmmsg(s, mmh_send + start, n, MSG_NOSIGNAL); + return n_err; } /** - * udp_update_hdr4() - Update headers for one IPv4 datagram + * udp_sock_recv() - Receive datagrams from a socket * @c: Execution context - * @n: Index of buffer in udp4_l2_buf pool - * @dstport: Destination port number - * @now: Current timestamp + * @s: Socket to receive from + * @events: epoll events bitmap + * @mmh mmsghdr array to receive into * - * Return: size of tap frame with headers + * Return: Number of datagrams received + * + * #syscalls recvmmsg arm:recvmmsg_time64 i686:recvmmsg_time64 */ -size_t udp_update_hdr4(const struct ctx *c, struct iphdr *iph, - size_t data_len, struct sockaddr_in *s_in, - in_port_t dstport, const struct timespec *now) +static int udp_sock_recv(const struct ctx *c, int s, uint32_t events, + struct mmsghdr *mmh) { - struct udphdr *uh = (struct udphdr *)(iph + 1); - const struct in_addr *src; - in_port_t src_port; - size_t ip_len; - - ip_len = data_len + sizeof(struct iphdr) + sizeof(struct udphdr); - - iph->tot_len = htons(ip_len); - iph->daddr = c->ip4.addr_seen.s_addr; - - src = &s_in->sin_addr; - src_port = ntohs(s_in->sin_port); - - if (!IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns_match) && - IN4_ARE_ADDR_EQUAL(src, &c->ip4.dns_host) && src_port == 53) { - src = &c->ip4.dns_match; - } else if (IN4_IS_ADDR_LOOPBACK(src) || - IN4_ARE_ADDR_EQUAL(src, &c->ip4.addr_seen)) { - udp_tap_map[V4][src_port].ts = now->tv_sec; - udp_tap_map[V4][src_port].flags |= PORT_LOCAL; + /* For not entirely clear reasons (data locality?) pasta gets better + * throughput if we receive tap datagrams one at a atime. For small + * splice datagrams throughput is slightly better if we do batch, but + * it's slightly worse for large splice datagrams. Since we don't know + * before we receive whether we'll use tap or splice, always go one at a + * time for pasta mode. + */ + int n = (c->mode == MODE_PASTA ? 1 : UDP_MAX_FRAMES); - if (IN4_IS_ADDR_LOOPBACK(src)) - udp_tap_map[V4][src_port].flags |= PORT_LOOPBACK; - else - udp_tap_map[V4][src_port].flags &= ~PORT_LOOPBACK; + ASSERT(!c->no_udp); - bitmap_set(udp_act[V4][UDP_ACT_TAP], src_port); + if (!(events & EPOLLIN)) + return 0; - src = &c->ip4.gw; + n = recvmmsg(s, mmh, n, 0, NULL); + if (n < 0) { + err_perror("Error receiving datagrams"); + return 0; } - iph->saddr = src->s_addr; - - iph->check = csum_ip4_header(iph->tot_len, IPPROTO_UDP, - *src, c->ip4.addr_seen); - uh->source = s_in->sin_port; - uh->dest = htons(dstport); - uh->len = htons(data_len + sizeof(struct udphdr)); - uh->check = 0; - return ip_len; + return n; } /** - * udp_update_hdr6() - Update headers for one IPv6 datagram + * udp_buf_listen_sock_handler() - Handle new data from socket * @c: Execution context - * @n: Index of buffer in udp6_l2_buf pool - * @dstport: Destination port number + * @ref: epoll reference + * @events: epoll events bitmap * @now: Current timestamp * - * Return: size of tap frame with headers + * #syscalls recvmmsg */ -size_t udp_update_hdr6(const struct ctx *c, struct ipv6hdr *ip6h, - size_t data_len, struct sockaddr_in6 *s_in6, - in_port_t dstport, const struct timespec *now) +static void udp_buf_listen_sock_handler(const struct ctx *c, + union epoll_ref ref, uint32_t events, + const struct timespec *now) { - struct udphdr *uh = (struct udphdr *)(ip6h + 1); - const struct in6_addr *src, *dst; - uint16_t payload_len; - in_port_t src_port; - size_t ip_len; - - dst = &c->ip6.addr_seen; - src = &s_in6->sin6_addr; - src_port = ntohs(s_in6->sin6_port); - - ip_len = data_len + sizeof(struct ipv6hdr) + sizeof(struct udphdr); - - payload_len = data_len + sizeof(struct udphdr); - ip6h->payload_len = htons(payload_len); - - if (IN6_IS_ADDR_LINKLOCAL(src)) { - dst = &c->ip6.addr_ll_seen; - } else if (!IN6_IS_ADDR_UNSPECIFIED(&c->ip6.dns_match) && - IN6_ARE_ADDR_EQUAL(src, &c->ip6.dns_host) && - src_port == 53) { - src = &c->ip6.dns_match; - } else if (IN6_IS_ADDR_LOOPBACK(src) || - IN6_ARE_ADDR_EQUAL(src, &c->ip6.addr_seen) || - IN6_ARE_ADDR_EQUAL(src, &c->ip6.addr)) { - udp_tap_map[V6][src_port].ts = now->tv_sec; - udp_tap_map[V6][src_port].flags |= PORT_LOCAL; - - if (IN6_IS_ADDR_LOOPBACK(src)) - udp_tap_map[V6][src_port].flags |= PORT_LOOPBACK; - else - udp_tap_map[V6][src_port].flags &= ~PORT_LOOPBACK; - - if (IN6_ARE_ADDR_EQUAL(src, &c->ip6.addr)) - udp_tap_map[V6][src_port].flags |= PORT_GUA; - else - udp_tap_map[V6][src_port].flags &= ~PORT_GUA; - - bitmap_set(udp_act[V6][UDP_ACT_TAP], src_port); - - dst = &c->ip6.addr_ll_seen; - - if (IN6_IS_ADDR_LINKLOCAL(&c->ip6.gw)) - src = &c->ip6.gw; - else - src = &c->ip6.addr_ll; + const socklen_t sasize = sizeof(udp_meta[0].s_in); + int n, i; + + if (udp_sock_errs(c, ref.fd, events) < 0) { + err("UDP: Unrecoverable error on listening socket:" + " (%s port %hu)", pif_name(ref.udp.pif), ref.udp.port); + /* FIXME: what now? close/re-open socket? */ + return; } - ip6h->daddr = *dst; - ip6h->saddr = *src; - ip6h->version = 6; - ip6h->nexthdr = IPPROTO_UDP; - ip6h->hop_limit = 255; - uh->source = s_in6->sin6_port; - uh->dest = htons(dstport); - uh->len = ip6h->payload_len; - uh->check = 0; - if (c->mode != MODE_VU) - uh->check = csum(uh, payload_len, - proto_ipv6_header_psum(payload_len, IPPROTO_UDP, - src, dst)); - else - uh->check = 0xffff; /* zero checksum is invalid with IPv6 */ - - return ip_len; + if ((n = udp_sock_recv(c, ref.fd, events, udp_mh_recv)) <= 0) + return; + + /* We divide datagrams into batches based on how we need to send them, + * determined by udp_meta[i].tosidx. To avoid either two passes through + * the array, or recalculating tosidx for a single entry, we have to + * populate it one entry *ahead* of the loop counter. + */ + udp_meta[0].tosidx = udp_flow_from_sock(c, ref, &udp_meta[0].s_in, now); + udp_mh_recv[0].msg_hdr.msg_namelen = sasize; + for (i = 0; i < n; ) { + flow_sidx_t batchsidx = udp_meta[i].tosidx; + uint8_t batchpif = pif_at_sidx(batchsidx); + int batchstart = i; + + do { + if (pif_is_socket(batchpif)) { + udp_splice_prepare(udp_mh_recv, i); + } else if (batchpif == PIF_TAP) { + udp_tap_prepare(udp_mh_recv, i, + flowside_at_sidx(batchsidx), + false); + } + + if (++i >= n) + break; + + udp_meta[i].tosidx = udp_flow_from_sock(c, ref, + &udp_meta[i].s_in, + now); + udp_mh_recv[i].msg_hdr.msg_namelen = sasize; + } while (flow_sidx_eq(udp_meta[i].tosidx, batchsidx)); + + if (pif_is_socket(batchpif)) { + udp_splice_send(c, batchstart, i - batchstart, + batchsidx); + } else if (batchpif == PIF_TAP) { + tap_send_frames(c, &udp_l2_iov[batchstart][0], + UDP_NUM_IOVS, i - batchstart); + } else if (flow_sidx_valid(batchsidx)) { + flow_sidx_t fromsidx = flow_sidx_opposite(batchsidx); + struct udp_flow *uflow = udp_at_sidx(batchsidx); + + flow_err(uflow, + "No support for forwarding UDP from %s to %s", + pif_name(pif_at_sidx(fromsidx)), + pif_name(batchpif)); + } else { + debug("Discarding %d datagrams without flow", + i - batchstart); + } + } } /** - * udp_tap_send() - Prepare UDP datagrams and send to tap interface + * udp_listen_sock_handler() - Handle new data from socket * @c: Execution context - * @start: Index of first datagram in udp[46]_l2_buf pool - * @n: Number of datagrams to send - * @dstport: Destination port number - * @v6: True if using IPv6 + * @ref: epoll reference + * @events: epoll events bitmap * @now: Current timestamp - * - * Return: size of tap frame with headers - */ -#pragma GCC diagnostic push -/* ignore unaligned pointer value warning for &udp6_l2_buf[i].ip6h and - * &udp4_l2_buf[i].iph */ -#pragma GCC diagnostic ignored "-Waddress-of-packed-member" -static void udp_tap_send(const struct ctx *c, - unsigned int start, unsigned int n, - in_port_t dstport, bool v6, const struct timespec *now) +void udp_listen_sock_handler(const struct ctx *c, + union epoll_ref ref, uint32_t events, + const struct timespec *now) { - struct iovec *tap_iov; - unsigned int i; - - if (v6) - tap_iov = udp6_l2_iov_tap; - else - tap_iov = udp4_l2_iov_tap; - - for (i = start; i < start + n; i++) { - size_t ip_len; - - if (v6) { - ip_len = udp_update_hdr6(c, &udp6_l2_buf[i].ip6h, - udp6_l2_mh_sock[i].msg_len, - &udp6_l2_buf[i].s_in6, dstport, - now); - tap_iov[i].iov_len = tap_iov_len(c, - &udp6_l2_buf[i].taph, - ip_len); - } else { - ip_len = udp_update_hdr4(c, &udp4_l2_buf[i].iph, - udp4_l2_mh_sock[i].msg_len, - &udp4_l2_buf[i].s_in, - dstport, now); - - tap_iov[i].iov_len = tap_iov_len(c, - &udp4_l2_buf[i].taph, - ip_len); - } + if (c->mode == MODE_VU) { + udp_vu_listen_sock_handler(c, ref, events, now); + return; } - tap_send_frames(c, tap_iov + start, n); + udp_buf_listen_sock_handler(c, ref, events, now); } -#pragma GCC diagnostic pop /** - * udp_buf_sock_handler() - Handle new data from socket + * udp_buf_reply_sock_handler() - Handle new data from flow specific socket * @c: Execution context * @ref: epoll reference * @events: epoll events bitmap @@ -754,65 +646,70 @@ static void udp_tap_send(const struct ctx *c, * * #syscalls recvmmsg */ -void udp_buf_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, - const struct timespec *now) +static void udp_buf_reply_sock_handler(const struct ctx *c, union epoll_ref ref, + uint32_t events, + const struct timespec *now) { - /* For not entirely clear reasons (data locality?) pasta gets - * better throughput if we receive tap datagrams one at a - * atime. For small splice datagrams throughput is slightly - * better if we do batch, but it's slightly worse for large - * splice datagrams. Since we don't know before we receive - * whether we'll use tap or splice, always go one at a time - * for pasta mode. - */ - ssize_t n = (c->mode == MODE_PASTA ? 1 : UDP_MAX_FRAMES); - in_port_t dstport = ref.udp.port; - bool v6 = ref.udp.v6; - struct mmsghdr *mmh_recv; - int i, m; + flow_sidx_t tosidx = flow_sidx_opposite(ref.flowside); + const struct flowside *toside = flowside_at_sidx(tosidx); + struct udp_flow *uflow = udp_at_sidx(ref.flowside); + uint8_t topif = pif_at_sidx(tosidx); + int n, i, from_s; - if (c->no_udp || !(events & EPOLLIN)) - return; + ASSERT(!c->no_udp && uflow); - if (ref.udp.pif == PIF_SPLICE) - dstport += c->udp.fwd_out.f.delta[dstport]; - else if (ref.udp.pif == PIF_HOST) - dstport += c->udp.fwd_in.f.delta[dstport]; + from_s = uflow->s[ref.flowside.sidei]; - if (v6) { - mmh_recv = udp6_l2_mh_sock; - udp6_localname.sin6_port = htons(dstport); - } else { - mmh_recv = udp4_l2_mh_sock; - udp4_localname.sin_port = htons(dstport); + if (udp_sock_errs(c, from_s, events) < 0) { + flow_err(uflow, "Unrecoverable error on reply socket"); + flow_err_details(uflow); + udp_flow_close(c, uflow); + return; } - n = recvmmsg(ref.fd, mmh_recv, n, 0, NULL); - if (n <= 0) + if ((n = udp_sock_recv(c, from_s, events, udp_mh_recv)) <= 0) return; - for (i = 0; i < n; i += m) { - int splicefrom = -1; - m = n; + flow_trace(uflow, "Received %d datagrams on reply socket", n); + uflow->ts = now->tv_sec; - if (ref.udp.splice) { - splicefrom = udp_mmh_splice_port(v6, mmh_recv + i); + for (i = 0; i < n; i++) { + if (pif_is_socket(topif)) + udp_splice_prepare(udp_mh_recv, i); + else if (topif == PIF_TAP) + udp_tap_prepare(udp_mh_recv, i, toside, false); + /* Restore sockaddr length clobbered by recvmsg() */ + udp_mh_recv[i].msg_hdr.msg_namelen = sizeof(udp_meta[i].s_in); + } - for (m = 1; i + m < n; m++) { - int p; + if (pif_is_socket(topif)) { + udp_splice_send(c, 0, n, tosidx); + } else if (topif == PIF_TAP) { + tap_send_frames(c, &udp_l2_iov[0][0], UDP_NUM_IOVS, n); + } else { + uint8_t frompif = pif_at_sidx(ref.flowside); - p = udp_mmh_splice_port(v6, mmh_recv + i + m); - if (p != splicefrom) - break; - } - } + flow_err(uflow, "No support for forwarding UDP from %s to %s", + pif_name(frompif), pif_name(topif)); + } +} - if (splicefrom >= 0) - udp_splice_sendfrom(c, i, m, splicefrom, dstport, - ref.udp.pif, v6, ref.udp.orig, now); - else - udp_tap_send(c, i, m, dstport, v6, now); +/** + * udp_reply_sock_handler() - Handle new data from flow specific socket + * @c: Execution context + * @ref: epoll reference + * @events: epoll events bitmap + * @now: Current timestamp + */ +void udp_reply_sock_handler(const struct ctx *c, union epoll_ref ref, + uint32_t events, const struct timespec *now) +{ + if (c->mode == MODE_VU) { + udp_vu_reply_sock_handler(c, ref, events, now); + return; } + + udp_buf_reply_sock_handler(c, ref, events, now); } /** @@ -830,23 +727,23 @@ void udp_buf_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t eve * * #syscalls sendmmsg */ -int udp_tap_handler(struct ctx *c, uint8_t pif, +int udp_tap_handler(const struct ctx *c, uint8_t pif, sa_family_t af, const void *saddr, const void *daddr, const struct pool *p, int idx, const struct timespec *now) { + const struct flowside *toside; struct mmsghdr mm[UIO_MAXIOV]; + union sockaddr_inany to_sa; struct iovec m[UIO_MAXIOV]; - struct sockaddr_in6 s_in6; - struct sockaddr_in s_in; const struct udphdr *uh; - struct sockaddr *sa; + struct udp_flow *uflow; int i, s, count = 0; + flow_sidx_t tosidx; in_port_t src, dst; + uint8_t topif; socklen_t sl; - (void)c; - (void)saddr; - (void)pif; + ASSERT(!c->no_udp); uh = packet_get(p, idx, 0, sizeof(*uh), NULL); if (!uh) @@ -858,107 +755,32 @@ int udp_tap_handler(struct ctx *c, uint8_t pif, src = ntohs(uh->source); dst = ntohs(uh->dest); - if (af == AF_INET) { - s_in = (struct sockaddr_in) { - .sin_family = AF_INET, - .sin_port = uh->dest, - .sin_addr = *(struct in_addr *)daddr, - }; - - sa = (struct sockaddr *)&s_in; - sl = sizeof(s_in); - - if (IN4_ARE_ADDR_EQUAL(&s_in.sin_addr, &c->ip4.dns_match) && - ntohs(s_in.sin_port) == 53) { - s_in.sin_addr = c->ip4.dns_host; - } else if (IN4_ARE_ADDR_EQUAL(&s_in.sin_addr, &c->ip4.gw) && - !c->no_map_gw) { - if (!(udp_tap_map[V4][dst].flags & PORT_LOCAL) || - (udp_tap_map[V4][dst].flags & PORT_LOOPBACK)) - s_in.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - else - s_in.sin_addr = c->ip4.addr_seen; - } - - debug("UDP from tap src=%hu dst=%hu, s=%d", - src, dst, udp_tap_map[V4][src].sock); - if ((s = udp_tap_map[V4][src].sock) < 0) { - struct in_addr bind_addr = IN4ADDR_ANY_INIT; - union udp_epoll_ref uref = { - .port = src, - .pif = PIF_HOST, - }; - const char *bind_if = NULL; - - if (!IN4_IS_ADDR_LOOPBACK(&s_in.sin_addr)) - bind_if = c->ip4.ifname_out; - - if (!IN4_IS_ADDR_LOOPBACK(&s_in.sin_addr)) - bind_addr = c->ip4.addr_out; - - s = sock_l4(c, AF_INET, IPPROTO_UDP, &bind_addr, - bind_if, src, uref.u32); - if (s < 0) - return p->count - idx; - - udp_tap_map[V4][src].sock = s; - bitmap_set(udp_act[V4][UDP_ACT_TAP], src); - } - - udp_tap_map[V4][src].ts = now->tv_sec; - } else { - s_in6 = (struct sockaddr_in6) { - .sin6_family = AF_INET6, - .sin6_port = uh->dest, - .sin6_addr = *(struct in6_addr *)daddr, - }; - const struct in6_addr *bind_addr = &in6addr_any; - - sa = (struct sockaddr *)&s_in6; - sl = sizeof(s_in6); - - if (IN6_ARE_ADDR_EQUAL(daddr, &c->ip6.dns_match) && - ntohs(s_in6.sin6_port) == 53) { - s_in6.sin6_addr = c->ip6.dns_host; - } else if (IN6_ARE_ADDR_EQUAL(daddr, &c->ip6.gw) && - !c->no_map_gw) { - if (!(udp_tap_map[V6][dst].flags & PORT_LOCAL) || - (udp_tap_map[V6][dst].flags & PORT_LOOPBACK)) - s_in6.sin6_addr = in6addr_loopback; - else if (udp_tap_map[V6][dst].flags & PORT_GUA) - s_in6.sin6_addr = c->ip6.addr; - else - s_in6.sin6_addr = c->ip6.addr_seen; - } else if (IN6_IS_ADDR_LINKLOCAL(&s_in6.sin6_addr)) { - bind_addr = &c->ip6.addr_ll; - } - - if ((s = udp_tap_map[V6][src].sock) < 0) { - union udp_epoll_ref uref = { - .v6 = 1, - .port = src, - .pif = PIF_HOST, - }; - const char *bind_if = NULL; + tosidx = udp_flow_from_tap(c, pif, af, saddr, daddr, src, dst, now); + if (!(uflow = udp_at_sidx(tosidx))) { + char sstr[INET6_ADDRSTRLEN], dstr[INET6_ADDRSTRLEN]; - if (!IN6_IS_ADDR_LOOPBACK(&s_in6.sin6_addr)) - bind_if = c->ip6.ifname_out; + debug("Dropping datagram with no flow %s %s:%hu -> %s:%hu", + pif_name(pif), + inet_ntop(af, saddr, sstr, sizeof(sstr)), src, + inet_ntop(af, daddr, dstr, sizeof(dstr)), dst); + return 1; + } - if (!IN6_IS_ADDR_LOOPBACK(&s_in6.sin6_addr) && - !IN6_IS_ADDR_LINKLOCAL(&s_in6.sin6_addr)) - bind_addr = &c->ip6.addr_out; + topif = pif_at_sidx(tosidx); + if (topif != PIF_HOST) { + flow_sidx_t fromsidx = flow_sidx_opposite(tosidx); + uint8_t frompif = pif_at_sidx(fromsidx); - s = sock_l4(c, AF_INET6, IPPROTO_UDP, bind_addr, - bind_if, src, uref.u32); - if (s < 0) - return p->count - idx; + flow_err(uflow, "No support for forwarding UDP from %s to %s", + pif_name(frompif), pif_name(topif)); + return 1; + } + toside = flowside_at_sidx(tosidx); - udp_tap_map[V6][src].sock = s; - bitmap_set(udp_act[V6][UDP_ACT_TAP], src); - } + s = udp_at_sidx(tosidx)->s[tosidx.sidei]; + ASSERT(s >= 0); - udp_tap_map[V6][src].ts = now->tv_sec; - } + pif_sockaddr(c, &to_sa, &sl, topif, &toside->eaddr, toside->eport); for (i = 0; i < (int)p->count - idx; i++) { struct udphdr *uh_send; @@ -968,7 +790,7 @@ int udp_tap_handler(struct ctx *c, uint8_t pif, if (!uh_send) return p->count - idx; - mm[i].msg_hdr.msg_name = sa; + mm[i].msg_hdr.msg_name = &to_sa; mm[i].msg_hdr.msg_namelen = sl; if (len) { @@ -1000,56 +822,62 @@ int udp_tap_handler(struct ctx *c, uint8_t pif, * udp_sock_init() - Initialise listening sockets for a given port * @c: Execution context * @ns: In pasta mode, if set, bind with loopback address in namespace - * @af: Address family to select a specific IP version, or AF_UNSPEC * @addr: Pointer to address for binding, NULL if not configured * @ifname: Name of interface to bind to, NULL if not configured * @port: Port, host order * * Return: 0 on (partial) success, negative error code on (complete) failure */ -int udp_sock_init(const struct ctx *c, int ns, sa_family_t af, - const void *addr, const char *ifname, in_port_t port) +int udp_sock_init(const struct ctx *c, int ns, const union inany_addr *addr, + const char *ifname, in_port_t port) { - union udp_epoll_ref uref = { .splice = (c->mode == MODE_PASTA), - .orig = true, .port = port }; - int s, r4 = FD_REF_MAX + 1, r6 = FD_REF_MAX + 1; - - if (ns) - uref.pif = PIF_SPLICE; - else - uref.pif = PIF_HOST; - - if ((af == AF_INET || af == AF_UNSPEC) && c->ifi4) { - uref.v6 = 0; + union udp_listen_epoll_ref uref = { + .pif = ns ? PIF_SPLICE : PIF_HOST, + .port = port, + }; + int r4 = FD_REF_MAX + 1, r6 = FD_REF_MAX + 1; + + ASSERT(!c->no_udp); + + if (!addr && c->ifi4 && c->ifi6 && !ns) { + int s; + + /* Attempt to get a dual stack socket */ + s = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, PIF_HOST, + NULL, ifname, port, uref.u32); + udp_splice_init[V4][port] = s < 0 ? -1 : s; + udp_splice_init[V6][port] = s < 0 ? -1 : s; + if (IN_INTERVAL(0, FD_REF_MAX, s)) + return 0; + } + if ((!addr || inany_v4(addr)) && c->ifi4) { if (!ns) { - r4 = s = sock_l4(c, AF_INET, IPPROTO_UDP, addr, - ifname, port, uref.u32); + r4 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, PIF_HOST, + addr ? addr : &inany_any4, ifname, + port, uref.u32); - udp_tap_map[V4][uref.port].sock = s < 0 ? -1 : s; - udp_splice_init[V4][port].sock = s < 0 ? -1 : s; + udp_splice_init[V4][port] = r4 < 0 ? -1 : r4; } else { - r4 = s = sock_l4(c, AF_INET, IPPROTO_UDP, - &in4addr_loopback, - ifname, port, uref.u32); - udp_splice_ns[V4][port].sock = s < 0 ? -1 : s; + r4 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, PIF_SPLICE, + &inany_loopback4, ifname, + port, uref.u32); + udp_splice_ns[V4][port] = r4 < 0 ? -1 : r4; } } - if ((af == AF_INET6 || af == AF_UNSPEC) && c->ifi6) { - uref.v6 = 1; - + if ((!addr || !inany_v4(addr)) && c->ifi6) { if (!ns) { - r6 = s = sock_l4(c, AF_INET6, IPPROTO_UDP, addr, - ifname, port, uref.u32); + r6 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, PIF_HOST, + addr ? addr : &inany_any6, ifname, + port, uref.u32); - udp_tap_map[V6][uref.port].sock = s < 0 ? -1 : s; - udp_splice_init[V6][port].sock = s < 0 ? -1 : s; + udp_splice_init[V6][port] = r6 < 0 ? -1 : r6; } else { - r6 = s = sock_l4(c, AF_INET6, IPPROTO_UDP, - &in6addr_loopback, - ifname, port, uref.u32); - udp_splice_ns[V6][port].sock = s < 0 ? -1 : s; + r6 = pif_sock_l4(c, EPOLL_TYPE_UDP_LISTEN, PIF_SPLICE, + &inany_loopback6, ifname, + port, uref.u32); + udp_splice_ns[V6][port] = r6 < 0 ? -1 : r6; } } @@ -1067,73 +895,15 @@ static void udp_splice_iov_init(void) int i; for (i = 0; i < UDP_MAX_FRAMES; i++) { - struct msghdr *mh4 = &udp4_mh_splice[i].msg_hdr; - struct msghdr *mh6 = &udp6_mh_splice[i].msg_hdr; - - mh4->msg_name = &udp4_localname; - mh4->msg_namelen = sizeof(udp4_localname); + struct msghdr *mh = &udp_mh_splice[i].msg_hdr; - mh6->msg_name = &udp6_localname; - mh6->msg_namelen = sizeof(udp6_localname); + mh->msg_name = &udp_splice_to; + mh->msg_namelen = sizeof(udp_splice_to); - udp4_iov_splice[i].iov_base = udp4_l2_buf[i].data; - udp6_iov_splice[i].iov_base = udp6_l2_buf[i].data; + udp_iov_splice[i].iov_base = udp_payload[i].data; - mh4->msg_iov = &udp4_iov_splice[i]; - mh6->msg_iov = &udp6_iov_splice[i]; - mh4->msg_iovlen = mh6->msg_iovlen = 1; - } -} - -/** - * udp_timer_one() - Handler for timed events on one port - * @c: Execution context - * @v6: Set for IPv6 connections - * @type: Socket type - * @port: Port number, host order - * @now: Current timestamp - */ -static void udp_timer_one(struct ctx *c, int v6, enum udp_act_type type, - in_port_t port, const struct timespec *now) -{ - struct udp_splice_port *sp; - struct udp_tap_port *tp; - int *sockp = NULL; - - switch (type) { - case UDP_ACT_TAP: - tp = &udp_tap_map[v6 ? V6 : V4][port]; - - if (now->tv_sec - tp->ts > UDP_CONN_TIMEOUT) { - sockp = &tp->sock; - tp->flags = 0; - } - - break; - case UDP_ACT_SPLICE_INIT: - sp = &udp_splice_init[v6 ? V6 : V4][port]; - - if (now->tv_sec - sp->ts > UDP_CONN_TIMEOUT) - sockp = &sp->sock; - - break; - case UDP_ACT_SPLICE_NS: - sp = &udp_splice_ns[v6 ? V6 : V4][port]; - - if (now->tv_sec - sp->ts > UDP_CONN_TIMEOUT) - sockp = &sp->sock; - - break; - default: - return; - } - - if (sockp && *sockp >= 0) { - int s = *sockp; - *sockp = -1; - epoll_ctl(c->epollfd, EPOLL_CTL_DEL, s, NULL); - close(s); - bitmap_clear(udp_act[v6 ? V6 : V4][type], port); + mh->msg_iov = &udp_iov_splice[i]; + mh->msg_iovlen = 1; } } @@ -1146,24 +916,23 @@ static void udp_timer_one(struct ctx *c, int v6, enum udp_act_type type, */ static void udp_port_rebind(struct ctx *c, bool outbound) { + int (*socks)[NUM_PORTS] = outbound ? udp_splice_ns : udp_splice_init; const uint8_t *fmap - = outbound ? c->udp.fwd_out.f.map : c->udp.fwd_in.f.map; + = outbound ? c->udp.fwd_out.map : c->udp.fwd_in.map; const uint8_t *rmap - = outbound ? c->udp.fwd_in.f.map : c->udp.fwd_out.f.map; - struct udp_splice_port (*socks)[NUM_PORTS] - = outbound ? udp_splice_ns : udp_splice_init; + = outbound ? c->udp.fwd_in.map : c->udp.fwd_out.map; unsigned port; for (port = 0; port < NUM_PORTS; port++) { if (!bitmap_isset(fmap, port)) { - if (socks[V4][port].sock >= 0) { - close(socks[V4][port].sock); - socks[V4][port].sock = -1; + if (socks[V4][port] >= 0) { + close(socks[V4][port]); + socks[V4][port] = -1; } - if (socks[V6][port].sock >= 0) { - close(socks[V6][port].sock); - socks[V6][port].sock = -1; + if (socks[V6][port] >= 0) { + close(socks[V6][port]); + socks[V6][port] = -1; } continue; @@ -1173,9 +942,9 @@ static void udp_port_rebind(struct ctx *c, bool outbound) if (bitmap_isset(rmap, port)) continue; - if ((c->ifi4 && socks[V4][port].sock == -1) || - (c->ifi6 && socks[V6][port].sock == -1)) - udp_sock_init(c, outbound, AF_UNSPEC, NULL, NULL, port); + if ((c->ifi4 && socks[V4][port] == -1) || + (c->ifi6 && socks[V6][port] == -1)) + udp_sock_init(c, outbound, NULL, NULL, port); } } @@ -1204,43 +973,23 @@ static int udp_port_rebind_outbound(void *arg) */ void udp_timer(struct ctx *c, const struct timespec *now) { - int n, t, v6 = 0; - unsigned int i; - long *word, tmp; + (void)now; + + ASSERT(!c->no_udp); if (c->mode == MODE_PASTA) { - if (c->udp.fwd_out.f.mode == FWD_AUTO) { - fwd_scan_ports_udp(&c->udp.fwd_out.f, &c->udp.fwd_in.f, + if (c->udp.fwd_out.mode == FWD_AUTO) { + fwd_scan_ports_udp(&c->udp.fwd_out, &c->udp.fwd_in, &c->tcp.fwd_out, &c->tcp.fwd_in); NS_CALL(udp_port_rebind_outbound, c); } - if (c->udp.fwd_in.f.mode == FWD_AUTO) { - fwd_scan_ports_udp(&c->udp.fwd_in.f, &c->udp.fwd_out.f, + if (c->udp.fwd_in.mode == FWD_AUTO) { + fwd_scan_ports_udp(&c->udp.fwd_in, &c->udp.fwd_out, &c->tcp.fwd_in, &c->tcp.fwd_out); udp_port_rebind(c, false); } } - - if (!c->ifi4) - v6 = 1; -v6: - for (t = 0; t < UDP_ACT_TYPE_MAX; t++) { - word = (long *)udp_act[v6 ? V6 : V4][t]; - for (i = 0; i < ARRAY_SIZE(udp_act[0][0]); - i += sizeof(long), word++) { - tmp = *word; - while ((n = ffsl(tmp))) { - tmp &= ~(1UL << (n - 1)); - udp_timer_one(c, v6, t, i * 8 + n - 1, now); - } - } - } - - if (!v6 && c->ifi6) { - v6 = 1; - goto v6; - } } /** @@ -1251,14 +1000,9 @@ v6: */ int udp_init(struct ctx *c) { - if (c->ifi4) - udp_sock4_iov_init(c); - - if (c->ifi6) - udp_sock6_iov_init(c); + ASSERT(!c->no_udp); - udp_invert_portmap(&c->udp.fwd_in); - udp_invert_portmap(&c->udp.fwd_out); + udp_iov_init(c); if (c->mode == MODE_PASTA) { udp_splice_iov_init(); @@ -9,58 +9,43 @@ #define UDP_TIMER_INTERVAL 1000 /* ms */ void udp_portmap_clear(void); -void udp_buf_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, - const struct timespec *now); -int udp_tap_handler(struct ctx *c, uint8_t pif, sa_family_t af, - const void *saddr, const void *daddr, +void udp_listen_sock_handler(const struct ctx *c, union epoll_ref ref, + uint32_t events, const struct timespec *now); +void udp_reply_sock_handler(const struct ctx *c, union epoll_ref ref, + uint32_t events, const struct timespec *now); +int udp_tap_handler(const struct ctx *c, uint8_t pif, + sa_family_t af, const void *saddr, const void *daddr, const struct pool *p, int idx, const struct timespec *now); -int udp_sock_init(const struct ctx *c, int ns, sa_family_t af, - const void *addr, const char *ifname, in_port_t port); +int udp_sock_init(const struct ctx *c, int ns, const union inany_addr *addr, + const char *ifname, in_port_t port); int udp_init(struct ctx *c); void udp_timer(struct ctx *c, const struct timespec *now); void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s); /** - * union udp_epoll_ref - epoll reference portion for TCP connections + * union udp_listen_epoll_ref - epoll reference for "listening" UDP sockets * @port: Source port for connected sockets, bound port otherwise * @pif: pif for this socket - * @bound: Set if this file descriptor is a bound socket - * @splice: Set if descriptor packets to be "spliced" - * @orig: Set if a spliced socket which can originate "connections" - * @v6: Set for IPv6 sockets or connections * @u32: Opaque u32 value of reference */ -union udp_epoll_ref { +union udp_listen_epoll_ref { struct { in_port_t port; uint8_t pif; - bool splice:1, - orig:1, - v6:1; }; uint32_t u32; }; /** - * udp_fwd_ports - UDP specific port forwarding configuration - * @f: Generic forwarding configuration - * @rdelta: Reversed delta map to translate source ports on return packets - */ -struct udp_fwd_ports { - struct fwd_ports f; - in_port_t rdelta[NUM_PORTS]; -}; - -/** * struct udp_ctx - Execution context for UDP * @fwd_in: Port forwarding configuration for inbound packets * @fwd_out: Port forwarding configuration for outbound packets * @timer_run: Timestamp of most recent timer run */ struct udp_ctx { - struct udp_fwd_ports fwd_in; - struct udp_fwd_ports fwd_out; + struct fwd_ports fwd_in; + struct fwd_ports fwd_out; struct timespec timer_run; }; diff --git a/udp_flow.c b/udp_flow.c new file mode 100644 index 0000000..b81be2c --- /dev/null +++ b/udp_flow.c @@ -0,0 +1,291 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * Copyright Red Hat + * Author: David Gibson <david@gibson.dropbear.id.au> + * + * UDP flow tracking functions + */ + +#include <errno.h> +#include <fcntl.h> +#include <sys/uio.h> +#include <unistd.h> + +#include "util.h" +#include "passt.h" +#include "flow_table.h" + +#define UDP_CONN_TIMEOUT 180 /* s, timeout for ephemeral or local bind */ + +/** + * udp_at_sidx() - Get UDP specific flow at given sidx + * @sidx: Flow and side to retrieve + * + * Return: UDP specific flow at @sidx, or NULL of @sidx is invalid. Asserts if + * the flow at @sidx is not FLOW_UDP. + */ +struct udp_flow *udp_at_sidx(flow_sidx_t sidx) +{ + union flow *flow = flow_at_sidx(sidx); + + if (!flow) + return NULL; + + ASSERT(flow->f.type == FLOW_UDP); + return &flow->udp; +} + +/* + * udp_flow_close() - Close and clean up UDP flow + * @c: Execution context + * @uflow: UDP flow + */ +void udp_flow_close(const struct ctx *c, struct udp_flow *uflow) +{ + if (uflow->closed) + return; /* Nothing to do */ + + if (uflow->s[INISIDE] >= 0) { + /* The listening socket needs to stay in epoll */ + close(uflow->s[INISIDE]); + uflow->s[INISIDE] = -1; + } + + if (uflow->s[TGTSIDE] >= 0) { + /* But the flow specific one needs to be removed */ + epoll_ctl(c->epollfd, EPOLL_CTL_DEL, uflow->s[TGTSIDE], NULL); + close(uflow->s[TGTSIDE]); + uflow->s[TGTSIDE] = -1; + } + flow_hash_remove(c, FLOW_SIDX(uflow, INISIDE)); + if (!pif_is_socket(uflow->f.pif[TGTSIDE])) + flow_hash_remove(c, FLOW_SIDX(uflow, TGTSIDE)); + + uflow->closed = true; +} + +/** + * udp_flow_new() - Common setup for a new UDP flow + * @c: Execution context + * @flow: Initiated flow + * @s_ini: Initiating socket (or -1) + * @now: Timestamp + * + * Return: UDP specific flow, if successful, NULL on failure + */ +static flow_sidx_t udp_flow_new(const struct ctx *c, union flow *flow, + int s_ini, const struct timespec *now) +{ + const struct flowside *ini = &flow->f.side[INISIDE]; + struct udp_flow *uflow = NULL; + const struct flowside *tgt; + uint8_t tgtpif; + + if (!inany_is_unicast(&ini->eaddr) || ini->eport == 0) { + flow_trace(flow, "Invalid endpoint to initiate UDP flow"); + goto cancel; + } + + if (!(tgt = flow_target(c, flow, IPPROTO_UDP))) + goto cancel; + tgtpif = flow->f.pif[TGTSIDE]; + + uflow = FLOW_SET_TYPE(flow, FLOW_UDP, udp); + uflow->ts = now->tv_sec; + uflow->s[INISIDE] = uflow->s[TGTSIDE] = -1; + + if (s_ini >= 0) { + /* When using auto port-scanning the listening port could go + * away, so we need to duplicate the socket + */ + uflow->s[INISIDE] = fcntl(s_ini, F_DUPFD_CLOEXEC, 0); + if (uflow->s[INISIDE] < 0) { + flow_err(uflow, + "Couldn't duplicate listening socket: %s", + strerror(errno)); + goto cancel; + } + } + + if (pif_is_socket(tgtpif)) { + struct mmsghdr discard[UIO_MAXIOV] = { 0 }; + union { + flow_sidx_t sidx; + uint32_t data; + } fref = { + .sidx = FLOW_SIDX(flow, TGTSIDE), + }; + int rc; + + uflow->s[TGTSIDE] = flowside_sock_l4(c, EPOLL_TYPE_UDP_REPLY, + tgtpif, tgt, fref.data); + if (uflow->s[TGTSIDE] < 0) { + flow_dbg(uflow, + "Couldn't open socket for spliced flow: %s", + strerror(errno)); + goto cancel; + } + + if (flowside_connect(c, uflow->s[TGTSIDE], tgtpif, tgt) < 0) { + flow_dbg(uflow, + "Couldn't connect flow socket: %s", + strerror(errno)); + goto cancel; + } + + /* It's possible, if unlikely, that we could receive some + * unrelated packets in between the bind() and connect() of this + * socket. For now we just discard these. We could consider + * trying to redirect these to an appropriate handler, if we + * need to. + */ + rc = recvmmsg(uflow->s[TGTSIDE], discard, ARRAY_SIZE(discard), + MSG_DONTWAIT, NULL); + if (rc >= ARRAY_SIZE(discard)) { + flow_dbg(uflow, + "Too many (%d) spurious reply datagrams", rc); + goto cancel; + } else if (rc > 0) { + flow_trace(uflow, + "Discarded %d spurious reply datagrams", rc); + } else if (errno != EAGAIN) { + flow_err(uflow, + "Unexpected error discarding datagrams: %s", + strerror(errno)); + } + } + + flow_hash_insert(c, FLOW_SIDX(uflow, INISIDE)); + + /* If the target side is a socket, it will be a reply socket that knows + * its own flowside. But if it's tap, then we need to look it up by + * hash. + */ + if (!pif_is_socket(tgtpif)) + flow_hash_insert(c, FLOW_SIDX(uflow, TGTSIDE)); + FLOW_ACTIVATE(uflow); + + return FLOW_SIDX(uflow, TGTSIDE); + +cancel: + if (uflow) + udp_flow_close(c, uflow); + flow_alloc_cancel(flow); + return FLOW_SIDX_NONE; +} + +/** + * udp_flow_from_sock() - Find or create UDP flow for "listening" socket + * @c: Execution context + * @ref: epoll reference of the receiving socket + * @s_in: Source socket address, filled in by recvmmsg() + * @now: Timestamp + * + * #syscalls fcntl arm:fcntl64 ppc64:fcntl64 i686:fcntl64 + * + * Return: sidx for the destination side of the flow for this packet, or + * FLOW_SIDX_NONE if we couldn't find or create a flow. + */ +flow_sidx_t udp_flow_from_sock(const struct ctx *c, union epoll_ref ref, + const union sockaddr_inany *s_in, + const struct timespec *now) +{ + struct udp_flow *uflow; + union flow *flow; + flow_sidx_t sidx; + + ASSERT(ref.type == EPOLL_TYPE_UDP_LISTEN); + + sidx = flow_lookup_sa(c, IPPROTO_UDP, ref.udp.pif, s_in, ref.udp.port); + if ((uflow = udp_at_sidx(sidx))) { + uflow->ts = now->tv_sec; + return flow_sidx_opposite(sidx); + } + + if (!(flow = flow_alloc())) { + char sastr[SOCKADDR_STRLEN]; + + debug("Couldn't allocate flow for UDP datagram from %s %s", + pif_name(ref.udp.pif), + sockaddr_ntop(s_in, sastr, sizeof(sastr))); + return FLOW_SIDX_NONE; + } + + flow_initiate_sa(flow, ref.udp.pif, s_in, ref.udp.port); + return udp_flow_new(c, flow, ref.fd, now); +} + +/** + * udp_flow_from_tap() - Find or create UDP flow for tap packets + * @c: Execution context + * @pif: pif on which the packet is arriving + * @af: Address family, AF_INET or AF_INET6 + * @saddr: Source address on guest side + * @daddr: Destination address guest side + * @srcport: Source port on guest side + * @dstport: Destination port on guest side + * + * Return: sidx for the destination side of the flow for this packet, or + * FLOW_SIDX_NONE if we couldn't find or create a flow. + */ +flow_sidx_t udp_flow_from_tap(const struct ctx *c, + uint8_t pif, sa_family_t af, + const void *saddr, const void *daddr, + in_port_t srcport, in_port_t dstport, + const struct timespec *now) +{ + struct udp_flow *uflow; + union flow *flow; + flow_sidx_t sidx; + + ASSERT(pif == PIF_TAP); + + sidx = flow_lookup_af(c, IPPROTO_UDP, pif, af, saddr, daddr, + srcport, dstport); + if ((uflow = udp_at_sidx(sidx))) { + uflow->ts = now->tv_sec; + return flow_sidx_opposite(sidx); + } + + if (!(flow = flow_alloc())) { + char sstr[INET6_ADDRSTRLEN], dstr[INET6_ADDRSTRLEN]; + + debug("Couldn't allocate flow for UDP datagram from %s %s:%hu -> %s:%hu", + pif_name(pif), + inet_ntop(af, saddr, sstr, sizeof(sstr)), srcport, + inet_ntop(af, daddr, dstr, sizeof(dstr)), dstport); + return FLOW_SIDX_NONE; + } + + flow_initiate_af(flow, PIF_TAP, af, saddr, srcport, daddr, dstport); + + return udp_flow_new(c, flow, -1, now); +} + +/** + * udp_flow_defer() - Deferred per-flow handling (clean up aborted flows) + * @uflow: Flow to handle + * + * Return: true if the connection is ready to free, false otherwise + */ +bool udp_flow_defer(const struct udp_flow *uflow) +{ + return uflow->closed; +} + +/** + * udp_flow_timer() - Handler for timed events related to a given flow + * @c: Execution context + * @uflow: UDP flow + * @now: Current timestamp + * + * Return: true if the flow is ready to free, false otherwise + */ +bool udp_flow_timer(const struct ctx *c, struct udp_flow *uflow, + const struct timespec *now) +{ + if (now->tv_sec - uflow->ts <= UDP_CONN_TIMEOUT) + return false; + + udp_flow_close(c, uflow); + return true; +} diff --git a/udp_flow.h b/udp_flow.h new file mode 100644 index 0000000..9a1b059 --- /dev/null +++ b/udp_flow.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * Copyright Red Hat + * Author: David Gibson <david@gibson.dropbear.id.au> + * + * UDP flow tracking data structures + */ +#ifndef UDP_FLOW_H +#define UDP_FLOW_H + +/** + * struct udp - Descriptor for a flow of UDP packets + * @f: Generic flow information + * @closed: Flow is already closed + * @ts: Activity timestamp + * @s: Socket fd (or -1) for each side of the flow + */ +struct udp_flow { + /* Must be first element */ + struct flow_common f; + + bool closed :1; + time_t ts; + int s[SIDES]; +}; + +struct udp_flow *udp_at_sidx(flow_sidx_t sidx); +flow_sidx_t udp_flow_from_sock(const struct ctx *c, union epoll_ref ref, + const union sockaddr_inany *s_in, + const struct timespec *now); +flow_sidx_t udp_flow_from_tap(const struct ctx *c, + uint8_t pif, sa_family_t af, + const void *saddr, const void *daddr, + in_port_t srcport, in_port_t dstport, + const struct timespec *now); +void udp_flow_close(const struct ctx *c, struct udp_flow *uflow); +bool udp_flow_defer(const struct udp_flow *uflow); +bool udp_flow_timer(const struct ctx *c, struct udp_flow *uflow, + const struct timespec *now); + +#endif /* UDP_FLOW_H */ diff --git a/udp_internal.h b/udp_internal.h index a09f3c6..cc80e30 100644 --- a/udp_internal.h +++ b/udp_internal.h @@ -6,16 +6,29 @@ #ifndef UDP_INTERNAL_H #define UDP_INTERNAL_H -#define UDP_CONN_TIMEOUT 180 /* s, timeout for ephemeral or local bind */ +#include "tap.h" /* needed by udp_meta_t */ + #define UDP_MAX_FRAMES 32 /* max # of frames to receive at once */ -extern struct sockaddr_in udp4_localname; -extern struct sockaddr_in6 udp6_localname; +/** + * struct udp_payload_t - UDP header and data for inbound messages + * @uh: UDP header + * @data: UDP data + */ +struct udp_payload_t { + struct udphdr uh; + char data[USHRT_MAX - sizeof(struct udphdr)]; +#ifdef __AVX2__ +} __attribute__ ((packed, aligned(32))); +#else +} __attribute__ ((packed, aligned(__alignof__(unsigned int)))); +#endif -size_t udp_update_hdr4(const struct ctx *c, struct iphdr *iph, - size_t data_len, struct sockaddr_in *s_in, - in_port_t dstport, const struct timespec *now); -size_t udp_update_hdr6(const struct ctx *c, struct ipv6hdr *ip6h, - size_t data_len, struct sockaddr_in6 *s_in6, - in_port_t dstport, const struct timespec *now); +size_t udp_update_hdr4(struct iphdr *ip4h, struct udp_payload_t *bp, + const struct flowside *toside, size_t dlen, + bool no_udp_csum); +size_t udp_update_hdr6(struct ipv6hdr *ip6h, struct udp_payload_t *bp, + const struct flowside *toside, size_t dlen, + bool no_udp_csum); +int udp_sock_errs(const struct ctx *c, int s, uint32_t events); #endif /* UDP_INTERNAL_H */ @@ -1,6 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-or-later +/* udp_vu.c - UDP L2 vhost-user management functions + * + * Copyright Red Hat + * Author: Laurent Vivier <lvivier@redhat.com> + */ #include <unistd.h> +#include <assert.h> #include <net/ethernet.h> #include <net/if.h> #include <netinet/in.h> @@ -14,205 +20,317 @@ #include "checksum.h" #include "util.h" #include "ip.h" +#include "siphash.h" +#include "inany.h" #include "passt.h" #include "pcap.h" #include "log.h" #include "vhost_user.h" #include "udp_internal.h" +#include "flow.h" +#include "flow_table.h" +#include "udp_flow.h" #include "udp_vu.h" +#include "vu_common.h" -/* vhost-user */ -static const struct virtio_net_hdr vu_header = { - .flags = VIRTIO_NET_HDR_F_DATA_VALID, - .gso_type = VIRTIO_NET_HDR_GSO_NONE, -}; - -static unsigned char buffer[65536]; static struct iovec iov_vu [VIRTQUEUE_MAX_SIZE]; -static unsigned int indexes [VIRTQUEUE_MAX_SIZE]; +static struct vu_virtq_element elem [VIRTQUEUE_MAX_SIZE]; + +/** + * udp_vu_hdrlen() - return the size of the header in level 2 frame (UDP) + * @v6: Set for IPv6 packet + * + * Return: Return the size of the header + */ +static size_t udp_vu_hdrlen(bool v6) +{ + size_t hdrlen; + + hdrlen = sizeof(struct virtio_net_hdr_mrg_rxbuf) + + sizeof(struct ethhdr) + sizeof(struct udphdr); -void udp_vu_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, - const struct timespec *now) + if (v6) + hdrlen += sizeof(struct ipv6hdr); + else + hdrlen += sizeof(struct iphdr); + + return hdrlen; +} + +static int udp_vu_sock_init(int s, union sockaddr_inany *s_in) { - VuDev *vdev = (VuDev *)&c->vdev; - VuVirtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE]; - size_t l2_hdrlen, vnet_hdrlen, fillsize; - ssize_t data_len; - in_port_t dstport = ref.udp.port; - bool has_mrg_rxbuf, v6 = ref.udp.v6; - struct msghdr msg; - int i, iov_count, iov_used, virtqueue_max; - - if (c->no_udp || !(events & EPOLLIN)) - return; + struct msghdr msg = { + .msg_name = s_in, + .msg_namelen = sizeof(union sockaddr_inany), + }; - has_mrg_rxbuf = vu_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF); - if (has_mrg_rxbuf) { - vnet_hdrlen = sizeof(struct virtio_net_hdr_mrg_rxbuf); - virtqueue_max = VIRTQUEUE_MAX_SIZE; - } else { - vnet_hdrlen = sizeof(struct virtio_net_hdr); - virtqueue_max = 1; - } - l2_hdrlen = vnet_hdrlen + sizeof(struct ethhdr) + sizeof(struct udphdr); + return recvmsg(s, &msg, MSG_PEEK | MSG_DONTWAIT); +} + +/** + * udp_vu_sock_recv() - Receive datagrams from socket into vhost-user buffers + * @c: Execution context + * @s: Socket to receive from + * @events: epoll events bitmap + * @v6: Set for IPv6 connections + * @dlen: Size of received data (output) + * + * Return: Number of iov entries used to store the datagram + */ +static int udp_vu_sock_recv(const struct ctx *c, int s, uint32_t events, + bool v6, ssize_t *dlen) +{ + struct vu_dev *vdev = c->vdev; + struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE]; + int iov_cnt, idx, iov_used; + struct msghdr msg = { 0 }; + size_t off, hdrlen; - if (v6) { - l2_hdrlen += sizeof(struct ipv6hdr); + ASSERT(!c->no_udp); - udp6_localname.sin6_port = htons(dstport); - msg.msg_name = &udp6_localname; - msg.msg_namelen = sizeof(udp6_localname); - } else { - l2_hdrlen += sizeof(struct iphdr); + if (!(events & EPOLLIN)) + return 0; + + /* compute L2 header length */ + hdrlen = udp_vu_hdrlen(v6); + + vu_init_elem(elem, iov_vu, VIRTQUEUE_MAX_SIZE); + + iov_cnt = vu_collect(vdev, vq, elem, VIRTQUEUE_MAX_SIZE, + IP_MAX_MTU - sizeof(struct udphdr) + hdrlen, + NULL); + if (iov_cnt == 0) + return 0; + + /* reserve space for the headers */ + iov_vu[0].iov_base = (char *)iov_vu[0].iov_base + hdrlen; + iov_vu[0].iov_len -= hdrlen; - udp4_localname.sin_port = htons(dstport); - msg.msg_name = &udp4_localname; - msg.msg_namelen = sizeof(udp4_localname); + /* read data from the socket */ + msg.msg_iov = iov_vu; + msg.msg_iovlen = iov_cnt; + + *dlen = recvmsg(s, &msg, 0); + if (*dlen < 0) { + vu_queue_rewind(vq, iov_cnt); + return 0; } - msg.msg_control = NULL; - msg.msg_controllen = 0; - msg.msg_flags = 0; + /* restore the pointer to the headers address */ + iov_vu[0].iov_base = (char *)iov_vu[0].iov_base - hdrlen; + iov_vu[0].iov_len += hdrlen; - for (i = 0; i < UDP_MAX_FRAMES; i++) { - struct virtio_net_hdr_mrg_rxbuf *vh; - struct ethhdr *eh; - char *base; - size_t size; - - fillsize = USHRT_MAX; - iov_count = 0; - while (fillsize && iov_count < virtqueue_max) { - VuVirtqElement *elem; - - elem = vu_queue_pop(vdev, vq, sizeof(VuVirtqElement), buffer); - if (!elem) - break; - - if (elem->in_num < 1) { - err("virtio-net receive queue contains no in buffers"); - vu_queue_rewind(vdev, vq, iov_count); - return; - } - ASSERT(elem->in_num == 1); - ASSERT(elem->in_sg[0].iov_len >= l2_hdrlen); + /* count the numbers of buffer filled by recvmsg() */ + idx = iov_skip_bytes(iov_vu, iov_cnt, *dlen + hdrlen, &off); - indexes[iov_count] = elem->index; - if (iov_count == 0) { - iov_vu[0].iov_base = (char *)elem->in_sg[0].iov_base + l2_hdrlen; - iov_vu[0].iov_len = elem->in_sg[0].iov_len - l2_hdrlen; - } else { - iov_vu[iov_count].iov_base = elem->in_sg[0].iov_base; - iov_vu[iov_count].iov_len = elem->in_sg[0].iov_len; - } + /* adjust last iov length */ + if (idx < iov_cnt) + iov_vu[idx].iov_len = off; + iov_used = idx + !!off; - if (iov_vu[iov_count].iov_len > fillsize) - iov_vu[iov_count].iov_len = fillsize; + vu_set_vnethdr(vdev, iov_vu[0].iov_base, iov_used); - fillsize -= iov_vu[iov_count].iov_len; + /* release unused buffers */ + vu_queue_rewind(vq, iov_cnt - iov_used); - iov_count++; - } - if (iov_count == 0) - break; + return iov_used; +} - msg.msg_iov = iov_vu; - msg.msg_iovlen = iov_count; +/** + * udp_vu_prepare() - Prepare the packet header + * @c: Execution context + * @toside: Address information for one side of the flow + * @dlen: Packet data length + * + * Return: Layer-4 length + */ +static size_t udp_vu_prepare(const struct ctx *c, + const struct flowside *toside, ssize_t dlen) +{ + struct ethhdr *eh; + size_t l4len; - data_len = recvmsg(ref.fd, &msg, 0); - if (data_len < 0) { - vu_queue_rewind(vdev, vq, iov_count); - return; - } + /* ethernet header */ + eh = vu_eth(iov_vu[0].iov_base); - iov_used = 0; - size = data_len; - while (size) { - if (iov_vu[iov_used].iov_len > size) - iov_vu[iov_used].iov_len = size; + memcpy(eh->h_dest, c->guest_mac, sizeof(eh->h_dest)); + memcpy(eh->h_source, c->our_tap_mac, sizeof(eh->h_source)); - size -= iov_vu[iov_used].iov_len; - iov_used++; - } + /* initialize header */ + if (inany_v4(&toside->eaddr) && inany_v4(&toside->oaddr)) { + struct iphdr *iph = vu_ip(iov_vu[0].iov_base); + struct udp_payload_t *bp = vu_payloadv4(iov_vu[0].iov_base); - base = (char *)iov_vu[0].iov_base - l2_hdrlen; - size = iov_vu[0].iov_len + l2_hdrlen; + eh->h_proto = htons(ETH_P_IP); - /* release unused buffers */ - vu_queue_rewind(vdev, vq, iov_count - iov_used); + *iph = (struct iphdr)L2_BUF_IP4_INIT(IPPROTO_UDP); - /* vnet_header */ - vh = (struct virtio_net_hdr_mrg_rxbuf *)base; - vh->hdr = vu_header; - if (has_mrg_rxbuf) - vh->num_buffers = htole16(iov_used); + l4len = udp_update_hdr4(iph, bp, toside, dlen, true); + } else { + struct ipv6hdr *ip6h = vu_ip(iov_vu[0].iov_base); + struct udp_payload_t *bp = vu_payloadv6(iov_vu[0].iov_base); - /* ethernet header */ - eh = (struct ethhdr *)(base + vnet_hdrlen); + eh->h_proto = htons(ETH_P_IPV6); - memcpy(eh->h_dest, c->mac_guest, sizeof(eh->h_dest)); - memcpy(eh->h_source, c->mac, sizeof(eh->h_source)); + *ip6h = (struct ipv6hdr)L2_BUF_IP6_INIT(IPPROTO_UDP); - /* initialize header */ - if (v6) { - struct ipv6hdr *ip6h = (struct ipv6hdr *)(eh + 1); - struct udphdr *uh = (struct udphdr *)(ip6h + 1); - uint32_t sum; + l4len = udp_update_hdr6(ip6h, bp, toside, dlen, true); + } - eh->h_proto = htons(ETH_P_IPV6); + return l4len; +} - *ip6h = (struct ipv6hdr)L2_BUF_IP6_INIT(IPPROTO_UDP); +/** + * udp_vu_csum() - Calculate and set checksum for a UDP packet + * @toside: ddress information for one side of the flow + * @l4len: IPv4 Payload length + * @iov_used: Length of the array + */ +static void udp_vu_csum(const struct flowside *toside, int iov_used) +{ + const struct in_addr *src4 = inany_v4(&toside->oaddr); + const struct in_addr *dst4 = inany_v4(&toside->eaddr); + char *base = iov_vu[0].iov_base; + struct udp_payload_t *bp; + + if (src4 && dst4) { + bp = vu_payloadv4(base); + csum_udp4(&bp->uh, *src4, *dst4, iov_vu, iov_used, + (char *)&bp->data - base); + } else { + bp = vu_payloadv6(base); + csum_udp6(&bp->uh, &toside->oaddr.a6, &toside->eaddr.a6, + iov_vu, iov_used, (char *)&bp->data - base); + } +} - udp_update_hdr6(c, ip6h, data_len, &udp6_localname, - dstport, now); - if (*c->pcap) { - sum = proto_ipv6_header_psum(ip6h->payload_len, - IPPROTO_UDP, - &ip6h->saddr, - &ip6h->daddr); +/** + * udp_vu_listen_sock_handler() - Handle new data from socket + * @c: Execution context + * @ref: epoll reference + * @events: epoll events bitmap + * @now: Current timestamp + */ +void udp_vu_listen_sock_handler(const struct ctx *c, union epoll_ref ref, + uint32_t events, const struct timespec *now) +{ + struct vu_dev *vdev = c->vdev; + struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE]; + int i; - iov_vu[0].iov_base = uh; - iov_vu[0].iov_len = size - l2_hdrlen + sizeof(*uh); - uh->check = csum_iov(iov_vu, iov_used, sum); - } - } else { - struct iphdr *iph = (struct iphdr *)(eh + 1); - struct udphdr *uh = (struct udphdr *)(iph + 1); - uint32_t sum; - - eh->h_proto = htons(ETH_P_IP); - - *iph = (struct iphdr)L2_BUF_IP4_INIT(IPPROTO_UDP); - - udp_update_hdr4(c, iph, data_len, &udp4_localname, - dstport, now); - if (*c->pcap) { - sum = proto_ipv4_header_psum(iph->tot_len, - IPPROTO_UDP, - (struct in_addr){ .s_addr = iph->saddr }, - (struct in_addr){ .s_addr = iph->daddr }); - - iov_vu[0].iov_base = uh; - iov_vu[0].iov_len = size - l2_hdrlen + sizeof(*uh); - uh->check = csum_iov(iov_vu, iov_used, sum); + if (udp_sock_errs(c, ref.fd, events) < 0) { + err("UDP: Unrecoverable error on listening socket:" + " (%s port %hu)", pif_name(ref.udp.pif), ref.udp.port); + return; + } + + for (i = 0; i < UDP_MAX_FRAMES; i++) { + const struct flowside *toside; + union sockaddr_inany s_in; + flow_sidx_t sidx; + uint8_t pif; + ssize_t dlen; + int iov_used; + bool v6; + + if (udp_vu_sock_init(ref.fd, &s_in) < 0) + break; + + sidx = udp_flow_from_sock(c, ref, &s_in, now); + pif = pif_at_sidx(sidx); + + if (pif != PIF_TAP) { + if (flow_sidx_valid(sidx)) { + flow_sidx_t fromsidx = flow_sidx_opposite(sidx); + struct udp_flow *uflow = udp_at_sidx(sidx); + + flow_err(uflow, + "No support for forwarding UDP from %s to %s", + pif_name(pif_at_sidx(fromsidx)), + pif_name(pif)); + } else { + debug("Discarding 1 datagram without flow"); } + + continue; + } + + toside = flowside_at_sidx(sidx); + + v6 = !(inany_v4(&toside->eaddr) && inany_v4(&toside->oaddr)); + + iov_used = udp_vu_sock_recv(c, ref.fd, events, v6, &dlen); + if (iov_used <= 0) + break; + + udp_vu_prepare(c, toside, dlen); + if (*c->pcap) { + udp_vu_csum(toside, iov_used); + pcap_iov(iov_vu, iov_used, + sizeof(struct virtio_net_hdr_mrg_rxbuf)); } + vu_flush(vdev, vq, elem, iov_used); + } +} + +/** + * udp_vu_reply_sock_handler() - Handle new data from flow specific socket + * @c: Execution context + * @ref: epoll reference + * @events: epoll events bitmap + * @now: Current timestamp + */ +void udp_vu_reply_sock_handler(const struct ctx *c, union epoll_ref ref, + uint32_t events, const struct timespec *now) +{ + flow_sidx_t tosidx = flow_sidx_opposite(ref.flowside); + const struct flowside *toside = flowside_at_sidx(tosidx); + struct udp_flow *uflow = udp_at_sidx(ref.flowside); + int from_s = uflow->s[ref.flowside.sidei]; + struct vu_dev *vdev = c->vdev; + struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE]; + int i; + + ASSERT(!c->no_udp); + + if (udp_sock_errs(c, from_s, events) < 0) { + flow_err(uflow, "Unrecoverable error on reply socket"); + flow_err_details(uflow); + udp_flow_close(c, uflow); + return; + } + + for (i = 0; i < UDP_MAX_FRAMES; i++) { + uint8_t topif = pif_at_sidx(tosidx); + ssize_t dlen; + int iov_used; + bool v6; - /* set iov for pcap logging */ - iov_vu[0].iov_base = base + vnet_hdrlen; - iov_vu[0].iov_len = size - vnet_hdrlen; - pcap_iov(iov_vu, iov_used); + ASSERT(uflow); - /* set iov_len for vu_queue_fill_by_index(); */ - iov_vu[0].iov_base = base; - iov_vu[0].iov_len = size; + if (topif != PIF_TAP) { + uint8_t frompif = pif_at_sidx(ref.flowside); - /* send packets */ - for (i = 0; i < iov_used; i++) - vu_queue_fill_by_index(vdev, vq, indexes[i], - iov_vu[i].iov_len, i); + flow_err(uflow, + "No support for forwarding UDP from %s to %s", + pif_name(frompif), pif_name(topif)); + continue; + } + + v6 = !(inany_v4(&toside->eaddr) && inany_v4(&toside->oaddr)); - vu_queue_flush(vdev, vq, iov_used); - vu_queue_notify(vdev, vq); + iov_used = udp_vu_sock_recv(c, from_s, events, v6, &dlen); + if (iov_used <= 0) + break; + flow_trace(uflow, "Received 1 datagram on reply socket"); + uflow->ts = now->tv_sec; + + udp_vu_prepare(c, toside, dlen); + if (*c->pcap) { + udp_vu_csum(toside, iov_used); + pcap_iov(iov_vu, iov_used, + sizeof(struct virtio_net_hdr_mrg_rxbuf)); + } + vu_flush(vdev, vq, elem, iov_used); } } @@ -1,8 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-or-later +/* Copyright Red Hat + * Author: Laurent Vivier <lvivier@redhat.com> + */ #ifndef UDP_VU_H #define UDP_VU_H -void udp_vu_sock_handler(const struct ctx *c, union epoll_ref ref, - uint32_t events, const struct timespec *now); +void udp_vu_listen_sock_handler(const struct ctx *c, union epoll_ref ref, + uint32_t events, const struct timespec *now); +void udp_vu_reply_sock_handler(const struct ctx *c, union epoll_ref ref, + uint32_t events, const struct timespec *now); #endif /* UDP_VU_H */ @@ -25,73 +25,68 @@ #include <time.h> #include <errno.h> #include <stdbool.h> +#include <linux/errqueue.h> +#include <getopt.h> +#include "linux_dep.h" #include "util.h" #include "iov.h" #include "passt.h" #include "packet.h" #include "log.h" +#ifdef HAS_GETRANDOM +#include <sys/random.h> +#endif /** - * sock_l4() - Create and bind socket for given L4, add to epoll list + * sock_l4_sa() - Create and bind socket to socket address, add to epoll list * @c: Execution context - * @af: Address family, AF_INET or AF_INET6 - * @proto: Protocol number - * @bind_addr: Address for binding, NULL for any + * @type: epoll type + * @sa: Socket address to bind to + * @sl: Length of @sa * @ifname: Interface for binding, NULL for any - * @port: Port, host order + * @v6only: Set IPV6_V6ONLY socket option * @data: epoll reference portion for protocol handlers * * Return: newly created socket, negative error code on failure */ -int sock_l4(const struct ctx *c, sa_family_t af, uint8_t proto, - const void *bind_addr, const char *ifname, uint16_t port, - uint32_t data) +int sock_l4_sa(const struct ctx *c, enum epoll_type type, + const void *sa, socklen_t sl, + const char *ifname, bool v6only, uint32_t data) { - union epoll_ref ref = { .data = data }; - struct sockaddr_in addr4 = { - .sin_family = AF_INET, - .sin_port = htons(port), - { 0 }, { 0 }, - }; - struct sockaddr_in6 addr6 = { - .sin6_family = AF_INET6, - .sin6_port = htons(port), - 0, IN6ADDR_ANY_INIT, 0, - }; - const struct sockaddr *sa; - bool dual_stack = false; - int fd, sl, y = 1, ret; + sa_family_t af = ((const struct sockaddr *)sa)->sa_family; + union epoll_ref ref = { .type = type, .data = data }; + bool freebind = false; struct epoll_event ev; - - switch (proto) { - case IPPROTO_TCP: - ref.type = EPOLL_TYPE_TCP_LISTEN; - break; - case IPPROTO_UDP: - ref.type = EPOLL_TYPE_UDP; + int fd, y = 1, ret; + uint8_t proto; + int socktype; + + switch (type) { + case EPOLL_TYPE_TCP_LISTEN: + proto = IPPROTO_TCP; + socktype = SOCK_STREAM | SOCK_NONBLOCK; + freebind = c->freebind; break; - case IPPROTO_ICMP: - ref.type = EPOLL_TYPE_ICMP; + case EPOLL_TYPE_UDP_LISTEN: + freebind = c->freebind; + /* fallthrough */ + case EPOLL_TYPE_UDP_REPLY: + proto = IPPROTO_UDP; + socktype = SOCK_DGRAM | SOCK_NONBLOCK; break; - case IPPROTO_ICMPV6: - ref.type = EPOLL_TYPE_ICMPV6; + case EPOLL_TYPE_PING: + if (af == AF_INET) + proto = IPPROTO_ICMP; + else + proto = IPPROTO_ICMPV6; + socktype = SOCK_DGRAM | SOCK_NONBLOCK; break; default: - return -EPFNOSUPPORT; /* Not implemented. */ + ASSERT(0); } - if (af == AF_UNSPEC) { - if (!DUAL_STACK_SOCKETS || bind_addr) - return -EINVAL; - dual_stack = true; - af = AF_INET6; - } - - if (proto == IPPROTO_TCP) - fd = socket(af, SOCK_STREAM | SOCK_NONBLOCK, proto); - else - fd = socket(af, SOCK_DGRAM | SOCK_NONBLOCK, proto); + fd = socket(af, socktype, proto); ret = -errno; if (fd < 0) { @@ -106,34 +101,21 @@ int sock_l4(const struct ctx *c, sa_family_t af, uint8_t proto, ref.fd = fd; - if (af == AF_INET) { - if (bind_addr) - addr4.sin_addr = *(struct in_addr *)bind_addr; + if (v6only) + if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &y, sizeof(y))) + debug("Failed to set IPV6_V6ONLY on socket %i", fd); - sa = (const struct sockaddr *)&addr4; - sl = sizeof(addr4); - } else { - if (bind_addr) { - addr6.sin6_addr = *(struct in6_addr *)bind_addr; - - if (!memcmp(bind_addr, &c->ip6.addr_ll, - sizeof(c->ip6.addr_ll))) - addr6.sin6_scope_id = c->ifi6; - } + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &y, sizeof(y))) + debug("Failed to set SO_REUSEADDR on socket %i", fd); - sa = (const struct sockaddr *)&addr6; - sl = sizeof(addr6); + if (proto == IPPROTO_UDP) { + int level = af == AF_INET ? IPPROTO_IP : IPPROTO_IPV6; + int opt = af == AF_INET ? IP_RECVERR : IPV6_RECVERR; - if (!dual_stack) - if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, - &y, sizeof(y))) - debug("Failed to set IPV6_V6ONLY on socket %i", - fd); + if (setsockopt(fd, level, opt, &y, sizeof(y))) + die_perror("Failed to set RECVERR on socket %i", fd); } - if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &y, sizeof(y))) - debug("Failed to set SO_REUSEADDR on socket %i", fd); - if (ifname && *ifname) { /* Supported since kernel version 5.7, commit c427bfec18f2 * ("net: core: enable SO_BINDTODEVICE for non-root users"). If @@ -142,28 +124,43 @@ int sock_l4(const struct ctx *c, sa_family_t af, uint8_t proto, */ if (setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE, ifname, strlen(ifname))) { + char str[SOCKADDR_STRLEN]; + ret = -errno; - warn("Can't bind %s socket for port %u to %s, closing", - EPOLL_TYPE_STR(proto), port, ifname); + warn("Can't bind %s socket for %s to %s, closing", + EPOLL_TYPE_STR(proto), + sockaddr_ntop(sa, str, sizeof(str)), ifname); close(fd); return ret; } } + if (freebind) { + int level = af == AF_INET ? IPPROTO_IP : IPPROTO_IPV6; + int opt = af == AF_INET ? IP_FREEBIND : IPV6_FREEBIND; + + if (setsockopt(fd, level, opt, &y, sizeof(y))) { + err_perror("Failed to set %s on socket %i", + af == AF_INET ? "IP_FREEBIND" + : "IPV6_FREEBIND", + fd); + } + } + if (bind(fd, sa, sl) < 0) { /* We'll fail to bind to low ports if we don't have enough * capabilities, and we'll fail to bind on already bound ports, * this is fine. This might also fail for ICMP because of a * broken SELinux policy, see icmp_tap_handler(). */ - if (proto != IPPROTO_ICMP && proto != IPPROTO_ICMPV6) { + if (type != EPOLL_TYPE_PING) { ret = -errno; close(fd); return ret; } } - if (proto == IPPROTO_TCP && listen(fd, 128) < 0) { + if (type == EPOLL_TYPE_TCP_LISTEN && listen(fd, 128) < 0) { ret = -errno; warn("TCP socket listen: %s", strerror(-ret)); close(fd); @@ -190,7 +187,8 @@ void sock_probe_mem(struct ctx *c) int v = INT_MAX / 2, s; socklen_t sl; - if ((s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { + s = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP); + if (s < 0) { c->low_wmem = c->low_rmem = 1; return; } @@ -210,23 +208,34 @@ void sock_probe_mem(struct ctx *c) close(s); } - /** - * timespec_diff_ms() - Report difference in milliseconds between two timestamps + * timespec_diff_us() - Report difference in microseconds between two timestamps * @a: Minuend timestamp * @b: Subtrahend timestamp * - * Return: difference in milliseconds + * Return: difference in microseconds (wraps after 2^63 / 10^6s ~= 292k years) */ -int timespec_diff_ms(const struct timespec *a, const struct timespec *b) +int64_t timespec_diff_us(const struct timespec *a, const struct timespec *b) { if (a->tv_nsec < b->tv_nsec) { - return (b->tv_nsec - a->tv_nsec) / 1000000 + - (a->tv_sec - b->tv_sec - 1) * 1000; + return (a->tv_nsec + 1000000000 - b->tv_nsec) / 1000 + + (a->tv_sec - b->tv_sec - 1) * 1000000; } - return (a->tv_nsec - b->tv_nsec) / 1000000 + - (a->tv_sec - b->tv_sec) * 1000; + return (a->tv_nsec - b->tv_nsec) / 1000 + + (a->tv_sec - b->tv_sec) * 1000000; +} + +/** + * timespec_diff_ms() - Report difference in milliseconds between two timestamps + * @a: Minuend timestamp + * @b: Subtrahend timestamp + * + * Return: difference in milliseconds + */ +long timespec_diff_ms(const struct timespec *a, const struct timespec *b) +{ + return timespec_diff_us(a, b) / 1000; } /** @@ -234,7 +243,7 @@ int timespec_diff_ms(const struct timespec *a, const struct timespec *b) * @map: Pointer to bitmap * @bit: Bit number to set */ -void bitmap_set(uint8_t *map, int bit) +void bitmap_set(uint8_t *map, unsigned bit) { unsigned long *word = (unsigned long *)map + BITMAP_WORD(bit); @@ -246,7 +255,7 @@ void bitmap_set(uint8_t *map, int bit) * @map: Pointer to bitmap * @bit: Bit number to clear */ -void bitmap_clear(uint8_t *map, int bit) +void bitmap_clear(uint8_t *map, unsigned bit) { unsigned long *word = (unsigned long *)map + BITMAP_WORD(bit); @@ -258,9 +267,9 @@ void bitmap_clear(uint8_t *map, int bit) * @map: Pointer to bitmap * @bit: Bit number to check * - * Return: one if given bit is set, zero if it's not + * Return: true if given bit is set, false if it's not */ -int bitmap_isset(const uint8_t *map, int bit) +bool bitmap_isset(const uint8_t *map, unsigned bit) { const unsigned long *word = (const unsigned long *)map + BITMAP_WORD(bit); @@ -300,7 +309,7 @@ void bitmap_or(uint8_t *dst, size_t size, const uint8_t *a, const uint8_t *b) void ns_enter(const struct ctx *c) { if (setns(c->pasta_netns_fd, CLONE_NEWNET)) - die("setns() failed entering netns: %s", strerror(errno)); + die_perror("setns() failed entering netns"); } /** @@ -315,10 +324,8 @@ bool ns_is_init(void) bool ret = true; int fd; - if ((fd = open("/proc/self/uid_map", O_RDONLY | O_CLOEXEC)) < 0) { - die("Can't determine if we're in init namespace: %s", - strerror(errno)); - } + if ((fd = open("/proc/self/uid_map", O_RDONLY | O_CLOEXEC)) < 0) + die_perror("Can't determine if we're in init namespace"); if (read(fd, buf, sizeof(root_uid_map)) != sizeof(root_uid_map) - 1 || strncmp(buf, root_uid_map, sizeof(root_uid_map))) @@ -382,11 +389,11 @@ int open_in_ns(const struct ctx *c, const char *path, int flags) } /** - * pid_file() - Write PID to file, if requested to do so, and close it + * pidfile_write() - Write PID to file, if requested to do so, and close it * @fd: Open PID file descriptor, closed on exit, -1 to skip writing it * @pid: PID value to write */ -void write_pidfile(int fd, pid_t pid) +void pidfile_write(int fd, pid_t pid) { char pid_buf[12]; int n; @@ -405,6 +412,23 @@ void write_pidfile(int fd, pid_t pid) } /** + * output_file_open() - Open file for output, if needed + * @path: Path for output file + * @flags: Flags for open() other than O_CREAT, O_TRUNC, O_CLOEXEC + * + * Return: file descriptor on success, -1 on failure with errno set by open() + */ +int output_file_open(const char *path, int flags) +{ + /* We use O_CLOEXEC here, but clang-tidy as of LLVM 16 to 19 looks for + * it in the 'mode' argument if we have one + */ + return open(path, O_CREAT | O_TRUNC | O_CLOEXEC | flags, + /* NOLINTNEXTLINE(android-cloexec-open) */ + S_IRUSR | S_IWUSR); +} + +/** * __daemon() - daemon()-like function writing PID file before parent exits * @pidfile_fd: Open PID file descriptor * @devnull_fd: Open file descriptor for /dev/null @@ -421,20 +445,15 @@ int __daemon(int pidfile_fd, int devnull_fd) } if (pid) { - write_pidfile(pidfile_fd, pid); + pidfile_write(pidfile_fd, pid); exit(EXIT_SUCCESS); } - errno = 0; - - setsid(); - - dup2(devnull_fd, STDIN_FILENO); - dup2(devnull_fd, STDOUT_FILENO); - dup2(devnull_fd, STDERR_FILENO); - close(devnull_fd); - - if (errno) + if (setsid() < 0 || + dup2(devnull_fd, STDIN_FILENO) < 0 || + dup2(devnull_fd, STDOUT_FILENO) < 0 || + dup2(devnull_fd, STDERR_FILENO) < 0 || + close(devnull_fd)) exit(EXIT_FAILURE); return 0; @@ -472,7 +491,7 @@ int write_file(const char *path, const char *buf) size_t len = strlen(buf); if (fd < 0) { - warn("Could not open %s: %s", path, strerror(errno)); + warn_perror("Could not open %s", path); return -1; } @@ -480,7 +499,7 @@ int write_file(const char *path, const char *buf) ssize_t rc = write(fd, buf, len); if (rc <= 0) { - warn("Couldn't write to %s: %s", path, strerror(errno)); + warn_perror("Couldn't write to %s", path); break; } @@ -522,6 +541,36 @@ int do_clone(int (*fn)(void *), char *stack_area, size_t stack_size, int flags, #endif } +/* write_all_buf() - write all of a buffer to an fd + * @fd: File descriptor + * @buf: Pointer to base of buffer + * @len: Length of buffer + * + * Return: 0 on success, -1 on error (with errno set) + * + * #syscalls write + */ +int write_all_buf(int fd, const void *buf, size_t len) +{ + const char *p = buf; + size_t left = len; + + while (left) { + ssize_t rc; + + do + rc = write(fd, p, left); + while ((rc < 0) && errno == EINTR); + + if (rc < 0) + return -1; + + p += rc; + left -= rc; + } + return 0; +} + /* write_remainder() - write the tail of an IO vector to an fd * @fd: File descriptor * @iov: IO vector @@ -530,27 +579,261 @@ int do_clone(int (*fn)(void *), char *stack_area, size_t stack_size, int flags, * * Return: 0 on success, -1 on error (with errno set) * - * #syscalls write writev + * #syscalls writev */ -int write_remainder(int fd, const struct iovec *iov, int iovcnt, size_t skip) +int write_remainder(int fd, const struct iovec *iov, size_t iovcnt, size_t skip) { - int i; + size_t i = 0, offset; - while ((i = iov_skip_bytes(iov, iovcnt, skip, &skip)) < iovcnt) { + while ((i += iov_skip_bytes(iov + i, iovcnt - i, skip, &offset)) < iovcnt) { ssize_t rc; - if (skip) { - rc = write(fd, (char *)iov[i].iov_base + skip, - iov[i].iov_len - skip); - } else { - rc = writev(fd, &iov[i], iovcnt - i); + if (offset) { + /* Write the remainder of the partially written buffer */ + if (write_all_buf(fd, (char *)iov[i].iov_base + offset, + iov[i].iov_len - offset) < 0) + return -1; + i++; } + /* Write as much of the remaining whole buffers as we can */ + rc = writev(fd, &iov[i], iovcnt - i); if (rc < 0) return -1; - skip += rc; + skip = rc; } - return 0; } + +/** sockaddr_ntop() - Convert a socket address to text format + * @sa: Socket address + * @dst: output buffer, minimum SOCKADDR_STRLEN bytes + * @size: size of buffer at @dst + * + * Return: On success, a non-null pointer to @dst, NULL on failure + */ +const char *sockaddr_ntop(const void *sa, char *dst, socklen_t size) +{ + sa_family_t family = ((const struct sockaddr *)sa)->sa_family; + socklen_t off = 0; + +#define IPRINTF(...) \ + do { \ + off += snprintf(dst + off, size - off, __VA_ARGS__); \ + if (off >= size) \ + return NULL; \ + } while (0) + +#define INTOP(af, addr) \ + do { \ + if (!inet_ntop((af), (addr), dst + off, size - off)) \ + return NULL; \ + off += strlen(dst + off); \ + } while (0) + + switch (family) { + case AF_UNSPEC: + IPRINTF("<unspecified>"); + break; + + case AF_INET: { + const struct sockaddr_in *sa4 = sa; + + INTOP(AF_INET, &sa4->sin_addr); + IPRINTF(":%hu", ntohs(sa4->sin_port)); + break; + } + + case AF_INET6: { + const struct sockaddr_in6 *sa6 = sa; + + IPRINTF("["); + INTOP(AF_INET6, &sa6->sin6_addr); + IPRINTF("]:%hu", ntohs(sa6->sin6_port)); + break; + } + + /* FIXME: Implement AF_UNIX */ + default: + errno = EAFNOSUPPORT; + return NULL; + } + +#undef IPRINTF +#undef INTOP + + return dst; +} + +/** eth_ntop() - Convert an Ethernet MAC address to text format + * @mac: MAC address + * @dst: Output buffer, minimum ETH_ADDRSTRLEN bytes + * @size: Size of buffer at @dst + * + * Return: On success, a non-null pointer to @dst, NULL on failure + */ +const char *eth_ntop(const unsigned char *mac, char *dst, size_t size) +{ + int len; + + len = snprintf(dst, size, "%02x:%02x:%02x:%02x:%02x:%02x", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); + if (len < 0 || (size_t)len >= size) + return NULL; + + return dst; +} + +/** str_ee_origin() - Convert socket extended error origin to a string + * @ee: Socket extended error structure + * + * Return: Static string describing error origin + */ +const char *str_ee_origin(const struct sock_extended_err *ee) +{ + const char *const desc[] = { + [SO_EE_ORIGIN_NONE] = "<no origin>", + [SO_EE_ORIGIN_LOCAL] = "Local", + [SO_EE_ORIGIN_ICMP] = "ICMP", + [SO_EE_ORIGIN_ICMP6] = "ICMPv6", + }; + + if (ee->ee_origin < ARRAY_SIZE(desc)) + return desc[ee->ee_origin]; + + return "<invalid>"; +} + +/** + * close_open_files() - Close leaked files, but not --fd, stdin, stdout, stderr + * @argc: Argument count + * @argv: Command line options, as we need to skip any file given via --fd + */ +void close_open_files(int argc, char **argv) +{ + const struct option optfd[] = { { "fd", required_argument, NULL, 'F' }, + { 0 }, + }; + long fd = -1; + int name, rc; + + do { + name = getopt_long(argc, argv, "-:F:", optfd, NULL); + + if (name == 'F') { + errno = 0; + fd = strtol(optarg, NULL, 0); + + if (errno || fd <= STDERR_FILENO || fd > INT_MAX) + die("Invalid --fd: %s", optarg); + } + } while (name != -1); + + if (fd == -1) { + rc = close_range(STDERR_FILENO + 1, ~0U, CLOSE_RANGE_UNSHARE); + } else if (fd == STDERR_FILENO + 1) { /* Still a single range */ + rc = close_range(STDERR_FILENO + 2, ~0U, CLOSE_RANGE_UNSHARE); + } else { + rc = close_range(STDERR_FILENO + 1, fd - 1, + CLOSE_RANGE_UNSHARE); + if (!rc) + rc = close_range(fd + 1, ~0U, CLOSE_RANGE_UNSHARE); + } + + if (rc) { + if (errno == ENOSYS || errno == EINVAL) { + /* This probably means close_range() or the + * CLOSE_RANGE_UNSHARE flag is not supported by the + * kernel. Not much we can do here except carry on and + * hope for the best. + */ + warn( +"Can't use close_range() to ensure no files leaked by parent"); + } else { + die_perror("Failed to close files leaked by parent"); + } + } + +} + +/** + * snprintf_check() - snprintf() wrapper, checking for truncation and errors + * @str: Output buffer + * @size: Maximum size to write to @str + * @format: Message + * + * Return: false on success, true on truncation or error, sets errno on failure + */ +bool snprintf_check(char *str, size_t size, const char *format, ...) +{ + va_list ap; + int rc; + + va_start(ap, format); + rc = vsnprintf(str, size, format, ap); + va_end(ap); + + if (rc < 0) { + errno = EIO; + return true; + } + + if ((size_t)rc >= size) { + errno = ENOBUFS; + return true; + } + + return false; +} + +#define DEV_RANDOM "/dev/random" + +/** + * raw_random() - Get high quality random bytes + * @buf: Buffer to fill with random bytes + * @buflen: Number of bytes of random data to put in @buf + * + * Assumes that the random data is essential, and will die() if unable to obtain + * it. + */ +void raw_random(void *buf, size_t buflen) +{ + size_t random_read = 0; +#ifndef HAS_GETRANDOM + int fd = open(DEV_RANDOM, O_RDONLY); + + if (fd < 0) + die_perror("Couldn't open %s", DEV_RANDOM); +#endif + + while (random_read < buflen) { + ssize_t ret; + +#ifdef HAS_GETRANDOM + ret = getrandom((char *)buf + random_read, + buflen - random_read, GRND_RANDOM); +#else + ret = read(dev_random, (char *)buf + random_read, + buflen - random_read); +#endif + + if (ret == -1 && errno == EINTR) + continue; + + if (ret < 0) + die_perror("Error on random data source"); + + if (ret == 0) + break; + + random_read += ret; + } + +#ifndef HAS_GETRANDOM + close(dev_random); +#endif + + if (random_read < buflen) + die("Unexpected EOF on random data source"); +} @@ -9,8 +9,14 @@ #include <stdlib.h> #include <stdarg.h> #include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> #include <string.h> #include <signal.h> +#include <arpa/inet.h> +#include <unistd.h> +#include <sys/syscall.h> #include "log.h" @@ -31,6 +37,9 @@ #ifndef ETH_MIN_MTU #define ETH_MIN_MTU 68 #endif +#ifndef IP_MAX_MTU +#define IP_MAX_MTU USHRT_MAX +#endif #ifndef MIN #define MIN(x, y) (((x) < (y)) ? (x) : (y)) @@ -40,12 +49,10 @@ #endif #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) +#define DIV_ROUND_CLOSEST(n, d) (((n) + (d) / 2) / (d)) #define ROUND_DOWN(x, y) ((x) & ~((y) - 1)) #define ROUND_UP(x, y) (((x) + (y) - 1) & ~((y) - 1)) -#define ALIGN_DOWN(n, m) ((n) / (m) * (m)) -#define ALIGN_UP(n, m) ALIGN_DOWN((n) + (m) - 1, (m)) - #define MAX_FROM_BITS(n) (((1U << (n)) - 1)) #define BIT(n) (1UL << (n)) @@ -60,6 +67,15 @@ #define STRINGIFY(x) #x #define STR(x) STRINGIFY(x) +#ifdef CPPCHECK_6936 +/* Some cppcheck versions get confused by aborts inside a loop, causing + * it to give false positive uninitialised variable warnings later in + * the function, because it doesn't realise the non-initialising path + * already exited. See https://trac.cppcheck.net/ticket/13227 + */ +#define ASSERT(expr) \ + ((expr) ? (void)0 : abort()) +#else #define ASSERT(expr) \ do { \ if (!(expr)) { \ @@ -71,6 +87,7 @@ abort(); \ } \ } while (0) +#endif #ifdef P_tmpdir #define TMPDIR P_tmpdir @@ -84,13 +101,13 @@ #define ARRAY_SIZE(a) ((int)(sizeof(a) / sizeof((a)[0]))) +#define foreach(item, array) \ + for ((item) = (array); (item) - (array) < ARRAY_SIZE(array); (item)++) + #define IN_INTERVAL(a, b, x) ((x) >= (a) && (x) <= (b)) #define FD_PROTO(x, proto) \ (IN_INTERVAL(c->proto.fd_min, c->proto.fd_max, (x))) -#define PORT_EPHEMERAL_MIN ((1 << 15) + (1 << 14)) /* RFC 6335 */ -#define PORT_IS_EPHEMERAL(port) ((port) >= PORT_EPHEMERAL_MIN) - #define MAC_ZERO ((uint8_t [ETH_ALEN]){ 0 }) #define MAC_IS_ZERO(addr) (!memcmp((addr), MAC_ZERO, ETH_ALEN)) @@ -113,7 +130,21 @@ #define htonl_constant(x) (__bswap_constant_32(x)) #endif -#define barrier() do { __asm__ __volatile__("" ::: "memory"); } while (0) +/** + * ntohl_unaligned() - Read 32-bit BE value from a possibly unaligned address + * @p: Pointer to the BE value in memory + * + * Returns: Host-order value of 32-bit BE quantity at @p + */ +static inline uint32_t ntohl_unaligned(const void *p) +{ + uint32_t val; + + memcpy(&val, p, sizeof(val)); + return ntohl(val); +} + +static inline void barrier(void) { __asm__ __volatile__("" ::: "memory"); } #define smp_mb() do { barrier(); __atomic_thread_fence(__ATOMIC_SEQ_CST); } while (0) #define smp_mb_release() do { barrier(); __atomic_thread_fence(__ATOMIC_RELEASE); } while (0) #define smp_mb_acquire() do { barrier(); __atomic_thread_fence(__ATOMIC_ACQUIRE); } while (0) @@ -121,50 +152,94 @@ #define smp_wmb() smp_mb_release() #define smp_rmb() smp_mb_acquire() -#define NS_FN_STACK_SIZE (RLIMIT_STACK_VAL * 1024 / 8) +#define NS_FN_STACK_SIZE (1024 * 1024) /* 1MiB */ + int do_clone(int (*fn)(void *), char *stack_area, size_t stack_size, int flags, void *arg); #define NS_CALL(fn, arg) \ do { \ - char ns_fn_stack[NS_FN_STACK_SIZE]; \ + char ns_fn_stack[NS_FN_STACK_SIZE] \ + __attribute__ ((aligned(__alignof__(max_align_t)))); \ \ do_clone((fn), ns_fn_stack, sizeof(ns_fn_stack), \ CLONE_VM | CLONE_VFORK | CLONE_FILES | SIGCHLD,\ (void *)(arg)); \ } while (0) -#define RCVBUF_BIG (2UL * 1024 * 1024) -#define SNDBUF_BIG (4UL * 1024 * 1024) -#define SNDBUF_SMALL (128UL * 1024) +#define RCVBUF_BIG (2ULL * 1024 * 1024) +#define SNDBUF_BIG (4ULL * 1024 * 1024) +#define SNDBUF_SMALL (128ULL * 1024) #include <net/if.h> #include <limits.h> #include <stdint.h> +#include "epoll_type.h" #include "packet.h" struct ctx; -/* cppcheck-suppress funcArgNamesDifferent */ -__attribute__ ((weak)) int ffsl(long int i) { return __builtin_ffsl(i); } -int sock_l4(const struct ctx *c, sa_family_t af, uint8_t proto, - const void *bind_addr, const char *ifname, uint16_t port, - uint32_t data); +int sock_l4_sa(const struct ctx *c, enum epoll_type type, + const void *sa, socklen_t sl, + const char *ifname, bool v6only, uint32_t data); void sock_probe_mem(struct ctx *c); -int timespec_diff_ms(const struct timespec *a, const struct timespec *b); -void bitmap_set(uint8_t *map, int bit); -void bitmap_clear(uint8_t *map, int bit); -int bitmap_isset(const uint8_t *map, int bit); +long timespec_diff_ms(const struct timespec *a, const struct timespec *b); +int64_t timespec_diff_us(const struct timespec *a, const struct timespec *b); +void bitmap_set(uint8_t *map, unsigned bit); +void bitmap_clear(uint8_t *map, unsigned bit); +bool bitmap_isset(const uint8_t *map, unsigned bit); void bitmap_or(uint8_t *dst, size_t size, const uint8_t *a, const uint8_t *b); char *line_read(char *buf, size_t len, int fd); void ns_enter(const struct ctx *c); bool ns_is_init(void); int open_in_ns(const struct ctx *c, const char *path, int flags); -void write_pidfile(int fd, pid_t pid); +int output_file_open(const char *path, int flags); +void pidfile_write(int fd, pid_t pid); int __daemon(int pidfile_fd, int devnull_fd); int fls(unsigned long x); int write_file(const char *path, const char *buf); -int write_remainder(int fd, const struct iovec *iov, int iovcnt, size_t skip); +int write_all_buf(int fd, const void *buf, size_t len); +int write_remainder(int fd, const struct iovec *iov, size_t iovcnt, size_t skip); +void close_open_files(int argc, char **argv); +bool snprintf_check(char *str, size_t size, const char *format, ...); + +/** + * af_name() - Return name of an address family + * @af: Address/protocol family (AF_INET or AF_INET6) + * + * Returns: Name of the protocol family as a string + */ +static inline const char *af_name(sa_family_t af) +{ + switch (af) { + case AF_INET: + return "IPv4"; + case AF_INET6: + return "IPv6"; + default: + return "<unknown address family>"; + } +} + +#define UINT16_STRLEN (sizeof("65535")) + +/* inet address (- '\0') + port (u16) (- '\0') + ':' + '\0' */ +#define SOCKADDR_INET_STRLEN \ + (INET_ADDRSTRLEN-1 + UINT16_STRLEN-1 + sizeof(":")) + +/* inet6 address (- '\0') + port (u16) (- '\0') + '[' + ']' + ':' + '\0' */ +#define SOCKADDR_INET6_STRLEN \ + (INET6_ADDRSTRLEN-1 + UINT16_STRLEN-1 + sizeof("[]:")) + +#define SOCKADDR_STRLEN MAX(SOCKADDR_INET_STRLEN, SOCKADDR_INET6_STRLEN) + +#define ETH_ADDRSTRLEN (sizeof("00:11:22:33:44:55")) + +struct sock_extended_err; + +const char *sockaddr_ntop(const void *sa, char *dst, socklen_t size); +const char *eth_ntop(const unsigned char *mac, char *dst, size_t size); +const char *str_ee_origin(const struct sock_extended_err *ee); /** * mod_sub() - Modular arithmetic subtraction @@ -194,6 +269,11 @@ static inline bool mod_between(unsigned x, unsigned i, unsigned j, unsigned m) return mod_sub(x, i, m) < mod_sub(j, i, m); } +/* FPRINTF() intentionally silences cert-err33-c clang-tidy warnings */ +#define FPRINTF(f, ...) (void)fprintf(f, __VA_ARGS__) + +void raw_random(void *buf, size_t buflen); + /* * Workarounds for https://github.com/llvm/llvm-project/issues/58992 * diff --git a/vhost_user.c b/vhost_user.c index 9cc07c8..51c90db 100644 --- a/vhost_user.c +++ b/vhost_user.c @@ -1,6 +1,24 @@ // SPDX-License-Identifier: GPL-2.0-or-later - -/* some parts from QEMU subprojects/libvhost-user/libvhost-user.c */ +/* + * vhost-user API, command management and virtio interface + * + * Copyright Red Hat + * Author: Laurent Vivier <lvivier@redhat.com> + * + * Some parts from QEMU subprojects/libvhost-user/libvhost-user.c + * licensed under the following terms: + * + * Copyright IBM, Corp. 2007 + * Copyright (c) 2016 Red Hat, Inc. + * + * Authors: + * Anthony Liguori <aliguori@us.ibm.com> + * Marc-André Lureau <mlureau@redhat.com> + * Victor Kaplansky <victork@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * later. See the COPYING file in the top-level directory. + */ #include <errno.h> #include <fcntl.h> @@ -25,91 +43,112 @@ #include "passt.h" #include "tap.h" #include "vhost_user.h" +#include "pcap.h" +/* vhost-user version we are compatible with */ #define VHOST_USER_VERSION 1 -static unsigned char buffer[65536][VHOST_USER_MAX_QUEUES]; +static struct vu_dev vdev_storage; +/** + * vu_print_capabilities() - print vhost-user capabilities + * this is part of the vhost-user backend + * convention. + */ void vu_print_capabilities(void) { - printf("{\n"); - printf(" \"type\": \"net\"\n"); - printf("}\n"); + info("{"); + info(" \"type\": \"net\""); + info("}"); exit(EXIT_SUCCESS); } -static const char * -vu_request_to_string(unsigned int req) +/** + * vu_request_to_string() - convert a vhost-user request number to its name + * @req: request number + * + * Return: the name of request number + */ +static const char *vu_request_to_string(unsigned int req) { + if (req < VHOST_USER_MAX) { #define REQ(req) [req] = #req - static const char *vu_request_str[] = { - REQ(VHOST_USER_NONE), - REQ(VHOST_USER_GET_FEATURES), - REQ(VHOST_USER_SET_FEATURES), - REQ(VHOST_USER_SET_OWNER), - REQ(VHOST_USER_RESET_OWNER), - REQ(VHOST_USER_SET_MEM_TABLE), - REQ(VHOST_USER_SET_LOG_BASE), - REQ(VHOST_USER_SET_LOG_FD), - REQ(VHOST_USER_SET_VRING_NUM), - REQ(VHOST_USER_SET_VRING_ADDR), - REQ(VHOST_USER_SET_VRING_BASE), - REQ(VHOST_USER_GET_VRING_BASE), - REQ(VHOST_USER_SET_VRING_KICK), - REQ(VHOST_USER_SET_VRING_CALL), - REQ(VHOST_USER_SET_VRING_ERR), - REQ(VHOST_USER_GET_PROTOCOL_FEATURES), - REQ(VHOST_USER_SET_PROTOCOL_FEATURES), - REQ(VHOST_USER_GET_QUEUE_NUM), - REQ(VHOST_USER_SET_VRING_ENABLE), - REQ(VHOST_USER_SEND_RARP), - REQ(VHOST_USER_NET_SET_MTU), - REQ(VHOST_USER_SET_BACKEND_REQ_FD), - REQ(VHOST_USER_IOTLB_MSG), - REQ(VHOST_USER_SET_VRING_ENDIAN), - REQ(VHOST_USER_GET_CONFIG), - REQ(VHOST_USER_SET_CONFIG), - REQ(VHOST_USER_POSTCOPY_ADVISE), - REQ(VHOST_USER_POSTCOPY_LISTEN), - REQ(VHOST_USER_POSTCOPY_END), - REQ(VHOST_USER_GET_INFLIGHT_FD), - REQ(VHOST_USER_SET_INFLIGHT_FD), - REQ(VHOST_USER_GPU_SET_SOCKET), - REQ(VHOST_USER_VRING_KICK), - REQ(VHOST_USER_GET_MAX_MEM_SLOTS), - REQ(VHOST_USER_ADD_MEM_REG), - REQ(VHOST_USER_REM_MEM_REG), - REQ(VHOST_USER_MAX), - }; + static const char * const vu_request_str[VHOST_USER_MAX] = { + REQ(VHOST_USER_NONE), + REQ(VHOST_USER_GET_FEATURES), + REQ(VHOST_USER_SET_FEATURES), + REQ(VHOST_USER_SET_OWNER), + REQ(VHOST_USER_RESET_OWNER), + REQ(VHOST_USER_SET_MEM_TABLE), + REQ(VHOST_USER_SET_LOG_BASE), + REQ(VHOST_USER_SET_LOG_FD), + REQ(VHOST_USER_SET_VRING_NUM), + REQ(VHOST_USER_SET_VRING_ADDR), + REQ(VHOST_USER_SET_VRING_BASE), + REQ(VHOST_USER_GET_VRING_BASE), + REQ(VHOST_USER_SET_VRING_KICK), + REQ(VHOST_USER_SET_VRING_CALL), + REQ(VHOST_USER_SET_VRING_ERR), + REQ(VHOST_USER_GET_PROTOCOL_FEATURES), + REQ(VHOST_USER_SET_PROTOCOL_FEATURES), + REQ(VHOST_USER_GET_QUEUE_NUM), + REQ(VHOST_USER_SET_VRING_ENABLE), + REQ(VHOST_USER_SEND_RARP), + REQ(VHOST_USER_NET_SET_MTU), + REQ(VHOST_USER_SET_BACKEND_REQ_FD), + REQ(VHOST_USER_IOTLB_MSG), + REQ(VHOST_USER_SET_VRING_ENDIAN), + REQ(VHOST_USER_GET_CONFIG), + REQ(VHOST_USER_SET_CONFIG), + REQ(VHOST_USER_POSTCOPY_ADVISE), + REQ(VHOST_USER_POSTCOPY_LISTEN), + REQ(VHOST_USER_POSTCOPY_END), + REQ(VHOST_USER_GET_INFLIGHT_FD), + REQ(VHOST_USER_SET_INFLIGHT_FD), + REQ(VHOST_USER_GPU_SET_SOCKET), + REQ(VHOST_USER_VRING_KICK), + REQ(VHOST_USER_GET_MAX_MEM_SLOTS), + REQ(VHOST_USER_ADD_MEM_REG), + REQ(VHOST_USER_REM_MEM_REG), + }; #undef REQ - - if (req < VHOST_USER_MAX) { return vu_request_str[req]; - } else { - return "unknown"; } + + return "unknown"; } -/* Translate qemu virtual address to our virtual address. */ -static void *qva_to_va(VuDev *dev, uint64_t qemu_addr) +/** + * qva_to_va() - Translate front-end (QEMU) virtual address to our virtual + * address + * @dev: vhost-user device + * @qemu_addr: front-end userspace address + * + * Return: the memory address in our process virtual address space. + */ +static void *qva_to_va(struct vu_dev *dev, uint64_t qemu_addr) { unsigned int i; /* Find matching memory region. */ for (i = 0; i < dev->nregions; i++) { - VuDevRegion *r = &dev->regions[i]; + const struct vu_dev_region *r = &dev->regions[i]; if ((qemu_addr >= r->qva) && (qemu_addr < (r->qva + r->size))) { - return (void *)(uintptr_t) - (qemu_addr - r->qva + r->mmap_addr + r->mmap_offset); + /* NOLINTNEXTLINE(performance-no-int-to-ptr) */ + return (void *)(qemu_addr - r->qva + r->mmap_addr + + r->mmap_offset); } } return NULL; } -static void -vmsg_close_fds(VhostUserMsg *vmsg) +/** + * vmsg_close_fds() - Close all file descriptors of a given message + * @vmsg: vhost-user message with the list of the file descriptors + */ +static void vmsg_close_fds(const struct vhost_user_msg *vmsg) { int i; @@ -117,15 +156,24 @@ vmsg_close_fds(VhostUserMsg *vmsg) close(vmsg->fds[i]); } -static void vu_remove_watch(VuDev *vdev, int fd) +/** + * vu_remove_watch() - Remove a file descriptor from our passt epoll + * file descriptor + * @vdev: vhost-user device + * @fd: file descriptor to remove + */ +static void vu_remove_watch(const struct vu_dev *vdev, int fd) { - struct ctx *c = (struct ctx *) ((char *)vdev - offsetof(struct ctx, vdev)); - - epoll_ctl(c->epollfd, EPOLL_CTL_DEL, fd, NULL); + epoll_ctl(vdev->context->epollfd, EPOLL_CTL_DEL, fd, NULL); } -/* Set reply payload.u64 and clear request flags and fd_num */ -static void vmsg_set_reply_u64(struct VhostUserMsg *vmsg, uint64_t val) +/** + * vmsg_set_reply_u64() - Set reply payload.u64 and clear request flags + * and fd_num + * @vmsg: vhost-user message + * @val: 64-bit value to reply + */ +static void vmsg_set_reply_u64(struct vhost_user_msg *vmsg, uint64_t val) { vmsg->hdr.flags = 0; /* defaults will be set by vu_send_reply() */ vmsg->hdr.size = sizeof(vmsg->payload.u64); @@ -133,7 +181,16 @@ static void vmsg_set_reply_u64(struct VhostUserMsg *vmsg, uint64_t val) vmsg->fd_num = 0; } -static ssize_t vu_message_read_default(VuDev *dev, int conn_fd, struct VhostUserMsg *vmsg) +/** + * vu_message_read_default() - Read incoming vhost-user message from the + * front-end + * @conn_fd: vhost-user command socket + * @vmsg: vhost-user message + * + * Return: 0 if recvmsg() has been interrupted or if there's no data to read, + * 1 if a message has been received + */ +static int vu_message_read_default(int conn_fd, struct vhost_user_msg *vmsg) { char control[CMSG_SPACE(VHOST_MEMORY_BASELINE_NREGIONS * sizeof(int))] = { 0 }; @@ -147,16 +204,14 @@ static ssize_t vu_message_read_default(VuDev *dev, int conn_fd, struct VhostUser .msg_control = control, .msg_controllen = sizeof(control), }; - size_t fd_size; - struct cmsghdr *cmsg; ssize_t ret, sz_payload; + struct cmsghdr *cmsg; ret = recvmsg(conn_fd, &msg, MSG_DONTWAIT); if (ret < 0) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) return 0; - vu_panic(dev, "Error while recvmsg: %s", strerror(errno)); - goto out; + die_perror("vhost-user message receive (recvmsg)"); } vmsg->fd_num = 0; @@ -164,7 +219,11 @@ static ssize_t vu_message_read_default(VuDev *dev, int conn_fd, struct VhostUser cmsg = CMSG_NXTHDR(&msg, cmsg)) { if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) { + size_t fd_size; + + ASSERT(cmsg->cmsg_len >= CMSG_LEN(0)); fd_size = cmsg->cmsg_len - CMSG_LEN(0); + ASSERT(fd_size <= sizeof(vmsg->fds)); vmsg->fd_num = fd_size / sizeof(int); memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size); break; @@ -173,99 +232,106 @@ static ssize_t vu_message_read_default(VuDev *dev, int conn_fd, struct VhostUser sz_payload = vmsg->hdr.size; if ((size_t)sz_payload > sizeof(vmsg->payload)) { - vu_panic(dev, - "Error: too big message request: %d, size: vmsg->size: %zd, " + die("vhost-user message request too big: %d," + " size: vmsg->size: %zd, " "while sizeof(vmsg->payload) = %zu", vmsg->hdr.request, sz_payload, sizeof(vmsg->payload)); - goto out; } if (sz_payload) { - do { + do ret = recv(conn_fd, &vmsg->payload, sz_payload, 0); - } while (ret < 0 && (errno == EINTR || errno == EAGAIN)); + while (ret < 0 && errno == EINTR); - if (ret < sz_payload) { - vu_panic(dev, "Error while reading: %s", strerror(errno)); - goto out; - } + if (ret < 0) + die_perror("vhost-user message receive"); + + if (ret == 0) + die("EOF on vhost-user message receive"); + + if (ret < sz_payload) + die("Short-read on vhost-user message receive"); } return 1; -out: - vmsg_close_fds(vmsg); - - return -ECONNRESET; } -static int vu_message_write(VuDev *dev, int conn_fd, struct VhostUserMsg *vmsg) +/** + * vu_message_write() - Send a message to the front-end + * @conn_fd: vhost-user command socket + * @vmsg: vhost-user message + * + * #syscalls:vu sendmsg + */ +static void vu_message_write(int conn_fd, struct vhost_user_msg *vmsg) { - int rc; - uint8_t *p = (uint8_t *)vmsg; char control[CMSG_SPACE(VHOST_MEMORY_BASELINE_NREGIONS * sizeof(int))] = { 0 }; struct iovec iov = { .iov_base = (char *)vmsg, - .iov_len = VHOST_USER_HDR_SIZE, + .iov_len = VHOST_USER_HDR_SIZE + vmsg->hdr.size, }; struct msghdr msg = { .msg_iov = &iov, .msg_iovlen = 1, .msg_control = control, }; - struct cmsghdr *cmsg; + int rc; - memset(control, 0, sizeof(control)); - assert(vmsg->fd_num <= VHOST_MEMORY_BASELINE_NREGIONS); + ASSERT(vmsg->fd_num <= VHOST_MEMORY_BASELINE_NREGIONS); if (vmsg->fd_num > 0) { size_t fdsize = vmsg->fd_num * sizeof(int); + struct cmsghdr *cmsg; + msg.msg_controllen = CMSG_SPACE(fdsize); cmsg = CMSG_FIRSTHDR(&msg); cmsg->cmsg_len = CMSG_LEN(fdsize); cmsg->cmsg_level = SOL_SOCKET; cmsg->cmsg_type = SCM_RIGHTS; memcpy(CMSG_DATA(cmsg), vmsg->fds, fdsize); - } else { - msg.msg_controllen = 0; } - do { + do rc = sendmsg(conn_fd, &msg, 0); - } while (rc < 0 && (errno == EINTR || errno == EAGAIN)); - - if (vmsg->hdr.size) { - do { - if (vmsg->data) { - rc = write(conn_fd, vmsg->data, vmsg->hdr.size); - } else { - rc = write(conn_fd, p + VHOST_USER_HDR_SIZE, vmsg->hdr.size); - } - } while (rc < 0 && (errno == EINTR || errno == EAGAIN)); - } + while (rc < 0 && errno == EINTR); - if (rc <= 0) { - vu_panic(dev, "Error while writing: %s", strerror(errno)); - return false; - } + if (rc < 0) + die_perror("vhost-user message send"); - return true; + if ((uint32_t)rc < VHOST_USER_HDR_SIZE + vmsg->hdr.size) + die("EOF on vhost-user message send"); } -static int vu_send_reply(VuDev *dev, int conn_fd, struct VhostUserMsg *msg) +/** + * vu_send_reply() - Update message flags and send it to front-end + * @conn_fd: vhost-user command socket + * @vmsg: vhost-user message + */ +static void vu_send_reply(int conn_fd, struct vhost_user_msg *msg) { msg->hdr.flags &= ~VHOST_USER_VERSION_MASK; msg->hdr.flags |= VHOST_USER_VERSION; msg->hdr.flags |= VHOST_USER_REPLY_MASK; - return vu_message_write(dev, conn_fd, msg); + vu_message_write(conn_fd, msg); } -static bool vu_get_features_exec(struct VhostUserMsg *msg) +/** + * vu_get_features_exec() - Provide back-end features bitmask to front-end + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: True as a reply is requested + */ +static bool vu_get_features_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { uint64_t features = 1ULL << VIRTIO_F_VERSION_1 | 1ULL << VIRTIO_NET_F_MRG_RXBUF | 1ULL << VHOST_USER_F_PROTOCOL_FEATURES; + (void)vdev; + vmsg_set_reply_u64(msg, features); debug("Sending back to guest u64: 0x%016"PRIx64, msg->payload.u64); @@ -273,54 +339,69 @@ static bool vu_get_features_exec(struct VhostUserMsg *msg) return true; } -static void -vu_set_enable_all_rings(VuDev *vdev, bool enabled) +/** + * vu_set_enable_all_rings() - Enable/disable all the virtqueues + * @vdev: vhost-user device + * @enable: New virtqueues state + */ +static void vu_set_enable_all_rings(struct vu_dev *vdev, bool enable) { uint16_t i; - for (i = 0; i < VHOST_USER_MAX_QUEUES; i++) { - vdev->vq[i].enable = enabled; - } + for (i = 0; i < VHOST_USER_MAX_QUEUES; i++) + vdev->vq[i].enable = enable; } -static bool -vu_set_features_exec(VuDev *vdev, struct VhostUserMsg *msg) +/** + * vu_set_features_exec() - Enable features of the back-end + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: False as no reply is requested + */ +static bool vu_set_features_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { debug("u64: 0x%016"PRIx64, msg->payload.u64); vdev->features = msg->payload.u64; - if (!vu_has_feature(vdev, VIRTIO_F_VERSION_1)) { - /* - * We only support devices conforming to VIRTIO 1.0 or - * later - */ - vu_panic(vdev, "virtio legacy devices aren't supported by passt"); - return false; - } + /* We only support devices conforming to VIRTIO 1.0 or + * later + */ + if (!vu_has_feature(vdev, VIRTIO_F_VERSION_1)) + die("virtio legacy devices aren't supported by passt"); - if (!vu_has_feature(vdev, VHOST_USER_F_PROTOCOL_FEATURES)) { + if (!vu_has_feature(vdev, VHOST_USER_F_PROTOCOL_FEATURES)) vu_set_enable_all_rings(vdev, true); - } - - /* virtio-net features */ - - if (vu_has_feature(vdev, VIRTIO_F_VERSION_1) || - vu_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) { - vdev->hdrlen = sizeof(struct virtio_net_hdr_mrg_rxbuf); - } else { - vdev->hdrlen = sizeof(struct virtio_net_hdr); - } return false; } -static bool -vu_set_owner_exec(void) +/** + * vu_set_owner_exec() - Session start flag, do nothing in our case + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: False as no reply is requested + */ +static bool vu_set_owner_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { + (void)vdev; + (void)msg; + return false; } -static bool map_ring(VuDev *vdev, VuVirtq *vq) +/** + * map_ring() - Convert ring front-end (QEMU) addresses to our process + * virtual address space. + * @vdev: vhost-user device + * @vq: Virtqueue + * + * Return: True if ring cannot be mapped to our address space + */ +static bool map_ring(struct vu_dev *vdev, struct vu_virtq *vq) { vq->vring.desc = qva_to_va(vdev, vq->vra.desc_user_addr); vq->vring.used = qva_to_va(vdev, vq->vra.used_user_addr); @@ -334,51 +415,39 @@ static bool map_ring(VuDev *vdev, VuVirtq *vq) return !(vq->vring.desc && vq->vring.used && vq->vring.avail); } -int vu_packet_check_range(void *buf, size_t offset, size_t len, const char *start, - const char *func, int line) -{ - VuDevRegion *dev_region; - - for (dev_region = buf; dev_region->mmap_addr; dev_region++) { - if ((char *)dev_region->mmap_addr <= start && - start + offset + len < (char *)dev_region->mmap_addr + - dev_region->mmap_offset + - dev_region->size) - return 0; - } - if (func) { - trace("cannot find region, %s:%i", func, line); - } - - return -1; -} - -/* - * #syscalls:passt mmap munmap +/** + * vu_set_mem_table_exec() - Sets the memory map regions to be able to + * translate the vring addresses. + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: False as no reply is requested + * + * #syscalls:vu mmap munmap */ - -static bool vu_set_mem_table_exec(VuDev *vdev, - struct VhostUserMsg *msg) +static bool vu_set_mem_table_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { + struct vhost_user_memory m = msg->payload.memory, *memory = &m; unsigned int i; - struct VhostUserMemory m = msg->payload.memory, *memory = &m; for (i = 0; i < vdev->nregions; i++) { - VuDevRegion *r = &vdev->regions[i]; - void *m = (void *) (uintptr_t) r->mmap_addr; + const struct vu_dev_region *r = &vdev->regions[i]; - if (m) - munmap(m, r->size + r->mmap_offset); + if (r->mmap_addr) { + /* NOLINTNEXTLINE(performance-no-int-to-ptr) */ + munmap((void *)r->mmap_addr, r->size + r->mmap_offset); + } } vdev->nregions = memory->nregions; - debug("Nregions: %u", memory->nregions); + debug("vhost-user nregions: %u", memory->nregions); for (i = 0; i < vdev->nregions; i++) { + struct vhost_user_memory_region *msg_region = &memory->regions[i]; + struct vu_dev_region *dev_region = &vdev->regions[i]; void *mmap_addr; - VhostUserMemory_region *msg_region = &memory->regions[i]; - VuDevRegion *dev_region = &vdev->regions[i]; - debug("Region %d", i); + debug("vhost-user region %d", i); debug(" guest_phys_addr: 0x%016"PRIx64, msg_region->guest_phys_addr); debug(" memory_size: 0x%016"PRIx64, @@ -394,76 +463,95 @@ static bool vu_set_mem_table_exec(VuDev *vdev, dev_region->mmap_offset = msg_region->mmap_offset; /* We don't use offset argument of mmap() since the - * mapped address has to be page aligned, and we use huge - * pages. */ + * mapped address has to be page aligned. + */ mmap_addr = mmap(0, dev_region->size + dev_region->mmap_offset, - PROT_READ | PROT_WRITE, MAP_SHARED | MAP_NORESERVE, - msg->fds[i], 0); - - if (mmap_addr == MAP_FAILED) { - vu_panic(vdev, "region mmap error: %s", strerror(errno)); - } else { - dev_region->mmap_addr = (uint64_t)(uintptr_t)mmap_addr; - debug(" mmap_addr: 0x%016"PRIx64, - dev_region->mmap_addr); - } + PROT_READ | PROT_WRITE, MAP_SHARED | + MAP_NORESERVE, msg->fds[i], 0); + + if (mmap_addr == MAP_FAILED) + die_perror("vhost-user region mmap error"); + + dev_region->mmap_addr = (uint64_t)(uintptr_t)mmap_addr; + debug(" mmap_addr: 0x%016"PRIx64, + dev_region->mmap_addr); close(msg->fds[i]); } for (i = 0; i < VHOST_USER_MAX_QUEUES; i++) { if (vdev->vq[i].vring.desc) { - if (map_ring(vdev, &vdev->vq[i])) { - vu_panic(vdev, "remapping queue %d during setmemtable", i); - } + if (map_ring(vdev, &vdev->vq[i])) + die("remapping queue %d during setmemtable", i); } } - /* XXX */ + /* As vu_packet_check_range() has no access to the number of + * memory regions, mark the end of the array with mmap_addr = 0 + */ ASSERT(vdev->nregions < VHOST_USER_MAX_RAM_SLOTS - 1); - vdev->regions[vdev->nregions].mmap_addr = 0; /* mark EOF for vu_packet_check_range() */ + vdev->regions[vdev->nregions].mmap_addr = 0; - tap_sock_update_buf(vdev->regions, 0); + tap_sock_update_pool(vdev->regions, 0); return false; } -static bool vu_set_vring_num_exec(VuDev *vdev, - struct VhostUserMsg *msg) +/** + * vu_set_vring_num_exec() - Set the size of the queue (vring size) + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: False as no reply is requested + */ +static bool vu_set_vring_num_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { - unsigned int index = msg->payload.state.index; + unsigned int idx = msg->payload.state.index; unsigned int num = msg->payload.state.num; - debug("State.index: %u", index); + debug("State.index: %u", idx); debug("State.num: %u", num); - vdev->vq[index].vring.num = num; + vdev->vq[idx].vring.num = num; return false; } -static bool vu_set_vring_addr_exec(VuDev *vdev, - struct VhostUserMsg *msg) +/** + * vu_set_vring_addr_exec() - Set the addresses of the vring + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: False as no reply is requested + */ +static bool vu_set_vring_addr_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { - struct vhost_vring_addr addr = msg->payload.addr, *vra = &addr; - unsigned int index = vra->index; - VuVirtq *vq = &vdev->vq[index]; + /* We need to copy the payload to vhost_vring_addr structure + * to access index because address of msg->payload.addr + * can be unaligned as it is packed. + */ + struct vhost_vring_addr addr = msg->payload.addr; + struct vu_virtq *vq = &vdev->vq[addr.index]; debug("vhost_vring_addr:"); - debug(" index: %d", vra->index); - debug(" flags: %d", vra->flags); - debug(" desc_user_addr: 0x%016" PRIx64, (uint64_t)vra->desc_user_addr); - debug(" used_user_addr: 0x%016" PRIx64, (uint64_t)vra->used_user_addr); - debug(" avail_user_addr: 0x%016" PRIx64, (uint64_t)vra->avail_user_addr); - debug(" log_guest_addr: 0x%016" PRIx64, (uint64_t)vra->log_guest_addr); - - vq->vra = *vra; - vq->vring.flags = vra->flags; - vq->vring.log_guest_addr = vra->log_guest_addr; - - if (map_ring(vdev, vq)) { - vu_panic(vdev, "Invalid vring_addr message"); - return false; - } + debug(" index: %d", addr.index); + debug(" flags: %d", addr.flags); + debug(" desc_user_addr: 0x%016" PRIx64, + (uint64_t)addr.desc_user_addr); + debug(" used_user_addr: 0x%016" PRIx64, + (uint64_t)addr.used_user_addr); + debug(" avail_user_addr: 0x%016" PRIx64, + (uint64_t)addr.avail_user_addr); + debug(" log_guest_addr: 0x%016" PRIx64, + (uint64_t)addr.log_guest_addr); + + vq->vra = msg->payload.addr; + vq->vring.flags = addr.flags; + vq->vring.log_guest_addr = addr.log_guest_addr; + + if (map_ring(vdev, vq)) + die("Invalid vring_addr message"); vq->used_idx = le16toh(vq->vring.used->idx); @@ -474,392 +562,232 @@ static bool vu_set_vring_addr_exec(VuDev *vdev, return false; } - -static bool vu_set_vring_base_exec(VuDev *vdev, - struct VhostUserMsg *msg) +/** + * vu_set_vring_base_exec() - Sets the next index to use for descriptors + * in this vring + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: False as no reply is requested + */ +static bool vu_set_vring_base_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { - unsigned int index = msg->payload.state.index; + unsigned int idx = msg->payload.state.index; unsigned int num = msg->payload.state.num; - debug("State.index: %u", index); + debug("State.index: %u", idx); debug("State.num: %u", num); - vdev->vq[index].shadow_avail_idx = vdev->vq[index].last_avail_idx = num; + vdev->vq[idx].shadow_avail_idx = vdev->vq[idx].last_avail_idx = num; return false; } -static bool vu_get_vring_base_exec(VuDev *vdev, - struct VhostUserMsg *msg) +/** + * vu_get_vring_base_exec() - Stops the vring and returns the current + * descriptor index or indices + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: True as a reply is requested + */ +static bool vu_get_vring_base_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { - unsigned int index = msg->payload.state.index; + unsigned int idx = msg->payload.state.index; - debug("State.index: %u", index); - msg->payload.state.num = vdev->vq[index].last_avail_idx; + debug("State.index: %u", idx); + msg->payload.state.num = vdev->vq[idx].last_avail_idx; msg->hdr.size = sizeof(msg->payload.state); - vdev->vq[index].started = false; + vdev->vq[idx].started = false; - if (vdev->vq[index].call_fd != -1) { - close(vdev->vq[index].call_fd); - vdev->vq[index].call_fd = -1; + if (vdev->vq[idx].call_fd != -1) { + close(vdev->vq[idx].call_fd); + vdev->vq[idx].call_fd = -1; } - if (vdev->vq[index].kick_fd != -1) { - vu_remove_watch(vdev, vdev->vq[index].kick_fd); - close(vdev->vq[index].kick_fd); - vdev->vq[index].kick_fd = -1; + if (vdev->vq[idx].kick_fd != -1) { + vu_remove_watch(vdev, vdev->vq[idx].kick_fd); + close(vdev->vq[idx].kick_fd); + vdev->vq[idx].kick_fd = -1; } return true; } -static void vu_set_watch(VuDev *vdev, int fd) +/** + * vu_set_watch() - Add a file descriptor to the passt epoll file descriptor + * @vdev: vhost-user device + * @idx: queue index of the file descriptor to add + */ +static void vu_set_watch(const struct vu_dev *vdev, int idx) { - struct ctx *c = (struct ctx *) ((char *)vdev - offsetof(struct ctx, vdev)); - union epoll_ref ref = { .type = EPOLL_TYPE_VHOST_KICK, .fd = fd }; + union epoll_ref ref = { + .type = EPOLL_TYPE_VHOST_KICK, + .fd = vdev->vq[idx].kick_fd, + .queue = idx + }; struct epoll_event ev = { 0 }; ev.data.u64 = ref.u64; ev.events = EPOLLIN; - epoll_ctl(c->epollfd, EPOLL_CTL_ADD, fd, &ev); -} - -int vu_send(const struct ctx *c, const void *buf, size_t size) -{ - VuDev *vdev = (VuDev *)&c->vdev; - size_t hdrlen = vdev->hdrlen; - VuVirtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE]; - unsigned int indexes[VIRTQUEUE_MAX_SIZE]; - size_t lens[VIRTQUEUE_MAX_SIZE]; - size_t offset; - int i, j; - __virtio16 *num_buffers_ptr; - - debug("vu_send size %zu hdrlen %zu", size, hdrlen); - - if (!vu_queue_enabled(vq) || !vu_queue_started(vq)) { - err("Got packet, but no available descriptors on RX virtq."); - return 0; - } - - offset = 0; - i = 0; - num_buffers_ptr = NULL; - while (offset < size) { - VuVirtqElement *elem; - size_t len; - int total; - - total = 0; - - if (i == VIRTQUEUE_MAX_SIZE) { - err("virtio-net unexpected long buffer chain"); - goto err; - } - - elem = vu_queue_pop(vdev, vq, sizeof(VuVirtqElement), - buffer[VHOST_USER_RX_QUEUE]); - if (!elem) { - if (!vdev->broken) { - eventfd_t kick_data; - ssize_t rc; - int status; - - /* wait the kernel to put new entries in the queue */ - - status = fcntl(vq->kick_fd, F_GETFL); - if (status != -1) { - fcntl(vq->kick_fd, F_SETFL, status & ~O_NONBLOCK); - rc = eventfd_read(vq->kick_fd, &kick_data); - fcntl(vq->kick_fd, F_SETFL, status); - if (rc != -1) - continue; - } - } - if (i) { - err("virtio-net unexpected empty queue: " - "i %d mergeable %d offset %zd, size %zd, " - "features 0x%" PRIx64, - i, vu_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF), - offset, size, vdev->features); - } - offset = -1; - goto err; - } - - if (elem->in_num < 1) { - err("virtio-net receive queue contains no in buffers"); - vu_queue_detach_element(vdev, vq, elem->index, 0); - offset = -1; - goto err; - } - - if (i == 0) { - struct virtio_net_hdr hdr = { - .flags = VIRTIO_NET_HDR_F_DATA_VALID, - .gso_type = VIRTIO_NET_HDR_GSO_NONE, - }; - - ASSERT(offset == 0); - ASSERT(elem->in_sg[0].iov_len >= hdrlen); - - len = iov_from_buf(elem->in_sg, elem->in_num, 0, &hdr, sizeof hdr); - - num_buffers_ptr = (__virtio16 *)((char *)elem->in_sg[0].iov_base + - len); - - total += hdrlen; - } - - len = iov_from_buf(elem->in_sg, elem->in_num, total, (char *)buf + offset, - size - offset); - - total += len; - offset += len; - - /* If buffers can't be merged, at this point we - * must have consumed the complete packet. - * Otherwise, drop it. - */ - if (!vu_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) && offset < size) { - vu_queue_unpop(vdev, vq, elem->index, total); - goto err; - } - - indexes[i] = elem->index; - lens[i] = total; - i++; - } - - if (num_buffers_ptr && vu_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) { - *num_buffers_ptr = htole16(i); - } - - for (j = 0; j < i; j++) { - debug("filling total %zd idx %d", lens[j], j); - vu_queue_fill_by_index(vdev, vq, indexes[j], lens[j], j); - } - - vu_queue_flush(vdev, vq, i); - vu_queue_notify(vdev, vq); - - debug("sent %zu", offset); - - return offset; -err: - for (j = 0; j < i; j++) { - vu_queue_detach_element(vdev, vq, indexes[j], lens[j]); - } - - return offset; + epoll_ctl(vdev->context->epollfd, EPOLL_CTL_ADD, ref.fd, &ev); } -size_t tap_send_frames_vu(const struct ctx *c, const struct iovec *iov, size_t n) -{ - size_t i; - int ret; - - debug("tap_send_frames_vu n %zd", n); - - for (i = 0; i < n; i++) { - ret = vu_send(c, iov[i].iov_base, iov[i].iov_len); - if (ret < 0) - break; - } - debug("count %zd", i); - return i; -} - -static void vu_handle_tx(VuDev *vdev, int index) -{ - struct ctx *c = (struct ctx *) ((char *)vdev - offsetof(struct ctx, vdev)); - VuVirtq *vq = &vdev->vq[index]; - int hdrlen = vdev->hdrlen; - struct timespec now; - unsigned int indexes[VIRTQUEUE_MAX_SIZE]; - int count; - - if (index % 2 != VHOST_USER_TX_QUEUE) { - debug("index %d is not an TX queue", index); - return; - } - - clock_gettime(CLOCK_MONOTONIC, &now); - - pool_flush_all(); - - count = 0; - while (1) { - VuVirtqElement *elem; - - ASSERT(index == VHOST_USER_TX_QUEUE); - elem = vu_queue_pop(vdev, vq, sizeof(VuVirtqElement), buffer[index]); - if (!elem) { - break; - } - - if (elem->out_num < 1) { - debug("virtio-net header not in first element"); - break; - } - ASSERT(elem->out_num == 1); - - packet_add_all(c, elem->out_sg[0].iov_len - hdrlen, - (char *)elem->out_sg[0].iov_base + hdrlen); - indexes[count] = elem->index; - count++; - } - tap_handler_all(c, &now); - - if (count) { - int i; - for (i = 0; i < count; i++) - vu_queue_fill_by_index(vdev, vq, indexes[i], 0, i); - vu_queue_flush(vdev, vq, count); - vu_queue_notify(vdev, vq); - } -} - -void vu_kick_cb(struct ctx *c, union epoll_ref ref) -{ - VuDev *vdev = &c->vdev; - eventfd_t kick_data; - ssize_t rc; - int index; - - for (index = 0; index < VHOST_USER_MAX_QUEUES; index++) - if (c->vdev.vq[index].kick_fd == ref.fd) - break; - - if (index == VHOST_USER_MAX_QUEUES) - return; - - rc = eventfd_read(ref.fd, &kick_data); - if (rc == -1) { - vu_panic(vdev, "kick eventfd_read(): %s", strerror(errno)); - vu_remove_watch(vdev, ref.fd); - } else { - debug("Got kick_data: %016"PRIx64" idx:%d", - kick_data, index); - if (index % 2 == VHOST_USER_TX_QUEUE) - vu_handle_tx(vdev, index); - } -} - -static bool vu_check_queue_msg_file(VuDev *vdev, struct VhostUserMsg *msg) +/** + * vu_check_queue_msg_file() - Check if a message is valid, + * close fds if NOFD bit is set + * @vmsg: vhost-user message + */ +static void vu_check_queue_msg_file(struct vhost_user_msg *msg) { - int index = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK; bool nofd = msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK; + int idx = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK; - if (index >= VHOST_USER_MAX_QUEUES) { - vmsg_close_fds(msg); - vu_panic(vdev, "Invalid queue index: %u", index); - return false; - } + if (idx >= VHOST_USER_MAX_QUEUES) + die("Invalid vhost-user queue index: %u", idx); if (nofd) { vmsg_close_fds(msg); - return true; - } - - if (msg->fd_num != 1) { - vmsg_close_fds(msg); - vu_panic(vdev, "Invalid fds in request: %d", msg->hdr.request); - return false; + return; } - return true; + if (msg->fd_num != 1) + die("Invalid fds in vhost-user request: %d", msg->hdr.request); } -static bool vu_set_vring_kick_exec(VuDev *vdev, - struct VhostUserMsg *msg) +/** + * vu_set_vring_kick_exec() - Set the event file descriptor for adding buffers + * to the vring + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: False as no reply is requested + */ +static bool vu_set_vring_kick_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { - int index = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK; bool nofd = msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK; + int idx = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK; debug("u64: 0x%016"PRIx64, msg->payload.u64); - if (!vu_check_queue_msg_file(vdev, msg)) - return false; + vu_check_queue_msg_file(msg); - if (vdev->vq[index].kick_fd != -1) { - vu_remove_watch(vdev, vdev->vq[index].kick_fd); - close(vdev->vq[index].kick_fd); - vdev->vq[index].kick_fd = -1; + if (vdev->vq[idx].kick_fd != -1) { + vu_remove_watch(vdev, vdev->vq[idx].kick_fd); + close(vdev->vq[idx].kick_fd); + vdev->vq[idx].kick_fd = -1; } - vdev->vq[index].kick_fd = nofd ? -1 : msg->fds[0]; - debug("Got kick_fd: %d for vq: %d", vdev->vq[index].kick_fd, index); + if (!nofd) + vdev->vq[idx].kick_fd = msg->fds[0]; - vdev->vq[index].started = true; + debug("Got kick_fd: %d for vq: %d", vdev->vq[idx].kick_fd, idx); - if (vdev->vq[index].kick_fd != -1 && index % 2 == VHOST_USER_TX_QUEUE) { - vu_set_watch(vdev, vdev->vq[index].kick_fd); + vdev->vq[idx].started = true; + + if (vdev->vq[idx].kick_fd != -1 && VHOST_USER_IS_QUEUE_TX(idx)) { + vu_set_watch(vdev, idx); debug("Waiting for kicks on fd: %d for vq: %d", - vdev->vq[index].kick_fd, index); + vdev->vq[idx].kick_fd, idx); } return false; } -static bool vu_set_vring_call_exec(VuDev *vdev, - struct VhostUserMsg *msg) +/** + * vu_set_vring_call_exec() - Set the event file descriptor to signal when + * buffers are used + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: False as no reply is requested + */ +static bool vu_set_vring_call_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { - int index = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK; bool nofd = msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK; + int idx = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK; debug("u64: 0x%016"PRIx64, msg->payload.u64); - if (!vu_check_queue_msg_file(vdev, msg)) - return false; + vu_check_queue_msg_file(msg); - if (vdev->vq[index].call_fd != -1) { - close(vdev->vq[index].call_fd); - vdev->vq[index].call_fd = -1; + if (vdev->vq[idx].call_fd != -1) { + close(vdev->vq[idx].call_fd); + vdev->vq[idx].call_fd = -1; } - vdev->vq[index].call_fd = nofd ? -1 : msg->fds[0]; + if (!nofd) + vdev->vq[idx].call_fd = msg->fds[0]; /* in case of I/O hang after reconnecting */ - if (vdev->vq[index].call_fd != -1) { + if (vdev->vq[idx].call_fd != -1) eventfd_write(msg->fds[0], 1); - } - debug("Got call_fd: %d for vq: %d", vdev->vq[index].call_fd, index); + debug("Got call_fd: %d for vq: %d", vdev->vq[idx].call_fd, idx); return false; } -static bool vu_set_vring_err_exec(VuDev *vdev, - struct VhostUserMsg *msg) +/** + * vu_set_vring_err_exec() - Set the event file descriptor to signal when + * error occurs + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: False as no reply is requested + */ +static bool vu_set_vring_err_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { - int index = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK; bool nofd = msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK; + int idx = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK; debug("u64: 0x%016"PRIx64, msg->payload.u64); - if (!vu_check_queue_msg_file(vdev, msg)) - return false; + vu_check_queue_msg_file(msg); - if (vdev->vq[index].err_fd != -1) { - close(vdev->vq[index].err_fd); - vdev->vq[index].err_fd = -1; + if (vdev->vq[idx].err_fd != -1) { + close(vdev->vq[idx].err_fd); + vdev->vq[idx].err_fd = -1; } - vdev->vq[index].err_fd = nofd ? -1 : msg->fds[0]; + if (!nofd) + vdev->vq[idx].err_fd = msg->fds[0]; return false; } -static bool vu_get_protocol_features_exec(struct VhostUserMsg *msg) +/** + * vu_get_protocol_features_exec() - Provide the protocol (vhost-user) features + * to the front-end + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: True as a reply is requested + */ +static bool vu_get_protocol_features_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { uint64_t features = 1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK; + (void)vdev; vmsg_set_reply_u64(msg, features); return true; } -static bool vu_set_protocol_features_exec(VuDev *vdev, struct VhostUserMsg *msg) +/** + * vu_set_protocol_features_exec() - Enable protocol (vhost-user) features + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: False as no reply is requested + */ +static bool vu_set_protocol_features_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { uint64_t features = msg->payload.u64; @@ -867,72 +795,80 @@ static bool vu_set_protocol_features_exec(VuDev *vdev, struct VhostUserMsg *msg) vdev->protocol_features = msg->payload.u64; - if (vu_has_protocol_feature(vdev, - VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS) && - (!vu_has_protocol_feature(vdev, VHOST_USER_PROTOCOL_F_BACKEND_REQ) || - !vu_has_protocol_feature(vdev, VHOST_USER_PROTOCOL_F_REPLY_ACK))) { - /* - * The use case for using messages for kick/call is simulation, to make - * the kick and call synchronous. To actually get that behaviour, both - * of the other features are required. - * Theoretically, one could use only kick messages, or do them without - * having F_REPLY_ACK, but too many (possibly pending) messages on the - * socket will eventually cause the master to hang, to avoid this in - * scenarios where not desired enforce that the settings are in a way - * that actually enables the simulation case. - */ - vu_panic(vdev, - "F_IN_BAND_NOTIFICATIONS requires F_BACKEND_REQ && F_REPLY_ACK"); - return false; - } - return false; } - -static bool vu_get_queue_num_exec(struct VhostUserMsg *msg) +/** + * vu_get_queue_num_exec() - Tell how many queues we support + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: True as a reply is requested + */ +static bool vu_get_queue_num_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { + (void)vdev; + vmsg_set_reply_u64(msg, VHOST_USER_MAX_QUEUES); + return true; } -static bool vu_set_vring_enable_exec(VuDev *vdev, struct VhostUserMsg *msg) +/** + * vu_set_vring_enable_exec() - Enable or disable corresponding vring + * @vdev: vhost-user device + * @vmsg: vhost-user message + * + * Return: False as no reply is requested + */ +static bool vu_set_vring_enable_exec(struct vu_dev *vdev, + struct vhost_user_msg *msg) { - unsigned int index = msg->payload.state.index; unsigned int enable = msg->payload.state.num; + unsigned int idx = msg->payload.state.index; - debug("State.index: %u", index); + debug("State.index: %u", idx); debug("State.enable: %u", enable); - if (index >= VHOST_USER_MAX_QUEUES) { - vu_panic(vdev, "Invalid vring_enable index: %u", index); - return false; - } + if (idx >= VHOST_USER_MAX_QUEUES) + die("Invalid vring_enable index: %u", idx); - vdev->vq[index].enable = enable; + vdev->vq[idx].enable = enable; return false; } +/** + * vu_init() - Initialize vhost-user device structure + * @c: execution context + * @vdev: vhost-user device + */ void vu_init(struct ctx *c) { int i; - c->vdev.hdrlen = 0; - for (i = 0; i < VHOST_USER_MAX_QUEUES; i++) - c->vdev.vq[i] = (VuVirtq){ + c->vdev = &vdev_storage; + c->vdev->context = c; + for (i = 0; i < VHOST_USER_MAX_QUEUES; i++) { + c->vdev->vq[i] = (struct vu_virtq){ .call_fd = -1, .kick_fd = -1, .err_fd = -1, .notification = true, }; + } } -static void vu_cleanup(VuDev *vdev) +/** + * vu_cleanup() - Reset vhost-user device + * @vdev: vhost-user device + */ +void vu_cleanup(struct vu_dev *vdev) { unsigned int i; for (i = 0; i < VHOST_USER_MAX_QUEUES; i++) { - VuVirtq *vq = &vdev->vq[i]; + struct vu_virtq *vq = &vdev->vq[i]; vq->started = false; vq->notification = true; @@ -946,7 +882,7 @@ static void vu_cleanup(VuDev *vdev) vq->err_fd = -1; } if (vq->kick_fd != -1) { - vu_remove_watch(vdev, vq->kick_fd); + vu_remove_watch(vdev, vq->kick_fd); close(vq->kick_fd); vq->kick_fd = -1; } @@ -955,40 +891,66 @@ static void vu_cleanup(VuDev *vdev) vq->vring.used = 0; vq->vring.avail = 0; } - vdev->hdrlen = 0; for (i = 0; i < vdev->nregions; i++) { - VuDevRegion *r = &vdev->regions[i]; - void *m = (void *) (uintptr_t) r->mmap_addr; + const struct vu_dev_region *r = &vdev->regions[i]; - if (m) - munmap(m, r->size + r->mmap_offset); + if (r->mmap_addr) { + /* NOLINTNEXTLINE(performance-no-int-to-ptr) */ + munmap((void *)r->mmap_addr, r->size + r->mmap_offset); + } } vdev->nregions = 0; } /** - * tap_handler_vu() - Packet handler for vhost-user - * @c: Execution context + * vu_sock_reset() - Reset connection socket + * @vdev: vhost-user device + */ +static void vu_sock_reset(struct vu_dev *vdev) +{ + tap_sock_reset(vdev->context); +} + +static bool (*vu_handle[VHOST_USER_MAX])(struct vu_dev *vdev, + struct vhost_user_msg *msg) = { + [VHOST_USER_GET_FEATURES] = vu_get_features_exec, + [VHOST_USER_SET_FEATURES] = vu_set_features_exec, + [VHOST_USER_GET_PROTOCOL_FEATURES] = vu_get_protocol_features_exec, + [VHOST_USER_SET_PROTOCOL_FEATURES] = vu_set_protocol_features_exec, + [VHOST_USER_GET_QUEUE_NUM] = vu_get_queue_num_exec, + [VHOST_USER_SET_OWNER] = vu_set_owner_exec, + [VHOST_USER_SET_MEM_TABLE] = vu_set_mem_table_exec, + [VHOST_USER_SET_VRING_NUM] = vu_set_vring_num_exec, + [VHOST_USER_SET_VRING_ADDR] = vu_set_vring_addr_exec, + [VHOST_USER_SET_VRING_BASE] = vu_set_vring_base_exec, + [VHOST_USER_GET_VRING_BASE] = vu_get_vring_base_exec, + [VHOST_USER_SET_VRING_KICK] = vu_set_vring_kick_exec, + [VHOST_USER_SET_VRING_CALL] = vu_set_vring_call_exec, + [VHOST_USER_SET_VRING_ERR] = vu_set_vring_err_exec, + [VHOST_USER_SET_VRING_ENABLE] = vu_set_vring_enable_exec, +}; + +/** + * vu_control_handler() - Handle control commands for vhost-user + * @vdev: vhost-user device + * @fd: vhost-user message socket * @events: epoll events */ -void tap_handler_vu(struct ctx *c, uint32_t events) +void vu_control_handler(struct vu_dev *vdev, int fd, uint32_t events) { - VuDev *dev = &c->vdev; - struct VhostUserMsg msg = { 0 }; + struct vhost_user_msg msg = { 0 }; bool need_reply, reply_requested; int ret; if (events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)) { - tap_sock_reset(c); + vu_sock_reset(vdev); return; } - - ret = vu_message_read_default(dev, c->fd_tap, &msg); - if (ret <= 0) { - if (errno != EINTR && errno != EAGAIN && errno != EWOULDBLOCK) - tap_sock_reset(c); + ret = vu_message_read_default(fd, &msg); + if (ret == 0) { + vu_sock_reset(vdev); return; } debug("================ Vhost user message ================"); @@ -998,60 +960,14 @@ void tap_handler_vu(struct ctx *c, uint32_t events) debug("Size: %u", msg.hdr.size); need_reply = msg.hdr.flags & VHOST_USER_NEED_REPLY_MASK; - switch (msg.hdr.request) { - case VHOST_USER_GET_FEATURES: - reply_requested = vu_get_features_exec(&msg); - break; - case VHOST_USER_SET_FEATURES: - reply_requested = vu_set_features_exec(dev, &msg); - break; - case VHOST_USER_GET_PROTOCOL_FEATURES: - reply_requested = vu_get_protocol_features_exec(&msg); - break; - case VHOST_USER_SET_PROTOCOL_FEATURES: - reply_requested = vu_set_protocol_features_exec(dev, &msg); - break; - case VHOST_USER_GET_QUEUE_NUM: - reply_requested = vu_get_queue_num_exec(&msg); - break; - case VHOST_USER_SET_OWNER: - reply_requested = vu_set_owner_exec(); - break; - case VHOST_USER_SET_MEM_TABLE: - reply_requested = vu_set_mem_table_exec(dev, &msg); - break; - case VHOST_USER_SET_VRING_NUM: - reply_requested = vu_set_vring_num_exec(dev, &msg); - break; - case VHOST_USER_SET_VRING_ADDR: - reply_requested = vu_set_vring_addr_exec(dev, &msg); - break; - case VHOST_USER_SET_VRING_BASE: - reply_requested = vu_set_vring_base_exec(dev, &msg); - break; - case VHOST_USER_GET_VRING_BASE: - reply_requested = vu_get_vring_base_exec(dev, &msg); - break; - case VHOST_USER_SET_VRING_KICK: - reply_requested = vu_set_vring_kick_exec(dev, &msg); - break; - case VHOST_USER_SET_VRING_CALL: - reply_requested = vu_set_vring_call_exec(dev, &msg); - break; - case VHOST_USER_SET_VRING_ERR: - reply_requested = vu_set_vring_err_exec(dev, &msg); - break; - case VHOST_USER_SET_VRING_ENABLE: - reply_requested = vu_set_vring_enable_exec(dev, &msg); - break; - case VHOST_USER_NONE: - vu_cleanup(dev); - return; - default: - vu_panic(dev, "Unhandled request: %d", msg.hdr.request); - return; - } + if (msg.hdr.request >= 0 && msg.hdr.request < VHOST_USER_MAX && + vu_handle[msg.hdr.request]) + reply_requested = vu_handle[msg.hdr.request](vdev, &msg); + else + die("Unhandled request: %d", msg.hdr.request); + + /* cppcheck-suppress legacyUninitvar */ if (!reply_requested && need_reply) { msg.payload.u64 = 0; msg.hdr.flags = 0; @@ -1061,6 +977,5 @@ void tap_handler_vu(struct ctx *c, uint32_t events) } if (reply_requested) - ret = vu_send_reply(dev, c->fd_tap, &msg); - free(msg.data); + vu_send_reply(fd, &msg); } diff --git a/vhost_user.h b/vhost_user.h index 25f0b61..464ba21 100644 --- a/vhost_user.h +++ b/vhost_user.h @@ -1,4 +1,10 @@ // SPDX-License-Identifier: GPL-2.0-or-later +/* + * vhost-user API, command management and virtio interface + * + * Copyright Red Hat + * Author: Laurent Vivier <lvivier@redhat.com> + */ /* some parts from subprojects/libvhost-user/libvhost-user.h */ @@ -12,6 +18,9 @@ #define VHOST_MEMORY_BASELINE_NREGIONS 8 +/** + * enum vhost_user_protocol_feature - List of available vhost-user features + */ enum vhost_user_protocol_feature { VHOST_USER_PROTOCOL_F_MQ = 0, VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1, @@ -32,6 +41,9 @@ enum vhost_user_protocol_feature { VHOST_USER_PROTOCOL_F_MAX }; +/** + * enum vhost_user_request - List of available vhost-user requests + */ enum vhost_user_request { VHOST_USER_NONE = 0, VHOST_USER_GET_FEATURES = 1, @@ -74,66 +86,121 @@ enum vhost_user_request { VHOST_USER_MAX }; -typedef struct { +/** + * struct vhost_user_header - vhost-user message header + * @request: Request type of the message + * @flags: Request flags + * @size: The following payload size + */ +struct vhost_user_header { enum vhost_user_request request; #define VHOST_USER_VERSION_MASK 0x3 #define VHOST_USER_REPLY_MASK (0x1 << 2) #define VHOST_USER_NEED_REPLY_MASK (0x1 << 3) uint32_t flags; - uint32_t size; /* the following payload size */ -} __attribute__ ((__packed__)) vhost_user_header; - -typedef struct VhostUserMemory_region { + uint32_t size; +} __attribute__ ((__packed__)); + +/** + * struct vhost_user_memory_region - Front-end shared memory region information + * @guest_phys_addr: Guest physical address of the region + * @memory_size: Memory size + * @userspace_addr: front-end (QEMU) userspace address + * @mmap_offset: region offset in the shared memory area + */ +struct vhost_user_memory_region { uint64_t guest_phys_addr; uint64_t memory_size; uint64_t userspace_addr; uint64_t mmap_offset; -} VhostUserMemory_region; +}; -struct VhostUserMemory { +/** + * struct vhost_user_memory - List of all the shared memory regions + * @nregions: Number of memory regions + * @padding: Padding + * @regions: Memory regions list + */ +struct vhost_user_memory { uint32_t nregions; uint32_t padding; - struct VhostUserMemory_region regions[VHOST_MEMORY_BASELINE_NREGIONS]; + struct vhost_user_memory_region regions[VHOST_MEMORY_BASELINE_NREGIONS]; }; -typedef union { +/** + * union vhost_user_payload - vhost-user message payload + * @u64: 64-bit payload + * @state: vring state payload + * @addr: vring addresses payload + * vhost_user_memory: Memory regions information payload + */ +union vhost_user_payload { #define VHOST_USER_VRING_IDX_MASK 0xff #define VHOST_USER_VRING_NOFD_MASK (0x1 << 8) uint64_t u64; struct vhost_vring_state state; struct vhost_vring_addr addr; - struct VhostUserMemory memory; -} vhost_user_payload; + struct vhost_user_memory memory; +}; -typedef struct VhostUserMsg { - vhost_user_header hdr; - vhost_user_payload payload; +/** + * struct vhost_user_msg - vhost-use message + * @hdr: Message header + * @payload: Message payload + * @fds: File descriptors associated with the message + * in the ancillary data. + * (shared memory or event file descriptors) + * @fd_num: Number of file descriptors + */ +struct vhost_user_msg { + struct vhost_user_header hdr; + union vhost_user_payload payload; int fds[VHOST_MEMORY_BASELINE_NREGIONS]; int fd_num; - uint8_t *data; -} __attribute__ ((__packed__)) VhostUserMsg; -#define VHOST_USER_HDR_SIZE sizeof(vhost_user_header) +} __attribute__ ((__packed__)); +#define VHOST_USER_HDR_SIZE sizeof(struct vhost_user_header) +/* index of the RX virtqueue */ #define VHOST_USER_RX_QUEUE 0 +/* index of the TX virtqueue */ #define VHOST_USER_TX_QUEUE 1 -static inline bool vu_queue_enabled(VuVirtq *vq) +/* in case of multiqueue, the RX and TX queues are interleaved */ +#define VHOST_USER_IS_QUEUE_TX(n) (n % 2) +#define VHOST_USER_IS_QUEUE_RX(n) (!(n % 2)) + +/* Default virtio-net header for passt */ +#define VU_HEADER ((struct virtio_net_hdr){ \ + .flags = VIRTIO_NET_HDR_F_DATA_VALID, \ + .gso_type = VIRTIO_NET_HDR_GSO_NONE, \ +}) + +/** + * vu_queue_enabled - Return state of a virtqueue + * @vq: virtqueue to check + * + * Return: true if the virqueue is enabled, false otherwise + */ +static inline bool vu_queue_enabled(const struct vu_virtq *vq) { return vq->enable; } -static inline bool vu_queue_started(const VuVirtq *vq) +/** + * vu_queue_started - Return state of a virtqueue + * @vq: virtqueue to check + * + * Return: true if the virqueue is started, false otherwise + */ +static inline bool vu_queue_started(const struct vu_virtq *vq) { return vq->started; } -size_t tap_send_frames_vu(const struct ctx *c, const struct iovec *iov, - size_t n); -int vu_send(const struct ctx *c, const void *data, size_t len); void vu_print_capabilities(void); void vu_init(struct ctx *c); -void vu_kick_cb(struct ctx *c, union epoll_ref ref); -void tap_handler_vu(struct ctx *c, uint32_t events); +void vu_cleanup(struct vu_dev *vdev); +void vu_control_handler(struct vu_dev *vdev, int fd, uint32_t events); #endif /* VHOST_USER_H */ @@ -1,6 +1,76 @@ -// SPDX-License-Identifier: GPL-2.0-or-later - -/* some parts copied from QEMU subprojects/libvhost-user/libvhost-user.c */ +// SPDX-License-Identifier: GPL-2.0-or-later AND BSD-3-Clause +/* + * virtio API, vring and virtqueue functions definition + * + * Copyright Red Hat + * Author: Laurent Vivier <lvivier@redhat.com> + */ + +/* Some parts copied from QEMU subprojects/libvhost-user/libvhost-user.c + * originally licensed under the following terms: + * + * -- + * + * Copyright IBM, Corp. 2007 + * Copyright (c) 2016 Red Hat, Inc. + * + * Authors: + * Anthony Liguori <aliguori@us.ibm.com> + * Marc-André Lureau <mlureau@redhat.com> + * Victor Kaplansky <victork@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * later. See the COPYING file in the top-level directory. + * + * Some parts copied from QEMU hw/virtio/virtio.c + * licensed under the following terms: + * + * Copyright IBM, Corp. 2007 + * + * Authors: + * Anthony Liguori <aliguori@us.ibm.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * -- + * + * virtq_used_event() and virtq_avail_event() from + * https://docs.oasis-open.org/virtio/virtio/v1.2/csd01/virtio-v1.2-csd01.html#x1-712000A + * licensed under the following terms: + * + * -- + * + * This header is BSD licensed so anyone can use the definitions + * to implement compatible drivers/servers. + * + * Copyright 2007, 2009, IBM Corporation + * Copyright 2011, Red Hat, Inc + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of IBM nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ‘‘AS IS’’ AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ #include <stddef.h> #include <endian.h> @@ -14,143 +84,223 @@ #define VIRTQUEUE_MAX_SIZE 1024 -/* Translate guest physical address to our virtual address. */ -static void *vu_gpa_to_va(VuDev *dev, uint64_t *plen, uint64_t guest_addr) +/** + * vu_gpa_to_va() - Translate guest physical address to our virtual address. + * @dev: Vhost-user device + * @plen: Physical length to map (input), capped to region (output) + * @guest_addr: Guest physical address + * + * Return: virtual address in our address space of the guest physical address + */ +static void *vu_gpa_to_va(struct vu_dev *dev, uint64_t *plen, uint64_t guest_addr) { unsigned int i; - if (*plen == 0) { + if (*plen == 0) return NULL; - } - /* Find matching memory region. */ + /* Find matching memory region. */ for (i = 0; i < dev->nregions; i++) { - VuDevRegion *r = &dev->regions[i]; + const struct vu_dev_region *r = &dev->regions[i]; - if ((guest_addr >= r->gpa) && (guest_addr < (r->gpa + r->size))) { - if ((guest_addr + *plen) > (r->gpa + r->size)) { + if ((guest_addr >= r->gpa) && + (guest_addr < (r->gpa + r->size))) { + if ((guest_addr + *plen) > (r->gpa + r->size)) *plen = r->gpa + r->size - guest_addr; - } - return (void *)(guest_addr - (uintptr_t)r->gpa + - (uintptr_t)r->mmap_addr + r->mmap_offset); + /* NOLINTNEXTLINE(performance-no-int-to-ptr) */ + return (void *)(guest_addr - r->gpa + r->mmap_addr + + r->mmap_offset); } } return NULL; } -static inline uint16_t vring_avail_flags(VuVirtq *vq) +/** + * vring_avail_flags() - Read the available ring flags + * @vq: Virtqueue + * + * Return: the available ring descriptor flags of the given virtqueue + */ +static inline uint16_t vring_avail_flags(const struct vu_virtq *vq) { return le16toh(vq->vring.avail->flags); } -static inline uint16_t vring_avail_idx(VuVirtq *vq) +/** + * vring_avail_idx() - Read the available ring index + * @vq: Virtqueue + * + * Return: the available ring index of the given virtqueue + */ +static inline uint16_t vring_avail_idx(struct vu_virtq *vq) { vq->shadow_avail_idx = le16toh(vq->vring.avail->idx); return vq->shadow_avail_idx; } -static inline uint16_t vring_avail_ring(VuVirtq *vq, int i) +/** + * vring_avail_ring() - Read an available ring entry + * @vq: Virtqueue + * @i: Index of the entry to read + * + * Return: the ring entry content (head of the descriptor chain) + */ +static inline uint16_t vring_avail_ring(const struct vu_virtq *vq, int i) { return le16toh(vq->vring.avail->ring[i]); } -static inline uint16_t vring_get_used_event(VuVirtq *vq) +/** + * virtq_used_event - Get location of used event indices + * (only with VIRTIO_F_EVENT_IDX) + * @vq Virtqueue + * + * Return: return the location of the used event index + */ +static inline uint16_t *virtq_used_event(const struct vu_virtq *vq) +{ + /* For backwards compat, used event index is at *end* of avail ring. */ + return &vq->vring.avail->ring[vq->vring.num]; +} + +/** + * vring_get_used_event() - Get the used event from the available ring + * @vq Virtqueue + * + * Return: the used event (available only if VIRTIO_RING_F_EVENT_IDX is set) + * used_event is a performant alternative where the driver + * specifies how far the device can progress before a notification + * is required. + */ +static inline uint16_t vring_get_used_event(const struct vu_virtq *vq) { - return vring_avail_ring(vq, vq->vring.num); + return le16toh(*virtq_used_event(vq)); } -static bool virtqueue_get_head(VuDev *dev, VuVirtq *vq, - unsigned int idx, unsigned int *head) +/** + * virtqueue_get_head() - Get the head of the descriptor chain for a given + * index + * @vq: Virtqueue + * @idx: Available ring entry index + * @head: Head of the descriptor chain + */ +static void virtqueue_get_head(const struct vu_virtq *vq, + unsigned int idx, unsigned int *head) { /* Grab the next descriptor number they're advertising, and increment - * the index we've seen. */ + * the index we've seen. + */ *head = vring_avail_ring(vq, idx % vq->vring.num); /* If their number is silly, that's a fatal mistake. */ - if (*head >= vq->vring.num) { - vu_panic(dev, "Guest says index %u is available", *head); - return false; - } - - return true; + if (*head >= vq->vring.num) + die("vhost-user: Guest says index %u is available", *head); } -static int -virtqueue_read_indirect_desc(VuDev *dev, struct vring_desc *desc, - uint64_t addr, size_t len) +/** + * virtqueue_read_indirect_desc() - Copy virtio ring descriptors from guest + * memory + * @dev: Vhost-user device + * @desc: Destination address to copy the descriptors to + * @addr: Guest memory address to copy from + * @len: Length of memory to copy + * + * Return: -1 if there is an error, 0 otherwise + */ +static int virtqueue_read_indirect_desc(struct vu_dev *dev, struct vring_desc *desc, + uint64_t addr, size_t len) { - struct vring_desc *ori_desc; uint64_t read_len; - if (len > (VIRTQUEUE_MAX_SIZE * sizeof(struct vring_desc))) { + if (len > (VIRTQUEUE_MAX_SIZE * sizeof(struct vring_desc))) return -1; - } - if (len == 0) { + if (len == 0) return -1; - } while (len) { + const struct vring_desc *orig_desc; + read_len = len; - ori_desc = vu_gpa_to_va(dev, &read_len, addr); - if (!ori_desc) { + orig_desc = vu_gpa_to_va(dev, &read_len, addr); + if (!orig_desc) return -1; - } - memcpy(desc, ori_desc, read_len); + memcpy(desc, orig_desc, read_len); len -= read_len; addr += read_len; - desc += read_len; + desc += read_len / sizeof(struct vring_desc); } return 0; } -enum { +/** + * enum virtqueue_read_desc_state - State in the descriptor chain + * @VIRTQUEUE_READ_DESC_ERROR Found an invalid descriptor + * @VIRTQUEUE_READ_DESC_DONE No more descriptors in the chain + * @VIRTQUEUE_READ_DESC_MORE there are more descriptors in the chain + */ +enum virtqueue_read_desc_state { VIRTQUEUE_READ_DESC_ERROR = -1, VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */ VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */ }; -static int -virtqueue_read_next_desc(VuDev *dev, struct vring_desc *desc, - int i, unsigned int max, unsigned int *next) +/** + * virtqueue_read_next_desc() - Read the the next descriptor in the chain + * @desc: Virtio ring descriptors + * @i: Index of the current descriptor + * @max: Maximum value of the descriptor index + * @next: Index of the next descriptor in the chain (output value) + * + * Return: current chain descriptor state (error, next, done) + */ +static int virtqueue_read_next_desc(const struct vring_desc *desc, + int i, unsigned int max, unsigned int *next) { /* If this descriptor says it doesn't chain, we're done. */ - if (!(le16toh(desc[i].flags) & VRING_DESC_F_NEXT)) { + if (!(le16toh(desc[i].flags) & VRING_DESC_F_NEXT)) return VIRTQUEUE_READ_DESC_DONE; - } /* Check they're not leading us off end of descriptors. */ *next = le16toh(desc[i].next); /* Make sure compiler knows to grab that: we don't want it changing! */ smp_wmb(); - if (*next >= max) { - vu_panic(dev, "Desc next is %u", *next); + if (*next >= max) return VIRTQUEUE_READ_DESC_ERROR; - } return VIRTQUEUE_READ_DESC_MORE; } -bool vu_queue_empty(VuDev *dev, VuVirtq *vq) +/** + * vu_queue_empty() - Check if virtqueue is empty + * @vq: Virtqueue + * + * Return: true if the virtqueue is empty, false otherwise + */ +bool vu_queue_empty(struct vu_virtq *vq) { - if (dev->broken || - !vq->vring.avail) { + if (!vq->vring.avail) return true; - } - if (vq->shadow_avail_idx != vq->last_avail_idx) { + if (vq->shadow_avail_idx != vq->last_avail_idx) return false; - } return vring_avail_idx(vq) == vq->last_avail_idx; } -static bool vring_notify(VuDev *dev, VuVirtq *vq) +/** + * vring_can_notify() - Check if a notification can be sent + * @dev: Vhost-user device + * @vq: Virtqueue + * + * Return: true if notification can be sent + */ +static bool vring_can_notify(const struct vu_dev *dev, struct vu_virtq *vq) { uint16_t old, new; bool v; @@ -160,13 +310,11 @@ static bool vring_notify(VuDev *dev, VuVirtq *vq) /* Always notify when queue is empty (when feature acknowledge) */ if (vu_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY) && - !vq->inuse && vu_queue_empty(dev, vq)) { + !vq->inuse && vu_queue_empty(vq)) return true; - } - if (!vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) { + if (!vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT); - } v = vq->signalled_used_valid; vq->signalled_used_valid = true; @@ -175,60 +323,88 @@ static bool vring_notify(VuDev *dev, VuVirtq *vq) return !v || vring_need_event(vring_get_used_event(vq), new, old); } -void vu_queue_notify(VuDev *dev, VuVirtq *vq) +/** + * vu_queue_notify() - Send a notification to the given virtqueue + * @dev: Vhost-user device + * @vq: Virtqueue + */ +void vu_queue_notify(const struct vu_dev *dev, struct vu_virtq *vq) { - if (dev->broken || !vq->vring.avail) { + if (!vq->vring.avail) return; - } - if (!vring_notify(dev, vq)) { - debug("skipped notify..."); + if (!vring_can_notify(dev, vq)) { + debug("vhost-user: virtqueue can skip notify..."); return; } - if (eventfd_write(vq->call_fd, 1) < 0) { - vu_panic(dev, "Error writing eventfd: %s", strerror(errno)); - } + if (eventfd_write(vq->call_fd, 1) < 0) + die_perror("Error writing vhost-user queue eventfd"); +} + +/* virtq_avail_event() - Get location of available event indices + * (only with VIRTIO_F_EVENT_IDX) + * @vq: Virtqueue + * + * Return: return the location of the available event index + */ +static inline uint16_t *virtq_avail_event(const struct vu_virtq *vq) +{ + /* For backwards compat, avail event index is at *end* of used ring. */ + return (uint16_t *)&vq->vring.used->ring[vq->vring.num]; } -static inline void vring_set_avail_event(VuVirtq *vq, uint16_t val) +/** + * vring_set_avail_event() - Set avail_event + * @vq: Virtqueue + * @val: Value to set to avail_event + * avail_event is used in the same way the used_event is in the + * avail_ring. + * avail_event is used to advise the driver that notifications + * are unnecessary until the driver writes entry with an index + * specified by avail_event into the available ring. + */ +static inline void vring_set_avail_event(const struct vu_virtq *vq, + uint16_t val) { uint16_t val_le = htole16(val); - if (!vq->notification) { + if (!vq->notification) return; - } - memcpy(&vq->vring.used->ring[vq->vring.num], &val_le, sizeof(uint16_t)); + memcpy(virtq_avail_event(vq), &val_le, sizeof(val_le)); } -static bool virtqueue_map_desc(VuDev *dev, +/** + * virtqueue_map_desc() - Translate descriptor ring physical address into our + * virtual address space + * @dev: Vhost-user device + * @p_num_sg: First iov entry to use (input), + * first iov entry not used (output) + * @iov: Iov array to use to store buffer virtual addresses + * @max_num_sg: Maximum number of iov entries + * @pa: Guest physical address of the buffer to map into our virtual + * address + * @sz: Size of the buffer + * + * Return: false on error, true otherwise + */ +static bool virtqueue_map_desc(struct vu_dev *dev, unsigned int *p_num_sg, struct iovec *iov, unsigned int max_num_sg, uint64_t pa, size_t sz) { - unsigned num_sg = *p_num_sg; + unsigned int num_sg = *p_num_sg; - ASSERT(num_sg <= max_num_sg); - - if (!sz) { - vu_panic(dev, "virtio: zero sized buffers are not allowed"); - return false; - } + ASSERT(num_sg < max_num_sg); + ASSERT(sz); while (sz) { uint64_t len = sz; - if (num_sg == max_num_sg) { - vu_panic(dev, "virtio: too many descriptors in indirect table"); - return false; - } - iov[num_sg].iov_base = vu_gpa_to_va(dev, &len, pa); - if (iov[num_sg].iov_base == NULL) { - vu_panic(dev, "virtio: invalid address for buffers"); - return false; - } + if (iov[num_sg].iov_base == NULL) + die("vhost-user: invalid address for buffers"); iov[num_sg].iov_len = len; num_sg++; sz -= len; @@ -239,43 +415,33 @@ static bool virtqueue_map_desc(VuDev *dev, return true; } -static void * virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num, unsigned char *buffer) +/** + * vu_queue_map_desc - Map the virtqueue descriptor ring into our virtual + * address space + * @dev: Vhost-user device + * @vq: Virtqueue + * @idx: First descriptor ring entry to map + * @elem: Virtqueue element to store descriptor ring iov + * + * Return: -1 if there is an error, 0 otherwise + */ +static int vu_queue_map_desc(struct vu_dev *dev, struct vu_virtq *vq, unsigned int idx, + struct vu_virtq_element *elem) { - VuVirtqElement *elem; - size_t in_sg_ofs = ALIGN_UP(sz, __alignof__(elem->in_sg[0])); - size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]); - size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]); - - if (out_sg_end > 65536) - return NULL; - - elem = (void *)buffer; - elem->out_num = out_num; - elem->in_num = in_num; - elem->in_sg = (struct iovec *)((uintptr_t)elem + in_sg_ofs); - elem->out_sg = (struct iovec *)((uintptr_t)elem + out_sg_ofs); - return elem; -} - -static void * -vu_queue_map_desc(VuDev *dev, VuVirtq *vq, unsigned int idx, size_t sz, unsigned char *buffer) -{ - struct vring_desc *desc = vq->vring.desc; - uint64_t desc_addr, read_len; - unsigned int desc_len; + const struct vring_desc *desc = vq->vring.desc; + struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE]; + unsigned int out_num = 0, in_num = 0; unsigned int max = vq->vring.num; unsigned int i = idx; - VuVirtqElement *elem; - unsigned int out_num = 0, in_num = 0; - struct iovec iov[VIRTQUEUE_MAX_SIZE]; - struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE]; + uint64_t read_len; int rc; if (le16toh(desc[i].flags) & VRING_DESC_F_INDIRECT) { - if (le32toh(desc[i].len) % sizeof(struct vring_desc)) { - vu_panic(dev, "Invalid size for indirect buffer table"); - return NULL; - } + unsigned int desc_len; + uint64_t desc_addr; + + if (le32toh(desc[i].len) % sizeof(struct vring_desc)) + die("vhost-user: Invalid size for indirect buffer table"); /* loop over the indirect descriptor table */ desc_addr = le64toh(desc[i].addr); @@ -286,152 +452,155 @@ vu_queue_map_desc(VuDev *dev, VuVirtq *vq, unsigned int idx, size_t sz, unsigned if (desc && read_len != desc_len) { /* Failed to use zero copy */ desc = NULL; - if (!virtqueue_read_indirect_desc(dev, desc_buf, desc_addr, desc_len)) { + if (!virtqueue_read_indirect_desc(dev, desc_buf, desc_addr, desc_len)) desc = desc_buf; - } - } - if (!desc) { - vu_panic(dev, "Invalid indirect buffer table"); - return NULL; } + if (!desc) + die("vhost-user: Invalid indirect buffer table"); i = 0; } /* Collect all the descriptors */ do { if (le16toh(desc[i].flags) & VRING_DESC_F_WRITE) { - if (!virtqueue_map_desc(dev, &in_num, iov + out_num, - VIRTQUEUE_MAX_SIZE - out_num, + if (!virtqueue_map_desc(dev, &in_num, elem->in_sg, + elem->in_num, le64toh(desc[i].addr), - le32toh(desc[i].len))) { - return NULL; - } + le32toh(desc[i].len))) + return -1; } else { - if (in_num) { - vu_panic(dev, "Incorrect order for descriptors"); - return NULL; - } - if (!virtqueue_map_desc(dev, &out_num, iov, - VIRTQUEUE_MAX_SIZE, + if (in_num) + die("Incorrect order for descriptors"); + if (!virtqueue_map_desc(dev, &out_num, elem->out_sg, + elem->out_num, le64toh(desc[i].addr), le32toh(desc[i].len))) { - return NULL; + return -1; } } /* If we've got too many, that implies a descriptor loop. */ - if ((in_num + out_num) > max) { - vu_panic(dev, "Looped descriptor"); - return NULL; - } - rc = virtqueue_read_next_desc(dev, desc, i, max, &i); + if ((in_num + out_num) > max) + die("vhost-user: Loop in queue descriptor list"); + rc = virtqueue_read_next_desc(desc, i, max, &i); } while (rc == VIRTQUEUE_READ_DESC_MORE); - if (rc == VIRTQUEUE_READ_DESC_ERROR) { - vu_panic(dev, "read descriptor error"); - return NULL; - } + if (rc == VIRTQUEUE_READ_DESC_ERROR) + die("vhost-user: Failed to read descriptor list"); - /* Now copy what we have collected and mapped */ - elem = virtqueue_alloc_element(sz, out_num, in_num, buffer); - if (!elem) { - return NULL; - } elem->index = idx; - for (i = 0; i < out_num; i++) { - elem->out_sg[i] = iov[i]; - } - for (i = 0; i < in_num; i++) { - elem->in_sg[i] = iov[out_num + i]; - } + elem->in_num = in_num; + elem->out_num = out_num; - return elem; + return 0; } -void *vu_queue_pop(VuDev *dev, VuVirtq *vq, size_t sz, unsigned char *buffer) +/** + * vu_queue_pop() - Pop an entry from the virtqueue + * @dev: Vhost-user device + * @vq: Virtqueue + * @elem: Virtqueue element to file with the entry information + * + * Return: -1 if there is an error, 0 otherwise + */ +int vu_queue_pop(struct vu_dev *dev, struct vu_virtq *vq, struct vu_virtq_element *elem) { unsigned int head; - VuVirtqElement *elem; + int ret; - if (dev->broken || !vq->vring.avail) { - return NULL; - } + if (!vq->vring.avail) + return -1; - if (vu_queue_empty(dev, vq)) { - return NULL; - } - /* - * Needed after virtio_queue_empty(), see comment in + if (vu_queue_empty(vq)) + return -1; + + /* Needed after vu_queue_empty(), see comment in * virtqueue_num_heads(). */ smp_rmb(); - if (vq->inuse >= vq->vring.num) { - vu_panic(dev, "Virtqueue size exceeded"); - return NULL; - } + if (vq->inuse >= vq->vring.num) + die("vhost-user queue size exceeded"); - if (!virtqueue_get_head(dev, vq, vq->last_avail_idx++, &head)) { - return NULL; - } + virtqueue_get_head(vq, vq->last_avail_idx++, &head); - if (vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) { + if (vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) vring_set_avail_event(vq, vq->last_avail_idx); - } - elem = vu_queue_map_desc(dev, vq, head, sz, buffer); + ret = vu_queue_map_desc(dev, vq, head, elem); - if (!elem) { - return NULL; - } + if (ret < 0) + return ret; vq->inuse++; - return elem; + return 0; } -void vu_queue_detach_element(VuDev *dev, VuVirtq *vq, - unsigned int index, size_t len) +/** + * vu_queue_detach_element() - Detach an element from the virqueue + * @vq: Virtqueue + */ +void vu_queue_detach_element(struct vu_virtq *vq) { - (void)dev; - (void)index; - (void)len; - vq->inuse--; /* unmap, when DMA support is added */ } -void vu_queue_unpop(VuDev *dev, VuVirtq *vq, unsigned int index, size_t len) +/** + * vu_queue_unpop() - Push back the previously popped element from the virqueue + * @vq: Virtqueue + */ +/* cppcheck-suppress unusedFunction */ +void vu_queue_unpop(struct vu_virtq *vq) { vq->last_avail_idx--; - vu_queue_detach_element(dev, vq, index, len); + vu_queue_detach_element(vq); } -bool vu_queue_rewind(VuDev *dev, VuVirtq *vq, unsigned int num) +/** + * vu_queue_rewind() - Push back a given number of popped elements + * @vq: Virtqueue + * @num: Number of element to unpop + */ +bool vu_queue_rewind(struct vu_virtq *vq, unsigned int num) { - (void)dev; - if (num > vq->inuse) { + if (num > vq->inuse) return false; - } + vq->last_avail_idx -= num; vq->inuse -= num; return true; } -static inline void vring_used_write(VuVirtq *vq, - struct vring_used_elem *uelem, int i) +/** + * vring_used_write() - Write an entry in the used ring + * @vq: Virtqueue + * @uelem: Entry to write + * @i: Index of the entry in the used ring + */ +static inline void vring_used_write(struct vu_virtq *vq, + const struct vring_used_elem *uelem, int i) { struct vring_used *used = vq->vring.used; used->ring[i] = *uelem; } -void vu_queue_fill_by_index(VuDev *dev, VuVirtq *vq, unsigned int index, - unsigned int len, unsigned int idx) +/** + * vu_queue_fill_by_index() - Update information of a descriptor ring entry + * in the used ring + * @vq: Virtqueue + * @index: Descriptor ring index + * @len: Size of the element + * @idx: Used ring entry index + */ +void vu_queue_fill_by_index(struct vu_virtq *vq, unsigned int index, + unsigned int len, unsigned int idx) { struct vring_used_elem uelem; - if (dev->broken || !vq->vring.avail) + if (!vq->vring.avail) return; idx = (idx + vq->used_idx) % vq->vring.num; @@ -441,27 +610,43 @@ void vu_queue_fill_by_index(VuDev *dev, VuVirtq *vq, unsigned int index, vring_used_write(vq, &uelem, idx); } -void vu_queue_fill(VuDev *dev, VuVirtq *vq, VuVirtqElement *elem, +/** + * vu_queue_fill() - Update information of a given element in the used ring + * @dev: Vhost-user device + * @vq: Virtqueue + * @elem: Element information to fill + * @len: Size of the element + * @idx: Used ring entry index + */ +void vu_queue_fill(struct vu_virtq *vq, const struct vu_virtq_element *elem, unsigned int len, unsigned int idx) { - vu_queue_fill_by_index(dev, vq, elem->index, len, idx); + vu_queue_fill_by_index(vq, elem->index, len, idx); } -static inline void vring_used_idx_set(VuVirtq *vq, uint16_t val) +/** + * vring_used_idx_set() - Set the descriptor ring current index + * @vq: Virtqueue + * @val: Value to set in the index + */ +static inline void vring_used_idx_set(struct vu_virtq *vq, uint16_t val) { vq->vring.used->idx = htole16(val); vq->used_idx = val; } -void vu_queue_flush(VuDev *dev, VuVirtq *vq, unsigned int count) +/** + * vu_queue_flush() - Flush the virtqueue + * @vq: Virtqueue + * @count: Number of entry to flush + */ +void vu_queue_flush(struct vu_virtq *vq, unsigned int count) { uint16_t old, new; - if (dev->broken || - !vq->vring.avail) { + if (!vq->vring.avail) return; - } /* Make sure buffer is written before we update index. */ smp_wmb(); @@ -470,15 +655,6 @@ void vu_queue_flush(VuDev *dev, VuVirtq *vq, unsigned int count) new = old + count; vring_used_idx_set(vq, new); vq->inuse -= count; - if ((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)) { + if ((uint16_t)(new - vq->signalled_used) < (uint16_t)(new - old)) vq->signalled_used_valid = false; - } } - -void vu_queue_push(VuDev *dev, VuVirtq *vq, - VuVirtqElement *elem, unsigned int len) -{ - vu_queue_fill(dev, vq, elem, len, 0); - vu_queue_flush(dev, vq, 1); -} - @@ -1,6 +1,10 @@ // SPDX-License-Identifier: GPL-2.0-or-later -// -/* come parts copied from QEMU subprojects/libvhost-user/libvhost-user.h */ +/* + * virtio API, vring and virtqueue functions definition + * + * Copyright Red Hat + * Author: Laurent Vivier <lvivier@redhat.com> + */ #ifndef VIRTIO_H #define VIRTIO_H @@ -8,61 +12,81 @@ #include <stdbool.h> #include <linux/vhost_types.h> +/* Maximum size of a virtqueue */ #define VIRTQUEUE_MAX_SIZE 1024 -#define vu_panic(vdev, ...) \ - do { \ - (vdev)->broken = true; \ - err( __VA_ARGS__ ); \ - } while (0) - -typedef struct VuRing { +/** + * struct vu_ring - Virtqueue rings + * @num: Size of the queue + * @desc: Descriptor ring + * @avail: Available ring + * @used: Used ring + * @log_guest_addr: Guest address for logging + * @flags: Vring flags + * VHOST_VRING_F_LOG is set if log address is valid + */ +struct vu_ring { unsigned int num; struct vring_desc *desc; struct vring_avail *avail; struct vring_used *used; uint64_t log_guest_addr; uint32_t flags; -} VuRing; - -typedef struct VuVirtq { - VuRing vring; - - /* Next head to pop */ +}; + +/** + * struct vu_virtq - Virtqueue definition + * @vring: Virtqueue rings + * @last_avail_idx: Next head to pop + * @shadow_avail_idx: Last avail_idx read from VQ. + * @used_idx: Descriptor ring current index + * @signalled_used: Last used index value we have signalled on + * @signalled_used_valid: True if signalled_used if valid + * @notification: True if the queues notify (via event + * index or interrupt) + * @inuse: Number of entries in use + * @call_fd: The event file descriptor to signal when + * buffers are used. + * @kick_fd: The event file descriptor for adding + * buffers to the vring + * @err_fd: The event file descriptor to signal when + * error occurs + * @enable: True if the virtqueue is enabled + * @started: True if the virtqueue is started + * @vra: QEMU address of our rings + */ +struct vu_virtq { + struct vu_ring vring; uint16_t last_avail_idx; - - /* Last avail_idx read from VQ. */ uint16_t shadow_avail_idx; - uint16_t used_idx; - - /* Last used index value we have signalled on */ uint16_t signalled_used; - - /* Last used index value we have signalled on */ bool signalled_used_valid; - bool notification; - unsigned int inuse; - int call_fd; int kick_fd; int err_fd; unsigned int enable; bool started; - - /* Guest addresses of our ring */ struct vhost_vring_addr vra; -} VuVirtq; - -typedef struct VuDevRegion { +}; + +/** + * struct vu_dev_region - guest shared memory region + * @gpa: Guest physical address of the region + * @size: Memory size in bytes + * @qva: QEMU virtual address + * @mmap_offset: Offset where the region starts in the mapped memory + * @mmap_addr: Address of the mapped memory + */ +struct vu_dev_region { uint64_t gpa; uint64_t size; uint64_t qva; uint64_t mmap_offset; uint64_t mmap_addr; -} VuDevRegion; +}; #define VHOST_USER_MAX_QUEUES 2 @@ -72,50 +96,89 @@ typedef struct VuDevRegion { */ #define VHOST_USER_MAX_RAM_SLOTS 32 -typedef struct VuDev { +/** + * struct vu_dev - vhost-user device information + * @context: Execution context + * @nregions: Number of shared memory regions + * @regions: Guest shared memory regions + * @features: Vhost-user features + * @protocol_features: Vhost-user protocol features + */ +struct vu_dev { + struct ctx *context; uint32_t nregions; - VuDevRegion regions[VHOST_USER_MAX_RAM_SLOTS]; - VuVirtq vq[VHOST_USER_MAX_QUEUES]; + struct vu_dev_region regions[VHOST_USER_MAX_RAM_SLOTS]; + struct vu_virtq vq[VHOST_USER_MAX_QUEUES]; uint64_t features; uint64_t protocol_features; - bool broken; - int hdrlen; -} VuDev; - -typedef struct VuVirtqElement { +}; + +/** + * struct vu_virtq_element - virtqueue element + * @index: Descriptor ring index + * @out_num: Number of outgoing iovec buffers + * @in_num: Number of incoming iovec buffers + * @in_sg: Incoming iovec buffers + * @out_sg: Outgoing iovec buffers + */ +struct vu_virtq_element { unsigned int index; unsigned int out_num; unsigned int in_num; struct iovec *in_sg; struct iovec *out_sg; -} VuVirtqElement; - +}; + +/** + * has_feature() - Check a feature bit in a features set + * @features: Features set + * @fb: Feature bit to check + * + * Return: True if the feature bit is set + */ static inline bool has_feature(uint64_t features, unsigned int fbit) { return !!(features & (1ULL << fbit)); } -static inline bool vu_has_feature(VuDev *vdev, unsigned int fbit) +/** + * vu_has_feature() - Check if a virtio-net feature is available + * @vdev: Vhost-user device + * @bit: Feature to check + * + * Return: True if the feature is available + */ +static inline bool vu_has_feature(const struct vu_dev *vdev, + unsigned int fbit) { return has_feature(vdev->features, fbit); } -static inline bool vu_has_protocol_feature(VuDev *vdev, unsigned int fbit) +/** + * vu_has_protocol_feature() - Check if a vhost-user feature is available + * @vdev: Vhost-user device + * @bit: Feature to check + * + * Return: True if the feature is available + */ +/* cppcheck-suppress unusedFunction */ +static inline bool vu_has_protocol_feature(const struct vu_dev *vdev, + unsigned int fbit) { return has_feature(vdev->protocol_features, fbit); } -bool vu_queue_empty(VuDev *dev, VuVirtq *vq); -void vu_queue_notify(VuDev *dev, VuVirtq *vq); -void *vu_queue_pop(VuDev *dev, VuVirtq *vq, size_t sz, unsigned char *buffer); -void vu_queue_detach_element(VuDev *dev, VuVirtq *vq, unsigned int index, size_t len); -void vu_queue_unpop(VuDev *dev, VuVirtq *vq, unsigned int index, size_t len); -bool vu_queue_rewind(VuDev *dev, VuVirtq *vq, unsigned int num); - -void vu_queue_fill_by_index(VuDev *dev, VuVirtq *vq, unsigned int index, +bool vu_queue_empty(struct vu_virtq *vq); +void vu_queue_notify(const struct vu_dev *dev, struct vu_virtq *vq); +int vu_queue_pop(struct vu_dev *dev, struct vu_virtq *vq, + struct vu_virtq_element *elem); +void vu_queue_detach_element(struct vu_virtq *vq); +void vu_queue_unpop(struct vu_virtq *vq); +bool vu_queue_rewind(struct vu_virtq *vq, unsigned int num); +void vu_queue_fill_by_index(struct vu_virtq *vq, unsigned int index, unsigned int len, unsigned int idx); -void vu_queue_fill(VuDev *dev, VuVirtq *vq, VuVirtqElement *elem, unsigned int len, +void vu_queue_fill(struct vu_virtq *vq, + const struct vu_virtq_element *elem, unsigned int len, unsigned int idx); -void vu_queue_flush(VuDev *dev, VuVirtq *vq, unsigned int count); -void vu_queue_push(VuDev *dev, VuVirtq *vq, VuVirtqElement *elem, unsigned int len); +void vu_queue_flush(struct vu_virtq *vq, unsigned int count); #endif /* VIRTIO_H */ diff --git a/vu_common.c b/vu_common.c new file mode 100644 index 0000000..4291fda --- /dev/null +++ b/vu_common.c @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Copyright Red Hat + * Author: Laurent Vivier <lvivier@redhat.com> + * + * common_vu.c - vhost-user common UDP and TCP functions + */ + +#include <unistd.h> +#include <sys/uio.h> +#include <sys/eventfd.h> +#include <linux/virtio_net.h> + +#include "util.h" +#include "passt.h" +#include "tap.h" +#include "vhost_user.h" +#include "pcap.h" +#include "vu_common.h" + +/** + * vu_packet_check_range() - Check if a given memory zone is contained in + * a mapped guest memory region + * @buf: Array of the available memory regions + * @offset: Offset of data range in packet descriptor + * @size: Length of desired data range + * @start: Start of the packet descriptor + * + * Return: 0 if the zone is in a mapped memory region, -1 otherwise + */ +int vu_packet_check_range(void *buf, size_t offset, size_t len, + const char *start) +{ + struct vu_dev_region *dev_region; + + for (dev_region = buf; dev_region->mmap_addr; dev_region++) { + /* NOLINTNEXTLINE(performance-no-int-to-ptr) */ + char *m = (char *)dev_region->mmap_addr; + + if (m <= start && + start + offset + len <= m + dev_region->mmap_offset + + dev_region->size) + return 0; + } + + return -1; +} + +/** + * vu_init_elem() - initialize an array of virtqueue element with 1 iov in each + * @elem: Array of virtqueue element to initialize + * @iov: Array of iovec to assign to virtqueue element + * @elem_cnt: Number of virtqueue element + */ +void vu_init_elem(struct vu_virtq_element *elem, struct iovec *iov, int elem_cnt) +{ + int i; + + for (i = 0; i < elem_cnt; i++) + vu_set_element(&elem[i], NULL, &iov[i]); +} + +/** + * vu_collect() - collect virtio buffers from a given virtqueue + * @vdev: vhost-user device + * @vq: virtqueue to collect from + * @elem: Array of virtqueue element + * each element must be initialized with one iovec entry + * in the in_sg array. + * @max_elem: Number of virtqueue element in the array + * @size: Maximum size of the data in the frame + * @frame_size: The total size of the buffers (output) + * + * Return: number of elements used to contain the frame + */ +int vu_collect(struct vu_dev *vdev, struct vu_virtq *vq, + struct vu_virtq_element *elem, int max_elem, + size_t size, size_t *frame_size) +{ + size_t current_size = 0; + int elem_cnt = 0; + + while (current_size < size && elem_cnt < max_elem) { + struct iovec *iov; + int ret; + + ret = vu_queue_pop(vdev, vq, &elem[elem_cnt]); + if (ret < 0) + break; + + if (elem[elem_cnt].in_num < 1) { + warn("virtio-net receive queue contains no in buffers"); + vu_queue_detach_element(vq); + break; + } + + iov = &elem[elem_cnt].in_sg[0]; + + if (iov->iov_len > size - current_size) + iov->iov_len = size - current_size; + + current_size += iov->iov_len; + elem_cnt++; + + if (!vu_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) + break; + } + + if (frame_size) + *frame_size = current_size; + + return elem_cnt; +} + +/** + * vu_set_vnethdr() - set virtio-net headers + * @vdev: vhost-user device + * @vnethdr: Address of the header to set + * @num_buffers: Number of guest buffers of the frame + */ +void vu_set_vnethdr(const struct vu_dev *vdev, + struct virtio_net_hdr_mrg_rxbuf *vnethdr, + int num_buffers) +{ + vnethdr->hdr = VU_HEADER; + if (vu_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) + vnethdr->num_buffers = htole16(num_buffers); +} + +/** + * vu_flush() - flush all the collected buffers to the vhost-user interface + * @vdev: vhost-user device + * @vq: vhost-user virtqueue + * @elem: virtqueue element array to send back to the virqueue + * @iov_used: Length of the array + */ +void vu_flush(const struct vu_dev *vdev, struct vu_virtq *vq, + struct vu_virtq_element *elem, int elem_cnt) +{ + int i; + + for (i = 0; i < elem_cnt; i++) + vu_queue_fill(vq, &elem[i], elem[i].in_sg[0].iov_len, i); + + vu_queue_flush(vq, elem_cnt); + vu_queue_notify(vdev, vq); +} + +/** + * vu_handle_tx() - Receive data from the TX virtqueue + * @vdev: vhost-user device + * @index: index of the virtqueue + * @now: Current timestamp + */ +static void vu_handle_tx(struct vu_dev *vdev, int index, + const struct timespec *now) +{ + struct vu_virtq_element elem[VIRTQUEUE_MAX_SIZE]; + struct iovec out_sg[VIRTQUEUE_MAX_SIZE]; + struct vu_virtq *vq = &vdev->vq[index]; + int hdrlen = sizeof(struct virtio_net_hdr_mrg_rxbuf); + int out_sg_count; + int count; + + if (!VHOST_USER_IS_QUEUE_TX(index)) { + debug("vhost-user: index %d is not a TX queue", index); + return; + } + + tap_flush_pools(); + + count = 0; + out_sg_count = 0; + while (count < VIRTQUEUE_MAX_SIZE) { + int ret; + + vu_set_element(&elem[count], &out_sg[out_sg_count], NULL); + ret = vu_queue_pop(vdev, vq, &elem[count]); + if (ret < 0) + break; + out_sg_count += elem[count].out_num; + + if (elem[count].out_num < 1) { + warn("virtio-net transmit queue contains no out buffers"); + break; + } + ASSERT(elem[count].out_num == 1); + + tap_add_packet(vdev->context, + elem[count].out_sg[0].iov_len - hdrlen, + (char *)elem[count].out_sg[0].iov_base + hdrlen); + count++; + } + tap_handler(vdev->context, now); + + if (count) { + int i; + + for (i = 0; i < count; i++) + vu_queue_fill(vq, &elem[i], 0, i); + vu_queue_flush(vq, count); + vu_queue_notify(vdev, vq); + } +} + +/** + * vu_kick_cb() - Called on a kick event to start to receive data + * @vdev: vhost-user device + * @ref: epoll reference information + * @now: Current timestamp + */ +void vu_kick_cb(struct vu_dev *vdev, union epoll_ref ref, + const struct timespec *now) +{ + eventfd_t kick_data; + ssize_t rc; + + rc = eventfd_read(ref.fd, &kick_data); + if (rc == -1) + die_perror("vhost-user kick eventfd_read()"); + + debug("vhost-user: ot kick_data: %016"PRIx64" idx:%d", + kick_data, ref.queue); + if (VHOST_USER_IS_QUEUE_TX(ref.queue)) + vu_handle_tx(vdev, ref.queue, now); +} + +/** + * vu_send_single() - Send a buffer to the front-end using the RX virtqueue + * @c: execution context + * @buf: address of the buffer + * @size: size of the buffer + * + * Return: number of bytes sent, -1 if there is an error + */ +int vu_send_single(const struct ctx *c, const void *buf, size_t size) +{ + struct vu_dev *vdev = c->vdev; + struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE]; + struct vu_virtq_element elem[VIRTQUEUE_MAX_SIZE]; + struct iovec in_sg[VIRTQUEUE_MAX_SIZE]; + size_t total; + int elem_cnt; + int i; + + debug("vu_send_single size %zu", size); + + if (!vu_queue_enabled(vq) || !vu_queue_started(vq)) { + err("Got packet, but RX virtqueue not usable yet"); + return 0; + } + + vu_init_elem(elem, in_sg, VIRTQUEUE_MAX_SIZE); + + size += sizeof(struct virtio_net_hdr_mrg_rxbuf); + elem_cnt = vu_collect(vdev, vq, elem, VIRTQUEUE_MAX_SIZE, size, &total); + if (total < size) { + debug("vu_send_single: no space to send the data " + "elem_cnt %d size %zd", elem_cnt, total); + goto err; + } + + vu_set_vnethdr(vdev, in_sg[0].iov_base, elem_cnt); + + total -= sizeof(struct virtio_net_hdr_mrg_rxbuf); + + /* copy data from the buffer to the iovec */ + iov_from_buf(in_sg, elem_cnt, sizeof(struct virtio_net_hdr_mrg_rxbuf), + buf, total); + + if (*c->pcap) { + pcap_iov(in_sg, elem_cnt, + sizeof(struct virtio_net_hdr_mrg_rxbuf)); + } + + vu_flush(vdev, vq, elem, elem_cnt); + + debug("vhost-user sent %zu", total); + + return total; +err: + for (i = 0; i < elem_cnt; i++) + vu_queue_detach_element(vq); + + return -1; +} diff --git a/vu_common.h b/vu_common.h new file mode 100644 index 0000000..901d972 --- /dev/null +++ b/vu_common.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * Copyright Red Hat + * Author: Laurent Vivier <lvivier@redhat.com> + * + * vhost-user common UDP and TCP functions + */ + +#ifndef VU_COMMON_H +#define VU_COMMON_H +#include <linux/virtio_net.h> + +static inline void *vu_eth(void *base) +{ + return ((char *)base + sizeof(struct virtio_net_hdr_mrg_rxbuf)); +} + +static inline void *vu_ip(void *base) +{ + return (struct ethhdr *)vu_eth(base) + 1; +} + +static inline void *vu_payloadv4(void *base) +{ + return (struct iphdr *)vu_ip(base) + 1; +} + +static inline void *vu_payloadv6(void *base) +{ + return (struct ipv6hdr *)vu_ip(base) + 1; +} + +/** + * vu_set_element() - Initialize a vu_virtq_element + * @elem: Element to initialize + * @out_sg: One out iovec entry to set in elem + * @in_sg: One in iovec entry to set in elem + */ +static inline void vu_set_element(struct vu_virtq_element *elem, + struct iovec *out_sg, struct iovec *in_sg) +{ + elem->out_num = !!out_sg; + elem->out_sg = out_sg; + elem->in_num = !!in_sg; + elem->in_sg = in_sg; +} + +void vu_init_elem(struct vu_virtq_element *elem, struct iovec *iov, + int elem_cnt); +int vu_collect(struct vu_dev *vdev, struct vu_virtq *vq, + struct vu_virtq_element *elem, int max_elem, size_t size, + size_t *frame_size); +void vu_set_vnethdr(const struct vu_dev *vdev, + struct virtio_net_hdr_mrg_rxbuf *vnethdr, + int num_buffers); +void vu_flush(const struct vu_dev *vdev, struct vu_virtq *vq, + struct vu_virtq_element *elem, int elem_cnt); +void vu_kick_cb(struct vu_dev *vdev, union epoll_ref ref, + const struct timespec *now); +int vu_send_single(const struct ctx *c, const void *buf, size_t size); +#endif /* VU_COMMON_H */ |