diff options
Diffstat (limited to 'tap.c')
| -rw-r--r-- | tap.c | 823 |
1 files changed, 395 insertions, 428 deletions
@@ -59,6 +59,7 @@ #include "tap.h" #include "log.h" #include "vhost_user.h" +#include "vu_common.h" /* IPv4 (plus ARP) and IPv6 message batches from tap/guest to IP handlers */ static PACKET_POOL_NOINIT(pool_tap4, TAP_MSGS, pkt_buf); @@ -68,42 +69,33 @@ static PACKET_POOL_NOINIT(pool_tap6, TAP_MSGS, pkt_buf); #define FRAGMENT_MSG_RATE 10 /* # seconds between fragment warnings */ /** - * tap_send() - Send frame, with qemu socket header if needed + * tap_send_single() - Send a single frame * @c: Execution context * @data: Packet buffer - * @len: Total L2 packet length - * - * Return: return code from send() or write() + * @l2len: Total L2 packet length */ -int tap_send(const struct ctx *c, const void *data, size_t len) +void tap_send_single(const struct ctx *c, const void *data, size_t l2len) { - int flags = MSG_NOSIGNAL | MSG_DONTWAIT; - uint32_t vnet_len = htonl(len); - - pcap(data, len); + uint32_t vnet_len = htonl(l2len); + struct iovec iov[2]; + size_t iovcnt = 0; switch (c->mode) { case MODE_PASST: - if (send(c->fd_tap, &vnet_len, 4, flags) < 0) - return -1; - return send(c->fd_tap, data, len, flags); + iov[iovcnt] = IOV_OF_LVALUE(vnet_len); + iovcnt++; + /* fall through */ case MODE_PASTA: - return write(c->fd_tap, (char *)data, len); + iov[iovcnt].iov_base = (void *)data; + iov[iovcnt].iov_len = l2len; + iovcnt++; + + tap_send_frames(c, iov, iovcnt, 1); + break; case MODE_VU: - return vu_send(c, data, len); + vu_send_single(c, data, l2len); + break; } - return 0; -} - -/** - * tap_ip4_daddr() - Normal IPv4 destination address for inbound packets - * @c: Execution context - * - * Return: IPv4 address, network order - */ -struct in_addr tap_ip4_daddr(const struct ctx *c) -{ - return c->ip4.addr_seen; } /** @@ -134,8 +126,8 @@ static void *tap_push_l2h(const struct ctx *c, void *buf, uint16_t proto) struct ethhdr *eh = (struct ethhdr *)buf; /* TODO: ARP table lookup */ - memcpy(eh->h_dest, c->mac_guest, ETH_ALEN); - memcpy(eh->h_source, c->mac, ETH_ALEN); + memcpy(eh->h_dest, c->guest_mac, ETH_ALEN); + memcpy(eh->h_source, c->our_tap_mac, ETH_ALEN); eh->h_proto = ntohs(proto); return eh + 1; } @@ -143,29 +135,29 @@ static void *tap_push_l2h(const struct ctx *c, void *buf, uint16_t proto) /** * tap_push_ip4h() - Build IPv4 header for inbound packet, with checksum * @c: Execution context - * @src: IPv4 source address, network order - * @dst: IPv4 destination address, network order - * @len: L4 payload length + * @src: IPv4 source address + * @dst: IPv4 destination address + * @l4len: IPv4 payload length * @proto: L4 protocol number * * Return: pointer at which to write the packet's payload */ -static void *tap_push_ip4h(char *buf, struct in_addr src, struct in_addr dst, - size_t len, uint8_t proto) +static void *tap_push_ip4h(struct iphdr *ip4h, struct in_addr src, + struct in_addr dst, size_t l4len, uint8_t proto) { - struct iphdr *ip4h = (struct iphdr *)buf; + uint16_t l3len = l4len + sizeof(*ip4h); ip4h->version = 4; ip4h->ihl = sizeof(struct iphdr) / 4; ip4h->tos = 0; - ip4h->tot_len = htons(len + sizeof(*ip4h)); + ip4h->tot_len = htons(l3len); ip4h->id = 0; ip4h->frag_off = 0; ip4h->ttl = 255; ip4h->protocol = proto; ip4h->saddr = src.s_addr; ip4h->daddr = dst.s_addr; - ip4h->check = csum_ip4_header(ip4h->tot_len, proto, src, dst); + ip4h->check = csum_ip4_header(l3len, proto, src, dst); return ip4h + 1; } @@ -177,27 +169,29 @@ static void *tap_push_ip4h(char *buf, struct in_addr src, struct in_addr dst, * @dst: IPv4 destination address * @dport: UDP destination port * @in: UDP payload contents (not including UDP header) - * @len: UDP payload length (not including UDP header) + * @dlen: UDP payload length (not including UDP header) */ void tap_udp4_send(const struct ctx *c, struct in_addr src, in_port_t sport, struct in_addr dst, in_port_t dport, - const void *in, size_t len) + const void *in, size_t dlen) { - size_t udplen = len + sizeof(struct udphdr); + size_t l4len = dlen + sizeof(struct udphdr); char buf[USHRT_MAX]; - void *ip4h = tap_push_l2h(c, buf, ETH_P_IP); - void *uhp = tap_push_ip4h(ip4h, src, dst, udplen, IPPROTO_UDP); - struct udphdr *uh = (struct udphdr *)uhp; + struct iphdr *ip4h = tap_push_l2h(c, buf, ETH_P_IP); + struct udphdr *uh = tap_push_ip4h(ip4h, src, dst, l4len, IPPROTO_UDP); char *data = (char *)(uh + 1); + const struct iovec iov = { + .iov_base = (void *)in, + .iov_len = dlen + }; uh->source = htons(sport); uh->dest = htons(dport); - uh->len = htons(udplen); - csum_udp4(uh, src, dst, in, len); - memcpy(data, in, len); + uh->len = htons(l4len); + csum_udp4(uh, src, dst, &iov, 1, 0); + memcpy(data, in, dlen); - if (tap_send(c, buf, len + (data - buf)) < 0) - debug("tap: failed to send %zu bytes (IPv4)", len); + tap_send_single(c, buf, dlen + (data - buf)); } /** @@ -206,21 +200,20 @@ void tap_udp4_send(const struct ctx *c, struct in_addr src, in_port_t sport, * @src: IPv4 source address * @dst: IPv4 destination address * @in: ICMP packet, including ICMP header - * @len: ICMP packet length, including ICMP header + * @l4len: ICMP packet length, including ICMP header */ void tap_icmp4_send(const struct ctx *c, struct in_addr src, struct in_addr dst, - const void *in, size_t len) + const void *in, size_t l4len) { char buf[USHRT_MAX]; - void *ip4h = tap_push_l2h(c, buf, ETH_P_IP); - char *data = tap_push_ip4h(ip4h, src, dst, len, IPPROTO_ICMP); - struct icmphdr *icmp4h = (struct icmphdr *)data; + struct iphdr *ip4h = tap_push_l2h(c, buf, ETH_P_IP); + struct icmphdr *icmp4h = tap_push_ip4h(ip4h, src, dst, + l4len, IPPROTO_ICMP); - memcpy(data, in, len); - csum_icmp4(icmp4h, icmp4h + 1, len - sizeof(*icmp4h)); + memcpy(icmp4h, in, l4len); + csum_icmp4(icmp4h, icmp4h + 1, l4len - sizeof(*icmp4h)); - if (tap_send(c, buf, len + (data - buf)) < 0) - debug("tap: failed to send %zu bytes (IPv4)", len); + tap_send_single(c, buf, l4len + ((char *)icmp4h - buf)); } /** @@ -228,20 +221,18 @@ void tap_icmp4_send(const struct ctx *c, struct in_addr src, struct in_addr dst, * @c: Execution context * @src: IPv6 source address * @dst: IPv6 destination address - * @len: L4 payload length + * @l4len: L4 payload length * @proto: L4 protocol number * @flow: IPv6 flow identifier * * Return: pointer at which to write the packet's payload */ -static void *tap_push_ip6h(char *buf, +static void *tap_push_ip6h(struct ipv6hdr *ip6h, const struct in6_addr *src, const struct in6_addr *dst, - size_t len, uint8_t proto, uint32_t flow) + size_t l4len, uint8_t proto, uint32_t flow) { - struct ipv6hdr *ip6h = (struct ipv6hdr *)buf; - - ip6h->payload_len = htons(len); + ip6h->payload_len = htons(l4len); ip6h->priority = 0; ip6h->version = 6; ip6h->nexthdr = proto; @@ -263,28 +254,31 @@ static void *tap_push_ip6h(char *buf, * @dport: UDP destination port * @flow: Flow label * @in: UDP payload contents (not including UDP header) - * @len: UDP payload length (not including UDP header) + * @dlen: UDP payload length (not including UDP header) */ void tap_udp6_send(const struct ctx *c, const struct in6_addr *src, in_port_t sport, const struct in6_addr *dst, in_port_t dport, - uint32_t flow, const void *in, size_t len) + uint32_t flow, void *in, size_t dlen) { - size_t udplen = len + sizeof(struct udphdr); + size_t l4len = dlen + sizeof(struct udphdr); char buf[USHRT_MAX]; - void *ip6h = tap_push_l2h(c, buf, ETH_P_IPV6); - void *uhp = tap_push_ip6h(ip6h, src, dst, udplen, IPPROTO_UDP, flow); - struct udphdr *uh = (struct udphdr *)uhp; + struct ipv6hdr *ip6h = tap_push_l2h(c, buf, ETH_P_IPV6); + struct udphdr *uh = tap_push_ip6h(ip6h, src, dst, + l4len, IPPROTO_UDP, flow); char *data = (char *)(uh + 1); + const struct iovec iov = { + .iov_base = in, + .iov_len = dlen + }; uh->source = htons(sport); uh->dest = htons(dport); - uh->len = htons(udplen); - csum_udp6(uh, src, dst, in, len); - memcpy(data, in, len); + uh->len = htons(l4len); + csum_udp6(uh, src, dst, &iov, 1, 0); + memcpy(data, in, dlen); - if (tap_send(c, buf, len + (data - buf)) < 1) - debug("tap: failed to send %zu bytes (IPv6)", len); + tap_send_single(c, buf, dlen + (data - buf)); } /** @@ -293,44 +287,50 @@ void tap_udp6_send(const struct ctx *c, * @src: IPv6 source address * @dst: IPv6 destination address * @in: ICMP packet, including ICMP header - * @len: ICMP packet length, including ICMP header + * @l4len: ICMP packet length, including ICMP header */ void tap_icmp6_send(const struct ctx *c, const struct in6_addr *src, const struct in6_addr *dst, - const void *in, size_t len) + const void *in, size_t l4len) { char buf[USHRT_MAX]; - void *ip6h = tap_push_l2h(c, buf, ETH_P_IPV6); - char *data = tap_push_ip6h(ip6h, src, dst, len, IPPROTO_ICMPV6, 0); - struct icmp6hdr *icmp6h = (struct icmp6hdr *)data; + struct ipv6hdr *ip6h = tap_push_l2h(c, buf, ETH_P_IPV6); + struct icmp6hdr *icmp6h = tap_push_ip6h(ip6h, src, dst, l4len, + IPPROTO_ICMPV6, 0); - memcpy(data, in, len); - csum_icmp6(icmp6h, src, dst, icmp6h + 1, len - sizeof(*icmp6h)); + memcpy(icmp6h, in, l4len); + csum_icmp6(icmp6h, src, dst, icmp6h + 1, l4len - sizeof(*icmp6h)); - if (tap_send(c, buf, len + (data - buf)) < 1) - debug("tap: failed to send %zu bytes (IPv6)", len); + tap_send_single(c, buf, l4len + ((char *)icmp6h - buf)); } /** * tap_send_frames_pasta() - Send multiple frames to the pasta tap - * @c: Execution context - * @iov: Array of buffers, each containing one frame - * @n: Number of buffers/frames in @iov + * @c: Execution context + * @iov: Array of buffers + * @bufs_per_frame: Number of buffers (iovec entries) per frame + * @nframes: Number of frames to send + * + * @iov must have total length @bufs_per_frame * @nframes, with each set of + * @bufs_per_frame contiguous buffers representing a single frame. * * Return: number of frames successfully sent * * #syscalls:pasta write */ static size_t tap_send_frames_pasta(const struct ctx *c, - const struct iovec *iov, size_t n) + const struct iovec *iov, + size_t bufs_per_frame, size_t nframes) { + size_t nbufs = bufs_per_frame * nframes; size_t i; - for (i = 0; i < n; i++) { - ssize_t rc = write(c->fd_tap, iov[i].iov_base, iov[i].iov_len); + for (i = 0; i < nbufs; i += bufs_per_frame) { + ssize_t rc = writev(c->fd_tap, iov + i, bufs_per_frame); + size_t framelen = iov_size(iov + i, bufs_per_frame); if (rc < 0) { - debug("tap write: %s", strerror(errno)); + debug_perror("tap write"); switch (errno) { case EAGAIN: @@ -340,60 +340,42 @@ static size_t tap_send_frames_pasta(const struct ctx *c, case EINTR: case ENOBUFS: case ENOSPC: + case EIO: /* interface down? */ break; default: die("Write error on tap device, exiting"); } - } else if ((size_t)rc < iov[i].iov_len) { - debug("short write on tuntap: %zd/%zu", - rc, iov[i].iov_len); + } else if ((size_t)rc < framelen) { + debug("short write on tuntap: %zd/%zu", rc, framelen); break; } } - return i; -} - -/** - * tap_send_iov_pasta() - Send out multiple prepared frames - * @c: Execution context - * @iov: Array of frames, each frames is divided in an array of iovecs. - * The first entry of the iovec is ignored - * @n: Number of frames in @iov - * - * Return: number of frames actually sent - */ -static size_t tap_send_iov_pasta(const struct ctx *c, - struct iovec iov[][TCP_IOV_NUM], size_t n) -{ - unsigned int i; - - for (i = 0; i < n; i++) { - if (!tap_send_frames_pasta(c, &iov[i][TCP_IOV_ETH], - TCP_IOV_NUM - TCP_IOV_ETH)) - break; - } - - return i; - + return i / bufs_per_frame; } /** * tap_send_frames_passt() - Send multiple frames to the passt tap - * @c: Execution context - * @iov: Array of buffers, each containing one frame - * @n: Number of buffers/frames in @iov + * @c: Execution context + * @iov: Array of buffers, each containing one frame + * @bufs_per_frame: Number of buffers (iovec entries) per frame + * @nframes: Number of frames to send + * + * @iov must have total length @bufs_per_frame * @nframes, with each set of + * @bufs_per_frame contiguous buffers representing a single frame. * * Return: number of frames successfully sent * * #syscalls:passt sendmsg */ static size_t tap_send_frames_passt(const struct ctx *c, - const struct iovec *iov, size_t n) + const struct iovec *iov, + size_t bufs_per_frame, size_t nframes) { + size_t nbufs = bufs_per_frame * nframes; struct msghdr mh = { .msg_iov = (void *)iov, - .msg_iovlen = n, + .msg_iovlen = nbufs, }; size_t buf_offset; unsigned int i; @@ -404,134 +386,61 @@ static size_t tap_send_frames_passt(const struct ctx *c, return 0; /* Check for any partial frames due to short send */ - i = iov_skip_bytes(iov, n, sent, &buf_offset); + i = iov_skip_bytes(iov, nbufs, sent, &buf_offset); - if (i < n && buf_offset) { - /* A partial frame was sent */ - if (write_remainder(c->fd_tap, &iov[i], 1, buf_offset) < 0) { - err("tap: partial frame send: %s", strerror(errno)); + if (i < nbufs && (buf_offset || (i % bufs_per_frame))) { + /* Number of unsent or partially sent buffers for the frame */ + size_t rembufs = bufs_per_frame - (i % bufs_per_frame); + + if (write_remainder(c->fd_tap, &iov[i], rembufs, buf_offset) < 0) { + err_perror("tap: partial frame send"); return i; } - i++; - } - - return i; -} - -/** - * tap_send_iov_passt() - Send out multiple prepared frames - * @c: Execution context - * @iov: Array of frames, each frames is divided in an array of iovecs. - * The first entry of the iovec is updated to point to an - * uint32_t storing the frame length. - * @n: Number of frames in @iov - * - * Return: number of frames actually sent - */ -static size_t tap_send_iov_passt(const struct ctx *c, - struct iovec iov[][TCP_IOV_NUM], - size_t n) -{ - unsigned int i; - - for (i = 0; i < n; i++) { - uint32_t vnet_len; - int j; - - vnet_len = 0; - for (j = TCP_IOV_ETH; j < TCP_IOV_NUM; j++) - vnet_len += iov[i][j].iov_len; - - vnet_len = htonl(vnet_len); - iov[i][TCP_IOV_VNET].iov_base = &vnet_len; - iov[i][TCP_IOV_VNET].iov_len = sizeof(vnet_len); - - if (!tap_send_frames_passt(c, iov[i], TCP_IOV_NUM)) - break; + i += rembufs; } - return i; - + return i / bufs_per_frame; } /** * tap_send_frames() - Send out multiple prepared frames - * @c: Execution context - * @iov: Array of buffers, each containing one frame (with L2 headers) - * @n: Number of buffers/frames in @iov + * @c: Execution context + * @iov: Array of buffers, each containing one frame (with L2 headers) + * @bufs_per_frame: Number of buffers (iovec entries) per frame + * @nframes: Number of frames to send + * + * @iov must have total length @bufs_per_frame * @nframes, with each set of + * @bufs_per_frame contiguous buffers representing a single frame. * * Return: number of frames actually sent */ -size_t tap_send_frames(const struct ctx *c, const struct iovec *iov, size_t n) +size_t tap_send_frames(const struct ctx *c, const struct iovec *iov, + size_t bufs_per_frame, size_t nframes) { size_t m; - if (!n) + if (!nframes) return 0; switch (c->mode) { case MODE_PASTA: - m = tap_send_frames_pasta(c, iov, n); + m = tap_send_frames_pasta(c, iov, bufs_per_frame, nframes); break; case MODE_PASST: - m = tap_send_frames_passt(c, iov, n); + m = tap_send_frames_passt(c, iov, bufs_per_frame, nframes); break; case MODE_VU: - m = tap_send_frames_vu(c, iov, n); - break; - default: - m = 0; - break; - } - - if (m < n) - debug("tap: failed to send %zu frames of %zu", n - m, n); - - pcap_multiple(iov, 1, n, c->mode == MODE_PASST ? sizeof(uint32_t) : 0); - - return m; -} - -/** - * tap_send_iov() - Send out multiple prepared frames - * @c: Execution context - * @iov: Array of frames, each frames is divided in an array of iovecs. - * iovec array is: - * TCP_IOV_VNET (0) vnet length - * TCP_IOV_ETH (1) ethernet header - * TCP_IOV_IP (2) IP (v4/v6) header - * TCP_IOV_PAYLOAD (3) IP payload (TCP header + data) - * TCP_IOV_NUM (4) is the number of entries in the iovec array - * TCP_IOV_VNET entry is updated with passt, ignored with pasta. - * @n: Number of frames in @iov - * - * Return: number of frames actually sent - */ -size_t tap_send_iov(const struct ctx *c, struct iovec iov[][TCP_IOV_NUM], - size_t n) -{ - size_t m; - unsigned int i; - - if (!n) - return 0; - - switch (c->mode) { - case MODE_PASST: - m = tap_send_iov_passt(c, iov, n); - break; - case MODE_PASTA: - m = tap_send_iov_pasta(c, iov, n); - break; + /* fall through */ default: ASSERT(0); } - if (m < n) - debug("tap: failed to send %zu frames of %zu", n - m, n); + if (m < nframes) + debug("tap: failed to send %zu frames of %zu", + nframes - m, nframes); - for (i = 0; i < m; i++) - pcap_iov(&iov[i][TCP_IOV_ETH], TCP_IOV_NUM - TCP_IOV_ETH); + pcap_multiple(iov, bufs_per_frame, m, + c->mode == MODE_PASST ? sizeof(uint32_t) : 0); return m; } @@ -695,21 +604,21 @@ static int tap4_handler(struct ctx *c, const struct pool *in, i = 0; resume: for (seq_count = 0, seq = NULL; i < in->count; i++) { - size_t l2_len, l3_len, hlen, l4_len; + size_t l2len, l3len, hlen, l4len; const struct ethhdr *eh; const struct udphdr *uh; struct iphdr *iph; const char *l4h; - packet_get(in, i, 0, 0, &l2_len); + packet_get(in, i, 0, 0, &l2len); - eh = packet_get(in, i, 0, sizeof(*eh), &l3_len); + eh = packet_get(in, i, 0, sizeof(*eh), &l3len); if (!eh) continue; if (ntohs(eh->h_proto) == ETH_P_ARP) { PACKET_POOL_P(pkt, 1, in->buf, in->buf_size); - packet_add(pkt, l2_len, (char *)eh); + packet_add(pkt, l2len, (char *)eh); arp(c, pkt); continue; } @@ -719,15 +628,15 @@ resume: continue; hlen = iph->ihl * 4UL; - if (hlen < sizeof(*iph) || htons(iph->tot_len) > l3_len || - hlen > l3_len) + if (hlen < sizeof(*iph) || htons(iph->tot_len) > l3len || + hlen > l3len) continue; /* We don't handle IP fragments, drop them */ if (tap4_is_fragment(iph, now)) continue; - l4_len = htons(iph->tot_len) - hlen; + l4len = htons(iph->tot_len) - hlen; if (IN4_IS_ADDR_LOOPBACK(&iph->saddr) || IN4_IS_ADDR_LOOPBACK(&iph->daddr)) { @@ -742,7 +651,7 @@ resume: if (iph->saddr && c->ip4.addr_seen.s_addr != iph->saddr) c->ip4.addr_seen.s_addr = iph->saddr; - l4h = packet_get(in, i, sizeof(*eh) + hlen, l4_len, NULL); + l4h = packet_get(in, i, sizeof(*eh) + hlen, l4len, NULL); if (!l4h) continue; @@ -754,7 +663,7 @@ resume: tap_packet_debug(iph, NULL, NULL, 0, NULL, 1); - packet_add(pkt, l4_len, l4h); + packet_add(pkt, l4len, l4h); icmp_tap_handler(c, PIF_TAP, AF_INET, &iph->saddr, &iph->daddr, pkt, now); @@ -768,7 +677,7 @@ resume: if (iph->protocol == IPPROTO_UDP) { PACKET_POOL_P(pkt, 1, in->buf, in->buf_size); - packet_add(pkt, l2_len, (char *)eh); + packet_add(pkt, l2len, (char *)eh); if (dhcp(c, pkt)) continue; } @@ -779,18 +688,18 @@ resume: continue; } -#define L4_MATCH(iph, uh, seq) \ - (seq->protocol == iph->protocol && \ - seq->source == uh->source && seq->dest == uh->dest && \ - seq->saddr.s_addr == iph->saddr && seq->daddr.s_addr == iph->daddr) +#define L4_MATCH(iph, uh, seq) \ + ((seq)->protocol == (iph)->protocol && \ + (seq)->source == (uh)->source && (seq)->dest == (uh)->dest && \ + (seq)->saddr.s_addr == (iph)->saddr && (seq)->daddr.s_addr == (iph)->daddr) #define L4_SET(iph, uh, seq) \ do { \ - seq->protocol = iph->protocol; \ - seq->source = uh->source; \ - seq->dest = uh->dest; \ - seq->saddr.s_addr = iph->saddr; \ - seq->daddr.s_addr = iph->daddr; \ + (seq)->protocol = (iph)->protocol; \ + (seq)->source = (uh)->source; \ + (seq)->dest = (uh)->dest; \ + (seq)->saddr.s_addr = (iph)->saddr; \ + (seq)->daddr.s_addr = (iph)->daddr; \ } while (0) if (seq && L4_MATCH(iph, uh, seq) && seq->p.count < UIO_MAXIOV) @@ -817,7 +726,7 @@ resume: #undef L4_SET append: - packet_add((struct pool *)&seq->p, l4_len, l4h); + packet_add((struct pool *)&seq->p, l4len, l4h); } for (j = 0, seq = tap4_l4; j < seq_count; j++, seq++) { @@ -869,7 +778,7 @@ static int tap6_handler(struct ctx *c, const struct pool *in, i = 0; resume: for (seq_count = 0, seq = NULL; i < in->count; i++) { - size_t l4_len, plen, check; + size_t l4len, plen, check; struct in6_addr *saddr, *daddr; const struct ethhdr *eh; const struct udphdr *uh; @@ -892,7 +801,7 @@ resume: if (plen != check) continue; - if (!(l4h = ipv6_l4hdr(in, i, sizeof(*eh), &proto, &l4_len))) + if (!(l4h = ipv6_l4hdr(in, i, sizeof(*eh), &proto, &l4len))) continue; if (IN6_IS_ADDR_LOOPBACK(saddr) || IN6_IS_ADDR_LOOPBACK(daddr)) { @@ -920,28 +829,29 @@ resume: if (c->no_icmp) continue; - if (l4_len < sizeof(struct icmp6hdr)) + if (l4len < sizeof(struct icmp6hdr)) continue; - if (ndp(c, (struct icmp6hdr *)l4h, saddr)) + packet_add(pkt, l4len, l4h); + + if (ndp(c, (struct icmp6hdr *)l4h, saddr, pkt)) continue; tap_packet_debug(NULL, ip6h, NULL, proto, NULL, 1); - packet_add(pkt, l4_len, l4h); icmp_tap_handler(c, PIF_TAP, AF_INET6, saddr, daddr, pkt, now); continue; } - if (l4_len < sizeof(*uh)) + if (l4len < sizeof(*uh)) continue; uh = (struct udphdr *)l4h; if (proto == IPPROTO_UDP) { PACKET_POOL_P(pkt, 1, in->buf, in->buf_size); - packet_add(pkt, l4_len, l4h); + packet_add(pkt, l4len, l4h); if (dhcpv6(c, pkt, saddr, daddr)) continue; @@ -953,18 +863,19 @@ resume: } #define L4_MATCH(ip6h, proto, uh, seq) \ - (seq->protocol == proto && \ - seq->source == uh->source && seq->dest == uh->dest && \ - IN6_ARE_ADDR_EQUAL(&seq->saddr, saddr) && \ - IN6_ARE_ADDR_EQUAL(&seq->daddr, daddr)) + ((seq)->protocol == (proto) && \ + (seq)->source == (uh)->source && \ + (seq)->dest == (uh)->dest && \ + IN6_ARE_ADDR_EQUAL(&(seq)->saddr, saddr) && \ + IN6_ARE_ADDR_EQUAL(&(seq)->daddr, daddr)) #define L4_SET(ip6h, proto, uh, seq) \ do { \ - seq->protocol = proto; \ - seq->source = uh->source; \ - seq->dest = uh->dest; \ - seq->saddr = *saddr; \ - seq->daddr = *daddr; \ + (seq)->protocol = (proto); \ + (seq)->source = (uh)->source; \ + (seq)->dest = (uh)->dest; \ + (seq)->saddr = *saddr; \ + (seq)->daddr = *daddr; \ } while (0) if (seq && L4_MATCH(ip6h, proto, uh, seq) && @@ -992,7 +903,7 @@ resume: #undef L4_SET append: - packet_add((struct pool *)&seq->p, l4_len, l4h); + packet_add((struct pool *)&seq->p, l4len, l4h); } for (j = 0, seq = tap6_l4; j < seq_count; j++, seq++) { @@ -1025,39 +936,52 @@ append: return in->count; } -void pool_flush_all(void) +/** + * tap_flush_pools() - Flush both IPv4 and IPv6 packet pools + */ +void tap_flush_pools(void) { pool_flush(pool_tap4); pool_flush(pool_tap6); } -void tap_handler_all(struct ctx *c, const struct timespec *now) +/** + * tap_handler() - IPv4/IPv6 and ARP packet handler for tap file descriptor + * @c: Execution context + * @now: Current timestamp + */ +void tap_handler(struct ctx *c, const struct timespec *now) { tap4_handler(c, pool_tap4, now); tap6_handler(c, pool_tap6, now); } -void packet_add_all_do(struct ctx *c, ssize_t len, char *p, - const char *func, int line) +/** + * tap_add_packet() - Queue/capture packet, update notion of guest MAC address + * @c: Execution context + * @l2len: Total L2 packet length + * @p: Packet buffer + */ +void tap_add_packet(struct ctx *c, ssize_t l2len, char *p) { const struct ethhdr *eh; - pcap(p, len); + pcap(p, l2len); eh = (struct ethhdr *)p; - if (memcmp(c->mac_guest, eh->h_source, ETH_ALEN)) { - memcpy(c->mac_guest, eh->h_source, ETH_ALEN); - proto_update_l2_buf(c->mac_guest, NULL); + if (memcmp(c->guest_mac, eh->h_source, ETH_ALEN)) { + memcpy(c->guest_mac, eh->h_source, ETH_ALEN); + proto_update_l2_buf(c->guest_mac, NULL); } switch (ntohs(eh->h_proto)) { case ETH_P_ARP: case ETH_P_IP: - packet_add_do(pool_tap4, len, p, func, line); + packet_add(pool_tap4, l2len, p); break; case ETH_P_IPV6: - packet_add_do(pool_tap6, len, p, func, line); + packet_add(pool_tap6, l2len, p); break; default: break; @@ -1070,174 +994,194 @@ void packet_add_all_do(struct ctx *c, ssize_t len, char *p, */ void tap_sock_reset(struct ctx *c) { - if (c->one_off) { - info("Client closed connection, exiting"); + info("Client connection closed%s", c->one_off ? ", exiting" : ""); + + if (c->one_off) exit(EXIT_SUCCESS); - } /* Close the connected socket, wait for a new connection */ epoll_ctl(c->epollfd, EPOLL_CTL_DEL, c->fd_tap, NULL); close(c->fd_tap); c->fd_tap = -1; + if (c->mode == MODE_VU) + vu_cleanup(c->vdev); } /** - * tap_handler_passt() - Packet handler for AF_UNIX file descriptor + * tap_passt_input() - Handler for new data on the socket to qemu * @c: Execution context - * @events: epoll events * @now: Current timestamp */ -void tap_handler_passt(struct ctx *c, uint32_t events, - const struct timespec *now) +static void tap_passt_input(struct ctx *c, const struct timespec *now) { - ssize_t n, rem; + static const char *partial_frame; + static ssize_t partial_len = 0; + ssize_t n; char *p; - if (events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)) { - tap_sock_reset(c); - return; - } + tap_flush_pools(); -redo: - p = pkt_buf; - rem = 0; + if (partial_len) { + /* We have a partial frame from an earlier pass. Move it to the + * start of the buffer, top up with new data, then process all + * of it. + */ + memmove(pkt_buf, partial_frame, partial_len); + } - pool_flush_all(); + do { + n = recv(c->fd_tap, pkt_buf + partial_len, + TAP_BUF_BYTES - partial_len, MSG_DONTWAIT); + } while ((n < 0) && errno == EINTR); - n = recv(c->fd_tap, p, TAP_BUF_FILL, MSG_DONTWAIT); if (n < 0) { - if (errno != EINTR && errno != EAGAIN && errno != EWOULDBLOCK) + if (errno != EAGAIN && errno != EWOULDBLOCK) { + err_perror("Receive error on guest connection, reset"); tap_sock_reset(c); + } return; } - while (n > (ssize_t)sizeof(uint32_t)) { - ssize_t len = ntohl(*(uint32_t *)p); + p = pkt_buf; + n += partial_len; - p += sizeof(uint32_t); - n -= sizeof(uint32_t); + while (n >= (ssize_t)sizeof(uint32_t)) { + uint32_t l2len = ntohl_unaligned(p); - /* At most one packet might not fit in a single read, and this - * needs to be blocking. - */ - if (len > n) { - rem = recv(c->fd_tap, p + n, len - n, 0); - if ((n += rem) != len) - return; + if (l2len < sizeof(struct ethhdr) || l2len > ETH_MAX_MTU) { + err("Bad frame size from guest, resetting connection"); + tap_sock_reset(c); + return; } - /* Complete the partial read above before discarding a malformed - * frame, otherwise the stream will be inconsistent. - */ - if (len < (ssize_t)sizeof(struct ethhdr) || - len > (ssize_t)ETH_MAX_MTU) - goto next; + if (l2len + sizeof(uint32_t) > (size_t)n) + /* Leave this incomplete frame for later */ + break; - packet_add_all(c, len, p); + p += sizeof(uint32_t); + n -= sizeof(uint32_t); -next: - p += len; - n -= len; + tap_add_packet(c, l2len, p); + + p += l2len; + n -= l2len; } - tap_handler_all(c, now); + partial_len = n; + partial_frame = p; - /* We can't use EPOLLET otherwise. */ - if (rem) - goto redo; + tap_handler(c, now); } /** - * tap_handler_pasta() - Packet handler for /dev/net/tun file descriptor + * tap_handler_passt() - Event handler for AF_UNIX file descriptor * @c: Execution context * @events: epoll events * @now: Current timestamp */ -void tap_handler_pasta(struct ctx *c, uint32_t events, +void tap_handler_passt(struct ctx *c, uint32_t events, const struct timespec *now) { - ssize_t n, len; - int ret; - - if (events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)) - die("Disconnect event on /dev/net/tun device, exiting"); + if (events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)) { + tap_sock_reset(c); + return; + } -redo: - n = 0; + if (events & EPOLLIN) + tap_passt_input(c, now); +} - pool_flush_all(); -restart: - while ((len = read(c->fd_tap, pkt_buf + n, TAP_BUF_BYTES - n)) > 0) { +/** + * tap_pasta_input() - Handler for new data on the socket to hypervisor + * @c: Execution context + * @now: Current timestamp + */ +static void tap_pasta_input(struct ctx *c, const struct timespec *now) +{ + ssize_t n, len; - if (len < (ssize_t)sizeof(struct ethhdr) || - len > (ssize_t)ETH_MAX_MTU) { - n += len; - continue; - } + tap_flush_pools(); + for (n = 0; n <= (ssize_t)(TAP_BUF_BYTES - ETH_MAX_MTU); n += len) { + len = read(c->fd_tap, pkt_buf + n, ETH_MAX_MTU); - packet_add_all(c, len, pkt_buf + n); + if (len == 0) { + die("EOF on tap device, exiting"); + } else if (len < 0) { + if (errno == EINTR) { + len = 0; + continue; + } - if ((n += len) == TAP_BUF_BYTES) - break; - } + if (errno == EAGAIN && errno == EWOULDBLOCK) + break; /* all done for now */ - if (len < 0 && errno == EINTR) - goto restart; + die("Error on tap device, exiting"); + } - ret = errno; + /* Ignore frames of bad length */ + if (len < (ssize_t)sizeof(struct ethhdr) || + len > (ssize_t)ETH_MAX_MTU) + continue; - tap_handler_all(c, now); + tap_add_packet(c, len, pkt_buf + n); + } - if (len > 0 || ret == EAGAIN) - return; + tap_handler(c, now); +} - if (n == TAP_BUF_BYTES) - goto redo; +/** + * tap_handler_pasta() - Packet handler for /dev/net/tun file descriptor + * @c: Execution context + * @events: epoll events + * @now: Current timestamp + */ +void tap_handler_pasta(struct ctx *c, uint32_t events, + const struct timespec *now) +{ + if (events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)) + die("Disconnect event on /dev/net/tun device, exiting"); - die("Error on tap device, exiting"); + if (events & EPOLLIN) + tap_pasta_input(c, now); } /** - * tap_sock_unix_init() - Create and bind AF_UNIX socket, listen for connection - * @c: Execution context + * tap_sock_unix_open() - Create and bind AF_UNIX socket + * @sock_path: Socket path. If empty, set on return (UNIX_SOCK_PATH as prefix) + * + * Return: socket descriptor on success, won't return on failure */ -static void tap_sock_unix_init(struct ctx *c) +int tap_sock_unix_open(char *sock_path) { - int fd = socket(AF_UNIX, SOCK_STREAM, 0); - union epoll_ref ref = { .type = EPOLL_TYPE_TAP_LISTEN }; - struct epoll_event ev = { 0 }; + int fd = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); struct sockaddr_un addr = { .sun_family = AF_UNIX, }; int i; if (fd < 0) - die("UNIX socket: %s", strerror(errno)); - - /* In passt mode, we don't know the guest's MAC until it sends - * us packets. Use the broadcast address so our first packets - * will reach it. - */ - memset(&c->mac_guest, 0xff, sizeof(c->mac_guest)); + die_perror("Failed to open UNIX domain socket"); for (i = 1; i < UNIX_SOCK_MAX; i++) { char *path = addr.sun_path; int ex, ret; - if (*c->sock_path) - memcpy(path, c->sock_path, UNIX_PATH_MAX); - else - snprintf(path, UNIX_PATH_MAX - 1, UNIX_SOCK_PATH, i); + if (*sock_path) + memcpy(path, sock_path, UNIX_PATH_MAX); + else if (snprintf_check(path, UNIX_PATH_MAX - 1, + UNIX_SOCK_PATH, i)) + die_perror("Can't build UNIX domain socket path"); - ex = socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0); + ex = socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, + 0); if (ex < 0) - die("UNIX domain socket check: %s", strerror(errno)); + die_perror("Failed to check for UNIX domain conflicts"); ret = connect(ex, (const struct sockaddr *)&addr, sizeof(addr)); if (!ret || (errno != ENOENT && errno != ECONNREFUSED && errno != EACCES)) { - if (*c->sock_path) + if (*sock_path) die("Socket path %s already in use", path); close(ex); @@ -1246,45 +1190,75 @@ static void tap_sock_unix_init(struct ctx *c) close(ex); unlink(path); - if (!bind(fd, (const struct sockaddr *)&addr, sizeof(addr)) || - *c->sock_path) + ret = bind(fd, (const struct sockaddr *)&addr, sizeof(addr)); + if (*sock_path && ret) + die_perror("Failed to bind UNIX domain socket"); + + if (!ret) break; } if (i == UNIX_SOCK_MAX) - die("UNIX socket bind: %s", strerror(errno)); - - info("UNIX domain socket bound at %s\n", addr.sun_path); + die_perror("Failed to bind UNIX domain socket"); - listen(fd, 0); + info("UNIX domain socket bound at %s", addr.sun_path); + if (!*sock_path) + memcpy(sock_path, addr.sun_path, UNIX_PATH_MAX); - ref.fd = c->fd_tap_listen = fd; - ev.events = EPOLLIN | EPOLLET; - ev.data.u64 = ref.u64; - epoll_ctl(c->epollfd, EPOLL_CTL_ADD, c->fd_tap_listen, &ev); + return fd; +} - if (c->mode == MODE_VU) { - info("You can start qemu with:"); - info(" kvm ... -chardev socket,id=chr0,path=%s -netdev vhost-user,id=netdev0,chardev=chr0 -device virtio-net,netdev=netdev0 -object memory-backend-memfd,id=memfd0,share=on,size=$RAMSIZE -numa node,memdev=memfd0\n", - addr.sun_path); - } else { - info("You can now start qemu (>= 7.2, with commit 13c6be96618c):"); +/** + * tap_backend_show_hints() - Give help information to start QEMU + * @c: Execution context + */ +static void tap_backend_show_hints(struct ctx *c) +{ + switch (c->mode) { + case MODE_PASTA: + /* No hints */ + break; + case MODE_PASST: + info("\nYou can now start qemu (>= 7.2, with commit 13c6be96618c):"); info(" kvm ... -device virtio-net-pci,netdev=s -netdev stream,id=s,server=off,addr.type=unix,addr.path=%s", - addr.sun_path); + c->sock_path); info("or qrap, for earlier qemu versions:"); info(" ./qrap 5 kvm ... -net socket,fd=5 -net nic,model=virtio"); + break; + case MODE_VU: + info("You can start qemu with:"); + info(" kvm ... -chardev socket,id=chr0,path=%s -netdev vhost-user,id=netdev0,chardev=chr0 -device virtio-net,netdev=netdev0 -object memory-backend-memfd,id=memfd0,share=on,size=$RAMSIZE -numa node,memdev=memfd0\n", + c->sock_path); + break; } } /** + * tap_sock_unix_init() - Start listening for connections on AF_UNIX socket + * @c: Execution context + */ +static void tap_sock_unix_init(const struct ctx *c) +{ + union epoll_ref ref = { .type = EPOLL_TYPE_TAP_LISTEN }; + struct epoll_event ev = { 0 }; + + listen(c->fd_tap_listen, 0); + + ref.fd = c->fd_tap_listen; + ev.events = EPOLLIN | EPOLLET; + ev.data.u64 = ref.u64; + epoll_ctl(c->epollfd, EPOLL_CTL_ADD, c->fd_tap_listen, &ev); +} + +/** * tap_listen_handler() - Handle new connection on listening socket * @c: Execution context * @events: epoll events */ void tap_listen_handler(struct ctx *c, uint32_t events) { - union epoll_ref ref; struct epoll_event ev = { 0 }; + union epoll_ref ref = { 0 }; int v = INT_MAX / 2; struct ucred ucred; socklen_t len; @@ -1324,13 +1298,11 @@ void tap_listen_handler(struct ctx *c, uint32_t events) trace("tap: failed to set SO_SNDBUF to %i", v); ref.fd = c->fd_tap; - if (c->mode == MODE_VU) { + if (c->mode == MODE_VU) ref.type = EPOLL_TYPE_VHOST_CMD; - ev.events = EPOLLIN | EPOLLRDHUP; - } else { + else ref.type = EPOLL_TYPE_TAP_PASST; - ev.events = EPOLLIN | EPOLLRDHUP | EPOLLET; - } + ev.events = EPOLLIN | EPOLLRDHUP; ev.data.u64 = ref.u64; epoll_ctl(c->epollfd, EPOLL_CTL_ADD, c->fd_tap, &ev); } @@ -1356,11 +1328,11 @@ static int tap_ns_tun(void *arg) fd = open("/dev/net/tun", flags); if (fd < 0) - die("Failed to open() /dev/net/tun: %s", strerror(errno)); + die_perror("Failed to open() /dev/net/tun"); - rc = ioctl(fd, TUNSETIFF, &ifr); + rc = ioctl(fd, (int)TUNSETIFF, &ifr); if (rc < 0) - die("TUNSETIFF failed: %s", strerror(errno)); + die_perror("TUNSETIFF ioctl on /dev/net/tun failed"); if (!(c->pasta_ifi = if_nametoindex(c->pasta_ifn))) die("Tap device opened but no network interface found"); @@ -1391,51 +1363,35 @@ static void tap_sock_tun_init(struct ctx *c) epoll_ctl(c->epollfd, EPOLL_CTL_ADD, c->fd_tap, &ev); } -void tap_sock_update_buf(void *base, size_t size) +/** + * tap_sock_update_pool() - Set the buffer base and size for the pool of packets + * @base: Buffer base + * @size Buffer size + */ +void tap_sock_update_pool(void *base, size_t size) { int i; - pool_tap4_storage.buf = base; - pool_tap4_storage.buf_size = size; - pool_tap6_storage.buf = base; - pool_tap6_storage.buf_size = size; + pool_tap4_storage = PACKET_INIT(pool_tap4, TAP_MSGS, base, size); + pool_tap6_storage = PACKET_INIT(pool_tap6, TAP_MSGS, base, size); for (i = 0; i < TAP_SEQS; i++) { - tap4_l4[i].p.buf = base; - tap4_l4[i].p.buf_size = size; - tap6_l4[i].p.buf = base; - tap6_l4[i].p.buf_size = size; + tap4_l4[i].p = PACKET_INIT(pool_l4, UIO_MAXIOV, base, size); + tap6_l4[i].p = PACKET_INIT(pool_l4, UIO_MAXIOV, base, size); } } /** - * tap_sock_init() - Create and set up AF_UNIX socket or tuntap file descriptor + * tap_backend_init() - Create and set up AF_UNIX socket or + * tuntap file descriptor * @c: Execution context */ -void tap_sock_init(struct ctx *c) +void tap_backend_init(struct ctx *c) { - size_t sz = sizeof(pkt_buf); - int i; - - pool_tap4_storage = PACKET_INIT(pool_tap4, TAP_MSGS, pkt_buf, sz); - pool_tap6_storage = PACKET_INIT(pool_tap6, TAP_MSGS, pkt_buf, sz); - if (c->mode == MODE_VU) { - pool_tap4_storage.buf = NULL; - pool_tap4_storage.buf_size = 0; - pool_tap6_storage.buf = NULL; - pool_tap6_storage.buf_size = 0; - } - - for (i = 0; i < TAP_SEQS; i++) { - tap4_l4[i].p = PACKET_INIT(pool_l4, UIO_MAXIOV, pkt_buf, sz); - tap6_l4[i].p = PACKET_INIT(pool_l4, UIO_MAXIOV, pkt_buf, sz); - if (c->mode == MODE_VU) { - tap4_l4[i].p.buf = NULL; - tap4_l4[i].p.buf_size = 0; - tap6_l4[i].p.buf = NULL; - tap6_l4[i].p.buf_size = 0; - } - } + if (c->mode == MODE_VU) + tap_sock_update_pool(NULL, 0); + else + tap_sock_update_pool(pkt_buf, sizeof(pkt_buf)); if (c->fd_tap != -1) { /* Passed as --fd */ struct epoll_event ev = { 0 }; @@ -1446,27 +1402,38 @@ void tap_sock_init(struct ctx *c) switch (c->mode) { case MODE_PASST: ref.type = EPOLL_TYPE_TAP_PASST; - ev.events = EPOLLIN | EPOLLET | EPOLLRDHUP; break; case MODE_PASTA: ref.type = EPOLL_TYPE_TAP_PASTA; - ev.events = EPOLLIN | EPOLLET | EPOLLRDHUP; break; case MODE_VU: ref.type = EPOLL_TYPE_VHOST_CMD; - ev.events = EPOLLIN | EPOLLRDHUP; break; } + ev.events = EPOLLIN | EPOLLRDHUP; ev.data.u64 = ref.u64; epoll_ctl(c->epollfd, EPOLL_CTL_ADD, c->fd_tap, &ev); return; } - if (c->mode == MODE_PASTA) { + switch (c->mode) { + case MODE_PASTA: tap_sock_tun_init(c); - } else { - if (c->fd_tap_listen == -1) - tap_sock_unix_init(c); + break; + case MODE_VU: + vu_init(c); + /* fall through */ + case MODE_PASST: + tap_sock_unix_init(c); + + /* In passt mode, we don't know the guest's MAC address until it + * sends us packets. Use the broadcast address so that our + * first packets will reach it. + */ + memset(&c->guest_mac, 0xff, sizeof(c->guest_mac)); + break; } + + tap_backend_show_hints(c); } |