diff options
Diffstat (limited to 'tcp_vu.c')
| -rw-r--r-- | tcp_vu.c | 460 |
1 files changed, 460 insertions, 0 deletions
diff --git a/tcp_vu.c b/tcp_vu.c new file mode 100644 index 0000000..b0a3993 --- /dev/null +++ b/tcp_vu.c @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include <errno.h> +#include <stddef.h> +#include <stdint.h> + +#include <netinet/ip.h> + +#include <sys/socket.h> + +#include <linux/tcp.h> +#include <linux/virtio_net.h> + +#include "util.h" +#include "ip.h" +#include "passt.h" +#include "siphash.h" +#include "inany.h" +#include "vhost_user.h" +#include "tcp.h" +#include "pcap.h" +#include "flow.h" +#include "tcp_conn.h" +#include "flow_table.h" +#include "tcp_vu.h" +#include "tcp_internal.h" +#include "checksum.h" + +#define CONN_V4(conn) (!!inany_v4(&(conn)->faddr)) +#define CONN_V6(conn) (!CONN_V4(conn)) + +/* vhost-user */ +static const struct virtio_net_hdr vu_header = { + .flags = VIRTIO_NET_HDR_F_DATA_VALID, + .gso_type = VIRTIO_NET_HDR_GSO_NONE, +}; + +static unsigned char buffer[65536]; +static struct iovec iov_vu [VIRTQUEUE_MAX_SIZE]; +static unsigned int indexes [VIRTQUEUE_MAX_SIZE]; + +uint16_t tcp_vu_conn_tap_mss(const struct tcp_tap_conn *conn) +{ + (void)conn; + return USHRT_MAX; +} + +int tcp_vu_send_flag(struct ctx *c, struct tcp_tap_conn *conn, int flags) +{ + VuDev *vdev = (VuDev *)&c->vdev; + VuVirtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE]; + size_t tlen, vnet_hdrlen, ip_len, optlen = 0; + struct virtio_net_hdr_mrg_rxbuf *vh; + VuVirtqElement *elem; + struct ethhdr *eh; + int nb_ack; + int ret; + + elem = vu_queue_pop(vdev, vq, sizeof(VuVirtqElement), buffer); + if (!elem) + return 0; + + if (elem->in_num < 1) { + err("virtio-net receive queue contains no in buffers"); + vu_queue_rewind(vdev, vq, 1); + return 0; + } + + vh = elem->in_sg[0].iov_base; + + vh->hdr = vu_header; + if (vu_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) { + vnet_hdrlen = sizeof(struct virtio_net_hdr_mrg_rxbuf); + vh->num_buffers = htole16(1); + } else { + vnet_hdrlen = sizeof(struct virtio_net_hdr); + } + eh = (struct ethhdr *)((char *)elem->in_sg[0].iov_base + vnet_hdrlen); + + memcpy(eh->h_dest, c->mac_guest, sizeof(eh->h_dest)); + memcpy(eh->h_source, c->mac, sizeof(eh->h_source)); + + if (CONN_V4(conn)) { + struct iphdr *iph = (struct iphdr *)(eh + 1); + struct tcphdr *th = (struct tcphdr *)(iph + 1); + char *data = (char *)(th + 1); + + eh->h_proto = htons(ETH_P_IP); + + *th = (struct tcphdr){ + .doff = sizeof(struct tcphdr) / 4, + .ack = 1 + }; + + *iph = (struct iphdr)L2_BUF_IP4_INIT(IPPROTO_TCP); + + ret = tcp_fill_flag_header(c, conn, flags, th, data, &optlen); + if (ret <= 0) { + vu_queue_rewind(vdev, vq, 1); + return ret; + } + + ip_len = tcp_fill_headers4(c, conn, iph, + (struct tcphdr *)(iph + 1), optlen, + NULL, conn->seq_to_tap); + + tlen = ip_len + sizeof(struct ethhdr); + + if (*c->pcap) { + uint32_t sum = proto_ipv4_header_psum(iph->tot_len, + IPPROTO_TCP, + (struct in_addr){ .s_addr = iph->saddr }, + (struct in_addr){ .s_addr = iph->daddr }); + + th->check = csum(th, optlen + sizeof(struct tcphdr), sum); + } + } else { + struct ipv6hdr *ip6h = (struct ipv6hdr *)(eh + 1); + struct tcphdr *th = (struct tcphdr *)(ip6h + 1); + char *data = (char *)(th + 1); + + eh->h_proto = htons(ETH_P_IPV6); + + *th = (struct tcphdr){ + .doff = sizeof(struct tcphdr) / 4, + .ack = 1 + }; + + *ip6h = (struct ipv6hdr)L2_BUF_IP6_INIT(IPPROTO_TCP); + + ret = tcp_fill_flag_header(c, conn, flags, th, data, &optlen); + if (ret <= 0) { + vu_queue_rewind(vdev, vq, 1); + return ret; + } + + ip_len = tcp_fill_headers6(c, conn, ip6h, + (struct tcphdr *)(ip6h + 1), + optlen, conn->seq_to_tap); + + tlen = ip_len + sizeof(struct ethhdr); + + if (*c->pcap) { + uint32_t sum = proto_ipv6_header_psum(ip6h->payload_len, + IPPROTO_TCP, + &ip6h->saddr, + &ip6h->daddr); + + th->check = csum(th, optlen + sizeof(struct tcphdr), sum); + } + } + + pcap((void *)eh, tlen); + + tlen += vnet_hdrlen; + vu_queue_fill(vdev, vq, elem, tlen, 0); + nb_ack = 1; + + if (flags & DUP_ACK) { + elem = vu_queue_pop(vdev, vq, sizeof(VuVirtqElement), buffer); + if (elem) { + if (elem->in_num < 1 || elem->in_sg[0].iov_len < tlen) { + vu_queue_rewind(vdev, vq, 1); + } else { + memcpy(elem->in_sg[0].iov_base, vh, tlen); + nb_ack++; + } + } + } + + vu_queue_flush(vdev, vq, nb_ack); + vu_queue_notify(vdev, vq); + + return 0; +} + +int tcp_vu_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn) +{ + uint32_t wnd_scaled = conn->wnd_from_tap << conn->ws_from_tap; + uint32_t already_sent; + VuDev *vdev = (VuDev *)&c->vdev; + VuVirtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE]; + int s = conn->sock, v4 = CONN_V4(conn); + int i, ret = 0, iov_count, iov_used; + struct msghdr mh_sock = { 0 }; + size_t l2_hdrlen, vnet_hdrlen, fillsize; + ssize_t len; + uint16_t *check; + uint16_t mss = MSS_GET(conn); + int num_buffers; + int segment_size; + struct iovec *first; + bool has_mrg_rxbuf; + + if (!vu_queue_enabled(vq) || !vu_queue_started(vq)) { + err("Got packet, but no available descriptors on RX virtq."); + return 0; + } + + already_sent = conn->seq_to_tap - conn->seq_ack_from_tap; + + if (SEQ_LT(already_sent, 0)) { + /* RFC 761, section 2.1. */ + flow_trace(conn, "ACK sequence gap: ACK for %u, sent: %u", + conn->seq_ack_from_tap, conn->seq_to_tap); + conn->seq_to_tap = conn->seq_ack_from_tap; + already_sent = 0; + } + + if (!wnd_scaled || already_sent >= wnd_scaled) { + conn_flag(c, conn, STALLED); + conn_flag(c, conn, ACK_FROM_TAP_DUE); + return 0; + } + + /* Set up buffer descriptors we'll fill completely and partially. */ + + fillsize = wnd_scaled; + + iov_vu[0].iov_base = tcp_buf_discard; + iov_vu[0].iov_len = already_sent; + fillsize -= already_sent; + + has_mrg_rxbuf = vu_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF); + if (has_mrg_rxbuf) { + vnet_hdrlen = sizeof(struct virtio_net_hdr_mrg_rxbuf); + } else { + vnet_hdrlen = sizeof(struct virtio_net_hdr); + } + l2_hdrlen = vnet_hdrlen + sizeof(struct ethhdr) + sizeof(struct tcphdr); + if (v4) { + l2_hdrlen += sizeof(struct iphdr); + } else { + l2_hdrlen += sizeof(struct ipv6hdr); + } + + iov_count = 0; + segment_size = 0; + while (fillsize > 0 && iov_count < VIRTQUEUE_MAX_SIZE - 1) { + VuVirtqElement *elem; + + elem = vu_queue_pop(vdev, vq, sizeof(VuVirtqElement), buffer); + if (!elem) + break; + + if (elem->in_num < 1) { + err("virtio-net receive queue contains no in buffers"); + goto err; + } + + ASSERT(elem->in_num == 1); + ASSERT(elem->in_sg[0].iov_len >= l2_hdrlen); + + indexes[iov_count] = elem->index; + + if (segment_size == 0) { + iov_vu[iov_count + 1].iov_base = + (char *)elem->in_sg[0].iov_base + l2_hdrlen; + iov_vu[iov_count + 1].iov_len = + elem->in_sg[0].iov_len - l2_hdrlen; + } else { + iov_vu[iov_count + 1].iov_base = elem->in_sg[0].iov_base; + iov_vu[iov_count + 1].iov_len = elem->in_sg[0].iov_len; + } + + if (iov_vu[iov_count + 1].iov_len > fillsize) + iov_vu[iov_count + 1].iov_len = fillsize; + + segment_size += iov_vu[iov_count + 1].iov_len; + if (!has_mrg_rxbuf) { + segment_size = 0; + } else if (segment_size >= mss) { + iov_vu[iov_count + 1].iov_len -= segment_size - mss; + segment_size = 0; + } + fillsize -= iov_vu[iov_count + 1].iov_len; + + iov_count++; + } + if (iov_count == 0) + return 0; + + mh_sock.msg_iov = iov_vu; + mh_sock.msg_iovlen = iov_count + 1; + + do + len = recvmsg(s, &mh_sock, MSG_PEEK); + while (len < 0 && errno == EINTR); + + if (len < 0) + goto err; + + if (!len) { + vu_queue_rewind(vdev, vq, iov_count); + if ((conn->events & (SOCK_FIN_RCVD | TAP_FIN_SENT)) == SOCK_FIN_RCVD) { + if ((ret = tcp_vu_send_flag(c, conn, FIN | ACK))) { + tcp_rst(c, conn); + return ret; + } + + conn_event(c, conn, TAP_FIN_SENT); + } + + return 0; + } + + len -= already_sent; + if (len <= 0) { + conn_flag(c, conn, STALLED); + vu_queue_rewind(vdev, vq, iov_count); + return 0; + } + + conn_flag(c, conn, ~STALLED); + + /* Likely, some new data was acked too. */ + tcp_update_seqack_wnd(c, conn, 0, NULL); + + /* initialize headers */ + iov_used = 0; + num_buffers = 0; + check = NULL; + segment_size = 0; + for (i = 0; i < iov_count && len; i++) { + + if (segment_size == 0) + first = &iov_vu[i + 1]; + + if (iov_vu[i + 1].iov_len > (size_t)len) + iov_vu[i + 1].iov_len = len; + + len -= iov_vu[i + 1].iov_len; + iov_used++; + + segment_size += iov_vu[i + 1].iov_len; + num_buffers++; + + if (segment_size >= mss || len == 0 || + i + 1 == iov_count || !has_mrg_rxbuf) { + + struct ethhdr *eh; + struct virtio_net_hdr_mrg_rxbuf *vh; + char *base = (char *)first->iov_base - l2_hdrlen; + size_t size = first->iov_len + l2_hdrlen; + + vh = (struct virtio_net_hdr_mrg_rxbuf *)base; + + vh->hdr = vu_header; + if (has_mrg_rxbuf) + vh->num_buffers = htole16(num_buffers); + + eh = (struct ethhdr *)((char *)base + vnet_hdrlen); + + memcpy(eh->h_dest, c->mac_guest, sizeof(eh->h_dest)); + memcpy(eh->h_source, c->mac, sizeof(eh->h_source)); + + /* initialize header */ + if (v4) { + struct iphdr *iph = (struct iphdr *)(eh + 1); + struct tcphdr *th = (struct tcphdr *)(iph + 1); + + eh->h_proto = htons(ETH_P_IP); + + *th = (struct tcphdr){ + .doff = sizeof(struct tcphdr) / 4, + .ack = 1 + }; + + *iph = (struct iphdr)L2_BUF_IP4_INIT(IPPROTO_TCP); + + tcp_fill_headers4(c, conn, iph, + (struct tcphdr *)(iph + 1), + segment_size, len ? check : NULL, + conn->seq_to_tap); + + if (*c->pcap) { + uint32_t sum = proto_ipv4_header_psum(iph->tot_len, + IPPROTO_TCP, + (struct in_addr){ .s_addr = iph->saddr }, + (struct in_addr){ .s_addr = iph->daddr }); + + first->iov_base = th; + first->iov_len = size - l2_hdrlen + sizeof(*th); + + th->check = csum_iov(first, num_buffers, sum); + } + + check = &iph->check; + } else { + struct ipv6hdr *ip6h = (struct ipv6hdr *)(eh + 1); + struct tcphdr *th = (struct tcphdr *)(ip6h + 1); + + eh->h_proto = htons(ETH_P_IPV6); + + *th = (struct tcphdr){ + .doff = sizeof(struct tcphdr) / 4, + .ack = 1 + }; + + *ip6h = (struct ipv6hdr)L2_BUF_IP6_INIT(IPPROTO_TCP); + + tcp_fill_headers6(c, conn, ip6h, + (struct tcphdr *)(ip6h + 1), + segment_size, conn->seq_to_tap); + if (*c->pcap) { + uint32_t sum = proto_ipv6_header_psum(ip6h->payload_len, + IPPROTO_TCP, + &ip6h->saddr, + &ip6h->daddr); + + first->iov_base = th; + first->iov_len = size - l2_hdrlen + sizeof(*th); + + th->check = csum_iov(first, num_buffers, sum); + } + } + + /* set iov for pcap logging */ + first->iov_base = eh; + first->iov_len = size - vnet_hdrlen; + + pcap_iov(first, num_buffers); + + /* set iov_len for vu_queue_fill_by_index(); */ + + first->iov_base = base; + first->iov_len = size; + + conn->seq_to_tap += segment_size; + + segment_size = 0; + num_buffers = 0; + } + } + + /* release unused buffers */ + vu_queue_rewind(vdev, vq, iov_count - iov_used); + + /* send packets */ + for (i = 0; i < iov_used; i++) { + vu_queue_fill_by_index(vdev, vq, indexes[i], + iov_vu[i + 1].iov_len, i); + } + + vu_queue_flush(vdev, vq, iov_used); + vu_queue_notify(vdev, vq); + + conn_flag(c, conn, ACK_FROM_TAP_DUE); + + return 0; +err: + vu_queue_rewind(vdev, vq, iov_count); + + if (errno != EAGAIN && errno != EWOULDBLOCK) { + ret = -errno; + tcp_rst(c, conn); + } + + return ret; +} |