aboutgitcodebugslistschat
path: root/udp.c
diff options
context:
space:
mode:
Diffstat (limited to 'udp.c')
-rw-r--r--udp.c312
1 files changed, 157 insertions, 155 deletions
diff --git a/udp.c b/udp.c
index 694424a..3abafc9 100644
--- a/udp.c
+++ b/udp.c
@@ -113,6 +113,7 @@
#include "checksum.h"
#include "util.h"
+#include "iov.h"
#include "ip.h"
#include "siphash.h"
#include "inany.h"
@@ -171,65 +172,72 @@ static uint8_t udp_act[IP_VERSIONS][UDP_ACT_TYPE_MAX][DIV_ROUND_UP(NUM_PORTS, 8)
/* Static buffers */
/**
- * udp4_l2_buf_t - Pre-cooked IPv4 packet buffers for tap connections
- * @s_in: Source socket address, filled in by recvmmsg()
- * @taph: Tap-level headers (partially pre-filled)
- * @iph: Pre-filled IP header (except for tot_len and saddr)
- * @uh: Headroom for UDP header
- * @data: Storage for UDP payload
+ * struct udp_payload_t - UDP header and data for inbound messages
+ * @uh: UDP header
+ * @data: UDP data
*/
-static struct udp4_l2_buf_t {
- struct sockaddr_in s_in;
-
- struct tap_hdr taph;
- struct iphdr iph;
+static struct udp_payload_t {
struct udphdr uh;
- uint8_t data[USHRT_MAX -
- (sizeof(struct iphdr) + sizeof(struct udphdr))];
+ char data[USHRT_MAX - sizeof(struct udphdr)];
+#ifdef __AVX2__
+} __attribute__ ((packed, aligned(32)))
+#else
} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
-udp4_l2_buf[UDP_MAX_FRAMES];
+#endif
+udp_payload[UDP_MAX_FRAMES];
+
+/* Ethernet header for IPv4 frames */
+static struct ethhdr udp4_eth_hdr;
+
+/* Ethernet header for IPv6 frames */
+static struct ethhdr udp6_eth_hdr;
/**
- * udp6_l2_buf_t - Pre-cooked IPv6 packet buffers for tap connections
- * @s_in6: Source socket address, filled in by recvmmsg()
- * @taph: Tap-level headers (partially pre-filled)
- * @ip6h: Pre-filled IP header (except for payload_len and addresses)
- * @uh: Headroom for UDP header
- * @data: Storage for UDP payload
+ * struct udp_meta_t - Pre-cooked headers and metadata for UDP packets
+ * @ip6h: Pre-filled IPv6 header (except for payload_len and addresses)
+ * @ip4h: Pre-filled IPv4 header (except for tot_len and saddr)
+ * @taph: Tap backend specific header
+ * @s_in: Source socket address, filled in by recvmmsg()
*/
-struct udp6_l2_buf_t {
- struct sockaddr_in6 s_in6;
-#ifdef __AVX2__
- /* Align ip6h to 32-byte boundary. */
- uint8_t pad[64 - (sizeof(struct sockaddr_in6) + sizeof(struct ethhdr) +
- sizeof(uint32_t))];
-#endif
-
- struct tap_hdr taph;
+static struct udp_meta_t {
struct ipv6hdr ip6h;
- struct udphdr uh;
- uint8_t data[USHRT_MAX -
- (sizeof(struct ipv6hdr) + sizeof(struct udphdr))];
+ struct iphdr ip4h;
+ struct tap_hdr taph;
+
+ union sockaddr_inany s_in;
+}
#ifdef __AVX2__
-} __attribute__ ((packed, aligned(32)))
-#else
-} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
+__attribute__ ((aligned(32)))
#endif
-udp6_l2_buf[UDP_MAX_FRAMES];
+udp_meta[UDP_MAX_FRAMES];
+
+/**
+ * enum udp_iov_idx - Indices for the buffers making up a single UDP frame
+ * @UDP_IOV_TAP tap specific header
+ * @UDP_IOV_ETH Ethernet header
+ * @UDP_IOV_IP IP (v4/v6) header
+ * @UDP_IOV_PAYLOAD IP payload (UDP header + data)
+ * @UDP_NUM_IOVS the number of entries in the iovec array
+ */
+enum udp_iov_idx {
+ UDP_IOV_TAP = 0,
+ UDP_IOV_ETH = 1,
+ UDP_IOV_IP = 2,
+ UDP_IOV_PAYLOAD = 3,
+ UDP_NUM_IOVS
+};
/* recvmmsg()/sendmmsg() data for tap */
-static struct iovec udp4_l2_iov_sock [UDP_MAX_FRAMES];
-static struct iovec udp6_l2_iov_sock [UDP_MAX_FRAMES];
+static struct iovec udp_l2_iov_sock [UDP_MAX_FRAMES];
-static struct iovec udp4_l2_iov_tap [UDP_MAX_FRAMES];
-static struct iovec udp6_l2_iov_tap [UDP_MAX_FRAMES];
+static struct iovec udp4_l2_iov_tap [UDP_MAX_FRAMES][UDP_NUM_IOVS];
+static struct iovec udp6_l2_iov_tap [UDP_MAX_FRAMES][UDP_NUM_IOVS];
static struct mmsghdr udp4_l2_mh_sock [UDP_MAX_FRAMES];
static struct mmsghdr udp6_l2_mh_sock [UDP_MAX_FRAMES];
/* recvmmsg()/sendmmsg() data for "spliced" connections */
-static struct iovec udp4_iov_splice [UDP_MAX_FRAMES];
-static struct iovec udp6_iov_splice [UDP_MAX_FRAMES];
+static struct iovec udp_iov_splice [UDP_MAX_FRAMES];
static struct sockaddr_in udp4_localname = {
.sin_family = AF_INET,
@@ -283,87 +291,71 @@ static void udp_invert_portmap(struct udp_fwd_ports *fwd)
*/
void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s)
{
- int i;
-
- for (i = 0; i < UDP_MAX_FRAMES; i++) {
- struct udp4_l2_buf_t *b4 = &udp4_l2_buf[i];
- struct udp6_l2_buf_t *b6 = &udp6_l2_buf[i];
-
- eth_update_mac(&b4->taph.eh, eth_d, eth_s);
- eth_update_mac(&b6->taph.eh, eth_d, eth_s);
- }
+ eth_update_mac(&udp4_eth_hdr, eth_d, eth_s);
+ eth_update_mac(&udp6_eth_hdr, eth_d, eth_s);
}
/**
- * udp_sock4_iov_init_one() - Initialise a scatter-gather L2 buffer for IPv4
+ * udp_iov_init_one() - Initialise scatter-gather lists for one buffer
* @c: Execution context
* @i: Index of buffer to initialize
*/
-static void udp_sock4_iov_init_one(const struct ctx *c, size_t i)
+static void udp_iov_init_one(const struct ctx *c, size_t i)
{
- struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
- struct udp4_l2_buf_t *buf = &udp4_l2_buf[i];
- struct iovec *siov = &udp4_l2_iov_sock[i];
- struct iovec *tiov = &udp4_l2_iov_tap[i];
-
- *buf = (struct udp4_l2_buf_t) {
- .taph = TAP_HDR_INIT(ETH_P_IP),
- .iph = L2_BUF_IP4_INIT(IPPROTO_UDP)
+ struct udp_payload_t *payload = &udp_payload[i];
+ struct iovec *siov = &udp_l2_iov_sock[i];
+ struct udp_meta_t *meta = &udp_meta[i];
+
+ *meta = (struct udp_meta_t) {
+ .ip4h = L2_BUF_IP4_INIT(IPPROTO_UDP),
+ .ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP),
};
- siov->iov_base = buf->data;
- siov->iov_len = sizeof(buf->data);
+ *siov = IOV_OF_LVALUE(payload->data);
+ udp4_eth_hdr.h_proto = htons_constant(ETH_P_IP);
+ udp6_eth_hdr.h_proto = htons_constant(ETH_P_IPV6);
- mh->msg_name = &buf->s_in;
- mh->msg_namelen = sizeof(buf->s_in);
- mh->msg_iov = siov;
- mh->msg_iovlen = 1;
+ if (c->ifi4) {
+ struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
+ struct iovec *tiov = udp4_l2_iov_tap[i];
- tiov->iov_base = tap_frame_base(c, &buf->taph);
-}
+ mh->msg_name = &meta->s_in;
+ mh->msg_namelen = sizeof(struct sockaddr_in);
+ mh->msg_iov = siov;
+ mh->msg_iovlen = 1;
-/**
- * udp_sock6_iov_init_one() - Initialise a scatter-gather L2 buffer for IPv6
- * @c: Execution context
- * @i: Index of buffer to initialize
- */
-static void udp_sock6_iov_init_one(const struct ctx *c, size_t i)
-{
- struct msghdr *mh = &udp6_l2_mh_sock[i].msg_hdr;
- struct udp6_l2_buf_t *buf = &udp6_l2_buf[i];
- struct iovec *siov = &udp6_l2_iov_sock[i];
- struct iovec *tiov = &udp6_l2_iov_tap[i];
-
- *buf = (struct udp6_l2_buf_t) {
- .taph = TAP_HDR_INIT(ETH_P_IPV6),
- .ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP)
- };
+ tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &meta->taph);
+ tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(udp4_eth_hdr);
+ tiov[UDP_IOV_IP] = IOV_OF_LVALUE(meta->ip4h);
+ tiov[UDP_IOV_PAYLOAD].iov_base = payload;
+ }
- siov->iov_base = buf->data;
- siov->iov_len = sizeof(buf->data);
+ if (c->ifi6) {
+ struct msghdr *mh = &udp6_l2_mh_sock[i].msg_hdr;
+ struct iovec *tiov = udp6_l2_iov_tap[i];
- mh->msg_name = &buf->s_in6;
- mh->msg_namelen = sizeof(buf->s_in6);
- mh->msg_iov = siov;
- mh->msg_iovlen = 1;
+ mh->msg_name = &meta->s_in;
+ mh->msg_namelen = sizeof(struct sockaddr_in6);
+ mh->msg_iov = siov;
+ mh->msg_iovlen = 1;
- tiov->iov_base = tap_frame_base(c, &buf->taph);
+ tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &meta->taph);
+ tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(udp6_eth_hdr);
+ tiov[UDP_IOV_IP] = IOV_OF_LVALUE(meta->ip6h);
+ tiov[UDP_IOV_PAYLOAD].iov_base = payload;
+ }
}
/**
- * udp_sock_iov_init() - Initialise scatter-gather L2 buffers
+ * udp_iov_init() - Initialise scatter-gather L2 buffers
* @c: Execution context
*/
-static void udp_sock_iov_init(const struct ctx *c)
+static void udp_iov_init(const struct ctx *c)
{
size_t i;
- for (i = 0; i < UDP_MAX_FRAMES; i++) {
- if (c->ifi4)
- udp_sock4_iov_init_one(c, i);
- if (c->ifi6)
- udp_sock6_iov_init_one(c, i);
- }
+ for (i = 0; i < UDP_MAX_FRAMES; i++)
+ udp_iov_init_one(c, i);
}
/**
@@ -564,20 +556,24 @@ static void udp_splice_sendfrom(const struct ctx *c, unsigned start, unsigned n,
/**
* udp_update_hdr4() - Update headers for one IPv4 datagram
* @c: Execution context
- * @b: Pointer to udp4_l2_buf to update
+ * @bm: Pointer to udp_meta_t to update
+ * @bp: Pointer to udp_payload_t to update
* @dstport: Destination port number
- * @datalen: Length of UDP payload
+ * @dlen: Length of UDP payload
* @now: Current timestamp
*
- * Return: size of tap frame with headers
+ * Return: size of IPv4 payload (UDP header + data)
*/
-static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
- in_port_t dstport, size_t datalen,
+static size_t udp_update_hdr4(const struct ctx *c,
+ struct udp_meta_t *bm, struct udp_payload_t *bp,
+ in_port_t dstport, size_t dlen,
const struct timespec *now)
{
- size_t ip_len = datalen + sizeof(b->iph) + sizeof(b->uh);
- in_port_t srcport = ntohs(b->s_in.sin_port);
- struct in_addr src = b->s_in.sin_addr;
+ in_port_t srcport = ntohs(bm->s_in.sa4.sin_port);
+ const struct in_addr dst = c->ip4.addr_seen;
+ struct in_addr src = bm->s_in.sa4.sin_addr;
+ size_t l4len = dlen + sizeof(bp->uh);
+ size_t l3len = l4len + sizeof(bm->ip4h);
if (!IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns_match) &&
IN4_ARE_ADDR_EQUAL(&src, &c->ip4.dns_host) && srcport == 53 &&
@@ -598,37 +594,40 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
src = c->ip4.gw;
}
- b->iph.tot_len = htons(ip_len);
- b->iph.daddr = c->ip4.addr_seen.s_addr;
- b->iph.saddr = src.s_addr;
- b->iph.check = csum_ip4_header(b->iph.tot_len, IPPROTO_UDP,
- src, c->ip4.addr_seen);
+ bm->ip4h.tot_len = htons(l3len);
+ bm->ip4h.daddr = dst.s_addr;
+ bm->ip4h.saddr = src.s_addr;
+ bm->ip4h.check = csum_ip4_header(l3len, IPPROTO_UDP, src, dst);
- b->uh.source = b->s_in.sin_port;
- b->uh.dest = htons(dstport);
- b->uh.len = htons(datalen + sizeof(b->uh));
+ bp->uh.source = bm->s_in.sa4.sin_port;
+ bp->uh.dest = htons(dstport);
+ bp->uh.len = htons(l4len);
+ csum_udp4(&bp->uh, src, dst, bp->data, dlen);
- return tap_frame_len(c, &b->taph, ip_len);
+ tap_hdr_update(&bm->taph, l3len + sizeof(udp4_eth_hdr));
+ return l4len;
}
/**
* udp_update_hdr6() - Update headers for one IPv6 datagram
* @c: Execution context
- * @b: Pointer to udp6_l2_buf to update
+ * @bm: Pointer to udp_meta_t to update
+ * @bp: Pointer to udp_payload_t to update
* @dstport: Destination port number
- * @datalen: Length of UDP payload
+ * @dlen: Length of UDP payload
* @now: Current timestamp
*
- * Return: size of tap frame with headers
+ * Return: size of IPv6 payload (UDP header + data)
*/
-static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *b,
- in_port_t dstport, size_t datalen,
+static size_t udp_update_hdr6(const struct ctx *c,
+ struct udp_meta_t *bm, struct udp_payload_t *bp,
+ in_port_t dstport, size_t dlen,
const struct timespec *now)
{
- const struct in6_addr *src = &b->s_in6.sin6_addr;
+ const struct in6_addr *src = &bm->s_in.sa6.sin6_addr;
const struct in6_addr *dst = &c->ip6.addr_seen;
- uint16_t payload_len = datalen + sizeof(b->uh);
- in_port_t srcport = ntohs(b->s_in6.sin6_port);
+ in_port_t srcport = ntohs(bm->s_in.sa6.sin6_port);
+ uint16_t l4len = dlen + sizeof(bp->uh);
if (IN6_IS_ADDR_LINKLOCAL(src)) {
dst = &c->ip6.addr_ll_seen;
@@ -664,19 +663,20 @@ static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *b,
}
- b->ip6h.payload_len = htons(payload_len);
- b->ip6h.daddr = *dst;
- b->ip6h.saddr = *src;
- b->ip6h.version = 6;
- b->ip6h.nexthdr = IPPROTO_UDP;
- b->ip6h.hop_limit = 255;
+ bm->ip6h.payload_len = htons(l4len);
+ bm->ip6h.daddr = *dst;
+ bm->ip6h.saddr = *src;
+ bm->ip6h.version = 6;
+ bm->ip6h.nexthdr = IPPROTO_UDP;
+ bm->ip6h.hop_limit = 255;
- b->uh.source = b->s_in6.sin6_port;
- b->uh.dest = htons(dstport);
- b->uh.len = b->ip6h.payload_len;
- csum_udp6(&b->uh, src, dst, b->data, datalen);
+ bp->uh.source = bm->s_in.sa6.sin6_port;
+ bp->uh.dest = htons(dstport);
+ bp->uh.len = bm->ip6h.payload_len;
+ csum_udp6(&bp->uh, src, dst, bp->data, dlen);
- return tap_frame_len(c, &b->taph, payload_len + sizeof(b->ip6h));
+ tap_hdr_update(&bm->taph, l4len + sizeof(bm->ip6h) + sizeof(udp6_eth_hdr));
+ return l4len;
}
/**
@@ -694,8 +694,8 @@ static void udp_tap_send(const struct ctx *c,
unsigned int start, unsigned int n,
in_port_t dstport, bool v6, const struct timespec *now)
{
- struct iovec *tap_iov;
- unsigned int i;
+ struct iovec (*tap_iov)[UDP_NUM_IOVS];
+ size_t i;
if (v6)
tap_iov = udp6_l2_iov_tap;
@@ -703,19 +703,21 @@ static void udp_tap_send(const struct ctx *c,
tap_iov = udp4_l2_iov_tap;
for (i = start; i < start + n; i++) {
- size_t buf_len;
-
- if (v6)
- buf_len = udp_update_hdr6(c, &udp6_l2_buf[i], dstport,
- udp6_l2_mh_sock[i].msg_len, now);
- else
- buf_len = udp_update_hdr4(c, &udp4_l2_buf[i], dstport,
- udp4_l2_mh_sock[i].msg_len, now);
+ struct udp_payload_t *bp = &udp_payload[i];
+ struct udp_meta_t *bm = &udp_meta[i];
+ size_t l4len;
- tap_iov[i].iov_len = buf_len;
+ if (v6) {
+ l4len = udp_update_hdr6(c, bm, bp, dstport,
+ udp6_l2_mh_sock[i].msg_len, now);
+ } else {
+ l4len = udp_update_hdr4(c, bm, bp, dstport,
+ udp4_l2_mh_sock[i].msg_len, now);
+ }
+ tap_iov[i][UDP_IOV_PAYLOAD].iov_len = l4len;
}
- tap_send_frames(c, tap_iov + start, 1, n);
+ tap_send_frames(c, &tap_iov[start][0], UDP_NUM_IOVS, n);
}
/**
@@ -829,6 +831,7 @@ int udp_tap_handler(struct ctx *c, uint8_t pif,
* and destination, so we can just take those from the first message.
*/
src = ntohs(uh->source);
+ src += c->udp.fwd_in.rdelta[src];
dst = ntohs(uh->dest);
if (af == AF_INET) {
@@ -1005,7 +1008,7 @@ int udp_sock_init(const struct ctx *c, int ns, sa_family_t af,
r4 = s = sock_l4(c, AF_INET, IPPROTO_UDP, addr,
ifname, port, uref.u32);
- udp_tap_map[V4][uref.port].sock = s < 0 ? -1 : s;
+ udp_tap_map[V4][port].sock = s < 0 ? -1 : s;
udp_splice_init[V4][port].sock = s < 0 ? -1 : s;
} else {
r4 = s = sock_l4(c, AF_INET, IPPROTO_UDP,
@@ -1022,7 +1025,7 @@ int udp_sock_init(const struct ctx *c, int ns, sa_family_t af,
r6 = s = sock_l4(c, AF_INET6, IPPROTO_UDP, addr,
ifname, port, uref.u32);
- udp_tap_map[V6][uref.port].sock = s < 0 ? -1 : s;
+ udp_tap_map[V6][port].sock = s < 0 ? -1 : s;
udp_splice_init[V6][port].sock = s < 0 ? -1 : s;
} else {
r6 = s = sock_l4(c, AF_INET6, IPPROTO_UDP,
@@ -1055,11 +1058,10 @@ static void udp_splice_iov_init(void)
mh6->msg_name = &udp6_localname;
mh6->msg_namelen = sizeof(udp6_localname);
- udp4_iov_splice[i].iov_base = udp4_l2_buf[i].data;
- udp6_iov_splice[i].iov_base = udp6_l2_buf[i].data;
+ udp_iov_splice[i].iov_base = udp_payload[i].data;
- mh4->msg_iov = &udp4_iov_splice[i];
- mh6->msg_iov = &udp6_iov_splice[i];
+ mh4->msg_iov = &udp_iov_splice[i];
+ mh6->msg_iov = &udp_iov_splice[i];
mh4->msg_iovlen = mh6->msg_iovlen = 1;
}
}
@@ -1230,7 +1232,7 @@ v6:
*/
int udp_init(struct ctx *c)
{
- udp_sock_iov_init(c);
+ udp_iov_init(c);
udp_invert_portmap(&c->udp.fwd_in);
udp_invert_portmap(&c->udp.fwd_out);
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-09 08:57:47 +0000