1 files changed, 47 insertions, 40 deletions

diff --git a/checksum.c b/checksum.c
index 006614f..0894eca 100644
--- a/checksum.c
+++ b/checksum.c
@@ -59,6 +59,7 @@
 #include "util.h"
 #include "ip.h"
 #include "checksum.h"
+#include "iov.h"
 
 /* Checksums are optional for UDP over IPv4, so we usually just set
  * them to 0.  Change this to 1 to calculate real UDP over IPv4
@@ -84,7 +85,7 @@
  */
 /* NOLINTNEXTLINE(clang-diagnostic-unknown-attributes) */
 __attribute__((optimize("-fno-strict-aliasing")))
-uint32_t sum_16b(const void *buf, size_t len)
+static uint32_t sum_16b(const void *buf, size_t len)
 {
 	const uint16_t *p = buf;
 	uint32_t sum = 0;
@@ -106,7 +107,7 @@ uint32_t sum_16b(const void *buf, size_t len)
  *
  * Return: 16-bit folded sum
  */
-uint16_t csum_fold(uint32_t sum)
+static uint16_t csum_fold(uint32_t sum)
 {
 	while (sum >> 16)
 		sum = (sum & 0xffff) + (sum >> 16);
@@ -161,26 +162,41 @@ uint32_t proto_ipv4_header_psum(uint16_t l4len, uint8_t protocol,
 }
 
 /**
+ * csum() - Compute TCP/IP-style checksum
+ * @buf:	Input buffer
+ * @len:	Input length
+ * @init:	Initial 32-bit checksum, 0 for no pre-computed checksum
+ *
+ * Return: 16-bit folded, complemented checksum
+ */
+/* NOLINTNEXTLINE(clang-diagnostic-unknown-attributes) */
+__attribute__((optimize("-fno-strict-aliasing")))	/* See csum_16b() */
+static uint16_t csum(const void *buf, size_t len, uint32_t init)
+{
+	return (uint16_t)~csum_fold(csum_unfolded(buf, len, init));
+}
+
+/**
  * csum_udp4() - Calculate and set checksum for a UDP over IPv4 packet
  * @udp4hr:	UDP header, initialised apart from checksum
  * @saddr:	IPv4 source address
  * @daddr:	IPv4 destination address
- * @payload:	UDP packet payload
- * @dlen:	Length of @payload (not including UDP header)
+ * @data:	UDP payload (as IO vector tail)
  */
 void csum_udp4(struct udphdr *udp4hr,
 	       struct in_addr saddr, struct in_addr daddr,
-	       const void *payload, size_t dlen)
+	       struct iov_tail *data)
 {
 	/* UDP checksums are optional, so don't bother */
 	udp4hr->check = 0;
 
 	if (UDP4_REAL_CHECKSUMS) {
-		uint16_t l4len = dlen + sizeof(struct udphdr);
+		uint16_t l4len = iov_tail_size(data) + sizeof(struct udphdr);
 		uint32_t psum = proto_ipv4_header_psum(l4len, IPPROTO_UDP,
 						       saddr, daddr);
+
 		psum = csum_unfolded(udp4hr, sizeof(struct udphdr), psum);
-		udp4hr->check = csum(payload, dlen, psum);
+		udp4hr->check = csum_iov_tail(data, psum);
 	}
 }
 
@@ -226,19 +242,22 @@ uint32_t proto_ipv6_header_psum(uint16_t payload_len, uint8_t protocol,
 /**
  * csum_udp6() - Calculate and set checksum for a UDP over IPv6 packet
  * @udp6hr:	UDP header, initialised apart from checksum
- * @payload:	UDP packet payload
- * @dlen:	Length of @payload (not including UDP header)
+ * @saddr:	Source address
+ * @daddr:	Destination address
+ * @data:	UDP payload (as IO vector tail)
  */
 void csum_udp6(struct udphdr *udp6hr,
 	       const struct in6_addr *saddr, const struct in6_addr *daddr,
-	       const void *payload, size_t dlen)
+	       struct iov_tail *data)
 {
-	uint32_t psum = proto_ipv6_header_psum(dlen + sizeof(struct udphdr),
-					       IPPROTO_UDP, saddr, daddr);
+	uint16_t l4len = iov_tail_size(data) + sizeof(struct udphdr);
+	uint32_t psum = proto_ipv6_header_psum(l4len, IPPROTO_UDP,
+					       saddr, daddr);
+
 	udp6hr->check = 0;
 
 	psum = csum_unfolded(udp6hr, sizeof(struct udphdr), psum);
-	udp6hr->check = csum(payload, dlen, psum);
+	udp6hr->check = csum_iov_tail(data, psum);
 }
 
 /**
@@ -448,7 +467,8 @@ uint32_t csum_unfolded(const void *buf, size_t len, uint32_t init)
 	intptr_t align = ROUND_UP((intptr_t)buf, sizeof(__m256i));
 	unsigned int pad = align - (intptr_t)buf;
 
-	if (len < pad)
+	/* Don't mix sum_16b() and csum_avx2() with odd padding lengths */
+	if (pad & 1 || len < pad)
 		pad = len;
 
 	if (pad)
@@ -478,36 +498,23 @@ uint32_t csum_unfolded(const void *buf, size_t len, uint32_t init)
 #endif /* !__AVX2__ */
 
 /**
- * csum() - Compute TCP/IP-style checksum
- * @buf:	Input buffer
- * @len:	Input length
- * @init:	Initial 32-bit checksum, 0 for no pre-computed checksum
- *
- * Return: 16-bit folded, complemented checksum
- */
-/* NOLINTNEXTLINE(clang-diagnostic-unknown-attributes) */
-__attribute__((optimize("-fno-strict-aliasing")))	/* See csum_16b() */
-uint16_t csum(const void *buf, size_t len, uint32_t init)
-{
-	return (uint16_t)~csum_fold(csum_unfolded(buf, len, init));
-}
-
-/**
- * csum_iov() - Calculates the unfolded checksum over an array of IO vectors
- *
- * @iov		Pointer to the array of IO vectors
- * @n		Length of the array
+ * csum_iov_tail() - Calculate unfolded checksum for the tail of an IO vector
+ * @tail:	IO vector tail to checksum
  * @init	Initial 32-bit checksum, 0 for no pre-computed checksum
  *
  * Return: 16-bit folded, complemented checksum
  */
-/* cppcheck-suppress unusedFunction */
-uint16_t csum_iov(const struct iovec *iov, size_t n, uint32_t init)
+uint16_t csum_iov_tail(struct iov_tail *tail, uint32_t init)
 {
-	unsigned int i;
-
-	for (i = 0; i < n; i++)
-		init = csum_unfolded(iov[i].iov_base, iov[i].iov_len, init);
-
+	if (iov_tail_prune(tail)) {
+		size_t i;
+
+		init = csum_unfolded((char *)tail->iov[0].iov_base + tail->off,
+				     tail->iov[0].iov_len - tail->off, init);
+		for (i = 1; i < tail->cnt; i++) {
+			const struct iovec *iov = &tail->iov[i];
+			init = csum_unfolded(iov->iov_base, iov->iov_len, init);
+		}
+	}
 	return (uint16_t)~csum_fold(init);
 }