aboutgitcodebugslistschat
path: root/tcp.c
diff options
context:
space:
mode:
Diffstat (limited to 'tcp.c')
-rw-r--r--tcp.c607
1 files changed, 400 insertions, 207 deletions
diff --git a/tcp.c b/tcp.c
index 8aa9740..5b4e408 100644
--- a/tcp.c
+++ b/tcp.c
@@ -148,57 +148,49 @@
* - TAP_SYN_SENT connect() in progress, triggered from tap
* - connect() completes SYN,ACK to tap > TAP_SYN_RCVD
* - connect() aborts RST to tap, close socket > CLOSED
- * - RST from tap close socket > CLOSED
*
* - SOCK_SYN_SENT new connected socket, SYN sent to tap
* - SYN,ACK from tap ACK to tap > ESTABLISHED
- * - socket error RST to tap, close socket > CLOSED
* - SYN,ACK timeout RST to tap, close socket > CLOSED
- * - RST from tap close socket > CLOSED
*
* - TAP_SYN_RCVD connect() completed, SYN,ACK sent to tap
* - FIN from tap write shutdown > FIN_WAIT_1
* - ACK from tap > ESTABLISHED
- * - socket error RST to tap, close socket > CLOSED
* - ACK timeout RST to tap, close socket > CLOSED
- * - RST from tap close socket > CLOSED
*
* - ESTABLISHED connection established, ready for data
+ * - EPOLLRDHUP read shutdown > ESTABLISHED_SOCK_FIN
* - FIN from tap write shutdown > FIN_WAIT_1
- * - EPOLLRDHUP/EPOLLHUP read shutdown > ESTABLISHED_SOCK_FIN
- * - socket error RST to tap, close socket > CLOSED
- * - data timeout FIN to tap > ESTABLISHED_SOCK_FIN
- * - RST from tap close socket > CLOSED
+ * - EPOLLHUP RST to tap, close socket > CLOSED
+ * - data timeout read shutdown, FIN to tap >
+ * ESTABLISHED_SOCK_FIN_SENT
*
* - ESTABLISHED_SOCK_FIN socket closing connection, reading half closed
- * - zero-sized socket read FIN, ACK to tap > CLOSE_WAIT
- * - ACK from tap > CLOSE_WAIT
- * - ACK timeout RST to tap, close socket > CLOSED
- * - RST from tap close socket > CLOSED
+ * - zero-sized socket read FIN,ACK to tap > ESTABLISHED_SOCK_FIN_SENT
+ *
+ * - ESTABLISHED_SOCK_FIN_SENT socket closing connection, FIN sent to tap
+ * - ACK (for FIN) from tap > CLOSE_WAIT
+ * - tap ACK timeout RST to tap, close socket > CLOSED
*
* - CLOSE_WAIT socket closing connection, ACK from tap
* - FIN from tap write shutdown > LAST_ACK
- * - socket error RST to tap, close socket > CLOSED
- * - FIN timeout RST to tap, close socket > CLOSED
- * - RST from tap close socket > CLOSED
+ * - data timeout RST to tap, close socket > CLOSED
*
* - LAST_ACK socket started close, tap completed it
- * - anything from socket close socket > CLOSED
- * - socket error RST to tap, close socket > CLOSED
+ * - any event from socket ACK to tap, close socket > CLOSED
* - ACK timeout RST to tap, close socket > CLOSED
- * - RST from tap close socket > CLOSED
*
* - FIN_WAIT_1 tap closing connection, FIN sent to socket
- * - zero-sized socket read FIN,ACK to tap, shutdown > FIN_WAIT_1_SOCK_FIN
- * - socket error RST to tap, close socket > CLOSED
- * - ACK timeout RST to tap, close socket > CLOSED
- * - RST from tap close socket > CLOSED
+ * - EPOLLRDHUP FIN,ACK to tap, shutdown > FIN_WAIT_1_SOCK_FIN
+ * - socket timeout RST to tap, close socket > CLOSED
*
* - FIN_WAIT_1_SOCK_FIN tap closing connection, FIN received from socket
* - ACK from tap close socket > CLOSED
- * - socket error RST to tap, close socket > CLOSED
- * - ACK timeout RST to tap, close socket > CLOSED
+ * - tap ACK timeout RST to tap, close socket > CLOSED
+ *
+ * - from any state
* - RST from tap close socket > CLOSED
+ * - socket error RST to tap, close socket > CLOSED
*
* Connection setup
* ----------------
@@ -387,6 +379,7 @@ enum tcp_state {
TAP_SYN_RCVD,
ESTABLISHED,
ESTABLISHED_SOCK_FIN,
+ ESTABLISHED_SOCK_FIN_SENT,
CLOSE_WAIT,
LAST_ACK,
FIN_WAIT_1,
@@ -394,14 +387,18 @@ enum tcp_state {
SPLICE_ACCEPTED,
SPLICE_CONNECT,
SPLICE_ESTABLISHED,
+ SPLICE_FIN_FROM,
+ SPLICE_FIN_TO,
+ SPLICE_FIN_BOTH,
};
-#define TCP_STATE_STR_SIZE (SPLICE_ESTABLISHED + 1)
+#define TCP_STATE_STR_SIZE (SPLICE_FIN_BOTH + 1)
static char *tcp_state_str[TCP_STATE_STR_SIZE] __attribute((__unused__)) = {
"CLOSED", "TAP_SYN_SENT", "SOCK_SYN_SENT", "TAP_SYN_RCVD",
- "ESTABLISHED", "ESTABLISHED_SOCK_FIN", "CLOSE_WAIT", "LAST_ACK",
- "FIN_WAIT_1", "FIN_WAIT_1_SOCK_FIN",
+ "ESTABLISHED", "ESTABLISHED_SOCK_FIN", "ESTABLISHED_SOCK_FIN_SENT",
+ "CLOSE_WAIT", "LAST_ACK", "FIN_WAIT_1", "FIN_WAIT_1_SOCK_FIN",
"SPLICE_ACCEPTED", "SPLICE_CONNECT", "SPLICE_ESTABLISHED",
+ "SPLICE_FIN_FROM", "SPLICE_FIN_TO", "SPLICE_FIN_BOTH",
};
#define FIN (1 << 0)
@@ -489,6 +486,9 @@ struct tcp_tap_conn {
struct timespec ts_ack_tap;
int mss_guest;
+ uint32_t sndbuf;
+
+ uint32_t events;
};
/**
@@ -505,7 +505,13 @@ struct tcp_splice_conn {
int to;
int pipe_to_from[2];
enum tcp_state state;
+ int from_fin_sent;
+ int to_fin_sent;
int v6;
+ uint64_t from_read;
+ uint64_t from_written;
+ uint64_t to_read;
+ uint64_t to_written;
};
/* Port re-mappings as delta, indexed by original destination port */
@@ -1037,15 +1043,35 @@ static struct tcp_tap_conn *tcp_hash_lookup(struct ctx *c, int af, void *addr,
}
/**
- * tcp_table_tap_compact - Perform compaction on tap connection table
+ * tcp_tap_epoll_mask() - Set new epoll event mask given a connection
+ * @c: Execution context
+ * @conn: Connection pointer
+ * @events: New epoll event bitmap
+ */
+static void tcp_tap_epoll_mask(struct ctx *c, struct tcp_tap_conn *conn,
+ uint32_t events)
+{
+ union epoll_ref ref = { .proto = IPPROTO_TCP, .s = conn->sock,
+ .tcp.index = conn - tt,
+ .tcp.v6 = !IN6_IS_ADDR_V4MAPPED(&conn->a.a6) };
+ struct epoll_event ev = { .data.u64 = ref.u64, .events = events };
+
+ if (conn->events == events)
+ return;
+
+ conn->events = events;
+ epoll_ctl(c->epollfd, EPOLL_CTL_MOD, conn->sock, &ev);
+}
+
+/**
+ * tcp_table_tap_compact() - Perform compaction on tap connection table
* @c: Execution context
* @hole: Pointer to recently closed connection
*/
static void tcp_table_tap_compact(struct ctx *c, struct tcp_tap_conn *hole)
{
- union epoll_ref ref = { .proto = IPPROTO_TCP, .tcp.index = hole - tt };
struct tcp_tap_conn *from, *to;
- struct epoll_event ev;
+ uint32_t events;
if ((hole - tt) == --c->tcp.tap_conn_count) {
bitmap_clear(tcp_act, hole - tt);
@@ -1061,17 +1087,9 @@ static void tcp_table_tap_compact(struct ctx *c, struct tcp_tap_conn *hole)
to = hole;
tcp_hash_update(from, to);
- if (to->state == SOCK_SYN_SENT)
- ev.events = EPOLLRDHUP;
- else if (to->state == TAP_SYN_SENT)
- ev.events = EPOLLOUT | EPOLLRDHUP;
- else
- ev.events = EPOLLIN | EPOLLRDHUP;
-
- ref.tcp.v6 = !IN6_IS_ADDR_V4MAPPED(&to->a.a6);
- ref.s = from->sock;
- ev.data.u64 = ref.u64;
- epoll_ctl(c->epollfd, EPOLL_CTL_MOD, from->sock, &ev);
+ events = hole->events;
+ hole->events = UINT_MAX;
+ tcp_tap_epoll_mask(c, hole, events);
debug("TCP: hash table compaction: old index %i, new index %i, "
"sock %i, from: %p, to: %p",
@@ -1117,7 +1135,8 @@ static int tcp_send_to_tap(struct ctx *c, struct tcp_tap_conn *conn,
socklen_t sl = sizeof(info);
struct tcphdr *th;
- if (!ack_offset && !flags) {
+ if (conn->state == ESTABLISHED && !ack_offset && !flags &&
+ conn->tcpi_snd_wnd) {
err = 0;
info.tcpi_bytes_acked = conn->tcpi_acked_last;
info.tcpi_snd_wnd = conn->tcpi_snd_wnd;
@@ -1131,6 +1150,16 @@ static int tcp_send_to_tap(struct ctx *c, struct tcp_tap_conn *conn,
return err;
}
+ if (info.tcpi_snd_wnd > conn->sndbuf) {
+ sl = sizeof(conn->sndbuf);
+ if (getsockopt(conn->sock, SOL_SOCKET, SO_SNDBUF,
+ &conn->sndbuf, &sl))
+ conn->sndbuf = USHRT_MAX;
+
+ info.tcpi_snd_wnd = MIN(info.tcpi_snd_wnd,
+ conn->sndbuf / 100 * 90);
+ }
+
conn->tcpi_snd_wnd = info.tcpi_snd_wnd;
}
@@ -1189,14 +1218,18 @@ static int tcp_send_to_tap(struct ctx *c, struct tcp_tap_conn *conn,
conn->tcpi_acked_last = info.tcpi_bytes_acked;
}
- if (conn->state == LAST_ACK ||
- conn->state == FIN_WAIT_1_SOCK_FIN)
- conn->seq_ack_to_tap = conn->seq_from_tap + 1;
-
- if (conn->state == LAST_ACK)
- th->seq = htonl(ntohl(th->seq) + 1);
-
- th->ack_seq = htonl(conn->seq_ack_to_tap);
+ /* seq_ack_to_tap matching seq_from_tap means, in these states,
+ * that we shut the writing half down, but the FIN segment
+ * wasn't acknowledged yet. We sent the FIN for sure, so adjust
+ * the sequence number in that case.
+ */
+ if ((conn->state == LAST_ACK ||
+ conn->state == FIN_WAIT_1_SOCK_FIN ||
+ conn->state == FIN_WAIT_1) &&
+ conn->seq_ack_to_tap == conn->seq_from_tap)
+ th->ack_seq = htonl(conn->seq_ack_to_tap + 1);
+ else
+ th->ack_seq = htonl(conn->seq_ack_to_tap);
} else {
if (!len && !flags)
return 0;
@@ -1221,6 +1254,9 @@ static int tcp_send_to_tap(struct ctx *c, struct tcp_tap_conn *conn,
th->window = htons(WINDOW_DEFAULT);
}
+ if (!th->window)
+ conn->tcpi_snd_wnd = 0;
+
th->urg_ptr = 0;
th->check = 0;
@@ -1229,6 +1265,9 @@ static int tcp_send_to_tap(struct ctx *c, struct tcp_tap_conn *conn,
tap_ip_send(c, &conn->a.a6, IPPROTO_TCP, buf, th->doff * 4 + len,
conn->seq_init_to_tap);
+ if (th->fin)
+ conn->seq_to_tap++;
+
return 0;
}
@@ -1373,10 +1412,10 @@ static void tcp_conn_from_tap(struct ctx *c, int af, void *addr,
.sin6_port = th->dest,
.sin6_addr = *(struct in6_addr *)addr,
};
- struct epoll_event ev = { .events = EPOLLIN | EPOLLRDHUP };
union epoll_ref ref = { .proto = IPPROTO_TCP };
const struct sockaddr *sa;
struct tcp_tap_conn *conn;
+ struct epoll_event ev;
socklen_t sl;
int s;
@@ -1404,6 +1443,10 @@ static void tcp_conn_from_tap(struct ctx *c, int af, void *addr,
conn = &tt[c->tcp.tap_conn_count++];
conn->sock = s;
+ sl = sizeof(conn->sndbuf);
+ if (getsockopt(s, SOL_SOCKET, SO_SNDBUF, &conn->sndbuf, &sl))
+ conn->sndbuf = USHRT_MAX;
+
conn->mss_guest = tcp_opt_get(th, len, OPT_MSS, NULL, NULL);
if (conn->mss_guest < 0)
conn->mss_guest = MSS_DEFAULT;
@@ -1468,8 +1511,11 @@ static void tcp_conn_from_tap(struct ctx *c, int af, void *addr,
if (tcp_send_to_tap(c, conn, SYN | ACK, NULL, 0))
return;
+
+ ev.events = EPOLLIN | EPOLLRDHUP;
}
+ conn->events = ev.events;
ref.tcp.index = conn - tt;
ev.data.u64 = ref.u64;
epoll_ctl(c->epollfd, EPOLL_CTL_ADD, s, &ev);
@@ -1522,13 +1568,16 @@ static void tcp_table_splice_compact(struct ctx *c,
}
/**
- * tcp_tap_destroy() - Close spliced connection and pipes, drop from epoll
+ * tcp_splice_destroy() - Close spliced connection and pipes, drop from epoll
* @c: Execution context
* @conn: Connection pointer
*/
static void tcp_splice_destroy(struct ctx *c, struct tcp_splice_conn *conn)
{
switch (conn->state) {
+ case SPLICE_FIN_BOTH:
+ case SPLICE_FIN_FROM:
+ case SPLICE_FIN_TO:
case SPLICE_ESTABLISHED:
if (conn->pipe_from_to[0] != -1) {
close(conn->pipe_from_to[0]);
@@ -1548,6 +1597,8 @@ static void tcp_splice_destroy(struct ctx *c, struct tcp_splice_conn *conn)
close(conn->from);
tcp_splice_state(conn, CLOSED);
tcp_table_splice_compact(c, conn);
+ conn->from_read = conn->from_written = 0;
+ conn->to_read = conn->to_written = 0;
return;
default:
return;
@@ -1604,8 +1655,16 @@ static int tcp_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn,
already_sent = conn->seq_to_tap - conn->seq_ack_from_tap;
- if (!conn->tap_window || already_sent >= conn->tap_window)
+ if (already_sent > MAX_WINDOW) {
+ /* RFC 761, section 2.1. */
+ seq_to_tap = conn->seq_to_tap = conn->seq_ack_from_tap;
+ already_sent = 0;
+ }
+
+ if (!conn->tap_window || already_sent >= conn->tap_window) {
+ tcp_tap_epoll_mask(c, conn, conn->events | EPOLLET);
return 0;
+ }
fill_bufs = DIV_ROUND_UP(conn->tap_window - already_sent,
conn->mss_guest);
@@ -1643,16 +1702,24 @@ static int tcp_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn,
tcp6_l2_mh_sock.msg_iovlen = fill_bufs + 1;
/* Don't dequeue until acknowledged by guest. */
+recvmmsg:
len = recvmsg(s, v4 ? &tcp4_l2_mh_sock : &tcp6_l2_mh_sock, MSG_PEEK);
- if (len < 0)
+ if (len < 0) {
+ if (errno == EINTR)
+ goto recvmmsg;
goto err;
+ }
if (!len)
goto zero_len;
send = len - already_sent;
- if (send <= 0)
+ if (send <= 0) {
+ tcp_tap_epoll_mask(c, conn, conn->events | EPOLLET);
goto out_restore_iov;
+ }
+
+ tcp_tap_epoll_mask(c, conn, conn->events & ~EPOLLET);
send_bufs = DIV_ROUND_UP(send, conn->mss_guest);
last_len = send - (send_bufs - 1) * conn->mss_guest;
@@ -1693,6 +1760,16 @@ static int tcp_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn,
&sl))
goto err;
+ if (info.tcpi_snd_wnd > conn->sndbuf) {
+ if (getsockopt(conn->sock, SOL_SOCKET,
+ SO_SNDBUF, &conn->sndbuf, &sl))
+ conn->sndbuf = USHRT_MAX;
+
+ info.tcpi_snd_wnd = MIN(info.tcpi_snd_wnd,
+ conn->sndbuf / 100
+ * 90);
+ }
+
conn->tcpi_acked_last = info.tcpi_bytes_acked;
conn->seq_ack_to_tap = info.tcpi_bytes_acked +
conn->seq_init_from_tap;
@@ -1805,8 +1882,12 @@ static int tcp_data_from_sock(struct ctx *c, struct tcp_tap_conn *conn,
if (c->mode == MODE_PASTA)
goto out;
+sendmmsg:
ret = sendmmsg(c->fd_tap, tcp_l2_mh_tap, mh - tcp_l2_mh_tap,
MSG_NOSIGNAL | MSG_DONTWAIT);
+ if (ret < 0 && ret == EINTR)
+ goto sendmmsg;
+
if (ret <= 0)
goto out;
@@ -1845,14 +1926,15 @@ err:
goto out_restore_iov;
zero_len:
- if (conn->state == FIN_WAIT_1) {
- tcp_tap_state(conn, FIN_WAIT_1_SOCK_FIN);
- } else if (conn->state == ESTABLISHED_SOCK_FIN) {
- tcp_send_to_tap(c, conn, FIN | ACK, NULL, 0);
- tcp_tap_state(conn, CLOSE_WAIT);
- }
+ if (conn->state == ESTABLISHED_SOCK_FIN) {
+ uint8_t probe;
- goto out_restore_iov;
+ if (!recv(conn->sock, &probe, 1, MSG_PEEK)) {
+ tcp_tap_epoll_mask(c, conn, EPOLLET);
+ tcp_send_to_tap(c, conn, FIN | ACK, NULL, 0);
+ tcp_tap_state(conn, ESTABLISHED_SOCK_FIN_SENT);
+ }
+ }
out:
conn->ts_sock = *now;
@@ -1878,7 +1960,7 @@ static void tcp_data_from_tap(struct ctx *c, struct tcp_tap_conn *conn,
struct tap_msg *msg, int count,
struct timespec *now)
{
- int i, iov_i, keep = -1, ack = 0, fin = 0, retr = 0;
+ int i, iov_i, ack = 0, fin = 0, retr = 0, keep = -1;
struct msghdr mh = { .msg_iov = tcp_tap_iov };
uint32_t max_ack_seq = conn->seq_ack_from_tap;
uint32_t seq_from_tap = conn->seq_from_tap;
@@ -1946,27 +2028,27 @@ static void tcp_data_from_tap(struct ctx *c, struct tcp_tap_conn *conn,
* |--------| <-- len |--------| <-- len
* '----' <-- offset ' <-- offset
* ^ seq ^ seq
- * (offset >= 0, seq + len > seq_from_tap)
+ * (offset >= 0, seq + len > seq_from_tap)
*
* discard in these two cases:
* , seq_from_tap , seq_from_tap
* |--------| <-- len |--------| <-- len
* '--------' <-- offset '-----| <- offset
* ^ seq ^ seq
- * (offset >= 0, seq + len <= seq_from_tap)
+ * (offset >= 0 i.e. < MAX_WINDOW, seq + len <= seq_from_tap)
*
* keep, look for another buffer, then go back, in this case:
- * , seq_from_tap
- * |--------| <-- len
- * '===' <-- offset
- * ^ seq
- * (offset < 0 i.e. > MAX_WINDOW)
+ * , seq_from_tap
+ * |--------| <-- len
+ * '===' <-- offset
+ * ^ seq
+ * (offset < 0 i.e. > MAX_WINDOW)
*/
if (seq_offset < MAX_WINDOW && seq + len <= seq_from_tap)
continue;
if (seq_offset > MAX_WINDOW) {
- if (keep != -1)
+ if (keep == -1)
keep = i;
continue;
}
@@ -1976,10 +2058,11 @@ static void tcp_data_from_tap(struct ctx *c, struct tcp_tap_conn *conn,
seq_from_tap += tcp_tap_iov[iov_i].iov_len;
iov_i++;
- if (keep == i) {
- i = keep + 1;
+ if (keep == i)
keep = -1;
- }
+
+ if (keep != -1)
+ i = keep - 1;
}
tcp_clamp_window(conn, NULL, 0, max_ack_seq_wnd, 0);
@@ -1992,19 +2075,24 @@ static void tcp_data_from_tap(struct ctx *c, struct tcp_tap_conn *conn,
if (retr) {
conn->seq_to_tap = max_ack_seq;
tcp_data_from_sock(c, conn, now);
+ return;
}
- if (!iov_i) {
- if (keep != -1) {
- tcp_send_to_tap(c, conn, ACK, NULL, 0);
- tcp_send_to_tap(c, conn, ACK, NULL, 0);
- }
- goto fin;
+ if (keep != -1) {
+ tcp_send_to_tap(c, conn, ACK, NULL, 0);
+ tcp_send_to_tap(c, conn, ACK, NULL, 0);
}
+ if (!iov_i)
+ goto fin_check;
+
mh.msg_iovlen = iov_i;
+eintr:
len = sendmsg(conn->sock, &mh, MSG_DONTWAIT | MSG_NOSIGNAL);
if (len < 0) {
+ if (errno == EINTR)
+ goto eintr;
+
if (errno == EAGAIN || errno == EWOULDBLOCK) {
tcp_send_to_tap(c, conn, ZERO_WINDOW, NULL, 0);
return;
@@ -2020,24 +2108,30 @@ static void tcp_data_from_tap(struct ctx *c, struct tcp_tap_conn *conn,
}
conn->seq_from_tap += len;
- if (!fin)
+
+fin_check:
+ if (keep != -1)
+ return;
+
+ if (ack) {
+ if (conn->state == ESTABLISHED_SOCK_FIN_SENT &&
+ conn->seq_ack_from_tap == conn->seq_to_tap)
+ tcp_tap_state(conn, CLOSE_WAIT);
+ }
+
+ if (!fin) {
tcp_send_to_tap(c, conn, 0, NULL, 0);
+ return;
+ }
-fin:
- if (conn->state == CLOSE_WAIT && fin && ack) {
- tcp_data_from_sock(c, conn, now);
- tcp_sock_consume(conn, conn->seq_ack_from_tap);
+ if (conn->state == ESTABLISHED) {
shutdown(conn->sock, SHUT_WR);
- tcp_tap_state(conn, LAST_ACK);
+ tcp_tap_state(conn, FIN_WAIT_1);
tcp_send_to_tap(c, conn, ACK, NULL, 0);
- } else if (fin) {
+ } else if (conn->state == CLOSE_WAIT) {
shutdown(conn->sock, SHUT_WR);
- if (conn->state == ESTABLISHED) {
- tcp_tap_state(conn, FIN_WAIT_1);
- tcp_data_from_sock(c, conn, now);
- } else {
- tcp_tap_state(conn, LAST_ACK);
- }
+ tcp_tap_state(conn, LAST_ACK);
+ tcp_send_to_tap(c, conn, ACK, NULL, 0);
}
}
@@ -2055,12 +2149,9 @@ fin:
int tcp_tap_handler(struct ctx *c, int af, void *addr,
struct tap_msg *msg, int count, struct timespec *now)
{
- union epoll_ref ref = { .proto = IPPROTO_TCP,
- .tcp.v6 = ( af == AF_INET6 ) };
struct tcphdr *th = (struct tcphdr *)msg[0].l4h;
size_t len = msg[0].l4_len, off;
struct tcp_tap_conn *conn;
- struct epoll_event ev;
int ws;
if (len < sizeof(*th))
@@ -2133,15 +2224,12 @@ int tcp_tap_handler(struct ctx *c, int af, void *addr,
tcp_data_from_sock(c, conn, now);
tcp_send_to_tap(c, conn, 0, NULL, 0);
- ev.events = EPOLLIN | EPOLLRDHUP;
- ref.s = conn->sock;
- ref.tcp.index = conn - tt;
- ev.data.u64 = ref.u64;
- epoll_ctl(c->epollfd, EPOLL_CTL_MOD, conn->sock, &ev);
+ tcp_tap_epoll_mask(c, conn, EPOLLIN | EPOLLRDHUP);
break;
case TAP_SYN_RCVD:
if (th->fin) {
shutdown(conn->sock, SHUT_WR);
+ tcp_send_to_tap(c, conn, ACK, NULL, 0);
tcp_tap_state(conn, FIN_WAIT_1);
break;
}
@@ -2157,21 +2245,21 @@ int tcp_tap_handler(struct ctx *c, int af, void *addr,
break;
case ESTABLISHED:
case ESTABLISHED_SOCK_FIN:
+ case ESTABLISHED_SOCK_FIN_SENT:
case CLOSE_WAIT:
+ case FIN_WAIT_1_SOCK_FIN:
+ case FIN_WAIT_1:
+ tcp_tap_epoll_mask(c, conn, conn->events & ~EPOLLET);
tcp_data_from_tap(c, conn, msg, count, now);
return count;
- case FIN_WAIT_1:
- tcp_send_to_tap(c, conn, ACK, NULL, 0);
- break;
- case FIN_WAIT_1_SOCK_FIN:
- if (th->ack)
- tcp_tap_destroy(c, conn);
- break;
case TAP_SYN_SENT:
case LAST_ACK:
case SPLICE_ACCEPTED:
case SPLICE_CONNECT:
case SPLICE_ESTABLISHED:
+ case SPLICE_FIN_FROM:
+ case SPLICE_FIN_TO:
+ case SPLICE_FIN_BOTH:
case CLOSED: /* ;) */
break;
}
@@ -2183,12 +2271,9 @@ int tcp_tap_handler(struct ctx *c, int af, void *addr,
* tcp_connect_finish() - Handle completion of connect() from EPOLLOUT event
* @c: Execution context
* @s: File descriptor number for socket
- * @ref: epoll reference
*/
-static void tcp_connect_finish(struct ctx *c, struct tcp_tap_conn *conn,
- union epoll_ref ref)
+static void tcp_connect_finish(struct ctx *c, struct tcp_tap_conn *conn)
{
- struct epoll_event ev;
socklen_t sl;
int so;
@@ -2202,9 +2287,7 @@ static void tcp_connect_finish(struct ctx *c, struct tcp_tap_conn *conn,
return;
/* Drop EPOLLOUT, only used to wait for connect() to complete */
- ev.events = EPOLLIN | EPOLLRDHUP;
- ev.data.u64 = ref.u64;
- epoll_ctl(c->epollfd, EPOLL_CTL_MOD, conn->sock, &ev);
+ tcp_tap_epoll_mask(c, conn, EPOLLIN | EPOLLRDHUP);
tcp_tap_state(conn, TAP_SYN_RCVD);
}
@@ -2236,15 +2319,6 @@ static void tcp_splice_connect_finish(struct ctx *c,
tcp_splice_destroy(c, conn);
return;
}
-
- tcp_splice_state(conn, SPLICE_ESTABLISHED);
-
- ev_from.events = ev_to.events = EPOLLIN | EPOLLET | EPOLLRDHUP;
- ev_from.data.u64 = ref_from.u64;
- ev_to.data.u64 = ref_to.u64;
-
- epoll_ctl(c->epollfd, EPOLL_CTL_MOD, conn->from, &ev_from);
- epoll_ctl(c->epollfd, EPOLL_CTL_MOD, conn->to, &ev_to);
}
conn->pipe_from_to[0] = conn->pipe_to_from[0] = -1;
@@ -2256,6 +2330,17 @@ static void tcp_splice_connect_finish(struct ctx *c,
fcntl(conn->pipe_from_to[0], F_SETPIPE_SZ, PIPE_SIZE);
fcntl(conn->pipe_to_from[0], F_SETPIPE_SZ, PIPE_SIZE);
+
+ if (conn->state == SPLICE_CONNECT) {
+ tcp_splice_state(conn, SPLICE_ESTABLISHED);
+
+ ev_from.events = ev_to.events = EPOLLIN | EPOLLET | EPOLLRDHUP;
+ ev_from.data.u64 = ref_from.u64;
+ ev_to.data.u64 = ref_to.u64;
+
+ epoll_ctl(c->epollfd, EPOLL_CTL_MOD, conn->from, &ev_from);
+ epoll_ctl(c->epollfd, EPOLL_CTL_MOD, conn->to, &ev_to);
+ }
}
/**
@@ -2278,9 +2363,9 @@ static int tcp_splice_connect(struct ctx *c, struct tcp_splice_conn *conn,
union epoll_ref ref_conn = { .proto = IPPROTO_TCP, .s = sock_conn,
.tcp = { .splice = 1, .v6 = v6,
.index = conn - ts } };
- struct epoll_event ev_accept = { .events = EPOLLRDHUP | EPOLLET,
+ struct epoll_event ev_accept = { .events = EPOLLET,
.data.u64 = ref_accept.u64 };
- struct epoll_event ev_conn = { .events = EPOLLRDHUP | EPOLLET,
+ struct epoll_event ev_conn = { .events = EPOLLET,
.data.u64 = ref_conn.u64 };
struct sockaddr_in6 addr6 = {
.sin6_family = AF_INET6,
@@ -2405,14 +2490,14 @@ static void tcp_conn_from_sock(struct ctx *c, union epoll_ref ref,
struct sockaddr_storage sa;
struct tcp_tap_conn *conn;
struct epoll_event ev;
- socklen_t sa_len;
+ socklen_t sl;
int s;
if (c->tcp.tap_conn_count >= MAX_TAP_CONNS)
return;
- sa_len = sizeof(sa);
- s = accept4(ref.s, (struct sockaddr *)&sa, &sa_len, SOCK_NONBLOCK);
+ sl = sizeof(sa);
+ s = accept4(ref.s, (struct sockaddr *)&sa, &sl, SOCK_NONBLOCK);
if (s < 0)
return;
@@ -2420,11 +2505,16 @@ static void tcp_conn_from_sock(struct ctx *c, union epoll_ref ref,
ref_conn.tcp.index = conn - tt;
ref_conn.s = conn->sock = s;
+ sl = sizeof(conn->sndbuf);
+ if (getsockopt(s, SOL_SOCKET, SO_SNDBUF, &conn->sndbuf, &sl))
+ conn->sndbuf = USHRT_MAX;
+
if (ref.tcp.v6) {
struct sockaddr_in6 *sa6 = (struct sockaddr_in6 *)&sa;
if (IN6_IS_ADDR_LOOPBACK(&sa6->sin6_addr) ||
- !memcmp(&sa6->sin6_addr, &c->addr6_seen, sizeof(c->addr6)))
+ !memcmp(&sa6->sin6_addr, &c->addr6_seen, sizeof(c->gw6)) ||
+ !memcmp(&sa6->sin6_addr, &c->addr6, sizeof(c->gw6)))
memcpy(&sa6->sin6_addr, &c->gw6, sizeof(c->gw6));
memcpy(&conn->a.a6, &sa6->sin6_addr, sizeof(conn->a.a6));
@@ -2447,7 +2537,7 @@ static void tcp_conn_from_sock(struct ctx *c, union epoll_ref ref,
memset(&conn->a.a4.one, 0xff, sizeof(conn->a.a4.one));
if (s_addr >> IN_CLASSA_NSHIFT == IN_LOOPBACKNET ||
- s_addr == INADDR_ANY || s_addr == c->addr4_seen)
+ s_addr == INADDR_ANY || s_addr == htonl(c->addr4_seen))
sa4->sin_addr.s_addr = c->gw4;
memcpy(&conn->a.a4.a, &sa4->sin_addr, sizeof(conn->a.a4.a));
@@ -2473,7 +2563,7 @@ static void tcp_conn_from_sock(struct ctx *c, union epoll_ref ref,
bitmap_set(tcp_act, conn - tt);
- ev.events = EPOLLRDHUP;
+ conn->events = ev.events = EPOLLRDHUP;
ev.data.u64 = ref_conn.u64;
epoll_ctl(c->epollfd, EPOLL_CTL_ADD, conn->sock, &ev);
@@ -2490,8 +2580,9 @@ static void tcp_conn_from_sock(struct ctx *c, union epoll_ref ref,
void tcp_sock_handler_splice(struct ctx *c, union epoll_ref ref,
uint32_t events)
{
- int move_from, move_to, *pipes;
+ int move_from, move_to, *pipes, eof;
struct tcp_splice_conn *conn;
+ struct epoll_event ev;
if (ref.tcp.listen) {
int s;
@@ -2514,10 +2605,11 @@ void tcp_sock_handler_splice(struct ctx *c, union epoll_ref ref,
conn = &ts[ref.tcp.index];
- if (events & EPOLLRDHUP || events & EPOLLHUP || events & EPOLLERR) {
- tcp_splice_destroy(c, conn);
- return;
- }
+ if (events & EPOLLERR)
+ goto close;
+
+ if (conn->state == SPLICE_CONNECT && (events & EPOLLHUP))
+ goto close;
if (events & EPOLLOUT) {
struct epoll_event ev = {
@@ -2525,12 +2617,11 @@ void tcp_sock_handler_splice(struct ctx *c, union epoll_ref ref,
.data.u64 = ref.u64,
};
- if (conn->state == SPLICE_CONNECT) {
+ if (conn->state == SPLICE_CONNECT)
tcp_splice_connect_finish(c, conn, ref.tcp.v6);
- return;
- }
- epoll_ctl(c->epollfd, EPOLL_CTL_MOD, ref.s, &ev);
+ if (conn->state == SPLICE_ESTABLISHED)
+ epoll_ctl(c->epollfd, EPOLL_CTL_MOD, ref.s, &ev);
move_to = ref.s;
if (ref.s == conn->to) {
@@ -2551,50 +2642,113 @@ void tcp_sock_handler_splice(struct ctx *c, union epoll_ref ref,
}
}
+ if (events & EPOLLHUP) {
+ tcp_splice_state(conn, SPLICE_FIN_BOTH);
+ } else if (events & EPOLLRDHUP) {
+ if (ref.s == conn->from) {
+ if (conn->state == SPLICE_ESTABLISHED)
+ tcp_splice_state(conn, SPLICE_FIN_FROM);
+ else if (conn->state == SPLICE_FIN_TO)
+ tcp_splice_state(conn, SPLICE_FIN_BOTH);
+ } else {
+ if (conn->state == SPLICE_ESTABLISHED)
+ tcp_splice_state(conn, SPLICE_FIN_TO);
+ else if (conn->state == SPLICE_FIN_FROM)
+ tcp_splice_state(conn, SPLICE_FIN_BOTH);
+ }
+ }
+
swap:
+ eof = 0;
+
while (1) {
- int retry_write = 1, no_read = 1;
- ssize_t ret, nr = 0, nw;
+ int never_read = 1, retry_write = 1;
+ ssize_t read, to_write = 0, written;
retry:
- ret = splice(move_from, NULL, pipes[1], NULL, PIPE_SIZE,
- SPLICE_F_MOVE);
- if (ret < 0) {
- if (errno == EAGAIN) {
- nr = PIPE_SIZE;
- } else {
- tcp_splice_destroy(c, conn);
- return;
- }
- } else if (!ret && no_read) {
- break;
- } else if (ret) {
- no_read = 0;
- nr += ret;
+ read = splice(move_from, NULL, pipes[1], NULL, PIPE_SIZE,
+ SPLICE_F_MOVE);
+ if (read < 0) {
+ if (errno == EINTR)
+ goto retry;
+
+ if (errno != EAGAIN)
+ goto close;
+
+ to_write = PIPE_SIZE;
+ } else if (!read) {
+ eof = 1;
+ to_write = PIPE_SIZE;
+ } else {
+ never_read = 0;
+ to_write += read;
+ if (move_from == conn->from)
+ conn->from_read += read;
+ else
+ conn->to_read += read;
}
- nw = splice(pipes[0], NULL, move_to, NULL, nr, SPLICE_F_MOVE);
- if (nw < 0) {
- if (errno == EAGAIN) {
- struct epoll_event ev = {
- .events = EPOLLIN | EPOLLOUT | EPOLLET |
- EPOLLRDHUP
- };
+eintr:
+ written = splice(pipes[0], NULL, move_to, NULL, to_write,
+ SPLICE_F_MOVE);
- if (no_read)
- break;
+ if (written > 0) {
+ if (move_from == conn->from)
+ conn->from_written += written;
+ else
+ conn->to_written += written;
+ }
+
+ if (written < 0) {
+ if (errno == EINTR)
+ goto eintr;
- if (retry_write--)
- goto retry;
+ if (errno != EAGAIN)
+ goto close;
- ref.s = move_to;
- ev.data.u64 = ref.u64,
- epoll_ctl(c->epollfd, EPOLL_CTL_MOD, move_to,
- &ev);
+ if (never_read)
break;
+
+ if (retry_write--)
+ goto retry;
+
+ ev.events = EPOLLIN | EPOLLOUT | EPOLLET | EPOLLRDHUP;
+ ref.s = move_to;
+ ev.data.u64 = ref.u64,
+ epoll_ctl(c->epollfd, EPOLL_CTL_MOD, move_to, &ev);
+ break;
+ } else if (never_read && written == PIPE_SIZE) {
+ goto retry;
+ } else if (!never_read &&written < to_write) {
+ to_write -= written;
+ goto retry;
+ }
+
+ if (eof)
+ break;
+ }
+
+ if (conn->state == SPLICE_FIN_BOTH ||
+ (conn->state == SPLICE_FIN_FROM && move_from == conn->from) ||
+ (conn->state == SPLICE_FIN_TO && move_from == conn->to)) {
+ if (move_from == conn->from &&
+ conn->from_read == conn->from_written) {
+ if (!conn->from_fin_sent) {
+ shutdown(move_to, SHUT_WR);
+ conn->from_fin_sent = 1;
}
- tcp_splice_destroy(c, conn);
- return;
+
+ if (conn->to_fin_sent)
+ goto close;
+ } else if (move_from == conn->to &&
+ conn->to_read == conn->to_written) {
+ if (!conn->to_fin_sent) {
+ shutdown(move_to, SHUT_WR);
+ conn->to_fin_sent = 1;
+ }
+
+ if (conn->from_fin_sent)
+ goto close;
}
}
@@ -2609,6 +2763,12 @@ retry:
goto swap;
}
+
+ return;
+
+close:
+ tcp_splice_destroy(c, conn);
+ return;
}
/**
@@ -2635,45 +2795,71 @@ void tcp_sock_handler(struct ctx *c, union epoll_ref ref, uint32_t events,
conn = &tt[ref.tcp.index];
- if (conn->state == LAST_ACK) {
- tcp_send_to_tap(c, conn, ACK, NULL, 0);
- tcp_tap_destroy(c, conn);
- return;
- }
-
- if (conn->state == SOCK_SYN_SENT) {
- /* This could also be a data packet already, don't reset yet. */
- return;
- }
-
if (events & EPOLLERR) {
if (conn->state != CLOSED)
tcp_rst(c, conn);
return;
}
- if (events & EPOLLOUT) { /* Implies TAP_SYN_SENT */
- tcp_connect_finish(c, conn, ref);
+ switch (conn->state) {
+ case TAP_SYN_SENT:
+ if (events & EPOLLOUT)
+ tcp_connect_finish(c, conn);
+ else
+ tcp_rst(c, conn);
return;
- }
-
- if (conn->state == ESTABLISHED || conn->state == ESTABLISHED_SOCK_FIN)
+ case ESTABLISHED_SOCK_FIN:
+ case ESTABLISHED_SOCK_FIN_SENT:
+ case ESTABLISHED:
tcp_data_from_sock(c, conn, now);
-
- if (events & (EPOLLRDHUP | EPOLLHUP)) {
- if (conn->state == ESTABLISHED) {
- tcp_tap_state(conn, ESTABLISHED_SOCK_FIN);
- shutdown(conn->sock, SHUT_RD);
- } else if (conn->state == FIN_WAIT_1) {
- tcp_tap_state(conn, FIN_WAIT_1_SOCK_FIN);
- shutdown(conn->sock, SHUT_RD);
+ if (events & EPOLLHUP) {
+ tcp_rst(c, conn);
+ } else if (events & EPOLLRDHUP) {
+ if (conn->state == ESTABLISHED)
+ tcp_tap_state(conn, ESTABLISHED_SOCK_FIN);
tcp_data_from_sock(c, conn, now);
- tcp_sock_consume(conn, conn->seq_ack_from_tap);
- tcp_send_to_tap(c, conn, FIN | ACK, NULL, 0);
- } else if (conn->state != ESTABLISHED_SOCK_FIN) {
+ }
+ return;
+ case LAST_ACK:
+ tcp_send_to_tap(c, conn, 0, NULL, 0);
+ if (conn->seq_ack_to_tap == conn->seq_from_tap + 1)
tcp_tap_destroy(c, conn);
+ return;
+ case FIN_WAIT_1:
+ if (events & EPOLLIN)
+ tcp_data_from_sock(c, conn, now);
+ if (events & EPOLLRDHUP) {
+ tcp_send_to_tap(c, conn, FIN | ACK, NULL, 0);
+ tcp_tap_state(conn, FIN_WAIT_1_SOCK_FIN);
+ }
+ return;
+ case CLOSE_WAIT:
+ case FIN_WAIT_1_SOCK_FIN:
+ if (events & EPOLLIN)
+ tcp_data_from_sock(c, conn, now);
+ if (events & EPOLLHUP) {
+ if ((conn->seq_ack_to_tap == conn->seq_from_tap + 1) &&
+ (conn->seq_ack_from_tap == conn->seq_to_tap)) {
+ tcp_tap_destroy(c, conn);
+ } else {
+ tcp_send_to_tap(c, conn, 0, NULL, 0);
+ }
}
+ return;
+ case TAP_SYN_RCVD:
+ case SOCK_SYN_SENT:
+ case SPLICE_ACCEPTED:
+ case SPLICE_CONNECT:
+ case SPLICE_ESTABLISHED:
+ case SPLICE_FIN_FROM:
+ case SPLICE_FIN_TO:
+ case SPLICE_FIN_BOTH:
+ case CLOSED:
+ break;
}
+
+ if (events & EPOLLHUP)
+ tcp_rst(c, conn);
}
/**
@@ -2793,13 +2979,14 @@ static void tcp_timer_one(struct ctx *c, struct tcp_tap_conn *conn,
tcp_rst(c, conn);
break;
- case ESTABLISHED_SOCK_FIN:
+ case ESTABLISHED_SOCK_FIN_SENT:
if (ack_tap_ms > FIN_TIMEOUT) {
tcp_rst(c, conn);
break;
}
/* Falls through */
case ESTABLISHED:
+ case ESTABLISHED_SOCK_FIN:
if (tap_ms > ACT_TIMEOUT && sock_ms > ACT_TIMEOUT) {
tcp_rst(c, conn);
break;
@@ -2832,16 +3019,19 @@ static void tcp_timer_one(struct ctx *c, struct tcp_tap_conn *conn,
if (conn->seq_from_tap == conn->seq_ack_to_tap)
conn->ts_sock = *ts;
+ if (!conn->tcpi_snd_wnd)
+ tcp_send_to_tap(c, conn, 0, NULL, 0);
+
break;
case CLOSE_WAIT:
- case FIN_WAIT_1:
- if (sock_ms > FIN_TIMEOUT)
- tcp_rst(c, conn);
- break;
case FIN_WAIT_1_SOCK_FIN:
if (ack_tap_ms > FIN_TIMEOUT)
tcp_rst(c, conn);
break;
+ case FIN_WAIT_1:
+ if (sock_ms > FIN_TIMEOUT)
+ tcp_rst(c, conn);
+ break;
case LAST_ACK:
if (sock_ms > LAST_ACK_TIMEOUT)
tcp_rst(c, conn);
@@ -2850,6 +3040,9 @@ static void tcp_timer_one(struct ctx *c, struct tcp_tap_conn *conn,
case SPLICE_ACCEPTED:
case SPLICE_CONNECT:
case SPLICE_ESTABLISHED:
+ case SPLICE_FIN_FROM:
+ case SPLICE_FIN_TO:
+ case SPLICE_FIN_BOTH:
case CLOSED:
break;
}