flow: Record the pifs for each side of each flow

Currently we have no generic information flows apart from the type and
state, everything else is specific to the flow type.  Start introducing
generic flow information by recording the pifs which the flow connects.

To keep track of what information is valid, introduce new flow states:
INI for when the initiating side information is complete, and TGT for
when both sides information is complete, but we haven't chosen the
flow type yet.  For now, these states don't do an awful lot, but
they'll become more important as we add more generic information.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>

7 files changed, 109 insertions, 16 deletions

diff --git a/flow.c b/flow.c
index 91c0046..d05aa49 100644
--- a/flow.c
+++ b/flow.c
@@ -21,6 +21,8 @@
 const char *flow_state_str[] = {
 	[FLOW_STATE_FREE]	= "FREE",
 	[FLOW_STATE_NEW]	= "NEW",
+	[FLOW_STATE_INI]	= "INI",
+	[FLOW_STATE_TGT]	= "TGT",
 	[FLOW_STATE_TYPED]	= "TYPED",
 	[FLOW_STATE_ACTIVE]	= "ACTIVE",
 };
@@ -146,22 +148,63 @@ static void flow_set_state(struct flow_common *f, enum flow_state state)
 	f->state = state;
 	flow_log_(f, LOG_DEBUG, "%s -> %s", flow_state_str[oldstate],
 		  FLOW_STATE(f));
+
+	if (MAX(state, oldstate) >= FLOW_STATE_TGT)
+		flow_log_(f, LOG_DEBUG, "%s => %s", pif_name(f->pif[INISIDE]),
+			                            pif_name(f->pif[TGTSIDE]));
+	else if (MAX(state, oldstate) >= FLOW_STATE_INI)
+		flow_log_(f, LOG_DEBUG, "%s => ?", pif_name(f->pif[INISIDE]));
+}
+
+/**
+ * flow_initiate() - Move flow to INI, setting INISIDE details
+ * @flow:	Flow to change state
+ * @pif:	pif of the initiating side
+ */
+void flow_initiate(union flow *flow, uint8_t pif)
+{
+	struct flow_common *f = &flow->f;
+
+	ASSERT(pif != PIF_NONE);
+	ASSERT(flow_new_entry == flow && f->state == FLOW_STATE_NEW);
+	ASSERT(f->type == FLOW_TYPE_NONE);
+	ASSERT(f->pif[INISIDE] == PIF_NONE && f->pif[TGTSIDE] == PIF_NONE);
+
+	f->pif[INISIDE] = pif;
+	flow_set_state(f, FLOW_STATE_INI);
+}
+
+/**
+ * flow_target() - Move flow to TGT, setting TGTSIDE details
+ * @flow:	Flow to change state
+ * @pif:	pif of the target side
+ */
+void flow_target(union flow *flow, uint8_t pif)
+{
+	struct flow_common *f = &flow->f;
+
+	ASSERT(pif != PIF_NONE);
+	ASSERT(flow_new_entry == flow && f->state == FLOW_STATE_INI);
+	ASSERT(f->type == FLOW_TYPE_NONE);
+	ASSERT(f->pif[INISIDE] != PIF_NONE && f->pif[TGTSIDE] == PIF_NONE);
+
+	f->pif[TGTSIDE] = pif;
+	flow_set_state(f, FLOW_STATE_TGT);
 }
 
 /**
  * flow_set_type() - Set type and move to TYPED
  * @flow:	Flow to change state
- * @type:	Type for new flow
- *
- * Return: @flow
+ * @pif:	pif of the initiating side
  */
 union flow *flow_set_type(union flow *flow, enum flow_type type)
 {
 	struct flow_common *f = &flow->f;
 
 	ASSERT(type != FLOW_TYPE_NONE);
-	ASSERT(flow_new_entry == flow && f->state == FLOW_STATE_NEW);
+	ASSERT(flow_new_entry == flow && f->state == FLOW_STATE_TGT);
 	ASSERT(f->type == FLOW_TYPE_NONE);
+	ASSERT(f->pif[INISIDE] != PIF_NONE && f->pif[TGTSIDE] != PIF_NONE);
 
 	f->type = type;
 	flow_set_state(f, FLOW_STATE_TYPED);
@@ -175,6 +218,7 @@ union flow *flow_set_type(union flow *flow, enum flow_type type)
 void flow_activate(struct flow_common *f)
 {
 	ASSERT(&flow_new_entry->f == f && f->state == FLOW_STATE_TYPED);
+	ASSERT(f->pif[INISIDE] != PIF_NONE && f->pif[TGTSIDE] != PIF_NONE);
 
 	flow_set_state(f, FLOW_STATE_ACTIVE);
 	flow_new_entry = NULL;
@@ -234,6 +278,8 @@ void flow_alloc_cancel(union flow *flow)
 {
 	ASSERT(flow_new_entry == flow);
 	ASSERT(flow->f.state == FLOW_STATE_NEW ||
+	       flow->f.state == FLOW_STATE_INI ||
+	       flow->f.state == FLOW_STATE_TGT ||
 	       flow->f.state == FLOW_STATE_TYPED);
 	ASSERT(flow_first_free > FLOW_IDX(flow));
 
@@ -296,6 +342,8 @@ void flow_defer_handler(const struct ctx *c, const struct timespec *now)
 		}
 
 		case FLOW_STATE_NEW:
+		case FLOW_STATE_INI:
+		case FLOW_STATE_TGT:
 		case FLOW_STATE_TYPED:
 			/* Incomplete flow at end of cycle */
 			ASSERT(false);
diff --git a/flow.h b/flow.h
index 9530938..29ef9f1 100644
--- a/flow.h
+++ b/flow.h
@@ -25,14 +25,42 @@
  *    NEW - Freshly allocated, uninitialised entry
  *        Operations:
  *            - flow_alloc_cancel() returns the entry to FREE
+ *            - flow_initiate() sets the entry's INISIDE details and moves to
+ *              INI
  *            - FLOW_SET_TYPE() sets the entry's type and moves to TYPED
  *        Caveats:
  *            - No fields other than state may be accessed
- *            - At most one entry may be NEW or TYPED at a time, so it's unsafe
- *              to use flow_alloc() again until this entry moves to ACTIVE or
- *              FREE
+ *            - At most one entry may be NEW, INI, TGT or TYPED at a time, so
+ *              it's unsafe to use flow_alloc() again until this entry moves to
+ *              ACTIVE or FREE
  *            - You may not return to the main epoll loop while any flow is NEW
  *
+ *    INI - An entry with INISIDE common information completed
+ *        Operations:
+ *            - Common fields related to INISIDE may be read
+ *            - flow_alloc_cancel() returns the entry to FREE
+ *            - flow_target() sets the entry's TGTSIDE details and moves to TGT
+ *        Caveats:
+ *            - Other common fields may not be read
+ *            - Type specific fields may not be read or written
+ *            - At most one entry may be NEW, INI, TGT or TYPED at a time, so
+ *              it's unsafe to use flow_alloc() again until this entry moves to
+ *              ACTIVE or FREE
+ *            - You may not return to the main epoll loop while any flow is INI
+ *
+ *    TGT - An entry with only INISIDE and TGTSIDE common information completed
+ *        Operations:
+ *            - Common fields related to INISIDE & TGTSIDE may be read
+ *            - flow_alloc_cancel() returns the entry to FREE
+ *            - FLOW_SET_TYPE() sets the entry's type and moves to TYPED
+ *        Caveats:
+ *            - Other common fields may not be read
+ *            - Type specific fields may not be read or written
+ *            - At most one entry may be NEW, INI, TGT or TYPED at a time, so
+ *              it's unsafe to use flow_alloc() again until this entry moves to
+ *              ACTIVE or FREE
+ *            - You may not return to the main epoll loop while any flow is TGT
+ *
  *    TYPED - Generic info initialised, type specific initialisation underway
  *        Operations:
  *            - All common fields may be read
@@ -40,9 +68,9 @@
  *            - flow_alloc_cancel() returns the entry to FREE
  *            - FLOW_ACTIVATE() moves the entry to ACTIVE
  *        Caveats:
- *            - At most one entry may be NEW or TYPED at a time, so it's unsafe
- *              to use flow_alloc() again until this entry moves to ACTIVE or
- *              FREE
+ *            - At most one entry may be NEW, INI, TGT or TYPED at a time, so
+ *              it's unsafe to use flow_alloc() again until this entry moves to
+ *              ACTIVE or FREE
  *            - You may not return to the main epoll loop while any flow is
  *              TYPED
  *
@@ -58,6 +86,8 @@
 enum flow_state {
 	FLOW_STATE_FREE,
 	FLOW_STATE_NEW,
+	FLOW_STATE_INI,
+	FLOW_STATE_TGT,
 	FLOW_STATE_TYPED,
 	FLOW_STATE_ACTIVE,
 
@@ -109,6 +139,7 @@ extern const uint8_t flow_proto[];
  * struct flow_common - Common fields for packet flows
  * @state:	State of the flow table entry
  * @type:	Type of packet flow
+ * @pif[]:	Interface for each side of the flow
  */
 struct flow_common {
 #ifdef __GNUC__
@@ -122,6 +153,7 @@ struct flow_common {
 	static_assert(sizeof(uint8_t) * 8 >= FLOW_TYPE_BITS,
 		      "Not enough bits for type field");
 #endif
+	uint8_t		pif[SIDES];
 };
 
 #define FLOW_INDEX_BITS		17	/* 128k - 1 */
diff --git a/flow_table.h b/flow_table.h
index 28e36b9..1b16349 100644
--- a/flow_table.h
+++ b/flow_table.h
@@ -107,6 +107,9 @@ static inline flow_sidx_t flow_sidx(const struct flow_common *f,
 union flow *flow_alloc(void);
 void flow_alloc_cancel(union flow *flow);
 
+void flow_initiate(union flow *flow, uint8_t pif);
+void flow_target(union flow *flow, uint8_t pif);
+
 union flow *flow_set_type(union flow *flow, enum flow_type type);
 #define FLOW_SET_TYPE(flow_, t_, var_)	(&flow_set_type((flow_), (t_))->var_)
 
diff --git a/icmp.c b/icmp.c
index 3567ad7..80330f6 100644
--- a/icmp.c
+++ b/icmp.c
@@ -163,6 +163,8 @@ static struct icmp_ping_flow *icmp_ping_new(const struct ctx *c,
 	if (!flow)
 		return NULL;
 
+	flow_initiate(flow, PIF_TAP);
+	flow_target(flow, PIF_HOST);
 	pingf = FLOW_SET_TYPE(flow, flowtype, ping);
 
 	pingf->seq = -1;
diff --git a/pif.h b/pif.h
index bd52936..ca85b34 100644
--- a/pif.h
+++ b/pif.h
@@ -38,7 +38,6 @@ static inline const char *pif_type(enum pif_type pt)
 		return "?";
 }
 
-/* cppcheck-suppress unusedFunction */
 static inline const char *pif_name(uint8_t pif)
 {
 	return pif_type(pif);
diff --git a/tcp.c b/tcp.c
index a15b6f5..969f5b1 100644
--- a/tcp.c
+++ b/tcp.c
@@ -1948,6 +1948,8 @@ static void tcp_conn_from_tap(struct ctx *c, sa_family_t af,
 	if (!(flow = flow_alloc()))
 		return;
 
+	flow_initiate(flow, PIF_TAP);
+
 	if (af == AF_INET) {
 		if (IN4_IS_ADDR_UNSPECIFIED(saddr) ||
 		    IN4_IS_ADDR_BROADCAST(saddr) ||
@@ -2000,6 +2002,7 @@ static void tcp_conn_from_tap(struct ctx *c, sa_family_t af,
 			goto cancel;
 	}
 
+	flow_target(flow, PIF_HOST);
 	conn = FLOW_SET_TYPE(flow, FLOW_TCP, tcp);
 	conn->tapside = INISIDE;
 	conn->sock = s;
@@ -2713,7 +2716,10 @@ static void tcp_tap_conn_from_sock(struct ctx *c, in_port_t dstport,
 				   const union sockaddr_inany *sa,
 				   const struct timespec *now)
 {
-	struct tcp_tap_conn *conn = FLOW_SET_TYPE(flow, FLOW_TCP, tcp);
+	struct tcp_tap_conn *conn;
+
+	flow_target(flow, PIF_TAP);
+	conn = FLOW_SET_TYPE(flow, FLOW_TCP, tcp);
 
 	conn->tapside = TGTSIDE;
 	conn->sock = s;
@@ -2763,6 +2769,8 @@ void tcp_listen_handler(struct ctx *c, union epoll_ref ref,
 	if (s < 0)
 		goto cancel;
 
+	flow_initiate(flow, ref.tcp_listen.pif);
+
 	if (sa.sa_family == AF_INET) {
 		const struct in_addr *addr = &sa.sa4.sin_addr;
 		in_port_t port = sa.sa4.sin_port;
diff --git a/tcp_splice.c b/tcp_splice.c
index 31e2173..b8f64a9 100644
--- a/tcp_splice.c
+++ b/tcp_splice.c
@@ -431,7 +431,7 @@ bool tcp_splice_conn_from_sock(const struct ctx *c,
 	union inany_addr src;
 	in_port_t srcport;
 	sa_family_t af;
-	uint8_t pif1;
+	uint8_t tgtpif;
 
 	if (c->mode != MODE_PASTA)
 		return false;
@@ -455,7 +455,7 @@ bool tcp_splice_conn_from_sock(const struct ctx *c,
 			return true;
 		}
 
-		pif1 = PIF_HOST;
+		tgtpif = PIF_HOST;
 		dstport += c->tcp.fwd_out.delta[dstport];
 		break;
 
@@ -463,7 +463,7 @@ bool tcp_splice_conn_from_sock(const struct ctx *c,
 		if (!inany_is_loopback(&src))
 			return false;
 
-		pif1 = PIF_SPLICE;
+		tgtpif = PIF_SPLICE;
 		dstport += c->tcp.fwd_in.delta[dstport];
 		break;
 
@@ -471,6 +471,7 @@ bool tcp_splice_conn_from_sock(const struct ctx *c,
 		return false;
 	}
 
+	flow_target(flow, tgtpif);
 	conn = FLOW_SET_TYPE(flow, FLOW_TCP_SPLICE, tcp_splice);
 
 	conn->flags = af == AF_INET ? 0 : SPLICE_V6;
@@ -482,7 +483,7 @@ bool tcp_splice_conn_from_sock(const struct ctx *c,
 	if (setsockopt(s0, SOL_TCP, TCP_QUICKACK, &((int){ 1 }), sizeof(int)))
 		flow_trace(conn, "failed to set TCP_QUICKACK on %i", s0);
 
-	if (tcp_splice_connect(c, conn, af, pif1, dstport))
+	if (tcp_splice_connect(c, conn, af, tgtpif, dstport))
 		conn_flag(c, conn, CLOSING);
 
 	FLOW_ACTIVATE(conn);