gk: prefetch the transmission fields of packets

This patch prefetches the transmission fields of a packet when
it is ready to be prepared for transmission.

gk/bpf.c

@@ -106,12 +106,13 @@ static const struct rte_bpf_xsym flow_handler_init_xsym[] = {
 };
 
 struct gk_bpf_pkt_frame {
-	uint64_t		password;
-	struct flow_entry	*fe;
-	struct ipacket          *packet;
-	struct gk_config	*gk_conf;
-	bool			ready_to_tx;
-	struct gk_bpf_pkt_ctx	ctx;
+	uint64_t              password;
+	struct flow_entry     *fe;
+	struct ipacket        *packet;
+	struct gk_co          *this_co;
+	bool                  pkt_part2_prefetched;
+	bool                  ready_to_tx;
+	struct gk_bpf_pkt_ctx ctx;
 };
 
 static const uint64_t pkt_password = 0xa2e329ba8b15af05;
@@ -199,6 +200,7 @@ gk_bpf_prep_for_tx(struct gk_bpf_pkt_ctx *ctx, int priority,
 	int direct_if_possible)
 {
 	int ret;
+	struct gatekeeper_if *back;
 	struct gk_bpf_pkt_frame *frame = pkt_ctx_to_frame(ctx);
 	if (unlikely(frame == NULL))
 		return -EINVAL;
@@ -208,11 +210,18 @@ gk_bpf_prep_for_tx(struct gk_bpf_pkt_ctx *ctx, int priority,
 	if (unlikely(priority < 0 || priority > PRIORITY_MAX))
 		return -EINVAL;
 
+	/* Prepare packet for transmission if needed. */
+	if (likely(!frame->pkt_part2_prefetched)) {
+		frame->pkt_part2_prefetched = true;
+		if (likely(rte_mbuf_prefetch_part2_non_temporal(
+				frame->packet->pkt)))
+			gk_yield_next(frame->this_co);
+	}
+
+	back = &frame->this_co->work->gk_conf->net->back;
 	ret = (direct_if_possible != 0 && priority == PRIORITY_GRANTED)
-		? update_pkt_priority(frame->packet, priority,
-			&frame->gk_conf->net->back)
-		: encapsulate(frame->packet->pkt, priority,
-			&frame->gk_conf->net->back,
+		? update_pkt_priority(frame->packet, priority, back)
+		: encapsulate(frame->packet->pkt, priority, back,
 			&frame->fe->grantor_fib->u.grantor.gt_addr);
 
 	frame->ready_to_tx = ret == 0;
@@ -486,23 +495,24 @@ parse_packet_further(struct ipacket *packet, struct gk_bpf_pkt_ctx *ctx)
 }
 
 int
-gk_bpf_decide_pkt(struct gk_config *gk_conf, uint8_t program_index,
+gk_bpf_decide_pkt(struct gk_co *this_co, uint8_t program_index,
 	struct flow_entry *fe, struct ipacket *packet, uint64_t now,
 	uint64_t *p_bpf_ret)
 {
 	struct gk_bpf_pkt_frame frame = {
 		.password = pkt_password,
 		.fe = fe,
 		.packet = packet,
-		.gk_conf = gk_conf,
+		.this_co = this_co,
+		.pkt_part2_prefetched = false,
 		.ready_to_tx = false,
 		.ctx = {
 			.now = now,
 			.expire_at = fe->u.bpf.expire_at,
 		},
 	};
 	const struct gk_bpf_flow_handler *handler =
-		&gk_conf->flow_handlers[program_index];
+		&this_co->work->gk_conf->flow_handlers[program_index];
 
 	if (unlikely(handler->f_pkt == NULL)) {
 		GK_LOG(WARNING,

gk/bpf.h

@@ -20,6 +20,7 @@
 #define _GATEKEEPER_GK_BPF_H_
 
 #include "gatekeeper_gk.h"
+#include "co.h"
 
 /*
  * Load the BPF program that handles flows into @gk_conf at
@@ -32,7 +33,7 @@
 int gk_load_bpf_flow_handler(struct gk_config *gk_conf, unsigned int index,
 	const char *filename, int jit);
 
-int gk_bpf_decide_pkt(struct gk_config *gk_conf, uint8_t program_index,
+int gk_bpf_decide_pkt(struct gk_co *this_co, uint8_t program_index,
 	struct flow_entry *fe, struct ipacket *packet, uint64_t now,
 	uint64_t *p_bpf_ret);
 

gk/co.c

@@ -40,8 +40,8 @@ get_next_co(struct gk_co *this_co)
 	return list_next_entry(this_co, co_list);
 }
 
-static void
-yield_next(struct gk_co *this_co)
+void
+gk_yield_next(struct gk_co *this_co)
 {
 	struct gk_co *next_co = get_next_co(this_co);
 	if (unlikely(this_co == next_co))
@@ -169,7 +169,7 @@ parse_front_pkt(struct gk_co *this_co,
 	int ret;
 
 	rte_mbuf_prefetch_part1_non_temporal(pkt);
-	yield_next(this_co);
+	gk_yield_next(this_co);
        /*
         * This prefetch is enough to load Ethernet header (14 bytes),
         * optional Ethernet VLAN header (8 bytes), and either
@@ -179,7 +179,7 @@ parse_front_pkt(struct gk_co *this_co,
         * IPv6: 14 + 8 + 40 = 62
         */
 	rte_prefetch_non_temporal(rte_pktmbuf_mtod_offset(pkt, void *, 0));
-	yield_next(this_co);
+	gk_yield_next(this_co);
 
 	ret = extract_packet_info(pkt, packet);
 	if (ret < 0) {
@@ -351,6 +351,10 @@ gk_process_request(struct gk_co *this_co, struct flow_entry *fe,
 
 	/* The assigned priority is @priority. */
 
+	/* Prepare packet for transmission. */
+	if (likely(rte_mbuf_prefetch_part2_non_temporal(pkt)))
+		gk_yield_next(this_co);
+
 	/* Encapsulate the packet as a request. */
 	ret = encapsulate(pkt, priority, back, &fib->u.grantor.gt_addr);
 	if (ret < 0)
@@ -423,6 +427,10 @@ gk_process_granted(struct gk_co *this_co, struct flow_entry *fe,
 		priority = PRIORITY_RENEW_CAP;
 	}
 
+	/* Prepare packet for transmission. */
+	if (likely(rte_mbuf_prefetch_part2_non_temporal(pkt)))
+		gk_yield_next(this_co);
+
 	/*
 	 * Encapsulate packet as a granted packet,
 	 * mark it as a capability renewal request if @renew_cap is true,
@@ -489,7 +497,6 @@ gk_process_bpf(struct gk_co *this_co, struct flow_entry *fe,
 {
 	struct rte_mbuf *pkt = packet->pkt;
 	struct gk_co_work *work = this_co->work;
-	struct gk_config *gk_conf = work->gk_conf;
 	struct gk_measurement_metrics *stats;
 	uint64_t bpf_ret;
 	int program_index, rc;
@@ -499,7 +506,7 @@ gk_process_bpf(struct gk_co *this_co, struct flow_entry *fe,
 		goto expired;
 
 	program_index = fe->program_index;
-	rc = gk_bpf_decide_pkt(gk_conf, program_index, fe, packet, now,
+	rc = gk_bpf_decide_pkt(this_co, program_index, fe, packet, now,
 		&bpf_ret);
 	if (unlikely(rc != 0)) {
 		GK_LOG(WARNING,
@@ -519,7 +526,7 @@ gk_process_bpf(struct gk_co *this_co, struct flow_entry *fe,
 		 * packet header space.
 		 */
 		if (pkt_copy_cached_eth_header(pkt, eth_cache,
-				gk_conf->net->back.l2_len_out))
+				work->gk_conf->net->back.l2_len_out))
 			goto drop_pkt;
 
 		stats->pkts_num_granted++;
@@ -998,14 +1005,14 @@ gk_co_process_front_pkt_final(struct gk_co *this_co,
 	/* Look up flow entry. */
 	rte_hash_prefetch_buckets_non_temporal(
 		work->instance->ip_flow_hash_table, ip_flow_hash_val);
-	yield_next(this_co);
+	gk_yield_next(this_co);
 	ret = rte_hash_lookup_with_hash(work->instance->ip_flow_hash_table,
 		&packet->flow, ip_flow_hash_val);
 	if (ret >= 0) {
 		fe = &work->instance->ip_flow_entry_table[ret];
 		/* TODO Break this prefetch into part1 and part2. */
 		prefetch_flow_entry(fe);
-		yield_next(this_co);
+		gk_yield_next(this_co);
 		process_flow_entry(this_co, fe, packet);
 		set_leftover_fe(leftover, fe);
 		return;
@@ -1081,7 +1088,7 @@ gk_co_scan_flow_table_final(struct gk_co *this_co,
 
 	rte_hash_prefetch_buckets_non_temporal(instance->ip_flow_hash_table,
 		task->task_hash);
-	yield_next(this_co);
+	gk_yield_next(this_co);
 
 	gk_del_flow_entry_from_hash(instance->ip_flow_hash_table, fe);
 	if (leftover->fe == fe)
@@ -1138,7 +1145,7 @@ gk_co_scan_flow_table(struct gk_co *this_co,
 	 * check if it's expired.
 	 */
 	rte_prefetch_non_temporal(fe);
-	yield_next(this_co);
+	gk_yield_next(this_co);
 
 	if (!fe->in_use || !is_flow_expired(fe, rte_rdtsc()))
 		return;

gk/co.h

@@ -217,4 +217,7 @@ void
 gk_co_process_front_pkt_software_rss(struct gk_co *this_co,
 	struct gk_co_task *task, struct gk_co_leftover *leftover);
 
+void
+gk_yield_next(struct gk_co *this_co);
+
 #endif /* _GATEKEEPER_GK_CO_H_ */

include/gatekeeper_main.h

@@ -20,6 +20,7 @@
 #define _GATEKEEPER_MAIN_H_
 
 #include <stdint.h>
+#include <stdbool.h>
 
 #include <rte_mbuf.h>
 #include <rte_prefetch.h>
@@ -72,4 +73,28 @@ rte_mbuf_prefetch_part1_non_temporal(struct rte_mbuf *m)
 	rte_prefetch_non_temporal(&m->cacheline0);
 }
 
+/* XXX #52 This should be part of DPDK. */
+/**
+ * Prefetch the second part of the mbuf
+ *
+ * The next 64 bytes of the mbuf corresponds to fields that are used in the
+ * transmit path. If the cache line of the architecture is higher than 64B,
+ * this function does nothing as it is expected that the full mbuf is
+ * already in cache.
+ *
+ * @param m
+ *   The pointer to the mbuf.
+ */
+static inline bool
+rte_mbuf_prefetch_part2_non_temporal(struct rte_mbuf *m)
+{
+#if RTE_CACHE_LINE_SIZE == 64
+	rte_prefetch_non_temporal(&m->cacheline1);
+	return true;
+#else
+	RTE_SET_USED(m);
+	return false;
+#endif
+}
+
 #endif /* _GATEKEEPER_MAIN_H_ */