diff --git a/gk/bpf.c b/gk/bpf.c index 16b09963b..2ffcdd913 100644 --- a/gk/bpf.c +++ b/gk/bpf.c @@ -106,12 +106,13 @@ static const struct rte_bpf_xsym flow_handler_init_xsym[] = { }; struct gk_bpf_pkt_frame { - uint64_t password; - struct flow_entry *fe; - struct ipacket *packet; - struct gk_config *gk_conf; - bool ready_to_tx; - struct gk_bpf_pkt_ctx ctx; + uint64_t password; + struct flow_entry *fe; + struct ipacket *packet; + struct gk_co *this_co; + bool pkt_part2_prefetched; + bool ready_to_tx; + struct gk_bpf_pkt_ctx ctx; }; static const uint64_t pkt_password = 0xa2e329ba8b15af05; @@ -199,6 +200,7 @@ gk_bpf_prep_for_tx(struct gk_bpf_pkt_ctx *ctx, int priority, int direct_if_possible) { int ret; + struct gatekeeper_if *back; struct gk_bpf_pkt_frame *frame = pkt_ctx_to_frame(ctx); if (unlikely(frame == NULL)) return -EINVAL; @@ -208,11 +210,18 @@ gk_bpf_prep_for_tx(struct gk_bpf_pkt_ctx *ctx, int priority, if (unlikely(priority < 0 || priority > PRIORITY_MAX)) return -EINVAL; + /* Prepare packet for transmission if needed. */ + if (likely(!frame->pkt_part2_prefetched)) { + frame->pkt_part2_prefetched = true; + if (likely(rte_mbuf_prefetch_part2_non_temporal( + frame->packet->pkt))) + gk_yield_next(frame->this_co); + } + + back = &frame->this_co->work->gk_conf->net->back; ret = (direct_if_possible != 0 && priority == PRIORITY_GRANTED) - ? update_pkt_priority(frame->packet, priority, - &frame->gk_conf->net->back) - : encapsulate(frame->packet->pkt, priority, - &frame->gk_conf->net->back, + ? update_pkt_priority(frame->packet, priority, back) + : encapsulate(frame->packet->pkt, priority, back, &frame->fe->grantor_fib->u.grantor.gt_addr); frame->ready_to_tx = ret == 0; @@ -486,7 +495,7 @@ parse_packet_further(struct ipacket *packet, struct gk_bpf_pkt_ctx *ctx) } int -gk_bpf_decide_pkt(struct gk_config *gk_conf, uint8_t program_index, +gk_bpf_decide_pkt(struct gk_co *this_co, uint8_t program_index, struct flow_entry *fe, struct ipacket *packet, uint64_t now, uint64_t *p_bpf_ret) { @@ -494,7 +503,8 @@ gk_bpf_decide_pkt(struct gk_config *gk_conf, uint8_t program_index, .password = pkt_password, .fe = fe, .packet = packet, - .gk_conf = gk_conf, + .this_co = this_co, + .pkt_part2_prefetched = false, .ready_to_tx = false, .ctx = { .now = now, @@ -502,7 +512,7 @@ gk_bpf_decide_pkt(struct gk_config *gk_conf, uint8_t program_index, }, }; const struct gk_bpf_flow_handler *handler = - &gk_conf->flow_handlers[program_index]; + &this_co->work->gk_conf->flow_handlers[program_index]; if (unlikely(handler->f_pkt == NULL)) { GK_LOG(WARNING, diff --git a/gk/bpf.h b/gk/bpf.h index f5c93e9ec..05cfd7f6d 100644 --- a/gk/bpf.h +++ b/gk/bpf.h @@ -20,6 +20,7 @@ #define _GATEKEEPER_GK_BPF_H_ #include "gatekeeper_gk.h" +#include "co.h" /* * Load the BPF program that handles flows into @gk_conf at @@ -32,7 +33,7 @@ int gk_load_bpf_flow_handler(struct gk_config *gk_conf, unsigned int index, const char *filename, int jit); -int gk_bpf_decide_pkt(struct gk_config *gk_conf, uint8_t program_index, +int gk_bpf_decide_pkt(struct gk_co *this_co, uint8_t program_index, struct flow_entry *fe, struct ipacket *packet, uint64_t now, uint64_t *p_bpf_ret); diff --git a/gk/co.c b/gk/co.c index d11eadf35..a6b7f6e73 100644 --- a/gk/co.c +++ b/gk/co.c @@ -40,8 +40,8 @@ get_next_co(struct gk_co *this_co) return list_next_entry(this_co, co_list); } -static void -yield_next(struct gk_co *this_co) +void +gk_yield_next(struct gk_co *this_co) { struct gk_co *next_co = get_next_co(this_co); if (unlikely(this_co == next_co)) @@ -169,7 +169,7 @@ parse_front_pkt(struct gk_co *this_co, int ret; rte_mbuf_prefetch_part1_non_temporal(pkt); - yield_next(this_co); + gk_yield_next(this_co); /* * This prefetch is enough to load Ethernet header (14 bytes), * optional Ethernet VLAN header (8 bytes), and either @@ -179,7 +179,7 @@ parse_front_pkt(struct gk_co *this_co, * IPv6: 14 + 8 + 40 = 62 */ rte_prefetch_non_temporal(rte_pktmbuf_mtod_offset(pkt, void *, 0)); - yield_next(this_co); + gk_yield_next(this_co); ret = extract_packet_info(pkt, packet); if (ret < 0) { @@ -351,6 +351,10 @@ gk_process_request(struct gk_co *this_co, struct flow_entry *fe, /* The assigned priority is @priority. */ + /* Prepare packet for transmission. */ + if (likely(rte_mbuf_prefetch_part2_non_temporal(pkt))) + gk_yield_next(this_co); + /* Encapsulate the packet as a request. */ ret = encapsulate(pkt, priority, back, &fib->u.grantor.gt_addr); if (ret < 0) @@ -423,6 +427,10 @@ gk_process_granted(struct gk_co *this_co, struct flow_entry *fe, priority = PRIORITY_RENEW_CAP; } + /* Prepare packet for transmission. */ + if (likely(rte_mbuf_prefetch_part2_non_temporal(pkt))) + gk_yield_next(this_co); + /* * Encapsulate packet as a granted packet, * mark it as a capability renewal request if @renew_cap is true, @@ -489,7 +497,6 @@ gk_process_bpf(struct gk_co *this_co, struct flow_entry *fe, { struct rte_mbuf *pkt = packet->pkt; struct gk_co_work *work = this_co->work; - struct gk_config *gk_conf = work->gk_conf; struct gk_measurement_metrics *stats; uint64_t bpf_ret; int program_index, rc; @@ -499,7 +506,7 @@ gk_process_bpf(struct gk_co *this_co, struct flow_entry *fe, goto expired; program_index = fe->program_index; - rc = gk_bpf_decide_pkt(gk_conf, program_index, fe, packet, now, + rc = gk_bpf_decide_pkt(this_co, program_index, fe, packet, now, &bpf_ret); if (unlikely(rc != 0)) { GK_LOG(WARNING, @@ -519,7 +526,7 @@ gk_process_bpf(struct gk_co *this_co, struct flow_entry *fe, * packet header space. */ if (pkt_copy_cached_eth_header(pkt, eth_cache, - gk_conf->net->back.l2_len_out)) + work->gk_conf->net->back.l2_len_out)) goto drop_pkt; stats->pkts_num_granted++; @@ -998,14 +1005,14 @@ gk_co_process_front_pkt_final(struct gk_co *this_co, /* Look up flow entry. */ rte_hash_prefetch_buckets_non_temporal( work->instance->ip_flow_hash_table, ip_flow_hash_val); - yield_next(this_co); + gk_yield_next(this_co); ret = rte_hash_lookup_with_hash(work->instance->ip_flow_hash_table, &packet->flow, ip_flow_hash_val); if (ret >= 0) { fe = &work->instance->ip_flow_entry_table[ret]; /* TODO Break this prefetch into part1 and part2. */ prefetch_flow_entry(fe); - yield_next(this_co); + gk_yield_next(this_co); process_flow_entry(this_co, fe, packet); set_leftover_fe(leftover, fe); return; @@ -1081,7 +1088,7 @@ gk_co_scan_flow_table_final(struct gk_co *this_co, rte_hash_prefetch_buckets_non_temporal(instance->ip_flow_hash_table, task->task_hash); - yield_next(this_co); + gk_yield_next(this_co); gk_del_flow_entry_from_hash(instance->ip_flow_hash_table, fe); if (leftover->fe == fe) @@ -1138,7 +1145,7 @@ gk_co_scan_flow_table(struct gk_co *this_co, * check if it's expired. */ rte_prefetch_non_temporal(fe); - yield_next(this_co); + gk_yield_next(this_co); if (!fe->in_use || !is_flow_expired(fe, rte_rdtsc())) return; diff --git a/gk/co.h b/gk/co.h index 86b69ada8..1feeb5b31 100644 --- a/gk/co.h +++ b/gk/co.h @@ -217,4 +217,7 @@ void gk_co_process_front_pkt_software_rss(struct gk_co *this_co, struct gk_co_task *task, struct gk_co_leftover *leftover); +void +gk_yield_next(struct gk_co *this_co); + #endif /* _GATEKEEPER_GK_CO_H_ */ diff --git a/include/gatekeeper_main.h b/include/gatekeeper_main.h index 37f1f0b9b..1c9396c15 100644 --- a/include/gatekeeper_main.h +++ b/include/gatekeeper_main.h @@ -20,6 +20,7 @@ #define _GATEKEEPER_MAIN_H_ #include +#include #include #include @@ -72,4 +73,28 @@ rte_mbuf_prefetch_part1_non_temporal(struct rte_mbuf *m) rte_prefetch_non_temporal(&m->cacheline0); } +/* XXX #52 This should be part of DPDK. */ +/** + * Prefetch the second part of the mbuf + * + * The next 64 bytes of the mbuf corresponds to fields that are used in the + * transmit path. If the cache line of the architecture is higher than 64B, + * this function does nothing as it is expected that the full mbuf is + * already in cache. + * + * @param m + * The pointer to the mbuf. + */ +static inline bool +rte_mbuf_prefetch_part2_non_temporal(struct rte_mbuf *m) +{ +#if RTE_CACHE_LINE_SIZE == 64 + rte_prefetch_non_temporal(&m->cacheline1); + return true; +#else + RTE_SET_USED(m); + return false; +#endif +} + #endif /* _GATEKEEPER_MAIN_H_ */