diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c index 4bfcb1f3def83c075b647f69904b12f9e09a49ee..0de59f04da84fef85e0b3387d33b2a64005c6513 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c @@ -1217,6 +1217,28 @@ static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) adjust_head = &nfp_prog->bpf->adjust_head; + /* Optimized version - 5 vs 14 cycles */ + if (nfp_prog->adjust_head_location != UINT_MAX) { + if (WARN_ON_ONCE(nfp_prog->adjust_head_location != meta->n)) + return -EINVAL; + + emit_alu(nfp_prog, pptr_reg(nfp_prog), + reg_a(2 * 2), ALU_OP_ADD, pptr_reg(nfp_prog)); + emit_alu(nfp_prog, plen_reg(nfp_prog), + plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); + emit_alu(nfp_prog, pv_len(nfp_prog), + pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); + + wrp_immed(nfp_prog, reg_both(0), 0); + wrp_immed(nfp_prog, reg_both(1), 0); + + /* TODO: when adjust head is guaranteed to succeed we can + * also eliminate the following if (r0 == 0) branch. + */ + + return 0; + } + ret_einval = nfp_prog_current_offset(nfp_prog) + 14; end = ret_einval + 2; diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.c b/drivers/net/ethernet/netronome/nfp/bpf/main.c index bd4a1dcc58b3c9a8faf7ebcd9c16a227b4b51717..7678e687a2b1463a3e56f8522b53a903b2b4da43 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/main.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/main.c @@ -172,6 +172,8 @@ nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value, bpf->adjust_head.flags = readl(&cap->flags); bpf->adjust_head.off_min = readl(&cap->off_min); bpf->adjust_head.off_max = readl(&cap->off_max); + bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub); + bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add); if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) { nfp_err(cpp, "invalid adjust_head TLV: min > max\n"); diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.h b/drivers/net/ethernet/netronome/nfp/bpf/main.h index 00a46258fb6d406f6c5a64df6a34c381b6a2d575..f49669bf6b44564cd7ed8344c53bd89a4ffe7c6d 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/main.h +++ b/drivers/net/ethernet/netronome/nfp/bpf/main.h @@ -86,6 +86,8 @@ enum pkt_vec { * @flags: extra flags for adjust head * @off_min: minimal packet offset within buffer required * @off_max: maximum packet offset within buffer required + * @guaranteed_sub: amount of negative adjustment guaranteed possible + * @guaranteed_add: amount of positive adjustment guaranteed possible */ struct nfp_app_bpf { struct nfp_app *app; @@ -94,6 +96,8 @@ struct nfp_app_bpf { u32 flags; int off_min; int off_max; + int guaranteed_sub; + int guaranteed_add; } adjust_head; }; @@ -116,6 +120,7 @@ typedef int (*instr_cb_t)(struct nfp_prog *, struct nfp_insn_meta *); * @ptr: pointer type for memory operations * @ldst_gather_len: memcpy length gathered from load/store sequence * @paired_st: the paired store insn at the head of the sequence + * @arg2: arg2 for call instructions * @ptr_not_const: pointer is not always constant * @jmp_dst: destination info for jump instructions * @off: index of first generated machine instruction (in nfp_prog.prog) @@ -135,6 +140,7 @@ struct nfp_insn_meta { bool ptr_not_const; }; struct nfp_insn_meta *jmp_dst; + struct bpf_reg_state arg2; }; unsigned int off; unsigned short n; @@ -193,6 +199,7 @@ static inline bool is_mbpf_store(const struct nfp_insn_meta *meta) * @n_translated: number of successfully translated instructions (for errors) * @error: error code if something went wrong * @stack_depth: max stack depth from the verifier + * @adjust_head_location: if program has single adjust head call - the insn no. * @insns: list of BPF instruction wrappers (struct nfp_insn_meta) */ struct nfp_prog { @@ -216,6 +223,7 @@ struct nfp_prog { int error; unsigned int stack_depth; + unsigned int adjust_head_location; struct list_head insns; }; diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c index 0a457d98666c8868038ba25016827378188fcb42..9c2608445bd879e8e36aeed9f6c1651a55939510 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c @@ -69,9 +69,47 @@ nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, return meta; } +static void +nfp_record_adjust_head(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, + struct nfp_insn_meta *meta, + const struct bpf_reg_state *reg2) +{ + unsigned int location = UINT_MAX; + int imm; + + /* Datapath usually can give us guarantees on how much adjust head + * can be done without the need for any checks. Optimize the simple + * case where there is only one adjust head by a constant. + */ + if (reg2->type != SCALAR_VALUE || !tnum_is_const(reg2->var_off)) + goto exit_set_location; + imm = reg2->var_off.value; + /* Translator will skip all checks, we need to guarantee min pkt len */ + if (imm > ETH_ZLEN - ETH_HLEN) + goto exit_set_location; + if (imm > (int)bpf->adjust_head.guaranteed_add || + imm < -bpf->adjust_head.guaranteed_sub) + goto exit_set_location; + + if (nfp_prog->adjust_head_location) { + /* Only one call per program allowed */ + if (nfp_prog->adjust_head_location != meta->n) + goto exit_set_location; + + if (meta->arg2.var_off.value != imm) + goto exit_set_location; + } + + location = meta->n; +exit_set_location: + nfp_prog->adjust_head_location = location; +} + static int -nfp_bpf_check_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) +nfp_bpf_check_call(struct nfp_prog *nfp_prog, struct bpf_verifier_env *env, + struct nfp_insn_meta *meta) { + const struct bpf_reg_state *reg2 = cur_regs(env) + BPF_REG_2; struct nfp_app_bpf *bpf = nfp_prog->bpf; u32 func_id = meta->insn.imm; @@ -85,12 +123,16 @@ nfp_bpf_check_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) pr_warn("adjust_head: FW requires shifting metadata, not supported by the driver\n"); return -EOPNOTSUPP; } + + nfp_record_adjust_head(bpf, nfp_prog, meta, reg2); break; default: pr_warn("unsupported function id: %d\n", func_id); return -EOPNOTSUPP; } + meta->arg2 = *reg2; + return 0; } @@ -204,7 +246,7 @@ nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx) } if (meta->insn.code == (BPF_JMP | BPF_CALL)) - return nfp_bpf_check_call(nfp_prog, meta); + return nfp_bpf_check_call(nfp_prog, env, meta); if (meta->insn.code == (BPF_JMP | BPF_EXIT)) return nfp_bpf_check_exit(nfp_prog, env);