offload.c 6.8 KB
Newer Older
1
/*
2
 * Copyright (C) 2016-2017 Netronome Systems, Inc.
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
 *
 * This software is dual licensed under the GNU General License Version 2,
 * June 1991 as shown in the file COPYING in the top-level directory of this
 * source tree or the BSD 2-Clause License provided below.  You have the
 * option to license this software under the complete terms of either license.
 *
 * The BSD 2-Clause License:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      1. Redistributions of source code must retain the above
 *         copyright notice, this list of conditions and the following
 *         disclaimer.
 *
 *      2. Redistributions in binary form must reproduce the above
 *         copyright notice, this list of conditions and the following
 *         disclaimer in the documentation and/or other materials
 *         provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

/*
 * nfp_net_offload.c
 * Netronome network device driver: TC offload functions for PF and VF
 */

#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/pci.h>
#include <linux/jiffies.h>
#include <linux/timer.h>
#include <linux/list.h>
45
#include <linux/mm.h>
46 47 48 49 50

#include <net/pkt_cls.h>
#include <net/tc_act/tc_gact.h>
#include <net/tc_act/tc_mirred.h>

51
#include "main.h"
52
#include "../nfp_app.h"
53 54
#include "../nfp_net_ctrl.h"
#include "../nfp_net.h"
55

56
static int
57 58 59
nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
		 unsigned int cnt)
{
60
	struct nfp_insn_meta *meta;
61 62 63 64 65 66 67 68 69 70 71 72 73
	unsigned int i;

	for (i = 0; i < cnt; i++) {
		meta = kzalloc(sizeof(*meta), GFP_KERNEL);
		if (!meta)
			return -ENOMEM;

		meta->insn = prog[i];
		meta->n = i;

		list_add_tail(&meta->l, &nfp_prog->insns);
	}

74
	nfp_bpf_jit_prepare(nfp_prog, cnt);
75

76 77 78
	return 0;
}

79
static void nfp_prog_free(struct nfp_prog *nfp_prog)
80 81 82 83 84 85 86 87 88 89
{
	struct nfp_insn_meta *meta, *tmp;

	list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
		list_del(&meta->l);
		kfree(meta);
	}
	kfree(nfp_prog);
}

90 91 92
static int
nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
		      struct netdev_bpf *bpf)
93
{
94
	struct bpf_prog *prog = bpf->verifier.prog;
95
	struct nfp_prog *nfp_prog;
96 97
	int ret;

98 99
	nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
	if (!nfp_prog)
100 101
		return -ENOMEM;
	prog->aux->offload->dev_priv = nfp_prog;
102 103 104

	INIT_LIST_HEAD(&nfp_prog->insns);
	nfp_prog->type = prog->type;
105
	nfp_prog->bpf = app->priv;
106

107 108 109 110
	ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len);
	if (ret)
		goto err_free;

111 112 113 114
	nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog);
	bpf->verifier.ops = &nfp_bpf_analyzer_ops;

	return 0;
115 116 117 118

err_free:
	nfp_prog_free(nfp_prog);

119
	return ret;
120 121
}

122
static int nfp_bpf_translate(struct nfp_net *nn, struct bpf_prog *prog)
123
{
124
	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
125 126
	unsigned int stack_size;
	unsigned int max_instr;
127

J
Jakub Kicinski 已提交
128
	stack_size = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;
J
Jakub Kicinski 已提交
129
	if (prog->aux->stack_depth > stack_size) {
J
Jakub Kicinski 已提交
130
		nn_info(nn, "stack too large: program %dB > FW stack %dB\n",
J
Jakub Kicinski 已提交
131
			prog->aux->stack_depth, stack_size);
J
Jakub Kicinski 已提交
132 133
		return -EOPNOTSUPP;
	}
134
	nfp_prog->stack_depth = round_up(prog->aux->stack_depth, 4);
135 136 137 138

	max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
	nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);

139
	nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);
140
	if (!nfp_prog->prog)
141 142
		return -ENOMEM;

143
	return nfp_bpf_jit(nfp_prog);
144 145
}

146
static int nfp_bpf_destroy(struct nfp_net *nn, struct bpf_prog *prog)
147
{
148 149
	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;

150
	kvfree(nfp_prog->prog);
151
	nfp_prog_free(nfp_prog);
152 153

	return 0;
154 155
}

156 157 158 159 160 161 162 163 164 165 166 167 168 169
int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf)
{
	switch (bpf->command) {
	case BPF_OFFLOAD_VERIFIER_PREP:
		return nfp_bpf_verifier_prep(app, nn, bpf);
	case BPF_OFFLOAD_TRANSLATE:
		return nfp_bpf_translate(nn, bpf->offload.prog);
	case BPF_OFFLOAD_DESTROY:
		return nfp_bpf_destroy(nn, bpf->offload.prog);
	default:
		return -EINVAL;
	}
}

170
static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog)
171
{
172
	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
173
	unsigned int max_mtu;
174
	dma_addr_t dma_addr;
175
	void *img;
176 177 178 179 180
	int err;

	max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
	if (max_mtu < nn->dp.netdev->mtu) {
		nn_info(nn, "BPF offload not supported with MTU larger than HW packet split boundary\n");
181
		return -EOPNOTSUPP;
182 183
	}

184 185 186 187 188
	img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);
	if (IS_ERR(img))
		return PTR_ERR(img);

	dma_addr = dma_map_single(nn->dp.dev, img,
189 190
				  nfp_prog->prog_len * sizeof(u64),
				  DMA_TO_DEVICE);
191 192
	if (dma_mapping_error(nn->dp.dev, dma_addr)) {
		kfree(img);
193
		return -ENOMEM;
194
	}
195

196
	nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
197
	nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
198 199 200 201 202 203

	/* Load up the JITed code */
	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
	if (err)
		nn_err(nn, "FW command error while loading BPF: %d\n", err);

204 205
	dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
			 DMA_TO_DEVICE);
206
	kfree(img);
207 208

	return err;
209 210 211 212 213 214
}

static void nfp_net_bpf_start(struct nfp_net *nn)
{
	int err;

215
	/* Enable passing packets through BPF function */
216 217
	nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF;
	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
218 219 220 221 222 223 224
	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
	if (err)
		nn_err(nn, "FW command error while enabling BPF: %d\n", err);
}

static int nfp_net_bpf_stop(struct nfp_net *nn)
{
225
	if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF))
226 227
		return 0;

228 229
	nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF;
	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
230 231 232 233

	return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
}

J
Jakub Kicinski 已提交
234
int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
235
			bool old_prog)
236
{
237 238
	int err;

239 240 241 242 243 244 245 246
	if (prog) {
		struct bpf_dev_offload *offload = prog->aux->offload;

		if (!offload)
			return -EINVAL;
		if (offload->netdev != nn->dp.netdev)
			return -EINVAL;
	}
247

248 249 250 251 252 253 254 255 256
	if (prog && old_prog) {
		u8 cap;

		cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP);
		if (!(cap & NFP_NET_BPF_CAP_RELO)) {
			nn_err(nn, "FW does not support live reload\n");
			return -EBUSY;
		}
	}
J
Jakub Kicinski 已提交
257 258 259 260 261

	/* Something else is loaded, different program type? */
	if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
		return -EBUSY;

262 263
	if (old_prog && !prog)
		return nfp_net_bpf_stop(nn);
264

265 266 267
	err = nfp_net_bpf_load(nn, prog);
	if (err)
		return err;
268

269 270
	if (!old_prog)
		nfp_net_bpf_start(nn);
271

J
Jakub Kicinski 已提交
272
	return 0;
273
}