offload.c 9.6 KB
Newer Older
1
/*
2
 * Copyright (C) 2016-2017 Netronome Systems, Inc.
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
 *
 * This software is dual licensed under the GNU General License Version 2,
 * June 1991 as shown in the file COPYING in the top-level directory of this
 * source tree or the BSD 2-Clause License provided below.  You have the
 * option to license this software under the complete terms of either license.
 *
 * The BSD 2-Clause License:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      1. Redistributions of source code must retain the above
 *         copyright notice, this list of conditions and the following
 *         disclaimer.
 *
 *      2. Redistributions in binary form must reproduce the above
 *         copyright notice, this list of conditions and the following
 *         disclaimer in the documentation and/or other materials
 *         provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

/*
 * nfp_net_offload.c
 * Netronome network device driver: TC offload functions for PF and VF
 */

39 40 41
#define pr_fmt(fmt)	"NFP net bpf: " fmt

#include <linux/bpf.h>
42 43 44 45 46 47
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/pci.h>
#include <linux/jiffies.h>
#include <linux/timer.h>
#include <linux/list.h>
48
#include <linux/mm.h>
49 50 51 52 53

#include <net/pkt_cls.h>
#include <net/tc_act/tc_gact.h>
#include <net/tc_act/tc_mirred.h>

54
#include "main.h"
55
#include "../nfp_app.h"
56 57
#include "../nfp_net_ctrl.h"
#include "../nfp_net.h"
58

59
static int
60 61 62
nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
		 unsigned int cnt)
{
63
	struct nfp_insn_meta *meta;
64 65 66 67 68 69 70 71 72 73 74 75 76
	unsigned int i;

	for (i = 0; i < cnt; i++) {
		meta = kzalloc(sizeof(*meta), GFP_KERNEL);
		if (!meta)
			return -ENOMEM;

		meta->insn = prog[i];
		meta->n = i;

		list_add_tail(&meta->l, &nfp_prog->insns);
	}

77
	nfp_bpf_jit_prepare(nfp_prog, cnt);
78

79 80 81
	return 0;
}

82
static void nfp_prog_free(struct nfp_prog *nfp_prog)
83 84 85 86 87 88 89 90 91 92
{
	struct nfp_insn_meta *meta, *tmp;

	list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
		list_del(&meta->l);
		kfree(meta);
	}
	kfree(nfp_prog);
}

93 94 95
static int
nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
		      struct netdev_bpf *bpf)
96
{
97
	struct bpf_prog *prog = bpf->verifier.prog;
98
	struct nfp_prog *nfp_prog;
99 100
	int ret;

101 102
	nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
	if (!nfp_prog)
103 104
		return -ENOMEM;
	prog->aux->offload->dev_priv = nfp_prog;
105 106 107

	INIT_LIST_HEAD(&nfp_prog->insns);
	nfp_prog->type = prog->type;
108
	nfp_prog->bpf = app->priv;
109

110 111 112 113
	ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len);
	if (ret)
		goto err_free;

114 115 116 117
	nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog);
	bpf->verifier.ops = &nfp_bpf_analyzer_ops;

	return 0;
118 119 120 121

err_free:
	nfp_prog_free(nfp_prog);

122
	return ret;
123 124
}

125
static int nfp_bpf_translate(struct nfp_net *nn, struct bpf_prog *prog)
126
{
127
	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
128 129
	unsigned int stack_size;
	unsigned int max_instr;
J
Jiong Wang 已提交
130
	int err;
131

J
Jakub Kicinski 已提交
132
	stack_size = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;
J
Jakub Kicinski 已提交
133
	if (prog->aux->stack_depth > stack_size) {
J
Jakub Kicinski 已提交
134
		nn_info(nn, "stack too large: program %dB > FW stack %dB\n",
J
Jakub Kicinski 已提交
135
			prog->aux->stack_depth, stack_size);
J
Jakub Kicinski 已提交
136 137
		return -EOPNOTSUPP;
	}
138
	nfp_prog->stack_depth = round_up(prog->aux->stack_depth, 4);
139 140 141 142

	max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
	nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);

143
	nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);
144
	if (!nfp_prog->prog)
145 146
		return -ENOMEM;

J
Jiong Wang 已提交
147 148 149 150 151 152 153 154
	err = nfp_bpf_jit(nfp_prog);
	if (err)
		return err;

	prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64);
	prog->aux->offload->jited_image = nfp_prog->prog;

	return 0;
155 156
}

157
static int nfp_bpf_destroy(struct nfp_net *nn, struct bpf_prog *prog)
158
{
159 160
	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;

161
	kvfree(nfp_prog->prog);
162
	nfp_prog_free(nfp_prog);
163 164

	return 0;
165 166
}

167 168 169 170 171 172 173 174 175 176 177 178
static int
nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap,
			 void *key, void *next_key)
{
	if (!key)
		return nfp_bpf_ctrl_getfirst_entry(offmap, next_key);
	return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key);
}

static int
nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
{
179 180
	if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
		return -EINVAL;
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
	return nfp_bpf_ctrl_del_entry(offmap, key);
}

static const struct bpf_map_dev_ops nfp_bpf_map_ops = {
	.map_get_next_key	= nfp_bpf_map_get_next_key,
	.map_lookup_elem	= nfp_bpf_ctrl_lookup_entry,
	.map_update_elem	= nfp_bpf_ctrl_update_entry,
	.map_delete_elem	= nfp_bpf_map_delete_elem,
};

static int
nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
{
	struct nfp_bpf_map *nfp_map;
	long long int res;

	if (!bpf->maps.types)
		return -EOPNOTSUPP;

	if (offmap->map.map_flags ||
	    offmap->map.numa_node != NUMA_NO_NODE) {
		pr_info("map flags are not supported\n");
		return -EINVAL;
	}

	if (!(bpf->maps.types & 1 << offmap->map.map_type)) {
		pr_info("map type not supported\n");
		return -EOPNOTSUPP;
	}
	if (bpf->maps.max_maps == bpf->maps_in_use) {
		pr_info("too many maps for a device\n");
		return -ENOMEM;
	}
	if (bpf->maps.max_elems - bpf->map_elems_in_use <
	    offmap->map.max_entries) {
		pr_info("map with too many elements: %u, left: %u\n",
			offmap->map.max_entries,
			bpf->maps.max_elems - bpf->map_elems_in_use);
		return -ENOMEM;
	}
	if (offmap->map.key_size > bpf->maps.max_key_sz ||
	    offmap->map.value_size > bpf->maps.max_val_sz ||
	    round_up(offmap->map.key_size, 8) +
	    round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) {
		pr_info("elements don't fit in device constraints\n");
		return -ENOMEM;
	}

	nfp_map = kzalloc(sizeof(*nfp_map), GFP_USER);
	if (!nfp_map)
		return -ENOMEM;

	offmap->dev_priv = nfp_map;
	nfp_map->offmap = offmap;
	nfp_map->bpf = bpf;

	res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map);
	if (res < 0) {
		kfree(nfp_map);
		return res;
	}

	nfp_map->tid = res;
	offmap->dev_ops = &nfp_bpf_map_ops;
	bpf->maps_in_use++;
	bpf->map_elems_in_use += offmap->map.max_entries;
	list_add_tail(&nfp_map->l, &bpf->map_list);

	return 0;
}

static int
nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
{
	struct nfp_bpf_map *nfp_map = offmap->dev_priv;

	nfp_bpf_ctrl_free_map(bpf, nfp_map);
	list_del_init(&nfp_map->l);
	bpf->map_elems_in_use -= offmap->map.max_entries;
	bpf->maps_in_use--;
	kfree(nfp_map);

	return 0;
}

266 267 268 269 270 271 272 273 274
int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf)
{
	switch (bpf->command) {
	case BPF_OFFLOAD_VERIFIER_PREP:
		return nfp_bpf_verifier_prep(app, nn, bpf);
	case BPF_OFFLOAD_TRANSLATE:
		return nfp_bpf_translate(nn, bpf->offload.prog);
	case BPF_OFFLOAD_DESTROY:
		return nfp_bpf_destroy(nn, bpf->offload.prog);
275 276 277 278
	case BPF_OFFLOAD_MAP_ALLOC:
		return nfp_bpf_map_alloc(app->priv, bpf->offmap);
	case BPF_OFFLOAD_MAP_FREE:
		return nfp_bpf_map_free(app->priv, bpf->offmap);
279 280 281 282 283
	default:
		return -EINVAL;
	}
}

284
static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog)
285
{
286
	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
287
	unsigned int max_mtu;
288
	dma_addr_t dma_addr;
289
	void *img;
290 291 292 293 294
	int err;

	max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
	if (max_mtu < nn->dp.netdev->mtu) {
		nn_info(nn, "BPF offload not supported with MTU larger than HW packet split boundary\n");
295
		return -EOPNOTSUPP;
296 297
	}

298 299 300 301 302
	img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);
	if (IS_ERR(img))
		return PTR_ERR(img);

	dma_addr = dma_map_single(nn->dp.dev, img,
303 304
				  nfp_prog->prog_len * sizeof(u64),
				  DMA_TO_DEVICE);
305 306
	if (dma_mapping_error(nn->dp.dev, dma_addr)) {
		kfree(img);
307
		return -ENOMEM;
308
	}
309

310
	nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
311
	nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
312 313 314 315 316 317

	/* Load up the JITed code */
	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
	if (err)
		nn_err(nn, "FW command error while loading BPF: %d\n", err);

318 319
	dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
			 DMA_TO_DEVICE);
320
	kfree(img);
321 322

	return err;
323 324 325 326 327 328
}

static void nfp_net_bpf_start(struct nfp_net *nn)
{
	int err;

329
	/* Enable passing packets through BPF function */
330 331
	nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF;
	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
332 333 334 335 336 337 338
	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
	if (err)
		nn_err(nn, "FW command error while enabling BPF: %d\n", err);
}

static int nfp_net_bpf_stop(struct nfp_net *nn)
{
339
	if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF))
340 341
		return 0;

342 343
	nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF;
	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
344 345 346 347

	return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
}

J
Jakub Kicinski 已提交
348
int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
349
			bool old_prog)
350
{
351 352
	int err;

353
	if (prog) {
354
		struct bpf_prog_offload *offload = prog->aux->offload;
355 356 357 358 359 360

		if (!offload)
			return -EINVAL;
		if (offload->netdev != nn->dp.netdev)
			return -EINVAL;
	}
361

362 363 364 365 366 367 368 369 370
	if (prog && old_prog) {
		u8 cap;

		cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP);
		if (!(cap & NFP_NET_BPF_CAP_RELO)) {
			nn_err(nn, "FW does not support live reload\n");
			return -EBUSY;
		}
	}
J
Jakub Kicinski 已提交
371 372 373 374 375

	/* Something else is loaded, different program type? */
	if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
		return -EBUSY;

376 377
	if (old_prog && !prog)
		return nfp_net_bpf_stop(nn);
378

379 380 381
	err = nfp_net_bpf_load(nn, prog);
	if (err)
		return err;
382

383 384
	if (!old_prog)
		nfp_net_bpf_start(nn);
385

J
Jakub Kicinski 已提交
386
	return 0;
387
}