offload.c 9.4 KB
Newer Older
1
/*
2
 * Copyright (C) 2016-2017 Netronome Systems, Inc.
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
 *
 * This software is dual licensed under the GNU General License Version 2,
 * June 1991 as shown in the file COPYING in the top-level directory of this
 * source tree or the BSD 2-Clause License provided below.  You have the
 * option to license this software under the complete terms of either license.
 *
 * The BSD 2-Clause License:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      1. Redistributions of source code must retain the above
 *         copyright notice, this list of conditions and the following
 *         disclaimer.
 *
 *      2. Redistributions in binary form must reproduce the above
 *         copyright notice, this list of conditions and the following
 *         disclaimer in the documentation and/or other materials
 *         provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

/*
 * nfp_net_offload.c
 * Netronome network device driver: TC offload functions for PF and VF
 */

39 40 41
#define pr_fmt(fmt)	"NFP net bpf: " fmt

#include <linux/bpf.h>
42 43 44 45 46 47
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/pci.h>
#include <linux/jiffies.h>
#include <linux/timer.h>
#include <linux/list.h>
48
#include <linux/mm.h>
49 50 51 52 53

#include <net/pkt_cls.h>
#include <net/tc_act/tc_gact.h>
#include <net/tc_act/tc_mirred.h>

54
#include "main.h"
55
#include "../nfp_app.h"
56 57
#include "../nfp_net_ctrl.h"
#include "../nfp_net.h"
58

59
static int
60 61 62
nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
		 unsigned int cnt)
{
63
	struct nfp_insn_meta *meta;
64 65 66 67 68 69 70 71 72 73 74 75 76
	unsigned int i;

	for (i = 0; i < cnt; i++) {
		meta = kzalloc(sizeof(*meta), GFP_KERNEL);
		if (!meta)
			return -ENOMEM;

		meta->insn = prog[i];
		meta->n = i;

		list_add_tail(&meta->l, &nfp_prog->insns);
	}

77
	nfp_bpf_jit_prepare(nfp_prog, cnt);
78

79 80 81
	return 0;
}

82
static void nfp_prog_free(struct nfp_prog *nfp_prog)
83 84 85 86 87 88 89 90 91 92
{
	struct nfp_insn_meta *meta, *tmp;

	list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
		list_del(&meta->l);
		kfree(meta);
	}
	kfree(nfp_prog);
}

93 94 95
static int
nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
		      struct netdev_bpf *bpf)
96
{
97
	struct bpf_prog *prog = bpf->verifier.prog;
98
	struct nfp_prog *nfp_prog;
99 100
	int ret;

101 102
	nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
	if (!nfp_prog)
103 104
		return -ENOMEM;
	prog->aux->offload->dev_priv = nfp_prog;
105 106 107

	INIT_LIST_HEAD(&nfp_prog->insns);
	nfp_prog->type = prog->type;
108
	nfp_prog->bpf = app->priv;
109

110 111 112 113
	ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len);
	if (ret)
		goto err_free;

114 115 116 117
	nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog);
	bpf->verifier.ops = &nfp_bpf_analyzer_ops;

	return 0;
118 119 120 121

err_free:
	nfp_prog_free(nfp_prog);

122
	return ret;
123 124
}

125
static int nfp_bpf_translate(struct nfp_net *nn, struct bpf_prog *prog)
126
{
127
	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
128 129
	unsigned int stack_size;
	unsigned int max_instr;
130

J
Jakub Kicinski 已提交
131
	stack_size = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;
J
Jakub Kicinski 已提交
132
	if (prog->aux->stack_depth > stack_size) {
J
Jakub Kicinski 已提交
133
		nn_info(nn, "stack too large: program %dB > FW stack %dB\n",
J
Jakub Kicinski 已提交
134
			prog->aux->stack_depth, stack_size);
J
Jakub Kicinski 已提交
135 136
		return -EOPNOTSUPP;
	}
137
	nfp_prog->stack_depth = round_up(prog->aux->stack_depth, 4);
138 139 140 141

	max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
	nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);

142
	nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);
143
	if (!nfp_prog->prog)
144 145
		return -ENOMEM;

146
	return nfp_bpf_jit(nfp_prog);
147 148
}

149
static int nfp_bpf_destroy(struct nfp_net *nn, struct bpf_prog *prog)
150
{
151 152
	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;

153
	kvfree(nfp_prog->prog);
154
	nfp_prog_free(nfp_prog);
155 156

	return 0;
157 158
}

159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
static int
nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap,
			 void *key, void *next_key)
{
	if (!key)
		return nfp_bpf_ctrl_getfirst_entry(offmap, next_key);
	return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key);
}

static int
nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
{
	return nfp_bpf_ctrl_del_entry(offmap, key);
}

static const struct bpf_map_dev_ops nfp_bpf_map_ops = {
	.map_get_next_key	= nfp_bpf_map_get_next_key,
	.map_lookup_elem	= nfp_bpf_ctrl_lookup_entry,
	.map_update_elem	= nfp_bpf_ctrl_update_entry,
	.map_delete_elem	= nfp_bpf_map_delete_elem,
};

static int
nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
{
	struct nfp_bpf_map *nfp_map;
	long long int res;

	if (!bpf->maps.types)
		return -EOPNOTSUPP;

	if (offmap->map.map_flags ||
	    offmap->map.numa_node != NUMA_NO_NODE) {
		pr_info("map flags are not supported\n");
		return -EINVAL;
	}

	if (!(bpf->maps.types & 1 << offmap->map.map_type)) {
		pr_info("map type not supported\n");
		return -EOPNOTSUPP;
	}
	if (bpf->maps.max_maps == bpf->maps_in_use) {
		pr_info("too many maps for a device\n");
		return -ENOMEM;
	}
	if (bpf->maps.max_elems - bpf->map_elems_in_use <
	    offmap->map.max_entries) {
		pr_info("map with too many elements: %u, left: %u\n",
			offmap->map.max_entries,
			bpf->maps.max_elems - bpf->map_elems_in_use);
		return -ENOMEM;
	}
	if (offmap->map.key_size > bpf->maps.max_key_sz ||
	    offmap->map.value_size > bpf->maps.max_val_sz ||
	    round_up(offmap->map.key_size, 8) +
	    round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) {
		pr_info("elements don't fit in device constraints\n");
		return -ENOMEM;
	}

	nfp_map = kzalloc(sizeof(*nfp_map), GFP_USER);
	if (!nfp_map)
		return -ENOMEM;

	offmap->dev_priv = nfp_map;
	nfp_map->offmap = offmap;
	nfp_map->bpf = bpf;

	res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map);
	if (res < 0) {
		kfree(nfp_map);
		return res;
	}

	nfp_map->tid = res;
	offmap->dev_ops = &nfp_bpf_map_ops;
	bpf->maps_in_use++;
	bpf->map_elems_in_use += offmap->map.max_entries;
	list_add_tail(&nfp_map->l, &bpf->map_list);

	return 0;
}

static int
nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
{
	struct nfp_bpf_map *nfp_map = offmap->dev_priv;

	nfp_bpf_ctrl_free_map(bpf, nfp_map);
	list_del_init(&nfp_map->l);
	bpf->map_elems_in_use -= offmap->map.max_entries;
	bpf->maps_in_use--;
	kfree(nfp_map);

	return 0;
}

256 257 258 259 260 261 262 263 264
int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf)
{
	switch (bpf->command) {
	case BPF_OFFLOAD_VERIFIER_PREP:
		return nfp_bpf_verifier_prep(app, nn, bpf);
	case BPF_OFFLOAD_TRANSLATE:
		return nfp_bpf_translate(nn, bpf->offload.prog);
	case BPF_OFFLOAD_DESTROY:
		return nfp_bpf_destroy(nn, bpf->offload.prog);
265 266 267 268
	case BPF_OFFLOAD_MAP_ALLOC:
		return nfp_bpf_map_alloc(app->priv, bpf->offmap);
	case BPF_OFFLOAD_MAP_FREE:
		return nfp_bpf_map_free(app->priv, bpf->offmap);
269 270 271 272 273
	default:
		return -EINVAL;
	}
}

274
static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog)
275
{
276
	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
277
	unsigned int max_mtu;
278
	dma_addr_t dma_addr;
279
	void *img;
280 281 282 283 284
	int err;

	max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
	if (max_mtu < nn->dp.netdev->mtu) {
		nn_info(nn, "BPF offload not supported with MTU larger than HW packet split boundary\n");
285
		return -EOPNOTSUPP;
286 287
	}

288 289 290 291 292
	img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);
	if (IS_ERR(img))
		return PTR_ERR(img);

	dma_addr = dma_map_single(nn->dp.dev, img,
293 294
				  nfp_prog->prog_len * sizeof(u64),
				  DMA_TO_DEVICE);
295 296
	if (dma_mapping_error(nn->dp.dev, dma_addr)) {
		kfree(img);
297
		return -ENOMEM;
298
	}
299

300
	nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
301
	nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
302 303 304 305 306 307

	/* Load up the JITed code */
	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
	if (err)
		nn_err(nn, "FW command error while loading BPF: %d\n", err);

308 309
	dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
			 DMA_TO_DEVICE);
310
	kfree(img);
311 312

	return err;
313 314 315 316 317 318
}

static void nfp_net_bpf_start(struct nfp_net *nn)
{
	int err;

319
	/* Enable passing packets through BPF function */
320 321
	nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF;
	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
322 323 324 325 326 327 328
	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
	if (err)
		nn_err(nn, "FW command error while enabling BPF: %d\n", err);
}

static int nfp_net_bpf_stop(struct nfp_net *nn)
{
329
	if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF))
330 331
		return 0;

332 333
	nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF;
	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
334 335 336 337

	return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
}

J
Jakub Kicinski 已提交
338
int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
339
			bool old_prog)
340
{
341 342
	int err;

343
	if (prog) {
344
		struct bpf_prog_offload *offload = prog->aux->offload;
345 346 347 348 349 350

		if (!offload)
			return -EINVAL;
		if (offload->netdev != nn->dp.netdev)
			return -EINVAL;
	}
351

352 353 354 355 356 357 358 359 360
	if (prog && old_prog) {
		u8 cap;

		cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP);
		if (!(cap & NFP_NET_BPF_CAP_RELO)) {
			nn_err(nn, "FW does not support live reload\n");
			return -EBUSY;
		}
	}
J
Jakub Kicinski 已提交
361 362 363 364 365

	/* Something else is loaded, different program type? */
	if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
		return -EBUSY;

366 367
	if (old_prog && !prog)
		return nfp_net_bpf_stop(nn);
368

369 370 371
	err = nfp_net_bpf_load(nn, prog);
	if (err)
		return err;
372

373 374
	if (!old_prog)
		nfp_net_bpf_start(nn);
375

J
Jakub Kicinski 已提交
376
	return 0;
377
}