nfp_net_offload.c 7.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
/*
 * Copyright (C) 2016 Netronome Systems, Inc.
 *
 * This software is dual licensed under the GNU General License Version 2,
 * June 1991 as shown in the file COPYING in the top-level directory of this
 * source tree or the BSD 2-Clause License provided below.  You have the
 * option to license this software under the complete terms of either license.
 *
 * The BSD 2-Clause License:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      1. Redistributions of source code must retain the above
 *         copyright notice, this list of conditions and the following
 *         disclaimer.
 *
 *      2. Redistributions in binary form must reproduce the above
 *         copyright notice, this list of conditions and the following
 *         disclaimer in the documentation and/or other materials
 *         provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

/*
 * nfp_net_offload.c
 * Netronome network device driver: TC offload functions for PF and VF
 */

#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/pci.h>
#include <linux/jiffies.h>
#include <linux/timer.h>
#include <linux/list.h>

#include <net/pkt_cls.h>
#include <net/tc_act/tc_gact.h>
#include <net/tc_act/tc_mirred.h>

#include "nfp_bpf.h"
#include "nfp_net_ctrl.h"
#include "nfp_net.h"

54 55 56 57 58 59 60
void nfp_net_filter_stats_timer(unsigned long data)
{
	struct nfp_net *nn = (void *)data;
	struct nfp_stat_pair latest;

	spin_lock_bh(&nn->rx_filter_lock);

61
	if (nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
		mod_timer(&nn->rx_filter_stats_timer,
			  jiffies + NFP_NET_STAT_POLL_IVL);

	spin_unlock_bh(&nn->rx_filter_lock);

	latest.pkts = nn_readq(nn, NFP_NET_CFG_STATS_APP1_FRAMES);
	latest.bytes = nn_readq(nn, NFP_NET_CFG_STATS_APP1_BYTES);

	if (latest.pkts != nn->rx_filter.pkts)
		nn->rx_filter_change = jiffies;

	nn->rx_filter = latest;
}

static void nfp_net_bpf_stats_reset(struct nfp_net *nn)
{
	nn->rx_filter.pkts = nn_readq(nn, NFP_NET_CFG_STATS_APP1_FRAMES);
	nn->rx_filter.bytes = nn_readq(nn, NFP_NET_CFG_STATS_APP1_BYTES);
	nn->rx_filter_prev = nn->rx_filter;
	nn->rx_filter_change = jiffies;
}

static int
nfp_net_bpf_stats_update(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf)
{
	u64 bytes, pkts;

	pkts = nn->rx_filter.pkts - nn->rx_filter_prev.pkts;
	bytes = nn->rx_filter.bytes - nn->rx_filter_prev.bytes;
	bytes -= pkts * ETH_HLEN;

	nn->rx_filter_prev = nn->rx_filter;

95 96
	tcf_exts_stats_update(cls_bpf->exts,
			      bytes, pkts, nn->rx_filter_change);
97 98 99 100

	return 0;
}

101 102 103 104 105 106
static int
nfp_net_bpf_get_act(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf)
{
	const struct tc_action *a;
	LIST_HEAD(actions);

107 108 109
	if (!cls_bpf->exts)
		return NN_ACT_XDP;

110
	/* TC direct action */
111 112 113 114
	if (cls_bpf->exts_integrated) {
		if (tc_no_actions(cls_bpf->exts))
			return NN_ACT_DIRECT;

115
		return -EOPNOTSUPP;
116
	}
117 118 119

	/* TC legacy mode */
	if (!tc_single_action(cls_bpf->exts))
120
		return -EOPNOTSUPP;
121 122 123 124 125

	tcf_exts_to_list(cls_bpf->exts, &actions);
	list_for_each_entry(a, &actions, list) {
		if (is_tcf_gact_shot(a))
			return NN_ACT_TC_DROP;
126

127
		if (is_tcf_mirred_egress_redirect(a) &&
128
		    tcf_mirred_ifindex(a) == nn->dp.netdev->ifindex)
129
			return NN_ACT_TC_REDIR;
130 131
	}

132
	return -EOPNOTSUPP;
133 134 135 136 137 138 139 140 141 142 143 144 145 146
}

static int
nfp_net_bpf_offload_prepare(struct nfp_net *nn,
			    struct tc_cls_bpf_offload *cls_bpf,
			    struct nfp_bpf_result *res,
			    void **code, dma_addr_t *dma_addr, u16 max_instr)
{
	unsigned int code_sz = max_instr * sizeof(u64);
	enum nfp_bpf_action_type act;
	u16 start_off, done_off;
	unsigned int max_mtu;
	int ret;

147
	if (!IS_ENABLED(CONFIG_BPF_SYSCALL))
148
		return -EOPNOTSUPP;
149

150 151 152 153 154 155
	ret = nfp_net_bpf_get_act(nn, cls_bpf);
	if (ret < 0)
		return ret;
	act = ret;

	max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
156
	if (max_mtu < nn->dp.netdev->mtu) {
157
		nn_info(nn, "BPF offload not supported with MTU larger than HW packet split boundary\n");
158
		return -EOPNOTSUPP;
159 160 161 162 163
	}

	start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
	done_off = nn_readw(nn, NFP_NET_CFG_BPF_DONE);

164
	*code = dma_zalloc_coherent(nn->dp.dev, code_sz, dma_addr, GFP_KERNEL);
165 166 167 168 169 170 171 172 173 174 175
	if (!*code)
		return -ENOMEM;

	ret = nfp_bpf_jit(cls_bpf->prog, *code, act, start_off, done_off,
			  max_instr, res);
	if (ret)
		goto out;

	return 0;

out:
176
	dma_free_coherent(nn->dp.dev, code_sz, *code, *dma_addr);
177 178 179 180 181 182 183 184 185 186 187 188
	return ret;
}

static void
nfp_net_bpf_load_and_start(struct nfp_net *nn, u32 tc_flags,
			   void *code, dma_addr_t dma_addr,
			   unsigned int code_sz, unsigned int n_instr,
			   bool dense_mode)
{
	u64 bpf_addr = dma_addr;
	int err;

189
	nn->dp.bpf_offload_skip_sw = !!(tc_flags & TCA_CLS_FLAGS_SKIP_SW);
190 191 192 193 194 195 196 197 198 199 200 201 202

	if (dense_mode)
		bpf_addr |= NFP_NET_CFG_BPF_CFG_8CTX;

	nn_writew(nn, NFP_NET_CFG_BPF_SIZE, n_instr);
	nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, bpf_addr);

	/* Load up the JITed code */
	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
	if (err)
		nn_err(nn, "FW command error while loading BPF: %d\n", err);

	/* Enable passing packets through BPF function */
203 204
	nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF;
	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
205 206 207 208
	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
	if (err)
		nn_err(nn, "FW command error while enabling BPF: %d\n", err);

209
	dma_free_coherent(nn->dp.dev, code_sz, code, dma_addr);
210 211 212

	nfp_net_bpf_stats_reset(nn);
	mod_timer(&nn->rx_filter_stats_timer, jiffies + NFP_NET_STAT_POLL_IVL);
213 214 215 216
}

static int nfp_net_bpf_stop(struct nfp_net *nn)
{
217
	if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF))
218 219
		return 0;

220
	spin_lock_bh(&nn->rx_filter_lock);
221
	nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF;
222
	spin_unlock_bh(&nn->rx_filter_lock);
223
	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
224

225
	del_timer_sync(&nn->rx_filter_stats_timer);
226
	nn->dp.bpf_offload_skip_sw = 0;
227 228 229 230

	return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
}

231
int nfp_net_bpf_offload(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf)
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
{
	struct nfp_bpf_result res;
	dma_addr_t dma_addr;
	u16 max_instr;
	void *code;
	int err;

	max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);

	switch (cls_bpf->command) {
	case TC_CLSBPF_REPLACE:
		/* There is nothing stopping us from implementing seamless
		 * replace but the simple method of loading I adopted in
		 * the firmware does not handle atomic replace (i.e. we have to
		 * stop the BPF offload and re-enable it).  Leaking-in a few
		 * frames which didn't have BPF applied in the hardware should
		 * be fine if software fallback is available, though.
		 */
250
		if (nn->dp.bpf_offload_skip_sw)
251 252 253 254 255 256 257 258 259 260 261 262 263 264
			return -EBUSY;

		err = nfp_net_bpf_offload_prepare(nn, cls_bpf, &res, &code,
						  &dma_addr, max_instr);
		if (err)
			return err;

		nfp_net_bpf_stop(nn);
		nfp_net_bpf_load_and_start(nn, cls_bpf->gen_flags, code,
					   dma_addr, max_instr * sizeof(u64),
					   res.n_instr, res.dense_mode);
		return 0;

	case TC_CLSBPF_ADD:
265
		if (nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
			return -EBUSY;

		err = nfp_net_bpf_offload_prepare(nn, cls_bpf, &res, &code,
						  &dma_addr, max_instr);
		if (err)
			return err;

		nfp_net_bpf_load_and_start(nn, cls_bpf->gen_flags, code,
					   dma_addr, max_instr * sizeof(u64),
					   res.n_instr, res.dense_mode);
		return 0;

	case TC_CLSBPF_DESTROY:
		return nfp_net_bpf_stop(nn);

281 282 283
	case TC_CLSBPF_STATS:
		return nfp_net_bpf_stats_update(nn, cls_bpf);

284
	default:
285
		return -EOPNOTSUPP;
286 287
	}
}