en_rx.c 39.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
/*
 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

34
#include <net/busy_poll.h>
35
#include <linux/bpf.h>
36
#include <linux/mlx4/cq.h>
37
#include <linux/slab.h>
38 39
#include <linux/mlx4/qp.h>
#include <linux/skbuff.h>
40
#include <linux/rculist.h>
41 42 43
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <linux/vmalloc.h>
44
#include <linux/irq.h>
45

46 47 48 49
#if IS_ENABLED(CONFIG_IPV6)
#include <net/ip6_checksum.h>
#endif

50 51
#include "mlx4_en.h"

52 53 54 55 56 57 58 59 60
static int mlx4_alloc_pages(struct mlx4_en_priv *priv,
			    struct mlx4_en_rx_alloc *page_alloc,
			    const struct mlx4_en_frag_info *frag_info,
			    gfp_t _gfp)
{
	int order;
	struct page *page;
	dma_addr_t dma;

61
	for (order = frag_info->order; ;) {
62 63 64
		gfp_t gfp = _gfp;

		if (order)
65
			gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NOMEMALLOC;
66 67 68 69 70 71 72 73
		page = alloc_pages(gfp, order);
		if (likely(page))
			break;
		if (--order < 0 ||
		    ((PAGE_SIZE << order) < frag_info->frag_size))
			return -ENOMEM;
	}
	dma = dma_map_page(priv->ddev, page, 0, PAGE_SIZE << order,
74
			   frag_info->dma_dir);
75
	if (unlikely(dma_mapping_error(priv->ddev, dma))) {
76 77 78
		put_page(page);
		return -ENOMEM;
	}
79
	page_alloc->page_size = PAGE_SIZE << order;
80 81
	page_alloc->page = page;
	page_alloc->dma = dma;
82
	page_alloc->page_offset = 0;
83
	/* Not doing get_page() for each frag is a big win
84
	 * on asymetric workloads. Note we can not use atomic_set().
85
	 */
86
	page_ref_add(page, page_alloc->page_size / frag_info->frag_stride - 1);
87 88 89
	return 0;
}

90 91 92
static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv,
			       struct mlx4_en_rx_desc *rx_desc,
			       struct mlx4_en_rx_alloc *frags,
93 94
			       struct mlx4_en_rx_alloc *ring_alloc,
			       gfp_t gfp)
95
{
96
	struct mlx4_en_rx_alloc page_alloc[MLX4_EN_MAX_RX_FRAGS];
97
	const struct mlx4_en_frag_info *frag_info;
98
	struct page *page;
99
	int i;
100

101 102
	for (i = 0; i < priv->num_frags; i++) {
		frag_info = &priv->frag_info[i];
103
		page_alloc[i] = ring_alloc[i];
104 105 106 107
		page_alloc[i].page_offset += frag_info->frag_stride;

		if (page_alloc[i].page_offset + frag_info->frag_stride <=
		    ring_alloc[i].page_size)
108
			continue;
109

110 111
		if (unlikely(mlx4_alloc_pages(priv, &page_alloc[i],
					      frag_info, gfp)))
112
			goto out;
113
	}
114

115 116
	for (i = 0; i < priv->num_frags; i++) {
		frags[i] = ring_alloc[i];
117 118 119
		frags[i].page_offset += priv->frag_info[i].rx_headroom;
		rx_desc->data[i].addr = cpu_to_be64(frags[i].dma +
						    frags[i].page_offset);
120
		ring_alloc[i] = page_alloc[i];
121
	}
122

123
	return 0;
124 125 126

out:
	while (i--) {
127
		if (page_alloc[i].page != ring_alloc[i].page) {
128
			dma_unmap_page(priv->ddev, page_alloc[i].dma,
129 130
				page_alloc[i].page_size,
				priv->frag_info[i].dma_dir);
131
			page = page_alloc[i].page;
132 133 134
			/* Revert changes done by mlx4_alloc_pages */
			page_ref_sub(page, page_alloc[i].page_size /
					   priv->frag_info[i].frag_stride - 1);
135 136
			put_page(page);
		}
137 138 139 140 141 142 143 144
	}
	return -ENOMEM;
}

static void mlx4_en_free_frag(struct mlx4_en_priv *priv,
			      struct mlx4_en_rx_alloc *frags,
			      int i)
{
145
	const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
146
	u32 next_frag_end = frags[i].page_offset + 2 * frag_info->frag_stride;
147

148 149

	if (next_frag_end > frags[i].page_size)
150
		dma_unmap_page(priv->ddev, frags[i].dma, frags[i].page_size,
151
			       frag_info->dma_dir);
152

153 154
	if (frags[i].page)
		put_page(frags[i].page);
155 156 157 158 159 160
}

static int mlx4_en_init_allocator(struct mlx4_en_priv *priv,
				  struct mlx4_en_rx_ring *ring)
{
	int i;
161
	struct mlx4_en_rx_alloc *page_alloc;
162 163

	for (i = 0; i < priv->num_frags; i++) {
164
		const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
165

166
		if (mlx4_alloc_pages(priv, &ring->page_alloc[i],
167
				     frag_info, GFP_KERNEL | __GFP_COLD))
168
			goto out;
169 170 171

		en_dbg(DRV, priv, "  frag %d allocator: - size:%d frags:%d\n",
		       i, ring->page_alloc[i].page_size,
172
		       page_ref_count(ring->page_alloc[i].page));
173 174 175 176 177
	}
	return 0;

out:
	while (i--) {
178 179
		struct page *page;

180
		page_alloc = &ring->page_alloc[i];
181
		dma_unmap_page(priv->ddev, page_alloc->dma,
182 183
			       page_alloc->page_size,
			       priv->frag_info[i].dma_dir);
184
		page = page_alloc->page;
185 186 187
		/* Revert changes done by mlx4_alloc_pages */
		page_ref_sub(page, page_alloc->page_size /
				   priv->frag_info[i].frag_stride - 1);
188
		put_page(page);
189 190 191 192 193 194 195 196 197 198 199 200
		page_alloc->page = NULL;
	}
	return -ENOMEM;
}

static void mlx4_en_destroy_allocator(struct mlx4_en_priv *priv,
				      struct mlx4_en_rx_ring *ring)
{
	struct mlx4_en_rx_alloc *page_alloc;
	int i;

	for (i = 0; i < priv->num_frags; i++) {
201 202
		const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];

203
		page_alloc = &ring->page_alloc[i];
204 205
		en_dbg(DRV, priv, "Freeing allocator:%d count:%d\n",
		       i, page_count(page_alloc->page));
206

207
		dma_unmap_page(priv->ddev, page_alloc->dma,
208
				page_alloc->page_size, frag_info->dma_dir);
209 210
		while (page_alloc->page_offset + frag_info->frag_stride <
		       page_alloc->page_size) {
211
			put_page(page_alloc->page);
212
			page_alloc->page_offset += frag_info->frag_stride;
213
		}
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
		page_alloc->page = NULL;
	}
}

static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv,
				 struct mlx4_en_rx_ring *ring, int index)
{
	struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index;
	int possible_frags;
	int i;

	/* Set size and memtype fields */
	for (i = 0; i < priv->num_frags; i++) {
		rx_desc->data[i].byte_count =
			cpu_to_be32(priv->frag_info[i].frag_size);
		rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key);
	}

	/* If the number of used fragments does not fill up the ring stride,
	 * remaining (unused) fragments must be padded with null address/size
	 * and a special memory key */
	possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE;
	for (i = priv->num_frags; i < possible_frags; i++) {
		rx_desc->data[i].byte_count = 0;
		rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD);
		rx_desc->data[i].addr = 0;
	}
}

static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv,
244 245
				   struct mlx4_en_rx_ring *ring, int index,
				   gfp_t gfp)
246 247
{
	struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride);
248 249
	struct mlx4_en_rx_alloc *frags = ring->rx_info +
					(index << priv->log_rx_info);
250

251 252
	if (ring->page_cache.index > 0) {
		frags[0] = ring->page_cache.buf[--ring->page_cache.index];
253 254
		rx_desc->data[0].addr = cpu_to_be64(frags[0].dma +
						    frags[0].page_offset);
255 256 257
		return 0;
	}

258
	return mlx4_en_alloc_frags(priv, rx_desc, frags, ring->page_alloc, gfp);
259 260
}

261 262 263 264 265
static inline bool mlx4_en_is_ring_empty(struct mlx4_en_rx_ring *ring)
{
	return ring->prod == ring->cons;
}

266 267 268 269 270
static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring)
{
	*ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff);
}

271 272 273 274
static void mlx4_en_free_rx_desc(struct mlx4_en_priv *priv,
				 struct mlx4_en_rx_ring *ring,
				 int index)
{
275
	struct mlx4_en_rx_alloc *frags;
276 277
	int nr;

278
	frags = ring->rx_info + (index << priv->log_rx_info);
279
	for (nr = 0; nr < priv->num_frags; nr++) {
280
		en_dbg(DRV, priv, "Freeing fragment:%d\n", nr);
281
		mlx4_en_free_frag(priv, frags, nr);
282 283 284
	}
}

285 286 287 288 289
static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv)
{
	struct mlx4_en_rx_ring *ring;
	int ring_ind;
	int buf_ind;
290
	int new_size;
291 292 293

	for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) {
		for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
294
			ring = priv->rx_ring[ring_ind];
295 296

			if (mlx4_en_prepare_rx_desc(priv, ring,
297
						    ring->actual_size,
298
						    GFP_KERNEL | __GFP_COLD)) {
299
				if (ring->actual_size < MLX4_EN_MIN_RX_SIZE) {
J
Joe Perches 已提交
300
					en_err(priv, "Failed to allocate enough rx buffers\n");
301 302
					return -ENOMEM;
				} else {
303
					new_size = rounddown_pow_of_two(ring->actual_size);
J
Joe Perches 已提交
304
					en_warn(priv, "Only %d buffers allocated reducing ring size to %d\n",
305
						ring->actual_size, new_size);
306
					goto reduce_rings;
307 308 309 310 311 312
				}
			}
			ring->actual_size++;
			ring->prod++;
		}
	}
313 314 315 316
	return 0;

reduce_rings:
	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
317
		ring = priv->rx_ring[ring_ind];
318 319 320 321 322 323 324
		while (ring->actual_size > new_size) {
			ring->actual_size--;
			ring->prod--;
			mlx4_en_free_rx_desc(priv, ring, ring->actual_size);
		}
	}

325 326 327 328 329 330 331 332
	return 0;
}

static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv,
				struct mlx4_en_rx_ring *ring)
{
	int index;

333 334
	en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n",
	       ring->cons, ring->prod);
335 336

	/* Unmap and free Rx buffers */
337
	while (!mlx4_en_is_ring_empty(ring)) {
338
		index = ring->cons & ring->size_mask;
339
		en_dbg(DRV, priv, "Processing descriptor:%d\n", index);
340
		mlx4_en_free_rx_desc(priv, ring, index);
341 342 343 344
		++ring->cons;
	}
}

345 346 347 348
void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev)
{
	int i;
	int num_of_eqs;
349
	int num_rx_rings;
350 351 352
	struct mlx4_dev *dev = mdev->dev;

	mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) {
M
Matan Barak 已提交
353 354 355 356
		num_of_eqs = max_t(int, MIN_RX_RINGS,
				   min_t(int,
					 mlx4_get_eqs_per_port(mdev->dev, i),
					 DEF_RX_RINGS));
357

358 359 360
		num_rx_rings = mlx4_low_memory_profile() ? MIN_RX_RINGS :
			min_t(int, num_of_eqs,
			      netif_get_num_default_rss_queues());
361
		mdev->profile.prof[i].rx_ring_num =
362
			rounddown_pow_of_two(num_rx_rings);
363 364 365
	}
}

366
int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
367
			   struct mlx4_en_rx_ring **pring,
368
			   u32 size, u16 stride, int node)
369 370
{
	struct mlx4_en_dev *mdev = priv->mdev;
371
	struct mlx4_en_rx_ring *ring;
372
	int err = -ENOMEM;
373 374
	int tmp;

375
	ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node);
376
	if (!ring) {
377 378 379 380 381
		ring = kzalloc(sizeof(*ring), GFP_KERNEL);
		if (!ring) {
			en_err(priv, "Failed to allocate RX ring structure\n");
			return -ENOMEM;
		}
382 383
	}

384 385 386 387 388 389
	ring->prod = 0;
	ring->cons = 0;
	ring->size = size;
	ring->size_mask = size - 1;
	ring->stride = stride;
	ring->log_stride = ffs(ring->stride) - 1;
390
	ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
391 392

	tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
393
					sizeof(struct mlx4_en_rx_alloc));
394
	ring->rx_info = vmalloc_node(tmp, node);
395
	if (!ring->rx_info) {
396 397 398 399 400
		ring->rx_info = vmalloc(tmp);
		if (!ring->rx_info) {
			err = -ENOMEM;
			goto err_ring;
		}
401
	}
402

403
	en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n",
404 405
		 ring->rx_info, tmp);

406
	/* Allocate HW buffers on provided NUMA node */
407
	set_dev_node(&mdev->dev->persist->pdev->dev, node);
408
	err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
409
	set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
410
	if (err)
411
		goto err_info;
412 413 414

	ring->buf = ring->wqres.buf.direct.buf;

415 416
	ring->hwtstamp_rx_filter = priv->hwtstamp_config.rx_filter;

417
	*pring = ring;
418 419
	return 0;

420
err_info:
421 422
	vfree(ring->rx_info);
	ring->rx_info = NULL;
423 424 425 426
err_ring:
	kfree(ring);
	*pring = NULL;

427 428 429 430 431 432 433 434 435 436 437 438 439
	return err;
}

int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
{
	struct mlx4_en_rx_ring *ring;
	int i;
	int ring_ind;
	int err;
	int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
					DS_SIZE * priv->num_frags);

	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
440
		ring = priv->rx_ring[ring_ind];
441 442 443 444

		ring->prod = 0;
		ring->cons = 0;
		ring->actual_size = 0;
445
		ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn;
446 447

		ring->stride = stride;
E
Eugenia Emantayev 已提交
448 449 450 451 452 453
		if (ring->stride <= TXBB_SIZE) {
			/* Stamp first unused send wqe */
			__be32 *ptr = (__be32 *)ring->buf;
			__be32 stamp = cpu_to_be32(1 << STAMP_SHIFT);
			*ptr = stamp;
			/* Move pointer to start of rx section */
454
			ring->buf += TXBB_SIZE;
E
Eugenia Emantayev 已提交
455
		}
456

457 458 459 460 461 462
		ring->log_stride = ffs(ring->stride) - 1;
		ring->buf_size = ring->size * ring->stride;

		memset(ring->buf, 0, ring->buf_size);
		mlx4_en_update_rx_prod_db(ring);

463
		/* Initialize all descriptors */
464 465 466 467 468 469
		for (i = 0; i < ring->size; i++)
			mlx4_en_init_rx_desc(priv, ring, i);

		/* Initialize page allocators */
		err = mlx4_en_init_allocator(priv, ring);
		if (err) {
470
			en_err(priv, "Failed initializing ring allocator\n");
471 472
			if (ring->stride <= TXBB_SIZE)
				ring->buf -= TXBB_SIZE;
473 474
			ring_ind--;
			goto err_allocator;
475 476
		}
	}
477 478
	err = mlx4_en_fill_rx_buffers(priv);
	if (err)
479 480 481
		goto err_buffers;

	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
482
		ring = priv->rx_ring[ring_ind];
483

484
		ring->size_mask = ring->actual_size - 1;
485 486 487 488 489 490 491
		mlx4_en_update_rx_prod_db(ring);
	}

	return 0;

err_buffers:
	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++)
492
		mlx4_en_free_rx_buf(priv, priv->rx_ring[ring_ind]);
493 494 495 496

	ring_ind = priv->rx_ring_num - 1;
err_allocator:
	while (ring_ind >= 0) {
497 498 499
		if (priv->rx_ring[ring_ind]->stride <= TXBB_SIZE)
			priv->rx_ring[ring_ind]->buf -= TXBB_SIZE;
		mlx4_en_destroy_allocator(priv, priv->rx_ring[ring_ind]);
500 501 502 503 504
		ring_ind--;
	}
	return err;
}

505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521
/* We recover from out of memory by scheduling our napi poll
 * function (mlx4_en_process_cq), which tries to allocate
 * all missing RX buffers (call to mlx4_en_refill_rx_buffers).
 */
void mlx4_en_recover_from_oom(struct mlx4_en_priv *priv)
{
	int ring;

	if (!priv->port_up)
		return;

	for (ring = 0; ring < priv->rx_ring_num; ring++) {
		if (mlx4_en_is_ring_empty(priv->rx_ring[ring]))
			napi_reschedule(&priv->rx_cq[ring]->napi);
	}
}

522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539
/* When the rx ring is running in page-per-packet mode, a released frame can go
 * directly into a small cache, to avoid unmapping or touching the page
 * allocator. In bpf prog performance scenarios, buffers are either forwarded
 * or dropped, never converted to skbs, so every page can come directly from
 * this cache when it is sized to be a multiple of the napi budget.
 */
bool mlx4_en_rx_recycle(struct mlx4_en_rx_ring *ring,
			struct mlx4_en_rx_alloc *frame)
{
	struct mlx4_en_page_cache *cache = &ring->page_cache;

	if (cache->index >= MLX4_EN_CACHE_SIZE)
		return false;

	cache->buf[cache->index++] = *frame;
	return true;
}

540
void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
541 542
			     struct mlx4_en_rx_ring **pring,
			     u32 size, u16 stride)
543 544
{
	struct mlx4_en_dev *mdev = priv->mdev;
545
	struct mlx4_en_rx_ring *ring = *pring;
546
	struct bpf_prog *old_prog;
547

548 549 550
	old_prog = rcu_dereference_protected(
					ring->xdp_prog,
					lockdep_is_held(&mdev->state_lock));
551 552
	if (old_prog)
		bpf_prog_put(old_prog);
553
	mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE);
554 555
	vfree(ring->rx_info);
	ring->rx_info = NULL;
556 557
	kfree(ring);
	*pring = NULL;
558 559 560 561 562
}

void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
				struct mlx4_en_rx_ring *ring)
{
563 564 565 566 567 568 569 570 571 572
	int i;

	for (i = 0; i < ring->page_cache.index; i++) {
		struct mlx4_en_rx_alloc *frame = &ring->page_cache.buf[i];

		dma_unmap_page(priv->ddev, frame->dma, frame->page_size,
			       priv->frag_info[0].dma_dir);
		put_page(frame->page);
	}
	ring->page_cache.index = 0;
573
	mlx4_en_free_rx_buf(priv, ring);
574 575
	if (ring->stride <= TXBB_SIZE)
		ring->buf -= TXBB_SIZE;
576 577 578 579 580 581
	mlx4_en_destroy_allocator(priv, ring);
}


static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv,
				    struct mlx4_en_rx_desc *rx_desc,
582
				    struct mlx4_en_rx_alloc *frags,
583
				    struct sk_buff *skb,
584 585
				    int length)
{
586
	struct skb_frag_struct *skb_frags_rx = skb_shinfo(skb)->frags;
587 588 589 590
	struct mlx4_en_frag_info *frag_info;
	int nr;
	dma_addr_t dma;

591
	/* Collect used fragments while replacing them in the HW descriptors */
592 593 594 595
	for (nr = 0; nr < priv->num_frags; nr++) {
		frag_info = &priv->frag_info[nr];
		if (length <= frag_info->frag_prefix_size)
			break;
596
		if (unlikely(!frags[nr].page))
597
			goto fail;
598 599

		dma = be64_to_cpu(rx_desc->data[nr].addr);
600 601
		dma_sync_single_for_cpu(priv->ddev, dma, frag_info->frag_size,
					DMA_FROM_DEVICE);
602

603 604 605
		/* Save page reference in skb */
		__skb_frag_set_page(&skb_frags_rx[nr], frags[nr].page);
		skb_frag_size_set(&skb_frags_rx[nr], frag_info->frag_size);
606
		skb_frags_rx[nr].page_offset = frags[nr].page_offset;
607
		skb->truesize += frag_info->frag_stride;
608
		frags[nr].page = NULL;
609 610
	}
	/* Adjust size of last fragment to match actual length */
611
	if (nr > 0)
E
Eric Dumazet 已提交
612 613
		skb_frag_size_set(&skb_frags_rx[nr - 1],
			length - priv->frag_info[nr - 1].frag_prefix_size);
614 615 616 617 618
	return nr;

fail:
	while (nr > 0) {
		nr--;
619
		__skb_frag_unref(&skb_frags_rx[nr]);
620 621 622 623 624 625 626
	}
	return 0;
}


static struct sk_buff *mlx4_en_rx_skb(struct mlx4_en_priv *priv,
				      struct mlx4_en_rx_desc *rx_desc,
627
				      struct mlx4_en_rx_alloc *frags,
628 629 630 631 632 633 634
				      unsigned int length)
{
	struct sk_buff *skb;
	void *va;
	int used_frags;
	dma_addr_t dma;

635
	skb = netdev_alloc_skb(priv->dev, SMALL_PACKET_SIZE + NET_IP_ALIGN);
636
	if (unlikely(!skb)) {
637
		en_dbg(RX_ERR, priv, "Failed allocating skb\n");
638 639 640 641 642 643 644
		return NULL;
	}
	skb_reserve(skb, NET_IP_ALIGN);
	skb->len = length;

	/* Get pointer to first fragment so we could copy the headers into the
	 * (linear part of the) skb */
645
	va = page_address(frags[0].page) + frags[0].page_offset;
646 647 648

	if (length <= SMALL_PACKET_SIZE) {
		/* We are copying all relevant data to the skb - temporarily
649
		 * sync buffers for the copy */
650
		dma = be64_to_cpu(rx_desc->data[0].addr);
651
		dma_sync_single_for_cpu(priv->ddev, dma, length,
652
					DMA_FROM_DEVICE);
653 654 655
		skb_copy_to_linear_data(skb, va, length);
		skb->tail += length;
	} else {
656 657
		unsigned int pull_len;

658
		/* Move relevant fragments to skb */
659 660
		used_frags = mlx4_en_complete_rx_desc(priv, rx_desc, frags,
							skb, length);
661 662 663 664
		if (unlikely(!used_frags)) {
			kfree_skb(skb);
			return NULL;
		}
665 666
		skb_shinfo(skb)->nr_frags = used_frags;

667
		pull_len = eth_get_headlen(va, SMALL_PACKET_SIZE);
668
		/* Copy headers into the skb linear buffer */
669 670
		memcpy(skb->data, va, pull_len);
		skb->tail += pull_len;
671 672

		/* Skip headers in first fragment */
673
		skb_shinfo(skb)->frags[0].page_offset += pull_len;
674 675

		/* Adjust size of first fragment */
676 677
		skb_frag_size_sub(&skb_shinfo(skb)->frags[0], pull_len);
		skb->data_len = length - pull_len;
678 679 680 681
	}
	return skb;
}

682 683 684 685 686 687 688 689 690 691 692 693 694 695 696
static void validate_loopback(struct mlx4_en_priv *priv, struct sk_buff *skb)
{
	int i;
	int offset = ETH_HLEN;

	for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) {
		if (*(skb->data + offset) != (unsigned char) (i & 0xff))
			goto out_loopback;
	}
	/* Loopback found */
	priv->loopback_ok = 1;

out_loopback:
	dev_kfree_skb_any(skb);
}
697

698 699
static bool mlx4_en_refill_rx_buffers(struct mlx4_en_priv *priv,
				      struct mlx4_en_rx_ring *ring)
700
{
701
	u32 missing = ring->actual_size - (ring->prod - ring->cons);
702

703 704 705 706 707 708
	/* Try to batch allocations, but not too much. */
	if (missing < 8)
		return false;
	do {
		if (mlx4_en_prepare_rx_desc(priv, ring,
					    ring->prod & ring->size_mask,
709
					    GFP_ATOMIC | __GFP_COLD))
710 711
			break;
		ring->prod++;
712 713 714
	} while (--missing);

	return true;
715 716
}

717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751
/* When hardware doesn't strip the vlan, we need to calculate the checksum
 * over it and add it to the hardware's checksum calculation
 */
static inline __wsum get_fixed_vlan_csum(__wsum hw_checksum,
					 struct vlan_hdr *vlanh)
{
	return csum_add(hw_checksum, *(__wsum *)vlanh);
}

/* Although the stack expects checksum which doesn't include the pseudo
 * header, the HW adds it. To address that, we are subtracting the pseudo
 * header checksum from the checksum value provided by the HW.
 */
static void get_fixed_ipv4_csum(__wsum hw_checksum, struct sk_buff *skb,
				struct iphdr *iph)
{
	__u16 length_for_csum = 0;
	__wsum csum_pseudo_header = 0;

	length_for_csum = (be16_to_cpu(iph->tot_len) - (iph->ihl << 2));
	csum_pseudo_header = csum_tcpudp_nofold(iph->saddr, iph->daddr,
						length_for_csum, iph->protocol, 0);
	skb->csum = csum_sub(hw_checksum, csum_pseudo_header);
}

#if IS_ENABLED(CONFIG_IPV6)
/* In IPv6 packets, besides subtracting the pseudo header checksum,
 * we also compute/add the IP header checksum which
 * is not added by the HW.
 */
static int get_fixed_ipv6_csum(__wsum hw_checksum, struct sk_buff *skb,
			       struct ipv6hdr *ipv6h)
{
	__wsum csum_pseudo_hdr = 0;

752 753
	if (unlikely(ipv6h->nexthdr == IPPROTO_FRAGMENT ||
		     ipv6h->nexthdr == IPPROTO_HOPOPTS))
754
		return -1;
755
	hw_checksum = csum_add(hw_checksum, (__force __wsum)htons(ipv6h->nexthdr));
756 757 758 759 760 761 762 763 764 765 766 767

	csum_pseudo_hdr = csum_partial(&ipv6h->saddr,
				       sizeof(ipv6h->saddr) + sizeof(ipv6h->daddr), 0);
	csum_pseudo_hdr = csum_add(csum_pseudo_hdr, (__force __wsum)ipv6h->payload_len);
	csum_pseudo_hdr = csum_add(csum_pseudo_hdr, (__force __wsum)ntohs(ipv6h->nexthdr));

	skb->csum = csum_sub(hw_checksum, csum_pseudo_hdr);
	skb->csum = csum_add(skb->csum, csum_partial(ipv6h, sizeof(struct ipv6hdr), 0));
	return 0;
}
#endif
static int check_csum(struct mlx4_cqe *cqe, struct sk_buff *skb, void *va,
768
		      netdev_features_t dev_features)
769 770 771 772 773 774 775
{
	__wsum hw_checksum = 0;

	void *hdr = (u8 *)va + sizeof(struct ethhdr);

	hw_checksum = csum_unfold((__force __sum16)cqe->checksum);

776
	if (cqe->vlan_my_qpn & cpu_to_be32(MLX4_CQE_CVLAN_PRESENT_MASK) &&
777
	    !(dev_features & NETIF_F_HW_VLAN_CTAG_RX)) {
778 779 780 781 782 783 784 785
		hw_checksum = get_fixed_vlan_csum(hw_checksum, hdr);
		hdr += sizeof(struct vlan_hdr);
	}

	if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV4))
		get_fixed_ipv4_csum(hw_checksum, skb, hdr);
#if IS_ENABLED(CONFIG_IPV6)
	else if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV6))
786
		if (unlikely(get_fixed_ipv6_csum(hw_checksum, skb, hdr)))
787 788 789 790 791
			return -1;
#endif
	return 0;
}

792 793 794
int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget)
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
795
	struct mlx4_en_dev *mdev = priv->mdev;
796
	struct mlx4_cqe *cqe;
797
	struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring];
798
	struct mlx4_en_rx_alloc *frags;
799
	struct mlx4_en_rx_desc *rx_desc;
800
	struct bpf_prog *xdp_prog;
801
	int doorbell_pending;
802 803 804 805 806 807
	struct sk_buff *skb;
	int index;
	int nr;
	unsigned int length;
	int polled = 0;
	int ip_summed;
O
Or Gerlitz 已提交
808
	int factor = priv->cqe_factor;
809
	u64 timestamp;
810
	bool l2_tunnel;
811

812
	if (unlikely(!priv->port_up))
813 814
		return 0;

815
	if (unlikely(budget <= 0))
816 817
		return polled;

818 819 820
	/* Protect accesses to: ring->xdp_prog, priv->mac_hash list */
	rcu_read_lock();
	xdp_prog = rcu_dereference(ring->xdp_prog);
821
	doorbell_pending = 0;
822

823 824 825 826
	/* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
	 * descriptor offset can be deduced from the CQE index instead of
	 * reading 'cqe->index' */
	index = cq->mcq.cons_index & ring->size_mask;
827
	cqe = mlx4_en_get_cqe(cq->buf, index, priv->cqe_size) + factor;
828 829 830 831 832

	/* Process all completed CQEs */
	while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
		    cq->mcq.cons_index & cq->size)) {

833
		frags = ring->rx_info + (index << priv->log_rx_info);
834 835 836 837 838
		rx_desc = ring->buf + (index << ring->log_stride);

		/*
		 * make sure we read the CQE after we read the ownership bit
		 */
839
		dma_rmb();
840 841 842 843

		/* Drop packet on bad receive or bad checksum */
		if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
						MLX4_CQE_OPCODE_ERROR)) {
J
Joe Perches 已提交
844 845 846
			en_err(priv, "CQE completed in error - vendor syndrom:%d syndrom:%d\n",
			       ((struct mlx4_err_cqe *)cqe)->vendor_err_syndrome,
			       ((struct mlx4_err_cqe *)cqe)->syndrome);
847 848 849
			goto next;
		}
		if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {
850
			en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");
851 852 853
			goto next;
		}

854 855 856 857 858 859 860 861 862 863 864 865 866
		/* Check if we need to drop the packet if SRIOV is not enabled
		 * and not performing the selftest or flb disabled
		 */
		if (priv->flags & MLX4_EN_FLAG_RX_FILTER_NEEDED) {
			struct ethhdr *ethh;
			dma_addr_t dma;
			/* Get pointer to first fragment since we haven't
			 * skb yet and cast it to ethhdr struct
			 */
			dma = be64_to_cpu(rx_desc->data[0].addr);
			dma_sync_single_for_cpu(priv->ddev, dma, sizeof(*ethh),
						DMA_FROM_DEVICE);
			ethh = (struct ethhdr *)(page_address(frags[0].page) +
867
						 frags[0].page_offset);
868

869 870 871 872 873 874 875 876
			if (is_multicast_ether_addr(ethh->h_dest)) {
				struct mlx4_mac_entry *entry;
				struct hlist_head *bucket;
				unsigned int mac_hash;

				/* Drop the packet, since HW loopback-ed it */
				mac_hash = ethh->h_source[MLX4_EN_MAC_HASH_IDX];
				bucket = &priv->mac_hash[mac_hash];
877
				hlist_for_each_entry_rcu(entry, bucket, hlist) {
878
					if (ether_addr_equal_64bits(entry->mac,
879
								    ethh->h_source))
880 881 882
						goto next;
				}
			}
883
		}
884

885 886 887 888
		/*
		 * Packet is OK - process it.
		 */
		length = be32_to_cpu(cqe->byte_cnt);
889
		length -= ring->fcs_del;
890 891
		l2_tunnel = (dev->hw_enc_features & NETIF_F_RXCSUM) &&
			(cqe->vlan_my_qpn & cpu_to_be32(MLX4_CQE_L2_TUNNEL));
892

893 894 895 896 897 898
		/* A bpf program gets first chance to drop the packet. It may
		 * read bytes but not past the end of the frag.
		 */
		if (xdp_prog) {
			struct xdp_buff xdp;
			dma_addr_t dma;
899
			void *orig_data;
900 901 902 903 904 905 906
			u32 act;

			dma = be64_to_cpu(rx_desc->data[0].addr);
			dma_sync_single_for_cpu(priv->ddev, dma,
						priv->frag_info[0].frag_size,
						DMA_FROM_DEVICE);

907 908
			xdp.data_hard_start = page_address(frags[0].page);
			xdp.data = xdp.data_hard_start + frags[0].page_offset;
909
			xdp.data_end = xdp.data + length;
910
			orig_data = xdp.data;
911 912

			act = bpf_prog_run_xdp(xdp_prog, &xdp);
913 914 915 916 917 918 919

			if (xdp.data != orig_data) {
				length = xdp.data_end - xdp.data;
				frags[0].page_offset = xdp.data -
					xdp.data_hard_start;
			}

920 921 922
			switch (act) {
			case XDP_PASS:
				break;
923
			case XDP_TX:
924
				if (likely(!mlx4_en_xmit_frame(ring, frags, dev,
925
							length, cq->ring,
926
							&doorbell_pending)))
927
					goto consumed;
928
				goto xdp_drop_no_cnt; /* Drop on xmit failure */
929 930 931 932
			default:
				bpf_warn_invalid_xdp_action(act);
			case XDP_ABORTED:
			case XDP_DROP:
933 934
				ring->xdp_drop++;
xdp_drop_no_cnt:
935
				if (likely(mlx4_en_rx_recycle(ring, frags)))
936
					goto consumed;
937 938 939 940
				goto next;
			}
		}

941 942 943
		ring->bytes += length;
		ring->packets++;

944
		if (likely(dev->features & NETIF_F_RXCSUM)) {
945 946 947 948 949 950 951 952 953 954
			if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_TCP |
						      MLX4_CQE_STATUS_UDP)) {
				if ((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
				    cqe->checksum == cpu_to_be16(0xffff)) {
					ip_summed = CHECKSUM_UNNECESSARY;
					ring->csum_ok++;
				} else {
					ip_summed = CHECKSUM_NONE;
					ring->csum_none++;
				}
955
			} else {
956 957 958 959 960 961 962 963 964
				if (priv->flags & MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP &&
				    (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 |
							       MLX4_CQE_STATUS_IPV6))) {
					ip_summed = CHECKSUM_COMPLETE;
					ring->csum_complete++;
				} else {
					ip_summed = CHECKSUM_NONE;
					ring->csum_none++;
				}
965 966 967
			}
		} else {
			ip_summed = CHECKSUM_NONE;
968
			ring->csum_none++;
969 970
		}

971 972 973 974 975 976
		/* This packet is eligible for GRO if it is:
		 * - DIX Ethernet (type interpretation)
		 * - TCP/IP (v4)
		 * - without IP options
		 * - not an IP fragment
		 */
977
		if (dev->features & NETIF_F_GRO) {
978 979 980 981 982 983 984 985 986 987
			struct sk_buff *gro_skb = napi_get_frags(&cq->napi);
			if (!gro_skb)
				goto next;

			nr = mlx4_en_complete_rx_desc(priv,
				rx_desc, frags, gro_skb,
				length);
			if (!nr)
				goto next;

988 989
			if (ip_summed == CHECKSUM_COMPLETE) {
				void *va = skb_frag_address(skb_shinfo(gro_skb)->frags);
990 991
				if (check_csum(cqe, gro_skb, va,
					       dev->features)) {
992 993 994 995 996 997
					ip_summed = CHECKSUM_NONE;
					ring->csum_none++;
					ring->csum_complete--;
				}
			}

998 999 1000 1001 1002 1003
			skb_shinfo(gro_skb)->nr_frags = nr;
			gro_skb->len = length;
			gro_skb->data_len = length;
			gro_skb->ip_summed = ip_summed;

			if (l2_tunnel && ip_summed == CHECKSUM_UNNECESSARY)
1004 1005
				gro_skb->csum_level = 1;

1006
			if ((cqe->vlan_my_qpn &
1007
			    cpu_to_be32(MLX4_CQE_CVLAN_PRESENT_MASK)) &&
1008 1009 1010 1011
			    (dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
				u16 vid = be16_to_cpu(cqe->sl_vid);

				__vlan_hwaccel_put_tag(gro_skb, htons(ETH_P_8021Q), vid);
1012 1013 1014 1015 1016 1017
			} else if ((be32_to_cpu(cqe->vlan_my_qpn) &
				  MLX4_CQE_SVLAN_PRESENT_MASK) &&
				 (dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
				__vlan_hwaccel_put_tag(gro_skb,
						       htons(ETH_P_8021AD),
						       be16_to_cpu(cqe->sl_vid));
1018 1019 1020 1021 1022
			}

			if (dev->features & NETIF_F_RXHASH)
				skb_set_hash(gro_skb,
					     be32_to_cpu(cqe->immed_rss_invalid),
1023 1024 1025
					     (ip_summed == CHECKSUM_UNNECESSARY) ?
						PKT_HASH_TYPE_L4 :
						PKT_HASH_TYPE_L3);
1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040

			skb_record_rx_queue(gro_skb, cq->ring);

			if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) {
				timestamp = mlx4_en_get_cqe_ts(cqe);
				mlx4_en_fill_hwtstamps(mdev,
						       skb_hwtstamps(gro_skb),
						       timestamp);
			}

			napi_gro_frags(&cq->napi);
			goto next;
		}

		/* GRO not possible, complete processing here */
1041
		skb = mlx4_en_rx_skb(priv, rx_desc, frags, length);
1042
		if (unlikely(!skb)) {
1043
			ring->dropped++;
1044 1045 1046
			goto next;
		}

K
Kamal Heib 已提交
1047
		if (unlikely(priv->validate_loopback)) {
1048 1049 1050 1051
			validate_loopback(priv, skb);
			goto next;
		}

1052
		if (ip_summed == CHECKSUM_COMPLETE) {
1053
			if (check_csum(cqe, skb, skb->data, dev->features)) {
1054 1055 1056 1057 1058 1059
				ip_summed = CHECKSUM_NONE;
				ring->csum_complete--;
				ring->csum_none++;
			}
		}

1060 1061
		skb->ip_summed = ip_summed;
		skb->protocol = eth_type_trans(skb, dev);
1062
		skb_record_rx_queue(skb, cq->ring);
1063

1064 1065
		if (l2_tunnel && ip_summed == CHECKSUM_UNNECESSARY)
			skb->csum_level = 1;
1066

Y
Yevgeny Petrilin 已提交
1067
		if (dev->features & NETIF_F_RXHASH)
T
Tom Herbert 已提交
1068 1069
			skb_set_hash(skb,
				     be32_to_cpu(cqe->immed_rss_invalid),
1070 1071 1072
				     (ip_summed == CHECKSUM_UNNECESSARY) ?
					PKT_HASH_TYPE_L4 :
					PKT_HASH_TYPE_L3);
Y
Yevgeny Petrilin 已提交
1073

1074
		if ((be32_to_cpu(cqe->vlan_my_qpn) &
1075
		    MLX4_CQE_CVLAN_PRESENT_MASK) &&
1076
		    (dev->features & NETIF_F_HW_VLAN_CTAG_RX))
1077
			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), be16_to_cpu(cqe->sl_vid));
1078 1079 1080 1081 1082
		else if ((be32_to_cpu(cqe->vlan_my_qpn) &
			  MLX4_CQE_SVLAN_PRESENT_MASK) &&
			 (dev->features & NETIF_F_HW_VLAN_STAG_RX))
			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021AD),
					       be16_to_cpu(cqe->sl_vid));
J
Jiri Pirko 已提交
1083

1084 1085 1086 1087 1088 1089
		if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) {
			timestamp = mlx4_en_get_cqe_ts(cqe);
			mlx4_en_fill_hwtstamps(mdev, skb_hwtstamps(skb),
					       timestamp);
		}

1090
		napi_gro_receive(&cq->napi, skb);
1091
next:
1092 1093 1094
		for (nr = 0; nr < priv->num_frags; nr++)
			mlx4_en_free_frag(priv, frags, nr);

1095
consumed:
1096 1097
		++cq->mcq.cons_index;
		index = (cq->mcq.cons_index) & ring->size_mask;
1098
		cqe = mlx4_en_get_cqe(cq->buf, index, priv->cqe_size) + factor;
1099
		if (++polled == budget)
1100 1101 1102 1103
			goto out;
	}

out:
1104
	rcu_read_unlock();
1105

1106 1107 1108 1109 1110 1111 1112 1113
	if (polled) {
		if (doorbell_pending)
			mlx4_en_xmit_doorbell(priv->tx_ring[TX_XDP][cq->ring]);

		mlx4_cq_set_ci(&cq->mcq);
		wmb(); /* ensure HW sees CQ consumer before we post new buffers */
		ring->cons = cq->mcq.cons_index;
	}
1114
	AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled);
1115 1116 1117 1118

	if (mlx4_en_refill_rx_buffers(priv, ring))
		mlx4_en_update_rx_prod_db(ring);

1119 1120 1121 1122 1123 1124 1125 1126 1127
	return polled;
}


void mlx4_en_rx_irq(struct mlx4_cq *mcq)
{
	struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
	struct mlx4_en_priv *priv = netdev_priv(cq->dev);

E
Eric Dumazet 已提交
1128 1129
	if (likely(priv->port_up))
		napi_schedule_irqoff(&cq->napi);
1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
	else
		mlx4_en_arm_cq(priv, cq);
}

/* Rx CQ polling - called by NAPI */
int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget)
{
	struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
	struct net_device *dev = cq->dev;
	struct mlx4_en_priv *priv = netdev_priv(dev);
	int done;

	done = mlx4_en_process_rx_cq(dev, cq, budget);

	/* If we used up all the quota - we're probably not done yet... */
1145
	if (done == budget) {
1146
		const struct cpumask *aff;
1147 1148
		struct irq_data *idata;
		int cpu_curr;
1149

1150
		INC_PERF_COUNTER(priv->pstats.napi_quota);
1151 1152

		cpu_curr = smp_processor_id();
1153 1154
		idata = irq_desc_get_irq_data(cq->irq_desc);
		aff = irq_data_get_affinity_mask(idata);
1155

1156 1157 1158 1159
		if (likely(cpumask_test_cpu(cpu_curr, aff)))
			return budget;

		/* Current cpu is not according to smp_irq_affinity -
1160 1161 1162 1163
		 * probably affinity changed. Need to stop this NAPI
		 * poll, and restart it on the right CPU.
		 * Try to avoid returning a too small value (like 0),
		 * to not fool net_rx_action() and its netdev_budget
1164
		 */
1165 1166
		if (done)
			done--;
1167
	}
E
Eric Dumazet 已提交
1168
	/* Done for now */
1169 1170
	if (napi_complete_done(napi, done))
		mlx4_en_arm_cq(priv, cq);
1171 1172 1173
	return done;
}

1174
static const int frag_sizes[] = {
1175 1176 1177 1178 1179 1180 1181 1182 1183
	FRAG_SZ0,
	FRAG_SZ1,
	FRAG_SZ2,
	FRAG_SZ3
};

void mlx4_en_calc_rx_buf(struct net_device *dev)
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
1184
	int eff_mtu = MLX4_EN_EFF_MTU(dev->mtu);
1185 1186
	int i = 0;

1187 1188 1189
	/* bpf requires buffers to be set up as 1 packet per page.
	 * This only works when num_frags == 1.
	 */
1190
	if (priv->tx_ring_num[TX_XDP]) {
1191 1192 1193 1194 1195
		priv->frag_info[0].order = 0;
		priv->frag_info[0].frag_size = eff_mtu;
		priv->frag_info[0].frag_prefix_size = 0;
		/* This will gain efficient xdp frame recycling at the
		 * expense of more costly truesize accounting
1196
		 */
1197 1198
		priv->frag_info[0].frag_stride = PAGE_SIZE;
		priv->frag_info[0].dma_dir = PCI_DMA_BIDIRECTIONAL;
1199
		priv->frag_info[0].rx_headroom = XDP_PACKET_HEADROOM;
1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
		i = 1;
	} else {
		int buf_size = 0;

		while (buf_size < eff_mtu) {
			priv->frag_info[i].order = MLX4_EN_ALLOC_PREFER_ORDER;
			priv->frag_info[i].frag_size =
				(eff_mtu > buf_size + frag_sizes[i]) ?
					frag_sizes[i] : eff_mtu - buf_size;
			priv->frag_info[i].frag_prefix_size = buf_size;
			priv->frag_info[i].frag_stride =
				ALIGN(priv->frag_info[i].frag_size,
				      SMP_CACHE_BYTES);
			priv->frag_info[i].dma_dir = PCI_DMA_FROMDEVICE;
1214
			priv->frag_info[i].rx_headroom = 0;
1215 1216 1217
			buf_size += priv->frag_info[i].frag_size;
			i++;
		}
1218 1219 1220 1221
	}

	priv->num_frags = i;
	priv->rx_skb_size = eff_mtu;
1222
	priv->log_rx_info = ROUNDUP_LOG2(i * sizeof(struct mlx4_en_rx_alloc));
1223

J
Joe Perches 已提交
1224 1225
	en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d num_frags:%d):\n",
	       eff_mtu, priv->num_frags);
1226
	for (i = 0; i < priv->num_frags; i++) {
1227
		en_err(priv,
1228
		       "  frag:%d - size:%d prefix:%d stride:%d\n",
1229 1230 1231 1232
		       i,
		       priv->frag_info[i].frag_size,
		       priv->frag_info[i].frag_prefix_size,
		       priv->frag_info[i].frag_stride);
1233 1234 1235 1236 1237
	}
}

/* RSS related functions */

1238 1239
static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn,
				 struct mlx4_en_rx_ring *ring,
1240 1241 1242 1243 1244 1245 1246
				 enum mlx4_qp_state *state,
				 struct mlx4_qp *qp)
{
	struct mlx4_en_dev *mdev = priv->mdev;
	struct mlx4_qp_context *context;
	int err = 0;

1247 1248
	context = kmalloc(sizeof(*context), GFP_KERNEL);
	if (!context)
1249 1250
		return -ENOMEM;

1251
	err = mlx4_qp_alloc(mdev->dev, qpn, qp, GFP_KERNEL);
1252
	if (err) {
1253
		en_err(priv, "Failed to allocate qp #%x\n", qpn);
1254 1255 1256 1257 1258
		goto out;
	}
	qp->event = mlx4_en_sqp_event;

	memset(context, 0, sizeof *context);
1259
	mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0,
1260
				qpn, ring->cqn, -1, context);
1261
	context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma);
1262

1263
	/* Cancel FCS removal if FW allows */
1264
	if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) {
1265
		context->param3 |= cpu_to_be32(1 << 29);
1266 1267 1268 1269
		if (priv->dev->features & NETIF_F_RXFCS)
			ring->fcs_del = 0;
		else
			ring->fcs_del = ETH_FCS_LEN;
1270 1271
	} else
		ring->fcs_del = 0;
1272

1273
	err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state);
1274 1275 1276 1277
	if (err) {
		mlx4_qp_remove(mdev->dev, qp);
		mlx4_qp_free(mdev->dev, qp);
	}
1278
	mlx4_en_update_rx_prod_db(ring);
1279 1280 1281 1282 1283
out:
	kfree(context);
	return err;
}

1284 1285 1286 1287 1288
int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv)
{
	int err;
	u32 qpn;

M
Matan Barak 已提交
1289 1290
	err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn,
				    MLX4_RESERVE_A0_QP);
1291 1292 1293 1294
	if (err) {
		en_err(priv, "Failed reserving drop qpn\n");
		return err;
	}
1295
	err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp, GFP_KERNEL);
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
	if (err) {
		en_err(priv, "Failed allocating drop qp\n");
		mlx4_qp_release_range(priv->mdev->dev, qpn, 1);
		return err;
	}

	return 0;
}

void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv)
{
	u32 qpn;

	qpn = priv->drop_qp.qpn;
	mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp);
	mlx4_qp_free(priv->mdev->dev, &priv->drop_qp);
	mlx4_qp_release_range(priv->mdev->dev, qpn, 1);
}

1315 1316 1317 1318 1319 1320
/* Allocate rx qp's and configure them according to rss map */
int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
{
	struct mlx4_en_dev *mdev = priv->mdev;
	struct mlx4_en_rss_map *rss_map = &priv->rss_map;
	struct mlx4_qp_context context;
1321
	struct mlx4_rss_context *rss_context;
1322
	int rss_rings;
1323
	void *ptr;
1324
	u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 |
1325
			MLX4_RSS_TCP_IPV6);
1326
	int i, qpn;
1327 1328 1329
	int err = 0;
	int good_qps = 0;

1330
	en_dbg(DRV, priv, "Configuring rss steering\n");
1331 1332
	err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,
				    priv->rx_ring_num,
1333
				    &rss_map->base_qpn, 0);
1334
	if (err) {
1335
		en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);
1336 1337 1338
		return err;
	}

1339
	for (i = 0; i < priv->rx_ring_num; i++) {
1340
		qpn = rss_map->base_qpn + i;
1341
		err = mlx4_en_config_rss_qp(priv, qpn, priv->rx_ring[i],
1342 1343 1344 1345 1346 1347 1348 1349 1350
					    &rss_map->state[i],
					    &rss_map->qps[i]);
		if (err)
			goto rss_err;

		++good_qps;
	}

	/* Configure RSS indirection qp */
1351
	err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp, GFP_KERNEL);
1352
	if (err) {
1353
		en_err(priv, "Failed to allocate RSS indirection QP\n");
1354
		goto rss_err;
1355 1356 1357
	}
	rss_map->indir_qp.event = mlx4_en_sqp_event;
	mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn,
1358
				priv->rx_ring[0]->cqn, -1, &context);
1359

1360 1361 1362 1363 1364
	if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num)
		rss_rings = priv->rx_ring_num;
	else
		rss_rings = priv->prof->rss_rings;

1365 1366
	ptr = ((void *) &context) + offsetof(struct mlx4_qp_context, pri_path)
					+ MLX4_RSS_OFFSET_IN_QPC_PRI_PATH;
1367
	rss_context = ptr;
1368
	rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 |
1369
					    (rss_map->base_qpn));
1370
	rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn);
1371 1372 1373 1374
	if (priv->mdev->profile.udp_rss) {
		rss_mask |=  MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6;
		rss_context->base_qpn_udp = rss_context->default_qpn;
	}
1375 1376 1377 1378 1379 1380

	if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
		en_info(priv, "Setting RSS context tunnel type to RSS on inner headers\n");
		rss_mask |= MLX4_RSS_BY_INNER_HEADERS;
	}

Y
Yevgeny Petrilin 已提交
1381
	rss_context->flags = rss_mask;
1382
	rss_context->hash_fn = MLX4_RSS_HASH_TOP;
1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393
	if (priv->rss_hash_fn == ETH_RSS_HASH_XOR) {
		rss_context->hash_fn = MLX4_RSS_HASH_XOR;
	} else if (priv->rss_hash_fn == ETH_RSS_HASH_TOP) {
		rss_context->hash_fn = MLX4_RSS_HASH_TOP;
		memcpy(rss_context->rss_key, priv->rss_key,
		       MLX4_EN_RSS_KEY_SIZE);
	} else {
		en_err(priv, "Unknown RSS hash function requested\n");
		err = -EINVAL;
		goto indir_err;
	}
1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412
	err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context,
			       &rss_map->indir_qp, &rss_map->indir_state);
	if (err)
		goto indir_err;

	return 0;

indir_err:
	mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
		       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
	mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
	mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
rss_err:
	for (i = 0; i < good_qps; i++) {
		mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
			       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
		mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
		mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
	}
1413
	mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
	return err;
}

void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv)
{
	struct mlx4_en_dev *mdev = priv->mdev;
	struct mlx4_en_rss_map *rss_map = &priv->rss_map;
	int i;

	mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
		       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
	mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
	mlx4_qp_free(mdev->dev, &rss_map->indir_qp);

1428
	for (i = 0; i < priv->rx_ring_num; i++) {
1429 1430 1431 1432 1433
		mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
			       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
		mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
		mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
	}
1434
	mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
1435
}