en_rx.c 39.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
/*
 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

34
#include <net/busy_poll.h>
35
#include <linux/bpf.h>
36
#include <linux/bpf_trace.h>
37
#include <linux/mlx4/cq.h>
38
#include <linux/slab.h>
39 40
#include <linux/mlx4/qp.h>
#include <linux/skbuff.h>
41
#include <linux/rculist.h>
42 43 44
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <linux/vmalloc.h>
45
#include <linux/irq.h>
46

47 48 49 50
#if IS_ENABLED(CONFIG_IPV6)
#include <net/ip6_checksum.h>
#endif

51 52
#include "mlx4_en.h"

53 54 55 56 57 58 59 60 61
static int mlx4_alloc_pages(struct mlx4_en_priv *priv,
			    struct mlx4_en_rx_alloc *page_alloc,
			    const struct mlx4_en_frag_info *frag_info,
			    gfp_t _gfp)
{
	int order;
	struct page *page;
	dma_addr_t dma;

62
	for (order = priv->rx_page_order; ;) {
63 64 65
		gfp_t gfp = _gfp;

		if (order)
66
			gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NOMEMALLOC;
67 68 69 70 71 72 73 74
		page = alloc_pages(gfp, order);
		if (likely(page))
			break;
		if (--order < 0 ||
		    ((PAGE_SIZE << order) < frag_info->frag_size))
			return -ENOMEM;
	}
	dma = dma_map_page(priv->ddev, page, 0, PAGE_SIZE << order,
75
			   priv->dma_dir);
76
	if (unlikely(dma_mapping_error(priv->ddev, dma))) {
77 78 79
		put_page(page);
		return -ENOMEM;
	}
80
	page_alloc->page_size = PAGE_SIZE << order;
81 82
	page_alloc->page = page;
	page_alloc->dma = dma;
83
	page_alloc->page_offset = 0;
84
	/* Not doing get_page() for each frag is a big win
85
	 * on asymetric workloads. Note we can not use atomic_set().
86
	 */
87
	page_ref_add(page, page_alloc->page_size / frag_info->frag_stride - 1);
88 89 90
	return 0;
}

91 92 93
static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv,
			       struct mlx4_en_rx_desc *rx_desc,
			       struct mlx4_en_rx_alloc *frags,
94 95
			       struct mlx4_en_rx_alloc *ring_alloc,
			       gfp_t gfp)
96
{
97
	struct mlx4_en_rx_alloc page_alloc[MLX4_EN_MAX_RX_FRAGS];
98
	const struct mlx4_en_frag_info *frag_info;
99
	struct page *page;
100
	int i;
101

102 103
	for (i = 0; i < priv->num_frags; i++) {
		frag_info = &priv->frag_info[i];
104
		page_alloc[i] = ring_alloc[i];
105 106 107 108
		page_alloc[i].page_offset += frag_info->frag_stride;

		if (page_alloc[i].page_offset + frag_info->frag_stride <=
		    ring_alloc[i].page_size)
109
			continue;
110

111 112
		if (unlikely(mlx4_alloc_pages(priv, &page_alloc[i],
					      frag_info, gfp)))
113
			goto out;
114
	}
115

116 117
	for (i = 0; i < priv->num_frags; i++) {
		frags[i] = ring_alloc[i];
118 119 120
		frags[i].page_offset += priv->frag_info[i].rx_headroom;
		rx_desc->data[i].addr = cpu_to_be64(frags[i].dma +
						    frags[i].page_offset);
121
		ring_alloc[i] = page_alloc[i];
122
	}
123

124
	return 0;
125 126 127

out:
	while (i--) {
128
		if (page_alloc[i].page != ring_alloc[i].page) {
129
			dma_unmap_page(priv->ddev, page_alloc[i].dma,
130
				page_alloc[i].page_size,
131
				priv->dma_dir);
132
			page = page_alloc[i].page;
133 134 135
			/* Revert changes done by mlx4_alloc_pages */
			page_ref_sub(page, page_alloc[i].page_size /
					   priv->frag_info[i].frag_stride - 1);
136 137
			put_page(page);
		}
138 139 140 141 142 143 144 145
	}
	return -ENOMEM;
}

static void mlx4_en_free_frag(struct mlx4_en_priv *priv,
			      struct mlx4_en_rx_alloc *frags,
			      int i)
{
146
	const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
147
	u32 next_frag_end = frags[i].page_offset + 2 * frag_info->frag_stride;
148

149 150

	if (next_frag_end > frags[i].page_size)
151
		dma_unmap_page(priv->ddev, frags[i].dma, frags[i].page_size,
152
			       priv->dma_dir);
153

154 155
	if (frags[i].page)
		put_page(frags[i].page);
156 157 158 159 160 161
}

static int mlx4_en_init_allocator(struct mlx4_en_priv *priv,
				  struct mlx4_en_rx_ring *ring)
{
	int i;
162
	struct mlx4_en_rx_alloc *page_alloc;
163 164

	for (i = 0; i < priv->num_frags; i++) {
165
		const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
166

167
		if (mlx4_alloc_pages(priv, &ring->page_alloc[i],
168
				     frag_info, GFP_KERNEL | __GFP_COLD))
169
			goto out;
170 171 172

		en_dbg(DRV, priv, "  frag %d allocator: - size:%d frags:%d\n",
		       i, ring->page_alloc[i].page_size,
173
		       page_ref_count(ring->page_alloc[i].page));
174 175 176 177 178
	}
	return 0;

out:
	while (i--) {
179 180
		struct page *page;

181
		page_alloc = &ring->page_alloc[i];
182
		dma_unmap_page(priv->ddev, page_alloc->dma,
183
			       page_alloc->page_size,
184
			       priv->dma_dir);
185
		page = page_alloc->page;
186 187 188
		/* Revert changes done by mlx4_alloc_pages */
		page_ref_sub(page, page_alloc->page_size /
				   priv->frag_info[i].frag_stride - 1);
189
		put_page(page);
190 191 192 193 194 195 196 197 198 199 200 201
		page_alloc->page = NULL;
	}
	return -ENOMEM;
}

static void mlx4_en_destroy_allocator(struct mlx4_en_priv *priv,
				      struct mlx4_en_rx_ring *ring)
{
	struct mlx4_en_rx_alloc *page_alloc;
	int i;

	for (i = 0; i < priv->num_frags; i++) {
202 203
		const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];

204
		page_alloc = &ring->page_alloc[i];
205 206
		en_dbg(DRV, priv, "Freeing allocator:%d count:%d\n",
		       i, page_count(page_alloc->page));
207

208
		dma_unmap_page(priv->ddev, page_alloc->dma,
209
				page_alloc->page_size, priv->dma_dir);
210 211
		while (page_alloc->page_offset + frag_info->frag_stride <
		       page_alloc->page_size) {
212
			put_page(page_alloc->page);
213
			page_alloc->page_offset += frag_info->frag_stride;
214
		}
215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
		page_alloc->page = NULL;
	}
}

static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv,
				 struct mlx4_en_rx_ring *ring, int index)
{
	struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index;
	int possible_frags;
	int i;

	/* Set size and memtype fields */
	for (i = 0; i < priv->num_frags; i++) {
		rx_desc->data[i].byte_count =
			cpu_to_be32(priv->frag_info[i].frag_size);
		rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key);
	}

	/* If the number of used fragments does not fill up the ring stride,
	 * remaining (unused) fragments must be padded with null address/size
	 * and a special memory key */
	possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE;
	for (i = priv->num_frags; i < possible_frags; i++) {
		rx_desc->data[i].byte_count = 0;
		rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD);
		rx_desc->data[i].addr = 0;
	}
}

static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv,
245 246
				   struct mlx4_en_rx_ring *ring, int index,
				   gfp_t gfp)
247 248
{
	struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride);
249 250
	struct mlx4_en_rx_alloc *frags = ring->rx_info +
					(index << priv->log_rx_info);
251

252 253
	if (ring->page_cache.index > 0) {
		frags[0] = ring->page_cache.buf[--ring->page_cache.index];
254 255
		rx_desc->data[0].addr = cpu_to_be64(frags[0].dma +
						    frags[0].page_offset);
256 257 258
		return 0;
	}

259
	return mlx4_en_alloc_frags(priv, rx_desc, frags, ring->page_alloc, gfp);
260 261
}

262 263 264 265 266
static inline bool mlx4_en_is_ring_empty(struct mlx4_en_rx_ring *ring)
{
	return ring->prod == ring->cons;
}

267 268 269 270 271
static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring)
{
	*ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff);
}

272 273 274 275
static void mlx4_en_free_rx_desc(struct mlx4_en_priv *priv,
				 struct mlx4_en_rx_ring *ring,
				 int index)
{
276
	struct mlx4_en_rx_alloc *frags;
277 278
	int nr;

279
	frags = ring->rx_info + (index << priv->log_rx_info);
280
	for (nr = 0; nr < priv->num_frags; nr++) {
281
		en_dbg(DRV, priv, "Freeing fragment:%d\n", nr);
282
		mlx4_en_free_frag(priv, frags, nr);
283 284 285
	}
}

286 287 288 289 290
static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv)
{
	struct mlx4_en_rx_ring *ring;
	int ring_ind;
	int buf_ind;
291
	int new_size;
292 293 294

	for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) {
		for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
295
			ring = priv->rx_ring[ring_ind];
296 297

			if (mlx4_en_prepare_rx_desc(priv, ring,
298
						    ring->actual_size,
299
						    GFP_KERNEL | __GFP_COLD)) {
300
				if (ring->actual_size < MLX4_EN_MIN_RX_SIZE) {
J
Joe Perches 已提交
301
					en_err(priv, "Failed to allocate enough rx buffers\n");
302 303
					return -ENOMEM;
				} else {
304
					new_size = rounddown_pow_of_two(ring->actual_size);
J
Joe Perches 已提交
305
					en_warn(priv, "Only %d buffers allocated reducing ring size to %d\n",
306
						ring->actual_size, new_size);
307
					goto reduce_rings;
308 309 310 311 312 313
				}
			}
			ring->actual_size++;
			ring->prod++;
		}
	}
314 315 316 317
	return 0;

reduce_rings:
	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
318
		ring = priv->rx_ring[ring_ind];
319 320 321 322 323 324 325
		while (ring->actual_size > new_size) {
			ring->actual_size--;
			ring->prod--;
			mlx4_en_free_rx_desc(priv, ring, ring->actual_size);
		}
	}

326 327 328 329 330 331 332 333
	return 0;
}

static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv,
				struct mlx4_en_rx_ring *ring)
{
	int index;

334 335
	en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n",
	       ring->cons, ring->prod);
336 337

	/* Unmap and free Rx buffers */
338
	while (!mlx4_en_is_ring_empty(ring)) {
339
		index = ring->cons & ring->size_mask;
340
		en_dbg(DRV, priv, "Processing descriptor:%d\n", index);
341
		mlx4_en_free_rx_desc(priv, ring, index);
342 343 344 345
		++ring->cons;
	}
}

346 347 348 349
void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev)
{
	int i;
	int num_of_eqs;
350
	int num_rx_rings;
351 352 353
	struct mlx4_dev *dev = mdev->dev;

	mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) {
M
Matan Barak 已提交
354 355 356 357
		num_of_eqs = max_t(int, MIN_RX_RINGS,
				   min_t(int,
					 mlx4_get_eqs_per_port(mdev->dev, i),
					 DEF_RX_RINGS));
358

359 360 361
		num_rx_rings = mlx4_low_memory_profile() ? MIN_RX_RINGS :
			min_t(int, num_of_eqs,
			      netif_get_num_default_rss_queues());
362
		mdev->profile.prof[i].rx_ring_num =
363
			rounddown_pow_of_two(num_rx_rings);
364 365 366
	}
}

367
int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
368
			   struct mlx4_en_rx_ring **pring,
369
			   u32 size, u16 stride, int node)
370 371
{
	struct mlx4_en_dev *mdev = priv->mdev;
372
	struct mlx4_en_rx_ring *ring;
373
	int err = -ENOMEM;
374 375
	int tmp;

376
	ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node);
377
	if (!ring) {
378 379 380 381 382
		ring = kzalloc(sizeof(*ring), GFP_KERNEL);
		if (!ring) {
			en_err(priv, "Failed to allocate RX ring structure\n");
			return -ENOMEM;
		}
383 384
	}

385 386 387 388 389 390
	ring->prod = 0;
	ring->cons = 0;
	ring->size = size;
	ring->size_mask = size - 1;
	ring->stride = stride;
	ring->log_stride = ffs(ring->stride) - 1;
391
	ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
392 393

	tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
394
					sizeof(struct mlx4_en_rx_alloc));
395
	ring->rx_info = vmalloc_node(tmp, node);
396
	if (!ring->rx_info) {
397 398 399 400 401
		ring->rx_info = vmalloc(tmp);
		if (!ring->rx_info) {
			err = -ENOMEM;
			goto err_ring;
		}
402
	}
403

404
	en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n",
405 406
		 ring->rx_info, tmp);

407
	/* Allocate HW buffers on provided NUMA node */
408
	set_dev_node(&mdev->dev->persist->pdev->dev, node);
409
	err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
410
	set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
411
	if (err)
412
		goto err_info;
413 414 415

	ring->buf = ring->wqres.buf.direct.buf;

416 417
	ring->hwtstamp_rx_filter = priv->hwtstamp_config.rx_filter;

418
	*pring = ring;
419 420
	return 0;

421
err_info:
422 423
	vfree(ring->rx_info);
	ring->rx_info = NULL;
424 425 426 427
err_ring:
	kfree(ring);
	*pring = NULL;

428 429 430 431 432 433 434 435 436 437 438 439 440
	return err;
}

int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
{
	struct mlx4_en_rx_ring *ring;
	int i;
	int ring_ind;
	int err;
	int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
					DS_SIZE * priv->num_frags);

	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
441
		ring = priv->rx_ring[ring_ind];
442 443 444 445

		ring->prod = 0;
		ring->cons = 0;
		ring->actual_size = 0;
446
		ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn;
447 448

		ring->stride = stride;
E
Eugenia Emantayev 已提交
449 450 451 452 453 454
		if (ring->stride <= TXBB_SIZE) {
			/* Stamp first unused send wqe */
			__be32 *ptr = (__be32 *)ring->buf;
			__be32 stamp = cpu_to_be32(1 << STAMP_SHIFT);
			*ptr = stamp;
			/* Move pointer to start of rx section */
455
			ring->buf += TXBB_SIZE;
E
Eugenia Emantayev 已提交
456
		}
457

458 459 460 461 462 463
		ring->log_stride = ffs(ring->stride) - 1;
		ring->buf_size = ring->size * ring->stride;

		memset(ring->buf, 0, ring->buf_size);
		mlx4_en_update_rx_prod_db(ring);

464
		/* Initialize all descriptors */
465 466 467 468 469 470
		for (i = 0; i < ring->size; i++)
			mlx4_en_init_rx_desc(priv, ring, i);

		/* Initialize page allocators */
		err = mlx4_en_init_allocator(priv, ring);
		if (err) {
471
			en_err(priv, "Failed initializing ring allocator\n");
472 473
			if (ring->stride <= TXBB_SIZE)
				ring->buf -= TXBB_SIZE;
474 475
			ring_ind--;
			goto err_allocator;
476 477
		}
	}
478 479
	err = mlx4_en_fill_rx_buffers(priv);
	if (err)
480 481 482
		goto err_buffers;

	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
483
		ring = priv->rx_ring[ring_ind];
484

485
		ring->size_mask = ring->actual_size - 1;
486 487 488 489 490 491 492
		mlx4_en_update_rx_prod_db(ring);
	}

	return 0;

err_buffers:
	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++)
493
		mlx4_en_free_rx_buf(priv, priv->rx_ring[ring_ind]);
494 495 496 497

	ring_ind = priv->rx_ring_num - 1;
err_allocator:
	while (ring_ind >= 0) {
498 499 500
		if (priv->rx_ring[ring_ind]->stride <= TXBB_SIZE)
			priv->rx_ring[ring_ind]->buf -= TXBB_SIZE;
		mlx4_en_destroy_allocator(priv, priv->rx_ring[ring_ind]);
501 502 503 504 505
		ring_ind--;
	}
	return err;
}

506 507 508 509 510 511 512 513 514 515 516 517
/* We recover from out of memory by scheduling our napi poll
 * function (mlx4_en_process_cq), which tries to allocate
 * all missing RX buffers (call to mlx4_en_refill_rx_buffers).
 */
void mlx4_en_recover_from_oom(struct mlx4_en_priv *priv)
{
	int ring;

	if (!priv->port_up)
		return;

	for (ring = 0; ring < priv->rx_ring_num; ring++) {
518 519
		if (mlx4_en_is_ring_empty(priv->rx_ring[ring])) {
			local_bh_disable();
520
			napi_reschedule(&priv->rx_cq[ring]->napi);
521 522
			local_bh_enable();
		}
523 524 525
	}
}

526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
/* When the rx ring is running in page-per-packet mode, a released frame can go
 * directly into a small cache, to avoid unmapping or touching the page
 * allocator. In bpf prog performance scenarios, buffers are either forwarded
 * or dropped, never converted to skbs, so every page can come directly from
 * this cache when it is sized to be a multiple of the napi budget.
 */
bool mlx4_en_rx_recycle(struct mlx4_en_rx_ring *ring,
			struct mlx4_en_rx_alloc *frame)
{
	struct mlx4_en_page_cache *cache = &ring->page_cache;

	if (cache->index >= MLX4_EN_CACHE_SIZE)
		return false;

	cache->buf[cache->index++] = *frame;
	return true;
}

544
void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
545 546
			     struct mlx4_en_rx_ring **pring,
			     u32 size, u16 stride)
547 548
{
	struct mlx4_en_dev *mdev = priv->mdev;
549
	struct mlx4_en_rx_ring *ring = *pring;
550
	struct bpf_prog *old_prog;
551

552 553 554
	old_prog = rcu_dereference_protected(
					ring->xdp_prog,
					lockdep_is_held(&mdev->state_lock));
555 556
	if (old_prog)
		bpf_prog_put(old_prog);
557
	mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE);
558 559
	vfree(ring->rx_info);
	ring->rx_info = NULL;
560 561
	kfree(ring);
	*pring = NULL;
562 563 564 565 566
}

void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
				struct mlx4_en_rx_ring *ring)
{
567 568 569 570 571 572
	int i;

	for (i = 0; i < ring->page_cache.index; i++) {
		struct mlx4_en_rx_alloc *frame = &ring->page_cache.buf[i];

		dma_unmap_page(priv->ddev, frame->dma, frame->page_size,
573
			       priv->dma_dir);
574 575 576
		put_page(frame->page);
	}
	ring->page_cache.index = 0;
577
	mlx4_en_free_rx_buf(priv, ring);
578 579
	if (ring->stride <= TXBB_SIZE)
		ring->buf -= TXBB_SIZE;
580 581 582 583 584 585
	mlx4_en_destroy_allocator(priv, ring);
}


static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv,
				    struct mlx4_en_rx_desc *rx_desc,
586
				    struct mlx4_en_rx_alloc *frags,
587
				    struct sk_buff *skb,
588 589
				    int length)
{
590
	struct skb_frag_struct *skb_frags_rx = skb_shinfo(skb)->frags;
E
Eric Dumazet 已提交
591 592
	struct mlx4_en_frag_info *frag_info = priv->frag_info;
	int nr, frag_size;
593 594
	dma_addr_t dma;

595
	/* Collect used fragments while replacing them in the HW descriptors */
E
Eric Dumazet 已提交
596 597 598
	for (nr = 0;;) {
		frag_size = min_t(int, length, frag_info->frag_size);

599
		if (unlikely(!frags[nr].page))
600
			goto fail;
601 602

		dma = be64_to_cpu(rx_desc->data[nr].addr);
603 604
		dma_sync_single_for_cpu(priv->ddev, dma, frag_info->frag_size,
					DMA_FROM_DEVICE);
605

606 607
		__skb_fill_page_desc(skb, nr, frags[nr].page,
				     frags[nr].page_offset,
E
Eric Dumazet 已提交
608
				     frag_size);
609

610
		skb->truesize += frag_info->frag_stride;
611
		frags[nr].page = NULL;
E
Eric Dumazet 已提交
612 613 614 615 616
		nr++;
		length -= frag_size;
		if (!length)
			break;
		frag_info++;
617 618 619 620 621 622
	}
	return nr;

fail:
	while (nr > 0) {
		nr--;
623
		__skb_frag_unref(&skb_frags_rx[nr]);
624 625 626 627 628 629 630
	}
	return 0;
}


static struct sk_buff *mlx4_en_rx_skb(struct mlx4_en_priv *priv,
				      struct mlx4_en_rx_desc *rx_desc,
631
				      struct mlx4_en_rx_alloc *frags,
632 633 634 635 636 637 638
				      unsigned int length)
{
	struct sk_buff *skb;
	void *va;
	int used_frags;
	dma_addr_t dma;

639
	skb = netdev_alloc_skb(priv->dev, SMALL_PACKET_SIZE + NET_IP_ALIGN);
640
	if (unlikely(!skb)) {
641
		en_dbg(RX_ERR, priv, "Failed allocating skb\n");
642 643 644 645 646 647 648
		return NULL;
	}
	skb_reserve(skb, NET_IP_ALIGN);
	skb->len = length;

	/* Get pointer to first fragment so we could copy the headers into the
	 * (linear part of the) skb */
649
	va = page_address(frags[0].page) + frags[0].page_offset;
650 651 652

	if (length <= SMALL_PACKET_SIZE) {
		/* We are copying all relevant data to the skb - temporarily
653
		 * sync buffers for the copy */
654
		dma = be64_to_cpu(rx_desc->data[0].addr);
655
		dma_sync_single_for_cpu(priv->ddev, dma, length,
656
					DMA_FROM_DEVICE);
657 658 659
		skb_copy_to_linear_data(skb, va, length);
		skb->tail += length;
	} else {
660 661
		unsigned int pull_len;

662
		/* Move relevant fragments to skb */
663 664
		used_frags = mlx4_en_complete_rx_desc(priv, rx_desc, frags,
							skb, length);
665 666 667 668
		if (unlikely(!used_frags)) {
			kfree_skb(skb);
			return NULL;
		}
669 670
		skb_shinfo(skb)->nr_frags = used_frags;

671
		pull_len = eth_get_headlen(va, SMALL_PACKET_SIZE);
672
		/* Copy headers into the skb linear buffer */
673 674
		memcpy(skb->data, va, pull_len);
		skb->tail += pull_len;
675 676

		/* Skip headers in first fragment */
677
		skb_shinfo(skb)->frags[0].page_offset += pull_len;
678 679

		/* Adjust size of first fragment */
680 681
		skb_frag_size_sub(&skb_shinfo(skb)->frags[0], pull_len);
		skb->data_len = length - pull_len;
682 683 684 685
	}
	return skb;
}

686 687 688 689 690 691 692 693 694 695 696 697 698 699 700
static void validate_loopback(struct mlx4_en_priv *priv, struct sk_buff *skb)
{
	int i;
	int offset = ETH_HLEN;

	for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) {
		if (*(skb->data + offset) != (unsigned char) (i & 0xff))
			goto out_loopback;
	}
	/* Loopback found */
	priv->loopback_ok = 1;

out_loopback:
	dev_kfree_skb_any(skb);
}
701

702 703
static bool mlx4_en_refill_rx_buffers(struct mlx4_en_priv *priv,
				      struct mlx4_en_rx_ring *ring)
704
{
705
	u32 missing = ring->actual_size - (ring->prod - ring->cons);
706

707 708 709 710 711 712
	/* Try to batch allocations, but not too much. */
	if (missing < 8)
		return false;
	do {
		if (mlx4_en_prepare_rx_desc(priv, ring,
					    ring->prod & ring->size_mask,
713 714
					    GFP_ATOMIC | __GFP_COLD |
					    __GFP_MEMALLOC))
715 716
			break;
		ring->prod++;
717 718 719
	} while (--missing);

	return true;
720 721
}

722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756
/* When hardware doesn't strip the vlan, we need to calculate the checksum
 * over it and add it to the hardware's checksum calculation
 */
static inline __wsum get_fixed_vlan_csum(__wsum hw_checksum,
					 struct vlan_hdr *vlanh)
{
	return csum_add(hw_checksum, *(__wsum *)vlanh);
}

/* Although the stack expects checksum which doesn't include the pseudo
 * header, the HW adds it. To address that, we are subtracting the pseudo
 * header checksum from the checksum value provided by the HW.
 */
static void get_fixed_ipv4_csum(__wsum hw_checksum, struct sk_buff *skb,
				struct iphdr *iph)
{
	__u16 length_for_csum = 0;
	__wsum csum_pseudo_header = 0;

	length_for_csum = (be16_to_cpu(iph->tot_len) - (iph->ihl << 2));
	csum_pseudo_header = csum_tcpudp_nofold(iph->saddr, iph->daddr,
						length_for_csum, iph->protocol, 0);
	skb->csum = csum_sub(hw_checksum, csum_pseudo_header);
}

#if IS_ENABLED(CONFIG_IPV6)
/* In IPv6 packets, besides subtracting the pseudo header checksum,
 * we also compute/add the IP header checksum which
 * is not added by the HW.
 */
static int get_fixed_ipv6_csum(__wsum hw_checksum, struct sk_buff *skb,
			       struct ipv6hdr *ipv6h)
{
	__wsum csum_pseudo_hdr = 0;

757 758
	if (unlikely(ipv6h->nexthdr == IPPROTO_FRAGMENT ||
		     ipv6h->nexthdr == IPPROTO_HOPOPTS))
759
		return -1;
760
	hw_checksum = csum_add(hw_checksum, (__force __wsum)htons(ipv6h->nexthdr));
761 762 763 764 765 766 767 768 769 770 771 772

	csum_pseudo_hdr = csum_partial(&ipv6h->saddr,
				       sizeof(ipv6h->saddr) + sizeof(ipv6h->daddr), 0);
	csum_pseudo_hdr = csum_add(csum_pseudo_hdr, (__force __wsum)ipv6h->payload_len);
	csum_pseudo_hdr = csum_add(csum_pseudo_hdr, (__force __wsum)ntohs(ipv6h->nexthdr));

	skb->csum = csum_sub(hw_checksum, csum_pseudo_hdr);
	skb->csum = csum_add(skb->csum, csum_partial(ipv6h, sizeof(struct ipv6hdr), 0));
	return 0;
}
#endif
static int check_csum(struct mlx4_cqe *cqe, struct sk_buff *skb, void *va,
773
		      netdev_features_t dev_features)
774 775 776 777 778 779 780
{
	__wsum hw_checksum = 0;

	void *hdr = (u8 *)va + sizeof(struct ethhdr);

	hw_checksum = csum_unfold((__force __sum16)cqe->checksum);

781
	if (cqe->vlan_my_qpn & cpu_to_be32(MLX4_CQE_CVLAN_PRESENT_MASK) &&
782
	    !(dev_features & NETIF_F_HW_VLAN_CTAG_RX)) {
783 784 785 786 787 788 789 790
		hw_checksum = get_fixed_vlan_csum(hw_checksum, hdr);
		hdr += sizeof(struct vlan_hdr);
	}

	if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV4))
		get_fixed_ipv4_csum(hw_checksum, skb, hdr);
#if IS_ENABLED(CONFIG_IPV6)
	else if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV6))
791
		if (unlikely(get_fixed_ipv6_csum(hw_checksum, skb, hdr)))
792 793 794 795 796
			return -1;
#endif
	return 0;
}

797 798 799
int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget)
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
800
	struct mlx4_en_dev *mdev = priv->mdev;
801
	struct mlx4_cqe *cqe;
802
	struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring];
803
	struct mlx4_en_rx_alloc *frags;
804
	struct mlx4_en_rx_desc *rx_desc;
805
	struct bpf_prog *xdp_prog;
806
	int doorbell_pending;
807 808 809 810 811 812
	struct sk_buff *skb;
	int index;
	int nr;
	unsigned int length;
	int polled = 0;
	int ip_summed;
O
Or Gerlitz 已提交
813
	int factor = priv->cqe_factor;
814
	u64 timestamp;
815
	bool l2_tunnel;
816

817
	if (unlikely(!priv->port_up))
818 819
		return 0;

820
	if (unlikely(budget <= 0))
821 822
		return polled;

823 824 825
	/* Protect accesses to: ring->xdp_prog, priv->mac_hash list */
	rcu_read_lock();
	xdp_prog = rcu_dereference(ring->xdp_prog);
826
	doorbell_pending = 0;
827

828 829 830 831
	/* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
	 * descriptor offset can be deduced from the CQE index instead of
	 * reading 'cqe->index' */
	index = cq->mcq.cons_index & ring->size_mask;
832
	cqe = mlx4_en_get_cqe(cq->buf, index, priv->cqe_size) + factor;
833 834 835 836 837

	/* Process all completed CQEs */
	while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
		    cq->mcq.cons_index & cq->size)) {

838
		frags = ring->rx_info + (index << priv->log_rx_info);
839 840 841 842 843
		rx_desc = ring->buf + (index << ring->log_stride);

		/*
		 * make sure we read the CQE after we read the ownership bit
		 */
844
		dma_rmb();
845 846 847 848

		/* Drop packet on bad receive or bad checksum */
		if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
						MLX4_CQE_OPCODE_ERROR)) {
J
Joe Perches 已提交
849 850 851
			en_err(priv, "CQE completed in error - vendor syndrom:%d syndrom:%d\n",
			       ((struct mlx4_err_cqe *)cqe)->vendor_err_syndrome,
			       ((struct mlx4_err_cqe *)cqe)->syndrome);
852 853 854
			goto next;
		}
		if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {
855
			en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");
856 857 858
			goto next;
		}

859 860 861 862 863 864 865 866 867 868 869 870 871
		/* Check if we need to drop the packet if SRIOV is not enabled
		 * and not performing the selftest or flb disabled
		 */
		if (priv->flags & MLX4_EN_FLAG_RX_FILTER_NEEDED) {
			struct ethhdr *ethh;
			dma_addr_t dma;
			/* Get pointer to first fragment since we haven't
			 * skb yet and cast it to ethhdr struct
			 */
			dma = be64_to_cpu(rx_desc->data[0].addr);
			dma_sync_single_for_cpu(priv->ddev, dma, sizeof(*ethh),
						DMA_FROM_DEVICE);
			ethh = (struct ethhdr *)(page_address(frags[0].page) +
872
						 frags[0].page_offset);
873

874 875 876 877 878 879 880 881
			if (is_multicast_ether_addr(ethh->h_dest)) {
				struct mlx4_mac_entry *entry;
				struct hlist_head *bucket;
				unsigned int mac_hash;

				/* Drop the packet, since HW loopback-ed it */
				mac_hash = ethh->h_source[MLX4_EN_MAC_HASH_IDX];
				bucket = &priv->mac_hash[mac_hash];
882
				hlist_for_each_entry_rcu(entry, bucket, hlist) {
883
					if (ether_addr_equal_64bits(entry->mac,
884
								    ethh->h_source))
885 886 887
						goto next;
				}
			}
888
		}
889

890 891 892 893
		/*
		 * Packet is OK - process it.
		 */
		length = be32_to_cpu(cqe->byte_cnt);
894
		length -= ring->fcs_del;
895 896
		l2_tunnel = (dev->hw_enc_features & NETIF_F_RXCSUM) &&
			(cqe->vlan_my_qpn & cpu_to_be32(MLX4_CQE_L2_TUNNEL));
897

898 899 900 901 902 903
		/* A bpf program gets first chance to drop the packet. It may
		 * read bytes but not past the end of the frag.
		 */
		if (xdp_prog) {
			struct xdp_buff xdp;
			dma_addr_t dma;
904
			void *orig_data;
905 906 907 908 909 910 911
			u32 act;

			dma = be64_to_cpu(rx_desc->data[0].addr);
			dma_sync_single_for_cpu(priv->ddev, dma,
						priv->frag_info[0].frag_size,
						DMA_FROM_DEVICE);

912 913
			xdp.data_hard_start = page_address(frags[0].page);
			xdp.data = xdp.data_hard_start + frags[0].page_offset;
914
			xdp.data_end = xdp.data + length;
915
			orig_data = xdp.data;
916 917

			act = bpf_prog_run_xdp(xdp_prog, &xdp);
918 919 920 921 922 923 924

			if (xdp.data != orig_data) {
				length = xdp.data_end - xdp.data;
				frags[0].page_offset = xdp.data -
					xdp.data_hard_start;
			}

925 926 927
			switch (act) {
			case XDP_PASS:
				break;
928
			case XDP_TX:
929
				if (likely(!mlx4_en_xmit_frame(ring, frags, dev,
930
							length, cq->ring,
931
							&doorbell_pending)))
932
					goto consumed;
933
				trace_xdp_exception(dev, xdp_prog, act);
934
				goto xdp_drop_no_cnt; /* Drop on xmit failure */
935 936 937
			default:
				bpf_warn_invalid_xdp_action(act);
			case XDP_ABORTED:
938
				trace_xdp_exception(dev, xdp_prog, act);
939
			case XDP_DROP:
940 941
				ring->xdp_drop++;
xdp_drop_no_cnt:
942
				if (likely(mlx4_en_rx_recycle(ring, frags)))
943
					goto consumed;
944 945 946 947
				goto next;
			}
		}

948 949 950
		ring->bytes += length;
		ring->packets++;

951
		if (likely(dev->features & NETIF_F_RXCSUM)) {
952 953 954 955 956 957 958 959 960 961
			if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_TCP |
						      MLX4_CQE_STATUS_UDP)) {
				if ((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
				    cqe->checksum == cpu_to_be16(0xffff)) {
					ip_summed = CHECKSUM_UNNECESSARY;
					ring->csum_ok++;
				} else {
					ip_summed = CHECKSUM_NONE;
					ring->csum_none++;
				}
962
			} else {
963 964 965 966 967 968 969 970 971
				if (priv->flags & MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP &&
				    (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 |
							       MLX4_CQE_STATUS_IPV6))) {
					ip_summed = CHECKSUM_COMPLETE;
					ring->csum_complete++;
				} else {
					ip_summed = CHECKSUM_NONE;
					ring->csum_none++;
				}
972 973 974
			}
		} else {
			ip_summed = CHECKSUM_NONE;
975
			ring->csum_none++;
976 977
		}

978 979 980 981 982 983
		/* This packet is eligible for GRO if it is:
		 * - DIX Ethernet (type interpretation)
		 * - TCP/IP (v4)
		 * - without IP options
		 * - not an IP fragment
		 */
984
		if (dev->features & NETIF_F_GRO) {
985 986 987 988 989 990 991 992 993 994
			struct sk_buff *gro_skb = napi_get_frags(&cq->napi);
			if (!gro_skb)
				goto next;

			nr = mlx4_en_complete_rx_desc(priv,
				rx_desc, frags, gro_skb,
				length);
			if (!nr)
				goto next;

995 996
			if (ip_summed == CHECKSUM_COMPLETE) {
				void *va = skb_frag_address(skb_shinfo(gro_skb)->frags);
997 998
				if (check_csum(cqe, gro_skb, va,
					       dev->features)) {
999 1000 1001 1002 1003 1004
					ip_summed = CHECKSUM_NONE;
					ring->csum_none++;
					ring->csum_complete--;
				}
			}

1005 1006 1007 1008 1009 1010
			skb_shinfo(gro_skb)->nr_frags = nr;
			gro_skb->len = length;
			gro_skb->data_len = length;
			gro_skb->ip_summed = ip_summed;

			if (l2_tunnel && ip_summed == CHECKSUM_UNNECESSARY)
1011 1012
				gro_skb->csum_level = 1;

1013
			if ((cqe->vlan_my_qpn &
1014
			    cpu_to_be32(MLX4_CQE_CVLAN_PRESENT_MASK)) &&
1015 1016 1017 1018
			    (dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
				u16 vid = be16_to_cpu(cqe->sl_vid);

				__vlan_hwaccel_put_tag(gro_skb, htons(ETH_P_8021Q), vid);
1019 1020 1021 1022 1023 1024
			} else if ((be32_to_cpu(cqe->vlan_my_qpn) &
				  MLX4_CQE_SVLAN_PRESENT_MASK) &&
				 (dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
				__vlan_hwaccel_put_tag(gro_skb,
						       htons(ETH_P_8021AD),
						       be16_to_cpu(cqe->sl_vid));
1025 1026 1027 1028 1029
			}

			if (dev->features & NETIF_F_RXHASH)
				skb_set_hash(gro_skb,
					     be32_to_cpu(cqe->immed_rss_invalid),
1030 1031 1032
					     (ip_summed == CHECKSUM_UNNECESSARY) ?
						PKT_HASH_TYPE_L4 :
						PKT_HASH_TYPE_L3);
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047

			skb_record_rx_queue(gro_skb, cq->ring);

			if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) {
				timestamp = mlx4_en_get_cqe_ts(cqe);
				mlx4_en_fill_hwtstamps(mdev,
						       skb_hwtstamps(gro_skb),
						       timestamp);
			}

			napi_gro_frags(&cq->napi);
			goto next;
		}

		/* GRO not possible, complete processing here */
1048
		skb = mlx4_en_rx_skb(priv, rx_desc, frags, length);
1049
		if (unlikely(!skb)) {
1050
			ring->dropped++;
1051 1052 1053
			goto next;
		}

K
Kamal Heib 已提交
1054
		if (unlikely(priv->validate_loopback)) {
1055 1056 1057 1058
			validate_loopback(priv, skb);
			goto next;
		}

1059
		if (ip_summed == CHECKSUM_COMPLETE) {
1060
			if (check_csum(cqe, skb, skb->data, dev->features)) {
1061 1062 1063 1064 1065 1066
				ip_summed = CHECKSUM_NONE;
				ring->csum_complete--;
				ring->csum_none++;
			}
		}

1067 1068
		skb->ip_summed = ip_summed;
		skb->protocol = eth_type_trans(skb, dev);
1069
		skb_record_rx_queue(skb, cq->ring);
1070

1071 1072
		if (l2_tunnel && ip_summed == CHECKSUM_UNNECESSARY)
			skb->csum_level = 1;
1073

Y
Yevgeny Petrilin 已提交
1074
		if (dev->features & NETIF_F_RXHASH)
T
Tom Herbert 已提交
1075 1076
			skb_set_hash(skb,
				     be32_to_cpu(cqe->immed_rss_invalid),
1077 1078 1079
				     (ip_summed == CHECKSUM_UNNECESSARY) ?
					PKT_HASH_TYPE_L4 :
					PKT_HASH_TYPE_L3);
Y
Yevgeny Petrilin 已提交
1080

1081
		if ((be32_to_cpu(cqe->vlan_my_qpn) &
1082
		    MLX4_CQE_CVLAN_PRESENT_MASK) &&
1083
		    (dev->features & NETIF_F_HW_VLAN_CTAG_RX))
1084
			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), be16_to_cpu(cqe->sl_vid));
1085 1086 1087 1088 1089
		else if ((be32_to_cpu(cqe->vlan_my_qpn) &
			  MLX4_CQE_SVLAN_PRESENT_MASK) &&
			 (dev->features & NETIF_F_HW_VLAN_STAG_RX))
			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021AD),
					       be16_to_cpu(cqe->sl_vid));
J
Jiri Pirko 已提交
1090

1091 1092 1093 1094 1095 1096
		if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) {
			timestamp = mlx4_en_get_cqe_ts(cqe);
			mlx4_en_fill_hwtstamps(mdev, skb_hwtstamps(skb),
					       timestamp);
		}

1097
		napi_gro_receive(&cq->napi, skb);
1098
next:
1099 1100 1101
		for (nr = 0; nr < priv->num_frags; nr++)
			mlx4_en_free_frag(priv, frags, nr);

1102
consumed:
1103 1104
		++cq->mcq.cons_index;
		index = (cq->mcq.cons_index) & ring->size_mask;
1105
		cqe = mlx4_en_get_cqe(cq->buf, index, priv->cqe_size) + factor;
1106
		if (++polled == budget)
1107 1108 1109 1110
			goto out;
	}

out:
1111
	rcu_read_unlock();
1112

1113 1114 1115 1116 1117 1118 1119 1120
	if (polled) {
		if (doorbell_pending)
			mlx4_en_xmit_doorbell(priv->tx_ring[TX_XDP][cq->ring]);

		mlx4_cq_set_ci(&cq->mcq);
		wmb(); /* ensure HW sees CQ consumer before we post new buffers */
		ring->cons = cq->mcq.cons_index;
	}
1121
	AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled);
1122 1123 1124 1125

	if (mlx4_en_refill_rx_buffers(priv, ring))
		mlx4_en_update_rx_prod_db(ring);

1126 1127 1128 1129 1130 1131 1132 1133 1134
	return polled;
}


void mlx4_en_rx_irq(struct mlx4_cq *mcq)
{
	struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
	struct mlx4_en_priv *priv = netdev_priv(cq->dev);

E
Eric Dumazet 已提交
1135 1136
	if (likely(priv->port_up))
		napi_schedule_irqoff(&cq->napi);
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151
	else
		mlx4_en_arm_cq(priv, cq);
}

/* Rx CQ polling - called by NAPI */
int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget)
{
	struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
	struct net_device *dev = cq->dev;
	struct mlx4_en_priv *priv = netdev_priv(dev);
	int done;

	done = mlx4_en_process_rx_cq(dev, cq, budget);

	/* If we used up all the quota - we're probably not done yet... */
1152
	if (done == budget) {
1153
		const struct cpumask *aff;
1154 1155
		struct irq_data *idata;
		int cpu_curr;
1156

1157
		INC_PERF_COUNTER(priv->pstats.napi_quota);
1158 1159

		cpu_curr = smp_processor_id();
1160 1161
		idata = irq_desc_get_irq_data(cq->irq_desc);
		aff = irq_data_get_affinity_mask(idata);
1162

1163 1164 1165 1166
		if (likely(cpumask_test_cpu(cpu_curr, aff)))
			return budget;

		/* Current cpu is not according to smp_irq_affinity -
1167 1168 1169 1170
		 * probably affinity changed. Need to stop this NAPI
		 * poll, and restart it on the right CPU.
		 * Try to avoid returning a too small value (like 0),
		 * to not fool net_rx_action() and its netdev_budget
1171
		 */
1172 1173
		if (done)
			done--;
1174
	}
E
Eric Dumazet 已提交
1175
	/* Done for now */
1176 1177
	if (napi_complete_done(napi, done))
		mlx4_en_arm_cq(priv, cq);
1178 1179 1180
	return done;
}

1181
static const int frag_sizes[] = {
1182 1183 1184 1185 1186 1187 1188 1189 1190
	FRAG_SZ0,
	FRAG_SZ1,
	FRAG_SZ2,
	FRAG_SZ3
};

void mlx4_en_calc_rx_buf(struct net_device *dev)
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
1191
	int eff_mtu = MLX4_EN_EFF_MTU(dev->mtu);
1192 1193
	int i = 0;

1194 1195 1196
	/* bpf requires buffers to be set up as 1 packet per page.
	 * This only works when num_frags == 1.
	 */
1197
	if (priv->tx_ring_num[TX_XDP]) {
1198
		priv->rx_page_order = 0;
1199 1200 1201
		priv->frag_info[0].frag_size = eff_mtu;
		/* This will gain efficient xdp frame recycling at the
		 * expense of more costly truesize accounting
1202
		 */
1203
		priv->frag_info[0].frag_stride = PAGE_SIZE;
1204
		priv->dma_dir = PCI_DMA_BIDIRECTIONAL;
1205
		priv->frag_info[0].rx_headroom = XDP_PACKET_HEADROOM;
1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216
		i = 1;
	} else {
		int buf_size = 0;

		while (buf_size < eff_mtu) {
			priv->frag_info[i].frag_size =
				(eff_mtu > buf_size + frag_sizes[i]) ?
					frag_sizes[i] : eff_mtu - buf_size;
			priv->frag_info[i].frag_stride =
				ALIGN(priv->frag_info[i].frag_size,
				      SMP_CACHE_BYTES);
1217
			priv->frag_info[i].rx_headroom = 0;
1218 1219 1220
			buf_size += priv->frag_info[i].frag_size;
			i++;
		}
1221
		priv->rx_page_order = MLX4_EN_ALLOC_PREFER_ORDER;
1222
		priv->dma_dir = PCI_DMA_FROMDEVICE;
1223 1224 1225 1226
	}

	priv->num_frags = i;
	priv->rx_skb_size = eff_mtu;
1227
	priv->log_rx_info = ROUNDUP_LOG2(i * sizeof(struct mlx4_en_rx_alloc));
1228

J
Joe Perches 已提交
1229 1230
	en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d num_frags:%d):\n",
	       eff_mtu, priv->num_frags);
1231
	for (i = 0; i < priv->num_frags; i++) {
1232
		en_err(priv,
E
Eric Dumazet 已提交
1233
		       "  frag:%d - size:%d stride:%d\n",
1234 1235 1236
		       i,
		       priv->frag_info[i].frag_size,
		       priv->frag_info[i].frag_stride);
1237 1238 1239 1240 1241
	}
}

/* RSS related functions */

1242 1243
static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn,
				 struct mlx4_en_rx_ring *ring,
1244 1245 1246 1247 1248 1249 1250
				 enum mlx4_qp_state *state,
				 struct mlx4_qp *qp)
{
	struct mlx4_en_dev *mdev = priv->mdev;
	struct mlx4_qp_context *context;
	int err = 0;

1251 1252
	context = kmalloc(sizeof(*context), GFP_KERNEL);
	if (!context)
1253 1254
		return -ENOMEM;

1255
	err = mlx4_qp_alloc(mdev->dev, qpn, qp, GFP_KERNEL);
1256
	if (err) {
1257
		en_err(priv, "Failed to allocate qp #%x\n", qpn);
1258 1259 1260 1261 1262
		goto out;
	}
	qp->event = mlx4_en_sqp_event;

	memset(context, 0, sizeof *context);
1263
	mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0,
1264
				qpn, ring->cqn, -1, context);
1265
	context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma);
1266

1267
	/* Cancel FCS removal if FW allows */
1268
	if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) {
1269
		context->param3 |= cpu_to_be32(1 << 29);
1270 1271 1272 1273
		if (priv->dev->features & NETIF_F_RXFCS)
			ring->fcs_del = 0;
		else
			ring->fcs_del = ETH_FCS_LEN;
1274 1275
	} else
		ring->fcs_del = 0;
1276

1277
	err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state);
1278 1279 1280 1281
	if (err) {
		mlx4_qp_remove(mdev->dev, qp);
		mlx4_qp_free(mdev->dev, qp);
	}
1282
	mlx4_en_update_rx_prod_db(ring);
1283 1284 1285 1286 1287
out:
	kfree(context);
	return err;
}

1288 1289 1290 1291 1292
int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv)
{
	int err;
	u32 qpn;

M
Matan Barak 已提交
1293 1294
	err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn,
				    MLX4_RESERVE_A0_QP);
1295 1296 1297 1298
	if (err) {
		en_err(priv, "Failed reserving drop qpn\n");
		return err;
	}
1299
	err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp, GFP_KERNEL);
1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318
	if (err) {
		en_err(priv, "Failed allocating drop qp\n");
		mlx4_qp_release_range(priv->mdev->dev, qpn, 1);
		return err;
	}

	return 0;
}

void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv)
{
	u32 qpn;

	qpn = priv->drop_qp.qpn;
	mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp);
	mlx4_qp_free(priv->mdev->dev, &priv->drop_qp);
	mlx4_qp_release_range(priv->mdev->dev, qpn, 1);
}

1319 1320 1321 1322 1323 1324
/* Allocate rx qp's and configure them according to rss map */
int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
{
	struct mlx4_en_dev *mdev = priv->mdev;
	struct mlx4_en_rss_map *rss_map = &priv->rss_map;
	struct mlx4_qp_context context;
1325
	struct mlx4_rss_context *rss_context;
1326
	int rss_rings;
1327
	void *ptr;
1328
	u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 |
1329
			MLX4_RSS_TCP_IPV6);
1330
	int i, qpn;
1331 1332 1333
	int err = 0;
	int good_qps = 0;

1334
	en_dbg(DRV, priv, "Configuring rss steering\n");
1335 1336
	err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,
				    priv->rx_ring_num,
1337
				    &rss_map->base_qpn, 0);
1338
	if (err) {
1339
		en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);
1340 1341 1342
		return err;
	}

1343
	for (i = 0; i < priv->rx_ring_num; i++) {
1344
		qpn = rss_map->base_qpn + i;
1345
		err = mlx4_en_config_rss_qp(priv, qpn, priv->rx_ring[i],
1346 1347 1348 1349 1350 1351 1352 1353 1354
					    &rss_map->state[i],
					    &rss_map->qps[i]);
		if (err)
			goto rss_err;

		++good_qps;
	}

	/* Configure RSS indirection qp */
1355
	err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp, GFP_KERNEL);
1356
	if (err) {
1357
		en_err(priv, "Failed to allocate RSS indirection QP\n");
1358
		goto rss_err;
1359 1360 1361
	}
	rss_map->indir_qp.event = mlx4_en_sqp_event;
	mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn,
1362
				priv->rx_ring[0]->cqn, -1, &context);
1363

1364 1365 1366 1367 1368
	if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num)
		rss_rings = priv->rx_ring_num;
	else
		rss_rings = priv->prof->rss_rings;

1369 1370
	ptr = ((void *) &context) + offsetof(struct mlx4_qp_context, pri_path)
					+ MLX4_RSS_OFFSET_IN_QPC_PRI_PATH;
1371
	rss_context = ptr;
1372
	rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 |
1373
					    (rss_map->base_qpn));
1374
	rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn);
1375 1376 1377 1378
	if (priv->mdev->profile.udp_rss) {
		rss_mask |=  MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6;
		rss_context->base_qpn_udp = rss_context->default_qpn;
	}
1379 1380 1381 1382 1383 1384

	if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
		en_info(priv, "Setting RSS context tunnel type to RSS on inner headers\n");
		rss_mask |= MLX4_RSS_BY_INNER_HEADERS;
	}

Y
Yevgeny Petrilin 已提交
1385
	rss_context->flags = rss_mask;
1386
	rss_context->hash_fn = MLX4_RSS_HASH_TOP;
1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397
	if (priv->rss_hash_fn == ETH_RSS_HASH_XOR) {
		rss_context->hash_fn = MLX4_RSS_HASH_XOR;
	} else if (priv->rss_hash_fn == ETH_RSS_HASH_TOP) {
		rss_context->hash_fn = MLX4_RSS_HASH_TOP;
		memcpy(rss_context->rss_key, priv->rss_key,
		       MLX4_EN_RSS_KEY_SIZE);
	} else {
		en_err(priv, "Unknown RSS hash function requested\n");
		err = -EINVAL;
		goto indir_err;
	}
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416
	err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context,
			       &rss_map->indir_qp, &rss_map->indir_state);
	if (err)
		goto indir_err;

	return 0;

indir_err:
	mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
		       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
	mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
	mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
rss_err:
	for (i = 0; i < good_qps; i++) {
		mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
			       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
		mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
		mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
	}
1417
	mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
	return err;
}

void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv)
{
	struct mlx4_en_dev *mdev = priv->mdev;
	struct mlx4_en_rss_map *rss_map = &priv->rss_map;
	int i;

	mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
		       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
	mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
	mlx4_qp_free(mdev->dev, &rss_map->indir_qp);

1432
	for (i = 0; i < priv->rx_ring_num; i++) {
1433 1434 1435 1436 1437
		mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
			       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
		mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
		mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
	}
1438
	mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
1439
}