en_rx.c 35.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
/*
 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

34
#include <net/busy_poll.h>
35
#include <linux/mlx4/cq.h>
36
#include <linux/slab.h>
37 38
#include <linux/mlx4/qp.h>
#include <linux/skbuff.h>
39
#include <linux/rculist.h>
40 41 42
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <linux/vmalloc.h>
43
#include <linux/irq.h>
44

45 46 47 48
#if IS_ENABLED(CONFIG_IPV6)
#include <net/ip6_checksum.h>
#endif

49 50
#include "mlx4_en.h"

51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
static int mlx4_alloc_pages(struct mlx4_en_priv *priv,
			    struct mlx4_en_rx_alloc *page_alloc,
			    const struct mlx4_en_frag_info *frag_info,
			    gfp_t _gfp)
{
	int order;
	struct page *page;
	dma_addr_t dma;

	for (order = MLX4_EN_ALLOC_PREFER_ORDER; ;) {
		gfp_t gfp = _gfp;

		if (order)
			gfp |= __GFP_COMP | __GFP_NOWARN;
		page = alloc_pages(gfp, order);
		if (likely(page))
			break;
		if (--order < 0 ||
		    ((PAGE_SIZE << order) < frag_info->frag_size))
			return -ENOMEM;
	}
	dma = dma_map_page(priv->ddev, page, 0, PAGE_SIZE << order,
			   PCI_DMA_FROMDEVICE);
	if (dma_mapping_error(priv->ddev, dma)) {
		put_page(page);
		return -ENOMEM;
	}
78
	page_alloc->page_size = PAGE_SIZE << order;
79 80
	page_alloc->page = page;
	page_alloc->dma = dma;
81
	page_alloc->page_offset = 0;
82
	/* Not doing get_page() for each frag is a big win
83
	 * on asymetric workloads. Note we can not use atomic_set().
84
	 */
85 86
	atomic_add(page_alloc->page_size / frag_info->frag_stride - 1,
		   &page->_count);
87 88 89
	return 0;
}

90 91 92
static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv,
			       struct mlx4_en_rx_desc *rx_desc,
			       struct mlx4_en_rx_alloc *frags,
93 94
			       struct mlx4_en_rx_alloc *ring_alloc,
			       gfp_t gfp)
95
{
96
	struct mlx4_en_rx_alloc page_alloc[MLX4_EN_MAX_RX_FRAGS];
97
	const struct mlx4_en_frag_info *frag_info;
98 99
	struct page *page;
	dma_addr_t dma;
100
	int i;
101

102 103
	for (i = 0; i < priv->num_frags; i++) {
		frag_info = &priv->frag_info[i];
104
		page_alloc[i] = ring_alloc[i];
105 106 107 108
		page_alloc[i].page_offset += frag_info->frag_stride;

		if (page_alloc[i].page_offset + frag_info->frag_stride <=
		    ring_alloc[i].page_size)
109
			continue;
110

111 112
		if (mlx4_alloc_pages(priv, &page_alloc[i], frag_info, gfp))
			goto out;
113
	}
114

115 116
	for (i = 0; i < priv->num_frags; i++) {
		frags[i] = ring_alloc[i];
117
		dma = ring_alloc[i].dma + ring_alloc[i].page_offset;
118 119
		ring_alloc[i] = page_alloc[i];
		rx_desc->data[i].addr = cpu_to_be64(dma);
120
	}
121

122
	return 0;
123 124 125

out:
	while (i--) {
126
		if (page_alloc[i].page != ring_alloc[i].page) {
127
			dma_unmap_page(priv->ddev, page_alloc[i].dma,
128
				page_alloc[i].page_size, PCI_DMA_FROMDEVICE);
129 130 131 132
			page = page_alloc[i].page;
			atomic_set(&page->_count, 1);
			put_page(page);
		}
133 134 135 136 137 138 139 140
	}
	return -ENOMEM;
}

static void mlx4_en_free_frag(struct mlx4_en_priv *priv,
			      struct mlx4_en_rx_alloc *frags,
			      int i)
{
141
	const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
142
	u32 next_frag_end = frags[i].page_offset + 2 * frag_info->frag_stride;
143

144 145

	if (next_frag_end > frags[i].page_size)
146 147
		dma_unmap_page(priv->ddev, frags[i].dma, frags[i].page_size,
			       PCI_DMA_FROMDEVICE);
148

149 150
	if (frags[i].page)
		put_page(frags[i].page);
151 152 153 154 155 156
}

static int mlx4_en_init_allocator(struct mlx4_en_priv *priv,
				  struct mlx4_en_rx_ring *ring)
{
	int i;
157
	struct mlx4_en_rx_alloc *page_alloc;
158 159

	for (i = 0; i < priv->num_frags; i++) {
160
		const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
161

162
		if (mlx4_alloc_pages(priv, &ring->page_alloc[i],
163
				     frag_info, GFP_KERNEL | __GFP_COLD))
164
			goto out;
165 166 167 168

		en_dbg(DRV, priv, "  frag %d allocator: - size:%d frags:%d\n",
		       i, ring->page_alloc[i].page_size,
		       atomic_read(&ring->page_alloc[i].page->_count));
169 170 171 172 173
	}
	return 0;

out:
	while (i--) {
174 175
		struct page *page;

176
		page_alloc = &ring->page_alloc[i];
177
		dma_unmap_page(priv->ddev, page_alloc->dma,
178
			       page_alloc->page_size, PCI_DMA_FROMDEVICE);
179 180 181
		page = page_alloc->page;
		atomic_set(&page->_count, 1);
		put_page(page);
182 183 184 185 186 187 188 189 190 191 192 193
		page_alloc->page = NULL;
	}
	return -ENOMEM;
}

static void mlx4_en_destroy_allocator(struct mlx4_en_priv *priv,
				      struct mlx4_en_rx_ring *ring)
{
	struct mlx4_en_rx_alloc *page_alloc;
	int i;

	for (i = 0; i < priv->num_frags; i++) {
194 195
		const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];

196
		page_alloc = &ring->page_alloc[i];
197 198
		en_dbg(DRV, priv, "Freeing allocator:%d count:%d\n",
		       i, page_count(page_alloc->page));
199

200
		dma_unmap_page(priv->ddev, page_alloc->dma,
201 202 203
				page_alloc->page_size, PCI_DMA_FROMDEVICE);
		while (page_alloc->page_offset + frag_info->frag_stride <
		       page_alloc->page_size) {
204
			put_page(page_alloc->page);
205
			page_alloc->page_offset += frag_info->frag_stride;
206
		}
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236
		page_alloc->page = NULL;
	}
}

static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv,
				 struct mlx4_en_rx_ring *ring, int index)
{
	struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index;
	int possible_frags;
	int i;

	/* Set size and memtype fields */
	for (i = 0; i < priv->num_frags; i++) {
		rx_desc->data[i].byte_count =
			cpu_to_be32(priv->frag_info[i].frag_size);
		rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key);
	}

	/* If the number of used fragments does not fill up the ring stride,
	 * remaining (unused) fragments must be padded with null address/size
	 * and a special memory key */
	possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE;
	for (i = priv->num_frags; i < possible_frags; i++) {
		rx_desc->data[i].byte_count = 0;
		rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD);
		rx_desc->data[i].addr = 0;
	}
}

static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv,
237 238
				   struct mlx4_en_rx_ring *ring, int index,
				   gfp_t gfp)
239 240
{
	struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride);
241 242
	struct mlx4_en_rx_alloc *frags = ring->rx_info +
					(index << priv->log_rx_info);
243

244
	return mlx4_en_alloc_frags(priv, rx_desc, frags, ring->page_alloc, gfp);
245 246
}

247 248 249 250 251 252
static inline bool mlx4_en_is_ring_empty(struct mlx4_en_rx_ring *ring)
{
	BUG_ON((u32)(ring->prod - ring->cons) > ring->actual_size);
	return ring->prod == ring->cons;
}

253 254 255 256 257
static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring)
{
	*ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff);
}

258 259 260 261
static void mlx4_en_free_rx_desc(struct mlx4_en_priv *priv,
				 struct mlx4_en_rx_ring *ring,
				 int index)
{
262
	struct mlx4_en_rx_alloc *frags;
263 264
	int nr;

265
	frags = ring->rx_info + (index << priv->log_rx_info);
266
	for (nr = 0; nr < priv->num_frags; nr++) {
267
		en_dbg(DRV, priv, "Freeing fragment:%d\n", nr);
268
		mlx4_en_free_frag(priv, frags, nr);
269 270 271
	}
}

272 273 274 275 276
static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv)
{
	struct mlx4_en_rx_ring *ring;
	int ring_ind;
	int buf_ind;
277
	int new_size;
278 279 280

	for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) {
		for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
281
			ring = priv->rx_ring[ring_ind];
282 283

			if (mlx4_en_prepare_rx_desc(priv, ring,
284
						    ring->actual_size,
285
						    GFP_KERNEL | __GFP_COLD)) {
286
				if (ring->actual_size < MLX4_EN_MIN_RX_SIZE) {
J
Joe Perches 已提交
287
					en_err(priv, "Failed to allocate enough rx buffers\n");
288 289
					return -ENOMEM;
				} else {
290
					new_size = rounddown_pow_of_two(ring->actual_size);
J
Joe Perches 已提交
291
					en_warn(priv, "Only %d buffers allocated reducing ring size to %d\n",
292
						ring->actual_size, new_size);
293
					goto reduce_rings;
294 295 296 297 298 299
				}
			}
			ring->actual_size++;
			ring->prod++;
		}
	}
300 301 302 303
	return 0;

reduce_rings:
	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
304
		ring = priv->rx_ring[ring_ind];
305 306 307 308 309 310 311
		while (ring->actual_size > new_size) {
			ring->actual_size--;
			ring->prod--;
			mlx4_en_free_rx_desc(priv, ring, ring->actual_size);
		}
	}

312 313 314 315 316 317 318 319
	return 0;
}

static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv,
				struct mlx4_en_rx_ring *ring)
{
	int index;

320 321
	en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n",
	       ring->cons, ring->prod);
322 323

	/* Unmap and free Rx buffers */
324
	while (!mlx4_en_is_ring_empty(ring)) {
325
		index = ring->cons & ring->size_mask;
326
		en_dbg(DRV, priv, "Processing descriptor:%d\n", index);
327
		mlx4_en_free_rx_desc(priv, ring, index);
328 329 330 331
		++ring->cons;
	}
}

332 333 334 335
void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev)
{
	int i;
	int num_of_eqs;
336
	int num_rx_rings;
337 338 339
	struct mlx4_dev *dev = mdev->dev;

	mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) {
M
Matan Barak 已提交
340 341 342 343
		num_of_eqs = max_t(int, MIN_RX_RINGS,
				   min_t(int,
					 mlx4_get_eqs_per_port(mdev->dev, i),
					 DEF_RX_RINGS));
344

345 346 347
		num_rx_rings = mlx4_low_memory_profile() ? MIN_RX_RINGS :
			min_t(int, num_of_eqs,
			      netif_get_num_default_rss_queues());
348
		mdev->profile.prof[i].rx_ring_num =
349
			rounddown_pow_of_two(num_rx_rings);
350 351 352
	}
}

353
int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
354
			   struct mlx4_en_rx_ring **pring,
355
			   u32 size, u16 stride, int node)
356 357
{
	struct mlx4_en_dev *mdev = priv->mdev;
358
	struct mlx4_en_rx_ring *ring;
359
	int err = -ENOMEM;
360 361
	int tmp;

362
	ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node);
363
	if (!ring) {
364 365 366 367 368
		ring = kzalloc(sizeof(*ring), GFP_KERNEL);
		if (!ring) {
			en_err(priv, "Failed to allocate RX ring structure\n");
			return -ENOMEM;
		}
369 370
	}

371 372 373 374 375 376
	ring->prod = 0;
	ring->cons = 0;
	ring->size = size;
	ring->size_mask = size - 1;
	ring->stride = stride;
	ring->log_stride = ffs(ring->stride) - 1;
377
	ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
378 379

	tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
380
					sizeof(struct mlx4_en_rx_alloc));
381
	ring->rx_info = vmalloc_node(tmp, node);
382
	if (!ring->rx_info) {
383 384 385 386 387
		ring->rx_info = vmalloc(tmp);
		if (!ring->rx_info) {
			err = -ENOMEM;
			goto err_ring;
		}
388
	}
389

390
	en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n",
391 392
		 ring->rx_info, tmp);

393
	/* Allocate HW buffers on provided NUMA node */
394
	set_dev_node(&mdev->dev->persist->pdev->dev, node);
395 396
	err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres,
				 ring->buf_size, 2 * PAGE_SIZE);
397
	set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
398
	if (err)
399
		goto err_info;
400 401 402

	err = mlx4_en_map_buffer(&ring->wqres.buf);
	if (err) {
403
		en_err(priv, "Failed to map RX buffer\n");
404 405 406 407
		goto err_hwq;
	}
	ring->buf = ring->wqres.buf.direct.buf;

408 409
	ring->hwtstamp_rx_filter = priv->hwtstamp_config.rx_filter;

410
	*pring = ring;
411 412 413 414
	return 0;

err_hwq:
	mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
415
err_info:
416 417
	vfree(ring->rx_info);
	ring->rx_info = NULL;
418 419 420 421
err_ring:
	kfree(ring);
	*pring = NULL;

422 423 424 425 426 427 428 429 430 431 432 433 434
	return err;
}

int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
{
	struct mlx4_en_rx_ring *ring;
	int i;
	int ring_ind;
	int err;
	int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
					DS_SIZE * priv->num_frags);

	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
435
		ring = priv->rx_ring[ring_ind];
436 437 438 439

		ring->prod = 0;
		ring->cons = 0;
		ring->actual_size = 0;
440
		ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn;
441 442

		ring->stride = stride;
443 444 445
		if (ring->stride <= TXBB_SIZE)
			ring->buf += TXBB_SIZE;

446 447 448 449 450 451
		ring->log_stride = ffs(ring->stride) - 1;
		ring->buf_size = ring->size * ring->stride;

		memset(ring->buf, 0, ring->buf_size);
		mlx4_en_update_rx_prod_db(ring);

452
		/* Initialize all descriptors */
453 454 455 456 457 458
		for (i = 0; i < ring->size; i++)
			mlx4_en_init_rx_desc(priv, ring, i);

		/* Initialize page allocators */
		err = mlx4_en_init_allocator(priv, ring);
		if (err) {
459
			en_err(priv, "Failed initializing ring allocator\n");
460 461
			if (ring->stride <= TXBB_SIZE)
				ring->buf -= TXBB_SIZE;
462 463
			ring_ind--;
			goto err_allocator;
464 465
		}
	}
466 467
	err = mlx4_en_fill_rx_buffers(priv);
	if (err)
468 469 470
		goto err_buffers;

	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
471
		ring = priv->rx_ring[ring_ind];
472

473
		ring->size_mask = ring->actual_size - 1;
474 475 476 477 478 479 480
		mlx4_en_update_rx_prod_db(ring);
	}

	return 0;

err_buffers:
	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++)
481
		mlx4_en_free_rx_buf(priv, priv->rx_ring[ring_ind]);
482 483 484 485

	ring_ind = priv->rx_ring_num - 1;
err_allocator:
	while (ring_ind >= 0) {
486 487 488
		if (priv->rx_ring[ring_ind]->stride <= TXBB_SIZE)
			priv->rx_ring[ring_ind]->buf -= TXBB_SIZE;
		mlx4_en_destroy_allocator(priv, priv->rx_ring[ring_ind]);
489 490 491 492 493
		ring_ind--;
	}
	return err;
}

494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510
/* We recover from out of memory by scheduling our napi poll
 * function (mlx4_en_process_cq), which tries to allocate
 * all missing RX buffers (call to mlx4_en_refill_rx_buffers).
 */
void mlx4_en_recover_from_oom(struct mlx4_en_priv *priv)
{
	int ring;

	if (!priv->port_up)
		return;

	for (ring = 0; ring < priv->rx_ring_num; ring++) {
		if (mlx4_en_is_ring_empty(priv->rx_ring[ring]))
			napi_reschedule(&priv->rx_cq[ring]->napi);
	}
}

511
void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
512 513
			     struct mlx4_en_rx_ring **pring,
			     u32 size, u16 stride)
514 515
{
	struct mlx4_en_dev *mdev = priv->mdev;
516
	struct mlx4_en_rx_ring *ring = *pring;
517 518

	mlx4_en_unmap_buffer(&ring->wqres.buf);
519
	mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE);
520 521
	vfree(ring->rx_info);
	ring->rx_info = NULL;
522 523
	kfree(ring);
	*pring = NULL;
524
#ifdef CONFIG_RFS_ACCEL
525
	mlx4_en_cleanup_filters(priv);
526
#endif
527 528 529 530 531 532
}

void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
				struct mlx4_en_rx_ring *ring)
{
	mlx4_en_free_rx_buf(priv, ring);
533 534
	if (ring->stride <= TXBB_SIZE)
		ring->buf -= TXBB_SIZE;
535 536 537 538 539 540
	mlx4_en_destroy_allocator(priv, ring);
}


static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv,
				    struct mlx4_en_rx_desc *rx_desc,
541
				    struct mlx4_en_rx_alloc *frags,
542
				    struct sk_buff *skb,
543 544
				    int length)
{
545
	struct skb_frag_struct *skb_frags_rx = skb_shinfo(skb)->frags;
546 547 548 549
	struct mlx4_en_frag_info *frag_info;
	int nr;
	dma_addr_t dma;

550
	/* Collect used fragments while replacing them in the HW descriptors */
551 552 553 554
	for (nr = 0; nr < priv->num_frags; nr++) {
		frag_info = &priv->frag_info[nr];
		if (length <= frag_info->frag_prefix_size)
			break;
555 556
		if (!frags[nr].page)
			goto fail;
557 558

		dma = be64_to_cpu(rx_desc->data[nr].addr);
559 560
		dma_sync_single_for_cpu(priv->ddev, dma, frag_info->frag_size,
					DMA_FROM_DEVICE);
561

562 563 564
		/* Save page reference in skb */
		__skb_frag_set_page(&skb_frags_rx[nr], frags[nr].page);
		skb_frag_size_set(&skb_frags_rx[nr], frag_info->frag_size);
565
		skb_frags_rx[nr].page_offset = frags[nr].page_offset;
566
		skb->truesize += frag_info->frag_stride;
567
		frags[nr].page = NULL;
568 569
	}
	/* Adjust size of last fragment to match actual length */
570
	if (nr > 0)
E
Eric Dumazet 已提交
571 572
		skb_frag_size_set(&skb_frags_rx[nr - 1],
			length - priv->frag_info[nr - 1].frag_prefix_size);
573 574 575 576 577
	return nr;

fail:
	while (nr > 0) {
		nr--;
578
		__skb_frag_unref(&skb_frags_rx[nr]);
579 580 581 582 583 584 585
	}
	return 0;
}


static struct sk_buff *mlx4_en_rx_skb(struct mlx4_en_priv *priv,
				      struct mlx4_en_rx_desc *rx_desc,
586
				      struct mlx4_en_rx_alloc *frags,
587 588 589 590 591 592 593
				      unsigned int length)
{
	struct sk_buff *skb;
	void *va;
	int used_frags;
	dma_addr_t dma;

594
	skb = netdev_alloc_skb(priv->dev, SMALL_PACKET_SIZE + NET_IP_ALIGN);
595
	if (!skb) {
596
		en_dbg(RX_ERR, priv, "Failed allocating skb\n");
597 598 599 600 601 602 603
		return NULL;
	}
	skb_reserve(skb, NET_IP_ALIGN);
	skb->len = length;

	/* Get pointer to first fragment so we could copy the headers into the
	 * (linear part of the) skb */
604
	va = page_address(frags[0].page) + frags[0].page_offset;
605 606 607

	if (length <= SMALL_PACKET_SIZE) {
		/* We are copying all relevant data to the skb - temporarily
608
		 * sync buffers for the copy */
609
		dma = be64_to_cpu(rx_desc->data[0].addr);
610
		dma_sync_single_for_cpu(priv->ddev, dma, length,
611
					DMA_FROM_DEVICE);
612 613 614
		skb_copy_to_linear_data(skb, va, length);
		skb->tail += length;
	} else {
615 616
		unsigned int pull_len;

617
		/* Move relevant fragments to skb */
618 619
		used_frags = mlx4_en_complete_rx_desc(priv, rx_desc, frags,
							skb, length);
620 621 622 623
		if (unlikely(!used_frags)) {
			kfree_skb(skb);
			return NULL;
		}
624 625
		skb_shinfo(skb)->nr_frags = used_frags;

626
		pull_len = eth_get_headlen(va, SMALL_PACKET_SIZE);
627
		/* Copy headers into the skb linear buffer */
628 629
		memcpy(skb->data, va, pull_len);
		skb->tail += pull_len;
630 631

		/* Skip headers in first fragment */
632
		skb_shinfo(skb)->frags[0].page_offset += pull_len;
633 634

		/* Adjust size of first fragment */
635 636
		skb_frag_size_sub(&skb_shinfo(skb)->frags[0], pull_len);
		skb->data_len = length - pull_len;
637 638 639 640
	}
	return skb;
}

641 642 643 644 645 646 647 648 649 650 651 652 653 654 655
static void validate_loopback(struct mlx4_en_priv *priv, struct sk_buff *skb)
{
	int i;
	int offset = ETH_HLEN;

	for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) {
		if (*(skb->data + offset) != (unsigned char) (i & 0xff))
			goto out_loopback;
	}
	/* Loopback found */
	priv->loopback_ok = 1;

out_loopback:
	dev_kfree_skb_any(skb);
}
656

657 658 659 660 661 662
static void mlx4_en_refill_rx_buffers(struct mlx4_en_priv *priv,
				     struct mlx4_en_rx_ring *ring)
{
	int index = ring->prod & ring->size_mask;

	while ((u32) (ring->prod - ring->cons) < ring->actual_size) {
663 664
		if (mlx4_en_prepare_rx_desc(priv, ring, index,
					    GFP_ATOMIC | __GFP_COLD))
665 666 667 668 669 670
			break;
		ring->prod++;
		index = ring->prod & ring->size_mask;
	}
}

671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750
/* When hardware doesn't strip the vlan, we need to calculate the checksum
 * over it and add it to the hardware's checksum calculation
 */
static inline __wsum get_fixed_vlan_csum(__wsum hw_checksum,
					 struct vlan_hdr *vlanh)
{
	return csum_add(hw_checksum, *(__wsum *)vlanh);
}

/* Although the stack expects checksum which doesn't include the pseudo
 * header, the HW adds it. To address that, we are subtracting the pseudo
 * header checksum from the checksum value provided by the HW.
 */
static void get_fixed_ipv4_csum(__wsum hw_checksum, struct sk_buff *skb,
				struct iphdr *iph)
{
	__u16 length_for_csum = 0;
	__wsum csum_pseudo_header = 0;

	length_for_csum = (be16_to_cpu(iph->tot_len) - (iph->ihl << 2));
	csum_pseudo_header = csum_tcpudp_nofold(iph->saddr, iph->daddr,
						length_for_csum, iph->protocol, 0);
	skb->csum = csum_sub(hw_checksum, csum_pseudo_header);
}

#if IS_ENABLED(CONFIG_IPV6)
/* In IPv6 packets, besides subtracting the pseudo header checksum,
 * we also compute/add the IP header checksum which
 * is not added by the HW.
 */
static int get_fixed_ipv6_csum(__wsum hw_checksum, struct sk_buff *skb,
			       struct ipv6hdr *ipv6h)
{
	__wsum csum_pseudo_hdr = 0;

	if (ipv6h->nexthdr == IPPROTO_FRAGMENT || ipv6h->nexthdr == IPPROTO_HOPOPTS)
		return -1;
	hw_checksum = csum_add(hw_checksum, (__force __wsum)(ipv6h->nexthdr << 8));

	csum_pseudo_hdr = csum_partial(&ipv6h->saddr,
				       sizeof(ipv6h->saddr) + sizeof(ipv6h->daddr), 0);
	csum_pseudo_hdr = csum_add(csum_pseudo_hdr, (__force __wsum)ipv6h->payload_len);
	csum_pseudo_hdr = csum_add(csum_pseudo_hdr, (__force __wsum)ntohs(ipv6h->nexthdr));

	skb->csum = csum_sub(hw_checksum, csum_pseudo_hdr);
	skb->csum = csum_add(skb->csum, csum_partial(ipv6h, sizeof(struct ipv6hdr), 0));
	return 0;
}
#endif
static int check_csum(struct mlx4_cqe *cqe, struct sk_buff *skb, void *va,
		      int hwtstamp_rx_filter)
{
	__wsum hw_checksum = 0;

	void *hdr = (u8 *)va + sizeof(struct ethhdr);

	hw_checksum = csum_unfold((__force __sum16)cqe->checksum);

	if (((struct ethhdr *)va)->h_proto == htons(ETH_P_8021Q) &&
	    hwtstamp_rx_filter != HWTSTAMP_FILTER_NONE) {
		/* next protocol non IPv4 or IPv6 */
		if (((struct vlan_hdr *)hdr)->h_vlan_encapsulated_proto
		    != htons(ETH_P_IP) &&
		    ((struct vlan_hdr *)hdr)->h_vlan_encapsulated_proto
		    != htons(ETH_P_IPV6))
			return -1;
		hw_checksum = get_fixed_vlan_csum(hw_checksum, hdr);
		hdr += sizeof(struct vlan_hdr);
	}

	if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV4))
		get_fixed_ipv4_csum(hw_checksum, skb, hdr);
#if IS_ENABLED(CONFIG_IPV6)
	else if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV6))
		if (get_fixed_ipv6_csum(hw_checksum, skb, hdr))
			return -1;
#endif
	return 0;
}

751 752 753
int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget)
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
754
	struct mlx4_en_dev *mdev = priv->mdev;
755
	struct mlx4_cqe *cqe;
756
	struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring];
757
	struct mlx4_en_rx_alloc *frags;
758 759 760 761 762 763 764
	struct mlx4_en_rx_desc *rx_desc;
	struct sk_buff *skb;
	int index;
	int nr;
	unsigned int length;
	int polled = 0;
	int ip_summed;
O
Or Gerlitz 已提交
765
	int factor = priv->cqe_factor;
766
	u64 timestamp;
767
	bool l2_tunnel;
768 769 770 771

	if (!priv->port_up)
		return 0;

772 773 774
	if (budget <= 0)
		return polled;

775 776 777 778
	/* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
	 * descriptor offset can be deduced from the CQE index instead of
	 * reading 'cqe->index' */
	index = cq->mcq.cons_index & ring->size_mask;
779
	cqe = mlx4_en_get_cqe(cq->buf, index, priv->cqe_size) + factor;
780 781 782 783 784

	/* Process all completed CQEs */
	while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
		    cq->mcq.cons_index & cq->size)) {

785
		frags = ring->rx_info + (index << priv->log_rx_info);
786 787 788 789 790
		rx_desc = ring->buf + (index << ring->log_stride);

		/*
		 * make sure we read the CQE after we read the ownership bit
		 */
791
		dma_rmb();
792 793 794 795

		/* Drop packet on bad receive or bad checksum */
		if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
						MLX4_CQE_OPCODE_ERROR)) {
J
Joe Perches 已提交
796 797 798
			en_err(priv, "CQE completed in error - vendor syndrom:%d syndrom:%d\n",
			       ((struct mlx4_err_cqe *)cqe)->vendor_err_syndrome,
			       ((struct mlx4_err_cqe *)cqe)->syndrome);
799 800 801
			goto next;
		}
		if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {
802
			en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");
803 804 805
			goto next;
		}

806 807 808 809 810 811 812 813 814 815 816 817 818
		/* Check if we need to drop the packet if SRIOV is not enabled
		 * and not performing the selftest or flb disabled
		 */
		if (priv->flags & MLX4_EN_FLAG_RX_FILTER_NEEDED) {
			struct ethhdr *ethh;
			dma_addr_t dma;
			/* Get pointer to first fragment since we haven't
			 * skb yet and cast it to ethhdr struct
			 */
			dma = be64_to_cpu(rx_desc->data[0].addr);
			dma_sync_single_for_cpu(priv->ddev, dma, sizeof(*ethh),
						DMA_FROM_DEVICE);
			ethh = (struct ethhdr *)(page_address(frags[0].page) +
819
						 frags[0].page_offset);
820

821 822 823 824 825 826 827 828 829
			if (is_multicast_ether_addr(ethh->h_dest)) {
				struct mlx4_mac_entry *entry;
				struct hlist_head *bucket;
				unsigned int mac_hash;

				/* Drop the packet, since HW loopback-ed it */
				mac_hash = ethh->h_source[MLX4_EN_MAC_HASH_IDX];
				bucket = &priv->mac_hash[mac_hash];
				rcu_read_lock();
830
				hlist_for_each_entry_rcu(entry, bucket, hlist) {
831 832 833 834 835 836 837 838
					if (ether_addr_equal_64bits(entry->mac,
								    ethh->h_source)) {
						rcu_read_unlock();
						goto next;
					}
				}
				rcu_read_unlock();
			}
839
		}
840

841 842 843 844
		/*
		 * Packet is OK - process it.
		 */
		length = be32_to_cpu(cqe->byte_cnt);
845
		length -= ring->fcs_del;
846 847
		ring->bytes += length;
		ring->packets++;
848 849
		l2_tunnel = (dev->hw_enc_features & NETIF_F_RXCSUM) &&
			(cqe->vlan_my_qpn & cpu_to_be32(MLX4_CQE_L2_TUNNEL));
850

851
		if (likely(dev->features & NETIF_F_RXCSUM)) {
852 853 854 855 856 857 858 859 860 861
			if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_TCP |
						      MLX4_CQE_STATUS_UDP)) {
				if ((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
				    cqe->checksum == cpu_to_be16(0xffff)) {
					ip_summed = CHECKSUM_UNNECESSARY;
					ring->csum_ok++;
				} else {
					ip_summed = CHECKSUM_NONE;
					ring->csum_none++;
				}
862
			} else {
863 864 865 866 867 868 869 870 871
				if (priv->flags & MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP &&
				    (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 |
							       MLX4_CQE_STATUS_IPV6))) {
					ip_summed = CHECKSUM_COMPLETE;
					ring->csum_complete++;
				} else {
					ip_summed = CHECKSUM_NONE;
					ring->csum_none++;
				}
872 873 874
			}
		} else {
			ip_summed = CHECKSUM_NONE;
875
			ring->csum_none++;
876 877
		}

878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896
		/* This packet is eligible for GRO if it is:
		 * - DIX Ethernet (type interpretation)
		 * - TCP/IP (v4)
		 * - without IP options
		 * - not an IP fragment
		 * - no LLS polling in progress
		 */
		if (!mlx4_en_cq_busy_polling(cq) &&
		    (dev->features & NETIF_F_GRO)) {
			struct sk_buff *gro_skb = napi_get_frags(&cq->napi);
			if (!gro_skb)
				goto next;

			nr = mlx4_en_complete_rx_desc(priv,
				rx_desc, frags, gro_skb,
				length);
			if (!nr)
				goto next;

897 898 899 900 901 902 903 904 905
			if (ip_summed == CHECKSUM_COMPLETE) {
				void *va = skb_frag_address(skb_shinfo(gro_skb)->frags);
				if (check_csum(cqe, gro_skb, va, ring->hwtstamp_rx_filter)) {
					ip_summed = CHECKSUM_NONE;
					ring->csum_none++;
					ring->csum_complete--;
				}
			}

906 907 908 909 910 911
			skb_shinfo(gro_skb)->nr_frags = nr;
			gro_skb->len = length;
			gro_skb->data_len = length;
			gro_skb->ip_summed = ip_summed;

			if (l2_tunnel && ip_summed == CHECKSUM_UNNECESSARY)
912 913
				gro_skb->csum_level = 1;

914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941
			if ((cqe->vlan_my_qpn &
			    cpu_to_be32(MLX4_CQE_VLAN_PRESENT_MASK)) &&
			    (dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
				u16 vid = be16_to_cpu(cqe->sl_vid);

				__vlan_hwaccel_put_tag(gro_skb, htons(ETH_P_8021Q), vid);
			}

			if (dev->features & NETIF_F_RXHASH)
				skb_set_hash(gro_skb,
					     be32_to_cpu(cqe->immed_rss_invalid),
					     PKT_HASH_TYPE_L3);

			skb_record_rx_queue(gro_skb, cq->ring);
			skb_mark_napi_id(gro_skb, &cq->napi);

			if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) {
				timestamp = mlx4_en_get_cqe_ts(cqe);
				mlx4_en_fill_hwtstamps(mdev,
						       skb_hwtstamps(gro_skb),
						       timestamp);
			}

			napi_gro_frags(&cq->napi);
			goto next;
		}

		/* GRO not possible, complete processing here */
942
		skb = mlx4_en_rx_skb(priv, rx_desc, frags, length);
943 944 945 946 947
		if (!skb) {
			priv->stats.rx_dropped++;
			goto next;
		}

948 949 950 951 952
                if (unlikely(priv->validate_loopback)) {
			validate_loopback(priv, skb);
			goto next;
		}

953 954 955 956 957 958 959 960
		if (ip_summed == CHECKSUM_COMPLETE) {
			if (check_csum(cqe, skb, skb->data, ring->hwtstamp_rx_filter)) {
				ip_summed = CHECKSUM_NONE;
				ring->csum_complete--;
				ring->csum_none++;
			}
		}

961 962
		skb->ip_summed = ip_summed;
		skb->protocol = eth_type_trans(skb, dev);
963
		skb_record_rx_queue(skb, cq->ring);
964

965 966
		if (l2_tunnel && ip_summed == CHECKSUM_UNNECESSARY)
			skb->csum_level = 1;
967

Y
Yevgeny Petrilin 已提交
968
		if (dev->features & NETIF_F_RXHASH)
T
Tom Herbert 已提交
969 970 971
			skb_set_hash(skb,
				     be32_to_cpu(cqe->immed_rss_invalid),
				     PKT_HASH_TYPE_L3);
Y
Yevgeny Petrilin 已提交
972

973 974 975
		if ((be32_to_cpu(cqe->vlan_my_qpn) &
		    MLX4_CQE_VLAN_PRESENT_MASK) &&
		    (dev->features & NETIF_F_HW_VLAN_CTAG_RX))
976
			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), be16_to_cpu(cqe->sl_vid));
J
Jiri Pirko 已提交
977

978 979 980 981 982 983
		if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) {
			timestamp = mlx4_en_get_cqe_ts(cqe);
			mlx4_en_fill_hwtstamps(mdev, skb_hwtstamps(skb),
					       timestamp);
		}

984
		skb_mark_napi_id(skb, &cq->napi);
985

986 987 988 989
		if (!mlx4_en_cq_busy_polling(cq))
			napi_gro_receive(&cq->napi, skb);
		else
			netif_receive_skb(skb);
990 991

next:
992 993 994
		for (nr = 0; nr < priv->num_frags; nr++)
			mlx4_en_free_frag(priv, frags, nr);

995 996
		++cq->mcq.cons_index;
		index = (cq->mcq.cons_index) & ring->size_mask;
997
		cqe = mlx4_en_get_cqe(cq->buf, index, priv->cqe_size) + factor;
998
		if (++polled == budget)
999 1000 1001 1002 1003 1004 1005 1006
			goto out;
	}

out:
	AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled);
	mlx4_cq_set_ci(&cq->mcq);
	wmb(); /* ensure HW sees CQ consumer before we post new buffers */
	ring->cons = cq->mcq.cons_index;
1007
	mlx4_en_refill_rx_buffers(priv, ring);
1008 1009 1010 1011 1012 1013 1014 1015 1016 1017
	mlx4_en_update_rx_prod_db(ring);
	return polled;
}


void mlx4_en_rx_irq(struct mlx4_cq *mcq)
{
	struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
	struct mlx4_en_priv *priv = netdev_priv(cq->dev);

E
Eric Dumazet 已提交
1018 1019
	if (likely(priv->port_up))
		napi_schedule_irqoff(&cq->napi);
1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
	else
		mlx4_en_arm_cq(priv, cq);
}

/* Rx CQ polling - called by NAPI */
int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget)
{
	struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
	struct net_device *dev = cq->dev;
	struct mlx4_en_priv *priv = netdev_priv(dev);
	int done;

1032 1033 1034
	if (!mlx4_en_cq_lock_napi(cq))
		return budget;

1035 1036
	done = mlx4_en_process_rx_cq(dev, cq, budget);

1037 1038
	mlx4_en_cq_unlock_napi(cq);

1039
	/* If we used up all the quota - we're probably not done yet... */
1040
	if (done == budget) {
1041 1042 1043
		int cpu_curr;
		const struct cpumask *aff;

1044
		INC_PERF_COUNTER(priv->pstats.napi_quota);
1045 1046 1047 1048

		cpu_curr = smp_processor_id();
		aff = irq_desc_get_irq_data(cq->irq_desc)->affinity;

1049 1050 1051 1052 1053 1054 1055 1056
		if (likely(cpumask_test_cpu(cpu_curr, aff)))
			return budget;

		/* Current cpu is not according to smp_irq_affinity -
		 * probably affinity changed. need to stop this NAPI
		 * poll, and restart it on the right CPU
		 */
		done = 0;
1057
	}
E
Eric Dumazet 已提交
1058 1059 1060
	/* Done for now */
	napi_complete_done(napi, done);
	mlx4_en_arm_cq(priv, cq);
1061 1062 1063
	return done;
}

1064
static const int frag_sizes[] = {
1065 1066 1067 1068 1069 1070 1071 1072 1073
	FRAG_SZ0,
	FRAG_SZ1,
	FRAG_SZ2,
	FRAG_SZ3
};

void mlx4_en_calc_rx_buf(struct net_device *dev)
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
1074
	int eff_mtu = dev->mtu + ETH_HLEN + VLAN_HLEN;
1075 1076 1077 1078 1079 1080 1081 1082
	int buf_size = 0;
	int i = 0;

	while (buf_size < eff_mtu) {
		priv->frag_info[i].frag_size =
			(eff_mtu > buf_size + frag_sizes[i]) ?
				frag_sizes[i] : eff_mtu - buf_size;
		priv->frag_info[i].frag_prefix_size = buf_size;
1083 1084 1085
		priv->frag_info[i].frag_stride =
				ALIGN(priv->frag_info[i].frag_size,
				      SMP_CACHE_BYTES);
1086 1087 1088 1089 1090 1091
		buf_size += priv->frag_info[i].frag_size;
		i++;
	}

	priv->num_frags = i;
	priv->rx_skb_size = eff_mtu;
1092
	priv->log_rx_info = ROUNDUP_LOG2(i * sizeof(struct mlx4_en_rx_alloc));
1093

J
Joe Perches 已提交
1094 1095
	en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d num_frags:%d):\n",
	       eff_mtu, priv->num_frags);
1096
	for (i = 0; i < priv->num_frags; i++) {
1097
		en_err(priv,
1098
		       "  frag:%d - size:%d prefix:%d stride:%d\n",
1099 1100 1101 1102
		       i,
		       priv->frag_info[i].frag_size,
		       priv->frag_info[i].frag_prefix_size,
		       priv->frag_info[i].frag_stride);
1103 1104 1105 1106 1107
	}
}

/* RSS related functions */

1108 1109
static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn,
				 struct mlx4_en_rx_ring *ring,
1110 1111 1112 1113 1114 1115 1116
				 enum mlx4_qp_state *state,
				 struct mlx4_qp *qp)
{
	struct mlx4_en_dev *mdev = priv->mdev;
	struct mlx4_qp_context *context;
	int err = 0;

1117 1118
	context = kmalloc(sizeof(*context), GFP_KERNEL);
	if (!context)
1119 1120
		return -ENOMEM;

1121
	err = mlx4_qp_alloc(mdev->dev, qpn, qp, GFP_KERNEL);
1122
	if (err) {
1123
		en_err(priv, "Failed to allocate qp #%x\n", qpn);
1124 1125 1126 1127 1128
		goto out;
	}
	qp->event = mlx4_en_sqp_event;

	memset(context, 0, sizeof *context);
1129
	mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0,
1130
				qpn, ring->cqn, -1, context);
1131
	context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma);
1132

1133
	/* Cancel FCS removal if FW allows */
1134
	if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) {
1135
		context->param3 |= cpu_to_be32(1 << 29);
1136 1137 1138 1139
		if (priv->dev->features & NETIF_F_RXFCS)
			ring->fcs_del = 0;
		else
			ring->fcs_del = ETH_FCS_LEN;
1140 1141
	} else
		ring->fcs_del = 0;
1142

1143
	err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state);
1144 1145 1146 1147
	if (err) {
		mlx4_qp_remove(mdev->dev, qp);
		mlx4_qp_free(mdev->dev, qp);
	}
1148
	mlx4_en_update_rx_prod_db(ring);
1149 1150 1151 1152 1153
out:
	kfree(context);
	return err;
}

1154 1155 1156 1157 1158
int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv)
{
	int err;
	u32 qpn;

M
Matan Barak 已提交
1159 1160
	err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn,
				    MLX4_RESERVE_A0_QP);
1161 1162 1163 1164
	if (err) {
		en_err(priv, "Failed reserving drop qpn\n");
		return err;
	}
1165
	err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp, GFP_KERNEL);
1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184
	if (err) {
		en_err(priv, "Failed allocating drop qp\n");
		mlx4_qp_release_range(priv->mdev->dev, qpn, 1);
		return err;
	}

	return 0;
}

void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv)
{
	u32 qpn;

	qpn = priv->drop_qp.qpn;
	mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp);
	mlx4_qp_free(priv->mdev->dev, &priv->drop_qp);
	mlx4_qp_release_range(priv->mdev->dev, qpn, 1);
}

1185 1186 1187 1188 1189 1190
/* Allocate rx qp's and configure them according to rss map */
int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
{
	struct mlx4_en_dev *mdev = priv->mdev;
	struct mlx4_en_rss_map *rss_map = &priv->rss_map;
	struct mlx4_qp_context context;
1191
	struct mlx4_rss_context *rss_context;
1192
	int rss_rings;
1193
	void *ptr;
1194
	u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 |
1195
			MLX4_RSS_TCP_IPV6);
1196
	int i, qpn;
1197 1198 1199
	int err = 0;
	int good_qps = 0;

1200
	en_dbg(DRV, priv, "Configuring rss steering\n");
1201 1202
	err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,
				    priv->rx_ring_num,
1203
				    &rss_map->base_qpn, 0);
1204
	if (err) {
1205
		en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);
1206 1207 1208
		return err;
	}

1209
	for (i = 0; i < priv->rx_ring_num; i++) {
1210
		qpn = rss_map->base_qpn + i;
1211
		err = mlx4_en_config_rss_qp(priv, qpn, priv->rx_ring[i],
1212 1213 1214 1215 1216 1217 1218 1219 1220
					    &rss_map->state[i],
					    &rss_map->qps[i]);
		if (err)
			goto rss_err;

		++good_qps;
	}

	/* Configure RSS indirection qp */
1221
	err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp, GFP_KERNEL);
1222
	if (err) {
1223
		en_err(priv, "Failed to allocate RSS indirection QP\n");
1224
		goto rss_err;
1225 1226 1227
	}
	rss_map->indir_qp.event = mlx4_en_sqp_event;
	mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn,
1228
				priv->rx_ring[0]->cqn, -1, &context);
1229

1230 1231 1232 1233 1234
	if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num)
		rss_rings = priv->rx_ring_num;
	else
		rss_rings = priv->prof->rss_rings;

1235 1236
	ptr = ((void *) &context) + offsetof(struct mlx4_qp_context, pri_path)
					+ MLX4_RSS_OFFSET_IN_QPC_PRI_PATH;
1237
	rss_context = ptr;
1238
	rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 |
1239
					    (rss_map->base_qpn));
1240
	rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn);
1241 1242 1243 1244
	if (priv->mdev->profile.udp_rss) {
		rss_mask |=  MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6;
		rss_context->base_qpn_udp = rss_context->default_qpn;
	}
1245 1246 1247 1248 1249 1250

	if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
		en_info(priv, "Setting RSS context tunnel type to RSS on inner headers\n");
		rss_mask |= MLX4_RSS_BY_INNER_HEADERS;
	}

Y
Yevgeny Petrilin 已提交
1251
	rss_context->flags = rss_mask;
1252
	rss_context->hash_fn = MLX4_RSS_HASH_TOP;
1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265
	if (priv->rss_hash_fn == ETH_RSS_HASH_XOR) {
		rss_context->hash_fn = MLX4_RSS_HASH_XOR;
	} else if (priv->rss_hash_fn == ETH_RSS_HASH_TOP) {
		rss_context->hash_fn = MLX4_RSS_HASH_TOP;
		memcpy(rss_context->rss_key, priv->rss_key,
		       MLX4_EN_RSS_KEY_SIZE);
		netdev_rss_key_fill(rss_context->rss_key,
				    MLX4_EN_RSS_KEY_SIZE);
	} else {
		en_err(priv, "Unknown RSS hash function requested\n");
		err = -EINVAL;
		goto indir_err;
	}
1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284
	err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context,
			       &rss_map->indir_qp, &rss_map->indir_state);
	if (err)
		goto indir_err;

	return 0;

indir_err:
	mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
		       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
	mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
	mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
rss_err:
	for (i = 0; i < good_qps; i++) {
		mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
			       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
		mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
		mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
	}
1285
	mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299
	return err;
}

void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv)
{
	struct mlx4_en_dev *mdev = priv->mdev;
	struct mlx4_en_rss_map *rss_map = &priv->rss_map;
	int i;

	mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
		       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
	mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
	mlx4_qp_free(mdev->dev, &rss_map->indir_qp);

1300
	for (i = 0; i < priv->rx_ring_num; i++) {
1301 1302 1303 1304 1305
		mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
			       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
		mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
		mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
	}
1306
	mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
1307
}