en_rx.c 28.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
/*
 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#include <linux/mlx4/cq.h>
35
#include <linux/slab.h>
36 37 38 39 40 41 42 43
#include <linux/mlx4/qp.h>
#include <linux/skbuff.h>
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <linux/vmalloc.h>

#include "mlx4_en.h"

44 45 46 47
static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv,
			       struct mlx4_en_rx_desc *rx_desc,
			       struct mlx4_en_rx_alloc *frags,
			       struct mlx4_en_rx_alloc *ring_alloc)
48
{
49 50
	struct mlx4_en_rx_alloc page_alloc[MLX4_EN_MAX_RX_FRAGS];
	struct mlx4_en_frag_info *frag_info;
51 52
	struct page *page;
	dma_addr_t dma;
53
	int i;
54

55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
	for (i = 0; i < priv->num_frags; i++) {
		frag_info = &priv->frag_info[i];
		if (ring_alloc[i].offset == frag_info->last_offset) {
			page = alloc_pages(GFP_ATOMIC | __GFP_COMP,
					MLX4_EN_ALLOC_ORDER);
			if (!page)
				goto out;
			dma = dma_map_page(priv->ddev, page, 0,
				MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE);
			if (dma_mapping_error(priv->ddev, dma)) {
				put_page(page);
				goto out;
			}
			page_alloc[i].page = page;
			page_alloc[i].dma = dma;
			page_alloc[i].offset = frag_info->frag_align;
		} else {
			page_alloc[i].page = ring_alloc[i].page;
			get_page(ring_alloc[i].page);
			page_alloc[i].dma = ring_alloc[i].dma;
			page_alloc[i].offset = ring_alloc[i].offset +
						frag_info->frag_stride;
		}
	}
79

80 81 82 83 84
	for (i = 0; i < priv->num_frags; i++) {
		frags[i] = ring_alloc[i];
		dma = ring_alloc[i].dma + ring_alloc[i].offset;
		ring_alloc[i] = page_alloc[i];
		rx_desc->data[i].addr = cpu_to_be64(dma);
85
	}
86

87
	return 0;
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112


out:
	while (i--) {
		frag_info = &priv->frag_info[i];
		if (ring_alloc[i].offset == frag_info->last_offset)
			dma_unmap_page(priv->ddev, page_alloc[i].dma,
				MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE);
		put_page(page_alloc[i].page);
	}
	return -ENOMEM;
}

static void mlx4_en_free_frag(struct mlx4_en_priv *priv,
			      struct mlx4_en_rx_alloc *frags,
			      int i)
{
	struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];

	if (frags[i].offset == frag_info->last_offset) {
		dma_unmap_page(priv->ddev, frags[i].dma, MLX4_EN_ALLOC_SIZE,
					 PCI_DMA_FROMDEVICE);
	}
	if (frags[i].page)
		put_page(frags[i].page);
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
}

static int mlx4_en_init_allocator(struct mlx4_en_priv *priv,
				  struct mlx4_en_rx_ring *ring)
{
	struct mlx4_en_rx_alloc *page_alloc;
	int i;

	for (i = 0; i < priv->num_frags; i++) {
		page_alloc = &ring->page_alloc[i];
		page_alloc->page = alloc_pages(GFP_ATOMIC | __GFP_COMP,
					       MLX4_EN_ALLOC_ORDER);
		if (!page_alloc->page)
			goto out;

128 129 130 131 132 133 134
		page_alloc->dma = dma_map_page(priv->ddev, page_alloc->page, 0,
					MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE);
		if (dma_mapping_error(priv->ddev, page_alloc->dma)) {
			put_page(page_alloc->page);
			page_alloc->page = NULL;
			goto out;
		}
135
		page_alloc->offset = priv->frag_info[i].frag_align;
136 137
		en_dbg(DRV, priv, "Initialized allocator:%d with page:%p\n",
		       i, page_alloc->page);
138 139 140 141 142 143
	}
	return 0;

out:
	while (i--) {
		page_alloc = &ring->page_alloc[i];
144 145
		dma_unmap_page(priv->ddev, page_alloc->dma,
				MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE);
146 147 148 149 150 151 152 153 154 155 156 157 158 159
		put_page(page_alloc->page);
		page_alloc->page = NULL;
	}
	return -ENOMEM;
}

static void mlx4_en_destroy_allocator(struct mlx4_en_priv *priv,
				      struct mlx4_en_rx_ring *ring)
{
	struct mlx4_en_rx_alloc *page_alloc;
	int i;

	for (i = 0; i < priv->num_frags; i++) {
		page_alloc = &ring->page_alloc[i];
160 161
		en_dbg(DRV, priv, "Freeing allocator:%d count:%d\n",
		       i, page_count(page_alloc->page));
162

163 164
		dma_unmap_page(priv->ddev, page_alloc->dma,
				MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE);
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
		put_page(page_alloc->page);
		page_alloc->page = NULL;
	}
}

static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv,
				 struct mlx4_en_rx_ring *ring, int index)
{
	struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index;
	int possible_frags;
	int i;

	/* Set size and memtype fields */
	for (i = 0; i < priv->num_frags; i++) {
		rx_desc->data[i].byte_count =
			cpu_to_be32(priv->frag_info[i].frag_size);
		rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key);
	}

	/* If the number of used fragments does not fill up the ring stride,
	 * remaining (unused) fragments must be padded with null address/size
	 * and a special memory key */
	possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE;
	for (i = priv->num_frags; i < possible_frags; i++) {
		rx_desc->data[i].byte_count = 0;
		rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD);
		rx_desc->data[i].addr = 0;
	}
}

static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv,
				   struct mlx4_en_rx_ring *ring, int index)
{
	struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride);
199 200
	struct mlx4_en_rx_alloc *frags = ring->rx_info +
					(index << priv->log_rx_info);
201

202
	return mlx4_en_alloc_frags(priv, rx_desc, frags, ring->page_alloc);
203 204 205 206 207 208 209
}

static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring)
{
	*ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff);
}

210 211 212 213
static void mlx4_en_free_rx_desc(struct mlx4_en_priv *priv,
				 struct mlx4_en_rx_ring *ring,
				 int index)
{
214
	struct mlx4_en_rx_alloc *frags;
215 216
	int nr;

217
	frags = ring->rx_info + (index << priv->log_rx_info);
218
	for (nr = 0; nr < priv->num_frags; nr++) {
219
		en_dbg(DRV, priv, "Freeing fragment:%d\n", nr);
220
		mlx4_en_free_frag(priv, frags, nr);
221 222 223
	}
}

224 225 226 227 228
static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv)
{
	struct mlx4_en_rx_ring *ring;
	int ring_ind;
	int buf_ind;
229
	int new_size;
230 231 232 233 234 235 236 237

	for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) {
		for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
			ring = &priv->rx_ring[ring_ind];

			if (mlx4_en_prepare_rx_desc(priv, ring,
						    ring->actual_size)) {
				if (ring->actual_size < MLX4_EN_MIN_RX_SIZE) {
238 239
					en_err(priv, "Failed to allocate "
						     "enough rx buffers\n");
240 241
					return -ENOMEM;
				} else {
242
					new_size = rounddown_pow_of_two(ring->actual_size);
243 244 245
					en_warn(priv, "Only %d buffers allocated "
						      "reducing ring size to %d",
						ring->actual_size, new_size);
246
					goto reduce_rings;
247 248 249 250 251 252
				}
			}
			ring->actual_size++;
			ring->prod++;
		}
	}
253 254 255 256 257 258 259 260 261 262 263 264
	return 0;

reduce_rings:
	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
		ring = &priv->rx_ring[ring_ind];
		while (ring->actual_size > new_size) {
			ring->actual_size--;
			ring->prod--;
			mlx4_en_free_rx_desc(priv, ring, ring->actual_size);
		}
	}

265 266 267 268 269 270 271 272
	return 0;
}

static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv,
				struct mlx4_en_rx_ring *ring)
{
	int index;

273 274
	en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n",
	       ring->cons, ring->prod);
275 276

	/* Unmap and free Rx buffers */
277
	BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size);
278 279
	while (ring->cons != ring->prod) {
		index = ring->cons & ring->size_mask;
280
		en_dbg(DRV, priv, "Processing descriptor:%d\n", index);
281
		mlx4_en_free_rx_desc(priv, ring, index);
282 283 284 285 286 287 288 289
		++ring->cons;
	}
}

int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
			   struct mlx4_en_rx_ring *ring, u32 size, u16 stride)
{
	struct mlx4_en_dev *mdev = priv->mdev;
290
	int err = -ENOMEM;
291 292 293 294 295 296 297 298
	int tmp;

	ring->prod = 0;
	ring->cons = 0;
	ring->size = size;
	ring->size_mask = size - 1;
	ring->stride = stride;
	ring->log_stride = ffs(ring->stride) - 1;
299
	ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
300 301

	tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
302
					sizeof(struct mlx4_en_rx_alloc));
303
	ring->rx_info = vmalloc(tmp);
304
	if (!ring->rx_info)
305
		return -ENOMEM;
306

307
	en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n",
308 309 310 311 312 313 314 315 316
		 ring->rx_info, tmp);

	err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres,
				 ring->buf_size, 2 * PAGE_SIZE);
	if (err)
		goto err_ring;

	err = mlx4_en_map_buffer(&ring->wqres.buf);
	if (err) {
317
		en_err(priv, "Failed to map RX buffer\n");
318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
		goto err_hwq;
	}
	ring->buf = ring->wqres.buf.direct.buf;

	return 0;

err_hwq:
	mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_ring:
	vfree(ring->rx_info);
	ring->rx_info = NULL;
	return err;
}

int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
{
	struct mlx4_en_rx_ring *ring;
	int i;
	int ring_ind;
	int err;
	int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
					DS_SIZE * priv->num_frags);

	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
		ring = &priv->rx_ring[ring_ind];

		ring->prod = 0;
		ring->cons = 0;
		ring->actual_size = 0;
		ring->cqn = priv->rx_cq[ring_ind].mcq.cqn;

		ring->stride = stride;
350 351 352
		if (ring->stride <= TXBB_SIZE)
			ring->buf += TXBB_SIZE;

353 354 355 356 357 358
		ring->log_stride = ffs(ring->stride) - 1;
		ring->buf_size = ring->size * ring->stride;

		memset(ring->buf, 0, ring->buf_size);
		mlx4_en_update_rx_prod_db(ring);

359
		/* Initialize all descriptors */
360 361 362 363 364 365
		for (i = 0; i < ring->size; i++)
			mlx4_en_init_rx_desc(priv, ring, i);

		/* Initialize page allocators */
		err = mlx4_en_init_allocator(priv, ring);
		if (err) {
366
			en_err(priv, "Failed initializing ring allocator\n");
367 368
			if (ring->stride <= TXBB_SIZE)
				ring->buf -= TXBB_SIZE;
369 370
			ring_ind--;
			goto err_allocator;
371 372
		}
	}
373 374
	err = mlx4_en_fill_rx_buffers(priv);
	if (err)
375 376 377 378 379
		goto err_buffers;

	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
		ring = &priv->rx_ring[ring_ind];

380
		ring->size_mask = ring->actual_size - 1;
381 382 383 384 385 386 387 388 389 390 391 392
		mlx4_en_update_rx_prod_db(ring);
	}

	return 0;

err_buffers:
	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++)
		mlx4_en_free_rx_buf(priv, &priv->rx_ring[ring_ind]);

	ring_ind = priv->rx_ring_num - 1;
err_allocator:
	while (ring_ind >= 0) {
393 394
		if (priv->rx_ring[ring_ind].stride <= TXBB_SIZE)
			priv->rx_ring[ring_ind].buf -= TXBB_SIZE;
395 396 397 398 399 400 401
		mlx4_en_destroy_allocator(priv, &priv->rx_ring[ring_ind]);
		ring_ind--;
	}
	return err;
}

void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
402
			     struct mlx4_en_rx_ring *ring, u32 size, u16 stride)
403 404 405 406
{
	struct mlx4_en_dev *mdev = priv->mdev;

	mlx4_en_unmap_buffer(&ring->wqres.buf);
407
	mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE);
408 409
	vfree(ring->rx_info);
	ring->rx_info = NULL;
410 411 412
#ifdef CONFIG_RFS_ACCEL
	mlx4_en_cleanup_filters(priv, ring);
#endif
413 414 415 416 417 418
}

void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
				struct mlx4_en_rx_ring *ring)
{
	mlx4_en_free_rx_buf(priv, ring);
419 420
	if (ring->stride <= TXBB_SIZE)
		ring->buf -= TXBB_SIZE;
421 422 423 424 425 426
	mlx4_en_destroy_allocator(priv, ring);
}


static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv,
				    struct mlx4_en_rx_desc *rx_desc,
427
				    struct mlx4_en_rx_alloc *frags,
428
				    struct sk_buff *skb,
429 430
				    int length)
{
431
	struct skb_frag_struct *skb_frags_rx = skb_shinfo(skb)->frags;
432 433 434 435
	struct mlx4_en_frag_info *frag_info;
	int nr;
	dma_addr_t dma;

436
	/* Collect used fragments while replacing them in the HW descriptors */
437 438 439 440
	for (nr = 0; nr < priv->num_frags; nr++) {
		frag_info = &priv->frag_info[nr];
		if (length <= frag_info->frag_prefix_size)
			break;
441 442
		if (!frags[nr].page)
			goto fail;
443 444

		dma = be64_to_cpu(rx_desc->data[nr].addr);
445 446
		dma_sync_single_for_cpu(priv->ddev, dma, frag_info->frag_size,
					DMA_FROM_DEVICE);
447

448 449 450 451 452 453
		/* Save page reference in skb */
		get_page(frags[nr].page);
		__skb_frag_set_page(&skb_frags_rx[nr], frags[nr].page);
		skb_frag_size_set(&skb_frags_rx[nr], frag_info->frag_size);
		skb_frags_rx[nr].page_offset = frags[nr].offset;
		skb->truesize += frag_info->frag_stride;
454 455
	}
	/* Adjust size of last fragment to match actual length */
456
	if (nr > 0)
E
Eric Dumazet 已提交
457 458
		skb_frag_size_set(&skb_frags_rx[nr - 1],
			length - priv->frag_info[nr - 1].frag_prefix_size);
459 460 461 462 463
	return nr;

fail:
	while (nr > 0) {
		nr--;
464
		__skb_frag_unref(&skb_frags_rx[nr]);
465 466 467 468 469 470 471
	}
	return 0;
}


static struct sk_buff *mlx4_en_rx_skb(struct mlx4_en_priv *priv,
				      struct mlx4_en_rx_desc *rx_desc,
472
				      struct mlx4_en_rx_alloc *frags,
473 474 475 476 477 478 479
				      unsigned int length)
{
	struct sk_buff *skb;
	void *va;
	int used_frags;
	dma_addr_t dma;

480
	skb = netdev_alloc_skb(priv->dev, SMALL_PACKET_SIZE + NET_IP_ALIGN);
481
	if (!skb) {
482
		en_dbg(RX_ERR, priv, "Failed allocating skb\n");
483 484 485 486 487 488 489
		return NULL;
	}
	skb_reserve(skb, NET_IP_ALIGN);
	skb->len = length;

	/* Get pointer to first fragment so we could copy the headers into the
	 * (linear part of the) skb */
490
	va = page_address(frags[0].page) + frags[0].offset;
491 492 493

	if (length <= SMALL_PACKET_SIZE) {
		/* We are copying all relevant data to the skb - temporarily
494
		 * sync buffers for the copy */
495
		dma = be64_to_cpu(rx_desc->data[0].addr);
496
		dma_sync_single_for_cpu(priv->ddev, dma, length,
497
					DMA_FROM_DEVICE);
498 499 500 501
		skb_copy_to_linear_data(skb, va, length);
		skb->tail += length;
	} else {
		/* Move relevant fragments to skb */
502 503
		used_frags = mlx4_en_complete_rx_desc(priv, rx_desc, frags,
							skb, length);
504 505 506 507
		if (unlikely(!used_frags)) {
			kfree_skb(skb);
			return NULL;
		}
508 509 510 511 512 513 514 515 516 517
		skb_shinfo(skb)->nr_frags = used_frags;

		/* Copy headers into the skb linear buffer */
		memcpy(skb->data, va, HEADER_COPY_SIZE);
		skb->tail += HEADER_COPY_SIZE;

		/* Skip headers in first fragment */
		skb_shinfo(skb)->frags[0].page_offset += HEADER_COPY_SIZE;

		/* Adjust size of first fragment */
E
Eric Dumazet 已提交
518
		skb_frag_size_sub(&skb_shinfo(skb)->frags[0], HEADER_COPY_SIZE);
519 520 521 522 523
		skb->data_len = length - HEADER_COPY_SIZE;
	}
	return skb;
}

524 525 526 527 528 529 530 531 532 533 534 535 536 537 538
static void validate_loopback(struct mlx4_en_priv *priv, struct sk_buff *skb)
{
	int i;
	int offset = ETH_HLEN;

	for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) {
		if (*(skb->data + offset) != (unsigned char) (i & 0xff))
			goto out_loopback;
	}
	/* Loopback found */
	priv->loopback_ok = 1;

out_loopback:
	dev_kfree_skb_any(skb);
}
539

540 541 542 543 544 545 546 547 548 549 550 551 552
static void mlx4_en_refill_rx_buffers(struct mlx4_en_priv *priv,
				     struct mlx4_en_rx_ring *ring)
{
	int index = ring->prod & ring->size_mask;

	while ((u32) (ring->prod - ring->cons) < ring->actual_size) {
		if (mlx4_en_prepare_rx_desc(priv, ring, index))
			break;
		ring->prod++;
		index = ring->prod & ring->size_mask;
	}
}

553 554 555 556 557
int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget)
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
	struct mlx4_cqe *cqe;
	struct mlx4_en_rx_ring *ring = &priv->rx_ring[cq->ring];
558
	struct mlx4_en_rx_alloc *frags;
559 560 561 562 563 564 565
	struct mlx4_en_rx_desc *rx_desc;
	struct sk_buff *skb;
	int index;
	int nr;
	unsigned int length;
	int polled = 0;
	int ip_summed;
O
Or Gerlitz 已提交
566
	int factor = priv->cqe_factor;
567 568 569 570 571 572 573 574

	if (!priv->port_up)
		return 0;

	/* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
	 * descriptor offset can be deduced from the CQE index instead of
	 * reading 'cqe->index' */
	index = cq->mcq.cons_index & ring->size_mask;
O
Or Gerlitz 已提交
575
	cqe = &cq->buf[(index << factor) + factor];
576 577 578 579 580

	/* Process all completed CQEs */
	while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
		    cq->mcq.cons_index & cq->size)) {

581
		frags = ring->rx_info + (index << priv->log_rx_info);
582 583 584 585 586 587 588 589 590 591
		rx_desc = ring->buf + (index << ring->log_stride);

		/*
		 * make sure we read the CQE after we read the ownership bit
		 */
		rmb();

		/* Drop packet on bad receive or bad checksum */
		if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
						MLX4_CQE_OPCODE_ERROR)) {
592
			en_err(priv, "CQE completed in error - vendor "
593 594 595 596 597 598
				  "syndrom:%d syndrom:%d\n",
				  ((struct mlx4_err_cqe *) cqe)->vendor_err_syndrome,
				  ((struct mlx4_err_cqe *) cqe)->syndrome);
			goto next;
		}
		if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {
599
			en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");
600 601 602
			goto next;
		}

603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
		/* Check if we need to drop the packet if SRIOV is not enabled
		 * and not performing the selftest or flb disabled
		 */
		if (priv->flags & MLX4_EN_FLAG_RX_FILTER_NEEDED) {
			struct ethhdr *ethh;
			dma_addr_t dma;
			/* Get pointer to first fragment since we haven't
			 * skb yet and cast it to ethhdr struct
			 */
			dma = be64_to_cpu(rx_desc->data[0].addr);
			dma_sync_single_for_cpu(priv->ddev, dma, sizeof(*ethh),
						DMA_FROM_DEVICE);
			ethh = (struct ethhdr *)(page_address(frags[0].page) +
						 frags[0].offset);

618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636
			if (is_multicast_ether_addr(ethh->h_dest)) {
				struct mlx4_mac_entry *entry;
				struct hlist_node *n;
				struct hlist_head *bucket;
				unsigned int mac_hash;

				/* Drop the packet, since HW loopback-ed it */
				mac_hash = ethh->h_source[MLX4_EN_MAC_HASH_IDX];
				bucket = &priv->mac_hash[mac_hash];
				rcu_read_lock();
				hlist_for_each_entry_rcu(entry, n, bucket, hlist) {
					if (ether_addr_equal_64bits(entry->mac,
								    ethh->h_source)) {
						rcu_read_unlock();
						goto next;
					}
				}
				rcu_read_unlock();
			}
637
		}
638

639 640 641 642
		/*
		 * Packet is OK - process it.
		 */
		length = be32_to_cpu(cqe->byte_cnt);
643
		length -= ring->fcs_del;
644 645 646
		ring->bytes += length;
		ring->packets++;

647
		if (likely(dev->features & NETIF_F_RXCSUM)) {
648 649
			if ((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
			    (cqe->checksum == cpu_to_be16(0xffff))) {
650
				ring->csum_ok++;
651
				/* This packet is eligible for GRO if it is:
652 653 654 655
				 * - DIX Ethernet (type interpretation)
				 * - TCP/IP (v4)
				 * - without IP options
				 * - not an IP fragment */
656 657
				if (dev->features & NETIF_F_GRO) {
					struct sk_buff *gro_skb = napi_get_frags(&cq->napi);
658 659
					if (!gro_skb)
						goto next;
660

661 662 663
					nr = mlx4_en_complete_rx_desc(priv,
						rx_desc, frags, gro_skb,
						length);
664 665 666
					if (!nr)
						goto next;

667 668 669 670 671
					skb_shinfo(gro_skb)->nr_frags = nr;
					gro_skb->len = length;
					gro_skb->data_len = length;
					gro_skb->ip_summed = CHECKSUM_UNNECESSARY;

J
Jiri Pirko 已提交
672 673 674 675 676 677 678
					if (cqe->vlan_my_qpn &
					    cpu_to_be32(MLX4_CQE_VLAN_PRESENT_MASK)) {
						u16 vid = be16_to_cpu(cqe->sl_vid);

						__vlan_hwaccel_put_tag(gro_skb, vid);
					}

Y
Yevgeny Petrilin 已提交
679 680 681
					if (dev->features & NETIF_F_RXHASH)
						gro_skb->rxhash = be32_to_cpu(cqe->immed_rss_invalid);

682
					skb_record_rx_queue(gro_skb, cq->ring);
J
Jiri Pirko 已提交
683
					napi_gro_frags(&cq->napi);
684 685 686 687

					goto next;
				}

688
				/* GRO not possible, complete processing here */
689 690 691
				ip_summed = CHECKSUM_UNNECESSARY;
			} else {
				ip_summed = CHECKSUM_NONE;
692
				ring->csum_none++;
693 694 695
			}
		} else {
			ip_summed = CHECKSUM_NONE;
696
			ring->csum_none++;
697 698
		}

699
		skb = mlx4_en_rx_skb(priv, rx_desc, frags, length);
700 701 702 703 704
		if (!skb) {
			priv->stats.rx_dropped++;
			goto next;
		}

705 706 707 708 709
                if (unlikely(priv->validate_loopback)) {
			validate_loopback(priv, skb);
			goto next;
		}

710 711
		skb->ip_summed = ip_summed;
		skb->protocol = eth_type_trans(skb, dev);
712
		skb_record_rx_queue(skb, cq->ring);
713

Y
Yevgeny Petrilin 已提交
714 715 716
		if (dev->features & NETIF_F_RXHASH)
			skb->rxhash = be32_to_cpu(cqe->immed_rss_invalid);

J
Jiri Pirko 已提交
717 718 719 720
		if (be32_to_cpu(cqe->vlan_my_qpn) &
		    MLX4_CQE_VLAN_PRESENT_MASK)
			__vlan_hwaccel_put_tag(skb, be16_to_cpu(cqe->sl_vid));

721
		/* Push it up the stack */
J
Jiri Pirko 已提交
722
		netif_receive_skb(skb);
723 724

next:
725 726 727
		for (nr = 0; nr < priv->num_frags; nr++)
			mlx4_en_free_frag(priv, frags, nr);

728 729
		++cq->mcq.cons_index;
		index = (cq->mcq.cons_index) & ring->size_mask;
O
Or Gerlitz 已提交
730
		cqe = &cq->buf[(index << factor) + factor];
731
		if (++polled == budget)
732 733 734 735 736 737 738 739
			goto out;
	}

out:
	AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled);
	mlx4_cq_set_ci(&cq->mcq);
	wmb(); /* ensure HW sees CQ consumer before we post new buffers */
	ring->cons = cq->mcq.cons_index;
740
	mlx4_en_refill_rx_buffers(priv, ring);
741 742 743 744 745 746 747 748 749 750 751
	mlx4_en_update_rx_prod_db(ring);
	return polled;
}


void mlx4_en_rx_irq(struct mlx4_cq *mcq)
{
	struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
	struct mlx4_en_priv *priv = netdev_priv(cq->dev);

	if (priv->port_up)
752
		napi_schedule(&cq->napi);
753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771
	else
		mlx4_en_arm_cq(priv, cq);
}

/* Rx CQ polling - called by NAPI */
int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget)
{
	struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
	struct net_device *dev = cq->dev;
	struct mlx4_en_priv *priv = netdev_priv(dev);
	int done;

	done = mlx4_en_process_rx_cq(dev, cq, budget);

	/* If we used up all the quota - we're probably not done yet... */
	if (done == budget)
		INC_PERF_COUNTER(priv->pstats.napi_quota);
	else {
		/* Done for now */
772
		napi_complete(napi);
773 774 775 776 777 778
		mlx4_en_arm_cq(priv, cq);
	}
	return done;
}


L
Lucas De Marchi 已提交
779
/* Calculate the last offset position that accommodates a full fragment
780 781 782 783 784 785
 * (assuming fagment size = stride-align) */
static int mlx4_en_last_alloc_offset(struct mlx4_en_priv *priv, u16 stride, u16 align)
{
	u16 res = MLX4_EN_ALLOC_SIZE % stride;
	u16 offset = MLX4_EN_ALLOC_SIZE - stride - res + align;

786
	en_dbg(DRV, priv, "Calculated last offset for stride:%d align:%d "
787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828
			    "res:%d offset:%d\n", stride, align, res, offset);
	return offset;
}


static int frag_sizes[] = {
	FRAG_SZ0,
	FRAG_SZ1,
	FRAG_SZ2,
	FRAG_SZ3
};

void mlx4_en_calc_rx_buf(struct net_device *dev)
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
	int eff_mtu = dev->mtu + ETH_HLEN + VLAN_HLEN + ETH_LLC_SNAP_SIZE;
	int buf_size = 0;
	int i = 0;

	while (buf_size < eff_mtu) {
		priv->frag_info[i].frag_size =
			(eff_mtu > buf_size + frag_sizes[i]) ?
				frag_sizes[i] : eff_mtu - buf_size;
		priv->frag_info[i].frag_prefix_size = buf_size;
		if (!i)	{
			priv->frag_info[i].frag_align = NET_IP_ALIGN;
			priv->frag_info[i].frag_stride =
				ALIGN(frag_sizes[i] + NET_IP_ALIGN, SMP_CACHE_BYTES);
		} else {
			priv->frag_info[i].frag_align = 0;
			priv->frag_info[i].frag_stride =
				ALIGN(frag_sizes[i], SMP_CACHE_BYTES);
		}
		priv->frag_info[i].last_offset = mlx4_en_last_alloc_offset(
						priv, priv->frag_info[i].frag_stride,
						priv->frag_info[i].frag_align);
		buf_size += priv->frag_info[i].frag_size;
		i++;
	}

	priv->num_frags = i;
	priv->rx_skb_size = eff_mtu;
829
	priv->log_rx_info = ROUNDUP_LOG2(i * sizeof(struct mlx4_en_rx_alloc));
830

831
	en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d "
832 833
		  "num_frags:%d):\n", eff_mtu, priv->num_frags);
	for (i = 0; i < priv->num_frags; i++) {
834
		en_dbg(DRV, priv, "  frag:%d - size:%d prefix:%d align:%d "
835 836 837 838 839 840 841 842 843 844 845
				"stride:%d last_offset:%d\n", i,
				priv->frag_info[i].frag_size,
				priv->frag_info[i].frag_prefix_size,
				priv->frag_info[i].frag_align,
				priv->frag_info[i].frag_stride,
				priv->frag_info[i].last_offset);
	}
}

/* RSS related functions */

846 847
static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn,
				 struct mlx4_en_rx_ring *ring,
848 849 850 851 852 853 854
				 enum mlx4_qp_state *state,
				 struct mlx4_qp *qp)
{
	struct mlx4_en_dev *mdev = priv->mdev;
	struct mlx4_qp_context *context;
	int err = 0;

855 856
	context = kmalloc(sizeof(*context), GFP_KERNEL);
	if (!context)
857 858 859 860
		return -ENOMEM;

	err = mlx4_qp_alloc(mdev->dev, qpn, qp);
	if (err) {
861
		en_err(priv, "Failed to allocate qp #%x\n", qpn);
862 863 864 865 866
		goto out;
	}
	qp->event = mlx4_en_sqp_event;

	memset(context, 0, sizeof *context);
867
	mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0,
868
				qpn, ring->cqn, -1, context);
869
	context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma);
870

871
	/* Cancel FCS removal if FW allows */
872
	if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) {
873
		context->param3 |= cpu_to_be32(1 << 29);
874 875 876
		ring->fcs_del = ETH_FCS_LEN;
	} else
		ring->fcs_del = 0;
877

878
	err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state);
879 880 881 882
	if (err) {
		mlx4_qp_remove(mdev->dev, qp);
		mlx4_qp_free(mdev->dev, qp);
	}
883
	mlx4_en_update_rx_prod_db(ring);
884 885 886 887 888
out:
	kfree(context);
	return err;
}

889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918
int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv)
{
	int err;
	u32 qpn;

	err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn);
	if (err) {
		en_err(priv, "Failed reserving drop qpn\n");
		return err;
	}
	err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp);
	if (err) {
		en_err(priv, "Failed allocating drop qp\n");
		mlx4_qp_release_range(priv->mdev->dev, qpn, 1);
		return err;
	}

	return 0;
}

void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv)
{
	u32 qpn;

	qpn = priv->drop_qp.qpn;
	mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp);
	mlx4_qp_free(priv->mdev->dev, &priv->drop_qp);
	mlx4_qp_release_range(priv->mdev->dev, qpn, 1);
}

919 920 921 922 923 924
/* Allocate rx qp's and configure them according to rss map */
int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
{
	struct mlx4_en_dev *mdev = priv->mdev;
	struct mlx4_en_rss_map *rss_map = &priv->rss_map;
	struct mlx4_qp_context context;
925
	struct mlx4_rss_context *rss_context;
926
	int rss_rings;
927
	void *ptr;
928
	u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 |
929
			MLX4_RSS_TCP_IPV6);
930
	int i, qpn;
931 932
	int err = 0;
	int good_qps = 0;
Y
Yevgeny Petrilin 已提交
933 934 935
	static const u32 rsskey[10] = { 0xD181C62C, 0xF7F4DB5B, 0x1983A2FC,
				0x943E1ADB, 0xD9389E6B, 0xD1039C2C, 0xA74499AD,
				0x593D56D9, 0xF3253C06, 0x2ADC1FFC};
936

937
	en_dbg(DRV, priv, "Configuring rss steering\n");
938 939 940
	err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,
				    priv->rx_ring_num,
				    &rss_map->base_qpn);
941
	if (err) {
942
		en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);
943 944 945
		return err;
	}

946
	for (i = 0; i < priv->rx_ring_num; i++) {
947
		qpn = rss_map->base_qpn + i;
948
		err = mlx4_en_config_rss_qp(priv, qpn, &priv->rx_ring[i],
949 950 951 952 953 954 955 956 957 958 959
					    &rss_map->state[i],
					    &rss_map->qps[i]);
		if (err)
			goto rss_err;

		++good_qps;
	}

	/* Configure RSS indirection qp */
	err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp);
	if (err) {
960
		en_err(priv, "Failed to allocate RSS indirection QP\n");
961
		goto rss_err;
962 963 964
	}
	rss_map->indir_qp.event = mlx4_en_sqp_event;
	mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn,
965
				priv->rx_ring[0].cqn, -1, &context);
966

967 968 969 970 971
	if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num)
		rss_rings = priv->rx_ring_num;
	else
		rss_rings = priv->prof->rss_rings;

972 973
	ptr = ((void *) &context) + offsetof(struct mlx4_qp_context, pri_path)
					+ MLX4_RSS_OFFSET_IN_QPC_PRI_PATH;
974
	rss_context = ptr;
975
	rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 |
976
					    (rss_map->base_qpn));
977
	rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn);
978 979 980 981
	if (priv->mdev->profile.udp_rss) {
		rss_mask |=  MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6;
		rss_context->base_qpn_udp = rss_context->default_qpn;
	}
Y
Yevgeny Petrilin 已提交
982
	rss_context->flags = rss_mask;
983
	rss_context->hash_fn = MLX4_RSS_HASH_TOP;
Y
Yevgeny Petrilin 已提交
984
	for (i = 0; i < 10; i++)
985
		rss_context->rss_key[i] = cpu_to_be32(rsskey[i]);
986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005

	err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context,
			       &rss_map->indir_qp, &rss_map->indir_state);
	if (err)
		goto indir_err;

	return 0;

indir_err:
	mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
		       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
	mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
	mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
rss_err:
	for (i = 0; i < good_qps; i++) {
		mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
			       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
		mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
		mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
	}
1006
	mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
	return err;
}

void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv)
{
	struct mlx4_en_dev *mdev = priv->mdev;
	struct mlx4_en_rss_map *rss_map = &priv->rss_map;
	int i;

	mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
		       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
	mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
	mlx4_qp_free(mdev->dev, &rss_map->indir_qp);

1021
	for (i = 0; i < priv->rx_ring_num; i++) {
1022 1023 1024 1025 1026
		mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
			       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
		mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
		mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
	}
1027
	mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
1028
}