i40e_txrx.c 62.3 KB
Newer Older
1 2 3
/*******************************************************************************
 *
 * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
4
 * Copyright(c) 2013 - 2016 Intel Corporation.
5 6 7 8 9 10 11 12 13 14
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
15 16 17
 * You should have received a copy of the GNU General Public License along
 * with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
18 19 20 21 22 23 24 25 26
 * The full GNU General Public License is included in this distribution in
 * the file called "COPYING".
 *
 * Contact Information:
 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
 *
 ******************************************************************************/

27
#include <linux/prefetch.h>
28
#include <net/busy_poll.h>
29

30
#include "i40evf.h"
31
#include "i40e_prototype.h"
32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53

static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
				u32 td_tag)
{
	return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
			   ((u64)td_cmd  << I40E_TXD_QW1_CMD_SHIFT) |
			   ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
			   ((u64)size  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
			   ((u64)td_tag  << I40E_TXD_QW1_L2TAG1_SHIFT));
}

#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)

/**
 * i40e_unmap_and_free_tx_resource - Release a Tx buffer
 * @ring:      the ring that owns the buffer
 * @tx_buffer: the buffer to free
 **/
static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
					    struct i40e_tx_buffer *tx_buffer)
{
	if (tx_buffer->skb) {
54 55 56 57
		if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
			kfree(tx_buffer->raw_buf);
		else
			dev_kfree_skb_any(tx_buffer->skb);
58 59 60 61 62 63 64 65 66 67 68
		if (dma_unmap_len(tx_buffer, len))
			dma_unmap_single(ring->dev,
					 dma_unmap_addr(tx_buffer, dma),
					 dma_unmap_len(tx_buffer, len),
					 DMA_TO_DEVICE);
	} else if (dma_unmap_len(tx_buffer, len)) {
		dma_unmap_page(ring->dev,
			       dma_unmap_addr(tx_buffer, dma),
			       dma_unmap_len(tx_buffer, len),
			       DMA_TO_DEVICE);
	}
69

70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
	tx_buffer->next_to_watch = NULL;
	tx_buffer->skb = NULL;
	dma_unmap_len_set(tx_buffer, len, 0);
	/* tx_buffer must be completely set up in the transmit path */
}

/**
 * i40evf_clean_tx_ring - Free any empty Tx buffers
 * @tx_ring: ring to be cleaned
 **/
void i40evf_clean_tx_ring(struct i40e_ring *tx_ring)
{
	unsigned long bi_size;
	u16 i;

	/* ring already cleared, nothing to do */
	if (!tx_ring->tx_bi)
		return;

	/* Free all the Tx ring sk_buffs */
	for (i = 0; i < tx_ring->count; i++)
		i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);

	bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
	memset(tx_ring->tx_bi, 0, bi_size);

	/* Zero out the descriptor ring */
	memset(tx_ring->desc, 0, tx_ring->size);

	tx_ring->next_to_use = 0;
	tx_ring->next_to_clean = 0;

	if (!tx_ring->netdev)
		return;

	/* cleanup Tx queue statistics */
106
	netdev_tx_reset_queue(txring_txq(tx_ring));
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
}

/**
 * i40evf_free_tx_resources - Free Tx resources per queue
 * @tx_ring: Tx descriptor ring for a specific queue
 *
 * Free all transmit software resources
 **/
void i40evf_free_tx_resources(struct i40e_ring *tx_ring)
{
	i40evf_clean_tx_ring(tx_ring);
	kfree(tx_ring->tx_bi);
	tx_ring->tx_bi = NULL;

	if (tx_ring->desc) {
		dma_free_coherent(tx_ring->dev, tx_ring->size,
				  tx_ring->desc, tx_ring->dma);
		tx_ring->desc = NULL;
	}
}

J
Jesse Brandeburg 已提交
128
/**
129 130
 * i40evf_get_tx_pending - how many Tx descriptors not processed
 * @tx_ring: the ring of descriptors
131
 * @in_sw: is tx_pending being checked in SW or HW
J
Jesse Brandeburg 已提交
132
 *
133 134
 * Since there is no access to the ring head register
 * in XL710, we need to use our local copies
J
Jesse Brandeburg 已提交
135
 **/
136
u32 i40evf_get_tx_pending(struct i40e_ring *ring, bool in_sw)
J
Jesse Brandeburg 已提交
137
{
138
	u32 head, tail;
J
Jesse Brandeburg 已提交
139

140 141 142 143
	if (!in_sw)
		head = i40e_get_head(ring);
	else
		head = ring->next_to_clean;
144 145 146 147 148 149 150
	tail = readl(ring->tail);

	if (head != tail)
		return (head < tail) ?
			tail - head : (tail + ring->count - head);

	return 0;
J
Jesse Brandeburg 已提交
151 152
}

153
#define WB_STRIDE 4
154

155 156
/**
 * i40e_clean_tx_irq - Reclaim resources after transmit completes
157 158 159
 * @vsi: the VSI we care about
 * @tx_ring: Tx ring to clean
 * @napi_budget: Used to determine if we are in netpoll
160 161 162
 *
 * Returns true if there's any budget left (e.g. the clean is finished)
 **/
163 164
static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,
			      struct i40e_ring *tx_ring, int napi_budget)
165 166 167
{
	u16 i = tx_ring->next_to_clean;
	struct i40e_tx_buffer *tx_buf;
168
	struct i40e_tx_desc *tx_head;
169
	struct i40e_tx_desc *tx_desc;
170 171
	unsigned int total_bytes = 0, total_packets = 0;
	unsigned int budget = vsi->work_limit;
172 173 174 175 176

	tx_buf = &tx_ring->tx_bi[i];
	tx_desc = I40E_TX_DESC(tx_ring, i);
	i -= tx_ring->count;

177 178
	tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring));

179 180 181 182 183 184 185 186 187 188
	do {
		struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;

		/* if next_to_watch is not set then there is no work pending */
		if (!eop_desc)
			break;

		/* prevent any other reads prior to eop_desc */
		read_barrier_depends();

189 190
		/* we have caught up to head, no work left to do */
		if (tx_head == tx_desc)
191 192 193 194 195 196 197 198 199 200
			break;

		/* clear next_to_watch to prevent false hangs */
		tx_buf->next_to_watch = NULL;

		/* update the statistics for this packet */
		total_bytes += tx_buf->bytecount;
		total_packets += tx_buf->gso_segs;

		/* free the skb */
201
		napi_consume_skb(tx_buf->skb, napi_budget);
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244

		/* unmap skb header data */
		dma_unmap_single(tx_ring->dev,
				 dma_unmap_addr(tx_buf, dma),
				 dma_unmap_len(tx_buf, len),
				 DMA_TO_DEVICE);

		/* clear tx_buffer data */
		tx_buf->skb = NULL;
		dma_unmap_len_set(tx_buf, len, 0);

		/* unmap remaining buffers */
		while (tx_desc != eop_desc) {

			tx_buf++;
			tx_desc++;
			i++;
			if (unlikely(!i)) {
				i -= tx_ring->count;
				tx_buf = tx_ring->tx_bi;
				tx_desc = I40E_TX_DESC(tx_ring, 0);
			}

			/* unmap any remaining paged data */
			if (dma_unmap_len(tx_buf, len)) {
				dma_unmap_page(tx_ring->dev,
					       dma_unmap_addr(tx_buf, dma),
					       dma_unmap_len(tx_buf, len),
					       DMA_TO_DEVICE);
				dma_unmap_len_set(tx_buf, len, 0);
			}
		}

		/* move us one more past the eop_desc for start of next pkt */
		tx_buf++;
		tx_desc++;
		i++;
		if (unlikely(!i)) {
			i -= tx_ring->count;
			tx_buf = tx_ring->tx_bi;
			tx_desc = I40E_TX_DESC(tx_ring, 0);
		}

245 246
		prefetch(tx_desc);

247 248 249 250 251 252 253 254 255 256 257 258 259
		/* update budget accounting */
		budget--;
	} while (likely(budget));

	i += tx_ring->count;
	tx_ring->next_to_clean = i;
	u64_stats_update_begin(&tx_ring->syncp);
	tx_ring->stats.bytes += total_bytes;
	tx_ring->stats.packets += total_packets;
	u64_stats_update_end(&tx_ring->syncp);
	tx_ring->q_vector->tx.total_bytes += total_bytes;
	tx_ring->q_vector->tx.total_packets += total_packets;

260 261 262 263 264 265
	if (tx_ring->flags & I40E_TXR_FLAGS_WB_ON_ITR) {
		/* check to see if there are < 4 descriptors
		 * waiting to be written back, then kick the hardware to force
		 * them to be written back in case we stay in NAPI.
		 * In this mode on X722 we do not enable Interrupt.
		 */
266
		unsigned int j = i40evf_get_tx_pending(tx_ring, false);
267 268

		if (budget &&
269
		    ((j / WB_STRIDE) == 0) && (j > 0) &&
270
		    !test_bit(__I40E_DOWN, &vsi->state) &&
271 272 273 274
		    (I40E_DESC_UNUSED(tx_ring) != tx_ring->count))
			tx_ring->arm_wb = true;
	}

275 276
	/* notify netdev of completed buffers */
	netdev_tx_completed_queue(txring_txq(tx_ring),
277 278 279 280 281 282 283 284 285 286 287
				  total_packets, total_bytes);

#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
	if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
		     (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
		/* Make sure that anybody stopping the queue after this
		 * sees the new next_to_clean.
		 */
		smp_mb();
		if (__netif_subqueue_stopped(tx_ring->netdev,
					     tx_ring->queue_index) &&
288
		   !test_bit(__I40E_DOWN, &vsi->state)) {
289 290 291 292 293 294
			netif_wake_subqueue(tx_ring->netdev,
					    tx_ring->queue_index);
			++tx_ring->tx_stats.restart_queue;
		}
	}

295
	return !!budget;
296 297
}

298
/**
299
 * i40evf_enable_wb_on_itr - Arm hardware to do a wb, interrupts are not enabled
300
 * @vsi: the VSI we care about
301
 * @q_vector: the vector on which to enable writeback
302 303
 *
 **/
304 305
static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi,
				  struct i40e_q_vector *q_vector)
306
{
307
	u16 flags = q_vector->tx.ring[0].flags;
308
	u32 val;
309

310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
	if (!(flags & I40E_TXR_FLAGS_WB_ON_ITR))
		return;

	if (q_vector->arm_wb_state)
		return;

	val = I40E_VFINT_DYN_CTLN1_WB_ON_ITR_MASK |
	      I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK; /* set noitr */

	wr32(&vsi->back->hw,
	     I40E_VFINT_DYN_CTLN1(q_vector->v_idx +
				  vsi->base_vector - 1), val);
	q_vector->arm_wb_state = true;
}

/**
 * i40evf_force_wb - Issue SW Interrupt so HW does a wb
 * @vsi: the VSI we care about
 * @q_vector: the vector  on which to force writeback
 *
 **/
void i40evf_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
{
	u32 val = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
		  I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK | /* set noitr */
		  I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK |
		  I40E_VFINT_DYN_CTLN1_SW_ITR_INDX_ENA_MASK
		  /* allow 00 to be written to the index */;

	wr32(&vsi->back->hw,
	     I40E_VFINT_DYN_CTLN1(q_vector->v_idx + vsi->base_vector - 1),
	     val);
342 343
}

344 345 346 347
/**
 * i40e_set_new_dynamic_itr - Find new ITR level
 * @rc: structure containing ring performance data
 *
348 349
 * Returns true if ITR changed, false if not
 *
350 351 352 353 354 355 356 357
 * Stores a new ITR value based on packets and byte counts during
 * the last interrupt.  The advantage of per interrupt computation
 * is faster updates and more accurate ITR for the current traffic
 * pattern.  Constants in this function were computed based on
 * theoretical maximum wire speed and thresholds were set based on
 * testing data as well as attempting to minimize response time
 * while increasing bulk throughput.
 **/
358
static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
359 360
{
	enum i40e_latency_range new_latency_range = rc->latency_range;
361
	struct i40e_q_vector *qv = rc->ring->q_vector;
362 363
	u32 new_itr = rc->itr;
	int bytes_per_int;
364
	int usecs;
365 366

	if (rc->total_packets == 0 || !rc->itr)
367
		return false;
368 369

	/* simple throttlerate management
370
	 *   0-10MB/s   lowest (50000 ints/s)
371
	 *  10-20MB/s   low    (20000 ints/s)
372 373
	 *  20-1249MB/s bulk   (18000 ints/s)
	 *  > 40000 Rx packets per second (8000 ints/s)
374 375 376 377
	 *
	 * The math works out because the divisor is in 10^(-6) which
	 * turns the bytes/us input value into MB/s values, but
	 * make sure to use usecs, as the register values written
378 379
	 * are in 2 usec increments in the ITR registers, and make sure
	 * to use the smoothed values that the countdown timer gives us.
380
	 */
381
	usecs = (rc->itr << 1) * ITR_COUNTDOWN_START;
382
	bytes_per_int = rc->total_bytes / usecs;
383

384
	switch (new_latency_range) {
385 386 387 388 389 390 391 392 393 394 395
	case I40E_LOWEST_LATENCY:
		if (bytes_per_int > 10)
			new_latency_range = I40E_LOW_LATENCY;
		break;
	case I40E_LOW_LATENCY:
		if (bytes_per_int > 20)
			new_latency_range = I40E_BULK_LATENCY;
		else if (bytes_per_int <= 10)
			new_latency_range = I40E_LOWEST_LATENCY;
		break;
	case I40E_BULK_LATENCY:
396
	case I40E_ULTRA_LATENCY:
397 398 399
	default:
		if (bytes_per_int <= 20)
			new_latency_range = I40E_LOW_LATENCY;
400 401
		break;
	}
402 403 404 405 406 407 408 409 410 411 412 413

	/* this is to adjust RX more aggressively when streaming small
	 * packets.  The value of 40000 was picked as it is just beyond
	 * what the hardware can receive per second if in low latency
	 * mode.
	 */
#define RX_ULTRA_PACKET_RATE 40000

	if ((((rc->total_packets * 1000000) / usecs) > RX_ULTRA_PACKET_RATE) &&
	    (&qv->rx == rc))
		new_latency_range = I40E_ULTRA_LATENCY;

414
	rc->latency_range = new_latency_range;
415 416 417

	switch (new_latency_range) {
	case I40E_LOWEST_LATENCY:
418
		new_itr = I40E_ITR_50K;
419 420 421 422 423
		break;
	case I40E_LOW_LATENCY:
		new_itr = I40E_ITR_20K;
		break;
	case I40E_BULK_LATENCY:
424 425 426
		new_itr = I40E_ITR_18K;
		break;
	case I40E_ULTRA_LATENCY:
427 428 429 430 431 432 433 434
		new_itr = I40E_ITR_8K;
		break;
	default:
		break;
	}

	rc->total_bytes = 0;
	rc->total_packets = 0;
435 436 437 438 439 440 441

	if (new_itr != rc->itr) {
		rc->itr = new_itr;
		return true;
	}

	return false;
442 443
}

J
Jesse Brandeburg 已提交
444
/**
445 446 447 448 449 450 451 452 453 454 455 456 457
 * i40evf_setup_tx_descriptors - Allocate the Tx descriptors
 * @tx_ring: the tx ring to set up
 *
 * Return 0 on success, negative on error
 **/
int i40evf_setup_tx_descriptors(struct i40e_ring *tx_ring)
{
	struct device *dev = tx_ring->dev;
	int bi_size;

	if (!dev)
		return -ENOMEM;

458 459
	/* warn if we are about to overwrite the pointer */
	WARN_ON(tx_ring->tx_bi);
460 461 462 463 464 465 466
	bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
	tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
	if (!tx_ring->tx_bi)
		goto err;

	/* round up to nearest 4K */
	tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
467 468 469 470
	/* add u32 for head writeback, align after this takes care of
	 * guaranteeing this is at least one cache line in size
	 */
	tx_ring->size += sizeof(u32);
471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
	tx_ring->size = ALIGN(tx_ring->size, 4096);
	tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
					   &tx_ring->dma, GFP_KERNEL);
	if (!tx_ring->desc) {
		dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
			 tx_ring->size);
		goto err;
	}

	tx_ring->next_to_use = 0;
	tx_ring->next_to_clean = 0;
	return 0;

err:
	kfree(tx_ring->tx_bi);
	tx_ring->tx_bi = NULL;
	return -ENOMEM;
}

/**
 * i40evf_clean_rx_ring - Free Rx buffers
 * @rx_ring: ring to be cleaned
 **/
void i40evf_clean_rx_ring(struct i40e_ring *rx_ring)
{
	unsigned long bi_size;
	u16 i;

	/* ring already cleared, nothing to do */
	if (!rx_ring->rx_bi)
		return;

503 504 505 506 507
	if (rx_ring->skb) {
		dev_kfree_skb(rx_ring->skb);
		rx_ring->skb = NULL;
	}

508 509
	/* Free all the Rx ring sk_buffs */
	for (i = 0; i < rx_ring->count; i++) {
510 511 512 513 514
		struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i];

		if (!rx_bi->page)
			continue;

515 516 517 518 519 520 521 522 523 524 525 526 527 528
		/* Invalidate cache lines that may have been written to by
		 * device so that we avoid corrupting memory.
		 */
		dma_sync_single_range_for_cpu(rx_ring->dev,
					      rx_bi->dma,
					      rx_bi->page_offset,
					      I40E_RXBUFFER_2048,
					      DMA_FROM_DEVICE);

		/* free resources associated with mapping */
		dma_unmap_page_attrs(rx_ring->dev, rx_bi->dma,
				     PAGE_SIZE,
				     DMA_FROM_DEVICE,
				     I40E_RX_DMA_ATTR);
529 530 531 532
		__free_pages(rx_bi->page, 0);

		rx_bi->page = NULL;
		rx_bi->page_offset = 0;
533 534 535 536 537 538 539 540
	}

	bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
	memset(rx_ring->rx_bi, 0, bi_size);

	/* Zero out the descriptor ring */
	memset(rx_ring->desc, 0, rx_ring->size);

541
	rx_ring->next_to_alloc = 0;
542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575
	rx_ring->next_to_clean = 0;
	rx_ring->next_to_use = 0;
}

/**
 * i40evf_free_rx_resources - Free Rx resources
 * @rx_ring: ring to clean the resources from
 *
 * Free all receive software resources
 **/
void i40evf_free_rx_resources(struct i40e_ring *rx_ring)
{
	i40evf_clean_rx_ring(rx_ring);
	kfree(rx_ring->rx_bi);
	rx_ring->rx_bi = NULL;

	if (rx_ring->desc) {
		dma_free_coherent(rx_ring->dev, rx_ring->size,
				  rx_ring->desc, rx_ring->dma);
		rx_ring->desc = NULL;
	}
}

/**
 * i40evf_setup_rx_descriptors - Allocate Rx descriptors
 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
 *
 * Returns 0 on success, negative on failure
 **/
int i40evf_setup_rx_descriptors(struct i40e_ring *rx_ring)
{
	struct device *dev = rx_ring->dev;
	int bi_size;

576 577
	/* warn if we are about to overwrite the pointer */
	WARN_ON(rx_ring->rx_bi);
578 579 580 581 582
	bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
	rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
	if (!rx_ring->rx_bi)
		goto err;

583
	u64_stats_init(&rx_ring->syncp);
584

585
	/* Round up to nearest 4K */
586
	rx_ring->size = rx_ring->count * sizeof(union i40e_32byte_rx_desc);
587 588 589 590 591 592 593 594 595 596
	rx_ring->size = ALIGN(rx_ring->size, 4096);
	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
					   &rx_ring->dma, GFP_KERNEL);

	if (!rx_ring->desc) {
		dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
			 rx_ring->size);
		goto err;
	}

597
	rx_ring->next_to_alloc = 0;
598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615
	rx_ring->next_to_clean = 0;
	rx_ring->next_to_use = 0;

	return 0;
err:
	kfree(rx_ring->rx_bi);
	rx_ring->rx_bi = NULL;
	return -ENOMEM;
}

/**
 * i40e_release_rx_desc - Store the new tail and head values
 * @rx_ring: ring to bump
 * @val: new head index
 **/
static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
{
	rx_ring->next_to_use = val;
616 617 618 619

	/* update next to alloc since we have filled the ring */
	rx_ring->next_to_alloc = val;

620 621 622 623 624 625 626 627 628 629
	/* Force memory writes to complete before letting h/w
	 * know there are new descriptors to fetch.  (Only
	 * applicable for weak-ordered memory model archs,
	 * such as IA-64).
	 */
	wmb();
	writel(val, rx_ring->tail);
}

/**
630 631 632
 * i40e_alloc_mapped_page - recycle or make a new page
 * @rx_ring: ring to use
 * @bi: rx_buffer struct to modify
633
 *
634 635
 * Returns true if the page was successfully allocated or
 * reused.
636
 **/
637 638
static bool i40e_alloc_mapped_page(struct i40e_ring *rx_ring,
				   struct i40e_rx_buffer *bi)
639
{
640 641
	struct page *page = bi->page;
	dma_addr_t dma;
642

643 644 645 646 647
	/* since we are recycling buffers we should seldom need to alloc */
	if (likely(page)) {
		rx_ring->rx_stats.page_reuse_count++;
		return true;
	}
648

649 650 651 652 653 654
	/* alloc new page for storage */
	page = dev_alloc_page();
	if (unlikely(!page)) {
		rx_ring->rx_stats.alloc_page_failed++;
		return false;
	}
655

656
	/* map page for use */
657 658 659 660
	dma = dma_map_page_attrs(rx_ring->dev, page, 0,
				 PAGE_SIZE,
				 DMA_FROM_DEVICE,
				 I40E_RX_DMA_ATTR);
661

662 663
	/* if mapping failed free memory back to system since
	 * there isn't much point in holding memory we can't use
664
	 */
665 666 667 668
	if (dma_mapping_error(rx_ring->dev, dma)) {
		__free_pages(page, 0);
		rx_ring->rx_stats.alloc_page_failed++;
		return false;
669 670
	}

671 672 673
	bi->dma = dma;
	bi->page = page;
	bi->page_offset = 0;
674

675 676
	return true;
}
677

678 679 680 681 682 683 684 685 686 687
/**
 * i40e_receive_skb - Send a completed packet up the stack
 * @rx_ring:  rx ring in play
 * @skb: packet to send up
 * @vlan_tag: vlan tag for packet
 **/
static void i40e_receive_skb(struct i40e_ring *rx_ring,
			     struct sk_buff *skb, u16 vlan_tag)
{
	struct i40e_q_vector *q_vector = rx_ring->q_vector;
688

689 690 691 692 693
	if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
	    (vlan_tag & VLAN_VID_MASK))
		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);

	napi_gro_receive(&q_vector->napi, skb);
694 695 696
}

/**
697
 * i40evf_alloc_rx_buffers - Replace used receive buffers
698 699
 * @rx_ring: ring to place buffers on
 * @cleaned_count: number of buffers to replace
700
 *
701
 * Returns false if all allocations were successful, true if any fail
702
 **/
703
bool i40evf_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
704
{
705
	u16 ntu = rx_ring->next_to_use;
706 707 708 709 710
	union i40e_rx_desc *rx_desc;
	struct i40e_rx_buffer *bi;

	/* do nothing if no valid netdev defined */
	if (!rx_ring->netdev || !cleaned_count)
711
		return false;
712

713 714
	rx_desc = I40E_RX_DESC(rx_ring, ntu);
	bi = &rx_ring->rx_bi[ntu];
715

716 717 718
	do {
		if (!i40e_alloc_mapped_page(rx_ring, bi))
			goto no_buffers;
719

720 721 722 723 724 725
		/* sync the buffer for use by the device */
		dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
						 bi->page_offset,
						 I40E_RXBUFFER_2048,
						 DMA_FROM_DEVICE);

726 727 728 729
		/* Refresh the desc even if buffer_addrs didn't change
		 * because each write-back erases this info.
		 */
		rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
730

731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
		rx_desc++;
		bi++;
		ntu++;
		if (unlikely(ntu == rx_ring->count)) {
			rx_desc = I40E_RX_DESC(rx_ring, 0);
			bi = rx_ring->rx_bi;
			ntu = 0;
		}

		/* clear the status bits for the next_to_use descriptor */
		rx_desc->wb.qword1.status_error_len = 0;

		cleaned_count--;
	} while (cleaned_count);

	if (rx_ring->next_to_use != ntu)
		i40e_release_rx_desc(rx_ring, ntu);
748 749 750

	return false;

751
no_buffers:
752 753
	if (rx_ring->next_to_use != ntu)
		i40e_release_rx_desc(rx_ring, ntu);
754 755 756 757 758

	/* make sure to come back via polling to try again after
	 * allocation failure
	 */
	return true;
759 760 761 762 763 764
}

/**
 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
 * @vsi: the VSI we care about
 * @skb: skb currently being received and modified
765 766 767
 * @rx_desc: the receive descriptor
 *
 * skb->protocol must be set before this function is called
768 769 770
 **/
static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
				    struct sk_buff *skb,
771
				    union i40e_rx_desc *rx_desc)
772
{
773 774
	struct i40e_rx_ptype_decoded decoded;
	u32 rx_error, rx_status;
775
	bool ipv4, ipv6;
776 777 778 779 780 781 782 783 784 785
	u8 ptype;
	u64 qword;

	qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
	ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT;
	rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
		   I40E_RXD_QW1_ERROR_SHIFT;
	rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
		    I40E_RXD_QW1_STATUS_SHIFT;
	decoded = decode_rx_desc_ptype(ptype);
786 787 788

	skb->ip_summed = CHECKSUM_NONE;

789 790
	skb_checksum_none_assert(skb);

791
	/* Rx csum enabled and ip headers found? */
792 793 794 795
	if (!(vsi->netdev->features & NETIF_F_RXCSUM))
		return;

	/* did the hardware decode the packet and checksum? */
796
	if (!(rx_status & BIT(I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
797 798 799 800
		return;

	/* both known and outer_ip must be set for the below code to work */
	if (!(decoded.known && decoded.outer_ip))
801 802
		return;

803 804 805 806
	ipv4 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
	       (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4);
	ipv6 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
	       (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6);
807 808

	if (ipv4 &&
809 810
	    (rx_error & (BIT(I40E_RX_DESC_ERROR_IPE_SHIFT) |
			 BIT(I40E_RX_DESC_ERROR_EIPE_SHIFT))))
811 812
		goto checksum_fail;

J
Jesse Brandeburg 已提交
813
	/* likely incorrect csum if alternate IP extension headers found */
814
	if (ipv6 &&
815
	    rx_status & BIT(I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
816
		/* don't increment checksum err here, non-fatal err */
817 818
		return;

819
	/* there was some L4 error, count error and punt packet to the stack */
820
	if (rx_error & BIT(I40E_RX_DESC_ERROR_L4E_SHIFT))
821 822 823 824 825 826
		goto checksum_fail;

	/* handle packets that were not able to be checksummed due
	 * to arrival speed, in this case the stack can compute
	 * the csum.
	 */
827
	if (rx_error & BIT(I40E_RX_DESC_ERROR_PPRS_SHIFT))
828 829
		return;

830 831 832
	/* If there is an outer header present that might contain a checksum
	 * we need to bump the checksum level by 1 to reflect the fact that
	 * we are indicating we validated the inner checksum.
833
	 */
834 835 836 837 838 839 840 841 842 843 844 845 846
	if (decoded.tunnel_type >= I40E_RX_PTYPE_TUNNEL_IP_GRENAT)
		skb->csum_level = 1;

	/* Only report checksum unnecessary for TCP, UDP, or SCTP */
	switch (decoded.inner_prot) {
	case I40E_RX_PTYPE_INNER_PROT_TCP:
	case I40E_RX_PTYPE_INNER_PROT_UDP:
	case I40E_RX_PTYPE_INNER_PROT_SCTP:
		skb->ip_summed = CHECKSUM_UNNECESSARY;
		/* fall though */
	default:
		break;
	}
847 848 849 850 851

	return;

checksum_fail:
	vsi->back->hw_csum_rx_error++;
852 853 854
}

/**
855
 * i40e_ptype_to_htype - get a hash type
856 857 858 859
 * @ptype: the ptype value from the descriptor
 *
 * Returns a hash type to be used by skb_set_hash
 **/
860
static inline int i40e_ptype_to_htype(u8 ptype)
861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
{
	struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);

	if (!decoded.known)
		return PKT_HASH_TYPE_NONE;

	if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
	    decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
		return PKT_HASH_TYPE_L4;
	else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
		 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
		return PKT_HASH_TYPE_L3;
	else
		return PKT_HASH_TYPE_L2;
}

877 878 879 880 881 882 883 884 885 886 887
/**
 * i40e_rx_hash - set the hash value in the skb
 * @ring: descriptor ring
 * @rx_desc: specific descriptor
 **/
static inline void i40e_rx_hash(struct i40e_ring *ring,
				union i40e_rx_desc *rx_desc,
				struct sk_buff *skb,
				u8 rx_ptype)
{
	u32 hash;
888
	const __le64 rss_mask =
889 890 891 892 893 894 895 896 897 898 899 900
		cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
			    I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);

	if (ring->netdev->features & NETIF_F_RXHASH)
		return;

	if ((rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) {
		hash = le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
		skb_set_hash(skb, hash, i40e_ptype_to_htype(rx_ptype));
	}
}

901
/**
902 903 904 905 906
 * i40evf_process_skb_fields - Populate skb header fields from Rx descriptor
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @rx_desc: pointer to the EOP Rx descriptor
 * @skb: pointer to current skb being populated
 * @rx_ptype: the packet type decoded by hardware
907
 *
908 909 910
 * This function checks the ring, descriptor, and packet information in
 * order to populate the hash, checksum, VLAN, protocol, and
 * other fields within the skb.
911
 **/
912 913 914 915
static inline
void i40evf_process_skb_fields(struct i40e_ring *rx_ring,
			       union i40e_rx_desc *rx_desc, struct sk_buff *skb,
			       u8 rx_ptype)
916
{
917
	i40e_rx_hash(rx_ring, rx_desc, skb, rx_ptype);
918

919 920
	/* modifies the skb - consumes the enet header */
	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
921

922
	i40e_rx_checksum(rx_ring->vsi, skb, rx_desc);
923

924 925
	skb_record_rx_queue(skb, rx_ring->queue_index);
}
926

927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944
/**
 * i40e_cleanup_headers - Correct empty headers
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @skb: pointer to current skb being fixed
 *
 * Also address the case where we are pulling data in on pages only
 * and as such no data is present in the skb header.
 *
 * In addition if skb is not at least 60 bytes we need to pad it so that
 * it is large enough to qualify as a valid Ethernet frame.
 *
 * Returns true if an error was encountered and skb was freed.
 **/
static bool i40e_cleanup_headers(struct i40e_ring *rx_ring, struct sk_buff *skb)
{
	/* if eth_skb_pad returns an error the skb was freed */
	if (eth_skb_pad(skb))
		return true;
945

946 947
	return false;
}
948

949 950 951 952 953 954 955 956 957 958 959 960
/**
 * i40e_reuse_rx_page - page flip buffer and store it back on the ring
 * @rx_ring: rx descriptor ring to store buffers on
 * @old_buff: donor buffer to have page reused
 *
 * Synchronizes page for reuse by the adapter
 **/
static void i40e_reuse_rx_page(struct i40e_ring *rx_ring,
			       struct i40e_rx_buffer *old_buff)
{
	struct i40e_rx_buffer *new_buff;
	u16 nta = rx_ring->next_to_alloc;
961

962
	new_buff = &rx_ring->rx_bi[nta];
963

964 965 966
	/* update, and store next to alloc */
	nta++;
	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
967

968 969 970 971 972
	/* transfer page from old buffer to new buffer */
	*new_buff = *old_buff;
}

/**
973
 * i40e_page_is_reusable - check if any reuse is possible
974
 * @page: page struct to check
975 976 977
 *
 * A page is not reusable if it was allocated under low memory
 * conditions, or it's not in the same NUMA node as this CPU.
978
 */
979
static inline bool i40e_page_is_reusable(struct page *page)
980
{
981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
	return (page_to_nid(page) == numa_mem_id()) &&
		!page_is_pfmemalloc(page);
}

/**
 * i40e_can_reuse_rx_page - Determine if this page can be reused by
 * the adapter for another receive
 *
 * @rx_buffer: buffer containing the page
 * @page: page address from rx_buffer
 * @truesize: actual size of the buffer in this page
 *
 * If page is reusable, rx_buffer->page_offset is adjusted to point to
 * an unused region in the page.
 *
 * For small pages, @truesize will be a constant value, half the size
 * of the memory at page.  We'll attempt to alternate between high and
 * low halves of the page, with one half ready for use by the hardware
 * and the other half being consumed by the stack.  We use the page
 * ref count to determine whether the stack has finished consuming the
 * portion of this page that was passed up with a previous packet.  If
 * the page ref count is >1, we'll assume the "other" half page is
 * still busy, and this page cannot be reused.
 *
 * For larger pages, @truesize will be the actual space used by the
 * received packet (adjusted upward to an even multiple of the cache
 * line size).  This will advance through the page by the amount
 * actually consumed by the received packets while there is still
 * space for a buffer.  Each region of larger pages will be used at
 * most once, after which the page will not be reused.
 *
 * In either case, if the page is reusable its refcount is increased.
 **/
static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer,
				   struct page *page,
				   const unsigned int truesize)
{
#if (PAGE_SIZE >= 8192)
	unsigned int last_offset = PAGE_SIZE - I40E_RXBUFFER_2048;
#endif

	/* Is any reuse possible? */
	if (unlikely(!i40e_page_is_reusable(page)))
		return false;

#if (PAGE_SIZE < 8192)
	/* if we are only owner of page we can reuse it */
	if (unlikely(page_count(page) != 1))
		return false;

	/* flip page offset to other buffer */
	rx_buffer->page_offset ^= truesize;
#else
	/* move offset up to the next cache line */
	rx_buffer->page_offset += truesize;

	if (rx_buffer->page_offset > last_offset)
		return false;
#endif

	/* Inc ref count on page before passing it up to the stack */
	get_page(page);

	return true;
1045 1046 1047 1048 1049 1050
}

/**
 * i40e_add_rx_frag - Add contents of Rx buffer to sk_buff
 * @rx_ring: rx descriptor ring to transact packets on
 * @rx_buffer: buffer containing page to add
1051
 * @size: packet length from rx_desc
1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
 * @skb: sk_buff to place the data into
 *
 * This function will add the data contained in rx_buffer->page to the skb.
 * This is done either through a direct copy if the data in the buffer is
 * less than the skb header size, otherwise it will just attach the page as
 * a frag to the skb.
 *
 * The function will then update the page offset if necessary and return
 * true if the buffer can be reused by the adapter.
 **/
static bool i40e_add_rx_frag(struct i40e_ring *rx_ring,
			     struct i40e_rx_buffer *rx_buffer,
1064
			     unsigned int size,
1065 1066 1067
			     struct sk_buff *skb)
{
	struct page *page = rx_buffer->page;
1068
	unsigned char *va = page_address(page) + rx_buffer->page_offset;
1069 1070 1071 1072
#if (PAGE_SIZE < 8192)
	unsigned int truesize = I40E_RXBUFFER_2048;
#else
	unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
1073
#endif
1074 1075 1076 1077
	unsigned int pull_len;

	if (unlikely(skb_is_nonlinear(skb)))
		goto add_tail_frag;
1078

1079 1080 1081
	/* will the data fit in the skb we allocated? if so, just
	 * copy it as it is pretty small anyway
	 */
1082
	if (size <= I40E_RX_HDR_SIZE) {
1083
		memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
1084

1085 1086
		/* page is reusable, we can reuse buffer as-is */
		if (likely(i40e_page_is_reusable(page)))
1087
			return true;
1088

1089 1090 1091 1092 1093
		/* this page cannot be reused so discard it */
		__free_pages(page, 0);
		return false;
	}

1094 1095 1096 1097 1098
	/* we need the header to contain the greater of either
	 * ETH_HLEN or 60 bytes if the skb->len is less than
	 * 60 for skb_pad.
	 */
	pull_len = eth_get_headlen(va, I40E_RX_HDR_SIZE);
1099

1100 1101 1102 1103
	/* align pull length to size of long to optimize
	 * memcpy performance
	 */
	memcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long)));
1104

1105 1106 1107
	/* update all of the pointers */
	va += pull_len;
	size -= pull_len;
1108

1109 1110 1111
add_tail_frag:
	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
			(unsigned long)va & ~PAGE_MASK, size, truesize);
1112

1113
	return i40e_can_reuse_rx_page(rx_buffer, page, truesize);
1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
}

/**
 * i40evf_fetch_rx_buffer - Allocate skb and populate it
 * @rx_ring: rx descriptor ring to transact packets on
 * @rx_desc: descriptor containing info written by hardware
 *
 * This function allocates an skb on the fly, and populates it with the page
 * data from the current receive descriptor, taking care to set up the skb
 * correctly, as well as handling calling the page recycle function if
 * necessary.
 */
static inline
struct sk_buff *i40evf_fetch_rx_buffer(struct i40e_ring *rx_ring,
1128 1129
				       union i40e_rx_desc *rx_desc,
				       struct sk_buff *skb)
1130
{
1131 1132 1133 1134 1135
	u64 local_status_error_len =
		le64_to_cpu(rx_desc->wb.qword1.status_error_len);
	unsigned int size =
		(local_status_error_len & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
		I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171
	struct i40e_rx_buffer *rx_buffer;
	struct page *page;

	rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean];
	page = rx_buffer->page;
	prefetchw(page);

	if (likely(!skb)) {
		void *page_addr = page_address(page) + rx_buffer->page_offset;

		/* prefetch first cache line of first page */
		prefetch(page_addr);
#if L1_CACHE_BYTES < 128
		prefetch(page_addr + L1_CACHE_BYTES);
#endif

		/* allocate a skb to store the frags */
		skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
				       I40E_RX_HDR_SIZE,
				       GFP_ATOMIC | __GFP_NOWARN);
		if (unlikely(!skb)) {
			rx_ring->rx_stats.alloc_buff_failed++;
			return NULL;
		}

		/* we will be copying header into skb->data in
		 * pskb_may_pull so it is in our interest to prefetch
		 * it now to avoid a possible cache miss
		 */
		prefetchw(skb->data);
	}

	/* we are reusing so sync this buffer for CPU use */
	dma_sync_single_range_for_cpu(rx_ring->dev,
				      rx_buffer->dma,
				      rx_buffer->page_offset,
1172
				      size,
1173 1174 1175
				      DMA_FROM_DEVICE);

	/* pull page into skb */
1176
	if (i40e_add_rx_frag(rx_ring, rx_buffer, size, skb)) {
1177 1178 1179 1180 1181
		/* hand second half of page back to the ring */
		i40e_reuse_rx_page(rx_ring, rx_buffer);
		rx_ring->rx_stats.page_reuse_count++;
	} else {
		/* we are not reusing the buffer so unmap it */
1182 1183
		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, PAGE_SIZE,
				     DMA_FROM_DEVICE, I40E_RX_DMA_ATTR);
1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222
	}

	/* clear contents of buffer_info */
	rx_buffer->page = NULL;

	return skb;
}

/**
 * i40e_is_non_eop - process handling of non-EOP buffers
 * @rx_ring: Rx ring being processed
 * @rx_desc: Rx descriptor for current buffer
 * @skb: Current socket buffer containing buffer in progress
 *
 * This function updates next to clean.  If the buffer is an EOP buffer
 * this function exits returning false, otherwise it will place the
 * sk_buff in the next buffer to be chained and return true indicating
 * that this is in fact a non-EOP buffer.
 **/
static bool i40e_is_non_eop(struct i40e_ring *rx_ring,
			    union i40e_rx_desc *rx_desc,
			    struct sk_buff *skb)
{
	u32 ntc = rx_ring->next_to_clean + 1;

	/* fetch, update, and store next to clean */
	ntc = (ntc < rx_ring->count) ? ntc : 0;
	rx_ring->next_to_clean = ntc;

	prefetch(I40E_RX_DESC(rx_ring, ntc));

	/* if we are the last buffer then there is nothing else to do */
#define I40E_RXD_EOF BIT(I40E_RX_DESC_STATUS_EOF_SHIFT)
	if (likely(i40e_test_staterr(rx_desc, I40E_RXD_EOF)))
		return false;

	rx_ring->rx_stats.non_eop_descs++;

	return true;
1223 1224 1225
}

/**
1226 1227 1228 1229 1230 1231 1232 1233
 * i40e_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
 * @rx_ring: rx descriptor ring to transact packets on
 * @budget: Total limit on number of packets to process
 *
 * This function provides a "bounce buffer" approach to Rx interrupt
 * processing.  The advantage to this is that on systems that have
 * expensive overhead for IOMMU access this provides a means of avoiding
 * it by maintaining the mapping of the page to the system.
1234
 *
1235
 * Returns amount of work completed
1236
 **/
1237
static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1238 1239
{
	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1240
	struct sk_buff *skb = rx_ring->skb;
1241
	u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1242
	bool failure = false;
1243

1244 1245
	while (likely(total_rx_packets < budget)) {
		union i40e_rx_desc *rx_desc;
1246
		u16 vlan_tag;
1247 1248 1249
		u8 rx_ptype;
		u64 qword;

1250 1251
		/* return some buffers to hardware, one at a time is too slow */
		if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1252
			failure = failure ||
1253
				  i40evf_alloc_rx_buffers(rx_ring, cleaned_count);
1254 1255 1256
			cleaned_count = 0;
		}

1257 1258 1259 1260 1261 1262 1263
		rx_desc = I40E_RX_DESC(rx_ring, rx_ring->next_to_clean);

		/* status_error_len will always be zero for unused descriptors
		 * because it's cleared in cleanup, and overlaps with hdr_addr
		 * which is always zero because packet split isn't used, if the
		 * hardware wrote DD then it will be non-zero
		 */
1264 1265
		if (!i40e_test_staterr(rx_desc,
				       BIT(I40E_RX_DESC_STATUS_DD_SHIFT)))
1266 1267
			break;

1268 1269 1270 1271
		/* This memory barrier is needed to keep us from reading
		 * any other fields out of the rx_desc until we know the
		 * DD bit is set.
		 */
1272
		dma_rmb();
1273

1274
		skb = i40evf_fetch_rx_buffer(rx_ring, rx_desc, skb);
1275 1276
		if (!skb)
			break;
1277 1278 1279

		cleaned_count++;

1280
		if (i40e_is_non_eop(rx_ring, rx_desc, skb))
1281 1282
			continue;

1283 1284 1285 1286 1287 1288
		/* ERR_MASK will only have valid bits if EOP set, and
		 * what we are doing here is actually checking
		 * I40E_RX_DESC_ERROR_RXE_SHIFT, since it is the zeroth bit in
		 * the error field
		 */
		if (unlikely(i40e_test_staterr(rx_desc, BIT(I40E_RXD_QW1_ERROR_SHIFT)))) {
1289 1290 1291 1292
			dev_kfree_skb_any(skb);
			continue;
		}

1293 1294
		if (i40e_cleanup_headers(rx_ring, skb)) {
			skb = NULL;
1295
			continue;
1296
		}
1297

1298 1299 1300
		/* probably a little skewed due to removing CRC */
		total_rx_bytes += skb->len;

1301 1302 1303 1304
		qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
		rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
			   I40E_RXD_QW1_PTYPE_SHIFT;

1305 1306
		/* populate checksum, VLAN, and protocol */
		i40evf_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype);
1307 1308


1309 1310 1311
		vlan_tag = (qword & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) ?
			   le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) : 0;

1312
		i40e_receive_skb(rx_ring, skb, vlan_tag);
1313
		skb = NULL;
1314

1315 1316 1317
		/* update budget accounting */
		total_rx_packets++;
	}
1318

1319 1320
	rx_ring->skb = skb;

1321 1322 1323 1324 1325 1326 1327
	u64_stats_update_begin(&rx_ring->syncp);
	rx_ring->stats.packets += total_rx_packets;
	rx_ring->stats.bytes += total_rx_bytes;
	u64_stats_update_end(&rx_ring->syncp);
	rx_ring->q_vector->rx.total_packets += total_rx_packets;
	rx_ring->q_vector->rx.total_bytes += total_rx_bytes;

1328
	/* guarantee a trip back through this routine if there was a failure */
1329
	return failure ? budget : total_rx_packets;
1330 1331
}

1332 1333 1334 1335 1336
static u32 i40e_buildreg_itr(const int type, const u16 itr)
{
	u32 val;

	val = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1337 1338 1339
	      /* Don't clear PBA because that can cause lost interrupts that
	       * came in while we were cleaning/polling
	       */
1340 1341 1342 1343 1344 1345 1346 1347
	      (type << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) |
	      (itr << I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT);

	return val;
}

/* a small macro to shorten up some long lines */
#define INTREG I40E_VFINT_DYN_CTLN1
1348
static inline int get_rx_itr(struct i40e_vsi *vsi, int idx)
1349 1350 1351
{
	struct i40evf_adapter *adapter = vsi->back;

1352
	return adapter->rx_rings[idx].rx_itr_setting;
1353 1354
}

1355
static inline int get_tx_itr(struct i40e_vsi *vsi, int idx)
1356 1357 1358
{
	struct i40evf_adapter *adapter = vsi->back;

1359
	return adapter->tx_rings[idx].tx_itr_setting;
1360
}
1361

1362 1363 1364 1365 1366 1367 1368 1369 1370 1371
/**
 * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt
 * @vsi: the VSI we care about
 * @q_vector: q_vector for which itr is being updated and interrupt enabled
 *
 **/
static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
					  struct i40e_q_vector *q_vector)
{
	struct i40e_hw *hw = &vsi->back->hw;
1372 1373
	bool rx = false, tx = false;
	u32 rxval, txval;
1374
	int vector;
1375 1376
	int idx = q_vector->v_idx;
	int rx_itr_setting, tx_itr_setting;
1377 1378

	vector = (q_vector->v_idx + vsi->base_vector);
1379 1380 1381 1382

	/* avoid dynamic calculation if in countdown mode OR if
	 * all dynamic is disabled
	 */
1383 1384
	rxval = txval = i40e_buildreg_itr(I40E_ITR_NONE, 0);

1385 1386
	rx_itr_setting = get_rx_itr(vsi, idx);
	tx_itr_setting = get_tx_itr(vsi, idx);
1387

1388
	if (q_vector->itr_countdown > 0 ||
1389 1390
	    (!ITR_IS_DYNAMIC(rx_itr_setting) &&
	     !ITR_IS_DYNAMIC(tx_itr_setting))) {
1391 1392 1393
		goto enable_int;
	}

1394
	if (ITR_IS_DYNAMIC(rx_itr_setting)) {
1395 1396
		rx = i40e_set_new_dynamic_itr(&q_vector->rx);
		rxval = i40e_buildreg_itr(I40E_RX_ITR, q_vector->rx.itr);
1397
	}
J
Jesse Brandeburg 已提交
1398

1399
	if (ITR_IS_DYNAMIC(tx_itr_setting)) {
1400 1401 1402
		tx = i40e_set_new_dynamic_itr(&q_vector->tx);
		txval = i40e_buildreg_itr(I40E_TX_ITR, q_vector->tx.itr);
	}
J
Jesse Brandeburg 已提交
1403

1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415
	if (rx || tx) {
		/* get the higher of the two ITR adjustments and
		 * use the same value for both ITR registers
		 * when in adaptive mode (Rx and/or Tx)
		 */
		u16 itr = max(q_vector->tx.itr, q_vector->rx.itr);

		q_vector->tx.itr = q_vector->rx.itr = itr;
		txval = i40e_buildreg_itr(I40E_TX_ITR, itr);
		tx = true;
		rxval = i40e_buildreg_itr(I40E_RX_ITR, itr);
		rx = true;
1416
	}
1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430

	/* only need to enable the interrupt once, but need
	 * to possibly update both ITR values
	 */
	if (rx) {
		/* set the INTENA_MSK_MASK so that this first write
		 * won't actually enable the interrupt, instead just
		 * updating the ITR (it's bit 31 PF and VF)
		 */
		rxval |= BIT(31);
		/* don't check _DOWN because interrupt isn't being enabled */
		wr32(hw, INTREG(vector - 1), rxval);
	}

1431
enable_int:
1432 1433
	if (!test_bit(__I40E_DOWN, &vsi->state))
		wr32(hw, INTREG(vector - 1), txval);
1434 1435 1436 1437 1438

	if (q_vector->itr_countdown)
		q_vector->itr_countdown--;
	else
		q_vector->itr_countdown = ITR_COUNTDOWN_START;
1439 1440
}

1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456
/**
 * i40evf_napi_poll - NAPI polling Rx/Tx cleanup routine
 * @napi: napi struct with our devices info in it
 * @budget: amount of work driver is allowed to do this pass, in packets
 *
 * This function will clean all queues associated with a q_vector.
 *
 * Returns the amount of work done
 **/
int i40evf_napi_poll(struct napi_struct *napi, int budget)
{
	struct i40e_q_vector *q_vector =
			       container_of(napi, struct i40e_q_vector, napi);
	struct i40e_vsi *vsi = q_vector->vsi;
	struct i40e_ring *ring;
	bool clean_complete = true;
1457
	bool arm_wb = false;
1458
	int budget_per_ring;
1459
	int work_done = 0;
1460 1461 1462 1463 1464 1465 1466 1467 1468

	if (test_bit(__I40E_DOWN, &vsi->state)) {
		napi_complete(napi);
		return 0;
	}

	/* Since the actual Tx work is minimal, we can give the Tx a larger
	 * budget and be more aggressive about cleaning up the Tx descriptors.
	 */
1469
	i40e_for_each_ring(ring, q_vector->tx) {
1470
		if (!i40e_clean_tx_irq(vsi, ring, budget)) {
1471 1472 1473 1474
			clean_complete = false;
			continue;
		}
		arm_wb |= ring->arm_wb;
1475
		ring->arm_wb = false;
1476
	}
1477

1478 1479 1480 1481
	/* Handle case where we are called by netpoll with a budget of 0 */
	if (budget <= 0)
		goto tx_only;

1482 1483 1484 1485 1486
	/* We attempt to distribute budget to each Rx queue fairly, but don't
	 * allow the budget to go below 1 because that would exit polling early.
	 */
	budget_per_ring = max(budget/q_vector->num_ringpairs, 1);

1487
	i40e_for_each_ring(ring, q_vector->rx) {
1488
		int cleaned = i40e_clean_rx_irq(ring, budget_per_ring);
1489 1490

		work_done += cleaned;
1491 1492 1493
		/* if we clean as many as budgeted, we must not be done */
		if (cleaned >= budget_per_ring)
			clean_complete = false;
1494
	}
1495 1496

	/* If work not completed, return budget and polling will return */
1497
	if (!clean_complete) {
1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508
		const cpumask_t *aff_mask = &q_vector->affinity_mask;
		int cpu_id = smp_processor_id();

		/* It is possible that the interrupt affinity has changed but,
		 * if the cpu is pegged at 100%, polling will never exit while
		 * traffic continues and the interrupt will be stuck on this
		 * cpu.  We check to make sure affinity is correct before we
		 * continue to poll, otherwise we must stop polling so the
		 * interrupt can move to the correct cpu.
		 */
		if (likely(cpumask_test_cpu(cpu_id, aff_mask))) {
1509
tx_only:
1510 1511 1512 1513 1514
			if (arm_wb) {
				q_vector->tx.ring[0].tx_stats.tx_force_wb++;
				i40e_enable_wb_on_itr(vsi, q_vector);
			}
			return budget;
1515
		}
1516
	}
1517

1518 1519 1520
	if (vsi->back->flags & I40E_TXR_FLAGS_WB_ON_ITR)
		q_vector->arm_wb_state = false;

1521
	/* Work is done so exit the polling mode and re-enable the interrupt */
1522
	napi_complete_done(napi, work_done);
1523 1524 1525 1526 1527 1528 1529 1530 1531 1532

	/* If we're prematurely stopping polling to fix the interrupt
	 * affinity we want to make sure polling starts back up so we
	 * issue a call to i40evf_force_wb which triggers a SW interrupt.
	 */
	if (!clean_complete)
		i40evf_force_wb(vsi, q_vector);
	else
		i40e_update_enable_itr(vsi, q_vector);

1533
	return min(work_done, budget - 1);
1534 1535 1536
}

/**
1537
 * i40evf_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
1538 1539 1540 1541 1542 1543 1544 1545 1546 1547
 * @skb:     send buffer
 * @tx_ring: ring to send buffer on
 * @flags:   the tx flags to be set
 *
 * Checks the skb and set up correspondingly several generic transmit flags
 * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
 *
 * Returns error code indicate the frame should be dropped upon error and the
 * otherwise  returns 0 to indicate the flags has been set properly.
 **/
1548 1549 1550
static inline int i40evf_tx_prepare_vlan_flags(struct sk_buff *skb,
					       struct i40e_ring *tx_ring,
					       u32 *flags)
1551 1552 1553 1554
{
	__be16 protocol = skb->protocol;
	u32  tx_flags = 0;

1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567
	if (protocol == htons(ETH_P_8021Q) &&
	    !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {
		/* When HW VLAN acceleration is turned off by the user the
		 * stack sets the protocol to 8021q so that the driver
		 * can take any steps required to support the SW only
		 * VLAN handling.  In our case the driver doesn't need
		 * to take any further steps so just set the protocol
		 * to the encapsulated ethertype.
		 */
		skb->protocol = vlan_get_protocol(skb);
		goto out;
	}

1568
	/* if we have a HW VLAN tag being added, default to the HW one */
1569 1570
	if (skb_vlan_tag_present(skb)) {
		tx_flags |= skb_vlan_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1571 1572 1573 1574
		tx_flags |= I40E_TX_FLAGS_HW_VLAN;
	/* else if it is a SW VLAN, check the next protocol and store the tag */
	} else if (protocol == htons(ETH_P_8021Q)) {
		struct vlan_hdr *vhdr, _vhdr;
J
Jesse Brandeburg 已提交
1575

1576 1577 1578 1579 1580 1581 1582 1583 1584
		vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
		if (!vhdr)
			return -EINVAL;

		protocol = vhdr->h_vlan_encapsulated_proto;
		tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
		tx_flags |= I40E_TX_FLAGS_SW_VLAN;
	}

1585
out:
1586 1587 1588 1589 1590 1591
	*flags = tx_flags;
	return 0;
}

/**
 * i40e_tso - set up the tso context descriptor
1592
 * @first:    pointer to first Tx buffer for xmit
1593
 * @hdr_len:  ptr to the size of the packet header
1594
 * @cd_type_cmd_tso_mss: Quad Word 1
1595 1596 1597
 *
 * Returns 0 if no TSO can happen, 1 if tso is going, or error
 **/
1598 1599
static int i40e_tso(struct i40e_tx_buffer *first, u8 *hdr_len,
		    u64 *cd_type_cmd_tso_mss)
1600
{
1601
	struct sk_buff *skb = first->skb;
1602
	u64 cd_cmd, cd_tso_len, cd_mss;
1603 1604 1605 1606 1607
	union {
		struct iphdr *v4;
		struct ipv6hdr *v6;
		unsigned char *hdr;
	} ip;
1608 1609
	union {
		struct tcphdr *tcp;
1610
		struct udphdr *udp;
1611 1612 1613
		unsigned char *hdr;
	} l4;
	u32 paylen, l4_offset;
1614
	u16 gso_segs, gso_size;
1615 1616
	int err;

1617 1618 1619
	if (skb->ip_summed != CHECKSUM_PARTIAL)
		return 0;

1620 1621 1622
	if (!skb_is_gso(skb))
		return 0;

1623 1624 1625
	err = skb_cow_head(skb, 0);
	if (err < 0)
		return err;
1626

1627 1628
	ip.hdr = skb_network_header(skb);
	l4.hdr = skb_transport_header(skb);
1629

1630 1631 1632 1633
	/* initialize outer IP header fields */
	if (ip.v4->version == 4) {
		ip.v4->tot_len = 0;
		ip.v4->check = 0;
1634
	} else {
1635 1636 1637
		ip.v6->payload_len = 0;
	}

1638
	if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1639
					 SKB_GSO_GRE_CSUM |
1640
					 SKB_GSO_IPXIP4 |
1641
					 SKB_GSO_IPXIP6 |
1642
					 SKB_GSO_UDP_TUNNEL |
1643
					 SKB_GSO_UDP_TUNNEL_CSUM)) {
1644 1645 1646 1647
		if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
		    (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) {
			l4.udp->len = 0;

1648 1649 1650 1651
			/* determine offset of outer transport header */
			l4_offset = l4.hdr - skb->data;

			/* remove payload length from outer checksum */
1652
			paylen = skb->len - l4_offset;
1653 1654
			csum_replace_by_diff(&l4.udp->check,
					     (__force __wsum)htonl(paylen));
1655 1656
		}

1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667
		/* reset pointers to inner headers */
		ip.hdr = skb_inner_network_header(skb);
		l4.hdr = skb_inner_transport_header(skb);

		/* initialize inner IP header fields */
		if (ip.v4->version == 4) {
			ip.v4->tot_len = 0;
			ip.v4->check = 0;
		} else {
			ip.v6->payload_len = 0;
		}
1668 1669
	}

1670 1671 1672 1673
	/* determine offset of inner transport header */
	l4_offset = l4.hdr - skb->data;

	/* remove payload length from inner checksum */
1674
	paylen = skb->len - l4_offset;
1675
	csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
1676 1677 1678

	/* compute length of segmentation header */
	*hdr_len = (l4.tcp->doff * 4) + l4_offset;
1679

1680 1681 1682 1683 1684 1685 1686 1687
	/* pull values out of skb_shinfo */
	gso_size = skb_shinfo(skb)->gso_size;
	gso_segs = skb_shinfo(skb)->gso_segs;

	/* update GSO size and bytecount with header size */
	first->gso_segs = gso_segs;
	first->bytecount += (first->gso_segs - 1) * *hdr_len;

1688 1689 1690
	/* find the field values */
	cd_cmd = I40E_TX_CTX_DESC_TSO;
	cd_tso_len = skb->len - *hdr_len;
1691
	cd_mss = gso_size;
1692 1693 1694
	*cd_type_cmd_tso_mss |= (cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
				(cd_tso_len << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
				(cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
1695 1696 1697 1698 1699 1700
	return 1;
}

/**
 * i40e_tx_enable_csum - Enable Tx checksum offloads
 * @skb: send buffer
1701
 * @tx_flags: pointer to Tx flags currently set
1702 1703
 * @td_cmd: Tx descriptor command bits to set
 * @td_offset: Tx descriptor header offsets to set
1704
 * @tx_ring: Tx descriptor ring
1705 1706
 * @cd_tunneling: ptr to context desc bits
 **/
1707 1708 1709 1710
static int i40e_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags,
			       u32 *td_cmd, u32 *td_offset,
			       struct i40e_ring *tx_ring,
			       u32 *cd_tunneling)
1711
{
1712 1713 1714 1715 1716 1717 1718 1719 1720 1721
	union {
		struct iphdr *v4;
		struct ipv6hdr *v6;
		unsigned char *hdr;
	} ip;
	union {
		struct tcphdr *tcp;
		struct udphdr *udp;
		unsigned char *hdr;
	} l4;
1722
	unsigned char *exthdr;
1723
	u32 offset, cmd = 0;
1724
	__be16 frag_off;
1725 1726
	u8 l4_proto = 0;

1727 1728 1729
	if (skb->ip_summed != CHECKSUM_PARTIAL)
		return 0;

1730 1731
	ip.hdr = skb_network_header(skb);
	l4.hdr = skb_transport_header(skb);
1732

1733 1734 1735
	/* compute outer L2 header size */
	offset = ((ip.hdr - skb->data) / 2) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;

1736
	if (skb->encapsulation) {
1737
		u32 tunnel = 0;
1738 1739
		/* define outer network header type */
		if (*tx_flags & I40E_TX_FLAGS_IPV4) {
1740 1741 1742 1743
			tunnel |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
				  I40E_TX_CTX_EXT_IP_IPV4 :
				  I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;

1744 1745
			l4_proto = ip.v4->protocol;
		} else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
1746
			tunnel |= I40E_TX_CTX_EXT_IP_IPV6;
1747 1748

			exthdr = ip.hdr + sizeof(*ip.v6);
1749
			l4_proto = ip.v6->nexthdr;
1750 1751 1752
			if (l4.hdr != exthdr)
				ipv6_skip_exthdr(skb, exthdr - skb->data,
						 &l4_proto, &frag_off);
1753 1754 1755 1756
		}

		/* define outer transport */
		switch (l4_proto) {
1757
		case IPPROTO_UDP:
1758
			tunnel |= I40E_TXD_CTX_UDP_TUNNELING;
1759
			*tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
1760
			break;
1761
		case IPPROTO_GRE:
1762
			tunnel |= I40E_TXD_CTX_GRE_TUNNELING;
1763 1764
			*tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
			break;
1765 1766 1767 1768 1769
		case IPPROTO_IPIP:
		case IPPROTO_IPV6:
			*tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
			l4.hdr = skb_inner_network_header(skb);
			break;
1770
		default:
1771 1772 1773 1774 1775
			if (*tx_flags & I40E_TX_FLAGS_TSO)
				return -1;

			skb_checksum_help(skb);
			return 0;
1776
		}
1777

1778 1779 1780 1781 1782 1783 1784
		/* compute outer L3 header size */
		tunnel |= ((l4.hdr - ip.hdr) / 4) <<
			  I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT;

		/* switch IP header pointer from outer to inner header */
		ip.hdr = skb_inner_network_header(skb);

1785 1786 1787 1788
		/* compute tunnel header size */
		tunnel |= ((ip.hdr - l4.hdr) / 2) <<
			  I40E_TXD_CTX_QW0_NATLEN_SHIFT;

1789 1790
		/* indicate if we need to offload outer UDP header */
		if ((*tx_flags & I40E_TX_FLAGS_TSO) &&
1791
		    !(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
1792 1793 1794
		    (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM))
			tunnel |= I40E_TXD_CTX_QW0_L4T_CS_MASK;

1795 1796 1797
		/* record tunnel offload values */
		*cd_tunneling |= tunnel;

1798 1799
		/* switch L4 header pointer from outer to inner */
		l4.hdr = skb_inner_transport_header(skb);
1800
		l4_proto = 0;
1801

1802 1803 1804 1805 1806
		/* reset type as we transition from outer to inner headers */
		*tx_flags &= ~(I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6);
		if (ip.v4->version == 4)
			*tx_flags |= I40E_TX_FLAGS_IPV4;
		if (ip.v6->version == 6)
1807
			*tx_flags |= I40E_TX_FLAGS_IPV6;
1808 1809 1810
	}

	/* Enable IP checksum offloads */
1811
	if (*tx_flags & I40E_TX_FLAGS_IPV4) {
1812
		l4_proto = ip.v4->protocol;
1813 1814 1815
		/* the stack computes the IP header already, the only time we
		 * need the hardware to recompute it is in the case of TSO.
		 */
1816 1817 1818
		cmd |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
		       I40E_TX_DESC_CMD_IIPT_IPV4_CSUM :
		       I40E_TX_DESC_CMD_IIPT_IPV4;
1819
	} else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
1820
		cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
1821 1822 1823 1824 1825 1826

		exthdr = ip.hdr + sizeof(*ip.v6);
		l4_proto = ip.v6->nexthdr;
		if (l4.hdr != exthdr)
			ipv6_skip_exthdr(skb, exthdr - skb->data,
					 &l4_proto, &frag_off);
1827
	}
1828

1829 1830
	/* compute inner L3 header size */
	offset |= ((l4.hdr - ip.hdr) / 4) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1831 1832

	/* Enable L4 checksum offloads */
1833
	switch (l4_proto) {
1834 1835
	case IPPROTO_TCP:
		/* enable checksum offloads */
1836 1837
		cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
		offset |= l4.tcp->doff << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1838 1839 1840
		break;
	case IPPROTO_SCTP:
		/* enable SCTP checksum offload */
1841 1842 1843
		cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
		offset |= (sizeof(struct sctphdr) >> 2) <<
			  I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1844 1845 1846
		break;
	case IPPROTO_UDP:
		/* enable UDP checksum offload */
1847 1848 1849
		cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
		offset |= (sizeof(struct udphdr) >> 2) <<
			  I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1850 1851
		break;
	default:
1852 1853 1854 1855
		if (*tx_flags & I40E_TX_FLAGS_TSO)
			return -1;
		skb_checksum_help(skb);
		return 0;
1856
	}
1857 1858 1859

	*td_cmd |= cmd;
	*td_offset |= offset;
1860 1861

	return 1;
1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877
}

/**
 * i40e_create_tx_ctx Build the Tx context descriptor
 * @tx_ring:  ring to create the descriptor on
 * @cd_type_cmd_tso_mss: Quad Word 1
 * @cd_tunneling: Quad Word 0 - bits 0-31
 * @cd_l2tag2: Quad Word 0 - bits 32-63
 **/
static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
			       const u64 cd_type_cmd_tso_mss,
			       const u32 cd_tunneling, const u32 cd_l2tag2)
{
	struct i40e_tx_context_desc *context_desc;
	int i = tx_ring->next_to_use;

1878 1879
	if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) &&
	    !cd_tunneling && !cd_l2tag2)
1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890
		return;

	/* grab the next descriptor */
	context_desc = I40E_TX_CTXTDESC(tx_ring, i);

	i++;
	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;

	/* cpu_to_le32 and assign to struct fields */
	context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
	context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
1891
	context_desc->rsvd = cpu_to_le16(0);
1892 1893 1894
	context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
}

J
Jesse Brandeburg 已提交
1895
/**
1896
 * __i40evf_chk_linearize - Check if there are more than 8 buffers per packet
1897 1898
 * @skb:      send buffer
 *
1899 1900 1901 1902 1903 1904 1905 1906
 * Note: Our HW can't DMA more than 8 buffers to build a packet on the wire
 * and so we need to figure out the cases where we need to linearize the skb.
 *
 * For TSO we need to count the TSO header and segment payload separately.
 * As such we need to check cases where we have 7 fragments or more as we
 * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for
 * the segment payload in the first descriptor, and another 7 for the
 * fragments.
1907
 **/
1908
bool __i40evf_chk_linearize(struct sk_buff *skb)
1909
{
1910
	const struct skb_frag_struct *frag, *stale;
1911
	int nr_frags, sum;
1912

1913
	/* no need to check if number of frags is less than 7 */
1914
	nr_frags = skb_shinfo(skb)->nr_frags;
1915
	if (nr_frags < (I40E_MAX_BUFFER_TXD - 1))
1916
		return false;
1917

1918
	/* We need to walk through the list and validate that each group
1919
	 * of 6 fragments totals at least gso_size.
1920
	 */
1921
	nr_frags -= I40E_MAX_BUFFER_TXD - 2;
1922 1923 1924 1925 1926 1927 1928 1929
	frag = &skb_shinfo(skb)->frags[0];

	/* Initialize size to the negative value of gso_size minus 1.  We
	 * use this as the worst case scenerio in which the frag ahead
	 * of us only provides one byte which is why we are limited to 6
	 * descriptors for a single transmit as the header and previous
	 * fragment are already consuming 2 descriptors.
	 */
1930
	sum = 1 - skb_shinfo(skb)->gso_size;
1931

1932 1933 1934 1935 1936 1937
	/* Add size of frags 0 through 4 to create our initial sum */
	sum += skb_frag_size(frag++);
	sum += skb_frag_size(frag++);
	sum += skb_frag_size(frag++);
	sum += skb_frag_size(frag++);
	sum += skb_frag_size(frag++);
1938 1939 1940 1941 1942 1943

	/* Walk through fragments adding latest fragment, testing it, and
	 * then removing stale fragments from the sum.
	 */
	stale = &skb_shinfo(skb)->frags[0];
	for (;;) {
1944
		sum += skb_frag_size(frag++);
1945 1946 1947 1948 1949

		/* if sum is negative we failed to make sufficient progress */
		if (sum < 0)
			return true;

1950
		if (!nr_frags--)
1951 1952
			break;

1953
		sum -= skb_frag_size(stale++);
1954 1955
	}

1956
	return false;
1957 1958
}

1959 1960 1961 1962 1963 1964 1965
/**
 * __i40evf_maybe_stop_tx - 2nd level check for tx stop conditions
 * @tx_ring: the ring to be checked
 * @size:    the size buffer we want to assure is available
 *
 * Returns -EBUSY if a stop is needed, else 0
 **/
1966
int __i40evf_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981
{
	netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
	/* Memory barrier before checking head and tail */
	smp_mb();

	/* Check again in a case another CPU has just made room available. */
	if (likely(I40E_DESC_UNUSED(tx_ring) < size))
		return -EBUSY;

	/* A reprieve! - use start_queue because it doesn't call schedule */
	netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
	++tx_ring->tx_stats.restart_queue;
	return 0;
}

1982
/**
1983
 * i40evf_tx_map - Build the Tx descriptor
1984 1985 1986 1987 1988 1989 1990 1991
 * @tx_ring:  ring to send buffer on
 * @skb:      send buffer
 * @first:    first buffer info buffer to use
 * @tx_flags: collected send information
 * @hdr_len:  size of the packet header
 * @td_cmd:   the command field in the descriptor
 * @td_offset: offset for checksum or crc
 **/
1992 1993 1994
static inline void i40evf_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
				 struct i40e_tx_buffer *first, u32 tx_flags,
				 const u8 hdr_len, u32 td_cmd, u32 td_offset)
1995 1996 1997 1998 1999 2000 2001 2002 2003
{
	unsigned int data_len = skb->data_len;
	unsigned int size = skb_headlen(skb);
	struct skb_frag_struct *frag;
	struct i40e_tx_buffer *tx_bi;
	struct i40e_tx_desc *tx_desc;
	u16 i = tx_ring->next_to_use;
	u32 td_tag = 0;
	dma_addr_t dma;
2004
	u16 desc_count = 1;
2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019

	if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
		td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
		td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
			 I40E_TX_FLAGS_VLAN_SHIFT;
	}

	first->tx_flags = tx_flags;

	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);

	tx_desc = I40E_TX_DESC(tx_ring, i);
	tx_bi = first;

	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2020 2021
		unsigned int max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;

2022 2023 2024 2025 2026 2027 2028
		if (dma_mapping_error(tx_ring->dev, dma))
			goto dma_error;

		/* record length, and DMA address */
		dma_unmap_len_set(tx_bi, len, size);
		dma_unmap_addr_set(tx_bi, dma, dma);

2029 2030
		/* align size to end of page */
		max_data += -dma & (I40E_MAX_READ_REQ_SIZE - 1);
2031 2032 2033 2034 2035
		tx_desc->buffer_addr = cpu_to_le64(dma);

		while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
			tx_desc->cmd_type_offset_bsz =
				build_ctob(td_cmd, td_offset,
2036
					   max_data, td_tag);
2037 2038 2039

			tx_desc++;
			i++;
2040 2041
			desc_count++;

2042 2043 2044 2045 2046
			if (i == tx_ring->count) {
				tx_desc = I40E_TX_DESC(tx_ring, 0);
				i = 0;
			}

2047 2048
			dma += max_data;
			size -= max_data;
2049

2050
			max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;
2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061
			tx_desc->buffer_addr = cpu_to_le64(dma);
		}

		if (likely(!data_len))
			break;

		tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
							  size, td_tag);

		tx_desc++;
		i++;
2062 2063
		desc_count++;

2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077
		if (i == tx_ring->count) {
			tx_desc = I40E_TX_DESC(tx_ring, 0);
			i = 0;
		}

		size = skb_frag_size(frag);
		data_len -= size;

		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
				       DMA_TO_DEVICE);

		tx_bi = &tx_ring->tx_bi[i];
	}

2078
	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
2079 2080 2081 2082 2083 2084 2085

	i++;
	if (i == tx_ring->count)
		i = 0;

	tx_ring->next_to_use = i;

2086
	i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2087

2088 2089 2090 2091 2092 2093 2094 2095 2096
	/* write last descriptor with EOP bit */
	td_cmd |= I40E_TX_DESC_CMD_EOP;

	/* We can OR these values together as they both are checked against
	 * 4 below and at this point desc_count will be used as a boolean value
	 * after this if/else block.
	 */
	desc_count |= ++tx_ring->packet_stride;

2097
	/* Algorithm to optimize tail and RS bit setting:
2098 2099 2100 2101 2102 2103
	 * if queue is stopped
	 *	mark RS bit
	 *	reset packet counter
	 * else if xmit_more is supported and is true
	 *	advance packet counter to 4
	 *	reset desc_count to 0
2104
	 *
2105 2106 2107 2108 2109
	 * if desc_count >= 4
	 *	mark RS bit
	 *	reset packet counter
	 * if desc_count > 0
	 *	update tail
2110
	 *
2111
	 * Note: If there are less than 4 descriptors
2112 2113 2114
	 * pending and interrupts were disabled the service task will
	 * trigger a force WB.
	 */
2115 2116 2117 2118 2119 2120 2121 2122 2123 2124
	if (netif_xmit_stopped(txring_txq(tx_ring))) {
		goto do_rs;
	} else if (skb->xmit_more) {
		/* set stride to arm on next packet and reset desc_count */
		tx_ring->packet_stride = WB_STRIDE;
		desc_count = 0;
	} else if (desc_count >= WB_STRIDE) {
do_rs:
		/* write last descriptor with RS bit set */
		td_cmd |= I40E_TX_DESC_CMD_RS;
2125 2126 2127 2128
		tx_ring->packet_stride = 0;
	}

	tx_desc->cmd_type_offset_bsz =
2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140
			build_ctob(td_cmd, td_offset, size, td_tag);

	/* Force memory writes to complete before letting h/w know there
	 * are new descriptors to fetch.
	 *
	 * We also use this memory barrier to make certain all of the
	 * status bits have been updated before next_to_watch is written.
	 */
	wmb();

	/* set next_to_watch value indicating a packet is present */
	first->next_to_watch = tx_desc;
2141

2142
	/* notify HW of packet */
2143
	if (desc_count) {
2144
		writel(i, tx_ring->tail);
2145 2146 2147 2148 2149

		/* we need this if more than one processor can write to our tail
		 * at a time, it synchronizes IO on IA64/Altix systems
		 */
		mmiowb();
2150
	}
2151

2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188
	return;

dma_error:
	dev_info(tx_ring->dev, "TX DMA map failed\n");

	/* clear dma mappings for failed tx_bi map */
	for (;;) {
		tx_bi = &tx_ring->tx_bi[i];
		i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
		if (tx_bi == first)
			break;
		if (i == 0)
			i = tx_ring->count;
		i--;
	}

	tx_ring->next_to_use = i;
}

/**
 * i40e_xmit_frame_ring - Sends buffer on Tx ring
 * @skb:     send buffer
 * @tx_ring: ring to send buffer on
 *
 * Returns NETDEV_TX_OK if sent, else an error code
 **/
static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
					struct i40e_ring *tx_ring)
{
	u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
	u32 cd_tunneling = 0, cd_l2tag2 = 0;
	struct i40e_tx_buffer *first;
	u32 td_offset = 0;
	u32 tx_flags = 0;
	__be16 protocol;
	u32 td_cmd = 0;
	u8 hdr_len = 0;
2189
	int tso, count;
J
Jesse Brandeburg 已提交
2190

2191 2192 2193
	/* prefetch the data, we'll need it later */
	prefetch(skb->data);

2194
	count = i40e_xmit_descriptor_count(skb);
2195
	if (i40e_chk_linearize(skb, count)) {
2196 2197 2198 2199
		if (__skb_linearize(skb)) {
			dev_kfree_skb_any(skb);
			return NETDEV_TX_OK;
		}
2200
		count = i40e_txd_use_count(skb->len);
2201 2202
		tx_ring->tx_stats.tx_linearize++;
	}
2203 2204 2205 2206 2207 2208 2209 2210 2211

	/* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
	 *       + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
	 *       + 4 desc gap to avoid the cache line where head is,
	 *       + 1 desc for context descriptor,
	 * otherwise try next time
	 */
	if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
		tx_ring->tx_stats.tx_busy++;
2212
		return NETDEV_TX_BUSY;
2213
	}
2214

2215 2216 2217 2218 2219 2220
	/* record the location of the first descriptor for this packet */
	first = &tx_ring->tx_bi[tx_ring->next_to_use];
	first->skb = skb;
	first->bytecount = skb->len;
	first->gso_segs = 1;

2221
	/* prepare the xmit flags */
2222
	if (i40evf_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2223 2224 2225
		goto out_drop;

	/* obtain protocol of skb */
2226
	protocol = vlan_get_protocol(skb);
2227 2228 2229 2230 2231 2232 2233

	/* setup IPv4/IPv6 offloads */
	if (protocol == htons(ETH_P_IP))
		tx_flags |= I40E_TX_FLAGS_IPV4;
	else if (protocol == htons(ETH_P_IPV6))
		tx_flags |= I40E_TX_FLAGS_IPV6;

2234
	tso = i40e_tso(first, &hdr_len, &cd_type_cmd_tso_mss);
2235 2236 2237 2238 2239 2240 2241

	if (tso < 0)
		goto out_drop;
	else if (tso)
		tx_flags |= I40E_TX_FLAGS_TSO;

	/* Always offload the checksum, since it's in the data descriptor */
2242 2243 2244 2245
	tso = i40e_tx_enable_csum(skb, &tx_flags, &td_cmd, &td_offset,
				  tx_ring, &cd_tunneling);
	if (tso < 0)
		goto out_drop;
2246

2247 2248 2249 2250 2251
	skb_tx_timestamp(skb);

	/* always enable CRC insertion offload */
	td_cmd |= I40E_TX_DESC_CMD_ICRC;

2252 2253 2254
	i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
			   cd_tunneling, cd_l2tag2);

2255 2256
	i40evf_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
		      td_cmd, td_offset);
2257 2258 2259 2260

	return NETDEV_TX_OK;

out_drop:
2261 2262
	dev_kfree_skb_any(first->skb);
	first->skb = NULL;
2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275
	return NETDEV_TX_OK;
}

/**
 * i40evf_xmit_frame - Selects the correct VSI and Tx queue to send buffer
 * @skb:    send buffer
 * @netdev: network interface device structure
 *
 * Returns NETDEV_TX_OK if sent, else an error code
 **/
netdev_tx_t i40evf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
{
	struct i40evf_adapter *adapter = netdev_priv(netdev);
2276
	struct i40e_ring *tx_ring = &adapter->tx_rings[skb->queue_mapping];
2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289

	/* hardware can't handle really short frames, hardware padding works
	 * beyond this point
	 */
	if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
		if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
			return NETDEV_TX_OK;
		skb->len = I40E_MIN_TX_LEN;
		skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
	}

	return i40e_xmit_frame_ring(skb, tx_ring);
}