i40e_txrx.c 61.1 KB
Newer Older
1 2 3
/*******************************************************************************
 *
 * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
4
 * Copyright(c) 2013 - 2016 Intel Corporation.
5 6 7 8 9 10 11 12 13 14
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
15 16 17
 * You should have received a copy of the GNU General Public License along
 * with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
18 19 20 21 22 23 24 25 26
 * The full GNU General Public License is included in this distribution in
 * the file called "COPYING".
 *
 * Contact Information:
 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
 *
 ******************************************************************************/

27
#include <linux/prefetch.h>
28
#include <net/busy_poll.h>
29

30
#include "i40evf.h"
31
#include "i40e_prototype.h"
32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53

static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
				u32 td_tag)
{
	return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
			   ((u64)td_cmd  << I40E_TXD_QW1_CMD_SHIFT) |
			   ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
			   ((u64)size  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
			   ((u64)td_tag  << I40E_TXD_QW1_L2TAG1_SHIFT));
}

#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)

/**
 * i40e_unmap_and_free_tx_resource - Release a Tx buffer
 * @ring:      the ring that owns the buffer
 * @tx_buffer: the buffer to free
 **/
static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
					    struct i40e_tx_buffer *tx_buffer)
{
	if (tx_buffer->skb) {
54 55 56 57
		if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
			kfree(tx_buffer->raw_buf);
		else
			dev_kfree_skb_any(tx_buffer->skb);
58 59 60 61 62 63 64 65 66 67 68
		if (dma_unmap_len(tx_buffer, len))
			dma_unmap_single(ring->dev,
					 dma_unmap_addr(tx_buffer, dma),
					 dma_unmap_len(tx_buffer, len),
					 DMA_TO_DEVICE);
	} else if (dma_unmap_len(tx_buffer, len)) {
		dma_unmap_page(ring->dev,
			       dma_unmap_addr(tx_buffer, dma),
			       dma_unmap_len(tx_buffer, len),
			       DMA_TO_DEVICE);
	}
69

70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
	tx_buffer->next_to_watch = NULL;
	tx_buffer->skb = NULL;
	dma_unmap_len_set(tx_buffer, len, 0);
	/* tx_buffer must be completely set up in the transmit path */
}

/**
 * i40evf_clean_tx_ring - Free any empty Tx buffers
 * @tx_ring: ring to be cleaned
 **/
void i40evf_clean_tx_ring(struct i40e_ring *tx_ring)
{
	unsigned long bi_size;
	u16 i;

	/* ring already cleared, nothing to do */
	if (!tx_ring->tx_bi)
		return;

	/* Free all the Tx ring sk_buffs */
	for (i = 0; i < tx_ring->count; i++)
		i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);

	bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
	memset(tx_ring->tx_bi, 0, bi_size);

	/* Zero out the descriptor ring */
	memset(tx_ring->desc, 0, tx_ring->size);

	tx_ring->next_to_use = 0;
	tx_ring->next_to_clean = 0;

	if (!tx_ring->netdev)
		return;

	/* cleanup Tx queue statistics */
106
	netdev_tx_reset_queue(txring_txq(tx_ring));
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
}

/**
 * i40evf_free_tx_resources - Free Tx resources per queue
 * @tx_ring: Tx descriptor ring for a specific queue
 *
 * Free all transmit software resources
 **/
void i40evf_free_tx_resources(struct i40e_ring *tx_ring)
{
	i40evf_clean_tx_ring(tx_ring);
	kfree(tx_ring->tx_bi);
	tx_ring->tx_bi = NULL;

	if (tx_ring->desc) {
		dma_free_coherent(tx_ring->dev, tx_ring->size,
				  tx_ring->desc, tx_ring->dma);
		tx_ring->desc = NULL;
	}
}

J
Jesse Brandeburg 已提交
128
/**
129 130
 * i40evf_get_tx_pending - how many Tx descriptors not processed
 * @tx_ring: the ring of descriptors
131
 * @in_sw: is tx_pending being checked in SW or HW
J
Jesse Brandeburg 已提交
132
 *
133 134
 * Since there is no access to the ring head register
 * in XL710, we need to use our local copies
J
Jesse Brandeburg 已提交
135
 **/
136
u32 i40evf_get_tx_pending(struct i40e_ring *ring, bool in_sw)
J
Jesse Brandeburg 已提交
137
{
138
	u32 head, tail;
J
Jesse Brandeburg 已提交
139

140 141 142 143
	if (!in_sw)
		head = i40e_get_head(ring);
	else
		head = ring->next_to_clean;
144 145 146 147 148 149 150
	tail = readl(ring->tail);

	if (head != tail)
		return (head < tail) ?
			tail - head : (tail + ring->count - head);

	return 0;
J
Jesse Brandeburg 已提交
151 152
}

153
#define WB_STRIDE 4
154

155 156
/**
 * i40e_clean_tx_irq - Reclaim resources after transmit completes
157 158 159
 * @vsi: the VSI we care about
 * @tx_ring: Tx ring to clean
 * @napi_budget: Used to determine if we are in netpoll
160 161 162
 *
 * Returns true if there's any budget left (e.g. the clean is finished)
 **/
163 164
static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,
			      struct i40e_ring *tx_ring, int napi_budget)
165 166 167
{
	u16 i = tx_ring->next_to_clean;
	struct i40e_tx_buffer *tx_buf;
168
	struct i40e_tx_desc *tx_head;
169
	struct i40e_tx_desc *tx_desc;
170 171
	unsigned int total_bytes = 0, total_packets = 0;
	unsigned int budget = vsi->work_limit;
172 173 174 175 176

	tx_buf = &tx_ring->tx_bi[i];
	tx_desc = I40E_TX_DESC(tx_ring, i);
	i -= tx_ring->count;

177 178
	tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring));

179 180 181 182 183 184 185 186 187 188
	do {
		struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;

		/* if next_to_watch is not set then there is no work pending */
		if (!eop_desc)
			break;

		/* prevent any other reads prior to eop_desc */
		read_barrier_depends();

189 190
		/* we have caught up to head, no work left to do */
		if (tx_head == tx_desc)
191 192 193 194 195 196 197 198 199 200
			break;

		/* clear next_to_watch to prevent false hangs */
		tx_buf->next_to_watch = NULL;

		/* update the statistics for this packet */
		total_bytes += tx_buf->bytecount;
		total_packets += tx_buf->gso_segs;

		/* free the skb */
201
		napi_consume_skb(tx_buf->skb, napi_budget);
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244

		/* unmap skb header data */
		dma_unmap_single(tx_ring->dev,
				 dma_unmap_addr(tx_buf, dma),
				 dma_unmap_len(tx_buf, len),
				 DMA_TO_DEVICE);

		/* clear tx_buffer data */
		tx_buf->skb = NULL;
		dma_unmap_len_set(tx_buf, len, 0);

		/* unmap remaining buffers */
		while (tx_desc != eop_desc) {

			tx_buf++;
			tx_desc++;
			i++;
			if (unlikely(!i)) {
				i -= tx_ring->count;
				tx_buf = tx_ring->tx_bi;
				tx_desc = I40E_TX_DESC(tx_ring, 0);
			}

			/* unmap any remaining paged data */
			if (dma_unmap_len(tx_buf, len)) {
				dma_unmap_page(tx_ring->dev,
					       dma_unmap_addr(tx_buf, dma),
					       dma_unmap_len(tx_buf, len),
					       DMA_TO_DEVICE);
				dma_unmap_len_set(tx_buf, len, 0);
			}
		}

		/* move us one more past the eop_desc for start of next pkt */
		tx_buf++;
		tx_desc++;
		i++;
		if (unlikely(!i)) {
			i -= tx_ring->count;
			tx_buf = tx_ring->tx_bi;
			tx_desc = I40E_TX_DESC(tx_ring, 0);
		}

245 246
		prefetch(tx_desc);

247 248 249 250 251 252 253 254 255 256 257 258 259
		/* update budget accounting */
		budget--;
	} while (likely(budget));

	i += tx_ring->count;
	tx_ring->next_to_clean = i;
	u64_stats_update_begin(&tx_ring->syncp);
	tx_ring->stats.bytes += total_bytes;
	tx_ring->stats.packets += total_packets;
	u64_stats_update_end(&tx_ring->syncp);
	tx_ring->q_vector->tx.total_bytes += total_bytes;
	tx_ring->q_vector->tx.total_packets += total_packets;

260 261 262 263 264 265
	if (tx_ring->flags & I40E_TXR_FLAGS_WB_ON_ITR) {
		/* check to see if there are < 4 descriptors
		 * waiting to be written back, then kick the hardware to force
		 * them to be written back in case we stay in NAPI.
		 * In this mode on X722 we do not enable Interrupt.
		 */
266
		unsigned int j = i40evf_get_tx_pending(tx_ring, false);
267 268

		if (budget &&
269
		    ((j / WB_STRIDE) == 0) && (j > 0) &&
270
		    !test_bit(__I40E_DOWN, &vsi->state) &&
271 272 273 274
		    (I40E_DESC_UNUSED(tx_ring) != tx_ring->count))
			tx_ring->arm_wb = true;
	}

275 276
	/* notify netdev of completed buffers */
	netdev_tx_completed_queue(txring_txq(tx_ring),
277 278 279 280 281 282 283 284 285 286 287
				  total_packets, total_bytes);

#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
	if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
		     (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
		/* Make sure that anybody stopping the queue after this
		 * sees the new next_to_clean.
		 */
		smp_mb();
		if (__netif_subqueue_stopped(tx_ring->netdev,
					     tx_ring->queue_index) &&
288
		   !test_bit(__I40E_DOWN, &vsi->state)) {
289 290 291 292 293 294
			netif_wake_subqueue(tx_ring->netdev,
					    tx_ring->queue_index);
			++tx_ring->tx_stats.restart_queue;
		}
	}

295
	return !!budget;
296 297
}

298
/**
299
 * i40evf_enable_wb_on_itr - Arm hardware to do a wb, interrupts are not enabled
300
 * @vsi: the VSI we care about
301
 * @q_vector: the vector on which to enable writeback
302 303
 *
 **/
304 305
static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi,
				  struct i40e_q_vector *q_vector)
306
{
307
	u16 flags = q_vector->tx.ring[0].flags;
308
	u32 val;
309

310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
	if (!(flags & I40E_TXR_FLAGS_WB_ON_ITR))
		return;

	if (q_vector->arm_wb_state)
		return;

	val = I40E_VFINT_DYN_CTLN1_WB_ON_ITR_MASK |
	      I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK; /* set noitr */

	wr32(&vsi->back->hw,
	     I40E_VFINT_DYN_CTLN1(q_vector->v_idx +
				  vsi->base_vector - 1), val);
	q_vector->arm_wb_state = true;
}

/**
 * i40evf_force_wb - Issue SW Interrupt so HW does a wb
 * @vsi: the VSI we care about
 * @q_vector: the vector  on which to force writeback
 *
 **/
void i40evf_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
{
	u32 val = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
		  I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK | /* set noitr */
		  I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK |
		  I40E_VFINT_DYN_CTLN1_SW_ITR_INDX_ENA_MASK
		  /* allow 00 to be written to the index */;

	wr32(&vsi->back->hw,
	     I40E_VFINT_DYN_CTLN1(q_vector->v_idx + vsi->base_vector - 1),
	     val);
342 343
}

344 345 346 347
/**
 * i40e_set_new_dynamic_itr - Find new ITR level
 * @rc: structure containing ring performance data
 *
348 349
 * Returns true if ITR changed, false if not
 *
350 351 352 353 354 355 356 357
 * Stores a new ITR value based on packets and byte counts during
 * the last interrupt.  The advantage of per interrupt computation
 * is faster updates and more accurate ITR for the current traffic
 * pattern.  Constants in this function were computed based on
 * theoretical maximum wire speed and thresholds were set based on
 * testing data as well as attempting to minimize response time
 * while increasing bulk throughput.
 **/
358
static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
359 360
{
	enum i40e_latency_range new_latency_range = rc->latency_range;
361
	struct i40e_q_vector *qv = rc->ring->q_vector;
362 363
	u32 new_itr = rc->itr;
	int bytes_per_int;
364
	int usecs;
365 366

	if (rc->total_packets == 0 || !rc->itr)
367
		return false;
368 369

	/* simple throttlerate management
370
	 *   0-10MB/s   lowest (50000 ints/s)
371
	 *  10-20MB/s   low    (20000 ints/s)
372 373
	 *  20-1249MB/s bulk   (18000 ints/s)
	 *  > 40000 Rx packets per second (8000 ints/s)
374 375 376 377
	 *
	 * The math works out because the divisor is in 10^(-6) which
	 * turns the bytes/us input value into MB/s values, but
	 * make sure to use usecs, as the register values written
378 379
	 * are in 2 usec increments in the ITR registers, and make sure
	 * to use the smoothed values that the countdown timer gives us.
380
	 */
381
	usecs = (rc->itr << 1) * ITR_COUNTDOWN_START;
382
	bytes_per_int = rc->total_bytes / usecs;
383

384
	switch (new_latency_range) {
385 386 387 388 389 390 391 392 393 394 395
	case I40E_LOWEST_LATENCY:
		if (bytes_per_int > 10)
			new_latency_range = I40E_LOW_LATENCY;
		break;
	case I40E_LOW_LATENCY:
		if (bytes_per_int > 20)
			new_latency_range = I40E_BULK_LATENCY;
		else if (bytes_per_int <= 10)
			new_latency_range = I40E_LOWEST_LATENCY;
		break;
	case I40E_BULK_LATENCY:
396
	case I40E_ULTRA_LATENCY:
397 398 399
	default:
		if (bytes_per_int <= 20)
			new_latency_range = I40E_LOW_LATENCY;
400 401
		break;
	}
402 403 404 405 406 407 408 409 410 411 412 413

	/* this is to adjust RX more aggressively when streaming small
	 * packets.  The value of 40000 was picked as it is just beyond
	 * what the hardware can receive per second if in low latency
	 * mode.
	 */
#define RX_ULTRA_PACKET_RATE 40000

	if ((((rc->total_packets * 1000000) / usecs) > RX_ULTRA_PACKET_RATE) &&
	    (&qv->rx == rc))
		new_latency_range = I40E_ULTRA_LATENCY;

414
	rc->latency_range = new_latency_range;
415 416 417

	switch (new_latency_range) {
	case I40E_LOWEST_LATENCY:
418
		new_itr = I40E_ITR_50K;
419 420 421 422 423
		break;
	case I40E_LOW_LATENCY:
		new_itr = I40E_ITR_20K;
		break;
	case I40E_BULK_LATENCY:
424 425 426
		new_itr = I40E_ITR_18K;
		break;
	case I40E_ULTRA_LATENCY:
427 428 429 430 431 432 433 434
		new_itr = I40E_ITR_8K;
		break;
	default:
		break;
	}

	rc->total_bytes = 0;
	rc->total_packets = 0;
435 436 437 438 439 440 441

	if (new_itr != rc->itr) {
		rc->itr = new_itr;
		return true;
	}

	return false;
442 443
}

J
Jesse Brandeburg 已提交
444
/**
445 446 447 448 449 450 451 452 453 454 455 456 457
 * i40evf_setup_tx_descriptors - Allocate the Tx descriptors
 * @tx_ring: the tx ring to set up
 *
 * Return 0 on success, negative on error
 **/
int i40evf_setup_tx_descriptors(struct i40e_ring *tx_ring)
{
	struct device *dev = tx_ring->dev;
	int bi_size;

	if (!dev)
		return -ENOMEM;

458 459
	/* warn if we are about to overwrite the pointer */
	WARN_ON(tx_ring->tx_bi);
460 461 462 463 464 465 466
	bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
	tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
	if (!tx_ring->tx_bi)
		goto err;

	/* round up to nearest 4K */
	tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
467 468 469 470
	/* add u32 for head writeback, align after this takes care of
	 * guaranteeing this is at least one cache line in size
	 */
	tx_ring->size += sizeof(u32);
471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505
	tx_ring->size = ALIGN(tx_ring->size, 4096);
	tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
					   &tx_ring->dma, GFP_KERNEL);
	if (!tx_ring->desc) {
		dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
			 tx_ring->size);
		goto err;
	}

	tx_ring->next_to_use = 0;
	tx_ring->next_to_clean = 0;
	return 0;

err:
	kfree(tx_ring->tx_bi);
	tx_ring->tx_bi = NULL;
	return -ENOMEM;
}

/**
 * i40evf_clean_rx_ring - Free Rx buffers
 * @rx_ring: ring to be cleaned
 **/
void i40evf_clean_rx_ring(struct i40e_ring *rx_ring)
{
	struct device *dev = rx_ring->dev;
	unsigned long bi_size;
	u16 i;

	/* ring already cleared, nothing to do */
	if (!rx_ring->rx_bi)
		return;

	/* Free all the Rx ring sk_buffs */
	for (i = 0; i < rx_ring->count; i++) {
506 507
		struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i];

508 509 510 511
		if (rx_bi->skb) {
			dev_kfree_skb(rx_bi->skb);
			rx_bi->skb = NULL;
		}
512 513 514 515 516 517 518 519
		if (!rx_bi->page)
			continue;

		dma_unmap_page(dev, rx_bi->dma, PAGE_SIZE, DMA_FROM_DEVICE);
		__free_pages(rx_bi->page, 0);

		rx_bi->page = NULL;
		rx_bi->page_offset = 0;
520 521 522 523 524 525 526 527
	}

	bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
	memset(rx_ring->rx_bi, 0, bi_size);

	/* Zero out the descriptor ring */
	memset(rx_ring->desc, 0, rx_ring->size);

528
	rx_ring->next_to_alloc = 0;
529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562
	rx_ring->next_to_clean = 0;
	rx_ring->next_to_use = 0;
}

/**
 * i40evf_free_rx_resources - Free Rx resources
 * @rx_ring: ring to clean the resources from
 *
 * Free all receive software resources
 **/
void i40evf_free_rx_resources(struct i40e_ring *rx_ring)
{
	i40evf_clean_rx_ring(rx_ring);
	kfree(rx_ring->rx_bi);
	rx_ring->rx_bi = NULL;

	if (rx_ring->desc) {
		dma_free_coherent(rx_ring->dev, rx_ring->size,
				  rx_ring->desc, rx_ring->dma);
		rx_ring->desc = NULL;
	}
}

/**
 * i40evf_setup_rx_descriptors - Allocate Rx descriptors
 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
 *
 * Returns 0 on success, negative on failure
 **/
int i40evf_setup_rx_descriptors(struct i40e_ring *rx_ring)
{
	struct device *dev = rx_ring->dev;
	int bi_size;

563 564
	/* warn if we are about to overwrite the pointer */
	WARN_ON(rx_ring->rx_bi);
565 566 567 568 569
	bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
	rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
	if (!rx_ring->rx_bi)
		goto err;

570
	u64_stats_init(&rx_ring->syncp);
571

572
	/* Round up to nearest 4K */
573
	rx_ring->size = rx_ring->count * sizeof(union i40e_32byte_rx_desc);
574 575 576 577 578 579 580 581 582 583
	rx_ring->size = ALIGN(rx_ring->size, 4096);
	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
					   &rx_ring->dma, GFP_KERNEL);

	if (!rx_ring->desc) {
		dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
			 rx_ring->size);
		goto err;
	}

584
	rx_ring->next_to_alloc = 0;
585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602
	rx_ring->next_to_clean = 0;
	rx_ring->next_to_use = 0;

	return 0;
err:
	kfree(rx_ring->rx_bi);
	rx_ring->rx_bi = NULL;
	return -ENOMEM;
}

/**
 * i40e_release_rx_desc - Store the new tail and head values
 * @rx_ring: ring to bump
 * @val: new head index
 **/
static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
{
	rx_ring->next_to_use = val;
603 604 605 606

	/* update next to alloc since we have filled the ring */
	rx_ring->next_to_alloc = val;

607 608 609 610 611 612 613 614 615 616
	/* Force memory writes to complete before letting h/w
	 * know there are new descriptors to fetch.  (Only
	 * applicable for weak-ordered memory model archs,
	 * such as IA-64).
	 */
	wmb();
	writel(val, rx_ring->tail);
}

/**
617 618 619
 * i40e_alloc_mapped_page - recycle or make a new page
 * @rx_ring: ring to use
 * @bi: rx_buffer struct to modify
620
 *
621 622
 * Returns true if the page was successfully allocated or
 * reused.
623
 **/
624 625
static bool i40e_alloc_mapped_page(struct i40e_ring *rx_ring,
				   struct i40e_rx_buffer *bi)
626
{
627 628
	struct page *page = bi->page;
	dma_addr_t dma;
629

630 631 632 633 634
	/* since we are recycling buffers we should seldom need to alloc */
	if (likely(page)) {
		rx_ring->rx_stats.page_reuse_count++;
		return true;
	}
635

636 637 638 639 640 641
	/* alloc new page for storage */
	page = dev_alloc_page();
	if (unlikely(!page)) {
		rx_ring->rx_stats.alloc_page_failed++;
		return false;
	}
642

643 644
	/* map page for use */
	dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
645

646 647
	/* if mapping failed free memory back to system since
	 * there isn't much point in holding memory we can't use
648
	 */
649 650 651 652
	if (dma_mapping_error(rx_ring->dev, dma)) {
		__free_pages(page, 0);
		rx_ring->rx_stats.alloc_page_failed++;
		return false;
653 654
	}

655 656 657
	bi->dma = dma;
	bi->page = page;
	bi->page_offset = 0;
658

659 660
	return true;
}
661

662 663 664 665 666 667 668 669 670 671
/**
 * i40e_receive_skb - Send a completed packet up the stack
 * @rx_ring:  rx ring in play
 * @skb: packet to send up
 * @vlan_tag: vlan tag for packet
 **/
static void i40e_receive_skb(struct i40e_ring *rx_ring,
			     struct sk_buff *skb, u16 vlan_tag)
{
	struct i40e_q_vector *q_vector = rx_ring->q_vector;
672

673 674 675 676 677
	if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
	    (vlan_tag & VLAN_VID_MASK))
		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);

	napi_gro_receive(&q_vector->napi, skb);
678 679 680
}

/**
681
 * i40evf_alloc_rx_buffers - Replace used receive buffers
682 683
 * @rx_ring: ring to place buffers on
 * @cleaned_count: number of buffers to replace
684
 *
685
 * Returns false if all allocations were successful, true if any fail
686
 **/
687
bool i40evf_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
688
{
689
	u16 ntu = rx_ring->next_to_use;
690 691 692 693 694
	union i40e_rx_desc *rx_desc;
	struct i40e_rx_buffer *bi;

	/* do nothing if no valid netdev defined */
	if (!rx_ring->netdev || !cleaned_count)
695
		return false;
696

697 698
	rx_desc = I40E_RX_DESC(rx_ring, ntu);
	bi = &rx_ring->rx_bi[ntu];
699

700 701 702
	do {
		if (!i40e_alloc_mapped_page(rx_ring, bi))
			goto no_buffers;
703

704 705 706 707
		/* Refresh the desc even if buffer_addrs didn't change
		 * because each write-back erases this info.
		 */
		rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
708

709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
		rx_desc++;
		bi++;
		ntu++;
		if (unlikely(ntu == rx_ring->count)) {
			rx_desc = I40E_RX_DESC(rx_ring, 0);
			bi = rx_ring->rx_bi;
			ntu = 0;
		}

		/* clear the status bits for the next_to_use descriptor */
		rx_desc->wb.qword1.status_error_len = 0;

		cleaned_count--;
	} while (cleaned_count);

	if (rx_ring->next_to_use != ntu)
		i40e_release_rx_desc(rx_ring, ntu);
726 727 728

	return false;

729
no_buffers:
730 731
	if (rx_ring->next_to_use != ntu)
		i40e_release_rx_desc(rx_ring, ntu);
732 733 734 735 736

	/* make sure to come back via polling to try again after
	 * allocation failure
	 */
	return true;
737 738 739 740 741 742
}

/**
 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
 * @vsi: the VSI we care about
 * @skb: skb currently being received and modified
743 744 745
 * @rx_desc: the receive descriptor
 *
 * skb->protocol must be set before this function is called
746 747 748
 **/
static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
				    struct sk_buff *skb,
749
				    union i40e_rx_desc *rx_desc)
750
{
751 752
	struct i40e_rx_ptype_decoded decoded;
	u32 rx_error, rx_status;
753
	bool ipv4, ipv6;
754 755 756 757 758 759 760 761 762 763
	u8 ptype;
	u64 qword;

	qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
	ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT;
	rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
		   I40E_RXD_QW1_ERROR_SHIFT;
	rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
		    I40E_RXD_QW1_STATUS_SHIFT;
	decoded = decode_rx_desc_ptype(ptype);
764 765 766

	skb->ip_summed = CHECKSUM_NONE;

767 768
	skb_checksum_none_assert(skb);

769
	/* Rx csum enabled and ip headers found? */
770 771 772 773
	if (!(vsi->netdev->features & NETIF_F_RXCSUM))
		return;

	/* did the hardware decode the packet and checksum? */
774
	if (!(rx_status & BIT(I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
775 776 777 778
		return;

	/* both known and outer_ip must be set for the below code to work */
	if (!(decoded.known && decoded.outer_ip))
779 780
		return;

781 782 783 784
	ipv4 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
	       (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4);
	ipv6 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
	       (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6);
785 786

	if (ipv4 &&
787 788
	    (rx_error & (BIT(I40E_RX_DESC_ERROR_IPE_SHIFT) |
			 BIT(I40E_RX_DESC_ERROR_EIPE_SHIFT))))
789 790
		goto checksum_fail;

J
Jesse Brandeburg 已提交
791
	/* likely incorrect csum if alternate IP extension headers found */
792
	if (ipv6 &&
793
	    rx_status & BIT(I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
794
		/* don't increment checksum err here, non-fatal err */
795 796
		return;

797
	/* there was some L4 error, count error and punt packet to the stack */
798
	if (rx_error & BIT(I40E_RX_DESC_ERROR_L4E_SHIFT))
799 800 801 802 803 804
		goto checksum_fail;

	/* handle packets that were not able to be checksummed due
	 * to arrival speed, in this case the stack can compute
	 * the csum.
	 */
805
	if (rx_error & BIT(I40E_RX_DESC_ERROR_PPRS_SHIFT))
806 807
		return;

808 809 810
	/* If there is an outer header present that might contain a checksum
	 * we need to bump the checksum level by 1 to reflect the fact that
	 * we are indicating we validated the inner checksum.
811
	 */
812 813 814 815 816 817 818 819 820 821 822 823 824
	if (decoded.tunnel_type >= I40E_RX_PTYPE_TUNNEL_IP_GRENAT)
		skb->csum_level = 1;

	/* Only report checksum unnecessary for TCP, UDP, or SCTP */
	switch (decoded.inner_prot) {
	case I40E_RX_PTYPE_INNER_PROT_TCP:
	case I40E_RX_PTYPE_INNER_PROT_UDP:
	case I40E_RX_PTYPE_INNER_PROT_SCTP:
		skb->ip_summed = CHECKSUM_UNNECESSARY;
		/* fall though */
	default:
		break;
	}
825 826 827 828 829

	return;

checksum_fail:
	vsi->back->hw_csum_rx_error++;
830 831 832
}

/**
833
 * i40e_ptype_to_htype - get a hash type
834 835 836 837
 * @ptype: the ptype value from the descriptor
 *
 * Returns a hash type to be used by skb_set_hash
 **/
838
static inline int i40e_ptype_to_htype(u8 ptype)
839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854
{
	struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);

	if (!decoded.known)
		return PKT_HASH_TYPE_NONE;

	if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
	    decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
		return PKT_HASH_TYPE_L4;
	else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
		 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
		return PKT_HASH_TYPE_L3;
	else
		return PKT_HASH_TYPE_L2;
}

855 856 857 858 859 860 861 862 863 864 865
/**
 * i40e_rx_hash - set the hash value in the skb
 * @ring: descriptor ring
 * @rx_desc: specific descriptor
 **/
static inline void i40e_rx_hash(struct i40e_ring *ring,
				union i40e_rx_desc *rx_desc,
				struct sk_buff *skb,
				u8 rx_ptype)
{
	u32 hash;
866
	const __le64 rss_mask =
867 868 869 870 871 872 873 874 875 876 877 878
		cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
			    I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);

	if (ring->netdev->features & NETIF_F_RXHASH)
		return;

	if ((rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) {
		hash = le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
		skb_set_hash(skb, hash, i40e_ptype_to_htype(rx_ptype));
	}
}

879
/**
880 881 882 883 884
 * i40evf_process_skb_fields - Populate skb header fields from Rx descriptor
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @rx_desc: pointer to the EOP Rx descriptor
 * @skb: pointer to current skb being populated
 * @rx_ptype: the packet type decoded by hardware
885
 *
886 887 888
 * This function checks the ring, descriptor, and packet information in
 * order to populate the hash, checksum, VLAN, protocol, and
 * other fields within the skb.
889
 **/
890 891 892 893
static inline
void i40evf_process_skb_fields(struct i40e_ring *rx_ring,
			       union i40e_rx_desc *rx_desc, struct sk_buff *skb,
			       u8 rx_ptype)
894
{
895
	i40e_rx_hash(rx_ring, rx_desc, skb, rx_ptype);
896

897 898
	/* modifies the skb - consumes the enet header */
	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
899

900
	i40e_rx_checksum(rx_ring->vsi, skb, rx_desc);
901

902 903
	skb_record_rx_queue(skb, rx_ring->queue_index);
}
904

905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921
/**
 * i40e_pull_tail - i40e specific version of skb_pull_tail
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @skb: pointer to current skb being adjusted
 *
 * This function is an i40e specific version of __pskb_pull_tail.  The
 * main difference between this version and the original function is that
 * this function can make several assumptions about the state of things
 * that allow for significant optimizations versus the standard function.
 * As a result we can do things like drop a frag and maintain an accurate
 * truesize for the skb.
 */
static void i40e_pull_tail(struct i40e_ring *rx_ring, struct sk_buff *skb)
{
	struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
	unsigned char *va;
	unsigned int pull_len;
922

923 924 925 926 927
	/* it is valid to use page_address instead of kmap since we are
	 * working with pages allocated out of the lomem pool per
	 * alloc_page(GFP_ATOMIC)
	 */
	va = skb_frag_address(frag);
928

929 930 931 932
	/* we need the header to contain the greater of either ETH_HLEN or
	 * 60 bytes if the skb->len is less than 60 for skb_pad.
	 */
	pull_len = eth_get_headlen(va, I40E_RX_HDR_SIZE);
933

934 935
	/* align pull length to size of long to optimize memcpy performance */
	skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
936

937 938 939 940 941 942
	/* update all of the pointers */
	skb_frag_size_sub(frag, pull_len);
	frag->page_offset += pull_len;
	skb->data_len -= pull_len;
	skb->tail += pull_len;
}
943

944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961
/**
 * i40e_cleanup_headers - Correct empty headers
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @skb: pointer to current skb being fixed
 *
 * Also address the case where we are pulling data in on pages only
 * and as such no data is present in the skb header.
 *
 * In addition if skb is not at least 60 bytes we need to pad it so that
 * it is large enough to qualify as a valid Ethernet frame.
 *
 * Returns true if an error was encountered and skb was freed.
 **/
static bool i40e_cleanup_headers(struct i40e_ring *rx_ring, struct sk_buff *skb)
{
	/* place header in linear portion of buffer */
	if (skb_is_nonlinear(skb))
		i40e_pull_tail(rx_ring, skb);
962

963 964 965
	/* if eth_skb_pad returns an error the skb was freed */
	if (eth_skb_pad(skb))
		return true;
966

967 968
	return false;
}
969

970 971 972 973 974 975 976 977 978 979 980 981
/**
 * i40e_reuse_rx_page - page flip buffer and store it back on the ring
 * @rx_ring: rx descriptor ring to store buffers on
 * @old_buff: donor buffer to have page reused
 *
 * Synchronizes page for reuse by the adapter
 **/
static void i40e_reuse_rx_page(struct i40e_ring *rx_ring,
			       struct i40e_rx_buffer *old_buff)
{
	struct i40e_rx_buffer *new_buff;
	u16 nta = rx_ring->next_to_alloc;
982

983
	new_buff = &rx_ring->rx_bi[nta];
984

985 986 987
	/* update, and store next to alloc */
	nta++;
	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
988

989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
	/* transfer page from old buffer to new buffer */
	*new_buff = *old_buff;
}

/**
 * i40e_page_is_reserved - check if reuse is possible
 * @page: page struct to check
 */
static inline bool i40e_page_is_reserved(struct page *page)
{
	return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
}

/**
 * i40e_add_rx_frag - Add contents of Rx buffer to sk_buff
 * @rx_ring: rx descriptor ring to transact packets on
 * @rx_buffer: buffer containing page to add
 * @rx_desc: descriptor containing length of buffer written by hardware
 * @skb: sk_buff to place the data into
 *
 * This function will add the data contained in rx_buffer->page to the skb.
 * This is done either through a direct copy if the data in the buffer is
 * less than the skb header size, otherwise it will just attach the page as
 * a frag to the skb.
 *
 * The function will then update the page offset if necessary and return
 * true if the buffer can be reused by the adapter.
 **/
static bool i40e_add_rx_frag(struct i40e_ring *rx_ring,
			     struct i40e_rx_buffer *rx_buffer,
			     union i40e_rx_desc *rx_desc,
			     struct sk_buff *skb)
{
	struct page *page = rx_buffer->page;
	u64 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
	unsigned int size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
			    I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
#if (PAGE_SIZE < 8192)
	unsigned int truesize = I40E_RXBUFFER_2048;
#else
	unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
	unsigned int last_offset = PAGE_SIZE - I40E_RXBUFFER_2048;
1031
#endif
1032

1033 1034 1035 1036 1037
	/* will the data fit in the skb we allocated? if so, just
	 * copy it as it is pretty small anyway
	 */
	if ((size <= I40E_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) {
		unsigned char *va = page_address(page) + rx_buffer->page_offset;
1038

1039
		memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
1040

1041 1042 1043
		/* page is not reserved, we can reuse buffer as-is */
		if (likely(!i40e_page_is_reserved(page)))
			return true;
1044

1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187
		/* this page cannot be reused so discard it */
		__free_pages(page, 0);
		return false;
	}

	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
			rx_buffer->page_offset, size, truesize);

	/* avoid re-using remote pages */
	if (unlikely(i40e_page_is_reserved(page)))
		return false;

#if (PAGE_SIZE < 8192)
	/* if we are only owner of page we can reuse it */
	if (unlikely(page_count(page) != 1))
		return false;

	/* flip page offset to other buffer */
	rx_buffer->page_offset ^= truesize;
#else
	/* move offset up to the next cache line */
	rx_buffer->page_offset += truesize;

	if (rx_buffer->page_offset > last_offset)
		return false;
#endif

	/* Even if we own the page, we are not allowed to use atomic_set()
	 * This would break get_page_unless_zero() users.
	 */
	get_page(rx_buffer->page);

	return true;
}

/**
 * i40evf_fetch_rx_buffer - Allocate skb and populate it
 * @rx_ring: rx descriptor ring to transact packets on
 * @rx_desc: descriptor containing info written by hardware
 *
 * This function allocates an skb on the fly, and populates it with the page
 * data from the current receive descriptor, taking care to set up the skb
 * correctly, as well as handling calling the page recycle function if
 * necessary.
 */
static inline
struct sk_buff *i40evf_fetch_rx_buffer(struct i40e_ring *rx_ring,
				       union i40e_rx_desc *rx_desc)
{
	struct i40e_rx_buffer *rx_buffer;
	struct sk_buff *skb;
	struct page *page;

	rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean];
	page = rx_buffer->page;
	prefetchw(page);

	skb = rx_buffer->skb;

	if (likely(!skb)) {
		void *page_addr = page_address(page) + rx_buffer->page_offset;

		/* prefetch first cache line of first page */
		prefetch(page_addr);
#if L1_CACHE_BYTES < 128
		prefetch(page_addr + L1_CACHE_BYTES);
#endif

		/* allocate a skb to store the frags */
		skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
				       I40E_RX_HDR_SIZE,
				       GFP_ATOMIC | __GFP_NOWARN);
		if (unlikely(!skb)) {
			rx_ring->rx_stats.alloc_buff_failed++;
			return NULL;
		}

		/* we will be copying header into skb->data in
		 * pskb_may_pull so it is in our interest to prefetch
		 * it now to avoid a possible cache miss
		 */
		prefetchw(skb->data);
	} else {
		rx_buffer->skb = NULL;
	}

	/* we are reusing so sync this buffer for CPU use */
	dma_sync_single_range_for_cpu(rx_ring->dev,
				      rx_buffer->dma,
				      rx_buffer->page_offset,
				      I40E_RXBUFFER_2048,
				      DMA_FROM_DEVICE);

	/* pull page into skb */
	if (i40e_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
		/* hand second half of page back to the ring */
		i40e_reuse_rx_page(rx_ring, rx_buffer);
		rx_ring->rx_stats.page_reuse_count++;
	} else {
		/* we are not reusing the buffer so unmap it */
		dma_unmap_page(rx_ring->dev, rx_buffer->dma, PAGE_SIZE,
			       DMA_FROM_DEVICE);
	}

	/* clear contents of buffer_info */
	rx_buffer->page = NULL;

	return skb;
}

/**
 * i40e_is_non_eop - process handling of non-EOP buffers
 * @rx_ring: Rx ring being processed
 * @rx_desc: Rx descriptor for current buffer
 * @skb: Current socket buffer containing buffer in progress
 *
 * This function updates next to clean.  If the buffer is an EOP buffer
 * this function exits returning false, otherwise it will place the
 * sk_buff in the next buffer to be chained and return true indicating
 * that this is in fact a non-EOP buffer.
 **/
static bool i40e_is_non_eop(struct i40e_ring *rx_ring,
			    union i40e_rx_desc *rx_desc,
			    struct sk_buff *skb)
{
	u32 ntc = rx_ring->next_to_clean + 1;

	/* fetch, update, and store next to clean */
	ntc = (ntc < rx_ring->count) ? ntc : 0;
	rx_ring->next_to_clean = ntc;

	prefetch(I40E_RX_DESC(rx_ring, ntc));

	/* if we are the last buffer then there is nothing else to do */
#define I40E_RXD_EOF BIT(I40E_RX_DESC_STATUS_EOF_SHIFT)
	if (likely(i40e_test_staterr(rx_desc, I40E_RXD_EOF)))
		return false;

	/* place skb in next buffer to be received */
	rx_ring->rx_bi[ntc].skb = skb;
	rx_ring->rx_stats.non_eop_descs++;

	return true;
1188 1189 1190
}

/**
1191 1192 1193 1194 1195 1196 1197 1198
 * i40e_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
 * @rx_ring: rx descriptor ring to transact packets on
 * @budget: Total limit on number of packets to process
 *
 * This function provides a "bounce buffer" approach to Rx interrupt
 * processing.  The advantage to this is that on systems that have
 * expensive overhead for IOMMU access this provides a means of avoiding
 * it by maintaining the mapping of the page to the system.
1199
 *
1200
 * Returns amount of work completed
1201
 **/
1202
static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1203 1204 1205
{
	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
	u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1206
	bool failure = false;
1207

1208 1209
	while (likely(total_rx_packets < budget)) {
		union i40e_rx_desc *rx_desc;
1210 1211
		struct sk_buff *skb;
		u16 vlan_tag;
1212 1213 1214
		u8 rx_ptype;
		u64 qword;

1215 1216
		/* return some buffers to hardware, one at a time is too slow */
		if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1217
			failure = failure ||
1218
				  i40evf_alloc_rx_buffers(rx_ring, cleaned_count);
1219 1220 1221
			cleaned_count = 0;
		}

1222 1223 1224 1225 1226 1227 1228
		rx_desc = I40E_RX_DESC(rx_ring, rx_ring->next_to_clean);

		/* status_error_len will always be zero for unused descriptors
		 * because it's cleared in cleanup, and overlaps with hdr_addr
		 * which is always zero because packet split isn't used, if the
		 * hardware wrote DD then it will be non-zero
		 */
1229 1230
		if (!i40e_test_staterr(rx_desc,
				       BIT(I40E_RX_DESC_STATUS_DD_SHIFT)))
1231 1232
			break;

1233 1234 1235 1236
		/* This memory barrier is needed to keep us from reading
		 * any other fields out of the rx_desc until we know the
		 * DD bit is set.
		 */
1237
		dma_rmb();
1238

1239 1240 1241
		skb = i40evf_fetch_rx_buffer(rx_ring, rx_desc);
		if (!skb)
			break;
1242 1243 1244

		cleaned_count++;

1245
		if (i40e_is_non_eop(rx_ring, rx_desc, skb))
1246 1247
			continue;

1248 1249 1250 1251 1252 1253
		/* ERR_MASK will only have valid bits if EOP set, and
		 * what we are doing here is actually checking
		 * I40E_RX_DESC_ERROR_RXE_SHIFT, since it is the zeroth bit in
		 * the error field
		 */
		if (unlikely(i40e_test_staterr(rx_desc, BIT(I40E_RXD_QW1_ERROR_SHIFT)))) {
1254 1255 1256 1257
			dev_kfree_skb_any(skb);
			continue;
		}

1258 1259 1260
		if (i40e_cleanup_headers(rx_ring, skb))
			continue;

1261 1262 1263
		/* probably a little skewed due to removing CRC */
		total_rx_bytes += skb->len;

1264 1265 1266 1267
		qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
		rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
			   I40E_RXD_QW1_PTYPE_SHIFT;

1268 1269
		/* populate checksum, VLAN, and protocol */
		i40evf_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype);
1270 1271


1272 1273 1274
		vlan_tag = (qword & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) ?
			   le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) : 0;

1275 1276
		i40e_receive_skb(rx_ring, skb, vlan_tag);

1277 1278 1279
		/* update budget accounting */
		total_rx_packets++;
	}
1280 1281 1282 1283 1284 1285 1286 1287

	u64_stats_update_begin(&rx_ring->syncp);
	rx_ring->stats.packets += total_rx_packets;
	rx_ring->stats.bytes += total_rx_bytes;
	u64_stats_update_end(&rx_ring->syncp);
	rx_ring->q_vector->rx.total_packets += total_rx_packets;
	rx_ring->q_vector->rx.total_bytes += total_rx_bytes;

1288
	/* guarantee a trip back through this routine if there was a failure */
1289
	return failure ? budget : total_rx_packets;
1290 1291
}

1292 1293 1294 1295 1296
static u32 i40e_buildreg_itr(const int type, const u16 itr)
{
	u32 val;

	val = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1297 1298 1299
	      /* Don't clear PBA because that can cause lost interrupts that
	       * came in while we were cleaning/polling
	       */
1300 1301 1302 1303 1304 1305 1306 1307
	      (type << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) |
	      (itr << I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT);

	return val;
}

/* a small macro to shorten up some long lines */
#define INTREG I40E_VFINT_DYN_CTLN1
1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320
static inline int get_rx_itr_enabled(struct i40e_vsi *vsi, int idx)
{
	struct i40evf_adapter *adapter = vsi->back;

	return !!(adapter->rx_rings[idx].rx_itr_setting);
}

static inline int get_tx_itr_enabled(struct i40e_vsi *vsi, int idx)
{
	struct i40evf_adapter *adapter = vsi->back;

	return !!(adapter->tx_rings[idx].tx_itr_setting);
}
1321

1322 1323 1324 1325 1326 1327 1328 1329 1330 1331
/**
 * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt
 * @vsi: the VSI we care about
 * @q_vector: q_vector for which itr is being updated and interrupt enabled
 *
 **/
static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
					  struct i40e_q_vector *q_vector)
{
	struct i40e_hw *hw = &vsi->back->hw;
1332 1333
	bool rx = false, tx = false;
	u32 rxval, txval;
1334
	int vector;
1335 1336
	int idx = q_vector->v_idx;
	int rx_itr_setting, tx_itr_setting;
1337 1338

	vector = (q_vector->v_idx + vsi->base_vector);
1339 1340 1341 1342

	/* avoid dynamic calculation if in countdown mode OR if
	 * all dynamic is disabled
	 */
1343 1344
	rxval = txval = i40e_buildreg_itr(I40E_ITR_NONE, 0);

1345 1346 1347
	rx_itr_setting = get_rx_itr_enabled(vsi, idx);
	tx_itr_setting = get_tx_itr_enabled(vsi, idx);

1348
	if (q_vector->itr_countdown > 0 ||
1349 1350
	    (!ITR_IS_DYNAMIC(rx_itr_setting) &&
	     !ITR_IS_DYNAMIC(tx_itr_setting))) {
1351 1352 1353
		goto enable_int;
	}

1354
	if (ITR_IS_DYNAMIC(rx_itr_setting)) {
1355 1356
		rx = i40e_set_new_dynamic_itr(&q_vector->rx);
		rxval = i40e_buildreg_itr(I40E_RX_ITR, q_vector->rx.itr);
1357
	}
J
Jesse Brandeburg 已提交
1358

1359
	if (ITR_IS_DYNAMIC(tx_itr_setting)) {
1360 1361 1362
		tx = i40e_set_new_dynamic_itr(&q_vector->tx);
		txval = i40e_buildreg_itr(I40E_TX_ITR, q_vector->tx.itr);
	}
J
Jesse Brandeburg 已提交
1363

1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375
	if (rx || tx) {
		/* get the higher of the two ITR adjustments and
		 * use the same value for both ITR registers
		 * when in adaptive mode (Rx and/or Tx)
		 */
		u16 itr = max(q_vector->tx.itr, q_vector->rx.itr);

		q_vector->tx.itr = q_vector->rx.itr = itr;
		txval = i40e_buildreg_itr(I40E_TX_ITR, itr);
		tx = true;
		rxval = i40e_buildreg_itr(I40E_RX_ITR, itr);
		rx = true;
1376
	}
1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390

	/* only need to enable the interrupt once, but need
	 * to possibly update both ITR values
	 */
	if (rx) {
		/* set the INTENA_MSK_MASK so that this first write
		 * won't actually enable the interrupt, instead just
		 * updating the ITR (it's bit 31 PF and VF)
		 */
		rxval |= BIT(31);
		/* don't check _DOWN because interrupt isn't being enabled */
		wr32(hw, INTREG(vector - 1), rxval);
	}

1391
enable_int:
1392 1393
	if (!test_bit(__I40E_DOWN, &vsi->state))
		wr32(hw, INTREG(vector - 1), txval);
1394 1395 1396 1397 1398

	if (q_vector->itr_countdown)
		q_vector->itr_countdown--;
	else
		q_vector->itr_countdown = ITR_COUNTDOWN_START;
1399 1400
}

1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416
/**
 * i40evf_napi_poll - NAPI polling Rx/Tx cleanup routine
 * @napi: napi struct with our devices info in it
 * @budget: amount of work driver is allowed to do this pass, in packets
 *
 * This function will clean all queues associated with a q_vector.
 *
 * Returns the amount of work done
 **/
int i40evf_napi_poll(struct napi_struct *napi, int budget)
{
	struct i40e_q_vector *q_vector =
			       container_of(napi, struct i40e_q_vector, napi);
	struct i40e_vsi *vsi = q_vector->vsi;
	struct i40e_ring *ring;
	bool clean_complete = true;
1417
	bool arm_wb = false;
1418
	int budget_per_ring;
1419
	int work_done = 0;
1420 1421 1422 1423 1424 1425 1426 1427 1428

	if (test_bit(__I40E_DOWN, &vsi->state)) {
		napi_complete(napi);
		return 0;
	}

	/* Since the actual Tx work is minimal, we can give the Tx a larger
	 * budget and be more aggressive about cleaning up the Tx descriptors.
	 */
1429
	i40e_for_each_ring(ring, q_vector->tx) {
1430
		if (!i40e_clean_tx_irq(vsi, ring, budget)) {
1431 1432 1433 1434
			clean_complete = false;
			continue;
		}
		arm_wb |= ring->arm_wb;
1435
		ring->arm_wb = false;
1436
	}
1437

1438 1439 1440 1441
	/* Handle case where we are called by netpoll with a budget of 0 */
	if (budget <= 0)
		goto tx_only;

1442 1443 1444 1445 1446
	/* We attempt to distribute budget to each Rx queue fairly, but don't
	 * allow the budget to go below 1 because that would exit polling early.
	 */
	budget_per_ring = max(budget/q_vector->num_ringpairs, 1);

1447
	i40e_for_each_ring(ring, q_vector->rx) {
1448
		int cleaned = i40e_clean_rx_irq(ring, budget_per_ring);
1449 1450

		work_done += cleaned;
1451 1452 1453
		/* if we clean as many as budgeted, we must not be done */
		if (cleaned >= budget_per_ring)
			clean_complete = false;
1454
	}
1455 1456

	/* If work not completed, return budget and polling will return */
1457
	if (!clean_complete) {
1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468
		const cpumask_t *aff_mask = &q_vector->affinity_mask;
		int cpu_id = smp_processor_id();

		/* It is possible that the interrupt affinity has changed but,
		 * if the cpu is pegged at 100%, polling will never exit while
		 * traffic continues and the interrupt will be stuck on this
		 * cpu.  We check to make sure affinity is correct before we
		 * continue to poll, otherwise we must stop polling so the
		 * interrupt can move to the correct cpu.
		 */
		if (likely(cpumask_test_cpu(cpu_id, aff_mask))) {
1469
tx_only:
1470 1471 1472 1473 1474
			if (arm_wb) {
				q_vector->tx.ring[0].tx_stats.tx_force_wb++;
				i40e_enable_wb_on_itr(vsi, q_vector);
			}
			return budget;
1475
		}
1476
	}
1477

1478 1479 1480
	if (vsi->back->flags & I40E_TXR_FLAGS_WB_ON_ITR)
		q_vector->arm_wb_state = false;

1481
	/* Work is done so exit the polling mode and re-enable the interrupt */
1482
	napi_complete_done(napi, work_done);
1483 1484 1485 1486 1487 1488 1489 1490 1491 1492

	/* If we're prematurely stopping polling to fix the interrupt
	 * affinity we want to make sure polling starts back up so we
	 * issue a call to i40evf_force_wb which triggers a SW interrupt.
	 */
	if (!clean_complete)
		i40evf_force_wb(vsi, q_vector);
	else
		i40e_update_enable_itr(vsi, q_vector);

1493
	return min(work_done, budget - 1);
1494 1495 1496
}

/**
1497
 * i40evf_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
1498 1499 1500 1501 1502 1503 1504 1505 1506 1507
 * @skb:     send buffer
 * @tx_ring: ring to send buffer on
 * @flags:   the tx flags to be set
 *
 * Checks the skb and set up correspondingly several generic transmit flags
 * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
 *
 * Returns error code indicate the frame should be dropped upon error and the
 * otherwise  returns 0 to indicate the flags has been set properly.
 **/
1508 1509 1510
static inline int i40evf_tx_prepare_vlan_flags(struct sk_buff *skb,
					       struct i40e_ring *tx_ring,
					       u32 *flags)
1511 1512 1513 1514
{
	__be16 protocol = skb->protocol;
	u32  tx_flags = 0;

1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527
	if (protocol == htons(ETH_P_8021Q) &&
	    !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {
		/* When HW VLAN acceleration is turned off by the user the
		 * stack sets the protocol to 8021q so that the driver
		 * can take any steps required to support the SW only
		 * VLAN handling.  In our case the driver doesn't need
		 * to take any further steps so just set the protocol
		 * to the encapsulated ethertype.
		 */
		skb->protocol = vlan_get_protocol(skb);
		goto out;
	}

1528
	/* if we have a HW VLAN tag being added, default to the HW one */
1529 1530
	if (skb_vlan_tag_present(skb)) {
		tx_flags |= skb_vlan_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1531 1532 1533 1534
		tx_flags |= I40E_TX_FLAGS_HW_VLAN;
	/* else if it is a SW VLAN, check the next protocol and store the tag */
	} else if (protocol == htons(ETH_P_8021Q)) {
		struct vlan_hdr *vhdr, _vhdr;
J
Jesse Brandeburg 已提交
1535

1536 1537 1538 1539 1540 1541 1542 1543 1544
		vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
		if (!vhdr)
			return -EINVAL;

		protocol = vhdr->h_vlan_encapsulated_proto;
		tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
		tx_flags |= I40E_TX_FLAGS_SW_VLAN;
	}

1545
out:
1546 1547 1548 1549 1550 1551 1552 1553
	*flags = tx_flags;
	return 0;
}

/**
 * i40e_tso - set up the tso context descriptor
 * @skb:      ptr to the skb we're sending
 * @hdr_len:  ptr to the size of the packet header
1554
 * @cd_type_cmd_tso_mss: Quad Word 1
1555 1556 1557
 *
 * Returns 0 if no TSO can happen, 1 if tso is going, or error
 **/
1558
static int i40e_tso(struct sk_buff *skb, u8 *hdr_len, u64 *cd_type_cmd_tso_mss)
1559
{
1560
	u64 cd_cmd, cd_tso_len, cd_mss;
1561 1562 1563 1564 1565
	union {
		struct iphdr *v4;
		struct ipv6hdr *v6;
		unsigned char *hdr;
	} ip;
1566 1567
	union {
		struct tcphdr *tcp;
1568
		struct udphdr *udp;
1569 1570 1571
		unsigned char *hdr;
	} l4;
	u32 paylen, l4_offset;
1572 1573
	int err;

1574 1575 1576
	if (skb->ip_summed != CHECKSUM_PARTIAL)
		return 0;

1577 1578 1579
	if (!skb_is_gso(skb))
		return 0;

1580 1581 1582
	err = skb_cow_head(skb, 0);
	if (err < 0)
		return err;
1583

1584 1585
	ip.hdr = skb_network_header(skb);
	l4.hdr = skb_transport_header(skb);
1586

1587 1588 1589 1590
	/* initialize outer IP header fields */
	if (ip.v4->version == 4) {
		ip.v4->tot_len = 0;
		ip.v4->check = 0;
1591
	} else {
1592 1593 1594
		ip.v6->payload_len = 0;
	}

1595
	if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1596
					 SKB_GSO_GRE_CSUM |
1597
					 SKB_GSO_IPXIP4 |
1598
					 SKB_GSO_IPXIP6 |
1599
					 SKB_GSO_UDP_TUNNEL |
1600
					 SKB_GSO_UDP_TUNNEL_CSUM)) {
1601 1602 1603 1604
		if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
		    (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) {
			l4.udp->len = 0;

1605 1606 1607 1608
			/* determine offset of outer transport header */
			l4_offset = l4.hdr - skb->data;

			/* remove payload length from outer checksum */
1609 1610
			paylen = skb->len - l4_offset;
			csum_replace_by_diff(&l4.udp->check, htonl(paylen));
1611 1612
		}

1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623
		/* reset pointers to inner headers */
		ip.hdr = skb_inner_network_header(skb);
		l4.hdr = skb_inner_transport_header(skb);

		/* initialize inner IP header fields */
		if (ip.v4->version == 4) {
			ip.v4->tot_len = 0;
			ip.v4->check = 0;
		} else {
			ip.v6->payload_len = 0;
		}
1624 1625
	}

1626 1627 1628 1629
	/* determine offset of inner transport header */
	l4_offset = l4.hdr - skb->data;

	/* remove payload length from inner checksum */
1630 1631
	paylen = skb->len - l4_offset;
	csum_replace_by_diff(&l4.tcp->check, htonl(paylen));
1632 1633 1634

	/* compute length of segmentation header */
	*hdr_len = (l4.tcp->doff * 4) + l4_offset;
1635 1636 1637 1638 1639

	/* find the field values */
	cd_cmd = I40E_TX_CTX_DESC_TSO;
	cd_tso_len = skb->len - *hdr_len;
	cd_mss = skb_shinfo(skb)->gso_size;
1640 1641 1642
	*cd_type_cmd_tso_mss |= (cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
				(cd_tso_len << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
				(cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
1643 1644 1645 1646 1647 1648
	return 1;
}

/**
 * i40e_tx_enable_csum - Enable Tx checksum offloads
 * @skb: send buffer
1649
 * @tx_flags: pointer to Tx flags currently set
1650 1651
 * @td_cmd: Tx descriptor command bits to set
 * @td_offset: Tx descriptor header offsets to set
1652
 * @tx_ring: Tx descriptor ring
1653 1654
 * @cd_tunneling: ptr to context desc bits
 **/
1655 1656 1657 1658
static int i40e_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags,
			       u32 *td_cmd, u32 *td_offset,
			       struct i40e_ring *tx_ring,
			       u32 *cd_tunneling)
1659
{
1660 1661 1662 1663 1664 1665 1666 1667 1668 1669
	union {
		struct iphdr *v4;
		struct ipv6hdr *v6;
		unsigned char *hdr;
	} ip;
	union {
		struct tcphdr *tcp;
		struct udphdr *udp;
		unsigned char *hdr;
	} l4;
1670
	unsigned char *exthdr;
1671
	u32 offset, cmd = 0;
1672
	__be16 frag_off;
1673 1674
	u8 l4_proto = 0;

1675 1676 1677
	if (skb->ip_summed != CHECKSUM_PARTIAL)
		return 0;

1678 1679
	ip.hdr = skb_network_header(skb);
	l4.hdr = skb_transport_header(skb);
1680

1681 1682 1683
	/* compute outer L2 header size */
	offset = ((ip.hdr - skb->data) / 2) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;

1684
	if (skb->encapsulation) {
1685
		u32 tunnel = 0;
1686 1687
		/* define outer network header type */
		if (*tx_flags & I40E_TX_FLAGS_IPV4) {
1688 1689 1690 1691
			tunnel |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
				  I40E_TX_CTX_EXT_IP_IPV4 :
				  I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;

1692 1693
			l4_proto = ip.v4->protocol;
		} else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
1694
			tunnel |= I40E_TX_CTX_EXT_IP_IPV6;
1695 1696

			exthdr = ip.hdr + sizeof(*ip.v6);
1697
			l4_proto = ip.v6->nexthdr;
1698 1699 1700
			if (l4.hdr != exthdr)
				ipv6_skip_exthdr(skb, exthdr - skb->data,
						 &l4_proto, &frag_off);
1701 1702 1703 1704
		}

		/* define outer transport */
		switch (l4_proto) {
1705
		case IPPROTO_UDP:
1706
			tunnel |= I40E_TXD_CTX_UDP_TUNNELING;
1707
			*tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
1708
			break;
1709
		case IPPROTO_GRE:
1710
			tunnel |= I40E_TXD_CTX_GRE_TUNNELING;
1711 1712
			*tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
			break;
1713 1714 1715 1716 1717
		case IPPROTO_IPIP:
		case IPPROTO_IPV6:
			*tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
			l4.hdr = skb_inner_network_header(skb);
			break;
1718
		default:
1719 1720 1721 1722 1723
			if (*tx_flags & I40E_TX_FLAGS_TSO)
				return -1;

			skb_checksum_help(skb);
			return 0;
1724
		}
1725

1726 1727 1728 1729 1730 1731 1732
		/* compute outer L3 header size */
		tunnel |= ((l4.hdr - ip.hdr) / 4) <<
			  I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT;

		/* switch IP header pointer from outer to inner header */
		ip.hdr = skb_inner_network_header(skb);

1733 1734 1735 1736
		/* compute tunnel header size */
		tunnel |= ((ip.hdr - l4.hdr) / 2) <<
			  I40E_TXD_CTX_QW0_NATLEN_SHIFT;

1737 1738
		/* indicate if we need to offload outer UDP header */
		if ((*tx_flags & I40E_TX_FLAGS_TSO) &&
1739
		    !(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
1740 1741 1742
		    (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM))
			tunnel |= I40E_TXD_CTX_QW0_L4T_CS_MASK;

1743 1744 1745
		/* record tunnel offload values */
		*cd_tunneling |= tunnel;

1746 1747
		/* switch L4 header pointer from outer to inner */
		l4.hdr = skb_inner_transport_header(skb);
1748
		l4_proto = 0;
1749

1750 1751 1752 1753 1754
		/* reset type as we transition from outer to inner headers */
		*tx_flags &= ~(I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6);
		if (ip.v4->version == 4)
			*tx_flags |= I40E_TX_FLAGS_IPV4;
		if (ip.v6->version == 6)
1755
			*tx_flags |= I40E_TX_FLAGS_IPV6;
1756 1757 1758
	}

	/* Enable IP checksum offloads */
1759
	if (*tx_flags & I40E_TX_FLAGS_IPV4) {
1760
		l4_proto = ip.v4->protocol;
1761 1762 1763
		/* the stack computes the IP header already, the only time we
		 * need the hardware to recompute it is in the case of TSO.
		 */
1764 1765 1766
		cmd |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
		       I40E_TX_DESC_CMD_IIPT_IPV4_CSUM :
		       I40E_TX_DESC_CMD_IIPT_IPV4;
1767
	} else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
1768
		cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
1769 1770 1771 1772 1773 1774

		exthdr = ip.hdr + sizeof(*ip.v6);
		l4_proto = ip.v6->nexthdr;
		if (l4.hdr != exthdr)
			ipv6_skip_exthdr(skb, exthdr - skb->data,
					 &l4_proto, &frag_off);
1775
	}
1776

1777 1778
	/* compute inner L3 header size */
	offset |= ((l4.hdr - ip.hdr) / 4) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1779 1780

	/* Enable L4 checksum offloads */
1781
	switch (l4_proto) {
1782 1783
	case IPPROTO_TCP:
		/* enable checksum offloads */
1784 1785
		cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
		offset |= l4.tcp->doff << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1786 1787 1788
		break;
	case IPPROTO_SCTP:
		/* enable SCTP checksum offload */
1789 1790 1791
		cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
		offset |= (sizeof(struct sctphdr) >> 2) <<
			  I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1792 1793 1794
		break;
	case IPPROTO_UDP:
		/* enable UDP checksum offload */
1795 1796 1797
		cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
		offset |= (sizeof(struct udphdr) >> 2) <<
			  I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1798 1799
		break;
	default:
1800 1801 1802 1803
		if (*tx_flags & I40E_TX_FLAGS_TSO)
			return -1;
		skb_checksum_help(skb);
		return 0;
1804
	}
1805 1806 1807

	*td_cmd |= cmd;
	*td_offset |= offset;
1808 1809

	return 1;
1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825
}

/**
 * i40e_create_tx_ctx Build the Tx context descriptor
 * @tx_ring:  ring to create the descriptor on
 * @cd_type_cmd_tso_mss: Quad Word 1
 * @cd_tunneling: Quad Word 0 - bits 0-31
 * @cd_l2tag2: Quad Word 0 - bits 32-63
 **/
static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
			       const u64 cd_type_cmd_tso_mss,
			       const u32 cd_tunneling, const u32 cd_l2tag2)
{
	struct i40e_tx_context_desc *context_desc;
	int i = tx_ring->next_to_use;

1826 1827
	if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) &&
	    !cd_tunneling && !cd_l2tag2)
1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838
		return;

	/* grab the next descriptor */
	context_desc = I40E_TX_CTXTDESC(tx_ring, i);

	i++;
	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;

	/* cpu_to_le32 and assign to struct fields */
	context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
	context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
1839
	context_desc->rsvd = cpu_to_le16(0);
1840 1841 1842
	context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
}

J
Jesse Brandeburg 已提交
1843
/**
1844
 * __i40evf_chk_linearize - Check if there are more than 8 buffers per packet
1845 1846
 * @skb:      send buffer
 *
1847 1848 1849 1850 1851 1852 1853 1854
 * Note: Our HW can't DMA more than 8 buffers to build a packet on the wire
 * and so we need to figure out the cases where we need to linearize the skb.
 *
 * For TSO we need to count the TSO header and segment payload separately.
 * As such we need to check cases where we have 7 fragments or more as we
 * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for
 * the segment payload in the first descriptor, and another 7 for the
 * fragments.
1855
 **/
1856
bool __i40evf_chk_linearize(struct sk_buff *skb)
1857
{
1858
	const struct skb_frag_struct *frag, *stale;
1859
	int nr_frags, sum;
1860

1861
	/* no need to check if number of frags is less than 7 */
1862
	nr_frags = skb_shinfo(skb)->nr_frags;
1863
	if (nr_frags < (I40E_MAX_BUFFER_TXD - 1))
1864
		return false;
1865

1866
	/* We need to walk through the list and validate that each group
1867
	 * of 6 fragments totals at least gso_size.
1868
	 */
1869
	nr_frags -= I40E_MAX_BUFFER_TXD - 2;
1870 1871 1872 1873 1874 1875 1876 1877
	frag = &skb_shinfo(skb)->frags[0];

	/* Initialize size to the negative value of gso_size minus 1.  We
	 * use this as the worst case scenerio in which the frag ahead
	 * of us only provides one byte which is why we are limited to 6
	 * descriptors for a single transmit as the header and previous
	 * fragment are already consuming 2 descriptors.
	 */
1878
	sum = 1 - skb_shinfo(skb)->gso_size;
1879

1880 1881 1882 1883 1884 1885
	/* Add size of frags 0 through 4 to create our initial sum */
	sum += skb_frag_size(frag++);
	sum += skb_frag_size(frag++);
	sum += skb_frag_size(frag++);
	sum += skb_frag_size(frag++);
	sum += skb_frag_size(frag++);
1886 1887 1888 1889 1890 1891

	/* Walk through fragments adding latest fragment, testing it, and
	 * then removing stale fragments from the sum.
	 */
	stale = &skb_shinfo(skb)->frags[0];
	for (;;) {
1892
		sum += skb_frag_size(frag++);
1893 1894 1895 1896 1897

		/* if sum is negative we failed to make sufficient progress */
		if (sum < 0)
			return true;

1898
		if (!nr_frags--)
1899 1900
			break;

1901
		sum -= skb_frag_size(stale++);
1902 1903
	}

1904
	return false;
1905 1906
}

1907 1908 1909 1910 1911 1912 1913
/**
 * __i40evf_maybe_stop_tx - 2nd level check for tx stop conditions
 * @tx_ring: the ring to be checked
 * @size:    the size buffer we want to assure is available
 *
 * Returns -EBUSY if a stop is needed, else 0
 **/
1914
int __i40evf_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929
{
	netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
	/* Memory barrier before checking head and tail */
	smp_mb();

	/* Check again in a case another CPU has just made room available. */
	if (likely(I40E_DESC_UNUSED(tx_ring) < size))
		return -EBUSY;

	/* A reprieve! - use start_queue because it doesn't call schedule */
	netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
	++tx_ring->tx_stats.restart_queue;
	return 0;
}

1930
/**
1931
 * i40evf_tx_map - Build the Tx descriptor
1932 1933 1934 1935 1936 1937 1938 1939
 * @tx_ring:  ring to send buffer on
 * @skb:      send buffer
 * @first:    first buffer info buffer to use
 * @tx_flags: collected send information
 * @hdr_len:  size of the packet header
 * @td_cmd:   the command field in the descriptor
 * @td_offset: offset for checksum or crc
 **/
1940 1941 1942
static inline void i40evf_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
				 struct i40e_tx_buffer *first, u32 tx_flags,
				 const u8 hdr_len, u32 td_cmd, u32 td_offset)
1943 1944 1945 1946 1947 1948 1949 1950 1951 1952
{
	unsigned int data_len = skb->data_len;
	unsigned int size = skb_headlen(skb);
	struct skb_frag_struct *frag;
	struct i40e_tx_buffer *tx_bi;
	struct i40e_tx_desc *tx_desc;
	u16 i = tx_ring->next_to_use;
	u32 td_tag = 0;
	dma_addr_t dma;
	u16 gso_segs;
1953
	u16 desc_count = 1;
1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977

	if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
		td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
		td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
			 I40E_TX_FLAGS_VLAN_SHIFT;
	}

	if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO))
		gso_segs = skb_shinfo(skb)->gso_segs;
	else
		gso_segs = 1;

	/* multiply data chunks by size of headers */
	first->bytecount = skb->len - hdr_len + (gso_segs * hdr_len);
	first->gso_segs = gso_segs;
	first->skb = skb;
	first->tx_flags = tx_flags;

	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);

	tx_desc = I40E_TX_DESC(tx_ring, i);
	tx_bi = first;

	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
1978 1979
		unsigned int max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;

1980 1981 1982 1983 1984 1985 1986
		if (dma_mapping_error(tx_ring->dev, dma))
			goto dma_error;

		/* record length, and DMA address */
		dma_unmap_len_set(tx_bi, len, size);
		dma_unmap_addr_set(tx_bi, dma, dma);

1987 1988
		/* align size to end of page */
		max_data += -dma & (I40E_MAX_READ_REQ_SIZE - 1);
1989 1990 1991 1992 1993
		tx_desc->buffer_addr = cpu_to_le64(dma);

		while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
			tx_desc->cmd_type_offset_bsz =
				build_ctob(td_cmd, td_offset,
1994
					   max_data, td_tag);
1995 1996 1997

			tx_desc++;
			i++;
1998 1999
			desc_count++;

2000 2001 2002 2003 2004
			if (i == tx_ring->count) {
				tx_desc = I40E_TX_DESC(tx_ring, 0);
				i = 0;
			}

2005 2006
			dma += max_data;
			size -= max_data;
2007

2008
			max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;
2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019
			tx_desc->buffer_addr = cpu_to_le64(dma);
		}

		if (likely(!data_len))
			break;

		tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
							  size, td_tag);

		tx_desc++;
		i++;
2020 2021
		desc_count++;

2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035
		if (i == tx_ring->count) {
			tx_desc = I40E_TX_DESC(tx_ring, 0);
			i = 0;
		}

		size = skb_frag_size(frag);
		data_len -= size;

		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
				       DMA_TO_DEVICE);

		tx_bi = &tx_ring->tx_bi[i];
	}

2036
	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
2037 2038 2039 2040 2041 2042 2043

	i++;
	if (i == tx_ring->count)
		i = 0;

	tx_ring->next_to_use = i;

2044
	i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2045

2046 2047 2048 2049 2050 2051 2052 2053 2054
	/* write last descriptor with EOP bit */
	td_cmd |= I40E_TX_DESC_CMD_EOP;

	/* We can OR these values together as they both are checked against
	 * 4 below and at this point desc_count will be used as a boolean value
	 * after this if/else block.
	 */
	desc_count |= ++tx_ring->packet_stride;

2055
	/* Algorithm to optimize tail and RS bit setting:
2056 2057 2058 2059 2060 2061
	 * if queue is stopped
	 *	mark RS bit
	 *	reset packet counter
	 * else if xmit_more is supported and is true
	 *	advance packet counter to 4
	 *	reset desc_count to 0
2062
	 *
2063 2064 2065 2066 2067
	 * if desc_count >= 4
	 *	mark RS bit
	 *	reset packet counter
	 * if desc_count > 0
	 *	update tail
2068
	 *
2069
	 * Note: If there are less than 4 descriptors
2070 2071 2072
	 * pending and interrupts were disabled the service task will
	 * trigger a force WB.
	 */
2073 2074 2075 2076 2077 2078 2079 2080 2081 2082
	if (netif_xmit_stopped(txring_txq(tx_ring))) {
		goto do_rs;
	} else if (skb->xmit_more) {
		/* set stride to arm on next packet and reset desc_count */
		tx_ring->packet_stride = WB_STRIDE;
		desc_count = 0;
	} else if (desc_count >= WB_STRIDE) {
do_rs:
		/* write last descriptor with RS bit set */
		td_cmd |= I40E_TX_DESC_CMD_RS;
2083 2084 2085 2086
		tx_ring->packet_stride = 0;
	}

	tx_desc->cmd_type_offset_bsz =
2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098
			build_ctob(td_cmd, td_offset, size, td_tag);

	/* Force memory writes to complete before letting h/w know there
	 * are new descriptors to fetch.
	 *
	 * We also use this memory barrier to make certain all of the
	 * status bits have been updated before next_to_watch is written.
	 */
	wmb();

	/* set next_to_watch value indicating a packet is present */
	first->next_to_watch = tx_desc;
2099

2100
	/* notify HW of packet */
2101
	if (desc_count) {
2102
		writel(i, tx_ring->tail);
2103 2104 2105 2106 2107

		/* we need this if more than one processor can write to our tail
		 * at a time, it synchronizes IO on IA64/Altix systems
		 */
		mmiowb();
2108
	}
2109

2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146
	return;

dma_error:
	dev_info(tx_ring->dev, "TX DMA map failed\n");

	/* clear dma mappings for failed tx_bi map */
	for (;;) {
		tx_bi = &tx_ring->tx_bi[i];
		i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
		if (tx_bi == first)
			break;
		if (i == 0)
			i = tx_ring->count;
		i--;
	}

	tx_ring->next_to_use = i;
}

/**
 * i40e_xmit_frame_ring - Sends buffer on Tx ring
 * @skb:     send buffer
 * @tx_ring: ring to send buffer on
 *
 * Returns NETDEV_TX_OK if sent, else an error code
 **/
static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
					struct i40e_ring *tx_ring)
{
	u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
	u32 cd_tunneling = 0, cd_l2tag2 = 0;
	struct i40e_tx_buffer *first;
	u32 td_offset = 0;
	u32 tx_flags = 0;
	__be16 protocol;
	u32 td_cmd = 0;
	u8 hdr_len = 0;
2147
	int tso, count;
J
Jesse Brandeburg 已提交
2148

2149 2150 2151
	/* prefetch the data, we'll need it later */
	prefetch(skb->data);

2152
	count = i40e_xmit_descriptor_count(skb);
2153 2154 2155
	if (i40e_chk_linearize(skb, count)) {
		if (__skb_linearize(skb))
			goto out_drop;
2156
		count = i40e_txd_use_count(skb->len);
2157 2158
		tx_ring->tx_stats.tx_linearize++;
	}
2159 2160 2161 2162 2163 2164 2165 2166 2167

	/* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
	 *       + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
	 *       + 4 desc gap to avoid the cache line where head is,
	 *       + 1 desc for context descriptor,
	 * otherwise try next time
	 */
	if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
		tx_ring->tx_stats.tx_busy++;
2168
		return NETDEV_TX_BUSY;
2169
	}
2170 2171

	/* prepare the xmit flags */
2172
	if (i40evf_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2173 2174 2175
		goto out_drop;

	/* obtain protocol of skb */
2176
	protocol = vlan_get_protocol(skb);
2177 2178 2179 2180 2181 2182 2183 2184 2185 2186

	/* record the location of the first descriptor for this packet */
	first = &tx_ring->tx_bi[tx_ring->next_to_use];

	/* setup IPv4/IPv6 offloads */
	if (protocol == htons(ETH_P_IP))
		tx_flags |= I40E_TX_FLAGS_IPV4;
	else if (protocol == htons(ETH_P_IPV6))
		tx_flags |= I40E_TX_FLAGS_IPV6;

2187
	tso = i40e_tso(skb, &hdr_len, &cd_type_cmd_tso_mss);
2188 2189 2190 2191 2192 2193 2194

	if (tso < 0)
		goto out_drop;
	else if (tso)
		tx_flags |= I40E_TX_FLAGS_TSO;

	/* Always offload the checksum, since it's in the data descriptor */
2195 2196 2197 2198
	tso = i40e_tx_enable_csum(skb, &tx_flags, &td_cmd, &td_offset,
				  tx_ring, &cd_tunneling);
	if (tso < 0)
		goto out_drop;
2199

2200 2201 2202 2203 2204
	skb_tx_timestamp(skb);

	/* always enable CRC insertion offload */
	td_cmd |= I40E_TX_DESC_CMD_ICRC;

2205 2206 2207
	i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
			   cd_tunneling, cd_l2tag2);

2208 2209
	i40evf_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
		      td_cmd, td_offset);
2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227

	return NETDEV_TX_OK;

out_drop:
	dev_kfree_skb_any(skb);
	return NETDEV_TX_OK;
}

/**
 * i40evf_xmit_frame - Selects the correct VSI and Tx queue to send buffer
 * @skb:    send buffer
 * @netdev: network interface device structure
 *
 * Returns NETDEV_TX_OK if sent, else an error code
 **/
netdev_tx_t i40evf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
{
	struct i40evf_adapter *adapter = netdev_priv(netdev);
2228
	struct i40e_ring *tx_ring = &adapter->tx_rings[skb->queue_mapping];
2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241

	/* hardware can't handle really short frames, hardware padding works
	 * beyond this point
	 */
	if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
		if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
			return NETDEV_TX_OK;
		skb->len = I40E_MIN_TX_LEN;
		skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
	}

	return i40e_xmit_frame_ring(skb, tx_ring);
}