en_tx.c 28.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
/*
 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#include <asm/page.h>
#include <linux/mlx4/cq.h>
36
#include <linux/slab.h>
37 38 39
#include <linux/mlx4/qp.h>
#include <linux/skbuff.h>
#include <linux/if_vlan.h>
40
#include <linux/prefetch.h>
41
#include <linux/vmalloc.h>
42
#include <linux/tcp.h>
43
#include <linux/ip.h>
44
#include <linux/ipv6.h>
45
#include <linux/moduleparam.h>
46 47 48 49

#include "mlx4_en.h"

int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
50
			   struct mlx4_en_tx_ring **pring, u32 size,
51
			   u16 stride, int node, int queue_index)
52 53
{
	struct mlx4_en_dev *mdev = priv->mdev;
54
	struct mlx4_en_tx_ring *ring;
55 56 57
	int tmp;
	int err;

58
	ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node);
59
	if (!ring) {
60 61 62 63 64
		ring = kzalloc(sizeof(*ring), GFP_KERNEL);
		if (!ring) {
			en_err(priv, "Failed allocating TX ring\n");
			return -ENOMEM;
		}
65 66
	}

67 68 69
	ring->size = size;
	ring->size_mask = size - 1;
	ring->stride = stride;
70
	ring->full_size = ring->size - HEADROOM - MAX_DESC_TXBBS;
71 72

	tmp = size * sizeof(struct mlx4_en_tx_info);
73
	ring->tx_info = kmalloc_node(tmp, GFP_KERNEL | __GFP_NOWARN, node);
74
	if (!ring->tx_info) {
75 76 77 78 79
		ring->tx_info = vmalloc(tmp);
		if (!ring->tx_info) {
			err = -ENOMEM;
			goto err_ring;
		}
80
	}
81

82
	en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
83 84
		 ring->tx_info, tmp);

85
	ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, GFP_KERNEL, node);
86
	if (!ring->bounce_buf) {
87 88 89 90 91
		ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL);
		if (!ring->bounce_buf) {
			err = -ENOMEM;
			goto err_info;
		}
92 93 94
	}
	ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE);

95
	/* Allocate HW buffers on provided NUMA node */
96
	set_dev_node(&mdev->dev->persist->pdev->dev, node);
97
	err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
98
	set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
99
	if (err) {
100
		en_err(priv, "Failed allocating hwq resources\n");
101 102 103 104 105
		goto err_bounce;
	}

	ring->buf = ring->wqres.buf.direct.buf;

J
Joe Perches 已提交
106 107 108
	en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d buf_size:%d dma:%llx\n",
	       ring, ring->buf, ring->size, ring->buf_size,
	       (unsigned long long) ring->wqres.buf.direct.map);
109

110 111 112 113
	err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
				    MLX4_RESERVE_ETH_BF_QP);
	if (err) {
		en_err(priv, "failed reserving qp for TX ring\n");
114
		goto err_hwq_res;
115 116
	}

117
	err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp, GFP_KERNEL);
118
	if (err) {
119
		en_err(priv, "Failed allocating qp %d\n", ring->qpn);
120
		goto err_reserve;
121
	}
122
	ring->qp.event = mlx4_en_sqp_event;
123

124
	err = mlx4_bf_alloc(mdev->dev, &ring->bf, node);
125
	if (err) {
J
Joe Perches 已提交
126
		en_dbg(DRV, priv, "working without blueflame (%d)\n", err);
127 128 129
		ring->bf.uar = &mdev->priv_uar;
		ring->bf.uar->map = mdev->uar_map;
		ring->bf_enabled = false;
130 131 132 133 134 135 136
		ring->bf_alloced = false;
		priv->pflags &= ~MLX4_EN_PRIV_FLAGS_BLUEFLAME;
	} else {
		ring->bf_alloced = true;
		ring->bf_enabled = !!(priv->pflags &
				      MLX4_EN_PRIV_FLAGS_BLUEFLAME);
	}
137

138
	ring->hwtstamp_tx_type = priv->hwtstamp_config.tx_type;
139 140
	ring->queue_index = queue_index;

141
	if (queue_index < priv->num_tx_rings_p_up)
142 143 144
		cpumask_set_cpu(cpumask_local_spread(queue_index,
						     priv->mdev->dev->numa_node),
				&ring->affinity_mask);
145

146
	*pring = ring;
147 148
	return 0;

149 150
err_reserve:
	mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
151 152 153 154 155
err_hwq_res:
	mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_bounce:
	kfree(ring->bounce_buf);
	ring->bounce_buf = NULL;
156
err_info:
157
	kvfree(ring->tx_info);
158
	ring->tx_info = NULL;
159 160 161
err_ring:
	kfree(ring);
	*pring = NULL;
162 163 164 165
	return err;
}

void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
166
			     struct mlx4_en_tx_ring **pring)
167 168
{
	struct mlx4_en_dev *mdev = priv->mdev;
169
	struct mlx4_en_tx_ring *ring = *pring;
170
	en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn);
171

172
	if (ring->bf_alloced)
173
		mlx4_bf_free(mdev->dev, &ring->bf);
174 175
	mlx4_qp_remove(mdev->dev, &ring->qp);
	mlx4_qp_free(mdev->dev, &ring->qp);
176
	mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1);
177 178 179
	mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
	kfree(ring->bounce_buf);
	ring->bounce_buf = NULL;
180
	kvfree(ring->tx_info);
181
	ring->tx_info = NULL;
182 183
	kfree(ring);
	*pring = NULL;
184 185 186 187
}

int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv,
			     struct mlx4_en_tx_ring *ring,
188
			     int cq, int user_prio)
189 190 191 192 193 194 195 196 197 198 199 200
{
	struct mlx4_en_dev *mdev = priv->mdev;
	int err;

	ring->cqn = cq;
	ring->prod = 0;
	ring->cons = 0xffffffff;
	ring->last_nr_txbb = 1;
	memset(ring->tx_info, 0, ring->size * sizeof(struct mlx4_en_tx_info));
	memset(ring->buf, 0, ring->buf_size);

	ring->qp_state = MLX4_QP_STATE_RST;
201 202
	ring->doorbell_qpn = cpu_to_be32(ring->qp.qpn << 8);
	ring->mr_key = cpu_to_be32(mdev->mr.key);
203 204

	mlx4_en_fill_qp_context(priv, ring->size, ring->stride, 1, 0, ring->qpn,
205
				ring->cqn, user_prio, &ring->context);
206
	if (ring->bf_alloced)
207 208 209
		ring->context.usr_page =
			cpu_to_be32(mlx4_to_hw_uar_index(mdev->dev,
							 ring->bf.uar->index));
210 211 212

	err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, &ring->context,
			       &ring->qp, &ring->qp_state);
213
	if (!cpumask_empty(&ring->affinity_mask))
214 215
		netif_set_xps_queue(priv->dev, &ring->affinity_mask,
				    ring->queue_index);
216 217 218 219 220 221 222 223 224 225 226 227 228

	return err;
}

void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv,
				struct mlx4_en_tx_ring *ring)
{
	struct mlx4_en_dev *mdev = priv->mdev;

	mlx4_qp_modify(mdev->dev, NULL, ring->qp_state,
		       MLX4_QP_STATE_RST, NULL, 0, 0, &ring->qp);
}

229 230 231 232 233
static inline bool mlx4_en_is_tx_ring_full(struct mlx4_en_tx_ring *ring)
{
	return ring->prod - ring->cons > ring->full_size;
}

234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
static void mlx4_en_stamp_wqe(struct mlx4_en_priv *priv,
			      struct mlx4_en_tx_ring *ring, int index,
			      u8 owner)
{
	__be32 stamp = cpu_to_be32(STAMP_VAL | (!!owner << STAMP_SHIFT));
	struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE;
	struct mlx4_en_tx_info *tx_info = &ring->tx_info[index];
	void *end = ring->buf + ring->buf_size;
	__be32 *ptr = (__be32 *)tx_desc;
	int i;

	/* Optimize the common case when there are no wraparounds */
	if (likely((void *)tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) {
		/* Stamp the freed descriptor */
		for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE;
		     i += STAMP_STRIDE) {
			*ptr = stamp;
			ptr += STAMP_DWORDS;
		}
	} else {
		/* Stamp the freed descriptor */
		for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE;
		     i += STAMP_STRIDE) {
			*ptr = stamp;
			ptr += STAMP_DWORDS;
			if ((void *)ptr >= end) {
				ptr = ring->buf;
				stamp ^= cpu_to_be32(0x80000000);
			}
		}
	}
}

267 268 269

static u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv,
				struct mlx4_en_tx_ring *ring,
270 271
				int index, u8 owner, u64 timestamp,
				int napi_mode)
272 273 274 275 276
{
	struct mlx4_en_tx_info *tx_info = &ring->tx_info[index];
	struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE;
	struct mlx4_wqe_data_seg *data = (void *) tx_desc + tx_info->data_offset;
	void *end = ring->buf + ring->buf_size;
277 278
	struct sk_buff *skb = tx_info->skb;
	int nr_maps = tx_info->nr_maps;
279
	int i;
280

281 282 283 284 285
	/* We do not touch skb here, so prefetch skb->users location
	 * to speedup consume_skb()
	 */
	prefetchw(&skb->users);

286 287 288 289
	if (unlikely(timestamp)) {
		struct skb_shared_hwtstamps hwts;

		mlx4_en_fill_hwtstamps(priv->mdev, &hwts, timestamp);
290 291
		skb_tstamp_tx(skb, &hwts);
	}
292 293 294

	/* Optimize the common case when there are no wraparounds */
	if (likely((void *) tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) {
295
		if (!tx_info->inl) {
296
			if (tx_info->linear)
297
				dma_unmap_single(priv->ddev,
298 299 300 301 302 303 304 305 306 307
						tx_info->map0_dma,
						tx_info->map0_byte_count,
						PCI_DMA_TODEVICE);
			else
				dma_unmap_page(priv->ddev,
					       tx_info->map0_dma,
					       tx_info->map0_byte_count,
					       PCI_DMA_TODEVICE);
			for (i = 1; i < nr_maps; i++) {
				data++;
308
				dma_unmap_page(priv->ddev,
309 310 311
					(dma_addr_t)be64_to_cpu(data->addr),
					be32_to_cpu(data->byte_count),
					PCI_DMA_TODEVICE);
312
			}
313 314
		}
	} else {
315 316
		if (!tx_info->inl) {
			if ((void *) data >= end) {
317
				data = ring->buf + ((void *)data - end);
318
			}
319

320
			if (tx_info->linear)
321
				dma_unmap_single(priv->ddev,
322 323 324 325 326 327 328 329 330 331
						tx_info->map0_dma,
						tx_info->map0_byte_count,
						PCI_DMA_TODEVICE);
			else
				dma_unmap_page(priv->ddev,
					       tx_info->map0_dma,
					       tx_info->map0_byte_count,
					       PCI_DMA_TODEVICE);
			for (i = 1; i < nr_maps; i++) {
				data++;
332 333
				/* Check for wraparound before unmapping */
				if ((void *) data >= end)
334
					data = ring->buf;
335
				dma_unmap_page(priv->ddev,
336 337 338
					(dma_addr_t)be64_to_cpu(data->addr),
					be32_to_cpu(data->byte_count),
					PCI_DMA_TODEVICE);
339
			}
340 341
		}
	}
342 343
	napi_consume_skb(skb, napi_mode);

344 345 346 347 348 349 350 351 352 353 354
	return tx_info->nr_txbb;
}


int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring)
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
	int cnt = 0;

	/* Skip last polled descriptor */
	ring->cons += ring->last_nr_txbb;
355
	en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n",
356 357 358 359
		 ring->cons, ring->prod);

	if ((u32) (ring->prod - ring->cons) > ring->size) {
		if (netif_msg_tx_err(priv))
360
			en_warn(priv, "Tx consumer passed producer!\n");
361 362 363 364 365 366
		return 0;
	}

	while (ring->cons != ring->prod) {
		ring->last_nr_txbb = mlx4_en_free_tx_desc(priv, ring,
						ring->cons & ring->size_mask,
367 368
						!!(ring->cons & ring->size), 0,
						0 /* Non-NAPI caller */);
369 370 371 372
		ring->cons += ring->last_nr_txbb;
		cnt++;
	}

373 374
	netdev_tx_reset_queue(ring->tx_queue);

375
	if (cnt)
376
		en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt);
377 378 379 380

	return cnt;
}

381
static bool mlx4_en_process_tx_cq(struct net_device *dev,
382
				  struct mlx4_en_cq *cq, int napi_budget)
383 384 385
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
	struct mlx4_cq *mcq = &cq->mcq;
386
	struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring];
387
	struct mlx4_cqe *cqe;
388
	u16 index;
389
	u16 new_index, ring_index, stamp_index;
390
	u32 txbbs_skipped = 0;
391
	u32 txbbs_stamp = 0;
392 393 394 395
	u32 cons_index = mcq->cons_index;
	int size = cq->size;
	u32 size_mask = ring->size_mask;
	struct mlx4_cqe *buf = cq->buf;
396 397
	u32 packets = 0;
	u32 bytes = 0;
O
Or Gerlitz 已提交
398
	int factor = priv->cqe_factor;
399
	int done = 0;
400
	int budget = priv->tx_work_limit;
401 402
	u32 last_nr_txbb;
	u32 ring_cons;
403 404

	if (!priv->port_up)
405
		return true;
406

407 408
	netdev_txq_bql_complete_prefetchw(ring->tx_queue);

409
	index = cons_index & size_mask;
410
	cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
411 412 413
	last_nr_txbb = ACCESS_ONCE(ring->last_nr_txbb);
	ring_cons = ACCESS_ONCE(ring->cons);
	ring_index = ring_cons & size_mask;
414
	stamp_index = ring_index;
415 416 417

	/* Process all completed CQEs */
	while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
418
			cons_index & size) && (done < budget)) {
419 420 421 422
		/*
		 * make sure we read the CQE after we read the
		 * ownership bit
		 */
423
		dma_rmb();
424

425 426 427 428 429 430 431 432 433
		if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
			     MLX4_CQE_OPCODE_ERROR)) {
			struct mlx4_err_cqe *cqe_err = (struct mlx4_err_cqe *)cqe;

			en_err(priv, "CQE error - vendor syndrome: 0x%x syndrome: 0x%x\n",
			       cqe_err->vendor_err_syndrome,
			       cqe_err->syndrome);
		}

434 435 436
		/* Skip over last polled CQE */
		new_index = be16_to_cpu(cqe->wqe_index) & size_mask;

437
		do {
438 439
			u64 timestamp = 0;

440 441
			txbbs_skipped += last_nr_txbb;
			ring_index = (ring_index + last_nr_txbb) & size_mask;
442 443

			if (unlikely(ring->tx_info[ring_index].ts_requested))
444 445
				timestamp = mlx4_en_get_cqe_ts(cqe);

446
			/* free next descriptor */
447
			last_nr_txbb = mlx4_en_free_tx_desc(
448
					priv, ring, ring_index,
449
					!!((ring_cons + txbbs_skipped) &
450
					ring->size), timestamp, napi_budget);
451 452

			mlx4_en_stamp_wqe(priv, ring, stamp_index,
453
					  !!((ring_cons + txbbs_stamp) &
454 455 456
						ring->size));
			stamp_index = ring_index;
			txbbs_stamp = txbbs_skipped;
457 458
			packets++;
			bytes += ring->tx_info[ring_index].nr_bytes;
459
		} while ((++done < budget) && (ring_index != new_index));
460 461 462

		++cons_index;
		index = cons_index & size_mask;
463
		cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
464
	}
465 466 467 468 469 470


	/*
	 * To prevent CQ overflow we first update CQ consumer and only then
	 * the ring consumer.
	 */
471
	mcq->cons_index = cons_index;
472 473
	mlx4_cq_set_ci(mcq);
	wmb();
474 475 476 477 478

	/* we want to dirty this cache line once */
	ACCESS_ONCE(ring->last_nr_txbb) = last_nr_txbb;
	ACCESS_ONCE(ring->cons) = ring_cons + txbbs_skipped;

479
	netdev_tx_completed_queue(ring->tx_queue, packets, bytes);
480

481
	/* Wakeup Tx queue if this stopped, and ring is not full.
482
	 */
483 484
	if (netif_tx_queue_stopped(ring->tx_queue) &&
	    !mlx4_en_is_tx_ring_full(ring)) {
485
		netif_tx_wake_queue(ring->tx_queue);
486
		ring->wake_queue++;
487
	}
488
	return done < budget;
489 490 491 492 493 494 495
}

void mlx4_en_tx_irq(struct mlx4_cq *mcq)
{
	struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
	struct mlx4_en_priv *priv = netdev_priv(cq->dev);

E
Eric Dumazet 已提交
496 497
	if (likely(priv->port_up))
		napi_schedule_irqoff(&cq->napi);
498 499
	else
		mlx4_en_arm_cq(priv, cq);
500 501
}

502 503 504 505 506 507
/* TX CQ polling - called by NAPI */
int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget)
{
	struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
	struct net_device *dev = cq->dev;
	struct mlx4_en_priv *priv = netdev_priv(dev);
508
	int clean_complete;
509

510
	clean_complete = mlx4_en_process_tx_cq(dev, cq, budget);
511 512
	if (!clean_complete)
		return budget;
513

514 515 516 517
	napi_complete(napi);
	mlx4_en_arm_cq(priv, cq);

	return 0;
518
}
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547

static struct mlx4_en_tx_desc *mlx4_en_bounce_to_desc(struct mlx4_en_priv *priv,
						      struct mlx4_en_tx_ring *ring,
						      u32 index,
						      unsigned int desc_size)
{
	u32 copy = (ring->size - index) * TXBB_SIZE;
	int i;

	for (i = desc_size - copy - 4; i >= 0; i -= 4) {
		if ((i & (TXBB_SIZE - 1)) == 0)
			wmb();

		*((u32 *) (ring->buf + i)) =
			*((u32 *) (ring->bounce_buf + copy + i));
	}

	for (i = copy - 4; i >= 4 ; i -= 4) {
		if ((i & (TXBB_SIZE - 1)) == 0)
			wmb();

		*((u32 *) (ring->buf + index * TXBB_SIZE + i)) =
			*((u32 *) (ring->bounce_buf + i));
	}

	/* Return real descriptor location */
	return ring->buf + index * TXBB_SIZE;
}

548 549 550 551 552 553 554
/* Decide if skb can be inlined in tx descriptor to avoid dma mapping
 *
 * It seems strange we do not simply use skb_copy_bits().
 * This would allow to inline all skbs iff skb->len <= inline_thold
 *
 * Note that caller already checked skb was not a gso packet
 */
555
static bool is_inline(int inline_thold, const struct sk_buff *skb,
556
		      const struct skb_shared_info *shinfo,
557
		      void **pfrag)
558 559 560
{
	void *ptr;

561 562
	if (skb->len > inline_thold || !inline_thold)
		return false;
563

564 565 566 567 568 569
	if (shinfo->nr_frags == 1) {
		ptr = skb_frag_address_safe(&shinfo->frags[0]);
		if (unlikely(!ptr))
			return false;
		*pfrag = ptr;
		return true;
570
	}
571 572 573
	if (shinfo->nr_frags)
		return false;
	return true;
574 575
}

576
static int inline_size(const struct sk_buff *skb)
577 578 579 580 581 582 583 584 585 586
{
	if (skb->len + CTRL_SIZE + sizeof(struct mlx4_wqe_inline_seg)
	    <= MLX4_INLINE_ALIGN)
		return ALIGN(skb->len + CTRL_SIZE +
			     sizeof(struct mlx4_wqe_inline_seg), 16);
	else
		return ALIGN(skb->len + CTRL_SIZE + 2 *
			     sizeof(struct mlx4_wqe_inline_seg), 16);
}

587
static int get_real_size(const struct sk_buff *skb,
588
			 const struct skb_shared_info *shinfo,
589
			 struct net_device *dev,
590 591 592
			 int *lso_header_size,
			 bool *inline_ok,
			 void **pfrag)
593 594 595 596
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
	int real_size;

597
	if (shinfo->gso_size) {
598
		*inline_ok = false;
599 600 601 602
		if (skb->encapsulation)
			*lso_header_size = (skb_inner_transport_header(skb) - skb->data) + inner_tcp_hdrlen(skb);
		else
			*lso_header_size = skb_transport_offset(skb) + tcp_hdrlen(skb);
603
		real_size = CTRL_SIZE + shinfo->nr_frags * DS_SIZE +
604 605 606 607 608 609 610 611
			ALIGN(*lso_header_size + 4, DS_SIZE);
		if (unlikely(*lso_header_size != skb_headlen(skb))) {
			/* We add a segment for the skb linear buffer only if
			 * it contains data */
			if (*lso_header_size < skb_headlen(skb))
				real_size += DS_SIZE;
			else {
				if (netif_msg_tx_err(priv))
612
					en_warn(priv, "Non-linear headers\n");
613 614 615 616 617
				return 0;
			}
		}
	} else {
		*lso_header_size = 0;
618 619 620 621
		*inline_ok = is_inline(priv->prof->inline_thold, skb,
				       shinfo, pfrag);

		if (*inline_ok)
622
			real_size = inline_size(skb);
623 624 625
		else
			real_size = CTRL_SIZE +
				    (shinfo->nr_frags + 1) * DS_SIZE;
626 627 628 629 630
	}

	return real_size;
}

631 632
static void build_inline_wqe(struct mlx4_en_tx_desc *tx_desc,
			     const struct sk_buff *skb,
633
			     const struct skb_shared_info *shinfo,
634 635
			     int real_size, u16 *vlan_tag,
			     int tx_ind, void *fragptr)
636 637 638
{
	struct mlx4_wqe_inline_seg *inl = &tx_desc->inl;
	int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - sizeof *inl;
639
	unsigned int hlen = skb_headlen(skb);
640 641

	if (skb->len <= spc) {
642 643 644 645 646 647 648
		if (likely(skb->len >= MIN_PKT_LEN)) {
			inl->byte_count = cpu_to_be32(1 << 31 | skb->len);
		} else {
			inl->byte_count = cpu_to_be32(1 << 31 | MIN_PKT_LEN);
			memset(((void *)(inl + 1)) + skb->len, 0,
			       MIN_PKT_LEN - skb->len);
		}
649
		skb_copy_from_linear_data(skb, inl + 1, hlen);
650
		if (shinfo->nr_frags)
651
			memcpy(((void *)(inl + 1)) + hlen, fragptr,
652
			       skb_frag_size(&shinfo->frags[0]));
653 654 655

	} else {
		inl->byte_count = cpu_to_be32(1 << 31 | spc);
656 657 658 659 660 661
		if (hlen <= spc) {
			skb_copy_from_linear_data(skb, inl + 1, hlen);
			if (hlen < spc) {
				memcpy(((void *)(inl + 1)) + hlen,
				       fragptr, spc - hlen);
				fragptr +=  spc - hlen;
662 663 664 665 666 667 668
			}
			inl = (void *) (inl + 1) + spc;
			memcpy(((void *)(inl + 1)), fragptr, skb->len - spc);
		} else {
			skb_copy_from_linear_data(skb, inl + 1, spc);
			inl = (void *) (inl + 1) + spc;
			skb_copy_from_linear_data_offset(skb, spc, inl + 1,
669
							 hlen - spc);
670
			if (shinfo->nr_frags)
671
				memcpy(((void *)(inl + 1)) + hlen - spc,
672 673
				       fragptr,
				       skb_frag_size(&shinfo->frags[0]));
674 675
		}

676
		dma_wmb();
677 678 679 680
		inl->byte_count = cpu_to_be32(1 << 31 | (skb->len - spc));
	}
}

681
u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb,
682
			 void *accel_priv, select_queue_fallback_t fallback)
683
{
684
	struct mlx4_en_priv *priv = netdev_priv(dev);
685
	u16 rings_p_up = priv->num_tx_rings_p_up;
686
	u8 up = 0;
687

688 689 690
	if (dev->num_tc)
		return skb_tx_hash(dev, skb);

691 692
	if (skb_vlan_tag_present(skb))
		up = skb_vlan_tag_get(skb) >> VLAN_PRIO_SHIFT;
Y
Yevgeny Petrilin 已提交
693

694
	return fallback(dev, skb) % rings_p_up + up * rings_p_up;
695 696
}

697 698
static void mlx4_bf_copy(void __iomem *dst, const void *src,
			 unsigned int bytecnt)
699 700 701 702
{
	__iowrite64_copy(dst, src, bytecnt / 8);
}

703
netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
704
{
705
	struct skb_shared_info *shinfo = skb_shinfo(skb);
706
	struct mlx4_en_priv *priv = netdev_priv(dev);
707
	struct device *ddev = priv->ddev;
708 709 710 711 712 713 714 715
	struct mlx4_en_tx_ring *ring;
	struct mlx4_en_tx_desc *tx_desc;
	struct mlx4_wqe_data_seg *data;
	struct mlx4_en_tx_info *tx_info;
	int tx_ind = 0;
	int nr_txbb;
	int desc_size;
	int real_size;
716
	u32 index, bf_index;
717
	__be32 op_own;
Y
Yevgeny Petrilin 已提交
718
	u16 vlan_tag = 0;
719
	u16 vlan_proto = 0;
720
	int i_frag;
721
	int lso_header_size;
722
	void *fragptr = NULL;
723
	bool bounce = false;
724
	bool send_doorbell;
E
Eric Dumazet 已提交
725
	bool stop_queue;
726
	bool inline_ok;
727
	u32 ring_cons;
728

729 730 731
	tx_ind = skb_get_queue_mapping(skb);
	ring = priv->tx_ring[tx_ind];

E
Eric Dumazet 已提交
732 733 734
	if (!priv->port_up)
		goto tx_drop;

735 736 737
	/* fetch ring->cons far ahead before needing it to avoid stall */
	ring_cons = ACCESS_ONCE(ring->cons);

738 739
	real_size = get_real_size(skb, shinfo, dev, &lso_header_size,
				  &inline_ok, &fragptr);
740
	if (unlikely(!real_size))
741
		goto tx_drop;
742

L
Lucas De Marchi 已提交
743
	/* Align descriptor to TXBB size */
744 745 746 747
	desc_size = ALIGN(real_size, TXBB_SIZE);
	nr_txbb = desc_size / TXBB_SIZE;
	if (unlikely(nr_txbb > MAX_DESC_TXBBS)) {
		if (netif_msg_tx_err(priv))
748
			en_warn(priv, "Oversized header or SG list\n");
749
		goto tx_drop;
750 751
	}

752
	if (skb_vlan_tag_present(skb)) {
753
		vlan_tag = skb_vlan_tag_get(skb);
754 755
		vlan_proto = be16_to_cpu(skb->vlan_proto);
	}
756

757
	netdev_txq_bql_enqueue_prefetchw(ring->tx_queue);
758

759 760
	/* Track current inflight packets for performance analysis */
	AVG_PERF_COUNTER(priv->pstats.inflight_avg,
761
			 (u32)(ring->prod - ring_cons - 1));
762 763 764

	/* Packet is good - grab an index and transmit it */
	index = ring->prod & ring->size_mask;
765
	bf_index = ring->prod;
766 767 768 769 770

	/* See if we have enough space for whole descriptor TXBB for setting
	 * SW ownership on next descriptor; if not, use a bounce buffer. */
	if (likely(index + nr_txbb <= ring->size))
		tx_desc = ring->buf + index * TXBB_SIZE;
771
	else {
772
		tx_desc = (struct mlx4_en_tx_desc *) ring->bounce_buf;
773 774
		bounce = true;
	}
775 776 777 778 779 780

	/* Save skb in tx_info ring */
	tx_info = &ring->tx_info[index];
	tx_info->skb = skb;
	tx_info->nr_txbb = nr_txbb;

781
	data = &tx_desc->data;
782 783 784 785 786 787 788
	if (lso_header_size)
		data = ((void *)&tx_desc->lso + ALIGN(lso_header_size + 4,
						      DS_SIZE));

	/* valid only for none inline segments */
	tx_info->data_offset = (void *)data - (void *)tx_desc;

789 790
	tx_info->inl = inline_ok;

791
	tx_info->linear = (lso_header_size < skb_headlen(skb) &&
792
			   !inline_ok) ? 1 : 0;
793

794
	tx_info->nr_maps = shinfo->nr_frags + tx_info->linear;
795
	data += tx_info->nr_maps - 1;
796

797
	if (!tx_info->inl) {
798 799 800
		dma_addr_t dma = 0;
		u32 byte_count = 0;

801
		/* Map fragments if any */
802
		for (i_frag = shinfo->nr_frags - 1; i_frag >= 0; i_frag--) {
803
			const struct skb_frag_struct *frag;
804 805

			frag = &shinfo->frags[i_frag];
806
			byte_count = skb_frag_size(frag);
807
			dma = skb_frag_dma_map(ddev, frag,
808
					       0, byte_count,
809 810 811 812 813
					       DMA_TO_DEVICE);
			if (dma_mapping_error(ddev, dma))
				goto tx_drop_unmap;

			data->addr = cpu_to_be64(dma);
814
			data->lkey = ring->mr_key;
815
			dma_wmb();
816
			data->byte_count = cpu_to_be32(byte_count);
817 818 819
			--data;
		}

820
		/* Map linear part if needed */
821
		if (tx_info->linear) {
822
			byte_count = skb_headlen(skb) - lso_header_size;
823

824 825 826 827 828 829 830
			dma = dma_map_single(ddev, skb->data +
					     lso_header_size, byte_count,
					     PCI_DMA_TODEVICE);
			if (dma_mapping_error(ddev, dma))
				goto tx_drop_unmap;

			data->addr = cpu_to_be64(dma);
831
			data->lkey = ring->mr_key;
832
			dma_wmb();
833 834
			data->byte_count = cpu_to_be32(byte_count);
		}
835 836 837
		/* tx completion can avoid cache line miss for common cases */
		tx_info->map0_dma = dma;
		tx_info->map0_byte_count = byte_count;
838 839
	}

840 841 842 843
	/*
	 * For timestamping add flag to skb_shinfo and
	 * set flag for further reference
	 */
844
	tx_info->ts_requested = 0;
845 846 847
	if (unlikely(ring->hwtstamp_tx_type == HWTSTAMP_TX_ON &&
		     shinfo->tx_flags & SKBTX_HW_TSTAMP)) {
		shinfo->tx_flags |= SKBTX_IN_PROGRESS;
848 849 850
		tx_info->ts_requested = 1;
	}

851 852
	/* Prepare ctrl segement apart opcode+ownership, which depends on
	 * whether LSO is used */
A
Amir Vadai 已提交
853
	tx_desc->ctrl.srcrb_flags = priv->ctrl_flags;
854
	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
855 856 857 858 859
		if (!skb->encapsulation)
			tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM |
								 MLX4_WQE_CTRL_TCP_UDP_CSUM);
		else
			tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM);
860
		ring->tx_csum++;
861 862
	}

863
	if (priv->flags & MLX4_EN_FLAG_ENABLE_HW_LOOPBACK) {
864 865
		struct ethhdr *ethh;

866 867 868 869 870 871 872 873
		/* Copy dst mac address to wqe. This allows loopback in eSwitch,
		 * so that VFs and PF can communicate with each other
		 */
		ethh = (struct ethhdr *)skb->data;
		tx_desc->ctrl.srcrb_flags16[0] = get_unaligned((__be16 *)ethh->h_dest);
		tx_desc->ctrl.imm = get_unaligned((__be32 *)(ethh->h_dest + 2));
	}

874 875
	/* Handle LSO (TSO) packets */
	if (lso_header_size) {
876 877
		int i;

878 879 880 881 882 883 884
		/* Mark opcode as LSO */
		op_own = cpu_to_be32(MLX4_OPCODE_LSO | (1 << 6)) |
			((ring->prod & ring->size) ?
				cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0);

		/* Fill in the LSO prefix */
		tx_desc->lso.mss_hdr_size = cpu_to_be32(
885
			shinfo->gso_size << 16 | lso_header_size);
886 887 888 889 890

		/* Copy headers;
		 * note that we already verified that it is linear */
		memcpy(tx_desc->lso.header, skb->data, lso_header_size);

E
Eric Dumazet 已提交
891
		ring->tso_packets++;
892 893 894

		i = ((skb->len - lso_header_size) / shinfo->gso_size) +
			!!((skb->len - lso_header_size) % shinfo->gso_size);
895
		tx_info->nr_bytes = skb->len + (i - 1) * lso_header_size;
896 897 898 899 900 901
		ring->packets += i;
	} else {
		/* Normal (Non LSO) packet */
		op_own = cpu_to_be32(MLX4_OPCODE_SEND) |
			((ring->prod & ring->size) ?
			 cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0);
902
		tx_info->nr_bytes = max_t(unsigned int, skb->len, ETH_ZLEN);
903 904
		ring->packets++;
	}
905 906
	ring->bytes += tx_info->nr_bytes;
	netdev_tx_sent_queue(ring->tx_queue, tx_info->nr_bytes);
907 908
	AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, skb->len);

909
	if (tx_info->inl)
910 911
		build_inline_wqe(tx_desc, skb, shinfo, real_size, &vlan_tag,
				 tx_ind, fragptr);
912

913
	if (skb->encapsulation) {
914 915 916 917 918 919 920 921 922 923 924 925
		union {
			struct iphdr *v4;
			struct ipv6hdr *v6;
			unsigned char *hdr;
		} ip;
		u8 proto;

		ip.hdr = skb_inner_network_header(skb);
		proto = (ip.v4->version == 4) ? ip.v4->protocol :
						ip.v6->nexthdr;

		if (proto == IPPROTO_TCP || proto == IPPROTO_UDP)
926 927 928 929 930
			op_own |= cpu_to_be32(MLX4_WQE_CTRL_IIP | MLX4_WQE_CTRL_ILP);
		else
			op_own |= cpu_to_be32(MLX4_WQE_CTRL_IIP);
	}

931 932 933
	ring->prod += nr_txbb;

	/* If we used a bounce buffer then copy descriptor back into place */
934
	if (unlikely(bounce))
935 936
		tx_desc = mlx4_en_bounce_to_desc(priv, ring, index, desc_size);

937 938
	skb_tx_timestamp(skb);

E
Eric Dumazet 已提交
939
	/* Check available TXBBs And 2K spare for prefetch */
940
	stop_queue = mlx4_en_is_tx_ring_full(ring);
E
Eric Dumazet 已提交
941 942 943 944
	if (unlikely(stop_queue)) {
		netif_tx_stop_queue(ring->tx_queue);
		ring->queue_stopped++;
	}
945 946
	send_doorbell = !skb->xmit_more || netif_xmit_stopped(ring->tx_queue);

947 948
	real_size = (real_size / 16) & 0x3f;

949
	if (ring->bf_enabled && desc_size <= MAX_BF && !bounce &&
950
	    !skb_vlan_tag_present(skb) && send_doorbell) {
951 952
		tx_desc->ctrl.bf_qpn = ring->doorbell_qpn |
				       cpu_to_be32(real_size);
953

954
		op_own |= htonl((bf_index & 0xffff) << 8);
955 956 957
		/* Ensure new descriptor hits memory
		 * before setting ownership of this descriptor to HW
		 */
958
		dma_wmb();
959
		tx_desc->ctrl.owner_opcode = op_own;
960

961 962
		wmb();

963 964
		mlx4_bf_copy(ring->bf.reg + ring->bf.offset, &tx_desc->ctrl,
			     desc_size);
965 966 967 968 969

		wmb();

		ring->bf.offset ^= ring->bf.buf_size;
	} else {
970
		tx_desc->ctrl.vlan_tag = cpu_to_be16(vlan_tag);
971 972 973 974
		if (vlan_proto == ETH_P_8021AD)
			tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_SVLAN;
		else if (vlan_proto == ETH_P_8021Q)
			tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_CVLAN;
975 976
		else
			tx_desc->ctrl.ins_vlan = 0;
977

978 979
		tx_desc->ctrl.fence_size = real_size;

980 981 982
		/* Ensure new descriptor hits memory
		 * before setting ownership of this descriptor to HW
		 */
983
		dma_wmb();
984
		tx_desc->ctrl.owner_opcode = op_own;
985 986
		if (send_doorbell) {
			wmb();
987 988 989 990 991 992 993 994 995 996 997
			/* Since there is no iowrite*_native() that writes the
			 * value as is, without byteswapping - using the one
			 * the doesn't do byteswapping in the relevant arch
			 * endianness.
			 */
#if defined(__LITTLE_ENDIAN)
			iowrite32(
#else
			iowrite32be(
#endif
				  ring->doorbell_qpn,
998
				  ring->bf.uar->map + MLX4_SEND_DOORBELL);
E
Eric Dumazet 已提交
999 1000
		} else {
			ring->xmit_more++;
1001
		}
1002
	}
1003

E
Eric Dumazet 已提交
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
	if (unlikely(stop_queue)) {
		/* If queue was emptied after the if (stop_queue) , and before
		 * the netif_tx_stop_queue() - need to wake the queue,
		 * or else it will remain stopped forever.
		 * Need a memory barrier to make sure ring->cons was not
		 * updated before queue was stopped.
		 */
		smp_rmb();

		ring_cons = ACCESS_ONCE(ring->cons);
1014
		if (unlikely(!mlx4_en_is_tx_ring_full(ring))) {
E
Eric Dumazet 已提交
1015 1016 1017 1018
			netif_tx_wake_queue(ring->tx_queue);
			ring->wake_queue++;
		}
	}
1019
	return NETDEV_TX_OK;
1020

1021 1022 1023
tx_drop_unmap:
	en_err(priv, "DMA mapping error\n");

1024 1025
	while (++i_frag < shinfo->nr_frags) {
		++data;
1026 1027 1028 1029 1030
		dma_unmap_page(ddev, (dma_addr_t) be64_to_cpu(data->addr),
			       be32_to_cpu(data->byte_count),
			       PCI_DMA_TODEVICE);
	}

1031 1032
tx_drop:
	dev_kfree_skb_any(skb);
E
Eric Dumazet 已提交
1033
	ring->tx_dropped++;
1034
	return NETDEV_TX_OK;
1035 1036
}