en_tx.c 31.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
/*
 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#include <asm/page.h>
#include <linux/mlx4/cq.h>
36
#include <linux/slab.h>
37 38 39
#include <linux/mlx4/qp.h>
#include <linux/skbuff.h>
#include <linux/if_vlan.h>
40
#include <linux/prefetch.h>
41
#include <linux/vmalloc.h>
42
#include <linux/tcp.h>
43
#include <linux/ip.h>
44
#include <linux/ipv6.h>
45
#include <linux/moduleparam.h>
46 47 48 49

#include "mlx4_en.h"

int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
50
			   struct mlx4_en_tx_ring **pring, u32 size,
51
			   u16 stride, int node, int queue_index)
52 53
{
	struct mlx4_en_dev *mdev = priv->mdev;
54
	struct mlx4_en_tx_ring *ring;
55 56 57
	int tmp;
	int err;

58
	ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node);
59
	if (!ring) {
60 61 62 63 64
		ring = kzalloc(sizeof(*ring), GFP_KERNEL);
		if (!ring) {
			en_err(priv, "Failed allocating TX ring\n");
			return -ENOMEM;
		}
65 66
	}

67 68
	ring->size = size;
	ring->size_mask = size - 1;
69
	ring->sp_stride = stride;
70
	ring->full_size = ring->size - HEADROOM - MAX_DESC_TXBBS;
71 72

	tmp = size * sizeof(struct mlx4_en_tx_info);
73
	ring->tx_info = kvmalloc_node(tmp, GFP_KERNEL, node);
74
	if (!ring->tx_info) {
75 76
		err = -ENOMEM;
		goto err_ring;
77
	}
78

79
	en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
80 81
		 ring->tx_info, tmp);

82
	ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, GFP_KERNEL, node);
83
	if (!ring->bounce_buf) {
84 85 86 87 88
		ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL);
		if (!ring->bounce_buf) {
			err = -ENOMEM;
			goto err_info;
		}
89
	}
90
	ring->buf_size = ALIGN(size * ring->sp_stride, MLX4_EN_PAGE_SIZE);
91

92
	/* Allocate HW buffers on provided NUMA node */
93
	set_dev_node(&mdev->dev->persist->pdev->dev, node);
94
	err = mlx4_alloc_hwq_res(mdev->dev, &ring->sp_wqres, ring->buf_size);
95
	set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
96
	if (err) {
97
		en_err(priv, "Failed allocating hwq resources\n");
98 99 100
		goto err_bounce;
	}

101
	ring->buf = ring->sp_wqres.buf.direct.buf;
102

J
Joe Perches 已提交
103 104
	en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d buf_size:%d dma:%llx\n",
	       ring, ring->buf, ring->size, ring->buf_size,
105
	       (unsigned long long) ring->sp_wqres.buf.direct.map);
106

107 108 109 110
	err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
				    MLX4_RESERVE_ETH_BF_QP);
	if (err) {
		en_err(priv, "failed reserving qp for TX ring\n");
111
		goto err_hwq_res;
112 113
	}

114
	err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->sp_qp, GFP_KERNEL);
115
	if (err) {
116
		en_err(priv, "Failed allocating qp %d\n", ring->qpn);
117
		goto err_reserve;
118
	}
119
	ring->sp_qp.event = mlx4_en_sqp_event;
120

121
	err = mlx4_bf_alloc(mdev->dev, &ring->bf, node);
122
	if (err) {
J
Joe Perches 已提交
123
		en_dbg(DRV, priv, "working without blueflame (%d)\n", err);
124 125 126
		ring->bf.uar = &mdev->priv_uar;
		ring->bf.uar->map = mdev->uar_map;
		ring->bf_enabled = false;
127 128 129 130 131 132 133
		ring->bf_alloced = false;
		priv->pflags &= ~MLX4_EN_PRIV_FLAGS_BLUEFLAME;
	} else {
		ring->bf_alloced = true;
		ring->bf_enabled = !!(priv->pflags &
				      MLX4_EN_PRIV_FLAGS_BLUEFLAME);
	}
134

135
	ring->hwtstamp_tx_type = priv->hwtstamp_config.tx_type;
136 137
	ring->queue_index = queue_index;

138
	if (queue_index < priv->num_tx_rings_p_up)
139 140
		cpumask_set_cpu(cpumask_local_spread(queue_index,
						     priv->mdev->dev->numa_node),
141
				&ring->sp_affinity_mask);
142

143
	*pring = ring;
144 145
	return 0;

146 147
err_reserve:
	mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
148
err_hwq_res:
149
	mlx4_free_hwq_res(mdev->dev, &ring->sp_wqres, ring->buf_size);
150 151 152
err_bounce:
	kfree(ring->bounce_buf);
	ring->bounce_buf = NULL;
153
err_info:
154
	kvfree(ring->tx_info);
155
	ring->tx_info = NULL;
156 157 158
err_ring:
	kfree(ring);
	*pring = NULL;
159 160 161 162
	return err;
}

void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
163
			     struct mlx4_en_tx_ring **pring)
164 165
{
	struct mlx4_en_dev *mdev = priv->mdev;
166
	struct mlx4_en_tx_ring *ring = *pring;
167
	en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn);
168

169
	if (ring->bf_alloced)
170
		mlx4_bf_free(mdev->dev, &ring->bf);
171 172
	mlx4_qp_remove(mdev->dev, &ring->sp_qp);
	mlx4_qp_free(mdev->dev, &ring->sp_qp);
173
	mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1);
174
	mlx4_free_hwq_res(mdev->dev, &ring->sp_wqres, ring->buf_size);
175 176
	kfree(ring->bounce_buf);
	ring->bounce_buf = NULL;
177
	kvfree(ring->tx_info);
178
	ring->tx_info = NULL;
179 180
	kfree(ring);
	*pring = NULL;
181 182 183 184
}

int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv,
			     struct mlx4_en_tx_ring *ring,
185
			     int cq, int user_prio)
186 187 188 189
{
	struct mlx4_en_dev *mdev = priv->mdev;
	int err;

190
	ring->sp_cqn = cq;
191 192 193 194 195
	ring->prod = 0;
	ring->cons = 0xffffffff;
	ring->last_nr_txbb = 1;
	memset(ring->tx_info, 0, ring->size * sizeof(struct mlx4_en_tx_info));
	memset(ring->buf, 0, ring->buf_size);
196
	ring->free_tx_desc = mlx4_en_free_tx_desc;
197

198 199
	ring->sp_qp_state = MLX4_QP_STATE_RST;
	ring->doorbell_qpn = cpu_to_be32(ring->sp_qp.qpn << 8);
200
	ring->mr_key = cpu_to_be32(mdev->mr.key);
201

202 203
	mlx4_en_fill_qp_context(priv, ring->size, ring->sp_stride, 1, 0, ring->qpn,
				ring->sp_cqn, user_prio, &ring->sp_context);
204
	if (ring->bf_alloced)
205
		ring->sp_context.usr_page =
206 207
			cpu_to_be32(mlx4_to_hw_uar_index(mdev->dev,
							 ring->bf.uar->index));
208

209 210 211 212
	err = mlx4_qp_to_ready(mdev->dev, &ring->sp_wqres.mtt, &ring->sp_context,
			       &ring->sp_qp, &ring->sp_qp_state);
	if (!cpumask_empty(&ring->sp_affinity_mask))
		netif_set_xps_queue(priv->dev, &ring->sp_affinity_mask,
213
				    ring->queue_index);
214 215 216 217 218 219 220 221 222

	return err;
}

void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv,
				struct mlx4_en_tx_ring *ring)
{
	struct mlx4_en_dev *mdev = priv->mdev;

223 224
	mlx4_qp_modify(mdev->dev, NULL, ring->sp_qp_state,
		       MLX4_QP_STATE_RST, NULL, 0, 0, &ring->sp_qp);
225 226
}

227 228 229 230 231
static inline bool mlx4_en_is_tx_ring_full(struct mlx4_en_tx_ring *ring)
{
	return ring->prod - ring->cons > ring->full_size;
}

232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264
static void mlx4_en_stamp_wqe(struct mlx4_en_priv *priv,
			      struct mlx4_en_tx_ring *ring, int index,
			      u8 owner)
{
	__be32 stamp = cpu_to_be32(STAMP_VAL | (!!owner << STAMP_SHIFT));
	struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE;
	struct mlx4_en_tx_info *tx_info = &ring->tx_info[index];
	void *end = ring->buf + ring->buf_size;
	__be32 *ptr = (__be32 *)tx_desc;
	int i;

	/* Optimize the common case when there are no wraparounds */
	if (likely((void *)tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) {
		/* Stamp the freed descriptor */
		for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE;
		     i += STAMP_STRIDE) {
			*ptr = stamp;
			ptr += STAMP_DWORDS;
		}
	} else {
		/* Stamp the freed descriptor */
		for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE;
		     i += STAMP_STRIDE) {
			*ptr = stamp;
			ptr += STAMP_DWORDS;
			if ((void *)ptr >= end) {
				ptr = ring->buf;
				stamp ^= cpu_to_be32(0x80000000);
			}
		}
	}
}

265

266 267
u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv,
			 struct mlx4_en_tx_ring *ring,
268
			 int index, u64 timestamp,
269
			 int napi_mode)
270 271 272 273 274
{
	struct mlx4_en_tx_info *tx_info = &ring->tx_info[index];
	struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE;
	struct mlx4_wqe_data_seg *data = (void *) tx_desc + tx_info->data_offset;
	void *end = ring->buf + ring->buf_size;
275 276
	struct sk_buff *skb = tx_info->skb;
	int nr_maps = tx_info->nr_maps;
277
	int i;
278

279 280 281 282 283
	/* We do not touch skb here, so prefetch skb->users location
	 * to speedup consume_skb()
	 */
	prefetchw(&skb->users);

284 285 286 287
	if (unlikely(timestamp)) {
		struct skb_shared_hwtstamps hwts;

		mlx4_en_fill_hwtstamps(priv->mdev, &hwts, timestamp);
288 289
		skb_tstamp_tx(skb, &hwts);
	}
290 291 292

	/* Optimize the common case when there are no wraparounds */
	if (likely((void *) tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) {
293
		if (!tx_info->inl) {
294
			if (tx_info->linear)
295
				dma_unmap_single(priv->ddev,
296 297 298 299 300 301 302 303 304 305
						tx_info->map0_dma,
						tx_info->map0_byte_count,
						PCI_DMA_TODEVICE);
			else
				dma_unmap_page(priv->ddev,
					       tx_info->map0_dma,
					       tx_info->map0_byte_count,
					       PCI_DMA_TODEVICE);
			for (i = 1; i < nr_maps; i++) {
				data++;
306
				dma_unmap_page(priv->ddev,
307 308 309
					(dma_addr_t)be64_to_cpu(data->addr),
					be32_to_cpu(data->byte_count),
					PCI_DMA_TODEVICE);
310
			}
311 312
		}
	} else {
313 314
		if (!tx_info->inl) {
			if ((void *) data >= end) {
315
				data = ring->buf + ((void *)data - end);
316
			}
317

318
			if (tx_info->linear)
319
				dma_unmap_single(priv->ddev,
320 321 322 323 324 325 326 327 328 329
						tx_info->map0_dma,
						tx_info->map0_byte_count,
						PCI_DMA_TODEVICE);
			else
				dma_unmap_page(priv->ddev,
					       tx_info->map0_dma,
					       tx_info->map0_byte_count,
					       PCI_DMA_TODEVICE);
			for (i = 1; i < nr_maps; i++) {
				data++;
330 331
				/* Check for wraparound before unmapping */
				if ((void *) data >= end)
332
					data = ring->buf;
333
				dma_unmap_page(priv->ddev,
334 335 336
					(dma_addr_t)be64_to_cpu(data->addr),
					be32_to_cpu(data->byte_count),
					PCI_DMA_TODEVICE);
337
			}
338 339
		}
	}
340 341
	napi_consume_skb(skb, napi_mode);

342 343 344
	return tx_info->nr_txbb;
}

345 346
u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv,
			    struct mlx4_en_tx_ring *ring,
347
			    int index, u64 timestamp,
348 349 350 351 352 353 354 355 356 357
			    int napi_mode)
{
	struct mlx4_en_tx_info *tx_info = &ring->tx_info[index];
	struct mlx4_en_rx_alloc frame = {
		.page = tx_info->page,
		.dma = tx_info->map0_dma,
	};

	if (!mlx4_en_rx_recycle(ring->recycle_ring, &frame)) {
		dma_unmap_page(priv->ddev, tx_info->map0_dma,
358
			       PAGE_SIZE, priv->dma_dir);
359 360 361 362 363
		put_page(tx_info->page);
	}

	return tx_info->nr_txbb;
}
364 365 366 367 368 369 370 371

int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring)
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
	int cnt = 0;

	/* Skip last polled descriptor */
	ring->cons += ring->last_nr_txbb;
372
	en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n",
373 374 375 376
		 ring->cons, ring->prod);

	if ((u32) (ring->prod - ring->cons) > ring->size) {
		if (netif_msg_tx_err(priv))
377
			en_warn(priv, "Tx consumer passed producer!\n");
378 379 380 381
		return 0;
	}

	while (ring->cons != ring->prod) {
382
		ring->last_nr_txbb = ring->free_tx_desc(priv, ring,
383
						ring->cons & ring->size_mask,
384
						0, 0 /* Non-NAPI caller */);
385 386 387 388
		ring->cons += ring->last_nr_txbb;
		cnt++;
	}

389 390
	if (ring->tx_queue)
		netdev_tx_reset_queue(ring->tx_queue);
391

392
	if (cnt)
393
		en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt);
394 395 396 397

	return cnt;
}

398
static bool mlx4_en_process_tx_cq(struct net_device *dev,
399
				  struct mlx4_en_cq *cq, int napi_budget)
400 401 402
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
	struct mlx4_cq *mcq = &cq->mcq;
403
	struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->type][cq->ring];
404
	struct mlx4_cqe *cqe;
405
	u16 index, ring_index, stamp_index;
406
	u32 txbbs_skipped = 0;
407
	u32 txbbs_stamp = 0;
408 409 410 411
	u32 cons_index = mcq->cons_index;
	int size = cq->size;
	u32 size_mask = ring->size_mask;
	struct mlx4_cqe *buf = cq->buf;
412 413
	u32 packets = 0;
	u32 bytes = 0;
O
Or Gerlitz 已提交
414
	int factor = priv->cqe_factor;
415
	int done = 0;
416
	int budget = priv->tx_work_limit;
417 418
	u32 last_nr_txbb;
	u32 ring_cons;
419

420
	if (unlikely(!priv->port_up))
421
		return true;
422

423 424
	netdev_txq_bql_complete_prefetchw(ring->tx_queue);

425
	index = cons_index & size_mask;
426
	cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
427 428 429
	last_nr_txbb = ACCESS_ONCE(ring->last_nr_txbb);
	ring_cons = ACCESS_ONCE(ring->cons);
	ring_index = ring_cons & size_mask;
430
	stamp_index = ring_index;
431 432 433

	/* Process all completed CQEs */
	while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
434
			cons_index & size) && (done < budget)) {
435 436
		u16 new_index;

437 438 439 440
		/*
		 * make sure we read the CQE after we read the
		 * ownership bit
		 */
441
		dma_rmb();
442

443 444 445 446 447 448 449 450 451
		if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
			     MLX4_CQE_OPCODE_ERROR)) {
			struct mlx4_err_cqe *cqe_err = (struct mlx4_err_cqe *)cqe;

			en_err(priv, "CQE error - vendor syndrome: 0x%x syndrome: 0x%x\n",
			       cqe_err->vendor_err_syndrome,
			       cqe_err->syndrome);
		}

452 453 454
		/* Skip over last polled CQE */
		new_index = be16_to_cpu(cqe->wqe_index) & size_mask;

455
		do {
456 457
			u64 timestamp = 0;

458 459
			txbbs_skipped += last_nr_txbb;
			ring_index = (ring_index + last_nr_txbb) & size_mask;
460 461

			if (unlikely(ring->tx_info[ring_index].ts_requested))
462 463
				timestamp = mlx4_en_get_cqe_ts(cqe);

464
			/* free next descriptor */
465
			last_nr_txbb = ring->free_tx_desc(
466
					priv, ring, ring_index,
467
					timestamp, napi_budget);
468 469

			mlx4_en_stamp_wqe(priv, ring, stamp_index,
470
					  !!((ring_cons + txbbs_stamp) &
471 472 473
						ring->size));
			stamp_index = ring_index;
			txbbs_stamp = txbbs_skipped;
474 475
			packets++;
			bytes += ring->tx_info[ring_index].nr_bytes;
476
		} while ((++done < budget) && (ring_index != new_index));
477 478 479

		++cons_index;
		index = cons_index & size_mask;
480
		cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
481
	}
482 483 484 485 486

	/*
	 * To prevent CQ overflow we first update CQ consumer and only then
	 * the ring consumer.
	 */
487
	mcq->cons_index = cons_index;
488 489
	mlx4_cq_set_ci(mcq);
	wmb();
490 491 492 493 494

	/* we want to dirty this cache line once */
	ACCESS_ONCE(ring->last_nr_txbb) = last_nr_txbb;
	ACCESS_ONCE(ring->cons) = ring_cons + txbbs_skipped;

495
	if (cq->type == TX_XDP)
496 497
		return done < budget;

498
	netdev_tx_completed_queue(ring->tx_queue, packets, bytes);
499

500
	/* Wakeup Tx queue if this stopped, and ring is not full.
501
	 */
502 503
	if (netif_tx_queue_stopped(ring->tx_queue) &&
	    !mlx4_en_is_tx_ring_full(ring)) {
504
		netif_tx_wake_queue(ring->tx_queue);
505
		ring->wake_queue++;
506
	}
507

508
	return done < budget;
509 510 511 512 513 514 515
}

void mlx4_en_tx_irq(struct mlx4_cq *mcq)
{
	struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
	struct mlx4_en_priv *priv = netdev_priv(cq->dev);

E
Eric Dumazet 已提交
516 517
	if (likely(priv->port_up))
		napi_schedule_irqoff(&cq->napi);
518 519
	else
		mlx4_en_arm_cq(priv, cq);
520 521
}

522 523 524 525 526 527
/* TX CQ polling - called by NAPI */
int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget)
{
	struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
	struct net_device *dev = cq->dev;
	struct mlx4_en_priv *priv = netdev_priv(dev);
528
	bool clean_complete;
529

530
	clean_complete = mlx4_en_process_tx_cq(dev, cq, budget);
531 532
	if (!clean_complete)
		return budget;
533

534 535 536 537
	napi_complete(napi);
	mlx4_en_arm_cq(priv, cq);

	return 0;
538
}
539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567

static struct mlx4_en_tx_desc *mlx4_en_bounce_to_desc(struct mlx4_en_priv *priv,
						      struct mlx4_en_tx_ring *ring,
						      u32 index,
						      unsigned int desc_size)
{
	u32 copy = (ring->size - index) * TXBB_SIZE;
	int i;

	for (i = desc_size - copy - 4; i >= 0; i -= 4) {
		if ((i & (TXBB_SIZE - 1)) == 0)
			wmb();

		*((u32 *) (ring->buf + i)) =
			*((u32 *) (ring->bounce_buf + copy + i));
	}

	for (i = copy - 4; i >= 4 ; i -= 4) {
		if ((i & (TXBB_SIZE - 1)) == 0)
			wmb();

		*((u32 *) (ring->buf + index * TXBB_SIZE + i)) =
			*((u32 *) (ring->bounce_buf + i));
	}

	/* Return real descriptor location */
	return ring->buf + index * TXBB_SIZE;
}

568 569 570 571 572 573 574
/* Decide if skb can be inlined in tx descriptor to avoid dma mapping
 *
 * It seems strange we do not simply use skb_copy_bits().
 * This would allow to inline all skbs iff skb->len <= inline_thold
 *
 * Note that caller already checked skb was not a gso packet
 */
575
static bool is_inline(int inline_thold, const struct sk_buff *skb,
576
		      const struct skb_shared_info *shinfo,
577
		      void **pfrag)
578 579 580
{
	void *ptr;

581 582
	if (skb->len > inline_thold || !inline_thold)
		return false;
583

584 585 586 587 588 589
	if (shinfo->nr_frags == 1) {
		ptr = skb_frag_address_safe(&shinfo->frags[0]);
		if (unlikely(!ptr))
			return false;
		*pfrag = ptr;
		return true;
590
	}
591 592 593
	if (shinfo->nr_frags)
		return false;
	return true;
594 595
}

596
static int inline_size(const struct sk_buff *skb)
597 598 599 600 601 602 603 604 605 606
{
	if (skb->len + CTRL_SIZE + sizeof(struct mlx4_wqe_inline_seg)
	    <= MLX4_INLINE_ALIGN)
		return ALIGN(skb->len + CTRL_SIZE +
			     sizeof(struct mlx4_wqe_inline_seg), 16);
	else
		return ALIGN(skb->len + CTRL_SIZE + 2 *
			     sizeof(struct mlx4_wqe_inline_seg), 16);
}

607
static int get_real_size(const struct sk_buff *skb,
608
			 const struct skb_shared_info *shinfo,
609
			 struct net_device *dev,
610 611 612
			 int *lso_header_size,
			 bool *inline_ok,
			 void **pfrag)
613 614 615 616
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
	int real_size;

617
	if (shinfo->gso_size) {
618
		*inline_ok = false;
619 620 621 622
		if (skb->encapsulation)
			*lso_header_size = (skb_inner_transport_header(skb) - skb->data) + inner_tcp_hdrlen(skb);
		else
			*lso_header_size = skb_transport_offset(skb) + tcp_hdrlen(skb);
623
		real_size = CTRL_SIZE + shinfo->nr_frags * DS_SIZE +
624 625 626 627 628 629 630 631
			ALIGN(*lso_header_size + 4, DS_SIZE);
		if (unlikely(*lso_header_size != skb_headlen(skb))) {
			/* We add a segment for the skb linear buffer only if
			 * it contains data */
			if (*lso_header_size < skb_headlen(skb))
				real_size += DS_SIZE;
			else {
				if (netif_msg_tx_err(priv))
632
					en_warn(priv, "Non-linear headers\n");
633 634 635 636 637
				return 0;
			}
		}
	} else {
		*lso_header_size = 0;
638 639 640 641
		*inline_ok = is_inline(priv->prof->inline_thold, skb,
				       shinfo, pfrag);

		if (*inline_ok)
642
			real_size = inline_size(skb);
643 644 645
		else
			real_size = CTRL_SIZE +
				    (shinfo->nr_frags + 1) * DS_SIZE;
646 647 648 649 650
	}

	return real_size;
}

651 652
static void build_inline_wqe(struct mlx4_en_tx_desc *tx_desc,
			     const struct sk_buff *skb,
653
			     const struct skb_shared_info *shinfo,
654
			     void *fragptr)
655 656 657
{
	struct mlx4_wqe_inline_seg *inl = &tx_desc->inl;
	int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - sizeof *inl;
658
	unsigned int hlen = skb_headlen(skb);
659 660

	if (skb->len <= spc) {
661 662 663 664 665 666 667
		if (likely(skb->len >= MIN_PKT_LEN)) {
			inl->byte_count = cpu_to_be32(1 << 31 | skb->len);
		} else {
			inl->byte_count = cpu_to_be32(1 << 31 | MIN_PKT_LEN);
			memset(((void *)(inl + 1)) + skb->len, 0,
			       MIN_PKT_LEN - skb->len);
		}
668
		skb_copy_from_linear_data(skb, inl + 1, hlen);
669
		if (shinfo->nr_frags)
670
			memcpy(((void *)(inl + 1)) + hlen, fragptr,
671
			       skb_frag_size(&shinfo->frags[0]));
672 673 674

	} else {
		inl->byte_count = cpu_to_be32(1 << 31 | spc);
675 676 677 678 679 680
		if (hlen <= spc) {
			skb_copy_from_linear_data(skb, inl + 1, hlen);
			if (hlen < spc) {
				memcpy(((void *)(inl + 1)) + hlen,
				       fragptr, spc - hlen);
				fragptr +=  spc - hlen;
681 682 683 684 685 686 687
			}
			inl = (void *) (inl + 1) + spc;
			memcpy(((void *)(inl + 1)), fragptr, skb->len - spc);
		} else {
			skb_copy_from_linear_data(skb, inl + 1, spc);
			inl = (void *) (inl + 1) + spc;
			skb_copy_from_linear_data_offset(skb, spc, inl + 1,
688
							 hlen - spc);
689
			if (shinfo->nr_frags)
690
				memcpy(((void *)(inl + 1)) + hlen - spc,
691 692
				       fragptr,
				       skb_frag_size(&shinfo->frags[0]));
693 694
		}

695
		dma_wmb();
696 697 698 699
		inl->byte_count = cpu_to_be32(1 << 31 | (skb->len - spc));
	}
}

700
u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb,
701
			 void *accel_priv, select_queue_fallback_t fallback)
702
{
703
	struct mlx4_en_priv *priv = netdev_priv(dev);
704
	u16 rings_p_up = priv->num_tx_rings_p_up;
705
	u8 up = 0;
706

707
	if (netdev_get_num_tc(dev))
708 709
		return skb_tx_hash(dev, skb);

710 711
	if (skb_vlan_tag_present(skb))
		up = skb_vlan_tag_get(skb) >> VLAN_PRIO_SHIFT;
Y
Yevgeny Petrilin 已提交
712

713
	return fallback(dev, skb) % rings_p_up + up * rings_p_up;
714 715
}

716 717
static void mlx4_bf_copy(void __iomem *dst, const void *src,
			 unsigned int bytecnt)
718 719 720 721
{
	__iowrite64_copy(dst, src, bytecnt / 8);
}

722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776
void mlx4_en_xmit_doorbell(struct mlx4_en_tx_ring *ring)
{
	wmb();
	/* Since there is no iowrite*_native() that writes the
	 * value as is, without byteswapping - using the one
	 * the doesn't do byteswapping in the relevant arch
	 * endianness.
	 */
#if defined(__LITTLE_ENDIAN)
	iowrite32(
#else
	iowrite32be(
#endif
		  ring->doorbell_qpn,
		  ring->bf.uar->map + MLX4_SEND_DOORBELL);
}

static void mlx4_en_tx_write_desc(struct mlx4_en_tx_ring *ring,
				  struct mlx4_en_tx_desc *tx_desc,
				  union mlx4_wqe_qpn_vlan qpn_vlan,
				  int desc_size, int bf_index,
				  __be32 op_own, bool bf_ok,
				  bool send_doorbell)
{
	tx_desc->ctrl.qpn_vlan = qpn_vlan;

	if (bf_ok) {
		op_own |= htonl((bf_index & 0xffff) << 8);
		/* Ensure new descriptor hits memory
		 * before setting ownership of this descriptor to HW
		 */
		dma_wmb();
		tx_desc->ctrl.owner_opcode = op_own;

		wmb();

		mlx4_bf_copy(ring->bf.reg + ring->bf.offset, &tx_desc->ctrl,
			     desc_size);

		wmb();

		ring->bf.offset ^= ring->bf.buf_size;
	} else {
		/* Ensure new descriptor hits memory
		 * before setting ownership of this descriptor to HW
		 */
		dma_wmb();
		tx_desc->ctrl.owner_opcode = op_own;
		if (send_doorbell)
			mlx4_en_xmit_doorbell(ring);
		else
			ring->xmit_more++;
	}
}

777
netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
778
{
779
	struct skb_shared_info *shinfo = skb_shinfo(skb);
780
	struct mlx4_en_priv *priv = netdev_priv(dev);
781
	union mlx4_wqe_qpn_vlan	qpn_vlan = {};
782
	struct device *ddev = priv->ddev;
783 784 785 786 787 788 789 790
	struct mlx4_en_tx_ring *ring;
	struct mlx4_en_tx_desc *tx_desc;
	struct mlx4_wqe_data_seg *data;
	struct mlx4_en_tx_info *tx_info;
	int tx_ind = 0;
	int nr_txbb;
	int desc_size;
	int real_size;
791
	u32 index, bf_index;
792
	__be32 op_own;
793
	u16 vlan_proto = 0;
794
	int i_frag;
795
	int lso_header_size;
796
	void *fragptr = NULL;
797
	bool bounce = false;
798
	bool send_doorbell;
E
Eric Dumazet 已提交
799
	bool stop_queue;
800
	bool inline_ok;
801
	u32 ring_cons;
802
	bool bf_ok;
803

804
	tx_ind = skb_get_queue_mapping(skb);
805
	ring = priv->tx_ring[TX][tx_ind];
806

E
Eric Dumazet 已提交
807 808 809
	if (!priv->port_up)
		goto tx_drop;

810 811 812
	/* fetch ring->cons far ahead before needing it to avoid stall */
	ring_cons = ACCESS_ONCE(ring->cons);

813 814
	real_size = get_real_size(skb, shinfo, dev, &lso_header_size,
				  &inline_ok, &fragptr);
815
	if (unlikely(!real_size))
816
		goto tx_drop_count;
817

L
Lucas De Marchi 已提交
818
	/* Align descriptor to TXBB size */
819 820 821 822
	desc_size = ALIGN(real_size, TXBB_SIZE);
	nr_txbb = desc_size / TXBB_SIZE;
	if (unlikely(nr_txbb > MAX_DESC_TXBBS)) {
		if (netif_msg_tx_err(priv))
823
			en_warn(priv, "Oversized header or SG list\n");
824
		goto tx_drop_count;
825 826
	}

827
	bf_ok = ring->bf_enabled;
828
	if (skb_vlan_tag_present(skb)) {
829
		qpn_vlan.vlan_tag = cpu_to_be16(skb_vlan_tag_get(skb));
830
		vlan_proto = be16_to_cpu(skb->vlan_proto);
831 832 833 834 835 836 837
		if (vlan_proto == ETH_P_8021AD)
			qpn_vlan.ins_vlan = MLX4_WQE_CTRL_INS_SVLAN;
		else if (vlan_proto == ETH_P_8021Q)
			qpn_vlan.ins_vlan = MLX4_WQE_CTRL_INS_CVLAN;
		else
			qpn_vlan.ins_vlan = 0;
		bf_ok = false;
838
	}
839

840
	netdev_txq_bql_enqueue_prefetchw(ring->tx_queue);
841

842 843
	/* Track current inflight packets for performance analysis */
	AVG_PERF_COUNTER(priv->pstats.inflight_avg,
844
			 (u32)(ring->prod - ring_cons - 1));
845 846 847

	/* Packet is good - grab an index and transmit it */
	index = ring->prod & ring->size_mask;
848
	bf_index = ring->prod;
849 850 851 852 853

	/* See if we have enough space for whole descriptor TXBB for setting
	 * SW ownership on next descriptor; if not, use a bounce buffer. */
	if (likely(index + nr_txbb <= ring->size))
		tx_desc = ring->buf + index * TXBB_SIZE;
854
	else {
855
		tx_desc = (struct mlx4_en_tx_desc *) ring->bounce_buf;
856
		bounce = true;
857
		bf_ok = false;
858
	}
859 860 861 862 863 864

	/* Save skb in tx_info ring */
	tx_info = &ring->tx_info[index];
	tx_info->skb = skb;
	tx_info->nr_txbb = nr_txbb;

865
	data = &tx_desc->data;
866 867 868 869 870 871 872
	if (lso_header_size)
		data = ((void *)&tx_desc->lso + ALIGN(lso_header_size + 4,
						      DS_SIZE));

	/* valid only for none inline segments */
	tx_info->data_offset = (void *)data - (void *)tx_desc;

873 874
	tx_info->inl = inline_ok;

875
	tx_info->linear = (lso_header_size < skb_headlen(skb) &&
876
			   !inline_ok) ? 1 : 0;
877

878
	tx_info->nr_maps = shinfo->nr_frags + tx_info->linear;
879
	data += tx_info->nr_maps - 1;
880

881
	if (!tx_info->inl) {
882 883 884
		dma_addr_t dma = 0;
		u32 byte_count = 0;

885
		/* Map fragments if any */
886
		for (i_frag = shinfo->nr_frags - 1; i_frag >= 0; i_frag--) {
887
			const struct skb_frag_struct *frag;
888 889

			frag = &shinfo->frags[i_frag];
890
			byte_count = skb_frag_size(frag);
891
			dma = skb_frag_dma_map(ddev, frag,
892
					       0, byte_count,
893 894 895 896 897
					       DMA_TO_DEVICE);
			if (dma_mapping_error(ddev, dma))
				goto tx_drop_unmap;

			data->addr = cpu_to_be64(dma);
898
			data->lkey = ring->mr_key;
899
			dma_wmb();
900
			data->byte_count = cpu_to_be32(byte_count);
901 902 903
			--data;
		}

904
		/* Map linear part if needed */
905
		if (tx_info->linear) {
906
			byte_count = skb_headlen(skb) - lso_header_size;
907

908 909 910 911 912 913 914
			dma = dma_map_single(ddev, skb->data +
					     lso_header_size, byte_count,
					     PCI_DMA_TODEVICE);
			if (dma_mapping_error(ddev, dma))
				goto tx_drop_unmap;

			data->addr = cpu_to_be64(dma);
915
			data->lkey = ring->mr_key;
916
			dma_wmb();
917 918
			data->byte_count = cpu_to_be32(byte_count);
		}
919 920 921
		/* tx completion can avoid cache line miss for common cases */
		tx_info->map0_dma = dma;
		tx_info->map0_byte_count = byte_count;
922 923
	}

924 925 926 927
	/*
	 * For timestamping add flag to skb_shinfo and
	 * set flag for further reference
	 */
928
	tx_info->ts_requested = 0;
929 930 931
	if (unlikely(ring->hwtstamp_tx_type == HWTSTAMP_TX_ON &&
		     shinfo->tx_flags & SKBTX_HW_TSTAMP)) {
		shinfo->tx_flags |= SKBTX_IN_PROGRESS;
932 933 934
		tx_info->ts_requested = 1;
	}

935 936
	/* Prepare ctrl segement apart opcode+ownership, which depends on
	 * whether LSO is used */
A
Amir Vadai 已提交
937
	tx_desc->ctrl.srcrb_flags = priv->ctrl_flags;
938
	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
939 940 941 942 943
		if (!skb->encapsulation)
			tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM |
								 MLX4_WQE_CTRL_TCP_UDP_CSUM);
		else
			tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM);
944
		ring->tx_csum++;
945 946
	}

947
	if (priv->flags & MLX4_EN_FLAG_ENABLE_HW_LOOPBACK) {
948 949
		struct ethhdr *ethh;

950 951 952 953 954 955 956 957
		/* Copy dst mac address to wqe. This allows loopback in eSwitch,
		 * so that VFs and PF can communicate with each other
		 */
		ethh = (struct ethhdr *)skb->data;
		tx_desc->ctrl.srcrb_flags16[0] = get_unaligned((__be16 *)ethh->h_dest);
		tx_desc->ctrl.imm = get_unaligned((__be32 *)(ethh->h_dest + 2));
	}

958 959
	/* Handle LSO (TSO) packets */
	if (lso_header_size) {
960 961
		int i;

962 963 964 965 966 967 968
		/* Mark opcode as LSO */
		op_own = cpu_to_be32(MLX4_OPCODE_LSO | (1 << 6)) |
			((ring->prod & ring->size) ?
				cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0);

		/* Fill in the LSO prefix */
		tx_desc->lso.mss_hdr_size = cpu_to_be32(
969
			shinfo->gso_size << 16 | lso_header_size);
970 971 972 973 974

		/* Copy headers;
		 * note that we already verified that it is linear */
		memcpy(tx_desc->lso.header, skb->data, lso_header_size);

E
Eric Dumazet 已提交
975
		ring->tso_packets++;
976

E
Eric Dumazet 已提交
977
		i = shinfo->gso_segs;
978
		tx_info->nr_bytes = skb->len + (i - 1) * lso_header_size;
979 980 981 982 983 984
		ring->packets += i;
	} else {
		/* Normal (Non LSO) packet */
		op_own = cpu_to_be32(MLX4_OPCODE_SEND) |
			((ring->prod & ring->size) ?
			 cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0);
985
		tx_info->nr_bytes = max_t(unsigned int, skb->len, ETH_ZLEN);
986 987
		ring->packets++;
	}
988 989
	ring->bytes += tx_info->nr_bytes;
	netdev_tx_sent_queue(ring->tx_queue, tx_info->nr_bytes);
990 991
	AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, skb->len);

992
	if (tx_info->inl)
993
		build_inline_wqe(tx_desc, skb, shinfo, fragptr);
994

995
	if (skb->encapsulation) {
996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
		union {
			struct iphdr *v4;
			struct ipv6hdr *v6;
			unsigned char *hdr;
		} ip;
		u8 proto;

		ip.hdr = skb_inner_network_header(skb);
		proto = (ip.v4->version == 4) ? ip.v4->protocol :
						ip.v6->nexthdr;

		if (proto == IPPROTO_TCP || proto == IPPROTO_UDP)
1008 1009 1010 1011 1012
			op_own |= cpu_to_be32(MLX4_WQE_CTRL_IIP | MLX4_WQE_CTRL_ILP);
		else
			op_own |= cpu_to_be32(MLX4_WQE_CTRL_IIP);
	}

1013 1014 1015
	ring->prod += nr_txbb;

	/* If we used a bounce buffer then copy descriptor back into place */
1016
	if (unlikely(bounce))
1017 1018
		tx_desc = mlx4_en_bounce_to_desc(priv, ring, index, desc_size);

1019 1020
	skb_tx_timestamp(skb);

E
Eric Dumazet 已提交
1021
	/* Check available TXBBs And 2K spare for prefetch */
1022
	stop_queue = mlx4_en_is_tx_ring_full(ring);
E
Eric Dumazet 已提交
1023 1024 1025 1026
	if (unlikely(stop_queue)) {
		netif_tx_stop_queue(ring->tx_queue);
		ring->queue_stopped++;
	}
1027 1028
	send_doorbell = !skb->xmit_more || netif_xmit_stopped(ring->tx_queue);

1029 1030
	real_size = (real_size / 16) & 0x3f;

1031
	bf_ok &= desc_size <= MAX_BF && send_doorbell;
1032

1033 1034 1035 1036
	if (bf_ok)
		qpn_vlan.bf_qpn = ring->doorbell_qpn | cpu_to_be32(real_size);
	else
		qpn_vlan.fence_size = real_size;
1037

1038 1039
	mlx4_en_tx_write_desc(ring, tx_desc, qpn_vlan, desc_size, bf_index,
			      op_own, bf_ok, send_doorbell);
1040

E
Eric Dumazet 已提交
1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
	if (unlikely(stop_queue)) {
		/* If queue was emptied after the if (stop_queue) , and before
		 * the netif_tx_stop_queue() - need to wake the queue,
		 * or else it will remain stopped forever.
		 * Need a memory barrier to make sure ring->cons was not
		 * updated before queue was stopped.
		 */
		smp_rmb();

		ring_cons = ACCESS_ONCE(ring->cons);
1051
		if (unlikely(!mlx4_en_is_tx_ring_full(ring))) {
E
Eric Dumazet 已提交
1052 1053 1054 1055
			netif_tx_wake_queue(ring->tx_queue);
			ring->wake_queue++;
		}
	}
1056
	return NETDEV_TX_OK;
1057

1058 1059 1060
tx_drop_unmap:
	en_err(priv, "DMA mapping error\n");

1061 1062
	while (++i_frag < shinfo->nr_frags) {
		++data;
1063 1064 1065 1066 1067
		dma_unmap_page(ddev, (dma_addr_t) be64_to_cpu(data->addr),
			       be32_to_cpu(data->byte_count),
			       PCI_DMA_TODEVICE);
	}

1068 1069
tx_drop_count:
	ring->tx_dropped++;
1070 1071 1072
tx_drop:
	dev_kfree_skb_any(skb);
	return NETDEV_TX_OK;
1073 1074
}

1075 1076
netdev_tx_t mlx4_en_xmit_frame(struct mlx4_en_rx_ring *rx_ring,
			       struct mlx4_en_rx_alloc *frame,
1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
			       struct net_device *dev, unsigned int length,
			       int tx_ind, int *doorbell_pending)
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
	union mlx4_wqe_qpn_vlan	qpn_vlan = {};
	struct mlx4_en_tx_ring *ring;
	struct mlx4_en_tx_desc *tx_desc;
	struct mlx4_wqe_data_seg *data;
	struct mlx4_en_tx_info *tx_info;
	int index, bf_index;
	bool send_doorbell;
	int nr_txbb = 1;
	bool stop_queue;
	dma_addr_t dma;
	int real_size;
	__be32 op_own;
	u32 ring_cons;
	bool bf_ok;

	BUILD_BUG_ON_MSG(ALIGN(CTRL_SIZE + DS_SIZE, TXBB_SIZE) != TXBB_SIZE,
			 "mlx4_en_xmit_frame requires minimum size tx desc");

1099
	ring = priv->tx_ring[TX_XDP][tx_ind];
1100 1101 1102 1103 1104

	if (!priv->port_up)
		goto tx_drop;

	if (mlx4_en_is_tx_ring_full(ring))
1105
		goto tx_drop_count;
1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127

	/* fetch ring->cons far ahead before needing it to avoid stall */
	ring_cons = READ_ONCE(ring->cons);

	index = ring->prod & ring->size_mask;
	tx_info = &ring->tx_info[index];

	bf_ok = ring->bf_enabled;

	/* Track current inflight packets for performance analysis */
	AVG_PERF_COUNTER(priv->pstats.inflight_avg,
			 (u32)(ring->prod - ring_cons - 1));

	bf_index = ring->prod;
	tx_desc = ring->buf + index * TXBB_SIZE;
	data = &tx_desc->data;

	dma = frame->dma;

	tx_info->page = frame->page;
	frame->page = NULL;
	tx_info->map0_dma = dma;
1128
	tx_info->map0_byte_count = PAGE_SIZE;
1129 1130 1131 1132 1133 1134 1135 1136
	tx_info->nr_txbb = nr_txbb;
	tx_info->nr_bytes = max_t(unsigned int, length, ETH_ZLEN);
	tx_info->data_offset = (void *)data - (void *)tx_desc;
	tx_info->ts_requested = 0;
	tx_info->nr_maps = 1;
	tx_info->linear = 1;
	tx_info->inl = 0;

1137 1138
	dma_sync_single_range_for_device(priv->ddev, dma, frame->page_offset,
					 length, PCI_DMA_TODEVICE);
1139

1140
	data->addr = cpu_to_be64(dma + frame->page_offset);
1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151
	data->lkey = ring->mr_key;
	dma_wmb();
	data->byte_count = cpu_to_be32(length);

	/* tx completion can avoid cache line miss for common cases */
	tx_desc->ctrl.srcrb_flags = priv->ctrl_flags;

	op_own = cpu_to_be32(MLX4_OPCODE_SEND) |
		((ring->prod & ring->size) ?
		 cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0);

1152
	rx_ring->xdp_tx++;
1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
	AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, length);

	ring->prod += nr_txbb;

	stop_queue = mlx4_en_is_tx_ring_full(ring);
	send_doorbell = stop_queue ||
				*doorbell_pending > MLX4_EN_DOORBELL_BUDGET;
	bf_ok &= send_doorbell;

	real_size = ((CTRL_SIZE + nr_txbb * DS_SIZE) / 16) & 0x3f;

	if (bf_ok)
		qpn_vlan.bf_qpn = ring->doorbell_qpn | cpu_to_be32(real_size);
	else
		qpn_vlan.fence_size = real_size;

	mlx4_en_tx_write_desc(ring, tx_desc, qpn_vlan, TXBB_SIZE, bf_index,
			      op_own, bf_ok, send_doorbell);
	*doorbell_pending = send_doorbell ? 0 : *doorbell_pending + 1;

	return NETDEV_TX_OK;

1175
tx_drop_count:
1176
	rx_ring->xdp_tx_full++;
1177
tx_drop:
1178 1179
	return NETDEV_TX_BUSY;
}