en_tx.c 31.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
/*
 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#include <asm/page.h>
#include <linux/mlx4/cq.h>
36
#include <linux/slab.h>
37 38 39
#include <linux/mlx4/qp.h>
#include <linux/skbuff.h>
#include <linux/if_vlan.h>
40
#include <linux/prefetch.h>
41
#include <linux/vmalloc.h>
42
#include <linux/tcp.h>
43
#include <linux/ip.h>
44
#include <linux/ipv6.h>
45
#include <linux/moduleparam.h>
46 47 48 49

#include "mlx4_en.h"

int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
50
			   struct mlx4_en_tx_ring **pring, u32 size,
51
			   u16 stride, int node, int queue_index)
52 53
{
	struct mlx4_en_dev *mdev = priv->mdev;
54
	struct mlx4_en_tx_ring *ring;
55 56 57
	int tmp;
	int err;

58
	ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node);
59
	if (!ring) {
60 61 62 63 64
		ring = kzalloc(sizeof(*ring), GFP_KERNEL);
		if (!ring) {
			en_err(priv, "Failed allocating TX ring\n");
			return -ENOMEM;
		}
65 66
	}

67 68
	ring->size = size;
	ring->size_mask = size - 1;
69
	ring->sp_stride = stride;
70
	ring->full_size = ring->size - HEADROOM - MAX_DESC_TXBBS;
71 72

	tmp = size * sizeof(struct mlx4_en_tx_info);
73
	ring->tx_info = kmalloc_node(tmp, GFP_KERNEL | __GFP_NOWARN, node);
74
	if (!ring->tx_info) {
75 76 77 78 79
		ring->tx_info = vmalloc(tmp);
		if (!ring->tx_info) {
			err = -ENOMEM;
			goto err_ring;
		}
80
	}
81

82
	en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
83 84
		 ring->tx_info, tmp);

85
	ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, GFP_KERNEL, node);
86
	if (!ring->bounce_buf) {
87 88 89 90 91
		ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL);
		if (!ring->bounce_buf) {
			err = -ENOMEM;
			goto err_info;
		}
92
	}
93
	ring->buf_size = ALIGN(size * ring->sp_stride, MLX4_EN_PAGE_SIZE);
94

95
	/* Allocate HW buffers on provided NUMA node */
96
	set_dev_node(&mdev->dev->persist->pdev->dev, node);
97
	err = mlx4_alloc_hwq_res(mdev->dev, &ring->sp_wqres, ring->buf_size);
98
	set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
99
	if (err) {
100
		en_err(priv, "Failed allocating hwq resources\n");
101 102 103
		goto err_bounce;
	}

104
	ring->buf = ring->sp_wqres.buf.direct.buf;
105

J
Joe Perches 已提交
106 107
	en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d buf_size:%d dma:%llx\n",
	       ring, ring->buf, ring->size, ring->buf_size,
108
	       (unsigned long long) ring->sp_wqres.buf.direct.map);
109

110 111 112 113
	err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
				    MLX4_RESERVE_ETH_BF_QP);
	if (err) {
		en_err(priv, "failed reserving qp for TX ring\n");
114
		goto err_hwq_res;
115 116
	}

117
	err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->sp_qp, GFP_KERNEL);
118
	if (err) {
119
		en_err(priv, "Failed allocating qp %d\n", ring->qpn);
120
		goto err_reserve;
121
	}
122
	ring->sp_qp.event = mlx4_en_sqp_event;
123

124
	err = mlx4_bf_alloc(mdev->dev, &ring->bf, node);
125
	if (err) {
J
Joe Perches 已提交
126
		en_dbg(DRV, priv, "working without blueflame (%d)\n", err);
127 128 129
		ring->bf.uar = &mdev->priv_uar;
		ring->bf.uar->map = mdev->uar_map;
		ring->bf_enabled = false;
130 131 132 133 134 135 136
		ring->bf_alloced = false;
		priv->pflags &= ~MLX4_EN_PRIV_FLAGS_BLUEFLAME;
	} else {
		ring->bf_alloced = true;
		ring->bf_enabled = !!(priv->pflags &
				      MLX4_EN_PRIV_FLAGS_BLUEFLAME);
	}
137

138
	ring->hwtstamp_tx_type = priv->hwtstamp_config.tx_type;
139 140
	ring->queue_index = queue_index;

141
	if (queue_index < priv->num_tx_rings_p_up)
142 143
		cpumask_set_cpu(cpumask_local_spread(queue_index,
						     priv->mdev->dev->numa_node),
144
				&ring->sp_affinity_mask);
145

146
	*pring = ring;
147 148
	return 0;

149 150
err_reserve:
	mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
151
err_hwq_res:
152
	mlx4_free_hwq_res(mdev->dev, &ring->sp_wqres, ring->buf_size);
153 154 155
err_bounce:
	kfree(ring->bounce_buf);
	ring->bounce_buf = NULL;
156
err_info:
157
	kvfree(ring->tx_info);
158
	ring->tx_info = NULL;
159 160 161
err_ring:
	kfree(ring);
	*pring = NULL;
162 163 164 165
	return err;
}

void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
166
			     struct mlx4_en_tx_ring **pring)
167 168
{
	struct mlx4_en_dev *mdev = priv->mdev;
169
	struct mlx4_en_tx_ring *ring = *pring;
170
	en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn);
171

172
	if (ring->bf_alloced)
173
		mlx4_bf_free(mdev->dev, &ring->bf);
174 175
	mlx4_qp_remove(mdev->dev, &ring->sp_qp);
	mlx4_qp_free(mdev->dev, &ring->sp_qp);
176
	mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1);
177
	mlx4_free_hwq_res(mdev->dev, &ring->sp_wqres, ring->buf_size);
178 179
	kfree(ring->bounce_buf);
	ring->bounce_buf = NULL;
180
	kvfree(ring->tx_info);
181
	ring->tx_info = NULL;
182 183
	kfree(ring);
	*pring = NULL;
184 185 186 187
}

int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv,
			     struct mlx4_en_tx_ring *ring,
188
			     int cq, int user_prio)
189 190 191 192
{
	struct mlx4_en_dev *mdev = priv->mdev;
	int err;

193
	ring->sp_cqn = cq;
194 195 196 197 198
	ring->prod = 0;
	ring->cons = 0xffffffff;
	ring->last_nr_txbb = 1;
	memset(ring->tx_info, 0, ring->size * sizeof(struct mlx4_en_tx_info));
	memset(ring->buf, 0, ring->buf_size);
199
	ring->free_tx_desc = mlx4_en_free_tx_desc;
200

201 202
	ring->sp_qp_state = MLX4_QP_STATE_RST;
	ring->doorbell_qpn = cpu_to_be32(ring->sp_qp.qpn << 8);
203
	ring->mr_key = cpu_to_be32(mdev->mr.key);
204

205 206
	mlx4_en_fill_qp_context(priv, ring->size, ring->sp_stride, 1, 0, ring->qpn,
				ring->sp_cqn, user_prio, &ring->sp_context);
207
	if (ring->bf_alloced)
208
		ring->sp_context.usr_page =
209 210
			cpu_to_be32(mlx4_to_hw_uar_index(mdev->dev,
							 ring->bf.uar->index));
211

212 213 214 215
	err = mlx4_qp_to_ready(mdev->dev, &ring->sp_wqres.mtt, &ring->sp_context,
			       &ring->sp_qp, &ring->sp_qp_state);
	if (!cpumask_empty(&ring->sp_affinity_mask))
		netif_set_xps_queue(priv->dev, &ring->sp_affinity_mask,
216
				    ring->queue_index);
217 218 219 220 221 222 223 224 225

	return err;
}

void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv,
				struct mlx4_en_tx_ring *ring)
{
	struct mlx4_en_dev *mdev = priv->mdev;

226 227
	mlx4_qp_modify(mdev->dev, NULL, ring->sp_qp_state,
		       MLX4_QP_STATE_RST, NULL, 0, 0, &ring->sp_qp);
228 229
}

230 231 232 233 234
static inline bool mlx4_en_is_tx_ring_full(struct mlx4_en_tx_ring *ring)
{
	return ring->prod - ring->cons > ring->full_size;
}

235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267
static void mlx4_en_stamp_wqe(struct mlx4_en_priv *priv,
			      struct mlx4_en_tx_ring *ring, int index,
			      u8 owner)
{
	__be32 stamp = cpu_to_be32(STAMP_VAL | (!!owner << STAMP_SHIFT));
	struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE;
	struct mlx4_en_tx_info *tx_info = &ring->tx_info[index];
	void *end = ring->buf + ring->buf_size;
	__be32 *ptr = (__be32 *)tx_desc;
	int i;

	/* Optimize the common case when there are no wraparounds */
	if (likely((void *)tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) {
		/* Stamp the freed descriptor */
		for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE;
		     i += STAMP_STRIDE) {
			*ptr = stamp;
			ptr += STAMP_DWORDS;
		}
	} else {
		/* Stamp the freed descriptor */
		for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE;
		     i += STAMP_STRIDE) {
			*ptr = stamp;
			ptr += STAMP_DWORDS;
			if ((void *)ptr >= end) {
				ptr = ring->buf;
				stamp ^= cpu_to_be32(0x80000000);
			}
		}
	}
}

268

269 270 271 272
u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv,
			 struct mlx4_en_tx_ring *ring,
			 int index, u8 owner, u64 timestamp,
			 int napi_mode)
273 274 275 276 277
{
	struct mlx4_en_tx_info *tx_info = &ring->tx_info[index];
	struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE;
	struct mlx4_wqe_data_seg *data = (void *) tx_desc + tx_info->data_offset;
	void *end = ring->buf + ring->buf_size;
278 279
	struct sk_buff *skb = tx_info->skb;
	int nr_maps = tx_info->nr_maps;
280
	int i;
281

282 283 284 285 286
	/* We do not touch skb here, so prefetch skb->users location
	 * to speedup consume_skb()
	 */
	prefetchw(&skb->users);

287 288 289 290
	if (unlikely(timestamp)) {
		struct skb_shared_hwtstamps hwts;

		mlx4_en_fill_hwtstamps(priv->mdev, &hwts, timestamp);
291 292
		skb_tstamp_tx(skb, &hwts);
	}
293 294 295

	/* Optimize the common case when there are no wraparounds */
	if (likely((void *) tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) {
296
		if (!tx_info->inl) {
297
			if (tx_info->linear)
298
				dma_unmap_single(priv->ddev,
299 300 301 302 303 304 305 306 307 308
						tx_info->map0_dma,
						tx_info->map0_byte_count,
						PCI_DMA_TODEVICE);
			else
				dma_unmap_page(priv->ddev,
					       tx_info->map0_dma,
					       tx_info->map0_byte_count,
					       PCI_DMA_TODEVICE);
			for (i = 1; i < nr_maps; i++) {
				data++;
309
				dma_unmap_page(priv->ddev,
310 311 312
					(dma_addr_t)be64_to_cpu(data->addr),
					be32_to_cpu(data->byte_count),
					PCI_DMA_TODEVICE);
313
			}
314 315
		}
	} else {
316 317
		if (!tx_info->inl) {
			if ((void *) data >= end) {
318
				data = ring->buf + ((void *)data - end);
319
			}
320

321
			if (tx_info->linear)
322
				dma_unmap_single(priv->ddev,
323 324 325 326 327 328 329 330 331 332
						tx_info->map0_dma,
						tx_info->map0_byte_count,
						PCI_DMA_TODEVICE);
			else
				dma_unmap_page(priv->ddev,
					       tx_info->map0_dma,
					       tx_info->map0_byte_count,
					       PCI_DMA_TODEVICE);
			for (i = 1; i < nr_maps; i++) {
				data++;
333 334
				/* Check for wraparound before unmapping */
				if ((void *) data >= end)
335
					data = ring->buf;
336
				dma_unmap_page(priv->ddev,
337 338 339
					(dma_addr_t)be64_to_cpu(data->addr),
					be32_to_cpu(data->byte_count),
					PCI_DMA_TODEVICE);
340
			}
341 342
		}
	}
343 344
	napi_consume_skb(skb, napi_mode);

345 346 347
	return tx_info->nr_txbb;
}

348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv,
			    struct mlx4_en_tx_ring *ring,
			    int index, u8 owner, u64 timestamp,
			    int napi_mode)
{
	struct mlx4_en_tx_info *tx_info = &ring->tx_info[index];
	struct mlx4_en_rx_alloc frame = {
		.page = tx_info->page,
		.dma = tx_info->map0_dma,
		.page_offset = 0,
		.page_size = PAGE_SIZE,
	};

	if (!mlx4_en_rx_recycle(ring->recycle_ring, &frame)) {
		dma_unmap_page(priv->ddev, tx_info->map0_dma,
			       PAGE_SIZE, priv->frag_info[0].dma_dir);
		put_page(tx_info->page);
	}

	return tx_info->nr_txbb;
}
369 370 371 372 373 374 375 376

int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring)
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
	int cnt = 0;

	/* Skip last polled descriptor */
	ring->cons += ring->last_nr_txbb;
377
	en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n",
378 379 380 381
		 ring->cons, ring->prod);

	if ((u32) (ring->prod - ring->cons) > ring->size) {
		if (netif_msg_tx_err(priv))
382
			en_warn(priv, "Tx consumer passed producer!\n");
383 384 385 386
		return 0;
	}

	while (ring->cons != ring->prod) {
387
		ring->last_nr_txbb = ring->free_tx_desc(priv, ring,
388
						ring->cons & ring->size_mask,
389 390
						!!(ring->cons & ring->size), 0,
						0 /* Non-NAPI caller */);
391 392 393 394
		ring->cons += ring->last_nr_txbb;
		cnt++;
	}

395 396
	if (ring->tx_queue)
		netdev_tx_reset_queue(ring->tx_queue);
397

398
	if (cnt)
399
		en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt);
400 401 402 403

	return cnt;
}

404
static bool mlx4_en_process_tx_cq(struct net_device *dev,
405
				  struct mlx4_en_cq *cq, int napi_budget)
406 407 408
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
	struct mlx4_cq *mcq = &cq->mcq;
409
	struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->type][cq->ring];
410
	struct mlx4_cqe *cqe;
411
	u16 index;
412
	u16 new_index, ring_index, stamp_index;
413
	u32 txbbs_skipped = 0;
414
	u32 txbbs_stamp = 0;
415 416 417 418
	u32 cons_index = mcq->cons_index;
	int size = cq->size;
	u32 size_mask = ring->size_mask;
	struct mlx4_cqe *buf = cq->buf;
419 420
	u32 packets = 0;
	u32 bytes = 0;
O
Or Gerlitz 已提交
421
	int factor = priv->cqe_factor;
422
	int done = 0;
423
	int budget = priv->tx_work_limit;
424 425
	u32 last_nr_txbb;
	u32 ring_cons;
426 427

	if (!priv->port_up)
428
		return true;
429

430 431
	netdev_txq_bql_complete_prefetchw(ring->tx_queue);

432
	index = cons_index & size_mask;
433
	cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
434 435 436
	last_nr_txbb = ACCESS_ONCE(ring->last_nr_txbb);
	ring_cons = ACCESS_ONCE(ring->cons);
	ring_index = ring_cons & size_mask;
437
	stamp_index = ring_index;
438 439 440

	/* Process all completed CQEs */
	while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
441
			cons_index & size) && (done < budget)) {
442 443 444 445
		/*
		 * make sure we read the CQE after we read the
		 * ownership bit
		 */
446
		dma_rmb();
447

448 449 450 451 452 453 454 455 456
		if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
			     MLX4_CQE_OPCODE_ERROR)) {
			struct mlx4_err_cqe *cqe_err = (struct mlx4_err_cqe *)cqe;

			en_err(priv, "CQE error - vendor syndrome: 0x%x syndrome: 0x%x\n",
			       cqe_err->vendor_err_syndrome,
			       cqe_err->syndrome);
		}

457 458 459
		/* Skip over last polled CQE */
		new_index = be16_to_cpu(cqe->wqe_index) & size_mask;

460
		do {
461 462
			u64 timestamp = 0;

463 464
			txbbs_skipped += last_nr_txbb;
			ring_index = (ring_index + last_nr_txbb) & size_mask;
465 466

			if (unlikely(ring->tx_info[ring_index].ts_requested))
467 468
				timestamp = mlx4_en_get_cqe_ts(cqe);

469
			/* free next descriptor */
470
			last_nr_txbb = ring->free_tx_desc(
471
					priv, ring, ring_index,
472
					!!((ring_cons + txbbs_skipped) &
473
					ring->size), timestamp, napi_budget);
474 475

			mlx4_en_stamp_wqe(priv, ring, stamp_index,
476
					  !!((ring_cons + txbbs_stamp) &
477 478 479
						ring->size));
			stamp_index = ring_index;
			txbbs_stamp = txbbs_skipped;
480 481
			packets++;
			bytes += ring->tx_info[ring_index].nr_bytes;
482
		} while ((++done < budget) && (ring_index != new_index));
483 484 485

		++cons_index;
		index = cons_index & size_mask;
486
		cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
487
	}
488 489 490 491 492 493


	/*
	 * To prevent CQ overflow we first update CQ consumer and only then
	 * the ring consumer.
	 */
494
	mcq->cons_index = cons_index;
495 496
	mlx4_cq_set_ci(mcq);
	wmb();
497 498 499 500 501

	/* we want to dirty this cache line once */
	ACCESS_ONCE(ring->last_nr_txbb) = last_nr_txbb;
	ACCESS_ONCE(ring->cons) = ring_cons + txbbs_skipped;

502 503 504
	if (ring->free_tx_desc == mlx4_en_recycle_tx_desc)
		return done < budget;

505
	netdev_tx_completed_queue(ring->tx_queue, packets, bytes);
506

507
	/* Wakeup Tx queue if this stopped, and ring is not full.
508
	 */
509 510
	if (netif_tx_queue_stopped(ring->tx_queue) &&
	    !mlx4_en_is_tx_ring_full(ring)) {
511
		netif_tx_wake_queue(ring->tx_queue);
512
		ring->wake_queue++;
513
	}
514
	return done < budget;
515 516 517 518 519 520 521
}

void mlx4_en_tx_irq(struct mlx4_cq *mcq)
{
	struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
	struct mlx4_en_priv *priv = netdev_priv(cq->dev);

E
Eric Dumazet 已提交
522 523
	if (likely(priv->port_up))
		napi_schedule_irqoff(&cq->napi);
524 525
	else
		mlx4_en_arm_cq(priv, cq);
526 527
}

528 529 530 531 532 533
/* TX CQ polling - called by NAPI */
int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget)
{
	struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
	struct net_device *dev = cq->dev;
	struct mlx4_en_priv *priv = netdev_priv(dev);
534
	int clean_complete;
535

536
	clean_complete = mlx4_en_process_tx_cq(dev, cq, budget);
537 538
	if (!clean_complete)
		return budget;
539

540 541 542 543
	napi_complete(napi);
	mlx4_en_arm_cq(priv, cq);

	return 0;
544
}
545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573

static struct mlx4_en_tx_desc *mlx4_en_bounce_to_desc(struct mlx4_en_priv *priv,
						      struct mlx4_en_tx_ring *ring,
						      u32 index,
						      unsigned int desc_size)
{
	u32 copy = (ring->size - index) * TXBB_SIZE;
	int i;

	for (i = desc_size - copy - 4; i >= 0; i -= 4) {
		if ((i & (TXBB_SIZE - 1)) == 0)
			wmb();

		*((u32 *) (ring->buf + i)) =
			*((u32 *) (ring->bounce_buf + copy + i));
	}

	for (i = copy - 4; i >= 4 ; i -= 4) {
		if ((i & (TXBB_SIZE - 1)) == 0)
			wmb();

		*((u32 *) (ring->buf + index * TXBB_SIZE + i)) =
			*((u32 *) (ring->bounce_buf + i));
	}

	/* Return real descriptor location */
	return ring->buf + index * TXBB_SIZE;
}

574 575 576 577 578 579 580
/* Decide if skb can be inlined in tx descriptor to avoid dma mapping
 *
 * It seems strange we do not simply use skb_copy_bits().
 * This would allow to inline all skbs iff skb->len <= inline_thold
 *
 * Note that caller already checked skb was not a gso packet
 */
581
static bool is_inline(int inline_thold, const struct sk_buff *skb,
582
		      const struct skb_shared_info *shinfo,
583
		      void **pfrag)
584 585 586
{
	void *ptr;

587 588
	if (skb->len > inline_thold || !inline_thold)
		return false;
589

590 591 592 593 594 595
	if (shinfo->nr_frags == 1) {
		ptr = skb_frag_address_safe(&shinfo->frags[0]);
		if (unlikely(!ptr))
			return false;
		*pfrag = ptr;
		return true;
596
	}
597 598 599
	if (shinfo->nr_frags)
		return false;
	return true;
600 601
}

602
static int inline_size(const struct sk_buff *skb)
603 604 605 606 607 608 609 610 611 612
{
	if (skb->len + CTRL_SIZE + sizeof(struct mlx4_wqe_inline_seg)
	    <= MLX4_INLINE_ALIGN)
		return ALIGN(skb->len + CTRL_SIZE +
			     sizeof(struct mlx4_wqe_inline_seg), 16);
	else
		return ALIGN(skb->len + CTRL_SIZE + 2 *
			     sizeof(struct mlx4_wqe_inline_seg), 16);
}

613
static int get_real_size(const struct sk_buff *skb,
614
			 const struct skb_shared_info *shinfo,
615
			 struct net_device *dev,
616 617 618
			 int *lso_header_size,
			 bool *inline_ok,
			 void **pfrag)
619 620 621 622
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
	int real_size;

623
	if (shinfo->gso_size) {
624
		*inline_ok = false;
625 626 627 628
		if (skb->encapsulation)
			*lso_header_size = (skb_inner_transport_header(skb) - skb->data) + inner_tcp_hdrlen(skb);
		else
			*lso_header_size = skb_transport_offset(skb) + tcp_hdrlen(skb);
629
		real_size = CTRL_SIZE + shinfo->nr_frags * DS_SIZE +
630 631 632 633 634 635 636 637
			ALIGN(*lso_header_size + 4, DS_SIZE);
		if (unlikely(*lso_header_size != skb_headlen(skb))) {
			/* We add a segment for the skb linear buffer only if
			 * it contains data */
			if (*lso_header_size < skb_headlen(skb))
				real_size += DS_SIZE;
			else {
				if (netif_msg_tx_err(priv))
638
					en_warn(priv, "Non-linear headers\n");
639 640 641 642 643
				return 0;
			}
		}
	} else {
		*lso_header_size = 0;
644 645 646 647
		*inline_ok = is_inline(priv->prof->inline_thold, skb,
				       shinfo, pfrag);

		if (*inline_ok)
648
			real_size = inline_size(skb);
649 650 651
		else
			real_size = CTRL_SIZE +
				    (shinfo->nr_frags + 1) * DS_SIZE;
652 653 654 655 656
	}

	return real_size;
}

657 658
static void build_inline_wqe(struct mlx4_en_tx_desc *tx_desc,
			     const struct sk_buff *skb,
659
			     const struct skb_shared_info *shinfo,
660
			     void *fragptr)
661 662 663
{
	struct mlx4_wqe_inline_seg *inl = &tx_desc->inl;
	int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - sizeof *inl;
664
	unsigned int hlen = skb_headlen(skb);
665 666

	if (skb->len <= spc) {
667 668 669 670 671 672 673
		if (likely(skb->len >= MIN_PKT_LEN)) {
			inl->byte_count = cpu_to_be32(1 << 31 | skb->len);
		} else {
			inl->byte_count = cpu_to_be32(1 << 31 | MIN_PKT_LEN);
			memset(((void *)(inl + 1)) + skb->len, 0,
			       MIN_PKT_LEN - skb->len);
		}
674
		skb_copy_from_linear_data(skb, inl + 1, hlen);
675
		if (shinfo->nr_frags)
676
			memcpy(((void *)(inl + 1)) + hlen, fragptr,
677
			       skb_frag_size(&shinfo->frags[0]));
678 679 680

	} else {
		inl->byte_count = cpu_to_be32(1 << 31 | spc);
681 682 683 684 685 686
		if (hlen <= spc) {
			skb_copy_from_linear_data(skb, inl + 1, hlen);
			if (hlen < spc) {
				memcpy(((void *)(inl + 1)) + hlen,
				       fragptr, spc - hlen);
				fragptr +=  spc - hlen;
687 688 689 690 691 692 693
			}
			inl = (void *) (inl + 1) + spc;
			memcpy(((void *)(inl + 1)), fragptr, skb->len - spc);
		} else {
			skb_copy_from_linear_data(skb, inl + 1, spc);
			inl = (void *) (inl + 1) + spc;
			skb_copy_from_linear_data_offset(skb, spc, inl + 1,
694
							 hlen - spc);
695
			if (shinfo->nr_frags)
696
				memcpy(((void *)(inl + 1)) + hlen - spc,
697 698
				       fragptr,
				       skb_frag_size(&shinfo->frags[0]));
699 700
		}

701
		dma_wmb();
702 703 704 705
		inl->byte_count = cpu_to_be32(1 << 31 | (skb->len - spc));
	}
}

706
u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb,
707
			 void *accel_priv, select_queue_fallback_t fallback)
708
{
709
	struct mlx4_en_priv *priv = netdev_priv(dev);
710
	u16 rings_p_up = priv->num_tx_rings_p_up;
711
	u8 up = 0;
712

713 714 715
	if (dev->num_tc)
		return skb_tx_hash(dev, skb);

716 717
	if (skb_vlan_tag_present(skb))
		up = skb_vlan_tag_get(skb) >> VLAN_PRIO_SHIFT;
Y
Yevgeny Petrilin 已提交
718

719
	return fallback(dev, skb) % rings_p_up + up * rings_p_up;
720 721
}

722 723
static void mlx4_bf_copy(void __iomem *dst, const void *src,
			 unsigned int bytecnt)
724 725 726 727
{
	__iowrite64_copy(dst, src, bytecnt / 8);
}

728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782
void mlx4_en_xmit_doorbell(struct mlx4_en_tx_ring *ring)
{
	wmb();
	/* Since there is no iowrite*_native() that writes the
	 * value as is, without byteswapping - using the one
	 * the doesn't do byteswapping in the relevant arch
	 * endianness.
	 */
#if defined(__LITTLE_ENDIAN)
	iowrite32(
#else
	iowrite32be(
#endif
		  ring->doorbell_qpn,
		  ring->bf.uar->map + MLX4_SEND_DOORBELL);
}

static void mlx4_en_tx_write_desc(struct mlx4_en_tx_ring *ring,
				  struct mlx4_en_tx_desc *tx_desc,
				  union mlx4_wqe_qpn_vlan qpn_vlan,
				  int desc_size, int bf_index,
				  __be32 op_own, bool bf_ok,
				  bool send_doorbell)
{
	tx_desc->ctrl.qpn_vlan = qpn_vlan;

	if (bf_ok) {
		op_own |= htonl((bf_index & 0xffff) << 8);
		/* Ensure new descriptor hits memory
		 * before setting ownership of this descriptor to HW
		 */
		dma_wmb();
		tx_desc->ctrl.owner_opcode = op_own;

		wmb();

		mlx4_bf_copy(ring->bf.reg + ring->bf.offset, &tx_desc->ctrl,
			     desc_size);

		wmb();

		ring->bf.offset ^= ring->bf.buf_size;
	} else {
		/* Ensure new descriptor hits memory
		 * before setting ownership of this descriptor to HW
		 */
		dma_wmb();
		tx_desc->ctrl.owner_opcode = op_own;
		if (send_doorbell)
			mlx4_en_xmit_doorbell(ring);
		else
			ring->xmit_more++;
	}
}

783
netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
784
{
785
	struct skb_shared_info *shinfo = skb_shinfo(skb);
786
	struct mlx4_en_priv *priv = netdev_priv(dev);
787
	union mlx4_wqe_qpn_vlan	qpn_vlan = {};
788
	struct device *ddev = priv->ddev;
789 790 791 792 793 794 795 796
	struct mlx4_en_tx_ring *ring;
	struct mlx4_en_tx_desc *tx_desc;
	struct mlx4_wqe_data_seg *data;
	struct mlx4_en_tx_info *tx_info;
	int tx_ind = 0;
	int nr_txbb;
	int desc_size;
	int real_size;
797
	u32 index, bf_index;
798
	__be32 op_own;
799
	u16 vlan_proto = 0;
800
	int i_frag;
801
	int lso_header_size;
802
	void *fragptr = NULL;
803
	bool bounce = false;
804
	bool send_doorbell;
E
Eric Dumazet 已提交
805
	bool stop_queue;
806
	bool inline_ok;
807
	u32 ring_cons;
808
	bool bf_ok;
809

810
	tx_ind = skb_get_queue_mapping(skb);
811
	ring = priv->tx_ring[TX][tx_ind];
812

E
Eric Dumazet 已提交
813 814 815
	if (!priv->port_up)
		goto tx_drop;

816 817 818
	/* fetch ring->cons far ahead before needing it to avoid stall */
	ring_cons = ACCESS_ONCE(ring->cons);

819 820
	real_size = get_real_size(skb, shinfo, dev, &lso_header_size,
				  &inline_ok, &fragptr);
821
	if (unlikely(!real_size))
822
		goto tx_drop_count;
823

L
Lucas De Marchi 已提交
824
	/* Align descriptor to TXBB size */
825 826 827 828
	desc_size = ALIGN(real_size, TXBB_SIZE);
	nr_txbb = desc_size / TXBB_SIZE;
	if (unlikely(nr_txbb > MAX_DESC_TXBBS)) {
		if (netif_msg_tx_err(priv))
829
			en_warn(priv, "Oversized header or SG list\n");
830
		goto tx_drop_count;
831 832
	}

833
	bf_ok = ring->bf_enabled;
834
	if (skb_vlan_tag_present(skb)) {
835
		qpn_vlan.vlan_tag = cpu_to_be16(skb_vlan_tag_get(skb));
836
		vlan_proto = be16_to_cpu(skb->vlan_proto);
837 838 839 840 841 842 843
		if (vlan_proto == ETH_P_8021AD)
			qpn_vlan.ins_vlan = MLX4_WQE_CTRL_INS_SVLAN;
		else if (vlan_proto == ETH_P_8021Q)
			qpn_vlan.ins_vlan = MLX4_WQE_CTRL_INS_CVLAN;
		else
			qpn_vlan.ins_vlan = 0;
		bf_ok = false;
844
	}
845

846
	netdev_txq_bql_enqueue_prefetchw(ring->tx_queue);
847

848 849
	/* Track current inflight packets for performance analysis */
	AVG_PERF_COUNTER(priv->pstats.inflight_avg,
850
			 (u32)(ring->prod - ring_cons - 1));
851 852 853

	/* Packet is good - grab an index and transmit it */
	index = ring->prod & ring->size_mask;
854
	bf_index = ring->prod;
855 856 857 858 859

	/* See if we have enough space for whole descriptor TXBB for setting
	 * SW ownership on next descriptor; if not, use a bounce buffer. */
	if (likely(index + nr_txbb <= ring->size))
		tx_desc = ring->buf + index * TXBB_SIZE;
860
	else {
861
		tx_desc = (struct mlx4_en_tx_desc *) ring->bounce_buf;
862
		bounce = true;
863
		bf_ok = false;
864
	}
865 866 867 868 869 870

	/* Save skb in tx_info ring */
	tx_info = &ring->tx_info[index];
	tx_info->skb = skb;
	tx_info->nr_txbb = nr_txbb;

871
	data = &tx_desc->data;
872 873 874 875 876 877 878
	if (lso_header_size)
		data = ((void *)&tx_desc->lso + ALIGN(lso_header_size + 4,
						      DS_SIZE));

	/* valid only for none inline segments */
	tx_info->data_offset = (void *)data - (void *)tx_desc;

879 880
	tx_info->inl = inline_ok;

881
	tx_info->linear = (lso_header_size < skb_headlen(skb) &&
882
			   !inline_ok) ? 1 : 0;
883

884
	tx_info->nr_maps = shinfo->nr_frags + tx_info->linear;
885
	data += tx_info->nr_maps - 1;
886

887
	if (!tx_info->inl) {
888 889 890
		dma_addr_t dma = 0;
		u32 byte_count = 0;

891
		/* Map fragments if any */
892
		for (i_frag = shinfo->nr_frags - 1; i_frag >= 0; i_frag--) {
893
			const struct skb_frag_struct *frag;
894 895

			frag = &shinfo->frags[i_frag];
896
			byte_count = skb_frag_size(frag);
897
			dma = skb_frag_dma_map(ddev, frag,
898
					       0, byte_count,
899 900 901 902 903
					       DMA_TO_DEVICE);
			if (dma_mapping_error(ddev, dma))
				goto tx_drop_unmap;

			data->addr = cpu_to_be64(dma);
904
			data->lkey = ring->mr_key;
905
			dma_wmb();
906
			data->byte_count = cpu_to_be32(byte_count);
907 908 909
			--data;
		}

910
		/* Map linear part if needed */
911
		if (tx_info->linear) {
912
			byte_count = skb_headlen(skb) - lso_header_size;
913

914 915 916 917 918 919 920
			dma = dma_map_single(ddev, skb->data +
					     lso_header_size, byte_count,
					     PCI_DMA_TODEVICE);
			if (dma_mapping_error(ddev, dma))
				goto tx_drop_unmap;

			data->addr = cpu_to_be64(dma);
921
			data->lkey = ring->mr_key;
922
			dma_wmb();
923 924
			data->byte_count = cpu_to_be32(byte_count);
		}
925 926 927
		/* tx completion can avoid cache line miss for common cases */
		tx_info->map0_dma = dma;
		tx_info->map0_byte_count = byte_count;
928 929
	}

930 931 932 933
	/*
	 * For timestamping add flag to skb_shinfo and
	 * set flag for further reference
	 */
934
	tx_info->ts_requested = 0;
935 936 937
	if (unlikely(ring->hwtstamp_tx_type == HWTSTAMP_TX_ON &&
		     shinfo->tx_flags & SKBTX_HW_TSTAMP)) {
		shinfo->tx_flags |= SKBTX_IN_PROGRESS;
938 939 940
		tx_info->ts_requested = 1;
	}

941 942
	/* Prepare ctrl segement apart opcode+ownership, which depends on
	 * whether LSO is used */
A
Amir Vadai 已提交
943
	tx_desc->ctrl.srcrb_flags = priv->ctrl_flags;
944
	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
945 946 947 948 949
		if (!skb->encapsulation)
			tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM |
								 MLX4_WQE_CTRL_TCP_UDP_CSUM);
		else
			tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM);
950
		ring->tx_csum++;
951 952
	}

953
	if (priv->flags & MLX4_EN_FLAG_ENABLE_HW_LOOPBACK) {
954 955
		struct ethhdr *ethh;

956 957 958 959 960 961 962 963
		/* Copy dst mac address to wqe. This allows loopback in eSwitch,
		 * so that VFs and PF can communicate with each other
		 */
		ethh = (struct ethhdr *)skb->data;
		tx_desc->ctrl.srcrb_flags16[0] = get_unaligned((__be16 *)ethh->h_dest);
		tx_desc->ctrl.imm = get_unaligned((__be32 *)(ethh->h_dest + 2));
	}

964 965
	/* Handle LSO (TSO) packets */
	if (lso_header_size) {
966 967
		int i;

968 969 970 971 972 973 974
		/* Mark opcode as LSO */
		op_own = cpu_to_be32(MLX4_OPCODE_LSO | (1 << 6)) |
			((ring->prod & ring->size) ?
				cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0);

		/* Fill in the LSO prefix */
		tx_desc->lso.mss_hdr_size = cpu_to_be32(
975
			shinfo->gso_size << 16 | lso_header_size);
976 977 978 979 980

		/* Copy headers;
		 * note that we already verified that it is linear */
		memcpy(tx_desc->lso.header, skb->data, lso_header_size);

E
Eric Dumazet 已提交
981
		ring->tso_packets++;
982 983 984

		i = ((skb->len - lso_header_size) / shinfo->gso_size) +
			!!((skb->len - lso_header_size) % shinfo->gso_size);
985
		tx_info->nr_bytes = skb->len + (i - 1) * lso_header_size;
986 987 988 989 990 991
		ring->packets += i;
	} else {
		/* Normal (Non LSO) packet */
		op_own = cpu_to_be32(MLX4_OPCODE_SEND) |
			((ring->prod & ring->size) ?
			 cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0);
992
		tx_info->nr_bytes = max_t(unsigned int, skb->len, ETH_ZLEN);
993 994
		ring->packets++;
	}
995 996
	ring->bytes += tx_info->nr_bytes;
	netdev_tx_sent_queue(ring->tx_queue, tx_info->nr_bytes);
997 998
	AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, skb->len);

999
	if (tx_info->inl)
1000
		build_inline_wqe(tx_desc, skb, shinfo, fragptr);
1001

1002
	if (skb->encapsulation) {
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
		union {
			struct iphdr *v4;
			struct ipv6hdr *v6;
			unsigned char *hdr;
		} ip;
		u8 proto;

		ip.hdr = skb_inner_network_header(skb);
		proto = (ip.v4->version == 4) ? ip.v4->protocol :
						ip.v6->nexthdr;

		if (proto == IPPROTO_TCP || proto == IPPROTO_UDP)
1015 1016 1017 1018 1019
			op_own |= cpu_to_be32(MLX4_WQE_CTRL_IIP | MLX4_WQE_CTRL_ILP);
		else
			op_own |= cpu_to_be32(MLX4_WQE_CTRL_IIP);
	}

1020 1021 1022
	ring->prod += nr_txbb;

	/* If we used a bounce buffer then copy descriptor back into place */
1023
	if (unlikely(bounce))
1024 1025
		tx_desc = mlx4_en_bounce_to_desc(priv, ring, index, desc_size);

1026 1027
	skb_tx_timestamp(skb);

E
Eric Dumazet 已提交
1028
	/* Check available TXBBs And 2K spare for prefetch */
1029
	stop_queue = mlx4_en_is_tx_ring_full(ring);
E
Eric Dumazet 已提交
1030 1031 1032 1033
	if (unlikely(stop_queue)) {
		netif_tx_stop_queue(ring->tx_queue);
		ring->queue_stopped++;
	}
1034 1035
	send_doorbell = !skb->xmit_more || netif_xmit_stopped(ring->tx_queue);

1036 1037
	real_size = (real_size / 16) & 0x3f;

1038
	bf_ok &= desc_size <= MAX_BF && send_doorbell;
1039

1040 1041 1042 1043
	if (bf_ok)
		qpn_vlan.bf_qpn = ring->doorbell_qpn | cpu_to_be32(real_size);
	else
		qpn_vlan.fence_size = real_size;
1044

1045 1046
	mlx4_en_tx_write_desc(ring, tx_desc, qpn_vlan, desc_size, bf_index,
			      op_own, bf_ok, send_doorbell);
1047

E
Eric Dumazet 已提交
1048 1049 1050 1051 1052 1053 1054 1055 1056 1057
	if (unlikely(stop_queue)) {
		/* If queue was emptied after the if (stop_queue) , and before
		 * the netif_tx_stop_queue() - need to wake the queue,
		 * or else it will remain stopped forever.
		 * Need a memory barrier to make sure ring->cons was not
		 * updated before queue was stopped.
		 */
		smp_rmb();

		ring_cons = ACCESS_ONCE(ring->cons);
1058
		if (unlikely(!mlx4_en_is_tx_ring_full(ring))) {
E
Eric Dumazet 已提交
1059 1060 1061 1062
			netif_tx_wake_queue(ring->tx_queue);
			ring->wake_queue++;
		}
	}
1063
	return NETDEV_TX_OK;
1064

1065 1066 1067
tx_drop_unmap:
	en_err(priv, "DMA mapping error\n");

1068 1069
	while (++i_frag < shinfo->nr_frags) {
		++data;
1070 1071 1072 1073 1074
		dma_unmap_page(ddev, (dma_addr_t) be64_to_cpu(data->addr),
			       be32_to_cpu(data->byte_count),
			       PCI_DMA_TODEVICE);
	}

1075 1076
tx_drop_count:
	ring->tx_dropped++;
1077 1078 1079
tx_drop:
	dev_kfree_skb_any(skb);
	return NETDEV_TX_OK;
1080 1081
}

1082 1083
netdev_tx_t mlx4_en_xmit_frame(struct mlx4_en_rx_ring *rx_ring,
			       struct mlx4_en_rx_alloc *frame,
1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
			       struct net_device *dev, unsigned int length,
			       int tx_ind, int *doorbell_pending)
{
	struct mlx4_en_priv *priv = netdev_priv(dev);
	union mlx4_wqe_qpn_vlan	qpn_vlan = {};
	struct mlx4_en_tx_ring *ring;
	struct mlx4_en_tx_desc *tx_desc;
	struct mlx4_wqe_data_seg *data;
	struct mlx4_en_tx_info *tx_info;
	int index, bf_index;
	bool send_doorbell;
	int nr_txbb = 1;
	bool stop_queue;
	dma_addr_t dma;
	int real_size;
	__be32 op_own;
	u32 ring_cons;
	bool bf_ok;

	BUILD_BUG_ON_MSG(ALIGN(CTRL_SIZE + DS_SIZE, TXBB_SIZE) != TXBB_SIZE,
			 "mlx4_en_xmit_frame requires minimum size tx desc");

1106
	ring = priv->tx_ring[TX_XDP][tx_ind];
1107 1108 1109 1110 1111

	if (!priv->port_up)
		goto tx_drop;

	if (mlx4_en_is_tx_ring_full(ring))
1112
		goto tx_drop_count;
1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157

	/* fetch ring->cons far ahead before needing it to avoid stall */
	ring_cons = READ_ONCE(ring->cons);

	index = ring->prod & ring->size_mask;
	tx_info = &ring->tx_info[index];

	bf_ok = ring->bf_enabled;

	/* Track current inflight packets for performance analysis */
	AVG_PERF_COUNTER(priv->pstats.inflight_avg,
			 (u32)(ring->prod - ring_cons - 1));

	bf_index = ring->prod;
	tx_desc = ring->buf + index * TXBB_SIZE;
	data = &tx_desc->data;

	dma = frame->dma;

	tx_info->page = frame->page;
	frame->page = NULL;
	tx_info->map0_dma = dma;
	tx_info->map0_byte_count = length;
	tx_info->nr_txbb = nr_txbb;
	tx_info->nr_bytes = max_t(unsigned int, length, ETH_ZLEN);
	tx_info->data_offset = (void *)data - (void *)tx_desc;
	tx_info->ts_requested = 0;
	tx_info->nr_maps = 1;
	tx_info->linear = 1;
	tx_info->inl = 0;

	dma_sync_single_for_device(priv->ddev, dma, length, PCI_DMA_TODEVICE);

	data->addr = cpu_to_be64(dma);
	data->lkey = ring->mr_key;
	dma_wmb();
	data->byte_count = cpu_to_be32(length);

	/* tx completion can avoid cache line miss for common cases */
	tx_desc->ctrl.srcrb_flags = priv->ctrl_flags;

	op_own = cpu_to_be32(MLX4_OPCODE_SEND) |
		((ring->prod & ring->size) ?
		 cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0);

1158
	rx_ring->xdp_tx++;
1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
	AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, length);

	ring->prod += nr_txbb;

	stop_queue = mlx4_en_is_tx_ring_full(ring);
	send_doorbell = stop_queue ||
				*doorbell_pending > MLX4_EN_DOORBELL_BUDGET;
	bf_ok &= send_doorbell;

	real_size = ((CTRL_SIZE + nr_txbb * DS_SIZE) / 16) & 0x3f;

	if (bf_ok)
		qpn_vlan.bf_qpn = ring->doorbell_qpn | cpu_to_be32(real_size);
	else
		qpn_vlan.fence_size = real_size;

	mlx4_en_tx_write_desc(ring, tx_desc, qpn_vlan, TXBB_SIZE, bf_index,
			      op_own, bf_ok, send_doorbell);
	*doorbell_pending = send_doorbell ? 0 : *doorbell_pending + 1;

	return NETDEV_TX_OK;

1181
tx_drop_count:
1182
	rx_ring->xdp_tx_full++;
1183
tx_drop:
1184 1185
	return NETDEV_TX_BUSY;
}