sge.c 92.3 KB
Newer Older
1 2 3
/*
 * This file is part of the Chelsio T4 Ethernet driver for Linux.
 *
4
 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/if_vlan.h>
#include <linux/ip.h>
#include <linux/dma-mapping.h>
#include <linux/jiffies.h>
42
#include <linux/prefetch.h>
43
#include <linux/export.h>
44 45
#include <net/ipv6.h>
#include <net/tcp.h>
46 47 48
#ifdef CONFIG_NET_RX_BUSY_POLL
#include <net/busy_poll.h>
#endif /* CONFIG_NET_RX_BUSY_POLL */
V
Varun Prakash 已提交
49 50 51
#ifdef CONFIG_CHELSIO_T4_FCOE
#include <scsi/fc/fc_fcoe.h>
#endif /* CONFIG_CHELSIO_T4_FCOE */
52 53
#include "cxgb4.h"
#include "t4_regs.h"
54
#include "t4_values.h"
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
#include "t4_msg.h"
#include "t4fw_api.h"

/*
 * Rx buffer size.  We use largish buffers if possible but settle for single
 * pages under memory shortage.
 */
#if PAGE_SHIFT >= 16
# define FL_PG_ORDER 0
#else
# define FL_PG_ORDER (16 - PAGE_SHIFT)
#endif

/* RX_PULL_LEN should be <= RX_COPY_THRES */
#define RX_COPY_THRES    256
#define RX_PULL_LEN      128

/*
 * Main body length for sk_buffs used for Rx Ethernet packets with fragments.
 * Should be >= RX_PULL_LEN but possibly bigger to give pskb_may_pull some room.
 */
#define RX_PKT_SKB_LEN   512

/*
 * Max number of Tx descriptors we clean up at a time.  Should be modest as
 * freeing skbs isn't cheap and it happens while holding locks.  We just need
 * to free packets faster than they arrive, we eventually catch up and keep
 * the amortized cost reasonable.  Must be >= 2 * TXQ_STOP_THRES.
 */
#define MAX_TX_RECLAIM 16

/*
 * Max number of Rx buffers we replenish at a time.  Again keep this modest,
 * allocating buffers isn't cheap either.
 */
#define MAX_RX_REFILL 16U

/*
 * Period of the Rx queue check timer.  This timer is infrequent as it has
 * something to do only when the system experiences severe memory shortage.
 */
#define RX_QCHECK_PERIOD (HZ / 2)

/*
 * Period of the Tx queue check timer.
 */
#define TX_QCHECK_PERIOD (HZ / 2)

/*
 * Max number of Tx descriptors to be reclaimed by the Tx timer.
 */
#define MAX_TIMER_TX_RECLAIM 100

/*
 * Timer index used when backing off due to memory shortage.
 */
#define NOMEM_TMR_IDX (SGE_NTIMERS - 1)

/*
 * Suspend an Ethernet Tx queue with fewer available descriptors than this.
 * This is the same as calc_tx_descs() for a TSO packet with
 * nr_frags == MAX_SKB_FRAGS.
 */
#define ETHTXQ_STOP_THRES \
	(1 + DIV_ROUND_UP((3 * MAX_SKB_FRAGS) / 2 + (MAX_SKB_FRAGS & 1), 8))

/*
 * Suspension threshold for non-Ethernet Tx queues.  We require enough room
 * for a full sized WR.
 */
#define TXQ_STOP_THRES (SGE_MAX_WR_LEN / sizeof(struct tx_desc))

/*
 * Max Tx descriptor space we allow for an Ethernet packet to be inlined
 * into a WR.
 */
131
#define MAX_IMM_TX_PKT_LEN 256
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148

/*
 * Max size of a WR sent through a control Tx queue.
 */
#define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN

struct tx_sw_desc {                /* SW state per Tx descriptor */
	struct sk_buff *skb;
	struct ulptx_sgl *sgl;
};

struct rx_sw_desc {                /* SW state per Rx descriptor */
	struct page *page;
	dma_addr_t dma_addr;
};

/*
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
 * Rx buffer sizes for "useskbs" Free List buffers (one ingress packet pe skb
 * buffer).  We currently only support two sizes for 1500- and 9000-byte MTUs.
 * We could easily support more but there doesn't seem to be much need for
 * that ...
 */
#define FL_MTU_SMALL 1500
#define FL_MTU_LARGE 9000

static inline unsigned int fl_mtu_bufsize(struct adapter *adapter,
					  unsigned int mtu)
{
	struct sge *s = &adapter->sge;

	return ALIGN(s->pktshift + ETH_HLEN + VLAN_HLEN + mtu, s->fl_align);
}

#define FL_MTU_SMALL_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_SMALL)
#define FL_MTU_LARGE_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_LARGE)

/*
 * Bits 0..3 of rx_sw_desc.dma_addr have special meaning.  The hardware uses
 * these to specify the buffer size as an index into the SGE Free List Buffer
 * Size register array.  We also use bit 4, when the buffer has been unmapped
 * for DMA, but this is of course never sent to the hardware and is only used
 * to prevent double unmappings.  All of the above requires that the Free List
 * Buffers which we allocate have the bottom 5 bits free (0) -- i.e. are
 * 32-byte or or a power of 2 greater in alignment.  Since the SGE's minimal
 * Free List Buffer alignment is 32 bytes, this works out for us ...
177 178
 */
enum {
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
	RX_BUF_FLAGS     = 0x1f,   /* bottom five bits are special */
	RX_BUF_SIZE      = 0x0f,   /* bottom three bits are for buf sizes */
	RX_UNMAPPED_BUF  = 0x10,   /* buffer is not mapped */

	/*
	 * XXX We shouldn't depend on being able to use these indices.
	 * XXX Especially when some other Master PF has initialized the
	 * XXX adapter or we use the Firmware Configuration File.  We
	 * XXX should really search through the Host Buffer Size register
	 * XXX array for the appropriately sized buffer indices.
	 */
	RX_SMALL_PG_BUF  = 0x0,   /* small (PAGE_SIZE) page buffer */
	RX_LARGE_PG_BUF  = 0x1,   /* buffer large (FL_PG_ORDER) page buffer */

	RX_SMALL_MTU_BUF = 0x2,   /* small MTU buffer */
	RX_LARGE_MTU_BUF = 0x3,   /* large MTU buffer */
195 196
};

197 198 199
static int timer_pkt_quota[] = {1, 1, 2, 3, 4, 5};
#define MIN_NAPI_WORK  1

200 201
static inline dma_addr_t get_buf_addr(const struct rx_sw_desc *d)
{
202
	return d->dma_addr & ~(dma_addr_t)RX_BUF_FLAGS;
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
}

static inline bool is_buf_mapped(const struct rx_sw_desc *d)
{
	return !(d->dma_addr & RX_UNMAPPED_BUF);
}

/**
 *	txq_avail - return the number of available slots in a Tx queue
 *	@q: the Tx queue
 *
 *	Returns the number of descriptors in a Tx queue available to write new
 *	packets.
 */
static inline unsigned int txq_avail(const struct sge_txq *q)
{
	return q->size - 1 - q->in_use;
}

/**
 *	fl_cap - return the capacity of a free-buffer list
 *	@fl: the FL
 *
 *	Returns the capacity of a free-buffer list.  The capacity is less than
 *	the size because one descriptor needs to be left unpopulated, otherwise
 *	HW will think the FL is empty.
 */
static inline unsigned int fl_cap(const struct sge_fl *fl)
{
	return fl->size - 8;   /* 1 descriptor = 8 buffers */
}

235 236 237 238 239 240 241 242 243 244 245
/**
 *	fl_starving - return whether a Free List is starving.
 *	@adapter: pointer to the adapter
 *	@fl: the Free List
 *
 *	Tests specified Free List to see whether the number of buffers
 *	available to the hardware has falled below our "starvation"
 *	threshold.
 */
static inline bool fl_starving(const struct adapter *adapter,
			       const struct sge_fl *fl)
246
{
247 248 249
	const struct sge *s = &adapter->sge;

	return fl->avail - fl->pend_cred <= s->fl_starve_thres;
250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
}

static int map_skb(struct device *dev, const struct sk_buff *skb,
		   dma_addr_t *addr)
{
	const skb_frag_t *fp, *end;
	const struct skb_shared_info *si;

	*addr = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE);
	if (dma_mapping_error(dev, *addr))
		goto out_err;

	si = skb_shinfo(skb);
	end = &si->frags[si->nr_frags];

	for (fp = si->frags; fp < end; fp++) {
266 267
		*++addr = skb_frag_dma_map(dev, fp, 0, skb_frag_size(fp),
					   DMA_TO_DEVICE);
268 269 270 271 272 273 274
		if (dma_mapping_error(dev, *addr))
			goto unwind;
	}
	return 0;

unwind:
	while (fp-- > si->frags)
E
Eric Dumazet 已提交
275
		dma_unmap_page(dev, *--addr, skb_frag_size(fp), DMA_TO_DEVICE);
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293

	dma_unmap_single(dev, addr[-1], skb_headlen(skb), DMA_TO_DEVICE);
out_err:
	return -ENOMEM;
}

#ifdef CONFIG_NEED_DMA_MAP_STATE
static void unmap_skb(struct device *dev, const struct sk_buff *skb,
		      const dma_addr_t *addr)
{
	const skb_frag_t *fp, *end;
	const struct skb_shared_info *si;

	dma_unmap_single(dev, *addr++, skb_headlen(skb), DMA_TO_DEVICE);

	si = skb_shinfo(skb);
	end = &si->frags[si->nr_frags];
	for (fp = si->frags; fp < end; fp++)
E
Eric Dumazet 已提交
294
		dma_unmap_page(dev, *addr++, skb_frag_size(fp), DMA_TO_DEVICE);
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
}

/**
 *	deferred_unmap_destructor - unmap a packet when it is freed
 *	@skb: the packet
 *
 *	This is the packet destructor used for Tx packets that need to remain
 *	mapped until they are freed rather than until their Tx descriptors are
 *	freed.
 */
static void deferred_unmap_destructor(struct sk_buff *skb)
{
	unmap_skb(skb->dev->dev.parent, skb, (dma_addr_t *)skb->head);
}
#endif

static void unmap_sgl(struct device *dev, const struct sk_buff *skb,
		      const struct ulptx_sgl *sgl, const struct sge_txq *q)
{
	const struct ulptx_sge_pair *p;
	unsigned int nfrags = skb_shinfo(skb)->nr_frags;

	if (likely(skb_headlen(skb)))
		dma_unmap_single(dev, be64_to_cpu(sgl->addr0), ntohl(sgl->len0),
				 DMA_TO_DEVICE);
	else {
		dma_unmap_page(dev, be64_to_cpu(sgl->addr0), ntohl(sgl->len0),
			       DMA_TO_DEVICE);
		nfrags--;
	}

	/*
	 * the complexity below is because of the possibility of a wrap-around
	 * in the middle of an SGL
	 */
	for (p = sgl->sge; nfrags >= 2; nfrags -= 2) {
		if (likely((u8 *)(p + 1) <= (u8 *)q->stat)) {
unmap:			dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
				       ntohl(p->len[0]), DMA_TO_DEVICE);
			dma_unmap_page(dev, be64_to_cpu(p->addr[1]),
				       ntohl(p->len[1]), DMA_TO_DEVICE);
			p++;
		} else if ((u8 *)p == (u8 *)q->stat) {
			p = (const struct ulptx_sge_pair *)q->desc;
			goto unmap;
		} else if ((u8 *)p + 8 == (u8 *)q->stat) {
			const __be64 *addr = (const __be64 *)q->desc;

			dma_unmap_page(dev, be64_to_cpu(addr[0]),
				       ntohl(p->len[0]), DMA_TO_DEVICE);
			dma_unmap_page(dev, be64_to_cpu(addr[1]),
				       ntohl(p->len[1]), DMA_TO_DEVICE);
			p = (const struct ulptx_sge_pair *)&addr[2];
		} else {
			const __be64 *addr = (const __be64 *)q->desc;

			dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
				       ntohl(p->len[0]), DMA_TO_DEVICE);
			dma_unmap_page(dev, be64_to_cpu(addr[0]),
				       ntohl(p->len[1]), DMA_TO_DEVICE);
			p = (const struct ulptx_sge_pair *)&addr[1];
		}
	}
	if (nfrags) {
		__be64 addr;

		if ((u8 *)p == (u8 *)q->stat)
			p = (const struct ulptx_sge_pair *)q->desc;
		addr = (u8 *)p + 16 <= (u8 *)q->stat ? p->addr[0] :
						       *(const __be64 *)q->desc;
		dma_unmap_page(dev, be64_to_cpu(addr), ntohl(p->len[0]),
			       DMA_TO_DEVICE);
	}
}

/**
 *	free_tx_desc - reclaims Tx descriptors and their buffers
 *	@adapter: the adapter
 *	@q: the Tx queue to reclaim descriptors from
 *	@n: the number of descriptors to reclaim
 *	@unmap: whether the buffers should be unmapped for DMA
 *
 *	Reclaims Tx descriptors from an SGE Tx queue and frees the associated
 *	Tx buffers.  Called with the Tx queue lock held.
 */
static void free_tx_desc(struct adapter *adap, struct sge_txq *q,
			 unsigned int n, bool unmap)
{
	struct tx_sw_desc *d;
	unsigned int cidx = q->cidx;
	struct device *dev = adap->pdev_dev;

	d = &q->sdesc[cidx];
	while (n--) {
		if (d->skb) {                       /* an SGL is present */
			if (unmap)
				unmap_sgl(dev, d->skb, d->sgl, q);
392
			dev_consume_skb_any(d->skb);
393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
			d->skb = NULL;
		}
		++d;
		if (++cidx == q->size) {
			cidx = 0;
			d = q->sdesc;
		}
	}
	q->cidx = cidx;
}

/*
 * Return the number of reclaimable descriptors in a Tx queue.
 */
static inline int reclaimable(const struct sge_txq *q)
{
409
	int hw_cidx = ntohs(ACCESS_ONCE(q->stat->cidx));
410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441
	hw_cidx -= q->cidx;
	return hw_cidx < 0 ? hw_cidx + q->size : hw_cidx;
}

/**
 *	reclaim_completed_tx - reclaims completed Tx descriptors
 *	@adap: the adapter
 *	@q: the Tx queue to reclaim completed descriptors from
 *	@unmap: whether the buffers should be unmapped for DMA
 *
 *	Reclaims Tx descriptors that the SGE has indicated it has processed,
 *	and frees the associated buffers if possible.  Called with the Tx
 *	queue locked.
 */
static inline void reclaim_completed_tx(struct adapter *adap, struct sge_txq *q,
					bool unmap)
{
	int avail = reclaimable(q);

	if (avail) {
		/*
		 * Limit the amount of clean up work we do at a time to keep
		 * the Tx lock hold time O(1).
		 */
		if (avail > MAX_TX_RECLAIM)
			avail = MAX_TX_RECLAIM;

		free_tx_desc(adap, q, avail, unmap);
		q->in_use -= avail;
	}
}

442 443
static inline int get_buf_size(struct adapter *adapter,
			       const struct rx_sw_desc *d)
444
{
445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470
	struct sge *s = &adapter->sge;
	unsigned int rx_buf_size_idx = d->dma_addr & RX_BUF_SIZE;
	int buf_size;

	switch (rx_buf_size_idx) {
	case RX_SMALL_PG_BUF:
		buf_size = PAGE_SIZE;
		break;

	case RX_LARGE_PG_BUF:
		buf_size = PAGE_SIZE << s->fl_pg_order;
		break;

	case RX_SMALL_MTU_BUF:
		buf_size = FL_MTU_SMALL_BUFSIZE(adapter);
		break;

	case RX_LARGE_MTU_BUF:
		buf_size = FL_MTU_LARGE_BUFSIZE(adapter);
		break;

	default:
		BUG_ON(1);
	}

	return buf_size;
471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488
}

/**
 *	free_rx_bufs - free the Rx buffers on an SGE free list
 *	@adap: the adapter
 *	@q: the SGE free list to free buffers from
 *	@n: how many buffers to free
 *
 *	Release the next @n buffers on an SGE free-buffer Rx queue.   The
 *	buffers must be made inaccessible to HW before calling this function.
 */
static void free_rx_bufs(struct adapter *adap, struct sge_fl *q, int n)
{
	while (n--) {
		struct rx_sw_desc *d = &q->sdesc[q->cidx];

		if (is_buf_mapped(d))
			dma_unmap_page(adap->pdev_dev, get_buf_addr(d),
489 490
				       get_buf_size(adap, d),
				       PCI_DMA_FROMDEVICE);
491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
		put_page(d->page);
		d->page = NULL;
		if (++q->cidx == q->size)
			q->cidx = 0;
		q->avail--;
	}
}

/**
 *	unmap_rx_buf - unmap the current Rx buffer on an SGE free list
 *	@adap: the adapter
 *	@q: the SGE free list
 *
 *	Unmap the current buffer on an SGE free-buffer Rx queue.   The
 *	buffer must be made inaccessible to HW before calling this function.
 *
 *	This is similar to @free_rx_bufs above but does not free the buffer.
 *	Do note that the FL still loses any further access to the buffer.
 */
static void unmap_rx_buf(struct adapter *adap, struct sge_fl *q)
{
	struct rx_sw_desc *d = &q->sdesc[q->cidx];

	if (is_buf_mapped(d))
		dma_unmap_page(adap->pdev_dev, get_buf_addr(d),
516
			       get_buf_size(adap, d), PCI_DMA_FROMDEVICE);
517 518 519 520 521 522 523 524 525
	d->page = NULL;
	if (++q->cidx == q->size)
		q->cidx = 0;
	q->avail--;
}

static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q)
{
	if (q->pend_cred >= 8) {
526 527
		u32 val = adap->params.arch.sge_fl_db;

528
		if (is_t4(adap->params.chip))
529
			val |= PIDX_V(q->pend_cred / 8);
530
		else
531
			val |= PIDX_T5_V(q->pend_cred / 8);
532 533 534 535

		/* Make sure all memory writes to the Free List queue are
		 * committed before we tell the hardware about them.
		 */
536
		wmb();
537

538 539 540
		/* If we don't have access to the new User Doorbell (T5+), use
		 * the old doorbell mechanism; otherwise use the new BAR2
		 * mechanism.
541
		 */
542
		if (unlikely(q->bar2_addr == NULL)) {
543 544
			t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
				     val | QID_V(q->cntxt_id));
545
		} else {
546
			writel(val | QID_V(q->bar2_qid),
547
			       q->bar2_addr + SGE_UDB_KDOORBELL);
548 549 550 551 552 553

			/* This Write memory Barrier will force the write to
			 * the User Doorbell area to be flushed.
			 */
			wmb();
		}
554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581
		q->pend_cred &= 7;
	}
}

static inline void set_rx_sw_desc(struct rx_sw_desc *sd, struct page *pg,
				  dma_addr_t mapping)
{
	sd->page = pg;
	sd->dma_addr = mapping;      /* includes size low bits */
}

/**
 *	refill_fl - refill an SGE Rx buffer ring
 *	@adap: the adapter
 *	@q: the ring to refill
 *	@n: the number of new buffers to allocate
 *	@gfp: the gfp flags for the allocations
 *
 *	(Re)populate an SGE free-buffer queue with up to @n new packet buffers,
 *	allocated with the supplied gfp flags.  The caller must assure that
 *	@n does not exceed the queue's capacity.  If afterwards the queue is
 *	found critically low mark it as starving in the bitmap of starving FLs.
 *
 *	Returns the number of buffers allocated.
 */
static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n,
			      gfp_t gfp)
{
582
	struct sge *s = &adap->sge;
583 584 585 586 587
	struct page *pg;
	dma_addr_t mapping;
	unsigned int cred = q->avail;
	__be64 *d = &q->desc[q->pidx];
	struct rx_sw_desc *sd = &q->sdesc[q->pidx];
588
	int node;
589

590 591 592 593 594
#ifdef CONFIG_DEBUG_FS
	if (test_bit(q->cntxt_id - adap->sge.egr_start, adap->sge.blocked_fl))
		goto out;
#endif

595
	gfp |= __GFP_NOWARN;
596
	node = dev_to_node(adap->pdev_dev);
597

598 599 600
	if (s->fl_pg_order == 0)
		goto alloc_small_pages;

601 602 603 604
	/*
	 * Prefer large buffers
	 */
	while (n) {
605
		pg = alloc_pages_node(node, gfp | __GFP_COMP, s->fl_pg_order);
606 607 608 609 610 611
		if (unlikely(!pg)) {
			q->large_alloc_failed++;
			break;       /* fall back to single pages */
		}

		mapping = dma_map_page(adap->pdev_dev, pg, 0,
612
				       PAGE_SIZE << s->fl_pg_order,
613 614
				       PCI_DMA_FROMDEVICE);
		if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) {
615
			__free_pages(pg, s->fl_pg_order);
616
			q->mapping_err++;
617 618
			goto out;   /* do not try small pages for this error */
		}
619
		mapping |= RX_LARGE_PG_BUF;
620 621 622 623 624 625 626 627 628 629 630 631 632 633
		*d++ = cpu_to_be64(mapping);

		set_rx_sw_desc(sd, pg, mapping);
		sd++;

		q->avail++;
		if (++q->pidx == q->size) {
			q->pidx = 0;
			sd = q->sdesc;
			d = q->desc;
		}
		n--;
	}

634
alloc_small_pages:
635
	while (n--) {
636
		pg = alloc_pages_node(node, gfp, 0);
637 638 639 640 641 642 643 644
		if (unlikely(!pg)) {
			q->alloc_failed++;
			break;
		}

		mapping = dma_map_page(adap->pdev_dev, pg, 0, PAGE_SIZE,
				       PCI_DMA_FROMDEVICE);
		if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) {
645
			put_page(pg);
646
			q->mapping_err++;
647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665
			goto out;
		}
		*d++ = cpu_to_be64(mapping);

		set_rx_sw_desc(sd, pg, mapping);
		sd++;

		q->avail++;
		if (++q->pidx == q->size) {
			q->pidx = 0;
			sd = q->sdesc;
			d = q->desc;
		}
	}

out:	cred = q->avail - cred;
	q->pend_cred += cred;
	ring_fl_db(adap, q);

666
	if (unlikely(fl_starving(adap, q))) {
667
		smp_wmb();
668
		q->low++;
669 670
		set_bit(q->cntxt_id - adap->sge.egr_start,
			adap->sge.starving_fl);
671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690
	}

	return cred;
}

static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
{
	refill_fl(adap, fl, min(MAX_RX_REFILL, fl_cap(fl) - fl->avail),
		  GFP_ATOMIC);
}

/**
 *	alloc_ring - allocate resources for an SGE descriptor ring
 *	@dev: the PCI device's core device
 *	@nelem: the number of descriptors
 *	@elem_size: the size of each descriptor
 *	@sw_size: the size of the SW state associated with each ring element
 *	@phys: the physical address of the allocated ring
 *	@metadata: address of the array holding the SW state for the ring
 *	@stat_size: extra space in HW ring for status information
691
 *	@node: preferred node for memory allocations
692 693 694 695 696 697 698 699 700 701 702
 *
 *	Allocates resources for an SGE descriptor ring, such as Tx queues,
 *	free buffer lists, or response queues.  Each SGE ring requires
 *	space for its HW descriptors plus, optionally, space for the SW state
 *	associated with each HW entry (the metadata).  The function returns
 *	three values: the virtual address for the HW ring (the return value
 *	of the function), the bus address of the HW ring, and the address
 *	of the SW ring.
 */
static void *alloc_ring(struct device *dev, size_t nelem, size_t elem_size,
			size_t sw_size, dma_addr_t *phys, void *metadata,
703
			size_t stat_size, int node)
704 705 706 707 708 709 710 711
{
	size_t len = nelem * elem_size + stat_size;
	void *s = NULL;
	void *p = dma_alloc_coherent(dev, len, phys, GFP_KERNEL);

	if (!p)
		return NULL;
	if (sw_size) {
712
		s = kzalloc_node(nelem * sw_size, GFP_KERNEL, node);
713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733

		if (!s) {
			dma_free_coherent(dev, len, p, *phys);
			return NULL;
		}
	}
	if (metadata)
		*(void **)metadata = s;
	memset(p, 0, len);
	return p;
}

/**
 *	sgl_len - calculates the size of an SGL of the given capacity
 *	@n: the number of SGL entries
 *
 *	Calculates the number of flits needed for a scatter/gather list that
 *	can hold the given number of entries.
 */
static inline unsigned int sgl_len(unsigned int n)
{
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749
	/* A Direct Scatter Gather List uses 32-bit lengths and 64-bit PCI DMA
	 * addresses.  The DSGL Work Request starts off with a 32-bit DSGL
	 * ULPTX header, then Length0, then Address0, then, for 1 <= i <= N,
	 * repeated sequences of { Length[i], Length[i+1], Address[i],
	 * Address[i+1] } (this ensures that all addresses are on 64-bit
	 * boundaries).  If N is even, then Length[N+1] should be set to 0 and
	 * Address[N+1] is omitted.
	 *
	 * The following calculation incorporates all of the above.  It's
	 * somewhat hard to follow but, briefly: the "+2" accounts for the
	 * first two flits which include the DSGL header, Length0 and
	 * Address0; the "(3*(n-1))/2" covers the main body of list entries (3
	 * flits for every pair of the remaining N) +1 if (n-1) is odd; and
	 * finally the "+((n-1)&1)" adds the one remaining flit needed if
	 * (n-1) is odd ...
	 */
750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771
	n--;
	return (3 * n) / 2 + (n & 1) + 2;
}

/**
 *	flits_to_desc - returns the num of Tx descriptors for the given flits
 *	@n: the number of flits
 *
 *	Returns the number of Tx descriptors needed for the supplied number
 *	of flits.
 */
static inline unsigned int flits_to_desc(unsigned int n)
{
	BUG_ON(n > SGE_MAX_WR_LEN / 8);
	return DIV_ROUND_UP(n, 8);
}

/**
 *	is_eth_imm - can an Ethernet packet be sent as immediate data?
 *	@skb: the packet
 *
 *	Returns whether an Ethernet packet is small enough to fit as
772
 *	immediate data. Return value corresponds to headroom required.
773 774 775
 */
static inline int is_eth_imm(const struct sk_buff *skb)
{
776 777 778 779 780 781 782
	int hdrlen = skb_shinfo(skb)->gso_size ?
			sizeof(struct cpl_tx_pkt_lso_core) : 0;

	hdrlen += sizeof(struct cpl_tx_pkt);
	if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen)
		return hdrlen;
	return 0;
783 784 785 786 787 788 789 790 791 792 793 794
}

/**
 *	calc_tx_flits - calculate the number of flits for a packet Tx WR
 *	@skb: the packet
 *
 *	Returns the number of flits needed for a Tx WR for the given Ethernet
 *	packet, including the needed WR and CPL headers.
 */
static inline unsigned int calc_tx_flits(const struct sk_buff *skb)
{
	unsigned int flits;
795
	int hdrlen = is_eth_imm(skb);
796

797 798 799 800 801
	/* If the skb is small enough, we can pump it out as a work request
	 * with only immediate data.  In that case we just have to have the
	 * TX Packet header plus the skb data in the Work Request.
	 */

802 803
	if (hdrlen)
		return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64));
804

805 806 807 808 809 810 811 812
	/* Otherwise, we're going to have to construct a Scatter gather list
	 * of the skb body and fragments.  We also include the flits necessary
	 * for the TX Packet Work Request and CPL.  We always have a firmware
	 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
	 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
	 * message or, if we're doing a Large Send Offload, an LSO CPL message
	 * with an embedded TX Packet Write CPL message.
	 */
813
	flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
814
	if (skb_shinfo(skb)->gso_size)
815 816 817 818 819 820
		flits += (sizeof(struct fw_eth_tx_pkt_wr) +
			  sizeof(struct cpl_tx_pkt_lso_core) +
			  sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
	else
		flits += (sizeof(struct fw_eth_tx_pkt_wr) +
			  sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868
	return flits;
}

/**
 *	calc_tx_descs - calculate the number of Tx descriptors for a packet
 *	@skb: the packet
 *
 *	Returns the number of Tx descriptors needed for the given Ethernet
 *	packet, including the needed WR and CPL headers.
 */
static inline unsigned int calc_tx_descs(const struct sk_buff *skb)
{
	return flits_to_desc(calc_tx_flits(skb));
}

/**
 *	write_sgl - populate a scatter/gather list for a packet
 *	@skb: the packet
 *	@q: the Tx queue we are writing into
 *	@sgl: starting location for writing the SGL
 *	@end: points right after the end of the SGL
 *	@start: start offset into skb main-body data to include in the SGL
 *	@addr: the list of bus addresses for the SGL elements
 *
 *	Generates a gather list for the buffers that make up a packet.
 *	The caller must provide adequate space for the SGL that will be written.
 *	The SGL includes all of the packet's page fragments and the data in its
 *	main body except for the first @start bytes.  @sgl must be 16-byte
 *	aligned and within a Tx descriptor with available space.  @end points
 *	right after the end of the SGL but does not account for any potential
 *	wrap around, i.e., @end > @sgl.
 */
static void write_sgl(const struct sk_buff *skb, struct sge_txq *q,
		      struct ulptx_sgl *sgl, u64 *end, unsigned int start,
		      const dma_addr_t *addr)
{
	unsigned int i, len;
	struct ulptx_sge_pair *to;
	const struct skb_shared_info *si = skb_shinfo(skb);
	unsigned int nfrags = si->nr_frags;
	struct ulptx_sge_pair buf[MAX_SKB_FRAGS / 2 + 1];

	len = skb_headlen(skb) - start;
	if (likely(len)) {
		sgl->len0 = htonl(len);
		sgl->addr0 = cpu_to_be64(addr[0] + start);
		nfrags++;
	} else {
E
Eric Dumazet 已提交
869
		sgl->len0 = htonl(skb_frag_size(&si->frags[0]));
870 871 872
		sgl->addr0 = cpu_to_be64(addr[1]);
	}

873 874
	sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
			      ULPTX_NSGE_V(nfrags));
875 876 877 878 879 880 881 882 883 884
	if (likely(--nfrags == 0))
		return;
	/*
	 * Most of the complexity below deals with the possibility we hit the
	 * end of the queue in the middle of writing the SGL.  For this case
	 * only we create the SGL in a temporary buffer and then copy it.
	 */
	to = (u8 *)end > (u8 *)q->stat ? buf : sgl->sge;

	for (i = (nfrags != si->nr_frags); nfrags >= 2; nfrags -= 2, to++) {
E
Eric Dumazet 已提交
885 886
		to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
		to->len[1] = cpu_to_be32(skb_frag_size(&si->frags[++i]));
887 888 889 890
		to->addr[0] = cpu_to_be64(addr[i]);
		to->addr[1] = cpu_to_be64(addr[++i]);
	}
	if (nfrags) {
E
Eric Dumazet 已提交
891
		to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
892 893 894 895 896 897 898 899 900 901 902 903 904
		to->len[1] = cpu_to_be32(0);
		to->addr[0] = cpu_to_be64(addr[i + 1]);
	}
	if (unlikely((u8 *)end > (u8 *)q->stat)) {
		unsigned int part0 = (u8 *)q->stat - (u8 *)sgl->sge, part1;

		if (likely(part0))
			memcpy(sgl->sge, buf, part0);
		part1 = (u8 *)end - (u8 *)q->stat;
		memcpy(q->desc, (u8 *)buf + part0, part1);
		end = (void *)q->desc + part1;
	}
	if ((uintptr_t)end & 8)           /* 0-pad to multiple of 16 */
905
		*end = 0;
906 907
}

908 909 910
/* This function copies 64 byte coalesced work request to
 * memory mapped BAR2 space. For coalesced WR SGE fetches
 * data from the FIFO instead of from Host.
911
 */
912
static void cxgb_pio_copy(u64 __iomem *dst, u64 *src)
913
{
914
	int count = 8;
915 916 917 918 919 920 921 922 923

	while (count) {
		writeq(*src, dst);
		src++;
		dst++;
		count--;
	}
}

924 925 926 927 928 929 930 931 932 933
/**
 *	ring_tx_db - check and potentially ring a Tx queue's doorbell
 *	@adap: the adapter
 *	@q: the Tx queue
 *	@n: number of new descriptors to give to HW
 *
 *	Ring the doorbel for a Tx queue.
 */
static inline void ring_tx_db(struct adapter *adap, struct sge_txq *q, int n)
{
934 935 936 937
	/* Make sure that all writes to the TX Descriptors are committed
	 * before we tell the hardware about them.
	 */
	wmb();
938

939 940 941 942
	/* If we don't have access to the new User Doorbell (T5+), use the old
	 * doorbell mechanism; otherwise use the new BAR2 mechanism.
	 */
	if (unlikely(q->bar2_addr == NULL)) {
943
		u32 val = PIDX_V(n);
944 945 946 947 948 949 950
		unsigned long flags;

		/* For T4 we need to participate in the Doorbell Recovery
		 * mechanism.
		 */
		spin_lock_irqsave(&q->db_lock, flags);
		if (!q->db_disabled)
951 952
			t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
				     QID_V(q->cntxt_id) | val);
953 954 955 956 957
		else
			q->db_pidx_inc += n;
		q->db_pidx = q->pidx;
		spin_unlock_irqrestore(&q->db_lock, flags);
	} else {
958
		u32 val = PIDX_T5_V(n);
959 960 961 962 963 964 965

		/* T4 and later chips share the same PIDX field offset within
		 * the doorbell, but T5 and later shrank the field in order to
		 * gain a bit for Doorbell Priority.  The field was absurdly
		 * large in the first place (14 bits) so we just use the T5
		 * and later limits and warn if a Queue ID is too large.
		 */
966
		WARN_ON(val & DBPRIO_F);
967

968 969 970
		/* If we're only writing a single TX Descriptor and we can use
		 * Inferred QID registers, we can use the Write Combining
		 * Gather Buffer; otherwise we use the simple doorbell.
971
		 */
972
		if (n == 1 && q->bar2_qid == 0) {
973 974 975
			int index = (q->pidx
				     ? (q->pidx - 1)
				     : (q->size - 1));
976
			u64 *wr = (u64 *)&q->desc[index];
977

978 979 980
			cxgb_pio_copy((u64 __iomem *)
				      (q->bar2_addr + SGE_UDB_WCDOORBELL),
				      wr);
981
		} else {
982
			writel(val | QID_V(q->bar2_qid),
983
			       q->bar2_addr + SGE_UDB_KDOORBELL);
984
		}
985 986 987 988 989 990 991 992 993 994 995 996 997

		/* This Write Memory Barrier will force the write to the User
		 * Doorbell area to be flushed.  This is needed to prevent
		 * writes on different CPUs for the same queue from hitting
		 * the adapter out of order.  This is required when some Work
		 * Requests take the Write Combine Gather Buffer path (user
		 * doorbell area offset [SGE_UDB_WCDOORBELL..+63]) and some
		 * take the traditional path where we simply increment the
		 * PIDX (User Doorbell area SGE_UDB_KDOORBELL) and have the
		 * hardware DMA read the actual Work Request.
		 */
		wmb();
	}
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
}

/**
 *	inline_tx_skb - inline a packet's data into Tx descriptors
 *	@skb: the packet
 *	@q: the Tx queue where the packet will be inlined
 *	@pos: starting position in the Tx queue where to inline the packet
 *
 *	Inline a packet's contents directly into Tx descriptors, starting at
 *	the given position within the Tx DMA ring.
 *	Most of the complexity of this operation is dealing with wrap arounds
 *	in the middle of the packet we want to inline.
 */
static void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q,
			  void *pos)
{
	u64 *p;
	int left = (void *)q->stat - pos;

	if (likely(skb->len <= left)) {
		if (likely(!skb->data_len))
			skb_copy_from_linear_data(skb, pos, skb->len);
		else
			skb_copy_bits(skb, 0, pos, skb->len);
		pos += skb->len;
	} else {
		skb_copy_bits(skb, 0, pos, left);
		skb_copy_bits(skb, left, q->desc, skb->len - left);
		pos = (void *)q->desc + (skb->len - left);
	}

	/* 0-pad to multiple of 16 */
	p = PTR_ALIGN(pos, 8);
	if ((uintptr_t)p & 8)
		*p = 0;
}

1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058
static void *inline_tx_skb_header(const struct sk_buff *skb,
				  const struct sge_txq *q,  void *pos,
				  int length)
{
	u64 *p;
	int left = (void *)q->stat - pos;

	if (likely(length <= left)) {
		memcpy(pos, skb->data, length);
		pos += length;
	} else {
		memcpy(pos, skb->data, left);
		memcpy(q->desc, skb->data + left, length - left);
		pos = (void *)q->desc + (length - left);
	}
	/* 0-pad to multiple of 16 */
	p = PTR_ALIGN(pos, 8);
	if ((uintptr_t)p & 8) {
		*p = 0;
		return p + 1;
	}
	return p;
}

1059 1060 1061 1062
/*
 * Figure out what HW csum a packet wants and return the appropriate control
 * bits.
 */
1063
static u64 hwcsum(enum chip_type chip, const struct sk_buff *skb)
1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
{
	int csum_type;
	const struct iphdr *iph = ip_hdr(skb);

	if (iph->version == 4) {
		if (iph->protocol == IPPROTO_TCP)
			csum_type = TX_CSUM_TCPIP;
		else if (iph->protocol == IPPROTO_UDP)
			csum_type = TX_CSUM_UDPIP;
		else {
nocsum:			/*
			 * unknown protocol, disable HW csum
			 * and hope a bad packet is detected
			 */
1078
			return TXPKT_L4CSUM_DIS_F;
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093
		}
	} else {
		/*
		 * this doesn't work with extension headers
		 */
		const struct ipv6hdr *ip6h = (const struct ipv6hdr *)iph;

		if (ip6h->nexthdr == IPPROTO_TCP)
			csum_type = TX_CSUM_TCPIP6;
		else if (ip6h->nexthdr == IPPROTO_UDP)
			csum_type = TX_CSUM_UDPIP6;
		else
			goto nocsum;
	}

1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
	if (likely(csum_type >= TX_CSUM_TCPIP)) {
		u64 hdr_len = TXPKT_IPHDR_LEN_V(skb_network_header_len(skb));
		int eth_hdr_len = skb_network_offset(skb) - ETH_HLEN;

		if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5)
			hdr_len |= TXPKT_ETHHDR_LEN_V(eth_hdr_len);
		else
			hdr_len |= T6_TXPKT_ETHHDR_LEN_V(eth_hdr_len);
		return TXPKT_CSUM_TYPE_V(csum_type) | hdr_len;
	} else {
1104 1105
		int start = skb_transport_offset(skb);

1106 1107 1108
		return TXPKT_CSUM_TYPE_V(csum_type) |
			TXPKT_CSUM_START_V(start) |
			TXPKT_CSUM_LOC_V(start + skb->csum_offset);
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
	}
}

static void eth_txq_stop(struct sge_eth_txq *q)
{
	netif_tx_stop_queue(q->txq);
	q->q.stops++;
}

static inline void txq_advance(struct sge_txq *q, unsigned int n)
{
	q->in_use += n;
	q->pidx += n;
	if (q->pidx >= q->size)
		q->pidx -= q->size;
}

V
Varun Prakash 已提交
1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148
#ifdef CONFIG_CHELSIO_T4_FCOE
static inline int
cxgb_fcoe_offload(struct sk_buff *skb, struct adapter *adap,
		  const struct port_info *pi, u64 *cntrl)
{
	const struct cxgb_fcoe *fcoe = &pi->fcoe;

	if (!(fcoe->flags & CXGB_FCOE_ENABLED))
		return 0;

	if (skb->protocol != htons(ETH_P_FCOE))
		return 0;

	skb_reset_mac_header(skb);
	skb->mac_len = sizeof(struct ethhdr);

	skb_set_network_header(skb, skb->mac_len);
	skb_set_transport_header(skb, skb->mac_len + sizeof(struct fcoe_hdr));

	if (!cxgb_fcoe_sof_eof_supported(adap, skb))
		return -ENOTSUPP;

	/* FC CRC offload */
1149 1150 1151 1152 1153
	*cntrl = TXPKT_CSUM_TYPE_V(TX_CSUM_FCOE) |
		     TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F |
		     TXPKT_CSUM_START_V(CXGB_FCOE_TXPKT_CSUM_START) |
		     TXPKT_CSUM_END_V(CXGB_FCOE_TXPKT_CSUM_END) |
		     TXPKT_CSUM_LOC_V(CXGB_FCOE_TXPKT_CSUM_END);
V
Varun Prakash 已提交
1154 1155 1156 1157
	return 0;
}
#endif /* CONFIG_CHELSIO_T4_FCOE */

1158 1159 1160 1161 1162 1163 1164 1165 1166
/**
 *	t4_eth_xmit - add a packet to an Ethernet Tx queue
 *	@skb: the packet
 *	@dev: the egress net device
 *
 *	Add a packet to an SGE Ethernet Tx queue.  Runs with softirqs disabled.
 */
netdev_tx_t t4_eth_xmit(struct sk_buff *skb, struct net_device *dev)
{
1167
	u32 wr_mid, ctrl0;
1168 1169 1170 1171 1172 1173 1174 1175 1176 1177
	u64 cntrl, *end;
	int qidx, credits;
	unsigned int flits, ndesc;
	struct adapter *adap;
	struct sge_eth_txq *q;
	const struct port_info *pi;
	struct fw_eth_tx_pkt_wr *wr;
	struct cpl_tx_pkt_core *cpl;
	const struct skb_shared_info *ssi;
	dma_addr_t addr[MAX_SKB_FRAGS + 1];
1178
	bool immediate = false;
1179
	int len, max_pkt_len;
V
Varun Prakash 已提交
1180 1181 1182
#ifdef CONFIG_CHELSIO_T4_FCOE
	int err;
#endif /* CONFIG_CHELSIO_T4_FCOE */
1183 1184 1185 1186 1187 1188

	/*
	 * The chip min packet length is 10 octets but play safe and reject
	 * anything shorter than an Ethernet header.
	 */
	if (unlikely(skb->len < ETH_HLEN)) {
1189
out_free:	dev_kfree_skb_any(skb);
1190 1191 1192
		return NETDEV_TX_OK;
	}

1193 1194 1195 1196 1197 1198 1199
	/* Discard the packet if the length is greater than mtu */
	max_pkt_len = ETH_HLEN + dev->mtu;
	if (skb_vlan_tag_present(skb))
		max_pkt_len += VLAN_HLEN;
	if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len)))
		goto out_free;

1200 1201 1202 1203 1204 1205
	pi = netdev_priv(dev);
	adap = pi->adapter;
	qidx = skb_get_queue_mapping(skb);
	q = &adap->sge.ethtxq[qidx + pi->first_qset];

	reclaim_completed_tx(adap, &q->q, true);
1206
	cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
V
Varun Prakash 已提交
1207 1208 1209 1210 1211 1212

#ifdef CONFIG_CHELSIO_T4_FCOE
	err = cxgb_fcoe_offload(skb, adap, pi, &cntrl);
	if (unlikely(err == -ENOTSUPP))
		goto out_free;
#endif /* CONFIG_CHELSIO_T4_FCOE */
1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225

	flits = calc_tx_flits(skb);
	ndesc = flits_to_desc(flits);
	credits = txq_avail(&q->q) - ndesc;

	if (unlikely(credits < 0)) {
		eth_txq_stop(q);
		dev_err(adap->pdev_dev,
			"%s: Tx ring %u full while queue awake!\n",
			dev->name, qidx);
		return NETDEV_TX_BUSY;
	}

1226 1227 1228 1229
	if (is_eth_imm(skb))
		immediate = true;

	if (!immediate &&
1230 1231 1232 1233 1234
	    unlikely(map_skb(adap->pdev_dev, skb, addr) < 0)) {
		q->mapping_err++;
		goto out_free;
	}

1235
	wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
1236 1237
	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
		eth_txq_stop(q);
1238
		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1239 1240 1241 1242 1243 1244 1245
	}

	wr = (void *)&q->q.desc[q->q.pidx];
	wr->equiq_to_len16 = htonl(wr_mid);
	wr->r3 = cpu_to_be64(0);
	end = (u64 *)wr + flits;

1246
	len = immediate ? skb->len : 0;
1247 1248
	ssi = skb_shinfo(skb);
	if (ssi->gso_size) {
1249
		struct cpl_tx_pkt_lso *lso = (void *)wr;
1250 1251 1252 1253
		bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0;
		int l3hdr_len = skb_network_header_len(skb);
		int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;

1254
		len += sizeof(*lso);
1255 1256
		wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
				       FW_WR_IMMDLEN_V(len));
1257 1258 1259 1260 1261 1262
		lso->c.lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) |
					LSO_FIRST_SLICE_F | LSO_LAST_SLICE_F |
					LSO_IPV6_V(v6) |
					LSO_ETHHDR_LEN_V(eth_xtra_len / 4) |
					LSO_IPHDR_LEN_V(l3hdr_len / 4) |
					LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
1263 1264 1265
		lso->c.ipid_ofst = htons(0);
		lso->c.mss = htons(ssi->gso_size);
		lso->c.seqno_offset = htonl(0);
1266 1267 1268
		if (is_t4(adap->params.chip))
			lso->c.len = htonl(skb->len);
		else
1269
			lso->c.len = htonl(LSO_T5_XFER_SIZE_V(skb->len));
1270
		cpl = (void *)(lso + 1);
1271 1272 1273 1274 1275 1276 1277 1278 1279

		if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
			cntrl =	TXPKT_ETHHDR_LEN_V(eth_xtra_len);
		else
			cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len);

		cntrl |= TXPKT_CSUM_TYPE_V(v6 ?
					   TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) |
			 TXPKT_IPHDR_LEN_V(l3hdr_len);
1280 1281 1282
		q->tso++;
		q->tx_cso += ssi->gso_segs;
	} else {
1283
		len += sizeof(*cpl);
1284 1285
		wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
				       FW_WR_IMMDLEN_V(len));
1286 1287
		cpl = (void *)(wr + 1);
		if (skb->ip_summed == CHECKSUM_PARTIAL) {
1288 1289
			cntrl = hwcsum(adap->params.chip, skb) |
				TXPKT_IPCSUM_DIS_F;
1290
			q->tx_cso++;
V
Varun Prakash 已提交
1291
		}
1292 1293
	}

1294
	if (skb_vlan_tag_present(skb)) {
1295
		q->vlan_ins++;
1296
		cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
V
Varun Prakash 已提交
1297 1298
#ifdef CONFIG_CHELSIO_T4_FCOE
		if (skb->protocol == htons(ETH_P_FCOE))
1299
			cntrl |= TXPKT_VLAN_V(
V
Varun Prakash 已提交
1300 1301
				 ((skb->priority & 0x7) << VLAN_PRIO_SHIFT));
#endif /* CONFIG_CHELSIO_T4_FCOE */
1302 1303
	}

1304 1305 1306 1307 1308 1309 1310 1311 1312
	ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) |
		TXPKT_PF_V(adap->pf);
#ifdef CONFIG_CHELSIO_T4_DCB
	if (is_t4(adap->params.chip))
		ctrl0 |= TXPKT_OVLAN_IDX_V(q->dcb_prio);
	else
		ctrl0 |= TXPKT_T5_OVLAN_IDX_V(q->dcb_prio);
#endif
	cpl->ctrl0 = htonl(ctrl0);
1313 1314 1315 1316
	cpl->pack = htons(0);
	cpl->len = htons(skb->len);
	cpl->ctrl1 = cpu_to_be64(cntrl);

1317
	if (immediate) {
1318
		inline_tx_skb(skb, &q->q, cpl + 1);
1319
		dev_consume_skb_any(skb);
1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349
	} else {
		int last_desc;

		write_sgl(skb, &q->q, (struct ulptx_sgl *)(cpl + 1), end, 0,
			  addr);
		skb_orphan(skb);

		last_desc = q->q.pidx + ndesc - 1;
		if (last_desc >= q->q.size)
			last_desc -= q->q.size;
		q->q.sdesc[last_desc].skb = skb;
		q->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)(cpl + 1);
	}

	txq_advance(&q->q, ndesc);

	ring_tx_db(adap, &q->q, ndesc);
	return NETDEV_TX_OK;
}

/**
 *	reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
 *	@q: the SGE control Tx queue
 *
 *	This is a variant of reclaim_completed_tx() that is used for Tx queues
 *	that send only immediate data (presently just the control queues) and
 *	thus do not have any sk_buffs to release.
 */
static inline void reclaim_completed_tx_imm(struct sge_txq *q)
{
1350
	int hw_cidx = ntohs(ACCESS_ONCE(q->stat->cidx));
1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384
	int reclaim = hw_cidx - q->cidx;

	if (reclaim < 0)
		reclaim += q->size;

	q->in_use -= reclaim;
	q->cidx = hw_cidx;
}

/**
 *	is_imm - check whether a packet can be sent as immediate data
 *	@skb: the packet
 *
 *	Returns true if a packet can be sent as a WR with immediate data.
 */
static inline int is_imm(const struct sk_buff *skb)
{
	return skb->len <= MAX_CTRL_WR_LEN;
}

/**
 *	ctrlq_check_stop - check if a control queue is full and should stop
 *	@q: the queue
 *	@wr: most recent WR written to the queue
 *
 *	Check if a control queue has become full and should be stopped.
 *	We clean up control queue descriptors very lazily, only when we are out.
 *	If the queue is still full after reclaiming any completed descriptors
 *	we suspend it and have the last WR wake it up.
 */
static void ctrlq_check_stop(struct sge_ctrl_txq *q, struct fw_wr_hdr *wr)
{
	reclaim_completed_tx_imm(&q->q);
	if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) {
1385
		wr->lo |= htonl(FW_WR_EQUEQ_F | FW_WR_EQUIQ_F);
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453
		q->q.stops++;
		q->full = 1;
	}
}

/**
 *	ctrl_xmit - send a packet through an SGE control Tx queue
 *	@q: the control queue
 *	@skb: the packet
 *
 *	Send a packet through an SGE control Tx queue.  Packets sent through
 *	a control queue must fit entirely as immediate data.
 */
static int ctrl_xmit(struct sge_ctrl_txq *q, struct sk_buff *skb)
{
	unsigned int ndesc;
	struct fw_wr_hdr *wr;

	if (unlikely(!is_imm(skb))) {
		WARN_ON(1);
		dev_kfree_skb(skb);
		return NET_XMIT_DROP;
	}

	ndesc = DIV_ROUND_UP(skb->len, sizeof(struct tx_desc));
	spin_lock(&q->sendq.lock);

	if (unlikely(q->full)) {
		skb->priority = ndesc;                  /* save for restart */
		__skb_queue_tail(&q->sendq, skb);
		spin_unlock(&q->sendq.lock);
		return NET_XMIT_CN;
	}

	wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx];
	inline_tx_skb(skb, &q->q, wr);

	txq_advance(&q->q, ndesc);
	if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES))
		ctrlq_check_stop(q, wr);

	ring_tx_db(q->adap, &q->q, ndesc);
	spin_unlock(&q->sendq.lock);

	kfree_skb(skb);
	return NET_XMIT_SUCCESS;
}

/**
 *	restart_ctrlq - restart a suspended control queue
 *	@data: the control queue to restart
 *
 *	Resumes transmission on a suspended Tx control queue.
 */
static void restart_ctrlq(unsigned long data)
{
	struct sk_buff *skb;
	unsigned int written = 0;
	struct sge_ctrl_txq *q = (struct sge_ctrl_txq *)data;

	spin_lock(&q->sendq.lock);
	reclaim_completed_tx_imm(&q->q);
	BUG_ON(txq_avail(&q->q) < TXQ_STOP_THRES);  /* q should be empty */

	while ((skb = __skb_dequeue(&q->sendq)) != NULL) {
		struct fw_wr_hdr *wr;
		unsigned int ndesc = skb->priority;     /* previously saved */

1454 1455
		written += ndesc;
		/* Write descriptors and free skbs outside the lock to limit
1456 1457
		 * wait times.  q->full is still set so new skbs will be queued.
		 */
1458 1459
		wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx];
		txq_advance(&q->q, ndesc);
1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531
		spin_unlock(&q->sendq.lock);

		inline_tx_skb(skb, &q->q, wr);
		kfree_skb(skb);

		if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) {
			unsigned long old = q->q.stops;

			ctrlq_check_stop(q, wr);
			if (q->q.stops != old) {          /* suspended anew */
				spin_lock(&q->sendq.lock);
				goto ringdb;
			}
		}
		if (written > 16) {
			ring_tx_db(q->adap, &q->q, written);
			written = 0;
		}
		spin_lock(&q->sendq.lock);
	}
	q->full = 0;
ringdb: if (written)
		ring_tx_db(q->adap, &q->q, written);
	spin_unlock(&q->sendq.lock);
}

/**
 *	t4_mgmt_tx - send a management message
 *	@adap: the adapter
 *	@skb: the packet containing the management message
 *
 *	Send a management message through control queue 0.
 */
int t4_mgmt_tx(struct adapter *adap, struct sk_buff *skb)
{
	int ret;

	local_bh_disable();
	ret = ctrl_xmit(&adap->sge.ctrlq[0], skb);
	local_bh_enable();
	return ret;
}

/**
 *	is_ofld_imm - check whether a packet can be sent as immediate data
 *	@skb: the packet
 *
 *	Returns true if a packet can be sent as an offload WR with immediate
 *	data.  We currently use the same limit as for Ethernet packets.
 */
static inline int is_ofld_imm(const struct sk_buff *skb)
{
	return skb->len <= MAX_IMM_TX_PKT_LEN;
}

/**
 *	calc_tx_flits_ofld - calculate # of flits for an offload packet
 *	@skb: the packet
 *
 *	Returns the number of flits needed for the given offload packet.
 *	These packets are already fully constructed and no additional headers
 *	will be added.
 */
static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
{
	unsigned int flits, cnt;

	if (is_ofld_imm(skb))
		return DIV_ROUND_UP(skb->len, 8);

	flits = skb_transport_offset(skb) / 8U;   /* headers */
	cnt = skb_shinfo(skb)->nr_frags;
1532
	if (skb_tail_pointer(skb) != skb_transport_header(skb))
1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549
		cnt++;
	return flits + sgl_len(cnt);
}

/**
 *	txq_stop_maperr - stop a Tx queue due to I/O MMU exhaustion
 *	@adap: the adapter
 *	@q: the queue to stop
 *
 *	Mark a Tx queue stopped due to I/O MMU exhaustion and resulting
 *	inability to map packets.  A periodic timer attempts to restart
 *	queues so marked.
 */
static void txq_stop_maperr(struct sge_ofld_txq *q)
{
	q->mapping_err++;
	q->q.stops++;
1550 1551
	set_bit(q->q.cntxt_id - q->adap->sge.egr_start,
		q->adap->sge.txq_maperr);
1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565
}

/**
 *	ofldtxq_stop - stop an offload Tx queue that has become full
 *	@q: the queue to stop
 *	@skb: the packet causing the queue to become full
 *
 *	Stops an offload Tx queue that has become full and modifies the packet
 *	being written to request a wakeup.
 */
static void ofldtxq_stop(struct sge_ofld_txq *q, struct sk_buff *skb)
{
	struct fw_wr_hdr *wr = (struct fw_wr_hdr *)skb->data;

1566
	wr->lo |= htonl(FW_WR_EQUEQ_F | FW_WR_EQUIQ_F);
1567 1568 1569 1570 1571
	q->q.stops++;
	q->full = 1;
}

/**
1572
 *	service_ofldq - service/restart a suspended offload queue
1573 1574
 *	@q: the offload queue
 *
1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587
 *	Services an offload Tx queue by moving packets from its Pending Send
 *	Queue to the Hardware TX ring.  The function starts and ends with the
 *	Send Queue locked, but drops the lock while putting the skb at the
 *	head of the Send Queue onto the Hardware TX Ring.  Dropping the lock
 *	allows more skbs to be added to the Send Queue by other threads.
 *	The packet being processed at the head of the Pending Send Queue is
 *	left on the queue in case we experience DMA Mapping errors, etc.
 *	and need to give up and restart later.
 *
 *	service_ofldq() can be thought of as a task which opportunistically
 *	uses other threads execution contexts.  We use the Offload Queue
 *	boolean "service_ofldq_running" to make sure that only one instance
 *	is ever running at a time ...
1588 1589 1590
 */
static void service_ofldq(struct sge_ofld_txq *q)
{
1591
	u64 *pos, *before, *end;
1592 1593
	int credits;
	struct sk_buff *skb;
1594 1595
	struct sge_txq *txq;
	unsigned int left;
1596 1597 1598
	unsigned int written = 0;
	unsigned int flits, ndesc;

1599 1600 1601 1602 1603 1604 1605 1606 1607 1608
	/* If another thread is currently in service_ofldq() processing the
	 * Pending Send Queue then there's nothing to do. Otherwise, flag
	 * that we're doing the work and continue.  Examining/modifying
	 * the Offload Queue boolean "service_ofldq_running" must be done
	 * while holding the Pending Send Queue Lock.
	 */
	if (q->service_ofldq_running)
		return;
	q->service_ofldq_running = true;

1609
	while ((skb = skb_peek(&q->sendq)) != NULL && !q->full) {
1610 1611 1612 1613 1614 1615
		/* We drop the lock while we're working with the skb at the
		 * head of the Pending Send Queue.  This allows more skbs to
		 * be added to the Pending Send Queue while we're working on
		 * this one.  We don't need to lock to guard the TX Ring
		 * updates because only one thread of execution is ever
		 * allowed into service_ofldq() at a time.
1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638
		 */
		spin_unlock(&q->sendq.lock);

		reclaim_completed_tx(q->adap, &q->q, false);

		flits = skb->priority;                /* previously saved */
		ndesc = flits_to_desc(flits);
		credits = txq_avail(&q->q) - ndesc;
		BUG_ON(credits < 0);
		if (unlikely(credits < TXQ_STOP_THRES))
			ofldtxq_stop(q, skb);

		pos = (u64 *)&q->q.desc[q->q.pidx];
		if (is_ofld_imm(skb))
			inline_tx_skb(skb, &q->q, pos);
		else if (map_skb(q->adap->pdev_dev, skb,
				 (dma_addr_t *)skb->head)) {
			txq_stop_maperr(q);
			spin_lock(&q->sendq.lock);
			break;
		} else {
			int last_desc, hdr_len = skb_transport_offset(skb);

1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664
			/* The WR headers  may not fit within one descriptor.
			 * So we need to deal with wrap-around here.
			 */
			before = (u64 *)pos;
			end = (u64 *)pos + flits;
			txq = &q->q;
			pos = (void *)inline_tx_skb_header(skb, &q->q,
							   (void *)pos,
							   hdr_len);
			if (before > (u64 *)pos) {
				left = (u8 *)end - (u8 *)txq->stat;
				end = (void *)txq->desc + left;
			}

			/* If current position is already at the end of the
			 * ofld queue, reset the current to point to
			 * start of the queue and update the end ptr as well.
			 */
			if (pos == (u64 *)txq->stat) {
				left = (u8 *)end - (u8 *)txq->stat;
				end = (void *)txq->desc + left;
				pos = (void *)txq->desc;
			}

			write_sgl(skb, &q->q, (void *)pos,
				  end, hdr_len,
1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682
				  (dma_addr_t *)skb->head);
#ifdef CONFIG_NEED_DMA_MAP_STATE
			skb->dev = q->adap->port[0];
			skb->destructor = deferred_unmap_destructor;
#endif
			last_desc = q->q.pidx + ndesc - 1;
			if (last_desc >= q->q.size)
				last_desc -= q->q.size;
			q->q.sdesc[last_desc].skb = skb;
		}

		txq_advance(&q->q, ndesc);
		written += ndesc;
		if (unlikely(written > 32)) {
			ring_tx_db(q->adap, &q->q, written);
			written = 0;
		}

1683 1684 1685 1686 1687
		/* Reacquire the Pending Send Queue Lock so we can unlink the
		 * skb we've just successfully transferred to the TX Ring and
		 * loop for the next skb which may be at the head of the
		 * Pending Send Queue.
		 */
1688 1689 1690 1691 1692 1693 1694
		spin_lock(&q->sendq.lock);
		__skb_unlink(skb, &q->sendq);
		if (is_ofld_imm(skb))
			kfree_skb(skb);
	}
	if (likely(written))
		ring_tx_db(q->adap, &q->q, written);
1695 1696 1697 1698 1699

	/*Indicate that no thread is processing the Pending Send Queue
	 * currently.
	 */
	q->service_ofldq_running = false;
1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712
}

/**
 *	ofld_xmit - send a packet through an offload queue
 *	@q: the Tx offload queue
 *	@skb: the packet
 *
 *	Send an offload packet through an SGE offload queue.
 */
static int ofld_xmit(struct sge_ofld_txq *q, struct sk_buff *skb)
{
	skb->priority = calc_tx_flits_ofld(skb);       /* save for restart */
	spin_lock(&q->sendq.lock);
1713 1714 1715 1716 1717 1718 1719 1720 1721

	/* Queue the new skb onto the Offload Queue's Pending Send Queue.  If
	 * that results in this new skb being the only one on the queue, start
	 * servicing it.  If there are other skbs already on the list, then
	 * either the queue is currently being processed or it's been stopped
	 * for some reason and it'll be restarted at a later time.  Restart
	 * paths are triggered by events like experiencing a DMA Mapping Error
	 * or filling the Hardware TX Ring.
	 */
1722 1723 1724
	__skb_queue_tail(&q->sendq, skb);
	if (q->sendq.qlen == 1)
		service_ofldq(q);
1725

1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773
	spin_unlock(&q->sendq.lock);
	return NET_XMIT_SUCCESS;
}

/**
 *	restart_ofldq - restart a suspended offload queue
 *	@data: the offload queue to restart
 *
 *	Resumes transmission on a suspended Tx offload queue.
 */
static void restart_ofldq(unsigned long data)
{
	struct sge_ofld_txq *q = (struct sge_ofld_txq *)data;

	spin_lock(&q->sendq.lock);
	q->full = 0;            /* the queue actually is completely empty now */
	service_ofldq(q);
	spin_unlock(&q->sendq.lock);
}

/**
 *	skb_txq - return the Tx queue an offload packet should use
 *	@skb: the packet
 *
 *	Returns the Tx queue an offload packet should use as indicated by bits
 *	1-15 in the packet's queue_mapping.
 */
static inline unsigned int skb_txq(const struct sk_buff *skb)
{
	return skb->queue_mapping >> 1;
}

/**
 *	is_ctrl_pkt - return whether an offload packet is a control packet
 *	@skb: the packet
 *
 *	Returns whether an offload packet should use an OFLD or a CTRL
 *	Tx queue as indicated by bit 0 in the packet's queue_mapping.
 */
static inline unsigned int is_ctrl_pkt(const struct sk_buff *skb)
{
	return skb->queue_mapping & 1;
}

static inline int ofld_send(struct adapter *adap, struct sk_buff *skb)
{
	unsigned int idx = skb_txq(skb);

1774 1775 1776 1777
	if (unlikely(is_ctrl_pkt(skb))) {
		/* Single ctrl queue is a requirement for LE workaround path */
		if (adap->tids.nsftids)
			idx = 0;
1778
		return ctrl_xmit(&adap->sge.ctrlq[idx], skb);
1779
	}
1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815
	return ofld_xmit(&adap->sge.ofldtxq[idx], skb);
}

/**
 *	t4_ofld_send - send an offload packet
 *	@adap: the adapter
 *	@skb: the packet
 *
 *	Sends an offload packet.  We use the packet queue_mapping to select the
 *	appropriate Tx queue as follows: bit 0 indicates whether the packet
 *	should be sent as regular or control, bits 1-15 select the queue.
 */
int t4_ofld_send(struct adapter *adap, struct sk_buff *skb)
{
	int ret;

	local_bh_disable();
	ret = ofld_send(adap, skb);
	local_bh_enable();
	return ret;
}

/**
 *	cxgb4_ofld_send - send an offload packet
 *	@dev: the net device
 *	@skb: the packet
 *
 *	Sends an offload packet.  This is an exported version of @t4_ofld_send,
 *	intended for ULDs.
 */
int cxgb4_ofld_send(struct net_device *dev, struct sk_buff *skb)
{
	return t4_ofld_send(netdev2adap(dev), skb);
}
EXPORT_SYMBOL(cxgb4_ofld_send);

1816
static inline void copy_frags(struct sk_buff *skb,
1817 1818
			      const struct pkt_gl *gl, unsigned int offset)
{
1819
	int i;
1820 1821

	/* usually there's just one frag */
1822 1823 1824 1825 1826 1827 1828 1829
	__skb_fill_page_desc(skb, 0, gl->frags[0].page,
			     gl->frags[0].offset + offset,
			     gl->frags[0].size - offset);
	skb_shinfo(skb)->nr_frags = gl->nfrags;
	for (i = 1; i < gl->nfrags; i++)
		__skb_fill_page_desc(skb, i, gl->frags[i].page,
				     gl->frags[i].offset,
				     gl->frags[i].size);
1830 1831

	/* get a reference to the last page, we don't own it */
1832
	get_page(gl->frags[gl->nfrags - 1].page);
1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866
}

/**
 *	cxgb4_pktgl_to_skb - build an sk_buff from a packet gather list
 *	@gl: the gather list
 *	@skb_len: size of sk_buff main body if it carries fragments
 *	@pull_len: amount of data to move to the sk_buff's main body
 *
 *	Builds an sk_buff from the given packet gather list.  Returns the
 *	sk_buff or %NULL if sk_buff allocation failed.
 */
struct sk_buff *cxgb4_pktgl_to_skb(const struct pkt_gl *gl,
				   unsigned int skb_len, unsigned int pull_len)
{
	struct sk_buff *skb;

	/*
	 * Below we rely on RX_COPY_THRES being less than the smallest Rx buffer
	 * size, which is expected since buffers are at least PAGE_SIZEd.
	 * In this case packets up to RX_COPY_THRES have only one fragment.
	 */
	if (gl->tot_len <= RX_COPY_THRES) {
		skb = dev_alloc_skb(gl->tot_len);
		if (unlikely(!skb))
			goto out;
		__skb_put(skb, gl->tot_len);
		skb_copy_to_linear_data(skb, gl->va, gl->tot_len);
	} else {
		skb = dev_alloc_skb(skb_len);
		if (unlikely(!skb))
			goto out;
		__skb_put(skb, pull_len);
		skb_copy_to_linear_data(skb, gl->va, pull_len);

1867
		copy_frags(skb, gl, pull_len);
1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882
		skb->len = gl->tot_len;
		skb->data_len = skb->len - pull_len;
		skb->truesize += skb->data_len;
	}
out:	return skb;
}
EXPORT_SYMBOL(cxgb4_pktgl_to_skb);

/**
 *	t4_pktgl_free - free a packet gather list
 *	@gl: the gather list
 *
 *	Releases the pages of a packet gather list.  We do not own the last
 *	page on the list and do not free it.
 */
1883
static void t4_pktgl_free(const struct pkt_gl *gl)
1884 1885
{
	int n;
1886
	const struct page_frag *p;
1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906

	for (p = gl->frags, n = gl->nfrags - 1; n--; p++)
		put_page(p->page);
}

/*
 * Process an MPS trace packet.  Give it an unused protocol number so it won't
 * be delivered to anyone and send it to the stack for capture.
 */
static noinline int handle_trace_pkt(struct adapter *adap,
				     const struct pkt_gl *gl)
{
	struct sk_buff *skb;

	skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
	if (unlikely(!skb)) {
		t4_pktgl_free(gl);
		return 0;
	}

1907
	if (is_t4(adap->params.chip))
S
Santosh Rastapur 已提交
1908 1909 1910 1911
		__skb_pull(skb, sizeof(struct cpl_trace_pkt));
	else
		__skb_pull(skb, sizeof(struct cpl_t5_trace_pkt));

1912 1913 1914 1915 1916 1917 1918
	skb_reset_mac_header(skb);
	skb->protocol = htons(0xffff);
	skb->dev = adap->port[0];
	netif_receive_skb(skb);
	return 0;
}

1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940
/**
 * cxgb4_sgetim_to_hwtstamp - convert sge time stamp to hw time stamp
 * @adap: the adapter
 * @hwtstamps: time stamp structure to update
 * @sgetstamp: 60bit iqe timestamp
 *
 * Every ingress queue entry has the 60-bit timestamp, convert that timestamp
 * which is in Core Clock ticks into ktime_t and assign it
 **/
static void cxgb4_sgetim_to_hwtstamp(struct adapter *adap,
				     struct skb_shared_hwtstamps *hwtstamps,
				     u64 sgetstamp)
{
	u64 ns;
	u64 tmp = (sgetstamp * 1000 * 1000 + adap->params.vpd.cclk / 2);

	ns = div_u64(tmp, adap->params.vpd.cclk);

	memset(hwtstamps, 0, sizeof(*hwtstamps));
	hwtstamps->hwtstamp = ns_to_ktime(ns);
}

1941 1942 1943
static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl,
		   const struct cpl_rx_pkt *pkt)
{
1944 1945
	struct adapter *adapter = rxq->rspq.adap;
	struct sge *s = &adapter->sge;
1946
	struct port_info *pi;
1947 1948 1949 1950 1951 1952 1953 1954 1955 1956
	int ret;
	struct sk_buff *skb;

	skb = napi_get_frags(&rxq->rspq.napi);
	if (unlikely(!skb)) {
		t4_pktgl_free(gl);
		rxq->stats.rx_drops++;
		return;
	}

1957 1958
	copy_frags(skb, gl, s->pktshift);
	skb->len = gl->tot_len - s->pktshift;
1959 1960 1961 1962
	skb->data_len = skb->len;
	skb->truesize += skb->data_len;
	skb->ip_summed = CHECKSUM_UNNECESSARY;
	skb_record_rx_queue(skb, rxq->rspq.idx);
1963 1964 1965 1966
	pi = netdev_priv(skb->dev);
	if (pi->rxtstamp)
		cxgb4_sgetim_to_hwtstamp(adapter, skb_hwtstamps(skb),
					 gl->sgetstamp);
D
Dimitris Michailidis 已提交
1967
	if (rxq->rspq.netdev->features & NETIF_F_RXHASH)
T
Tom Herbert 已提交
1968 1969
		skb_set_hash(skb, (__force u32)pkt->rsshdr.hash_val,
			     PKT_HASH_TYPE_L3);
1970 1971

	if (unlikely(pkt->vlan_ex)) {
1972
		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(pkt->vlan));
1973 1974 1975
		rxq->stats.vlan_ex++;
	}
	ret = napi_gro_frags(&rxq->rspq.napi);
1976
	if (ret == GRO_HELD)
1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998
		rxq->stats.lro_pkts++;
	else if (ret == GRO_MERGED || ret == GRO_MERGED_FREE)
		rxq->stats.lro_merged++;
	rxq->stats.pkts++;
	rxq->stats.rx_cso++;
}

/**
 *	t4_ethrx_handler - process an ingress ethernet packet
 *	@q: the response queue that received the packet
 *	@rsp: the response queue descriptor holding the RX_PKT message
 *	@si: the gather list of packet fragments
 *
 *	Process an ingress ethernet packet and deliver it to the stack.
 */
int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp,
		     const struct pkt_gl *si)
{
	bool csum_ok;
	struct sk_buff *skb;
	const struct cpl_rx_pkt *pkt;
	struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq);
1999
	struct sge *s = &q->adap->sge;
2000
	int cpl_trace_pkt = is_t4(q->adap->params.chip) ?
S
Santosh Rastapur 已提交
2001
			    CPL_TRACE_PKT : CPL_TRACE_PKT_T5;
V
Varun Prakash 已提交
2002
	struct port_info *pi;
2003

S
Santosh Rastapur 已提交
2004
	if (unlikely(*(u8 *)rsp == cpl_trace_pkt))
2005 2006
		return handle_trace_pkt(q->adap, si);

D
Dimitris Michailidis 已提交
2007
	pkt = (const struct cpl_rx_pkt *)rsp;
2008 2009
	csum_ok = pkt->csum_calc && !pkt->err_vec &&
		  (q->netdev->features & NETIF_F_RXCSUM);
2010
	if ((pkt->l2info & htonl(RXF_TCP_F)) &&
2011
	    !(cxgb_poll_busy_polling(q)) &&
2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023
	    (q->netdev->features & NETIF_F_GRO) && csum_ok && !pkt->ip_frag) {
		do_gro(rxq, si, pkt);
		return 0;
	}

	skb = cxgb4_pktgl_to_skb(si, RX_PKT_SKB_LEN, RX_PULL_LEN);
	if (unlikely(!skb)) {
		t4_pktgl_free(si);
		rxq->stats.rx_drops++;
		return 0;
	}

2024
	__skb_pull(skb, s->pktshift);      /* remove ethernet header padding */
2025 2026
	skb->protocol = eth_type_trans(skb, q->netdev);
	skb_record_rx_queue(skb, q->idx);
D
Dimitris Michailidis 已提交
2027
	if (skb->dev->features & NETIF_F_RXHASH)
T
Tom Herbert 已提交
2028 2029
		skb_set_hash(skb, (__force u32)pkt->rsshdr.hash_val,
			     PKT_HASH_TYPE_L3);
D
Dimitris Michailidis 已提交
2030

2031 2032
	rxq->stats.pkts++;

2033 2034 2035 2036
	pi = netdev_priv(skb->dev);
	if (pi->rxtstamp)
		cxgb4_sgetim_to_hwtstamp(q->adap, skb_hwtstamps(skb),
					 si->sgetstamp);
2037
	if (csum_ok && (pkt->l2info & htonl(RXF_UDP_F | RXF_TCP_F))) {
2038
		if (!pkt->ip_frag) {
2039
			skb->ip_summed = CHECKSUM_UNNECESSARY;
2040
			rxq->stats.rx_cso++;
2041
		} else if (pkt->l2info & htonl(RXF_IP_F)) {
2042 2043 2044
			__sum16 c = (__force __sum16)pkt->csum;
			skb->csum = csum_unfold(c);
			skb->ip_summed = CHECKSUM_COMPLETE;
2045
			rxq->stats.rx_cso++;
2046
		}
V
Varun Prakash 已提交
2047
	} else {
2048
		skb_checksum_none_assert(skb);
V
Varun Prakash 已提交
2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063
#ifdef CONFIG_CHELSIO_T4_FCOE
#define CPL_RX_PKT_FLAGS (RXF_PSH_F | RXF_SYN_F | RXF_UDP_F | \
			  RXF_TCP_F | RXF_IP_F | RXF_IP6_F | RXF_LRO_F)

		if (!(pkt->l2info & cpu_to_be32(CPL_RX_PKT_FLAGS))) {
			if ((pkt->l2info & cpu_to_be32(RXF_FCOE_F)) &&
			    (pi->fcoe.flags & CXGB_FCOE_ENABLED)) {
				if (!(pkt->err_vec & cpu_to_be16(RXERR_CSUM_F)))
					skb->ip_summed = CHECKSUM_UNNECESSARY;
			}
		}

#undef CPL_RX_PKT_FLAGS
#endif /* CONFIG_CHELSIO_T4_FCOE */
	}
2064 2065

	if (unlikely(pkt->vlan_ex)) {
2066
		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(pkt->vlan));
2067
		rxq->stats.vlan_ex++;
2068
	}
2069
	skb_mark_napi_id(skb, &q->napi);
2070
	netif_receive_skb(skb);
2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116
	return 0;
}

/**
 *	restore_rx_bufs - put back a packet's Rx buffers
 *	@si: the packet gather list
 *	@q: the SGE free list
 *	@frags: number of FL buffers to restore
 *
 *	Puts back on an FL the Rx buffers associated with @si.  The buffers
 *	have already been unmapped and are left unmapped, we mark them so to
 *	prevent further unmapping attempts.
 *
 *	This function undoes a series of @unmap_rx_buf calls when we find out
 *	that the current packet can't be processed right away afterall and we
 *	need to come back to it later.  This is a very rare event and there's
 *	no effort to make this particularly efficient.
 */
static void restore_rx_bufs(const struct pkt_gl *si, struct sge_fl *q,
			    int frags)
{
	struct rx_sw_desc *d;

	while (frags--) {
		if (q->cidx == 0)
			q->cidx = q->size - 1;
		else
			q->cidx--;
		d = &q->sdesc[q->cidx];
		d->page = si->frags[frags].page;
		d->dma_addr |= RX_UNMAPPED_BUF;
		q->avail++;
	}
}

/**
 *	is_new_response - check if a response is newly written
 *	@r: the response descriptor
 *	@q: the response queue
 *
 *	Returns true if a response descriptor contains a yet unprocessed
 *	response.
 */
static inline bool is_new_response(const struct rsp_ctrl *r,
				   const struct sge_rspq *q)
{
2117
	return (r->type_gen >> RSPD_GEN_S) == q->gen;
2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154
}

/**
 *	rspq_next - advance to the next entry in a response queue
 *	@q: the queue
 *
 *	Updates the state of a response queue to advance it to the next entry.
 */
static inline void rspq_next(struct sge_rspq *q)
{
	q->cur_desc = (void *)q->cur_desc + q->iqe_len;
	if (unlikely(++q->cidx == q->size)) {
		q->cidx = 0;
		q->gen ^= 1;
		q->cur_desc = q->desc;
	}
}

/**
 *	process_responses - process responses from an SGE response queue
 *	@q: the ingress queue to process
 *	@budget: how many responses can be processed in this round
 *
 *	Process responses from an SGE response queue up to the supplied budget.
 *	Responses include received packets as well as control messages from FW
 *	or HW.
 *
 *	Additionally choose the interrupt holdoff time for the next interrupt
 *	on this queue.  If the system is under memory shortage use a fairly
 *	long delay to help recovery.
 */
static int process_responses(struct sge_rspq *q, int budget)
{
	int ret, rsp_type;
	int budget_left = budget;
	const struct rsp_ctrl *rc;
	struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq);
2155 2156
	struct adapter *adapter = q->adap;
	struct sge *s = &adapter->sge;
2157 2158 2159 2160 2161 2162

	while (likely(budget_left)) {
		rc = (void *)q->cur_desc + (q->iqe_len - sizeof(*rc));
		if (!is_new_response(rc, q))
			break;

2163
		dma_rmb();
2164 2165
		rsp_type = RSPD_TYPE_G(rc->type_gen);
		if (likely(rsp_type == RSPD_TYPE_FLBUF_X)) {
2166
			struct page_frag *fp;
2167 2168 2169 2170
			struct pkt_gl si;
			const struct rx_sw_desc *rsd;
			u32 len = ntohl(rc->pldbuflen_qid), bufsz, frags;

2171
			if (len & RSPD_NEWBUF_F) {
2172 2173 2174 2175
				if (likely(q->offset > 0)) {
					free_rx_bufs(q->adap, &rxq->fl, 1);
					q->offset = 0;
				}
2176
				len = RSPD_LEN_G(len);
2177 2178 2179 2180 2181 2182
			}
			si.tot_len = len;

			/* gather packet fragments */
			for (frags = 0, fp = si.frags; ; frags++, fp++) {
				rsd = &rxq->fl.sdesc[rxq->fl.cidx];
2183
				bufsz = get_buf_size(adapter, rsd);
2184
				fp->page = rsd->page;
2185 2186 2187
				fp->offset = q->offset;
				fp->size = min(bufsz, len);
				len -= fp->size;
2188 2189 2190 2191 2192
				if (!len)
					break;
				unmap_rx_buf(q->adap, &rxq->fl);
			}

2193 2194
			si.sgetstamp = SGE_TIMESTAMP_G(
					be64_to_cpu(rc->last_flit));
2195 2196 2197 2198 2199 2200
			/*
			 * Last buffer remains mapped so explicitly make it
			 * coherent for CPU access.
			 */
			dma_sync_single_for_cpu(q->adap->pdev_dev,
						get_buf_addr(rsd),
2201
						fp->size, DMA_FROM_DEVICE);
2202 2203

			si.va = page_address(si.frags[0].page) +
2204
				si.frags[0].offset;
2205 2206 2207 2208 2209
			prefetch(si.va);

			si.nfrags = frags + 1;
			ret = q->handler(q, q->cur_desc, &si);
			if (likely(ret == 0))
2210
				q->offset += ALIGN(fp->size, s->fl_align);
2211 2212
			else
				restore_rx_bufs(&si, &rxq->fl, frags);
2213
		} else if (likely(rsp_type == RSPD_TYPE_CPL_X)) {
2214 2215 2216 2217 2218 2219 2220
			ret = q->handler(q, q->cur_desc, NULL);
		} else {
			ret = q->handler(q, (const __be64 *)rc, CXGB4_MSG_AN);
		}

		if (unlikely(ret)) {
			/* couldn't process descriptor, back off for recovery */
2221
			q->next_intr_params = QINTR_TIMER_IDX_V(NOMEM_TMR_IDX);
2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233
			break;
		}

		rspq_next(q);
		budget_left--;
	}

	if (q->offset >= 0 && rxq->fl.size - rxq->fl.avail >= 16)
		__refill_fl(q->adap, &rxq->fl);
	return budget - budget_left;
}

2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244
#ifdef CONFIG_NET_RX_BUSY_POLL
int cxgb_busy_poll(struct napi_struct *napi)
{
	struct sge_rspq *q = container_of(napi, struct sge_rspq, napi);
	unsigned int params, work_done;
	u32 val;

	if (!cxgb_poll_lock_poll(q))
		return LL_FLUSH_BUSY;

	work_done = process_responses(q, 4);
2245
	params = QINTR_TIMER_IDX_V(TIMERREG_COUNTER0_X) | QINTR_CNT_EN_V(1);
2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265
	q->next_intr_params = params;
	val = CIDXINC_V(work_done) | SEINTARM_V(params);

	/* If we don't have access to the new User GTS (T5+), use the old
	 * doorbell mechanism; otherwise use the new BAR2 mechanism.
	 */
	if (unlikely(!q->bar2_addr))
		t4_write_reg(q->adap, MYPF_REG(SGE_PF_GTS_A),
			     val | INGRESSQID_V((u32)q->cntxt_id));
	else {
		writel(val | INGRESSQID_V(q->bar2_qid),
		       q->bar2_addr + SGE_UDB_GTS);
		wmb();
	}

	cxgb_poll_unlock_poll(q);
	return work_done;
}
#endif /* CONFIG_NET_RX_BUSY_POLL */

2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280
/**
 *	napi_rx_handler - the NAPI handler for Rx processing
 *	@napi: the napi instance
 *	@budget: how many packets we can process in this round
 *
 *	Handler for new data events when using NAPI.  This does not need any
 *	locking or protection from interrupts as data interrupts are off at
 *	this point and other adapter interrupts do not interfere (the latter
 *	in not a concern at all with MSI-X as non-data interrupts then have
 *	a separate handler).
 */
static int napi_rx_handler(struct napi_struct *napi, int budget)
{
	unsigned int params;
	struct sge_rspq *q = container_of(napi, struct sge_rspq, napi);
2281
	int work_done;
2282
	u32 val;
2283

2284 2285 2286 2287
	if (!cxgb_poll_lock_napi(q))
		return budget;

	work_done = process_responses(q, budget);
2288
	if (likely(work_done < budget)) {
2289 2290
		int timer_index;

2291
		napi_complete(napi);
2292
		timer_index = QINTR_TIMER_IDX_G(q->next_intr_params);
2293 2294 2295 2296 2297 2298 2299 2300 2301

		if (q->adaptive_rx) {
			if (work_done > max(timer_pkt_quota[timer_index],
					    MIN_NAPI_WORK))
				timer_index = (timer_index + 1);
			else
				timer_index = timer_index - 1;

			timer_index = clamp(timer_index, 0, SGE_TIMERREGS - 1);
2302 2303 2304
			q->next_intr_params =
					QINTR_TIMER_IDX_V(timer_index) |
					QINTR_CNT_EN_V(0);
2305 2306 2307 2308 2309
			params = q->next_intr_params;
		} else {
			params = q->next_intr_params;
			q->next_intr_params = q->intr_params;
		}
2310
	} else
2311
		params = QINTR_TIMER_IDX_V(7);
2312

2313
	val = CIDXINC_V(work_done) | SEINTARM_V(params);
2314 2315 2316 2317 2318

	/* If we don't have access to the new User GTS (T5+), use the old
	 * doorbell mechanism; otherwise use the new BAR2 mechanism.
	 */
	if (unlikely(q->bar2_addr == NULL)) {
2319 2320
		t4_write_reg(q->adap, MYPF_REG(SGE_PF_GTS_A),
			     val | INGRESSQID_V((u32)q->cntxt_id));
2321
	} else {
2322
		writel(val | INGRESSQID_V(q->bar2_qid),
2323
		       q->bar2_addr + SGE_UDB_GTS);
2324 2325
		wmb();
	}
2326
	cxgb_poll_unlock_napi(q);
2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349
	return work_done;
}

/*
 * The MSI-X interrupt handler for an SGE response queue.
 */
irqreturn_t t4_sge_intr_msix(int irq, void *cookie)
{
	struct sge_rspq *q = cookie;

	napi_schedule(&q->napi);
	return IRQ_HANDLED;
}

/*
 * Process the indirect interrupt entries in the interrupt queue and kick off
 * NAPI for each queue that has generated an entry.
 */
static unsigned int process_intrq(struct adapter *adap)
{
	unsigned int credits;
	const struct rsp_ctrl *rc;
	struct sge_rspq *q = &adap->sge.intrq;
2350
	u32 val;
2351 2352 2353 2354 2355 2356 2357

	spin_lock(&adap->sge.intrq_lock);
	for (credits = 0; ; credits++) {
		rc = (void *)q->cur_desc + (q->iqe_len - sizeof(*rc));
		if (!is_new_response(rc, q))
			break;

2358
		dma_rmb();
2359
		if (RSPD_TYPE_G(rc->type_gen) == RSPD_TYPE_INTR_X) {
2360 2361
			unsigned int qid = ntohl(rc->pldbuflen_qid);

2362
			qid -= adap->sge.ingr_start;
2363 2364 2365 2366 2367 2368
			napi_schedule(&adap->sge.ingr_map[qid]->napi);
		}

		rspq_next(q);
	}

2369
	val =  CIDXINC_V(credits) | SEINTARM_V(q->intr_params);
2370 2371 2372 2373 2374

	/* If we don't have access to the new User GTS (T5+), use the old
	 * doorbell mechanism; otherwise use the new BAR2 mechanism.
	 */
	if (unlikely(q->bar2_addr == NULL)) {
2375 2376
		t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
			     val | INGRESSQID_V(q->cntxt_id));
2377
	} else {
2378
		writel(val | INGRESSQID_V(q->bar2_qid),
2379
		       q->bar2_addr + SGE_UDB_GTS);
2380 2381
		wmb();
	}
2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393
	spin_unlock(&adap->sge.intrq_lock);
	return credits;
}

/*
 * The MSI interrupt handler, which handles data events from SGE response queues
 * as well as error and other async events as they all use the same MSI vector.
 */
static irqreturn_t t4_intr_msi(int irq, void *cookie)
{
	struct adapter *adap = cookie;

2394 2395
	if (adap->flags & MASTER_PF)
		t4_slow_intr_handler(adap);
2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408
	process_intrq(adap);
	return IRQ_HANDLED;
}

/*
 * Interrupt handler for legacy INTx interrupts.
 * Handles data events from SGE response queues as well as error and other
 * async events as they all use the same interrupt line.
 */
static irqreturn_t t4_intr_intx(int irq, void *cookie)
{
	struct adapter *adap = cookie;

2409
	t4_write_reg(adap, MYPF_REG(PCIE_PF_CLI_A), 0);
2410 2411
	if (((adap->flags & MASTER_PF) && t4_slow_intr_handler(adap)) |
	    process_intrq(adap))
2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434
		return IRQ_HANDLED;
	return IRQ_NONE;             /* probably shared interrupt */
}

/**
 *	t4_intr_handler - select the top-level interrupt handler
 *	@adap: the adapter
 *
 *	Selects the top-level interrupt handler based on the type of interrupts
 *	(MSI-X, MSI, or INTx).
 */
irq_handler_t t4_intr_handler(struct adapter *adap)
{
	if (adap->flags & USING_MSIX)
		return t4_sge_intr_msix;
	if (adap->flags & USING_MSI)
		return t4_intr_msi;
	return t4_intr_intx;
}

static void sge_rx_timer_cb(unsigned long data)
{
	unsigned long m;
2435
	unsigned int i;
2436 2437 2438
	struct adapter *adap = (struct adapter *)data;
	struct sge *s = &adap->sge;

2439
	for (i = 0; i < BITS_TO_LONGS(s->egr_sz); i++)
2440 2441 2442 2443 2444 2445
		for (m = s->starving_fl[i]; m; m &= m - 1) {
			struct sge_eth_rxq *rxq;
			unsigned int id = __ffs(m) + i * BITS_PER_LONG;
			struct sge_fl *fl = s->egr_map[id];

			clear_bit(id, s->starving_fl);
2446
			smp_mb__after_atomic();
2447

2448
			if (fl_starving(adap, fl)) {
2449 2450 2451 2452 2453 2454 2455
				rxq = container_of(fl, struct sge_eth_rxq, fl);
				if (napi_reschedule(&rxq->rspq.napi))
					fl->starving++;
				else
					set_bit(id, s->starving_fl);
			}
		}
2456 2457 2458 2459 2460 2461
	/* The remainder of the SGE RX Timer Callback routine is dedicated to
	 * global Master PF activities like checking for chip ingress stalls,
	 * etc.
	 */
	if (!(adap->flags & MASTER_PF))
		goto done;
2462

2463
	t4_idma_monitor(adap, &s->idma_monitor, HZ, RX_QCHECK_PERIOD);
2464

2465
done:
2466 2467 2468 2469 2470 2471 2472 2473 2474 2475
	mod_timer(&s->rx_timer, jiffies + RX_QCHECK_PERIOD);
}

static void sge_tx_timer_cb(unsigned long data)
{
	unsigned long m;
	unsigned int i, budget;
	struct adapter *adap = (struct adapter *)data;
	struct sge *s = &adap->sge;

2476
	for (i = 0; i < BITS_TO_LONGS(s->egr_sz); i++)
2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512
		for (m = s->txq_maperr[i]; m; m &= m - 1) {
			unsigned long id = __ffs(m) + i * BITS_PER_LONG;
			struct sge_ofld_txq *txq = s->egr_map[id];

			clear_bit(id, s->txq_maperr);
			tasklet_schedule(&txq->qresume_tsk);
		}

	budget = MAX_TIMER_TX_RECLAIM;
	i = s->ethtxq_rover;
	do {
		struct sge_eth_txq *q = &s->ethtxq[i];

		if (q->q.in_use &&
		    time_after_eq(jiffies, q->txq->trans_start + HZ / 100) &&
		    __netif_tx_trylock(q->txq)) {
			int avail = reclaimable(&q->q);

			if (avail) {
				if (avail > budget)
					avail = budget;

				free_tx_desc(adap, &q->q, avail, true);
				q->q.in_use -= avail;
				budget -= avail;
			}
			__netif_tx_unlock(q->txq);
		}

		if (++i >= s->ethqsets)
			i = 0;
	} while (budget && i != s->ethtxq_rover);
	s->ethtxq_rover = i;
	mod_timer(&s->tx_timer, jiffies + (budget ? TX_QCHECK_PERIOD : 2));
}

2513
/**
2514 2515 2516 2517 2518
 *	bar2_address - return the BAR2 address for an SGE Queue's Registers
 *	@adapter: the adapter
 *	@qid: the SGE Queue ID
 *	@qtype: the SGE Queue Type (Egress or Ingress)
 *	@pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues
2519
 *
2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532
 *	Returns the BAR2 address for the SGE Queue Registers associated with
 *	@qid.  If BAR2 SGE Registers aren't available, returns NULL.  Also
 *	returns the BAR2 Queue ID to be used with writes to the BAR2 SGE
 *	Queue Registers.  If the BAR2 Queue ID is 0, then "Inferred Queue ID"
 *	Registers are supported (e.g. the Write Combining Doorbell Buffer).
 */
static void __iomem *bar2_address(struct adapter *adapter,
				  unsigned int qid,
				  enum t4_bar2_qtype qtype,
				  unsigned int *pbar2_qid)
{
	u64 bar2_qoffset;
	int ret;
2533

2534
	ret = t4_bar2_sge_qregs(adapter, qid, qtype, 0,
2535 2536 2537
				&bar2_qoffset, pbar2_qid);
	if (ret)
		return NULL;
2538

2539
	return adapter->bar2 + bar2_qoffset;
2540 2541
}

2542 2543 2544
/* @intr_idx: MSI/MSI-X vector if >=0, -(absolute qid + 1) if < 0
 * @cong: < 0 -> no congestion feedback, >= 0 -> congestion channel map
 */
2545 2546
int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
		     struct net_device *dev, int intr_idx,
2547
		     struct sge_fl *fl, rspq_handler_t hnd, int cong)
2548 2549 2550
{
	int ret, flsz = 0;
	struct fw_iq_cmd c;
2551
	struct sge *s = &adap->sge;
2552 2553 2554 2555 2556 2557
	struct port_info *pi = netdev_priv(dev);

	/* Size needs to be multiple of 16, including status entry. */
	iq->size = roundup(iq->size, 16);

	iq->desc = alloc_ring(adap->pdev_dev, iq->size, iq->iqe_len, 0,
2558
			      &iq->phys_addr, NULL, 0, NUMA_NO_NODE);
2559 2560 2561 2562
	if (!iq->desc)
		return -ENOMEM;

	memset(&c, 0, sizeof(c));
2563 2564
	c.op_to_vfn = htonl(FW_CMD_OP_V(FW_IQ_CMD) | FW_CMD_REQUEST_F |
			    FW_CMD_WRITE_F | FW_CMD_EXEC_F |
2565
			    FW_IQ_CMD_PFN_V(adap->pf) | FW_IQ_CMD_VFN_V(0));
2566
	c.alloc_to_len16 = htonl(FW_IQ_CMD_ALLOC_F | FW_IQ_CMD_IQSTART_F |
2567
				 FW_LEN16(c));
2568 2569
	c.type_to_iqandstindex = htonl(FW_IQ_CMD_TYPE_V(FW_IQ_TYPE_FL_INT_CAP) |
		FW_IQ_CMD_IQASYNCH_V(fwevtq) | FW_IQ_CMD_VIID_V(pi->viid) |
2570 2571
		FW_IQ_CMD_IQANDST_V(intr_idx < 0) |
		FW_IQ_CMD_IQANUD_V(UPDATEDELIVERY_INTERRUPT_X) |
2572
		FW_IQ_CMD_IQANDSTINDEX_V(intr_idx >= 0 ? intr_idx :
2573
							-intr_idx - 1));
2574 2575 2576 2577
	c.iqdroprss_to_iqesize = htons(FW_IQ_CMD_IQPCIECH_V(pi->tx_chan) |
		FW_IQ_CMD_IQGTSMODE_F |
		FW_IQ_CMD_IQINTCNTTHRESH_V(iq->pktcnt_idx) |
		FW_IQ_CMD_IQESIZE_V(ilog2(iq->iqe_len) - 4));
2578 2579
	c.iqsize = htons(iq->size);
	c.iqaddr = cpu_to_be64(iq->phys_addr);
2580 2581
	if (cong >= 0)
		c.iqns_to_fl0congen = htonl(FW_IQ_CMD_IQFLINTCONGEN_F);
2582 2583

	if (fl) {
2584 2585
		enum chip_type chip = CHELSIO_CHIP_VERSION(adap->params.chip);

2586 2587 2588 2589 2590 2591 2592 2593 2594
		/* Allocate the ring for the hardware free list (with space
		 * for its status page) along with the associated software
		 * descriptor ring.  The free list size needs to be a multiple
		 * of the Egress Queue Unit and at least 2 Egress Units larger
		 * than the SGE's Egress Congrestion Threshold
		 * (fl_starve_thres - 1).
		 */
		if (fl->size < s->fl_starve_thres - 1 + 2 * 8)
			fl->size = s->fl_starve_thres - 1 + 2 * 8;
2595 2596 2597
		fl->size = roundup(fl->size, 8);
		fl->desc = alloc_ring(adap->pdev_dev, fl->size, sizeof(__be64),
				      sizeof(struct rx_sw_desc), &fl->addr,
2598
				      &fl->sdesc, s->stat_len, NUMA_NO_NODE);
2599 2600 2601
		if (!fl->desc)
			goto fl_nomem;

2602
		flsz = fl->size / 8 + s->stat_len / sizeof(struct tx_desc);
2603 2604 2605 2606 2607 2608 2609 2610 2611
		c.iqns_to_fl0congen |= htonl(FW_IQ_CMD_FL0PACKEN_F |
					     FW_IQ_CMD_FL0FETCHRO_F |
					     FW_IQ_CMD_FL0DATARO_F |
					     FW_IQ_CMD_FL0PADEN_F);
		if (cong >= 0)
			c.iqns_to_fl0congen |=
				htonl(FW_IQ_CMD_FL0CNGCHMAP_V(cong) |
				      FW_IQ_CMD_FL0CONGCIF_F |
				      FW_IQ_CMD_FL0CONGEN_F);
2612 2613
		c.fl0dcaen_to_fl0cidxfthresh =
			htons(FW_IQ_CMD_FL0FBMIN_V(FETCHBURSTMIN_64B_X) |
2614 2615 2616
			      FW_IQ_CMD_FL0FBMAX_V((chip <= CHELSIO_T5) ?
						   FETCHBURSTMAX_512B_X :
						   FETCHBURSTMAX_256B_X));
2617 2618 2619 2620
		c.fl0size = htons(flsz);
		c.fl0addr = cpu_to_be64(fl->addr);
	}

2621
	ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
2622 2623 2624 2625 2626 2627 2628 2629 2630 2631
	if (ret)
		goto err;

	netif_napi_add(dev, &iq->napi, napi_rx_handler, 64);
	iq->cur_desc = iq->desc;
	iq->cidx = 0;
	iq->gen = 1;
	iq->next_intr_params = iq->intr_params;
	iq->cntxt_id = ntohs(c.iqid);
	iq->abs_id = ntohs(c.physiqid);
2632 2633 2634 2635
	iq->bar2_addr = bar2_address(adap,
				     iq->cntxt_id,
				     T4_BAR2_QTYPE_INGRESS,
				     &iq->bar2_qid);
2636 2637 2638 2639 2640 2641 2642
	iq->size--;                           /* subtract status entry */
	iq->netdev = dev;
	iq->handler = hnd;

	/* set offset to -1 to distinguish ingress queues without FL */
	iq->offset = fl ? 0 : -1;

2643
	adap->sge.ingr_map[iq->cntxt_id - adap->sge.ingr_start] = iq;
2644 2645

	if (fl) {
2646
		fl->cntxt_id = ntohs(c.fl0id);
2647 2648 2649
		fl->avail = fl->pend_cred = 0;
		fl->pidx = fl->cidx = 0;
		fl->alloc_failed = fl->large_alloc_failed = fl->starving = 0;
2650
		adap->sge.egr_map[fl->cntxt_id - adap->sge.egr_start] = fl;
2651

2652 2653
		/* Note, we must initialize the BAR2 Free List User Doorbell
		 * information before refilling the Free List!
2654
		 */
2655 2656 2657 2658
		fl->bar2_addr = bar2_address(adap,
					     fl->cntxt_id,
					     T4_BAR2_QTYPE_EGRESS,
					     &fl->bar2_qid);
2659 2660
		refill_fl(adap, fl, fl_cap(fl), GFP_KERNEL);
	}
2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687

	/* For T5 and later we attempt to set up the Congestion Manager values
	 * of the new RX Ethernet Queue.  This should really be handled by
	 * firmware because it's more complex than any host driver wants to
	 * get involved with and it's different per chip and this is almost
	 * certainly wrong.  Firmware would be wrong as well, but it would be
	 * a lot easier to fix in one place ...  For now we do something very
	 * simple (and hopefully less wrong).
	 */
	if (!is_t4(adap->params.chip) && cong >= 0) {
		u32 param, val;
		int i;

		param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
			 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
			 FW_PARAMS_PARAM_YZ_V(iq->cntxt_id));
		if (cong == 0) {
			val = CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_QUEUE_X);
		} else {
			val =
			    CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_CHANNEL_X);
			for (i = 0; i < 4; i++) {
				if (cong & (1 << i))
					val |=
					     CONMCTXT_CNGCHMAP_V(1 << (i << 2));
			}
		}
2688
		ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1,
2689 2690 2691 2692 2693 2694 2695
				    &param, &val);
		if (ret)
			dev_warn(adap->pdev_dev, "Failed to set Congestion"
				 " Manager Context for Ingress Queue %d: %d\n",
				 iq->cntxt_id, -ret);
	}

2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717
	return 0;

fl_nomem:
	ret = -ENOMEM;
err:
	if (iq->desc) {
		dma_free_coherent(adap->pdev_dev, iq->size * iq->iqe_len,
				  iq->desc, iq->phys_addr);
		iq->desc = NULL;
	}
	if (fl && fl->desc) {
		kfree(fl->sdesc);
		fl->sdesc = NULL;
		dma_free_coherent(adap->pdev_dev, flsz * sizeof(struct tx_desc),
				  fl->desc, fl->addr);
		fl->desc = NULL;
	}
	return ret;
}

static void init_txq(struct adapter *adap, struct sge_txq *q, unsigned int id)
{
2718
	q->cntxt_id = id;
2719 2720 2721 2722
	q->bar2_addr = bar2_address(adap,
				    q->cntxt_id,
				    T4_BAR2_QTYPE_EGRESS,
				    &q->bar2_qid);
2723 2724 2725 2726
	q->in_use = 0;
	q->cidx = q->pidx = 0;
	q->stops = q->restarts = 0;
	q->stat = (void *)&q->desc[q->size];
2727
	spin_lock_init(&q->db_lock);
2728
	adap->sge.egr_map[id - adap->sge.egr_start] = q;
2729 2730 2731 2732 2733 2734 2735 2736
}

int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq,
			 struct net_device *dev, struct netdev_queue *netdevq,
			 unsigned int iqid)
{
	int ret, nentries;
	struct fw_eq_eth_cmd c;
2737
	struct sge *s = &adap->sge;
2738 2739 2740
	struct port_info *pi = netdev_priv(dev);

	/* Add status entries */
2741
	nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
2742 2743 2744

	txq->q.desc = alloc_ring(adap->pdev_dev, txq->q.size,
			sizeof(struct tx_desc), sizeof(struct tx_sw_desc),
2745
			&txq->q.phys_addr, &txq->q.sdesc, s->stat_len,
2746
			netdev_queue_numa_node_read(netdevq));
2747 2748 2749 2750
	if (!txq->q.desc)
		return -ENOMEM;

	memset(&c, 0, sizeof(c));
2751 2752
	c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_ETH_CMD) | FW_CMD_REQUEST_F |
			    FW_CMD_WRITE_F | FW_CMD_EXEC_F |
2753
			    FW_EQ_ETH_CMD_PFN_V(adap->pf) |
2754 2755 2756 2757 2758
			    FW_EQ_ETH_CMD_VFN_V(0));
	c.alloc_to_len16 = htonl(FW_EQ_ETH_CMD_ALLOC_F |
				 FW_EQ_ETH_CMD_EQSTART_F | FW_LEN16(c));
	c.viid_pkd = htonl(FW_EQ_ETH_CMD_AUTOEQUEQE_F |
			   FW_EQ_ETH_CMD_VIID_V(pi->viid));
2759 2760 2761 2762 2763 2764 2765 2766 2767
	c.fetchszm_to_iqid =
		htonl(FW_EQ_ETH_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) |
		      FW_EQ_ETH_CMD_PCIECHN_V(pi->tx_chan) |
		      FW_EQ_ETH_CMD_FETCHRO_F | FW_EQ_ETH_CMD_IQID_V(iqid));
	c.dcaen_to_eqsize =
		htonl(FW_EQ_ETH_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) |
		      FW_EQ_ETH_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
		      FW_EQ_ETH_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) |
		      FW_EQ_ETH_CMD_EQSIZE_V(nentries));
2768 2769
	c.eqaddr = cpu_to_be64(txq->q.phys_addr);

2770
	ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
2771 2772 2773 2774 2775 2776 2777 2778 2779 2780
	if (ret) {
		kfree(txq->q.sdesc);
		txq->q.sdesc = NULL;
		dma_free_coherent(adap->pdev_dev,
				  nentries * sizeof(struct tx_desc),
				  txq->q.desc, txq->q.phys_addr);
		txq->q.desc = NULL;
		return ret;
	}

2781
	init_txq(adap, &txq->q, FW_EQ_ETH_CMD_EQID_G(ntohl(c.eqid_pkd)));
2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793
	txq->txq = netdevq;
	txq->tso = txq->tx_cso = txq->vlan_ins = 0;
	txq->mapping_err = 0;
	return 0;
}

int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq,
			  struct net_device *dev, unsigned int iqid,
			  unsigned int cmplqid)
{
	int ret, nentries;
	struct fw_eq_ctrl_cmd c;
2794
	struct sge *s = &adap->sge;
2795 2796 2797
	struct port_info *pi = netdev_priv(dev);

	/* Add status entries */
2798
	nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
2799 2800 2801

	txq->q.desc = alloc_ring(adap->pdev_dev, nentries,
				 sizeof(struct tx_desc), 0, &txq->q.phys_addr,
2802
				 NULL, 0, dev_to_node(adap->pdev_dev));
2803 2804 2805
	if (!txq->q.desc)
		return -ENOMEM;

2806 2807
	c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_CTRL_CMD) | FW_CMD_REQUEST_F |
			    FW_CMD_WRITE_F | FW_CMD_EXEC_F |
2808
			    FW_EQ_CTRL_CMD_PFN_V(adap->pf) |
2809 2810 2811 2812
			    FW_EQ_CTRL_CMD_VFN_V(0));
	c.alloc_to_len16 = htonl(FW_EQ_CTRL_CMD_ALLOC_F |
				 FW_EQ_CTRL_CMD_EQSTART_F | FW_LEN16(c));
	c.cmpliqid_eqid = htonl(FW_EQ_CTRL_CMD_CMPLIQID_V(cmplqid));
2813
	c.physeqid_pkd = htonl(0);
2814 2815 2816 2817 2818 2819 2820 2821 2822
	c.fetchszm_to_iqid =
		htonl(FW_EQ_CTRL_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) |
		      FW_EQ_CTRL_CMD_PCIECHN_V(pi->tx_chan) |
		      FW_EQ_CTRL_CMD_FETCHRO_F | FW_EQ_CTRL_CMD_IQID_V(iqid));
	c.dcaen_to_eqsize =
		htonl(FW_EQ_CTRL_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) |
		      FW_EQ_CTRL_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
		      FW_EQ_CTRL_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) |
		      FW_EQ_CTRL_CMD_EQSIZE_V(nentries));
2823 2824
	c.eqaddr = cpu_to_be64(txq->q.phys_addr);

2825
	ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
2826 2827 2828 2829 2830 2831 2832 2833
	if (ret) {
		dma_free_coherent(adap->pdev_dev,
				  nentries * sizeof(struct tx_desc),
				  txq->q.desc, txq->q.phys_addr);
		txq->q.desc = NULL;
		return ret;
	}

2834
	init_txq(adap, &txq->q, FW_EQ_CTRL_CMD_EQID_G(ntohl(c.cmpliqid_eqid)));
2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846
	txq->adap = adap;
	skb_queue_head_init(&txq->sendq);
	tasklet_init(&txq->qresume_tsk, restart_ctrlq, (unsigned long)txq);
	txq->full = 0;
	return 0;
}

int t4_sge_alloc_ofld_txq(struct adapter *adap, struct sge_ofld_txq *txq,
			  struct net_device *dev, unsigned int iqid)
{
	int ret, nentries;
	struct fw_eq_ofld_cmd c;
2847
	struct sge *s = &adap->sge;
2848 2849 2850
	struct port_info *pi = netdev_priv(dev);

	/* Add status entries */
2851
	nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
2852 2853 2854

	txq->q.desc = alloc_ring(adap->pdev_dev, txq->q.size,
			sizeof(struct tx_desc), sizeof(struct tx_sw_desc),
2855
			&txq->q.phys_addr, &txq->q.sdesc, s->stat_len,
2856
			NUMA_NO_NODE);
2857 2858 2859 2860
	if (!txq->q.desc)
		return -ENOMEM;

	memset(&c, 0, sizeof(c));
2861 2862
	c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_OFLD_CMD) | FW_CMD_REQUEST_F |
			    FW_CMD_WRITE_F | FW_CMD_EXEC_F |
2863
			    FW_EQ_OFLD_CMD_PFN_V(adap->pf) |
2864 2865 2866
			    FW_EQ_OFLD_CMD_VFN_V(0));
	c.alloc_to_len16 = htonl(FW_EQ_OFLD_CMD_ALLOC_F |
				 FW_EQ_OFLD_CMD_EQSTART_F | FW_LEN16(c));
2867 2868 2869 2870 2871 2872 2873 2874 2875
	c.fetchszm_to_iqid =
		htonl(FW_EQ_OFLD_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) |
		      FW_EQ_OFLD_CMD_PCIECHN_V(pi->tx_chan) |
		      FW_EQ_OFLD_CMD_FETCHRO_F | FW_EQ_OFLD_CMD_IQID_V(iqid));
	c.dcaen_to_eqsize =
		htonl(FW_EQ_OFLD_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) |
		      FW_EQ_OFLD_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
		      FW_EQ_OFLD_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) |
		      FW_EQ_OFLD_CMD_EQSIZE_V(nentries));
2876 2877
	c.eqaddr = cpu_to_be64(txq->q.phys_addr);

2878
	ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
2879 2880 2881 2882 2883 2884 2885 2886 2887 2888
	if (ret) {
		kfree(txq->q.sdesc);
		txq->q.sdesc = NULL;
		dma_free_coherent(adap->pdev_dev,
				  nentries * sizeof(struct tx_desc),
				  txq->q.desc, txq->q.phys_addr);
		txq->q.desc = NULL;
		return ret;
	}

2889
	init_txq(adap, &txq->q, FW_EQ_OFLD_CMD_EQID_G(ntohl(c.eqid_pkd)));
2890 2891 2892 2893 2894 2895 2896 2897 2898 2899
	txq->adap = adap;
	skb_queue_head_init(&txq->sendq);
	tasklet_init(&txq->qresume_tsk, restart_ofldq, (unsigned long)txq);
	txq->full = 0;
	txq->mapping_err = 0;
	return 0;
}

static void free_txq(struct adapter *adap, struct sge_txq *q)
{
2900 2901
	struct sge *s = &adap->sge;

2902
	dma_free_coherent(adap->pdev_dev,
2903
			  q->size * sizeof(struct tx_desc) + s->stat_len,
2904 2905 2906 2907 2908 2909 2910 2911 2912
			  q->desc, q->phys_addr);
	q->cntxt_id = 0;
	q->sdesc = NULL;
	q->desc = NULL;
}

static void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq,
			 struct sge_fl *fl)
{
2913
	struct sge *s = &adap->sge;
2914 2915
	unsigned int fl_id = fl ? fl->cntxt_id : 0xffff;

2916
	adap->sge.ingr_map[rq->cntxt_id - adap->sge.ingr_start] = NULL;
2917
	t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
2918
		   rq->cntxt_id, fl_id, 0xffff);
2919 2920
	dma_free_coherent(adap->pdev_dev, (rq->size + 1) * rq->iqe_len,
			  rq->desc, rq->phys_addr);
2921
	napi_hash_del(&rq->napi);
2922 2923 2924 2925 2926 2927 2928
	netif_napi_del(&rq->napi);
	rq->netdev = NULL;
	rq->cntxt_id = rq->abs_id = 0;
	rq->desc = NULL;

	if (fl) {
		free_rx_bufs(adap, fl, fl->avail);
2929
		dma_free_coherent(adap->pdev_dev, fl->size * 8 + s->stat_len,
2930 2931 2932 2933 2934 2935 2936 2937
				  fl->desc, fl->addr);
		kfree(fl->sdesc);
		fl->sdesc = NULL;
		fl->cntxt_id = 0;
		fl->desc = NULL;
	}
}

2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953
/**
 *      t4_free_ofld_rxqs - free a block of consecutive Rx queues
 *      @adap: the adapter
 *      @n: number of queues
 *      @q: pointer to first queue
 *
 *      Release the resources of a consecutive block of offload Rx queues.
 */
void t4_free_ofld_rxqs(struct adapter *adap, int n, struct sge_ofld_rxq *q)
{
	for ( ; n; n--, q++)
		if (q->rspq.desc)
			free_rspq_fl(adap, &q->rspq,
				     q->fl.size ? &q->fl : NULL);
}

2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968
/**
 *	t4_free_sge_resources - free SGE resources
 *	@adap: the adapter
 *
 *	Frees resources used by the SGE queue sets.
 */
void t4_free_sge_resources(struct adapter *adap)
{
	int i;
	struct sge_eth_rxq *eq = adap->sge.ethrxq;
	struct sge_eth_txq *etq = adap->sge.ethtxq;

	/* clean up Ethernet Tx/Rx queues */
	for (i = 0; i < adap->sge.ethqsets; i++, eq++, etq++) {
		if (eq->rspq.desc)
2969 2970
			free_rspq_fl(adap, &eq->rspq,
				     eq->fl.size ? &eq->fl : NULL);
2971
		if (etq->q.desc) {
2972
			t4_eth_eq_free(adap, adap->mbox, adap->pf, 0,
2973
				       etq->q.cntxt_id);
2974 2975 2976 2977 2978 2979 2980
			free_tx_desc(adap, &etq->q, etq->q.in_use, true);
			kfree(etq->q.sdesc);
			free_txq(adap, &etq->q);
		}
	}

	/* clean up RDMA and iSCSI Rx queues */
2981 2982 2983
	t4_free_ofld_rxqs(adap, adap->sge.ofldqsets, adap->sge.ofldrxq);
	t4_free_ofld_rxqs(adap, adap->sge.rdmaqs, adap->sge.rdmarxq);
	t4_free_ofld_rxqs(adap, adap->sge.rdmaciqs, adap->sge.rdmaciq);
2984 2985 2986 2987 2988 2989 2990

	/* clean up offload Tx queues */
	for (i = 0; i < ARRAY_SIZE(adap->sge.ofldtxq); i++) {
		struct sge_ofld_txq *q = &adap->sge.ofldtxq[i];

		if (q->q.desc) {
			tasklet_kill(&q->qresume_tsk);
2991
			t4_ofld_eq_free(adap, adap->mbox, adap->pf, 0,
2992
					q->q.cntxt_id);
2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005
			free_tx_desc(adap, &q->q, q->q.in_use, false);
			kfree(q->q.sdesc);
			__skb_queue_purge(&q->sendq);
			free_txq(adap, &q->q);
		}
	}

	/* clean up control Tx queues */
	for (i = 0; i < ARRAY_SIZE(adap->sge.ctrlq); i++) {
		struct sge_ctrl_txq *cq = &adap->sge.ctrlq[i];

		if (cq->q.desc) {
			tasklet_kill(&cq->qresume_tsk);
3006
			t4_ctrl_eq_free(adap, adap->mbox, adap->pf, 0,
3007
					cq->q.cntxt_id);
3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019
			__skb_queue_purge(&cq->sendq);
			free_txq(adap, &cq->q);
		}
	}

	if (adap->sge.fw_evtq.desc)
		free_rspq_fl(adap, &adap->sge.fw_evtq, NULL);

	if (adap->sge.intrq.desc)
		free_rspq_fl(adap, &adap->sge.intrq, NULL);

	/* clear the reverse egress queue map */
3020 3021
	memset(adap->sge.egr_map, 0,
	       adap->sge.egr_sz * sizeof(*adap->sge.egr_map));
3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066
}

void t4_sge_start(struct adapter *adap)
{
	adap->sge.ethtxq_rover = 0;
	mod_timer(&adap->sge.rx_timer, jiffies + RX_QCHECK_PERIOD);
	mod_timer(&adap->sge.tx_timer, jiffies + TX_QCHECK_PERIOD);
}

/**
 *	t4_sge_stop - disable SGE operation
 *	@adap: the adapter
 *
 *	Stop tasklets and timers associated with the DMA engine.  Note that
 *	this is effective only if measures have been taken to disable any HW
 *	events that may restart them.
 */
void t4_sge_stop(struct adapter *adap)
{
	int i;
	struct sge *s = &adap->sge;

	if (in_interrupt())  /* actions below require waiting */
		return;

	if (s->rx_timer.function)
		del_timer_sync(&s->rx_timer);
	if (s->tx_timer.function)
		del_timer_sync(&s->tx_timer);

	for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++) {
		struct sge_ofld_txq *q = &s->ofldtxq[i];

		if (q->q.desc)
			tasklet_kill(&q->qresume_tsk);
	}
	for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) {
		struct sge_ctrl_txq *cq = &s->ctrlq[i];

		if (cq->q.desc)
			tasklet_kill(&cq->qresume_tsk);
	}
}

/**
3067
 *	t4_sge_init_soft - grab core SGE values needed by SGE code
3068 3069
 *	@adap: the adapter
 *
3070 3071
 *	We need to grab the SGE operating parameters that we need to have
 *	in order to do our job and make sure we can live with them.
3072
 */
3073 3074

static int t4_sge_init_soft(struct adapter *adap)
3075 3076
{
	struct sge *s = &adap->sge;
3077 3078 3079
	u32 fl_small_pg, fl_large_pg, fl_small_mtu, fl_large_mtu;
	u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5;
	u32 ingress_rx_threshold;
3080

3081 3082 3083 3084 3085
	/*
	 * Verify that CPL messages are going to the Ingress Queue for
	 * process_responses() and that only packet data is going to the
	 * Free Lists.
	 */
3086 3087
	if ((t4_read_reg(adap, SGE_CONTROL_A) & RXPKTCPLMODE_F) !=
	    RXPKTCPLMODE_V(RXPKTCPLMODE_SPLIT_X)) {
3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100
		dev_err(adap->pdev_dev, "bad SGE CPL MODE\n");
		return -EINVAL;
	}

	/*
	 * Validate the Host Buffer Register Array indices that we want to
	 * use ...
	 *
	 * XXX Note that we should really read through the Host Buffer Size
	 * XXX register array and find the indices of the Buffer Sizes which
	 * XXX meet our needs!
	 */
	#define READ_FL_BUF(x) \
3101
		t4_read_reg(adap, SGE_FL_BUFFER_SIZE0_A+(x)*sizeof(u32))
3102 3103 3104 3105 3106 3107

	fl_small_pg = READ_FL_BUF(RX_SMALL_PG_BUF);
	fl_large_pg = READ_FL_BUF(RX_LARGE_PG_BUF);
	fl_small_mtu = READ_FL_BUF(RX_SMALL_MTU_BUF);
	fl_large_mtu = READ_FL_BUF(RX_LARGE_MTU_BUF);

3108 3109 3110 3111 3112 3113
	/* We only bother using the Large Page logic if the Large Page Buffer
	 * is larger than our Page Size Buffer.
	 */
	if (fl_large_pg <= fl_small_pg)
		fl_large_pg = 0;

3114 3115
	#undef READ_FL_BUF

3116 3117 3118
	/* The Page Size Buffer must be exactly equal to our Page Size and the
	 * Large Page Size Buffer should be 0 (per above) or a power of 2.
	 */
3119
	if (fl_small_pg != PAGE_SIZE ||
3120
	    (fl_large_pg & (fl_large_pg-1)) != 0) {
3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138
		dev_err(adap->pdev_dev, "bad SGE FL page buffer sizes [%d, %d]\n",
			fl_small_pg, fl_large_pg);
		return -EINVAL;
	}
	if (fl_large_pg)
		s->fl_pg_order = ilog2(fl_large_pg) - PAGE_SHIFT;

	if (fl_small_mtu < FL_MTU_SMALL_BUFSIZE(adap) ||
	    fl_large_mtu < FL_MTU_LARGE_BUFSIZE(adap)) {
		dev_err(adap->pdev_dev, "bad SGE FL MTU sizes [%d, %d]\n",
			fl_small_mtu, fl_large_mtu);
		return -EINVAL;
	}

	/*
	 * Retrieve our RX interrupt holdoff timer values and counter
	 * threshold values from the SGE parameters.
	 */
3139 3140 3141
	timer_value_0_and_1 = t4_read_reg(adap, SGE_TIMER_VALUE_0_AND_1_A);
	timer_value_2_and_3 = t4_read_reg(adap, SGE_TIMER_VALUE_2_AND_3_A);
	timer_value_4_and_5 = t4_read_reg(adap, SGE_TIMER_VALUE_4_AND_5_A);
3142
	s->timer_val[0] = core_ticks_to_us(adap,
3143
		TIMERVALUE0_G(timer_value_0_and_1));
3144
	s->timer_val[1] = core_ticks_to_us(adap,
3145
		TIMERVALUE1_G(timer_value_0_and_1));
3146
	s->timer_val[2] = core_ticks_to_us(adap,
3147
		TIMERVALUE2_G(timer_value_2_and_3));
3148
	s->timer_val[3] = core_ticks_to_us(adap,
3149
		TIMERVALUE3_G(timer_value_2_and_3));
3150
	s->timer_val[4] = core_ticks_to_us(adap,
3151
		TIMERVALUE4_G(timer_value_4_and_5));
3152
	s->timer_val[5] = core_ticks_to_us(adap,
3153
		TIMERVALUE5_G(timer_value_4_and_5));
3154

3155 3156 3157 3158 3159
	ingress_rx_threshold = t4_read_reg(adap, SGE_INGRESS_RX_THRESHOLD_A);
	s->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold);
	s->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold);
	s->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold);
	s->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold);
3160 3161 3162 3163

	return 0;
}

3164 3165 3166 3167 3168 3169 3170
/**
 *     t4_sge_init - initialize SGE
 *     @adap: the adapter
 *
 *     Perform low-level SGE code initialization needed every time after a
 *     chip reset.
 */
3171 3172 3173
int t4_sge_init(struct adapter *adap)
{
	struct sge *s = &adap->sge;
3174 3175
	u32 sge_control, sge_control2, sge_conm_ctrl;
	unsigned int ingpadboundary, ingpackboundary;
3176
	int ret, egress_threshold;
3177 3178 3179 3180 3181

	/*
	 * Ingress Padding Boundary and Egress Status Page Size are set up by
	 * t4_fixup_host_params().
	 */
3182 3183 3184
	sge_control = t4_read_reg(adap, SGE_CONTROL_A);
	s->pktshift = PKTSHIFT_G(sge_control);
	s->stat_len = (sge_control & EGRSTATUSPAGESIZE_F) ? 128 : 64;
3185 3186 3187 3188 3189

	/* T4 uses a single control field to specify both the PCIe Padding and
	 * Packing Boundary.  T5 introduced the ability to specify these
	 * separately.  The actual Ingress Packet Data alignment boundary
	 * within Packed Buffer Mode is the maximum of these two
3190 3191 3192 3193 3194
	 * specifications.  (Note that it makes no real practical sense to
	 * have the Pading Boudary be larger than the Packing Boundary but you
	 * could set the chip up that way and, in fact, legacy T4 code would
	 * end doing this because it would initialize the Padding Boundary and
	 * leave the Packing Boundary initialized to 0 (16 bytes).)
3195
	 */
3196 3197
	ingpadboundary = 1 << (INGPADBOUNDARY_G(sge_control) +
			       INGPADBOUNDARY_SHIFT_X);
3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213
	if (is_t4(adap->params.chip)) {
		s->fl_align = ingpadboundary;
	} else {
		/* T5 has a different interpretation of one of the PCIe Packing
		 * Boundary values.
		 */
		sge_control2 = t4_read_reg(adap, SGE_CONTROL2_A);
		ingpackboundary = INGPACKBOUNDARY_G(sge_control2);
		if (ingpackboundary == INGPACKBOUNDARY_16B_X)
			ingpackboundary = 16;
		else
			ingpackboundary = 1 << (ingpackboundary +
						INGPACKBOUNDARY_SHIFT_X);

		s->fl_align = max(ingpadboundary, ingpackboundary);
	}
3214

3215
	ret = t4_sge_init_soft(adap);
3216 3217 3218 3219 3220 3221 3222 3223 3224
	if (ret < 0)
		return ret;

	/*
	 * A FL with <= fl_starve_thres buffers is starving and a periodic
	 * timer will attempt to refill it.  This needs to be larger than the
	 * SGE's Egress Congestion Threshold.  If it isn't, then we can get
	 * stuck waiting for new packets while the SGE is waiting for us to
	 * give it more Free List entries.  (Note that the SGE's Egress
3225 3226 3227 3228 3229
	 * Congestion Threshold is in units of 2 Free List pointers.) For T4,
	 * there was only a single field to control this.  For T5 there's the
	 * original field which now only applies to Unpacked Mode Free List
	 * buffers and a new field which only applies to Packed Mode Free List
	 * buffers.
3230
	 */
3231
	sge_conm_ctrl = t4_read_reg(adap, SGE_CONM_CTRL_A);
3232
	if (is_t4(adap->params.chip))
3233
		egress_threshold = EGRTHRESHOLD_G(sge_conm_ctrl);
3234
	else
3235
		egress_threshold = EGRTHRESHOLDPACKING_G(sge_conm_ctrl);
3236
	s->fl_starve_thres = 2*egress_threshold + 1;
3237

3238 3239
	t4_idma_monitor_init(adap, &s->idma_monitor);

3240 3241 3242
	/* Set up timers used for recuring callbacks to process RX and TX
	 * administrative tasks.
	 */
3243 3244
	setup_timer(&s->rx_timer, sge_rx_timer_cb, (unsigned long)adap);
	setup_timer(&s->tx_timer, sge_tx_timer_cb, (unsigned long)adap);
3245

3246
	spin_lock_init(&s->intrq_lock);
3247 3248

	return 0;
3249
}