cxgb4_main.c 177.2 KB
Newer Older
1 2 3
/*
 * This file is part of the Chelsio T4 Ethernet driver for Linux.
 *
4
 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/bitmap.h>
#include <linux/crc32.h>
#include <linux/ctype.h>
#include <linux/debugfs.h>
#include <linux/err.h>
#include <linux/etherdevice.h>
#include <linux/firmware.h>
44
#include <linux/if.h>
45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
#include <linux/if_vlan.h>
#include <linux/init.h>
#include <linux/log2.h>
#include <linux/mdio.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/mutex.h>
#include <linux/netdevice.h>
#include <linux/pci.h>
#include <linux/aer.h>
#include <linux/rtnetlink.h>
#include <linux/sched.h>
#include <linux/seq_file.h>
#include <linux/sockios.h>
#include <linux/vmalloc.h>
#include <linux/workqueue.h>
#include <net/neighbour.h>
#include <net/netevent.h>
63
#include <net/addrconf.h>
64
#include <net/bonding.h>
65
#include <linux/uaccess.h>
66
#include <linux/crash_dump.h>
67
#include <net/udp_tunnel.h>
68
#include <net/xfrm.h>
69 70

#include "cxgb4.h"
71
#include "cxgb4_filter.h"
72
#include "t4_regs.h"
73
#include "t4_values.h"
74 75
#include "t4_msg.h"
#include "t4fw_api.h"
76
#include "t4fw_version.h"
77
#include "cxgb4_dcb.h"
78
#include "srq.h"
79
#include "cxgb4_debugfs.h"
80
#include "clip_tbl.h"
81
#include "l2t.h"
82
#include "smt.h"
83
#include "sched.h"
84
#include "cxgb4_tc_u32.h"
85
#include "cxgb4_tc_flower.h"
86
#include "cxgb4_tc_mqprio.h"
87
#include "cxgb4_tc_matchall.h"
88
#include "cxgb4_ptp.h"
89
#include "cxgb4_cudbg.h"
90

91 92
char cxgb4_driver_name[] = KBUILD_MODNAME;

93 94 95
#ifdef DRV_VERSION
#undef DRV_VERSION
#endif
96
#define DRV_VERSION "2.0.0-ko"
97
const char cxgb4_driver_version[] = DRV_VERSION;
98
#define DRV_DESC "Chelsio T4/T5/T6 Network Driver"
99 100 101 102 103

#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
			 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
			 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)

104 105 106
/* Macros needed to support the PCI Device ID Table ...
 */
#define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
107
	static const struct pci_device_id cxgb4_pci_tbl[] = {
G
Ganesh Goudar 已提交
108 109 110
#define CXGB4_UNIFIED_PF 0x4

#define CH_PCI_DEVICE_ID_FUNCTION CXGB4_UNIFIED_PF
111

112 113 114 115 116 117
/* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is
 * called for both.
 */
#define CH_PCI_DEVICE_ID_FUNCTION2 0x0

#define CH_PCI_ID_TABLE_ENTRY(devid) \
G
Ganesh Goudar 已提交
118
		{PCI_VDEVICE(CHELSIO, (devid)), CXGB4_UNIFIED_PF}
119 120 121 122 123 124

#define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
		{ 0, } \
	}

#include "t4_pci_id_tbl.h"
125

126
#define FW4_FNAME "cxgb4/t4fw.bin"
S
Santosh Rastapur 已提交
127
#define FW5_FNAME "cxgb4/t5fw.bin"
128
#define FW6_FNAME "cxgb4/t6fw.bin"
129
#define FW4_CFNAME "cxgb4/t4-config.txt"
S
Santosh Rastapur 已提交
130
#define FW5_CFNAME "cxgb4/t5-config.txt"
131
#define FW6_CFNAME "cxgb4/t6-config.txt"
132 133 134 135
#define PHY_AQ1202_FIRMWARE "cxgb4/aq1202_fw.cld"
#define PHY_BCM84834_FIRMWARE "cxgb4/bcm8483.bin"
#define PHY_AQ1202_DEVICEID 0x4409
#define PHY_BCM84834_DEVICEID 0x4486
136 137 138 139 140 141

MODULE_DESCRIPTION(DRV_DESC);
MODULE_AUTHOR("Chelsio Communications");
MODULE_LICENSE("Dual BSD/GPL");
MODULE_VERSION(DRV_VERSION);
MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
142
MODULE_FIRMWARE(FW4_FNAME);
S
Santosh Rastapur 已提交
143
MODULE_FIRMWARE(FW5_FNAME);
144
MODULE_FIRMWARE(FW6_FNAME);
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159

/*
 * The driver uses the best interrupt scheme available on a platform in the
 * order MSI-X, MSI, legacy INTx interrupts.  This parameter determines which
 * of these schemes the driver may consider as follows:
 *
 * msi = 2: choose from among all three options
 * msi = 1: only consider MSI and INTx interrupts
 * msi = 0: force INTx interrupts
 */
static int msi = 2;

module_param(msi, int, 0644);
MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");

160 161 162 163 164 165 166 167 168 169 170 171 172 173
/*
 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
 * offset by 2 bytes in order to have the IP headers line up on 4-byte
 * boundaries.  This is a requirement for many architectures which will throw
 * a machine check fault if an attempt is made to access one of the 4-byte IP
 * header fields on a non-4-byte boundary.  And it's a major performance issue
 * even on some architectures which allow it like some implementations of the
 * x86 ISA.  However, some architectures don't mind this and for some very
 * edge-case performance sensitive applications (like forwarding large volumes
 * of small packets), setting this DMA offset to 0 will decrease the number of
 * PCI-E Bus transfers enough to measurably affect performance.
 */
static int rx_dma_offset = 2;

174 175 176 177 178 179 180 181 182 183 184
/* TX Queue select used to determine what algorithm to use for selecting TX
 * queue. Select between the kernel provided function (select_queue=0) or user
 * cxgb_select_queue function (select_queue=1)
 *
 * Default: select_queue=0
 */
static int select_queue;
module_param(select_queue, int, 0644);
MODULE_PARM_DESC(select_queue,
		 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method.");

185 186
static struct dentry *cxgb4_debugfs_root;

187 188
LIST_HEAD(adapter_list);
DEFINE_MUTEX(uld_mutex);
189

V
Vishal Kulkarni 已提交
190 191
static int cfg_queues(struct adapter *adap);

192 193 194 195 196 197 198
static void link_report(struct net_device *dev)
{
	if (!netif_carrier_ok(dev))
		netdev_info(dev, "link down\n");
	else {
		static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };

199
		const char *s;
200 201 202
		const struct port_info *p = netdev_priv(dev);

		switch (p->link_cfg.speed) {
203 204
		case 100:
			s = "100Mbps";
205
			break;
206
		case 1000:
207
			s = "1Gbps";
208
			break;
209 210 211 212 213
		case 10000:
			s = "10Gbps";
			break;
		case 25000:
			s = "25Gbps";
214
			break;
215
		case 40000:
216 217
			s = "40Gbps";
			break;
218 219 220
		case 50000:
			s = "50Gbps";
			break;
221 222 223
		case 100000:
			s = "100Gbps";
			break;
224 225 226 227
		default:
			pr_info("%s: unsupported speed: %d\n",
				dev->name, p->link_cfg.speed);
			return;
228 229 230 231 232 233 234
		}

		netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
			    fc[p->link_cfg.fc]);
	}
}

235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250
#ifdef CONFIG_CHELSIO_T4_DCB
/* Set up/tear down Data Center Bridging Priority mapping for a net device. */
static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable)
{
	struct port_info *pi = netdev_priv(dev);
	struct adapter *adap = pi->adapter;
	struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset];
	int i;

	/* We use a simple mapping of Port TX Queue Index to DCB
	 * Priority when we're enabling DCB.
	 */
	for (i = 0; i < pi->nqsets; i++, txq++) {
		u32 name, value;
		int err;

251 252 253 254
		name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
			FW_PARAMS_PARAM_X_V(
				FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) |
			FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id));
255 256 257 258 259 260
		value = enable ? i : 0xffffffff;

		/* Since we can be called while atomic (from "interrupt
		 * level") we need to issue the Set Parameters Commannd
		 * without sleeping (timeout < 0).
		 */
261
		err = t4_set_params_timeout(adap, adap->mbox, adap->pf, 0, 1,
262 263
					    &name, &value,
					    -FW_CMD_MAX_TIMEOUT);
264 265 266 267 268

		if (err)
			dev_err(adap->pdev_dev,
				"Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n",
				enable ? "set" : "unset", pi->port_id, i, -err);
269
		else
270
			txq->dcb_prio = enable ? value : 0;
271 272 273
	}
}

274
int cxgb4_dcb_enabled(const struct net_device *dev)
275 276 277 278 279 280 281 282 283
{
	struct port_info *pi = netdev_priv(dev);

	if (!pi->dcb.enabled)
		return 0;

	return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) ||
		(pi->dcb.state == CXGB4_DCB_STATE_HOST));
}
A
Arnd Bergmann 已提交
284
#endif /* CONFIG_CHELSIO_T4_DCB */
285

286 287 288 289 290 291 292 293
void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
{
	struct net_device *dev = adapter->port[port_id];

	/* Skip changes from disabled ports. */
	if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
		if (link_stat)
			netif_carrier_on(dev);
294 295
		else {
#ifdef CONFIG_CHELSIO_T4_DCB
296
			if (cxgb4_dcb_enabled(dev)) {
297
				cxgb4_dcb_reset(dev);
298 299
				dcb_tx_queue_prio_enable(dev, false);
			}
300
#endif /* CONFIG_CHELSIO_T4_DCB */
301
			netif_carrier_off(dev);
302
		}
303 304 305 306 307

		link_report(dev);
	}
}

308
void t4_os_portmod_changed(struct adapter *adap, int port_id)
309 310
{
	static const char *mod_str[] = {
311
		NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
312 313
	};

314 315
	struct net_device *dev = adap->port[port_id];
	struct port_info *pi = netdev_priv(dev);
316 317 318

	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
		netdev_info(dev, "port module unplugged\n");
319
	else if (pi->mod_type < ARRAY_SIZE(mod_str))
320
		netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
321 322 323 324 325 326 327 328 329 330 331
	else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
		netdev_info(dev, "%s: unsupported port module inserted\n",
			    dev->name);
	else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
		netdev_info(dev, "%s: unknown port module inserted\n",
			    dev->name);
	else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR)
		netdev_info(dev, "%s: transceiver module error\n", dev->name);
	else
		netdev_info(dev, "%s: unknown module type %d inserted\n",
			    dev->name, pi->mod_type);
332 333 334 335 336

	/* If the interface is running, then we'll need any "sticky" Link
	 * Parameters redone with a new Transceiver Module.
	 */
	pi->link_cfg.redo_l1cfg = netif_running(dev);
337 338
}

339 340 341 342
int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
module_param(dbfifo_int_thresh, int, 0644);
MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");

343
/*
344
 * usecs to sleep while draining the dbfifo
345
 */
346 347 348 349 350 351
static int dbfifo_drain_delay = 1000;
module_param(dbfifo_drain_delay, int, 0644);
MODULE_PARM_DESC(dbfifo_drain_delay,
		 "usecs to sleep while draining the dbfifo");

static inline int cxgb4_set_addr_hash(struct port_info *pi)
352
{
353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
	struct adapter *adap = pi->adapter;
	u64 vec = 0;
	bool ucast = false;
	struct hash_mac_addr *entry;

	/* Calculate the hash vector for the updated list and program it */
	list_for_each_entry(entry, &adap->mac_hlist, list) {
		ucast |= is_unicast_ether_addr(entry->addr);
		vec |= (1ULL << hash_mac_addr(entry->addr));
	}
	return t4_set_addr_hash(adap, adap->mbox, pi->viid, ucast,
				vec, false);
}

static int cxgb4_mac_sync(struct net_device *netdev, const u8 *mac_addr)
{
	struct port_info *pi = netdev_priv(netdev);
	struct adapter *adap = pi->adapter;
	int ret;
372 373
	u64 mhash = 0;
	u64 uhash = 0;
374 375 376 377 378 379
	/* idx stores the index of allocated filters,
	 * its size should be modified based on the number of
	 * MAC addresses that we allocate filters for
	 */

	u16 idx[1] = {};
380 381 382 383 384
	bool free = false;
	bool ucast = is_unicast_ether_addr(mac_addr);
	const u8 *maclist[1] = {mac_addr};
	struct hash_mac_addr *new_entry;

385 386
	ret = cxgb4_alloc_mac_filt(adap, pi->viid, free, 1, maclist,
				   idx, ucast ? &uhash : &mhash, false);
387 388 389 390 391 392 393 394 395 396 397 398 399
	if (ret < 0)
		goto out;
	/* if hash != 0, then add the addr to hash addr list
	 * so on the end we will calculate the hash for the
	 * list and program it
	 */
	if (uhash || mhash) {
		new_entry = kzalloc(sizeof(*new_entry), GFP_ATOMIC);
		if (!new_entry)
			return -ENOMEM;
		ether_addr_copy(new_entry->addr, mac_addr);
		list_add_tail(&new_entry->list, &adap->mac_hlist);
		ret = cxgb4_set_addr_hash(pi);
400
	}
401 402 403
out:
	return ret < 0 ? ret : 0;
}
404

405 406 407 408 409 410 411
static int cxgb4_mac_unsync(struct net_device *netdev, const u8 *mac_addr)
{
	struct port_info *pi = netdev_priv(netdev);
	struct adapter *adap = pi->adapter;
	int ret;
	const u8 *maclist[1] = {mac_addr};
	struct hash_mac_addr *entry, *tmp;
412

413 414 415 416 417 418 419 420
	/* If the MAC address to be removed is in the hash addr
	 * list, delete it from the list and update hash vector
	 */
	list_for_each_entry_safe(entry, tmp, &adap->mac_hlist, list) {
		if (ether_addr_equal(entry->addr, mac_addr)) {
			list_del(&entry->list);
			kfree(entry);
			return cxgb4_set_addr_hash(pi);
421 422 423
		}
	}

424
	ret = cxgb4_free_mac_filt(adap, pi->viid, 1, maclist, false);
425
	return ret < 0 ? -EINVAL : 0;
426 427 428 429 430 431 432 433 434
}

/*
 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
 * If @mtu is -1 it is left unchanged.
 */
static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
{
	struct port_info *pi = netdev_priv(dev);
435
	struct adapter *adapter = pi->adapter;
436

437 438
	__dev_uc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync);
	__dev_mc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync);
439 440 441 442 443

	return t4_set_rxmode(adapter, adapter->mbox, pi->viid, mtu,
			     (dev->flags & IFF_PROMISC) ? 1 : 0,
			     (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
			     sleep_ok);
444 445 446
}

/**
447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462
 *	cxgb4_change_mac - Update match filter for a MAC address.
 *	@pi: the port_info
 *	@viid: the VI id
 *	@tcam_idx: TCAM index of existing filter for old value of MAC address,
 *		   or -1
 *	@addr: the new MAC address value
 *	@persist: whether a new MAC allocation should be persistent
 *	@add_smt: if true also add the address to the HW SMT
 *
 *	Modifies an MPS filter and sets it to the new MAC address if
 *	@tcam_idx >= 0, or adds the MAC address to a new filter if
 *	@tcam_idx < 0. In the latter case the address is added persistently
 *	if @persist is %true.
 *	Addresses are programmed to hash region, if tcam runs out of entries.
 *
 */
463 464 465
int cxgb4_change_mac(struct port_info *pi, unsigned int viid,
		     int *tcam_idx, const u8 *addr, bool persist,
		     u8 *smt_idx)
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500
{
	struct adapter *adapter = pi->adapter;
	struct hash_mac_addr *entry, *new_entry;
	int ret;

	ret = t4_change_mac(adapter, adapter->mbox, viid,
			    *tcam_idx, addr, persist, smt_idx);
	/* We ran out of TCAM entries. try programming hash region. */
	if (ret == -ENOMEM) {
		/* If the MAC address to be updated is in the hash addr
		 * list, update it from the list
		 */
		list_for_each_entry(entry, &adapter->mac_hlist, list) {
			if (entry->iface_mac) {
				ether_addr_copy(entry->addr, addr);
				goto set_hash;
			}
		}
		new_entry = kzalloc(sizeof(*new_entry), GFP_KERNEL);
		if (!new_entry)
			return -ENOMEM;
		ether_addr_copy(new_entry->addr, addr);
		new_entry->iface_mac = true;
		list_add_tail(&new_entry->list, &adapter->mac_hlist);
set_hash:
		ret = cxgb4_set_addr_hash(pi);
	} else if (ret >= 0) {
		*tcam_idx = ret;
		ret = 0;
	}

	return ret;
}

/*
501 502 503 504 505 506 507 508 509
 *	link_start - enable a port
 *	@dev: the port to enable
 *
 *	Performs the MAC and PHY actions needed to enable a port.
 */
static int link_start(struct net_device *dev)
{
	int ret;
	struct port_info *pi = netdev_priv(dev);
510
	unsigned int mb = pi->adapter->pf;
511 512 513 514 515

	/*
	 * We do not set address filters and promiscuity here, the stack does
	 * that step explicitly.
	 */
516
	ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
517
			    !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
518
	if (ret == 0)
519 520
		ret = cxgb4_update_mac_filt(pi, pi->viid, &pi->xact_addr_filt,
					    dev->dev_addr, true, &pi->smt_idx);
521
	if (ret == 0)
522
		ret = t4_link_l1cfg(pi->adapter, mb, pi->tx_chan,
523
				    &pi->link_cfg);
524 525
	if (ret == 0) {
		local_bh_disable();
526
		ret = t4_enable_pi_params(pi->adapter, mb, pi, true,
527
					  true, CXGB4_DCB_ENABLED);
528 529
		local_bh_enable();
	}
530

531 532 533
	return ret;
}

534 535 536 537
#ifdef CONFIG_CHELSIO_T4_DCB
/* Handle a Data Center Bridging update message from the firmware. */
static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd)
{
538
	int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid));
539
	struct net_device *dev = adap->port[adap->chan_map[port]];
540 541 542 543 544 545 546 547 548 549 550 551 552 553 554
	int old_dcb_enabled = cxgb4_dcb_enabled(dev);
	int new_dcb_enabled;

	cxgb4_dcb_handle_fw_update(adap, pcmd);
	new_dcb_enabled = cxgb4_dcb_enabled(dev);

	/* If the DCB has become enabled or disabled on the port then we're
	 * going to need to set up/tear down DCB Priority parameters for the
	 * TX Queues associated with the port.
	 */
	if (new_dcb_enabled != old_dcb_enabled)
		dcb_tx_queue_prio_enable(dev, new_dcb_enabled);
}
#endif /* CONFIG_CHELSIO_T4_DCB */

V
Vipul Pandya 已提交
555
/* Response queue handler for the FW event queue.
556 557 558 559 560 561 562
 */
static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
			  const struct pkt_gl *gl)
{
	u8 opcode = ((const struct rss_header *)rsp)->opcode;

	rsp++;                                          /* skip RSS header */
563 564 565 566 567 568 569 570 571 572 573 574 575 576 577

	/* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
	 */
	if (unlikely(opcode == CPL_FW4_MSG &&
	   ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
		rsp++;
		opcode = ((const struct rss_header *)rsp)->opcode;
		rsp++;
		if (opcode != CPL_SGE_EGR_UPDATE) {
			dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
				, opcode);
			goto out;
		}
	}

578 579
	if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
		const struct cpl_sge_egr_update *p = (void *)rsp;
580
		unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid));
581
		struct sge_txq *txq;
582

583
		txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
584
		txq->restarts++;
585
		if (txq->q_type == CXGB4_TXQ_ETH) {
586 587 588
			struct sge_eth_txq *eq;

			eq = container_of(txq, struct sge_eth_txq, q);
589
			t4_sge_eth_txq_egress_update(q->adap, eq, -1);
590
		} else {
591
			struct sge_uld_txq *oq;
592

593
			oq = container_of(txq, struct sge_uld_txq, q);
594 595 596 597 598
			tasklet_schedule(&oq->qresume_tsk);
		}
	} else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
		const struct cpl_fw6_msg *p = (void *)rsp;

599 600
#ifdef CONFIG_CHELSIO_T4_DCB
		const struct fw_port_cmd *pcmd = (const void *)p->data;
601
		unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid));
602
		unsigned int action =
603
			FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16));
604 605

		if (cmd == FW_PORT_CMD &&
606 607
		    (action == FW_PORT_ACTION_GET_PORT_INFO ||
		     action == FW_PORT_ACTION_GET_PORT_INFO32)) {
608
			int port = FW_PORT_CMD_PORTID_G(
609
					be32_to_cpu(pcmd->op_to_portid));
610 611 612 613 614
			struct net_device *dev;
			int dcbxdis, state_input;

			dev = q->adap->port[q->adap->chan_map[port]];
			dcbxdis = (action == FW_PORT_ACTION_GET_PORT_INFO
615 616 617
			  ? !!(pcmd->u.info.dcbxdis_pkd & FW_PORT_CMD_DCBXDIS_F)
			  : !!(be32_to_cpu(pcmd->u.info32.lstatus32_to_cbllen32)
			       & FW_PORT_CMD_DCBXDIS32_F));
618 619 620
			state_input = (dcbxdis
				       ? CXGB4_DCB_INPUT_FW_DISABLED
				       : CXGB4_DCB_INPUT_FW_ENABLED);
621 622 623 624 625 626 627 628 629 630 631

			cxgb4_dcb_state_fsm(dev, state_input);
		}

		if (cmd == FW_PORT_CMD &&
		    action == FW_PORT_ACTION_L2_DCB_CFG)
			dcb_rpl(q->adap, pcmd);
		else
#endif
			if (p->type == 0)
				t4_handle_fw_rpl(q->adap, p->data);
632 633 634 635
	} else if (opcode == CPL_L2T_WRITE_RPL) {
		const struct cpl_l2t_write_rpl *p = (void *)rsp;

		do_l2t_write_rpl(q->adap, p);
636 637 638 639
	} else if (opcode == CPL_SMT_WRITE_RPL) {
		const struct cpl_smt_write_rpl *p = (void *)rsp;

		do_smt_write_rpl(q->adap, p);
V
Vipul Pandya 已提交
640 641 642 643
	} else if (opcode == CPL_SET_TCB_RPL) {
		const struct cpl_set_tcb_rpl *p = (void *)rsp;

		filter_rpl(q->adap, p);
644 645 646 647
	} else if (opcode == CPL_ACT_OPEN_RPL) {
		const struct cpl_act_open_rpl *p = (void *)rsp;

		hash_filter_rpl(q->adap, p);
648 649 650 651
	} else if (opcode == CPL_ABORT_RPL_RSS) {
		const struct cpl_abort_rpl_rss *p = (void *)rsp;

		hash_del_filter_rpl(q->adap, p);
652 653 654 655
	} else if (opcode == CPL_SRQ_TABLE_RPL) {
		const struct cpl_srq_table_rpl *p = (void *)rsp;

		do_srq_table_rpl(q->adap, p);
656 657 658
	} else
		dev_err(q->adap->pdev_dev,
			"unexpected CPL %#x on FW event queue\n", opcode);
659
out:
660 661 662 663 664
	return 0;
}

static void disable_msi(struct adapter *adapter)
{
665
	if (adapter->flags & CXGB4_USING_MSIX) {
666
		pci_disable_msix(adapter->pdev);
667 668
		adapter->flags &= ~CXGB4_USING_MSIX;
	} else if (adapter->flags & CXGB4_USING_MSI) {
669
		pci_disable_msi(adapter->pdev);
670
		adapter->flags &= ~CXGB4_USING_MSI;
671 672 673 674 675 676 677 678 679
	}
}

/*
 * Interrupt handler for non-data events used with MSI-X.
 */
static irqreturn_t t4_nondata_intr(int irq, void *cookie)
{
	struct adapter *adap = cookie;
680
	u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A));
681

682
	if (v & PFSW_F) {
683
		adap->swintr = 1;
684
		t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v);
685
	}
686
	if (adap->flags & CXGB4_MASTER_PF)
687
		t4_slow_intr_handler(adap);
688 689 690
	return IRQ_HANDLED;
}

691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
int cxgb4_set_msix_aff(struct adapter *adap, unsigned short vec,
		       cpumask_var_t *aff_mask, int idx)
{
	int rv;

	if (!zalloc_cpumask_var(aff_mask, GFP_KERNEL)) {
		dev_err(adap->pdev_dev, "alloc_cpumask_var failed\n");
		return -ENOMEM;
	}

	cpumask_set_cpu(cpumask_local_spread(idx, dev_to_node(adap->pdev_dev)),
			*aff_mask);

	rv = irq_set_affinity_hint(vec, *aff_mask);
	if (rv)
		dev_warn(adap->pdev_dev,
			 "irq_set_affinity_hint %u failed %d\n",
			 vec, rv);

	return 0;
}

void cxgb4_clear_msix_aff(unsigned short vec, cpumask_var_t aff_mask)
{
	irq_set_affinity_hint(vec, NULL);
	free_cpumask_var(aff_mask);
}

719 720 721
static int request_msix_queue_irqs(struct adapter *adap)
{
	struct sge *s = &adap->sge;
722
	struct msix_info *minfo;
723
	int err, ethqidx;
724

725 726 727 728 729 730 731
	if (s->fwevtq_msix_idx < 0)
		return -ENOMEM;

	err = request_irq(adap->msix_info[s->fwevtq_msix_idx].vec,
			  t4_sge_intr_msix, 0,
			  adap->msix_info[s->fwevtq_msix_idx].desc,
			  &s->fw_evtq);
732 733 734 735
	if (err)
		return err;

	for_each_ethrxq(s, ethqidx) {
736
		minfo = s->ethrxq[ethqidx].msix;
737
		err = request_irq(minfo->vec,
738
				  t4_sge_intr_msix, 0,
739
				  minfo->desc,
740 741 742
				  &s->ethrxq[ethqidx].rspq);
		if (err)
			goto unwind;
743 744 745

		cxgb4_set_msix_aff(adap, minfo->vec,
				   &minfo->aff_mask, ethqidx);
746 747 748 749
	}
	return 0;

unwind:
750
	while (--ethqidx >= 0) {
751
		minfo = s->ethrxq[ethqidx].msix;
752 753 754
		cxgb4_clear_msix_aff(minfo->vec, minfo->aff_mask);
		free_irq(minfo->vec, &s->ethrxq[ethqidx].rspq);
	}
755
	free_irq(adap->msix_info[s->fwevtq_msix_idx].vec, &s->fw_evtq);
756 757 758 759 760 761
	return err;
}

static void free_msix_queue_irqs(struct adapter *adap)
{
	struct sge *s = &adap->sge;
762
	struct msix_info *minfo;
763
	int i;
764

765
	free_irq(adap->msix_info[s->fwevtq_msix_idx].vec, &s->fw_evtq);
766
	for_each_ethrxq(s, i) {
767
		minfo = s->ethrxq[i].msix;
768 769 770
		cxgb4_clear_msix_aff(minfo->vec, minfo->aff_mask);
		free_irq(minfo->vec, &s->ethrxq[i].rspq);
	}
771 772
}

773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806
static int setup_ppod_edram(struct adapter *adap)
{
	unsigned int param, val;
	int ret;

	/* Driver sends FW_PARAMS_PARAM_DEV_PPOD_EDRAM read command to check
	 * if firmware supports ppod edram feature or not. If firmware
	 * returns 1, then driver can enable this feature by sending
	 * FW_PARAMS_PARAM_DEV_PPOD_EDRAM write command with value 1 to
	 * enable ppod edram feature.
	 */
	param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
		FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PPOD_EDRAM));

	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &param, &val);
	if (ret < 0) {
		dev_warn(adap->pdev_dev,
			 "querying PPOD_EDRAM support failed: %d\n",
			 ret);
		return -1;
	}

	if (val != 1)
		return -1;

	ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, &param, &val);
	if (ret < 0) {
		dev_err(adap->pdev_dev,
			"setting PPOD_EDRAM failed: %d\n", ret);
		return -1;
	}
	return 0;
}

807
/**
808
 *	cxgb4_write_rss - write the RSS table for a given port
809 810 811 812 813
 *	@pi: the port
 *	@queues: array of queue indices for RSS
 *
 *	Sets up the portion of the HW RSS table for the port's VI to distribute
 *	packets to the Rx queues in @queues.
814
 *	Should never be called before setting up sge eth rx queues
815
 */
816
int cxgb4_write_rss(const struct port_info *pi, const u16 *queues)
817 818 819
{
	u16 *rss;
	int i, err;
820 821
	struct adapter *adapter = pi->adapter;
	const struct sge_eth_rxq *rxq;
822

823
	rxq = &adapter->sge.ethrxq[pi->first_qset];
824
	rss = kmalloc_array(pi->rss_size, sizeof(u16), GFP_KERNEL);
825 826 827 828 829
	if (!rss)
		return -ENOMEM;

	/* map the queue indices to queue ids */
	for (i = 0; i < pi->rss_size; i++, queues++)
830
		rss[i] = rxq[*queues].rspq.abs_id;
831

832
	err = t4_config_rss_range(adapter, adapter->pf, pi->viid, 0,
833
				  pi->rss_size, rss, pi->rss_size);
834 835 836 837 838 839 840 841 842 843 844 845 846
	/* If Tunnel All Lookup isn't specified in the global RSS
	 * Configuration, then we need to specify a default Ingress
	 * Queue for any ingress packets which aren't hashed.  We'll
	 * use our first ingress queue ...
	 */
	if (!err)
		err = t4_config_vi_rss(adapter, adapter->mbox, pi->viid,
				       FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F |
				       FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F |
				       FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F |
				       FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F |
				       FW_RSS_VI_CONFIG_CMD_UDPEN_F,
				       rss[0]);
847 848 849 850
	kfree(rss);
	return err;
}

851 852 853 854
/**
 *	setup_rss - configure RSS
 *	@adap: the adapter
 *
855
 *	Sets up RSS for each port.
856 857 858
 */
static int setup_rss(struct adapter *adap)
{
859
	int i, j, err;
860 861 862 863

	for_each_port(adap, i) {
		const struct port_info *pi = adap2pinfo(adap, i);

864 865 866 867
		/* Fill default values with equal distribution */
		for (j = 0; j < pi->rss_size; j++)
			pi->rss[j] = j % pi->nqsets;

868
		err = cxgb4_write_rss(pi, pi->rss);
869 870 871 872 873 874
		if (err)
			return err;
	}
	return 0;
}

875 876 877 878 879 880 881 882 883
/*
 * Return the channel of the ingress queue with the given qid.
 */
static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
{
	qid -= p->ingr_start;
	return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
}

884 885 886 887 888 889
void cxgb4_quiesce_rx(struct sge_rspq *q)
{
	if (q->handler)
		napi_disable(&q->napi);
}

890 891 892 893 894 895 896
/*
 * Wait until all NAPI handlers are descheduled.
 */
static void quiesce_rx(struct adapter *adap)
{
	int i;

897
	for (i = 0; i < adap->sge.ingr_sz; i++) {
898 899
		struct sge_rspq *q = adap->sge.ingr_map[i];

900 901 902 903
		if (!q)
			continue;

		cxgb4_quiesce_rx(q);
904 905 906
	}
}

907 908 909
/* Disable interrupt and napi handler */
static void disable_interrupts(struct adapter *adap)
{
910 911
	struct sge *s = &adap->sge;

912
	if (adap->flags & CXGB4_FULL_INIT_DONE) {
913
		t4_intr_disable(adap);
914
		if (adap->flags & CXGB4_USING_MSIX) {
915
			free_msix_queue_irqs(adap);
916 917
			free_irq(adap->msix_info[s->nd_msix_idx].vec,
				 adap);
918 919 920 921 922 923 924
		} else {
			free_irq(adap->pdev->irq, adap);
		}
		quiesce_rx(adap);
	}
}

925 926 927 928 929 930 931 932 933 934 935
void cxgb4_enable_rx(struct adapter *adap, struct sge_rspq *q)
{
	if (q->handler)
		napi_enable(&q->napi);

	/* 0-increment GTS to start the timer and enable interrupts */
	t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
		     SEINTARM_V(q->intr_params) |
		     INGRESSQID_V(q->cntxt_id));
}

936 937 938 939 940 941 942
/*
 * Enable NAPI scheduling and interrupt generation for all Rx queues.
 */
static void enable_rx(struct adapter *adap)
{
	int i;

943
	for (i = 0; i < adap->sge.ingr_sz; i++) {
944 945 946 947
		struct sge_rspq *q = adap->sge.ingr_map[i];

		if (!q)
			continue;
948

949
		cxgb4_enable_rx(adap, q);
950 951 952
	}
}

953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972
static int setup_non_data_intr(struct adapter *adap)
{
	int msix;

	adap->sge.nd_msix_idx = -1;
	if (!(adap->flags & CXGB4_USING_MSIX))
		return 0;

	/* Request MSI-X vector for non-data interrupt */
	msix = cxgb4_get_msix_idx_from_bmap(adap);
	if (msix < 0)
		return -ENOMEM;

	snprintf(adap->msix_info[msix].desc,
		 sizeof(adap->msix_info[msix].desc),
		 "%s", adap->port[0]->name);

	adap->sge.nd_msix_idx = msix;
	return 0;
}
973

974
static int setup_fw_sge_queues(struct adapter *adap)
975 976
{
	struct sge *s = &adap->sge;
977
	int msix, err = 0;
978

979 980
	bitmap_zero(s->starving_fl, s->egr_sz);
	bitmap_zero(s->txq_maperr, s->egr_sz);
981

982 983 984 985 986 987 988 989 990 991
	if (adap->flags & CXGB4_USING_MSIX) {
		s->fwevtq_msix_idx = -1;
		msix = cxgb4_get_msix_idx_from_bmap(adap);
		if (msix < 0)
			return -ENOMEM;

		snprintf(adap->msix_info[msix].desc,
			 sizeof(adap->msix_info[msix].desc),
			 "%s-FWeventq", adap->port[0]->name);
	} else {
992
		err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
993
				       NULL, NULL, NULL, -1);
994 995
		if (err)
			return err;
996
		msix = -((int)s->intrq.abs_id + 1);
997 998 999
	}

	err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
1000 1001 1002 1003 1004
			       msix, NULL, fwevtq_handler, NULL, -1);
	if (err && msix >= 0)
		cxgb4_free_msix_idx_in_bmap(adap, msix);

	s->fwevtq_msix_idx = msix;
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017
	return err;
}

/**
 *	setup_sge_queues - configure SGE Tx/Rx/response queues
 *	@adap: the adapter
 *
 *	Determines how many sets of SGE queues to use and initializes them.
 *	We support multiple queue sets per port if we have MSI-X, otherwise
 *	just one queue set per port.
 */
static int setup_sge_queues(struct adapter *adap)
{
G
Ganesh Goudar 已提交
1018
	struct sge_uld_rxq_info *rxq_info = NULL;
1019
	struct sge *s = &adap->sge;
1020
	unsigned int cmplqid = 0;
1021
	int err, i, j, msix = 0;
1022

G
Ganesh Goudar 已提交
1023 1024 1025
	if (is_uld(adap))
		rxq_info = s->uld_rxq_info[CXGB4_ULD_RDMA];

1026 1027 1028
	if (!(adap->flags & CXGB4_USING_MSIX))
		msix = -((int)s->intrq.abs_id + 1);

1029 1030 1031 1032 1033 1034 1035
	for_each_port(adap, i) {
		struct net_device *dev = adap->port[i];
		struct port_info *pi = netdev_priv(dev);
		struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
		struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];

		for (j = 0; j < pi->nqsets; j++, q++) {
1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
			if (msix >= 0) {
				msix = cxgb4_get_msix_idx_from_bmap(adap);
				if (msix < 0) {
					err = msix;
					goto freeout;
				}

				snprintf(adap->msix_info[msix].desc,
					 sizeof(adap->msix_info[msix].desc),
					 "%s-Rx%d", dev->name, j);
				q->msix = &adap->msix_info[msix];
			}

1049
			err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1050
					       msix, &q->fl,
1051
					       t4_ethrx_handler,
1052
					       NULL,
1053 1054
					       t4_get_tp_ch_map(adap,
								pi->tx_chan));
1055 1056 1057 1058 1059
			if (err)
				goto freeout;
			q->rspq.idx = j;
			memset(&q->stats, 0, sizeof(q->stats));
		}
1060 1061 1062

		q = &s->ethrxq[pi->first_qset];
		for (j = 0; j < pi->nqsets; j++, t++, q++) {
1063 1064
			err = t4_sge_alloc_eth_txq(adap, t, dev,
					netdev_get_tx_queue(dev, j),
1065
					q->rspq.cntxt_id,
1066
					!!(adap->flags & CXGB4_SGE_DBQ_TIMER));
1067 1068 1069 1070 1071 1072
			if (err)
				goto freeout;
		}
	}

	for_each_port(adap, i) {
1073
		/* Note that cmplqid below is 0 if we don't
1074 1075
		 * have RDMA queues, and that's the right value.
		 */
1076 1077 1078
		if (rxq_info)
			cmplqid	= rxq_info->uldrxq[i].rspq.cntxt_id;

1079
		err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1080
					    s->fw_evtq.cntxt_id, cmplqid);
1081 1082 1083 1084
		if (err)
			goto freeout;
	}

1085 1086 1087
	if (!is_t4(adap->params.chip)) {
		err = t4_sge_alloc_eth_txq(adap, &s->ptptxq, adap->port[0],
					   netdev_get_tx_queue(adap->port[0], 0)
1088
					   , s->fw_evtq.cntxt_id, false);
1089 1090 1091 1092
		if (err)
			goto freeout;
	}

1093
	t4_write_reg(adap, is_t4(adap->params.chip) ?
1094 1095 1096 1097
				MPS_TRC_RSS_CONTROL_A :
				MPS_T5_TRC_RSS_CONTROL_A,
		     RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) |
		     QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id));
1098
	return 0;
1099
freeout:
1100
	dev_err(adap->pdev_dev, "Can't allocate queues, err=%d\n", -err);
1101 1102
	t4_free_sge_resources(adap);
	return err;
1103 1104
}

1105
static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
1106
			     struct net_device *sb_dev)
1107 1108 1109 1110 1111 1112 1113 1114 1115
{
	int txq;

#ifdef CONFIG_CHELSIO_T4_DCB
	/* If a Data Center Bridging has been successfully negotiated on this
	 * link then we'll use the skb's priority to map it to a TX Queue.
	 * The skb's priority is determined via the VLAN Tag Priority Code
	 * Point field.
	 */
1116
	if (cxgb4_dcb_enabled(dev) && !is_kdump_kernel()) {
1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
		u16 vlan_tci;
		int err;

		err = vlan_get_tag(skb, &vlan_tci);
		if (unlikely(err)) {
			if (net_ratelimit())
				netdev_warn(dev,
					    "TX Packet without VLAN Tag on DCB Link\n");
			txq = 0;
		} else {
			txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
V
Varun Prakash 已提交
1128 1129 1130 1131
#ifdef CONFIG_CHELSIO_T4_FCOE
			if (skb->protocol == htons(ETH_P_FCOE))
				txq = skb->priority & 0x7;
#endif /* CONFIG_CHELSIO_T4_FCOE */
1132 1133 1134 1135 1136
		}
		return txq;
	}
#endif /* CONFIG_CHELSIO_T4_DCB */

1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148
	if (dev->num_tc) {
		struct port_info *pi = netdev2pinfo(dev);

		/* Send unsupported traffic pattern to normal NIC queues. */
		txq = netdev_pick_tx(dev, skb, sb_dev);
		if (xfrm_offload(skb) || is_ptp_enabled(skb, dev) ||
		    ip_hdr(skb)->protocol != IPPROTO_TCP)
			txq = txq % pi->nqsets;

		return txq;
	}

1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
	if (select_queue) {
		txq = (skb_rx_queue_recorded(skb)
			? skb_get_rx_queue(skb)
			: smp_processor_id());

		while (unlikely(txq >= dev->real_num_tx_queues))
			txq -= dev->real_num_tx_queues;

		return txq;
	}

1160
	return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues;
1161 1162
}

1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195
static int closest_timer(const struct sge *s, int time)
{
	int i, delta, match = 0, min_delta = INT_MAX;

	for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
		delta = time - s->timer_val[i];
		if (delta < 0)
			delta = -delta;
		if (delta < min_delta) {
			min_delta = delta;
			match = i;
		}
	}
	return match;
}

static int closest_thres(const struct sge *s, int thres)
{
	int i, delta, match = 0, min_delta = INT_MAX;

	for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
		delta = thres - s->counter_val[i];
		if (delta < 0)
			delta = -delta;
		if (delta < min_delta) {
			min_delta = delta;
			match = i;
		}
	}
	return match;
}

/**
1196
 *	cxgb4_set_rspq_intr_params - set a queue's interrupt holdoff parameters
1197 1198 1199 1200 1201 1202 1203
 *	@q: the Rx queue
 *	@us: the hold-off time in us, or 0 to disable timer
 *	@cnt: the hold-off packet count, or 0 to disable counter
 *
 *	Sets an Rx queue's interrupt hold-off time and packet count.  At least
 *	one of the two needs to be enabled for the queue to generate interrupts.
 */
1204 1205
int cxgb4_set_rspq_intr_params(struct sge_rspq *q,
			       unsigned int us, unsigned int cnt)
1206
{
1207 1208
	struct adapter *adap = q->adap;

1209 1210 1211 1212 1213 1214 1215 1216 1217 1218
	if ((us | cnt) == 0)
		cnt = 1;

	if (cnt) {
		int err;
		u32 v, new_idx;

		new_idx = closest_thres(&adap->sge, cnt);
		if (q->desc && q->pktcnt_idx != new_idx) {
			/* the queue has already been created, update it */
1219 1220 1221 1222
			v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
			    FW_PARAMS_PARAM_X_V(
					FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
			    FW_PARAMS_PARAM_YZ_V(q->cntxt_id);
1223 1224
			err = t4_set_params(adap, adap->mbox, adap->pf, 0, 1,
					    &v, &new_idx);
1225 1226 1227 1228 1229 1230 1231
			if (err)
				return err;
		}
		q->pktcnt_idx = new_idx;
	}

	us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1232
	q->intr_params = QINTR_TIMER_IDX_V(us) | QINTR_CNT_EN_V(cnt > 0);
1233 1234 1235
	return 0;
}

1236
static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
D
Dimitris Michailidis 已提交
1237
{
1238
	const struct port_info *pi = netdev_priv(dev);
1239
	netdev_features_t changed = dev->features ^ features;
1240 1241
	int err;

1242
	if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
1243
		return 0;
1244

1245
	err = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, -1,
1246
			    -1, -1, -1,
1247
			    !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
1248
	if (unlikely(err))
1249
		dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
1250
	return err;
D
Dimitris Michailidis 已提交
1251 1252
}

B
Bill Pemberton 已提交
1253
static int setup_debugfs(struct adapter *adap)
1254 1255 1256 1257
{
	if (IS_ERR_OR_NULL(adap->debugfs_root))
		return -1;

1258 1259 1260
#ifdef CONFIG_DEBUG_FS
	t4_setup_debugfs(adap);
#endif
1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
	return 0;
}

/*
 * upper-layer driver support
 */

/*
 * Allocate an active-open TID and set it to the supplied value.
 */
int cxgb4_alloc_atid(struct tid_info *t, void *data)
{
	int atid = -1;

	spin_lock_bh(&t->atid_lock);
	if (t->afree) {
		union aopen_entry *p = t->afree;

V
Vipul Pandya 已提交
1279
		atid = (p - t->atid_tab) + t->atid_base;
1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293
		t->afree = p->next;
		p->data = data;
		t->atids_in_use++;
	}
	spin_unlock_bh(&t->atid_lock);
	return atid;
}
EXPORT_SYMBOL(cxgb4_alloc_atid);

/*
 * Release an active-open TID.
 */
void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
{
V
Vipul Pandya 已提交
1294
	union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318

	spin_lock_bh(&t->atid_lock);
	p->next = t->afree;
	t->afree = p;
	t->atids_in_use--;
	spin_unlock_bh(&t->atid_lock);
}
EXPORT_SYMBOL(cxgb4_free_atid);

/*
 * Allocate a server TID and set it to the supplied value.
 */
int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
{
	int stid;

	spin_lock_bh(&t->stid_lock);
	if (family == PF_INET) {
		stid = find_first_zero_bit(t->stid_bmap, t->nstids);
		if (stid < t->nstids)
			__set_bit(stid, t->stid_bmap);
		else
			stid = -1;
	} else {
1319
		stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 1);
1320 1321 1322 1323 1324 1325
		if (stid < 0)
			stid = -1;
	}
	if (stid >= 0) {
		t->stid_tab[stid].data = data;
		stid += t->stid_base;
1326 1327 1328 1329
		/* IPv6 requires max of 520 bits or 16 cells in TCAM
		 * This is equivalent to 4 TIDs. With CLIP enabled it
		 * needs 2 TIDs.
		 */
1330
		if (family == PF_INET6) {
1331
			t->stids_in_use += 2;
1332 1333 1334 1335
			t->v6_stids_in_use += 2;
		} else {
			t->stids_in_use++;
		}
1336 1337 1338 1339 1340 1341
	}
	spin_unlock_bh(&t->stid_lock);
	return stid;
}
EXPORT_SYMBOL(cxgb4_alloc_stid);

1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360
/* Allocate a server filter TID and set it to the supplied value.
 */
int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
{
	int stid;

	spin_lock_bh(&t->stid_lock);
	if (family == PF_INET) {
		stid = find_next_zero_bit(t->stid_bmap,
				t->nstids + t->nsftids, t->nstids);
		if (stid < (t->nstids + t->nsftids))
			__set_bit(stid, t->stid_bmap);
		else
			stid = -1;
	} else {
		stid = -1;
	}
	if (stid >= 0) {
		t->stid_tab[stid].data = data;
1361 1362
		stid -= t->nstids;
		stid += t->sftid_base;
1363
		t->sftids_in_use++;
1364 1365 1366 1367 1368 1369 1370
	}
	spin_unlock_bh(&t->stid_lock);
	return stid;
}
EXPORT_SYMBOL(cxgb4_alloc_sftid);

/* Release a server TID.
1371 1372 1373
 */
void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
{
1374 1375 1376 1377 1378 1379 1380 1381
	/* Is it a server filter TID? */
	if (t->nsftids && (stid >= t->sftid_base)) {
		stid -= t->sftid_base;
		stid += t->nstids;
	} else {
		stid -= t->stid_base;
	}

1382 1383 1384 1385
	spin_lock_bh(&t->stid_lock);
	if (family == PF_INET)
		__clear_bit(stid, t->stid_bmap);
	else
1386
		bitmap_release_region(t->stid_bmap, stid, 1);
1387
	t->stid_tab[stid].data = NULL;
1388
	if (stid < t->nstids) {
1389
		if (family == PF_INET6) {
1390
			t->stids_in_use -= 2;
1391 1392 1393 1394
			t->v6_stids_in_use -= 2;
		} else {
			t->stids_in_use--;
		}
1395 1396 1397
	} else {
		t->sftids_in_use--;
	}
1398

1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
	spin_unlock_bh(&t->stid_lock);
}
EXPORT_SYMBOL(cxgb4_free_stid);

/*
 * Populate a TID_RELEASE WR.  Caller must properly size the skb.
 */
static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
			   unsigned int tid)
{
	struct cpl_tid_release *req;

	set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
1412
	req = __skb_put(skb, sizeof(*req));
1413 1414 1415 1416 1417 1418 1419 1420
	INIT_TP_WR(req, tid);
	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
}

/*
 * Queue a TID release request and if necessary schedule a work queue to
 * process it.
 */
1421 1422
static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
				    unsigned int tid)
1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
{
	void **p = &t->tid_tab[tid];
	struct adapter *adap = container_of(t, struct adapter, tids);

	spin_lock_bh(&adap->tid_release_lock);
	*p = adap->tid_release_head;
	/* Low 2 bits encode the Tx channel number */
	adap->tid_release_head = (void **)((uintptr_t)p | chan);
	if (!adap->tid_release_task_busy) {
		adap->tid_release_task_busy = true;
1433
		queue_work(adap->workq, &adap->tid_release_task);
1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473
	}
	spin_unlock_bh(&adap->tid_release_lock);
}

/*
 * Process the list of pending TID release requests.
 */
static void process_tid_release_list(struct work_struct *work)
{
	struct sk_buff *skb;
	struct adapter *adap;

	adap = container_of(work, struct adapter, tid_release_task);

	spin_lock_bh(&adap->tid_release_lock);
	while (adap->tid_release_head) {
		void **p = adap->tid_release_head;
		unsigned int chan = (uintptr_t)p & 3;
		p = (void *)p - chan;

		adap->tid_release_head = *p;
		*p = NULL;
		spin_unlock_bh(&adap->tid_release_lock);

		while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
					 GFP_KERNEL)))
			schedule_timeout_uninterruptible(1);

		mk_tid_release(skb, chan, p - adap->tids.tid_tab);
		t4_ofld_send(adap, skb);
		spin_lock_bh(&adap->tid_release_lock);
	}
	adap->tid_release_task_busy = false;
	spin_unlock_bh(&adap->tid_release_lock);
}

/*
 * Release a TID and inform HW.  If we are unable to allocate the release
 * message we defer to a work queue.
 */
1474 1475
void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid,
		      unsigned short family)
1476 1477 1478 1479
{
	struct sk_buff *skb;
	struct adapter *adap = container_of(t, struct adapter, tids);

1480 1481 1482 1483
	WARN_ON(tid >= t->ntids);

	if (t->tid_tab[tid]) {
		t->tid_tab[tid] = NULL;
1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495
		atomic_dec(&t->conns_in_use);
		if (t->hash_base && (tid >= t->hash_base)) {
			if (family == AF_INET6)
				atomic_sub(2, &t->hash_tids_in_use);
			else
				atomic_dec(&t->hash_tids_in_use);
		} else {
			if (family == AF_INET6)
				atomic_sub(2, &t->tids_in_use);
			else
				atomic_dec(&t->tids_in_use);
		}
1496 1497
	}

1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511
	skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
	if (likely(skb)) {
		mk_tid_release(skb, chan, tid);
		t4_ofld_send(adap, skb);
	} else
		cxgb4_queue_tid_release(t, chan, tid);
}
EXPORT_SYMBOL(cxgb4_remove_tid);

/*
 * Allocate and initialize the TID tables.  Returns 0 on success.
 */
static int tid_init(struct tid_info *t)
{
1512
	struct adapter *adap = container_of(t, struct adapter, tids);
1513 1514
	unsigned int max_ftids = t->nftids + t->nsftids;
	unsigned int natids = t->natids;
1515
	unsigned int eotid_bmap_size;
1516 1517 1518
	unsigned int stid_bmap_size;
	unsigned int ftid_bmap_size;
	size_t size;
1519

1520
	stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
1521
	ftid_bmap_size = BITS_TO_LONGS(t->nftids);
1522
	eotid_bmap_size = BITS_TO_LONGS(t->neotids);
V
Vipul Pandya 已提交
1523 1524
	size = t->ntids * sizeof(*t->tid_tab) +
	       natids * sizeof(*t->atid_tab) +
1525
	       t->nstids * sizeof(*t->stid_tab) +
1526
	       t->nsftids * sizeof(*t->stid_tab) +
V
Vipul Pandya 已提交
1527
	       stid_bmap_size * sizeof(long) +
1528
	       max_ftids * sizeof(*t->ftid_tab) +
1529 1530 1531
	       ftid_bmap_size * sizeof(long) +
	       t->neotids * sizeof(*t->eotid_tab) +
	       eotid_bmap_size * sizeof(long);
V
Vipul Pandya 已提交
1532

1533
	t->tid_tab = kvzalloc(size, GFP_KERNEL);
1534 1535 1536 1537 1538
	if (!t->tid_tab)
		return -ENOMEM;

	t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
	t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
1539
	t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
V
Vipul Pandya 已提交
1540
	t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
1541
	t->ftid_bmap = (unsigned long *)&t->ftid_tab[max_ftids];
1542 1543
	t->eotid_tab = (struct eotid_entry *)&t->ftid_bmap[ftid_bmap_size];
	t->eotid_bmap = (unsigned long *)&t->eotid_tab[t->neotids];
1544 1545
	spin_lock_init(&t->stid_lock);
	spin_lock_init(&t->atid_lock);
1546
	spin_lock_init(&t->ftid_lock);
1547 1548

	t->stids_in_use = 0;
1549
	t->v6_stids_in_use = 0;
1550
	t->sftids_in_use = 0;
1551 1552 1553
	t->afree = NULL;
	t->atids_in_use = 0;
	atomic_set(&t->tids_in_use, 0);
1554
	atomic_set(&t->conns_in_use, 0);
1555
	atomic_set(&t->hash_tids_in_use, 0);
1556 1557 1558 1559 1560 1561 1562

	/* Setup the free list for atid_tab and clear the stid bitmap. */
	if (natids) {
		while (--natids)
			t->atid_tab[natids - 1].next = &t->atid_tab[natids];
		t->afree = t->atid_tab;
	}
1563

1564 1565 1566 1567 1568 1569
	if (is_offload(adap)) {
		bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
		/* Reserve stid 0 for T4/T5 adapters */
		if (!t->stid_base &&
		    CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
			__set_bit(0, t->stid_bmap);
1570 1571 1572

		if (t->neotids)
			bitmap_zero(t->eotid_bmap, t->neotids);
1573 1574 1575
	}

	bitmap_zero(t->ftid_bmap, t->nftids);
1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590
	return 0;
}

/**
 *	cxgb4_create_server - create an IP server
 *	@dev: the device
 *	@stid: the server TID
 *	@sip: local IP address to bind server to
 *	@sport: the server's TCP port
 *	@queue: queue to direct messages from this server to
 *
 *	Create an IP server for the given port and address.
 *	Returns <0 on error and one of the %NET_XMIT_* values on success.
 */
int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
1591 1592
			__be32 sip, __be16 sport, __be16 vlan,
			unsigned int queue)
1593 1594 1595 1596 1597
{
	unsigned int chan;
	struct sk_buff *skb;
	struct adapter *adap;
	struct cpl_pass_open_req *req;
1598
	int ret;
1599 1600 1601 1602 1603 1604

	skb = alloc_skb(sizeof(*req), GFP_KERNEL);
	if (!skb)
		return -ENOMEM;

	adap = netdev2adap(dev);
1605
	req = __skb_put(skb, sizeof(*req));
1606 1607 1608 1609 1610 1611
	INIT_TP_WR(req, 0);
	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
	req->local_port = sport;
	req->peer_port = htons(0);
	req->local_ip = sip;
	req->peer_ip = htonl(0);
1612
	chan = rxq_to_chan(&adap->sge, queue);
1613
	req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
1614 1615
	req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
				SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
1616 1617
	ret = t4_mgmt_tx(adap, skb);
	return net_xmit_eval(ret);
1618 1619 1620
}
EXPORT_SYMBOL(cxgb4_create_server);

1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645
/*	cxgb4_create_server6 - create an IPv6 server
 *	@dev: the device
 *	@stid: the server TID
 *	@sip: local IPv6 address to bind server to
 *	@sport: the server's TCP port
 *	@queue: queue to direct messages from this server to
 *
 *	Create an IPv6 server for the given port and address.
 *	Returns <0 on error and one of the %NET_XMIT_* values on success.
 */
int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
			 const struct in6_addr *sip, __be16 sport,
			 unsigned int queue)
{
	unsigned int chan;
	struct sk_buff *skb;
	struct adapter *adap;
	struct cpl_pass_open_req6 *req;
	int ret;

	skb = alloc_skb(sizeof(*req), GFP_KERNEL);
	if (!skb)
		return -ENOMEM;

	adap = netdev2adap(dev);
1646
	req = __skb_put(skb, sizeof(*req));
1647 1648 1649 1650 1651 1652 1653 1654 1655
	INIT_TP_WR(req, 0);
	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
	req->local_port = sport;
	req->peer_port = htons(0);
	req->local_ip_hi = *(__be64 *)(sip->s6_addr);
	req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
	req->peer_ip_hi = cpu_to_be64(0);
	req->peer_ip_lo = cpu_to_be64(0);
	chan = rxq_to_chan(&adap->sge, queue);
1656
	req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
1657 1658
	req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
				SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677
	ret = t4_mgmt_tx(adap, skb);
	return net_xmit_eval(ret);
}
EXPORT_SYMBOL(cxgb4_create_server6);

int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
			unsigned int queue, bool ipv6)
{
	struct sk_buff *skb;
	struct adapter *adap;
	struct cpl_close_listsvr_req *req;
	int ret;

	adap = netdev2adap(dev);

	skb = alloc_skb(sizeof(*req), GFP_KERNEL);
	if (!skb)
		return -ENOMEM;

1678
	req = __skb_put(skb, sizeof(*req));
1679 1680
	INIT_TP_WR(req, 0);
	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
1681 1682
	req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) :
				LISTSVR_IPV6_V(0)) | QUEUENO_V(queue));
1683 1684 1685 1686 1687
	ret = t4_mgmt_tx(adap, skb);
	return net_xmit_eval(ret);
}
EXPORT_SYMBOL(cxgb4_remove_server);

1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710
/**
 *	cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
 *	@mtus: the HW MTU table
 *	@mtu: the target MTU
 *	@idx: index of selected entry in the MTU table
 *
 *	Returns the index and the value in the HW MTU table that is closest to
 *	but does not exceed @mtu, unless @mtu is smaller than any value in the
 *	table, in which case that smallest available value is selected.
 */
unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
			    unsigned int *idx)
{
	unsigned int i = 0;

	while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
		++i;
	if (idx)
		*idx = i;
	return mtus[i];
}
EXPORT_SYMBOL(cxgb4_best_mtu);

1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781
/**
 *     cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned
 *     @mtus: the HW MTU table
 *     @header_size: Header Size
 *     @data_size_max: maximum Data Segment Size
 *     @data_size_align: desired Data Segment Size Alignment (2^N)
 *     @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL)
 *
 *     Similar to cxgb4_best_mtu() but instead of searching the Hardware
 *     MTU Table based solely on a Maximum MTU parameter, we break that
 *     parameter up into a Header Size and Maximum Data Segment Size, and
 *     provide a desired Data Segment Size Alignment.  If we find an MTU in
 *     the Hardware MTU Table which will result in a Data Segment Size with
 *     the requested alignment _and_ that MTU isn't "too far" from the
 *     closest MTU, then we'll return that rather than the closest MTU.
 */
unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus,
				    unsigned short header_size,
				    unsigned short data_size_max,
				    unsigned short data_size_align,
				    unsigned int *mtu_idxp)
{
	unsigned short max_mtu = header_size + data_size_max;
	unsigned short data_size_align_mask = data_size_align - 1;
	int mtu_idx, aligned_mtu_idx;

	/* Scan the MTU Table till we find an MTU which is larger than our
	 * Maximum MTU or we reach the end of the table.  Along the way,
	 * record the last MTU found, if any, which will result in a Data
	 * Segment Length matching the requested alignment.
	 */
	for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) {
		unsigned short data_size = mtus[mtu_idx] - header_size;

		/* If this MTU minus the Header Size would result in a
		 * Data Segment Size of the desired alignment, remember it.
		 */
		if ((data_size & data_size_align_mask) == 0)
			aligned_mtu_idx = mtu_idx;

		/* If we're not at the end of the Hardware MTU Table and the
		 * next element is larger than our Maximum MTU, drop out of
		 * the loop.
		 */
		if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu)
			break;
	}

	/* If we fell out of the loop because we ran to the end of the table,
	 * then we just have to use the last [largest] entry.
	 */
	if (mtu_idx == NMTUS)
		mtu_idx--;

	/* If we found an MTU which resulted in the requested Data Segment
	 * Length alignment and that's "not far" from the largest MTU which is
	 * less than or equal to the maximum MTU, then use that.
	 */
	if (aligned_mtu_idx >= 0 &&
	    mtu_idx - aligned_mtu_idx <= 1)
		mtu_idx = aligned_mtu_idx;

	/* If the caller has passed in an MTU Index pointer, pass the
	 * MTU Index back.  Return the MTU value.
	 */
	if (mtu_idxp)
		*mtu_idxp = mtu_idx;
	return mtus[mtu_idx];
}
EXPORT_SYMBOL(cxgb4_best_aligned_mtu);

1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793
/**
 *	cxgb4_port_chan - get the HW channel of a port
 *	@dev: the net device for the port
 *
 *	Return the HW Tx channel of the given port.
 */
unsigned int cxgb4_port_chan(const struct net_device *dev)
{
	return netdev2pinfo(dev)->tx_chan;
}
EXPORT_SYMBOL(cxgb4_port_chan);

1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805
/**
 *      cxgb4_port_e2cchan - get the HW c-channel of a port
 *      @dev: the net device for the port
 *
 *      Return the HW RX c-channel of the given port.
 */
unsigned int cxgb4_port_e2cchan(const struct net_device *dev)
{
	return netdev2pinfo(dev)->rx_cchan;
}
EXPORT_SYMBOL(cxgb4_port_e2cchan);

1806 1807 1808
unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
{
	struct adapter *adap = netdev2adap(dev);
1809
	u32 v1, v2, lp_count, hp_count;
1810

1811 1812
	v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
	v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
1813
	if (is_t4(adap->params.chip)) {
1814 1815
		lp_count = LP_COUNT_G(v1);
		hp_count = HP_COUNT_G(v1);
1816
	} else {
1817 1818
		lp_count = LP_COUNT_T5_G(v1);
		hp_count = HP_COUNT_T5_G(v2);
1819 1820
	}
	return lpfifo ? lp_count : hp_count;
1821 1822 1823
}
EXPORT_SYMBOL(cxgb4_dbfifo_count);

1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853
/**
 *	cxgb4_port_viid - get the VI id of a port
 *	@dev: the net device for the port
 *
 *	Return the VI id of the given port.
 */
unsigned int cxgb4_port_viid(const struct net_device *dev)
{
	return netdev2pinfo(dev)->viid;
}
EXPORT_SYMBOL(cxgb4_port_viid);

/**
 *	cxgb4_port_idx - get the index of a port
 *	@dev: the net device for the port
 *
 *	Return the index of the given port.
 */
unsigned int cxgb4_port_idx(const struct net_device *dev)
{
	return netdev2pinfo(dev)->port_id;
}
EXPORT_SYMBOL(cxgb4_port_idx);

void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
			 struct tp_tcp_stats *v6)
{
	struct adapter *adap = pci_get_drvdata(pdev);

	spin_lock(&adap->stats_lock);
1854
	t4_tp_get_tcp_stats(adap, v4, v6, false);
1855 1856 1857 1858 1859 1860 1861 1862 1863
	spin_unlock(&adap->stats_lock);
}
EXPORT_SYMBOL(cxgb4_get_tcp_stats);

void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
		      const unsigned int *pgsz_order)
{
	struct adapter *adap = netdev2adap(dev);

1864 1865 1866 1867
	t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask);
	t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) |
		     HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) |
		     HPZ3_V(pgsz_order[3]));
1868 1869 1870
}
EXPORT_SYMBOL(cxgb4_iscsi_init);

1871 1872 1873 1874
int cxgb4_flush_eq_cache(struct net_device *dev)
{
	struct adapter *adap = netdev2adap(dev);

1875
	return t4_sge_ctxt_flush(adap, adap->mbox, CTXT_EGRESS);
1876 1877 1878 1879 1880
}
EXPORT_SYMBOL(cxgb4_flush_eq_cache);

static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
{
1881
	u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8;
1882 1883 1884
	__be64 indices;
	int ret;

1885 1886 1887 1888 1889
	spin_lock(&adap->win0_lock);
	ret = t4_memory_rw(adap, 0, MEM_EDC0, addr,
			   sizeof(indices), (__be32 *)&indices,
			   T4_MEMORY_READ);
	spin_unlock(&adap->win0_lock);
1890
	if (!ret) {
1891 1892
		*cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
		*pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909
	}
	return ret;
}

int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
			u16 size)
{
	struct adapter *adap = netdev2adap(dev);
	u16 hw_pidx, hw_cidx;
	int ret;

	ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
	if (ret)
		goto out;

	if (pidx != hw_pidx) {
		u16 delta;
1910
		u32 val;
1911 1912 1913 1914 1915

		if (pidx >= hw_pidx)
			delta = pidx - hw_pidx;
		else
			delta = size - hw_pidx + pidx;
1916 1917 1918 1919 1920

		if (is_t4(adap->params.chip))
			val = PIDX_V(delta);
		else
			val = PIDX_T5_V(delta);
1921
		wmb();
1922 1923
		t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
			     QID_V(qid) | val);
1924 1925 1926 1927 1928 1929
	}
out:
	return ret;
}
EXPORT_SYMBOL(cxgb4_sync_txq_pidx);

1930 1931
int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte)
{
1932
	u32 edc0_size, edc1_size, mc0_size, mc1_size, size;
1933
	u32 edc0_end, edc1_end, mc0_end, mc1_end;
A
Arjun Vynipadath 已提交
1934 1935 1936
	u32 offset, memtype, memaddr;
	struct adapter *adap;
	u32 hma_size = 0;
1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948
	int ret;

	adap = netdev2adap(dev);

	offset = ((stag >> 8) * 32) + adap->vres.stag.start;

	/* Figure out where the offset lands in the Memory Type/Address scheme.
	 * This code assumes that the memory is laid out starting at offset 0
	 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0
	 * and EDC1.  Some cards will have neither MC0 nor MC1, most cards have
	 * MC0, and some have both MC0 and MC1.
	 */
1949 1950 1951 1952 1953 1954
	size = t4_read_reg(adap, MA_EDRAM0_BAR_A);
	edc0_size = EDRAM0_SIZE_G(size) << 20;
	size = t4_read_reg(adap, MA_EDRAM1_BAR_A);
	edc1_size = EDRAM1_SIZE_G(size) << 20;
	size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A);
	mc0_size = EXT_MEM0_SIZE_G(size) << 20;
1955

A
Arjun Vynipadath 已提交
1956 1957 1958 1959
	if (t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A) & HMA_MUX_F) {
		size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
		hma_size = EXT_MEM1_SIZE_G(size) << 20;
	}
1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970
	edc0_end = edc0_size;
	edc1_end = edc0_end + edc1_size;
	mc0_end = edc1_end + mc0_size;

	if (offset < edc0_end) {
		memtype = MEM_EDC0;
		memaddr = offset;
	} else if (offset < edc1_end) {
		memtype = MEM_EDC1;
		memaddr = offset - edc0_end;
	} else {
A
Arjun Vynipadath 已提交
1971 1972 1973 1974
		if (hma_size && (offset < (edc1_end + hma_size))) {
			memtype = MEM_HMA;
			memaddr = offset - edc1_end;
		} else if (offset < mc0_end) {
1975 1976
			memtype = MEM_MC0;
			memaddr = offset - edc1_end;
1977
		} else if (is_t5(adap->params.chip)) {
1978 1979
			size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
			mc1_size = EXT_MEM1_SIZE_G(size) << 20;
1980 1981 1982 1983 1984 1985 1986 1987
			mc1_end = mc0_end + mc1_size;
			if (offset < mc1_end) {
				memtype = MEM_MC1;
				memaddr = offset - mc0_end;
			} else {
				/* offset beyond the end of any memory */
				goto err;
			}
1988 1989 1990
		} else {
			/* T4/T6 only has a single memory channel */
			goto err;
1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005
		}
	}

	spin_lock(&adap->win0_lock);
	ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ);
	spin_unlock(&adap->win0_lock);
	return ret;

err:
	dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n",
		stag, offset);
	return -EINVAL;
}
EXPORT_SYMBOL(cxgb4_read_tpte);

2006 2007 2008 2009 2010 2011
u64 cxgb4_read_sge_timestamp(struct net_device *dev)
{
	u32 hi, lo;
	struct adapter *adap;

	adap = netdev2adap(dev);
2012 2013
	lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A);
	hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A));
2014 2015 2016 2017 2018

	return ((u64)hi << 32) | (u64)lo;
}
EXPORT_SYMBOL(cxgb4_read_sge_timestamp);

2019 2020 2021
int cxgb4_bar2_sge_qregs(struct net_device *dev,
			 unsigned int qid,
			 enum cxgb4_bar2_qtype qtype,
2022
			 int user,
2023 2024 2025
			 u64 *pbar2_qoffset,
			 unsigned int *pbar2_qid)
{
2026
	return t4_bar2_sge_qregs(netdev2adap(dev),
2027 2028 2029 2030
				 qid,
				 (qtype == CXGB4_BAR2_QTYPE_EGRESS
				  ? T4_BAR2_QTYPE_EGRESS
				  : T4_BAR2_QTYPE_INGRESS),
2031
				 user,
2032 2033 2034 2035 2036
				 pbar2_qoffset,
				 pbar2_qid);
}
EXPORT_SYMBOL(cxgb4_bar2_sge_qregs);

2037 2038 2039 2040 2041 2042 2043
static struct pci_driver cxgb4_driver;

static void check_neigh_update(struct neighbour *neigh)
{
	const struct device *parent;
	const struct net_device *netdev = neigh->dev;

2044
	if (is_vlan_dev(netdev))
2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069
		netdev = vlan_dev_real_dev(netdev);
	parent = netdev->dev.parent;
	if (parent && parent->driver == &cxgb4_driver.driver)
		t4_l2t_update(dev_get_drvdata(parent), neigh);
}

static int netevent_cb(struct notifier_block *nb, unsigned long event,
		       void *data)
{
	switch (event) {
	case NETEVENT_NEIGH_UPDATE:
		check_neigh_update(data);
		break;
	case NETEVENT_REDIRECT:
	default:
		break;
	}
	return 0;
}

static bool netevent_registered;
static struct notifier_block cxgb4_netevent_nb = {
	.notifier_call = netevent_cb
};

2070 2071
static void drain_db_fifo(struct adapter *adap, int usecs)
{
2072
	u32 v1, v2, lp_count, hp_count;
2073 2074

	do {
2075 2076
		v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
		v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
2077
		if (is_t4(adap->params.chip)) {
2078 2079
			lp_count = LP_COUNT_G(v1);
			hp_count = HP_COUNT_G(v1);
2080
		} else {
2081 2082
			lp_count = LP_COUNT_T5_G(v1);
			hp_count = HP_COUNT_T5_G(v2);
2083 2084 2085 2086
		}

		if (lp_count == 0 && hp_count == 0)
			break;
2087 2088 2089 2090 2091 2092 2093
		set_current_state(TASK_UNINTERRUPTIBLE);
		schedule_timeout(usecs_to_jiffies(usecs));
	} while (1);
}

static void disable_txq_db(struct sge_txq *q)
{
2094 2095 2096
	unsigned long flags;

	spin_lock_irqsave(&q->db_lock, flags);
2097
	q->db_disabled = 1;
2098
	spin_unlock_irqrestore(&q->db_lock, flags);
2099 2100
}

2101
static void enable_txq_db(struct adapter *adap, struct sge_txq *q)
2102 2103
{
	spin_lock_irq(&q->db_lock);
2104 2105 2106 2107 2108
	if (q->db_pidx_inc) {
		/* Make sure that all writes to the TX descriptors
		 * are committed before we tell HW about them.
		 */
		wmb();
2109 2110
		t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
			     QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc));
2111 2112
		q->db_pidx_inc = 0;
	}
2113 2114 2115 2116 2117 2118 2119 2120 2121 2122
	q->db_disabled = 0;
	spin_unlock_irq(&q->db_lock);
}

static void disable_dbs(struct adapter *adap)
{
	int i;

	for_each_ethrxq(&adap->sge, i)
		disable_txq_db(&adap->sge.ethtxq[i].q);
2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134
	if (is_offload(adap)) {
		struct sge_uld_txq_info *txq_info =
			adap->sge.uld_txq_info[CXGB4_TX_OFLD];

		if (txq_info) {
			for_each_ofldtxq(&adap->sge, i) {
				struct sge_uld_txq *txq = &txq_info->uldtxq[i];

				disable_txq_db(&txq->q);
			}
		}
	}
2135 2136 2137 2138 2139 2140 2141 2142 2143
	for_each_port(adap, i)
		disable_txq_db(&adap->sge.ctrlq[i].q);
}

static void enable_dbs(struct adapter *adap)
{
	int i;

	for_each_ethrxq(&adap->sge, i)
2144
		enable_txq_db(adap, &adap->sge.ethtxq[i].q);
2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156
	if (is_offload(adap)) {
		struct sge_uld_txq_info *txq_info =
			adap->sge.uld_txq_info[CXGB4_TX_OFLD];

		if (txq_info) {
			for_each_ofldtxq(&adap->sge, i) {
				struct sge_uld_txq *txq = &txq_info->uldtxq[i];

				enable_txq_db(adap, &txq->q);
			}
		}
	}
2157
	for_each_port(adap, i)
2158 2159 2160 2161 2162
		enable_txq_db(adap, &adap->sge.ctrlq[i].q);
}

static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
{
2163 2164 2165 2166
	enum cxgb4_uld type = CXGB4_ULD_RDMA;

	if (adap->uld && adap->uld[type].handle)
		adap->uld[type].control(adap->uld[type].handle, cmd);
2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177
}

static void process_db_full(struct work_struct *work)
{
	struct adapter *adap;

	adap = container_of(work, struct adapter, db_full_task);

	drain_db_fifo(adap, dbfifo_drain_delay);
	enable_dbs(adap);
	notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
2178 2179 2180 2181 2182 2183 2184
	if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
		t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
				 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F,
				 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F);
	else
		t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
				 DBFIFO_LP_INT_F, DBFIFO_LP_INT_F);
2185 2186 2187 2188 2189 2190 2191
}

static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
{
	u16 hw_pidx, hw_cidx;
	int ret;

2192
	spin_lock_irq(&q->db_lock);
2193 2194 2195 2196 2197
	ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
	if (ret)
		goto out;
	if (q->db_pidx != hw_pidx) {
		u16 delta;
2198
		u32 val;
2199 2200 2201 2202 2203

		if (q->db_pidx >= hw_pidx)
			delta = q->db_pidx - hw_pidx;
		else
			delta = q->size - hw_pidx + q->db_pidx;
2204 2205 2206 2207 2208

		if (is_t4(adap->params.chip))
			val = PIDX_V(delta);
		else
			val = PIDX_T5_V(delta);
2209
		wmb();
2210 2211
		t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
			     QID_V(q->cntxt_id) | val);
2212 2213 2214
	}
out:
	q->db_disabled = 0;
2215 2216
	q->db_pidx_inc = 0;
	spin_unlock_irq(&q->db_lock);
2217 2218 2219
	if (ret)
		CH_WARN(adap, "DB drop recovery failed.\n");
}
2220

2221 2222 2223 2224 2225 2226
static void recover_all_queues(struct adapter *adap)
{
	int i;

	for_each_ethrxq(&adap->sge, i)
		sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237
	if (is_offload(adap)) {
		struct sge_uld_txq_info *txq_info =
			adap->sge.uld_txq_info[CXGB4_TX_OFLD];
		if (txq_info) {
			for_each_ofldtxq(&adap->sge, i) {
				struct sge_uld_txq *txq = &txq_info->uldtxq[i];

				sync_txq_pidx(adap, &txq->q);
			}
		}
	}
2238 2239 2240 2241
	for_each_port(adap, i)
		sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
}

2242 2243 2244 2245
static void process_db_drop(struct work_struct *work)
{
	struct adapter *adap;

2246
	adap = container_of(work, struct adapter, db_drop_task);
2247

2248
	if (is_t4(adap->params.chip)) {
2249
		drain_db_fifo(adap, dbfifo_drain_delay);
2250
		notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
2251
		drain_db_fifo(adap, dbfifo_drain_delay);
2252
		recover_all_queues(adap);
2253
		drain_db_fifo(adap, dbfifo_drain_delay);
2254
		enable_dbs(adap);
2255
		notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
2256
	} else if (is_t5(adap->params.chip)) {
2257 2258 2259
		u32 dropped_db = t4_read_reg(adap, 0x010ac);
		u16 qid = (dropped_db >> 15) & 0x1ffff;
		u16 pidx_inc = dropped_db & 0x1fff;
2260 2261 2262
		u64 bar2_qoffset;
		unsigned int bar2_qid;
		int ret;
2263

2264
		ret = t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS,
2265
					0, &bar2_qoffset, &bar2_qid);
2266 2267 2268 2269
		if (ret)
			dev_err(adap->pdev_dev, "doorbell drop recovery: "
				"qid=%d, pidx_inc=%d\n", qid, pidx_inc);
		else
2270
			writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid),
2271
			       adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL);
2272 2273 2274 2275 2276

		/* Re-enable BAR2 WC */
		t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
	}

2277 2278
	if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
		t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0);
2279 2280 2281 2282
}

void t4_db_full(struct adapter *adap)
{
2283
	if (is_t4(adap->params.chip)) {
2284 2285
		disable_dbs(adap);
		notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
2286 2287
		t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
				 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0);
2288
		queue_work(adap->workq, &adap->db_full_task);
2289
	}
2290 2291 2292 2293
}

void t4_db_dropped(struct adapter *adap)
{
2294 2295 2296 2297
	if (is_t4(adap->params.chip)) {
		disable_dbs(adap);
		notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
	}
2298
	queue_work(adap->workq, &adap->db_drop_task);
2299 2300
}

2301 2302
void t4_register_netevent_notifier(void)
{
2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314
	if (!netevent_registered) {
		register_netevent_notifier(&cxgb4_netevent_nb);
		netevent_registered = true;
	}
}

static void detach_ulds(struct adapter *adap)
{
	unsigned int i;

	mutex_lock(&uld_mutex);
	list_del(&adap->list_node);
2315

2316
	for (i = 0; i < CXGB4_ULD_MAX; i++)
2317
		if (adap->uld && adap->uld[i].handle)
2318 2319
			adap->uld[i].state_change(adap->uld[i].handle,
					     CXGB4_STATE_DETACH);
2320

2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333
	if (netevent_registered && list_empty(&adapter_list)) {
		unregister_netevent_notifier(&cxgb4_netevent_nb);
		netevent_registered = false;
	}
	mutex_unlock(&uld_mutex);
}

static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
{
	unsigned int i;

	mutex_lock(&uld_mutex);
	for (i = 0; i < CXGB4_ULD_MAX; i++)
2334 2335 2336
		if (adap->uld && adap->uld[i].handle)
			adap->uld[i].state_change(adap->uld[i].handle,
						  new_state);
2337 2338 2339
	mutex_unlock(&uld_mutex);
}

2340
#if IS_ENABLED(CONFIG_IPV6)
2341 2342
static int cxgb4_inet6addr_handler(struct notifier_block *this,
				   unsigned long event, void *data)
2343
{
2344 2345 2346 2347
	struct inet6_ifaddr *ifa = data;
	struct net_device *event_dev = ifa->idev->dev;
	const struct device *parent = NULL;
#if IS_ENABLED(CONFIG_BONDING)
2348
	struct adapter *adap;
2349
#endif
2350
	if (is_vlan_dev(event_dev))
2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370
		event_dev = vlan_dev_real_dev(event_dev);
#if IS_ENABLED(CONFIG_BONDING)
	if (event_dev->flags & IFF_MASTER) {
		list_for_each_entry(adap, &adapter_list, list_node) {
			switch (event) {
			case NETDEV_UP:
				cxgb4_clip_get(adap->port[0],
					       (const u32 *)ifa, 1);
				break;
			case NETDEV_DOWN:
				cxgb4_clip_release(adap->port[0],
						   (const u32 *)ifa, 1);
				break;
			default:
				break;
			}
		}
		return NOTIFY_OK;
	}
#endif
2371

2372 2373
	if (event_dev)
		parent = event_dev->dev.parent;
2374

2375
	if (parent && parent->driver == &cxgb4_driver.driver) {
2376 2377
		switch (event) {
		case NETDEV_UP:
2378
			cxgb4_clip_get(event_dev, (const u32 *)ifa, 1);
2379 2380
			break;
		case NETDEV_DOWN:
2381
			cxgb4_clip_release(event_dev, (const u32 *)ifa, 1);
2382 2383 2384 2385 2386
			break;
		default:
			break;
		}
	}
2387
	return NOTIFY_OK;
2388 2389
}

2390
static bool inet6addr_registered;
2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407
static struct notifier_block cxgb4_inet6addr_notifier = {
	.notifier_call = cxgb4_inet6addr_handler
};

static void update_clip(const struct adapter *adap)
{
	int i;
	struct net_device *dev;
	int ret;

	rcu_read_lock();

	for (i = 0; i < MAX_NPORTS; i++) {
		dev = adap->port[i];
		ret = 0;

		if (dev)
2408
			ret = cxgb4_update_root_dev_clip(dev);
2409 2410 2411 2412 2413 2414

		if (ret < 0)
			break;
	}
	rcu_read_unlock();
}
2415
#endif /* IS_ENABLED(CONFIG_IPV6) */
2416

2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428
/**
 *	cxgb_up - enable the adapter
 *	@adap: adapter being enabled
 *
 *	Called when the first port is enabled, this function performs the
 *	actions necessary to make an adapter operational, such as completing
 *	the initialization of HW modules, and enabling interrupts.
 *
 *	Must be called with the rtnl lock held.
 */
static int cxgb_up(struct adapter *adap)
{
2429
	struct sge *s = &adap->sge;
2430
	int err;
2431

2432
	mutex_lock(&uld_mutex);
2433 2434
	err = setup_sge_queues(adap);
	if (err)
2435
		goto rel_lock;
2436 2437 2438
	err = setup_rss(adap);
	if (err)
		goto freeq;
2439

2440
	if (adap->flags & CXGB4_USING_MSIX) {
2441 2442 2443 2444 2445 2446 2447 2448
		if (s->nd_msix_idx < 0) {
			err = -ENOMEM;
			goto irq_err;
		}

		err = request_irq(adap->msix_info[s->nd_msix_idx].vec,
				  t4_nondata_intr, 0,
				  adap->msix_info[s->nd_msix_idx].desc, adap);
2449 2450
		if (err)
			goto irq_err;
2451

2452
		err = request_msix_queue_irqs(adap);
2453 2454
		if (err)
			goto irq_err_free_nd_msix;
2455 2456
	} else {
		err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2457 2458
				  (adap->flags & CXGB4_USING_MSI) ? 0
								  : IRQF_SHARED,
2459
				  adap->port[0]->name, adap);
2460 2461 2462
		if (err)
			goto irq_err;
	}
2463

2464 2465 2466
	enable_rx(adap);
	t4_sge_start(adap);
	t4_intr_enable(adap);
2467
	adap->flags |= CXGB4_FULL_INIT_DONE;
2468 2469
	mutex_unlock(&uld_mutex);

2470
	notify_ulds(adap, CXGB4_STATE_UP);
2471
#if IS_ENABLED(CONFIG_IPV6)
2472
	update_clip(adap);
2473
#endif
2474
	return err;
2475

2476 2477 2478
irq_err_free_nd_msix:
	free_irq(adap->msix_info[s->nd_msix_idx].vec, adap);
irq_err:
2479
	dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
2480
freeq:
2481
	t4_free_sge_resources(adap);
2482
rel_lock:
2483 2484
	mutex_unlock(&uld_mutex);
	return err;
2485 2486 2487 2488 2489
}

static void cxgb_down(struct adapter *adapter)
{
	cancel_work_sync(&adapter->tid_release_task);
2490 2491
	cancel_work_sync(&adapter->db_full_task);
	cancel_work_sync(&adapter->db_drop_task);
2492
	adapter->tid_release_task_busy = false;
D
Dimitris Michailidis 已提交
2493
	adapter->tid_release_head = NULL;
2494

2495 2496
	t4_sge_stop(adapter);
	t4_free_sge_resources(adapter);
A
Arjun Vynipadath 已提交
2497

2498
	adapter->flags &= ~CXGB4_FULL_INIT_DONE;
2499 2500 2501 2502 2503
}

/*
 * net_device operations
 */
2504
int cxgb_open(struct net_device *dev)
2505 2506 2507
{
	struct port_info *pi = netdev_priv(dev);
	struct adapter *adapter = pi->adapter;
2508
	int err;
2509

2510 2511
	netif_carrier_off(dev);

2512
	if (!(adapter->flags & CXGB4_FULL_INIT_DONE)) {
2513 2514 2515 2516
		err = cxgb_up(adapter);
		if (err < 0)
			return err;
	}
2517

2518 2519 2520 2521 2522 2523 2524
	/* It's possible that the basic port information could have
	 * changed since we first read it.
	 */
	err = t4_update_port_info(pi);
	if (err < 0)
		return err;

2525 2526 2527 2528
	err = link_start(dev);
	if (!err)
		netif_tx_start_all_queues(dev);
	return err;
2529 2530
}

2531
int cxgb_close(struct net_device *dev)
2532 2533 2534
{
	struct port_info *pi = netdev_priv(dev);
	struct adapter *adapter = pi->adapter;
2535
	int ret;
2536 2537 2538

	netif_tx_stop_all_queues(dev);
	netif_carrier_off(dev);
2539 2540
	ret = t4_enable_pi_params(adapter, adapter->pf, pi,
				  false, false, false);
2541 2542 2543 2544 2545
#ifdef CONFIG_CHELSIO_T4_DCB
	cxgb4_dcb_reset(dev);
	dcb_tx_queue_prio_enable(dev, false);
#endif
	return ret;
2546 2547
}

2548
int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
2549 2550
		__be32 sip, __be16 sport, __be16 vlan,
		unsigned int queue, unsigned char port, unsigned char mask)
2551 2552 2553 2554 2555 2556 2557 2558 2559
{
	int ret;
	struct filter_entry *f;
	struct adapter *adap;
	int i;
	u8 *val;

	adap = netdev2adap(dev);

2560
	/* Adjust stid to correct filter index */
2561
	stid -= adap->tids.sftid_base;
2562 2563
	stid += adap->tids.nftids;

2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581
	/* Check to make sure the filter requested is writable ...
	 */
	f = &adap->tids.ftid_tab[stid];
	ret = writable_filter(f);
	if (ret)
		return ret;

	/* Clear out any old resources being used by the filter before
	 * we start constructing the new filter.
	 */
	if (f->valid)
		clear_filter(adap, f);

	/* Clear out filter specifications */
	memset(&f->fs, 0, sizeof(struct ch_filter_specification));
	f->fs.val.lport = cpu_to_be16(sport);
	f->fs.mask.lport  = ~0;
	val = (u8 *)&sip;
2582
	if ((val[0] | val[1] | val[2] | val[3]) != 0) {
2583 2584 2585 2586
		for (i = 0; i < 4; i++) {
			f->fs.val.lip[i] = val[i];
			f->fs.mask.lip[i] = ~0;
		}
2587
		if (adap->params.tp.vlan_pri_map & PORT_F) {
2588 2589 2590 2591
			f->fs.val.iport = port;
			f->fs.mask.iport = mask;
		}
	}
2592

2593
	if (adap->params.tp.vlan_pri_map & PROTOCOL_F) {
2594 2595 2596 2597
		f->fs.val.proto = IPPROTO_TCP;
		f->fs.mask.proto = ~0;
	}

2598 2599 2600 2601 2602 2603
	f->fs.dirsteer = 1;
	f->fs.iq = queue;
	/* Mark filter as locked */
	f->locked = 1;
	f->fs.rpttid = 1;

2604 2605 2606 2607
	/* Save the actual tid. We need this to get the corresponding
	 * filter entry structure in filter_rpl.
	 */
	f->tid = stid + adap->tids.ftid_base;
2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624
	ret = set_filter_wr(adap, stid);
	if (ret) {
		clear_filter(adap, f);
		return ret;
	}

	return 0;
}
EXPORT_SYMBOL(cxgb4_create_server_filter);

int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
		unsigned int queue, bool ipv6)
{
	struct filter_entry *f;
	struct adapter *adap;

	adap = netdev2adap(dev);
2625 2626

	/* Adjust stid to correct filter index */
2627
	stid -= adap->tids.sftid_base;
2628 2629
	stid += adap->tids.nftids;

2630 2631 2632 2633
	f = &adap->tids.ftid_tab[stid];
	/* Unlock the filter */
	f->locked = 0;

2634
	return delete_filter(adap, stid);
2635 2636 2637
}
EXPORT_SYMBOL(cxgb4_remove_server_filter);

2638 2639
static void cxgb_get_stats(struct net_device *dev,
			   struct rtnl_link_stats64 *ns)
2640 2641 2642 2643 2644
{
	struct port_stats stats;
	struct port_info *p = netdev_priv(dev);
	struct adapter *adapter = p->adapter;

2645 2646 2647 2648
	/* Block retrieving statistics during EEH error
	 * recovery. Otherwise, the recovery might fail
	 * and the PCI device will be removed permanently
	 */
2649
	spin_lock(&adapter->stats_lock);
2650 2651
	if (!netif_device_present(dev)) {
		spin_unlock(&adapter->stats_lock);
2652
		return;
2653
	}
2654 2655
	t4_get_port_stats_offset(adapter, p->tx_chan, &stats,
				 &p->stats_base);
2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669
	spin_unlock(&adapter->stats_lock);

	ns->tx_bytes   = stats.tx_octets;
	ns->tx_packets = stats.tx_frames;
	ns->rx_bytes   = stats.rx_octets;
	ns->rx_packets = stats.rx_frames;
	ns->multicast  = stats.rx_mcast_frames;

	/* detailed rx_errors */
	ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
			       stats.rx_runt;
	ns->rx_over_errors   = 0;
	ns->rx_crc_errors    = stats.rx_fcs_err;
	ns->rx_frame_errors  = stats.rx_symbol_err;
2670
	ns->rx_dropped	     = stats.rx_ovflow0 + stats.rx_ovflow1 +
2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689
			       stats.rx_ovflow2 + stats.rx_ovflow3 +
			       stats.rx_trunc0 + stats.rx_trunc1 +
			       stats.rx_trunc2 + stats.rx_trunc3;
	ns->rx_missed_errors = 0;

	/* detailed tx_errors */
	ns->tx_aborted_errors   = 0;
	ns->tx_carrier_errors   = 0;
	ns->tx_fifo_errors      = 0;
	ns->tx_heartbeat_errors = 0;
	ns->tx_window_errors    = 0;

	ns->tx_errors = stats.tx_error_frames;
	ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
		ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
}

static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
{
2690
	unsigned int mbox;
2691 2692
	int ret = 0, prtad, devad;
	struct port_info *pi = netdev_priv(dev);
2693
	struct adapter *adapter = pi->adapter;
2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713
	struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;

	switch (cmd) {
	case SIOCGMIIPHY:
		if (pi->mdio_addr < 0)
			return -EOPNOTSUPP;
		data->phy_id = pi->mdio_addr;
		break;
	case SIOCGMIIREG:
	case SIOCSMIIREG:
		if (mdio_phy_id_is_c45(data->phy_id)) {
			prtad = mdio_phy_id_prtad(data->phy_id);
			devad = mdio_phy_id_devad(data->phy_id);
		} else if (data->phy_id < 32) {
			prtad = data->phy_id;
			devad = 0;
			data->reg_num &= 0x1f;
		} else
			return -EINVAL;

2714
		mbox = pi->adapter->pf;
2715
		if (cmd == SIOCGMIIREG)
2716
			ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
2717 2718
					 data->reg_num, &data->val_out);
		else
2719
			ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
2720 2721
					 data->reg_num, data->val_in);
		break;
2722 2723 2724 2725 2726 2727 2728 2729 2730
	case SIOCGHWTSTAMP:
		return copy_to_user(req->ifr_data, &pi->tstamp_config,
				    sizeof(pi->tstamp_config)) ?
			-EFAULT : 0;
	case SIOCSHWTSTAMP:
		if (copy_from_user(&pi->tstamp_config, req->ifr_data,
				   sizeof(pi->tstamp_config)))
			return -EFAULT;

2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782
		if (!is_t4(adapter->params.chip)) {
			switch (pi->tstamp_config.tx_type) {
			case HWTSTAMP_TX_OFF:
			case HWTSTAMP_TX_ON:
				break;
			default:
				return -ERANGE;
			}

			switch (pi->tstamp_config.rx_filter) {
			case HWTSTAMP_FILTER_NONE:
				pi->rxtstamp = false;
				break;
			case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
			case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
				cxgb4_ptprx_timestamping(pi, pi->port_id,
							 PTP_TS_L4);
				break;
			case HWTSTAMP_FILTER_PTP_V2_EVENT:
				cxgb4_ptprx_timestamping(pi, pi->port_id,
							 PTP_TS_L2_L4);
				break;
			case HWTSTAMP_FILTER_ALL:
			case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
			case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
			case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
			case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
				pi->rxtstamp = true;
				break;
			default:
				pi->tstamp_config.rx_filter =
					HWTSTAMP_FILTER_NONE;
				return -ERANGE;
			}

			if ((pi->tstamp_config.tx_type == HWTSTAMP_TX_OFF) &&
			    (pi->tstamp_config.rx_filter ==
				HWTSTAMP_FILTER_NONE)) {
				if (cxgb4_ptp_txtype(adapter, pi->port_id) >= 0)
					pi->ptp_enable = false;
			}

			if (pi->tstamp_config.rx_filter !=
				HWTSTAMP_FILTER_NONE) {
				if (cxgb4_ptp_redirect_rx_packet(adapter,
								 pi) >= 0)
					pi->ptp_enable = true;
			}
		} else {
			/* For T4 Adapters */
			switch (pi->tstamp_config.rx_filter) {
			case HWTSTAMP_FILTER_NONE:
2783 2784
			pi->rxtstamp = false;
			break;
2785
			case HWTSTAMP_FILTER_ALL:
2786 2787
			pi->rxtstamp = true;
			break;
2788 2789 2790
			default:
			pi->tstamp_config.rx_filter =
			HWTSTAMP_FILTER_NONE;
2791
			return -ERANGE;
2792
			}
2793 2794 2795 2796
		}
		return copy_to_user(req->ifr_data, &pi->tstamp_config,
				    sizeof(pi->tstamp_config)) ?
			-EFAULT : 0;
2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813
	default:
		return -EOPNOTSUPP;
	}
	return ret;
}

static void cxgb_set_rxmode(struct net_device *dev)
{
	/* unfortunately we can't return errors to the stack */
	set_rxmode(dev, -1, false);
}

static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
{
	int ret;
	struct port_info *pi = netdev_priv(dev);

2814
	ret = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, new_mtu, -1,
2815
			    -1, -1, -1, true);
2816 2817 2818 2819 2820
	if (!ret)
		dev->mtu = new_mtu;
	return ret;
}

2821
#ifdef CONFIG_PCI_IOV
G
Ganesh Goudar 已提交
2822
static int cxgb4_mgmt_open(struct net_device *dev)
2823 2824 2825 2826 2827 2828 2829 2830
{
	/* Turn carrier off since we don't have to transmit anything on this
	 * interface.
	 */
	netif_carrier_off(dev);
	return 0;
}

2831
/* Fill MAC address that will be assigned by the FW */
G
Ganesh Goudar 已提交
2832
static void cxgb4_mgmt_fill_vf_station_mac_addr(struct adapter *adap)
2833 2834
{
	u8 hw_addr[ETH_ALEN], macaddr[ETH_ALEN];
G
Ganesh Goudar 已提交
2835 2836
	unsigned int i, vf, nvfs;
	u16 a, b;
2837 2838 2839
	int err;
	u8 *na;

G
Ganesh Goudar 已提交
2840 2841
	adap->params.pci.vpd_cap_addr = pci_find_capability(adap->pdev,
							    PCI_CAP_ID_VPD);
2842
	err = t4_get_raw_vpd_params(adap, &adap->params.vpd);
G
Ganesh Goudar 已提交
2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863
	if (err)
		return;

	na = adap->params.vpd.na;
	for (i = 0; i < ETH_ALEN; i++)
		hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 +
			      hex2val(na[2 * i + 1]));

	a = (hw_addr[0] << 8) | hw_addr[1];
	b = (hw_addr[1] << 8) | hw_addr[2];
	a ^= b;
	a |= 0x0200;    /* locally assigned Ethernet MAC address */
	a &= ~0x0100;   /* not a multicast Ethernet MAC address */
	macaddr[0] = a >> 8;
	macaddr[1] = a & 0xff;

	for (i = 2; i < 5; i++)
		macaddr[i] = hw_addr[i + 1];

	for (vf = 0, nvfs = pci_sriov_get_totalvfs(adap->pdev);
		vf < nvfs; vf++) {
2864
		macaddr[5] = adap->pf * nvfs + vf;
G
Ganesh Goudar 已提交
2865
		ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, macaddr);
2866 2867 2868
	}
}

G
Ganesh Goudar 已提交
2869
static int cxgb4_mgmt_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
2870 2871 2872
{
	struct port_info *pi = netdev_priv(dev);
	struct adapter *adap = pi->adapter;
2873
	int ret;
2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884

	/* verify MAC addr is valid */
	if (!is_valid_ether_addr(mac)) {
		dev_err(pi->adapter->pdev_dev,
			"Invalid Ethernet address %pM for VF %d\n",
			mac, vf);
		return -EINVAL;
	}

	dev_info(pi->adapter->pdev_dev,
		 "Setting MAC %pM on VF %d\n", mac, vf);
2885 2886 2887 2888 2889 2890
	ret = t4_set_vf_mac_acl(adap, vf + 1, 1, mac);
	if (!ret)
		ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, mac);
	return ret;
}

G
Ganesh Goudar 已提交
2891 2892
static int cxgb4_mgmt_get_vf_config(struct net_device *dev,
				    int vf, struct ifla_vf_info *ivi)
2893 2894 2895
{
	struct port_info *pi = netdev_priv(dev);
	struct adapter *adap = pi->adapter;
2896
	struct vf_info *vfinfo;
2897 2898 2899

	if (vf >= adap->num_vfs)
		return -EINVAL;
2900 2901
	vfinfo = &adap->vfinfo[vf];

2902
	ivi->vf = vf;
2903
	ivi->max_tx_rate = vfinfo->tx_rate;
2904
	ivi->min_tx_rate = 0;
2905 2906
	ether_addr_copy(ivi->mac, vfinfo->vf_mac_addr);
	ivi->vlan = vfinfo->vlan;
2907
	ivi->linkstate = vfinfo->link_state;
2908
	return 0;
2909
}
2910

G
Ganesh Goudar 已提交
2911 2912
static int cxgb4_mgmt_get_phys_port_id(struct net_device *dev,
				       struct netdev_phys_item_id *ppid)
2913 2914 2915 2916 2917 2918 2919 2920 2921 2922
{
	struct port_info *pi = netdev_priv(dev);
	unsigned int phy_port_id;

	phy_port_id = pi->adapter->adap_idx * 10 + pi->port_id;
	ppid->id_len = sizeof(phy_port_id);
	memcpy(ppid->id, &phy_port_id, ppid->id_len);
	return 0;
}

G
Ganesh Goudar 已提交
2923 2924
static int cxgb4_mgmt_set_vf_rate(struct net_device *dev, int vf,
				  int min_tx_rate, int max_tx_rate)
2925 2926 2927
{
	struct port_info *pi = netdev_priv(dev);
	struct adapter *adap = pi->adapter;
2928
	unsigned int link_ok, speed, mtu;
2929 2930
	u32 fw_pfvf, fw_class;
	int class_id = vf;
2931
	int ret;
2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942
	u16 pktsize;

	if (vf >= adap->num_vfs)
		return -EINVAL;

	if (min_tx_rate) {
		dev_err(adap->pdev_dev,
			"Min tx rate (%d) (> 0) for VF %d is Invalid.\n",
			min_tx_rate, vf);
		return -EINVAL;
	}
2943

2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964
	if (max_tx_rate == 0) {
		/* unbind VF to to any Traffic Class */
		fw_pfvf =
		    (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) |
		     FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_SCHEDCLASS_ETH));
		fw_class = 0xffffffff;
		ret = t4_set_params(adap, adap->mbox, adap->pf, vf + 1, 1,
				    &fw_pfvf, &fw_class);
		if (ret) {
			dev_err(adap->pdev_dev,
				"Err %d in unbinding PF %d VF %d from TX Rate Limiting\n",
				ret, adap->pf, vf);
			return -EINVAL;
		}
		dev_info(adap->pdev_dev,
			 "PF %d VF %d is unbound from TX Rate Limiting\n",
			 adap->pf, vf);
		adap->vfinfo[vf].tx_rate = 0;
		return 0;
	}

2965
	ret = t4_get_link_params(pi, &link_ok, &speed, &mtu);
2966 2967
	if (ret != FW_SUCCESS) {
		dev_err(adap->pdev_dev,
2968
			"Failed to get link information for VF %d\n", vf);
2969 2970
		return -EINVAL;
	}
2971

2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982
	if (!link_ok) {
		dev_err(adap->pdev_dev, "Link down for VF %d\n", vf);
		return -EINVAL;
	}

	if (max_tx_rate > speed) {
		dev_err(adap->pdev_dev,
			"Max tx rate %d for VF %d can't be > link-speed %u",
			max_tx_rate, vf, speed);
		return -EINVAL;
	}
2983 2984

	pktsize = mtu;
2985 2986 2987 2988 2989 2990 2991 2992 2993 2994
	/* subtract ethhdr size and 4 bytes crc since, f/w appends it */
	pktsize = pktsize - sizeof(struct ethhdr) - 4;
	/* subtract ipv4 hdr size, tcp hdr size to get typical IPv4 MSS size */
	pktsize = pktsize - sizeof(struct iphdr) - sizeof(struct tcphdr);
	/* configure Traffic Class for rate-limiting */
	ret = t4_sched_params(adap, SCHED_CLASS_TYPE_PACKET,
			      SCHED_CLASS_LEVEL_CL_RL,
			      SCHED_CLASS_MODE_CLASS,
			      SCHED_CLASS_RATEUNIT_BITS,
			      SCHED_CLASS_RATEMODE_ABS,
2995
			      pi->tx_chan, class_id, 0,
2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013
			      max_tx_rate * 1000, 0, pktsize);
	if (ret) {
		dev_err(adap->pdev_dev, "Err %d for Traffic Class config\n",
			ret);
		return -EINVAL;
	}
	dev_info(adap->pdev_dev,
		 "Class %d with MSS %u configured with rate %u\n",
		 class_id, pktsize, max_tx_rate);

	/* bind VF to configured Traffic Class */
	fw_pfvf = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) |
		   FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_SCHEDCLASS_ETH));
	fw_class = class_id;
	ret = t4_set_params(adap, adap->mbox, adap->pf, vf + 1, 1, &fw_pfvf,
			    &fw_class);
	if (ret) {
		dev_err(adap->pdev_dev,
3014 3015
			"Err %d in binding PF %d VF %d to Traffic Class %d\n",
			ret, adap->pf, vf, class_id);
3016 3017 3018 3019 3020 3021 3022 3023
		return -EINVAL;
	}
	dev_info(adap->pdev_dev, "PF %d VF %d is bound to Class %d\n",
		 adap->pf, vf, class_id);
	adap->vfinfo[vf].tx_rate = max_tx_rate;
	return 0;
}

3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046
static int cxgb4_mgmt_set_vf_vlan(struct net_device *dev, int vf,
				  u16 vlan, u8 qos, __be16 vlan_proto)
{
	struct port_info *pi = netdev_priv(dev);
	struct adapter *adap = pi->adapter;
	int ret;

	if (vf >= adap->num_vfs || vlan > 4095 || qos > 7)
		return -EINVAL;

	if (vlan_proto != htons(ETH_P_8021Q) || qos != 0)
		return -EPROTONOSUPPORT;

	ret = t4_set_vlan_acl(adap, adap->mbox, vf + 1, vlan);
	if (!ret) {
		adap->vfinfo[vf].vlan = vlan;
		return 0;
	}

	dev_err(adap->pdev_dev, "Err %d %s VLAN ACL for PF/VF %d/%d\n",
		ret, (vlan ? "setting" : "clearing"), adap->pf, vf);
	return ret;
}
3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089

static int cxgb4_mgmt_set_vf_link_state(struct net_device *dev, int vf,
					int link)
{
	struct port_info *pi = netdev_priv(dev);
	struct adapter *adap = pi->adapter;
	u32 param, val;
	int ret = 0;

	if (vf >= adap->num_vfs)
		return -EINVAL;

	switch (link) {
	case IFLA_VF_LINK_STATE_AUTO:
		val = FW_VF_LINK_STATE_AUTO;
		break;

	case IFLA_VF_LINK_STATE_ENABLE:
		val = FW_VF_LINK_STATE_ENABLE;
		break;

	case IFLA_VF_LINK_STATE_DISABLE:
		val = FW_VF_LINK_STATE_DISABLE;
		break;

	default:
		return -EINVAL;
	}

	param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) |
		 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_LINK_STATE));
	ret = t4_set_params(adap, adap->mbox, adap->pf, vf + 1, 1,
			    &param, &val);
	if (ret) {
		dev_err(adap->pdev_dev,
			"Error %d in setting PF %d VF %d link state\n",
			ret, adap->pf, vf);
		return -EINVAL;
	}

	adap->vfinfo[vf].link_state = link;
	return ret;
}
3090
#endif /* CONFIG_PCI_IOV */
3091

3092 3093 3094 3095 3096 3097 3098
static int cxgb_set_mac_addr(struct net_device *dev, void *p)
{
	int ret;
	struct sockaddr *addr = p;
	struct port_info *pi = netdev_priv(dev);

	if (!is_valid_ether_addr(addr->sa_data))
3099
		return -EADDRNOTAVAIL;
3100

3101 3102
	ret = cxgb4_update_mac_filt(pi, pi->viid, &pi->xact_addr_filt,
				    addr->sa_data, true, &pi->smt_idx);
3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116
	if (ret < 0)
		return ret;

	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
	pi->xact_addr_filt = ret;
	return 0;
}

#ifdef CONFIG_NET_POLL_CONTROLLER
static void cxgb_netpoll(struct net_device *dev)
{
	struct port_info *pi = netdev_priv(dev);
	struct adapter *adap = pi->adapter;

3117
	if (adap->flags & CXGB4_USING_MSIX) {
3118 3119 3120 3121 3122 3123 3124 3125 3126 3127
		int i;
		struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];

		for (i = pi->nqsets; i; i--, rx++)
			t4_sge_intr_msix(0, &rx->rspq);
	} else
		t4_intr_handler(adap)(0, adap);
}
#endif

3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143
static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
{
	struct port_info *pi = netdev_priv(dev);
	struct adapter *adap = pi->adapter;
	struct sched_class *e;
	struct ch_sched_params p;
	struct ch_sched_queue qe;
	u32 req_rate;
	int err = 0;

	if (!can_sched(dev))
		return -ENOTSUPP;

	if (index < 0 || index > pi->nqsets - 1)
		return -EINVAL;

3144
	if (!(adap->flags & CXGB4_FULL_INIT_DONE)) {
3145 3146 3147 3148 3149 3150 3151
		dev_err(adap->pdev_dev,
			"Failed to rate limit on queue %d. Link Down?\n",
			index);
		return -EINVAL;
	}

	/* Convert from Mbps to Kbps */
3152
	req_rate = rate * 1000;
3153

3154
	/* Max rate is 100 Gbps */
3155
	if (req_rate > SCHED_MAX_RATE_KBPS) {
3156
		dev_err(adap->pdev_dev,
3157
			"Invalid rate %u Mbps, Max rate is %u Mbps\n",
3158
			rate, SCHED_MAX_RATE_KBPS / 1000);
3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208
		return -ERANGE;
	}

	/* First unbind the queue from any existing class */
	memset(&qe, 0, sizeof(qe));
	qe.queue = index;
	qe.class = SCHED_CLS_NONE;

	err = cxgb4_sched_class_unbind(dev, (void *)(&qe), SCHED_QUEUE);
	if (err) {
		dev_err(adap->pdev_dev,
			"Unbinding Queue %d on port %d fail. Err: %d\n",
			index, pi->port_id, err);
		return err;
	}

	/* Queue already unbound */
	if (!req_rate)
		return 0;

	/* Fetch any available unused or matching scheduling class */
	memset(&p, 0, sizeof(p));
	p.type = SCHED_CLASS_TYPE_PACKET;
	p.u.params.level    = SCHED_CLASS_LEVEL_CL_RL;
	p.u.params.mode     = SCHED_CLASS_MODE_CLASS;
	p.u.params.rateunit = SCHED_CLASS_RATEUNIT_BITS;
	p.u.params.ratemode = SCHED_CLASS_RATEMODE_ABS;
	p.u.params.channel  = pi->tx_chan;
	p.u.params.class    = SCHED_CLS_NONE;
	p.u.params.minrate  = 0;
	p.u.params.maxrate  = req_rate;
	p.u.params.weight   = 0;
	p.u.params.pktsize  = dev->mtu;

	e = cxgb4_sched_class_alloc(dev, &p);
	if (!e)
		return -ENOMEM;

	/* Bind the queue to a scheduling class */
	memset(&qe, 0, sizeof(qe));
	qe.queue = index;
	qe.class = e->idx;

	err = cxgb4_sched_class_bind(dev, (void *)(&qe), SCHED_QUEUE);
	if (err)
		dev_err(adap->pdev_dev,
			"Queue rate limiting failed. Err: %d\n", err);
	return err;
}

3209
static int cxgb_setup_tc_flower(struct net_device *dev,
3210
				struct flow_cls_offload *cls_flower)
3211 3212
{
	switch (cls_flower->command) {
3213
	case FLOW_CLS_REPLACE:
3214
		return cxgb4_tc_flower_replace(dev, cls_flower);
3215
	case FLOW_CLS_DESTROY:
3216
		return cxgb4_tc_flower_destroy(dev, cls_flower);
3217
	case FLOW_CLS_STATS:
3218 3219 3220 3221 3222 3223
		return cxgb4_tc_flower_stats(dev, cls_flower);
	default:
		return -EOPNOTSUPP;
	}
}

3224 3225 3226 3227 3228 3229
static int cxgb_setup_tc_cls_u32(struct net_device *dev,
				 struct tc_cls_u32_offload *cls_u32)
{
	switch (cls_u32->command) {
	case TC_CLSU32_NEW_KNODE:
	case TC_CLSU32_REPLACE_KNODE:
3230
		return cxgb4_config_knode(dev, cls_u32);
3231
	case TC_CLSU32_DELETE_KNODE:
3232
		return cxgb4_delete_knode(dev, cls_u32);
3233 3234 3235 3236 3237
	default:
		return -EOPNOTSUPP;
	}
}

3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259
static int cxgb_setup_tc_matchall(struct net_device *dev,
				  struct tc_cls_matchall_offload *cls_matchall)
{
	struct adapter *adap = netdev2adap(dev);

	if (!adap->tc_matchall)
		return -ENOMEM;

	switch (cls_matchall->command) {
	case TC_CLSMATCHALL_REPLACE:
		return cxgb4_tc_matchall_replace(dev, cls_matchall);
	case TC_CLSMATCHALL_DESTROY:
		return cxgb4_tc_matchall_destroy(dev, cls_matchall);
	default:
		break;
	}

	return -EOPNOTSUPP;
}

static int cxgb_setup_tc_block_ingress_cb(enum tc_setup_type type,
					  void *type_data, void *cb_priv)
3260
{
3261
	struct net_device *dev = cb_priv;
3262 3263 3264
	struct port_info *pi = netdev2pinfo(dev);
	struct adapter *adap = netdev2adap(dev);

3265
	if (!(adap->flags & CXGB4_FULL_INIT_DONE)) {
3266 3267 3268 3269 3270 3271
		dev_err(adap->pdev_dev,
			"Failed to setup tc on port %d. Link Down?\n",
			pi->port_id);
		return -EINVAL;
	}

3272
	if (!tc_cls_can_offload_and_chain0(dev, type_data))
3273 3274
		return -EOPNOTSUPP;

3275 3276
	switch (type) {
	case TC_SETUP_CLSU32:
3277
		return cxgb_setup_tc_cls_u32(dev, type_data);
3278 3279
	case TC_SETUP_CLSFLOWER:
		return cxgb_setup_tc_flower(dev, type_data);
3280 3281
	default:
		return -EOPNOTSUPP;
3282 3283 3284
	}
}

3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311
static int cxgb_setup_tc_block_egress_cb(enum tc_setup_type type,
					 void *type_data, void *cb_priv)
{
	struct net_device *dev = cb_priv;
	struct port_info *pi = netdev2pinfo(dev);
	struct adapter *adap = netdev2adap(dev);

	if (!(adap->flags & CXGB4_FULL_INIT_DONE)) {
		dev_err(adap->pdev_dev,
			"Failed to setup tc on port %d. Link Down?\n",
			pi->port_id);
		return -EINVAL;
	}

	if (!tc_cls_can_offload_and_chain0(dev, type_data))
		return -EOPNOTSUPP;

	switch (type) {
	case TC_SETUP_CLSMATCHALL:
		return cxgb_setup_tc_matchall(dev, type_data);
	default:
		break;
	}

	return -EOPNOTSUPP;
}

3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322
static int cxgb_setup_tc_mqprio(struct net_device *dev,
				struct tc_mqprio_qopt_offload *mqprio)
{
	struct adapter *adap = netdev2adap(dev);

	if (!is_ethofld(adap) || !adap->tc_mqprio)
		return -ENOMEM;

	return cxgb4_setup_tc_mqprio(dev, mqprio);
}

3323 3324
static LIST_HEAD(cxgb_block_cb_list);

3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344
static int cxgb_setup_tc_block(struct net_device *dev,
			       struct flow_block_offload *f)
{
	struct port_info *pi = netdev_priv(dev);
	flow_setup_cb_t *cb;
	bool ingress_only;

	pi->tc_block_shared = f->block_shared;
	if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) {
		cb = cxgb_setup_tc_block_egress_cb;
		ingress_only = false;
	} else {
		cb = cxgb_setup_tc_block_ingress_cb;
		ingress_only = true;
	}

	return flow_block_cb_setup_simple(f, &cxgb_block_cb_list,
					  cb, pi, dev, ingress_only);
}

3345 3346 3347 3348
static int cxgb_setup_tc(struct net_device *dev, enum tc_setup_type type,
			 void *type_data)
{
	switch (type) {
3349 3350
	case TC_SETUP_QDISC_MQPRIO:
		return cxgb_setup_tc_mqprio(dev, type_data);
3351
	case TC_SETUP_BLOCK:
3352
		return cxgb_setup_tc_block(dev, type_data);
3353 3354 3355 3356 3357
	default:
		return -EOPNOTSUPP;
	}
}

3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382
static void cxgb_del_udp_tunnel(struct net_device *netdev,
				struct udp_tunnel_info *ti)
{
	struct port_info *pi = netdev_priv(netdev);
	struct adapter *adapter = pi->adapter;
	unsigned int chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip);
	u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 };
	int ret = 0, i;

	if (chip_ver < CHELSIO_T6)
		return;

	switch (ti->type) {
	case UDP_TUNNEL_TYPE_VXLAN:
		if (!adapter->vxlan_port_cnt ||
		    adapter->vxlan_port != ti->port)
			return; /* Invalid VxLAN destination port */

		adapter->vxlan_port_cnt--;
		if (adapter->vxlan_port_cnt)
			return;

		adapter->vxlan_port = 0;
		t4_write_reg(adapter, MPS_RX_VXLAN_TYPE_A, 0);
		break;
3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393
	case UDP_TUNNEL_TYPE_GENEVE:
		if (!adapter->geneve_port_cnt ||
		    adapter->geneve_port != ti->port)
			return; /* Invalid GENEVE destination port */

		adapter->geneve_port_cnt--;
		if (adapter->geneve_port_cnt)
			return;

		adapter->geneve_port = 0;
		t4_write_reg(adapter, MPS_RX_GENEVE_TYPE_A, 0);
3394
		break;
3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409
	default:
		return;
	}

	/* Matchall mac entries can be deleted only after all tunnel ports
	 * are brought down or removed.
	 */
	if (!adapter->rawf_cnt)
		return;
	for_each_port(adapter, i) {
		pi = adap2pinfo(adapter, i);
		ret = t4_free_raw_mac_filt(adapter, pi->viid,
					   match_all_mac, match_all_mac,
					   adapter->rawf_start +
					    pi->port_id,
3410
					   1, pi->port_id, false);
3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427
		if (ret < 0) {
			netdev_info(netdev, "Failed to free mac filter entry, for port %d\n",
				    i);
			return;
		}
	}
}

static void cxgb_add_udp_tunnel(struct net_device *netdev,
				struct udp_tunnel_info *ti)
{
	struct port_info *pi = netdev_priv(netdev);
	struct adapter *adapter = pi->adapter;
	unsigned int chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip);
	u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 };
	int i, ret;

3428
	if (chip_ver < CHELSIO_T6 || !adapter->rawf_cnt)
3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457
		return;

	switch (ti->type) {
	case UDP_TUNNEL_TYPE_VXLAN:
		/* Callback for adding vxlan port can be called with the same
		 * port for both IPv4 and IPv6. We should not disable the
		 * offloading when the same port for both protocols is added
		 * and later one of them is removed.
		 */
		if (adapter->vxlan_port_cnt &&
		    adapter->vxlan_port == ti->port) {
			adapter->vxlan_port_cnt++;
			return;
		}

		/* We will support only one VxLAN port */
		if (adapter->vxlan_port_cnt) {
			netdev_info(netdev, "UDP port %d already offloaded, not adding port %d\n",
				    be16_to_cpu(adapter->vxlan_port),
				    be16_to_cpu(ti->port));
			return;
		}

		adapter->vxlan_port = ti->port;
		adapter->vxlan_port_cnt = 1;

		t4_write_reg(adapter, MPS_RX_VXLAN_TYPE_A,
			     VXLAN_V(be16_to_cpu(ti->port)) | VXLAN_EN_F);
		break;
3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477
	case UDP_TUNNEL_TYPE_GENEVE:
		if (adapter->geneve_port_cnt &&
		    adapter->geneve_port == ti->port) {
			adapter->geneve_port_cnt++;
			return;
		}

		/* We will support only one GENEVE port */
		if (adapter->geneve_port_cnt) {
			netdev_info(netdev, "UDP port %d already offloaded, not adding port %d\n",
				    be16_to_cpu(adapter->geneve_port),
				    be16_to_cpu(ti->port));
			return;
		}

		adapter->geneve_port = ti->port;
		adapter->geneve_port_cnt = 1;

		t4_write_reg(adapter, MPS_RX_GENEVE_TYPE_A,
			     GENEVE_V(be16_to_cpu(ti->port)) | GENEVE_EN_F);
3478
		break;
3479 3480 3481 3482 3483 3484 3485 3486 3487 3488
	default:
		return;
	}

	/* Create a 'match all' mac filter entry for inner mac,
	 * if raw mac interface is supported. Once the linux kernel provides
	 * driver entry points for adding/deleting the inner mac addresses,
	 * we will remove this 'match all' entry and fallback to adding
	 * exact match filters.
	 */
3489 3490 3491 3492 3493 3494 3495 3496
	for_each_port(adapter, i) {
		pi = adap2pinfo(adapter, i);

		ret = t4_alloc_raw_mac_filt(adapter, pi->viid,
					    match_all_mac,
					    match_all_mac,
					    adapter->rawf_start +
					    pi->port_id,
3497
					    1, pi->port_id, false);
3498 3499 3500 3501 3502
		if (ret < 0) {
			netdev_info(netdev, "Failed to allocate a mac filter entry, not adding port %d\n",
				    be16_to_cpu(ti->port));
			cxgb_del_udp_tunnel(netdev, ti);
			return;
3503 3504 3505 3506
		}
	}
}

3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524
static netdev_features_t cxgb_features_check(struct sk_buff *skb,
					     struct net_device *dev,
					     netdev_features_t features)
{
	struct port_info *pi = netdev_priv(dev);
	struct adapter *adapter = pi->adapter;

	if (CHELSIO_CHIP_VERSION(adapter->params.chip) < CHELSIO_T6)
		return features;

	/* Check if hw supports offload for this packet */
	if (!skb->encapsulation || cxgb_encap_offload_supported(skb))
		return features;

	/* Offload is not supported for this encapsulated packet */
	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
}

3525 3526 3527 3528 3529 3530 3531 3532 3533 3534
static netdev_features_t cxgb_fix_features(struct net_device *dev,
					   netdev_features_t features)
{
	/* Disable GRO, if RX_CSUM is disabled */
	if (!(features & NETIF_F_RXCSUM))
		features &= ~NETIF_F_GRO;

	return features;
}

3535 3536 3537
static const struct net_device_ops cxgb4_netdev_ops = {
	.ndo_open             = cxgb_open,
	.ndo_stop             = cxgb_close,
3538
	.ndo_start_xmit       = t4_start_xmit,
3539
	.ndo_select_queue     =	cxgb_select_queue,
3540
	.ndo_get_stats64      = cxgb_get_stats,
3541 3542
	.ndo_set_rx_mode      = cxgb_set_rxmode,
	.ndo_set_mac_address  = cxgb_set_mac_addr,
3543
	.ndo_set_features     = cxgb_set_features,
3544 3545 3546 3547 3548 3549
	.ndo_validate_addr    = eth_validate_addr,
	.ndo_do_ioctl         = cxgb_ioctl,
	.ndo_change_mtu       = cxgb_change_mtu,
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller  = cxgb_netpoll,
#endif
V
Varun Prakash 已提交
3550 3551 3552 3553
#ifdef CONFIG_CHELSIO_T4_FCOE
	.ndo_fcoe_enable      = cxgb_fcoe_enable,
	.ndo_fcoe_disable     = cxgb_fcoe_disable,
#endif /* CONFIG_CHELSIO_T4_FCOE */
3554
	.ndo_set_tx_maxrate   = cxgb_set_tx_maxrate,
3555
	.ndo_setup_tc         = cxgb_setup_tc,
3556 3557
	.ndo_udp_tunnel_add   = cxgb_add_udp_tunnel,
	.ndo_udp_tunnel_del   = cxgb_del_udp_tunnel,
3558
	.ndo_features_check   = cxgb_features_check,
3559
	.ndo_fix_features     = cxgb_fix_features,
3560 3561
};

3562
#ifdef CONFIG_PCI_IOV
3563
static const struct net_device_ops cxgb4_mgmt_netdev_ops = {
3564 3565 3566 3567 3568 3569 3570
	.ndo_open               = cxgb4_mgmt_open,
	.ndo_set_vf_mac         = cxgb4_mgmt_set_vf_mac,
	.ndo_get_vf_config      = cxgb4_mgmt_get_vf_config,
	.ndo_set_vf_rate        = cxgb4_mgmt_set_vf_rate,
	.ndo_get_phys_port_id   = cxgb4_mgmt_get_phys_port_id,
	.ndo_set_vf_vlan        = cxgb4_mgmt_set_vf_vlan,
	.ndo_set_vf_link_state	= cxgb4_mgmt_set_vf_link_state,
3571
};
3572
#endif
3573

G
Ganesh Goudar 已提交
3574 3575
static void cxgb4_mgmt_get_drvinfo(struct net_device *dev,
				   struct ethtool_drvinfo *info)
3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586
{
	struct adapter *adapter = netdev2adap(dev);

	strlcpy(info->driver, cxgb4_driver_name, sizeof(info->driver));
	strlcpy(info->version, cxgb4_driver_version,
		sizeof(info->version));
	strlcpy(info->bus_info, pci_name(adapter->pdev),
		sizeof(info->bus_info));
}

static const struct ethtool_ops cxgb4_mgmt_ethtool_ops = {
G
Ganesh Goudar 已提交
3587
	.get_drvinfo       = cxgb4_mgmt_get_drvinfo,
3588 3589
};

3590 3591 3592 3593 3594 3595 3596 3597
static void notify_fatal_err(struct work_struct *work)
{
	struct adapter *adap;

	adap = container_of(work, struct adapter, fatal_err_notify_task);
	notify_ulds(adap, CXGB4_STATE_FATAL_ERROR);
}

3598 3599
void t4_fatal_err(struct adapter *adap)
{
3600 3601
	int port;

3602 3603 3604
	if (pci_channel_offline(adap->pdev))
		return;

3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620
	/* Disable the SGE since ULDs are going to free resources that
	 * could be exposed to the adapter.  RDMA MWs for example...
	 */
	t4_shutdown_adapter(adap);
	for_each_port(adap, port) {
		struct net_device *dev = adap->port[port];

		/* If we get here in very early initialization the network
		 * devices may not have been set up yet.
		 */
		if (!dev)
			continue;

		netif_tx_stop_all_queues(dev);
		netif_carrier_off(dev);
	}
3621
	dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
3622
	queue_work(adap->workq, &adap->fatal_err_notify_task);
3623 3624 3625 3626
}

static void setup_memwin(struct adapter *adap)
{
3627
	u32 nic_win_base = t4_get_util_window(adap);
3628

3629
	t4_setup_memwin(adap, nic_win_base, MEMWIN_NIC);
3630 3631 3632 3633
}

static void setup_memwin_rdma(struct adapter *adap)
{
3634
	if (adap->vres.ocq.size) {
3635 3636
		u32 start;
		unsigned int sz_kb;
3637

3638 3639 3640
		start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2);
		start &= PCI_BASE_ADDRESS_MEM_MASK;
		start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
3641 3642
		sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
		t4_write_reg(adap,
3643 3644
			     PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3),
			     start | BIR_V(1) | WINDOW_V(ilog2(sz_kb)));
3645
		t4_write_reg(adap,
3646
			     PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3),
3647 3648
			     adap->vres.ocq.start);
		t4_read_reg(adap,
3649
			    PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3));
3650
	}
3651 3652
}

A
Arjun Vynipadath 已提交
3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760
/* HMA Definitions */

/* The maximum number of address that can be send in a single FW cmd */
#define HMA_MAX_ADDR_IN_CMD	5

#define HMA_PAGE_SIZE		PAGE_SIZE

#define HMA_MAX_NO_FW_ADDRESS	(16 << 10)  /* FW supports 16K addresses */

#define HMA_PAGE_ORDER					\
	((HMA_PAGE_SIZE < HMA_MAX_NO_FW_ADDRESS) ?	\
	ilog2(HMA_MAX_NO_FW_ADDRESS / HMA_PAGE_SIZE) : 0)

/* The minimum and maximum possible HMA sizes that can be specified in the FW
 * configuration(in units of MB).
 */
#define HMA_MIN_TOTAL_SIZE	1
#define HMA_MAX_TOTAL_SIZE				\
	(((HMA_PAGE_SIZE << HMA_PAGE_ORDER) *		\
	  HMA_MAX_NO_FW_ADDRESS) >> 20)

static void adap_free_hma_mem(struct adapter *adapter)
{
	struct scatterlist *iter;
	struct page *page;
	int i;

	if (!adapter->hma.sgt)
		return;

	if (adapter->hma.flags & HMA_DMA_MAPPED_FLAG) {
		dma_unmap_sg(adapter->pdev_dev, adapter->hma.sgt->sgl,
			     adapter->hma.sgt->nents, PCI_DMA_BIDIRECTIONAL);
		adapter->hma.flags &= ~HMA_DMA_MAPPED_FLAG;
	}

	for_each_sg(adapter->hma.sgt->sgl, iter,
		    adapter->hma.sgt->orig_nents, i) {
		page = sg_page(iter);
		if (page)
			__free_pages(page, HMA_PAGE_ORDER);
	}

	kfree(adapter->hma.phy_addr);
	sg_free_table(adapter->hma.sgt);
	kfree(adapter->hma.sgt);
	adapter->hma.sgt = NULL;
}

static int adap_config_hma(struct adapter *adapter)
{
	struct scatterlist *sgl, *iter;
	struct sg_table *sgt;
	struct page *newpage;
	unsigned int i, j, k;
	u32 param, hma_size;
	unsigned int ncmds;
	size_t page_size;
	u32 page_order;
	int node, ret;

	/* HMA is supported only for T6+ cards.
	 * Avoid initializing HMA in kdump kernels.
	 */
	if (is_kdump_kernel() ||
	    CHELSIO_CHIP_VERSION(adapter->params.chip) < CHELSIO_T6)
		return 0;

	/* Get the HMA region size required by fw */
	param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
		 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_HMA_SIZE));
	ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
			      1, &param, &hma_size);
	/* An error means card has its own memory or HMA is not supported by
	 * the firmware. Return without any errors.
	 */
	if (ret || !hma_size)
		return 0;

	if (hma_size < HMA_MIN_TOTAL_SIZE ||
	    hma_size > HMA_MAX_TOTAL_SIZE) {
		dev_err(adapter->pdev_dev,
			"HMA size %uMB beyond bounds(%u-%lu)MB\n",
			hma_size, HMA_MIN_TOTAL_SIZE, HMA_MAX_TOTAL_SIZE);
		return -EINVAL;
	}

	page_size = HMA_PAGE_SIZE;
	page_order = HMA_PAGE_ORDER;
	adapter->hma.sgt = kzalloc(sizeof(*adapter->hma.sgt), GFP_KERNEL);
	if (unlikely(!adapter->hma.sgt)) {
		dev_err(adapter->pdev_dev, "HMA SG table allocation failed\n");
		return -ENOMEM;
	}
	sgt = adapter->hma.sgt;
	/* FW returned value will be in MB's
	 */
	sgt->orig_nents = (hma_size << 20) / (page_size << page_order);
	if (sg_alloc_table(sgt, sgt->orig_nents, GFP_KERNEL)) {
		dev_err(adapter->pdev_dev, "HMA SGL allocation failed\n");
		kfree(adapter->hma.sgt);
		adapter->hma.sgt = NULL;
		return -ENOMEM;
	}

	sgl = adapter->hma.sgt->sgl;
	node = dev_to_node(adapter->pdev_dev);
	for_each_sg(sgl, iter, sgt->orig_nents, i) {
G
Ganesh Goudar 已提交
3761 3762
		newpage = alloc_pages_node(node, __GFP_NOWARN | GFP_KERNEL |
					   __GFP_ZERO, page_order);
A
Arjun Vynipadath 已提交
3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852
		if (!newpage) {
			dev_err(adapter->pdev_dev,
				"Not enough memory for HMA page allocation\n");
			ret = -ENOMEM;
			goto free_hma;
		}
		sg_set_page(iter, newpage, page_size << page_order, 0);
	}

	sgt->nents = dma_map_sg(adapter->pdev_dev, sgl, sgt->orig_nents,
				DMA_BIDIRECTIONAL);
	if (!sgt->nents) {
		dev_err(adapter->pdev_dev,
			"Not enough memory for HMA DMA mapping");
		ret = -ENOMEM;
		goto free_hma;
	}
	adapter->hma.flags |= HMA_DMA_MAPPED_FLAG;

	adapter->hma.phy_addr = kcalloc(sgt->nents, sizeof(dma_addr_t),
					GFP_KERNEL);
	if (unlikely(!adapter->hma.phy_addr))
		goto free_hma;

	for_each_sg(sgl, iter, sgt->nents, i) {
		newpage = sg_page(iter);
		adapter->hma.phy_addr[i] = sg_dma_address(iter);
	}

	ncmds = DIV_ROUND_UP(sgt->nents, HMA_MAX_ADDR_IN_CMD);
	/* Pass on the addresses to firmware */
	for (i = 0, k = 0; i < ncmds; i++, k += HMA_MAX_ADDR_IN_CMD) {
		struct fw_hma_cmd hma_cmd;
		u8 naddr = HMA_MAX_ADDR_IN_CMD;
		u8 soc = 0, eoc = 0;
		u8 hma_mode = 1; /* Presently we support only Page table mode */

		soc = (i == 0) ? 1 : 0;
		eoc = (i == ncmds - 1) ? 1 : 0;

		/* For last cmd, set naddr corresponding to remaining
		 * addresses
		 */
		if (i == ncmds - 1) {
			naddr = sgt->nents % HMA_MAX_ADDR_IN_CMD;
			naddr = naddr ? naddr : HMA_MAX_ADDR_IN_CMD;
		}
		memset(&hma_cmd, 0, sizeof(hma_cmd));
		hma_cmd.op_pkd = htonl(FW_CMD_OP_V(FW_HMA_CMD) |
				       FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
		hma_cmd.retval_len16 = htonl(FW_LEN16(hma_cmd));

		hma_cmd.mode_to_pcie_params =
			htonl(FW_HMA_CMD_MODE_V(hma_mode) |
			      FW_HMA_CMD_SOC_V(soc) | FW_HMA_CMD_EOC_V(eoc));

		/* HMA cmd size specified in MB's */
		hma_cmd.naddr_size =
			htonl(FW_HMA_CMD_SIZE_V(hma_size) |
			      FW_HMA_CMD_NADDR_V(naddr));

		/* Total Page size specified in units of 4K */
		hma_cmd.addr_size_pkd =
			htonl(FW_HMA_CMD_ADDR_SIZE_V
				((page_size << page_order) >> 12));

		/* Fill the 5 addresses */
		for (j = 0; j < naddr; j++) {
			hma_cmd.phy_address[j] =
				cpu_to_be64(adapter->hma.phy_addr[j + k]);
		}
		ret = t4_wr_mbox(adapter, adapter->mbox, &hma_cmd,
				 sizeof(hma_cmd), &hma_cmd);
		if (ret) {
			dev_err(adapter->pdev_dev,
				"HMA FW command failed with err %d\n", ret);
			goto free_hma;
		}
	}

	if (!ret)
		dev_info(adapter->pdev_dev,
			 "Reserved %uMB host memory for HMA\n", hma_size);
	return ret;

free_hma:
	adap_free_hma_mem(adapter);
	return ret;
}

3853 3854 3855 3856 3857
static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
{
	u32 v;
	int ret;

3858 3859 3860 3861 3862 3863 3864 3865 3866 3867
	/* Now that we've successfully configured and initialized the adapter
	 * can ask the Firmware what resources it has provisioned for us.
	 */
	ret = t4_get_pfres(adap);
	if (ret) {
		dev_err(adap->pdev_dev,
			"Unable to retrieve resource provisioning information\n");
		return ret;
	}

3868 3869
	/* get device capabilities */
	memset(c, 0, sizeof(*c));
3870 3871
	c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
			       FW_CMD_REQUEST_F | FW_CMD_READ_F);
3872
	c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
3873
	ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), c);
3874 3875 3876
	if (ret < 0)
		return ret;

3877 3878
	c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
			       FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
3879
	ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), NULL);
3880 3881 3882
	if (ret < 0)
		return ret;

3883
	ret = t4_config_glbl_rss(adap, adap->pf,
3884
				 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
3885 3886
				 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F |
				 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F);
3887 3888 3889
	if (ret < 0)
		return ret;

3890
	ret = t4_cfg_pfvf(adap, adap->mbox, adap->pf, 0, adap->sge.egr_sz, 64,
3891 3892
			  MAX_INGQ, 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF,
			  FW_CMD_CAP_PF);
3893 3894 3895 3896 3897 3898
	if (ret < 0)
		return ret;

	t4_sge_init(adap);

	/* tweak some settings */
3899
	t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849);
3900
	t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12));
3901 3902 3903
	t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A);
	v = t4_read_reg(adap, TP_PIO_DATA_A);
	t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F);
3904

3905 3906
	/* first 4 Tx modulation queues point to consecutive Tx channels */
	adap->params.tp.tx_modq_map = 0xE4;
3907 3908
	t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A,
		     TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map));
3909 3910 3911

	/* associate each Tx modulation queue with consecutive Tx channels */
	v = 0x84218421;
3912
	t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
3913
			  &v, 1, TP_TX_SCHED_HDR_A);
3914
	t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
3915
			  &v, 1, TP_TX_SCHED_FIFO_A);
3916
	t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
3917
			  &v, 1, TP_TX_SCHED_PCMD_A);
3918 3919 3920

#define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
	if (is_offload(adap)) {
3921 3922 3923 3924 3925 3926 3927 3928 3929 3930
		t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A,
			     TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
			     TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
			     TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
			     TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
		t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A,
			     TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
			     TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
			     TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
			     TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
3931 3932
	}

3933
	/* get basic stuff going */
3934
	return t4_early_init(adap, adap->pf);
3935 3936
}

3937 3938 3939 3940 3941
/*
 * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
 */
#define MAX_ATIDS 8192U

3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975
/*
 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
 *
 * If the firmware we're dealing with has Configuration File support, then
 * we use that to perform all configuration
 */

/*
 * Tweak configuration based on module parameters, etc.  Most of these have
 * defaults assigned to them by Firmware Configuration Files (if we're using
 * them) but need to be explicitly set if we're using hard-coded
 * initialization.  But even in the case of using Firmware Configuration
 * Files, we'd like to expose the ability to change these via module
 * parameters so these are essentially common tweaks/settings for
 * Configuration Files and hard-coded initialization ...
 */
static int adap_init0_tweaks(struct adapter *adapter)
{
	/*
	 * Fix up various Host-Dependent Parameters like Page Size, Cache
	 * Line Size, etc.  The firmware default is for a 4KB Page Size and
	 * 64B Cache Line Size ...
	 */
	t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);

	/*
	 * Process module parameters which affect early initialization.
	 */
	if (rx_dma_offset != 2 && rx_dma_offset != 0) {
		dev_err(&adapter->pdev->dev,
			"Ignoring illegal rx_dma_offset=%d, using 2\n",
			rx_dma_offset);
		rx_dma_offset = 2;
	}
3976 3977 3978
	t4_set_reg_field(adapter, SGE_CONTROL_A,
			 PKTSHIFT_V(PKTSHIFT_M),
			 PKTSHIFT_V(rx_dma_offset));
3979 3980 3981 3982 3983

	/*
	 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
	 * adds the pseudo header itself.
	 */
3984 3985
	t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A,
			       CSUM_HAS_PSEUDO_HDR_F, 0);
3986 3987 3988 3989

	return 0;
}

3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125
/* 10Gb/s-BT PHY Support. chip-external 10Gb/s-BT PHYs are complex chips
 * unto themselves and they contain their own firmware to perform their
 * tasks ...
 */
static int phy_aq1202_version(const u8 *phy_fw_data,
			      size_t phy_fw_size)
{
	int offset;

	/* At offset 0x8 you're looking for the primary image's
	 * starting offset which is 3 Bytes wide
	 *
	 * At offset 0xa of the primary image, you look for the offset
	 * of the DRAM segment which is 3 Bytes wide.
	 *
	 * The FW version is at offset 0x27e of the DRAM and is 2 Bytes
	 * wide
	 */
	#define be16(__p) (((__p)[0] << 8) | (__p)[1])
	#define le16(__p) ((__p)[0] | ((__p)[1] << 8))
	#define le24(__p) (le16(__p) | ((__p)[2] << 16))

	offset = le24(phy_fw_data + 0x8) << 12;
	offset = le24(phy_fw_data + offset + 0xa);
	return be16(phy_fw_data + offset + 0x27e);

	#undef be16
	#undef le16
	#undef le24
}

static struct info_10gbt_phy_fw {
	unsigned int phy_fw_id;		/* PCI Device ID */
	char *phy_fw_file;		/* /lib/firmware/ PHY Firmware file */
	int (*phy_fw_version)(const u8 *phy_fw_data, size_t phy_fw_size);
	int phy_flash;			/* Has FLASH for PHY Firmware */
} phy_info_array[] = {
	{
		PHY_AQ1202_DEVICEID,
		PHY_AQ1202_FIRMWARE,
		phy_aq1202_version,
		1,
	},
	{
		PHY_BCM84834_DEVICEID,
		PHY_BCM84834_FIRMWARE,
		NULL,
		0,
	},
	{ 0, NULL, NULL },
};

static struct info_10gbt_phy_fw *find_phy_info(int devid)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(phy_info_array); i++) {
		if (phy_info_array[i].phy_fw_id == devid)
			return &phy_info_array[i];
	}
	return NULL;
}

/* Handle updating of chip-external 10Gb/s-BT PHY firmware.  This needs to
 * happen after the FW_RESET_CMD but before the FW_INITIALIZE_CMD.  On error
 * we return a negative error number.  If we transfer new firmware we return 1
 * (from t4_load_phy_fw()).  If we don't do anything we return 0.
 */
static int adap_init0_phy(struct adapter *adap)
{
	const struct firmware *phyf;
	int ret;
	struct info_10gbt_phy_fw *phy_info;

	/* Use the device ID to determine which PHY file to flash.
	 */
	phy_info = find_phy_info(adap->pdev->device);
	if (!phy_info) {
		dev_warn(adap->pdev_dev,
			 "No PHY Firmware file found for this PHY\n");
		return -EOPNOTSUPP;
	}

	/* If we have a T4 PHY firmware file under /lib/firmware/cxgb4/, then
	 * use that. The adapter firmware provides us with a memory buffer
	 * where we can load a PHY firmware file from the host if we want to
	 * override the PHY firmware File in flash.
	 */
	ret = request_firmware_direct(&phyf, phy_info->phy_fw_file,
				      adap->pdev_dev);
	if (ret < 0) {
		/* For adapters without FLASH attached to PHY for their
		 * firmware, it's obviously a fatal error if we can't get the
		 * firmware to the adapter.  For adapters with PHY firmware
		 * FLASH storage, it's worth a warning if we can't find the
		 * PHY Firmware but we'll neuter the error ...
		 */
		dev_err(adap->pdev_dev, "unable to find PHY Firmware image "
			"/lib/firmware/%s, error %d\n",
			phy_info->phy_fw_file, -ret);
		if (phy_info->phy_flash) {
			int cur_phy_fw_ver = 0;

			t4_phy_fw_ver(adap, &cur_phy_fw_ver);
			dev_warn(adap->pdev_dev, "continuing with, on-adapter "
				 "FLASH copy, version %#x\n", cur_phy_fw_ver);
			ret = 0;
		}

		return ret;
	}

	/* Load PHY Firmware onto adapter.
	 */
	ret = t4_load_phy_fw(adap, MEMWIN_NIC, &adap->win0_lock,
			     phy_info->phy_fw_version,
			     (u8 *)phyf->data, phyf->size);
	if (ret < 0)
		dev_err(adap->pdev_dev, "PHY Firmware transfer error %d\n",
			-ret);
	else if (ret > 0) {
		int new_phy_fw_ver = 0;

		if (phy_info->phy_fw_version)
			new_phy_fw_ver = phy_info->phy_fw_version(phyf->data,
								  phyf->size);
		dev_info(adap->pdev_dev, "Successfully transferred PHY "
			 "Firmware /lib/firmware/%s, version %#x\n",
			 phy_info->phy_fw_file, new_phy_fw_ver);
	}

	release_firmware(phyf);

	return ret;
}

4126 4127 4128 4129 4130
/*
 * Attempt to initialize the adapter via a Firmware Configuration File.
 */
static int adap_init0_config(struct adapter *adapter, int reset)
{
4131 4132
	char *fw_config_file, fw_config_file_path[256];
	u32 finiver, finicsum, cfcsum, param, val;
4133 4134
	struct fw_caps_config_cmd caps_cmd;
	unsigned long mtype = 0, maddr = 0;
4135
	const struct firmware *cf;
4136
	char *config_name = NULL;
4137 4138
	int config_issued = 0;
	int ret;
4139 4140 4141 4142 4143 4144

	/*
	 * Reset device if necessary.
	 */
	if (reset) {
		ret = t4_fw_reset(adapter, adapter->mbox,
4145
				  PIORSTMODE_F | PIORST_F);
4146 4147 4148 4149
		if (ret < 0)
			goto bye;
	}

4150 4151 4152 4153 4154 4155 4156 4157 4158 4159
	/* If this is a 10Gb/s-BT adapter make sure the chip-external
	 * 10Gb/s-BT PHYs have up-to-date firmware.  Note that this step needs
	 * to be performed after any global adapter RESET above since some
	 * PHYs only have local RAM copies of the PHY firmware.
	 */
	if (is_10gbt_device(adapter->pdev->device)) {
		ret = adap_init0_phy(adapter);
		if (ret < 0)
			goto bye;
	}
4160 4161 4162 4163 4164
	/*
	 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
	 * then use that.  Otherwise, use the configuration file stored
	 * in the adapter flash ...
	 */
4165
	switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
S
Santosh Rastapur 已提交
4166
	case CHELSIO_T4:
4167
		fw_config_file = FW4_CFNAME;
S
Santosh Rastapur 已提交
4168 4169 4170 4171
		break;
	case CHELSIO_T5:
		fw_config_file = FW5_CFNAME;
		break;
4172 4173 4174
	case CHELSIO_T6:
		fw_config_file = FW6_CFNAME;
		break;
S
Santosh Rastapur 已提交
4175 4176 4177 4178 4179 4180 4181 4182
	default:
		dev_err(adapter->pdev_dev, "Device %d is not supported\n",
		       adapter->pdev->device);
		ret = -EINVAL;
		goto bye;
	}

	ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
4183
	if (ret < 0) {
4184
		config_name = "On FLASH";
4185 4186 4187 4188 4189
		mtype = FW_MEMTYPE_CF_FLASH;
		maddr = t4_flash_cfg_addr(adapter);
	} else {
		u32 params[7], val[7];

4190 4191 4192 4193
		sprintf(fw_config_file_path,
			"/lib/firmware/%s", fw_config_file);
		config_name = fw_config_file_path;

4194 4195 4196
		if (cf->size >= FLASH_CFG_MAX_SIZE)
			ret = -ENOMEM;
		else {
4197 4198
			params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
			     FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
4199
			ret = t4_query_params(adapter, adapter->mbox,
4200
					      adapter->pf, 0, 1, params, val);
4201 4202
			if (ret == 0) {
				/*
4203
				 * For t4_memory_rw() below addresses and
4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215
				 * sizes have to be in terms of multiples of 4
				 * bytes.  So, if the Configuration File isn't
				 * a multiple of 4 bytes in length we'll have
				 * to write that out separately since we can't
				 * guarantee that the bytes following the
				 * residual byte in the buffer returned by
				 * request_firmware() are zeroed out ...
				 */
				size_t resid = cf->size & 0x3;
				size_t size = cf->size & ~0x3;
				__be32 *data = (__be32 *)cf->data;

4216 4217
				mtype = FW_PARAMS_PARAM_Y_G(val[0]);
				maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16;
4218

4219 4220 4221
				spin_lock(&adapter->win0_lock);
				ret = t4_memory_rw(adapter, 0, mtype, maddr,
						   size, data, T4_MEMORY_WRITE);
4222 4223 4224 4225 4226 4227 4228 4229 4230 4231
				if (ret == 0 && resid != 0) {
					union {
						__be32 word;
						char buf[4];
					} last;
					int i;

					last.word = data[size >> 2];
					for (i = resid; i < 4; i++)
						last.buf[i] = 0;
4232 4233 4234 4235
					ret = t4_memory_rw(adapter, 0, mtype,
							   maddr + size,
							   4, &last.word,
							   T4_MEMORY_WRITE);
4236
				}
4237
				spin_unlock(&adapter->win0_lock);
4238 4239 4240 4241 4242 4243 4244 4245
			}
		}

		release_firmware(cf);
		if (ret)
			goto bye;
	}

4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263
	val = 0;

	/* Ofld + Hash filter is supported. Older fw will fail this request and
	 * it is fine.
	 */
	param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
		 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_HASHFILTER_WITH_OFLD));
	ret = t4_set_params(adapter, adapter->mbox, adapter->pf, 0,
			    1, &param, &val);

	/* FW doesn't know about Hash filter + ofld support,
	 * it's not a problem, don't return an error.
	 */
	if (ret < 0) {
		dev_warn(adapter->pdev_dev,
			 "Hash filter with ofld is not supported by FW\n");
	}

4264 4265 4266 4267 4268 4269 4270 4271
	/*
	 * Issue a Capability Configuration command to the firmware to get it
	 * to parse the Configuration File.  We don't use t4_fw_config_file()
	 * because we want the ability to modify various features after we've
	 * processed the configuration file ...
	 */
	memset(&caps_cmd, 0, sizeof(caps_cmd));
	caps_cmd.op_to_write =
4272 4273 4274
		htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
		      FW_CMD_REQUEST_F |
		      FW_CMD_READ_F);
4275
	caps_cmd.cfvalid_to_len16 =
4276 4277 4278
		htonl(FW_CAPS_CONFIG_CMD_CFVALID_F |
		      FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) |
		      FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) |
4279 4280 4281
		      FW_LEN16(caps_cmd));
	ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
			 &caps_cmd);
4282 4283 4284 4285 4286 4287 4288 4289 4290 4291

	/* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
	 * Configuration File in FLASH), our last gasp effort is to use the
	 * Firmware Configuration File which is embedded in the firmware.  A
	 * very few early versions of the firmware didn't have one embedded
	 * but we can ignore those.
	 */
	if (ret == -ENOENT) {
		memset(&caps_cmd, 0, sizeof(caps_cmd));
		caps_cmd.op_to_write =
4292 4293 4294
			htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
					FW_CMD_REQUEST_F |
					FW_CMD_READ_F);
4295 4296 4297 4298 4299 4300 4301
		caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
		ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
				sizeof(caps_cmd), &caps_cmd);
		config_name = "Firmware Default";
	}

	config_issued = 1;
4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316
	if (ret < 0)
		goto bye;

	finiver = ntohl(caps_cmd.finiver);
	finicsum = ntohl(caps_cmd.finicsum);
	cfcsum = ntohl(caps_cmd.cfcsum);
	if (finicsum != cfcsum)
		dev_warn(adapter->pdev_dev, "Configuration File checksum "\
			 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
			 finicsum, cfcsum);

	/*
	 * And now tell the firmware to use the configuration we just loaded.
	 */
	caps_cmd.op_to_write =
4317 4318 4319
		htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
		      FW_CMD_REQUEST_F |
		      FW_CMD_WRITE_F);
4320
	caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333
	ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
			 NULL);
	if (ret < 0)
		goto bye;

	/*
	 * Tweak configuration based on system architecture, module
	 * parameters, etc.
	 */
	ret = adap_init0_tweaks(adapter);
	if (ret < 0)
		goto bye;

A
Arjun Vynipadath 已提交
4334 4335 4336 4337 4338 4339
	/* We will proceed even if HMA init fails. */
	ret = adap_config_hma(adapter);
	if (ret)
		dev_err(adapter->pdev_dev,
			"HMA configuration failed with error %d\n", ret);

4340 4341 4342 4343 4344 4345 4346
	if (is_t6(adapter->params.chip)) {
		ret = setup_ppod_edram(adapter);
		if (!ret)
			dev_info(adapter->pdev_dev, "Successfully enabled "
				 "ppod edram feature\n");
	}

4347 4348 4349 4350 4351 4352 4353 4354
	/*
	 * And finally tell the firmware to initialize itself using the
	 * parameters from the Configuration File.
	 */
	ret = t4_fw_initialize(adapter, adapter->mbox);
	if (ret < 0)
		goto bye;

4355 4356
	/* Emit Firmware Configuration File information and return
	 * successfully.
4357 4358
	 */
	dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
4359 4360
		 "Configuration File \"%s\", version %#x, computed checksum %#x\n",
		 config_name, finiver, cfcsum);
4361 4362 4363 4364 4365 4366 4367 4368
	return 0;

	/*
	 * Something bad happened.  Return the error ...  (If the "error"
	 * is that there's no Configuration File on the adapter we don't
	 * want to issue a warning since this is fairly common.)
	 */
bye:
4369 4370 4371
	if (config_issued && ret != -ENOENT)
		dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
			 config_name, -ret);
4372 4373 4374
	return ret;
}

4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401
static struct fw_info fw_info_array[] = {
	{
		.chip = CHELSIO_T4,
		.fs_name = FW4_CFNAME,
		.fw_mod_name = FW4_FNAME,
		.fw_hdr = {
			.chip = FW_HDR_CHIP_T4,
			.fw_ver = __cpu_to_be32(FW_VERSION(T4)),
			.intfver_nic = FW_INTFVER(T4, NIC),
			.intfver_vnic = FW_INTFVER(T4, VNIC),
			.intfver_ri = FW_INTFVER(T4, RI),
			.intfver_iscsi = FW_INTFVER(T4, ISCSI),
			.intfver_fcoe = FW_INTFVER(T4, FCOE),
		},
	}, {
		.chip = CHELSIO_T5,
		.fs_name = FW5_CFNAME,
		.fw_mod_name = FW5_FNAME,
		.fw_hdr = {
			.chip = FW_HDR_CHIP_T5,
			.fw_ver = __cpu_to_be32(FW_VERSION(T5)),
			.intfver_nic = FW_INTFVER(T5, NIC),
			.intfver_vnic = FW_INTFVER(T5, VNIC),
			.intfver_ri = FW_INTFVER(T5, RI),
			.intfver_iscsi = FW_INTFVER(T5, ISCSI),
			.intfver_fcoe = FW_INTFVER(T5, FCOE),
		},
4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417
	}, {
		.chip = CHELSIO_T6,
		.fs_name = FW6_CFNAME,
		.fw_mod_name = FW6_FNAME,
		.fw_hdr = {
			.chip = FW_HDR_CHIP_T6,
			.fw_ver = __cpu_to_be32(FW_VERSION(T6)),
			.intfver_nic = FW_INTFVER(T6, NIC),
			.intfver_vnic = FW_INTFVER(T6, VNIC),
			.intfver_ofld = FW_INTFVER(T6, OFLD),
			.intfver_ri = FW_INTFVER(T6, RI),
			.intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU),
			.intfver_iscsi = FW_INTFVER(T6, ISCSI),
			.intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU),
			.intfver_fcoe = FW_INTFVER(T6, FCOE),
		},
4418
	}
4419

4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432
};

static struct fw_info *find_fw_info(int chip)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
		if (fw_info_array[i].chip == chip)
			return &fw_info_array[i];
	}
	return NULL;
}

4433 4434 4435
/*
 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
 */
V
Vishal Kulkarni 已提交
4436
static int adap_init0(struct adapter *adap, int vpd_skip)
4437
{
4438
	struct fw_caps_config_cmd caps_cmd;
V
Vishal Kulkarni 已提交
4439 4440 4441
	u32 params[7], val[7];
	enum dev_state state;
	u32 v, port_vec;
4442
	int reset = 1;
V
Vishal Kulkarni 已提交
4443
	int ret;
4444

4445 4446 4447 4448 4449 4450 4451
	/* Grab Firmware Device Log parameters as early as possible so we have
	 * access to it for debugging, etc.
	 */
	ret = t4_init_devlog_params(adap);
	if (ret < 0)
		return ret;

4452
	/* Contact FW, advertising Master capability */
4453 4454
	ret = t4_fw_hello(adap, adap->mbox, adap->mbox,
			  is_kdump_kernel() ? MASTER_MUST : MASTER_MAY, &state);
4455 4456 4457 4458 4459
	if (ret < 0) {
		dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
			ret);
		return ret;
	}
4460
	if (ret == adap->mbox)
4461
		adap->flags |= CXGB4_MASTER_PF;
4462

4463 4464 4465 4466 4467 4468 4469
	/*
	 * If we're the Master PF Driver and the device is uninitialized,
	 * then let's consider upgrading the firmware ...  (We always want
	 * to check the firmware version number in order to A. get it for
	 * later reporting and B. to warn if the currently loaded firmware
	 * is excessively mismatched relative to the driver.)
	 */
4470

4471
	t4_get_version_info(adap);
4472 4473
	ret = t4_check_fw_version(adap);
	/* If firmware is too old (not supported by driver) force an update. */
4474
	if (ret)
4475
		state = DEV_STATE_UNINIT;
4476
	if ((adap->flags & CXGB4_MASTER_PF) && state != DEV_STATE_INIT) {
4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491
		struct fw_info *fw_info;
		struct fw_hdr *card_fw;
		const struct firmware *fw;
		const u8 *fw_data = NULL;
		unsigned int fw_size = 0;

		/* This is the firmware whose headers the driver was compiled
		 * against
		 */
		fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
		if (fw_info == NULL) {
			dev_err(adap->pdev_dev,
				"unable to get firmware info for chip %d.\n",
				CHELSIO_CHIP_VERSION(adap->params.chip));
			return -EINVAL;
4492
		}
4493 4494 4495 4496

		/* allocate memory to read the header of the firmware on the
		 * card
		 */
4497
		card_fw = kvzalloc(sizeof(*card_fw), GFP_KERNEL);
4498 4499 4500 4501
		if (!card_fw) {
			ret = -ENOMEM;
			goto bye;
		}
4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519

		/* Get FW from from /lib/firmware/ */
		ret = request_firmware(&fw, fw_info->fw_mod_name,
				       adap->pdev_dev);
		if (ret < 0) {
			dev_err(adap->pdev_dev,
				"unable to load firmware image %s, error %d\n",
				fw_info->fw_mod_name, ret);
		} else {
			fw_data = fw->data;
			fw_size = fw->size;
		}

		/* upgrade FW logic */
		ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
				 state, &reset);

		/* Cleaning up */
4520
		release_firmware(fw);
4521
		kvfree(card_fw);
4522

4523
		if (ret < 0)
4524
			goto bye;
4525
	}
4526

4527 4528
	/* If the firmware is initialized already, emit a simply note to that
	 * effect. Otherwise, it's time to try initializing the adapter.
4529 4530
	 */
	if (state == DEV_STATE_INIT) {
A
Arjun Vynipadath 已提交
4531 4532 4533 4534 4535
		ret = adap_config_hma(adap);
		if (ret)
			dev_err(adap->pdev_dev,
				"HMA configuration failed with error %d\n",
				ret);
4536 4537
		dev_info(adap->pdev_dev, "Coming up as %s: "\
			 "Adapter already initialized\n",
4538
			 adap->flags & CXGB4_MASTER_PF ? "MASTER" : "SLAVE");
4539 4540 4541
	} else {
		dev_info(adap->pdev_dev, "Coming up as MASTER: "\
			 "Initializing adapter\n");
4542 4543 4544

		/* Find out whether we're dealing with a version of the
		 * firmware which has configuration file support.
4545
		 */
4546 4547
		params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
			     FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
4548
		ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1,
4549
				      params, val);
4550

4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568
		/* If the firmware doesn't support Configuration Files,
		 * return an error.
		 */
		if (ret < 0) {
			dev_err(adap->pdev_dev, "firmware doesn't support "
				"Firmware Configuration Files\n");
			goto bye;
		}

		/* The firmware provides us with a memory buffer where we can
		 * load a Configuration File from the host if we want to
		 * override the Configuration File in flash.
		 */
		ret = adap_init0_config(adap, reset);
		if (ret == -ENOENT) {
			dev_err(adap->pdev_dev, "no Configuration File "
				"present on adapter.\n");
			goto bye;
4569 4570
		}
		if (ret < 0) {
4571 4572
			dev_err(adap->pdev_dev, "could not initialize "
				"adapter, error %d\n", -ret);
4573 4574 4575 4576
			goto bye;
		}
	}

4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597
	/* Now that we've successfully configured and initialized the adapter
	 * (or found it already initialized), we can ask the Firmware what
	 * resources it has provisioned for us.
	 */
	ret = t4_get_pfres(adap);
	if (ret) {
		dev_err(adap->pdev_dev,
			"Unable to retrieve resource provisioning information\n");
		goto bye;
	}

	/* Grab VPD parameters.  This should be done after we establish a
	 * connection to the firmware since some of the VPD parameters
	 * (notably the Core Clock frequency) are retrieved via requests to
	 * the firmware.  On the other hand, we need these fairly early on
	 * so we do this right after getting ahold of the firmware.
	 *
	 * We need to do this after initializing the adapter because someone
	 * could have FLASHed a new VPD which won't be read by the firmware
	 * until we do the RESET ...
	 */
V
Vishal Kulkarni 已提交
4598 4599 4600 4601 4602
	if (!vpd_skip) {
		ret = t4_get_vpd_params(adap, &adap->params.vpd);
		if (ret < 0)
			goto bye;
	}
4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617

	/* Find out what ports are available to us.  Note that we need to do
	 * this before calling adap_init0_no_config() since it needs nports
	 * and portvec ...
	 */
	v =
	    FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
	    FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC);
	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &v, &port_vec);
	if (ret < 0)
		goto bye;

	adap->params.nports = hweight32(port_vec);
	adap->params.portvec = port_vec;

4618 4619 4620
	/* Give the SGE code a chance to pull in anything that it needs ...
	 * Note that this must be called after we retrieve our VPD parameters
	 * in order to know how to convert core ticks to seconds, etc.
4621
	 */
4622 4623 4624
	ret = t4_sge_init(adap);
	if (ret < 0)
		goto bye;
4625

4626 4627 4628
	/* Grab the SGE Doorbell Queue Timer values.  If successful, that
	 * indicates that the Firmware and Hardware support this.
	 */
4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640
	params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
		    FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_DBQ_TIMERTICK));
	ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
			      1, params, val);

	if (!ret) {
		adap->sge.dbqtimer_tick = val[0];
		ret = t4_read_sge_dbqtimers(adap,
					    ARRAY_SIZE(adap->sge.dbqtimer_val),
					    adap->sge.dbqtimer_val);
	}

4641
	if (!ret)
4642
		adap->flags |= CXGB4_SGE_DBQ_TIMER;
4643

4644 4645 4646
	if (is_bypass_device(adap->pdev->device))
		adap->params.bypass = 1;

4647 4648 4649 4650
	/*
	 * Grab some of our basic fundamental operating parameters.
	 */
#define FW_PARAM_DEV(param) \
4651 4652
	(FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \
	FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param))
4653

4654
#define FW_PARAM_PFVF(param) \
4655 4656 4657 4658
	FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \
	FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)|  \
	FW_PARAMS_PARAM_Y_V(0) | \
	FW_PARAMS_PARAM_Z_V(0)
4659

4660
	params[0] = FW_PARAM_PFVF(EQ_START);
4661 4662 4663 4664
	params[1] = FW_PARAM_PFVF(L2T_START);
	params[2] = FW_PARAM_PFVF(L2T_END);
	params[3] = FW_PARAM_PFVF(FILTER_START);
	params[4] = FW_PARAM_PFVF(FILTER_END);
4665
	params[5] = FW_PARAM_PFVF(IQFLINT_START);
4666
	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, val);
4667 4668
	if (ret < 0)
		goto bye;
4669 4670 4671
	adap->sge.egr_start = val[0];
	adap->l2t_start = val[1];
	adap->l2t_end = val[2];
4672 4673
	adap->tids.ftid_base = val[3];
	adap->tids.nftids = val[4] - val[3] + 1;
4674
	adap->sge.ingr_start = val[5];
4675

4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689
	if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) {
		/* Read the raw mps entries. In T6, the last 2 tcam entries
		 * are reserved for raw mac addresses (rawf = 2, one per port).
		 */
		params[0] = FW_PARAM_PFVF(RAWF_START);
		params[1] = FW_PARAM_PFVF(RAWF_END);
		ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
				      params, val);
		if (ret == 0) {
			adap->rawf_start = val[0];
			adap->rawf_cnt = val[1] - val[0] + 1;
		}
	}

4690 4691 4692 4693 4694 4695 4696 4697
	/* qids (ingress/egress) returned from firmware can be anywhere
	 * in the range from EQ(IQFLINT)_START to EQ(IQFLINT)_END.
	 * Hence driver needs to allocate memory for this range to
	 * store the queue info. Get the highest IQFLINT/EQ index returned
	 * in FW_EQ_*_CMD.alloc command.
	 */
	params[0] = FW_PARAM_PFVF(EQ_END);
	params[1] = FW_PARAM_PFVF(IQFLINT_END);
4698
	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val);
4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718
	if (ret < 0)
		goto bye;
	adap->sge.egr_sz = val[0] - adap->sge.egr_start + 1;
	adap->sge.ingr_sz = val[1] - adap->sge.ingr_start + 1;

	adap->sge.egr_map = kcalloc(adap->sge.egr_sz,
				    sizeof(*adap->sge.egr_map), GFP_KERNEL);
	if (!adap->sge.egr_map) {
		ret = -ENOMEM;
		goto bye;
	}

	adap->sge.ingr_map = kcalloc(adap->sge.ingr_sz,
				     sizeof(*adap->sge.ingr_map), GFP_KERNEL);
	if (!adap->sge.ingr_map) {
		ret = -ENOMEM;
		goto bye;
	}

	/* Allocate the memory for the vaious egress queue bitmaps
4719
	 * ie starving_fl, txq_maperr and blocked_fl.
4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734
	 */
	adap->sge.starving_fl =	kcalloc(BITS_TO_LONGS(adap->sge.egr_sz),
					sizeof(long), GFP_KERNEL);
	if (!adap->sge.starving_fl) {
		ret = -ENOMEM;
		goto bye;
	}

	adap->sge.txq_maperr = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz),
				       sizeof(long), GFP_KERNEL);
	if (!adap->sge.txq_maperr) {
		ret = -ENOMEM;
		goto bye;
	}

4735 4736 4737 4738 4739 4740 4741 4742 4743
#ifdef CONFIG_DEBUG_FS
	adap->sge.blocked_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz),
				       sizeof(long), GFP_KERNEL);
	if (!adap->sge.blocked_fl) {
		ret = -ENOMEM;
		goto bye;
	}
#endif

4744 4745
	params[0] = FW_PARAM_PFVF(CLIP_START);
	params[1] = FW_PARAM_PFVF(CLIP_END);
4746
	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val);
4747 4748 4749 4750 4751
	if (ret < 0)
		goto bye;
	adap->clipt_start = val[0];
	adap->clipt_end = val[1];

4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763
	/* Get the supported number of traffic classes */
	params[0] = FW_PARAM_DEV(NUM_TM_CLASS);
	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, params, val);
	if (ret < 0) {
		/* We couldn't retrieve the number of Traffic Classes
		 * supported by the hardware/firmware. So we hard
		 * code it here.
		 */
		adap->params.nsched_cls = is_t4(adap->params.chip) ? 15 : 16;
	} else {
		adap->params.nsched_cls = val[0];
	}
4764

4765 4766 4767
	/* query params related to active filter region */
	params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
	params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
4768
	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val);
4769 4770 4771 4772
	/* If Active filter size is set we enable establishing
	 * offload connection through firmware work request
	 */
	if ((val[0] != val[1]) && (ret >= 0)) {
4773
		adap->flags |= CXGB4_FW_OFLD_CONN;
4774 4775 4776 4777
		adap->tids.aftid_base = val[0];
		adap->tids.aftid_end = val[1];
	}

4778 4779 4780 4781 4782 4783 4784
	/* If we're running on newer firmware, let it know that we're
	 * prepared to deal with encapsulated CPL messages.  Older
	 * firmware won't understand this and we'll just get
	 * unencapsulated messages ...
	 */
	params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
	val[0] = 1;
4785
	(void)t4_set_params(adap, adap->mbox, adap->pf, 0, 1, params, val);
4786

4787 4788 4789 4790 4791 4792 4793 4794 4795 4796
	/*
	 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
	 * capability.  Earlier versions of the firmware didn't have the
	 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
	 * permission to use ULPTX MEMWRITE DSGL.
	 */
	if (is_t4(adap->params.chip)) {
		adap->params.ulptx_memwrite_dsgl = false;
	} else {
		params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
4797
		ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
4798 4799 4800 4801
				      1, params, val);
		adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
	}

4802 4803 4804 4805 4806 4807
	/* See if FW supports FW_RI_FR_NSMR_TPTE_WR work request */
	params[0] = FW_PARAM_DEV(RI_FR_NSMR_TPTE_WR);
	ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
			      1, params, val);
	adap->params.fr_nsmr_tpte_wr_support = (ret == 0 && val[0] != 0);

4808 4809 4810 4811 4812 4813 4814 4815 4816 4817
	/* See if FW supports FW_FILTER2 work request */
	if (is_t4(adap->params.chip)) {
		adap->params.filter2_wr_support = 0;
	} else {
		params[0] = FW_PARAM_DEV(FILTER2_WR);
		ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
				      1, params, val);
		adap->params.filter2_wr_support = (ret == 0 && val[0] != 0);
	}

4818 4819 4820 4821 4822 4823 4824 4825 4826
	/* Check if FW supports returning vin and smt index.
	 * If this is not supported, driver will interpret
	 * these values from viid.
	 */
	params[0] = FW_PARAM_DEV(OPAQUE_VIID_SMT_EXTN);
	ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
			      1, params, val);
	adap->params.viid_smt_extn_support = (ret == 0 && val[0] != 0);

4827 4828 4829 4830 4831
	/*
	 * Get device capabilities so we can determine what resources we need
	 * to manage.
	 */
	memset(&caps_cmd, 0, sizeof(caps_cmd));
4832 4833
	caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
				     FW_CMD_REQUEST_F | FW_CMD_READ_F);
4834
	caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4835 4836 4837 4838 4839
	ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
			 &caps_cmd);
	if (ret < 0)
		goto bye;

4840 4841 4842 4843 4844 4845 4846
	/* hash filter has some mandatory register settings to be tested and for
	 * that it needs to test whether offload is enabled or not, hence
	 * checking and setting it here.
	 */
	if (caps_cmd.ofldcaps)
		adap->params.offload = 1;

4847
	if (caps_cmd.ofldcaps ||
4848 4849
	    (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_HASHFILTER)) ||
	    (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_ETHOFLD))) {
4850 4851 4852 4853 4854 4855 4856
		/* query offload-related parameters */
		params[0] = FW_PARAM_DEV(NTID);
		params[1] = FW_PARAM_PFVF(SERVER_START);
		params[2] = FW_PARAM_PFVF(SERVER_END);
		params[3] = FW_PARAM_PFVF(TDDP_START);
		params[4] = FW_PARAM_PFVF(TDDP_END);
		params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
4857
		ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6,
4858
				      params, val);
4859 4860 4861 4862 4863 4864
		if (ret < 0)
			goto bye;
		adap->tids.ntids = val[0];
		adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
		adap->tids.stid_base = val[1];
		adap->tids.nstids = val[2] - val[1] + 1;
4865
		/*
4866
		 * Setup server filter region. Divide the available filter
4867 4868 4869 4870 4871 4872 4873
		 * region into two parts. Regular filters get 1/3rd and server
		 * filters get 2/3rd part. This is only enabled if workarond
		 * path is enabled.
		 * 1. For regular filters.
		 * 2. Server filter: This are special filters which are used
		 * to redirect SYN packets to offload queue.
		 */
4874
		if (adap->flags & CXGB4_FW_OFLD_CONN && !is_bypass(adap)) {
4875 4876 4877 4878 4879 4880 4881
			adap->tids.sftid_base = adap->tids.ftid_base +
					DIV_ROUND_UP(adap->tids.nftids, 3);
			adap->tids.nsftids = adap->tids.nftids -
					 DIV_ROUND_UP(adap->tids.nftids, 3);
			adap->tids.nftids = adap->tids.sftid_base -
						adap->tids.ftid_base;
		}
4882 4883 4884
		adap->vres.ddp.start = val[3];
		adap->vres.ddp.size = val[4] - val[3] + 1;
		adap->params.ofldq_wr_cred = val[5];
4885

4886
		if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_HASHFILTER)) {
4887
			init_hash_filter(adap);
4888 4889 4890
		} else {
			adap->num_ofld_uld += 1;
		}
4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903

		if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_ETHOFLD)) {
			params[0] = FW_PARAM_PFVF(ETHOFLD_START);
			params[1] = FW_PARAM_PFVF(ETHOFLD_END);
			ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
					      params, val);
			if (!ret) {
				adap->tids.eotid_base = val[0];
				adap->tids.neotids = min_t(u32, MAX_ATIDS,
							   val[1] - val[0] + 1);
				adap->params.ethofld = 1;
			}
		}
4904
	}
4905
	if (caps_cmd.rdmacaps) {
4906 4907 4908 4909 4910 4911
		params[0] = FW_PARAM_PFVF(STAG_START);
		params[1] = FW_PARAM_PFVF(STAG_END);
		params[2] = FW_PARAM_PFVF(RQ_START);
		params[3] = FW_PARAM_PFVF(RQ_END);
		params[4] = FW_PARAM_PFVF(PBL_START);
		params[5] = FW_PARAM_PFVF(PBL_END);
4912
		ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6,
4913
				      params, val);
4914 4915 4916 4917 4918 4919 4920 4921
		if (ret < 0)
			goto bye;
		adap->vres.stag.start = val[0];
		adap->vres.stag.size = val[1] - val[0] + 1;
		adap->vres.rq.start = val[2];
		adap->vres.rq.size = val[3] - val[2] + 1;
		adap->vres.pbl.start = val[4];
		adap->vres.pbl.size = val[5] - val[4] + 1;
4922

4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936
		params[0] = FW_PARAM_PFVF(SRQ_START);
		params[1] = FW_PARAM_PFVF(SRQ_END);
		ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
				      params, val);
		if (!ret) {
			adap->vres.srq.start = val[0];
			adap->vres.srq.size = val[1] - val[0] + 1;
		}
		if (adap->vres.srq.size) {
			adap->srq = t4_init_srq(adap->vres.srq.size);
			if (!adap->srq)
				dev_warn(&adap->pdev->dev, "could not allocate SRQ, continuing\n");
		}

4937 4938 4939 4940
		params[0] = FW_PARAM_PFVF(SQRQ_START);
		params[1] = FW_PARAM_PFVF(SQRQ_END);
		params[2] = FW_PARAM_PFVF(CQ_START);
		params[3] = FW_PARAM_PFVF(CQ_END);
4941 4942
		params[4] = FW_PARAM_PFVF(OCQ_START);
		params[5] = FW_PARAM_PFVF(OCQ_END);
4943
		ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params,
4944
				      val);
4945 4946 4947 4948 4949 4950
		if (ret < 0)
			goto bye;
		adap->vres.qp.start = val[0];
		adap->vres.qp.size = val[1] - val[0] + 1;
		adap->vres.cq.start = val[2];
		adap->vres.cq.size = val[3] - val[2] + 1;
4951 4952
		adap->vres.ocq.start = val[4];
		adap->vres.ocq.size = val[5] - val[4] + 1;
4953 4954 4955

		params[0] = FW_PARAM_DEV(MAXORDIRD_QP);
		params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER);
4956
		ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params,
4957
				      val);
4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969
		if (ret < 0) {
			adap->params.max_ordird_qp = 8;
			adap->params.max_ird_adapter = 32 * adap->tids.ntids;
			ret = 0;
		} else {
			adap->params.max_ordird_qp = val[0];
			adap->params.max_ird_adapter = val[1];
		}
		dev_info(adap->pdev_dev,
			 "max_ordird_qp %d max_ird_adapter %d\n",
			 adap->params.max_ordird_qp,
			 adap->params.max_ird_adapter);
4970 4971 4972 4973 4974 4975

		/* Enable write_with_immediate if FW supports it */
		params[0] = FW_PARAM_DEV(RDMA_WRITE_WITH_IMM);
		ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, params,
				      val);
		adap->params.write_w_imm_support = (ret == 0 && val[0] != 0);
4976 4977 4978 4979 4980 4981

		/* Enable write_cmpl if FW supports it */
		params[0] = FW_PARAM_DEV(RI_WRITE_CMPL_WR);
		ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, params,
				      val);
		adap->params.write_cmpl_support = (ret == 0 && val[0] != 0);
4982
		adap->num_ofld_uld += 2;
4983
	}
4984
	if (caps_cmd.iscsicaps) {
4985 4986
		params[0] = FW_PARAM_PFVF(ISCSI_START);
		params[1] = FW_PARAM_PFVF(ISCSI_END);
4987
		ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
4988
				      params, val);
4989 4990 4991 4992
		if (ret < 0)
			goto bye;
		adap->vres.iscsi.start = val[0];
		adap->vres.iscsi.size = val[1] - val[0] + 1;
4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008
		if (is_t6(adap->params.chip)) {
			params[0] = FW_PARAM_PFVF(PPOD_EDRAM_START);
			params[1] = FW_PARAM_PFVF(PPOD_EDRAM_END);
			ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
					      params, val);
			if (!ret) {
				adap->vres.ppod_edram.start = val[0];
				adap->vres.ppod_edram.size =
					val[1] - val[0] + 1;

				dev_info(adap->pdev_dev,
					 "ppod edram start 0x%x end 0x%x size 0x%x\n",
					 val[0], val[1],
					 adap->vres.ppod_edram.size);
			}
		}
5009 5010
		/* LIO target and cxgb4i initiaitor */
		adap->num_ofld_uld += 2;
5011
	}
5012
	if (caps_cmd.cryptocaps) {
5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032
		if (ntohs(caps_cmd.cryptocaps) &
		    FW_CAPS_CONFIG_CRYPTO_LOOKASIDE) {
			params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE);
			ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
					      2, params, val);
			if (ret < 0) {
				if (ret != -EINVAL)
					goto bye;
			} else {
				adap->vres.ncrypto_fc = val[0];
			}
			adap->num_ofld_uld += 1;
		}
		if (ntohs(caps_cmd.cryptocaps) &
		    FW_CAPS_CONFIG_TLS_INLINE) {
			params[0] = FW_PARAM_PFVF(TLS_START);
			params[1] = FW_PARAM_PFVF(TLS_END);
			ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
					      2, params, val);
			if (ret < 0)
H
Harsh Jain 已提交
5033
				goto bye;
5034 5035 5036
			adap->vres.key.start = val[0];
			adap->vres.key.size = val[1] - val[0] + 1;
			adap->num_uld += 1;
H
Harsh Jain 已提交
5037
		}
5038
		adap->params.crypto = ntohs(caps_cmd.cryptocaps);
5039
	}
5040 5041 5042
#undef FW_PARAM_PFVF
#undef FW_PARAM_DEV

5043 5044 5045 5046
	/* The MTU/MSS Table is initialized by now, so load their values.  If
	 * we're initializing the adapter, then we'll make any modifications
	 * we want to the MTU/MSS Table and also initialize the congestion
	 * parameters.
5047
	 */
5048
	t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073
	if (state != DEV_STATE_INIT) {
		int i;

		/* The default MTU Table contains values 1492 and 1500.
		 * However, for TCP, it's better to have two values which are
		 * a multiple of 8 +/- 4 bytes apart near this popular MTU.
		 * This allows us to have a TCP Data Payload which is a
		 * multiple of 8 regardless of what combination of TCP Options
		 * are in use (always a multiple of 4 bytes) which is
		 * important for performance reasons.  For instance, if no
		 * options are in use, then we have a 20-byte IP header and a
		 * 20-byte TCP header.  In this case, a 1500-byte MSS would
		 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes
		 * which is not a multiple of 8.  So using an MSS of 1488 in
		 * this case results in a TCP Data Payload of 1448 bytes which
		 * is a multiple of 8.  On the other hand, if 12-byte TCP Time
		 * Stamps have been negotiated, then an MTU of 1500 bytes
		 * results in a TCP Data Payload of 1448 bytes which, as
		 * above, is a multiple of 8 bytes ...
		 */
		for (i = 0; i < NMTUS; i++)
			if (adap->params.mtus[i] == 1492) {
				adap->params.mtus[i] = 1488;
				break;
			}
5074

5075 5076 5077
		t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
			     adap->params.b_wnd);
	}
5078
	t4_init_sge_params(adap);
5079
	adap->flags |= CXGB4_FW_OK;
5080
	t4_init_tp_params(adap, true);
5081 5082 5083
	return 0;

	/*
5084 5085 5086
	 * Something bad happened.  If a command timed out or failed with EIO
	 * FW does not operate within its spec or something catastrophic
	 * happened to HW/FW, stop issuing commands.
5087
	 */
5088
bye:
A
Arjun Vynipadath 已提交
5089
	adap_free_hma_mem(adap);
5090 5091 5092 5093
	kfree(adap->sge.egr_map);
	kfree(adap->sge.ingr_map);
	kfree(adap->sge.starving_fl);
	kfree(adap->sge.txq_maperr);
5094 5095 5096
#ifdef CONFIG_DEBUG_FS
	kfree(adap->sge.blocked_fl);
#endif
5097 5098
	if (ret != -ETIMEDOUT && ret != -EIO)
		t4_fw_bye(adap, adap->mbox);
5099 5100 5101
	return ret;
}

D
Dimitris Michailidis 已提交
5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113
/* EEH callbacks */

static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
					 pci_channel_state_t state)
{
	int i;
	struct adapter *adap = pci_get_drvdata(pdev);

	if (!adap)
		goto out;

	rtnl_lock();
5114
	adap->flags &= ~CXGB4_FW_OK;
D
Dimitris Michailidis 已提交
5115
	notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
5116
	spin_lock(&adap->stats_lock);
D
Dimitris Michailidis 已提交
5117 5118
	for_each_port(adap, i) {
		struct net_device *dev = adap->port[i];
5119 5120 5121 5122
		if (dev) {
			netif_device_detach(dev);
			netif_carrier_off(dev);
		}
D
Dimitris Michailidis 已提交
5123
	}
5124
	spin_unlock(&adap->stats_lock);
5125
	disable_interrupts(adap);
5126
	if (adap->flags & CXGB4_FULL_INIT_DONE)
D
Dimitris Michailidis 已提交
5127 5128
		cxgb_down(adap);
	rtnl_unlock();
5129
	if ((adap->flags & CXGB4_DEV_ENABLED)) {
5130
		pci_disable_device(pdev);
5131
		adap->flags &= ~CXGB4_DEV_ENABLED;
5132
	}
D
Dimitris Michailidis 已提交
5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148
out:	return state == pci_channel_io_perm_failure ?
		PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
}

static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
{
	int i, ret;
	struct fw_caps_config_cmd c;
	struct adapter *adap = pci_get_drvdata(pdev);

	if (!adap) {
		pci_restore_state(pdev);
		pci_save_state(pdev);
		return PCI_ERS_RESULT_RECOVERED;
	}

5149
	if (!(adap->flags & CXGB4_DEV_ENABLED)) {
5150 5151 5152 5153 5154
		if (pci_enable_device(pdev)) {
			dev_err(&pdev->dev, "Cannot reenable PCI "
					    "device after reset\n");
			return PCI_ERS_RESULT_DISCONNECT;
		}
5155
		adap->flags |= CXGB4_DEV_ENABLED;
D
Dimitris Michailidis 已提交
5156 5157 5158 5159 5160 5161
	}

	pci_set_master(pdev);
	pci_restore_state(pdev);
	pci_save_state(pdev);

5162
	if (t4_wait_dev_ready(adap->regs) < 0)
D
Dimitris Michailidis 已提交
5163
		return PCI_ERS_RESULT_DISCONNECT;
5164
	if (t4_fw_hello(adap, adap->mbox, adap->pf, MASTER_MUST, NULL) < 0)
D
Dimitris Michailidis 已提交
5165
		return PCI_ERS_RESULT_DISCONNECT;
5166
	adap->flags |= CXGB4_FW_OK;
D
Dimitris Michailidis 已提交
5167 5168 5169 5170
	if (adap_init1(adap, &c))
		return PCI_ERS_RESULT_DISCONNECT;

	for_each_port(adap, i) {
5171 5172
		struct port_info *pi = adap2pinfo(adap, i);
		u8 vivld = 0, vin = 0;
D
Dimitris Michailidis 已提交
5173

5174 5175
		ret = t4_alloc_vi(adap, adap->mbox, pi->tx_chan, adap->pf, 0, 1,
				  NULL, NULL, &vivld, &vin);
D
Dimitris Michailidis 已提交
5176 5177
		if (ret < 0)
			return PCI_ERS_RESULT_DISCONNECT;
5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190
		pi->viid = ret;
		pi->xact_addr_filt = -1;
		/* If fw supports returning the VIN as part of FW_VI_CMD,
		 * save the returned values.
		 */
		if (adap->params.viid_smt_extn_support) {
			pi->vivld = vivld;
			pi->vin = vin;
		} else {
			/* Retrieve the values from VIID */
			pi->vivld = FW_VIID_VIVLD_G(pi->viid);
			pi->vin = FW_VIID_VIN_G(pi->viid);
		}
D
Dimitris Michailidis 已提交
5191 5192 5193 5194
	}

	t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
		     adap->params.b_wnd);
5195
	setup_memwin(adap);
D
Dimitris Michailidis 已提交
5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211
	if (cxgb_up(adap))
		return PCI_ERS_RESULT_DISCONNECT;
	return PCI_ERS_RESULT_RECOVERED;
}

static void eeh_resume(struct pci_dev *pdev)
{
	int i;
	struct adapter *adap = pci_get_drvdata(pdev);

	if (!adap)
		return;

	rtnl_lock();
	for_each_port(adap, i) {
		struct net_device *dev = adap->port[i];
5212 5213 5214 5215 5216 5217
		if (dev) {
			if (netif_running(dev)) {
				link_start(dev);
				cxgb_set_rxmode(dev);
			}
			netif_device_attach(dev);
D
Dimitris Michailidis 已提交
5218 5219 5220 5221 5222
		}
	}
	rtnl_unlock();
}

V
Vishal Kulkarni 已提交
5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303
static void eeh_reset_prepare(struct pci_dev *pdev)
{
	struct adapter *adapter = pci_get_drvdata(pdev);
	int i;

	if (adapter->pf != 4)
		return;

	adapter->flags &= ~CXGB4_FW_OK;

	notify_ulds(adapter, CXGB4_STATE_DOWN);

	for_each_port(adapter, i)
		if (adapter->port[i]->reg_state == NETREG_REGISTERED)
			cxgb_close(adapter->port[i]);

	disable_interrupts(adapter);
	cxgb4_free_mps_ref_entries(adapter);

	adap_free_hma_mem(adapter);

	if (adapter->flags & CXGB4_FULL_INIT_DONE)
		cxgb_down(adapter);
}

static void eeh_reset_done(struct pci_dev *pdev)
{
	struct adapter *adapter = pci_get_drvdata(pdev);
	int err, i;

	if (adapter->pf != 4)
		return;

	err = t4_wait_dev_ready(adapter->regs);
	if (err < 0) {
		dev_err(adapter->pdev_dev,
			"Device not ready, err %d", err);
		return;
	}

	setup_memwin(adapter);

	err = adap_init0(adapter, 1);
	if (err) {
		dev_err(adapter->pdev_dev,
			"Adapter init failed, err %d", err);
		return;
	}

	setup_memwin_rdma(adapter);

	if (adapter->flags & CXGB4_FW_OK) {
		err = t4_port_init(adapter, adapter->pf, adapter->pf, 0);
		if (err) {
			dev_err(adapter->pdev_dev,
				"Port init failed, err %d", err);
			return;
		}
	}

	err = cfg_queues(adapter);
	if (err) {
		dev_err(adapter->pdev_dev,
			"Config queues failed, err %d", err);
		return;
	}

	cxgb4_init_mps_ref_entries(adapter);

	err = setup_fw_sge_queues(adapter);
	if (err) {
		dev_err(adapter->pdev_dev,
			"FW sge queue allocation failed, err %d", err);
		return;
	}

	for_each_port(adapter, i)
		if (adapter->port[i]->reg_state == NETREG_REGISTERED)
			cxgb_open(adapter->port[i]);
}

5304
static const struct pci_error_handlers cxgb4_eeh = {
D
Dimitris Michailidis 已提交
5305 5306 5307
	.error_detected = eeh_err_detected,
	.slot_reset     = eeh_slot_reset,
	.resume         = eeh_resume,
V
Vishal Kulkarni 已提交
5308 5309
	.reset_prepare  = eeh_reset_prepare,
	.reset_done     = eeh_reset_done,
D
Dimitris Michailidis 已提交
5310 5311
};

5312 5313 5314
/* Return true if the Link Configuration supports "High Speeds" (those greater
 * than 1Gb/s).
 */
5315
static inline bool is_x_10g_port(const struct link_config *lc)
5316
{
5317 5318
	unsigned int speeds, high_speeds;

5319 5320 5321
	speeds = FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_G(lc->pcaps));
	high_speeds = speeds &
			~(FW_PORT_CAP32_SPEED_100M | FW_PORT_CAP32_SPEED_1G);
5322 5323

	return high_speeds != 0;
5324 5325
}

5326
/* Perform default configuration of DMA queues depending on the number and type
5327 5328 5329
 * of ports we found and the number of available CPUs.  Most settings can be
 * modified by the admin prior to actual use.
 */
5330
static int cfg_queues(struct adapter *adap)
5331
{
5332 5333
	u32 avail_qsets, avail_eth_qsets, avail_uld_qsets;
	u32 niqflint, neq, num_ulds;
5334
	struct sge *s = &adap->sge;
5335
	u32 i, n10g = 0, qidx = 0;
5336 5337 5338
#ifndef CONFIG_CHELSIO_T4_DCB
	int q10g = 0;
#endif
5339

5340
	/* Reduce memory usage in kdump environment, disable all offload. */
5341
	if (is_kdump_kernel() || (is_uld(adap) && t4_uld_mem_alloc(adap))) {
5342
		adap->params.offload = 0;
5343
		adap->params.crypto = 0;
5344
		adap->params.ethofld = 0;
5345 5346
	}

5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359
	/* Calculate the number of Ethernet Queue Sets available based on
	 * resources provisioned for us.  We always have an Asynchronous
	 * Firmware Event Ingress Queue.  If we're operating in MSI or Legacy
	 * IRQ Pin Interrupt mode, then we'll also have a Forwarded Interrupt
	 * Ingress Queue.  Meanwhile, we need two Egress Queues for each
	 * Queue Set: one for the Free List and one for the Ethernet TX Queue.
	 *
	 * Note that we should also take into account all of the various
	 * Offload Queues.  But, in any situation where we're operating in
	 * a Resource Constrained Provisioning environment, doing any Offload
	 * at all is problematic ...
	 */
	niqflint = adap->params.pfres.niqflint - 1;
5360
	if (!(adap->flags & CXGB4_USING_MSIX))
5361 5362
		niqflint--;
	neq = adap->params.pfres.neq / 2;
5363
	avail_qsets = min(niqflint, neq);
5364

5365
	if (avail_qsets < adap->params.nports) {
5366
		dev_err(adap->pdev_dev, "avail_eth_qsets=%d < nports=%d\n",
5367
			avail_qsets, adap->params.nports);
5368 5369 5370 5371 5372 5373 5374
		return -ENOMEM;
	}

	/* Count the number of 10Gb/s or better ports */
	for_each_port(adap, i)
		n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);

5375
	avail_eth_qsets = min_t(u32, avail_qsets, MAX_ETH_QSETS);
5376 5377 5378 5379 5380
#ifdef CONFIG_CHELSIO_T4_DCB
	/* For Data Center Bridging support we need to be able to support up
	 * to 8 Traffic Priorities; each of which will be assigned to its
	 * own TX Queue in order to prevent Head-Of-Line Blocking.
	 */
5381 5382 5383 5384
	if (adap->params.nports * 8 > avail_eth_qsets) {
		dev_err(adap->pdev_dev, "DCB avail_eth_qsets=%d < %d!\n",
			avail_eth_qsets, adap->params.nports * 8);
		return -ENOMEM;
5385
	}
5386

5387 5388 5389 5390
	for_each_port(adap, i) {
		struct port_info *pi = adap2pinfo(adap, i);

		pi->first_qset = qidx;
5391
		pi->nqsets = is_kdump_kernel() ? 1 : 8;
5392 5393 5394
		qidx += pi->nqsets;
	}
#else /* !CONFIG_CHELSIO_T4_DCB */
5395
	/* We default to 1 queue per non-10G port and up to # of cores queues
5396 5397 5398
	 * per 10G port.
	 */
	if (n10g)
5399
		q10g = (avail_eth_qsets - (adap->params.nports - n10g)) / n10g;
5400 5401
	if (q10g > netif_get_num_default_rss_queues())
		q10g = netif_get_num_default_rss_queues();
5402

5403 5404 5405
	if (is_kdump_kernel())
		q10g = 1;

5406 5407 5408 5409
	for_each_port(adap, i) {
		struct port_info *pi = adap2pinfo(adap, i);

		pi->first_qset = qidx;
5410
		pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
5411 5412
		qidx += pi->nqsets;
	}
5413
#endif /* !CONFIG_CHELSIO_T4_DCB */
5414 5415 5416

	s->ethqsets = qidx;
	s->max_ethqsets = qidx;   /* MSI-X may lower it later */
5417
	avail_qsets -= qidx;
5418

5419
	if (is_uld(adap)) {
5420
		/* For offload we use 1 queue/channel if all ports are up to 1G,
5421 5422 5423
		 * otherwise we divide all available queues amongst the channels
		 * capped by the number of available cores.
		 */
5424 5425 5426 5427 5428 5429 5430 5431
		num_ulds = adap->num_uld + adap->num_ofld_uld;
		i = min_t(u32, MAX_OFLD_QSETS, num_online_cpus());
		avail_uld_qsets = roundup(i, adap->params.nports);
		if (avail_qsets < num_ulds * adap->params.nports) {
			adap->params.offload = 0;
			adap->params.crypto = 0;
			s->ofldqsets = 0;
		} else if (avail_qsets < num_ulds * avail_uld_qsets || !n10g) {
5432
			s->ofldqsets = adap->params.nports;
5433 5434
		} else {
			s->ofldqsets = avail_uld_qsets;
5435
		}
5436 5437

		avail_qsets -= num_ulds * s->ofldqsets;
5438 5439
	}

5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452
	/* ETHOFLD Queues used for QoS offload should follow same
	 * allocation scheme as normal Ethernet Queues.
	 */
	if (is_ethofld(adap)) {
		if (avail_qsets < s->max_ethqsets) {
			adap->params.ethofld = 0;
			s->eoqsets = 0;
		} else {
			s->eoqsets = s->max_ethqsets;
		}
		avail_qsets -= s->eoqsets;
	}

5453 5454 5455
	for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
		struct sge_eth_rxq *r = &s->ethrxq[i];

5456
		init_rspq(adap, &r->rspq, 5, 10, 1024, 64);
5457 5458 5459 5460 5461 5462 5463 5464 5465
		r->fl.size = 72;
	}

	for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
		s->ethtxq[i].q.size = 1024;

	for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
		s->ctrlq[i].q.size = 512;

5466 5467 5468
	if (!is_t4(adap->params.chip))
		s->ptptxq.q.size = 8;

5469
	init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64);
5470
	init_rspq(adap, &s->intrq, 0, 1, 512, 64);
5471 5472

	return 0;
5473 5474 5475 5476 5477 5478
}

/*
 * Reduce the number of Ethernet queues across all ports to at most n.
 * n provides at least one queue per port.
 */
B
Bill Pemberton 已提交
5479
static void reduce_ethqs(struct adapter *adap, int n)
5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502
{
	int i;
	struct port_info *pi;

	while (n < adap->sge.ethqsets)
		for_each_port(adap, i) {
			pi = adap2pinfo(adap, i);
			if (pi->nqsets > 1) {
				pi->nqsets--;
				adap->sge.ethqsets--;
				if (adap->sge.ethqsets <= n)
					break;
			}
		}

	n = 0;
	for_each_port(adap, i) {
		pi = adap2pinfo(adap, i);
		pi->first_qset = n;
		n += pi->nqsets;
	}
}

5503
static int alloc_msix_info(struct adapter *adap, u32 num_vec)
5504
{
5505
	struct msix_info *msix_info;
5506

5507
	msix_info = kcalloc(num_vec, sizeof(*msix_info), GFP_KERNEL);
5508 5509 5510
	if (!msix_info)
		return -ENOMEM;

5511 5512 5513
	adap->msix_bmap.msix_bmap = kcalloc(BITS_TO_LONGS(num_vec),
					    sizeof(long), GFP_KERNEL);
	if (!adap->msix_bmap.msix_bmap) {
5514 5515 5516
		kfree(msix_info);
		return -ENOMEM;
	}
5517 5518 5519 5520 5521

	spin_lock_init(&adap->msix_bmap.lock);
	adap->msix_bmap.mapsize = num_vec;

	adap->msix_info = msix_info;
5522 5523 5524 5525 5526
	return 0;
}

static void free_msix_info(struct adapter *adap)
{
5527 5528 5529 5530 5531 5532 5533 5534 5535
	kfree(adap->msix_bmap.msix_bmap);
	kfree(adap->msix_info);
}

int cxgb4_get_msix_idx_from_bmap(struct adapter *adap)
{
	struct msix_bmap *bmap = &adap->msix_bmap;
	unsigned int msix_idx;
	unsigned long flags;
5536

5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558
	spin_lock_irqsave(&bmap->lock, flags);
	msix_idx = find_first_zero_bit(bmap->msix_bmap, bmap->mapsize);
	if (msix_idx < bmap->mapsize) {
		__set_bit(msix_idx, bmap->msix_bmap);
	} else {
		spin_unlock_irqrestore(&bmap->lock, flags);
		return -ENOSPC;
	}

	spin_unlock_irqrestore(&bmap->lock, flags);
	return msix_idx;
}

void cxgb4_free_msix_idx_in_bmap(struct adapter *adap,
				 unsigned int msix_idx)
{
	struct msix_bmap *bmap = &adap->msix_bmap;
	unsigned long flags;

	spin_lock_irqsave(&bmap->lock, flags);
	__clear_bit(msix_idx, bmap->msix_bmap);
	spin_unlock_irqrestore(&bmap->lock, flags);
5559 5560
}

5561 5562 5563
/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
#define EXTRA_VECS 2

B
Bill Pemberton 已提交
5564
static int enable_msix(struct adapter *adap)
5565
{
5566 5567
	u32 eth_need, uld_need = 0, ethofld_need = 0;
	u32 ethqsets = 0, ofldqsets = 0, eoqsets = 0;
5568 5569
	u8 num_uld = 0, nchan = adap->params.nports;
	u32 i, want, need, num_vec;
5570
	struct sge *s = &adap->sge;
5571
	struct msix_entry *entries;
5572 5573
	struct port_info *pi;
	int allocated, ret;
5574

5575
	want = s->max_ethqsets;
5576 5577 5578 5579
#ifdef CONFIG_CHELSIO_T4_DCB
	/* For Data Center Bridging we need 8 Ethernet TX Priority Queues for
	 * each port.
	 */
5580
	need = 8 * nchan;
5581
#else
5582
	need = nchan;
5583
#endif
5584 5585 5586 5587 5588 5589 5590 5591
	eth_need = need;
	if (is_uld(adap)) {
		num_uld = adap->num_ofld_uld + adap->num_uld;
		want += num_uld * s->ofldqsets;
		uld_need = num_uld * nchan;
		need += uld_need;
	}

5592 5593 5594 5595 5596 5597
	if (is_ethofld(adap)) {
		want += s->eoqsets;
		ethofld_need = eth_need;
		need += ethofld_need;
	}

5598 5599 5600 5601 5602 5603 5604 5605 5606 5607
	want += EXTRA_VECS;
	need += EXTRA_VECS;

	entries = kmalloc_array(want, sizeof(*entries), GFP_KERNEL);
	if (!entries)
		return -ENOMEM;

	for (i = 0; i < want; i++)
		entries[i].entry = i;

5608 5609
	allocated = pci_enable_msix_range(adap->pdev, entries, need, want);
	if (allocated < 0) {
5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629
		/* Disable offload and attempt to get vectors for NIC
		 * only mode.
		 */
		want = s->max_ethqsets + EXTRA_VECS;
		need = eth_need + EXTRA_VECS;
		allocated = pci_enable_msix_range(adap->pdev, entries,
						  need, want);
		if (allocated < 0) {
			dev_info(adap->pdev_dev,
				 "Disabling MSI-X due to insufficient MSI-X vectors\n");
			ret = allocated;
			goto out_free;
		}

		dev_info(adap->pdev_dev,
			 "Disabling offload due to insufficient MSI-X vectors\n");
		adap->params.offload = 0;
		adap->params.crypto = 0;
		adap->params.ethofld = 0;
		s->ofldqsets = 0;
5630
		s->eoqsets = 0;
5631
		uld_need = 0;
5632
		ethofld_need = 0;
5633
	}
5634

5635 5636 5637 5638 5639 5640 5641 5642 5643
	num_vec = allocated;
	if (num_vec < want) {
		/* Distribute available vectors to the various queue groups.
		 * Every group gets its minimum requirement and NIC gets top
		 * priority for leftovers.
		 */
		ethqsets = eth_need;
		if (is_uld(adap))
			ofldqsets = nchan;
5644 5645
		if (is_ethofld(adap))
			eoqsets = ethofld_need;
5646 5647 5648

		num_vec -= need;
		while (num_vec) {
5649
			if (num_vec < eth_need + ethofld_need ||
5650 5651 5652 5653 5654 5655 5656 5657 5658 5659
			    ethqsets > s->max_ethqsets)
				break;

			for_each_port(adap, i) {
				pi = adap2pinfo(adap, i);
				if (pi->nqsets < 2)
					continue;

				ethqsets++;
				num_vec--;
5660 5661 5662 5663
				if (ethofld_need) {
					eoqsets++;
					num_vec--;
				}
5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680
			}
		}

		if (is_uld(adap)) {
			while (num_vec) {
				if (num_vec < uld_need ||
				    ofldqsets > s->ofldqsets)
					break;

				ofldqsets++;
				num_vec -= uld_need;
			}
		}
	} else {
		ethqsets = s->max_ethqsets;
		if (is_uld(adap))
			ofldqsets = s->ofldqsets;
5681 5682
		if (is_ethofld(adap))
			eoqsets = s->eoqsets;
5683 5684 5685 5686 5687
	}

	if (ethqsets < s->max_ethqsets) {
		s->max_ethqsets = ethqsets;
		reduce_ethqs(adap, ethqsets);
5688
	}
5689

5690
	if (is_uld(adap)) {
5691 5692
		s->ofldqsets = ofldqsets;
		s->nqs_per_uld = s->ofldqsets;
5693 5694
	}

5695 5696 5697
	if (is_ethofld(adap))
		s->eoqsets = eoqsets;

5698 5699 5700 5701 5702 5703
	/* map for msix */
	ret = alloc_msix_info(adap, allocated);
	if (ret)
		goto out_disable_msix;

	for (i = 0; i < allocated; i++) {
5704
		adap->msix_info[i].vec = entries[i].vector;
5705
		adap->msix_info[i].idx = i;
5706
	}
5707 5708

	dev_info(adap->pdev_dev,
5709 5710
		 "%d MSI-X vectors allocated, nic %d eoqsets %d per uld %d\n",
		 allocated, s->max_ethqsets, s->eoqsets, s->nqs_per_uld);
5711

5712
	kfree(entries);
5713
	return 0;
5714 5715 5716 5717 5718 5719 5720

out_disable_msix:
	pci_disable_msix(adap->pdev);

out_free:
	kfree(entries);
	return ret;
5721 5722 5723 5724
}

#undef EXTRA_VECS

B
Bill Pemberton 已提交
5725
static int init_rss(struct adapter *adap)
5726
{
5727 5728 5729 5730 5731 5732
	unsigned int i;
	int err;

	err = t4_init_rss_mode(adap, adap->mbox);
	if (err)
		return err;
5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743

	for_each_port(adap, i) {
		struct port_info *pi = adap2pinfo(adap, i);

		pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
		if (!pi->rss)
			return -ENOMEM;
	}
	return 0;
}

5744 5745 5746
/* Dump basic information about the adapter */
static void print_adapter_info(struct adapter *adapter)
{
5747 5748
	/* Hardware/Firmware/etc. Version/Revision IDs */
	t4_dump_version_info(adapter);
5749 5750 5751 5752

	/* Software/Hardware configuration */
	dev_info(adapter->pdev_dev, "Configuration: %sNIC %s, %s capable\n",
		 is_offload(adapter) ? "R" : "",
5753 5754
		 ((adapter->flags & CXGB4_USING_MSIX) ? "MSI-X" :
		  (adapter->flags & CXGB4_USING_MSI) ? "MSI" : ""),
5755 5756 5757
		 is_offload(adapter) ? "Offload" : "non-Offload");
}

B
Bill Pemberton 已提交
5758
static void print_port_info(const struct net_device *dev)
5759 5760
{
	char buf[80];
5761 5762 5763
	char *bufp = buf;
	const struct port_info *pi = netdev_priv(dev);
	const struct adapter *adap = pi->adapter;
5764

5765
	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100M)
5766
		bufp += sprintf(bufp, "100M/");
5767
	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_1G)
5768
		bufp += sprintf(bufp, "1G/");
5769
	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_10G)
5770
		bufp += sprintf(bufp, "10G/");
5771
	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_25G)
5772
		bufp += sprintf(bufp, "25G/");
5773
	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_40G)
5774
		bufp += sprintf(bufp, "40G/");
5775 5776 5777
	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_50G)
		bufp += sprintf(bufp, "50G/");
	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100G)
5778
		bufp += sprintf(bufp, "100G/");
5779 5780 5781 5782
	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_200G)
		bufp += sprintf(bufp, "200G/");
	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_400G)
		bufp += sprintf(bufp, "400G/");
5783 5784
	if (bufp != buf)
		--bufp;
5785
	sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
5786

5787 5788
	netdev_info(dev, "%s: Chelsio %s (%s) %s\n",
		    dev->name, adap->params.vpd.id, adap->name, buf);
5789 5790
}

5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801
/*
 * Free the following resources:
 * - memory used for tables
 * - MSI/MSI-X
 * - net devices
 * - resources FW is holding for us
 */
static void free_some_resources(struct adapter *adapter)
{
	unsigned int i;

5802
	kvfree(adapter->smt);
5803
	kvfree(adapter->l2t);
5804
	kvfree(adapter->srq);
5805
	t4_cleanup_sched(adapter);
5806
	kvfree(adapter->tids.tid_tab);
5807
	cxgb4_cleanup_tc_matchall(adapter);
5808
	cxgb4_cleanup_tc_mqprio(adapter);
5809
	cxgb4_cleanup_tc_flower(adapter);
5810
	cxgb4_cleanup_tc_u32(adapter);
5811 5812 5813 5814
	kfree(adapter->sge.egr_map);
	kfree(adapter->sge.ingr_map);
	kfree(adapter->sge.starving_fl);
	kfree(adapter->sge.txq_maperr);
5815 5816 5817
#ifdef CONFIG_DEBUG_FS
	kfree(adapter->sge.blocked_fl);
#endif
5818 5819 5820
	disable_msi(adapter);

	for_each_port(adapter, i)
5821
		if (adapter->port[i]) {
5822 5823 5824 5825 5826
			struct port_info *pi = adap2pinfo(adapter, i);

			if (pi->viid != 0)
				t4_free_vi(adapter, adapter->mbox, adapter->pf,
					   0, pi->viid);
5827
			kfree(adap2pinfo(adapter, i)->rss);
5828
			free_netdev(adapter->port[i]);
5829
		}
5830
	if (adapter->flags & CXGB4_FW_OK)
5831
		t4_fw_bye(adapter, adapter->pf);
5832 5833
}

5834
#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
5835
#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
5836
		   NETIF_F_GRO | NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
5837
#define SEGMENT_SIZE 128
5838

G
Ganesh Goudar 已提交
5839
static int t4_get_chip_type(struct adapter *adap, int ver)
5840
{
G
Ganesh Goudar 已提交
5841
	u32 pl_rev = REV_G(t4_read_reg(adap, PL_REV_A));
5842

G
Ganesh Goudar 已提交
5843
	switch (ver) {
5844
	case CHELSIO_T4:
5845
		return CHELSIO_CHIP_CODE(CHELSIO_T4, pl_rev);
5846
	case CHELSIO_T5:
5847
		return CHELSIO_CHIP_CODE(CHELSIO_T5, pl_rev);
5848
	case CHELSIO_T6:
5849
		return CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev);
5850
	default:
G
Ganesh Goudar 已提交
5851
		break;
5852
	}
5853
	return -EINVAL;
5854 5855
}

5856
#ifdef CONFIG_PCI_IOV
G
Ganesh Goudar 已提交
5857
static void cxgb4_mgmt_setup(struct net_device *dev)
5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871
{
	dev->type = ARPHRD_NONE;
	dev->mtu = 0;
	dev->hard_header_len = 0;
	dev->addr_len = 0;
	dev->tx_queue_len = 0;
	dev->flags |= IFF_NOARP;
	dev->priv_flags |= IFF_NO_QUEUE;

	/* Initialize the device structure. */
	dev->netdev_ops = &cxgb4_mgmt_netdev_ops;
	dev->ethtool_ops = &cxgb4_mgmt_ethtool_ops;
}

5872 5873
static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
{
5874
	struct adapter *adap = pci_get_drvdata(pdev);
5875 5876 5877 5878
	int err = 0;
	int current_vfs = pci_num_vf(pdev);
	u32 pcie_fw;

5879
	pcie_fw = readl(adap->regs + PCIE_FW_A);
5880 5881 5882
	/* Check if fw is initialized */
	if (!(pcie_fw & PCIE_FW_INIT_F)) {
		dev_warn(&pdev->dev, "Device not initialized\n");
5883 5884 5885 5886 5887 5888 5889 5890 5891
		return -EOPNOTSUPP;
	}

	/* If any of the VF's is already assigned to Guest OS, then
	 * SRIOV for the same cannot be modified
	 */
	if (current_vfs && pci_vfs_assigned(pdev)) {
		dev_err(&pdev->dev,
			"Cannot modify SR-IOV while VFs are assigned\n");
G
Ganesh Goudar 已提交
5892
		return current_vfs;
5893
	}
G
Ganesh Goudar 已提交
5894 5895 5896
	/* Note that the upper-level code ensures that we're never called with
	 * a non-zero "num_vfs" when we already have VFs instantiated.  But
	 * it never hurts to code defensively.
5897
	 */
G
Ganesh Goudar 已提交
5898 5899 5900 5901 5902 5903 5904 5905
	if (num_vfs != 0 && current_vfs != 0)
		return -EBUSY;

	/* Nothing to do for no change. */
	if (num_vfs == current_vfs)
		return num_vfs;

	/* Disable SRIOV when zero is passed. */
5906 5907
	if (!num_vfs) {
		pci_disable_sriov(pdev);
G
Ganesh Goudar 已提交
5908 5909 5910 5911 5912
		/* free VF Management Interface */
		unregister_netdev(adap->port[0]);
		free_netdev(adap->port[0]);
		adap->port[0] = NULL;

5913
		/* free VF resources */
G
Ganesh Goudar 已提交
5914
		adap->num_vfs = 0;
5915 5916
		kfree(adap->vfinfo);
		adap->vfinfo = NULL;
G
Ganesh Goudar 已提交
5917
		return 0;
5918 5919
	}

G
Ganesh Goudar 已提交
5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935
	if (!current_vfs) {
		struct fw_pfvf_cmd port_cmd, port_rpl;
		struct net_device *netdev;
		unsigned int pmask, port;
		struct pci_dev *pbridge;
		struct port_info *pi;
		char name[IFNAMSIZ];
		u32 devcap2;
		u16 flags;

		/* If we want to instantiate Virtual Functions, then our
		 * parent bridge's PCI-E needs to support Alternative Routing
		 * ID (ARI) because our VFs will show up at function offset 8
		 * and above.
		 */
		pbridge = pdev->bus->self;
5936 5937
		pcie_capability_read_word(pbridge, PCI_EXP_FLAGS, &flags);
		pcie_capability_read_dword(pbridge, PCI_EXP_DEVCAP2, &devcap2);
G
Ganesh Goudar 已提交
5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958

		if ((flags & PCI_EXP_FLAGS_VERS) < 2 ||
		    !(devcap2 & PCI_EXP_DEVCAP2_ARI)) {
			/* Our parent bridge does not support ARI so issue a
			 * warning and skip instantiating the VFs.  They
			 * won't be reachable.
			 */
			dev_warn(&pdev->dev, "Parent bridge %02x:%02x.%x doesn't support ARI; can't instantiate Virtual Functions\n",
				 pbridge->bus->number, PCI_SLOT(pbridge->devfn),
				 PCI_FUNC(pbridge->devfn));
			return -ENOTSUPP;
		}
		memset(&port_cmd, 0, sizeof(port_cmd));
		port_cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PFVF_CMD) |
						 FW_CMD_REQUEST_F |
						 FW_CMD_READ_F |
						 FW_PFVF_CMD_PFN_V(adap->pf) |
						 FW_PFVF_CMD_VFN_V(0));
		port_cmd.retval_len16 = cpu_to_be32(FW_LEN16(port_cmd));
		err = t4_wr_mbox(adap, adap->mbox, &port_cmd, sizeof(port_cmd),
				 &port_rpl);
5959 5960
		if (err)
			return err;
G
Ganesh Goudar 已提交
5961 5962 5963 5964 5965 5966 5967 5968 5969
		pmask = FW_PFVF_CMD_PMASK_G(be32_to_cpu(port_rpl.type_to_neq));
		port = ffs(pmask) - 1;
		/* Allocate VF Management Interface. */
		snprintf(name, IFNAMSIZ, "mgmtpf%d,%d", adap->adap_idx,
			 adap->pf);
		netdev = alloc_netdev(sizeof(struct port_info),
				      name, NET_NAME_UNKNOWN, cxgb4_mgmt_setup);
		if (!netdev)
			return -ENOMEM;
5970

G
Ganesh Goudar 已提交
5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984
		pi = netdev_priv(netdev);
		pi->adapter = adap;
		pi->lport = port;
		pi->tx_chan = port;
		SET_NETDEV_DEV(netdev, &pdev->dev);

		adap->port[0] = netdev;
		pi->port_id = 0;

		err = register_netdev(adap->port[0]);
		if (err) {
			pr_info("Unable to register VF mgmt netdev %s\n", name);
			free_netdev(adap->port[0]);
			adap->port[0] = NULL;
5985
			return err;
G
Ganesh Goudar 已提交
5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009
		}
		/* Allocate and set up VF Information. */
		adap->vfinfo = kcalloc(pci_sriov_get_totalvfs(pdev),
				       sizeof(struct vf_info), GFP_KERNEL);
		if (!adap->vfinfo) {
			unregister_netdev(adap->port[0]);
			free_netdev(adap->port[0]);
			adap->port[0] = NULL;
			return -ENOMEM;
		}
		cxgb4_mgmt_fill_vf_station_mac_addr(adap);
	}
	/* Instantiate the requested number of VFs. */
	err = pci_enable_sriov(pdev, num_vfs);
	if (err) {
		pr_info("Unable to instantiate %d VFs\n", num_vfs);
		if (!current_vfs) {
			unregister_netdev(adap->port[0]);
			free_netdev(adap->port[0]);
			adap->port[0] = NULL;
			kfree(adap->vfinfo);
			adap->vfinfo = NULL;
		}
		return err;
6010
	}
6011

G
Ganesh Goudar 已提交
6012
	adap->num_vfs = num_vfs;
6013 6014
	return num_vfs;
}
G
Ganesh Goudar 已提交
6015
#endif /* CONFIG_PCI_IOV */
6016

6017
static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
6018
{
G
Ganesh Goudar 已提交
6019 6020 6021 6022
	struct net_device *netdev;
	struct adapter *adapter;
	static int adap_idx = 1;
	int s_qpp, qpp, num_seg;
6023
	struct port_info *pi;
6024
	bool highdma = false;
6025
	enum chip_type chip;
G
Ganesh Goudar 已提交
6026 6027 6028 6029 6030
	void __iomem *regs;
	int func, chip_ver;
	u16 device_id;
	int i, err;
	u32 whoami;
6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046

	printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);

	err = pci_request_regions(pdev, KBUILD_MODNAME);
	if (err) {
		/* Just info, some other driver may have claimed the device. */
		dev_info(&pdev->dev, "cannot obtain PCI resources\n");
		return err;
	}

	err = pci_enable_device(pdev);
	if (err) {
		dev_err(&pdev->dev, "cannot enable PCI device\n");
		goto out_release_regions;
	}

6047 6048 6049 6050 6051 6052 6053
	regs = pci_ioremap_bar(pdev, 0);
	if (!regs) {
		dev_err(&pdev->dev, "cannot map device registers\n");
		err = -ENOMEM;
		goto out_disable_device;
	}

G
Ganesh Goudar 已提交
6054 6055 6056 6057 6058 6059 6060
	adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
	if (!adapter) {
		err = -ENOMEM;
		goto out_unmap_bar0;
	}

	adapter->regs = regs;
6061 6062
	err = t4_wait_dev_ready(regs);
	if (err < 0)
6063
		goto out_free_adapter;
6064

6065
	/* We control everything through one PF */
G
Ganesh Goudar 已提交
6066 6067 6068
	whoami = t4_read_reg(adapter, PL_WHOAMI_A);
	pci_read_config_word(pdev, PCI_DEVICE_ID, &device_id);
	chip = t4_get_chip_type(adapter, CHELSIO_PCI_ID_VER(device_id));
6069
	if ((int)chip < 0) {
G
Ganesh Goudar 已提交
6070 6071 6072 6073 6074 6075 6076
		dev_err(&pdev->dev, "Device %d is not supported\n", device_id);
		err = chip;
		goto out_free_adapter;
	}
	chip_ver = CHELSIO_CHIP_VERSION(chip);
	func = chip_ver <= CHELSIO_T5 ?
	       SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami);
G
Ganesh Goudar 已提交
6077 6078 6079 6080 6081 6082

	adapter->pdev = pdev;
	adapter->pdev_dev = &pdev->dev;
	adapter->name = pci_name(pdev);
	adapter->mbox = func;
	adapter->pf = func;
6083 6084
	adapter->params.chip = chip;
	adapter->adap_idx = adap_idx;
G
Ganesh Goudar 已提交
6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095
	adapter->msg_enable = DFLT_MSG_ENABLE;
	adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) +
				    (sizeof(struct mbox_cmd) *
				     T4_OS_LOG_MBOX_CMDS),
				    GFP_KERNEL);
	if (!adapter->mbox_log) {
		err = -ENOMEM;
		goto out_free_adapter;
	}
	spin_lock_init(&adapter->mbox_lock);
	INIT_LIST_HEAD(&adapter->mlist.list);
6096
	adapter->mbox_log->size = T4_OS_LOG_MBOX_CMDS;
G
Ganesh Goudar 已提交
6097 6098
	pci_set_drvdata(pdev, adapter);

6099 6100 6101
	if (func != ent->driver_data) {
		pci_disable_device(pdev);
		pci_save_state(pdev);        /* to restore SR-IOV later */
G
Ganesh Goudar 已提交
6102
		return 0;
6103 6104
	}

6105
	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
6106
		highdma = true;
6107 6108 6109 6110
		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
		if (err) {
			dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
				"coherent allocations\n");
G
Ganesh Goudar 已提交
6111
			goto out_free_adapter;
6112 6113 6114 6115 6116
		}
	} else {
		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
		if (err) {
			dev_err(&pdev->dev, "no usable DMA configuration\n");
G
Ganesh Goudar 已提交
6117
			goto out_free_adapter;
6118 6119 6120 6121 6122 6123
		}
	}

	pci_enable_pcie_error_reporting(pdev);
	pci_set_master(pdev);
	pci_save_state(pdev);
6124
	adap_idx++;
6125 6126 6127 6128 6129 6130
	adapter->workq = create_singlethread_workqueue("cxgb4");
	if (!adapter->workq) {
		err = -ENOMEM;
		goto out_free_adapter;
	}

6131
	/* PCI device has been enabled */
6132
	adapter->flags |= CXGB4_DEV_ENABLED;
6133 6134
	memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));

6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149
	/* If possible, we use PCIe Relaxed Ordering Attribute to deliver
	 * Ingress Packet Data to Free List Buffers in order to allow for
	 * chipset performance optimizations between the Root Complex and
	 * Memory Controllers.  (Messages to the associated Ingress Queue
	 * notifying new Packet Placement in the Free Lists Buffers will be
	 * send without the Relaxed Ordering Attribute thus guaranteeing that
	 * all preceding PCIe Transaction Layer Packets will be processed
	 * first.)  But some Root Complexes have various issues with Upstream
	 * Transaction Layer Packets with the Relaxed Ordering Attribute set.
	 * The PCIe devices which under the Root Complexes will be cleared the
	 * Relaxed Ordering bit in the configuration space, So we check our
	 * PCIe configuration space to see if it's flagged with advice against
	 * using Relaxed Ordering.
	 */
	if (!pcie_relaxed_ordering_enabled(pdev))
6150
		adapter->flags |= CXGB4_ROOT_NO_RELAXED_ORDERING;
6151

6152 6153
	spin_lock_init(&adapter->stats_lock);
	spin_lock_init(&adapter->tid_release_lock);
6154
	spin_lock_init(&adapter->win0_lock);
6155 6156

	INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
6157 6158
	INIT_WORK(&adapter->db_full_task, process_db_full);
	INIT_WORK(&adapter->db_drop_task, process_db_drop);
6159
	INIT_WORK(&adapter->fatal_err_notify_task, notify_fatal_err);
6160 6161 6162

	err = t4_prep_adapter(adapter);
	if (err)
6163 6164
		goto out_free_adapter;

6165 6166 6167 6168 6169 6170 6171 6172 6173 6174
	if (is_kdump_kernel()) {
		/* Collect hardware state and append to /proc/vmcore */
		err = cxgb4_cudbg_vmcore_add_dump(adapter);
		if (err) {
			dev_warn(adapter->pdev_dev,
				 "Fail collecting vmcore device dump, err: %d. Continuing\n",
				 err);
			err = 0;
		}
	}
6175

6176
	if (!is_t4(adapter->params.chip)) {
6177 6178
		s_qpp = (QUEUESPERPAGEPF0_S +
			(QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) *
6179
			adapter->pf);
6180 6181
		qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter,
		      SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp);
6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192
		num_seg = PAGE_SIZE / SEGMENT_SIZE;

		/* Each segment size is 128B. Write coalescing is enabled only
		 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
		 * queue is less no of segments that can be accommodated in
		 * a page size.
		 */
		if (qpp > num_seg) {
			dev_err(&pdev->dev,
				"Incorrect number of egress queues per page\n");
			err = -EINVAL;
6193
			goto out_free_adapter;
6194 6195 6196 6197 6198 6199
		}
		adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
		pci_resource_len(pdev, 2));
		if (!adapter->bar2) {
			dev_err(&pdev->dev, "cannot map device bar2 region\n");
			err = -ENOMEM;
6200
			goto out_free_adapter;
6201 6202 6203
		}
	}

6204
	setup_memwin(adapter);
V
Vishal Kulkarni 已提交
6205
	err = adap_init0(adapter, 0);
6206 6207 6208
#ifdef CONFIG_DEBUG_FS
	bitmap_zero(adapter->sge.blocked_fl, adapter->sge.egr_sz);
#endif
6209
	setup_memwin_rdma(adapter);
6210 6211 6212
	if (err)
		goto out_unmap_bar;

6213 6214
	/* configure SGE_STAT_CFG_A to read WC stats */
	if (!is_t4(adapter->params.chip))
6215 6216 6217
		t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7) |
			     (is_t5(adapter->params.chip) ? STATMODE_V(0) :
			      T6_STATMODE_V(0)));
6218

6219 6220 6221
	/* Initialize hash mac addr list */
	INIT_LIST_HEAD(&adapter->mac_hlist);

6222
	for_each_port(adapter, i) {
6223 6224 6225 6226 6227 6228
		/* For supporting MQPRIO Offload, need some extra
		 * queues for each ETHOFLD TIDs. Keep it equal to
		 * MAX_ATIDs for now. Once we connect to firmware
		 * later and query the EOTID params, we'll come to
		 * know the actual # of EOTIDs supported.
		 */
6229
		netdev = alloc_etherdev_mq(sizeof(struct port_info),
6230
					   MAX_ETH_QSETS + MAX_ATIDS);
6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244
		if (!netdev) {
			err = -ENOMEM;
			goto out_free_dev;
		}

		SET_NETDEV_DEV(netdev, &pdev->dev);

		adapter->port[i] = netdev;
		pi = netdev_priv(netdev);
		pi->adapter = adapter;
		pi->xact_addr_filt = -1;
		pi->port_id = i;
		netdev->irq = pdev->irq;

6245 6246
		netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
			NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6247
			NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_GRO |
6248 6249
			NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
			NETIF_F_HW_TC;
6250

G
Ganesh Goudar 已提交
6251
		if (chip_ver > CHELSIO_T5) {
6252 6253 6254 6255
			netdev->hw_enc_features |= NETIF_F_IP_CSUM |
						   NETIF_F_IPV6_CSUM |
						   NETIF_F_RXCSUM |
						   NETIF_F_GSO_UDP_TUNNEL |
6256
						   NETIF_F_GSO_UDP_TUNNEL_CSUM |
6257 6258
						   NETIF_F_TSO | NETIF_F_TSO6;

A
Atul Gupta 已提交
6259
			netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
6260
					       NETIF_F_GSO_UDP_TUNNEL_CSUM |
A
Atul Gupta 已提交
6261
					       NETIF_F_HW_TLS_RECORD;
6262
		}
6263

6264 6265 6266
		if (highdma)
			netdev->hw_features |= NETIF_F_HIGHDMA;
		netdev->features |= netdev->hw_features;
6267 6268
		netdev->vlan_features = netdev->features & VLAN_FEAT;

6269 6270
		netdev->priv_flags |= IFF_UNICAST_FLT;

6271
		/* MTU range: 81 - 9600 */
6272
		netdev->min_mtu = 81;              /* accommodate SACK */
6273 6274
		netdev->max_mtu = MAX_MTU;

6275
		netdev->netdev_ops = &cxgb4_netdev_ops;
6276 6277 6278
#ifdef CONFIG_CHELSIO_T4_DCB
		netdev->dcbnl_ops = &cxgb4_dcb_ops;
		cxgb4_dcb_state_init(netdev);
6279
		cxgb4_dcb_version_init(netdev);
6280
#endif
6281
		cxgb4_set_ethtool_ops(netdev);
6282 6283
	}

6284 6285
	cxgb4_init_ethtool_dump(adapter);

6286 6287
	pci_set_drvdata(pdev, adapter);

6288
	if (adapter->flags & CXGB4_FW_OK) {
6289
		err = t4_port_init(adapter, func, func, 0);
6290 6291
		if (err)
			goto out_free_dev;
6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307
	} else if (adapter->params.nports == 1) {
		/* If we don't have a connection to the firmware -- possibly
		 * because of an error -- grab the raw VPD parameters so we
		 * can set the proper MAC Address on the debug network
		 * interface that we've created.
		 */
		u8 hw_addr[ETH_ALEN];
		u8 *na = adapter->params.vpd.na;

		err = t4_get_raw_vpd_params(adapter, &adapter->params.vpd);
		if (!err) {
			for (i = 0; i < ETH_ALEN; i++)
				hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 +
					      hex2val(na[2 * i + 1]));
			t4_set_hw_addr(adapter, 0, hw_addr);
		}
6308 6309
	}

6310
	if (!(adapter->flags & CXGB4_FW_OK))
6311 6312
		goto fw_attach_fail;

6313
	/* Configure queues and allocate tables now, they can be needed as
6314 6315
	 * soon as the first register_netdev completes.
	 */
6316 6317 6318
	err = cfg_queues(adapter);
	if (err)
		goto out_free_dev;
6319

6320 6321 6322 6323 6324 6325
	adapter->smt = t4_init_smt();
	if (!adapter->smt) {
		/* We tolerate a lack of SMT, giving up some functionality */
		dev_warn(&pdev->dev, "could not allocate SMT, continuing\n");
	}

6326
	adapter->l2t = t4_init_l2t(adapter->l2t_start, adapter->l2t_end);
6327 6328 6329 6330 6331 6332
	if (!adapter->l2t) {
		/* We tolerate a lack of L2T, giving up some functionality */
		dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
		adapter->params.offload = 0;
	}

6333
#if IS_ENABLED(CONFIG_IPV6)
G
Ganesh Goudar 已提交
6334
	if (chip_ver <= CHELSIO_T5 &&
6335 6336 6337
	    (!(t4_read_reg(adapter, LE_DB_CONFIG_A) & ASLIPCOMPEN_F))) {
		/* CLIP functionality is not present in hardware,
		 * hence disable all offload features
6338 6339
		 */
		dev_warn(&pdev->dev,
6340
			 "CLIP not enabled in hardware, continuing\n");
6341
		adapter->params.offload = 0;
6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352
	} else {
		adapter->clipt = t4_init_clip_tbl(adapter->clipt_start,
						  adapter->clipt_end);
		if (!adapter->clipt) {
			/* We tolerate a lack of clip_table, giving up
			 * some functionality
			 */
			dev_warn(&pdev->dev,
				 "could not allocate Clip table, continuing\n");
			adapter->params.offload = 0;
		}
6353 6354
	}
#endif
6355 6356 6357 6358 6359 6360 6361 6362 6363 6364

	for_each_port(adapter, i) {
		pi = adap2pinfo(adapter, i);
		pi->sched_tbl = t4_init_sched(adapter->params.nsched_cls);
		if (!pi->sched_tbl)
			dev_warn(&pdev->dev,
				 "could not activate scheduling on port %d\n",
				 i);
	}

6365
	if (tid_init(&adapter->tids) < 0) {
6366 6367 6368
		dev_warn(&pdev->dev, "could not allocate TID table, "
			 "continuing\n");
		adapter->params.offload = 0;
6369
	} else {
6370
		adapter->tc_u32 = cxgb4_init_tc_u32(adapter);
6371 6372 6373
		if (!adapter->tc_u32)
			dev_warn(&pdev->dev,
				 "could not offload tc u32, continuing\n");
6374

6375 6376 6377
		if (cxgb4_init_tc_flower(adapter))
			dev_warn(&pdev->dev,
				 "could not offload tc flower, continuing\n");
6378 6379 6380 6381

		if (cxgb4_init_tc_mqprio(adapter))
			dev_warn(&pdev->dev,
				 "could not offload tc mqprio, continuing\n");
6382 6383 6384 6385

		if (cxgb4_init_tc_matchall(adapter))
			dev_warn(&pdev->dev,
				 "could not offload tc matchall, continuing\n");
6386 6387
	}

6388
	if (is_offload(adapter) || is_hashfilter(adapter)) {
6389 6390 6391
		if (t4_read_reg(adapter, LE_DB_CONFIG_A) & HASHEN_F) {
			u32 hash_base, hash_reg;

6392
			if (chip_ver <= CHELSIO_T5) {
6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403
				hash_reg = LE_DB_TID_HASHBASE_A;
				hash_base = t4_read_reg(adapter, hash_reg);
				adapter->tids.hash_base = hash_base / 4;
			} else {
				hash_reg = T6_LE_DB_HASH_TID_BASE_A;
				hash_base = t4_read_reg(adapter, hash_reg);
				adapter->tids.hash_base = hash_base;
			}
		}
	}

6404 6405
	/* See what interrupts we'll be using */
	if (msi > 1 && enable_msix(adapter) == 0)
6406
		adapter->flags |= CXGB4_USING_MSIX;
6407
	else if (msi > 0 && pci_enable_msi(pdev) == 0) {
6408
		adapter->flags |= CXGB4_USING_MSI;
6409 6410 6411
		if (msi > 1)
			free_msix_info(adapter);
	}
6412

6413
	/* check for PCI Express bandwidth capabiltites */
6414
	pcie_print_link_status(pdev);
6415

6416 6417
	cxgb4_init_mps_ref_entries(adapter);

6418 6419 6420 6421
	err = init_rss(adapter);
	if (err)
		goto out_free_dev;

6422 6423 6424 6425 6426 6427 6428
	err = setup_non_data_intr(adapter);
	if (err) {
		dev_err(adapter->pdev_dev,
			"Non Data interrupt allocation failed, err: %d\n", err);
		goto out_free_dev;
	}

6429 6430 6431 6432 6433 6434 6435
	err = setup_fw_sge_queues(adapter);
	if (err) {
		dev_err(adapter->pdev_dev,
			"FW sge queue allocation failed, err %d", err);
		goto out_free_dev;
	}

6436
fw_attach_fail:
6437 6438 6439 6440 6441 6442 6443
	/*
	 * The card is now ready to go.  If any errors occur during device
	 * registration we do not fail the whole card but rather proceed only
	 * with the ports we manage to register successfully.  However we must
	 * register at least one net device.
	 */
	for_each_port(adapter, i) {
6444
		pi = adap2pinfo(adapter, i);
6445
		adapter->port[i]->dev_port = pi->lport;
6446 6447 6448
		netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
		netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);

6449 6450
		netif_carrier_off(adapter->port[i]);

6451 6452
		err = register_netdev(adapter->port[i]);
		if (err)
6453 6454 6455
			break;
		adapter->chan_map[pi->tx_chan] = i;
		print_port_info(adapter->port[i]);
6456
	}
6457
	if (i == 0) {
6458 6459 6460
		dev_err(&pdev->dev, "could not register any net devices\n");
		goto out_free_dev;
	}
6461 6462 6463
	if (err) {
		dev_warn(&pdev->dev, "only %d net devices registered\n", i);
		err = 0;
6464
	}
6465 6466 6467 6468 6469 6470 6471

	if (cxgb4_debugfs_root) {
		adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
							   cxgb4_debugfs_root);
		setup_debugfs(adapter);
	}

D
Divy Le Ray 已提交
6472 6473 6474
	/* PCIe EEH recovery on powerpc platforms needs fundamental reset */
	pdev->needs_freset = 1;

6475 6476 6477 6478 6479
	if (is_uld(adapter)) {
		mutex_lock(&uld_mutex);
		list_add_tail(&adapter->list_node, &adapter_list);
		mutex_unlock(&uld_mutex);
	}
6480

6481 6482 6483
	if (!is_t4(adapter->params.chip))
		cxgb4_ptp_init(adapter);

6484
	if (IS_REACHABLE(CONFIG_THERMAL) &&
6485
	    !is_t4(adapter->params.chip) && (adapter->flags & CXGB4_FW_OK))
G
Ganesh Goudar 已提交
6486 6487
		cxgb4_thermal_init(adapter);

6488
	print_adapter_info(adapter);
6489
	return 0;
6490

6491
 out_free_dev:
6492
	t4_free_sge_resources(adapter);
6493
	free_some_resources(adapter);
6494
	if (adapter->flags & CXGB4_USING_MSIX)
6495
		free_msix_info(adapter);
6496 6497
	if (adapter->num_uld || adapter->num_ofld_uld)
		t4_uld_mem_free(adapter);
6498
 out_unmap_bar:
6499
	if (!is_t4(adapter->params.chip))
6500
		iounmap(adapter->bar2);
6501
 out_free_adapter:
6502 6503 6504
	if (adapter->workq)
		destroy_workqueue(adapter->workq);

6505
	kfree(adapter->mbox_log);
6506
	kfree(adapter);
6507 6508
 out_unmap_bar0:
	iounmap(regs);
6509 6510 6511 6512 6513 6514 6515 6516
 out_disable_device:
	pci_disable_pcie_error_reporting(pdev);
	pci_disable_device(pdev);
 out_release_regions:
	pci_release_regions(pdev);
	return err;
}

B
Bill Pemberton 已提交
6517
static void remove_one(struct pci_dev *pdev)
6518 6519
{
	struct adapter *adapter = pci_get_drvdata(pdev);
6520
	struct hash_mac_addr *entry, *tmp;
6521

6522 6523 6524 6525
	if (!adapter) {
		pci_release_regions(pdev);
		return;
	}
6526

6527 6528 6529 6530 6531
	/* If we allocated filters, free up state associated with any
	 * valid filters ...
	 */
	clear_all_filters(adapter);

6532
	adapter->flags |= CXGB4_SHUTTING_DOWN;
6533

6534
	if (adapter->pf == 4) {
6535 6536
		int i;

6537 6538 6539 6540 6541
		/* Tear down per-adapter Work Queue first since it can contain
		 * references to our adapter data structure.
		 */
		destroy_workqueue(adapter->workq);

6542
		if (is_uld(adapter)) {
6543
			detach_ulds(adapter);
6544 6545
			t4_uld_clean_up(adapter);
		}
6546

A
Arjun Vynipadath 已提交
6547 6548
		adap_free_hma_mem(adapter);

6549 6550
		disable_interrupts(adapter);

6551 6552
		cxgb4_free_mps_ref_entries(adapter);

6553
		for_each_port(adapter, i)
D
Dimitris Michailidis 已提交
6554
			if (adapter->port[i]->reg_state == NETREG_REGISTERED)
6555 6556
				unregister_netdev(adapter->port[i]);

6557
		debugfs_remove_recursive(adapter->debugfs_root);
6558

6559 6560
		if (!is_t4(adapter->params.chip))
			cxgb4_ptp_stop(adapter);
6561
		if (IS_REACHABLE(CONFIG_THERMAL))
6562
			cxgb4_thermal_remove(adapter);
6563

6564
		if (adapter->flags & CXGB4_FULL_INIT_DONE)
6565
			cxgb_down(adapter);
6566

6567
		if (adapter->flags & CXGB4_USING_MSIX)
6568
			free_msix_info(adapter);
6569 6570
		if (adapter->num_uld || adapter->num_ofld_uld)
			t4_uld_mem_free(adapter);
6571
		free_some_resources(adapter);
6572 6573 6574 6575 6576 6577
		list_for_each_entry_safe(entry, tmp, &adapter->mac_hlist,
					 list) {
			list_del(&entry->list);
			kfree(entry);
		}

6578 6579 6580
#if IS_ENABLED(CONFIG_IPV6)
		t4_cleanup_clip_tbl(adapter);
#endif
6581
		if (!is_t4(adapter->params.chip))
6582
			iounmap(adapter->bar2);
6583 6584 6585
	}
#ifdef CONFIG_PCI_IOV
	else {
G
Ganesh Goudar 已提交
6586
		cxgb4_iov_configure(adapter->pdev, 0);
6587 6588
	}
#endif
6589 6590
	iounmap(adapter->regs);
	pci_disable_pcie_error_reporting(pdev);
6591
	if ((adapter->flags & CXGB4_DEV_ENABLED)) {
6592
		pci_disable_device(pdev);
6593
		adapter->flags &= ~CXGB4_DEV_ENABLED;
6594 6595 6596 6597 6598
	}
	pci_release_regions(pdev);
	kfree(adapter->mbox_log);
	synchronize_rcu();
	kfree(adapter);
6599 6600
}

6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618
/* "Shutdown" quiesces the device, stopping Ingress Packet and Interrupt
 * delivery.  This is essentially a stripped down version of the PCI remove()
 * function where we do the minimal amount of work necessary to shutdown any
 * further activity.
 */
static void shutdown_one(struct pci_dev *pdev)
{
	struct adapter *adapter = pci_get_drvdata(pdev);

	/* As with remove_one() above (see extended comment), we only want do
	 * do cleanup on PCI Devices which went all the way through init_one()
	 * ...
	 */
	if (!adapter) {
		pci_release_regions(pdev);
		return;
	}

6619
	adapter->flags |= CXGB4_SHUTTING_DOWN;
6620

6621 6622 6623 6624 6625 6626 6627
	if (adapter->pf == 4) {
		int i;

		for_each_port(adapter, i)
			if (adapter->port[i]->reg_state == NETREG_REGISTERED)
				cxgb_close(adapter->port[i]);

6628 6629 6630 6631 6632
		if (is_uld(adapter)) {
			detach_ulds(adapter);
			t4_uld_clean_up(adapter);
		}

6633 6634 6635 6636
		disable_interrupts(adapter);
		disable_msi(adapter);

		t4_sge_stop(adapter);
6637
		if (adapter->flags & CXGB4_FW_OK)
6638 6639 6640 6641
			t4_fw_bye(adapter, adapter->mbox);
	}
}

6642 6643 6644 6645
static struct pci_driver cxgb4_driver = {
	.name     = KBUILD_MODNAME,
	.id_table = cxgb4_pci_tbl,
	.probe    = init_one,
B
Bill Pemberton 已提交
6646
	.remove   = remove_one,
6647
	.shutdown = shutdown_one,
6648 6649 6650
#ifdef CONFIG_PCI_IOV
	.sriov_configure = cxgb4_iov_configure,
#endif
D
Dimitris Michailidis 已提交
6651
	.err_handler = &cxgb4_eeh,
6652 6653 6654 6655 6656 6657 6658 6659 6660
};

static int __init cxgb4_init_module(void)
{
	int ret;

	cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);

	ret = pci_register_driver(&cxgb4_driver);
6661
	if (ret < 0)
6662
		goto err_pci;
6663

6664
#if IS_ENABLED(CONFIG_IPV6)
6665
	if (!inet6addr_registered) {
6666 6667 6668 6669 6670
		ret = register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
		if (ret)
			pci_unregister_driver(&cxgb4_driver);
		else
			inet6addr_registered = true;
6671
	}
6672
#endif
6673

6674 6675 6676 6677 6678 6679
	if (ret == 0)
		return ret;

err_pci:
	debugfs_remove(cxgb4_debugfs_root);

6680 6681 6682 6683 6684
	return ret;
}

static void __exit cxgb4_cleanup_module(void)
{
6685
#if IS_ENABLED(CONFIG_IPV6)
6686
	if (inet6addr_registered) {
6687 6688 6689
		unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
		inet6addr_registered = false;
	}
6690
#endif
6691 6692 6693 6694 6695 6696
	pci_unregister_driver(&cxgb4_driver);
	debugfs_remove(cxgb4_debugfs_root);  /* NULL ok */
}

module_init(cxgb4_init_module);
module_exit(cxgb4_cleanup_module);