igb_main.c 217.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/* Intel(R) Gigabit Ethernet Linux driver
 * Copyright(c) 2007-2014 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, see <http://www.gnu.org/licenses/>.
 *
 * The full GNU General Public License is included in this distribution in
 * the file called "COPYING".
 *
 * Contact Information:
 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
 */
23

J
Jeff Kirsher 已提交
24 25
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

26 27 28
#include <linux/module.h>
#include <linux/types.h>
#include <linux/init.h>
J
Jiri Pirko 已提交
29
#include <linux/bitops.h>
30 31 32 33
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/netdevice.h>
#include <linux/ipv6.h>
34
#include <linux/slab.h>
35 36
#include <net/checksum.h>
#include <net/ip6_checksum.h>
37
#include <linux/net_tstamp.h>
38 39
#include <linux/mii.h>
#include <linux/ethtool.h>
40
#include <linux/if.h>
41 42
#include <linux/if_vlan.h>
#include <linux/pci.h>
43
#include <linux/pci-aspm.h>
44 45
#include <linux/delay.h>
#include <linux/interrupt.h>
46 47 48
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/sctp.h>
49
#include <linux/if_ether.h>
50
#include <linux/aer.h>
51
#include <linux/prefetch.h>
Y
Yan, Zheng 已提交
52
#include <linux/pm_runtime.h>
53
#ifdef CONFIG_IGB_DCA
J
Jeb Cramer 已提交
54 55
#include <linux/dca.h>
#endif
C
Carolyn Wyborny 已提交
56
#include <linux/i2c.h>
57 58
#include "igb.h"

C
Carolyn Wyborny 已提交
59 60
#define MAJ 5
#define MIN 0
A
Akeem G Abodunrin 已提交
61
#define BUILD 5
C
Carolyn Wyborny 已提交
62
#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
63
__stringify(BUILD) "-k"
64 65 66 67
char igb_driver_name[] = "igb";
char igb_driver_version[] = DRV_VERSION;
static const char igb_driver_string[] =
				"Intel(R) Gigabit Ethernet Network Driver";
68
static const char igb_copyright[] =
69
				"Copyright (c) 2007-2014 Intel Corporation.";
70 71 72 73 74

static const struct e1000_info *igb_info_tbl[] = {
	[board_82575] = &e1000_82575_info,
};

75
static const struct pci_device_id igb_pci_tbl[] = {
76 77 78
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
79 80 81 82 83
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
84 85
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 },
86 87 88 89
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
90 91
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
92
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
93 94 95
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
96 97
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
G
Gasparakis, Joseph 已提交
98 99
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
A
Alexander Duyck 已提交
100
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
101
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
102
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
A
Alexander Duyck 已提交
103 104
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
105
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
106
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
107
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
108 109 110 111 112 113 114 115 116 117 118 119 120
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
	/* required last entry */
	{0, }
};

MODULE_DEVICE_TABLE(pci, igb_pci_tbl);

static int igb_setup_all_tx_resources(struct igb_adapter *);
static int igb_setup_all_rx_resources(struct igb_adapter *);
static void igb_free_all_tx_resources(struct igb_adapter *);
static void igb_free_all_rx_resources(struct igb_adapter *);
121
static void igb_setup_mrqc(struct igb_adapter *);
122
static int igb_probe(struct pci_dev *, const struct pci_device_id *);
123
static void igb_remove(struct pci_dev *pdev);
124 125 126
static int igb_sw_init(struct igb_adapter *);
static int igb_open(struct net_device *);
static int igb_close(struct net_device *);
127
static void igb_configure(struct igb_adapter *);
128 129 130 131
static void igb_configure_tx(struct igb_adapter *);
static void igb_configure_rx(struct igb_adapter *);
static void igb_clean_all_tx_rings(struct igb_adapter *);
static void igb_clean_all_rx_rings(struct igb_adapter *);
132 133
static void igb_clean_tx_ring(struct igb_ring *);
static void igb_clean_rx_ring(struct igb_ring *);
134
static void igb_set_rx_mode(struct net_device *);
135 136 137
static void igb_update_phy_info(unsigned long);
static void igb_watchdog(unsigned long);
static void igb_watchdog_task(struct work_struct *);
138
static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
E
Eric Dumazet 已提交
139
static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
140
					  struct rtnl_link_stats64 *stats);
141 142
static int igb_change_mtu(struct net_device *, int);
static int igb_set_mac(struct net_device *, void *);
143
static void igb_set_uta(struct igb_adapter *adapter);
144 145 146
static irqreturn_t igb_intr(int irq, void *);
static irqreturn_t igb_intr_msi(int irq, void *);
static irqreturn_t igb_msix_other(int irq, void *);
147
static irqreturn_t igb_msix_ring(int irq, void *);
148
#ifdef CONFIG_IGB_DCA
149
static void igb_update_dca(struct igb_q_vector *);
J
Jeb Cramer 已提交
150
static void igb_setup_dca(struct igb_adapter *);
151
#endif /* CONFIG_IGB_DCA */
152
static int igb_poll(struct napi_struct *, int);
153
static bool igb_clean_tx_irq(struct igb_q_vector *);
154
static bool igb_clean_rx_irq(struct igb_q_vector *, int);
155 156 157
static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
static void igb_tx_timeout(struct net_device *);
static void igb_reset_task(struct work_struct *);
158 159
static void igb_vlan_mode(struct net_device *netdev,
			  netdev_features_t features);
160 161
static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
162
static void igb_restore_vlan(struct igb_adapter *);
163
static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
164 165 166
static void igb_ping_all_vfs(struct igb_adapter *);
static void igb_msg_task(struct igb_adapter *);
static void igb_vmm_control(struct igb_adapter *);
167
static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
168
static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
169 170 171
static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
static int igb_ndo_set_vf_vlan(struct net_device *netdev,
			       int vf, u16 vlan, u8 qos);
172
static int igb_ndo_set_vf_bw(struct net_device *, int, int, int);
L
Lior Levy 已提交
173 174
static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
				   bool setting);
175 176
static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
				 struct ifla_vf_info *ivi);
177
static void igb_check_vf_rate_limit(struct igb_adapter *);
R
RongQing Li 已提交
178 179

#ifdef CONFIG_PCI_IOV
180
static int igb_vf_configure(struct igb_adapter *adapter, int vf);
181
static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs);
R
RongQing Li 已提交
182
#endif
183 184

#ifdef CONFIG_PM
185
#ifdef CONFIG_PM_SLEEP
Y
Yan, Zheng 已提交
186
static int igb_suspend(struct device *);
187
#endif
Y
Yan, Zheng 已提交
188 189 190 191 192 193 194 195 196 197 198
static int igb_resume(struct device *);
#ifdef CONFIG_PM_RUNTIME
static int igb_runtime_suspend(struct device *dev);
static int igb_runtime_resume(struct device *dev);
static int igb_runtime_idle(struct device *dev);
#endif
static const struct dev_pm_ops igb_pm_ops = {
	SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
	SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
			igb_runtime_idle)
};
199 200
#endif
static void igb_shutdown(struct pci_dev *);
201
static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs);
202
#ifdef CONFIG_IGB_DCA
J
Jeb Cramer 已提交
203 204 205 206 207 208 209
static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
static struct notifier_block dca_notifier = {
	.notifier_call	= igb_notify_dca,
	.next		= NULL,
	.priority	= 0
};
#endif
210 211 212 213
#ifdef CONFIG_NET_POLL_CONTROLLER
/* for netdump / net console */
static void igb_netpoll(struct net_device *);
#endif
214
#ifdef CONFIG_PCI_IOV
215
static unsigned int max_vfs;
216
module_param(max_vfs, uint, 0);
C
Carolyn Wyborny 已提交
217
MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate per physical function");
218 219
#endif /* CONFIG_PCI_IOV */

220 221 222 223 224
static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
		     pci_channel_state_t);
static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
static void igb_io_resume(struct pci_dev *);

225
static const struct pci_error_handlers igb_err_handler = {
226 227 228 229 230
	.error_detected = igb_io_error_detected,
	.slot_reset = igb_io_slot_reset,
	.resume = igb_io_resume,
};

231
static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
232 233 234 235 236

static struct pci_driver igb_driver = {
	.name     = igb_driver_name,
	.id_table = igb_pci_tbl,
	.probe    = igb_probe,
237
	.remove   = igb_remove,
238
#ifdef CONFIG_PM
Y
Yan, Zheng 已提交
239
	.driver.pm = &igb_pm_ops,
240 241
#endif
	.shutdown = igb_shutdown,
242
	.sriov_configure = igb_pci_sriov_configure,
243 244 245 246 247 248 249 250
	.err_handler = &igb_err_handler
};

MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRV_VERSION);

251 252 253 254 255
#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
static int debug = -1;
module_param(debug, int, 0);
MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");

256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
struct igb_reg_info {
	u32 ofs;
	char *name;
};

static const struct igb_reg_info igb_reg_info_tbl[] = {

	/* General Registers */
	{E1000_CTRL, "CTRL"},
	{E1000_STATUS, "STATUS"},
	{E1000_CTRL_EXT, "CTRL_EXT"},

	/* Interrupt Registers */
	{E1000_ICR, "ICR"},

	/* RX Registers */
	{E1000_RCTL, "RCTL"},
	{E1000_RDLEN(0), "RDLEN"},
	{E1000_RDH(0), "RDH"},
	{E1000_RDT(0), "RDT"},
	{E1000_RXDCTL(0), "RXDCTL"},
	{E1000_RDBAL(0), "RDBAL"},
	{E1000_RDBAH(0), "RDBAH"},

	/* TX Registers */
	{E1000_TCTL, "TCTL"},
	{E1000_TDBAL(0), "TDBAL"},
	{E1000_TDBAH(0), "TDBAH"},
	{E1000_TDLEN(0), "TDLEN"},
	{E1000_TDH(0), "TDH"},
	{E1000_TDT(0), "TDT"},
	{E1000_TXDCTL(0), "TXDCTL"},
	{E1000_TDFH, "TDFH"},
	{E1000_TDFT, "TDFT"},
	{E1000_TDFHS, "TDFHS"},
	{E1000_TDFPC, "TDFPC"},

	/* List Terminator */
	{}
};

297
/* igb_regdump - register printout routine */
298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353
static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
{
	int n = 0;
	char rname[16];
	u32 regs[8];

	switch (reginfo->ofs) {
	case E1000_RDLEN(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RDLEN(n));
		break;
	case E1000_RDH(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RDH(n));
		break;
	case E1000_RDT(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RDT(n));
		break;
	case E1000_RXDCTL(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RXDCTL(n));
		break;
	case E1000_RDBAL(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RDBAL(n));
		break;
	case E1000_RDBAH(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RDBAH(n));
		break;
	case E1000_TDBAL(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RDBAL(n));
		break;
	case E1000_TDBAH(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_TDBAH(n));
		break;
	case E1000_TDLEN(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_TDLEN(n));
		break;
	case E1000_TDH(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_TDH(n));
		break;
	case E1000_TDT(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_TDT(n));
		break;
	case E1000_TXDCTL(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_TXDCTL(n));
		break;
	default:
J
Jeff Kirsher 已提交
354
		pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
355 356 357 358
		return;
	}

	snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
J
Jeff Kirsher 已提交
359 360
	pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
		regs[2], regs[3]);
361 362
}

363
/* igb_dump - Print registers, Tx-rings and Rx-rings */
364 365 366 367 368 369 370 371 372 373 374
static void igb_dump(struct igb_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
	struct e1000_hw *hw = &adapter->hw;
	struct igb_reg_info *reginfo;
	struct igb_ring *tx_ring;
	union e1000_adv_tx_desc *tx_desc;
	struct my_u0 { u64 a; u64 b; } *u0;
	struct igb_ring *rx_ring;
	union e1000_adv_rx_desc *rx_desc;
	u32 staterr;
375
	u16 i, n;
376 377 378 379 380 381 382

	if (!netif_msg_hw(adapter))
		return;

	/* Print netdevice Info */
	if (netdev) {
		dev_info(&adapter->pdev->dev, "Net device Info\n");
C
Carolyn Wyborny 已提交
383
		pr_info("Device Name     state            trans_start      last_rx\n");
J
Jeff Kirsher 已提交
384 385
		pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
			netdev->state, netdev->trans_start, netdev->last_rx);
386 387 388 389
	}

	/* Print Registers */
	dev_info(&adapter->pdev->dev, "Register Dump\n");
J
Jeff Kirsher 已提交
390
	pr_info(" Register Name   Value\n");
391 392 393 394 395 396 397 398 399 400
	for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
	     reginfo->name; reginfo++) {
		igb_regdump(hw, reginfo);
	}

	/* Print TX Ring Summary */
	if (!netdev || !netif_running(netdev))
		goto exit;

	dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
J
Jeff Kirsher 已提交
401
	pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
402
	for (n = 0; n < adapter->num_tx_queues; n++) {
403
		struct igb_tx_buffer *buffer_info;
404
		tx_ring = adapter->tx_ring[n];
405
		buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
J
Jeff Kirsher 已提交
406 407
		pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
			n, tx_ring->next_to_use, tx_ring->next_to_clean,
408 409
			(u64)dma_unmap_addr(buffer_info, dma),
			dma_unmap_len(buffer_info, len),
J
Jeff Kirsher 已提交
410 411
			buffer_info->next_to_watch,
			(u64)buffer_info->time_stamp);
412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
	}

	/* Print TX Rings */
	if (!netif_msg_tx_done(adapter))
		goto rx_ring_summary;

	dev_info(&adapter->pdev->dev, "TX Rings Dump\n");

	/* Transmit Descriptor Formats
	 *
	 * Advanced Transmit Descriptor
	 *   +--------------------------------------------------------------+
	 * 0 |         Buffer Address [63:0]                                |
	 *   +--------------------------------------------------------------+
	 * 8 | PAYLEN  | PORTS  |CC|IDX | STA | DCMD  |DTYP|MAC|RSV| DTALEN |
	 *   +--------------------------------------------------------------+
	 *   63      46 45    40 39 38 36 35 32 31   24             15       0
	 */

	for (n = 0; n < adapter->num_tx_queues; n++) {
		tx_ring = adapter->tx_ring[n];
J
Jeff Kirsher 已提交
433 434 435
		pr_info("------------------------------------\n");
		pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
		pr_info("------------------------------------\n");
C
Carolyn Wyborny 已提交
436
		pr_info("T [desc]     [address 63:0  ] [PlPOCIStDDM Ln] [bi->dma       ] leng  ntw timestamp        bi->skb\n");
437 438

		for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
J
Jeff Kirsher 已提交
439
			const char *next_desc;
440
			struct igb_tx_buffer *buffer_info;
441
			tx_desc = IGB_TX_DESC(tx_ring, i);
442
			buffer_info = &tx_ring->tx_buffer_info[i];
443
			u0 = (struct my_u0 *)tx_desc;
J
Jeff Kirsher 已提交
444 445 446 447 448 449 450 451 452 453
			if (i == tx_ring->next_to_use &&
			    i == tx_ring->next_to_clean)
				next_desc = " NTC/U";
			else if (i == tx_ring->next_to_use)
				next_desc = " NTU";
			else if (i == tx_ring->next_to_clean)
				next_desc = " NTC";
			else
				next_desc = "";

C
Carolyn Wyborny 已提交
454 455
			pr_info("T [0x%03X]    %016llX %016llX %016llX %04X  %p %016llX %p%s\n",
				i, le64_to_cpu(u0->a),
456
				le64_to_cpu(u0->b),
457 458
				(u64)dma_unmap_addr(buffer_info, dma),
				dma_unmap_len(buffer_info, len),
459 460
				buffer_info->next_to_watch,
				(u64)buffer_info->time_stamp,
J
Jeff Kirsher 已提交
461
				buffer_info->skb, next_desc);
462

463
			if (netif_msg_pktdata(adapter) && buffer_info->skb)
464 465
				print_hex_dump(KERN_INFO, "",
					DUMP_PREFIX_ADDRESS,
466
					16, 1, buffer_info->skb->data,
467 468
					dma_unmap_len(buffer_info, len),
					true);
469 470 471 472 473 474
		}
	}

	/* Print RX Rings Summary */
rx_ring_summary:
	dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
J
Jeff Kirsher 已提交
475
	pr_info("Queue [NTU] [NTC]\n");
476 477
	for (n = 0; n < adapter->num_rx_queues; n++) {
		rx_ring = adapter->rx_ring[n];
J
Jeff Kirsher 已提交
478 479
		pr_info(" %5d %5X %5X\n",
			n, rx_ring->next_to_use, rx_ring->next_to_clean);
480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510
	}

	/* Print RX Rings */
	if (!netif_msg_rx_status(adapter))
		goto exit;

	dev_info(&adapter->pdev->dev, "RX Rings Dump\n");

	/* Advanced Receive Descriptor (Read) Format
	 *    63                                           1        0
	 *    +-----------------------------------------------------+
	 *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
	 *    +----------------------------------------------+------+
	 *  8 |       Header Buffer Address [63:1]           |  DD  |
	 *    +-----------------------------------------------------+
	 *
	 *
	 * Advanced Receive Descriptor (Write-Back) Format
	 *
	 *   63       48 47    32 31  30      21 20 17 16   4 3     0
	 *   +------------------------------------------------------+
	 * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
	 *   | Checksum   Ident  |   |           |    | Type | Type |
	 *   +------------------------------------------------------+
	 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
	 *   +------------------------------------------------------+
	 *   63       48 47    32 31            20 19               0
	 */

	for (n = 0; n < adapter->num_rx_queues; n++) {
		rx_ring = adapter->rx_ring[n];
J
Jeff Kirsher 已提交
511 512 513
		pr_info("------------------------------------\n");
		pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
		pr_info("------------------------------------\n");
C
Carolyn Wyborny 已提交
514 515
		pr_info("R  [desc]      [ PktBuf     A0] [  HeadBuf   DD] [bi->dma       ] [bi->skb] <-- Adv Rx Read format\n");
		pr_info("RWB[desc]      [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n");
516 517

		for (i = 0; i < rx_ring->count; i++) {
J
Jeff Kirsher 已提交
518
			const char *next_desc;
519 520
			struct igb_rx_buffer *buffer_info;
			buffer_info = &rx_ring->rx_buffer_info[i];
521
			rx_desc = IGB_RX_DESC(rx_ring, i);
522 523
			u0 = (struct my_u0 *)rx_desc;
			staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
J
Jeff Kirsher 已提交
524 525 526 527 528 529 530 531

			if (i == rx_ring->next_to_use)
				next_desc = " NTU";
			else if (i == rx_ring->next_to_clean)
				next_desc = " NTC";
			else
				next_desc = "";

532 533
			if (staterr & E1000_RXD_STAT_DD) {
				/* Descriptor Done */
534 535
				pr_info("%s[0x%03X]     %016llX %016llX ---------------- %s\n",
					"RWB", i,
536 537
					le64_to_cpu(u0->a),
					le64_to_cpu(u0->b),
538
					next_desc);
539
			} else {
540 541
				pr_info("%s[0x%03X]     %016llX %016llX %016llX %s\n",
					"R  ", i,
542 543 544
					le64_to_cpu(u0->a),
					le64_to_cpu(u0->b),
					(u64)buffer_info->dma,
545
					next_desc);
546

547
				if (netif_msg_pktdata(adapter) &&
548
				    buffer_info->dma && buffer_info->page) {
549 550 551
					print_hex_dump(KERN_INFO, "",
					  DUMP_PREFIX_ADDRESS,
					  16, 1,
552 553
					  page_address(buffer_info->page) +
						      buffer_info->page_offset,
554
					  IGB_RX_BUFSZ, true);
555 556 557 558 559 560 561 562 563
				}
			}
		}
	}

exit:
	return;
}

564 565
/**
 *  igb_get_i2c_data - Reads the I2C SDA data bit
C
Carolyn Wyborny 已提交
566 567 568 569
 *  @hw: pointer to hardware structure
 *  @i2cctl: Current value of I2CCTL register
 *
 *  Returns the I2C data bit value
570
 **/
C
Carolyn Wyborny 已提交
571 572 573 574 575 576
static int igb_get_i2c_data(void *data)
{
	struct igb_adapter *adapter = (struct igb_adapter *)data;
	struct e1000_hw *hw = &adapter->hw;
	s32 i2cctl = rd32(E1000_I2CPARAMS);

577
	return !!(i2cctl & E1000_I2C_DATA_IN);
C
Carolyn Wyborny 已提交
578 579
}

580 581
/**
 *  igb_set_i2c_data - Sets the I2C data bit
C
Carolyn Wyborny 已提交
582 583 584 585
 *  @data: pointer to hardware structure
 *  @state: I2C data value (0 or 1) to set
 *
 *  Sets the I2C data bit
586
 **/
C
Carolyn Wyborny 已提交
587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604
static void igb_set_i2c_data(void *data, int state)
{
	struct igb_adapter *adapter = (struct igb_adapter *)data;
	struct e1000_hw *hw = &adapter->hw;
	s32 i2cctl = rd32(E1000_I2CPARAMS);

	if (state)
		i2cctl |= E1000_I2C_DATA_OUT;
	else
		i2cctl &= ~E1000_I2C_DATA_OUT;

	i2cctl &= ~E1000_I2C_DATA_OE_N;
	i2cctl |= E1000_I2C_CLK_OE_N;
	wr32(E1000_I2CPARAMS, i2cctl);
	wrfl();

}

605 606
/**
 *  igb_set_i2c_clk - Sets the I2C SCL clock
C
Carolyn Wyborny 已提交
607 608 609 610
 *  @data: pointer to hardware structure
 *  @state: state to set clock
 *
 *  Sets the I2C clock line to state
611
 **/
C
Carolyn Wyborny 已提交
612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628
static void igb_set_i2c_clk(void *data, int state)
{
	struct igb_adapter *adapter = (struct igb_adapter *)data;
	struct e1000_hw *hw = &adapter->hw;
	s32 i2cctl = rd32(E1000_I2CPARAMS);

	if (state) {
		i2cctl |= E1000_I2C_CLK_OUT;
		i2cctl &= ~E1000_I2C_CLK_OE_N;
	} else {
		i2cctl &= ~E1000_I2C_CLK_OUT;
		i2cctl &= ~E1000_I2C_CLK_OE_N;
	}
	wr32(E1000_I2CPARAMS, i2cctl);
	wrfl();
}

629 630
/**
 *  igb_get_i2c_clk - Gets the I2C SCL clock state
C
Carolyn Wyborny 已提交
631 632 633
 *  @data: pointer to hardware structure
 *
 *  Gets the I2C clock state
634
 **/
C
Carolyn Wyborny 已提交
635 636 637 638 639 640
static int igb_get_i2c_clk(void *data)
{
	struct igb_adapter *adapter = (struct igb_adapter *)data;
	struct e1000_hw *hw = &adapter->hw;
	s32 i2cctl = rd32(E1000_I2CPARAMS);

641
	return !!(i2cctl & E1000_I2C_CLK_IN);
C
Carolyn Wyborny 已提交
642 643 644 645 646 647 648 649 650 651 652
}

static const struct i2c_algo_bit_data igb_i2c_algo = {
	.setsda		= igb_set_i2c_data,
	.setscl		= igb_set_i2c_clk,
	.getsda		= igb_get_i2c_data,
	.getscl		= igb_get_i2c_clk,
	.udelay		= 5,
	.timeout	= 20,
};

653
/**
654 655 656 657
 *  igb_get_hw_dev - return device
 *  @hw: pointer to hardware structure
 *
 *  used by hardware layer to print debugging information
658
 **/
659
struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
660 661
{
	struct igb_adapter *adapter = hw->back;
662
	return adapter->netdev;
663
}
P
Patrick Ohly 已提交
664

665
/**
666
 *  igb_init_module - Driver Registration Routine
667
 *
668 669
 *  igb_init_module is the first routine called when the driver is
 *  loaded. All it does is register with the PCI subsystem.
670 671 672 673
 **/
static int __init igb_init_module(void)
{
	int ret;
674

J
Jeff Kirsher 已提交
675
	pr_info("%s - version %s\n",
676
	       igb_driver_string, igb_driver_version);
J
Jeff Kirsher 已提交
677
	pr_info("%s\n", igb_copyright);
678

679
#ifdef CONFIG_IGB_DCA
J
Jeb Cramer 已提交
680 681
	dca_register_notify(&dca_notifier);
#endif
682
	ret = pci_register_driver(&igb_driver);
683 684 685 686 687 688
	return ret;
}

module_init(igb_init_module);

/**
689
 *  igb_exit_module - Driver Exit Cleanup Routine
690
 *
691 692
 *  igb_exit_module is called just before the driver is removed
 *  from memory.
693 694 695
 **/
static void __exit igb_exit_module(void)
{
696
#ifdef CONFIG_IGB_DCA
J
Jeb Cramer 已提交
697 698
	dca_unregister_notify(&dca_notifier);
#endif
699 700 701 702 703
	pci_unregister_driver(&igb_driver);
}

module_exit(igb_exit_module);

704 705
#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
/**
706 707
 *  igb_cache_ring_register - Descriptor ring to register mapping
 *  @adapter: board private structure to initialize
708
 *
709 710
 *  Once we know the feature-set enabled for the device, we'll cache
 *  the register offset the descriptor ring is assigned to.
711 712 713
 **/
static void igb_cache_ring_register(struct igb_adapter *adapter)
{
714
	int i = 0, j = 0;
715
	u32 rbase_offset = adapter->vfs_allocated_count;
716 717 718 719 720 721 722 723

	switch (adapter->hw.mac.type) {
	case e1000_82576:
		/* The queues are allocated for virtualization such that VF 0
		 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
		 * In order to avoid collision we start at the first free queue
		 * and continue consuming queues in the same sequence
		 */
724
		if (adapter->vfs_allocated_count) {
725
			for (; i < adapter->rss_queues; i++)
726
				adapter->rx_ring[i]->reg_idx = rbase_offset +
727
							       Q_IDX_82576(i);
728
		}
729
		/* Fall through */
730
	case e1000_82575:
731
	case e1000_82580:
732
	case e1000_i350:
733
	case e1000_i354:
734 735
	case e1000_i210:
	case e1000_i211:
736
		/* Fall through */
737
	default:
738
		for (; i < adapter->num_rx_queues; i++)
739
			adapter->rx_ring[i]->reg_idx = rbase_offset + i;
740
		for (; j < adapter->num_tx_queues; j++)
741
			adapter->tx_ring[j]->reg_idx = rbase_offset + j;
742 743 744 745
		break;
	}
}

746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
u32 igb_rd32(struct e1000_hw *hw, u32 reg)
{
	struct igb_adapter *igb = container_of(hw, struct igb_adapter, hw);
	u8 __iomem *hw_addr = ACCESS_ONCE(hw->hw_addr);
	u32 value = 0;

	if (E1000_REMOVED(hw_addr))
		return ~value;

	value = readl(&hw_addr[reg]);

	/* reads should not return all F's */
	if (!(~value) && (!reg || !(~readl(hw_addr)))) {
		struct net_device *netdev = igb->netdev;
		hw->hw_addr = NULL;
		netif_device_detach(netdev);
		netdev_err(netdev, "PCIe link lost, device now detached\n");
	}

	return value;
}

A
Alexander Duyck 已提交
768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793
/**
 *  igb_write_ivar - configure ivar for given MSI-X vector
 *  @hw: pointer to the HW structure
 *  @msix_vector: vector number we are allocating to a given ring
 *  @index: row index of IVAR register to write within IVAR table
 *  @offset: column offset of in IVAR, should be multiple of 8
 *
 *  This function is intended to handle the writing of the IVAR register
 *  for adapters 82576 and newer.  The IVAR table consists of 2 columns,
 *  each containing an cause allocation for an Rx and Tx ring, and a
 *  variable number of rows depending on the number of queues supported.
 **/
static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
			   int index, int offset)
{
	u32 ivar = array_rd32(E1000_IVAR0, index);

	/* clear any bits that are currently set */
	ivar &= ~((u32)0xFF << offset);

	/* write vector and valid bit */
	ivar |= (msix_vector | E1000_IVAR_VALID) << offset;

	array_wr32(E1000_IVAR0, index, ivar);
}

794
#define IGB_N0_QUEUE -1
795
static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
796
{
797
	struct igb_adapter *adapter = q_vector->adapter;
798
	struct e1000_hw *hw = &adapter->hw;
799 800
	int rx_queue = IGB_N0_QUEUE;
	int tx_queue = IGB_N0_QUEUE;
A
Alexander Duyck 已提交
801
	u32 msixbm = 0;
802

803 804 805 806
	if (q_vector->rx.ring)
		rx_queue = q_vector->rx.ring->reg_idx;
	if (q_vector->tx.ring)
		tx_queue = q_vector->tx.ring->reg_idx;
A
Alexander Duyck 已提交
807 808 809

	switch (hw->mac.type) {
	case e1000_82575:
810
		/* The 82575 assigns vectors using a bitmask, which matches the
811 812 813 814
		 * bitmask for the EICR/EIMS/EIMC registers.  To assign one
		 * or more queues to a vector, we write the appropriate bits
		 * into the MSIXBM register for that vector.
		 */
815
		if (rx_queue > IGB_N0_QUEUE)
816
			msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
817
		if (tx_queue > IGB_N0_QUEUE)
818
			msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
819
		if (!(adapter->flags & IGB_FLAG_HAS_MSIX) && msix_vector == 0)
820
			msixbm |= E1000_EIMS_OTHER;
821
		array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
822
		q_vector->eims_value = msixbm;
A
Alexander Duyck 已提交
823 824
		break;
	case e1000_82576:
825
		/* 82576 uses a table that essentially consists of 2 columns
A
Alexander Duyck 已提交
826 827 828 829 830 831 832 833 834 835 836 837
		 * with 8 rows.  The ordering is column-major so we use the
		 * lower 3 bits as the row index, and the 4th bit as the
		 * column offset.
		 */
		if (rx_queue > IGB_N0_QUEUE)
			igb_write_ivar(hw, msix_vector,
				       rx_queue & 0x7,
				       (rx_queue & 0x8) << 1);
		if (tx_queue > IGB_N0_QUEUE)
			igb_write_ivar(hw, msix_vector,
				       tx_queue & 0x7,
				       ((tx_queue & 0x8) << 1) + 8);
838
		q_vector->eims_value = 1 << msix_vector;
A
Alexander Duyck 已提交
839
		break;
840
	case e1000_82580:
841
	case e1000_i350:
842
	case e1000_i354:
843 844
	case e1000_i210:
	case e1000_i211:
845
		/* On 82580 and newer adapters the scheme is similar to 82576
A
Alexander Duyck 已提交
846 847 848 849 850 851 852 853 854 855 856 857 858
		 * however instead of ordering column-major we have things
		 * ordered row-major.  So we traverse the table by using
		 * bit 0 as the column offset, and the remaining bits as the
		 * row index.
		 */
		if (rx_queue > IGB_N0_QUEUE)
			igb_write_ivar(hw, msix_vector,
				       rx_queue >> 1,
				       (rx_queue & 0x1) << 4);
		if (tx_queue > IGB_N0_QUEUE)
			igb_write_ivar(hw, msix_vector,
				       tx_queue >> 1,
				       ((tx_queue & 0x1) << 4) + 8);
859 860
		q_vector->eims_value = 1 << msix_vector;
		break;
A
Alexander Duyck 已提交
861 862 863 864
	default:
		BUG();
		break;
	}
865 866 867 868 869 870

	/* add q_vector eims value to global eims_enable_mask */
	adapter->eims_enable_mask |= q_vector->eims_value;

	/* configure q_vector to set itr on first interrupt */
	q_vector->set_itr = 1;
871 872 873
}

/**
874 875
 *  igb_configure_msix - Configure MSI-X hardware
 *  @adapter: board private structure to initialize
876
 *
877 878
 *  igb_configure_msix sets up the hardware to properly
 *  generate MSI-X interrupts.
879 880 881 882 883 884 885 886 887 888
 **/
static void igb_configure_msix(struct igb_adapter *adapter)
{
	u32 tmp;
	int i, vector = 0;
	struct e1000_hw *hw = &adapter->hw;

	adapter->eims_enable_mask = 0;

	/* set vector for other causes, i.e. link changes */
A
Alexander Duyck 已提交
889 890
	switch (hw->mac.type) {
	case e1000_82575:
891 892 893 894 895 896 897 898 899
		tmp = rd32(E1000_CTRL_EXT);
		/* enable MSI-X PBA support*/
		tmp |= E1000_CTRL_EXT_PBA_CLR;

		/* Auto-Mask interrupts upon ICR read. */
		tmp |= E1000_CTRL_EXT_EIAME;
		tmp |= E1000_CTRL_EXT_IRCA;

		wr32(E1000_CTRL_EXT, tmp);
900 901

		/* enable msix_other interrupt */
902
		array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER);
P
PJ Waskiewicz 已提交
903
		adapter->eims_other = E1000_EIMS_OTHER;
904

A
Alexander Duyck 已提交
905 906 907
		break;

	case e1000_82576:
908
	case e1000_82580:
909
	case e1000_i350:
910
	case e1000_i354:
911 912
	case e1000_i210:
	case e1000_i211:
913
		/* Turn on MSI-X capability first, or our settings
914 915
		 * won't stick.  And it will take days to debug.
		 */
916
		wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
917 918
		     E1000_GPIE_PBA | E1000_GPIE_EIAME |
		     E1000_GPIE_NSICR);
919 920 921

		/* enable msix_other interrupt */
		adapter->eims_other = 1 << vector;
A
Alexander Duyck 已提交
922 923
		tmp = (vector++ | E1000_IVAR_VALID) << 8;

924
		wr32(E1000_IVAR_MISC, tmp);
A
Alexander Duyck 已提交
925 926 927 928 929
		break;
	default:
		/* do nothing, since nothing else supports MSI-X */
		break;
	} /* switch (hw->mac.type) */
930 931 932

	adapter->eims_enable_mask |= adapter->eims_other;

933 934
	for (i = 0; i < adapter->num_q_vectors; i++)
		igb_assign_vector(adapter->q_vector[i], vector++);
935

936 937 938 939
	wrfl();
}

/**
940 941
 *  igb_request_msix - Initialize MSI-X interrupts
 *  @adapter: board private structure to initialize
942
 *
943 944
 *  igb_request_msix allocates MSI-X vectors and requests interrupts from the
 *  kernel.
945 946 947 948
 **/
static int igb_request_msix(struct igb_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
949
	struct e1000_hw *hw = &adapter->hw;
950
	int i, err = 0, vector = 0, free_vector = 0;
951

952
	err = request_irq(adapter->msix_entries[vector].vector,
953
			  igb_msix_other, 0, netdev->name, adapter);
954
	if (err)
955
		goto err_out;
956 957 958 959

	for (i = 0; i < adapter->num_q_vectors; i++) {
		struct igb_q_vector *q_vector = adapter->q_vector[i];

960 961
		vector++;

962 963
		q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);

964
		if (q_vector->rx.ring && q_vector->tx.ring)
965
			sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
966 967
				q_vector->rx.ring->queue_index);
		else if (q_vector->tx.ring)
968
			sprintf(q_vector->name, "%s-tx-%u", netdev->name,
969 970
				q_vector->tx.ring->queue_index);
		else if (q_vector->rx.ring)
971
			sprintf(q_vector->name, "%s-rx-%u", netdev->name,
972
				q_vector->rx.ring->queue_index);
973
		else
974 975
			sprintf(q_vector->name, "%s-unused", netdev->name);

976
		err = request_irq(adapter->msix_entries[vector].vector,
977 978
				  igb_msix_ring, 0, q_vector->name,
				  q_vector);
979
		if (err)
980
			goto err_free;
981 982 983 984
	}

	igb_configure_msix(adapter);
	return 0;
985 986 987 988 989 990 991 992 993 994 995

err_free:
	/* free already assigned IRQs */
	free_irq(adapter->msix_entries[free_vector++].vector, adapter);

	vector--;
	for (i = 0; i < vector; i++) {
		free_irq(adapter->msix_entries[free_vector++].vector,
			 adapter->q_vector[i]);
	}
err_out:
996 997 998
	return err;
}

999
/**
1000 1001 1002
 *  igb_free_q_vector - Free memory allocated for specific interrupt vector
 *  @adapter: board private structure to initialize
 *  @v_idx: Index of vector to be freed
1003
 *
1004
 *  This function frees the memory allocated to the q_vector.
1005 1006 1007 1008 1009
 **/
static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx)
{
	struct igb_q_vector *q_vector = adapter->q_vector[v_idx];

1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
	adapter->q_vector[v_idx] = NULL;

	/* igb_get_stats64() might access the rings on this vector,
	 * we must wait a grace period before freeing it.
	 */
	kfree_rcu(q_vector, rcu);
}

/**
 *  igb_reset_q_vector - Reset config for interrupt vector
 *  @adapter: board private structure to initialize
 *  @v_idx: Index of vector to be reset
 *
 *  If NAPI is enabled it will delete any references to the
 *  NAPI struct. This is preparation for igb_free_q_vector.
 **/
static void igb_reset_q_vector(struct igb_adapter *adapter, int v_idx)
{
	struct igb_q_vector *q_vector = adapter->q_vector[v_idx];

1030 1031 1032 1033 1034 1035
	/* Coming from igb_set_interrupt_capability, the vectors are not yet
	 * allocated. So, q_vector is NULL so we should stop here.
	 */
	if (!q_vector)
		return;

1036 1037 1038 1039 1040 1041 1042 1043
	if (q_vector->tx.ring)
		adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;

	if (q_vector->rx.ring)
		adapter->tx_ring[q_vector->rx.ring->queue_index] = NULL;

	netif_napi_del(&q_vector->napi);

1044 1045 1046 1047 1048 1049
}

static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
{
	int v_idx = adapter->num_q_vectors;

1050
	if (adapter->flags & IGB_FLAG_HAS_MSIX)
1051
		pci_disable_msix(adapter->pdev);
1052
	else if (adapter->flags & IGB_FLAG_HAS_MSI)
1053 1054 1055 1056
		pci_disable_msi(adapter->pdev);

	while (v_idx--)
		igb_reset_q_vector(adapter, v_idx);
1057 1058
}

1059
/**
1060 1061
 *  igb_free_q_vectors - Free memory allocated for interrupt vectors
 *  @adapter: board private structure to initialize
1062
 *
1063 1064 1065
 *  This function frees the memory allocated to the q_vectors.  In addition if
 *  NAPI is enabled it will delete any references to the NAPI struct prior
 *  to freeing the q_vector.
1066 1067 1068
 **/
static void igb_free_q_vectors(struct igb_adapter *adapter)
{
1069 1070 1071 1072
	int v_idx = adapter->num_q_vectors;

	adapter->num_tx_queues = 0;
	adapter->num_rx_queues = 0;
1073
	adapter->num_q_vectors = 0;
1074

1075 1076
	while (v_idx--) {
		igb_reset_q_vector(adapter, v_idx);
1077
		igb_free_q_vector(adapter, v_idx);
1078
	}
1079 1080 1081
}

/**
1082 1083
 *  igb_clear_interrupt_scheme - reset the device to a state of no interrupts
 *  @adapter: board private structure to initialize
1084
 *
1085 1086
 *  This function resets the device so that it has 0 Rx queues, Tx queues, and
 *  MSI-X interrupts allocated.
1087 1088 1089 1090 1091 1092
 */
static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
{
	igb_free_q_vectors(adapter);
	igb_reset_interrupt_capability(adapter);
}
1093 1094

/**
1095 1096 1097
 *  igb_set_interrupt_capability - set MSI or MSI-X if supported
 *  @adapter: board private structure to initialize
 *  @msix: boolean value of MSIX capability
1098
 *
1099 1100
 *  Attempt to configure interrupts using the best available
 *  capabilities of the hardware and kernel.
1101
 **/
1102
static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix)
1103 1104 1105 1106
{
	int err;
	int numvecs, i;

1107 1108
	if (!msix)
		goto msi_only;
1109
	adapter->flags |= IGB_FLAG_HAS_MSIX;
1110

1111
	/* Number of supported queues. */
1112
	adapter->num_rx_queues = adapter->rss_queues;
1113 1114 1115 1116
	if (adapter->vfs_allocated_count)
		adapter->num_tx_queues = 1;
	else
		adapter->num_tx_queues = adapter->rss_queues;
1117

1118
	/* start with one vector for every Rx queue */
1119 1120
	numvecs = adapter->num_rx_queues;

1121
	/* if Tx handler is separate add 1 for every Tx queue */
1122 1123
	if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
		numvecs += adapter->num_tx_queues;
1124 1125 1126 1127 1128 1129

	/* store the number of vectors reserved for queues */
	adapter->num_q_vectors = numvecs;

	/* add 1 vector for link status interrupts */
	numvecs++;
1130 1131 1132
	for (i = 0; i < numvecs; i++)
		adapter->msix_entries[i].entry = i;

1133 1134 1135 1136 1137
	err = pci_enable_msix_range(adapter->pdev,
				    adapter->msix_entries,
				    numvecs,
				    numvecs);
	if (err > 0)
1138
		return;
1139 1140 1141 1142 1143

	igb_reset_interrupt_capability(adapter);

	/* If we can't do MSI-X, try MSI */
msi_only:
1144
	adapter->flags &= ~IGB_FLAG_HAS_MSIX;
1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155
#ifdef CONFIG_PCI_IOV
	/* disable SR-IOV for non MSI-X configurations */
	if (adapter->vf_data) {
		struct e1000_hw *hw = &adapter->hw;
		/* disable iov and allow time for transactions to clear */
		pci_disable_sriov(adapter->pdev);
		msleep(500);

		kfree(adapter->vf_data);
		adapter->vf_data = NULL;
		wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1156
		wrfl();
1157 1158 1159 1160
		msleep(100);
		dev_info(&adapter->pdev->dev, "IOV Disabled\n");
	}
#endif
1161
	adapter->vfs_allocated_count = 0;
1162
	adapter->rss_queues = 1;
1163
	adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1164
	adapter->num_rx_queues = 1;
1165
	adapter->num_tx_queues = 1;
1166
	adapter->num_q_vectors = 1;
1167
	if (!pci_enable_msi(adapter->pdev))
1168
		adapter->flags |= IGB_FLAG_HAS_MSI;
1169 1170
}

1171 1172 1173 1174 1175 1176 1177
static void igb_add_ring(struct igb_ring *ring,
			 struct igb_ring_container *head)
{
	head->ring = ring;
	head->count++;
}

1178
/**
1179 1180 1181 1182 1183 1184 1185 1186
 *  igb_alloc_q_vector - Allocate memory for a single interrupt vector
 *  @adapter: board private structure to initialize
 *  @v_count: q_vectors allocated on adapter, used for ring interleaving
 *  @v_idx: index of vector in adapter struct
 *  @txr_count: total number of Tx rings to allocate
 *  @txr_idx: index of first Tx ring to allocate
 *  @rxr_count: total number of Rx rings to allocate
 *  @rxr_idx: index of first Rx ring to allocate
1187
 *
1188
 *  We allocate one q_vector.  If allocation fails we return -ENOMEM.
1189
 **/
1190 1191 1192 1193
static int igb_alloc_q_vector(struct igb_adapter *adapter,
			      int v_count, int v_idx,
			      int txr_count, int txr_idx,
			      int rxr_count, int rxr_idx)
1194 1195
{
	struct igb_q_vector *q_vector;
1196 1197
	struct igb_ring *ring;
	int ring_count, size;
1198

1199 1200 1201 1202 1203 1204 1205 1206 1207
	/* igb only supports 1 Tx and/or 1 Rx queue per vector */
	if (txr_count > 1 || rxr_count > 1)
		return -ENOMEM;

	ring_count = txr_count + rxr_count;
	size = sizeof(struct igb_q_vector) +
	       (sizeof(struct igb_ring) * ring_count);

	/* allocate q_vector and rings */
1208 1209 1210
	q_vector = adapter->q_vector[v_idx];
	if (!q_vector)
		q_vector = kzalloc(size, GFP_KERNEL);
1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231
	if (!q_vector)
		return -ENOMEM;

	/* initialize NAPI */
	netif_napi_add(adapter->netdev, &q_vector->napi,
		       igb_poll, 64);

	/* tie q_vector and adapter together */
	adapter->q_vector[v_idx] = q_vector;
	q_vector->adapter = adapter;

	/* initialize work limits */
	q_vector->tx.work_limit = adapter->tx_work_limit;

	/* initialize ITR configuration */
	q_vector->itr_register = adapter->hw.hw_addr + E1000_EITR(0);
	q_vector->itr_val = IGB_START_ITR;

	/* initialize pointer to rings */
	ring = q_vector->ring;

1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242
	/* intialize ITR */
	if (rxr_count) {
		/* rx or rx/tx vector */
		if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
			q_vector->itr_val = adapter->rx_itr_setting;
	} else {
		/* tx only vector */
		if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
			q_vector->itr_val = adapter->tx_itr_setting;
	}

1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261
	if (txr_count) {
		/* assign generic ring traits */
		ring->dev = &adapter->pdev->dev;
		ring->netdev = adapter->netdev;

		/* configure backlink on ring */
		ring->q_vector = q_vector;

		/* update q_vector Tx values */
		igb_add_ring(ring, &q_vector->tx);

		/* For 82575, context index must be unique per ring. */
		if (adapter->hw.mac.type == e1000_82575)
			set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);

		/* apply Tx specific ring traits */
		ring->count = adapter->tx_ring_count;
		ring->queue_index = txr_idx;

1262 1263 1264
		u64_stats_init(&ring->tx_syncp);
		u64_stats_init(&ring->tx_syncp2);

1265 1266 1267 1268 1269
		/* assign ring to adapter */
		adapter->tx_ring[txr_idx] = ring;

		/* push pointer to next ring */
		ring++;
1270
	}
1271

1272 1273 1274 1275
	if (rxr_count) {
		/* assign generic ring traits */
		ring->dev = &adapter->pdev->dev;
		ring->netdev = adapter->netdev;
1276

1277 1278
		/* configure backlink on ring */
		ring->q_vector = q_vector;
1279

1280 1281
		/* update q_vector Rx values */
		igb_add_ring(ring, &q_vector->rx);
1282

1283 1284 1285
		/* set flag indicating ring supports SCTP checksum offload */
		if (adapter->hw.mac.type >= e1000_82576)
			set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
1286

1287
		/* On i350, i354, i210, and i211, loopback VLAN packets
1288
		 * have the tag byte-swapped.
1289
		 */
1290 1291
		if (adapter->hw.mac.type >= e1000_i350)
			set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
1292

1293 1294 1295 1296
		/* apply Rx specific ring traits */
		ring->count = adapter->rx_ring_count;
		ring->queue_index = rxr_idx;

1297 1298
		u64_stats_init(&ring->rx_syncp);

1299 1300 1301 1302 1303
		/* assign ring to adapter */
		adapter->rx_ring[rxr_idx] = ring;
	}

	return 0;
1304 1305
}

1306

1307
/**
1308 1309
 *  igb_alloc_q_vectors - Allocate memory for interrupt vectors
 *  @adapter: board private structure to initialize
1310
 *
1311 1312
 *  We allocate one q_vector per queue interrupt.  If allocation fails we
 *  return -ENOMEM.
1313
 **/
1314
static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1315
{
1316 1317 1318 1319 1320
	int q_vectors = adapter->num_q_vectors;
	int rxr_remaining = adapter->num_rx_queues;
	int txr_remaining = adapter->num_tx_queues;
	int rxr_idx = 0, txr_idx = 0, v_idx = 0;
	int err;
1321

1322 1323 1324 1325
	if (q_vectors >= (rxr_remaining + txr_remaining)) {
		for (; rxr_remaining; v_idx++) {
			err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
						 0, 0, 1, rxr_idx);
1326

1327 1328 1329 1330 1331 1332
			if (err)
				goto err_out;

			/* update counts and index */
			rxr_remaining--;
			rxr_idx++;
1333 1334
		}
	}
1335 1336 1337 1338

	for (; v_idx < q_vectors; v_idx++) {
		int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
		int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1339

1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352
		err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
					 tqpv, txr_idx, rqpv, rxr_idx);

		if (err)
			goto err_out;

		/* update counts and index */
		rxr_remaining -= rqpv;
		txr_remaining -= tqpv;
		rxr_idx++;
		txr_idx++;
	}

1353
	return 0;
1354 1355 1356 1357 1358 1359 1360 1361 1362 1363

err_out:
	adapter->num_tx_queues = 0;
	adapter->num_rx_queues = 0;
	adapter->num_q_vectors = 0;

	while (v_idx--)
		igb_free_q_vector(adapter, v_idx);

	return -ENOMEM;
1364 1365 1366
}

/**
1367 1368 1369
 *  igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
 *  @adapter: board private structure to initialize
 *  @msix: boolean value of MSIX capability
1370
 *
1371
 *  This function initializes the interrupts and allocates all of the queues.
1372
 **/
1373
static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix)
1374 1375 1376 1377
{
	struct pci_dev *pdev = adapter->pdev;
	int err;

1378
	igb_set_interrupt_capability(adapter, msix);
1379 1380 1381 1382 1383 1384 1385

	err = igb_alloc_q_vectors(adapter);
	if (err) {
		dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
		goto err_alloc_q_vectors;
	}

1386
	igb_cache_ring_register(adapter);
1387 1388

	return 0;
1389

1390 1391 1392 1393 1394
err_alloc_q_vectors:
	igb_reset_interrupt_capability(adapter);
	return err;
}

1395
/**
1396 1397
 *  igb_request_irq - initialize interrupts
 *  @adapter: board private structure to initialize
1398
 *
1399 1400
 *  Attempts to configure interrupts using the best available
 *  capabilities of the hardware and kernel.
1401 1402 1403 1404
 **/
static int igb_request_irq(struct igb_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
1405
	struct pci_dev *pdev = adapter->pdev;
1406 1407
	int err = 0;

1408
	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1409
		err = igb_request_msix(adapter);
P
PJ Waskiewicz 已提交
1410
		if (!err)
1411 1412
			goto request_done;
		/* fall back to MSI */
1413 1414
		igb_free_all_tx_resources(adapter);
		igb_free_all_rx_resources(adapter);
1415

1416
		igb_clear_interrupt_scheme(adapter);
1417 1418
		err = igb_init_interrupt_scheme(adapter, false);
		if (err)
1419
			goto request_done;
1420

1421 1422
		igb_setup_all_tx_resources(adapter);
		igb_setup_all_rx_resources(adapter);
1423
		igb_configure(adapter);
1424
	}
P
PJ Waskiewicz 已提交
1425

1426 1427
	igb_assign_vector(adapter->q_vector[0], 0);

1428
	if (adapter->flags & IGB_FLAG_HAS_MSI) {
1429
		err = request_irq(pdev->irq, igb_intr_msi, 0,
1430
				  netdev->name, adapter);
1431 1432
		if (!err)
			goto request_done;
1433

1434 1435
		/* fall back to legacy interrupts */
		igb_reset_interrupt_capability(adapter);
1436
		adapter->flags &= ~IGB_FLAG_HAS_MSI;
1437 1438
	}

1439
	err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
1440
			  netdev->name, adapter);
1441

A
Andy Gospodarek 已提交
1442
	if (err)
1443
		dev_err(&pdev->dev, "Error %d getting interrupt\n",
1444 1445 1446 1447 1448 1449 1450 1451
			err);

request_done:
	return err;
}

static void igb_free_irq(struct igb_adapter *adapter)
{
1452
	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1453 1454
		int vector = 0, i;

1455
		free_irq(adapter->msix_entries[vector++].vector, adapter);
1456

1457
		for (i = 0; i < adapter->num_q_vectors; i++)
1458
			free_irq(adapter->msix_entries[vector++].vector,
1459
				 adapter->q_vector[i]);
1460 1461
	} else {
		free_irq(adapter->pdev->irq, adapter);
1462 1463 1464 1465
	}
}

/**
1466 1467
 *  igb_irq_disable - Mask off interrupt generation on the NIC
 *  @adapter: board private structure
1468 1469 1470 1471 1472
 **/
static void igb_irq_disable(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;

1473
	/* we need to be careful when disabling interrupts.  The VFs are also
1474 1475 1476
	 * mapped into these registers and so clearing the bits can cause
	 * issues on the VF drivers so we only need to clear what we set
	 */
1477
	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1478
		u32 regval = rd32(E1000_EIAM);
1479

1480 1481 1482 1483
		wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
		wr32(E1000_EIMC, adapter->eims_enable_mask);
		regval = rd32(E1000_EIAC);
		wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1484
	}
P
PJ Waskiewicz 已提交
1485 1486

	wr32(E1000_IAM, 0);
1487 1488
	wr32(E1000_IMC, ~0);
	wrfl();
1489
	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1490
		int i;
1491

1492 1493 1494 1495 1496
		for (i = 0; i < adapter->num_q_vectors; i++)
			synchronize_irq(adapter->msix_entries[i].vector);
	} else {
		synchronize_irq(adapter->pdev->irq);
	}
1497 1498 1499
}

/**
1500 1501
 *  igb_irq_enable - Enable default interrupt generation settings
 *  @adapter: board private structure
1502 1503 1504 1505 1506
 **/
static void igb_irq_enable(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;

1507
	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1508
		u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1509
		u32 regval = rd32(E1000_EIAC);
1510

1511 1512 1513
		wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
		regval = rd32(E1000_EIAM);
		wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
P
PJ Waskiewicz 已提交
1514
		wr32(E1000_EIMS, adapter->eims_enable_mask);
1515
		if (adapter->vfs_allocated_count) {
1516
			wr32(E1000_MBVFIMR, 0xFF);
1517 1518 1519
			ims |= E1000_IMS_VMMB;
		}
		wr32(E1000_IMS, ims);
P
PJ Waskiewicz 已提交
1520
	} else {
1521 1522 1523 1524
		wr32(E1000_IMS, IMS_ENABLE_MASK |
				E1000_IMS_DRSTA);
		wr32(E1000_IAM, IMS_ENABLE_MASK |
				E1000_IMS_DRSTA);
P
PJ Waskiewicz 已提交
1525
	}
1526 1527 1528 1529
}

static void igb_update_mng_vlan(struct igb_adapter *adapter)
{
1530
	struct e1000_hw *hw = &adapter->hw;
1531 1532
	u16 vid = adapter->hw.mng_cookie.vlan_id;
	u16 old_vid = adapter->mng_vlan_id;
1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543

	if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
		/* add VID to filter table */
		igb_vfta_set(hw, vid, true);
		adapter->mng_vlan_id = vid;
	} else {
		adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
	}

	if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
	    (vid != old_vid) &&
J
Jiri Pirko 已提交
1544
	    !test_bit(old_vid, adapter->active_vlans)) {
1545 1546
		/* remove VID from filter table */
		igb_vfta_set(hw, old_vid, false);
1547 1548 1549 1550
	}
}

/**
1551 1552
 *  igb_release_hw_control - release control of the h/w to f/w
 *  @adapter: address of board private structure
1553
 *
1554 1555 1556
 *  igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
 *  For ASF and Pass Through versions of f/w this means that the
 *  driver is no longer loaded.
1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569
 **/
static void igb_release_hw_control(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 ctrl_ext;

	/* Let firmware take over control of h/w */
	ctrl_ext = rd32(E1000_CTRL_EXT);
	wr32(E1000_CTRL_EXT,
			ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
}

/**
1570 1571
 *  igb_get_hw_control - get control of the h/w from f/w
 *  @adapter: address of board private structure
1572
 *
1573 1574 1575
 *  igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
 *  For ASF and Pass Through versions of f/w this means that
 *  the driver is loaded.
1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588
 **/
static void igb_get_hw_control(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 ctrl_ext;

	/* Let firmware know the driver has taken over */
	ctrl_ext = rd32(E1000_CTRL_EXT);
	wr32(E1000_CTRL_EXT,
			ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
}

/**
1589 1590
 *  igb_configure - configure the hardware for RX and TX
 *  @adapter: private board structure
1591 1592 1593 1594 1595 1596 1597
 **/
static void igb_configure(struct igb_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
	int i;

	igb_get_hw_control(adapter);
1598
	igb_set_rx_mode(netdev);
1599 1600 1601

	igb_restore_vlan(adapter);

1602
	igb_setup_tctl(adapter);
1603
	igb_setup_mrqc(adapter);
1604
	igb_setup_rctl(adapter);
1605 1606

	igb_configure_tx(adapter);
1607
	igb_configure_rx(adapter);
1608 1609 1610

	igb_rx_fifo_flush_82575(&adapter->hw);

1611
	/* call igb_desc_unused which always leaves
1612
	 * at least 1 descriptor unused to make sure
1613 1614
	 * next_to_use != next_to_clean
	 */
1615
	for (i = 0; i < adapter->num_rx_queues; i++) {
1616
		struct igb_ring *ring = adapter->rx_ring[i];
1617
		igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
1618 1619 1620
	}
}

1621
/**
1622 1623
 *  igb_power_up_link - Power up the phy/serdes link
 *  @adapter: address of board private structure
1624 1625 1626
 **/
void igb_power_up_link(struct igb_adapter *adapter)
{
1627 1628
	igb_reset_phy(&adapter->hw);

1629 1630 1631 1632 1633 1634 1635
	if (adapter->hw.phy.media_type == e1000_media_type_copper)
		igb_power_up_phy_copper(&adapter->hw);
	else
		igb_power_up_serdes_link_82575(&adapter->hw);
}

/**
1636 1637
 *  igb_power_down_link - Power down the phy/serdes link
 *  @adapter: address of board private structure
1638 1639 1640 1641 1642 1643 1644 1645
 */
static void igb_power_down_link(struct igb_adapter *adapter)
{
	if (adapter->hw.phy.media_type == e1000_media_type_copper)
		igb_power_down_phy_copper_82575(&adapter->hw);
	else
		igb_shutdown_serdes_link_82575(&adapter->hw);
}
1646

1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713
/**
 * Detect and switch function for Media Auto Sense
 * @adapter: address of the board private structure
 **/
static void igb_check_swap_media(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 ctrl_ext, connsw;
	bool swap_now = false;

	ctrl_ext = rd32(E1000_CTRL_EXT);
	connsw = rd32(E1000_CONNSW);

	/* need to live swap if current media is copper and we have fiber/serdes
	 * to go to.
	 */

	if ((hw->phy.media_type == e1000_media_type_copper) &&
	    (!(connsw & E1000_CONNSW_AUTOSENSE_EN))) {
		swap_now = true;
	} else if (!(connsw & E1000_CONNSW_SERDESD)) {
		/* copper signal takes time to appear */
		if (adapter->copper_tries < 4) {
			adapter->copper_tries++;
			connsw |= E1000_CONNSW_AUTOSENSE_CONF;
			wr32(E1000_CONNSW, connsw);
			return;
		} else {
			adapter->copper_tries = 0;
			if ((connsw & E1000_CONNSW_PHYSD) &&
			    (!(connsw & E1000_CONNSW_PHY_PDN))) {
				swap_now = true;
				connsw &= ~E1000_CONNSW_AUTOSENSE_CONF;
				wr32(E1000_CONNSW, connsw);
			}
		}
	}

	if (!swap_now)
		return;

	switch (hw->phy.media_type) {
	case e1000_media_type_copper:
		netdev_info(adapter->netdev,
			"MAS: changing media to fiber/serdes\n");
		ctrl_ext |=
			E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
		adapter->flags |= IGB_FLAG_MEDIA_RESET;
		adapter->copper_tries = 0;
		break;
	case e1000_media_type_internal_serdes:
	case e1000_media_type_fiber:
		netdev_info(adapter->netdev,
			"MAS: changing media to copper\n");
		ctrl_ext &=
			~E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
		adapter->flags |= IGB_FLAG_MEDIA_RESET;
		break;
	default:
		/* shouldn't get here during regular operation */
		netdev_err(adapter->netdev,
			"AMS: Invalid media type found, returning\n");
		break;
	}
	wr32(E1000_CTRL_EXT, ctrl_ext);
}

1714
/**
1715 1716
 *  igb_up - Open the interface and prepare it to handle traffic
 *  @adapter: board private structure
1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727
 **/
int igb_up(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	int i;

	/* hardware has been reset, we need to reload some things */
	igb_configure(adapter);

	clear_bit(__IGB_DOWN, &adapter->state);

1728 1729 1730
	for (i = 0; i < adapter->num_q_vectors; i++)
		napi_enable(&(adapter->q_vector[i]->napi));

1731
	if (adapter->flags & IGB_FLAG_HAS_MSIX)
1732
		igb_configure_msix(adapter);
1733 1734
	else
		igb_assign_vector(adapter->q_vector[0], 0);
1735 1736 1737 1738 1739

	/* Clear any pending interrupts. */
	rd32(E1000_ICR);
	igb_irq_enable(adapter);

1740 1741 1742
	/* notify VFs that reset has been completed */
	if (adapter->vfs_allocated_count) {
		u32 reg_data = rd32(E1000_CTRL_EXT);
1743

1744 1745 1746 1747
		reg_data |= E1000_CTRL_EXT_PFRSTD;
		wr32(E1000_CTRL_EXT, reg_data);
	}

1748 1749
	netif_tx_start_all_queues(adapter->netdev);

1750 1751 1752 1753
	/* start the watchdog. */
	hw->mac.get_link_status = 1;
	schedule_work(&adapter->watchdog_task);

1754 1755 1756 1757
	if ((adapter->flags & IGB_FLAG_EEE) &&
	    (!hw->dev_spec._82575.eee_disable))
		adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T;

1758 1759 1760 1761 1762 1763
	return 0;
}

void igb_down(struct igb_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
1764
	struct e1000_hw *hw = &adapter->hw;
1765 1766 1767 1768
	u32 tctl, rctl;
	int i;

	/* signal that we're down so the interrupt handler does not
1769 1770
	 * reschedule our watchdog timer
	 */
1771 1772 1773 1774 1775 1776 1777
	set_bit(__IGB_DOWN, &adapter->state);

	/* disable receives in the hardware */
	rctl = rd32(E1000_RCTL);
	wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
	/* flush and sleep below */

1778
	netif_tx_stop_all_queues(netdev);
1779 1780 1781 1782 1783 1784 1785

	/* disable transmits in the hardware */
	tctl = rd32(E1000_TCTL);
	tctl &= ~E1000_TCTL_EN;
	wr32(E1000_TCTL, tctl);
	/* flush both disables and wait for them to finish */
	wrfl();
1786
	usleep_range(10000, 11000);
1787

1788 1789
	igb_irq_disable(adapter);

1790 1791
	adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;

1792 1793
	for (i = 0; i < adapter->num_q_vectors; i++) {
		napi_synchronize(&(adapter->q_vector[i]->napi));
1794
		napi_disable(&(adapter->q_vector[i]->napi));
1795
	}
1796 1797 1798 1799 1800 1801


	del_timer_sync(&adapter->watchdog_timer);
	del_timer_sync(&adapter->phy_info_timer);

	netif_carrier_off(netdev);
1802 1803

	/* record the stats before reset*/
E
Eric Dumazet 已提交
1804 1805 1806
	spin_lock(&adapter->stats64_lock);
	igb_update_stats(adapter, &adapter->stats64);
	spin_unlock(&adapter->stats64_lock);
1807

1808 1809 1810
	adapter->link_speed = 0;
	adapter->link_duplex = 0;

1811 1812
	if (!pci_channel_offline(adapter->pdev))
		igb_reset(adapter);
1813 1814
	igb_clean_all_tx_rings(adapter);
	igb_clean_all_rx_rings(adapter);
1815 1816 1817 1818 1819
#ifdef CONFIG_IGB_DCA

	/* since we reset the hardware DCA settings were cleared */
	igb_setup_dca(adapter);
#endif
1820 1821 1822 1823 1824 1825
}

void igb_reinit_locked(struct igb_adapter *adapter)
{
	WARN_ON(in_interrupt());
	while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1826
		usleep_range(1000, 2000);
1827 1828 1829 1830 1831
	igb_down(adapter);
	igb_up(adapter);
	clear_bit(__IGB_RESETTING, &adapter->state);
}

1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862
/** igb_enable_mas - Media Autosense re-enable after swap
 *
 * @adapter: adapter struct
 **/
static s32 igb_enable_mas(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 connsw;
	s32 ret_val = 0;

	connsw = rd32(E1000_CONNSW);
	if (!(hw->phy.media_type == e1000_media_type_copper))
		return ret_val;

	/* configure for SerDes media detect */
	if (!(connsw & E1000_CONNSW_SERDESD)) {
		connsw |= E1000_CONNSW_ENRGSRC;
		connsw |= E1000_CONNSW_AUTOSENSE_EN;
		wr32(E1000_CONNSW, connsw);
		wrfl();
	} else if (connsw & E1000_CONNSW_SERDESD) {
		/* already SerDes, no need to enable anything */
		return ret_val;
	} else {
		netdev_info(adapter->netdev,
			"MAS: Unable to configure feature, disabling..\n");
		adapter->flags &= ~IGB_FLAG_MAS_ENABLE;
	}
	return ret_val;
}

1863 1864
void igb_reset(struct igb_adapter *adapter)
{
1865
	struct pci_dev *pdev = adapter->pdev;
1866
	struct e1000_hw *hw = &adapter->hw;
A
Alexander Duyck 已提交
1867 1868
	struct e1000_mac_info *mac = &hw->mac;
	struct e1000_fc_info *fc = &hw->fc;
1869
	u32 pba = 0, tx_space, min_tx_space, min_rx_space, hwm;
1870 1871 1872 1873

	/* Repartition Pba for greater than 9k mtu
	 * To take effect CTRL.RST is required.
	 */
1874
	switch (mac->type) {
1875
	case e1000_i350:
1876
	case e1000_i354:
1877 1878 1879 1880
	case e1000_82580:
		pba = rd32(E1000_RXPBS);
		pba = igb_rxpbs_adjust_82580(pba);
		break;
1881
	case e1000_82576:
1882 1883
		pba = rd32(E1000_RXPBS);
		pba &= E1000_RXPBS_SIZE_MASK_82576;
1884 1885
		break;
	case e1000_82575:
1886 1887
	case e1000_i210:
	case e1000_i211:
1888 1889 1890
	default:
		pba = E1000_PBA_34K;
		break;
A
Alexander Duyck 已提交
1891
	}
1892

A
Alexander Duyck 已提交
1893 1894
	if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
	    (mac->type < e1000_82576)) {
1895 1896 1897 1898 1899 1900 1901 1902
		/* adjust PBA for jumbo frames */
		wr32(E1000_PBA, pba);

		/* To maintain wire speed transmits, the Tx FIFO should be
		 * large enough to accommodate two full transmit packets,
		 * rounded up to the next 1KB and expressed in KB.  Likewise,
		 * the Rx FIFO should be large enough to accommodate at least
		 * one full receive packet and is similarly rounded up and
1903 1904
		 * expressed in KB.
		 */
1905 1906 1907 1908 1909
		pba = rd32(E1000_PBA);
		/* upper 16 bits has Tx packet buffer allocation size in KB */
		tx_space = pba >> 16;
		/* lower 16 bits has Rx packet buffer allocation size in KB */
		pba &= 0xffff;
1910 1911 1912
		/* the Tx fifo also stores 16 bytes of information about the Tx
		 * but don't include ethernet FCS because hardware appends it
		 */
1913
		min_tx_space = (adapter->max_frame_size +
1914
				sizeof(union e1000_adv_tx_desc) -
1915 1916 1917 1918 1919 1920 1921 1922 1923 1924
				ETH_FCS_LEN) * 2;
		min_tx_space = ALIGN(min_tx_space, 1024);
		min_tx_space >>= 10;
		/* software strips receive CRC, so leave room for it */
		min_rx_space = adapter->max_frame_size;
		min_rx_space = ALIGN(min_rx_space, 1024);
		min_rx_space >>= 10;

		/* If current Tx allocation is less than the min Tx FIFO size,
		 * and the min Tx FIFO size is less than the current Rx FIFO
1925 1926
		 * allocation, take space away from current Rx allocation
		 */
1927 1928 1929 1930
		if (tx_space < min_tx_space &&
		    ((min_tx_space - tx_space) < pba)) {
			pba = pba - (min_tx_space - tx_space);

1931 1932 1933
			/* if short on Rx space, Rx wins and must trump Tx
			 * adjustment
			 */
1934 1935 1936
			if (pba < min_rx_space)
				pba = min_rx_space;
		}
A
Alexander Duyck 已提交
1937
		wr32(E1000_PBA, pba);
1938 1939 1940 1941 1942 1943 1944
	}

	/* flow control settings */
	/* The high water mark must be low enough to fit one full frame
	 * (or the size used for early receive) above it in the Rx FIFO.
	 * Set it to the lower of:
	 * - 90% of the Rx FIFO size, or
1945 1946
	 * - the full Rx FIFO size minus one full frame
	 */
1947
	hwm = min(((pba << 10) * 9 / 10),
A
Alexander Duyck 已提交
1948
			((pba << 10) - 2 * adapter->max_frame_size));
1949

1950
	fc->high_water = hwm & 0xFFFFFFF0;	/* 16-byte granularity */
1951
	fc->low_water = fc->high_water - 16;
1952 1953
	fc->pause_time = 0xFFFF;
	fc->send_xon = 1;
1954
	fc->current_mode = fc->requested_mode;
1955

1956 1957 1958
	/* disable receive for all VFs and wait one second */
	if (adapter->vfs_allocated_count) {
		int i;
1959

1960
		for (i = 0 ; i < adapter->vfs_allocated_count; i++)
G
Greg Rose 已提交
1961
			adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1962 1963

		/* ping all the active vfs to let them know we are going down */
1964
		igb_ping_all_vfs(adapter);
1965 1966 1967 1968 1969 1970

		/* disable transmits and receives */
		wr32(E1000_VFRE, 0);
		wr32(E1000_VFTE, 0);
	}

1971
	/* Allow time for pending master requests to run */
1972
	hw->mac.ops.reset_hw(hw);
1973 1974
	wr32(E1000_WUC, 0);

1975 1976 1977 1978 1979 1980 1981 1982 1983 1984
	if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
		/* need to resetup here after media swap */
		adapter->ei.get_invariants(hw);
		adapter->flags &= ~IGB_FLAG_MEDIA_RESET;
	}
	if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
		if (igb_enable_mas(adapter))
			dev_err(&pdev->dev,
				"Error enabling Media Auto Sense\n");
	}
1985
	if (hw->mac.ops.init_hw(hw))
1986
		dev_err(&pdev->dev, "Hardware Error\n");
1987

1988
	/* Flow control settings reset on hardware reset, so guarantee flow
1989 1990 1991 1992 1993
	 * control is off when forcing speed.
	 */
	if (!hw->mac.autoneg)
		igb_force_mac_fc(hw);

1994
	igb_init_dmac(adapter, pba);
1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006
#ifdef CONFIG_IGB_HWMON
	/* Re-initialize the thermal sensor on i350 devices. */
	if (!test_bit(__IGB_DOWN, &adapter->state)) {
		if (mac->type == e1000_i350 && hw->bus.func == 0) {
			/* If present, re-initialize the external thermal sensor
			 * interface.
			 */
			if (adapter->ets)
				mac->ops.init_thermal_sensor_thresh(hw);
		}
	}
#endif
J
Jeff Kirsher 已提交
2007
	/* Re-establish EEE setting */
2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021
	if (hw->phy.media_type == e1000_media_type_copper) {
		switch (mac->type) {
		case e1000_i350:
		case e1000_i210:
		case e1000_i211:
			igb_set_eee_i350(hw);
			break;
		case e1000_i354:
			igb_set_eee_i354(hw);
			break;
		default:
			break;
		}
	}
2022 2023 2024
	if (!netif_running(adapter->netdev))
		igb_power_down_link(adapter);

2025 2026 2027 2028 2029
	igb_update_mng_vlan(adapter);

	/* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
	wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);

2030 2031 2032
	/* Re-enable PTP, where applicable. */
	igb_ptp_reset(adapter);

2033
	igb_get_phy_info(hw);
2034 2035
}

2036 2037
static netdev_features_t igb_fix_features(struct net_device *netdev,
	netdev_features_t features)
J
Jiri Pirko 已提交
2038
{
2039 2040
	/* Since there is no support for separate Rx/Tx vlan accel
	 * enable/disable make sure Tx flag is always in same state as Rx.
J
Jiri Pirko 已提交
2041
	 */
2042 2043
	if (features & NETIF_F_HW_VLAN_CTAG_RX)
		features |= NETIF_F_HW_VLAN_CTAG_TX;
J
Jiri Pirko 已提交
2044
	else
2045
		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
J
Jiri Pirko 已提交
2046 2047 2048 2049

	return features;
}

2050 2051
static int igb_set_features(struct net_device *netdev,
	netdev_features_t features)
2052
{
2053
	netdev_features_t changed = netdev->features ^ features;
B
Ben Greear 已提交
2054
	struct igb_adapter *adapter = netdev_priv(netdev);
2055

2056
	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
J
Jiri Pirko 已提交
2057 2058
		igb_vlan_mode(netdev, features);

B
Ben Greear 已提交
2059 2060 2061 2062 2063 2064 2065 2066 2067 2068
	if (!(changed & NETIF_F_RXALL))
		return 0;

	netdev->features = features;

	if (netif_running(netdev))
		igb_reinit_locked(adapter);
	else
		igb_reset(adapter);

2069 2070 2071
	return 0;
}

S
Stephen Hemminger 已提交
2072
static const struct net_device_ops igb_netdev_ops = {
2073
	.ndo_open		= igb_open,
S
Stephen Hemminger 已提交
2074
	.ndo_stop		= igb_close,
2075
	.ndo_start_xmit		= igb_xmit_frame,
E
Eric Dumazet 已提交
2076
	.ndo_get_stats64	= igb_get_stats64,
2077
	.ndo_set_rx_mode	= igb_set_rx_mode,
S
Stephen Hemminger 已提交
2078 2079 2080 2081 2082 2083 2084
	.ndo_set_mac_address	= igb_set_mac,
	.ndo_change_mtu		= igb_change_mtu,
	.ndo_do_ioctl		= igb_ioctl,
	.ndo_tx_timeout		= igb_tx_timeout,
	.ndo_validate_addr	= eth_validate_addr,
	.ndo_vlan_rx_add_vid	= igb_vlan_rx_add_vid,
	.ndo_vlan_rx_kill_vid	= igb_vlan_rx_kill_vid,
2085 2086
	.ndo_set_vf_mac		= igb_ndo_set_vf_mac,
	.ndo_set_vf_vlan	= igb_ndo_set_vf_vlan,
2087
	.ndo_set_vf_rate	= igb_ndo_set_vf_bw,
L
Lior Levy 已提交
2088
	.ndo_set_vf_spoofchk	= igb_ndo_set_vf_spoofchk,
2089
	.ndo_get_vf_config	= igb_ndo_get_vf_config,
S
Stephen Hemminger 已提交
2090 2091 2092
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller	= igb_netpoll,
#endif
J
Jiri Pirko 已提交
2093 2094
	.ndo_fix_features	= igb_fix_features,
	.ndo_set_features	= igb_set_features,
S
Stephen Hemminger 已提交
2095 2096
};

2097 2098 2099 2100 2101 2102 2103
/**
 * igb_set_fw_version - Configure version string for ethtool
 * @adapter: adapter struct
 **/
void igb_set_fw_version(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
2104 2105 2106 2107 2108
	struct e1000_fw_version fw;

	igb_get_fw_version(hw, &fw);

	switch (hw->mac.type) {
2109
	case e1000_i210:
2110
	case e1000_i211:
2111 2112 2113 2114 2115 2116 2117 2118 2119
		if (!(igb_get_flash_presence_i210(hw))) {
			snprintf(adapter->fw_version,
				 sizeof(adapter->fw_version),
				 "%2d.%2d-%d",
				 fw.invm_major, fw.invm_minor,
				 fw.invm_img_type);
			break;
		}
		/* fall through */
2120 2121 2122 2123 2124 2125 2126 2127 2128
	default:
		/* if option is rom valid, display its version too */
		if (fw.or_valid) {
			snprintf(adapter->fw_version,
				 sizeof(adapter->fw_version),
				 "%d.%d, 0x%08x, %d.%d.%d",
				 fw.eep_major, fw.eep_minor, fw.etrack_id,
				 fw.or_major, fw.or_build, fw.or_patch);
		/* no option rom */
2129
		} else if (fw.etrack_id != 0X0000) {
2130
			snprintf(adapter->fw_version,
2131 2132 2133 2134 2135 2136 2137 2138
			    sizeof(adapter->fw_version),
			    "%d.%d, 0x%08x",
			    fw.eep_major, fw.eep_minor, fw.etrack_id);
		} else {
		snprintf(adapter->fw_version,
		    sizeof(adapter->fw_version),
		    "%d.%d.%d",
		    fw.eep_major, fw.eep_minor, fw.eep_build);
2139 2140
		}
		break;
2141 2142 2143
	}
}

2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195
/**
 * igb_init_mas - init Media Autosense feature if enabled in the NVM
 *
 * @adapter: adapter struct
 **/
static void igb_init_mas(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u16 eeprom_data;

	hw->nvm.ops.read(hw, NVM_COMPAT, 1, &eeprom_data);
	switch (hw->bus.func) {
	case E1000_FUNC_0:
		if (eeprom_data & IGB_MAS_ENABLE_0) {
			adapter->flags |= IGB_FLAG_MAS_ENABLE;
			netdev_info(adapter->netdev,
				"MAS: Enabling Media Autosense for port %d\n",
				hw->bus.func);
		}
		break;
	case E1000_FUNC_1:
		if (eeprom_data & IGB_MAS_ENABLE_1) {
			adapter->flags |= IGB_FLAG_MAS_ENABLE;
			netdev_info(adapter->netdev,
				"MAS: Enabling Media Autosense for port %d\n",
				hw->bus.func);
		}
		break;
	case E1000_FUNC_2:
		if (eeprom_data & IGB_MAS_ENABLE_2) {
			adapter->flags |= IGB_FLAG_MAS_ENABLE;
			netdev_info(adapter->netdev,
				"MAS: Enabling Media Autosense for port %d\n",
				hw->bus.func);
		}
		break;
	case E1000_FUNC_3:
		if (eeprom_data & IGB_MAS_ENABLE_3) {
			adapter->flags |= IGB_FLAG_MAS_ENABLE;
			netdev_info(adapter->netdev,
				"MAS: Enabling Media Autosense for port %d\n",
				hw->bus.func);
		}
		break;
	default:
		/* Shouldn't get here */
		netdev_err(adapter->netdev,
			"MAS: Invalid port configuration, returning\n");
		break;
	}
}

2196 2197
/**
 *  igb_init_i2c - Init I2C interface
C
Carolyn Wyborny 已提交
2198
 *  @adapter: pointer to adapter structure
2199
 **/
C
Carolyn Wyborny 已提交
2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222
static s32 igb_init_i2c(struct igb_adapter *adapter)
{
	s32 status = E1000_SUCCESS;

	/* I2C interface supported on i350 devices */
	if (adapter->hw.mac.type != e1000_i350)
		return E1000_SUCCESS;

	/* Initialize the i2c bus which is controlled by the registers.
	 * This bus will use the i2c_algo_bit structue that implements
	 * the protocol through toggling of the 4 bits in the register.
	 */
	adapter->i2c_adap.owner = THIS_MODULE;
	adapter->i2c_algo = igb_i2c_algo;
	adapter->i2c_algo.data = adapter;
	adapter->i2c_adap.algo_data = &adapter->i2c_algo;
	adapter->i2c_adap.dev.parent = &adapter->pdev->dev;
	strlcpy(adapter->i2c_adap.name, "igb BB",
		sizeof(adapter->i2c_adap.name));
	status = i2c_bit_add_bus(&adapter->i2c_adap);
	return status;
}

2223
/**
2224 2225 2226
 *  igb_probe - Device Initialization Routine
 *  @pdev: PCI device information struct
 *  @ent: entry in igb_pci_tbl
2227
 *
2228
 *  Returns 0 on success, negative on failure
2229
 *
2230 2231 2232
 *  igb_probe initializes an adapter identified by a pci_dev structure.
 *  The OS initialization, configuring of the adapter private structure,
 *  and a hardware reset occur.
2233
 **/
2234
static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2235 2236 2237 2238
{
	struct net_device *netdev;
	struct igb_adapter *adapter;
	struct e1000_hw *hw;
2239
	u16 eeprom_data = 0;
2240
	s32 ret_val;
2241
	static int global_quad_port_a; /* global quad port a indication */
2242
	const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
2243
	int err, pci_using_dac;
2244
	u8 part_str[E1000_PBANUM_LENGTH];
2245

2246 2247 2248 2249 2250
	/* Catch broken hardware that put the wrong VF device ID in
	 * the PCIe SR-IOV capability.
	 */
	if (pdev->is_virtfn) {
		WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
2251
			pci_name(pdev), pdev->vendor, pdev->device);
2252 2253 2254
		return -EINVAL;
	}

2255
	err = pci_enable_device_mem(pdev);
2256 2257 2258 2259
	if (err)
		return err;

	pci_using_dac = 0;
2260
	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2261
	if (!err) {
2262
		pci_using_dac = 1;
2263
	} else {
2264
		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
2265
		if (err) {
2266 2267 2268
			dev_err(&pdev->dev,
				"No usable DMA configuration, aborting\n");
			goto err_dma;
2269 2270 2271
		}
	}

2272
	err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
2273 2274
					   IORESOURCE_MEM),
					   igb_driver_name);
2275 2276 2277
	if (err)
		goto err_pci_reg;

2278
	pci_enable_pcie_error_reporting(pdev);
2279

2280
	pci_set_master(pdev);
2281
	pci_save_state(pdev);
2282 2283

	err = -ENOMEM;
2284
	netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
2285
				   IGB_MAX_TX_QUEUES);
2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296
	if (!netdev)
		goto err_alloc_etherdev;

	SET_NETDEV_DEV(netdev, &pdev->dev);

	pci_set_drvdata(pdev, netdev);
	adapter = netdev_priv(netdev);
	adapter->netdev = netdev;
	adapter->pdev = pdev;
	hw = &adapter->hw;
	hw->back = adapter;
2297
	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
2298 2299

	err = -EIO;
2300
	hw->hw_addr = pci_iomap(pdev, 0, 0);
2301
	if (!hw->hw_addr)
2302 2303
		goto err_ioremap;

S
Stephen Hemminger 已提交
2304
	netdev->netdev_ops = &igb_netdev_ops;
2305 2306 2307 2308 2309
	igb_set_ethtool_ops(netdev);
	netdev->watchdog_timeo = 5 * HZ;

	strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);

2310 2311
	netdev->mem_start = pci_resource_start(pdev, 0);
	netdev->mem_end = pci_resource_end(pdev, 0);
2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326

	/* PCI config space info */
	hw->vendor_id = pdev->vendor;
	hw->device_id = pdev->device;
	hw->revision_id = pdev->revision;
	hw->subsystem_vendor_id = pdev->subsystem_vendor;
	hw->subsystem_device_id = pdev->subsystem_device;

	/* Copy the default MAC, PHY and NVM function pointers */
	memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
	memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
	memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
	/* Initialize skew-specific constants */
	err = ei->get_invariants(hw);
	if (err)
2327
		goto err_sw_init;
2328

2329
	/* setup the private structure */
2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348
	err = igb_sw_init(adapter);
	if (err)
		goto err_sw_init;

	igb_get_bus_info_pcie(hw);

	hw->phy.autoneg_wait_to_complete = false;

	/* Copper options */
	if (hw->phy.media_type == e1000_media_type_copper) {
		hw->phy.mdix = AUTO_ALL_MODES;
		hw->phy.disable_polarity_correction = false;
		hw->phy.ms_type = e1000_ms_hw_default;
	}

	if (igb_check_reset_block(hw))
		dev_info(&pdev->dev,
			"PHY reset is blocked due to SOL/IDER session.\n");

2349
	/* features is initialized to 0 in allocation, it might have bits
2350 2351 2352 2353 2354 2355 2356 2357 2358 2359
	 * set by igb_sw_init so we should use an or instead of an
	 * assignment.
	 */
	netdev->features |= NETIF_F_SG |
			    NETIF_F_IP_CSUM |
			    NETIF_F_IPV6_CSUM |
			    NETIF_F_TSO |
			    NETIF_F_TSO6 |
			    NETIF_F_RXHASH |
			    NETIF_F_RXCSUM |
2360 2361
			    NETIF_F_HW_VLAN_CTAG_RX |
			    NETIF_F_HW_VLAN_CTAG_TX;
2362 2363 2364

	/* copy netdev features into list of user selectable features */
	netdev->hw_features |= netdev->features;
B
Ben Greear 已提交
2365
	netdev->hw_features |= NETIF_F_RXALL;
2366 2367

	/* set this bit last since it cannot be part of hw_features */
2368
	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2369 2370 2371 2372 2373 2374

	netdev->vlan_features |= NETIF_F_TSO |
				 NETIF_F_TSO6 |
				 NETIF_F_IP_CSUM |
				 NETIF_F_IPV6_CSUM |
				 NETIF_F_SG;
2375

2376 2377
	netdev->priv_flags |= IFF_SUPP_NOFCS;

2378
	if (pci_using_dac) {
2379
		netdev->features |= NETIF_F_HIGHDMA;
2380 2381
		netdev->vlan_features |= NETIF_F_HIGHDMA;
	}
2382

2383 2384
	if (hw->mac.type >= e1000_82576) {
		netdev->hw_features |= NETIF_F_SCTP_CSUM;
2385
		netdev->features |= NETIF_F_SCTP_CSUM;
2386
	}
2387

2388 2389
	netdev->priv_flags |= IFF_UNICAST_FLT;

2390
	adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
2391 2392

	/* before reading the NVM, reset the controller to put the device in a
2393 2394
	 * known good starting state
	 */
2395 2396
	hw->mac.ops.reset_hw(hw);

2397 2398
	/* make sure the NVM is good , i211/i210 parts can have special NVM
	 * that doesn't contain a checksum
2399
	 */
2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412
	switch (hw->mac.type) {
	case e1000_i210:
	case e1000_i211:
		if (igb_get_flash_presence_i210(hw)) {
			if (hw->nvm.ops.validate(hw) < 0) {
				dev_err(&pdev->dev,
					"The NVM Checksum Is Not Valid\n");
				err = -EIO;
				goto err_eeprom;
			}
		}
		break;
	default:
2413 2414 2415 2416 2417
		if (hw->nvm.ops.validate(hw) < 0) {
			dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
			err = -EIO;
			goto err_eeprom;
		}
2418
		break;
2419 2420 2421 2422 2423 2424 2425 2426
	}

	/* copy the MAC address out of the NVM */
	if (hw->mac.ops.read_mac_addr(hw))
		dev_err(&pdev->dev, "NVM Read Error\n");

	memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);

2427
	if (!is_valid_ether_addr(netdev->dev_addr)) {
2428 2429 2430 2431 2432
		dev_err(&pdev->dev, "Invalid MAC Address\n");
		err = -EIO;
		goto err_eeprom;
	}

2433 2434 2435
	/* get firmware version for ethtool -i */
	igb_set_fw_version(adapter);

2436
	setup_timer(&adapter->watchdog_timer, igb_watchdog,
2437
		    (unsigned long) adapter);
2438
	setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
2439
		    (unsigned long) adapter);
2440 2441 2442 2443

	INIT_WORK(&adapter->reset_task, igb_reset_task);
	INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);

2444
	/* Initialize link properties that are user-changeable */
2445 2446 2447 2448
	adapter->fc_autoneg = true;
	hw->mac.autoneg = true;
	hw->phy.autoneg_advertised = 0x2f;

2449 2450
	hw->fc.requested_mode = e1000_fc_default;
	hw->fc.current_mode = e1000_fc_default;
2451 2452 2453

	igb_validate_mdi_setting(hw);

2454
	/* By default, support wake on port A */
2455
	if (hw->bus.func == 0)
2456 2457 2458 2459
		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;

	/* Check the NVM for wake support on non-port A ports */
	if (hw->mac.type >= e1000_82580)
2460
		hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2461 2462
				 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
				 &eeprom_data);
2463 2464
	else if (hw->bus.func == 1)
		hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2465

2466 2467
	if (eeprom_data & IGB_EEPROM_APME)
		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
2468 2469 2470

	/* now that we have the eeprom settings, apply the special cases where
	 * the eeprom may be wrong or the board simply won't support wake on
2471 2472
	 * lan on a particular port
	 */
2473 2474
	switch (pdev->device) {
	case E1000_DEV_ID_82575GB_QUAD_COPPER:
2475
		adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2476 2477
		break;
	case E1000_DEV_ID_82575EB_FIBER_SERDES:
A
Alexander Duyck 已提交
2478 2479
	case E1000_DEV_ID_82576_FIBER:
	case E1000_DEV_ID_82576_SERDES:
2480
		/* Wake events only supported on port A for dual fiber
2481 2482
		 * regardless of eeprom setting
		 */
2483
		if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2484
			adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2485
		break;
2486
	case E1000_DEV_ID_82576_QUAD_COPPER:
2487
	case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2488 2489
		/* if quad port adapter, disable WoL on all but port A */
		if (global_quad_port_a != 0)
2490
			adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2491 2492 2493 2494 2495 2496
		else
			adapter->flags |= IGB_FLAG_QUAD_PORT_A;
		/* Reset for multiple quad port adapters */
		if (++global_quad_port_a == 4)
			global_quad_port_a = 0;
		break;
2497 2498 2499 2500
	default:
		/* If the device can't wake, don't set software support */
		if (!device_can_wakeup(&adapter->pdev->dev))
			adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2501 2502 2503
	}

	/* initialize the wol settings based on the eeprom settings */
2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515
	if (adapter->flags & IGB_FLAG_WOL_SUPPORTED)
		adapter->wol |= E1000_WUFC_MAG;

	/* Some vendors want WoL disabled by default, but still supported */
	if ((hw->mac.type == e1000_i350) &&
	    (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) {
		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
		adapter->wol = 0;
	}

	device_set_wakeup_enable(&adapter->pdev->dev,
				 adapter->flags & IGB_FLAG_WOL_SUPPORTED);
2516 2517 2518 2519

	/* reset the hardware with the new settings */
	igb_reset(adapter);

C
Carolyn Wyborny 已提交
2520 2521 2522 2523 2524 2525 2526
	/* Init the I2C interface */
	err = igb_init_i2c(adapter);
	if (err) {
		dev_err(&pdev->dev, "failed to init i2c interface\n");
		goto err_eeprom;
	}

2527
	/* let the f/w know that the h/w is now under the control of the
2528 2529
	 * driver.
	 */
2530 2531 2532 2533 2534 2535 2536
	igb_get_hw_control(adapter);

	strcpy(netdev->name, "eth%d");
	err = register_netdev(netdev);
	if (err)
		goto err_register;

2537 2538 2539
	/* carrier off reporting is important to ethtool even BEFORE open */
	netif_carrier_off(netdev);

2540
#ifdef CONFIG_IGB_DCA
2541
	if (dca_add_requester(&pdev->dev) == 0) {
2542
		adapter->flags |= IGB_FLAG_DCA_ENABLED;
J
Jeb Cramer 已提交
2543 2544 2545 2546
		dev_info(&pdev->dev, "DCA enabled\n");
		igb_setup_dca(adapter);
	}

P
Patrick Ohly 已提交
2547
#endif
2548 2549 2550 2551
#ifdef CONFIG_IGB_HWMON
	/* Initialize the thermal sensor on i350 devices. */
	if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
		u16 ets_word;
2552

2553
		/* Read the NVM to determine if this i350 device supports an
2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567
		 * external thermal sensor.
		 */
		hw->nvm.ops.read(hw, NVM_ETS_CFG, 1, &ets_word);
		if (ets_word != 0x0000 && ets_word != 0xFFFF)
			adapter->ets = true;
		else
			adapter->ets = false;
		if (igb_sysfs_init(adapter))
			dev_err(&pdev->dev,
				"failed to allocate sysfs resources\n");
	} else {
		adapter->ets = false;
	}
#endif
2568 2569 2570 2571 2572
	/* Check if Media Autosense is enabled */
	adapter->ei = *ei;
	if (hw->dev_spec._82575.mas_capable)
		igb_init_mas(adapter);

A
Anders Berggren 已提交
2573
	/* do hw tstamp init after resetting */
2574
	igb_ptp_init(adapter);
A
Anders Berggren 已提交
2575

2576
	dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590
	/* print bus type/speed/width info, not applicable to i354 */
	if (hw->mac.type != e1000_i354) {
		dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
			 netdev->name,
			 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
			  (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
			   "unknown"),
			 ((hw->bus.width == e1000_bus_width_pcie_x4) ?
			  "Width x4" :
			  (hw->bus.width == e1000_bus_width_pcie_x2) ?
			  "Width x2" :
			  (hw->bus.width == e1000_bus_width_pcie_x1) ?
			  "Width x1" : "unknown"), netdev->dev_addr);
	}
2591

2592 2593 2594 2595 2596 2597 2598 2599
	if ((hw->mac.type >= e1000_i210 ||
	     igb_get_flash_presence_i210(hw))) {
		ret_val = igb_read_part_string(hw, part_str,
					       E1000_PBANUM_LENGTH);
	} else {
		ret_val = -E1000_ERR_INVM_VALUE_NOT_FOUND;
	}

2600 2601 2602
	if (ret_val)
		strcpy(part_str, "Unknown");
	dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
2603 2604
	dev_info(&pdev->dev,
		"Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2605
		(adapter->flags & IGB_FLAG_HAS_MSIX) ? "MSI-X" :
2606
		(adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2607
		adapter->num_rx_queues, adapter->num_tx_queues);
2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622
	if (hw->phy.media_type == e1000_media_type_copper) {
		switch (hw->mac.type) {
		case e1000_i350:
		case e1000_i210:
		case e1000_i211:
			/* Enable EEE for internal copper PHY devices */
			err = igb_set_eee_i350(hw);
			if ((!err) &&
			    (!hw->dev_spec._82575.eee_disable)) {
				adapter->eee_advert =
					MDIO_EEE_100TX | MDIO_EEE_1000T;
				adapter->flags |= IGB_FLAG_EEE;
			}
			break;
		case e1000_i354:
2623
			if ((rd32(E1000_CTRL_EXT) &
2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635
			    E1000_CTRL_EXT_LINK_MODE_SGMII)) {
				err = igb_set_eee_i354(hw);
				if ((!err) &&
					(!hw->dev_spec._82575.eee_disable)) {
					adapter->eee_advert =
					   MDIO_EEE_100TX | MDIO_EEE_1000T;
					adapter->flags |= IGB_FLAG_EEE;
				}
			}
			break;
		default:
			break;
2636
		}
2637
	}
Y
Yan, Zheng 已提交
2638
	pm_runtime_put_noidle(&pdev->dev);
2639 2640 2641 2642
	return 0;

err_register:
	igb_release_hw_control(adapter);
C
Carolyn Wyborny 已提交
2643
	memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap));
2644 2645
err_eeprom:
	if (!igb_check_reset_block(hw))
2646
		igb_reset_phy(hw);
2647 2648 2649 2650

	if (hw->flash_address)
		iounmap(hw->flash_address);
err_sw_init:
2651
	igb_clear_interrupt_scheme(adapter);
2652
	pci_iounmap(pdev, hw->hw_addr);
2653 2654 2655
err_ioremap:
	free_netdev(netdev);
err_alloc_etherdev:
2656
	pci_release_selected_regions(pdev,
2657
				     pci_select_bars(pdev, IORESOURCE_MEM));
2658 2659 2660 2661 2662 2663
err_pci_reg:
err_dma:
	pci_disable_device(pdev);
	return err;
}

2664
#ifdef CONFIG_PCI_IOV
2665
static int igb_disable_sriov(struct pci_dev *pdev)
2666 2667 2668 2669 2670 2671 2672 2673
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;

	/* reclaim resources allocated to VFs */
	if (adapter->vf_data) {
		/* disable iov and allow time for transactions to clear */
2674
		if (pci_vfs_assigned(pdev)) {
2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705
			dev_warn(&pdev->dev,
				 "Cannot deallocate SR-IOV virtual functions while they are assigned - VFs will not be deallocated\n");
			return -EPERM;
		} else {
			pci_disable_sriov(pdev);
			msleep(500);
		}

		kfree(adapter->vf_data);
		adapter->vf_data = NULL;
		adapter->vfs_allocated_count = 0;
		wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
		wrfl();
		msleep(100);
		dev_info(&pdev->dev, "IOV Disabled\n");

		/* Re-enable DMA Coalescing flag since IOV is turned off */
		adapter->flags |= IGB_FLAG_DMAC;
	}

	return 0;
}

static int igb_enable_sriov(struct pci_dev *pdev, int num_vfs)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);
	int old_vfs = pci_num_vf(pdev);
	int err = 0;
	int i;

2706
	if (!(adapter->flags & IGB_FLAG_HAS_MSIX) || num_vfs > 7) {
2707 2708 2709
		err = -EPERM;
		goto out;
	}
2710 2711 2712
	if (!num_vfs)
		goto out;

2713 2714 2715 2716 2717 2718
	if (old_vfs) {
		dev_info(&pdev->dev, "%d pre-allocated VFs found - override max_vfs setting of %d\n",
			 old_vfs, max_vfs);
		adapter->vfs_allocated_count = old_vfs;
	} else
		adapter->vfs_allocated_count = num_vfs;
2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731

	adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
				sizeof(struct vf_data_storage), GFP_KERNEL);

	/* if allocation failed then we do not support SR-IOV */
	if (!adapter->vf_data) {
		adapter->vfs_allocated_count = 0;
		dev_err(&pdev->dev,
			"Unable to allocate memory for VF Data Storage\n");
		err = -ENOMEM;
		goto out;
	}

2732 2733 2734 2735 2736 2737
	/* only call pci_enable_sriov() if no VFs are allocated already */
	if (!old_vfs) {
		err = pci_enable_sriov(pdev, adapter->vfs_allocated_count);
		if (err)
			goto err_out;
	}
2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755
	dev_info(&pdev->dev, "%d VFs allocated\n",
		 adapter->vfs_allocated_count);
	for (i = 0; i < adapter->vfs_allocated_count; i++)
		igb_vf_configure(adapter, i);

	/* DMA Coalescing is not supported in IOV mode. */
	adapter->flags &= ~IGB_FLAG_DMAC;
	goto out;

err_out:
	kfree(adapter->vf_data);
	adapter->vf_data = NULL;
	adapter->vfs_allocated_count = 0;
out:
	return err;
}

#endif
2756
/**
C
Carolyn Wyborny 已提交
2757 2758
 *  igb_remove_i2c - Cleanup  I2C interface
 *  @adapter: pointer to adapter structure
2759
 **/
C
Carolyn Wyborny 已提交
2760 2761 2762 2763 2764 2765
static void igb_remove_i2c(struct igb_adapter *adapter)
{
	/* free the adapter bus structure */
	i2c_del_adapter(&adapter->i2c_adap);
}

2766
/**
2767 2768
 *  igb_remove - Device Removal Routine
 *  @pdev: PCI device information struct
2769
 *
2770 2771 2772 2773
 *  igb_remove is called by the PCI subsystem to alert the driver
 *  that it should release a PCI device.  The could be caused by a
 *  Hot-Plug event, or because the driver is going to be removed from
 *  memory.
2774
 **/
2775
static void igb_remove(struct pci_dev *pdev)
2776 2777 2778
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);
J
Jeb Cramer 已提交
2779
	struct e1000_hw *hw = &adapter->hw;
2780

Y
Yan, Zheng 已提交
2781
	pm_runtime_get_noresume(&pdev->dev);
2782 2783 2784
#ifdef CONFIG_IGB_HWMON
	igb_sysfs_exit(adapter);
#endif
C
Carolyn Wyborny 已提交
2785
	igb_remove_i2c(adapter);
2786
	igb_ptp_stop(adapter);
2787
	/* The watchdog timer may be rescheduled, so explicitly
2788 2789
	 * disable watchdog from being rescheduled.
	 */
2790 2791 2792 2793
	set_bit(__IGB_DOWN, &adapter->state);
	del_timer_sync(&adapter->watchdog_timer);
	del_timer_sync(&adapter->phy_info_timer);

2794 2795
	cancel_work_sync(&adapter->reset_task);
	cancel_work_sync(&adapter->watchdog_task);
2796

2797
#ifdef CONFIG_IGB_DCA
2798
	if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
J
Jeb Cramer 已提交
2799 2800
		dev_info(&pdev->dev, "DCA disabled\n");
		dca_remove_requester(&pdev->dev);
2801
		adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
A
Alexander Duyck 已提交
2802
		wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
J
Jeb Cramer 已提交
2803 2804 2805
	}
#endif

2806
	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
2807 2808
	 * would have already happened in close and is redundant.
	 */
2809 2810 2811 2812
	igb_release_hw_control(adapter);

	unregister_netdev(netdev);

2813
	igb_clear_interrupt_scheme(adapter);
2814

2815
#ifdef CONFIG_PCI_IOV
2816
	igb_disable_sriov(pdev);
2817
#endif
2818

2819
	pci_iounmap(pdev, hw->hw_addr);
2820 2821
	if (hw->flash_address)
		iounmap(hw->flash_address);
2822
	pci_release_selected_regions(pdev,
2823
				     pci_select_bars(pdev, IORESOURCE_MEM));
2824

2825
	kfree(adapter->shadow_vfta);
2826 2827
	free_netdev(netdev);

2828
	pci_disable_pcie_error_reporting(pdev);
2829

2830 2831 2832
	pci_disable_device(pdev);
}

2833
/**
2834 2835
 *  igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
 *  @adapter: board private structure to initialize
2836
 *
2837 2838 2839 2840
 *  This function initializes the vf specific data storage and then attempts to
 *  allocate the VFs.  The reason for ordering it this way is because it is much
 *  mor expensive time wise to disable SR-IOV than it is to allocate and free
 *  the memory for the VFs.
2841
 **/
2842
static void igb_probe_vfs(struct igb_adapter *adapter)
2843 2844 2845
{
#ifdef CONFIG_PCI_IOV
	struct pci_dev *pdev = adapter->pdev;
2846
	struct e1000_hw *hw = &adapter->hw;
2847

2848 2849 2850 2851
	/* Virtualization features not supported on i210 family. */
	if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
		return;

2852
	pci_sriov_set_totalvfs(pdev, 7);
2853
	igb_pci_enable_sriov(pdev, max_vfs);
2854

2855 2856 2857
#endif /* CONFIG_PCI_IOV */
}

2858
static void igb_init_queue_configuration(struct igb_adapter *adapter)
2859 2860
{
	struct e1000_hw *hw = &adapter->hw;
2861
	u32 max_rss_queues;
2862

2863
	/* Determine the maximum number of RSS queues supported. */
2864
	switch (hw->mac.type) {
2865 2866 2867 2868
	case e1000_i211:
		max_rss_queues = IGB_MAX_RX_QUEUES_I211;
		break;
	case e1000_82575:
2869
	case e1000_i210:
2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885
		max_rss_queues = IGB_MAX_RX_QUEUES_82575;
		break;
	case e1000_i350:
		/* I350 cannot do RSS and SR-IOV at the same time */
		if (!!adapter->vfs_allocated_count) {
			max_rss_queues = 1;
			break;
		}
		/* fall through */
	case e1000_82576:
		if (!!adapter->vfs_allocated_count) {
			max_rss_queues = 2;
			break;
		}
		/* fall through */
	case e1000_82580:
2886
	case e1000_i354:
2887 2888
	default:
		max_rss_queues = IGB_MAX_RX_QUEUES;
2889
		break;
2890 2891 2892 2893 2894 2895 2896
	}

	adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());

	/* Determine if we need to pair queues. */
	switch (hw->mac.type) {
	case e1000_82575:
2897
	case e1000_i211:
2898
		/* Device supports enough interrupts without queue pairing. */
2899
		break;
2900
	case e1000_82576:
2901
		/* If VFs are going to be allocated with RSS queues then we
2902 2903 2904 2905 2906 2907 2908 2909 2910
		 * should pair the queues in order to conserve interrupts due
		 * to limited supply.
		 */
		if ((adapter->rss_queues > 1) &&
		    (adapter->vfs_allocated_count > 6))
			adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
		/* fall through */
	case e1000_82580:
	case e1000_i350:
2911
	case e1000_i354:
2912
	case e1000_i210:
2913
	default:
2914
		/* If rss_queues > half of max_rss_queues, pair the queues in
2915 2916 2917 2918
		 * order to conserve interrupts due to limited supply.
		 */
		if (adapter->rss_queues > (max_rss_queues / 2))
			adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2919 2920
		break;
	}
2921 2922 2923
}

/**
2924 2925
 *  igb_sw_init - Initialize general software structures (struct igb_adapter)
 *  @adapter: board private structure to initialize
2926
 *
2927 2928 2929
 *  igb_sw_init initializes the Adapter private data structure.
 *  Fields are initialized based on PCI device information and
 *  OS network device settings (MTU size).
2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961
 **/
static int igb_sw_init(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;

	pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);

	/* set default ring sizes */
	adapter->tx_ring_count = IGB_DEFAULT_TXD;
	adapter->rx_ring_count = IGB_DEFAULT_RXD;

	/* set default ITR values */
	adapter->rx_itr_setting = IGB_DEFAULT_ITR;
	adapter->tx_itr_setting = IGB_DEFAULT_ITR;

	/* set default work limits */
	adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;

	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
				  VLAN_HLEN;
	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;

	spin_lock_init(&adapter->stats64_lock);
#ifdef CONFIG_PCI_IOV
	switch (hw->mac.type) {
	case e1000_82576:
	case e1000_i350:
		if (max_vfs > 7) {
			dev_warn(&pdev->dev,
				 "Maximum of 7 VFs per PF, using max\n");
2962
			max_vfs = adapter->vfs_allocated_count = 7;
2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974
		} else
			adapter->vfs_allocated_count = max_vfs;
		if (adapter->vfs_allocated_count)
			dev_warn(&pdev->dev,
				 "Enabling SR-IOV VFs using the module parameter is deprecated - please use the pci sysfs interface.\n");
		break;
	default:
		break;
	}
#endif /* CONFIG_PCI_IOV */

	igb_init_queue_configuration(adapter);
2975

2976
	/* Setup and initialize a copy of the hw vlan table array */
2977 2978
	adapter->shadow_vfta = kcalloc(E1000_VLAN_FILTER_TBL_SIZE, sizeof(u32),
				       GFP_ATOMIC);
2979

2980
	/* This call may decrease the number of queues */
2981
	if (igb_init_interrupt_scheme(adapter, true)) {
2982 2983 2984 2985
		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
		return -ENOMEM;
	}

2986 2987
	igb_probe_vfs(adapter);

2988 2989 2990
	/* Explicitly disable IRQ since the NIC can be in any state. */
	igb_irq_disable(adapter);

2991
	if (hw->mac.type >= e1000_i350)
2992 2993
		adapter->flags &= ~IGB_FLAG_DMAC;

2994 2995 2996 2997 2998
	set_bit(__IGB_DOWN, &adapter->state);
	return 0;
}

/**
2999 3000
 *  igb_open - Called when a network interface is made active
 *  @netdev: network interface device structure
3001
 *
3002
 *  Returns 0 on success, negative value on failure
3003
 *
3004 3005 3006 3007 3008
 *  The open entry point is called when a network interface is made
 *  active by the system (IFF_UP).  At this point all resources needed
 *  for transmit and receive operations are allocated, the interrupt
 *  handler is registered with the OS, the watchdog timer is started,
 *  and the stack is notified that the interface is ready.
3009
 **/
Y
Yan, Zheng 已提交
3010
static int __igb_open(struct net_device *netdev, bool resuming)
3011 3012 3013
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
Y
Yan, Zheng 已提交
3014
	struct pci_dev *pdev = adapter->pdev;
3015 3016 3017 3018
	int err;
	int i;

	/* disallow open during test */
Y
Yan, Zheng 已提交
3019 3020
	if (test_bit(__IGB_TESTING, &adapter->state)) {
		WARN_ON(resuming);
3021
		return -EBUSY;
Y
Yan, Zheng 已提交
3022 3023 3024 3025
	}

	if (!resuming)
		pm_runtime_get_sync(&pdev->dev);
3026

3027 3028
	netif_carrier_off(netdev);

3029 3030 3031 3032 3033 3034 3035 3036 3037 3038
	/* allocate transmit descriptors */
	err = igb_setup_all_tx_resources(adapter);
	if (err)
		goto err_setup_tx;

	/* allocate receive descriptors */
	err = igb_setup_all_rx_resources(adapter);
	if (err)
		goto err_setup_rx;

3039
	igb_power_up_link(adapter);
3040 3041 3042 3043

	/* before we allocate an interrupt, we must be ready to handle it.
	 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
	 * as soon as we call pci_request_irq, so we have to setup our
3044 3045
	 * clean_rx handler before we do so.
	 */
3046 3047 3048 3049 3050 3051
	igb_configure(adapter);

	err = igb_request_irq(adapter);
	if (err)
		goto err_req_irq;

3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062
	/* Notify the stack of the actual queue counts. */
	err = netif_set_real_num_tx_queues(adapter->netdev,
					   adapter->num_tx_queues);
	if (err)
		goto err_set_queues;

	err = netif_set_real_num_rx_queues(adapter->netdev,
					   adapter->num_rx_queues);
	if (err)
		goto err_set_queues;

3063 3064 3065
	/* From here on the code is the same as igb_up() */
	clear_bit(__IGB_DOWN, &adapter->state);

3066 3067
	for (i = 0; i < adapter->num_q_vectors; i++)
		napi_enable(&(adapter->q_vector[i]->napi));
3068 3069 3070

	/* Clear any pending interrupts. */
	rd32(E1000_ICR);
P
PJ Waskiewicz 已提交
3071 3072 3073

	igb_irq_enable(adapter);

3074 3075 3076
	/* notify VFs that reset has been completed */
	if (adapter->vfs_allocated_count) {
		u32 reg_data = rd32(E1000_CTRL_EXT);
3077

3078 3079 3080 3081
		reg_data |= E1000_CTRL_EXT_PFRSTD;
		wr32(E1000_CTRL_EXT, reg_data);
	}

3082 3083
	netif_tx_start_all_queues(netdev);

Y
Yan, Zheng 已提交
3084 3085 3086
	if (!resuming)
		pm_runtime_put(&pdev->dev);

3087 3088 3089
	/* start the watchdog. */
	hw->mac.get_link_status = 1;
	schedule_work(&adapter->watchdog_task);
3090 3091 3092

	return 0;

3093 3094
err_set_queues:
	igb_free_irq(adapter);
3095 3096
err_req_irq:
	igb_release_hw_control(adapter);
3097
	igb_power_down_link(adapter);
3098 3099 3100 3101 3102
	igb_free_all_rx_resources(adapter);
err_setup_rx:
	igb_free_all_tx_resources(adapter);
err_setup_tx:
	igb_reset(adapter);
Y
Yan, Zheng 已提交
3103 3104
	if (!resuming)
		pm_runtime_put(&pdev->dev);
3105 3106 3107 3108

	return err;
}

Y
Yan, Zheng 已提交
3109 3110 3111 3112 3113
static int igb_open(struct net_device *netdev)
{
	return __igb_open(netdev, false);
}

3114
/**
3115 3116
 *  igb_close - Disables a network interface
 *  @netdev: network interface device structure
3117
 *
3118
 *  Returns 0, this is not allowed to fail
3119
 *
3120 3121 3122 3123
 *  The close entry point is called when an interface is de-activated
 *  by the OS.  The hardware is still under the driver's control, but
 *  needs to be disabled.  A global MAC reset is issued to stop the
 *  hardware, and all transmit and receive resources are freed.
3124
 **/
Y
Yan, Zheng 已提交
3125
static int __igb_close(struct net_device *netdev, bool suspending)
3126 3127
{
	struct igb_adapter *adapter = netdev_priv(netdev);
Y
Yan, Zheng 已提交
3128
	struct pci_dev *pdev = adapter->pdev;
3129 3130 3131

	WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));

Y
Yan, Zheng 已提交
3132 3133 3134 3135
	if (!suspending)
		pm_runtime_get_sync(&pdev->dev);

	igb_down(adapter);
3136 3137 3138 3139 3140
	igb_free_irq(adapter);

	igb_free_all_tx_resources(adapter);
	igb_free_all_rx_resources(adapter);

Y
Yan, Zheng 已提交
3141 3142
	if (!suspending)
		pm_runtime_put_sync(&pdev->dev);
3143 3144 3145
	return 0;
}

Y
Yan, Zheng 已提交
3146 3147 3148 3149 3150
static int igb_close(struct net_device *netdev)
{
	return __igb_close(netdev, false);
}

3151
/**
3152 3153
 *  igb_setup_tx_resources - allocate Tx resources (Descriptors)
 *  @tx_ring: tx descriptor ring (for a specific queue) to setup
3154
 *
3155
 *  Return 0 on success, negative on failure
3156
 **/
3157
int igb_setup_tx_resources(struct igb_ring *tx_ring)
3158
{
3159
	struct device *dev = tx_ring->dev;
3160 3161
	int size;

3162
	size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3163 3164

	tx_ring->tx_buffer_info = vzalloc(size);
3165
	if (!tx_ring->tx_buffer_info)
3166 3167 3168
		goto err;

	/* round up to nearest 4K */
3169
	tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
3170 3171
	tx_ring->size = ALIGN(tx_ring->size, 4096);

3172 3173
	tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
					   &tx_ring->dma, GFP_KERNEL);
3174 3175 3176 3177 3178
	if (!tx_ring->desc)
		goto err;

	tx_ring->next_to_use = 0;
	tx_ring->next_to_clean = 0;
3179

3180 3181 3182
	return 0;

err:
3183
	vfree(tx_ring->tx_buffer_info);
3184 3185
	tx_ring->tx_buffer_info = NULL;
	dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
3186 3187 3188 3189
	return -ENOMEM;
}

/**
3190 3191 3192
 *  igb_setup_all_tx_resources - wrapper to allocate Tx resources
 *				 (Descriptors) for all queues
 *  @adapter: board private structure
3193
 *
3194
 *  Return 0 on success, negative on failure
3195 3196 3197
 **/
static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
{
3198
	struct pci_dev *pdev = adapter->pdev;
3199 3200 3201
	int i, err = 0;

	for (i = 0; i < adapter->num_tx_queues; i++) {
3202
		err = igb_setup_tx_resources(adapter->tx_ring[i]);
3203
		if (err) {
3204
			dev_err(&pdev->dev,
3205 3206
				"Allocation for Tx Queue %u failed\n", i);
			for (i--; i >= 0; i--)
3207
				igb_free_tx_resources(adapter->tx_ring[i]);
3208 3209 3210 3211 3212 3213 3214 3215
			break;
		}
	}

	return err;
}

/**
3216 3217
 *  igb_setup_tctl - configure the transmit control registers
 *  @adapter: Board private structure
3218
 **/
3219
void igb_setup_tctl(struct igb_adapter *adapter)
3220 3221 3222 3223
{
	struct e1000_hw *hw = &adapter->hw;
	u32 tctl;

3224 3225
	/* disable queue 0 which is enabled by default on 82575 and 82576 */
	wr32(E1000_TXDCTL(0), 0);
3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240

	/* Program the Transmit Control Register */
	tctl = rd32(E1000_TCTL);
	tctl &= ~E1000_TCTL_CT;
	tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
		(E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);

	igb_config_collision_dist(hw);

	/* Enable transmits */
	tctl |= E1000_TCTL_EN;

	wr32(E1000_TCTL, tctl);
}

3241
/**
3242 3243 3244
 *  igb_configure_tx_ring - Configure transmit ring after Reset
 *  @adapter: board private structure
 *  @ring: tx ring to configure
3245
 *
3246
 *  Configure a transmit ring after a reset.
3247
 **/
3248
void igb_configure_tx_ring(struct igb_adapter *adapter,
3249
			   struct igb_ring *ring)
3250 3251
{
	struct e1000_hw *hw = &adapter->hw;
3252
	u32 txdctl = 0;
3253 3254 3255 3256
	u64 tdba = ring->dma;
	int reg_idx = ring->reg_idx;

	/* disable the queue */
3257
	wr32(E1000_TXDCTL(reg_idx), 0);
3258 3259 3260 3261
	wrfl();
	mdelay(10);

	wr32(E1000_TDLEN(reg_idx),
3262
	     ring->count * sizeof(union e1000_adv_tx_desc));
3263
	wr32(E1000_TDBAL(reg_idx),
3264
	     tdba & 0x00000000ffffffffULL);
3265 3266
	wr32(E1000_TDBAH(reg_idx), tdba >> 32);

3267
	ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
3268
	wr32(E1000_TDH(reg_idx), 0);
3269
	writel(0, ring->tail);
3270 3271 3272 3273 3274 3275 3276 3277 3278 3279

	txdctl |= IGB_TX_PTHRESH;
	txdctl |= IGB_TX_HTHRESH << 8;
	txdctl |= IGB_TX_WTHRESH << 16;

	txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
	wr32(E1000_TXDCTL(reg_idx), txdctl);
}

/**
3280 3281
 *  igb_configure_tx - Configure transmit Unit after Reset
 *  @adapter: board private structure
3282
 *
3283
 *  Configure the Tx unit of the MAC after a reset.
3284 3285 3286 3287 3288 3289
 **/
static void igb_configure_tx(struct igb_adapter *adapter)
{
	int i;

	for (i = 0; i < adapter->num_tx_queues; i++)
3290
		igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
3291 3292
}

3293
/**
3294 3295
 *  igb_setup_rx_resources - allocate Rx resources (Descriptors)
 *  @rx_ring: Rx descriptor ring (for a specific queue) to setup
3296
 *
3297
 *  Returns 0 on success, negative on failure
3298
 **/
3299
int igb_setup_rx_resources(struct igb_ring *rx_ring)
3300
{
3301
	struct device *dev = rx_ring->dev;
3302
	int size;
3303

3304
	size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3305 3306

	rx_ring->rx_buffer_info = vzalloc(size);
3307
	if (!rx_ring->rx_buffer_info)
3308 3309 3310
		goto err;

	/* Round up to nearest 4K */
3311
	rx_ring->size = rx_ring->count * sizeof(union e1000_adv_rx_desc);
3312 3313
	rx_ring->size = ALIGN(rx_ring->size, 4096);

3314 3315
	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
					   &rx_ring->dma, GFP_KERNEL);
3316 3317 3318
	if (!rx_ring->desc)
		goto err;

3319
	rx_ring->next_to_alloc = 0;
3320 3321 3322 3323 3324 3325
	rx_ring->next_to_clean = 0;
	rx_ring->next_to_use = 0;

	return 0;

err:
3326 3327
	vfree(rx_ring->rx_buffer_info);
	rx_ring->rx_buffer_info = NULL;
3328
	dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
3329 3330 3331 3332
	return -ENOMEM;
}

/**
3333 3334 3335
 *  igb_setup_all_rx_resources - wrapper to allocate Rx resources
 *				 (Descriptors) for all queues
 *  @adapter: board private structure
3336
 *
3337
 *  Return 0 on success, negative on failure
3338 3339 3340
 **/
static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
{
3341
	struct pci_dev *pdev = adapter->pdev;
3342 3343 3344
	int i, err = 0;

	for (i = 0; i < adapter->num_rx_queues; i++) {
3345
		err = igb_setup_rx_resources(adapter->rx_ring[i]);
3346
		if (err) {
3347
			dev_err(&pdev->dev,
3348 3349
				"Allocation for Rx Queue %u failed\n", i);
			for (i--; i >= 0; i--)
3350
				igb_free_rx_resources(adapter->rx_ring[i]);
3351 3352 3353 3354 3355 3356 3357
			break;
		}
	}

	return err;
}

3358
/**
3359 3360
 *  igb_setup_mrqc - configure the multiple receive queue control registers
 *  @adapter: Board private structure
3361 3362 3363 3364 3365
 **/
static void igb_setup_mrqc(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 mrqc, rxcsum;
3366
	u32 j, num_rx_queues;
3367 3368 3369 3370
	static const u32 rsskey[10] = { 0xDA565A6D, 0xC20E5B25, 0x3D256741,
					0xB08FA343, 0xCB2BCAD0, 0xB4307BAE,
					0xA32DCB77, 0x0CF23080, 0x3BB7426A,
					0xFA01ACBE };
3371 3372

	/* Fill out hash function seeds */
3373 3374
	for (j = 0; j < 10; j++)
		wr32(E1000_RSSRK(j), rsskey[j]);
3375

3376
	num_rx_queues = adapter->rss_queues;
3377

3378 3379 3380
	switch (hw->mac.type) {
	case e1000_82576:
		/* 82576 supports 2 RSS queues for SR-IOV */
3381
		if (adapter->vfs_allocated_count)
3382
			num_rx_queues = 2;
3383 3384 3385
		break;
	default:
		break;
3386 3387
	}

3388 3389
	if (adapter->rss_indir_tbl_init != num_rx_queues) {
		for (j = 0; j < IGB_RETA_SIZE; j++)
3390 3391
			adapter->rss_indir_tbl[j] =
			(j * num_rx_queues) / IGB_RETA_SIZE;
3392
		adapter->rss_indir_tbl_init = num_rx_queues;
3393
	}
3394
	igb_write_rss_indir_tbl(adapter);
3395

3396
	/* Disable raw packet checksumming so that RSS hash is placed in
3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408
	 * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
	 * offloads as they are enabled by default
	 */
	rxcsum = rd32(E1000_RXCSUM);
	rxcsum |= E1000_RXCSUM_PCSD;

	if (adapter->hw.mac.type >= e1000_82576)
		/* Enable Receive Checksum Offload for SCTP */
		rxcsum |= E1000_RXCSUM_CRCOFL;

	/* Don't need to set TUOFL or IPOFL, they default to 1 */
	wr32(E1000_RXCSUM, rxcsum);
3409

3410 3411 3412
	/* Generate RSS hash based on packet types, TCP/UDP
	 * port numbers and/or IPv4/v6 src and dst addresses
	 */
3413 3414 3415 3416 3417
	mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
	       E1000_MRQC_RSS_FIELD_IPV4_TCP |
	       E1000_MRQC_RSS_FIELD_IPV6 |
	       E1000_MRQC_RSS_FIELD_IPV6_TCP |
	       E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
3418

3419 3420 3421 3422 3423
	if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV4_UDP)
		mrqc |= E1000_MRQC_RSS_FIELD_IPV4_UDP;
	if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV6_UDP)
		mrqc |= E1000_MRQC_RSS_FIELD_IPV6_UDP;

3424 3425
	/* If VMDq is enabled then we set the appropriate mode for that, else
	 * we default to RSS so that an RSS hash is calculated per packet even
3426 3427
	 * if we are only using one queue
	 */
3428 3429 3430 3431
	if (adapter->vfs_allocated_count) {
		if (hw->mac.type > e1000_82575) {
			/* Set the default pool for the PF's first queue */
			u32 vtctl = rd32(E1000_VT_CTL);
3432

3433 3434 3435 3436 3437 3438
			vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
				   E1000_VT_CTL_DISABLE_DEF_POOL);
			vtctl |= adapter->vfs_allocated_count <<
				E1000_VT_CTL_DEFAULT_POOL_SHIFT;
			wr32(E1000_VT_CTL, vtctl);
		}
3439
		if (adapter->rss_queues > 1)
3440
			mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
3441
		else
3442
			mrqc |= E1000_MRQC_ENABLE_VMDQ;
3443
	} else {
3444 3445
		if (hw->mac.type != e1000_i211)
			mrqc |= E1000_MRQC_ENABLE_RSS_4Q;
3446 3447 3448 3449 3450 3451
	}
	igb_vmm_control(adapter);

	wr32(E1000_MRQC, mrqc);
}

3452
/**
3453 3454
 *  igb_setup_rctl - configure the receive control registers
 *  @adapter: Board private structure
3455
 **/
3456
void igb_setup_rctl(struct igb_adapter *adapter)
3457 3458 3459 3460 3461 3462 3463
{
	struct e1000_hw *hw = &adapter->hw;
	u32 rctl;

	rctl = rd32(E1000_RCTL);

	rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3464
	rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
3465

3466
	rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
3467
		(hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3468

3469
	/* enable stripping of CRC. It's unlikely this will break BMC
3470 3471
	 * redirection as it did with e1000. Newer features require
	 * that the HW strips the CRC.
3472
	 */
3473
	rctl |= E1000_RCTL_SECRC;
3474

3475
	/* disable store bad packets and clear size bits. */
3476
	rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
3477

A
Alexander Duyck 已提交
3478 3479
	/* enable LPE to prevent packets larger than max_frame_size */
	rctl |= E1000_RCTL_LPE;
3480

3481 3482
	/* disable queue 0 to prevent tail write w/o re-config */
	wr32(E1000_RXDCTL(0), 0);
3483

3484 3485 3486 3487 3488 3489 3490 3491 3492
	/* Attention!!!  For SR-IOV PF driver operations you must enable
	 * queue drop for all VF and PF queues to prevent head of line blocking
	 * if an un-trusted VF does not provide descriptors to hardware.
	 */
	if (adapter->vfs_allocated_count) {
		/* set all queue drop enable bits */
		wr32(E1000_QDE, ALL_QUEUES);
	}

B
Ben Greear 已提交
3493 3494 3495
	/* This is useful for sniffing bad packets. */
	if (adapter->netdev->features & NETIF_F_RXALL) {
		/* UPE and MPE will be handled by normal PROMISC logic
3496 3497
		 * in e1000e_set_rx_mode
		 */
B
Ben Greear 已提交
3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509
		rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
			 E1000_RCTL_BAM | /* RX All Bcast Pkts */
			 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */

		rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
			  E1000_RCTL_DPF | /* Allow filtered pause */
			  E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
		/* Do not mess with E1000_CTRL_VME, it affects transmit as well,
		 * and that breaks VLANs.
		 */
	}

3510 3511 3512
	wr32(E1000_RCTL, rctl);
}

3513
static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3514
				   int vfn)
3515 3516 3517 3518 3519
{
	struct e1000_hw *hw = &adapter->hw;
	u32 vmolr;

	/* if it isn't the PF check to see if VFs are enabled and
3520 3521
	 * increase the size to support vlan tags
	 */
3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533
	if (vfn < adapter->vfs_allocated_count &&
	    adapter->vf_data[vfn].vlans_enabled)
		size += VLAN_TAG_SIZE;

	vmolr = rd32(E1000_VMOLR(vfn));
	vmolr &= ~E1000_VMOLR_RLPML_MASK;
	vmolr |= size | E1000_VMOLR_LPE;
	wr32(E1000_VMOLR(vfn), vmolr);

	return 0;
}

3534
/**
3535 3536
 *  igb_rlpml_set - set maximum receive packet size
 *  @adapter: board private structure
3537
 *
3538
 *  Configure maximum receivable packet size.
3539 3540 3541
 **/
static void igb_rlpml_set(struct igb_adapter *adapter)
{
3542
	u32 max_frame_size = adapter->max_frame_size;
3543 3544 3545 3546 3547
	struct e1000_hw *hw = &adapter->hw;
	u16 pf_id = adapter->vfs_allocated_count;

	if (pf_id) {
		igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
3548
		/* If we're in VMDQ or SR-IOV mode, then set global RLPML
3549 3550 3551 3552 3553
		 * to our max jumbo frame size, in case we need to enable
		 * jumbo frames on one of the rings later.
		 * This will not pass over-length frames into the default
		 * queue because it's gated by the VMOLR.RLPML.
		 */
3554
		max_frame_size = MAX_JUMBO_FRAME_SIZE;
3555 3556 3557 3558 3559
	}

	wr32(E1000_RLPML, max_frame_size);
}

3560 3561
static inline void igb_set_vmolr(struct igb_adapter *adapter,
				 int vfn, bool aupe)
3562 3563 3564 3565
{
	struct e1000_hw *hw = &adapter->hw;
	u32 vmolr;

3566
	/* This register exists only on 82576 and newer so if we are older then
3567 3568 3569 3570 3571 3572
	 * we should exit and do nothing
	 */
	if (hw->mac.type < e1000_82576)
		return;

	vmolr = rd32(E1000_VMOLR(vfn));
3573
	vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3574 3575 3576 3577 3578 3579 3580
	if (hw->mac.type == e1000_i350) {
		u32 dvmolr;

		dvmolr = rd32(E1000_DVMOLR(vfn));
		dvmolr |= E1000_DVMOLR_STRVLAN;
		wr32(E1000_DVMOLR(vfn), dvmolr);
	}
3581
	if (aupe)
3582
		vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3583 3584
	else
		vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3585 3586 3587 3588

	/* clear all bits that might not be set */
	vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);

3589
	if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3590
		vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3591
	/* for VMDq only allow the VFs and pool 0 to accept broadcast and
3592 3593 3594
	 * multicast packets
	 */
	if (vfn <= adapter->vfs_allocated_count)
3595
		vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3596 3597 3598 3599

	wr32(E1000_VMOLR(vfn), vmolr);
}

3600
/**
3601 3602 3603
 *  igb_configure_rx_ring - Configure a receive ring after Reset
 *  @adapter: board private structure
 *  @ring: receive ring to be configured
3604
 *
3605
 *  Configure the Rx unit of the MAC after a reset.
3606
 **/
3607
void igb_configure_rx_ring(struct igb_adapter *adapter,
3608
			   struct igb_ring *ring)
3609 3610 3611 3612
{
	struct e1000_hw *hw = &adapter->hw;
	u64 rdba = ring->dma;
	int reg_idx = ring->reg_idx;
3613
	u32 srrctl = 0, rxdctl = 0;
3614 3615

	/* disable the queue */
3616
	wr32(E1000_RXDCTL(reg_idx), 0);
3617 3618 3619 3620 3621 3622

	/* Set DMA base address registers */
	wr32(E1000_RDBAL(reg_idx),
	     rdba & 0x00000000ffffffffULL);
	wr32(E1000_RDBAH(reg_idx), rdba >> 32);
	wr32(E1000_RDLEN(reg_idx),
3623
	     ring->count * sizeof(union e1000_adv_rx_desc));
3624 3625

	/* initialize head and tail */
3626
	ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3627
	wr32(E1000_RDH(reg_idx), 0);
3628
	writel(0, ring->tail);
3629

3630
	/* set descriptor configuration */
3631
	srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3632
	srrctl |= IGB_RX_BUFSZ >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3633
	srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3634
	if (hw->mac.type >= e1000_82580)
N
Nick Nunley 已提交
3635
		srrctl |= E1000_SRRCTL_TIMESTAMP;
3636 3637 3638
	/* Only set Drop Enable if we are supporting multiple queues */
	if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
		srrctl |= E1000_SRRCTL_DROP_EN;
3639 3640 3641

	wr32(E1000_SRRCTL(reg_idx), srrctl);

3642
	/* set filtering for VMDQ pools */
3643
	igb_set_vmolr(adapter, reg_idx & 0x7, true);
3644

3645 3646 3647
	rxdctl |= IGB_RX_PTHRESH;
	rxdctl |= IGB_RX_HTHRESH << 8;
	rxdctl |= IGB_RX_WTHRESH << 16;
3648 3649 3650

	/* enable receive descriptor fetching */
	rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3651 3652 3653
	wr32(E1000_RXDCTL(reg_idx), rxdctl);
}

3654
/**
3655 3656
 *  igb_configure_rx - Configure receive Unit after Reset
 *  @adapter: board private structure
3657
 *
3658
 *  Configure the Rx unit of the MAC after a reset.
3659 3660 3661
 **/
static void igb_configure_rx(struct igb_adapter *adapter)
{
3662
	int i;
3663

3664 3665 3666
	/* set UTA to appropriate mode */
	igb_set_uta(adapter);

3667 3668
	/* set the correct pool for the PF default MAC address in entry 0 */
	igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3669
			 adapter->vfs_allocated_count);
3670

3671
	/* Setup the HW Rx Head and Tail Descriptor Pointers and
3672 3673
	 * the Base and Length of the Rx Descriptor Ring
	 */
3674 3675
	for (i = 0; i < adapter->num_rx_queues; i++)
		igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
3676 3677 3678
}

/**
3679 3680
 *  igb_free_tx_resources - Free Tx Resources per Queue
 *  @tx_ring: Tx descriptor ring for a specific queue
3681
 *
3682
 *  Free all transmit software resources
3683
 **/
3684
void igb_free_tx_resources(struct igb_ring *tx_ring)
3685
{
3686
	igb_clean_tx_ring(tx_ring);
3687

3688 3689
	vfree(tx_ring->tx_buffer_info);
	tx_ring->tx_buffer_info = NULL;
3690

3691 3692 3693 3694
	/* if not set, then don't free */
	if (!tx_ring->desc)
		return;

3695 3696
	dma_free_coherent(tx_ring->dev, tx_ring->size,
			  tx_ring->desc, tx_ring->dma);
3697 3698 3699 3700 3701

	tx_ring->desc = NULL;
}

/**
3702 3703
 *  igb_free_all_tx_resources - Free Tx Resources for All Queues
 *  @adapter: board private structure
3704
 *
3705
 *  Free all transmit software resources
3706 3707 3708 3709 3710 3711
 **/
static void igb_free_all_tx_resources(struct igb_adapter *adapter)
{
	int i;

	for (i = 0; i < adapter->num_tx_queues; i++)
3712
		igb_free_tx_resources(adapter->tx_ring[i]);
3713 3714
}

3715 3716 3717 3718 3719
void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
				    struct igb_tx_buffer *tx_buffer)
{
	if (tx_buffer->skb) {
		dev_kfree_skb_any(tx_buffer->skb);
3720
		if (dma_unmap_len(tx_buffer, len))
3721
			dma_unmap_single(ring->dev,
3722 3723
					 dma_unmap_addr(tx_buffer, dma),
					 dma_unmap_len(tx_buffer, len),
3724
					 DMA_TO_DEVICE);
3725
	} else if (dma_unmap_len(tx_buffer, len)) {
3726
		dma_unmap_page(ring->dev,
3727 3728
			       dma_unmap_addr(tx_buffer, dma),
			       dma_unmap_len(tx_buffer, len),
3729 3730 3731 3732
			       DMA_TO_DEVICE);
	}
	tx_buffer->next_to_watch = NULL;
	tx_buffer->skb = NULL;
3733
	dma_unmap_len_set(tx_buffer, len, 0);
3734
	/* buffer_info must be completely set up in the transmit path */
3735 3736 3737
}

/**
3738 3739
 *  igb_clean_tx_ring - Free Tx Buffers
 *  @tx_ring: ring to be cleaned
3740
 **/
3741
static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3742
{
3743
	struct igb_tx_buffer *buffer_info;
3744
	unsigned long size;
3745
	u16 i;
3746

3747
	if (!tx_ring->tx_buffer_info)
3748 3749 3750 3751
		return;
	/* Free all the Tx ring sk_buffs */

	for (i = 0; i < tx_ring->count; i++) {
3752
		buffer_info = &tx_ring->tx_buffer_info[i];
3753
		igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3754 3755
	}

3756 3757
	netdev_tx_reset_queue(txring_txq(tx_ring));

3758 3759
	size = sizeof(struct igb_tx_buffer) * tx_ring->count;
	memset(tx_ring->tx_buffer_info, 0, size);
3760 3761 3762 3763 3764 3765 3766 3767 3768

	/* Zero out the descriptor ring */
	memset(tx_ring->desc, 0, tx_ring->size);

	tx_ring->next_to_use = 0;
	tx_ring->next_to_clean = 0;
}

/**
3769 3770
 *  igb_clean_all_tx_rings - Free Tx Buffers for all queues
 *  @adapter: board private structure
3771 3772 3773 3774 3775 3776
 **/
static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
{
	int i;

	for (i = 0; i < adapter->num_tx_queues; i++)
3777
		igb_clean_tx_ring(adapter->tx_ring[i]);
3778 3779 3780
}

/**
3781 3782
 *  igb_free_rx_resources - Free Rx Resources
 *  @rx_ring: ring to clean the resources from
3783
 *
3784
 *  Free all receive software resources
3785
 **/
3786
void igb_free_rx_resources(struct igb_ring *rx_ring)
3787
{
3788
	igb_clean_rx_ring(rx_ring);
3789

3790 3791
	vfree(rx_ring->rx_buffer_info);
	rx_ring->rx_buffer_info = NULL;
3792

3793 3794 3795 3796
	/* if not set, then don't free */
	if (!rx_ring->desc)
		return;

3797 3798
	dma_free_coherent(rx_ring->dev, rx_ring->size,
			  rx_ring->desc, rx_ring->dma);
3799 3800 3801 3802 3803

	rx_ring->desc = NULL;
}

/**
3804 3805
 *  igb_free_all_rx_resources - Free Rx Resources for All Queues
 *  @adapter: board private structure
3806
 *
3807
 *  Free all receive software resources
3808 3809 3810 3811 3812 3813
 **/
static void igb_free_all_rx_resources(struct igb_adapter *adapter)
{
	int i;

	for (i = 0; i < adapter->num_rx_queues; i++)
3814
		igb_free_rx_resources(adapter->rx_ring[i]);
3815 3816 3817
}

/**
3818 3819
 *  igb_clean_rx_ring - Free Rx Buffers per Queue
 *  @rx_ring: ring to free buffers from
3820
 **/
3821
static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3822 3823
{
	unsigned long size;
3824
	u16 i;
3825

3826 3827 3828 3829
	if (rx_ring->skb)
		dev_kfree_skb(rx_ring->skb);
	rx_ring->skb = NULL;

3830
	if (!rx_ring->rx_buffer_info)
3831
		return;
3832

3833 3834
	/* Free all the Rx ring sk_buffs */
	for (i = 0; i < rx_ring->count; i++) {
3835
		struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
3836

3837 3838 3839 3840 3841 3842 3843 3844 3845
		if (!buffer_info->page)
			continue;

		dma_unmap_page(rx_ring->dev,
			       buffer_info->dma,
			       PAGE_SIZE,
			       DMA_FROM_DEVICE);
		__free_page(buffer_info->page);

3846
		buffer_info->page = NULL;
3847 3848
	}

3849 3850
	size = sizeof(struct igb_rx_buffer) * rx_ring->count;
	memset(rx_ring->rx_buffer_info, 0, size);
3851 3852 3853 3854

	/* Zero out the descriptor ring */
	memset(rx_ring->desc, 0, rx_ring->size);

3855
	rx_ring->next_to_alloc = 0;
3856 3857 3858 3859 3860
	rx_ring->next_to_clean = 0;
	rx_ring->next_to_use = 0;
}

/**
3861 3862
 *  igb_clean_all_rx_rings - Free Rx Buffers for all queues
 *  @adapter: board private structure
3863 3864 3865 3866 3867 3868
 **/
static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
{
	int i;

	for (i = 0; i < adapter->num_rx_queues; i++)
3869
		igb_clean_rx_ring(adapter->rx_ring[i]);
3870 3871 3872
}

/**
3873 3874 3875
 *  igb_set_mac - Change the Ethernet Address of the NIC
 *  @netdev: network interface device structure
 *  @p: pointer to an address structure
3876
 *
3877
 *  Returns 0 on success, negative on failure
3878 3879 3880 3881
 **/
static int igb_set_mac(struct net_device *netdev, void *p)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
3882
	struct e1000_hw *hw = &adapter->hw;
3883 3884 3885 3886 3887 3888
	struct sockaddr *addr = p;

	if (!is_valid_ether_addr(addr->sa_data))
		return -EADDRNOTAVAIL;

	memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3889
	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3890

3891 3892
	/* set the correct pool for the new PF MAC address in entry 0 */
	igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3893
			 adapter->vfs_allocated_count);
3894

3895 3896 3897 3898
	return 0;
}

/**
3899 3900
 *  igb_write_mc_addr_list - write multicast addresses to MTA
 *  @netdev: network interface device structure
3901
 *
3902 3903 3904 3905
 *  Writes multicast address list to the MTA hash table.
 *  Returns: -ENOMEM on failure
 *           0 on no addresses written
 *           X on writing X addresses to MTA
3906
 **/
3907
static int igb_write_mc_addr_list(struct net_device *netdev)
3908 3909 3910
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
3911
	struct netdev_hw_addr *ha;
3912
	u8  *mta_list;
3913 3914
	int i;

3915
	if (netdev_mc_empty(netdev)) {
3916 3917 3918 3919 3920
		/* nothing to program, so clear mc list */
		igb_update_mc_addr_list(hw, NULL, 0);
		igb_restore_vf_multicasts(adapter);
		return 0;
	}
3921

3922
	mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3923 3924
	if (!mta_list)
		return -ENOMEM;
3925

3926
	/* The shared function expects a packed array of only addresses. */
3927
	i = 0;
3928 3929
	netdev_for_each_mc_addr(ha, netdev)
		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3930 3931 3932 3933

	igb_update_mc_addr_list(hw, mta_list, i);
	kfree(mta_list);

3934
	return netdev_mc_count(netdev);
3935 3936 3937
}

/**
3938 3939
 *  igb_write_uc_addr_list - write unicast addresses to RAR table
 *  @netdev: network interface device structure
3940
 *
3941 3942 3943 3944
 *  Writes unicast address list to the RAR table.
 *  Returns: -ENOMEM on failure/insufficient address space
 *           0 on no addresses written
 *           X on writing X addresses to the RAR table
3945 3946 3947 3948 3949 3950 3951 3952 3953 3954
 **/
static int igb_write_uc_addr_list(struct net_device *netdev)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
	unsigned int vfn = adapter->vfs_allocated_count;
	unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
	int count = 0;

	/* return ENOMEM indicating insufficient memory for addresses */
3955
	if (netdev_uc_count(netdev) > rar_entries)
3956
		return -ENOMEM;
3957

3958
	if (!netdev_uc_empty(netdev) && rar_entries) {
3959
		struct netdev_hw_addr *ha;
3960 3961

		netdev_for_each_uc_addr(ha, netdev) {
3962 3963
			if (!rar_entries)
				break;
3964
			igb_rar_set_qsel(adapter, ha->addr,
3965 3966
					 rar_entries--,
					 vfn);
3967
			count++;
3968 3969 3970 3971 3972 3973 3974 3975 3976
		}
	}
	/* write the addresses in reverse order to avoid write combining */
	for (; rar_entries > 0 ; rar_entries--) {
		wr32(E1000_RAH(rar_entries), 0);
		wr32(E1000_RAL(rar_entries), 0);
	}
	wrfl();

3977 3978 3979 3980
	return count;
}

/**
3981 3982
 *  igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
 *  @netdev: network interface device structure
3983
 *
3984 3985 3986 3987
 *  The set_rx_mode entry point is called whenever the unicast or multicast
 *  address lists or the network interface flags are updated.  This routine is
 *  responsible for configuring the hardware for proper unicast, multicast,
 *  promiscuous mode, and all-multi behavior.
3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003
 **/
static void igb_set_rx_mode(struct net_device *netdev)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
	unsigned int vfn = adapter->vfs_allocated_count;
	u32 rctl, vmolr = 0;
	int count;

	/* Check for Promiscuous and All Multicast modes */
	rctl = rd32(E1000_RCTL);

	/* clear the effected bits */
	rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);

	if (netdev->flags & IFF_PROMISC) {
4004
		/* retain VLAN HW filtering if in VT mode */
4005
		if (adapter->vfs_allocated_count)
4006
			rctl |= E1000_RCTL_VFE;
4007 4008 4009 4010 4011 4012 4013
		rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
		vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
	} else {
		if (netdev->flags & IFF_ALLMULTI) {
			rctl |= E1000_RCTL_MPE;
			vmolr |= E1000_VMOLR_MPME;
		} else {
4014
			/* Write addresses to the MTA, if the attempt fails
L
Lucas De Marchi 已提交
4015
			 * then we should just turn on promiscuous mode so
4016 4017 4018 4019 4020 4021 4022 4023 4024 4025
			 * that we can at least receive multicast traffic
			 */
			count = igb_write_mc_addr_list(netdev);
			if (count < 0) {
				rctl |= E1000_RCTL_MPE;
				vmolr |= E1000_VMOLR_MPME;
			} else if (count) {
				vmolr |= E1000_VMOLR_ROMPE;
			}
		}
4026
		/* Write addresses to available RAR registers, if there is not
4027
		 * sufficient space to store all the addresses then enable
L
Lucas De Marchi 已提交
4028
		 * unicast promiscuous mode
4029 4030 4031 4032 4033 4034 4035
		 */
		count = igb_write_uc_addr_list(netdev);
		if (count < 0) {
			rctl |= E1000_RCTL_UPE;
			vmolr |= E1000_VMOLR_ROPE;
		}
		rctl |= E1000_RCTL_VFE;
4036
	}
4037
	wr32(E1000_RCTL, rctl);
4038

4039
	/* In order to support SR-IOV and eventually VMDq it is necessary to set
4040 4041 4042 4043
	 * the VMOLR to enable the appropriate modes.  Without this workaround
	 * we will have issues with VLAN tag stripping not being done for frames
	 * that are only arriving because we are the default pool
	 */
4044
	if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
4045
		return;
4046

4047
	vmolr |= rd32(E1000_VMOLR(vfn)) &
4048
		 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
4049
	wr32(E1000_VMOLR(vfn), vmolr);
4050
	igb_restore_vf_multicasts(adapter);
4051 4052
}

G
Greg Rose 已提交
4053 4054 4055 4056 4057 4058 4059 4060
static void igb_check_wvbr(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 wvbr = 0;

	switch (hw->mac.type) {
	case e1000_82576:
	case e1000_i350:
4061 4062
		wvbr = rd32(E1000_WVBR);
		if (!wvbr)
G
Greg Rose 已提交
4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080
			return;
		break;
	default:
		break;
	}

	adapter->wvbr |= wvbr;
}

#define IGB_STAGGERED_QUEUE_OFFSET 8

static void igb_spoof_check(struct igb_adapter *adapter)
{
	int j;

	if (!adapter->wvbr)
		return;

4081
	for (j = 0; j < adapter->vfs_allocated_count; j++) {
G
Greg Rose 已提交
4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092
		if (adapter->wvbr & (1 << j) ||
		    adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
			dev_warn(&adapter->pdev->dev,
				"Spoof event(s) detected on VF %d\n", j);
			adapter->wvbr &=
				~((1 << j) |
				  (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
		}
	}
}

4093
/* Need to wait a few seconds after link up to get diagnostic information from
4094 4095
 * the phy
 */
4096 4097 4098
static void igb_update_phy_info(unsigned long data)
{
	struct igb_adapter *adapter = (struct igb_adapter *) data;
4099
	igb_get_phy_info(&adapter->hw);
4100 4101
}

A
Alexander Duyck 已提交
4102
/**
4103 4104
 *  igb_has_link - check shared code for link and determine up/down
 *  @adapter: pointer to driver private info
A
Alexander Duyck 已提交
4105
 **/
4106
bool igb_has_link(struct igb_adapter *adapter)
A
Alexander Duyck 已提交
4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117
{
	struct e1000_hw *hw = &adapter->hw;
	bool link_active = false;

	/* get_link_status is set on LSC (link status) interrupt or
	 * rx sequence error interrupt.  get_link_status will stay
	 * false until the e1000_check_for_link establishes link
	 * for copper adapters ONLY
	 */
	switch (hw->phy.media_type) {
	case e1000_media_type_copper:
4118 4119
		if (!hw->mac.get_link_status)
			return true;
A
Alexander Duyck 已提交
4120
	case e1000_media_type_internal_serdes:
4121 4122
		hw->mac.ops.check_for_link(hw);
		link_active = !hw->mac.get_link_status;
A
Alexander Duyck 已提交
4123 4124 4125 4126 4127 4128
		break;
	default:
	case e1000_media_type_unknown:
		break;
	}

4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139
	if (((hw->mac.type == e1000_i210) ||
	     (hw->mac.type == e1000_i211)) &&
	     (hw->phy.id == I210_I_PHY_ID)) {
		if (!netif_carrier_ok(adapter->netdev)) {
			adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
		} else if (!(adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)) {
			adapter->flags |= IGB_FLAG_NEED_LINK_UPDATE;
			adapter->link_check_timeout = jiffies;
		}
	}

A
Alexander Duyck 已提交
4140 4141 4142
	return link_active;
}

4143 4144 4145 4146 4147
static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
{
	bool ret = false;
	u32 ctrl_ext, thstat;

4148
	/* check for thermal sensor event on i350 copper only */
4149 4150 4151 4152 4153
	if (hw->mac.type == e1000_i350) {
		thstat = rd32(E1000_THSTAT);
		ctrl_ext = rd32(E1000_CTRL_EXT);

		if ((hw->phy.media_type == e1000_media_type_copper) &&
4154
		    !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII))
4155 4156 4157 4158 4159 4160
			ret = !!(thstat & event);
	}

	return ret;
}

4161
/**
4162 4163
 *  igb_watchdog - Timer Call-back
 *  @data: pointer to adapter cast into an unsigned long
4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174
 **/
static void igb_watchdog(unsigned long data)
{
	struct igb_adapter *adapter = (struct igb_adapter *)data;
	/* Do the rest outside of interrupt context */
	schedule_work(&adapter->watchdog_task);
}

static void igb_watchdog_task(struct work_struct *work)
{
	struct igb_adapter *adapter = container_of(work,
4175 4176
						   struct igb_adapter,
						   watchdog_task);
4177
	struct e1000_hw *hw = &adapter->hw;
4178
	struct e1000_phy_info *phy = &hw->phy;
4179
	struct net_device *netdev = adapter->netdev;
4180
	u32 link;
4181
	int i;
4182
	u32 connsw;
4183

A
Alexander Duyck 已提交
4184
	link = igb_has_link(adapter);
4185 4186 4187 4188 4189 4190 4191 4192

	if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) {
		if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
			adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
		else
			link = false;
	}

4193 4194 4195 4196 4197 4198 4199 4200
	/* Force link down if we have fiber to swap to */
	if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
		if (hw->phy.media_type == e1000_media_type_copper) {
			connsw = rd32(E1000_CONNSW);
			if (!(connsw & E1000_CONNSW_AUTOSENSE_EN))
				link = 0;
		}
	}
4201
	if (link) {
4202 4203 4204 4205 4206 4207
		/* Perform a reset if the media type changed. */
		if (hw->dev_spec._82575.media_changed) {
			hw->dev_spec._82575.media_changed = false;
			adapter->flags |= IGB_FLAG_MEDIA_RESET;
			igb_reset(adapter);
		}
Y
Yan, Zheng 已提交
4208 4209 4210
		/* Cancel scheduled suspend requests. */
		pm_runtime_resume(netdev->dev.parent);

4211 4212
		if (!netif_carrier_ok(netdev)) {
			u32 ctrl;
4213

4214
			hw->mac.ops.get_speed_and_duplex(hw,
4215 4216
							 &adapter->link_speed,
							 &adapter->link_duplex);
4217 4218

			ctrl = rd32(E1000_CTRL);
4219
			/* Links status message must follow this format */
C
Carolyn Wyborny 已提交
4220 4221
			netdev_info(netdev,
			       "igb: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
4222 4223 4224
			       netdev->name,
			       adapter->link_speed,
			       adapter->link_duplex == FULL_DUPLEX ?
J
Jeff Kirsher 已提交
4225 4226 4227 4228 4229
			       "Full" : "Half",
			       (ctrl & E1000_CTRL_TFCE) &&
			       (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
			       (ctrl & E1000_CTRL_RFCE) ?  "RX" :
			       (ctrl & E1000_CTRL_TFCE) ?  "TX" : "None");
4230

4231 4232 4233 4234 4235 4236 4237 4238 4239
			/* disable EEE if enabled */
			if ((adapter->flags & IGB_FLAG_EEE) &&
				(adapter->link_duplex == HALF_DUPLEX)) {
				dev_info(&adapter->pdev->dev,
				"EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex.\n");
				adapter->hw.dev_spec._82575.eee_disable = true;
				adapter->flags &= ~IGB_FLAG_EEE;
			}

4240 4241 4242 4243 4244
			/* check if SmartSpeed worked */
			igb_check_downshift(hw);
			if (phy->speed_downgraded)
				netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");

4245
			/* check for thermal sensor event */
J
Jeff Kirsher 已提交
4246
			if (igb_thermal_sensor_event(hw,
4247
			    E1000_THSTAT_LINK_THROTTLE))
C
Carolyn Wyborny 已提交
4248
				netdev_info(netdev, "The network adapter link speed was downshifted because it overheated\n");
4249

4250
			/* adjust timeout factor according to speed/duplex */
4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262
			adapter->tx_timeout_factor = 1;
			switch (adapter->link_speed) {
			case SPEED_10:
				adapter->tx_timeout_factor = 14;
				break;
			case SPEED_100:
				/* maybe add some timeout factor ? */
				break;
			}

			netif_carrier_on(netdev);

4263
			igb_ping_all_vfs(adapter);
4264
			igb_check_vf_rate_limit(adapter);
4265

4266
			/* link state has changed, schedule phy info update */
4267 4268 4269 4270 4271 4272 4273 4274
			if (!test_bit(__IGB_DOWN, &adapter->state))
				mod_timer(&adapter->phy_info_timer,
					  round_jiffies(jiffies + 2 * HZ));
		}
	} else {
		if (netif_carrier_ok(netdev)) {
			adapter->link_speed = 0;
			adapter->link_duplex = 0;
4275 4276

			/* check for thermal sensor event */
J
Jeff Kirsher 已提交
4277 4278
			if (igb_thermal_sensor_event(hw,
			    E1000_THSTAT_PWR_DOWN)) {
C
Carolyn Wyborny 已提交
4279
				netdev_err(netdev, "The network adapter was stopped because it overheated\n");
4280
			}
4281

4282
			/* Links status message must follow this format */
C
Carolyn Wyborny 已提交
4283
			netdev_info(netdev, "igb: %s NIC Link is Down\n",
4284
			       netdev->name);
4285
			netif_carrier_off(netdev);
4286

4287 4288
			igb_ping_all_vfs(adapter);

4289
			/* link state has changed, schedule phy info update */
4290 4291 4292
			if (!test_bit(__IGB_DOWN, &adapter->state))
				mod_timer(&adapter->phy_info_timer,
					  round_jiffies(jiffies + 2 * HZ));
Y
Yan, Zheng 已提交
4293

4294 4295 4296 4297 4298 4299 4300 4301 4302
			/* link is down, time to check for alternate media */
			if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
				igb_check_swap_media(adapter);
				if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
					schedule_work(&adapter->reset_task);
					/* return immediately */
					return;
				}
			}
Y
Yan, Zheng 已提交
4303 4304
			pm_schedule_suspend(netdev->dev.parent,
					    MSEC_PER_SEC * 5);
4305 4306 4307 4308 4309 4310 4311 4312 4313 4314

		/* also check for alternate media here */
		} else if (!netif_carrier_ok(netdev) &&
			   (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
			igb_check_swap_media(adapter);
			if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
				schedule_work(&adapter->reset_task);
				/* return immediately */
				return;
			}
4315 4316 4317
		}
	}

E
Eric Dumazet 已提交
4318 4319 4320
	spin_lock(&adapter->stats64_lock);
	igb_update_stats(adapter, &adapter->stats64);
	spin_unlock(&adapter->stats64_lock);
4321

4322
	for (i = 0; i < adapter->num_tx_queues; i++) {
4323
		struct igb_ring *tx_ring = adapter->tx_ring[i];
4324
		if (!netif_carrier_ok(netdev)) {
4325 4326 4327
			/* We've lost link, so the controller stops DMA,
			 * but we've got queued Tx work that's never going
			 * to get done, so reset controller to flush Tx.
4328 4329
			 * (Do the reset outside of interrupt context).
			 */
4330 4331 4332 4333 4334 4335
			if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
				adapter->tx_timeout_count++;
				schedule_work(&adapter->reset_task);
				/* return immediately since reset is imminent */
				return;
			}
4336 4337
		}

4338
		/* Force detection of hung controller every watchdog period */
4339
		set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
4340
	}
4341

4342
	/* Cause software interrupt to ensure Rx ring is cleaned */
4343
	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
4344
		u32 eics = 0;
4345

4346 4347
		for (i = 0; i < adapter->num_q_vectors; i++)
			eics |= adapter->q_vector[i]->eims_value;
4348 4349 4350 4351
		wr32(E1000_EICS, eics);
	} else {
		wr32(E1000_ICS, E1000_ICS_RXDMT0);
	}
4352

G
Greg Rose 已提交
4353
	igb_spoof_check(adapter);
4354
	igb_ptp_rx_hang(adapter);
G
Greg Rose 已提交
4355

4356
	/* Reset the timer */
4357 4358 4359 4360 4361 4362 4363 4364
	if (!test_bit(__IGB_DOWN, &adapter->state)) {
		if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)
			mod_timer(&adapter->watchdog_timer,
				  round_jiffies(jiffies +  HZ));
		else
			mod_timer(&adapter->watchdog_timer,
				  round_jiffies(jiffies + 2 * HZ));
	}
4365 4366 4367 4368 4369 4370 4371 4372 4373
}

enum latency_range {
	lowest_latency = 0,
	low_latency = 1,
	bulk_latency = 2,
	latency_invalid = 255
};

4374
/**
4375 4376
 *  igb_update_ring_itr - update the dynamic ITR value based on packet size
 *  @q_vector: pointer to q_vector
4377
 *
4378 4379 4380 4381 4382 4383 4384
 *  Stores a new ITR value based on strictly on packet size.  This
 *  algorithm is less sophisticated than that used in igb_update_itr,
 *  due to the difficulty of synchronizing statistics across multiple
 *  receive rings.  The divisors and thresholds used by this function
 *  were determined based on theoretical maximum wire speed and testing
 *  data, in order to minimize response time while increasing bulk
 *  throughput.
4385
 *  This functionality is controlled by ethtool's coalescing settings.
4386 4387
 *  NOTE:  This function is called only when operating in a multiqueue
 *         receive environment.
4388
 **/
4389
static void igb_update_ring_itr(struct igb_q_vector *q_vector)
4390
{
4391
	int new_val = q_vector->itr_val;
4392
	int avg_wire_size = 0;
4393
	struct igb_adapter *adapter = q_vector->adapter;
E
Eric Dumazet 已提交
4394
	unsigned int packets;
4395

4396 4397 4398 4399
	/* For non-gigabit speeds, just fix the interrupt rate at 4000
	 * ints/sec - ITR timer value of 120 ticks.
	 */
	if (adapter->link_speed != SPEED_1000) {
4400
		new_val = IGB_4K_ITR;
4401
		goto set_itr_val;
4402
	}
4403

4404 4405 4406
	packets = q_vector->rx.total_packets;
	if (packets)
		avg_wire_size = q_vector->rx.total_bytes / packets;
4407

4408 4409 4410 4411
	packets = q_vector->tx.total_packets;
	if (packets)
		avg_wire_size = max_t(u32, avg_wire_size,
				      q_vector->tx.total_bytes / packets);
4412 4413 4414 4415

	/* if avg_wire_size isn't set no work was done */
	if (!avg_wire_size)
		goto clear_counts;
4416

4417 4418 4419 4420 4421
	/* Add 24 bytes to size to account for CRC, preamble, and gap */
	avg_wire_size += 24;

	/* Don't starve jumbo frames */
	avg_wire_size = min(avg_wire_size, 3000);
4422

4423 4424 4425 4426 4427
	/* Give a little boost to mid-size frames */
	if ((avg_wire_size > 300) && (avg_wire_size < 1200))
		new_val = avg_wire_size / 3;
	else
		new_val = avg_wire_size / 2;
4428

4429 4430 4431 4432 4433
	/* conservative mode (itr 3) eliminates the lowest_latency setting */
	if (new_val < IGB_20K_ITR &&
	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
	     (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
		new_val = IGB_20K_ITR;
4434

4435
set_itr_val:
4436 4437 4438
	if (new_val != q_vector->itr_val) {
		q_vector->itr_val = new_val;
		q_vector->set_itr = 1;
4439
	}
4440
clear_counts:
4441 4442 4443 4444
	q_vector->rx.total_bytes = 0;
	q_vector->rx.total_packets = 0;
	q_vector->tx.total_bytes = 0;
	q_vector->tx.total_packets = 0;
4445 4446 4447
}

/**
4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458
 *  igb_update_itr - update the dynamic ITR value based on statistics
 *  @q_vector: pointer to q_vector
 *  @ring_container: ring info to update the itr for
 *
 *  Stores a new ITR value based on packets and byte
 *  counts during the last interrupt.  The advantage of per interrupt
 *  computation is faster updates and more accurate ITR for the current
 *  traffic pattern.  Constants in this function were computed
 *  based on theoretical maximum wire speed and thresholds were set based
 *  on testing data as well as attempting to minimize response time
 *  while increasing bulk throughput.
4459
 *  This functionality is controlled by ethtool's coalescing settings.
4460 4461
 *  NOTE:  These calculations are only valid when operating in a single-
 *         queue environment.
4462
 **/
4463 4464
static void igb_update_itr(struct igb_q_vector *q_vector,
			   struct igb_ring_container *ring_container)
4465
{
4466 4467 4468
	unsigned int packets = ring_container->total_packets;
	unsigned int bytes = ring_container->total_bytes;
	u8 itrval = ring_container->itr;
4469

4470
	/* no packets, exit with status unchanged */
4471
	if (packets == 0)
4472
		return;
4473

4474
	switch (itrval) {
4475 4476 4477
	case lowest_latency:
		/* handle TSO and jumbo frames */
		if (bytes/packets > 8000)
4478
			itrval = bulk_latency;
4479
		else if ((packets < 5) && (bytes > 512))
4480
			itrval = low_latency;
4481 4482 4483 4484
		break;
	case low_latency:  /* 50 usec aka 20000 ints/s */
		if (bytes > 10000) {
			/* this if handles the TSO accounting */
4485
			if (bytes/packets > 8000)
4486
				itrval = bulk_latency;
4487
			else if ((packets < 10) || ((bytes/packets) > 1200))
4488
				itrval = bulk_latency;
4489
			else if ((packets > 35))
4490
				itrval = lowest_latency;
4491
		} else if (bytes/packets > 2000) {
4492
			itrval = bulk_latency;
4493
		} else if (packets <= 2 && bytes < 512) {
4494
			itrval = lowest_latency;
4495 4496 4497 4498 4499
		}
		break;
	case bulk_latency: /* 250 usec aka 4000 ints/s */
		if (bytes > 25000) {
			if (packets > 35)
4500
				itrval = low_latency;
4501
		} else if (bytes < 1500) {
4502
			itrval = low_latency;
4503 4504 4505 4506
		}
		break;
	}

4507 4508 4509 4510 4511 4512
	/* clear work counters since we have the values we need */
	ring_container->total_bytes = 0;
	ring_container->total_packets = 0;

	/* write updated itr to ring container */
	ring_container->itr = itrval;
4513 4514
}

4515
static void igb_set_itr(struct igb_q_vector *q_vector)
4516
{
4517
	struct igb_adapter *adapter = q_vector->adapter;
4518
	u32 new_itr = q_vector->itr_val;
4519
	u8 current_itr = 0;
4520 4521 4522 4523

	/* for non-gigabit speeds, just fix the interrupt rate at 4000 */
	if (adapter->link_speed != SPEED_1000) {
		current_itr = 0;
4524
		new_itr = IGB_4K_ITR;
4525 4526 4527
		goto set_itr_now;
	}

4528 4529
	igb_update_itr(q_vector, &q_vector->tx);
	igb_update_itr(q_vector, &q_vector->rx);
4530

4531
	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4532

4533
	/* conservative mode (itr 3) eliminates the lowest_latency setting */
4534 4535 4536
	if (current_itr == lowest_latency &&
	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
	     (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4537 4538
		current_itr = low_latency;

4539 4540 4541
	switch (current_itr) {
	/* counts and packets in update_itr are dependent on these numbers */
	case lowest_latency:
4542
		new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
4543 4544
		break;
	case low_latency:
4545
		new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
4546 4547
		break;
	case bulk_latency:
4548
		new_itr = IGB_4K_ITR;  /* 4,000 ints/sec */
4549 4550 4551 4552 4553 4554
		break;
	default:
		break;
	}

set_itr_now:
4555
	if (new_itr != q_vector->itr_val) {
4556 4557
		/* this attempts to bias the interrupt rate towards Bulk
		 * by adding intermediate steps when interrupt rate is
4558 4559
		 * increasing
		 */
4560
		new_itr = new_itr > q_vector->itr_val ?
4561 4562 4563
			  max((new_itr * q_vector->itr_val) /
			  (new_itr + (q_vector->itr_val >> 2)),
			  new_itr) : new_itr;
4564 4565 4566 4567 4568 4569
		/* Don't write the value here; it resets the adapter's
		 * internal timer, and causes us to delay far longer than
		 * we should between interrupts.  Instead, we write the ITR
		 * value at the beginning of the next interrupt so the timing
		 * ends up being correct.
		 */
4570 4571
		q_vector->itr_val = new_itr;
		q_vector->set_itr = 1;
4572 4573 4574
	}
}

4575 4576
static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
			    u32 type_tucmd, u32 mss_l4len_idx)
4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589
{
	struct e1000_adv_tx_context_desc *context_desc;
	u16 i = tx_ring->next_to_use;

	context_desc = IGB_TX_CTXTDESC(tx_ring, i);

	i++;
	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;

	/* set bits to identify this as an advanced context descriptor */
	type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;

	/* For 82575, context index must be unique per ring. */
4590
	if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4591 4592 4593 4594 4595 4596 4597 4598
		mss_l4len_idx |= tx_ring->reg_idx << 4;

	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
	context_desc->seqnum_seed	= 0;
	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
}

4599 4600 4601
static int igb_tso(struct igb_ring *tx_ring,
		   struct igb_tx_buffer *first,
		   u8 *hdr_len)
4602
{
4603
	struct sk_buff *skb = first->skb;
4604 4605
	u32 vlan_macip_lens, type_tucmd;
	u32 mss_l4len_idx, l4len;
4606
	int err;
4607

4608 4609 4610
	if (skb->ip_summed != CHECKSUM_PARTIAL)
		return 0;

4611 4612
	if (!skb_is_gso(skb))
		return 0;
4613

4614 4615 4616
	err = skb_cow_head(skb, 0);
	if (err < 0)
		return err;
4617

4618 4619
	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
	type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
4620

4621
	if (first->protocol == htons(ETH_P_IP)) {
4622 4623 4624 4625 4626 4627 4628
		struct iphdr *iph = ip_hdr(skb);
		iph->tot_len = 0;
		iph->check = 0;
		tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
							 iph->daddr, 0,
							 IPPROTO_TCP,
							 0);
4629
		type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4630 4631 4632
		first->tx_flags |= IGB_TX_FLAGS_TSO |
				   IGB_TX_FLAGS_CSUM |
				   IGB_TX_FLAGS_IPV4;
4633
	} else if (skb_is_gso_v6(skb)) {
4634 4635 4636 4637
		ipv6_hdr(skb)->payload_len = 0;
		tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
						       &ipv6_hdr(skb)->daddr,
						       0, IPPROTO_TCP, 0);
4638 4639
		first->tx_flags |= IGB_TX_FLAGS_TSO |
				   IGB_TX_FLAGS_CSUM;
4640 4641
	}

4642
	/* compute header lengths */
4643 4644
	l4len = tcp_hdrlen(skb);
	*hdr_len = skb_transport_offset(skb) + l4len;
4645

4646 4647 4648 4649
	/* update gso size and bytecount with header size */
	first->gso_segs = skb_shinfo(skb)->gso_segs;
	first->bytecount += (first->gso_segs - 1) * *hdr_len;

4650
	/* MSS L4LEN IDX */
4651 4652
	mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
	mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
4653

4654 4655 4656
	/* VLAN MACLEN IPLEN */
	vlan_macip_lens = skb_network_header_len(skb);
	vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4657
	vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4658

4659
	igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4660

4661
	return 1;
4662 4663
}

4664
static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
4665
{
4666
	struct sk_buff *skb = first->skb;
4667 4668 4669
	u32 vlan_macip_lens = 0;
	u32 mss_l4len_idx = 0;
	u32 type_tucmd = 0;
4670

4671
	if (skb->ip_summed != CHECKSUM_PARTIAL) {
4672 4673
		if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
			return;
4674 4675
	} else {
		u8 l4_hdr = 0;
4676

4677
		switch (first->protocol) {
4678
		case htons(ETH_P_IP):
4679 4680 4681 4682
			vlan_macip_lens |= skb_network_header_len(skb);
			type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
			l4_hdr = ip_hdr(skb)->protocol;
			break;
4683
		case htons(ETH_P_IPV6):
4684 4685 4686 4687 4688 4689
			vlan_macip_lens |= skb_network_header_len(skb);
			l4_hdr = ipv6_hdr(skb)->nexthdr;
			break;
		default:
			if (unlikely(net_ratelimit())) {
				dev_warn(tx_ring->dev,
4690 4691
					 "partial checksum but proto=%x!\n",
					 first->protocol);
4692
			}
4693 4694
			break;
		}
4695

4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713
		switch (l4_hdr) {
		case IPPROTO_TCP:
			type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
			mss_l4len_idx = tcp_hdrlen(skb) <<
					E1000_ADVTXD_L4LEN_SHIFT;
			break;
		case IPPROTO_SCTP:
			type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
			mss_l4len_idx = sizeof(struct sctphdr) <<
					E1000_ADVTXD_L4LEN_SHIFT;
			break;
		case IPPROTO_UDP:
			mss_l4len_idx = sizeof(struct udphdr) <<
					E1000_ADVTXD_L4LEN_SHIFT;
			break;
		default:
			if (unlikely(net_ratelimit())) {
				dev_warn(tx_ring->dev,
4714 4715
					 "partial checksum but l4 proto=%x!\n",
					 l4_hdr);
4716
			}
4717
			break;
4718
		}
4719 4720 4721

		/* update TX checksum flag */
		first->tx_flags |= IGB_TX_FLAGS_CSUM;
4722
	}
4723

4724
	vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4725
	vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4726

4727
	igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4728 4729
}

4730 4731 4732 4733 4734 4735
#define IGB_SET_FLAG(_input, _flag, _result) \
	((_flag <= _result) ? \
	 ((u32)(_input & _flag) * (_result / _flag)) : \
	 ((u32)(_input & _flag) / (_flag / _result)))

static u32 igb_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
4736 4737
{
	/* set type for advanced descriptor with frame checksum insertion */
4738 4739 4740
	u32 cmd_type = E1000_ADVTXD_DTYP_DATA |
		       E1000_ADVTXD_DCMD_DEXT |
		       E1000_ADVTXD_DCMD_IFCS;
4741 4742

	/* set HW vlan bit if vlan is present */
4743 4744 4745 4746 4747 4748
	cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_VLAN,
				 (E1000_ADVTXD_DCMD_VLE));

	/* set segmentation bits for TSO */
	cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSO,
				 (E1000_ADVTXD_DCMD_TSE));
4749 4750

	/* set timestamp bit if present */
4751 4752
	cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSTAMP,
				 (E1000_ADVTXD_MAC_TSTAMP));
4753

4754 4755
	/* insert frame checksum */
	cmd_type ^= IGB_SET_FLAG(skb->no_fcs, 1, E1000_ADVTXD_DCMD_IFCS);
4756 4757 4758 4759

	return cmd_type;
}

4760 4761 4762
static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
				 union e1000_adv_tx_desc *tx_desc,
				 u32 tx_flags, unsigned int paylen)
4763 4764 4765
{
	u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;

4766 4767
	/* 82575 requires a unique index per ring */
	if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4768 4769 4770
		olinfo_status |= tx_ring->reg_idx << 4;

	/* insert L4 checksum */
4771 4772 4773
	olinfo_status |= IGB_SET_FLAG(tx_flags,
				      IGB_TX_FLAGS_CSUM,
				      (E1000_TXD_POPTS_TXSM << 8));
4774

4775 4776 4777 4778
	/* insert IPv4 checksum */
	olinfo_status |= IGB_SET_FLAG(tx_flags,
				      IGB_TX_FLAGS_IPV4,
				      (E1000_TXD_POPTS_IXSM << 8));
4779

4780
	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4781 4782
}

4783 4784
static void igb_tx_map(struct igb_ring *tx_ring,
		       struct igb_tx_buffer *first,
4785
		       const u8 hdr_len)
4786
{
4787
	struct sk_buff *skb = first->skb;
4788
	struct igb_tx_buffer *tx_buffer;
4789
	union e1000_adv_tx_desc *tx_desc;
4790
	struct skb_frag_struct *frag;
4791
	dma_addr_t dma;
4792
	unsigned int data_len, size;
4793
	u32 tx_flags = first->tx_flags;
4794
	u32 cmd_type = igb_tx_cmd_type(skb, tx_flags);
4795 4796 4797 4798
	u16 i = tx_ring->next_to_use;

	tx_desc = IGB_TX_DESC(tx_ring, i);

4799 4800 4801 4802
	igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);

	size = skb_headlen(skb);
	data_len = skb->data_len;
4803 4804

	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4805

4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816
	tx_buffer = first;

	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
		if (dma_mapping_error(tx_ring->dev, dma))
			goto dma_error;

		/* record length, and DMA address */
		dma_unmap_len_set(tx_buffer, len, size);
		dma_unmap_addr_set(tx_buffer, dma, dma);

		tx_desc->read.buffer_addr = cpu_to_le64(dma);
4817 4818 4819

		while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
			tx_desc->read.cmd_type_len =
4820
				cpu_to_le32(cmd_type ^ IGB_MAX_DATA_PER_TXD);
4821 4822 4823 4824 4825 4826 4827

			i++;
			tx_desc++;
			if (i == tx_ring->count) {
				tx_desc = IGB_TX_DESC(tx_ring, 0);
				i = 0;
			}
4828
			tx_desc->read.olinfo_status = 0;
4829 4830 4831 4832 4833 4834 4835 4836 4837

			dma += IGB_MAX_DATA_PER_TXD;
			size -= IGB_MAX_DATA_PER_TXD;

			tx_desc->read.buffer_addr = cpu_to_le64(dma);
		}

		if (likely(!data_len))
			break;
4838

4839
		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
4840

4841
		i++;
4842 4843 4844
		tx_desc++;
		if (i == tx_ring->count) {
			tx_desc = IGB_TX_DESC(tx_ring, 0);
4845
			i = 0;
4846
		}
4847
		tx_desc->read.olinfo_status = 0;
4848

E
Eric Dumazet 已提交
4849
		size = skb_frag_size(frag);
4850 4851 4852
		data_len -= size;

		dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4853
				       size, DMA_TO_DEVICE);
4854

4855
		tx_buffer = &tx_ring->tx_buffer_info[i];
4856 4857
	}

4858
	/* write last descriptor with RS and EOP bits */
4859 4860
	cmd_type |= size | IGB_TXD_DCMD;
	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
4861

4862 4863
	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);

4864 4865 4866
	/* set the timestamp */
	first->time_stamp = jiffies;

4867
	/* Force memory writes to complete before letting h/w know there
4868 4869 4870 4871 4872 4873 4874 4875
	 * are new descriptors to fetch.  (Only applicable for weak-ordered
	 * memory model archs, such as IA-64).
	 *
	 * We also need this memory barrier to make certain all of the
	 * status bits have been updated before next_to_watch is written.
	 */
	wmb();

4876
	/* set next_to_watch value indicating a packet is present */
4877
	first->next_to_watch = tx_desc;
4878

4879 4880 4881
	i++;
	if (i == tx_ring->count)
		i = 0;
4882

4883
	tx_ring->next_to_use = i;
4884

4885
	writel(i, tx_ring->tail);
4886

4887
	/* we need this if more than one processor can write to our tail
4888 4889
	 * at a time, it synchronizes IO on IA64/Altix systems
	 */
4890 4891 4892 4893 4894 4895 4896 4897 4898
	mmiowb();

	return;

dma_error:
	dev_err(tx_ring->dev, "TX DMA map failed\n");

	/* clear dma mappings for failed tx_buffer_info map */
	for (;;) {
4899 4900 4901
		tx_buffer = &tx_ring->tx_buffer_info[i];
		igb_unmap_and_free_tx_resource(tx_ring, tx_buffer);
		if (tx_buffer == first)
4902
			break;
4903 4904
		if (i == 0)
			i = tx_ring->count;
4905 4906 4907
		i--;
	}

4908 4909 4910
	tx_ring->next_to_use = i;
}

4911
static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4912
{
4913 4914
	struct net_device *netdev = tx_ring->netdev;

4915 4916
	netif_stop_subqueue(netdev, tx_ring->queue_index);

4917 4918
	/* Herbert's original patch had:
	 *  smp_mb__after_netif_stop_queue();
4919 4920
	 * but since that doesn't exist yet, just open code it.
	 */
4921 4922 4923
	smp_mb();

	/* We need to check again in a case another CPU has just
4924 4925
	 * made room available.
	 */
4926
	if (igb_desc_unused(tx_ring) < size)
4927 4928 4929
		return -EBUSY;

	/* A reprieve! */
4930
	netif_wake_subqueue(netdev, tx_ring->queue_index);
E
Eric Dumazet 已提交
4931 4932 4933 4934 4935

	u64_stats_update_begin(&tx_ring->tx_syncp2);
	tx_ring->tx_stats.restart_queue2++;
	u64_stats_update_end(&tx_ring->tx_syncp2);

4936 4937 4938
	return 0;
}

4939
static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4940
{
4941
	if (igb_desc_unused(tx_ring) >= size)
4942
		return 0;
4943
	return __igb_maybe_stop_tx(tx_ring, size);
4944 4945
}

4946 4947
netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
				struct igb_ring *tx_ring)
4948
{
4949
	struct igb_tx_buffer *first;
4950
	int tso;
N
Nick Nunley 已提交
4951
	u32 tx_flags = 0;
4952
	u16 count = TXD_USE_COUNT(skb_headlen(skb));
4953
	__be16 protocol = vlan_get_protocol(skb);
N
Nick Nunley 已提交
4954
	u8 hdr_len = 0;
4955

4956 4957
	/* need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD,
	 *       + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD,
4958 4959
	 *       + 2 desc gap to keep tail from touching head,
	 *       + 1 desc for context descriptor,
4960 4961 4962 4963
	 * otherwise try next time
	 */
	if (NETDEV_FRAG_PAGE_MAX_SIZE > IGB_MAX_DATA_PER_TXD) {
		unsigned short f;
4964

4965 4966 4967 4968 4969 4970 4971
		for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
			count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
	} else {
		count += skb_shinfo(skb)->nr_frags;
	}

	if (igb_maybe_stop_tx(tx_ring, count + 3)) {
4972 4973 4974
		/* this is a hard error */
		return NETDEV_TX_BUSY;
	}
4975

4976 4977 4978 4979 4980 4981
	/* record the location of the first descriptor for this packet */
	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
	first->skb = skb;
	first->bytecount = skb->len;
	first->gso_segs = 1;

4982 4983
	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
		struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
4984

4985 4986
		if (!test_and_set_bit_lock(__IGB_PTP_TX_IN_PROGRESS,
					   &adapter->state)) {
4987 4988 4989 4990 4991 4992 4993 4994
			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
			tx_flags |= IGB_TX_FLAGS_TSTAMP;

			adapter->ptp_tx_skb = skb_get(skb);
			adapter->ptp_tx_start = jiffies;
			if (adapter->hw.mac.type == e1000_82576)
				schedule_work(&adapter->ptp_tx_work);
		}
4995
	}
4996

4997 4998
	skb_tx_timestamp(skb);

4999
	if (vlan_tx_tag_present(skb)) {
5000 5001 5002 5003
		tx_flags |= IGB_TX_FLAGS_VLAN;
		tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
	}

5004 5005 5006
	/* record initial flags and protocol */
	first->tx_flags = tx_flags;
	first->protocol = protocol;
A
Alexander Duyck 已提交
5007

5008 5009
	tso = igb_tso(tx_ring, first, &hdr_len);
	if (tso < 0)
5010
		goto out_drop;
5011 5012
	else if (!tso)
		igb_tx_csum(tx_ring, first);
5013

5014
	igb_tx_map(tx_ring, first, hdr_len);
5015 5016

	/* Make sure there is space in the ring for the next send. */
5017
	igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
5018

5019
	return NETDEV_TX_OK;
5020 5021

out_drop:
5022 5023
	igb_unmap_and_free_tx_resource(tx_ring, first);

5024
	return NETDEV_TX_OK;
5025 5026
}

5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037
static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
						    struct sk_buff *skb)
{
	unsigned int r_idx = skb->queue_mapping;

	if (r_idx >= adapter->num_tx_queues)
		r_idx = r_idx % adapter->num_tx_queues;

	return adapter->tx_ring[r_idx];
}

5038 5039
static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
				  struct net_device *netdev)
5040 5041
{
	struct igb_adapter *adapter = netdev_priv(netdev);
5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052

	if (test_bit(__IGB_DOWN, &adapter->state)) {
		dev_kfree_skb_any(skb);
		return NETDEV_TX_OK;
	}

	if (skb->len <= 0) {
		dev_kfree_skb_any(skb);
		return NETDEV_TX_OK;
	}

5053
	/* The minimum packet size with TCTL.PSP set is 17 so pad the skb
5054 5055
	 * in order to meet this minimum size requirement.
	 */
5056 5057
	if (unlikely(skb->len < 17)) {
		if (skb_pad(skb, 17 - skb->len))
5058 5059
			return NETDEV_TX_OK;
		skb->len = 17;
5060
		skb_set_tail_pointer(skb, 17);
5061
	}
5062

5063
	return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
5064 5065 5066
}

/**
5067 5068
 *  igb_tx_timeout - Respond to a Tx Hang
 *  @netdev: network interface device structure
5069 5070 5071 5072 5073 5074 5075 5076
 **/
static void igb_tx_timeout(struct net_device *netdev)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;

	/* Do the reset outside of interrupt context */
	adapter->tx_timeout_count++;
5077

5078
	if (hw->mac.type >= e1000_82580)
5079 5080
		hw->dev_spec._82575.global_device_reset = true;

5081
	schedule_work(&adapter->reset_task);
5082 5083
	wr32(E1000_EICS,
	     (adapter->eims_enable_mask & ~adapter->eims_other));
5084 5085 5086 5087 5088 5089 5090
}

static void igb_reset_task(struct work_struct *work)
{
	struct igb_adapter *adapter;
	adapter = container_of(work, struct igb_adapter, reset_task);

5091 5092
	igb_dump(adapter);
	netdev_err(adapter->netdev, "Reset adapter\n");
5093 5094 5095 5096
	igb_reinit_locked(adapter);
}

/**
5097 5098 5099
 *  igb_get_stats64 - Get System Network Statistics
 *  @netdev: network interface device structure
 *  @stats: rtnl_link_stats64 pointer
5100
 **/
E
Eric Dumazet 已提交
5101
static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
5102
						struct rtnl_link_stats64 *stats)
5103
{
E
Eric Dumazet 已提交
5104 5105 5106 5107 5108 5109 5110 5111
	struct igb_adapter *adapter = netdev_priv(netdev);

	spin_lock(&adapter->stats64_lock);
	igb_update_stats(adapter, &adapter->stats64);
	memcpy(stats, &adapter->stats64, sizeof(*stats));
	spin_unlock(&adapter->stats64_lock);

	return stats;
5112 5113 5114
}

/**
5115 5116 5117
 *  igb_change_mtu - Change the Maximum Transfer Unit
 *  @netdev: network interface device structure
 *  @new_mtu: new value for maximum frame size
5118
 *
5119
 *  Returns 0 on success, negative on failure
5120 5121 5122 5123
 **/
static int igb_change_mtu(struct net_device *netdev, int new_mtu)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
5124
	struct pci_dev *pdev = adapter->pdev;
5125
	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
5126

5127
	if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
5128
		dev_err(&pdev->dev, "Invalid MTU setting\n");
5129 5130 5131
		return -EINVAL;
	}

5132
#define MAX_STD_JUMBO_FRAME_SIZE 9238
5133
	if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
5134
		dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
5135 5136 5137
		return -EINVAL;
	}

5138 5139 5140 5141
	/* adjust max frame to be at least the size of a standard frame */
	if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
		max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;

5142
	while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
5143
		usleep_range(1000, 2000);
5144

5145 5146
	/* igb_down has a dependency on max_frame_size */
	adapter->max_frame_size = max_frame;
5147

5148 5149
	if (netif_running(netdev))
		igb_down(adapter);
5150

5151
	dev_info(&pdev->dev, "changing MTU from %d to %d\n",
5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165
		 netdev->mtu, new_mtu);
	netdev->mtu = new_mtu;

	if (netif_running(netdev))
		igb_up(adapter);
	else
		igb_reset(adapter);

	clear_bit(__IGB_RESETTING, &adapter->state);

	return 0;
}

/**
5166 5167
 *  igb_update_stats - Update the board statistics counters
 *  @adapter: board private structure
5168
 **/
E
Eric Dumazet 已提交
5169 5170
void igb_update_stats(struct igb_adapter *adapter,
		      struct rtnl_link_stats64 *net_stats)
5171 5172 5173
{
	struct e1000_hw *hw = &adapter->hw;
	struct pci_dev *pdev = adapter->pdev;
5174
	u32 reg, mpc;
5175
	u16 phy_tmp;
5176 5177
	int i;
	u64 bytes, packets;
E
Eric Dumazet 已提交
5178 5179
	unsigned int start;
	u64 _bytes, _packets;
5180 5181 5182

#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF

5183
	/* Prevent stats update while adapter is being reset, or if the pci
5184 5185 5186 5187 5188 5189 5190
	 * connection is down.
	 */
	if (adapter->link_speed == 0)
		return;
	if (pci_channel_offline(pdev))
		return;

5191 5192
	bytes = 0;
	packets = 0;
5193 5194

	rcu_read_lock();
5195
	for (i = 0; i < adapter->num_rx_queues; i++) {
5196
		struct igb_ring *ring = adapter->rx_ring[i];
5197 5198 5199
		u32 rqdpc = rd32(E1000_RQDPC(i));
		if (hw->mac.type >= e1000_i210)
			wr32(E1000_RQDPC(i), 0);
E
Eric Dumazet 已提交
5200

5201 5202 5203 5204
		if (rqdpc) {
			ring->rx_stats.drops += rqdpc;
			net_stats->rx_fifo_errors += rqdpc;
		}
E
Eric Dumazet 已提交
5205 5206

		do {
5207
			start = u64_stats_fetch_begin_irq(&ring->rx_syncp);
E
Eric Dumazet 已提交
5208 5209
			_bytes = ring->rx_stats.bytes;
			_packets = ring->rx_stats.packets;
5210
		} while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start));
E
Eric Dumazet 已提交
5211 5212
		bytes += _bytes;
		packets += _packets;
5213 5214
	}

5215 5216
	net_stats->rx_bytes = bytes;
	net_stats->rx_packets = packets;
5217 5218 5219 5220

	bytes = 0;
	packets = 0;
	for (i = 0; i < adapter->num_tx_queues; i++) {
5221
		struct igb_ring *ring = adapter->tx_ring[i];
E
Eric Dumazet 已提交
5222
		do {
5223
			start = u64_stats_fetch_begin_irq(&ring->tx_syncp);
E
Eric Dumazet 已提交
5224 5225
			_bytes = ring->tx_stats.bytes;
			_packets = ring->tx_stats.packets;
5226
		} while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start));
E
Eric Dumazet 已提交
5227 5228
		bytes += _bytes;
		packets += _packets;
5229
	}
5230 5231
	net_stats->tx_bytes = bytes;
	net_stats->tx_packets = packets;
5232
	rcu_read_unlock();
5233 5234

	/* read stats registers */
5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251
	adapter->stats.crcerrs += rd32(E1000_CRCERRS);
	adapter->stats.gprc += rd32(E1000_GPRC);
	adapter->stats.gorc += rd32(E1000_GORCL);
	rd32(E1000_GORCH); /* clear GORCL */
	adapter->stats.bprc += rd32(E1000_BPRC);
	adapter->stats.mprc += rd32(E1000_MPRC);
	adapter->stats.roc += rd32(E1000_ROC);

	adapter->stats.prc64 += rd32(E1000_PRC64);
	adapter->stats.prc127 += rd32(E1000_PRC127);
	adapter->stats.prc255 += rd32(E1000_PRC255);
	adapter->stats.prc511 += rd32(E1000_PRC511);
	adapter->stats.prc1023 += rd32(E1000_PRC1023);
	adapter->stats.prc1522 += rd32(E1000_PRC1522);
	adapter->stats.symerrs += rd32(E1000_SYMERRS);
	adapter->stats.sec += rd32(E1000_SEC);

5252 5253 5254
	mpc = rd32(E1000_MPC);
	adapter->stats.mpc += mpc;
	net_stats->rx_fifo_errors += mpc;
5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268
	adapter->stats.scc += rd32(E1000_SCC);
	adapter->stats.ecol += rd32(E1000_ECOL);
	adapter->stats.mcc += rd32(E1000_MCC);
	adapter->stats.latecol += rd32(E1000_LATECOL);
	adapter->stats.dc += rd32(E1000_DC);
	adapter->stats.rlec += rd32(E1000_RLEC);
	adapter->stats.xonrxc += rd32(E1000_XONRXC);
	adapter->stats.xontxc += rd32(E1000_XONTXC);
	adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
	adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
	adapter->stats.fcruc += rd32(E1000_FCRUC);
	adapter->stats.gptc += rd32(E1000_GPTC);
	adapter->stats.gotc += rd32(E1000_GOTCL);
	rd32(E1000_GOTCH); /* clear GOTCL */
5269
	adapter->stats.rnbc += rd32(E1000_RNBC);
5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286
	adapter->stats.ruc += rd32(E1000_RUC);
	adapter->stats.rfc += rd32(E1000_RFC);
	adapter->stats.rjc += rd32(E1000_RJC);
	adapter->stats.tor += rd32(E1000_TORH);
	adapter->stats.tot += rd32(E1000_TOTH);
	adapter->stats.tpr += rd32(E1000_TPR);

	adapter->stats.ptc64 += rd32(E1000_PTC64);
	adapter->stats.ptc127 += rd32(E1000_PTC127);
	adapter->stats.ptc255 += rd32(E1000_PTC255);
	adapter->stats.ptc511 += rd32(E1000_PTC511);
	adapter->stats.ptc1023 += rd32(E1000_PTC1023);
	adapter->stats.ptc1522 += rd32(E1000_PTC1522);

	adapter->stats.mptc += rd32(E1000_MPTC);
	adapter->stats.bptc += rd32(E1000_BPTC);

5287 5288
	adapter->stats.tpt += rd32(E1000_TPT);
	adapter->stats.colc += rd32(E1000_COLC);
5289 5290

	adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
5291 5292 5293 5294
	/* read internal phy specific stats */
	reg = rd32(E1000_CTRL_EXT);
	if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
		adapter->stats.rxerrc += rd32(E1000_RXERRC);
5295 5296 5297 5298 5299

		/* this stat has invalid values on i210/i211 */
		if ((hw->mac.type != e1000_i210) &&
		    (hw->mac.type != e1000_i211))
			adapter->stats.tncrs += rd32(E1000_TNCRS);
5300 5301
	}

5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315
	adapter->stats.tsctc += rd32(E1000_TSCTC);
	adapter->stats.tsctfc += rd32(E1000_TSCTFC);

	adapter->stats.iac += rd32(E1000_IAC);
	adapter->stats.icrxoc += rd32(E1000_ICRXOC);
	adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
	adapter->stats.icrxatc += rd32(E1000_ICRXATC);
	adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
	adapter->stats.ictxatc += rd32(E1000_ICTXATC);
	adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
	adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
	adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);

	/* Fill out the OS statistics structure */
5316 5317
	net_stats->multicast = adapter->stats.mprc;
	net_stats->collisions = adapter->stats.colc;
5318 5319 5320 5321

	/* Rx Errors */

	/* RLEC on some newer hardware can be incorrect so build
5322 5323
	 * our own version based on RUC and ROC
	 */
5324
	net_stats->rx_errors = adapter->stats.rxerrc +
5325 5326 5327
		adapter->stats.crcerrs + adapter->stats.algnerrc +
		adapter->stats.ruc + adapter->stats.roc +
		adapter->stats.cexterr;
5328 5329 5330 5331 5332
	net_stats->rx_length_errors = adapter->stats.ruc +
				      adapter->stats.roc;
	net_stats->rx_crc_errors = adapter->stats.crcerrs;
	net_stats->rx_frame_errors = adapter->stats.algnerrc;
	net_stats->rx_missed_errors = adapter->stats.mpc;
5333 5334

	/* Tx Errors */
5335 5336 5337 5338 5339
	net_stats->tx_errors = adapter->stats.ecol +
			       adapter->stats.latecol;
	net_stats->tx_aborted_errors = adapter->stats.ecol;
	net_stats->tx_window_errors = adapter->stats.latecol;
	net_stats->tx_carrier_errors = adapter->stats.tncrs;
5340 5341 5342 5343 5344 5345

	/* Tx Dropped needs to be maintained elsewhere */

	/* Phy Stats */
	if (hw->phy.media_type == e1000_media_type_copper) {
		if ((adapter->link_speed == SPEED_1000) &&
5346
		   (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
5347 5348 5349 5350 5351 5352 5353 5354 5355
			phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
			adapter->phy_stats.idle_errors += phy_tmp;
		}
	}

	/* Management Stats */
	adapter->stats.mgptc += rd32(E1000_MGTPTC);
	adapter->stats.mgprc += rd32(E1000_MGTPRC);
	adapter->stats.mgpdc += rd32(E1000_MGTPDC);
5356 5357 5358 5359 5360 5361 5362 5363 5364

	/* OS2BMC Stats */
	reg = rd32(E1000_MANC);
	if (reg & E1000_MANC_EN_BMC2OS) {
		adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
		adapter->stats.o2bspc += rd32(E1000_O2BSPC);
		adapter->stats.b2ospc += rd32(E1000_B2OSPC);
		adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
	}
5365 5366 5367 5368
}

static irqreturn_t igb_msix_other(int irq, void *data)
{
5369
	struct igb_adapter *adapter = data;
5370
	struct e1000_hw *hw = &adapter->hw;
P
PJ Waskiewicz 已提交
5371 5372
	u32 icr = rd32(E1000_ICR);
	/* reading ICR causes bit 31 of EICR to be cleared */
5373

5374 5375 5376
	if (icr & E1000_ICR_DRSTA)
		schedule_work(&adapter->reset_task);

5377
	if (icr & E1000_ICR_DOUTSYNC) {
5378 5379
		/* HW is reporting DMA is out of sync */
		adapter->stats.doosync++;
G
Greg Rose 已提交
5380 5381
		/* The DMA Out of Sync is also indication of a spoof event
		 * in IOV mode. Check the Wrong VM Behavior register to
5382 5383
		 * see if it is really a spoof event.
		 */
G
Greg Rose 已提交
5384
		igb_check_wvbr(adapter);
5385
	}
5386

5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397
	/* Check for a mailbox event */
	if (icr & E1000_ICR_VMMB)
		igb_msg_task(adapter);

	if (icr & E1000_ICR_LSC) {
		hw->mac.get_link_status = 1;
		/* guard against interrupt when we're going down */
		if (!test_bit(__IGB_DOWN, &adapter->state))
			mod_timer(&adapter->watchdog_timer, jiffies + 1);
	}

5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408
	if (icr & E1000_ICR_TS) {
		u32 tsicr = rd32(E1000_TSICR);

		if (tsicr & E1000_TSICR_TXTS) {
			/* acknowledge the interrupt */
			wr32(E1000_TSICR, E1000_TSICR_TXTS);
			/* retrieve hardware timestamp */
			schedule_work(&adapter->ptp_tx_work);
		}
	}

P
PJ Waskiewicz 已提交
5409
	wr32(E1000_EIMS, adapter->eims_other);
5410 5411 5412 5413

	return IRQ_HANDLED;
}

5414
static void igb_write_itr(struct igb_q_vector *q_vector)
5415
{
5416
	struct igb_adapter *adapter = q_vector->adapter;
5417
	u32 itr_val = q_vector->itr_val & 0x7FFC;
5418

5419 5420
	if (!q_vector->set_itr)
		return;
5421

5422 5423
	if (!itr_val)
		itr_val = 0x4;
5424

5425 5426
	if (adapter->hw.mac.type == e1000_82575)
		itr_val |= itr_val << 16;
5427
	else
5428
		itr_val |= E1000_EITR_CNT_IGNR;
5429

5430 5431
	writel(itr_val, q_vector->itr_register);
	q_vector->set_itr = 0;
5432 5433
}

5434
static irqreturn_t igb_msix_ring(int irq, void *data)
5435
{
5436
	struct igb_q_vector *q_vector = data;
5437

5438 5439
	/* Write the ITR value calculated from the previous interrupt. */
	igb_write_itr(q_vector);
5440

5441
	napi_schedule(&q_vector->napi);
P
PJ Waskiewicz 已提交
5442

5443
	return IRQ_HANDLED;
J
Jeb Cramer 已提交
5444 5445
}

5446
#ifdef CONFIG_IGB_DCA
5447 5448 5449 5450 5451 5452 5453 5454 5455 5456
static void igb_update_tx_dca(struct igb_adapter *adapter,
			      struct igb_ring *tx_ring,
			      int cpu)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);

	if (hw->mac.type != e1000_82575)
		txctrl <<= E1000_DCA_TXCTRL_CPUID_SHIFT;

5457
	/* We can enable relaxed ordering for reads, but not writes when
5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477
	 * DCA is enabled.  This is due to a known issue in some chipsets
	 * which will cause the DCA tag to be cleared.
	 */
	txctrl |= E1000_DCA_TXCTRL_DESC_RRO_EN |
		  E1000_DCA_TXCTRL_DATA_RRO_EN |
		  E1000_DCA_TXCTRL_DESC_DCA_EN;

	wr32(E1000_DCA_TXCTRL(tx_ring->reg_idx), txctrl);
}

static void igb_update_rx_dca(struct igb_adapter *adapter,
			      struct igb_ring *rx_ring,
			      int cpu)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu);

	if (hw->mac.type != e1000_82575)
		rxctrl <<= E1000_DCA_RXCTRL_CPUID_SHIFT;

5478
	/* We can enable relaxed ordering for reads, but not writes when
5479 5480 5481 5482 5483 5484 5485 5486 5487
	 * DCA is enabled.  This is due to a known issue in some chipsets
	 * which will cause the DCA tag to be cleared.
	 */
	rxctrl |= E1000_DCA_RXCTRL_DESC_RRO_EN |
		  E1000_DCA_RXCTRL_DESC_DCA_EN;

	wr32(E1000_DCA_RXCTRL(rx_ring->reg_idx), rxctrl);
}

5488
static void igb_update_dca(struct igb_q_vector *q_vector)
J
Jeb Cramer 已提交
5489
{
5490
	struct igb_adapter *adapter = q_vector->adapter;
J
Jeb Cramer 已提交
5491 5492
	int cpu = get_cpu();

5493 5494 5495
	if (q_vector->cpu == cpu)
		goto out_no_update;

5496 5497 5498 5499 5500 5501
	if (q_vector->tx.ring)
		igb_update_tx_dca(adapter, q_vector->tx.ring, cpu);

	if (q_vector->rx.ring)
		igb_update_rx_dca(adapter, q_vector->rx.ring, cpu);

5502 5503
	q_vector->cpu = cpu;
out_no_update:
J
Jeb Cramer 已提交
5504 5505 5506 5507 5508
	put_cpu();
}

static void igb_setup_dca(struct igb_adapter *adapter)
{
5509
	struct e1000_hw *hw = &adapter->hw;
J
Jeb Cramer 已提交
5510 5511
	int i;

5512
	if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
J
Jeb Cramer 已提交
5513 5514
		return;

5515 5516 5517
	/* Always use CB2 mode, difference is masked in the CB driver. */
	wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);

5518
	for (i = 0; i < adapter->num_q_vectors; i++) {
5519 5520
		adapter->q_vector[i]->cpu = -1;
		igb_update_dca(adapter->q_vector[i]);
J
Jeb Cramer 已提交
5521 5522 5523 5524 5525 5526 5527
	}
}

static int __igb_notify_dca(struct device *dev, void *data)
{
	struct net_device *netdev = dev_get_drvdata(dev);
	struct igb_adapter *adapter = netdev_priv(netdev);
5528
	struct pci_dev *pdev = adapter->pdev;
J
Jeb Cramer 已提交
5529 5530 5531 5532 5533 5534
	struct e1000_hw *hw = &adapter->hw;
	unsigned long event = *(unsigned long *)data;

	switch (event) {
	case DCA_PROVIDER_ADD:
		/* if already enabled, don't do it again */
5535
		if (adapter->flags & IGB_FLAG_DCA_ENABLED)
J
Jeb Cramer 已提交
5536 5537
			break;
		if (dca_add_requester(dev) == 0) {
5538
			adapter->flags |= IGB_FLAG_DCA_ENABLED;
5539
			dev_info(&pdev->dev, "DCA enabled\n");
J
Jeb Cramer 已提交
5540 5541 5542 5543 5544
			igb_setup_dca(adapter);
			break;
		}
		/* Fall Through since DCA is disabled. */
	case DCA_PROVIDER_REMOVE:
5545
		if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
J
Jeb Cramer 已提交
5546
			/* without this a class_device is left
5547 5548
			 * hanging around in the sysfs model
			 */
J
Jeb Cramer 已提交
5549
			dca_remove_requester(dev);
5550
			dev_info(&pdev->dev, "DCA disabled\n");
5551
			adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
A
Alexander Duyck 已提交
5552
			wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
J
Jeb Cramer 已提交
5553 5554 5555
		}
		break;
	}
5556

J
Jeb Cramer 已提交
5557
	return 0;
5558 5559
}

J
Jeb Cramer 已提交
5560
static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
5561
			  void *p)
J
Jeb Cramer 已提交
5562 5563 5564 5565
{
	int ret_val;

	ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
5566
					 __igb_notify_dca);
J
Jeb Cramer 已提交
5567 5568 5569

	return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
}
5570
#endif /* CONFIG_IGB_DCA */
5571

5572 5573 5574 5575 5576
#ifdef CONFIG_PCI_IOV
static int igb_vf_configure(struct igb_adapter *adapter, int vf)
{
	unsigned char mac_addr[ETH_ALEN];

5577
	eth_zero_addr(mac_addr);
5578 5579
	igb_set_vf_mac(adapter, vf, mac_addr);

L
Lior Levy 已提交
5580 5581 5582
	/* By default spoof check is enabled for all VFs */
	adapter->vf_data[vf].spoofchk_enabled = true;

5583
	return 0;
5584 5585 5586
}

#endif
5587 5588 5589 5590 5591 5592 5593 5594
static void igb_ping_all_vfs(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 ping;
	int i;

	for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
		ping = E1000_PF_CONTROL_MSG;
5595
		if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
5596 5597 5598 5599 5600
			ping |= E1000_VT_MSGTYPE_CTS;
		igb_write_mbx(hw, &ping, 1, i);
	}
}

5601 5602 5603 5604 5605 5606
static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 vmolr = rd32(E1000_VMOLR(vf));
	struct vf_data_storage *vf_data = &adapter->vf_data[vf];

5607
	vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
5608
			    IGB_VF_FLAG_MULTI_PROMISC);
5609 5610 5611 5612
	vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);

	if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
		vmolr |= E1000_VMOLR_MPME;
5613
		vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
5614 5615
		*msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
	} else {
5616
		/* if we have hashes and we are clearing a multicast promisc
5617 5618 5619 5620 5621 5622 5623
		 * flag we need to write the hashes to the MTA as this step
		 * was previously skipped
		 */
		if (vf_data->num_vf_mc_hashes > 30) {
			vmolr |= E1000_VMOLR_MPME;
		} else if (vf_data->num_vf_mc_hashes) {
			int j;
5624

5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639
			vmolr |= E1000_VMOLR_ROMPE;
			for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
				igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
		}
	}

	wr32(E1000_VMOLR(vf), vmolr);

	/* there are flags left unprocessed, likely not supported */
	if (*msgbuf & E1000_VT_MSGINFO_MASK)
		return -EINVAL;

	return 0;
}

5640 5641 5642 5643 5644 5645 5646 5647
static int igb_set_vf_multicasts(struct igb_adapter *adapter,
				  u32 *msgbuf, u32 vf)
{
	int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
	u16 *hash_list = (u16 *)&msgbuf[1];
	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
	int i;

5648
	/* salt away the number of multicast addresses assigned
5649 5650 5651 5652 5653
	 * to this VF for later use to restore when the PF multi cast
	 * list changes
	 */
	vf_data->num_vf_mc_hashes = n;

5654 5655 5656 5657 5658
	/* only up to 30 hash values supported */
	if (n > 30)
		n = 30;

	/* store the hashes for later use */
5659
	for (i = 0; i < n; i++)
5660
		vf_data->vf_mc_hashes[i] = hash_list[i];
5661 5662

	/* Flush and reset the mta with the new values */
5663
	igb_set_rx_mode(adapter->netdev);
5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674

	return 0;
}

static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	struct vf_data_storage *vf_data;
	int i, j;

	for (i = 0; i < adapter->vfs_allocated_count; i++) {
5675
		u32 vmolr = rd32(E1000_VMOLR(i));
5676

5677 5678
		vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);

5679
		vf_data = &adapter->vf_data[i];
5680 5681 5682 5683 5684 5685 5686 5687 5688 5689

		if ((vf_data->num_vf_mc_hashes > 30) ||
		    (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
			vmolr |= E1000_VMOLR_MPME;
		} else if (vf_data->num_vf_mc_hashes) {
			vmolr |= E1000_VMOLR_ROMPE;
			for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
				igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
		}
		wr32(E1000_VMOLR(i), vmolr);
5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717
	}
}

static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 pool_mask, reg, vid;
	int i;

	pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);

	/* Find the vlan filter for this id */
	for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
		reg = rd32(E1000_VLVF(i));

		/* remove the vf from the pool */
		reg &= ~pool_mask;

		/* if pool is empty then remove entry from vfta */
		if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
		    (reg & E1000_VLVF_VLANID_ENABLE)) {
			reg = 0;
			vid = reg & E1000_VLVF_VLANID_MASK;
			igb_vfta_set(hw, vid, false);
		}

		wr32(E1000_VLVF(i), reg);
	}
5718 5719

	adapter->vf_data[vf].vlans_enabled = 0;
5720 5721 5722 5723 5724 5725 5726
}

static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 reg, i;

5727 5728 5729 5730 5731
	/* The vlvf table only exists on 82576 hardware and newer */
	if (hw->mac.type < e1000_82576)
		return -1;

	/* we only need to do this if VMDq is enabled */
5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760
	if (!adapter->vfs_allocated_count)
		return -1;

	/* Find the vlan filter for this id */
	for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
		reg = rd32(E1000_VLVF(i));
		if ((reg & E1000_VLVF_VLANID_ENABLE) &&
		    vid == (reg & E1000_VLVF_VLANID_MASK))
			break;
	}

	if (add) {
		if (i == E1000_VLVF_ARRAY_SIZE) {
			/* Did not find a matching VLAN ID entry that was
			 * enabled.  Search for a free filter entry, i.e.
			 * one without the enable bit set
			 */
			for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
				reg = rd32(E1000_VLVF(i));
				if (!(reg & E1000_VLVF_VLANID_ENABLE))
					break;
			}
		}
		if (i < E1000_VLVF_ARRAY_SIZE) {
			/* Found an enabled/available entry */
			reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);

			/* if !enabled we need to set this up in vfta */
			if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
5761 5762
				/* add VID to filter table */
				igb_vfta_set(hw, vid, true);
5763 5764
				reg |= E1000_VLVF_VLANID_ENABLE;
			}
A
Alexander Duyck 已提交
5765 5766
			reg &= ~E1000_VLVF_VLANID_MASK;
			reg |= vid;
5767
			wr32(E1000_VLVF(i), reg);
5768 5769 5770 5771 5772 5773 5774

			/* do not modify RLPML for PF devices */
			if (vf >= adapter->vfs_allocated_count)
				return 0;

			if (!adapter->vf_data[vf].vlans_enabled) {
				u32 size;
5775

5776 5777 5778 5779 5780 5781 5782 5783
				reg = rd32(E1000_VMOLR(vf));
				size = reg & E1000_VMOLR_RLPML_MASK;
				size += 4;
				reg &= ~E1000_VMOLR_RLPML_MASK;
				reg |= size;
				wr32(E1000_VMOLR(vf), reg);
			}

5784
			adapter->vf_data[vf].vlans_enabled++;
5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795
		}
	} else {
		if (i < E1000_VLVF_ARRAY_SIZE) {
			/* remove vf from the pool */
			reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
			/* if pool is empty then remove entry from vfta */
			if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
				reg = 0;
				igb_vfta_set(hw, vid, false);
			}
			wr32(E1000_VLVF(i), reg);
5796 5797 5798 5799 5800 5801 5802 5803

			/* do not modify RLPML for PF devices */
			if (vf >= adapter->vfs_allocated_count)
				return 0;

			adapter->vf_data[vf].vlans_enabled--;
			if (!adapter->vf_data[vf].vlans_enabled) {
				u32 size;
5804

5805 5806 5807 5808 5809 5810 5811
				reg = rd32(E1000_VMOLR(vf));
				size = reg & E1000_VMOLR_RLPML_MASK;
				size -= 4;
				reg &= ~E1000_VMOLR_RLPML_MASK;
				reg |= size;
				wr32(E1000_VMOLR(vf), reg);
			}
5812 5813
		}
	}
5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846
	return 0;
}

static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
{
	struct e1000_hw *hw = &adapter->hw;

	if (vid)
		wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
	else
		wr32(E1000_VMVIR(vf), 0);
}

static int igb_ndo_set_vf_vlan(struct net_device *netdev,
			       int vf, u16 vlan, u8 qos)
{
	int err = 0;
	struct igb_adapter *adapter = netdev_priv(netdev);

	if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
		return -EINVAL;
	if (vlan || qos) {
		err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
		if (err)
			goto out;
		igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
		igb_set_vmolr(adapter, vf, !vlan);
		adapter->vf_data[vf].pf_vlan = vlan;
		adapter->vf_data[vf].pf_qos = qos;
		dev_info(&adapter->pdev->dev,
			 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
		if (test_bit(__IGB_DOWN, &adapter->state)) {
			dev_warn(&adapter->pdev->dev,
5847
				 "The VF VLAN has been set, but the PF device is not up.\n");
5848
			dev_warn(&adapter->pdev->dev,
5849
				 "Bring the PF device up before attempting to use the VF device.\n");
5850 5851 5852
		}
	} else {
		igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5853
			     false, vf);
5854 5855 5856 5857
		igb_set_vmvir(adapter, vlan, vf);
		igb_set_vmolr(adapter, vf, true);
		adapter->vf_data[vf].pf_vlan = 0;
		adapter->vf_data[vf].pf_qos = 0;
5858
	}
5859
out:
5860
	return err;
5861 5862
}

5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882
static int igb_find_vlvf_entry(struct igb_adapter *adapter, int vid)
{
	struct e1000_hw *hw = &adapter->hw;
	int i;
	u32 reg;

	/* Find the vlan filter for this id */
	for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
		reg = rd32(E1000_VLVF(i));
		if ((reg & E1000_VLVF_VLANID_ENABLE) &&
		    vid == (reg & E1000_VLVF_VLANID_MASK))
			break;
	}

	if (i >= E1000_VLVF_ARRAY_SIZE)
		i = -1;

	return i;
}

5883 5884
static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
{
5885
	struct e1000_hw *hw = &adapter->hw;
5886 5887
	int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
	int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5888
	int err = 0;
5889

5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909
	/* If in promiscuous mode we need to make sure the PF also has
	 * the VLAN filter set.
	 */
	if (add && (adapter->netdev->flags & IFF_PROMISC))
		err = igb_vlvf_set(adapter, vid, add,
				   adapter->vfs_allocated_count);
	if (err)
		goto out;

	err = igb_vlvf_set(adapter, vid, add, vf);

	if (err)
		goto out;

	/* Go through all the checks to see if the VLAN filter should
	 * be wiped completely.
	 */
	if (!add && (adapter->netdev->flags & IFF_PROMISC)) {
		u32 vlvf, bits;
		int regndx = igb_find_vlvf_entry(adapter, vid);
5910

5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931
		if (regndx < 0)
			goto out;
		/* See if any other pools are set for this VLAN filter
		 * entry other than the PF.
		 */
		vlvf = bits = rd32(E1000_VLVF(regndx));
		bits &= 1 << (E1000_VLVF_POOLSEL_SHIFT +
			      adapter->vfs_allocated_count);
		/* If the filter was removed then ensure PF pool bit
		 * is cleared if the PF only added itself to the pool
		 * because the PF is in promiscuous mode.
		 */
		if ((vlvf & VLAN_VID_MASK) == vid &&
		    !test_bit(vid, adapter->active_vlans) &&
		    !bits)
			igb_vlvf_set(adapter, vid, add,
				     adapter->vfs_allocated_count);
	}

out:
	return err;
5932 5933
}

5934
static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
5935
{
G
Greg Rose 已提交
5936 5937
	/* clear flags - except flag that indicates PF has set the MAC */
	adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
5938
	adapter->vf_data[vf].last_nack = jiffies;
5939 5940

	/* reset offloads to defaults */
5941
	igb_set_vmolr(adapter, vf, true);
5942 5943 5944

	/* reset vlans for device */
	igb_clear_vf_vfta(adapter, vf);
5945 5946 5947 5948 5949 5950
	if (adapter->vf_data[vf].pf_vlan)
		igb_ndo_set_vf_vlan(adapter->netdev, vf,
				    adapter->vf_data[vf].pf_vlan,
				    adapter->vf_data[vf].pf_qos);
	else
		igb_clear_vf_vfta(adapter, vf);
5951 5952 5953 5954 5955

	/* reset multicast table array for vf */
	adapter->vf_data[vf].num_vf_mc_hashes = 0;

	/* Flush and reset the mta with the new values */
5956
	igb_set_rx_mode(adapter->netdev);
5957 5958
}

5959 5960 5961 5962
static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
{
	unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;

5963
	/* clear mac address as we were hotplug removed/added */
5964
	if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5965
		eth_zero_addr(vf_mac);
5966 5967 5968 5969 5970 5971

	/* process remaining reset events */
	igb_vf_reset(adapter, vf);
}

static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5972 5973 5974
{
	struct e1000_hw *hw = &adapter->hw;
	unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5975
	int rar_entry = hw->mac.rar_entry_count - (vf + 1);
5976 5977 5978 5979
	u32 reg, msgbuf[3];
	u8 *addr = (u8 *)(&msgbuf[1]);

	/* process all the same items cleared in a function level reset */
5980
	igb_vf_reset(adapter, vf);
5981 5982

	/* set vf mac address */
5983
	igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
5984 5985 5986 5987 5988 5989 5990

	/* enable transmit and receive for vf */
	reg = rd32(E1000_VFTE);
	wr32(E1000_VFTE, reg | (1 << vf));
	reg = rd32(E1000_VFRE);
	wr32(E1000_VFRE, reg | (1 << vf));

G
Greg Rose 已提交
5991
	adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
5992 5993 5994

	/* reply to reset with ack and vf mac address */
	msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5995
	memcpy(addr, vf_mac, ETH_ALEN);
5996 5997 5998 5999 6000
	igb_write_mbx(hw, msgbuf, 3, vf);
}

static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
{
6001
	/* The VF MAC Address is stored in a packed array of bytes
G
Greg Rose 已提交
6002 6003
	 * starting at the second 32 bit word of the msg array
	 */
6004 6005
	unsigned char *addr = (char *)&msg[1];
	int err = -1;
6006

6007 6008
	if (is_valid_ether_addr(addr))
		err = igb_set_vf_mac(adapter, vf, addr);
6009

6010
	return err;
6011 6012 6013 6014 6015
}

static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
{
	struct e1000_hw *hw = &adapter->hw;
6016
	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6017 6018 6019
	u32 msg = E1000_VT_MSGTYPE_NACK;

	/* if device isn't clear to send it shouldn't be reading either */
6020 6021
	if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
	    time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
6022
		igb_write_mbx(hw, &msg, 1, vf);
6023
		vf_data->last_nack = jiffies;
6024 6025 6026
	}
}

6027
static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
6028
{
6029 6030
	struct pci_dev *pdev = adapter->pdev;
	u32 msgbuf[E1000_VFMAILBOX_SIZE];
6031
	struct e1000_hw *hw = &adapter->hw;
6032
	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6033 6034
	s32 retval;

6035
	retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
6036

6037 6038
	if (retval) {
		/* if receive failed revoke VF CTS stats and restart init */
6039
		dev_err(&pdev->dev, "Error receiving message from VF\n");
6040 6041 6042 6043 6044
		vf_data->flags &= ~IGB_VF_FLAG_CTS;
		if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
			return;
		goto out;
	}
6045 6046 6047

	/* this is a message we already processed, do nothing */
	if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
6048
		return;
6049

6050
	/* until the vf completes a reset it should not be
6051 6052 6053 6054
	 * allowed to start any configuration.
	 */
	if (msgbuf[0] == E1000_VF_RESET) {
		igb_vf_reset_msg(adapter, vf);
6055
		return;
6056 6057
	}

6058
	if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
6059 6060 6061 6062
		if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
			return;
		retval = -1;
		goto out;
6063 6064 6065 6066
	}

	switch ((msgbuf[0] & 0xFFFF)) {
	case E1000_VF_SET_MAC_ADDR:
6067 6068 6069 6070 6071
		retval = -EINVAL;
		if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
			retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
		else
			dev_warn(&pdev->dev,
6072 6073
				 "VF %d attempted to override administratively set MAC address\nReload the VF driver to resume operations\n",
				 vf);
6074
		break;
6075 6076 6077
	case E1000_VF_SET_PROMISC:
		retval = igb_set_vf_promisc(adapter, msgbuf, vf);
		break;
6078 6079 6080 6081 6082 6083 6084
	case E1000_VF_SET_MULTICAST:
		retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
		break;
	case E1000_VF_SET_LPE:
		retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
		break;
	case E1000_VF_SET_VLAN:
6085 6086 6087
		retval = -1;
		if (vf_data->pf_vlan)
			dev_warn(&pdev->dev,
6088 6089
				 "VF %d attempted to override administratively set VLAN tag\nReload the VF driver to resume operations\n",
				 vf);
6090 6091
		else
			retval = igb_set_vf_vlan(adapter, msgbuf, vf);
6092 6093
		break;
	default:
6094
		dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
6095 6096 6097 6098
		retval = -1;
		break;
	}

6099 6100
	msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
out:
6101 6102 6103 6104 6105 6106 6107
	/* notify the VF of the results of what it sent us */
	if (retval)
		msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
	else
		msgbuf[0] |= E1000_VT_MSGTYPE_ACK;

	igb_write_mbx(hw, msgbuf, 1, vf);
6108
}
6109

6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127
static void igb_msg_task(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 vf;

	for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
		/* process any reset requests */
		if (!igb_check_for_rst(hw, vf))
			igb_vf_reset_event(adapter, vf);

		/* process any messages pending */
		if (!igb_check_for_msg(hw, vf))
			igb_rcv_msg_from_vf(adapter, vf);

		/* process any acks */
		if (!igb_check_for_ack(hw, vf))
			igb_rcv_ack_from_vf(adapter, vf);
	}
6128 6129
}

6130 6131 6132 6133 6134 6135 6136
/**
 *  igb_set_uta - Set unicast filter table address
 *  @adapter: board private structure
 *
 *  The unicast table address is a register array of 32-bit registers.
 *  The table is meant to be used in a way similar to how the MTA is used
 *  however due to certain limitations in the hardware it is necessary to
L
Lucas De Marchi 已提交
6137 6138
 *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
 *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156
 **/
static void igb_set_uta(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	int i;

	/* The UTA table only exists on 82576 hardware and newer */
	if (hw->mac.type < e1000_82576)
		return;

	/* we only need to do this if VMDq is enabled */
	if (!adapter->vfs_allocated_count)
		return;

	for (i = 0; i < hw->mac.uta_reg_count; i++)
		array_wr32(E1000_UTA, i, ~0);
}

6157
/**
6158 6159 6160
 *  igb_intr_msi - Interrupt Handler
 *  @irq: interrupt number
 *  @data: pointer to a network interface device structure
6161 6162 6163
 **/
static irqreturn_t igb_intr_msi(int irq, void *data)
{
6164 6165
	struct igb_adapter *adapter = data;
	struct igb_q_vector *q_vector = adapter->q_vector[0];
6166 6167 6168 6169
	struct e1000_hw *hw = &adapter->hw;
	/* read ICR disables interrupts using IAM */
	u32 icr = rd32(E1000_ICR);

6170
	igb_write_itr(q_vector);
6171

6172 6173 6174
	if (icr & E1000_ICR_DRSTA)
		schedule_work(&adapter->reset_task);

6175
	if (icr & E1000_ICR_DOUTSYNC) {
6176 6177 6178 6179
		/* HW is reporting DMA is out of sync */
		adapter->stats.doosync++;
	}

6180 6181 6182 6183 6184 6185
	if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
		hw->mac.get_link_status = 1;
		if (!test_bit(__IGB_DOWN, &adapter->state))
			mod_timer(&adapter->watchdog_timer, jiffies + 1);
	}

6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196
	if (icr & E1000_ICR_TS) {
		u32 tsicr = rd32(E1000_TSICR);

		if (tsicr & E1000_TSICR_TXTS) {
			/* acknowledge the interrupt */
			wr32(E1000_TSICR, E1000_TSICR_TXTS);
			/* retrieve hardware timestamp */
			schedule_work(&adapter->ptp_tx_work);
		}
	}

6197
	napi_schedule(&q_vector->napi);
6198 6199 6200 6201 6202

	return IRQ_HANDLED;
}

/**
6203 6204 6205
 *  igb_intr - Legacy Interrupt Handler
 *  @irq: interrupt number
 *  @data: pointer to a network interface device structure
6206 6207 6208
 **/
static irqreturn_t igb_intr(int irq, void *data)
{
6209 6210
	struct igb_adapter *adapter = data;
	struct igb_q_vector *q_vector = adapter->q_vector[0];
6211 6212
	struct e1000_hw *hw = &adapter->hw;
	/* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
6213 6214
	 * need for the IMC write
	 */
6215 6216 6217
	u32 icr = rd32(E1000_ICR);

	/* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
6218 6219
	 * not set, then the adapter didn't send an interrupt
	 */
6220 6221 6222
	if (!(icr & E1000_ICR_INT_ASSERTED))
		return IRQ_NONE;

6223 6224
	igb_write_itr(q_vector);

6225 6226 6227
	if (icr & E1000_ICR_DRSTA)
		schedule_work(&adapter->reset_task);

6228
	if (icr & E1000_ICR_DOUTSYNC) {
6229 6230 6231 6232
		/* HW is reporting DMA is out of sync */
		adapter->stats.doosync++;
	}

6233 6234 6235 6236 6237 6238 6239
	if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
		hw->mac.get_link_status = 1;
		/* guard against interrupt when we're going down */
		if (!test_bit(__IGB_DOWN, &adapter->state))
			mod_timer(&adapter->watchdog_timer, jiffies + 1);
	}

6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250
	if (icr & E1000_ICR_TS) {
		u32 tsicr = rd32(E1000_TSICR);

		if (tsicr & E1000_TSICR_TXTS) {
			/* acknowledge the interrupt */
			wr32(E1000_TSICR, E1000_TSICR_TXTS);
			/* retrieve hardware timestamp */
			schedule_work(&adapter->ptp_tx_work);
		}
	}

6251
	napi_schedule(&q_vector->napi);
6252 6253 6254 6255

	return IRQ_HANDLED;
}

6256
static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
6257
{
6258
	struct igb_adapter *adapter = q_vector->adapter;
6259
	struct e1000_hw *hw = &adapter->hw;
6260

6261 6262 6263 6264
	if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
	    (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
		if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
			igb_set_itr(q_vector);
6265
		else
6266
			igb_update_ring_itr(q_vector);
6267 6268
	}

6269
	if (!test_bit(__IGB_DOWN, &adapter->state)) {
6270
		if (adapter->flags & IGB_FLAG_HAS_MSIX)
6271
			wr32(E1000_EIMS, q_vector->eims_value);
6272 6273 6274
		else
			igb_irq_enable(adapter);
	}
6275 6276
}

6277
/**
6278 6279 6280
 *  igb_poll - NAPI Rx polling callback
 *  @napi: napi polling structure
 *  @budget: count of how many packets we should handle
6281 6282
 **/
static int igb_poll(struct napi_struct *napi, int budget)
6283
{
6284
	struct igb_q_vector *q_vector = container_of(napi,
6285 6286
						     struct igb_q_vector,
						     napi);
6287
	bool clean_complete = true;
6288

6289
#ifdef CONFIG_IGB_DCA
6290 6291
	if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
		igb_update_dca(q_vector);
J
Jeb Cramer 已提交
6292
#endif
6293
	if (q_vector->tx.ring)
6294
		clean_complete = igb_clean_tx_irq(q_vector);
6295

6296
	if (q_vector->rx.ring)
6297
		clean_complete &= igb_clean_rx_irq(q_vector, budget);
6298

6299 6300 6301
	/* If all work not completed, return budget and keep polling */
	if (!clean_complete)
		return budget;
6302

6303
	/* If not enough Rx work done, exit the polling mode */
6304 6305
	napi_complete(napi);
	igb_ring_irq_enable(q_vector);
6306

6307
	return 0;
6308
}
A
Al Viro 已提交
6309

6310
/**
6311 6312
 *  igb_clean_tx_irq - Reclaim resources after transmit completes
 *  @q_vector: pointer to q_vector containing needed info
6313
 *
6314
 *  returns true if ring is completely cleaned
6315
 **/
6316
static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
6317
{
6318
	struct igb_adapter *adapter = q_vector->adapter;
6319
	struct igb_ring *tx_ring = q_vector->tx.ring;
6320
	struct igb_tx_buffer *tx_buffer;
6321
	union e1000_adv_tx_desc *tx_desc;
6322
	unsigned int total_bytes = 0, total_packets = 0;
6323
	unsigned int budget = q_vector->tx.work_limit;
6324
	unsigned int i = tx_ring->next_to_clean;
6325

6326 6327
	if (test_bit(__IGB_DOWN, &adapter->state))
		return true;
A
Alexander Duyck 已提交
6328

6329
	tx_buffer = &tx_ring->tx_buffer_info[i];
6330
	tx_desc = IGB_TX_DESC(tx_ring, i);
6331
	i -= tx_ring->count;
6332

6333 6334
	do {
		union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
6335 6336 6337 6338

		/* if next_to_watch is not set then there is no work pending */
		if (!eop_desc)
			break;
6339

6340
		/* prevent any other reads prior to eop_desc */
6341
		read_barrier_depends();
6342

6343 6344 6345 6346
		/* if DD is not set pending work has not been completed */
		if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
			break;

6347 6348
		/* clear next_to_watch to prevent false hangs */
		tx_buffer->next_to_watch = NULL;
6349

6350 6351 6352
		/* update the statistics for this packet */
		total_bytes += tx_buffer->bytecount;
		total_packets += tx_buffer->gso_segs;
6353

6354 6355
		/* free the skb */
		dev_kfree_skb_any(tx_buffer->skb);
6356

6357 6358
		/* unmap skb header data */
		dma_unmap_single(tx_ring->dev,
6359 6360
				 dma_unmap_addr(tx_buffer, dma),
				 dma_unmap_len(tx_buffer, len),
6361 6362
				 DMA_TO_DEVICE);

6363 6364 6365 6366
		/* clear tx_buffer data */
		tx_buffer->skb = NULL;
		dma_unmap_len_set(tx_buffer, len, 0);

6367 6368
		/* clear last DMA location and unmap remaining buffers */
		while (tx_desc != eop_desc) {
6369 6370
			tx_buffer++;
			tx_desc++;
6371
			i++;
6372 6373
			if (unlikely(!i)) {
				i -= tx_ring->count;
6374
				tx_buffer = tx_ring->tx_buffer_info;
6375 6376
				tx_desc = IGB_TX_DESC(tx_ring, 0);
			}
6377 6378

			/* unmap any remaining paged data */
6379
			if (dma_unmap_len(tx_buffer, len)) {
6380
				dma_unmap_page(tx_ring->dev,
6381 6382
					       dma_unmap_addr(tx_buffer, dma),
					       dma_unmap_len(tx_buffer, len),
6383
					       DMA_TO_DEVICE);
6384
				dma_unmap_len_set(tx_buffer, len, 0);
6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396
			}
		}

		/* move us one more past the eop_desc for start of next pkt */
		tx_buffer++;
		tx_desc++;
		i++;
		if (unlikely(!i)) {
			i -= tx_ring->count;
			tx_buffer = tx_ring->tx_buffer_info;
			tx_desc = IGB_TX_DESC(tx_ring, 0);
		}
6397 6398 6399 6400 6401 6402 6403

		/* issue prefetch for next Tx descriptor */
		prefetch(tx_desc);

		/* update budget accounting */
		budget--;
	} while (likely(budget));
A
Alexander Duyck 已提交
6404

6405 6406
	netdev_tx_completed_queue(txring_txq(tx_ring),
				  total_packets, total_bytes);
6407
	i += tx_ring->count;
6408
	tx_ring->next_to_clean = i;
6409 6410 6411 6412
	u64_stats_update_begin(&tx_ring->tx_syncp);
	tx_ring->tx_stats.bytes += total_bytes;
	tx_ring->tx_stats.packets += total_packets;
	u64_stats_update_end(&tx_ring->tx_syncp);
6413 6414
	q_vector->tx.total_bytes += total_bytes;
	q_vector->tx.total_packets += total_packets;
6415

6416
	if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
6417
		struct e1000_hw *hw = &adapter->hw;
E
Eric Dumazet 已提交
6418

6419
		/* Detect a transmit hang in hardware, this serializes the
6420 6421
		 * check with the clearing of time_stamp and movement of i
		 */
6422
		clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
6423
		if (tx_buffer->next_to_watch &&
6424
		    time_after(jiffies, tx_buffer->time_stamp +
6425 6426
			       (adapter->tx_timeout_factor * HZ)) &&
		    !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
6427 6428

			/* detected Tx unit hang */
6429
			dev_err(tx_ring->dev,
6430
				"Detected Tx Unit Hang\n"
A
Alexander Duyck 已提交
6431
				"  Tx Queue             <%d>\n"
6432 6433 6434 6435 6436 6437
				"  TDH                  <%x>\n"
				"  TDT                  <%x>\n"
				"  next_to_use          <%x>\n"
				"  next_to_clean        <%x>\n"
				"buffer_info[next_to_clean]\n"
				"  time_stamp           <%lx>\n"
6438
				"  next_to_watch        <%p>\n"
6439 6440
				"  jiffies              <%lx>\n"
				"  desc.status          <%x>\n",
A
Alexander Duyck 已提交
6441
				tx_ring->queue_index,
6442
				rd32(E1000_TDH(tx_ring->reg_idx)),
6443
				readl(tx_ring->tail),
6444 6445
				tx_ring->next_to_use,
				tx_ring->next_to_clean,
6446
				tx_buffer->time_stamp,
6447
				tx_buffer->next_to_watch,
6448
				jiffies,
6449
				tx_buffer->next_to_watch->wb.status);
6450 6451 6452 6453 6454
			netif_stop_subqueue(tx_ring->netdev,
					    tx_ring->queue_index);

			/* we are about to reset, no point in enabling stuff */
			return true;
6455 6456
		}
	}
6457

6458
#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
6459
	if (unlikely(total_packets &&
6460 6461
	    netif_carrier_ok(tx_ring->netdev) &&
	    igb_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478
		/* Make sure that anybody stopping the queue after this
		 * sees the new next_to_clean.
		 */
		smp_mb();
		if (__netif_subqueue_stopped(tx_ring->netdev,
					     tx_ring->queue_index) &&
		    !(test_bit(__IGB_DOWN, &adapter->state))) {
			netif_wake_subqueue(tx_ring->netdev,
					    tx_ring->queue_index);

			u64_stats_update_begin(&tx_ring->tx_syncp);
			tx_ring->tx_stats.restart_queue++;
			u64_stats_update_end(&tx_ring->tx_syncp);
		}
	}

	return !!budget;
6479 6480
}

6481
/**
6482 6483 6484
 *  igb_reuse_rx_page - page flip buffer and store it back on the ring
 *  @rx_ring: rx descriptor ring to store buffers on
 *  @old_buff: donor buffer to have page reused
6485
 *
6486
 *  Synchronizes page for reuse by the adapter
6487 6488 6489 6490 6491 6492 6493 6494 6495 6496 6497 6498 6499 6500
 **/
static void igb_reuse_rx_page(struct igb_ring *rx_ring,
			      struct igb_rx_buffer *old_buff)
{
	struct igb_rx_buffer *new_buff;
	u16 nta = rx_ring->next_to_alloc;

	new_buff = &rx_ring->rx_buffer_info[nta];

	/* update, and store next to alloc */
	nta++;
	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;

	/* transfer page from old buffer to new buffer */
6501
	*new_buff = *old_buff;
6502 6503 6504 6505

	/* sync the buffer for use by the device */
	dma_sync_single_range_for_device(rx_ring->dev, old_buff->dma,
					 old_buff->page_offset,
6506
					 IGB_RX_BUFSZ,
6507 6508 6509
					 DMA_FROM_DEVICE);
}

6510 6511 6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544
static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
				  struct page *page,
				  unsigned int truesize)
{
	/* avoid re-using remote pages */
	if (unlikely(page_to_nid(page) != numa_node_id()))
		return false;

#if (PAGE_SIZE < 8192)
	/* if we are only owner of page we can reuse it */
	if (unlikely(page_count(page) != 1))
		return false;

	/* flip page offset to other buffer */
	rx_buffer->page_offset ^= IGB_RX_BUFSZ;

	/* since we are the only owner of the page and we need to
	 * increment it, just set the value to 2 in order to avoid
	 * an unnecessary locked operation
	 */
	atomic_set(&page->_count, 2);
#else
	/* move offset up to the next cache line */
	rx_buffer->page_offset += truesize;

	if (rx_buffer->page_offset > (PAGE_SIZE - IGB_RX_BUFSZ))
		return false;

	/* bump ref count on page before it is given to the stack */
	get_page(page);
#endif

	return true;
}

6545
/**
6546 6547 6548 6549 6550
 *  igb_add_rx_frag - Add contents of Rx buffer to sk_buff
 *  @rx_ring: rx descriptor ring to transact packets on
 *  @rx_buffer: buffer containing page to add
 *  @rx_desc: descriptor containing length of buffer written by hardware
 *  @skb: sk_buff to place the data into
6551
 *
6552 6553 6554 6555
 *  This function will add the data contained in rx_buffer->page to the skb.
 *  This is done either through a direct copy if the data in the buffer is
 *  less than the skb header size, otherwise it will just attach the page as
 *  a frag to the skb.
6556
 *
6557 6558
 *  The function will then update the page offset if necessary and return
 *  true if the buffer can be reused by the adapter.
6559 6560 6561 6562 6563 6564 6565 6566
 **/
static bool igb_add_rx_frag(struct igb_ring *rx_ring,
			    struct igb_rx_buffer *rx_buffer,
			    union e1000_adv_rx_desc *rx_desc,
			    struct sk_buff *skb)
{
	struct page *page = rx_buffer->page;
	unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
6567 6568 6569 6570 6571
#if (PAGE_SIZE < 8192)
	unsigned int truesize = IGB_RX_BUFSZ;
#else
	unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
#endif
6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593

	if ((size <= IGB_RX_HDR_LEN) && !skb_is_nonlinear(skb)) {
		unsigned char *va = page_address(page) + rx_buffer->page_offset;

		if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
			igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);
			va += IGB_TS_HDR_LEN;
			size -= IGB_TS_HDR_LEN;
		}

		memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));

		/* we can reuse buffer as-is, just make sure it is local */
		if (likely(page_to_nid(page) == numa_node_id()))
			return true;

		/* this page cannot be reused so discard it */
		put_page(page);
		return false;
	}

	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
6594
			rx_buffer->page_offset, size, truesize);
6595

6596 6597
	return igb_can_reuse_rx_page(rx_buffer, page, truesize);
}
6598

6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628
static struct sk_buff *igb_fetch_rx_buffer(struct igb_ring *rx_ring,
					   union e1000_adv_rx_desc *rx_desc,
					   struct sk_buff *skb)
{
	struct igb_rx_buffer *rx_buffer;
	struct page *page;

	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];

	page = rx_buffer->page;
	prefetchw(page);

	if (likely(!skb)) {
		void *page_addr = page_address(page) +
				  rx_buffer->page_offset;

		/* prefetch first cache line of first page */
		prefetch(page_addr);
#if L1_CACHE_BYTES < 128
		prefetch(page_addr + L1_CACHE_BYTES);
#endif

		/* allocate a skb to store the frags */
		skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
						IGB_RX_HDR_LEN);
		if (unlikely(!skb)) {
			rx_ring->rx_stats.alloc_failed++;
			return NULL;
		}

6629
		/* we will be copying header into skb->data in
6630 6631 6632 6633 6634 6635 6636 6637 6638 6639
		 * pskb_may_pull so it is in our interest to prefetch
		 * it now to avoid a possible cache miss
		 */
		prefetchw(skb->data);
	}

	/* we are reusing so sync this buffer for CPU use */
	dma_sync_single_range_for_cpu(rx_ring->dev,
				      rx_buffer->dma,
				      rx_buffer->page_offset,
6640
				      IGB_RX_BUFSZ,
6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658
				      DMA_FROM_DEVICE);

	/* pull page into skb */
	if (igb_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
		/* hand second half of page back to the ring */
		igb_reuse_rx_page(rx_ring, rx_buffer);
	} else {
		/* we are not reusing the buffer so unmap it */
		dma_unmap_page(rx_ring->dev, rx_buffer->dma,
			       PAGE_SIZE, DMA_FROM_DEVICE);
	}

	/* clear contents of rx_buffer */
	rx_buffer->page = NULL;

	return skb;
}

6659
static inline void igb_rx_checksum(struct igb_ring *ring,
6660 6661
				   union e1000_adv_rx_desc *rx_desc,
				   struct sk_buff *skb)
6662
{
6663
	skb_checksum_none_assert(skb);
6664

6665
	/* Ignore Checksum bit is set */
6666
	if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
6667 6668 6669 6670
		return;

	/* Rx checksum disabled via ethtool */
	if (!(ring->netdev->features & NETIF_F_RXCSUM))
6671
		return;
6672

6673
	/* TCP/UDP checksum error bit is set */
6674 6675 6676
	if (igb_test_staterr(rx_desc,
			     E1000_RXDEXT_STATERR_TCPE |
			     E1000_RXDEXT_STATERR_IPE)) {
6677
		/* work around errata with sctp packets where the TCPE aka
6678 6679 6680
		 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
		 * packets, (aka let the stack check the crc32c)
		 */
6681 6682
		if (!((skb->len == 60) &&
		      test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
E
Eric Dumazet 已提交
6683
			u64_stats_update_begin(&ring->rx_syncp);
6684
			ring->rx_stats.csum_err++;
E
Eric Dumazet 已提交
6685 6686
			u64_stats_update_end(&ring->rx_syncp);
		}
6687 6688 6689 6690
		/* let the stack verify checksum errors */
		return;
	}
	/* It must be a TCP or UDP packet with a valid checksum */
6691 6692
	if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
				      E1000_RXD_STAT_UDPCS))
6693 6694
		skb->ip_summed = CHECKSUM_UNNECESSARY;

6695 6696
	dev_dbg(ring->dev, "cksum success: bits %08X\n",
		le32_to_cpu(rx_desc->wb.upper.status_error));
6697 6698
}

6699 6700 6701 6702 6703
static inline void igb_rx_hash(struct igb_ring *ring,
			       union e1000_adv_rx_desc *rx_desc,
			       struct sk_buff *skb)
{
	if (ring->netdev->features & NETIF_F_RXHASH)
T
Tom Herbert 已提交
6704 6705 6706
		skb_set_hash(skb,
			     le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
			     PKT_HASH_TYPE_L3);
6707 6708
}

6709
/**
6710 6711 6712 6713
 *  igb_is_non_eop - process handling of non-EOP buffers
 *  @rx_ring: Rx ring being processed
 *  @rx_desc: Rx descriptor for current buffer
 *  @skb: current socket buffer containing buffer in progress
6714
 *
6715 6716 6717 6718
 *  This function updates next to clean.  If the buffer is an EOP buffer
 *  this function exits returning false, otherwise it will place the
 *  sk_buff in the next buffer to be chained and return true indicating
 *  that this is in fact a non-EOP buffer.
6719 6720 6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736
 **/
static bool igb_is_non_eop(struct igb_ring *rx_ring,
			   union e1000_adv_rx_desc *rx_desc)
{
	u32 ntc = rx_ring->next_to_clean + 1;

	/* fetch, update, and store next to clean */
	ntc = (ntc < rx_ring->count) ? ntc : 0;
	rx_ring->next_to_clean = ntc;

	prefetch(IGB_RX_DESC(rx_ring, ntc));

	if (likely(igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)))
		return false;

	return true;
}

6737
/**
6738 6739 6740
 *  igb_get_headlen - determine size of header for LRO/GRO
 *  @data: pointer to the start of the headers
 *  @max_len: total length of section to find headers in
6741
 *
6742 6743 6744 6745 6746
 *  This function is meant to determine the length of headers that will
 *  be recognized by hardware for LRO, and GRO offloads.  The main
 *  motivation of doing this is to only perform one pull for IPv4 TCP
 *  packets so that we can do basic things like calculating the gso_size
 *  based on the average data per packet.
6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775
 **/
static unsigned int igb_get_headlen(unsigned char *data,
				    unsigned int max_len)
{
	union {
		unsigned char *network;
		/* l2 headers */
		struct ethhdr *eth;
		struct vlan_hdr *vlan;
		/* l3 headers */
		struct iphdr *ipv4;
		struct ipv6hdr *ipv6;
	} hdr;
	__be16 protocol;
	u8 nexthdr = 0;	/* default to not TCP */
	u8 hlen;

	/* this should never happen, but better safe than sorry */
	if (max_len < ETH_HLEN)
		return max_len;

	/* initialize network frame pointer */
	hdr.network = data;

	/* set first protocol and move network header forward */
	protocol = hdr.eth->h_proto;
	hdr.network += ETH_HLEN;

	/* handle any vlan tag if present */
6776
	if (protocol == htons(ETH_P_8021Q)) {
6777 6778 6779 6780 6781 6782 6783 6784
		if ((hdr.network - data) > (max_len - VLAN_HLEN))
			return max_len;

		protocol = hdr.vlan->h_vlan_encapsulated_proto;
		hdr.network += VLAN_HLEN;
	}

	/* handle L3 protocols */
6785
	if (protocol == htons(ETH_P_IP)) {
6786 6787 6788 6789 6790 6791 6792 6793 6794 6795
		if ((hdr.network - data) > (max_len - sizeof(struct iphdr)))
			return max_len;

		/* access ihl as a u8 to avoid unaligned access on ia64 */
		hlen = (hdr.network[0] & 0x0F) << 2;

		/* verify hlen meets minimum size requirements */
		if (hlen < sizeof(struct iphdr))
			return hdr.network - data;

6796
		/* record next protocol if header is present */
6797
		if (!(hdr.ipv4->frag_off & htons(IP_OFFSET)))
6798
			nexthdr = hdr.ipv4->protocol;
6799
	} else if (protocol == htons(ETH_P_IPV6)) {
6800 6801 6802 6803 6804
		if ((hdr.network - data) > (max_len - sizeof(struct ipv6hdr)))
			return max_len;

		/* record next protocol */
		nexthdr = hdr.ipv6->nexthdr;
6805
		hlen = sizeof(struct ipv6hdr);
6806 6807 6808 6809
	} else {
		return hdr.network - data;
	}

6810 6811 6812
	/* relocate pointer to start of L4 header */
	hdr.network += hlen;

6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832
	/* finally sort out TCP */
	if (nexthdr == IPPROTO_TCP) {
		if ((hdr.network - data) > (max_len - sizeof(struct tcphdr)))
			return max_len;

		/* access doff as a u8 to avoid unaligned access on ia64 */
		hlen = (hdr.network[12] & 0xF0) >> 2;

		/* verify hlen meets minimum size requirements */
		if (hlen < sizeof(struct tcphdr))
			return hdr.network - data;

		hdr.network += hlen;
	} else if (nexthdr == IPPROTO_UDP) {
		if ((hdr.network - data) > (max_len - sizeof(struct udphdr)))
			return max_len;

		hdr.network += sizeof(struct udphdr);
	}

6833
	/* If everything has gone correctly hdr.network should be the
6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844
	 * data section of the packet and will be the end of the header.
	 * If not then it probably represents the end of the last recognized
	 * header.
	 */
	if ((hdr.network - data) < max_len)
		return hdr.network - data;
	else
		return max_len;
}

/**
6845 6846 6847 6848
 *  igb_pull_tail - igb specific version of skb_pull_tail
 *  @rx_ring: rx descriptor ring packet is being transacted on
 *  @rx_desc: pointer to the EOP Rx descriptor
 *  @skb: pointer to current skb being adjusted
6849
 *
6850 6851 6852 6853 6854 6855
 *  This function is an igb specific version of __pskb_pull_tail.  The
 *  main difference between this version and the original function is that
 *  this function can make several assumptions about the state of things
 *  that allow for significant optimizations versus the standard function.
 *  As a result we can do things like drop a frag and maintain an accurate
 *  truesize for the skb.
6856 6857 6858 6859
 */
static void igb_pull_tail(struct igb_ring *rx_ring,
			  union e1000_adv_rx_desc *rx_desc,
			  struct sk_buff *skb)
6860
{
6861 6862 6863 6864
	struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
	unsigned char *va;
	unsigned int pull_len;

6865
	/* it is valid to use page_address instead of kmap since we are
6866 6867
	 * working with pages allocated out of the lomem pool per
	 * alloc_page(GFP_ATOMIC)
6868
	 */
6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884
	va = skb_frag_address(frag);

	if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
		/* retrieve timestamp from buffer */
		igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);

		/* update pointers to remove timestamp header */
		skb_frag_size_sub(frag, IGB_TS_HDR_LEN);
		frag->page_offset += IGB_TS_HDR_LEN;
		skb->data_len -= IGB_TS_HDR_LEN;
		skb->len -= IGB_TS_HDR_LEN;

		/* move va to start of packet data */
		va += IGB_TS_HDR_LEN;
	}

6885
	/* we need the header to contain the greater of either ETH_HLEN or
6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900
	 * 60 bytes if the skb->len is less than 60 for skb_pad.
	 */
	pull_len = igb_get_headlen(va, IGB_RX_HDR_LEN);

	/* align pull length to size of long to optimize memcpy performance */
	skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));

	/* update all of the pointers */
	skb_frag_size_sub(frag, pull_len);
	frag->page_offset += pull_len;
	skb->data_len -= pull_len;
	skb->tail += pull_len;
}

/**
6901 6902 6903 6904
 *  igb_cleanup_headers - Correct corrupted or empty headers
 *  @rx_ring: rx descriptor ring packet is being transacted on
 *  @rx_desc: pointer to the EOP Rx descriptor
 *  @skb: pointer to current skb being fixed
6905
 *
6906 6907
 *  Address the case where we are pulling data in on pages only
 *  and as such no data is present in the skb header.
6908
 *
6909 6910
 *  In addition if skb is not at least 60 bytes we need to pad it so that
 *  it is large enough to qualify as a valid Ethernet frame.
6911
 *
6912
 *  Returns true if an error was encountered and skb was freed.
6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940
 **/
static bool igb_cleanup_headers(struct igb_ring *rx_ring,
				union e1000_adv_rx_desc *rx_desc,
				struct sk_buff *skb)
{
	if (unlikely((igb_test_staterr(rx_desc,
				       E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) {
		struct net_device *netdev = rx_ring->netdev;
		if (!(netdev->features & NETIF_F_RXALL)) {
			dev_kfree_skb_any(skb);
			return true;
		}
	}

	/* place header in linear portion of buffer */
	if (skb_is_nonlinear(skb))
		igb_pull_tail(rx_ring, rx_desc, skb);

	/* if skb_pad returns an error the skb was freed */
	if (unlikely(skb->len < 60)) {
		int pad_len = 60 - skb->len;

		if (skb_pad(skb, pad_len))
			return true;
		__skb_put(skb, pad_len);
	}

	return false;
6941 6942
}

6943
/**
6944 6945 6946 6947
 *  igb_process_skb_fields - Populate skb header fields from Rx descriptor
 *  @rx_ring: rx descriptor ring packet is being transacted on
 *  @rx_desc: pointer to the EOP Rx descriptor
 *  @skb: pointer to current skb being populated
6948
 *
6949 6950 6951
 *  This function checks the ring, descriptor, and packet information in
 *  order to populate the hash, checksum, VLAN, timestamp, protocol, and
 *  other fields within the skb.
6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962
 **/
static void igb_process_skb_fields(struct igb_ring *rx_ring,
				   union e1000_adv_rx_desc *rx_desc,
				   struct sk_buff *skb)
{
	struct net_device *dev = rx_ring->netdev;

	igb_rx_hash(rx_ring, rx_desc, skb);

	igb_rx_checksum(rx_ring, rx_desc, skb);

6963 6964 6965
	if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TS) &&
	    !igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))
		igb_ptp_rx_rgtstamp(rx_ring->q_vector, skb);
6966

6967
	if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
6968 6969
	    igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
		u16 vid;
6970

6971 6972 6973 6974 6975 6976
		if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
		    test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
			vid = be16_to_cpu(rx_desc->wb.upper.vlan);
		else
			vid = le16_to_cpu(rx_desc->wb.upper.vlan);

6977
		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
6978 6979 6980 6981 6982 6983 6984
	}

	skb_record_rx_queue(skb, rx_ring->queue_index);

	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
}

6985
static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
6986
{
6987
	struct igb_ring *rx_ring = q_vector->rx.ring;
6988
	struct sk_buff *skb = rx_ring->skb;
6989
	unsigned int total_bytes = 0, total_packets = 0;
6990
	u16 cleaned_count = igb_desc_unused(rx_ring);
6991

6992
	while (likely(total_packets < budget)) {
6993
		union e1000_adv_rx_desc *rx_desc;
6994

6995 6996 6997 6998 6999
		/* return some buffers to hardware, one at a time is too slow */
		if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
			igb_alloc_rx_buffers(rx_ring, cleaned_count);
			cleaned_count = 0;
		}
7000

7001
		rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean);
7002

7003 7004
		if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_DD))
			break;
7005

7006 7007 7008 7009 7010 7011
		/* This memory barrier is needed to keep us from reading
		 * any other fields out of the rx_desc until we know the
		 * RXD_STAT_DD bit is set
		 */
		rmb();

7012
		/* retrieve a buffer from the ring */
7013
		skb = igb_fetch_rx_buffer(rx_ring, rx_desc, skb);
7014

7015 7016 7017
		/* exit if we failed to retrieve a buffer */
		if (!skb)
			break;
7018

7019
		cleaned_count++;
7020

7021 7022 7023
		/* fetch next buffer in frame if non-eop */
		if (igb_is_non_eop(rx_ring, rx_desc))
			continue;
7024 7025 7026 7027 7028

		/* verify the packet layout is correct */
		if (igb_cleanup_headers(rx_ring, rx_desc, skb)) {
			skb = NULL;
			continue;
7029 7030
		}

7031
		/* probably a little skewed due to removing CRC */
7032 7033
		total_bytes += skb->len;

7034 7035
		/* populate checksum, timestamp, VLAN, and protocol */
		igb_process_skb_fields(rx_ring, rx_desc, skb);
7036

J
Jiri Pirko 已提交
7037
		napi_gro_receive(&q_vector->napi, skb);
7038

7039 7040 7041
		/* reset skb pointer */
		skb = NULL;

7042 7043
		/* update budget accounting */
		total_packets++;
7044
	}
7045

7046 7047 7048
	/* place incomplete frames back on ring for completion */
	rx_ring->skb = skb;

E
Eric Dumazet 已提交
7049
	u64_stats_update_begin(&rx_ring->rx_syncp);
7050 7051
	rx_ring->rx_stats.packets += total_packets;
	rx_ring->rx_stats.bytes += total_bytes;
E
Eric Dumazet 已提交
7052
	u64_stats_update_end(&rx_ring->rx_syncp);
7053 7054
	q_vector->rx.total_packets += total_packets;
	q_vector->rx.total_bytes += total_bytes;
7055 7056

	if (cleaned_count)
7057
		igb_alloc_rx_buffers(rx_ring, cleaned_count);
7058

7059
	return total_packets < budget;
7060 7061
}

7062
static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
7063
				  struct igb_rx_buffer *bi)
7064 7065
{
	struct page *page = bi->page;
7066
	dma_addr_t dma;
7067

7068 7069
	/* since we are recycling buffers we should seldom need to alloc */
	if (likely(page))
7070 7071
		return true;

7072 7073 7074 7075 7076
	/* alloc new page for storage */
	page = __skb_alloc_page(GFP_ATOMIC | __GFP_COLD, NULL);
	if (unlikely(!page)) {
		rx_ring->rx_stats.alloc_failed++;
		return false;
7077 7078
	}

7079 7080
	/* map page for use */
	dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
7081

7082
	/* if mapping failed free memory back to system since
7083 7084
	 * there isn't much point in holding memory we can't use
	 */
7085
	if (dma_mapping_error(rx_ring->dev, dma)) {
7086 7087
		__free_page(page);

7088 7089 7090 7091
		rx_ring->rx_stats.alloc_failed++;
		return false;
	}

7092
	bi->dma = dma;
7093 7094
	bi->page = page;
	bi->page_offset = 0;
7095

7096 7097 7098
	return true;
}

7099
/**
7100 7101
 *  igb_alloc_rx_buffers - Replace used receive buffers; packet split
 *  @adapter: address of board private structure
7102
 **/
7103
void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
7104 7105
{
	union e1000_adv_rx_desc *rx_desc;
7106
	struct igb_rx_buffer *bi;
7107
	u16 i = rx_ring->next_to_use;
7108

7109 7110 7111 7112
	/* nothing to do */
	if (!cleaned_count)
		return;

7113
	rx_desc = IGB_RX_DESC(rx_ring, i);
7114
	bi = &rx_ring->rx_buffer_info[i];
7115
	i -= rx_ring->count;
7116

7117
	do {
7118
		if (!igb_alloc_mapped_page(rx_ring, bi))
7119
			break;
7120

7121
		/* Refresh the desc even if buffer_addrs didn't change
7122 7123
		 * because each write-back erases this info.
		 */
7124
		rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
7125

7126 7127
		rx_desc++;
		bi++;
7128
		i++;
7129
		if (unlikely(!i)) {
7130
			rx_desc = IGB_RX_DESC(rx_ring, 0);
7131
			bi = rx_ring->rx_buffer_info;
7132 7133 7134 7135 7136
			i -= rx_ring->count;
		}

		/* clear the hdr_addr for the next_to_use descriptor */
		rx_desc->read.hdr_addr = 0;
7137 7138 7139

		cleaned_count--;
	} while (cleaned_count);
7140

7141 7142
	i += rx_ring->count;

7143
	if (rx_ring->next_to_use != i) {
7144
		/* record the next descriptor to use */
7145 7146
		rx_ring->next_to_use = i;

7147 7148 7149
		/* update next to alloc since we have filled the ring */
		rx_ring->next_to_alloc = i;

7150
		/* Force memory writes to complete before letting h/w
7151 7152
		 * know there are new descriptors to fetch.  (Only
		 * applicable for weak-ordered memory model archs,
7153 7154
		 * such as IA-64).
		 */
7155
		wmb();
7156
		writel(i, rx_ring->tail);
7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178
	}
}

/**
 * igb_mii_ioctl -
 * @netdev:
 * @ifreq:
 * @cmd:
 **/
static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct mii_ioctl_data *data = if_mii(ifr);

	if (adapter->hw.phy.media_type != e1000_media_type_copper)
		return -EOPNOTSUPP;

	switch (cmd) {
	case SIOCGMIIPHY:
		data->phy_id = adapter->hw.phy.addr;
		break;
	case SIOCGMIIREG:
7179
		if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
7180
				     &data->val_out))
7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202
			return -EIO;
		break;
	case SIOCSMIIREG:
	default:
		return -EOPNOTSUPP;
	}
	return 0;
}

/**
 * igb_ioctl -
 * @netdev:
 * @ifreq:
 * @cmd:
 **/
static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
{
	switch (cmd) {
	case SIOCGMIIPHY:
	case SIOCGMIIREG:
	case SIOCSMIIREG:
		return igb_mii_ioctl(netdev, ifr, cmd);
7203 7204
	case SIOCGHWTSTAMP:
		return igb_ptp_get_ts_config(netdev, ifr);
7205
	case SIOCSHWTSTAMP:
7206
		return igb_ptp_set_ts_config(netdev, ifr);
7207 7208 7209 7210 7211
	default:
		return -EOPNOTSUPP;
	}
}

7212 7213 7214 7215
s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
{
	struct igb_adapter *adapter = hw->back;

7216
	if (pcie_capability_read_word(adapter->pdev, reg, value))
7217 7218 7219 7220 7221 7222 7223 7224 7225
		return -E1000_ERR_CONFIG;

	return 0;
}

s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
{
	struct igb_adapter *adapter = hw->back;

7226
	if (pcie_capability_write_word(adapter->pdev, reg, *value))
7227 7228 7229 7230 7231
		return -E1000_ERR_CONFIG;

	return 0;
}

7232
static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
7233 7234 7235 7236
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
	u32 ctrl, rctl;
7237
	bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
7238

7239
	if (enable) {
7240 7241 7242 7243 7244
		/* enable VLAN tag insert/strip */
		ctrl = rd32(E1000_CTRL);
		ctrl |= E1000_CTRL_VME;
		wr32(E1000_CTRL, ctrl);

7245
		/* Disable CFI check */
7246 7247 7248 7249 7250 7251 7252 7253 7254 7255
		rctl = rd32(E1000_RCTL);
		rctl &= ~E1000_RCTL_CFIEN;
		wr32(E1000_RCTL, rctl);
	} else {
		/* disable VLAN tag insert/strip */
		ctrl = rd32(E1000_CTRL);
		ctrl &= ~E1000_CTRL_VME;
		wr32(E1000_CTRL, ctrl);
	}

7256
	igb_rlpml_set(adapter);
7257 7258
}

7259 7260
static int igb_vlan_rx_add_vid(struct net_device *netdev,
			       __be16 proto, u16 vid)
7261 7262 7263
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
7264
	int pf_id = adapter->vfs_allocated_count;
7265

7266 7267
	/* attempt to add filter to vlvf array */
	igb_vlvf_set(adapter, vid, true, pf_id);
7268

7269 7270
	/* add the filter since PF can receive vlans w/o entry in vlvf */
	igb_vfta_set(hw, vid, true);
J
Jiri Pirko 已提交
7271 7272

	set_bit(vid, adapter->active_vlans);
7273 7274

	return 0;
7275 7276
}

7277 7278
static int igb_vlan_rx_kill_vid(struct net_device *netdev,
				__be16 proto, u16 vid)
7279 7280 7281
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
7282
	int pf_id = adapter->vfs_allocated_count;
7283
	s32 err;
7284

7285 7286
	/* remove vlan from VLVF table array */
	err = igb_vlvf_set(adapter, vid, false, pf_id);
7287

7288 7289
	/* if vid was not present in VLVF just remove it from table */
	if (err)
7290
		igb_vfta_set(hw, vid, false);
J
Jiri Pirko 已提交
7291 7292

	clear_bit(vid, adapter->active_vlans);
7293 7294

	return 0;
7295 7296 7297 7298
}

static void igb_restore_vlan(struct igb_adapter *adapter)
{
J
Jiri Pirko 已提交
7299
	u16 vid;
7300

7301 7302
	igb_vlan_mode(adapter->netdev, adapter->netdev->features);

J
Jiri Pirko 已提交
7303
	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
7304
		igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
7305 7306
}

7307
int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
7308
{
7309
	struct pci_dev *pdev = adapter->pdev;
7310 7311 7312 7313
	struct e1000_mac_info *mac = &adapter->hw.mac;

	mac->autoneg = 0;

7314
	/* Make sure dplx is at most 1 bit and lsb of speed is not set
7315 7316
	 * for the switch() below to work
	 */
7317 7318 7319
	if ((spd & 1) || (dplx & ~1))
		goto err_inval;

7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332
	/* Fiber NIC's only allow 1000 gbps Full duplex
	 * and 100Mbps Full duplex for 100baseFx sfp
	 */
	if (adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
		switch (spd + dplx) {
		case SPEED_10 + DUPLEX_HALF:
		case SPEED_10 + DUPLEX_FULL:
		case SPEED_100 + DUPLEX_HALF:
			goto err_inval;
		default:
			break;
		}
	}
7333

7334
	switch (spd + dplx) {
7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352
	case SPEED_10 + DUPLEX_HALF:
		mac->forced_speed_duplex = ADVERTISE_10_HALF;
		break;
	case SPEED_10 + DUPLEX_FULL:
		mac->forced_speed_duplex = ADVERTISE_10_FULL;
		break;
	case SPEED_100 + DUPLEX_HALF:
		mac->forced_speed_duplex = ADVERTISE_100_HALF;
		break;
	case SPEED_100 + DUPLEX_FULL:
		mac->forced_speed_duplex = ADVERTISE_100_FULL;
		break;
	case SPEED_1000 + DUPLEX_FULL:
		mac->autoneg = 1;
		adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
		break;
	case SPEED_1000 + DUPLEX_HALF: /* not supported */
	default:
7353
		goto err_inval;
7354
	}
7355 7356 7357 7358

	/* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
	adapter->hw.phy.mdix = AUTO_ALL_MODES;

7359
	return 0;
7360 7361 7362 7363

err_inval:
	dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
	return -EINVAL;
7364 7365
}

Y
Yan, Zheng 已提交
7366 7367
static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
			  bool runtime)
7368 7369 7370 7371
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
A
Alexander Duyck 已提交
7372
	u32 ctrl, rctl, status;
Y
Yan, Zheng 已提交
7373
	u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
7374 7375 7376 7377 7378 7379
#ifdef CONFIG_PM
	int retval = 0;
#endif

	netif_device_detach(netdev);

A
Alexander Duyck 已提交
7380
	if (netif_running(netdev))
Y
Yan, Zheng 已提交
7381
		__igb_close(netdev, true);
A
Alexander Duyck 已提交
7382

7383
	igb_clear_interrupt_scheme(adapter);
7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396

#ifdef CONFIG_PM
	retval = pci_save_state(pdev);
	if (retval)
		return retval;
#endif

	status = rd32(E1000_STATUS);
	if (status & E1000_STATUS_LU)
		wufc &= ~E1000_WUFC_LNKC;

	if (wufc) {
		igb_setup_rctl(adapter);
7397
		igb_set_rx_mode(netdev);
7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414

		/* turn on all-multi mode if wake on multicast is enabled */
		if (wufc & E1000_WUFC_MC) {
			rctl = rd32(E1000_RCTL);
			rctl |= E1000_RCTL_MPE;
			wr32(E1000_RCTL, rctl);
		}

		ctrl = rd32(E1000_CTRL);
		/* advertise wake from D3Cold */
		#define E1000_CTRL_ADVD3WUC 0x00100000
		/* phy power management enable */
		#define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
		ctrl |= E1000_CTRL_ADVD3WUC;
		wr32(E1000_CTRL, ctrl);

		/* Allow time for pending master requests to run */
7415
		igb_disable_pcie_master(hw);
7416 7417 7418 7419 7420 7421 7422 7423

		wr32(E1000_WUC, E1000_WUC_PME_EN);
		wr32(E1000_WUFC, wufc);
	} else {
		wr32(E1000_WUC, 0);
		wr32(E1000_WUFC, 0);
	}

7424 7425
	*enable_wake = wufc || adapter->en_mng_pt;
	if (!*enable_wake)
7426 7427 7428
		igb_power_down_link(adapter);
	else
		igb_power_up_link(adapter);
7429 7430

	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
7431 7432
	 * would have already happened in close and is redundant.
	 */
7433 7434 7435 7436 7437 7438 7439 7440
	igb_release_hw_control(adapter);

	pci_disable_device(pdev);

	return 0;
}

#ifdef CONFIG_PM
7441
#ifdef CONFIG_PM_SLEEP
Y
Yan, Zheng 已提交
7442
static int igb_suspend(struct device *dev)
7443 7444 7445
{
	int retval;
	bool wake;
Y
Yan, Zheng 已提交
7446
	struct pci_dev *pdev = to_pci_dev(dev);
7447

Y
Yan, Zheng 已提交
7448
	retval = __igb_shutdown(pdev, &wake, 0);
7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460
	if (retval)
		return retval;

	if (wake) {
		pci_prepare_to_sleep(pdev);
	} else {
		pci_wake_from_d3(pdev, false);
		pci_set_power_state(pdev, PCI_D3hot);
	}

	return 0;
}
7461
#endif /* CONFIG_PM_SLEEP */
7462

Y
Yan, Zheng 已提交
7463
static int igb_resume(struct device *dev)
7464
{
Y
Yan, Zheng 已提交
7465
	struct pci_dev *pdev = to_pci_dev(dev);
7466 7467 7468 7469 7470 7471 7472
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
	u32 err;

	pci_set_power_state(pdev, PCI_D0);
	pci_restore_state(pdev);
7473
	pci_save_state(pdev);
T
Taku Izumi 已提交
7474

7475
	err = pci_enable_device_mem(pdev);
7476 7477 7478 7479 7480 7481 7482 7483 7484 7485
	if (err) {
		dev_err(&pdev->dev,
			"igb: Cannot enable PCI device from suspend\n");
		return err;
	}
	pci_set_master(pdev);

	pci_enable_wake(pdev, PCI_D3hot, 0);
	pci_enable_wake(pdev, PCI_D3cold, 0);

7486
	if (igb_init_interrupt_scheme(adapter, true)) {
A
Alexander Duyck 已提交
7487 7488
		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
		return -ENOMEM;
7489 7490 7491
	}

	igb_reset(adapter);
7492 7493

	/* let the f/w know that the h/w is now under the control of the
7494 7495
	 * driver.
	 */
7496 7497
	igb_get_hw_control(adapter);

7498 7499
	wr32(E1000_WUS, ~0);

Y
Yan, Zheng 已提交
7500
	if (netdev->flags & IFF_UP) {
7501
		rtnl_lock();
Y
Yan, Zheng 已提交
7502
		err = __igb_open(netdev, true);
7503
		rtnl_unlock();
A
Alexander Duyck 已提交
7504 7505 7506
		if (err)
			return err;
	}
7507 7508

	netif_device_attach(netdev);
Y
Yan, Zheng 已提交
7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540
	return 0;
}

#ifdef CONFIG_PM_RUNTIME
static int igb_runtime_idle(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);

	if (!igb_has_link(adapter))
		pm_schedule_suspend(dev, MSEC_PER_SEC * 5);

	return -EBUSY;
}

static int igb_runtime_suspend(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	int retval;
	bool wake;

	retval = __igb_shutdown(pdev, &wake, 1);
	if (retval)
		return retval;

	if (wake) {
		pci_prepare_to_sleep(pdev);
	} else {
		pci_wake_from_d3(pdev, false);
		pci_set_power_state(pdev, PCI_D3hot);
	}
7541 7542 7543

	return 0;
}
Y
Yan, Zheng 已提交
7544 7545 7546 7547 7548 7549

static int igb_runtime_resume(struct device *dev)
{
	return igb_resume(dev);
}
#endif /* CONFIG_PM_RUNTIME */
7550 7551 7552 7553
#endif

static void igb_shutdown(struct pci_dev *pdev)
{
7554 7555
	bool wake;

Y
Yan, Zheng 已提交
7556
	__igb_shutdown(pdev, &wake, 0);
7557 7558 7559 7560 7561

	if (system_state == SYSTEM_POWER_OFF) {
		pci_wake_from_d3(pdev, wake);
		pci_set_power_state(pdev, PCI_D3hot);
	}
7562 7563
}

7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577 7578 7579 7580 7581 7582 7583 7584 7585 7586 7587 7588 7589 7590 7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602 7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629
#ifdef CONFIG_PCI_IOV
static int igb_sriov_reinit(struct pci_dev *dev)
{
	struct net_device *netdev = pci_get_drvdata(dev);
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct pci_dev *pdev = adapter->pdev;

	rtnl_lock();

	if (netif_running(netdev))
		igb_close(netdev);

	igb_clear_interrupt_scheme(adapter);

	igb_init_queue_configuration(adapter);

	if (igb_init_interrupt_scheme(adapter, true)) {
		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
		return -ENOMEM;
	}

	if (netif_running(netdev))
		igb_open(netdev);

	rtnl_unlock();

	return 0;
}

static int igb_pci_disable_sriov(struct pci_dev *dev)
{
	int err = igb_disable_sriov(dev);

	if (!err)
		err = igb_sriov_reinit(dev);

	return err;
}

static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs)
{
	int err = igb_enable_sriov(dev, num_vfs);

	if (err)
		goto out;

	err = igb_sriov_reinit(dev);
	if (!err)
		return num_vfs;

out:
	return err;
}

#endif
static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
{
#ifdef CONFIG_PCI_IOV
	if (num_vfs == 0)
		return igb_pci_disable_sriov(dev);
	else
		return igb_pci_enable_sriov(dev, num_vfs);
#endif
	return 0;
}

7630
#ifdef CONFIG_NET_POLL_CONTROLLER
7631
/* Polling 'interrupt' - used by things like netconsole to send skbs
7632 7633 7634 7635 7636 7637
 * without having to re-enable interrupts. It's not called while
 * the interrupt routine is executing.
 */
static void igb_netpoll(struct net_device *netdev)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
7638
	struct e1000_hw *hw = &adapter->hw;
7639
	struct igb_q_vector *q_vector;
7640 7641
	int i;

7642
	for (i = 0; i < adapter->num_q_vectors; i++) {
7643
		q_vector = adapter->q_vector[i];
7644
		if (adapter->flags & IGB_FLAG_HAS_MSIX)
7645 7646 7647
			wr32(E1000_EIMC, q_vector->eims_value);
		else
			igb_irq_disable(adapter);
7648
		napi_schedule(&q_vector->napi);
7649
	}
7650 7651 7652 7653
}
#endif /* CONFIG_NET_POLL_CONTROLLER */

/**
7654 7655 7656
 *  igb_io_error_detected - called when PCI error is detected
 *  @pdev: Pointer to PCI device
 *  @state: The current pci connection state
7657
 *
7658 7659 7660
 *  This function is called after a PCI bus error affecting
 *  this device has been detected.
 **/
7661 7662 7663 7664 7665 7666 7667 7668
static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
					      pci_channel_state_t state)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);

	netif_device_detach(netdev);

7669 7670 7671
	if (state == pci_channel_io_perm_failure)
		return PCI_ERS_RESULT_DISCONNECT;

7672 7673 7674 7675 7676 7677 7678 7679 7680
	if (netif_running(netdev))
		igb_down(adapter);
	pci_disable_device(pdev);

	/* Request a slot slot reset. */
	return PCI_ERS_RESULT_NEED_RESET;
}

/**
7681 7682
 *  igb_io_slot_reset - called after the pci bus has been reset.
 *  @pdev: Pointer to PCI device
7683
 *
7684 7685 7686
 *  Restart the card from scratch, as if from a cold-boot. Implementation
 *  resembles the first-half of the igb_resume routine.
 **/
7687 7688 7689 7690 7691
static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
7692
	pci_ers_result_t result;
T
Taku Izumi 已提交
7693
	int err;
7694

7695
	if (pci_enable_device_mem(pdev)) {
7696 7697
		dev_err(&pdev->dev,
			"Cannot re-enable PCI device after reset.\n");
7698 7699 7700 7701
		result = PCI_ERS_RESULT_DISCONNECT;
	} else {
		pci_set_master(pdev);
		pci_restore_state(pdev);
7702
		pci_save_state(pdev);
7703

7704 7705
		pci_enable_wake(pdev, PCI_D3hot, 0);
		pci_enable_wake(pdev, PCI_D3cold, 0);
7706

7707 7708 7709 7710
		igb_reset(adapter);
		wr32(E1000_WUS, ~0);
		result = PCI_ERS_RESULT_RECOVERED;
	}
7711

7712 7713
	err = pci_cleanup_aer_uncorrect_error_status(pdev);
	if (err) {
7714 7715 7716
		dev_err(&pdev->dev,
			"pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
			err);
7717 7718
		/* non-fatal, continue */
	}
7719 7720

	return result;
7721 7722 7723
}

/**
7724 7725
 *  igb_io_resume - called when traffic can start flowing again.
 *  @pdev: Pointer to PCI device
7726
 *
7727 7728 7729
 *  This callback is called when the error recovery driver tells us that
 *  its OK to resume normal operation. Implementation resembles the
 *  second-half of the igb_resume routine.
7730 7731 7732 7733 7734 7735 7736 7737 7738 7739 7740 7741 7742 7743 7744 7745
 */
static void igb_io_resume(struct pci_dev *pdev)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);

	if (netif_running(netdev)) {
		if (igb_up(adapter)) {
			dev_err(&pdev->dev, "igb_up failed after reset\n");
			return;
		}
	}

	netif_device_attach(netdev);

	/* let the f/w know that the h/w is now under the control of the
7746 7747
	 * driver.
	 */
7748 7749 7750
	igb_get_hw_control(adapter);
}

7751
static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
7752
			     u8 qsel)
7753 7754 7755 7756 7757 7758 7759 7760
{
	u32 rar_low, rar_high;
	struct e1000_hw *hw = &adapter->hw;

	/* HW expects these in little endian so we reverse the byte order
	 * from network order (big endian) to little endian
	 */
	rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
7761
		   ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
7762 7763 7764 7765 7766 7767 7768 7769 7770 7771 7772 7773 7774 7775 7776 7777
	rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));

	/* Indicate to hardware the Address is Valid. */
	rar_high |= E1000_RAH_AV;

	if (hw->mac.type == e1000_82575)
		rar_high |= E1000_RAH_POOL_1 * qsel;
	else
		rar_high |= E1000_RAH_POOL_1 << qsel;

	wr32(E1000_RAL(index), rar_low);
	wrfl();
	wr32(E1000_RAH(index), rar_high);
	wrfl();
}

7778
static int igb_set_vf_mac(struct igb_adapter *adapter,
7779
			  int vf, unsigned char *mac_addr)
7780 7781
{
	struct e1000_hw *hw = &adapter->hw;
7782
	/* VF MAC addresses start at end of receive addresses and moves
7783 7784
	 * towards the first, as a result a collision should not be possible
	 */
7785
	int rar_entry = hw->mac.rar_entry_count - (vf + 1);
7786

7787
	memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
7788

7789
	igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
7790 7791 7792 7793

	return 0;
}

7794 7795 7796 7797 7798 7799 7800
static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
		return -EINVAL;
	adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
	dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
7801 7802
	dev_info(&adapter->pdev->dev,
		 "Reload the VF driver to make this change effective.");
7803
	if (test_bit(__IGB_DOWN, &adapter->state)) {
7804 7805 7806 7807
		dev_warn(&adapter->pdev->dev,
			 "The VF MAC address has been set, but the PF device is not up.\n");
		dev_warn(&adapter->pdev->dev,
			 "Bring the PF device up before attempting to use the VF device.\n");
7808 7809 7810 7811
	}
	return igb_set_vf_mac(adapter, vf, mac);
}

7812 7813 7814 7815 7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833
static int igb_link_mbps(int internal_link_speed)
{
	switch (internal_link_speed) {
	case SPEED_100:
		return 100;
	case SPEED_1000:
		return 1000;
	default:
		return 0;
	}
}

static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
				  int link_speed)
{
	int rf_dec, rf_int;
	u32 bcnrc_val;

	if (tx_rate != 0) {
		/* Calculate the rate factor values to set */
		rf_int = link_speed / tx_rate;
		rf_dec = (link_speed - (rf_int * tx_rate));
7834 7835
		rf_dec = (rf_dec * (1 << E1000_RTTBCNRC_RF_INT_SHIFT)) /
			 tx_rate;
7836 7837

		bcnrc_val = E1000_RTTBCNRC_RS_ENA;
7838 7839
		bcnrc_val |= ((rf_int << E1000_RTTBCNRC_RF_INT_SHIFT) &
			      E1000_RTTBCNRC_RF_INT_MASK);
7840 7841 7842 7843 7844 7845
		bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
	} else {
		bcnrc_val = 0;
	}

	wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
7846
	/* Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
L
Lior Levy 已提交
7847 7848 7849
	 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
	 */
	wr32(E1000_RTTBCNRM, 0x14);
7850 7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861 7862 7863 7864 7865 7866 7867
	wr32(E1000_RTTBCNRC, bcnrc_val);
}

static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
{
	int actual_link_speed, i;
	bool reset_rate = false;

	/* VF TX rate limit was not set or not supported */
	if ((adapter->vf_rate_link_speed == 0) ||
	    (adapter->hw.mac.type != e1000_82576))
		return;

	actual_link_speed = igb_link_mbps(adapter->link_speed);
	if (actual_link_speed != adapter->vf_rate_link_speed) {
		reset_rate = true;
		adapter->vf_rate_link_speed = 0;
		dev_info(&adapter->pdev->dev,
7868
			 "Link speed has been changed. VF Transmit rate is disabled\n");
7869 7870 7871 7872 7873 7874 7875
	}

	for (i = 0; i < adapter->vfs_allocated_count; i++) {
		if (reset_rate)
			adapter->vf_data[i].tx_rate = 0;

		igb_set_vf_rate_limit(&adapter->hw, i,
7876 7877
				      adapter->vf_data[i].tx_rate,
				      actual_link_speed);
7878 7879 7880
	}
}

7881 7882
static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf,
			     int min_tx_rate, int max_tx_rate)
7883
{
7884 7885 7886 7887 7888 7889 7890
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
	int actual_link_speed;

	if (hw->mac.type != e1000_82576)
		return -EOPNOTSUPP;

7891 7892 7893
	if (min_tx_rate)
		return -EINVAL;

7894 7895 7896
	actual_link_speed = igb_link_mbps(adapter->link_speed);
	if ((vf >= adapter->vfs_allocated_count) ||
	    (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
7897 7898
	    (max_tx_rate < 0) ||
	    (max_tx_rate > actual_link_speed))
7899 7900 7901
		return -EINVAL;

	adapter->vf_rate_link_speed = actual_link_speed;
7902 7903
	adapter->vf_data[vf].tx_rate = (u16)max_tx_rate;
	igb_set_vf_rate_limit(hw, vf, max_tx_rate, actual_link_speed);
7904 7905

	return 0;
7906 7907
}

L
Lior Levy 已提交
7908 7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931 7932 7933 7934
static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
				   bool setting)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
	u32 reg_val, reg_offset;

	if (!adapter->vfs_allocated_count)
		return -EOPNOTSUPP;

	if (vf >= adapter->vfs_allocated_count)
		return -EINVAL;

	reg_offset = (hw->mac.type == e1000_82576) ? E1000_DTXSWC : E1000_TXSWC;
	reg_val = rd32(reg_offset);
	if (setting)
		reg_val |= ((1 << vf) |
			    (1 << (vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT)));
	else
		reg_val &= ~((1 << vf) |
			     (1 << (vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT)));
	wr32(reg_offset, reg_val);

	adapter->vf_data[vf].spoofchk_enabled = setting;
	return E1000_SUCCESS;
}

7935 7936 7937 7938 7939 7940 7941 7942
static int igb_ndo_get_vf_config(struct net_device *netdev,
				 int vf, struct ifla_vf_info *ivi)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	if (vf >= adapter->vfs_allocated_count)
		return -EINVAL;
	ivi->vf = vf;
	memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
7943 7944
	ivi->max_tx_rate = adapter->vf_data[vf].tx_rate;
	ivi->min_tx_rate = 0;
7945 7946
	ivi->vlan = adapter->vf_data[vf].pf_vlan;
	ivi->qos = adapter->vf_data[vf].pf_qos;
L
Lior Levy 已提交
7947
	ivi->spoofchk = adapter->vf_data[vf].spoofchk_enabled;
7948 7949 7950
	return 0;
}

7951 7952 7953
static void igb_vmm_control(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
7954
	u32 reg;
7955

7956 7957
	switch (hw->mac.type) {
	case e1000_82575:
7958 7959
	case e1000_i210:
	case e1000_i211:
7960
	case e1000_i354:
7961 7962
	default:
		/* replication is not supported for 82575 */
7963
		return;
7964 7965 7966 7967 7968
	case e1000_82576:
		/* notify HW that the MAC is adding vlan tags */
		reg = rd32(E1000_DTXCTL);
		reg |= E1000_DTXCTL_VLAN_ADDED;
		wr32(E1000_DTXCTL, reg);
7969
		/* Fall through */
7970 7971 7972 7973 7974
	case e1000_82580:
		/* enable replication vlan tag stripping */
		reg = rd32(E1000_RPLOLR);
		reg |= E1000_RPLOLR_STRVLAN;
		wr32(E1000_RPLOLR, reg);
7975
		/* Fall through */
7976 7977
	case e1000_i350:
		/* none of the above registers are supported by i350 */
7978 7979
		break;
	}
7980

7981 7982 7983
	if (adapter->vfs_allocated_count) {
		igb_vmdq_set_loopback_pf(hw, true);
		igb_vmdq_set_replication_pf(hw, true);
G
Greg Rose 已提交
7984
		igb_vmdq_set_anti_spoofing_pf(hw, true,
7985
					      adapter->vfs_allocated_count);
7986 7987 7988 7989
	} else {
		igb_vmdq_set_loopback_pf(hw, false);
		igb_vmdq_set_replication_pf(hw, false);
	}
7990 7991
}

7992 7993 7994 7995 7996 7997 7998 7999 8000 8001 8002 8003 8004
static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 dmac_thr;
	u16 hwm;

	if (hw->mac.type > e1000_82580) {
		if (adapter->flags & IGB_FLAG_DMAC) {
			u32 reg;

			/* force threshold to 0. */
			wr32(E1000_DMCTXTH, 0);

8005
			/* DMA Coalescing high water mark needs to be greater
8006 8007
			 * than the Rx threshold. Set hwm to PBA - max frame
			 * size in 16B units, capping it at PBA - 6KB.
8008
			 */
8009 8010 8011 8012 8013 8014 8015 8016 8017
			hwm = 64 * pba - adapter->max_frame_size / 16;
			if (hwm < 64 * (pba - 6))
				hwm = 64 * (pba - 6);
			reg = rd32(E1000_FCRTC);
			reg &= ~E1000_FCRTC_RTH_COAL_MASK;
			reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
				& E1000_FCRTC_RTH_COAL_MASK);
			wr32(E1000_FCRTC, reg);

8018
			/* Set the DMA Coalescing Rx threshold to PBA - 2 * max
8019 8020 8021 8022 8023
			 * frame size, capping it at PBA - 10KB.
			 */
			dmac_thr = pba - adapter->max_frame_size / 512;
			if (dmac_thr < pba - 10)
				dmac_thr = pba - 10;
8024 8025 8026 8027 8028 8029 8030 8031 8032 8033
			reg = rd32(E1000_DMACR);
			reg &= ~E1000_DMACR_DMACTHR_MASK;
			reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
				& E1000_DMACR_DMACTHR_MASK);

			/* transition to L0x or L1 if available..*/
			reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);

			/* watchdog timer= +-1000 usec in 32usec intervals */
			reg |= (1000 >> 5);
8034 8035

			/* Disable BMC-to-OS Watchdog Enable */
8036 8037 8038
			if (hw->mac.type != e1000_i354)
				reg &= ~E1000_DMACR_DC_BMC2OSW_EN;

8039 8040
			wr32(E1000_DMACR, reg);

8041
			/* no lower threshold to disable
8042 8043 8044 8045 8046 8047 8048 8049
			 * coalescing(smart fifb)-UTRESH=0
			 */
			wr32(E1000_DMCRTRH, 0);

			reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);

			wr32(E1000_DMCTLX, reg);

8050
			/* free space in tx packet buffer to wake from
8051 8052 8053 8054 8055
			 * DMA coal
			 */
			wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
			     (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);

8056
			/* make low power state decision controlled
8057 8058 8059 8060 8061 8062 8063 8064
			 * by DMA coal
			 */
			reg = rd32(E1000_PCIEMISC);
			reg &= ~E1000_PCIEMISC_LX_DECISION;
			wr32(E1000_PCIEMISC, reg);
		} /* endif adapter->dmac is not disabled */
	} else if (hw->mac.type == e1000_82580) {
		u32 reg = rd32(E1000_PCIEMISC);
8065

8066 8067 8068 8069 8070
		wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
		wr32(E1000_DMACR, 0);
	}
}

8071 8072
/**
 *  igb_read_i2c_byte - Reads 8 bit word over I2C
C
Carolyn Wyborny 已提交
8073 8074 8075 8076 8077 8078 8079
 *  @hw: pointer to hardware structure
 *  @byte_offset: byte offset to read
 *  @dev_addr: device address
 *  @data: value read
 *
 *  Performs byte read operation over I2C interface at
 *  a specified device address.
8080
 **/
C
Carolyn Wyborny 已提交
8081
s32 igb_read_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
8082
		      u8 dev_addr, u8 *data)
C
Carolyn Wyborny 已提交
8083 8084
{
	struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
8085
	struct i2c_client *this_client = adapter->i2c_client;
C
Carolyn Wyborny 已提交
8086 8087 8088 8089 8090 8091 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104 8105 8106 8107 8108
	s32 status;
	u16 swfw_mask = 0;

	if (!this_client)
		return E1000_ERR_I2C;

	swfw_mask = E1000_SWFW_PHY0_SM;

	if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)
	    != E1000_SUCCESS)
		return E1000_ERR_SWFW_SYNC;

	status = i2c_smbus_read_byte_data(this_client, byte_offset);
	hw->mac.ops.release_swfw_sync(hw, swfw_mask);

	if (status < 0)
		return E1000_ERR_I2C;
	else {
		*data = status;
		return E1000_SUCCESS;
	}
}

8109 8110
/**
 *  igb_write_i2c_byte - Writes 8 bit word over I2C
C
Carolyn Wyborny 已提交
8111 8112 8113 8114 8115 8116 8117
 *  @hw: pointer to hardware structure
 *  @byte_offset: byte offset to write
 *  @dev_addr: device address
 *  @data: value to write
 *
 *  Performs byte write operation over I2C interface at
 *  a specified device address.
8118
 **/
C
Carolyn Wyborny 已提交
8119
s32 igb_write_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
8120
		       u8 dev_addr, u8 data)
C
Carolyn Wyborny 已提交
8121 8122
{
	struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
8123
	struct i2c_client *this_client = adapter->i2c_client;
C
Carolyn Wyborny 已提交
8124 8125 8126 8127 8128 8129 8130 8131 8132 8133 8134 8135 8136 8137 8138 8139 8140
	s32 status;
	u16 swfw_mask = E1000_SWFW_PHY0_SM;

	if (!this_client)
		return E1000_ERR_I2C;

	if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask) != E1000_SUCCESS)
		return E1000_ERR_SWFW_SYNC;
	status = i2c_smbus_write_byte_data(this_client, byte_offset, data);
	hw->mac.ops.release_swfw_sync(hw, swfw_mask);

	if (status)
		return E1000_ERR_I2C;
	else
		return E1000_SUCCESS;

}
8141 8142 8143 8144 8145 8146 8147 8148 8149 8150

int igb_reinit_queues(struct igb_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
	int err = 0;

	if (netif_running(netdev))
		igb_close(netdev);

8151
	igb_reset_interrupt_capability(adapter);
8152 8153 8154 8155 8156 8157 8158 8159 8160 8161 8162

	if (igb_init_interrupt_scheme(adapter, true)) {
		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
		return -ENOMEM;
	}

	if (netif_running(netdev))
		err = igb_open(netdev);

	return err;
}
8163
/* igb_main.c */