e1000_main.c 142.6 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*******************************************************************************

3 4 5 6 7 8 9 10 11 12
  Intel PRO/1000 Linux driver
  Copyright(c) 1999 - 2006 Intel Corporation.

  This program is free software; you can redistribute it and/or modify it
  under the terms and conditions of the GNU General Public License,
  version 2, as published by the Free Software Foundation.

  This program is distributed in the hope it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
L
Linus Torvalds 已提交
13
  more details.
14

L
Linus Torvalds 已提交
15
  You should have received a copy of the GNU General Public License along with
16 17 18 19 20 21
  this program; if not, write to the Free Software Foundation, Inc.,
  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.

  The full GNU General Public License is included in this distribution in
  the file called "COPYING".

L
Linus Torvalds 已提交
22 23
  Contact Information:
  Linux NICS <linux.nics@intel.com>
24
  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
L
Linus Torvalds 已提交
25 26 27 28 29
  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497

*******************************************************************************/

#include "e1000.h"
A
Andrew Morton 已提交
30
#include <net/ip6_checksum.h>
31
#include <linux/io.h>
32
#include <linux/prefetch.h>
J
Jiri Pirko 已提交
33 34
#include <linux/bitops.h>
#include <linux/if_vlan.h>
35

L
Linus Torvalds 已提交
36
char e1000_driver_name[] = "e1000";
37
static char e1000_driver_string[] = "Intel(R) PRO/1000 Network Driver";
A
Anupam Chanda 已提交
38
#define DRV_VERSION "7.3.21-k8-NAPI"
S
Stephen Hemminger 已提交
39 40
const char e1000_driver_version[] = DRV_VERSION;
static const char e1000_copyright[] = "Copyright (c) 1999-2006 Intel Corporation.";
L
Linus Torvalds 已提交
41 42 43 44 45 46 47 48

/* e1000_pci_tbl - PCI Device ID Table
 *
 * Last entry must be all 0s
 *
 * Macro expands to...
 *   {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
 */
49
static DEFINE_PCI_DEVICE_TABLE(e1000_pci_tbl) = {
L
Linus Torvalds 已提交
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
	INTEL_E1000_ETHERNET_DEVICE(0x1000),
	INTEL_E1000_ETHERNET_DEVICE(0x1001),
	INTEL_E1000_ETHERNET_DEVICE(0x1004),
	INTEL_E1000_ETHERNET_DEVICE(0x1008),
	INTEL_E1000_ETHERNET_DEVICE(0x1009),
	INTEL_E1000_ETHERNET_DEVICE(0x100C),
	INTEL_E1000_ETHERNET_DEVICE(0x100D),
	INTEL_E1000_ETHERNET_DEVICE(0x100E),
	INTEL_E1000_ETHERNET_DEVICE(0x100F),
	INTEL_E1000_ETHERNET_DEVICE(0x1010),
	INTEL_E1000_ETHERNET_DEVICE(0x1011),
	INTEL_E1000_ETHERNET_DEVICE(0x1012),
	INTEL_E1000_ETHERNET_DEVICE(0x1013),
	INTEL_E1000_ETHERNET_DEVICE(0x1014),
	INTEL_E1000_ETHERNET_DEVICE(0x1015),
	INTEL_E1000_ETHERNET_DEVICE(0x1016),
	INTEL_E1000_ETHERNET_DEVICE(0x1017),
	INTEL_E1000_ETHERNET_DEVICE(0x1018),
	INTEL_E1000_ETHERNET_DEVICE(0x1019),
69
	INTEL_E1000_ETHERNET_DEVICE(0x101A),
L
Linus Torvalds 已提交
70 71 72 73 74 75 76 77 78 79 80 81 82 83
	INTEL_E1000_ETHERNET_DEVICE(0x101D),
	INTEL_E1000_ETHERNET_DEVICE(0x101E),
	INTEL_E1000_ETHERNET_DEVICE(0x1026),
	INTEL_E1000_ETHERNET_DEVICE(0x1027),
	INTEL_E1000_ETHERNET_DEVICE(0x1028),
	INTEL_E1000_ETHERNET_DEVICE(0x1075),
	INTEL_E1000_ETHERNET_DEVICE(0x1076),
	INTEL_E1000_ETHERNET_DEVICE(0x1077),
	INTEL_E1000_ETHERNET_DEVICE(0x1078),
	INTEL_E1000_ETHERNET_DEVICE(0x1079),
	INTEL_E1000_ETHERNET_DEVICE(0x107A),
	INTEL_E1000_ETHERNET_DEVICE(0x107B),
	INTEL_E1000_ETHERNET_DEVICE(0x107C),
	INTEL_E1000_ETHERNET_DEVICE(0x108A),
84 85
	INTEL_E1000_ETHERNET_DEVICE(0x1099),
	INTEL_E1000_ETHERNET_DEVICE(0x10B5),
86
	INTEL_E1000_ETHERNET_DEVICE(0x2E6E),
L
Linus Torvalds 已提交
87 88 89 90 91 92
	/* required last entry */
	{0,}
};

MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);

93 94 95 96 97 98 99 100
int e1000_up(struct e1000_adapter *adapter);
void e1000_down(struct e1000_adapter *adapter);
void e1000_reinit_locked(struct e1000_adapter *adapter);
void e1000_reset(struct e1000_adapter *adapter);
int e1000_setup_all_tx_resources(struct e1000_adapter *adapter);
int e1000_setup_all_rx_resources(struct e1000_adapter *adapter);
void e1000_free_all_tx_resources(struct e1000_adapter *adapter);
void e1000_free_all_rx_resources(struct e1000_adapter *adapter);
101
static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
102
                             struct e1000_tx_ring *txdr);
103
static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
104
                             struct e1000_rx_ring *rxdr);
105
static void e1000_free_tx_resources(struct e1000_adapter *adapter,
106
                             struct e1000_tx_ring *tx_ring);
107
static void e1000_free_rx_resources(struct e1000_adapter *adapter,
108 109
                             struct e1000_rx_ring *rx_ring);
void e1000_update_stats(struct e1000_adapter *adapter);
L
Linus Torvalds 已提交
110 111 112 113 114

static int e1000_init_module(void);
static void e1000_exit_module(void);
static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
static void __devexit e1000_remove(struct pci_dev *pdev);
115
static int e1000_alloc_queues(struct e1000_adapter *adapter);
L
Linus Torvalds 已提交
116 117 118 119 120 121
static int e1000_sw_init(struct e1000_adapter *adapter);
static int e1000_open(struct net_device *netdev);
static int e1000_close(struct net_device *netdev);
static void e1000_configure_tx(struct e1000_adapter *adapter);
static void e1000_configure_rx(struct e1000_adapter *adapter);
static void e1000_setup_rctl(struct e1000_adapter *adapter);
122 123 124 125 126 127
static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter);
static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter);
static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
                                struct e1000_tx_ring *tx_ring);
static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
                                struct e1000_rx_ring *rx_ring);
128
static void e1000_set_rx_mode(struct net_device *netdev);
J
Jesse Brandeburg 已提交
129
static void e1000_update_phy_info_task(struct work_struct *work);
130
static void e1000_watchdog(struct work_struct *work);
J
Jesse Brandeburg 已提交
131
static void e1000_82547_tx_fifo_stall_task(struct work_struct *work);
132 133
static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
				    struct net_device *netdev);
L
Linus Torvalds 已提交
134 135 136
static struct net_device_stats * e1000_get_stats(struct net_device *netdev);
static int e1000_change_mtu(struct net_device *netdev, int new_mtu);
static int e1000_set_mac(struct net_device *netdev, void *p);
137
static irqreturn_t e1000_intr(int irq, void *data);
J
Joe Perches 已提交
138 139
static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
			       struct e1000_tx_ring *tx_ring);
140
static int e1000_clean(struct napi_struct *napi, int budget);
J
Joe Perches 已提交
141 142 143
static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
			       struct e1000_rx_ring *rx_ring,
			       int *work_done, int work_to_do);
144 145 146
static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
				     struct e1000_rx_ring *rx_ring,
				     int *work_done, int work_to_do);
147
static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
148
				   struct e1000_rx_ring *rx_ring,
149
				   int cleaned_count);
150 151 152
static void e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
					 struct e1000_rx_ring *rx_ring,
					 int cleaned_count);
L
Linus Torvalds 已提交
153 154 155 156 157 158
static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd);
static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
			   int cmd);
static void e1000_enter_82542_rst(struct e1000_adapter *adapter);
static void e1000_leave_82542_rst(struct e1000_adapter *adapter);
static void e1000_tx_timeout(struct net_device *dev);
159
static void e1000_reset_task(struct work_struct *work);
L
Linus Torvalds 已提交
160
static void e1000_smartspeed(struct e1000_adapter *adapter);
161 162
static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
                                       struct sk_buff *skb);
L
Linus Torvalds 已提交
163

J
Jiri Pirko 已提交
164
static bool e1000_vlan_used(struct e1000_adapter *adapter);
165 166
static void e1000_vlan_mode(struct net_device *netdev,
			    netdev_features_t features);
167 168
static int e1000_vlan_rx_add_vid(struct net_device *netdev, u16 vid);
static int e1000_vlan_rx_kill_vid(struct net_device *netdev, u16 vid);
L
Linus Torvalds 已提交
169 170
static void e1000_restore_vlan(struct e1000_adapter *adapter);

A
Auke Kok 已提交
171
#ifdef CONFIG_PM
172
static int e1000_suspend(struct pci_dev *pdev, pm_message_t state);
L
Linus Torvalds 已提交
173 174
static int e1000_resume(struct pci_dev *pdev);
#endif
175
static void e1000_shutdown(struct pci_dev *pdev);
L
Linus Torvalds 已提交
176 177 178 179 180 181

#ifdef CONFIG_NET_POLL_CONTROLLER
/* for netdump / net console */
static void e1000_netpoll (struct net_device *netdev);
#endif

182 183 184 185 186 187
#define COPYBREAK_DEFAULT 256
static unsigned int copybreak __read_mostly = COPYBREAK_DEFAULT;
module_param(copybreak, uint, 0644);
MODULE_PARM_DESC(copybreak,
	"Maximum size of packet that is copied to a new buffer on receive");

A
Auke Kok 已提交
188 189 190 191 192 193 194 195 196 197
static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
                     pci_channel_state_t state);
static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev);
static void e1000_io_resume(struct pci_dev *pdev);

static struct pci_error_handlers e1000_err_handler = {
	.error_detected = e1000_io_error_detected,
	.slot_reset = e1000_io_slot_reset,
	.resume = e1000_io_resume,
};
198

L
Linus Torvalds 已提交
199 200 201 202 203
static struct pci_driver e1000_driver = {
	.name     = e1000_driver_name,
	.id_table = e1000_pci_tbl,
	.probe    = e1000_probe,
	.remove   = __devexit_p(e1000_remove),
204
#ifdef CONFIG_PM
L
Lucas De Marchi 已提交
205
	/* Power Management Hooks */
L
Linus Torvalds 已提交
206
	.suspend  = e1000_suspend,
207
	.resume   = e1000_resume,
L
Linus Torvalds 已提交
208
#endif
A
Auke Kok 已提交
209 210
	.shutdown = e1000_shutdown,
	.err_handler = &e1000_err_handler
L
Linus Torvalds 已提交
211 212 213 214 215 216 217 218 219 220 221
};

MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRV_VERSION);

static int debug = NETIF_MSG_DRV | NETIF_MSG_PROBE;
module_param(debug, int, 0);
MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");

222 223 224 225 226 227 228 229 230 231 232
/**
 * e1000_get_hw_dev - return device
 * used by hardware layer to print debugging information
 *
 **/
struct net_device *e1000_get_hw_dev(struct e1000_hw *hw)
{
	struct e1000_adapter *adapter = hw->back;
	return adapter->netdev;
}

L
Linus Torvalds 已提交
233 234 235 236 237 238 239
/**
 * e1000_init_module - Driver Registration Routine
 *
 * e1000_init_module is the first routine called when the driver is
 * loaded. All it does is register with the PCI subsystem.
 **/

240
static int __init e1000_init_module(void)
L
Linus Torvalds 已提交
241 242
{
	int ret;
243
	pr_info("%s - version %s\n", e1000_driver_string, e1000_driver_version);
L
Linus Torvalds 已提交
244

245
	pr_info("%s\n", e1000_copyright);
L
Linus Torvalds 已提交
246

247
	ret = pci_register_driver(&e1000_driver);
248 249
	if (copybreak != COPYBREAK_DEFAULT) {
		if (copybreak == 0)
250
			pr_info("copybreak disabled\n");
251
		else
252 253
			pr_info("copybreak enabled for "
				   "packets <= %u bytes\n", copybreak);
254
	}
L
Linus Torvalds 已提交
255 256 257 258 259 260 261 262 263 264 265 266
	return ret;
}

module_init(e1000_init_module);

/**
 * e1000_exit_module - Driver Exit Cleanup Routine
 *
 * e1000_exit_module is called just before the driver is removed
 * from memory.
 **/

267
static void __exit e1000_exit_module(void)
L
Linus Torvalds 已提交
268 269 270 271 272 273
{
	pci_unregister_driver(&e1000_driver);
}

module_exit(e1000_exit_module);

274 275 276
static int e1000_request_irq(struct e1000_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
A
Al Viro 已提交
277
	irq_handler_t handler = e1000_intr;
278 279
	int irq_flags = IRQF_SHARED;
	int err;
280

281 282 283
	err = request_irq(adapter->pdev->irq, handler, irq_flags, netdev->name,
	                  netdev);
	if (err) {
284
		e_err(probe, "Unable to allocate interrupt Error: %d\n", err);
285
	}
286 287 288 289 290 291 292 293 294 295 296

	return err;
}

static void e1000_free_irq(struct e1000_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;

	free_irq(adapter->pdev->irq, netdev);
}

L
Linus Torvalds 已提交
297 298 299 300 301
/**
 * e1000_irq_disable - Mask off interrupt generation on the NIC
 * @adapter: board private structure
 **/

302
static void e1000_irq_disable(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
303
{
J
Joe Perches 已提交
304 305 306 307
	struct e1000_hw *hw = &adapter->hw;

	ew32(IMC, ~0);
	E1000_WRITE_FLUSH();
L
Linus Torvalds 已提交
308 309 310 311 312 313 314 315
	synchronize_irq(adapter->pdev->irq);
}

/**
 * e1000_irq_enable - Enable default interrupt generation settings
 * @adapter: board private structure
 **/

316
static void e1000_irq_enable(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
317
{
J
Joe Perches 已提交
318 319 320 321
	struct e1000_hw *hw = &adapter->hw;

	ew32(IMS, IMS_ENABLE_MASK);
	E1000_WRITE_FLUSH();
L
Linus Torvalds 已提交
322
}
323

324
static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
325
{
J
Joe Perches 已提交
326
	struct e1000_hw *hw = &adapter->hw;
327
	struct net_device *netdev = adapter->netdev;
J
Joe Perches 已提交
328
	u16 vid = hw->mng_cookie.vlan_id;
329
	u16 old_vid = adapter->mng_vlan_id;
J
Jesse Brandeburg 已提交
330

J
Jiri Pirko 已提交
331 332 333 334 335 336 337
	if (!e1000_vlan_used(adapter))
		return;

	if (!test_bit(vid, adapter->active_vlans)) {
		if (hw->mng_cookie.status &
		    E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) {
			e1000_vlan_rx_add_vid(netdev, vid);
338
			adapter->mng_vlan_id = vid;
J
Jiri Pirko 已提交
339 340 341 342 343 344 345 346 347
		} else {
			adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
		}
		if ((old_vid != (u16)E1000_MNG_VLAN_NONE) &&
		    (vid != old_vid) &&
		    !test_bit(old_vid, adapter->active_vlans))
			e1000_vlan_rx_kill_vid(netdev, old_vid);
	} else {
		adapter->mng_vlan_id = vid;
348 349
	}
}
350

351
static void e1000_init_manageability(struct e1000_adapter *adapter)
352
{
J
Joe Perches 已提交
353 354
	struct e1000_hw *hw = &adapter->hw;

355
	if (adapter->en_mng_pt) {
J
Joe Perches 已提交
356
		u32 manc = er32(MANC);
357 358 359 360

		/* disable hardware interception of ARP */
		manc &= ~(E1000_MANC_ARP_EN);

J
Joe Perches 已提交
361
		ew32(MANC, manc);
362 363 364
	}
}

365
static void e1000_release_manageability(struct e1000_adapter *adapter)
366
{
J
Joe Perches 已提交
367 368
	struct e1000_hw *hw = &adapter->hw;

369
	if (adapter->en_mng_pt) {
J
Joe Perches 已提交
370
		u32 manc = er32(MANC);
371 372 373 374

		/* re-enable hardware interception of ARP */
		manc |= E1000_MANC_ARP_EN;

J
Joe Perches 已提交
375
		ew32(MANC, manc);
376 377 378
	}
}

379 380 381 382 383
/**
 * e1000_configure - configure the hardware for RX and TX
 * @adapter = private board structure
 **/
static void e1000_configure(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
384 385
{
	struct net_device *netdev = adapter->netdev;
386
	int i;
L
Linus Torvalds 已提交
387

388
	e1000_set_rx_mode(netdev);
L
Linus Torvalds 已提交
389 390

	e1000_restore_vlan(adapter);
391
	e1000_init_manageability(adapter);
L
Linus Torvalds 已提交
392 393 394 395

	e1000_configure_tx(adapter);
	e1000_setup_rctl(adapter);
	e1000_configure_rx(adapter);
396 397 398
	/* call E1000_DESC_UNUSED which always leaves
	 * at least 1 descriptor unused to make sure
	 * next_to_use != next_to_clean */
399
	for (i = 0; i < adapter->num_rx_queues; i++) {
400
		struct e1000_rx_ring *ring = &adapter->rx_ring[i];
401 402
		adapter->alloc_rx_buf(adapter, ring,
		                      E1000_DESC_UNUSED(ring));
403
	}
404 405 406 407
}

int e1000_up(struct e1000_adapter *adapter)
{
J
Joe Perches 已提交
408 409
	struct e1000_hw *hw = &adapter->hw;

410 411 412 413
	/* hardware has been reset, we need to reload some things */
	e1000_configure(adapter);

	clear_bit(__E1000_DOWN, &adapter->flags);
414

415
	napi_enable(&adapter->napi);
416

417 418
	e1000_irq_enable(adapter);

419 420
	netif_wake_queue(adapter->netdev);

421
	/* fire a link change interrupt to start the watchdog */
J
Joe Perches 已提交
422
	ew32(ICS, E1000_ICS_LSC);
L
Linus Torvalds 已提交
423 424 425
	return 0;
}

426 427 428 429 430 431 432 433 434 435
/**
 * e1000_power_up_phy - restore link in case the phy was powered down
 * @adapter: address of board private structure
 *
 * The phy may be powered down to save power and turn off link when the
 * driver is unloaded and wake on lan is not enabled (among others)
 * *** this routine MUST be followed by a call to e1000_reset ***
 *
 **/

436
void e1000_power_up_phy(struct e1000_adapter *adapter)
437
{
J
Joe Perches 已提交
438
	struct e1000_hw *hw = &adapter->hw;
439
	u16 mii_reg = 0;
440 441

	/* Just clear the power down bit to wake the phy back up */
J
Joe Perches 已提交
442
	if (hw->media_type == e1000_media_type_copper) {
443 444
		/* according to the manual, the phy will retain its
		 * settings across a power-down/up cycle */
J
Joe Perches 已提交
445
		e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
446
		mii_reg &= ~MII_CR_POWER_DOWN;
J
Joe Perches 已提交
447
		e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
448 449 450 451 452
	}
}

static void e1000_power_down_phy(struct e1000_adapter *adapter)
{
J
Joe Perches 已提交
453 454
	struct e1000_hw *hw = &adapter->hw;

455
	/* Power down the PHY so no link is implied when interface is down *
J
Joe Perches 已提交
456
	 * The PHY cannot be powered down if any of the following is true *
457 458 459
	 * (a) WoL is enabled
	 * (b) AMT is active
	 * (c) SoL/IDER session is active */
J
Joe Perches 已提交
460 461
	if (!adapter->wol && hw->mac_type >= e1000_82540 &&
	   hw->media_type == e1000_media_type_copper) {
462
		u16 mii_reg = 0;
463

J
Joe Perches 已提交
464
		switch (hw->mac_type) {
465 466 467 468
		case e1000_82540:
		case e1000_82545:
		case e1000_82545_rev_3:
		case e1000_82546:
469
		case e1000_ce4100:
470 471 472 473 474
		case e1000_82546_rev_3:
		case e1000_82541:
		case e1000_82541_rev_2:
		case e1000_82547:
		case e1000_82547_rev_2:
J
Joe Perches 已提交
475
			if (er32(MANC) & E1000_MANC_SMBUS_EN)
476 477 478 479 480
				goto out;
			break;
		default:
			goto out;
		}
J
Joe Perches 已提交
481
		e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
482
		mii_reg |= MII_CR_POWER_DOWN;
J
Joe Perches 已提交
483
		e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
J
Jesse Brandeburg 已提交
484
		msleep(1);
485
	}
486 487
out:
	return;
488 489
}

490 491 492 493 494 495 496 497 498
static void e1000_down_and_stop(struct e1000_adapter *adapter)
{
	set_bit(__E1000_DOWN, &adapter->flags);
	cancel_work_sync(&adapter->reset_task);
	cancel_delayed_work_sync(&adapter->watchdog_task);
	cancel_delayed_work_sync(&adapter->phy_info_task);
	cancel_delayed_work_sync(&adapter->fifo_stall_task);
}

499
void e1000_down(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
500
{
501
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
502
	struct net_device *netdev = adapter->netdev;
503
	u32 rctl, tctl;
L
Linus Torvalds 已提交
504

A
Auke Kok 已提交
505

506 507 508 509 510
	/* disable receives in the hardware */
	rctl = er32(RCTL);
	ew32(RCTL, rctl & ~E1000_RCTL_EN);
	/* flush and sleep below */

J
Jesse Brandeburg 已提交
511
	netif_tx_disable(netdev);
512 513 514 515 516 517 518 519 520

	/* disable transmits in the hardware */
	tctl = er32(TCTL);
	tctl &= ~E1000_TCTL_EN;
	ew32(TCTL, tctl);
	/* flush both disables and wait for them to finish */
	E1000_WRITE_FLUSH();
	msleep(10);

521
	napi_disable(&adapter->napi);
522

L
Linus Torvalds 已提交
523
	e1000_irq_disable(adapter);
524

A
Anupam Chanda 已提交
525 526 527
	/*
	 * Setting DOWN must be after irq_disable to prevent
	 * a screaming interrupt.  Setting DOWN also prevents
528
	 * tasks from rescheduling.
A
Anupam Chanda 已提交
529
	 */
530
	e1000_down_and_stop(adapter);
L
Linus Torvalds 已提交
531 532 533 534 535 536

	adapter->link_speed = 0;
	adapter->link_duplex = 0;
	netif_carrier_off(netdev);

	e1000_reset(adapter);
537 538
	e1000_clean_all_tx_rings(adapter);
	e1000_clean_all_rx_rings(adapter);
L
Linus Torvalds 已提交
539 540
}

541
static void e1000_reinit_safe(struct e1000_adapter *adapter)
542 543 544
{
	while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
		msleep(1);
545
	mutex_lock(&adapter->mutex);
546 547
	e1000_down(adapter);
	e1000_up(adapter);
548
	mutex_unlock(&adapter->mutex);
549 550 551
	clear_bit(__E1000_RESETTING, &adapter->flags);
}

552
void e1000_reinit_locked(struct e1000_adapter *adapter)
553
{
554 555
	/* if rtnl_lock is not held the call path is bogus */
	ASSERT_RTNL();
556 557 558 559 560 561
	WARN_ON(in_interrupt());
	while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
		msleep(1);
	e1000_down(adapter);
	e1000_up(adapter);
	clear_bit(__E1000_RESETTING, &adapter->flags);
L
Linus Torvalds 已提交
562 563
}

564
void e1000_reset(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
565
{
J
Joe Perches 已提交
566
	struct e1000_hw *hw = &adapter->hw;
567
	u32 pba = 0, tx_space, min_tx_space, min_rx_space;
J
Joe Perches 已提交
568
	bool legacy_pba_adjust = false;
569
	u16 hwm;
L
Linus Torvalds 已提交
570 571 572 573 574

	/* Repartition Pba for greater than 9k mtu
	 * To take effect CTRL.RST is required.
	 */

J
Joe Perches 已提交
575
	switch (hw->mac_type) {
576 577 578 579 580 581 582
	case e1000_82542_rev2_0:
	case e1000_82542_rev2_1:
	case e1000_82543:
	case e1000_82544:
	case e1000_82540:
	case e1000_82541:
	case e1000_82541_rev_2:
J
Joe Perches 已提交
583
		legacy_pba_adjust = true;
584 585 586 587 588
		pba = E1000_PBA_48K;
		break;
	case e1000_82545:
	case e1000_82545_rev_3:
	case e1000_82546:
589
	case e1000_ce4100:
590 591 592
	case e1000_82546_rev_3:
		pba = E1000_PBA_48K;
		break;
593
	case e1000_82547:
594
	case e1000_82547_rev_2:
J
Joe Perches 已提交
595
		legacy_pba_adjust = true;
596 597
		pba = E1000_PBA_30K;
		break;
598 599
	case e1000_undefined:
	case e1000_num_macs:
600 601 602
		break;
	}

J
Joe Perches 已提交
603
	if (legacy_pba_adjust) {
604
		if (hw->max_frame_size > E1000_RXBUFFER_8192)
605
			pba -= 8; /* allocate more FIFO for Tx */
606

J
Joe Perches 已提交
607
		if (hw->mac_type == e1000_82547) {
608 609 610 611 612 613
			adapter->tx_fifo_head = 0;
			adapter->tx_head_addr = pba << E1000_TX_HEAD_ADDR_SHIFT;
			adapter->tx_fifo_size =
				(E1000_PBA_40K - pba) << E1000_PBA_BYTES_SHIFT;
			atomic_set(&adapter->tx_fifo_stall, 0);
		}
614
	} else if (hw->max_frame_size >  ETH_FRAME_LEN + ETH_FCS_LEN) {
615
		/* adjust PBA for jumbo frames */
J
Joe Perches 已提交
616
		ew32(PBA, pba);
617 618

		/* To maintain wire speed transmits, the Tx FIFO should be
619
		 * large enough to accommodate two full transmit packets,
620
		 * rounded up to the next 1KB and expressed in KB.  Likewise,
621
		 * the Rx FIFO should be large enough to accommodate at least
622 623
		 * one full receive packet and is similarly rounded up and
		 * expressed in KB. */
J
Joe Perches 已提交
624
		pba = er32(PBA);
625 626 627 628
		/* upper 16 bits has Tx packet buffer allocation size in KB */
		tx_space = pba >> 16;
		/* lower 16 bits has Rx packet buffer allocation size in KB */
		pba &= 0xffff;
629 630 631 632 633 634 635
		/*
		 * the tx fifo also stores 16 bytes of information about the tx
		 * but don't include ethernet FCS because hardware appends it
		 */
		min_tx_space = (hw->max_frame_size +
		                sizeof(struct e1000_tx_desc) -
		                ETH_FCS_LEN) * 2;
636
		min_tx_space = ALIGN(min_tx_space, 1024);
637
		min_tx_space >>= 10;
638 639
		/* software strips receive CRC, so leave room for it */
		min_rx_space = hw->max_frame_size;
640
		min_rx_space = ALIGN(min_rx_space, 1024);
641 642 643 644 645 646 647 648 649 650
		min_rx_space >>= 10;

		/* If current Tx allocation is less than the min Tx FIFO size,
		 * and the min Tx FIFO size is less than the current Rx FIFO
		 * allocation, take space away from current Rx allocation */
		if (tx_space < min_tx_space &&
		    ((min_tx_space - tx_space) < pba)) {
			pba = pba - (min_tx_space - tx_space);

			/* PCI/PCIx hardware has PBA alignment constraints */
J
Joe Perches 已提交
651
			switch (hw->mac_type) {
652 653 654 655 656 657 658 659 660
			case e1000_82545 ... e1000_82546_rev_3:
				pba &= ~(E1000_PBA_8K - 1);
				break;
			default:
				break;
			}

			/* if short on rx space, rx wins and must trump tx
			 * adjustment or use Early Receive if available */
661 662
			if (pba < min_rx_space)
				pba = min_rx_space;
663
		}
L
Linus Torvalds 已提交
664
	}
665

J
Joe Perches 已提交
666
	ew32(PBA, pba);
L
Linus Torvalds 已提交
667

668 669 670 671 672 673 674 675 676 677 678 679 680 681 682
	/*
	 * flow control settings:
	 * The high water mark must be low enough to fit one full frame
	 * (or the size used for early receive) above it in the Rx FIFO.
	 * Set it to the lower of:
	 * - 90% of the Rx FIFO size, and
	 * - the full Rx FIFO size minus the early receive size (for parts
	 *   with ERT support assuming ERT set to E1000_ERT_2048), or
	 * - the full Rx FIFO size minus one full frame
	 */
	hwm = min(((pba << 10) * 9 / 10),
		  ((pba << 10) - hw->max_frame_size));

	hw->fc_high_water = hwm & 0xFFF8;	/* 8-byte granularity */
	hw->fc_low_water = hw->fc_high_water - 8;
683
	hw->fc_pause_time = E1000_FC_PAUSE_TIME;
J
Joe Perches 已提交
684 685
	hw->fc_send_xon = 1;
	hw->fc = hw->original_fc;
L
Linus Torvalds 已提交
686

687
	/* Allow time for pending master requests to run */
J
Joe Perches 已提交
688 689 690
	e1000_reset_hw(hw);
	if (hw->mac_type >= e1000_82544)
		ew32(WUC, 0);
691

J
Joe Perches 已提交
692
	if (e1000_init_hw(hw))
693
		e_dev_err("Hardware Error\n");
694
	e1000_update_mng_vlan(adapter);
695 696

	/* if (adapter->hwflags & HWFLAGS_PHY_PWR_BIT) { */
J
Joe Perches 已提交
697 698 699 700
	if (hw->mac_type >= e1000_82544 &&
	    hw->autoneg == 1 &&
	    hw->autoneg_advertised == ADVERTISE_1000_FULL) {
		u32 ctrl = er32(CTRL);
701 702 703 704
		/* clear phy power management bit if we are in gig only mode,
		 * which if enabled will attempt negotiation to 100Mb, which
		 * can cause a loss of link at power off or driver unload */
		ctrl &= ~E1000_CTRL_SWDPIN3;
J
Joe Perches 已提交
705
		ew32(CTRL, ctrl);
706 707
	}

L
Linus Torvalds 已提交
708
	/* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
J
Joe Perches 已提交
709
	ew32(VET, ETHERNET_IEEE_VLAN_TYPE);
L
Linus Torvalds 已提交
710

J
Joe Perches 已提交
711 712
	e1000_reset_adaptive(hw);
	e1000_phy_get_info(hw, &adapter->phy_info);
A
Auke Kok 已提交
713

714
	e1000_release_manageability(adapter);
L
Linus Torvalds 已提交
715 716
}

717 718 719
/**
 *  Dump the eeprom for users having checksum issues
 **/
720
static void e1000_dump_eeprom(struct e1000_adapter *adapter)
721 722 723 724 725 726 727 728 729 730 731 732
{
	struct net_device *netdev = adapter->netdev;
	struct ethtool_eeprom eeprom;
	const struct ethtool_ops *ops = netdev->ethtool_ops;
	u8 *data;
	int i;
	u16 csum_old, csum_new = 0;

	eeprom.len = ops->get_eeprom_len(netdev);
	eeprom.offset = 0;

	data = kmalloc(eeprom.len, GFP_KERNEL);
733
	if (!data)
734 735 736 737 738 739 740 741 742 743
		return;

	ops->get_eeprom(netdev, &eeprom, data);

	csum_old = (data[EEPROM_CHECKSUM_REG * 2]) +
		   (data[EEPROM_CHECKSUM_REG * 2 + 1] << 8);
	for (i = 0; i < EEPROM_CHECKSUM_REG * 2; i += 2)
		csum_new += data[i] + (data[i + 1] << 8);
	csum_new = EEPROM_SUM - csum_new;

744 745 746
	pr_err("/*********************/\n");
	pr_err("Current EEPROM Checksum : 0x%04x\n", csum_old);
	pr_err("Calculated              : 0x%04x\n", csum_new);
747

748 749
	pr_err("Offset    Values\n");
	pr_err("========  ======\n");
750 751
	print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, data, 128, 0);

752 753 754 755 756 757 758 759 760 761 762
	pr_err("Include this output when contacting your support provider.\n");
	pr_err("This is not a software error! Something bad happened to\n");
	pr_err("your hardware or EEPROM image. Ignoring this problem could\n");
	pr_err("result in further problems, possibly loss of data,\n");
	pr_err("corruption or system hangs!\n");
	pr_err("The MAC Address will be reset to 00:00:00:00:00:00,\n");
	pr_err("which is invalid and requires you to set the proper MAC\n");
	pr_err("address manually before continuing to enable this network\n");
	pr_err("device. Please inspect the EEPROM dump and report the\n");
	pr_err("issue to your hardware vendor or Intel Customer Support.\n");
	pr_err("/*********************/\n");
763 764 765 766

	kfree(data);
}

T
Taku Izumi 已提交
767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802
/**
 * e1000_is_need_ioport - determine if an adapter needs ioport resources or not
 * @pdev: PCI device information struct
 *
 * Return true if an adapter needs ioport resources
 **/
static int e1000_is_need_ioport(struct pci_dev *pdev)
{
	switch (pdev->device) {
	case E1000_DEV_ID_82540EM:
	case E1000_DEV_ID_82540EM_LOM:
	case E1000_DEV_ID_82540EP:
	case E1000_DEV_ID_82540EP_LOM:
	case E1000_DEV_ID_82540EP_LP:
	case E1000_DEV_ID_82541EI:
	case E1000_DEV_ID_82541EI_MOBILE:
	case E1000_DEV_ID_82541ER:
	case E1000_DEV_ID_82541ER_LOM:
	case E1000_DEV_ID_82541GI:
	case E1000_DEV_ID_82541GI_LF:
	case E1000_DEV_ID_82541GI_MOBILE:
	case E1000_DEV_ID_82544EI_COPPER:
	case E1000_DEV_ID_82544EI_FIBER:
	case E1000_DEV_ID_82544GC_COPPER:
	case E1000_DEV_ID_82544GC_LOM:
	case E1000_DEV_ID_82545EM_COPPER:
	case E1000_DEV_ID_82545EM_FIBER:
	case E1000_DEV_ID_82546EB_COPPER:
	case E1000_DEV_ID_82546EB_FIBER:
	case E1000_DEV_ID_82546EB_QUAD_COPPER:
		return true;
	default:
		return false;
	}
}

803 804
static netdev_features_t e1000_fix_features(struct net_device *netdev,
	netdev_features_t features)
J
Jiri Pirko 已提交
805 806 807 808 809 810 811 812 813 814 815 816 817
{
	/*
	 * Since there is no support for separate rx/tx vlan accel
	 * enable/disable make sure tx flag is always in same state as rx.
	 */
	if (features & NETIF_F_HW_VLAN_RX)
		features |= NETIF_F_HW_VLAN_TX;
	else
		features &= ~NETIF_F_HW_VLAN_TX;

	return features;
}

818 819
static int e1000_set_features(struct net_device *netdev,
	netdev_features_t features)
820 821
{
	struct e1000_adapter *adapter = netdev_priv(netdev);
822
	netdev_features_t changed = features ^ netdev->features;
823

J
Jiri Pirko 已提交
824 825 826
	if (changed & NETIF_F_HW_VLAN_RX)
		e1000_vlan_mode(netdev, features);

827 828 829 830 831 832 833 834 835 836 837 838 839
	if (!(changed & NETIF_F_RXCSUM))
		return 0;

	adapter->rx_csum = !!(features & NETIF_F_RXCSUM);

	if (netif_running(netdev))
		e1000_reinit_locked(adapter);
	else
		e1000_reset(adapter);

	return 0;
}

840 841 842
static const struct net_device_ops e1000_netdev_ops = {
	.ndo_open		= e1000_open,
	.ndo_stop		= e1000_close,
843
	.ndo_start_xmit		= e1000_xmit_frame,
844 845 846
	.ndo_get_stats		= e1000_get_stats,
	.ndo_set_rx_mode	= e1000_set_rx_mode,
	.ndo_set_mac_address	= e1000_set_mac,
J
Jiri Pirko 已提交
847
	.ndo_tx_timeout		= e1000_tx_timeout,
848 849 850 851 852 853 854 855
	.ndo_change_mtu		= e1000_change_mtu,
	.ndo_do_ioctl		= e1000_ioctl,
	.ndo_validate_addr	= eth_validate_addr,
	.ndo_vlan_rx_add_vid	= e1000_vlan_rx_add_vid,
	.ndo_vlan_rx_kill_vid	= e1000_vlan_rx_kill_vid,
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller	= e1000_netpoll,
#endif
J
Jiri Pirko 已提交
856 857
	.ndo_fix_features	= e1000_fix_features,
	.ndo_set_features	= e1000_set_features,
858 859
};

860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923
/**
 * e1000_init_hw_struct - initialize members of hw struct
 * @adapter: board private struct
 * @hw: structure used by e1000_hw.c
 *
 * Factors out initialization of the e1000_hw struct to its own function
 * that can be called very early at init (just after struct allocation).
 * Fields are initialized based on PCI device information and
 * OS network device settings (MTU size).
 * Returns negative error codes if MAC type setup fails.
 */
static int e1000_init_hw_struct(struct e1000_adapter *adapter,
				struct e1000_hw *hw)
{
	struct pci_dev *pdev = adapter->pdev;

	/* PCI config space info */
	hw->vendor_id = pdev->vendor;
	hw->device_id = pdev->device;
	hw->subsystem_vendor_id = pdev->subsystem_vendor;
	hw->subsystem_id = pdev->subsystem_device;
	hw->revision_id = pdev->revision;

	pci_read_config_word(pdev, PCI_COMMAND, &hw->pci_cmd_word);

	hw->max_frame_size = adapter->netdev->mtu +
			     ENET_HEADER_SIZE + ETHERNET_FCS_SIZE;
	hw->min_frame_size = MINIMUM_ETHERNET_FRAME_SIZE;

	/* identify the MAC */
	if (e1000_set_mac_type(hw)) {
		e_err(probe, "Unknown MAC Type\n");
		return -EIO;
	}

	switch (hw->mac_type) {
	default:
		break;
	case e1000_82541:
	case e1000_82547:
	case e1000_82541_rev_2:
	case e1000_82547_rev_2:
		hw->phy_init_script = 1;
		break;
	}

	e1000_set_media_type(hw);
	e1000_get_bus_info(hw);

	hw->wait_autoneg_complete = false;
	hw->tbi_compatibility_en = true;
	hw->adaptive_ifs = true;

	/* Copper options */

	if (hw->media_type == e1000_media_type_copper) {
		hw->mdix = AUTO_ALL_MODES;
		hw->disable_polarity_correction = false;
		hw->master_slave = E1000_MASTER_SLAVE;
	}

	return 0;
}

L
Linus Torvalds 已提交
924 925 926 927 928 929 930 931 932 933 934
/**
 * e1000_probe - Device Initialization Routine
 * @pdev: PCI device information struct
 * @ent: entry in e1000_pci_tbl
 *
 * Returns 0 on success, negative on failure
 *
 * e1000_probe initializes an adapter identified by a pci_dev structure.
 * The OS initialization, configuring of the adapter private structure,
 * and a hardware reset occur.
 **/
J
Joe Perches 已提交
935 936
static int __devinit e1000_probe(struct pci_dev *pdev,
				 const struct pci_device_id *ent)
L
Linus Torvalds 已提交
937 938 939
{
	struct net_device *netdev;
	struct e1000_adapter *adapter;
J
Joe Perches 已提交
940
	struct e1000_hw *hw;
941

L
Linus Torvalds 已提交
942
	static int cards_found = 0;
943
	static int global_quad_port_a = 0; /* global ksp3 port a indication */
944
	int i, err, pci_using_dac;
945
	u16 eeprom_data = 0;
946
	u16 tmp = 0;
947
	u16 eeprom_apme_mask = E1000_EEPROM_APME;
T
Taku Izumi 已提交
948
	int bars, need_ioport;
949

T
Taku Izumi 已提交
950 951 952 953 954 955 956
	/* do not allocate ioport bars when not needed */
	need_ioport = e1000_is_need_ioport(pdev);
	if (need_ioport) {
		bars = pci_select_bars(pdev, IORESOURCE_MEM | IORESOURCE_IO);
		err = pci_enable_device(pdev);
	} else {
		bars = pci_select_bars(pdev, IORESOURCE_MEM);
957
		err = pci_enable_device_mem(pdev);
T
Taku Izumi 已提交
958
	}
959
	if (err)
L
Linus Torvalds 已提交
960 961
		return err;

T
Taku Izumi 已提交
962
	err = pci_request_selected_regions(pdev, bars, e1000_driver_name);
963
	if (err)
964
		goto err_pci_reg;
L
Linus Torvalds 已提交
965 966

	pci_set_master(pdev);
967 968 969
	err = pci_save_state(pdev);
	if (err)
		goto err_alloc_etherdev;
L
Linus Torvalds 已提交
970

971
	err = -ENOMEM;
L
Linus Torvalds 已提交
972
	netdev = alloc_etherdev(sizeof(struct e1000_adapter));
973
	if (!netdev)
L
Linus Torvalds 已提交
974 975 976 977 978
		goto err_alloc_etherdev;

	SET_NETDEV_DEV(netdev, &pdev->dev);

	pci_set_drvdata(pdev, netdev);
979
	adapter = netdev_priv(netdev);
L
Linus Torvalds 已提交
980 981 982
	adapter->netdev = netdev;
	adapter->pdev = pdev;
	adapter->msg_enable = (1 << debug) - 1;
T
Taku Izumi 已提交
983 984
	adapter->bars = bars;
	adapter->need_ioport = need_ioport;
L
Linus Torvalds 已提交
985

J
Joe Perches 已提交
986 987 988
	hw = &adapter->hw;
	hw->back = adapter;

989
	err = -EIO;
990
	hw->hw_addr = pci_ioremap_bar(pdev, BAR_0);
J
Joe Perches 已提交
991
	if (!hw->hw_addr)
L
Linus Torvalds 已提交
992 993
		goto err_ioremap;

T
Taku Izumi 已提交
994 995 996 997 998 999 1000 1001
	if (adapter->need_ioport) {
		for (i = BAR_1; i <= BAR_5; i++) {
			if (pci_resource_len(pdev, i) == 0)
				continue;
			if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
				hw->io_base = pci_resource_start(pdev, i);
				break;
			}
L
Linus Torvalds 已提交
1002 1003 1004
		}
	}

1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
	/* make ready for any if (hw->...) below */
	err = e1000_init_hw_struct(adapter, hw);
	if (err)
		goto err_sw_init;

	/*
	 * there is a workaround being applied below that limits
	 * 64-bit DMA addresses to 64-bit hardware.  There are some
	 * 32-bit adapters that Tx hang when given 64-bit DMA addresses
	 */
	pci_using_dac = 0;
	if ((hw->bus_type == e1000_bus_type_pcix) &&
	    !dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) {
		/*
		 * according to DMA-API-HOWTO, coherent calls will always
		 * succeed if the set call did
		 */
		dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
		pci_using_dac = 1;
	} else {
1025 1026 1027 1028 1029 1030
		err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
		if (err) {
			pr_err("No usable DMA config, aborting\n");
			goto err_dma;
		}
		dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1031 1032
	}

1033
	netdev->netdev_ops = &e1000_netdev_ops;
L
Linus Torvalds 已提交
1034 1035
	e1000_set_ethtool_ops(netdev);
	netdev->watchdog_timeo = 5 * HZ;
1036
	netif_napi_add(netdev, &adapter->napi, e1000_clean, 64);
1037

1038
	strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
L
Linus Torvalds 已提交
1039 1040 1041 1042 1043

	adapter->bd_number = cards_found;

	/* setup the private structure */

1044 1045
	err = e1000_sw_init(adapter);
	if (err)
L
Linus Torvalds 已提交
1046 1047
		goto err_sw_init;

1048
	err = -EIO;
1049
	if (hw->mac_type == e1000_ce4100) {
1050 1051
		hw->ce4100_gbe_mdio_base_virt =
					ioremap(pci_resource_start(pdev, BAR_1),
1052 1053
		                                pci_resource_len(pdev, BAR_1));

1054
		if (!hw->ce4100_gbe_mdio_base_virt)
1055 1056
			goto err_mdio_ioremap;
	}
1057

J
Joe Perches 已提交
1058
	if (hw->mac_type >= e1000_82543) {
1059
		netdev->hw_features = NETIF_F_SG |
J
Jiri Pirko 已提交
1060 1061
				   NETIF_F_HW_CSUM |
				   NETIF_F_HW_VLAN_RX;
1062
		netdev->features = NETIF_F_HW_VLAN_TX |
L
Linus Torvalds 已提交
1063 1064 1065
				   NETIF_F_HW_VLAN_FILTER;
	}

J
Joe Perches 已提交
1066 1067
	if ((hw->mac_type >= e1000_82544) &&
	   (hw->mac_type != e1000_82547))
1068 1069 1070 1071
		netdev->hw_features |= NETIF_F_TSO;

	netdev->features |= netdev->hw_features;
	netdev->hw_features |= NETIF_F_RXCSUM;
1072

1073
	if (pci_using_dac) {
L
Linus Torvalds 已提交
1074
		netdev->features |= NETIF_F_HIGHDMA;
1075 1076
		netdev->vlan_features |= NETIF_F_HIGHDMA;
	}
L
Linus Torvalds 已提交
1077

1078 1079 1080 1081
	netdev->vlan_features |= NETIF_F_TSO;
	netdev->vlan_features |= NETIF_F_HW_CSUM;
	netdev->vlan_features |= NETIF_F_SG;

1082 1083
	netdev->priv_flags |= IFF_UNICAST_FLT;

J
Joe Perches 已提交
1084
	adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
1085

1086
	/* initialize eeprom parameters */
J
Joe Perches 已提交
1087
	if (e1000_init_eeprom_params(hw)) {
1088
		e_err(probe, "EEPROM initialization failed\n");
1089
		goto err_eeprom;
1090 1091
	}

J
Jesse Brandeburg 已提交
1092
	/* before reading the EEPROM, reset the controller to
L
Linus Torvalds 已提交
1093
	 * put the device in a known good starting state */
J
Jesse Brandeburg 已提交
1094

J
Joe Perches 已提交
1095
	e1000_reset_hw(hw);
L
Linus Torvalds 已提交
1096 1097

	/* make sure the EEPROM is good */
J
Joe Perches 已提交
1098
	if (e1000_validate_eeprom_checksum(hw) < 0) {
1099
		e_err(probe, "The EEPROM Checksum Is Not Valid\n");
1100 1101 1102 1103 1104 1105 1106 1107 1108
		e1000_dump_eeprom(adapter);
		/*
		 * set MAC address to all zeroes to invalidate and temporary
		 * disable this device for the user. This blocks regular
		 * traffic while still permitting ethtool ioctls from reaching
		 * the hardware as well as allowing the user to run the
		 * interface after manually setting a hw addr using
		 * `ip set address`
		 */
J
Joe Perches 已提交
1109
		memset(hw->mac_addr, 0, netdev->addr_len);
1110 1111
	} else {
		/* copy the MAC address out of the EEPROM */
J
Joe Perches 已提交
1112
		if (e1000_read_mac_addr(hw))
1113
			e_err(probe, "EEPROM Read Error\n");
L
Linus Torvalds 已提交
1114
	}
1115
	/* don't block initalization here due to bad MAC address */
J
Joe Perches 已提交
1116 1117
	memcpy(netdev->dev_addr, hw->mac_addr, netdev->addr_len);
	memcpy(netdev->perm_addr, hw->mac_addr, netdev->addr_len);
L
Linus Torvalds 已提交
1118

1119
	if (!is_valid_ether_addr(netdev->perm_addr))
1120
		e_err(probe, "Invalid MAC Address\n");
L
Linus Torvalds 已提交
1121 1122


1123 1124 1125 1126
	INIT_DELAYED_WORK(&adapter->watchdog_task, e1000_watchdog);
	INIT_DELAYED_WORK(&adapter->fifo_stall_task,
			  e1000_82547_tx_fifo_stall_task);
	INIT_DELAYED_WORK(&adapter->phy_info_task, e1000_update_phy_info_task);
1127
	INIT_WORK(&adapter->reset_task, e1000_reset_task);
L
Linus Torvalds 已提交
1128 1129 1130 1131 1132 1133 1134 1135

	e1000_check_options(adapter);

	/* Initial Wake on LAN setting
	 * If APM wake is enabled in the EEPROM,
	 * enable the ACPI Magic Packet filter
	 */

J
Joe Perches 已提交
1136
	switch (hw->mac_type) {
L
Linus Torvalds 已提交
1137 1138 1139 1140 1141
	case e1000_82542_rev2_0:
	case e1000_82542_rev2_1:
	case e1000_82543:
		break;
	case e1000_82544:
J
Joe Perches 已提交
1142
		e1000_read_eeprom(hw,
L
Linus Torvalds 已提交
1143 1144 1145 1146 1147
			EEPROM_INIT_CONTROL2_REG, 1, &eeprom_data);
		eeprom_apme_mask = E1000_EEPROM_82544_APM;
		break;
	case e1000_82546:
	case e1000_82546_rev_3:
J
Joe Perches 已提交
1148 1149
		if (er32(STATUS) & E1000_STATUS_FUNC_1){
			e1000_read_eeprom(hw,
L
Linus Torvalds 已提交
1150 1151 1152 1153 1154
				EEPROM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
			break;
		}
		/* Fall Through */
	default:
J
Joe Perches 已提交
1155
		e1000_read_eeprom(hw,
L
Linus Torvalds 已提交
1156 1157 1158
			EEPROM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
		break;
	}
J
Jesse Brandeburg 已提交
1159
	if (eeprom_data & eeprom_apme_mask)
1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172
		adapter->eeprom_wol |= E1000_WUFC_MAG;

	/* now that we have the eeprom settings, apply the special cases
	 * where the eeprom may be wrong or the board simply won't support
	 * wake on lan on a particular port */
	switch (pdev->device) {
	case E1000_DEV_ID_82546GB_PCIE:
		adapter->eeprom_wol = 0;
		break;
	case E1000_DEV_ID_82546EB_FIBER:
	case E1000_DEV_ID_82546GB_FIBER:
		/* Wake events only supported on port A for dual fiber
		 * regardless of eeprom setting */
J
Joe Perches 已提交
1173
		if (er32(STATUS) & E1000_STATUS_FUNC_1)
1174 1175 1176 1177 1178 1179 1180
			adapter->eeprom_wol = 0;
		break;
	case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
		/* if quad port adapter, disable WoL on all but port A */
		if (global_quad_port_a != 0)
			adapter->eeprom_wol = 0;
		else
1181
			adapter->quad_port_a = true;
1182 1183 1184 1185 1186 1187 1188 1189
		/* Reset for multiple quad port adapters */
		if (++global_quad_port_a == 4)
			global_quad_port_a = 0;
		break;
	}

	/* initialize the wol settings based on the eeprom settings */
	adapter->wol = adapter->eeprom_wol;
1190
	device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
L
Linus Torvalds 已提交
1191

1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205
	/* Auto detect PHY address */
	if (hw->mac_type == e1000_ce4100) {
		for (i = 0; i < 32; i++) {
			hw->phy_addr = i;
			e1000_read_phy_reg(hw, PHY_ID2, &tmp);
			if (tmp == 0 || tmp == 0xFF) {
				if (i == 31)
					goto err_eeprom;
				continue;
			} else
				break;
		}
	}

1206 1207 1208 1209 1210 1211 1212 1213
	/* reset the hardware with the new settings */
	e1000_reset(adapter);

	strcpy(netdev->name, "eth%d");
	err = register_netdev(netdev);
	if (err)
		goto err_register;

J
Jiri Pirko 已提交
1214 1215
	e1000_vlan_mode(netdev, netdev->features);

1216
	/* print bus type/speed/width info */
1217
	e_info(probe, "(PCI%s:%dMHz:%d-bit) %pM\n",
J
Joe Perches 已提交
1218 1219 1220 1221 1222 1223 1224
	       ((hw->bus_type == e1000_bus_type_pcix) ? "-X" : ""),
	       ((hw->bus_speed == e1000_bus_speed_133) ? 133 :
		(hw->bus_speed == e1000_bus_speed_120) ? 120 :
		(hw->bus_speed == e1000_bus_speed_100) ? 100 :
		(hw->bus_speed == e1000_bus_speed_66) ? 66 : 33),
	       ((hw->bus_width == e1000_bus_width_64) ? 64 : 32),
	       netdev->dev_addr);
A
Auke Kok 已提交
1225

1226 1227 1228
	/* carrier off reporting is important to ethtool even BEFORE open */
	netif_carrier_off(netdev);

1229
	e_info(probe, "Intel(R) PRO/1000 Network Connection\n");
L
Linus Torvalds 已提交
1230 1231 1232 1233 1234

	cards_found++;
	return 0;

err_register:
1235
err_eeprom:
1236
	e1000_phy_hw_reset(hw);
1237

J
Joe Perches 已提交
1238 1239
	if (hw->flash_address)
		iounmap(hw->flash_address);
1240 1241
	kfree(adapter->tx_ring);
	kfree(adapter->rx_ring);
1242
err_dma:
L
Linus Torvalds 已提交
1243
err_sw_init:
1244
err_mdio_ioremap:
1245
	iounmap(hw->ce4100_gbe_mdio_base_virt);
J
Joe Perches 已提交
1246
	iounmap(hw->hw_addr);
L
Linus Torvalds 已提交
1247 1248 1249
err_ioremap:
	free_netdev(netdev);
err_alloc_etherdev:
T
Taku Izumi 已提交
1250
	pci_release_selected_regions(pdev, bars);
1251 1252
err_pci_reg:
	pci_disable_device(pdev);
L
Linus Torvalds 已提交
1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265
	return err;
}

/**
 * e1000_remove - Device Removal Routine
 * @pdev: PCI device information struct
 *
 * e1000_remove is called by the PCI subsystem to alert the driver
 * that it should release a PCI device.  The could be caused by a
 * Hot-Plug event, or because the driver is going to be removed from
 * memory.
 **/

1266
static void __devexit e1000_remove(struct pci_dev *pdev)
L
Linus Torvalds 已提交
1267 1268
{
	struct net_device *netdev = pci_get_drvdata(pdev);
1269
	struct e1000_adapter *adapter = netdev_priv(netdev);
J
Joe Perches 已提交
1270
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
1271

1272
	e1000_down_and_stop(adapter);
1273
	e1000_release_manageability(adapter);
L
Linus Torvalds 已提交
1274

1275 1276
	unregister_netdev(netdev);

1277
	e1000_phy_hw_reset(hw);
L
Linus Torvalds 已提交
1278

1279 1280 1281
	kfree(adapter->tx_ring);
	kfree(adapter->rx_ring);

1282
	if (hw->mac_type == e1000_ce4100)
1283
		iounmap(hw->ce4100_gbe_mdio_base_virt);
J
Joe Perches 已提交
1284 1285 1286
	iounmap(hw->hw_addr);
	if (hw->flash_address)
		iounmap(hw->flash_address);
T
Taku Izumi 已提交
1287
	pci_release_selected_regions(pdev, adapter->bars);
L
Linus Torvalds 已提交
1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298

	free_netdev(netdev);

	pci_disable_device(pdev);
}

/**
 * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
 * @adapter: board private structure to initialize
 *
 * e1000_sw_init initializes the Adapter private data structure.
1299
 * e1000_init_hw_struct MUST be called before this function
L
Linus Torvalds 已提交
1300 1301
 **/

1302
static int __devinit e1000_sw_init(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
1303
{
1304
	adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
L
Linus Torvalds 已提交
1305

1306 1307
	adapter->num_tx_queues = 1;
	adapter->num_rx_queues = 1;
1308 1309

	if (e1000_alloc_queues(adapter)) {
1310
		e_err(probe, "Unable to allocate memory for queues\n");
1311 1312 1313
		return -ENOMEM;
	}

1314 1315 1316
	/* Explicitly disable IRQ since the NIC can be in any state. */
	e1000_irq_disable(adapter);

L
Linus Torvalds 已提交
1317
	spin_lock_init(&adapter->stats_lock);
1318
	mutex_init(&adapter->mutex);
L
Linus Torvalds 已提交
1319

A
Auke Kok 已提交
1320 1321
	set_bit(__E1000_DOWN, &adapter->flags);

L
Linus Torvalds 已提交
1322 1323 1324
	return 0;
}

1325 1326 1327 1328 1329
/**
 * e1000_alloc_queues - Allocate memory for all rings
 * @adapter: board private structure to initialize
 *
 * We allocate one ring per queue at run-time since we don't know the
1330
 * number of queues at compile-time.
1331 1332
 **/

1333
static int __devinit e1000_alloc_queues(struct e1000_adapter *adapter)
1334
{
Y
Yan Burman 已提交
1335 1336
	adapter->tx_ring = kcalloc(adapter->num_tx_queues,
	                           sizeof(struct e1000_tx_ring), GFP_KERNEL);
1337 1338 1339
	if (!adapter->tx_ring)
		return -ENOMEM;

Y
Yan Burman 已提交
1340 1341
	adapter->rx_ring = kcalloc(adapter->num_rx_queues,
	                           sizeof(struct e1000_rx_ring), GFP_KERNEL);
1342 1343 1344 1345 1346 1347 1348 1349
	if (!adapter->rx_ring) {
		kfree(adapter->tx_ring);
		return -ENOMEM;
	}

	return E1000_SUCCESS;
}

L
Linus Torvalds 已提交
1350 1351 1352 1353 1354 1355 1356 1357 1358
/**
 * e1000_open - Called when a network interface is made active
 * @netdev: network interface device structure
 *
 * Returns 0 on success, negative value on failure
 *
 * The open entry point is called when a network interface is made
 * active by the system (IFF_UP).  At this point all resources needed
 * for transmit and receive operations are allocated, the interrupt
1359
 * handler is registered with the OS, the watchdog task is started,
L
Linus Torvalds 已提交
1360 1361 1362
 * and the stack is notified that the interface is ready.
 **/

1363
static int e1000_open(struct net_device *netdev)
L
Linus Torvalds 已提交
1364
{
1365
	struct e1000_adapter *adapter = netdev_priv(netdev);
J
Joe Perches 已提交
1366
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
1367 1368
	int err;

1369
	/* disallow open during test */
A
Auke Kok 已提交
1370
	if (test_bit(__E1000_TESTING, &adapter->flags))
1371 1372
		return -EBUSY;

1373 1374
	netif_carrier_off(netdev);

L
Linus Torvalds 已提交
1375
	/* allocate transmit descriptors */
1376 1377
	err = e1000_setup_all_tx_resources(adapter);
	if (err)
L
Linus Torvalds 已提交
1378 1379 1380
		goto err_setup_tx;

	/* allocate receive descriptors */
1381
	err = e1000_setup_all_rx_resources(adapter);
1382
	if (err)
1383
		goto err_setup_rx;
1384

1385 1386
	e1000_power_up_phy(adapter);

1387
	adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
J
Joe Perches 已提交
1388
	if ((hw->mng_cookie.status &
1389 1390 1391
			  E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT)) {
		e1000_update_mng_vlan(adapter);
	}
L
Linus Torvalds 已提交
1392

1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405
	/* before we allocate an interrupt, we must be ready to handle it.
	 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
	 * as soon as we call pci_request_irq, so we have to setup our
	 * clean_rx handler before we do so.  */
	e1000_configure(adapter);

	err = e1000_request_irq(adapter);
	if (err)
		goto err_req_irq;

	/* From here on the code is the same as e1000_up() */
	clear_bit(__E1000_DOWN, &adapter->flags);

1406
	napi_enable(&adapter->napi);
1407

1408 1409
	e1000_irq_enable(adapter);

B
Ben Hutchings 已提交
1410 1411
	netif_start_queue(netdev);

1412
	/* fire a link status change interrupt to start the watchdog */
J
Joe Perches 已提交
1413
	ew32(ICS, E1000_ICS_LSC);
1414

L
Linus Torvalds 已提交
1415 1416
	return E1000_SUCCESS;

1417
err_req_irq:
1418
	e1000_power_down_phy(adapter);
1419
	e1000_free_all_rx_resources(adapter);
L
Linus Torvalds 已提交
1420
err_setup_rx:
1421
	e1000_free_all_tx_resources(adapter);
L
Linus Torvalds 已提交
1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439
err_setup_tx:
	e1000_reset(adapter);

	return err;
}

/**
 * e1000_close - Disables a network interface
 * @netdev: network interface device structure
 *
 * Returns 0, this is not allowed to fail
 *
 * The close entry point is called when an interface is de-activated
 * by the OS.  The hardware is still under the drivers control, but
 * needs to be disabled.  A global MAC reset is issued to stop the
 * hardware, and all transmit and receive resources are freed.
 **/

1440
static int e1000_close(struct net_device *netdev)
L
Linus Torvalds 已提交
1441
{
1442
	struct e1000_adapter *adapter = netdev_priv(netdev);
J
Joe Perches 已提交
1443
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
1444

1445
	WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
L
Linus Torvalds 已提交
1446
	e1000_down(adapter);
1447
	e1000_power_down_phy(adapter);
1448
	e1000_free_irq(adapter);
L
Linus Torvalds 已提交
1449

1450 1451
	e1000_free_all_tx_resources(adapter);
	e1000_free_all_rx_resources(adapter);
L
Linus Torvalds 已提交
1452

1453 1454
	/* kill manageability vlan ID if supported, but not if a vlan with
	 * the same ID is registered on the host OS (let 8021q kill it) */
J
Joe Perches 已提交
1455
	if ((hw->mng_cookie.status &
1456
			  E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
J
Jiri Pirko 已提交
1457
	     !test_bit(adapter->mng_vlan_id, adapter->active_vlans)) {
1458 1459
		e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
	}
1460

L
Linus Torvalds 已提交
1461 1462 1463 1464 1465 1466
	return 0;
}

/**
 * e1000_check_64k_bound - check that memory doesn't cross 64kB boundary
 * @adapter: address of board private structure
1467 1468
 * @start: address of beginning of memory
 * @len: length of memory
L
Linus Torvalds 已提交
1469
 **/
1470 1471
static bool e1000_check_64k_bound(struct e1000_adapter *adapter, void *start,
				  unsigned long len)
L
Linus Torvalds 已提交
1472
{
J
Joe Perches 已提交
1473
	struct e1000_hw *hw = &adapter->hw;
1474
	unsigned long begin = (unsigned long)start;
L
Linus Torvalds 已提交
1475 1476
	unsigned long end = begin + len;

1477 1478
	/* First rev 82545 and 82546 need to not allow any memory
	 * write location to cross 64k boundary due to errata 23 */
J
Joe Perches 已提交
1479
	if (hw->mac_type == e1000_82545 ||
1480
	    hw->mac_type == e1000_ce4100 ||
J
Joe Perches 已提交
1481
	    hw->mac_type == e1000_82546) {
J
Joe Perches 已提交
1482
		return ((begin ^ (end - 1)) >> 16) != 0 ? false : true;
L
Linus Torvalds 已提交
1483 1484
	}

J
Joe Perches 已提交
1485
	return true;
L
Linus Torvalds 已提交
1486 1487 1488 1489 1490
}

/**
 * e1000_setup_tx_resources - allocate Tx resources (Descriptors)
 * @adapter: board private structure
1491
 * @txdr:    tx descriptor ring (for a specific queue) to setup
L
Linus Torvalds 已提交
1492 1493 1494 1495
 *
 * Return 0 on success, negative on failure
 **/

1496 1497
static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
				    struct e1000_tx_ring *txdr)
L
Linus Torvalds 已提交
1498 1499 1500 1501 1502
{
	struct pci_dev *pdev = adapter->pdev;
	int size;

	size = sizeof(struct e1000_buffer) * txdr->count;
E
Eric Dumazet 已提交
1503
	txdr->buffer_info = vzalloc(size);
J
Jesse Brandeburg 已提交
1504
	if (!txdr->buffer_info) {
1505 1506
		e_err(probe, "Unable to allocate memory for the Tx descriptor "
		      "ring\n");
L
Linus Torvalds 已提交
1507 1508 1509 1510 1511 1512
		return -ENOMEM;
	}

	/* round up to nearest 4K */

	txdr->size = txdr->count * sizeof(struct e1000_tx_desc);
1513
	txdr->size = ALIGN(txdr->size, 4096);
L
Linus Torvalds 已提交
1514

1515 1516
	txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size, &txdr->dma,
					GFP_KERNEL);
J
Jesse Brandeburg 已提交
1517
	if (!txdr->desc) {
L
Linus Torvalds 已提交
1518 1519
setup_tx_desc_die:
		vfree(txdr->buffer_info);
1520 1521
		e_err(probe, "Unable to allocate memory for the Tx descriptor "
		      "ring\n");
L
Linus Torvalds 已提交
1522 1523 1524
		return -ENOMEM;
	}

1525
	/* Fix for errata 23, can't cross 64kB boundary */
L
Linus Torvalds 已提交
1526 1527 1528
	if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
		void *olddesc = txdr->desc;
		dma_addr_t olddma = txdr->dma;
1529
		e_err(tx_err, "txdr align check failed: %u bytes at %p\n",
1530
		      txdr->size, txdr->desc);
1531
		/* Try again, without freeing the previous */
1532 1533
		txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size,
						&txdr->dma, GFP_KERNEL);
1534
		/* Failed allocation, critical failure */
J
Jesse Brandeburg 已提交
1535
		if (!txdr->desc) {
1536 1537
			dma_free_coherent(&pdev->dev, txdr->size, olddesc,
					  olddma);
L
Linus Torvalds 已提交
1538 1539 1540 1541 1542
			goto setup_tx_desc_die;
		}

		if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
			/* give up */
1543 1544 1545 1546
			dma_free_coherent(&pdev->dev, txdr->size, txdr->desc,
					  txdr->dma);
			dma_free_coherent(&pdev->dev, txdr->size, olddesc,
					  olddma);
1547
			e_err(probe, "Unable to allocate aligned memory "
1548
			      "for the transmit descriptor ring\n");
L
Linus Torvalds 已提交
1549 1550 1551
			vfree(txdr->buffer_info);
			return -ENOMEM;
		} else {
1552
			/* Free old allocation, new allocation was successful */
1553 1554
			dma_free_coherent(&pdev->dev, txdr->size, olddesc,
					  olddma);
L
Linus Torvalds 已提交
1555 1556 1557 1558 1559 1560 1561 1562 1563 1564
		}
	}
	memset(txdr->desc, 0, txdr->size);

	txdr->next_to_use = 0;
	txdr->next_to_clean = 0;

	return 0;
}

1565 1566 1567 1568 1569 1570 1571 1572
/**
 * e1000_setup_all_tx_resources - wrapper to allocate Tx resources
 * 				  (Descriptors) for all queues
 * @adapter: board private structure
 *
 * Return 0 on success, negative on failure
 **/

1573
int e1000_setup_all_tx_resources(struct e1000_adapter *adapter)
1574 1575 1576
{
	int i, err = 0;

1577
	for (i = 0; i < adapter->num_tx_queues; i++) {
1578 1579
		err = e1000_setup_tx_resources(adapter, &adapter->tx_ring[i]);
		if (err) {
1580
			e_err(probe, "Allocation for Tx Queue %u failed\n", i);
1581 1582 1583
			for (i-- ; i >= 0; i--)
				e1000_free_tx_resources(adapter,
							&adapter->tx_ring[i]);
1584 1585 1586 1587 1588 1589 1590
			break;
		}
	}

	return err;
}

L
Linus Torvalds 已提交
1591 1592 1593 1594 1595 1596 1597
/**
 * e1000_configure_tx - Configure 8254x Transmit Unit after Reset
 * @adapter: board private structure
 *
 * Configure the Tx unit of the MAC after a reset.
 **/

1598
static void e1000_configure_tx(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
1599
{
1600
	u64 tdba;
1601
	struct e1000_hw *hw = &adapter->hw;
1602
	u32 tdlen, tctl, tipg;
1603
	u32 ipgr1, ipgr2;
L
Linus Torvalds 已提交
1604 1605 1606

	/* Setup the HW Tx Head and Tail descriptor pointers */

1607
	switch (adapter->num_tx_queues) {
1608 1609
	case 1:
	default:
1610 1611 1612
		tdba = adapter->tx_ring[0].dma;
		tdlen = adapter->tx_ring[0].count *
			sizeof(struct e1000_tx_desc);
J
Joe Perches 已提交
1613 1614 1615 1616 1617
		ew32(TDLEN, tdlen);
		ew32(TDBAH, (tdba >> 32));
		ew32(TDBAL, (tdba & 0x00000000ffffffffULL));
		ew32(TDT, 0);
		ew32(TDH, 0);
1618 1619
		adapter->tx_ring[0].tdh = ((hw->mac_type >= e1000_82543) ? E1000_TDH : E1000_82542_TDH);
		adapter->tx_ring[0].tdt = ((hw->mac_type >= e1000_82543) ? E1000_TDT : E1000_82542_TDT);
1620 1621
		break;
	}
L
Linus Torvalds 已提交
1622 1623

	/* Set the default values for the Tx Inter Packet Gap timer */
1624
	if ((hw->media_type == e1000_media_type_fiber ||
1625
	     hw->media_type == e1000_media_type_internal_serdes))
1626 1627 1628 1629
		tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
	else
		tipg = DEFAULT_82543_TIPG_IPGT_COPPER;

1630
	switch (hw->mac_type) {
L
Linus Torvalds 已提交
1631 1632 1633
	case e1000_82542_rev2_0:
	case e1000_82542_rev2_1:
		tipg = DEFAULT_82542_TIPG_IPGT;
1634 1635
		ipgr1 = DEFAULT_82542_TIPG_IPGR1;
		ipgr2 = DEFAULT_82542_TIPG_IPGR2;
L
Linus Torvalds 已提交
1636 1637
		break;
	default:
1638 1639 1640
		ipgr1 = DEFAULT_82543_TIPG_IPGR1;
		ipgr2 = DEFAULT_82543_TIPG_IPGR2;
		break;
L
Linus Torvalds 已提交
1641
	}
1642 1643
	tipg |= ipgr1 << E1000_TIPG_IPGR1_SHIFT;
	tipg |= ipgr2 << E1000_TIPG_IPGR2_SHIFT;
J
Joe Perches 已提交
1644
	ew32(TIPG, tipg);
L
Linus Torvalds 已提交
1645 1646 1647

	/* Set the Tx Interrupt Delay register */

J
Joe Perches 已提交
1648
	ew32(TIDV, adapter->tx_int_delay);
1649
	if (hw->mac_type >= e1000_82540)
J
Joe Perches 已提交
1650
		ew32(TADV, adapter->tx_abs_int_delay);
L
Linus Torvalds 已提交
1651 1652 1653

	/* Program the Transmit Control Register */

J
Joe Perches 已提交
1654
	tctl = er32(TCTL);
L
Linus Torvalds 已提交
1655
	tctl &= ~E1000_TCTL_CT;
1656
	tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
L
Linus Torvalds 已提交
1657 1658
		(E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);

1659
	e1000_config_collision_dist(hw);
L
Linus Torvalds 已提交
1660 1661

	/* Setup Transmit Descriptor Settings for eop descriptor */
1662 1663 1664 1665 1666
	adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;

	/* only set IDE if we are delaying interrupts using the timers */
	if (adapter->tx_int_delay)
		adapter->txd_cmd |= E1000_TXD_CMD_IDE;
L
Linus Torvalds 已提交
1667

1668
	if (hw->mac_type < e1000_82543)
L
Linus Torvalds 已提交
1669 1670 1671 1672 1673 1674
		adapter->txd_cmd |= E1000_TXD_CMD_RPS;
	else
		adapter->txd_cmd |= E1000_TXD_CMD_RS;

	/* Cache if we're 82544 running in PCI-X because we'll
	 * need this to apply a workaround later in the send path. */
1675 1676
	if (hw->mac_type == e1000_82544 &&
	    hw->bus_type == e1000_bus_type_pcix)
1677
		adapter->pcix_82544 = true;
1678

J
Joe Perches 已提交
1679
	ew32(TCTL, tctl);
1680

L
Linus Torvalds 已提交
1681 1682 1683 1684 1685
}

/**
 * e1000_setup_rx_resources - allocate Rx resources (Descriptors)
 * @adapter: board private structure
1686
 * @rxdr:    rx descriptor ring (for a specific queue) to setup
L
Linus Torvalds 已提交
1687 1688 1689 1690
 *
 * Returns 0 on success, negative on failure
 **/

1691 1692
static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
				    struct e1000_rx_ring *rxdr)
L
Linus Torvalds 已提交
1693 1694
{
	struct pci_dev *pdev = adapter->pdev;
1695
	int size, desc_len;
L
Linus Torvalds 已提交
1696 1697

	size = sizeof(struct e1000_buffer) * rxdr->count;
E
Eric Dumazet 已提交
1698
	rxdr->buffer_info = vzalloc(size);
1699
	if (!rxdr->buffer_info) {
1700 1701
		e_err(probe, "Unable to allocate memory for the Rx descriptor "
		      "ring\n");
L
Linus Torvalds 已提交
1702 1703 1704
		return -ENOMEM;
	}

1705
	desc_len = sizeof(struct e1000_rx_desc);
1706

L
Linus Torvalds 已提交
1707 1708
	/* Round up to nearest 4K */

1709
	rxdr->size = rxdr->count * desc_len;
1710
	rxdr->size = ALIGN(rxdr->size, 4096);
L
Linus Torvalds 已提交
1711

1712 1713
	rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size, &rxdr->dma,
					GFP_KERNEL);
L
Linus Torvalds 已提交
1714

1715
	if (!rxdr->desc) {
1716 1717
		e_err(probe, "Unable to allocate memory for the Rx descriptor "
		      "ring\n");
L
Linus Torvalds 已提交
1718 1719 1720 1721 1722
setup_rx_desc_die:
		vfree(rxdr->buffer_info);
		return -ENOMEM;
	}

1723
	/* Fix for errata 23, can't cross 64kB boundary */
L
Linus Torvalds 已提交
1724 1725 1726
	if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
		void *olddesc = rxdr->desc;
		dma_addr_t olddma = rxdr->dma;
1727
		e_err(rx_err, "rxdr align check failed: %u bytes at %p\n",
1728
		      rxdr->size, rxdr->desc);
1729
		/* Try again, without freeing the previous */
1730 1731
		rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size,
						&rxdr->dma, GFP_KERNEL);
1732
		/* Failed allocation, critical failure */
1733
		if (!rxdr->desc) {
1734 1735
			dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
					  olddma);
1736 1737
			e_err(probe, "Unable to allocate memory for the Rx "
			      "descriptor ring\n");
L
Linus Torvalds 已提交
1738 1739 1740 1741 1742
			goto setup_rx_desc_die;
		}

		if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
			/* give up */
1743 1744 1745 1746
			dma_free_coherent(&pdev->dev, rxdr->size, rxdr->desc,
					  rxdr->dma);
			dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
					  olddma);
1747 1748
			e_err(probe, "Unable to allocate aligned memory for "
			      "the Rx descriptor ring\n");
1749
			goto setup_rx_desc_die;
L
Linus Torvalds 已提交
1750
		} else {
1751
			/* Free old allocation, new allocation was successful */
1752 1753
			dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
					  olddma);
L
Linus Torvalds 已提交
1754 1755 1756 1757 1758 1759
		}
	}
	memset(rxdr->desc, 0, rxdr->size);

	rxdr->next_to_clean = 0;
	rxdr->next_to_use = 0;
1760
	rxdr->rx_skb_top = NULL;
L
Linus Torvalds 已提交
1761 1762 1763 1764

	return 0;
}

1765 1766 1767 1768 1769 1770 1771 1772
/**
 * e1000_setup_all_rx_resources - wrapper to allocate Rx resources
 * 				  (Descriptors) for all queues
 * @adapter: board private structure
 *
 * Return 0 on success, negative on failure
 **/

1773
int e1000_setup_all_rx_resources(struct e1000_adapter *adapter)
1774 1775 1776
{
	int i, err = 0;

1777
	for (i = 0; i < adapter->num_rx_queues; i++) {
1778 1779
		err = e1000_setup_rx_resources(adapter, &adapter->rx_ring[i]);
		if (err) {
1780
			e_err(probe, "Allocation for Rx Queue %u failed\n", i);
1781 1782 1783
			for (i-- ; i >= 0; i--)
				e1000_free_rx_resources(adapter,
							&adapter->rx_ring[i]);
1784 1785 1786 1787 1788 1789 1790
			break;
		}
	}

	return err;
}

L
Linus Torvalds 已提交
1791
/**
1792
 * e1000_setup_rctl - configure the receive control registers
L
Linus Torvalds 已提交
1793 1794
 * @adapter: Board private structure
 **/
1795
static void e1000_setup_rctl(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
1796
{
J
Joe Perches 已提交
1797
	struct e1000_hw *hw = &adapter->hw;
1798
	u32 rctl;
L
Linus Torvalds 已提交
1799

J
Joe Perches 已提交
1800
	rctl = er32(RCTL);
L
Linus Torvalds 已提交
1801 1802 1803

	rctl &= ~(3 << E1000_RCTL_MO_SHIFT);

1804 1805
	rctl |= E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
		E1000_RCTL_RDMTS_HALF |
J
Joe Perches 已提交
1806
		(hw->mc_filter_type << E1000_RCTL_MO_SHIFT);
L
Linus Torvalds 已提交
1807

J
Joe Perches 已提交
1808
	if (hw->tbi_compatibility_on == 1)
L
Linus Torvalds 已提交
1809 1810 1811 1812
		rctl |= E1000_RCTL_SBP;
	else
		rctl &= ~E1000_RCTL_SBP;

1813 1814 1815 1816 1817
	if (adapter->netdev->mtu <= ETH_DATA_LEN)
		rctl &= ~E1000_RCTL_LPE;
	else
		rctl |= E1000_RCTL_LPE;

L
Linus Torvalds 已提交
1818
	/* Setup buffer sizes */
1819 1820 1821
	rctl &= ~E1000_RCTL_SZ_4096;
	rctl |= E1000_RCTL_BSEX;
	switch (adapter->rx_buffer_len) {
1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835
		case E1000_RXBUFFER_2048:
		default:
			rctl |= E1000_RCTL_SZ_2048;
			rctl &= ~E1000_RCTL_BSEX;
			break;
		case E1000_RXBUFFER_4096:
			rctl |= E1000_RCTL_SZ_4096;
			break;
		case E1000_RXBUFFER_8192:
			rctl |= E1000_RCTL_SZ_8192;
			break;
		case E1000_RXBUFFER_16384:
			rctl |= E1000_RCTL_SZ_16384;
			break;
1836 1837
	}

J
Joe Perches 已提交
1838
	ew32(RCTL, rctl);
L
Linus Torvalds 已提交
1839 1840 1841 1842 1843 1844 1845 1846 1847
}

/**
 * e1000_configure_rx - Configure 8254x Receive Unit after Reset
 * @adapter: board private structure
 *
 * Configure the Rx unit of the MAC after a reset.
 **/

1848
static void e1000_configure_rx(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
1849
{
1850
	u64 rdba;
1851
	struct e1000_hw *hw = &adapter->hw;
1852
	u32 rdlen, rctl, rxcsum;
1853

1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864
	if (adapter->netdev->mtu > ETH_DATA_LEN) {
		rdlen = adapter->rx_ring[0].count *
		        sizeof(struct e1000_rx_desc);
		adapter->clean_rx = e1000_clean_jumbo_rx_irq;
		adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
	} else {
		rdlen = adapter->rx_ring[0].count *
		        sizeof(struct e1000_rx_desc);
		adapter->clean_rx = e1000_clean_rx_irq;
		adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
	}
L
Linus Torvalds 已提交
1865 1866

	/* disable receives while setting up the descriptors */
J
Joe Perches 已提交
1867 1868
	rctl = er32(RCTL);
	ew32(RCTL, rctl & ~E1000_RCTL_EN);
L
Linus Torvalds 已提交
1869 1870

	/* set the Receive Delay Timer Register */
J
Joe Perches 已提交
1871
	ew32(RDTR, adapter->rx_int_delay);
L
Linus Torvalds 已提交
1872

1873
	if (hw->mac_type >= e1000_82540) {
J
Joe Perches 已提交
1874
		ew32(RADV, adapter->rx_abs_int_delay);
J
Jesse Brandeburg 已提交
1875
		if (adapter->itr_setting != 0)
J
Joe Perches 已提交
1876
			ew32(ITR, 1000000000 / (adapter->itr * 256));
L
Linus Torvalds 已提交
1877 1878
	}

1879 1880
	/* Setup the HW Rx Head and Tail Descriptor Pointers and
	 * the Base and Length of the Rx Descriptor Ring */
1881
	switch (adapter->num_rx_queues) {
1882 1883
	case 1:
	default:
1884
		rdba = adapter->rx_ring[0].dma;
J
Joe Perches 已提交
1885 1886 1887 1888 1889
		ew32(RDLEN, rdlen);
		ew32(RDBAH, (rdba >> 32));
		ew32(RDBAL, (rdba & 0x00000000ffffffffULL));
		ew32(RDT, 0);
		ew32(RDH, 0);
1890 1891
		adapter->rx_ring[0].rdh = ((hw->mac_type >= e1000_82543) ? E1000_RDH : E1000_82542_RDH);
		adapter->rx_ring[0].rdt = ((hw->mac_type >= e1000_82543) ? E1000_RDT : E1000_82542_RDT);
1892
		break;
1893 1894
	}

L
Linus Torvalds 已提交
1895
	/* Enable 82543 Receive Checksum Offload for TCP and UDP */
1896
	if (hw->mac_type >= e1000_82543) {
J
Joe Perches 已提交
1897
		rxcsum = er32(RXCSUM);
1898
		if (adapter->rx_csum)
1899
			rxcsum |= E1000_RXCSUM_TUOFL;
1900
		else
1901
			/* don't need to clear IPPCSE as it defaults to 0 */
1902
			rxcsum &= ~E1000_RXCSUM_TUOFL;
J
Joe Perches 已提交
1903
		ew32(RXCSUM, rxcsum);
L
Linus Torvalds 已提交
1904 1905 1906
	}

	/* Enable Receives */
1907
	ew32(RCTL, rctl | E1000_RCTL_EN);
L
Linus Torvalds 已提交
1908 1909 1910
}

/**
1911
 * e1000_free_tx_resources - Free Tx Resources per Queue
L
Linus Torvalds 已提交
1912
 * @adapter: board private structure
1913
 * @tx_ring: Tx descriptor ring for a specific queue
L
Linus Torvalds 已提交
1914 1915 1916 1917
 *
 * Free all transmit software resources
 **/

1918 1919
static void e1000_free_tx_resources(struct e1000_adapter *adapter,
				    struct e1000_tx_ring *tx_ring)
L
Linus Torvalds 已提交
1920 1921 1922
{
	struct pci_dev *pdev = adapter->pdev;

1923
	e1000_clean_tx_ring(adapter, tx_ring);
L
Linus Torvalds 已提交
1924

1925 1926
	vfree(tx_ring->buffer_info);
	tx_ring->buffer_info = NULL;
L
Linus Torvalds 已提交
1927

1928 1929
	dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
			  tx_ring->dma);
L
Linus Torvalds 已提交
1930

1931 1932 1933 1934 1935 1936 1937 1938 1939 1940
	tx_ring->desc = NULL;
}

/**
 * e1000_free_all_tx_resources - Free Tx Resources for All Queues
 * @adapter: board private structure
 *
 * Free all transmit software resources
 **/

1941
void e1000_free_all_tx_resources(struct e1000_adapter *adapter)
1942 1943 1944
{
	int i;

1945
	for (i = 0; i < adapter->num_tx_queues; i++)
1946
		e1000_free_tx_resources(adapter, &adapter->tx_ring[i]);
L
Linus Torvalds 已提交
1947 1948
}

1949 1950
static void e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter,
					     struct e1000_buffer *buffer_info)
L
Linus Torvalds 已提交
1951
{
1952 1953
	if (buffer_info->dma) {
		if (buffer_info->mapped_as_page)
1954 1955
			dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
				       buffer_info->length, DMA_TO_DEVICE);
1956
		else
1957
			dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1958
					 buffer_info->length,
1959
					 DMA_TO_DEVICE);
1960 1961
		buffer_info->dma = 0;
	}
1962
	if (buffer_info->skb) {
L
Linus Torvalds 已提交
1963
		dev_kfree_skb_any(buffer_info->skb);
1964 1965
		buffer_info->skb = NULL;
	}
1966
	buffer_info->time_stamp = 0;
1967
	/* buffer_info must be completely set up in the transmit path */
L
Linus Torvalds 已提交
1968 1969 1970 1971 1972
}

/**
 * e1000_clean_tx_ring - Free Tx Buffers
 * @adapter: board private structure
1973
 * @tx_ring: ring to be cleaned
L
Linus Torvalds 已提交
1974 1975
 **/

1976 1977
static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
				struct e1000_tx_ring *tx_ring)
L
Linus Torvalds 已提交
1978
{
J
Joe Perches 已提交
1979
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
1980 1981 1982 1983 1984 1985
	struct e1000_buffer *buffer_info;
	unsigned long size;
	unsigned int i;

	/* Free all the Tx ring sk_buffs */

J
Jesse Brandeburg 已提交
1986
	for (i = 0; i < tx_ring->count; i++) {
L
Linus Torvalds 已提交
1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999
		buffer_info = &tx_ring->buffer_info[i];
		e1000_unmap_and_free_tx_resource(adapter, buffer_info);
	}

	size = sizeof(struct e1000_buffer) * tx_ring->count;
	memset(tx_ring->buffer_info, 0, size);

	/* Zero out the descriptor ring */

	memset(tx_ring->desc, 0, tx_ring->size);

	tx_ring->next_to_use = 0;
	tx_ring->next_to_clean = 0;
2000
	tx_ring->last_tx_tso = false;
L
Linus Torvalds 已提交
2001

J
Joe Perches 已提交
2002 2003
	writel(0, hw->hw_addr + tx_ring->tdh);
	writel(0, hw->hw_addr + tx_ring->tdt);
2004 2005 2006 2007 2008 2009 2010
}

/**
 * e1000_clean_all_tx_rings - Free Tx Buffers for all queues
 * @adapter: board private structure
 **/

2011
static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter)
2012 2013 2014
{
	int i;

2015
	for (i = 0; i < adapter->num_tx_queues; i++)
2016
		e1000_clean_tx_ring(adapter, &adapter->tx_ring[i]);
L
Linus Torvalds 已提交
2017 2018 2019 2020 2021
}

/**
 * e1000_free_rx_resources - Free Rx Resources
 * @adapter: board private structure
2022
 * @rx_ring: ring to clean the resources from
L
Linus Torvalds 已提交
2023 2024 2025 2026
 *
 * Free all receive software resources
 **/

2027 2028
static void e1000_free_rx_resources(struct e1000_adapter *adapter,
				    struct e1000_rx_ring *rx_ring)
L
Linus Torvalds 已提交
2029 2030 2031
{
	struct pci_dev *pdev = adapter->pdev;

2032
	e1000_clean_rx_ring(adapter, rx_ring);
L
Linus Torvalds 已提交
2033 2034 2035 2036

	vfree(rx_ring->buffer_info);
	rx_ring->buffer_info = NULL;

2037 2038
	dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
			  rx_ring->dma);
L
Linus Torvalds 已提交
2039 2040 2041 2042 2043

	rx_ring->desc = NULL;
}

/**
2044
 * e1000_free_all_rx_resources - Free Rx Resources for All Queues
L
Linus Torvalds 已提交
2045
 * @adapter: board private structure
2046 2047 2048 2049
 *
 * Free all receive software resources
 **/

2050
void e1000_free_all_rx_resources(struct e1000_adapter *adapter)
2051 2052 2053
{
	int i;

2054
	for (i = 0; i < adapter->num_rx_queues; i++)
2055 2056 2057 2058 2059 2060 2061
		e1000_free_rx_resources(adapter, &adapter->rx_ring[i]);
}

/**
 * e1000_clean_rx_ring - Free Rx Buffers per Queue
 * @adapter: board private structure
 * @rx_ring: ring to free buffers from
L
Linus Torvalds 已提交
2062 2063
 **/

2064 2065
static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
				struct e1000_rx_ring *rx_ring)
L
Linus Torvalds 已提交
2066
{
J
Joe Perches 已提交
2067
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
2068 2069 2070
	struct e1000_buffer *buffer_info;
	struct pci_dev *pdev = adapter->pdev;
	unsigned long size;
2071
	unsigned int i;
L
Linus Torvalds 已提交
2072 2073

	/* Free all the Rx ring sk_buffs */
J
Jesse Brandeburg 已提交
2074
	for (i = 0; i < rx_ring->count; i++) {
L
Linus Torvalds 已提交
2075
		buffer_info = &rx_ring->buffer_info[i];
2076 2077
		if (buffer_info->dma &&
		    adapter->clean_rx == e1000_clean_rx_irq) {
2078
			dma_unmap_single(&pdev->dev, buffer_info->dma,
2079
			                 buffer_info->length,
2080
					 DMA_FROM_DEVICE);
2081 2082
		} else if (buffer_info->dma &&
		           adapter->clean_rx == e1000_clean_jumbo_rx_irq) {
2083 2084 2085
			dma_unmap_page(&pdev->dev, buffer_info->dma,
				       buffer_info->length,
				       DMA_FROM_DEVICE);
J
Jesse Brandeburg 已提交
2086
		}
L
Linus Torvalds 已提交
2087

J
Jesse Brandeburg 已提交
2088
		buffer_info->dma = 0;
2089 2090 2091 2092
		if (buffer_info->page) {
			put_page(buffer_info->page);
			buffer_info->page = NULL;
		}
J
Jesse Brandeburg 已提交
2093
		if (buffer_info->skb) {
L
Linus Torvalds 已提交
2094 2095
			dev_kfree_skb(buffer_info->skb);
			buffer_info->skb = NULL;
2096
		}
L
Linus Torvalds 已提交
2097 2098
	}

2099 2100 2101 2102 2103 2104
	/* there also may be some cached data from a chained receive */
	if (rx_ring->rx_skb_top) {
		dev_kfree_skb(rx_ring->rx_skb_top);
		rx_ring->rx_skb_top = NULL;
	}

L
Linus Torvalds 已提交
2105 2106 2107 2108 2109 2110 2111 2112 2113
	size = sizeof(struct e1000_buffer) * rx_ring->count;
	memset(rx_ring->buffer_info, 0, size);

	/* Zero out the descriptor ring */
	memset(rx_ring->desc, 0, rx_ring->size);

	rx_ring->next_to_clean = 0;
	rx_ring->next_to_use = 0;

J
Joe Perches 已提交
2114 2115
	writel(0, hw->hw_addr + rx_ring->rdh);
	writel(0, hw->hw_addr + rx_ring->rdt);
2116 2117 2118 2119 2120 2121 2122
}

/**
 * e1000_clean_all_rx_rings - Free Rx Buffers for all queues
 * @adapter: board private structure
 **/

2123
static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter)
2124 2125 2126
{
	int i;

2127
	for (i = 0; i < adapter->num_rx_queues; i++)
2128
		e1000_clean_rx_ring(adapter, &adapter->rx_ring[i]);
L
Linus Torvalds 已提交
2129 2130 2131 2132 2133
}

/* The 82542 2.0 (revision 2) needs to have the receive unit in reset
 * and memory write and invalidate disabled for certain operations
 */
2134
static void e1000_enter_82542_rst(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
2135
{
J
Joe Perches 已提交
2136
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
2137
	struct net_device *netdev = adapter->netdev;
2138
	u32 rctl;
L
Linus Torvalds 已提交
2139

J
Joe Perches 已提交
2140
	e1000_pci_clear_mwi(hw);
L
Linus Torvalds 已提交
2141

J
Joe Perches 已提交
2142
	rctl = er32(RCTL);
L
Linus Torvalds 已提交
2143
	rctl |= E1000_RCTL_RST;
J
Joe Perches 已提交
2144 2145
	ew32(RCTL, rctl);
	E1000_WRITE_FLUSH();
L
Linus Torvalds 已提交
2146 2147
	mdelay(5);

J
Jesse Brandeburg 已提交
2148
	if (netif_running(netdev))
2149
		e1000_clean_all_rx_rings(adapter);
L
Linus Torvalds 已提交
2150 2151
}

2152
static void e1000_leave_82542_rst(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
2153
{
J
Joe Perches 已提交
2154
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
2155
	struct net_device *netdev = adapter->netdev;
2156
	u32 rctl;
L
Linus Torvalds 已提交
2157

J
Joe Perches 已提交
2158
	rctl = er32(RCTL);
L
Linus Torvalds 已提交
2159
	rctl &= ~E1000_RCTL_RST;
J
Joe Perches 已提交
2160 2161
	ew32(RCTL, rctl);
	E1000_WRITE_FLUSH();
L
Linus Torvalds 已提交
2162 2163
	mdelay(5);

J
Joe Perches 已提交
2164 2165
	if (hw->pci_cmd_word & PCI_COMMAND_INVALIDATE)
		e1000_pci_set_mwi(hw);
L
Linus Torvalds 已提交
2166

J
Jesse Brandeburg 已提交
2167
	if (netif_running(netdev)) {
2168 2169
		/* No need to loop, because 82542 supports only 1 queue */
		struct e1000_rx_ring *ring = &adapter->rx_ring[0];
2170
		e1000_configure_rx(adapter);
2171
		adapter->alloc_rx_buf(adapter, ring, E1000_DESC_UNUSED(ring));
L
Linus Torvalds 已提交
2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182
	}
}

/**
 * e1000_set_mac - Change the Ethernet Address of the NIC
 * @netdev: network interface device structure
 * @p: pointer to an address structure
 *
 * Returns 0 on success, negative on failure
 **/

2183
static int e1000_set_mac(struct net_device *netdev, void *p)
L
Linus Torvalds 已提交
2184
{
2185
	struct e1000_adapter *adapter = netdev_priv(netdev);
J
Joe Perches 已提交
2186
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
2187 2188
	struct sockaddr *addr = p;

J
Jesse Brandeburg 已提交
2189
	if (!is_valid_ether_addr(addr->sa_data))
L
Linus Torvalds 已提交
2190 2191 2192 2193
		return -EADDRNOTAVAIL;

	/* 82542 2.0 needs to be in reset to write receive address registers */

J
Joe Perches 已提交
2194
	if (hw->mac_type == e1000_82542_rev2_0)
L
Linus Torvalds 已提交
2195 2196 2197
		e1000_enter_82542_rst(adapter);

	memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
J
Joe Perches 已提交
2198
	memcpy(hw->mac_addr, addr->sa_data, netdev->addr_len);
L
Linus Torvalds 已提交
2199

J
Joe Perches 已提交
2200
	e1000_rar_set(hw, hw->mac_addr, 0);
L
Linus Torvalds 已提交
2201

J
Joe Perches 已提交
2202
	if (hw->mac_type == e1000_82542_rev2_0)
L
Linus Torvalds 已提交
2203 2204 2205 2206 2207 2208
		e1000_leave_82542_rst(adapter);

	return 0;
}

/**
2209
 * e1000_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
L
Linus Torvalds 已提交
2210 2211
 * @netdev: network interface device structure
 *
2212 2213 2214
 * The set_rx_mode entry point is called whenever the unicast or multicast
 * address lists or the network interface flags are updated. This routine is
 * responsible for configuring the hardware for proper unicast, multicast,
L
Linus Torvalds 已提交
2215 2216 2217
 * promiscuous mode, and all-multi behavior.
 **/

2218
static void e1000_set_rx_mode(struct net_device *netdev)
L
Linus Torvalds 已提交
2219
{
2220
	struct e1000_adapter *adapter = netdev_priv(netdev);
L
Linus Torvalds 已提交
2221
	struct e1000_hw *hw = &adapter->hw;
J
Jiri Pirko 已提交
2222 2223
	struct netdev_hw_addr *ha;
	bool use_uc = false;
2224 2225
	u32 rctl;
	u32 hash_value;
2226
	int i, rar_entries = E1000_RAR_ENTRIES;
2227
	int mta_reg_count = E1000_NUM_MTA_REGISTERS;
2228 2229 2230
	u32 *mcarray = kcalloc(mta_reg_count, sizeof(u32), GFP_ATOMIC);

	if (!mcarray) {
2231
		e_err(probe, "memory allocation failed\n");
2232 2233
		return;
	}
2234

2235 2236
	/* Check for Promiscuous and All Multicast modes */

J
Joe Perches 已提交
2237
	rctl = er32(RCTL);
L
Linus Torvalds 已提交
2238

J
Jesse Brandeburg 已提交
2239
	if (netdev->flags & IFF_PROMISC) {
L
Linus Torvalds 已提交
2240
		rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2241
		rctl &= ~E1000_RCTL_VFE;
L
Linus Torvalds 已提交
2242
	} else {
2243
		if (netdev->flags & IFF_ALLMULTI)
2244
			rctl |= E1000_RCTL_MPE;
2245
		else
2246
			rctl &= ~E1000_RCTL_MPE;
2247
		/* Enable VLAN filter if there is a VLAN */
J
Jiri Pirko 已提交
2248
		if (e1000_vlan_used(adapter))
2249
			rctl |= E1000_RCTL_VFE;
2250 2251
	}

2252
	if (netdev_uc_count(netdev) > rar_entries - 1) {
2253 2254 2255
		rctl |= E1000_RCTL_UPE;
	} else if (!(netdev->flags & IFF_PROMISC)) {
		rctl &= ~E1000_RCTL_UPE;
J
Jiri Pirko 已提交
2256
		use_uc = true;
L
Linus Torvalds 已提交
2257 2258
	}

J
Joe Perches 已提交
2259
	ew32(RCTL, rctl);
L
Linus Torvalds 已提交
2260 2261 2262

	/* 82542 2.0 needs to be in reset to write receive address registers */

J
Jesse Brandeburg 已提交
2263
	if (hw->mac_type == e1000_82542_rev2_0)
L
Linus Torvalds 已提交
2264 2265
		e1000_enter_82542_rst(adapter);

2266 2267 2268 2269
	/* load the first 14 addresses into the exact filters 1-14. Unicast
	 * addresses take precedence to avoid disabling unicast filtering
	 * when possible.
	 *
2270
	 * RAR 0 is used for the station MAC address
L
Linus Torvalds 已提交
2271 2272
	 * if there are not 14 addresses, go ahead and clear the filters
	 */
J
Jiri Pirko 已提交
2273 2274
	i = 1;
	if (use_uc)
2275
		netdev_for_each_uc_addr(ha, netdev) {
J
Jiri Pirko 已提交
2276 2277 2278 2279 2280
			if (i == rar_entries)
				break;
			e1000_rar_set(hw, ha->addr, i++);
		}

2281
	netdev_for_each_mc_addr(ha, netdev) {
2282 2283 2284
		if (i == rar_entries) {
			/* load any remaining addresses into the hash table */
			u32 hash_reg, hash_bit, mta;
2285
			hash_value = e1000_hash_mc_addr(hw, ha->addr);
2286 2287 2288 2289
			hash_reg = (hash_value >> 5) & 0x7F;
			hash_bit = hash_value & 0x1F;
			mta = (1 << hash_bit);
			mcarray[hash_reg] |= mta;
2290
		} else {
2291
			e1000_rar_set(hw, ha->addr, i++);
L
Linus Torvalds 已提交
2292 2293 2294
		}
	}

2295 2296 2297 2298 2299
	for (; i < rar_entries; i++) {
		E1000_WRITE_REG_ARRAY(hw, RA, i << 1, 0);
		E1000_WRITE_FLUSH();
		E1000_WRITE_REG_ARRAY(hw, RA, (i << 1) + 1, 0);
		E1000_WRITE_FLUSH();
L
Linus Torvalds 已提交
2300 2301
	}

2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314
	/* write the hash table completely, write from bottom to avoid
	 * both stupid write combining chipsets, and flushing each write */
	for (i = mta_reg_count - 1; i >= 0 ; i--) {
		/*
		 * If we are on an 82544 has an errata where writing odd
		 * offsets overwrites the previous even offset, but writing
		 * backwards over the range solves the issue by always
		 * writing the odd offset first
		 */
		E1000_WRITE_REG_ARRAY(hw, MTA, i, mcarray[i]);
	}
	E1000_WRITE_FLUSH();

J
Jesse Brandeburg 已提交
2315
	if (hw->mac_type == e1000_82542_rev2_0)
L
Linus Torvalds 已提交
2316
		e1000_leave_82542_rst(adapter);
2317 2318

	kfree(mcarray);
L
Linus Torvalds 已提交
2319 2320
}

2321 2322 2323 2324 2325 2326 2327
/**
 * e1000_update_phy_info_task - get phy info
 * @work: work struct contained inside adapter struct
 *
 * Need to wait a few seconds after link up to get diagnostic information from
 * the phy
 */
J
Jesse Brandeburg 已提交
2328 2329 2330
static void e1000_update_phy_info_task(struct work_struct *work)
{
	struct e1000_adapter *adapter = container_of(work,
2331 2332
						     struct e1000_adapter,
						     phy_info_task.work);
2333 2334 2335
	if (test_bit(__E1000_DOWN, &adapter->flags))
		return;
	mutex_lock(&adapter->mutex);
2336
	e1000_phy_get_info(&adapter->hw, &adapter->phy_info);
2337
	mutex_unlock(&adapter->mutex);
L
Linus Torvalds 已提交
2338 2339
}

J
Jesse Brandeburg 已提交
2340 2341 2342 2343 2344 2345 2346
/**
 * e1000_82547_tx_fifo_stall_task - task to complete work
 * @work: work struct contained inside adapter struct
 **/
static void e1000_82547_tx_fifo_stall_task(struct work_struct *work)
{
	struct e1000_adapter *adapter = container_of(work,
2347 2348
						     struct e1000_adapter,
						     fifo_stall_task.work);
J
Joe Perches 已提交
2349
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
2350
	struct net_device *netdev = adapter->netdev;
2351
	u32 tctl;
L
Linus Torvalds 已提交
2352

2353 2354 2355
	if (test_bit(__E1000_DOWN, &adapter->flags))
		return;
	mutex_lock(&adapter->mutex);
J
Jesse Brandeburg 已提交
2356
	if (atomic_read(&adapter->tx_fifo_stall)) {
J
Joe Perches 已提交
2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367
		if ((er32(TDT) == er32(TDH)) &&
		   (er32(TDFT) == er32(TDFH)) &&
		   (er32(TDFTS) == er32(TDFHS))) {
			tctl = er32(TCTL);
			ew32(TCTL, tctl & ~E1000_TCTL_EN);
			ew32(TDFT, adapter->tx_head_addr);
			ew32(TDFH, adapter->tx_head_addr);
			ew32(TDFTS, adapter->tx_head_addr);
			ew32(TDFHS, adapter->tx_head_addr);
			ew32(TCTL, tctl);
			E1000_WRITE_FLUSH();
L
Linus Torvalds 已提交
2368 2369 2370 2371

			adapter->tx_fifo_head = 0;
			atomic_set(&adapter->tx_fifo_stall, 0);
			netif_wake_queue(netdev);
2372
		} else if (!test_bit(__E1000_DOWN, &adapter->flags)) {
2373
			schedule_delayed_work(&adapter->fifo_stall_task, 1);
L
Linus Torvalds 已提交
2374 2375
		}
	}
2376
	mutex_unlock(&adapter->mutex);
L
Linus Torvalds 已提交
2377 2378
}

2379
bool e1000_has_link(struct e1000_adapter *adapter)
2380 2381 2382 2383
{
	struct e1000_hw *hw = &adapter->hw;
	bool link_active = false;

2384 2385 2386 2387 2388
	/* get_link_status is set on LSC (link status) interrupt or rx
	 * sequence error interrupt (except on intel ce4100).
	 * get_link_status will stay false until the
	 * e1000_check_for_link establishes link for copper adapters
	 * ONLY
2389 2390 2391
	 */
	switch (hw->media_type) {
	case e1000_media_type_copper:
2392 2393
		if (hw->mac_type == e1000_ce4100)
			hw->get_link_status = 1;
2394
		if (hw->get_link_status) {
2395
			e1000_check_for_link(hw);
2396 2397 2398 2399 2400 2401
			link_active = !hw->get_link_status;
		} else {
			link_active = true;
		}
		break;
	case e1000_media_type_fiber:
2402
		e1000_check_for_link(hw);
2403 2404 2405
		link_active = !!(er32(STATUS) & E1000_STATUS_LU);
		break;
	case e1000_media_type_internal_serdes:
2406
		e1000_check_for_link(hw);
2407 2408 2409 2410 2411 2412 2413 2414 2415
		link_active = hw->serdes_has_link;
		break;
	default:
		break;
	}

	return link_active;
}

L
Linus Torvalds 已提交
2416
/**
2417 2418
 * e1000_watchdog - work function
 * @work: work struct contained inside adapter struct
L
Linus Torvalds 已提交
2419
 **/
2420
static void e1000_watchdog(struct work_struct *work)
L
Linus Torvalds 已提交
2421
{
2422 2423 2424
	struct e1000_adapter *adapter = container_of(work,
						     struct e1000_adapter,
						     watchdog_task.work);
J
Joe Perches 已提交
2425
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
2426
	struct net_device *netdev = adapter->netdev;
J
Jeff Kirsher 已提交
2427
	struct e1000_tx_ring *txdr = adapter->tx_ring;
2428
	u32 link, tctl;
2429

2430 2431 2432 2433
	if (test_bit(__E1000_DOWN, &adapter->flags))
		return;

	mutex_lock(&adapter->mutex);
2434 2435 2436
	link = e1000_has_link(adapter);
	if ((netif_carrier_ok(netdev)) && link)
		goto link_up;
L
Linus Torvalds 已提交
2437

J
Jesse Brandeburg 已提交
2438 2439
	if (link) {
		if (!netif_carrier_ok(netdev)) {
2440
			u32 ctrl;
J
Joe Perches 已提交
2441
			bool txb2b = true;
2442
			/* update snapshot of PHY registers on LSC */
J
Joe Perches 已提交
2443
			e1000_get_speed_and_duplex(hw,
L
Linus Torvalds 已提交
2444 2445 2446
			                           &adapter->link_speed,
			                           &adapter->link_duplex);

J
Joe Perches 已提交
2447
			ctrl = er32(CTRL);
2448 2449 2450 2451 2452 2453 2454 2455 2456 2457
			pr_info("%s NIC Link is Up %d Mbps %s, "
				"Flow Control: %s\n",
				netdev->name,
				adapter->link_speed,
				adapter->link_duplex == FULL_DUPLEX ?
				"Full Duplex" : "Half Duplex",
				((ctrl & E1000_CTRL_TFCE) && (ctrl &
				E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
				E1000_CTRL_RFCE) ? "RX" : ((ctrl &
				E1000_CTRL_TFCE) ? "TX" : "None")));
L
Linus Torvalds 已提交
2458

2459
			/* adjust timeout factor according to speed/duplex */
2460
			adapter->tx_timeout_factor = 1;
2461 2462
			switch (adapter->link_speed) {
			case SPEED_10:
J
Joe Perches 已提交
2463
				txb2b = false;
2464
				adapter->tx_timeout_factor = 16;
2465 2466
				break;
			case SPEED_100:
J
Joe Perches 已提交
2467
				txb2b = false;
2468 2469 2470 2471
				/* maybe add some timeout factor ? */
				break;
			}

2472
			/* enable transmits in the hardware */
J
Joe Perches 已提交
2473
			tctl = er32(TCTL);
2474
			tctl |= E1000_TCTL_EN;
J
Joe Perches 已提交
2475
			ew32(TCTL, tctl);
2476

L
Linus Torvalds 已提交
2477
			netif_carrier_on(netdev);
2478
			if (!test_bit(__E1000_DOWN, &adapter->flags))
2479 2480
				schedule_delayed_work(&adapter->phy_info_task,
						      2 * HZ);
L
Linus Torvalds 已提交
2481 2482 2483
			adapter->smartspeed = 0;
		}
	} else {
J
Jesse Brandeburg 已提交
2484
		if (netif_carrier_ok(netdev)) {
L
Linus Torvalds 已提交
2485 2486
			adapter->link_speed = 0;
			adapter->link_duplex = 0;
2487 2488
			pr_info("%s NIC Link is Down\n",
				netdev->name);
L
Linus Torvalds 已提交
2489
			netif_carrier_off(netdev);
2490 2491

			if (!test_bit(__E1000_DOWN, &adapter->flags))
2492 2493
				schedule_delayed_work(&adapter->phy_info_task,
						      2 * HZ);
L
Linus Torvalds 已提交
2494 2495 2496 2497 2498
		}

		e1000_smartspeed(adapter);
	}

2499
link_up:
L
Linus Torvalds 已提交
2500 2501
	e1000_update_stats(adapter);

J
Joe Perches 已提交
2502
	hw->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
L
Linus Torvalds 已提交
2503
	adapter->tpt_old = adapter->stats.tpt;
J
Joe Perches 已提交
2504
	hw->collision_delta = adapter->stats.colc - adapter->colc_old;
L
Linus Torvalds 已提交
2505 2506 2507 2508 2509 2510 2511
	adapter->colc_old = adapter->stats.colc;

	adapter->gorcl = adapter->stats.gorcl - adapter->gorcl_old;
	adapter->gorcl_old = adapter->stats.gorcl;
	adapter->gotcl = adapter->stats.gotcl - adapter->gotcl_old;
	adapter->gotcl_old = adapter->stats.gotcl;

J
Joe Perches 已提交
2512
	e1000_update_adaptive(hw);
L
Linus Torvalds 已提交
2513

2514
	if (!netif_carrier_ok(netdev)) {
2515
		if (E1000_DESC_UNUSED(txdr) + 1 < txdr->count) {
L
Linus Torvalds 已提交
2516 2517 2518 2519
			/* We've lost link, so the controller stops DMA,
			 * but we've got queued Tx work that's never going
			 * to get done, so reset controller to flush Tx.
			 * (Do the reset outside of interrupt context). */
2520 2521
			adapter->tx_timeout_count++;
			schedule_work(&adapter->reset_task);
2522 2523
			/* exit immediately since reset is imminent */
			goto unlock;
L
Linus Torvalds 已提交
2524 2525 2526
		}
	}

2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542
	/* Simple mode for Interrupt Throttle Rate (ITR) */
	if (hw->mac_type >= e1000_82540 && adapter->itr_setting == 4) {
		/*
		 * Symmetric Tx/Rx gets a reduced ITR=2000;
		 * Total asymmetrical Tx or Rx gets ITR=8000;
		 * everyone else is between 2000-8000.
		 */
		u32 goc = (adapter->gotcl + adapter->gorcl) / 10000;
		u32 dif = (adapter->gotcl > adapter->gorcl ?
			    adapter->gotcl - adapter->gorcl :
			    adapter->gorcl - adapter->gotcl) / 10000;
		u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;

		ew32(ITR, 1000000000 / (itr * 256));
	}

L
Linus Torvalds 已提交
2543
	/* Cause software interrupt to ensure rx ring is cleaned */
J
Joe Perches 已提交
2544
	ew32(ICS, E1000_ICS_RXDMT0);
L
Linus Torvalds 已提交
2545

2546
	/* Force detection of hung controller every watchdog period */
J
Joe Perches 已提交
2547
	adapter->detect_tx_hung = true;
L
Linus Torvalds 已提交
2548

2549
	/* Reschedule the task */
2550
	if (!test_bit(__E1000_DOWN, &adapter->flags))
2551
		schedule_delayed_work(&adapter->watchdog_task, 2 * HZ);
2552 2553 2554

unlock:
	mutex_unlock(&adapter->mutex);
L
Linus Torvalds 已提交
2555 2556
}

J
Jesse Brandeburg 已提交
2557 2558 2559 2560 2561 2562 2563 2564 2565
enum latency_range {
	lowest_latency = 0,
	low_latency = 1,
	bulk_latency = 2,
	latency_invalid = 255
};

/**
 * e1000_update_itr - update the dynamic ITR value based on statistics
2566 2567 2568 2569 2570
 * @adapter: pointer to adapter
 * @itr_setting: current adapter->itr
 * @packets: the number of packets during this measurement interval
 * @bytes: the number of bytes during this measurement interval
 *
J
Jesse Brandeburg 已提交
2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581
 *      Stores a new ITR value based on packets and byte
 *      counts during the last interrupt.  The advantage of per interrupt
 *      computation is faster updates and more accurate ITR for the current
 *      traffic pattern.  Constants in this function were computed
 *      based on theoretical maximum wire speed and thresholds were set based
 *      on testing data as well as attempting to minimize response time
 *      while increasing bulk throughput.
 *      this functionality is controlled by the InterruptThrottleRate module
 *      parameter (see e1000_param.c)
 **/
static unsigned int e1000_update_itr(struct e1000_adapter *adapter,
2582
				     u16 itr_setting, int packets, int bytes)
J
Jesse Brandeburg 已提交
2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594
{
	unsigned int retval = itr_setting;
	struct e1000_hw *hw = &adapter->hw;

	if (unlikely(hw->mac_type < e1000_82540))
		goto update_itr_done;

	if (packets == 0)
		goto update_itr_done;

	switch (itr_setting) {
	case lowest_latency:
2595 2596 2597 2598
		/* jumbo frames get bulk treatment*/
		if (bytes/packets > 8000)
			retval = bulk_latency;
		else if ((packets < 5) && (bytes > 512))
J
Jesse Brandeburg 已提交
2599 2600 2601 2602
			retval = low_latency;
		break;
	case low_latency:  /* 50 usec aka 20000 ints/s */
		if (bytes > 10000) {
2603 2604 2605 2606
			/* jumbo frames need bulk latency setting */
			if (bytes/packets > 8000)
				retval = bulk_latency;
			else if ((packets < 10) || ((bytes/packets) > 1200))
J
Jesse Brandeburg 已提交
2607 2608 2609
				retval = bulk_latency;
			else if ((packets > 35))
				retval = lowest_latency;
2610 2611 2612
		} else if (bytes/packets > 2000)
			retval = bulk_latency;
		else if (packets <= 2 && bytes < 512)
J
Jesse Brandeburg 已提交
2613 2614 2615 2616 2617 2618
			retval = lowest_latency;
		break;
	case bulk_latency: /* 250 usec aka 4000 ints/s */
		if (bytes > 25000) {
			if (packets > 35)
				retval = low_latency;
2619 2620
		} else if (bytes < 6000) {
			retval = low_latency;
J
Jesse Brandeburg 已提交
2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631
		}
		break;
	}

update_itr_done:
	return retval;
}

static void e1000_set_itr(struct e1000_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
2632 2633
	u16 current_itr;
	u32 new_itr = adapter->itr;
J
Jesse Brandeburg 已提交
2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648

	if (unlikely(hw->mac_type < e1000_82540))
		return;

	/* for non-gigabit speeds, just fix the interrupt rate at 4000 */
	if (unlikely(adapter->link_speed != SPEED_1000)) {
		current_itr = 0;
		new_itr = 4000;
		goto set_itr_now;
	}

	adapter->tx_itr = e1000_update_itr(adapter,
	                            adapter->tx_itr,
	                            adapter->total_tx_packets,
	                            adapter->total_tx_bytes);
2649 2650 2651 2652
	/* conservative mode (itr 3) eliminates the lowest_latency setting */
	if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
		adapter->tx_itr = low_latency;

J
Jesse Brandeburg 已提交
2653 2654 2655 2656
	adapter->rx_itr = e1000_update_itr(adapter,
	                            adapter->rx_itr,
	                            adapter->total_rx_packets,
	                            adapter->total_rx_bytes);
2657 2658 2659
	/* conservative mode (itr 3) eliminates the lowest_latency setting */
	if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
		adapter->rx_itr = low_latency;
J
Jesse Brandeburg 已提交
2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686

	current_itr = max(adapter->rx_itr, adapter->tx_itr);

	switch (current_itr) {
	/* counts and packets in update_itr are dependent on these numbers */
	case lowest_latency:
		new_itr = 70000;
		break;
	case low_latency:
		new_itr = 20000; /* aka hwitr = ~200 */
		break;
	case bulk_latency:
		new_itr = 4000;
		break;
	default:
		break;
	}

set_itr_now:
	if (new_itr != adapter->itr) {
		/* this attempts to bias the interrupt rate towards Bulk
		 * by adding intermediate steps when interrupt rate is
		 * increasing */
		new_itr = new_itr > adapter->itr ?
		             min(adapter->itr + (new_itr >> 2), new_itr) :
		             new_itr;
		adapter->itr = new_itr;
J
Joe Perches 已提交
2687
		ew32(ITR, 1000000000 / (new_itr * 256));
J
Jesse Brandeburg 已提交
2688 2689 2690
	}
}

L
Linus Torvalds 已提交
2691 2692 2693
#define E1000_TX_FLAGS_CSUM		0x00000001
#define E1000_TX_FLAGS_VLAN		0x00000002
#define E1000_TX_FLAGS_TSO		0x00000004
2694
#define E1000_TX_FLAGS_IPV4		0x00000008
L
Linus Torvalds 已提交
2695 2696 2697
#define E1000_TX_FLAGS_VLAN_MASK	0xffff0000
#define E1000_TX_FLAGS_VLAN_SHIFT	16

2698 2699
static int e1000_tso(struct e1000_adapter *adapter,
		     struct e1000_tx_ring *tx_ring, struct sk_buff *skb)
L
Linus Torvalds 已提交
2700 2701
{
	struct e1000_context_desc *context_desc;
J
Jeff Kirsher 已提交
2702
	struct e1000_buffer *buffer_info;
L
Linus Torvalds 已提交
2703
	unsigned int i;
2704 2705 2706
	u32 cmd_length = 0;
	u16 ipcse = 0, tucse, mss;
	u8 ipcss, ipcso, tucss, tucso, hdr_len;
L
Linus Torvalds 已提交
2707 2708
	int err;

H
Herbert Xu 已提交
2709
	if (skb_is_gso(skb)) {
L
Linus Torvalds 已提交
2710 2711 2712 2713 2714 2715
		if (skb_header_cloned(skb)) {
			err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
			if (err)
				return err;
		}

2716
		hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2717
		mss = skb_shinfo(skb)->gso_size;
A
Alexey Dobriyan 已提交
2718
		if (skb->protocol == htons(ETH_P_IP)) {
2719 2720 2721
			struct iphdr *iph = ip_hdr(skb);
			iph->tot_len = 0;
			iph->check = 0;
2722 2723 2724 2725
			tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
								 iph->daddr, 0,
								 IPPROTO_TCP,
								 0);
2726
			cmd_length = E1000_TXD_CMD_IP;
2727
			ipcse = skb_transport_offset(skb) - 1;
2728
		} else if (skb->protocol == htons(ETH_P_IPV6)) {
2729
			ipv6_hdr(skb)->payload_len = 0;
2730
			tcp_hdr(skb)->check =
2731 2732 2733
				~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
						 &ipv6_hdr(skb)->daddr,
						 0, IPPROTO_TCP, 0);
2734 2735
			ipcse = 0;
		}
2736
		ipcss = skb_network_offset(skb);
2737
		ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
2738
		tucss = skb_transport_offset(skb);
2739
		tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
L
Linus Torvalds 已提交
2740 2741 2742
		tucse = 0;

		cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
2743
			       E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
L
Linus Torvalds 已提交
2744

2745 2746
		i = tx_ring->next_to_use;
		context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
J
Jeff Kirsher 已提交
2747
		buffer_info = &tx_ring->buffer_info[i];
L
Linus Torvalds 已提交
2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758

		context_desc->lower_setup.ip_fields.ipcss  = ipcss;
		context_desc->lower_setup.ip_fields.ipcso  = ipcso;
		context_desc->lower_setup.ip_fields.ipcse  = cpu_to_le16(ipcse);
		context_desc->upper_setup.tcp_fields.tucss = tucss;
		context_desc->upper_setup.tcp_fields.tucso = tucso;
		context_desc->upper_setup.tcp_fields.tucse = cpu_to_le16(tucse);
		context_desc->tcp_seg_setup.fields.mss     = cpu_to_le16(mss);
		context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
		context_desc->cmd_and_length = cpu_to_le32(cmd_length);

J
Jeff Kirsher 已提交
2759
		buffer_info->time_stamp = jiffies;
2760
		buffer_info->next_to_watch = i;
J
Jeff Kirsher 已提交
2761

2762 2763
		if (++i == tx_ring->count) i = 0;
		tx_ring->next_to_use = i;
L
Linus Torvalds 已提交
2764

J
Joe Perches 已提交
2765
		return true;
L
Linus Torvalds 已提交
2766
	}
J
Joe Perches 已提交
2767
	return false;
L
Linus Torvalds 已提交
2768 2769
}

2770 2771
static bool e1000_tx_csum(struct e1000_adapter *adapter,
			  struct e1000_tx_ring *tx_ring, struct sk_buff *skb)
L
Linus Torvalds 已提交
2772 2773
{
	struct e1000_context_desc *context_desc;
J
Jeff Kirsher 已提交
2774
	struct e1000_buffer *buffer_info;
L
Linus Torvalds 已提交
2775
	unsigned int i;
2776
	u8 css;
2777
	u32 cmd_len = E1000_TXD_CMD_DEXT;
L
Linus Torvalds 已提交
2778

2779 2780
	if (skb->ip_summed != CHECKSUM_PARTIAL)
		return false;
L
Linus Torvalds 已提交
2781

2782
	switch (skb->protocol) {
2783
	case cpu_to_be16(ETH_P_IP):
2784 2785 2786
		if (ip_hdr(skb)->protocol == IPPROTO_TCP)
			cmd_len |= E1000_TXD_CMD_TCP;
		break;
2787
	case cpu_to_be16(ETH_P_IPV6):
2788 2789 2790 2791 2792 2793
		/* XXX not handling all IPV6 headers */
		if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
			cmd_len |= E1000_TXD_CMD_TCP;
		break;
	default:
		if (unlikely(net_ratelimit()))
2794 2795
			e_warn(drv, "checksum_partial proto=%x!\n",
			       skb->protocol);
2796 2797
		break;
	}
L
Linus Torvalds 已提交
2798

2799
	css = skb_checksum_start_offset(skb);
L
Linus Torvalds 已提交
2800

2801 2802 2803
	i = tx_ring->next_to_use;
	buffer_info = &tx_ring->buffer_info[i];
	context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
J
Jeff Kirsher 已提交
2804

2805 2806 2807 2808 2809 2810 2811
	context_desc->lower_setup.ip_config = 0;
	context_desc->upper_setup.tcp_fields.tucss = css;
	context_desc->upper_setup.tcp_fields.tucso =
		css + skb->csum_offset;
	context_desc->upper_setup.tcp_fields.tucse = 0;
	context_desc->tcp_seg_setup.data = 0;
	context_desc->cmd_and_length = cpu_to_le32(cmd_len);
L
Linus Torvalds 已提交
2812

2813 2814
	buffer_info->time_stamp = jiffies;
	buffer_info->next_to_watch = i;
L
Linus Torvalds 已提交
2815

2816 2817 2818 2819
	if (unlikely(++i == tx_ring->count)) i = 0;
	tx_ring->next_to_use = i;

	return true;
L
Linus Torvalds 已提交
2820 2821 2822 2823 2824
}

#define E1000_MAX_TXD_PWR	12
#define E1000_MAX_DATA_PER_TXD	(1<<E1000_MAX_TXD_PWR)

2825 2826 2827 2828 2829
static int e1000_tx_map(struct e1000_adapter *adapter,
			struct e1000_tx_ring *tx_ring,
			struct sk_buff *skb, unsigned int first,
			unsigned int max_per_txd, unsigned int nr_frags,
			unsigned int mss)
L
Linus Torvalds 已提交
2830
{
J
Joe Perches 已提交
2831
	struct e1000_hw *hw = &adapter->hw;
2832
	struct pci_dev *pdev = adapter->pdev;
2833
	struct e1000_buffer *buffer_info;
J
Jesse Brandeburg 已提交
2834
	unsigned int len = skb_headlen(skb);
2835
	unsigned int offset = 0, size, count = 0, i;
2836
	unsigned int f, bytecount, segs;
L
Linus Torvalds 已提交
2837 2838 2839

	i = tx_ring->next_to_use;

J
Jesse Brandeburg 已提交
2840
	while (len) {
2841
		buffer_info = &tx_ring->buffer_info[i];
L
Linus Torvalds 已提交
2842
		size = min(len, max_per_txd);
J
Jeff Kirsher 已提交
2843 2844 2845
		/* Workaround for Controller erratum --
		 * descriptor for non-tso packet in a linear SKB that follows a
		 * tso gets written back prematurely before the data is fully
2846
		 * DMA'd to the controller */
J
Jeff Kirsher 已提交
2847
		if (!skb->data_len && tx_ring->last_tx_tso &&
H
Herbert Xu 已提交
2848
		    !skb_is_gso(skb)) {
2849
			tx_ring->last_tx_tso = false;
J
Jeff Kirsher 已提交
2850 2851 2852
			size -= 4;
		}

L
Linus Torvalds 已提交
2853 2854
		/* Workaround for premature desc write-backs
		 * in TSO mode.  Append 4-byte sentinel desc */
J
Jesse Brandeburg 已提交
2855
		if (unlikely(mss && !nr_frags && size == len && size > 8))
L
Linus Torvalds 已提交
2856
			size -= 4;
2857 2858 2859 2860 2861
		/* work-around for errata 10 and it applies
		 * to all controllers in PCI-X mode
		 * The fix is to make sure that the first descriptor of a
		 * packet is smaller than 2048 - 16 - 16 (or 2016) bytes
		 */
J
Joe Perches 已提交
2862
		if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
2863 2864
		                (size > 2015) && count == 0))
		        size = 2015;
J
Jesse Brandeburg 已提交
2865

L
Linus Torvalds 已提交
2866 2867
		/* Workaround for potential 82544 hang in PCI-X.  Avoid
		 * terminating buffers within evenly-aligned dwords. */
J
Jesse Brandeburg 已提交
2868
		if (unlikely(adapter->pcix_82544 &&
L
Linus Torvalds 已提交
2869 2870 2871 2872 2873
		   !((unsigned long)(skb->data + offset + size - 1) & 4) &&
		   size > 4))
			size -= 4;

		buffer_info->length = size;
2874
		/* set time_stamp *before* dma to help avoid a possible race */
L
Linus Torvalds 已提交
2875
		buffer_info->time_stamp = jiffies;
2876
		buffer_info->mapped_as_page = false;
2877 2878 2879 2880
		buffer_info->dma = dma_map_single(&pdev->dev,
						  skb->data + offset,
						  size,	DMA_TO_DEVICE);
		if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2881
			goto dma_error;
2882
		buffer_info->next_to_watch = i;
L
Linus Torvalds 已提交
2883 2884 2885 2886

		len -= size;
		offset += size;
		count++;
2887 2888 2889 2890 2891
		if (len) {
			i++;
			if (unlikely(i == tx_ring->count))
				i = 0;
		}
L
Linus Torvalds 已提交
2892 2893
	}

J
Jesse Brandeburg 已提交
2894
	for (f = 0; f < nr_frags; f++) {
E
Eric Dumazet 已提交
2895
		const struct skb_frag_struct *frag;
L
Linus Torvalds 已提交
2896 2897

		frag = &skb_shinfo(skb)->frags[f];
E
Eric Dumazet 已提交
2898
		len = skb_frag_size(frag);
2899
		offset = 0;
L
Linus Torvalds 已提交
2900

J
Jesse Brandeburg 已提交
2901
		while (len) {
2902
			unsigned long bufend;
2903 2904 2905 2906
			i++;
			if (unlikely(i == tx_ring->count))
				i = 0;

L
Linus Torvalds 已提交
2907 2908 2909 2910
			buffer_info = &tx_ring->buffer_info[i];
			size = min(len, max_per_txd);
			/* Workaround for premature desc write-backs
			 * in TSO mode.  Append 4-byte sentinel desc */
J
Jesse Brandeburg 已提交
2911
			if (unlikely(mss && f == (nr_frags-1) && size == len && size > 8))
L
Linus Torvalds 已提交
2912 2913 2914 2915
				size -= 4;
			/* Workaround for potential 82544 hang in PCI-X.
			 * Avoid terminating buffers within evenly-aligned
			 * dwords. */
2916 2917 2918
			bufend = (unsigned long)
				page_to_phys(skb_frag_page(frag));
			bufend += offset + size - 1;
J
Jesse Brandeburg 已提交
2919
			if (unlikely(adapter->pcix_82544 &&
2920 2921
				     !(bufend & 4) &&
				     size > 4))
L
Linus Torvalds 已提交
2922 2923 2924 2925
				size -= 4;

			buffer_info->length = size;
			buffer_info->time_stamp = jiffies;
2926
			buffer_info->mapped_as_page = true;
2927 2928
			buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
						offset, size, DMA_TO_DEVICE);
2929
			if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2930
				goto dma_error;
2931
			buffer_info->next_to_watch = i;
L
Linus Torvalds 已提交
2932 2933 2934 2935 2936 2937 2938

			len -= size;
			offset += size;
			count++;
		}
	}

2939 2940 2941 2942
	segs = skb_shinfo(skb)->gso_segs ?: 1;
	/* multiply data chunks by size of headers */
	bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;

L
Linus Torvalds 已提交
2943
	tx_ring->buffer_info[i].skb = skb;
2944 2945
	tx_ring->buffer_info[i].segs = segs;
	tx_ring->buffer_info[i].bytecount = bytecount;
L
Linus Torvalds 已提交
2946 2947 2948
	tx_ring->buffer_info[first].next_to_watch = i;

	return count;
2949 2950 2951 2952

dma_error:
	dev_err(&pdev->dev, "TX DMA map failed\n");
	buffer_info->dma = 0;
2953
	if (count)
2954
		count--;
2955 2956 2957

	while (count--) {
		if (i==0)
2958
			i += tx_ring->count;
2959
		i--;
2960 2961 2962 2963 2964
		buffer_info = &tx_ring->buffer_info[i];
		e1000_unmap_and_free_tx_resource(adapter, buffer_info);
	}

	return 0;
L
Linus Torvalds 已提交
2965 2966
}

2967 2968 2969
static void e1000_tx_queue(struct e1000_adapter *adapter,
			   struct e1000_tx_ring *tx_ring, int tx_flags,
			   int count)
L
Linus Torvalds 已提交
2970
{
J
Joe Perches 已提交
2971
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
2972 2973
	struct e1000_tx_desc *tx_desc = NULL;
	struct e1000_buffer *buffer_info;
2974
	u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
L
Linus Torvalds 已提交
2975 2976
	unsigned int i;

J
Jesse Brandeburg 已提交
2977
	if (likely(tx_flags & E1000_TX_FLAGS_TSO)) {
L
Linus Torvalds 已提交
2978 2979
		txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
		             E1000_TXD_CMD_TSE;
2980 2981
		txd_upper |= E1000_TXD_POPTS_TXSM << 8;

J
Jesse Brandeburg 已提交
2982
		if (likely(tx_flags & E1000_TX_FLAGS_IPV4))
2983
			txd_upper |= E1000_TXD_POPTS_IXSM << 8;
L
Linus Torvalds 已提交
2984 2985
	}

J
Jesse Brandeburg 已提交
2986
	if (likely(tx_flags & E1000_TX_FLAGS_CSUM)) {
L
Linus Torvalds 已提交
2987 2988 2989 2990
		txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
		txd_upper |= E1000_TXD_POPTS_TXSM << 8;
	}

J
Jesse Brandeburg 已提交
2991
	if (unlikely(tx_flags & E1000_TX_FLAGS_VLAN)) {
L
Linus Torvalds 已提交
2992 2993 2994 2995 2996 2997
		txd_lower |= E1000_TXD_CMD_VLE;
		txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
	}

	i = tx_ring->next_to_use;

J
Jesse Brandeburg 已提交
2998
	while (count--) {
L
Linus Torvalds 已提交
2999 3000 3001 3002 3003 3004
		buffer_info = &tx_ring->buffer_info[i];
		tx_desc = E1000_TX_DESC(*tx_ring, i);
		tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
		tx_desc->lower.data =
			cpu_to_le32(txd_lower | buffer_info->length);
		tx_desc->upper.data = cpu_to_le32(txd_upper);
J
Jesse Brandeburg 已提交
3005
		if (unlikely(++i == tx_ring->count)) i = 0;
L
Linus Torvalds 已提交
3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016
	}

	tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);

	/* Force memory writes to complete before letting h/w
	 * know there are new descriptors to fetch.  (Only
	 * applicable for weak-ordered memory model archs,
	 * such as IA-64). */
	wmb();

	tx_ring->next_to_use = i;
J
Joe Perches 已提交
3017
	writel(i, hw->hw_addr + tx_ring->tdt);
3018 3019 3020
	/* we need this if more than one processor can write to our tail
	 * at a time, it syncronizes IO on IA64/Altix systems */
	mmiowb();
L
Linus Torvalds 已提交
3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034
}

/**
 * 82547 workaround to avoid controller hang in half-duplex environment.
 * The workaround is to avoid queuing a large packet that would span
 * the internal Tx FIFO ring boundary by notifying the stack to resend
 * the packet at a later time.  This gives the Tx FIFO an opportunity to
 * flush all packets.  When that occurs, we reset the Tx FIFO pointers
 * to the beginning of the Tx FIFO.
 **/

#define E1000_FIFO_HDR			0x10
#define E1000_82547_PAD_LEN		0x3E0

3035 3036
static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
				       struct sk_buff *skb)
L
Linus Torvalds 已提交
3037
{
3038 3039
	u32 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
	u32 skb_fifo_len = skb->len + E1000_FIFO_HDR;
L
Linus Torvalds 已提交
3040

3041
	skb_fifo_len = ALIGN(skb_fifo_len, E1000_FIFO_HDR);
L
Linus Torvalds 已提交
3042

J
Jesse Brandeburg 已提交
3043
	if (adapter->link_duplex != HALF_DUPLEX)
L
Linus Torvalds 已提交
3044 3045
		goto no_fifo_stall_required;

J
Jesse Brandeburg 已提交
3046
	if (atomic_read(&adapter->tx_fifo_stall))
L
Linus Torvalds 已提交
3047 3048
		return 1;

J
Jesse Brandeburg 已提交
3049
	if (skb_fifo_len >= (E1000_82547_PAD_LEN + fifo_space)) {
L
Linus Torvalds 已提交
3050 3051 3052 3053 3054 3055
		atomic_set(&adapter->tx_fifo_stall, 1);
		return 1;
	}

no_fifo_stall_required:
	adapter->tx_fifo_head += skb_fifo_len;
J
Jesse Brandeburg 已提交
3056
	if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
L
Linus Torvalds 已提交
3057 3058 3059 3060
		adapter->tx_fifo_head -= adapter->tx_fifo_size;
	return 0;
}

3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078
static int __e1000_maybe_stop_tx(struct net_device *netdev, int size)
{
	struct e1000_adapter *adapter = netdev_priv(netdev);
	struct e1000_tx_ring *tx_ring = adapter->tx_ring;

	netif_stop_queue(netdev);
	/* Herbert's original patch had:
	 *  smp_mb__after_netif_stop_queue();
	 * but since that doesn't exist yet, just open code it. */
	smp_mb();

	/* We need to check again in a case another CPU has just
	 * made room available. */
	if (likely(E1000_DESC_UNUSED(tx_ring) < size))
		return -EBUSY;

	/* A reprieve! */
	netif_start_queue(netdev);
3079
	++adapter->restart_queue;
3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090
	return 0;
}

static int e1000_maybe_stop_tx(struct net_device *netdev,
                               struct e1000_tx_ring *tx_ring, int size)
{
	if (likely(E1000_DESC_UNUSED(tx_ring) >= size))
		return 0;
	return __e1000_maybe_stop_tx(netdev, size);
}

L
Linus Torvalds 已提交
3091
#define TXD_USE_COUNT(S, X) (((S) >> (X)) + 1 )
3092 3093
static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
				    struct net_device *netdev)
L
Linus Torvalds 已提交
3094
{
3095
	struct e1000_adapter *adapter = netdev_priv(netdev);
J
Joe Perches 已提交
3096
	struct e1000_hw *hw = &adapter->hw;
3097
	struct e1000_tx_ring *tx_ring;
L
Linus Torvalds 已提交
3098 3099 3100
	unsigned int first, max_per_txd = E1000_MAX_DATA_PER_TXD;
	unsigned int max_txd_pwr = E1000_MAX_TXD_PWR;
	unsigned int tx_flags = 0;
E
Eric Dumazet 已提交
3101
	unsigned int len = skb_headlen(skb);
3102 3103
	unsigned int nr_frags;
	unsigned int mss;
L
Linus Torvalds 已提交
3104
	int count = 0;
3105
	int tso;
L
Linus Torvalds 已提交
3106 3107
	unsigned int f;

3108 3109 3110 3111
	/* This goes back to the question of how to logically map a tx queue
	 * to a flow.  Right now, performance is impacted slightly negatively
	 * if using multiple tx queues.  If the stack breaks away from a
	 * single qdisc implementation, we can look at this again. */
3112
	tx_ring = adapter->tx_ring;
3113

3114
	if (unlikely(skb->len <= 0)) {
L
Linus Torvalds 已提交
3115 3116 3117 3118
		dev_kfree_skb_any(skb);
		return NETDEV_TX_OK;
	}

3119
	mss = skb_shinfo(skb)->gso_size;
3120
	/* The controller does a simple calculation to
L
Linus Torvalds 已提交
3121 3122 3123 3124 3125
	 * make sure there is enough room in the FIFO before
	 * initiating the DMA for each buffer.  The calc is:
	 * 4 = ceil(buffer len/mss).  To make sure we don't
	 * overrun the FIFO, adjust the max buffer len if mss
	 * drops. */
J
Jesse Brandeburg 已提交
3126
	if (mss) {
3127
		u8 hdr_len;
L
Linus Torvalds 已提交
3128 3129
		max_per_txd = min(mss << 2, max_per_txd);
		max_txd_pwr = fls(max_per_txd) - 1;
J
Jeff Kirsher 已提交
3130

3131
		hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
3132
		if (skb->data_len && hdr_len == len) {
J
Joe Perches 已提交
3133
			switch (hw->mac_type) {
3134
				unsigned int pull_size;
3135 3136 3137 3138 3139 3140 3141
			case e1000_82544:
				/* Make sure we have room to chop off 4 bytes,
				 * and that the end alignment will work out to
				 * this hardware's requirements
				 * NOTE: this is a TSO only workaround
				 * if end byte alignment not correct move us
				 * into the next dword */
3142
				if ((unsigned long)(skb_tail_pointer(skb) - 1) & 4)
3143 3144
					break;
				/* fall through */
3145 3146
				pull_size = min((unsigned int)4, skb->data_len);
				if (!__pskb_pull_tail(skb, pull_size)) {
3147 3148
					e_err(drv, "__pskb_pull_tail "
					      "failed.\n");
3149
					dev_kfree_skb_any(skb);
J
Jeff Garzik 已提交
3150
					return NETDEV_TX_OK;
3151
				}
E
Eric Dumazet 已提交
3152
				len = skb_headlen(skb);
3153 3154 3155 3156
				break;
			default:
				/* do nothing */
				break;
3157
			}
J
Jeff Kirsher 已提交
3158
		}
L
Linus Torvalds 已提交
3159 3160
	}

J
Jeff Kirsher 已提交
3161
	/* reserve a descriptor for the offload context */
3162
	if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
L
Linus Torvalds 已提交
3163
		count++;
3164
	count++;
J
Jeff Kirsher 已提交
3165 3166

	/* Controller Erratum workaround */
H
Herbert Xu 已提交
3167
	if (!skb->data_len && tx_ring->last_tx_tso && !skb_is_gso(skb))
J
Jeff Kirsher 已提交
3168 3169
		count++;

L
Linus Torvalds 已提交
3170 3171
	count += TXD_USE_COUNT(len, max_txd_pwr);

J
Jesse Brandeburg 已提交
3172
	if (adapter->pcix_82544)
L
Linus Torvalds 已提交
3173 3174
		count++;

J
Jesse Brandeburg 已提交
3175
	/* work-around for errata 10 and it applies to all controllers
3176 3177
	 * in PCI-X mode, so add one more descriptor to the count
	 */
J
Joe Perches 已提交
3178
	if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
3179 3180 3181
			(len > 2015)))
		count++;

L
Linus Torvalds 已提交
3182
	nr_frags = skb_shinfo(skb)->nr_frags;
J
Jesse Brandeburg 已提交
3183
	for (f = 0; f < nr_frags; f++)
E
Eric Dumazet 已提交
3184
		count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->frags[f]),
L
Linus Torvalds 已提交
3185
				       max_txd_pwr);
J
Jesse Brandeburg 已提交
3186
	if (adapter->pcix_82544)
L
Linus Torvalds 已提交
3187 3188 3189 3190
		count += nr_frags;

	/* need: count + 2 desc gap to keep tail from touching
	 * head, otherwise try next time */
3191
	if (unlikely(e1000_maybe_stop_tx(netdev, tx_ring, count + 2)))
L
Linus Torvalds 已提交
3192 3193
		return NETDEV_TX_BUSY;

3194 3195 3196 3197 3198 3199
	if (unlikely((hw->mac_type == e1000_82547) &&
		     (e1000_82547_fifo_workaround(adapter, skb)))) {
		netif_stop_queue(netdev);
		if (!test_bit(__E1000_DOWN, &adapter->flags))
			schedule_delayed_work(&adapter->fifo_stall_task, 1);
		return NETDEV_TX_BUSY;
L
Linus Torvalds 已提交
3200 3201
	}

J
Jiri Pirko 已提交
3202
	if (vlan_tx_tag_present(skb)) {
L
Linus Torvalds 已提交
3203 3204 3205 3206
		tx_flags |= E1000_TX_FLAGS_VLAN;
		tx_flags |= (vlan_tx_tag_get(skb) << E1000_TX_FLAGS_VLAN_SHIFT);
	}

3207
	first = tx_ring->next_to_use;
J
Jesse Brandeburg 已提交
3208

3209
	tso = e1000_tso(adapter, tx_ring, skb);
L
Linus Torvalds 已提交
3210 3211 3212 3213 3214
	if (tso < 0) {
		dev_kfree_skb_any(skb);
		return NETDEV_TX_OK;
	}

J
Jeff Kirsher 已提交
3215
	if (likely(tso)) {
3216
		if (likely(hw->mac_type != e1000_82544))
3217
			tx_ring->last_tx_tso = true;
L
Linus Torvalds 已提交
3218
		tx_flags |= E1000_TX_FLAGS_TSO;
J
Jeff Kirsher 已提交
3219
	} else if (likely(e1000_tx_csum(adapter, tx_ring, skb)))
L
Linus Torvalds 已提交
3220 3221
		tx_flags |= E1000_TX_FLAGS_CSUM;

A
Alexey Dobriyan 已提交
3222
	if (likely(skb->protocol == htons(ETH_P_IP)))
3223 3224
		tx_flags |= E1000_TX_FLAGS_IPV4;

3225 3226
	count = e1000_tx_map(adapter, tx_ring, skb, first, max_per_txd,
	                     nr_frags, mss);
L
Linus Torvalds 已提交
3227

3228 3229 3230 3231
	if (count) {
		e1000_tx_queue(adapter, tx_ring, tx_flags, count);
		/* Make sure there is space in the ring for the next send. */
		e1000_maybe_stop_tx(netdev, tx_ring, MAX_SKB_FRAGS + 2);
L
Linus Torvalds 已提交
3232

3233 3234 3235 3236 3237
	} else {
		dev_kfree_skb_any(skb);
		tx_ring->buffer_info[first].time_stamp = 0;
		tx_ring->next_to_use = first;
	}
L
Linus Torvalds 已提交
3238 3239 3240 3241

	return NETDEV_TX_OK;
}

T
Tushar Dave 已提交
3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463
#define NUM_REGS 38 /* 1 based count */
static void e1000_regdump(struct e1000_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 regs[NUM_REGS];
	u32 *regs_buff = regs;
	int i = 0;

	char *reg_name[] = {
	"CTRL",  "STATUS",
	"RCTL", "RDLEN", "RDH", "RDT", "RDTR",
	"TCTL", "TDBAL", "TDBAH", "TDLEN", "TDH", "TDT",
	"TIDV", "TXDCTL", "TADV", "TARC0",
	"TDBAL1", "TDBAH1", "TDLEN1", "TDH1", "TDT1",
	"TXDCTL1", "TARC1",
	"CTRL_EXT", "ERT", "RDBAL", "RDBAH",
	"TDFH", "TDFT", "TDFHS", "TDFTS", "TDFPC",
	"RDFH", "RDFT", "RDFHS", "RDFTS", "RDFPC"
	};

	regs_buff[0]  = er32(CTRL);
	regs_buff[1]  = er32(STATUS);

	regs_buff[2]  = er32(RCTL);
	regs_buff[3]  = er32(RDLEN);
	regs_buff[4]  = er32(RDH);
	regs_buff[5]  = er32(RDT);
	regs_buff[6]  = er32(RDTR);

	regs_buff[7]  = er32(TCTL);
	regs_buff[8]  = er32(TDBAL);
	regs_buff[9]  = er32(TDBAH);
	regs_buff[10] = er32(TDLEN);
	regs_buff[11] = er32(TDH);
	regs_buff[12] = er32(TDT);
	regs_buff[13] = er32(TIDV);
	regs_buff[14] = er32(TXDCTL);
	regs_buff[15] = er32(TADV);
	regs_buff[16] = er32(TARC0);

	regs_buff[17] = er32(TDBAL1);
	regs_buff[18] = er32(TDBAH1);
	regs_buff[19] = er32(TDLEN1);
	regs_buff[20] = er32(TDH1);
	regs_buff[21] = er32(TDT1);
	regs_buff[22] = er32(TXDCTL1);
	regs_buff[23] = er32(TARC1);
	regs_buff[24] = er32(CTRL_EXT);
	regs_buff[25] = er32(ERT);
	regs_buff[26] = er32(RDBAL0);
	regs_buff[27] = er32(RDBAH0);
	regs_buff[28] = er32(TDFH);
	regs_buff[29] = er32(TDFT);
	regs_buff[30] = er32(TDFHS);
	regs_buff[31] = er32(TDFTS);
	regs_buff[32] = er32(TDFPC);
	regs_buff[33] = er32(RDFH);
	regs_buff[34] = er32(RDFT);
	regs_buff[35] = er32(RDFHS);
	regs_buff[36] = er32(RDFTS);
	regs_buff[37] = er32(RDFPC);

	pr_info("Register dump\n");
	for (i = 0; i < NUM_REGS; i++) {
		printk(KERN_INFO "%-15s  %08x\n",
		reg_name[i], regs_buff[i]);
	}
}

/*
 * e1000_dump: Print registers, tx ring and rx ring
 */
static void e1000_dump(struct e1000_adapter *adapter)
{
	/* this code doesn't handle multiple rings */
	struct e1000_tx_ring *tx_ring = adapter->tx_ring;
	struct e1000_rx_ring *rx_ring = adapter->rx_ring;
	int i;

	if (!netif_msg_hw(adapter))
		return;

	/* Print Registers */
	e1000_regdump(adapter);

	/*
	 * transmit dump
	 */
	pr_info("TX Desc ring0 dump\n");

	/* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
	 *
	 * Legacy Transmit Descriptor
	 *   +--------------------------------------------------------------+
	 * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
	 *   +--------------------------------------------------------------+
	 * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
	 *   +--------------------------------------------------------------+
	 *   63       48 47        36 35    32 31     24 23    16 15        0
	 *
	 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
	 *   63      48 47    40 39       32 31             16 15    8 7      0
	 *   +----------------------------------------------------------------+
	 * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
	 *   +----------------------------------------------------------------+
	 * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
	 *   +----------------------------------------------------------------+
	 *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
	 *
	 * Extended Data Descriptor (DTYP=0x1)
	 *   +----------------------------------------------------------------+
	 * 0 |                     Buffer Address [63:0]                      |
	 *   +----------------------------------------------------------------+
	 * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
	 *   +----------------------------------------------------------------+
	 *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
	 */
	printk(KERN_INFO "Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ]"
	       " leng  ntw timestmp         bi->skb\n");
	printk(KERN_INFO "Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ]"
	       " leng  ntw timestmp         bi->skb\n");

	if (!netif_msg_tx_done(adapter))
		goto rx_ring_summary;

	for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
		struct e1000_tx_desc *tx_desc = E1000_TX_DESC(*tx_ring, i);
		struct e1000_buffer *buffer_info = &tx_ring->buffer_info[i];
		struct my_u { u64 a; u64 b; };
		struct my_u *u = (struct my_u *)tx_desc;
		printk(KERN_INFO "T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X "
		       "%016llX %p",
		       ((le64_to_cpu(u->b) & (1<<20)) ? 'd' : 'c'), i,
		       le64_to_cpu(u->a), le64_to_cpu(u->b),
		       (u64)buffer_info->dma, buffer_info->length,
		       buffer_info->next_to_watch, (u64)buffer_info->time_stamp,
		       buffer_info->skb);
		if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
			printk(KERN_CONT" NTC/U\n");
		else if (i == tx_ring->next_to_use)
			printk(KERN_CONT " NTU\n");
		else if (i == tx_ring->next_to_clean)
			printk(KERN_CONT " NTC\n");
		else
			printk(KERN_CONT "\n");


		if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
			print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
					16, 1, phys_to_virt(buffer_info->dma),
					buffer_info->length, true);
	}

rx_ring_summary:
	/*
	 * receive dump
	 */
	pr_info("\nRX Desc ring dump\n");

	/* Legacy Receive Descriptor Format
	 *
	 * +-----------------------------------------------------+
	 * |                Buffer Address [63:0]                |
	 * +-----------------------------------------------------+
	 * | VLAN Tag | Errors | Status 0 | Packet csum | Length |
	 * +-----------------------------------------------------+
	 * 63       48 47    40 39      32 31         16 15      0
	 */
	printk(KERN_INFO "R[desc]      [address 63:0  ] [vl er S cks ln] "
		"[bi->dma       ] [bi->skb]\n");

	if (!netif_msg_rx_status(adapter))
		goto exit;

	for (i = 0; rx_ring->desc && (i < rx_ring->count); i++) {
		struct e1000_rx_desc *rx_desc = E1000_RX_DESC(*rx_ring, i);
		struct e1000_buffer *buffer_info = &rx_ring->buffer_info[i];
		struct my_u { u64 a; u64 b; };
		struct my_u *u = (struct my_u *)rx_desc;
		printk(KERN_INFO "R[0x%03X]     %016llX %016llX %016llX %p",
			i, le64_to_cpu(u->a), le64_to_cpu(u->b),
			(u64)buffer_info->dma, buffer_info->skb);
		if (i == rx_ring->next_to_use)
			printk(KERN_CONT " NTU\n");
		else if (i == rx_ring->next_to_clean)
			printk(KERN_CONT " NTC\n");
		else
			printk(KERN_CONT "\n");

		if (netif_msg_pktdata(adapter))
			print_hex_dump(KERN_INFO, "",
				DUMP_PREFIX_ADDRESS, 16, 1,
				phys_to_virt(buffer_info->dma),
				buffer_info->length, true);

	} /* for */

	/* dump the descriptor caches */
	/* rx */
	printk(KERN_INFO "e1000: Rx descriptor cache in 64bit format\n");
	for (i = 0x6000; i <= 0x63FF ; i += 0x10) {
		printk(KERN_INFO "R%04X: %08X|%08X %08X|%08X\n",
				i,
				readl(adapter->hw.hw_addr + i+4),
				readl(adapter->hw.hw_addr + i),
				readl(adapter->hw.hw_addr + i+12),
				readl(adapter->hw.hw_addr + i+8));
	}
	/* tx */
	printk(KERN_INFO "e1000: Tx descriptor cache in 64bit format\n");
	for (i = 0x7000; i <= 0x73FF ; i += 0x10) {
		printk(KERN_INFO "T%04X: %08X|%08X %08X|%08X\n",
				i,
				readl(adapter->hw.hw_addr + i+4),
				readl(adapter->hw.hw_addr + i),
				readl(adapter->hw.hw_addr + i+12),
				readl(adapter->hw.hw_addr + i+8));
	}
exit:
	return;
}

L
Linus Torvalds 已提交
3464 3465 3466 3467 3468
/**
 * e1000_tx_timeout - Respond to a Tx Hang
 * @netdev: network interface device structure
 **/

3469
static void e1000_tx_timeout(struct net_device *netdev)
L
Linus Torvalds 已提交
3470
{
3471
	struct e1000_adapter *adapter = netdev_priv(netdev);
L
Linus Torvalds 已提交
3472 3473

	/* Do the reset outside of interrupt context */
3474 3475
	adapter->tx_timeout_count++;
	schedule_work(&adapter->reset_task);
L
Linus Torvalds 已提交
3476 3477
}

3478
static void e1000_reset_task(struct work_struct *work)
L
Linus Torvalds 已提交
3479
{
3480 3481
	struct e1000_adapter *adapter =
		container_of(work, struct e1000_adapter, reset_task);
L
Linus Torvalds 已提交
3482

3483 3484
	if (test_bit(__E1000_DOWN, &adapter->flags))
		return;
T
Tushar Dave 已提交
3485
	e_err(drv, "Reset adapter\n");
3486
	e1000_reinit_safe(adapter);
L
Linus Torvalds 已提交
3487 3488 3489 3490 3491 3492 3493
}

/**
 * e1000_get_stats - Get System Network Statistics
 * @netdev: network interface device structure
 *
 * Returns the address of the device statistics structure.
3494
 * The statistics are actually updated from the watchdog.
L
Linus Torvalds 已提交
3495 3496
 **/

3497
static struct net_device_stats *e1000_get_stats(struct net_device *netdev)
L
Linus Torvalds 已提交
3498
{
J
Jeff Kirsher 已提交
3499
	/* only return the current stats */
3500
	return &netdev->stats;
L
Linus Torvalds 已提交
3501 3502 3503 3504 3505 3506 3507 3508 3509 3510
}

/**
 * e1000_change_mtu - Change the Maximum Transfer Unit
 * @netdev: network interface device structure
 * @new_mtu: new value for maximum frame size
 *
 * Returns 0 on success, negative on failure
 **/

3511
static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
L
Linus Torvalds 已提交
3512
{
3513
	struct e1000_adapter *adapter = netdev_priv(netdev);
J
Joe Perches 已提交
3514
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
3515 3516
	int max_frame = new_mtu + ENET_HEADER_SIZE + ETHERNET_FCS_SIZE;

J
Jesse Brandeburg 已提交
3517 3518
	if ((max_frame < MINIMUM_ETHERNET_FRAME_SIZE) ||
	    (max_frame > MAX_JUMBO_FRAME_SIZE)) {
3519
		e_err(probe, "Invalid MTU setting\n");
L
Linus Torvalds 已提交
3520
		return -EINVAL;
3521
	}
L
Linus Torvalds 已提交
3522

3523
	/* Adapter-specific max frame size limits. */
J
Joe Perches 已提交
3524
	switch (hw->mac_type) {
3525
	case e1000_undefined ... e1000_82542_rev2_1:
3526
		if (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)) {
3527
			e_err(probe, "Jumbo Frames not supported.\n");
3528 3529
			return -EINVAL;
		}
3530 3531 3532 3533
		break;
	default:
		/* Capable of supporting up to MAX_JUMBO_FRAME_SIZE limit. */
		break;
L
Linus Torvalds 已提交
3534 3535
	}

3536 3537 3538 3539 3540 3541 3542
	while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
		msleep(1);
	/* e1000_down has a dependency on max_frame_size */
	hw->max_frame_size = max_frame;
	if (netif_running(netdev))
		e1000_down(adapter);

3543
	/* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3544
	 * means we reserve 2 more, this pushes us to allocate from the next
3545 3546 3547 3548
	 * larger slab size.
	 * i.e. RXBUFFER_2048 --> size-4096 slab
	 *  however with the new *_jumbo_rx* routines, jumbo receives will use
	 *  fragmented skbs */
3549

3550
	if (max_frame <= E1000_RXBUFFER_2048)
3551
		adapter->rx_buffer_len = E1000_RXBUFFER_2048;
3552 3553
	else
#if (PAGE_SIZE >= E1000_RXBUFFER_16384)
3554
		adapter->rx_buffer_len = E1000_RXBUFFER_16384;
3555 3556 3557
#elif (PAGE_SIZE >= E1000_RXBUFFER_4096)
		adapter->rx_buffer_len = PAGE_SIZE;
#endif
3558 3559

	/* adjust allocation if LPE protects us, and we aren't using SBP */
J
Joe Perches 已提交
3560
	if (!hw->tbi_compatibility_on &&
3561
	    ((max_frame == (ETH_FRAME_LEN + ETH_FCS_LEN)) ||
3562 3563
	     (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE)))
		adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3564

3565 3566
	pr_info("%s changing MTU from %d to %d\n",
		netdev->name, netdev->mtu, new_mtu);
3567 3568
	netdev->mtu = new_mtu;

3569
	if (netif_running(netdev))
3570 3571 3572 3573 3574
		e1000_up(adapter);
	else
		e1000_reset(adapter);

	clear_bit(__E1000_RESETTING, &adapter->flags);
L
Linus Torvalds 已提交
3575 3576 3577 3578 3579 3580 3581 3582 3583

	return 0;
}

/**
 * e1000_update_stats - Update the board statistics counters
 * @adapter: board private structure
 **/

3584
void e1000_update_stats(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
3585
{
3586
	struct net_device *netdev = adapter->netdev;
L
Linus Torvalds 已提交
3587
	struct e1000_hw *hw = &adapter->hw;
3588
	struct pci_dev *pdev = adapter->pdev;
L
Linus Torvalds 已提交
3589
	unsigned long flags;
3590
	u16 phy_tmp;
L
Linus Torvalds 已提交
3591 3592 3593

#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF

3594 3595 3596 3597
	/*
	 * Prevent stats update while adapter is being reset, or if the pci
	 * connection is down.
	 */
A
Auke Kok 已提交
3598
	if (adapter->link_speed == 0)
3599
		return;
3600
	if (pci_channel_offline(pdev))
A
Auke Kok 已提交
3601 3602
		return;

L
Linus Torvalds 已提交
3603 3604
	spin_lock_irqsave(&adapter->stats_lock, flags);

3605
	/* these counters are modified from e1000_tbi_adjust_stats,
L
Linus Torvalds 已提交
3606 3607 3608 3609
	 * called from the interrupt context, so they must only
	 * be written while holding adapter->stats_lock
	 */

J
Joe Perches 已提交
3610 3611 3612 3613 3614 3615 3616 3617
	adapter->stats.crcerrs += er32(CRCERRS);
	adapter->stats.gprc += er32(GPRC);
	adapter->stats.gorcl += er32(GORCL);
	adapter->stats.gorch += er32(GORCH);
	adapter->stats.bprc += er32(BPRC);
	adapter->stats.mprc += er32(MPRC);
	adapter->stats.roc += er32(ROC);

3618 3619 3620 3621 3622 3623
	adapter->stats.prc64 += er32(PRC64);
	adapter->stats.prc127 += er32(PRC127);
	adapter->stats.prc255 += er32(PRC255);
	adapter->stats.prc511 += er32(PRC511);
	adapter->stats.prc1023 += er32(PRC1023);
	adapter->stats.prc1522 += er32(PRC1522);
J
Joe Perches 已提交
3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651

	adapter->stats.symerrs += er32(SYMERRS);
	adapter->stats.mpc += er32(MPC);
	adapter->stats.scc += er32(SCC);
	adapter->stats.ecol += er32(ECOL);
	adapter->stats.mcc += er32(MCC);
	adapter->stats.latecol += er32(LATECOL);
	adapter->stats.dc += er32(DC);
	adapter->stats.sec += er32(SEC);
	adapter->stats.rlec += er32(RLEC);
	adapter->stats.xonrxc += er32(XONRXC);
	adapter->stats.xontxc += er32(XONTXC);
	adapter->stats.xoffrxc += er32(XOFFRXC);
	adapter->stats.xofftxc += er32(XOFFTXC);
	adapter->stats.fcruc += er32(FCRUC);
	adapter->stats.gptc += er32(GPTC);
	adapter->stats.gotcl += er32(GOTCL);
	adapter->stats.gotch += er32(GOTCH);
	adapter->stats.rnbc += er32(RNBC);
	adapter->stats.ruc += er32(RUC);
	adapter->stats.rfc += er32(RFC);
	adapter->stats.rjc += er32(RJC);
	adapter->stats.torl += er32(TORL);
	adapter->stats.torh += er32(TORH);
	adapter->stats.totl += er32(TOTL);
	adapter->stats.toth += er32(TOTH);
	adapter->stats.tpr += er32(TPR);

3652 3653 3654 3655 3656 3657
	adapter->stats.ptc64 += er32(PTC64);
	adapter->stats.ptc127 += er32(PTC127);
	adapter->stats.ptc255 += er32(PTC255);
	adapter->stats.ptc511 += er32(PTC511);
	adapter->stats.ptc1023 += er32(PTC1023);
	adapter->stats.ptc1522 += er32(PTC1522);
J
Joe Perches 已提交
3658 3659 3660

	adapter->stats.mptc += er32(MPTC);
	adapter->stats.bptc += er32(BPTC);
L
Linus Torvalds 已提交
3661 3662 3663

	/* used for adaptive IFS */

J
Joe Perches 已提交
3664
	hw->tx_packet_delta = er32(TPT);
L
Linus Torvalds 已提交
3665
	adapter->stats.tpt += hw->tx_packet_delta;
J
Joe Perches 已提交
3666
	hw->collision_delta = er32(COLC);
L
Linus Torvalds 已提交
3667 3668
	adapter->stats.colc += hw->collision_delta;

J
Jesse Brandeburg 已提交
3669
	if (hw->mac_type >= e1000_82543) {
J
Joe Perches 已提交
3670 3671 3672 3673 3674 3675
		adapter->stats.algnerrc += er32(ALGNERRC);
		adapter->stats.rxerrc += er32(RXERRC);
		adapter->stats.tncrs += er32(TNCRS);
		adapter->stats.cexterr += er32(CEXTERR);
		adapter->stats.tsctc += er32(TSCTC);
		adapter->stats.tsctfc += er32(TSCTFC);
L
Linus Torvalds 已提交
3676 3677 3678
	}

	/* Fill out the OS statistics structure */
3679 3680
	netdev->stats.multicast = adapter->stats.mprc;
	netdev->stats.collisions = adapter->stats.colc;
L
Linus Torvalds 已提交
3681 3682 3683

	/* Rx Errors */

3684 3685
	/* RLEC on some newer hardware can be incorrect so build
	* our own version based on RUC and ROC */
3686
	netdev->stats.rx_errors = adapter->stats.rxerrc +
L
Linus Torvalds 已提交
3687
		adapter->stats.crcerrs + adapter->stats.algnerrc +
3688 3689
		adapter->stats.ruc + adapter->stats.roc +
		adapter->stats.cexterr;
3690
	adapter->stats.rlerrc = adapter->stats.ruc + adapter->stats.roc;
3691 3692 3693 3694
	netdev->stats.rx_length_errors = adapter->stats.rlerrc;
	netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
	netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
	netdev->stats.rx_missed_errors = adapter->stats.mpc;
L
Linus Torvalds 已提交
3695 3696

	/* Tx Errors */
3697
	adapter->stats.txerrc = adapter->stats.ecol + adapter->stats.latecol;
3698 3699 3700 3701
	netdev->stats.tx_errors = adapter->stats.txerrc;
	netdev->stats.tx_aborted_errors = adapter->stats.ecol;
	netdev->stats.tx_window_errors = adapter->stats.latecol;
	netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
J
Joe Perches 已提交
3702
	if (hw->bad_tx_carr_stats_fd &&
3703
	    adapter->link_duplex == FULL_DUPLEX) {
3704
		netdev->stats.tx_carrier_errors = 0;
3705 3706
		adapter->stats.tncrs = 0;
	}
L
Linus Torvalds 已提交
3707 3708 3709 3710

	/* Tx Dropped needs to be maintained elsewhere */

	/* Phy Stats */
J
Jesse Brandeburg 已提交
3711 3712
	if (hw->media_type == e1000_media_type_copper) {
		if ((adapter->link_speed == SPEED_1000) &&
L
Linus Torvalds 已提交
3713 3714 3715 3716 3717
		   (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
			phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
			adapter->phy_stats.idle_errors += phy_tmp;
		}

J
Jesse Brandeburg 已提交
3718
		if ((hw->mac_type <= e1000_82546) &&
L
Linus Torvalds 已提交
3719 3720 3721 3722 3723
		   (hw->phy_type == e1000_phy_m88) &&
		   !e1000_read_phy_reg(hw, M88E1000_RX_ERR_CNTR, &phy_tmp))
			adapter->phy_stats.receive_errors += phy_tmp;
	}

3724
	/* Management Stats */
J
Joe Perches 已提交
3725 3726 3727 3728
	if (hw->has_smbus) {
		adapter->stats.mgptc += er32(MGTPTC);
		adapter->stats.mgprc += er32(MGTPRC);
		adapter->stats.mgpdc += er32(MGTPDC);
3729 3730
	}

L
Linus Torvalds 已提交
3731 3732
	spin_unlock_irqrestore(&adapter->stats_lock, flags);
}
3733

L
Linus Torvalds 已提交
3734 3735 3736 3737 3738 3739
/**
 * e1000_intr - Interrupt Handler
 * @irq: interrupt number
 * @data: pointer to a network interface device structure
 **/

3740
static irqreturn_t e1000_intr(int irq, void *data)
L
Linus Torvalds 已提交
3741 3742
{
	struct net_device *netdev = data;
3743
	struct e1000_adapter *adapter = netdev_priv(netdev);
L
Linus Torvalds 已提交
3744
	struct e1000_hw *hw = &adapter->hw;
3745
	u32 icr = er32(ICR);
3746

J
Jesse Brandeburg 已提交
3747
	if (unlikely((!icr)))
J
Jesse Brandeburg 已提交
3748 3749
		return IRQ_NONE;  /* Not our interrupt */

J
Jesse Brandeburg 已提交
3750 3751 3752 3753 3754 3755 3756 3757
	/*
	 * we might have caused the interrupt, but the above
	 * read cleared it, and just in case the driver is
	 * down there is nothing to do so return handled
	 */
	if (unlikely(test_bit(__E1000_DOWN, &adapter->flags)))
		return IRQ_HANDLED;

J
Jesse Brandeburg 已提交
3758
	if (unlikely(icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))) {
L
Linus Torvalds 已提交
3759
		hw->get_link_status = 1;
A
Auke Kok 已提交
3760 3761
		/* guard against interrupt when we're going down */
		if (!test_bit(__E1000_DOWN, &adapter->flags))
3762
			schedule_delayed_work(&adapter->watchdog_task, 1);
L
Linus Torvalds 已提交
3763 3764
	}

3765 3766 3767 3768
	/* disable interrupts, without the synchronize_irq bit */
	ew32(IMC, ~0);
	E1000_WRITE_FLUSH();

3769
	if (likely(napi_schedule_prep(&adapter->napi))) {
J
Jesse Brandeburg 已提交
3770 3771 3772 3773
		adapter->total_tx_bytes = 0;
		adapter->total_tx_packets = 0;
		adapter->total_rx_bytes = 0;
		adapter->total_rx_packets = 0;
3774
		__napi_schedule(&adapter->napi);
3775
	} else {
3776 3777
		/* this really should not happen! if it does it is basically a
		 * bug, but not a hard error, so enable ints and continue */
3778 3779 3780
		if (!test_bit(__E1000_DOWN, &adapter->flags))
			e1000_irq_enable(adapter);
	}
L
Linus Torvalds 已提交
3781 3782 3783 3784 3785 3786 3787 3788

	return IRQ_HANDLED;
}

/**
 * e1000_clean - NAPI Rx polling callback
 * @adapter: board private structure
 **/
3789
static int e1000_clean(struct napi_struct *napi, int budget)
L
Linus Torvalds 已提交
3790
{
3791
	struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter, napi);
3792
	int tx_clean_complete = 0, work_done = 0;
3793

3794
	tx_clean_complete = e1000_clean_tx_irq(adapter, &adapter->tx_ring[0]);
3795

3796
	adapter->clean_rx(adapter, &adapter->rx_ring[0], &work_done, budget);
3797

3798
	if (!tx_clean_complete)
3799 3800
		work_done = budget;

3801 3802
	/* If budget not fully consumed, exit the polling mode */
	if (work_done < budget) {
J
Jesse Brandeburg 已提交
3803 3804
		if (likely(adapter->itr_setting & 3))
			e1000_set_itr(adapter);
3805
		napi_complete(napi);
3806 3807
		if (!test_bit(__E1000_DOWN, &adapter->flags))
			e1000_irq_enable(adapter);
L
Linus Torvalds 已提交
3808 3809
	}

3810
	return work_done;
L
Linus Torvalds 已提交
3811 3812 3813 3814 3815 3816
}

/**
 * e1000_clean_tx_irq - Reclaim resources after transmit completes
 * @adapter: board private structure
 **/
3817 3818
static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
			       struct e1000_tx_ring *tx_ring)
L
Linus Torvalds 已提交
3819
{
J
Joe Perches 已提交
3820
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
3821 3822 3823 3824
	struct net_device *netdev = adapter->netdev;
	struct e1000_tx_desc *tx_desc, *eop_desc;
	struct e1000_buffer *buffer_info;
	unsigned int i, eop;
3825
	unsigned int count = 0;
J
Jesse Brandeburg 已提交
3826
	unsigned int total_tx_bytes=0, total_tx_packets=0;
L
Linus Torvalds 已提交
3827 3828 3829 3830 3831

	i = tx_ring->next_to_clean;
	eop = tx_ring->buffer_info[i].next_to_watch;
	eop_desc = E1000_TX_DESC(*tx_ring, eop);

3832 3833
	while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
	       (count < tx_ring->count)) {
3834
		bool cleaned = false;
3835
		rmb();	/* read buffer_info after eop_desc */
3836
		for ( ; !cleaned; count++) {
L
Linus Torvalds 已提交
3837 3838 3839 3840
			tx_desc = E1000_TX_DESC(*tx_ring, i);
			buffer_info = &tx_ring->buffer_info[i];
			cleaned = (i == eop);

J
Jesse Brandeburg 已提交
3841
			if (cleaned) {
3842 3843
				total_tx_packets += buffer_info->segs;
				total_tx_bytes += buffer_info->bytecount;
J
Jesse Brandeburg 已提交
3844
			}
J
Jeff Kirsher 已提交
3845
			e1000_unmap_and_free_tx_resource(adapter, buffer_info);
3846
			tx_desc->upper.data = 0;
L
Linus Torvalds 已提交
3847

J
Jesse Brandeburg 已提交
3848
			if (unlikely(++i == tx_ring->count)) i = 0;
L
Linus Torvalds 已提交
3849
		}
3850

L
Linus Torvalds 已提交
3851 3852 3853 3854 3855 3856
		eop = tx_ring->buffer_info[i].next_to_watch;
		eop_desc = E1000_TX_DESC(*tx_ring, eop);
	}

	tx_ring->next_to_clean = i;

3857
#define TX_WAKE_THRESHOLD 32
3858
	if (unlikely(count && netif_carrier_ok(netdev) &&
3859 3860 3861 3862 3863
		     E1000_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD)) {
		/* Make sure that anybody stopping the queue after this
		 * sees the new next_to_clean.
		 */
		smp_mb();
3864 3865 3866

		if (netif_queue_stopped(netdev) &&
		    !(test_bit(__E1000_DOWN, &adapter->flags))) {
3867
			netif_wake_queue(netdev);
3868 3869
			++adapter->restart_queue;
		}
3870
	}
3871

3872
	if (adapter->detect_tx_hung) {
3873
		/* Detect a transmit hang in hardware, this serializes the
L
Linus Torvalds 已提交
3874
		 * check with the clearing of time_stamp and movement of i */
J
Joe Perches 已提交
3875
		adapter->detect_tx_hung = false;
3876 3877
		if (tx_ring->buffer_info[eop].time_stamp &&
		    time_after(jiffies, tx_ring->buffer_info[eop].time_stamp +
3878 3879
		               (adapter->tx_timeout_factor * HZ)) &&
		    !(er32(STATUS) & E1000_STATUS_TXOFF)) {
3880 3881

			/* detected Tx unit hang */
3882
			e_err(drv, "Detected Tx Unit Hang\n"
3883 3884 3885 3886 3887 3888 3889 3890 3891 3892
			      "  Tx Queue             <%lu>\n"
			      "  TDH                  <%x>\n"
			      "  TDT                  <%x>\n"
			      "  next_to_use          <%x>\n"
			      "  next_to_clean        <%x>\n"
			      "buffer_info[next_to_clean]\n"
			      "  time_stamp           <%lx>\n"
			      "  next_to_watch        <%x>\n"
			      "  jiffies              <%lx>\n"
			      "  next_to_watch.status <%x>\n",
3893 3894
				(unsigned long)((tx_ring - adapter->tx_ring) /
					sizeof(struct e1000_tx_ring)),
J
Joe Perches 已提交
3895 3896
				readl(hw->hw_addr + tx_ring->tdh),
				readl(hw->hw_addr + tx_ring->tdt),
3897
				tx_ring->next_to_use,
3898
				tx_ring->next_to_clean,
3899
				tx_ring->buffer_info[eop].time_stamp,
3900 3901 3902
				eop,
				jiffies,
				eop_desc->upper.fields.status);
T
Tushar Dave 已提交
3903
			e1000_dump(adapter);
L
Linus Torvalds 已提交
3904
			netif_stop_queue(netdev);
3905
		}
L
Linus Torvalds 已提交
3906
	}
J
Jesse Brandeburg 已提交
3907 3908
	adapter->total_tx_bytes += total_tx_bytes;
	adapter->total_tx_packets += total_tx_packets;
3909 3910
	netdev->stats.tx_bytes += total_tx_bytes;
	netdev->stats.tx_packets += total_tx_packets;
3911
	return count < tx_ring->count;
L
Linus Torvalds 已提交
3912 3913 3914 3915
}

/**
 * e1000_rx_checksum - Receive Checksum Offload for 82543
3916 3917 3918 3919
 * @adapter:     board private structure
 * @status_err:  receive descriptor status and error fields
 * @csum:        receive descriptor csum field
 * @sk_buff:     socket buffer with received data
L
Linus Torvalds 已提交
3920 3921
 **/

3922 3923
static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
			      u32 csum, struct sk_buff *skb)
L
Linus Torvalds 已提交
3924
{
J
Joe Perches 已提交
3925
	struct e1000_hw *hw = &adapter->hw;
3926 3927
	u16 status = (u16)status_err;
	u8 errors = (u8)(status_err >> 24);
3928 3929

	skb_checksum_none_assert(skb);
3930

L
Linus Torvalds 已提交
3931
	/* 82543 or newer only */
J
Joe Perches 已提交
3932
	if (unlikely(hw->mac_type < e1000_82543)) return;
L
Linus Torvalds 已提交
3933
	/* Ignore Checksum bit is set */
J
Jesse Brandeburg 已提交
3934
	if (unlikely(status & E1000_RXD_STAT_IXSM)) return;
3935
	/* TCP/UDP checksum error bit is set */
J
Jesse Brandeburg 已提交
3936
	if (unlikely(errors & E1000_RXD_ERR_TCPE)) {
L
Linus Torvalds 已提交
3937 3938
		/* let the stack verify checksum errors */
		adapter->hw_csum_err++;
3939 3940 3941
		return;
	}
	/* TCP/UDP Checksum has not been calculated */
3942 3943 3944
	if (!(status & E1000_RXD_STAT_TCPCS))
		return;

3945 3946
	/* It must be a TCP or UDP packet with a valid checksum */
	if (likely(status & E1000_RXD_STAT_TCPCS)) {
L
Linus Torvalds 已提交
3947 3948 3949
		/* TCP checksum is good */
		skb->ip_summed = CHECKSUM_UNNECESSARY;
	}
3950
	adapter->hw_csum_good++;
L
Linus Torvalds 已提交
3951 3952
}

3953 3954 3955 3956 3957 3958 3959 3960 3961
/**
 * e1000_consume_page - helper function
 **/
static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb,
                               u16 length)
{
	bi->page = NULL;
	skb->len += length;
	skb->data_len += length;
3962
	skb->truesize += PAGE_SIZE;
3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974
}

/**
 * e1000_receive_skb - helper function to handle rx indications
 * @adapter: board private structure
 * @status: descriptor status field as written by hardware
 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
 * @skb: pointer to sk_buff to be indicated to stack
 */
static void e1000_receive_skb(struct e1000_adapter *adapter, u8 status,
			      __le16 vlan, struct sk_buff *skb)
{
J
Jesse Brandeburg 已提交
3975 3976
	skb->protocol = eth_type_trans(skb, adapter->netdev);

J
Jiri Pirko 已提交
3977 3978 3979 3980 3981 3982
	if (status & E1000_RXD_STAT_VP) {
		u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;

		__vlan_hwaccel_put_tag(skb, vid);
	}
	napi_gro_receive(&adapter->napi, skb);
3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021
}

/**
 * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
 * @adapter: board private structure
 * @rx_ring: ring to clean
 * @work_done: amount of napi work completed this call
 * @work_to_do: max amount of work allowed for this call to do
 *
 * the return value indicates whether actual cleaning was done, there
 * is no guarantee that everything was cleaned
 */
static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
				     struct e1000_rx_ring *rx_ring,
				     int *work_done, int work_to_do)
{
	struct e1000_hw *hw = &adapter->hw;
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
	struct e1000_rx_desc *rx_desc, *next_rxd;
	struct e1000_buffer *buffer_info, *next_buffer;
	unsigned long irq_flags;
	u32 length;
	unsigned int i;
	int cleaned_count = 0;
	bool cleaned = false;
	unsigned int total_rx_bytes=0, total_rx_packets=0;

	i = rx_ring->next_to_clean;
	rx_desc = E1000_RX_DESC(*rx_ring, i);
	buffer_info = &rx_ring->buffer_info[i];

	while (rx_desc->status & E1000_RXD_STAT_DD) {
		struct sk_buff *skb;
		u8 status;

		if (*work_done >= work_to_do)
			break;
		(*work_done)++;
4022
		rmb(); /* read descriptor and rx_buffer_info after status DD */
4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035

		status = rx_desc->status;
		skb = buffer_info->skb;
		buffer_info->skb = NULL;

		if (++i == rx_ring->count) i = 0;
		next_rxd = E1000_RX_DESC(*rx_ring, i);
		prefetch(next_rxd);

		next_buffer = &rx_ring->buffer_info[i];

		cleaned = true;
		cleaned_count++;
4036 4037
		dma_unmap_page(&pdev->dev, buffer_info->dma,
			       buffer_info->length, DMA_FROM_DEVICE);
4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134
		buffer_info->dma = 0;

		length = le16_to_cpu(rx_desc->length);

		/* errors is only valid for DD + EOP descriptors */
		if (unlikely((status & E1000_RXD_STAT_EOP) &&
		    (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK))) {
			u8 last_byte = *(skb->data + length - 1);
			if (TBI_ACCEPT(hw, status, rx_desc->errors, length,
				       last_byte)) {
				spin_lock_irqsave(&adapter->stats_lock,
				                  irq_flags);
				e1000_tbi_adjust_stats(hw, &adapter->stats,
				                       length, skb->data);
				spin_unlock_irqrestore(&adapter->stats_lock,
				                       irq_flags);
				length--;
			} else {
				/* recycle both page and skb */
				buffer_info->skb = skb;
				/* an error means any chain goes out the window
				 * too */
				if (rx_ring->rx_skb_top)
					dev_kfree_skb(rx_ring->rx_skb_top);
				rx_ring->rx_skb_top = NULL;
				goto next_desc;
			}
		}

#define rxtop rx_ring->rx_skb_top
		if (!(status & E1000_RXD_STAT_EOP)) {
			/* this descriptor is only the beginning (or middle) */
			if (!rxtop) {
				/* this is the beginning of a chain */
				rxtop = skb;
				skb_fill_page_desc(rxtop, 0, buffer_info->page,
				                   0, length);
			} else {
				/* this is the middle of a chain */
				skb_fill_page_desc(rxtop,
				    skb_shinfo(rxtop)->nr_frags,
				    buffer_info->page, 0, length);
				/* re-use the skb, only consumed the page */
				buffer_info->skb = skb;
			}
			e1000_consume_page(buffer_info, rxtop, length);
			goto next_desc;
		} else {
			if (rxtop) {
				/* end of the chain */
				skb_fill_page_desc(rxtop,
				    skb_shinfo(rxtop)->nr_frags,
				    buffer_info->page, 0, length);
				/* re-use the current skb, we only consumed the
				 * page */
				buffer_info->skb = skb;
				skb = rxtop;
				rxtop = NULL;
				e1000_consume_page(buffer_info, skb, length);
			} else {
				/* no chain, got EOP, this buf is the packet
				 * copybreak to save the put_page/alloc_page */
				if (length <= copybreak &&
				    skb_tailroom(skb) >= length) {
					u8 *vaddr;
					vaddr = kmap_atomic(buffer_info->page,
					                    KM_SKB_DATA_SOFTIRQ);
					memcpy(skb_tail_pointer(skb), vaddr, length);
					kunmap_atomic(vaddr,
					              KM_SKB_DATA_SOFTIRQ);
					/* re-use the page, so don't erase
					 * buffer_info->page */
					skb_put(skb, length);
				} else {
					skb_fill_page_desc(skb, 0,
					                   buffer_info->page, 0,
				                           length);
					e1000_consume_page(buffer_info, skb,
					                   length);
				}
			}
		}

		/* Receive Checksum Offload XXX recompute due to CRC strip? */
		e1000_rx_checksum(adapter,
		                  (u32)(status) |
		                  ((u32)(rx_desc->errors) << 24),
		                  le16_to_cpu(rx_desc->csum), skb);

		pskb_trim(skb, skb->len - 4);

		/* probably a little skewed due to removing CRC */
		total_rx_bytes += skb->len;
		total_rx_packets++;

		/* eth type trans needs skb->data to point to something */
		if (!pskb_may_pull(skb, ETH_HLEN)) {
4135
			e_err(drv, "pskb_may_pull failed.\n");
4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162
			dev_kfree_skb(skb);
			goto next_desc;
		}

		e1000_receive_skb(adapter, status, rx_desc->special, skb);

next_desc:
		rx_desc->status = 0;

		/* return some buffers to hardware, one at a time is too slow */
		if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
			adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
			cleaned_count = 0;
		}

		/* use prefetched values */
		rx_desc = next_rxd;
		buffer_info = next_buffer;
	}
	rx_ring->next_to_clean = i;

	cleaned_count = E1000_DESC_UNUSED(rx_ring);
	if (cleaned_count)
		adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);

	adapter->total_rx_packets += total_rx_packets;
	adapter->total_rx_bytes += total_rx_bytes;
4163 4164
	netdev->stats.rx_bytes += total_rx_bytes;
	netdev->stats.rx_packets += total_rx_packets;
4165 4166 4167
	return cleaned;
}

4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192
/*
 * this should improve performance for small packets with large amounts
 * of reassembly being done in the stack
 */
static void e1000_check_copybreak(struct net_device *netdev,
				 struct e1000_buffer *buffer_info,
				 u32 length, struct sk_buff **skb)
{
	struct sk_buff *new_skb;

	if (length > copybreak)
		return;

	new_skb = netdev_alloc_skb_ip_align(netdev, length);
	if (!new_skb)
		return;

	skb_copy_to_linear_data_offset(new_skb, -NET_IP_ALIGN,
				       (*skb)->data - NET_IP_ALIGN,
				       length + NET_IP_ALIGN);
	/* save the skb in buffer_info as good */
	buffer_info->skb = *skb;
	*skb = new_skb;
}

L
Linus Torvalds 已提交
4193
/**
4194
 * e1000_clean_rx_irq - Send received data up the network stack; legacy
L
Linus Torvalds 已提交
4195
 * @adapter: board private structure
4196 4197 4198 4199
 * @rx_ring: ring to clean
 * @work_done: amount of napi work completed this call
 * @work_to_do: max amount of work allowed for this call to do
 */
4200 4201 4202
static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
			       struct e1000_rx_ring *rx_ring,
			       int *work_done, int work_to_do)
L
Linus Torvalds 已提交
4203
{
J
Joe Perches 已提交
4204
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
4205 4206
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
4207 4208
	struct e1000_rx_desc *rx_desc, *next_rxd;
	struct e1000_buffer *buffer_info, *next_buffer;
L
Linus Torvalds 已提交
4209
	unsigned long flags;
4210
	u32 length;
L
Linus Torvalds 已提交
4211
	unsigned int i;
4212
	int cleaned_count = 0;
J
Joe Perches 已提交
4213
	bool cleaned = false;
J
Jesse Brandeburg 已提交
4214
	unsigned int total_rx_bytes=0, total_rx_packets=0;
L
Linus Torvalds 已提交
4215 4216 4217

	i = rx_ring->next_to_clean;
	rx_desc = E1000_RX_DESC(*rx_ring, i);
4218
	buffer_info = &rx_ring->buffer_info[i];
L
Linus Torvalds 已提交
4219

4220
	while (rx_desc->status & E1000_RXD_STAT_DD) {
4221
		struct sk_buff *skb;
4222
		u8 status;
4223

J
Jesse Brandeburg 已提交
4224
		if (*work_done >= work_to_do)
L
Linus Torvalds 已提交
4225 4226
			break;
		(*work_done)++;
4227
		rmb(); /* read descriptor and rx_buffer_info after status DD */
4228

4229
		status = rx_desc->status;
4230
		skb = buffer_info->skb;
4231 4232
		buffer_info->skb = NULL;

4233 4234
		prefetch(skb->data - NET_IP_ALIGN);

4235 4236
		if (++i == rx_ring->count) i = 0;
		next_rxd = E1000_RX_DESC(*rx_ring, i);
4237 4238
		prefetch(next_rxd);

4239 4240
		next_buffer = &rx_ring->buffer_info[i];

J
Joe Perches 已提交
4241
		cleaned = true;
4242
		cleaned_count++;
4243 4244
		dma_unmap_single(&pdev->dev, buffer_info->dma,
				 buffer_info->length, DMA_FROM_DEVICE);
J
Jesse Brandeburg 已提交
4245
		buffer_info->dma = 0;
L
Linus Torvalds 已提交
4246 4247

		length = le16_to_cpu(rx_desc->length);
4248
		/* !EOP means multiple descriptors were used to store a single
4249 4250 4251 4252 4253 4254 4255 4256 4257
		 * packet, if thats the case we need to toss it.  In fact, we
		 * to toss every packet with the EOP bit clear and the next
		 * frame that _does_ have the EOP bit set, as it is by
		 * definition only a frame fragment
		 */
		if (unlikely(!(status & E1000_RXD_STAT_EOP)))
			adapter->discarding = true;

		if (adapter->discarding) {
4258
			/* All receives must fit into a single buffer */
4259
			e_dbg("Receive packet consumed multiple buffers\n");
A
Auke Kok 已提交
4260
			/* recycle */
4261
			buffer_info->skb = skb;
4262 4263
			if (status & E1000_RXD_STAT_EOP)
				adapter->discarding = false;
L
Linus Torvalds 已提交
4264 4265 4266
			goto next_desc;
		}

J
Jesse Brandeburg 已提交
4267
		if (unlikely(rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK)) {
4268
			u8 last_byte = *(skb->data + length - 1);
J
Joe Perches 已提交
4269 4270
			if (TBI_ACCEPT(hw, status, rx_desc->errors, length,
				       last_byte)) {
L
Linus Torvalds 已提交
4271
				spin_lock_irqsave(&adapter->stats_lock, flags);
J
Joe Perches 已提交
4272
				e1000_tbi_adjust_stats(hw, &adapter->stats,
L
Linus Torvalds 已提交
4273 4274 4275 4276 4277
				                       length, skb->data);
				spin_unlock_irqrestore(&adapter->stats_lock,
				                       flags);
				length--;
			} else {
4278 4279
				/* recycle */
				buffer_info->skb = skb;
L
Linus Torvalds 已提交
4280 4281
				goto next_desc;
			}
A
Auke Kok 已提交
4282
		}
L
Linus Torvalds 已提交
4283

4284 4285 4286 4287
		/* adjust length to remove Ethernet CRC, this must be
		 * done after the TBI_ACCEPT workaround above */
		length -= 4;

J
Jesse Brandeburg 已提交
4288 4289 4290 4291
		/* probably a little skewed due to removing CRC */
		total_rx_bytes += length;
		total_rx_packets++;

4292 4293
		e1000_check_copybreak(netdev, buffer_info, length, &skb);

A
Auke Kok 已提交
4294
		skb_put(skb, length);
L
Linus Torvalds 已提交
4295 4296

		/* Receive Checksum Offload */
4297
		e1000_rx_checksum(adapter,
4298 4299
				  (u32)(status) |
				  ((u32)(rx_desc->errors) << 24),
D
David S. Miller 已提交
4300
				  le16_to_cpu(rx_desc->csum), skb);
J
Jesse Brandeburg 已提交
4301

4302
		e1000_receive_skb(adapter, status, rx_desc->special, skb);
4303

L
Linus Torvalds 已提交
4304 4305 4306
next_desc:
		rx_desc->status = 0;

4307 4308 4309 4310 4311 4312
		/* return some buffers to hardware, one at a time is too slow */
		if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
			adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
			cleaned_count = 0;
		}

4313
		/* use prefetched values */
4314 4315
		rx_desc = next_rxd;
		buffer_info = next_buffer;
L
Linus Torvalds 已提交
4316 4317
	}
	rx_ring->next_to_clean = i;
4318 4319 4320 4321

	cleaned_count = E1000_DESC_UNUSED(rx_ring);
	if (cleaned_count)
		adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4322

J
Jesse Brandeburg 已提交
4323 4324
	adapter->total_rx_packets += total_rx_packets;
	adapter->total_rx_bytes += total_rx_bytes;
4325 4326
	netdev->stats.rx_bytes += total_rx_bytes;
	netdev->stats.rx_packets += total_rx_packets;
4327 4328 4329
	return cleaned;
}

4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346
/**
 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
 * @adapter: address of board private structure
 * @rx_ring: pointer to receive ring structure
 * @cleaned_count: number of buffers to allocate this pass
 **/

static void
e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
                             struct e1000_rx_ring *rx_ring, int cleaned_count)
{
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
	struct e1000_rx_desc *rx_desc;
	struct e1000_buffer *buffer_info;
	struct sk_buff *skb;
	unsigned int i;
4347
	unsigned int bufsz = 256 - 16 /*for skb_reserve */ ;
4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358

	i = rx_ring->next_to_use;
	buffer_info = &rx_ring->buffer_info[i];

	while (cleaned_count--) {
		skb = buffer_info->skb;
		if (skb) {
			skb_trim(skb, 0);
			goto check_page;
		}

4359
		skb = netdev_alloc_skb_ip_align(netdev, bufsz);
4360 4361 4362 4363 4364 4365 4366 4367 4368
		if (unlikely(!skb)) {
			/* Better luck next round */
			adapter->alloc_rx_buff_failed++;
			break;
		}

		/* Fix for errata 23, can't cross 64kB boundary */
		if (!e1000_check_64k_bound(adapter, skb->data, bufsz)) {
			struct sk_buff *oldskb = skb;
4369 4370
			e_err(rx_err, "skb align check failed: %u bytes at "
			      "%p\n", bufsz, skb->data);
4371
			/* Try again, without freeing the previous */
4372
			skb = netdev_alloc_skb_ip_align(netdev, bufsz);
4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401
			/* Failed allocation, critical failure */
			if (!skb) {
				dev_kfree_skb(oldskb);
				adapter->alloc_rx_buff_failed++;
				break;
			}

			if (!e1000_check_64k_bound(adapter, skb->data, bufsz)) {
				/* give up */
				dev_kfree_skb(skb);
				dev_kfree_skb(oldskb);
				break; /* while (cleaned_count--) */
			}

			/* Use new allocation */
			dev_kfree_skb(oldskb);
		}
		buffer_info->skb = skb;
		buffer_info->length = adapter->rx_buffer_len;
check_page:
		/* allocate a new page if necessary */
		if (!buffer_info->page) {
			buffer_info->page = alloc_page(GFP_ATOMIC);
			if (unlikely(!buffer_info->page)) {
				adapter->alloc_rx_buff_failed++;
				break;
			}
		}

4402
		if (!buffer_info->dma) {
4403
			buffer_info->dma = dma_map_page(&pdev->dev,
4404
			                                buffer_info->page, 0,
4405 4406 4407
							buffer_info->length,
							DMA_FROM_DEVICE);
			if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4408 4409 4410 4411 4412 4413 4414 4415 4416
				put_page(buffer_info->page);
				dev_kfree_skb(skb);
				buffer_info->page = NULL;
				buffer_info->skb = NULL;
				buffer_info->dma = 0;
				adapter->alloc_rx_buff_failed++;
				break; /* while !buffer_info->skb */
			}
		}
4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439

		rx_desc = E1000_RX_DESC(*rx_ring, i);
		rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);

		if (unlikely(++i == rx_ring->count))
			i = 0;
		buffer_info = &rx_ring->buffer_info[i];
	}

	if (likely(rx_ring->next_to_use != i)) {
		rx_ring->next_to_use = i;
		if (unlikely(i-- == 0))
			i = (rx_ring->count - 1);

		/* Force memory writes to complete before letting h/w
		 * know there are new descriptors to fetch.  (Only
		 * applicable for weak-ordered memory model archs,
		 * such as IA-64). */
		wmb();
		writel(i, adapter->hw.hw_addr + rx_ring->rdt);
	}
}

L
Linus Torvalds 已提交
4440
/**
4441
 * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended
L
Linus Torvalds 已提交
4442 4443 4444
 * @adapter: address of board private structure
 **/

4445 4446 4447
static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
				   struct e1000_rx_ring *rx_ring,
				   int cleaned_count)
L
Linus Torvalds 已提交
4448
{
J
Joe Perches 已提交
4449
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
4450 4451 4452 4453 4454
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
	struct e1000_rx_desc *rx_desc;
	struct e1000_buffer *buffer_info;
	struct sk_buff *skb;
4455
	unsigned int i;
4456
	unsigned int bufsz = adapter->rx_buffer_len;
L
Linus Torvalds 已提交
4457 4458 4459 4460

	i = rx_ring->next_to_use;
	buffer_info = &rx_ring->buffer_info[i];

4461
	while (cleaned_count--) {
4462 4463
		skb = buffer_info->skb;
		if (skb) {
4464 4465 4466 4467
			skb_trim(skb, 0);
			goto map_skb;
		}

4468
		skb = netdev_alloc_skb_ip_align(netdev, bufsz);
J
Jesse Brandeburg 已提交
4469
		if (unlikely(!skb)) {
L
Linus Torvalds 已提交
4470
			/* Better luck next round */
4471
			adapter->alloc_rx_buff_failed++;
L
Linus Torvalds 已提交
4472 4473 4474
			break;
		}

4475
		/* Fix for errata 23, can't cross 64kB boundary */
L
Linus Torvalds 已提交
4476 4477
		if (!e1000_check_64k_bound(adapter, skb->data, bufsz)) {
			struct sk_buff *oldskb = skb;
4478 4479
			e_err(rx_err, "skb align check failed: %u bytes at "
			      "%p\n", bufsz, skb->data);
4480
			/* Try again, without freeing the previous */
4481
			skb = netdev_alloc_skb_ip_align(netdev, bufsz);
4482
			/* Failed allocation, critical failure */
L
Linus Torvalds 已提交
4483 4484
			if (!skb) {
				dev_kfree_skb(oldskb);
4485
				adapter->alloc_rx_buff_failed++;
L
Linus Torvalds 已提交
4486 4487
				break;
			}
4488

L
Linus Torvalds 已提交
4489 4490 4491 4492
			if (!e1000_check_64k_bound(adapter, skb->data, bufsz)) {
				/* give up */
				dev_kfree_skb(skb);
				dev_kfree_skb(oldskb);
4493
				adapter->alloc_rx_buff_failed++;
L
Linus Torvalds 已提交
4494 4495
				break; /* while !buffer_info->skb */
			}
4496 4497 4498

			/* Use new allocation */
			dev_kfree_skb(oldskb);
L
Linus Torvalds 已提交
4499 4500 4501
		}
		buffer_info->skb = skb;
		buffer_info->length = adapter->rx_buffer_len;
4502
map_skb:
4503
		buffer_info->dma = dma_map_single(&pdev->dev,
L
Linus Torvalds 已提交
4504
						  skb->data,
4505
						  buffer_info->length,
4506 4507
						  DMA_FROM_DEVICE);
		if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4508 4509 4510 4511 4512 4513
			dev_kfree_skb(skb);
			buffer_info->skb = NULL;
			buffer_info->dma = 0;
			adapter->alloc_rx_buff_failed++;
			break; /* while !buffer_info->skb */
		}
L
Linus Torvalds 已提交
4514

4515 4516 4517 4518 4519
		/*
		 * XXX if it was allocated cleanly it will never map to a
		 * boundary crossing
		 */

4520 4521 4522 4523
		/* Fix for errata 23, can't cross 64kB boundary */
		if (!e1000_check_64k_bound(adapter,
					(void *)(unsigned long)buffer_info->dma,
					adapter->rx_buffer_len)) {
4524 4525
			e_err(rx_err, "dma align check failed: %u bytes at "
			      "%p\n", adapter->rx_buffer_len,
4526
			      (void *)(unsigned long)buffer_info->dma);
L
Linus Torvalds 已提交
4527 4528 4529
			dev_kfree_skb(skb);
			buffer_info->skb = NULL;

4530
			dma_unmap_single(&pdev->dev, buffer_info->dma,
L
Linus Torvalds 已提交
4531
					 adapter->rx_buffer_len,
4532
					 DMA_FROM_DEVICE);
J
Jesse Brandeburg 已提交
4533
			buffer_info->dma = 0;
L
Linus Torvalds 已提交
4534

4535
			adapter->alloc_rx_buff_failed++;
L
Linus Torvalds 已提交
4536 4537 4538 4539 4540
			break; /* while !buffer_info->skb */
		}
		rx_desc = E1000_RX_DESC(*rx_ring, i);
		rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);

J
Jesse Brandeburg 已提交
4541 4542
		if (unlikely(++i == rx_ring->count))
			i = 0;
L
Linus Torvalds 已提交
4543 4544 4545
		buffer_info = &rx_ring->buffer_info[i];
	}

4546 4547 4548 4549 4550 4551 4552 4553 4554 4555
	if (likely(rx_ring->next_to_use != i)) {
		rx_ring->next_to_use = i;
		if (unlikely(i-- == 0))
			i = (rx_ring->count - 1);

		/* Force memory writes to complete before letting h/w
		 * know there are new descriptors to fetch.  (Only
		 * applicable for weak-ordered memory model archs,
		 * such as IA-64). */
		wmb();
J
Joe Perches 已提交
4556
		writel(i, hw->hw_addr + rx_ring->rdt);
4557
	}
L
Linus Torvalds 已提交
4558 4559 4560 4561 4562 4563 4564
}

/**
 * e1000_smartspeed - Workaround for SmartSpeed on 82541 and 82547 controllers.
 * @adapter:
 **/

4565
static void e1000_smartspeed(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
4566
{
J
Joe Perches 已提交
4567
	struct e1000_hw *hw = &adapter->hw;
4568 4569
	u16 phy_status;
	u16 phy_ctrl;
L
Linus Torvalds 已提交
4570

J
Joe Perches 已提交
4571 4572
	if ((hw->phy_type != e1000_phy_igp) || !hw->autoneg ||
	   !(hw->autoneg_advertised & ADVERTISE_1000_FULL))
L
Linus Torvalds 已提交
4573 4574
		return;

J
Jesse Brandeburg 已提交
4575
	if (adapter->smartspeed == 0) {
L
Linus Torvalds 已提交
4576 4577
		/* If Master/Slave config fault is asserted twice,
		 * we assume back-to-back */
J
Joe Perches 已提交
4578
		e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
J
Jesse Brandeburg 已提交
4579
		if (!(phy_status & SR_1000T_MS_CONFIG_FAULT)) return;
J
Joe Perches 已提交
4580
		e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
J
Jesse Brandeburg 已提交
4581
		if (!(phy_status & SR_1000T_MS_CONFIG_FAULT)) return;
J
Joe Perches 已提交
4582
		e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
J
Jesse Brandeburg 已提交
4583
		if (phy_ctrl & CR_1000T_MS_ENABLE) {
L
Linus Torvalds 已提交
4584
			phy_ctrl &= ~CR_1000T_MS_ENABLE;
J
Joe Perches 已提交
4585
			e1000_write_phy_reg(hw, PHY_1000T_CTRL,
L
Linus Torvalds 已提交
4586 4587
					    phy_ctrl);
			adapter->smartspeed++;
J
Joe Perches 已提交
4588 4589
			if (!e1000_phy_setup_autoneg(hw) &&
			   !e1000_read_phy_reg(hw, PHY_CTRL,
L
Linus Torvalds 已提交
4590 4591 4592
				   	       &phy_ctrl)) {
				phy_ctrl |= (MII_CR_AUTO_NEG_EN |
					     MII_CR_RESTART_AUTO_NEG);
J
Joe Perches 已提交
4593
				e1000_write_phy_reg(hw, PHY_CTRL,
L
Linus Torvalds 已提交
4594 4595 4596 4597
						    phy_ctrl);
			}
		}
		return;
J
Jesse Brandeburg 已提交
4598
	} else if (adapter->smartspeed == E1000_SMARTSPEED_DOWNSHIFT) {
L
Linus Torvalds 已提交
4599
		/* If still no link, perhaps using 2/3 pair cable */
J
Joe Perches 已提交
4600
		e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
L
Linus Torvalds 已提交
4601
		phy_ctrl |= CR_1000T_MS_ENABLE;
J
Joe Perches 已提交
4602 4603 4604
		e1000_write_phy_reg(hw, PHY_1000T_CTRL, phy_ctrl);
		if (!e1000_phy_setup_autoneg(hw) &&
		   !e1000_read_phy_reg(hw, PHY_CTRL, &phy_ctrl)) {
L
Linus Torvalds 已提交
4605 4606
			phy_ctrl |= (MII_CR_AUTO_NEG_EN |
				     MII_CR_RESTART_AUTO_NEG);
J
Joe Perches 已提交
4607
			e1000_write_phy_reg(hw, PHY_CTRL, phy_ctrl);
L
Linus Torvalds 已提交
4608 4609 4610
		}
	}
	/* Restart process after E1000_SMARTSPEED_MAX iterations */
J
Jesse Brandeburg 已提交
4611
	if (adapter->smartspeed++ == E1000_SMARTSPEED_MAX)
L
Linus Torvalds 已提交
4612 4613 4614 4615 4616 4617 4618 4619 4620 4621
		adapter->smartspeed = 0;
}

/**
 * e1000_ioctl -
 * @netdev:
 * @ifreq:
 * @cmd:
 **/

4622
static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
L
Linus Torvalds 已提交
4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640
{
	switch (cmd) {
	case SIOCGMIIPHY:
	case SIOCGMIIREG:
	case SIOCSMIIREG:
		return e1000_mii_ioctl(netdev, ifr, cmd);
	default:
		return -EOPNOTSUPP;
	}
}

/**
 * e1000_mii_ioctl -
 * @netdev:
 * @ifreq:
 * @cmd:
 **/

4641 4642
static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
			   int cmd)
L
Linus Torvalds 已提交
4643
{
4644
	struct e1000_adapter *adapter = netdev_priv(netdev);
J
Joe Perches 已提交
4645
	struct e1000_hw *hw = &adapter->hw;
L
Linus Torvalds 已提交
4646 4647
	struct mii_ioctl_data *data = if_mii(ifr);
	int retval;
4648
	u16 mii_reg;
4649
	unsigned long flags;
L
Linus Torvalds 已提交
4650

J
Joe Perches 已提交
4651
	if (hw->media_type != e1000_media_type_copper)
L
Linus Torvalds 已提交
4652 4653 4654 4655
		return -EOPNOTSUPP;

	switch (cmd) {
	case SIOCGMIIPHY:
J
Joe Perches 已提交
4656
		data->phy_id = hw->phy_addr;
L
Linus Torvalds 已提交
4657 4658
		break;
	case SIOCGMIIREG:
4659
		spin_lock_irqsave(&adapter->stats_lock, flags);
J
Joe Perches 已提交
4660
		if (e1000_read_phy_reg(hw, data->reg_num & 0x1F,
4661 4662
				   &data->val_out)) {
			spin_unlock_irqrestore(&adapter->stats_lock, flags);
L
Linus Torvalds 已提交
4663
			return -EIO;
4664 4665
		}
		spin_unlock_irqrestore(&adapter->stats_lock, flags);
L
Linus Torvalds 已提交
4666 4667
		break;
	case SIOCSMIIREG:
J
Jesse Brandeburg 已提交
4668
		if (data->reg_num & ~(0x1F))
L
Linus Torvalds 已提交
4669 4670
			return -EFAULT;
		mii_reg = data->val_in;
4671
		spin_lock_irqsave(&adapter->stats_lock, flags);
J
Joe Perches 已提交
4672
		if (e1000_write_phy_reg(hw, data->reg_num,
4673 4674
					mii_reg)) {
			spin_unlock_irqrestore(&adapter->stats_lock, flags);
L
Linus Torvalds 已提交
4675
			return -EIO;
4676
		}
4677
		spin_unlock_irqrestore(&adapter->stats_lock, flags);
J
Joe Perches 已提交
4678
		if (hw->media_type == e1000_media_type_copper) {
L
Linus Torvalds 已提交
4679 4680
			switch (data->reg_num) {
			case PHY_CTRL:
J
Jesse Brandeburg 已提交
4681
				if (mii_reg & MII_CR_POWER_DOWN)
L
Linus Torvalds 已提交
4682
					break;
J
Jesse Brandeburg 已提交
4683
				if (mii_reg & MII_CR_AUTO_NEG_EN) {
J
Joe Perches 已提交
4684 4685
					hw->autoneg = 1;
					hw->autoneg_advertised = 0x2F;
L
Linus Torvalds 已提交
4686
				} else {
4687
					u32 speed;
L
Linus Torvalds 已提交
4688
					if (mii_reg & 0x40)
4689
						speed = SPEED_1000;
L
Linus Torvalds 已提交
4690
					else if (mii_reg & 0x2000)
4691
						speed = SPEED_100;
L
Linus Torvalds 已提交
4692
					else
4693 4694 4695 4696 4697 4698
						speed = SPEED_10;
					retval = e1000_set_spd_dplx(
						adapter, speed,
						((mii_reg & 0x100)
						 ? DUPLEX_FULL :
						 DUPLEX_HALF));
4699
					if (retval)
L
Linus Torvalds 已提交
4700 4701
						return retval;
				}
4702 4703 4704
				if (netif_running(adapter->netdev))
					e1000_reinit_locked(adapter);
				else
L
Linus Torvalds 已提交
4705 4706 4707 4708
					e1000_reset(adapter);
				break;
			case M88E1000_PHY_SPEC_CTRL:
			case M88E1000_EXT_PHY_SPEC_CTRL:
J
Joe Perches 已提交
4709
				if (e1000_phy_reset(hw))
L
Linus Torvalds 已提交
4710 4711 4712 4713 4714 4715
					return -EIO;
				break;
			}
		} else {
			switch (data->reg_num) {
			case PHY_CTRL:
J
Jesse Brandeburg 已提交
4716
				if (mii_reg & MII_CR_POWER_DOWN)
L
Linus Torvalds 已提交
4717
					break;
4718 4719 4720
				if (netif_running(adapter->netdev))
					e1000_reinit_locked(adapter);
				else
L
Linus Torvalds 已提交
4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731
					e1000_reset(adapter);
				break;
			}
		}
		break;
	default:
		return -EOPNOTSUPP;
	}
	return E1000_SUCCESS;
}

4732
void e1000_pci_set_mwi(struct e1000_hw *hw)
L
Linus Torvalds 已提交
4733 4734
{
	struct e1000_adapter *adapter = hw->back;
4735
	int ret_val = pci_set_mwi(adapter->pdev);
L
Linus Torvalds 已提交
4736

J
Jesse Brandeburg 已提交
4737
	if (ret_val)
4738
		e_err(probe, "Error in setting MWI\n");
L
Linus Torvalds 已提交
4739 4740
}

4741
void e1000_pci_clear_mwi(struct e1000_hw *hw)
L
Linus Torvalds 已提交
4742 4743 4744 4745 4746 4747
{
	struct e1000_adapter *adapter = hw->back;

	pci_clear_mwi(adapter->pdev);
}

4748
int e1000_pcix_get_mmrbc(struct e1000_hw *hw)
4749 4750 4751 4752 4753
{
	struct e1000_adapter *adapter = hw->back;
	return pcix_get_mmrbc(adapter->pdev);
}

4754
void e1000_pcix_set_mmrbc(struct e1000_hw *hw, int mmrbc)
4755 4756 4757 4758 4759
{
	struct e1000_adapter *adapter = hw->back;
	pcix_set_mmrbc(adapter->pdev, mmrbc);
}

4760
void e1000_io_write(struct e1000_hw *hw, unsigned long port, u32 value)
L
Linus Torvalds 已提交
4761 4762 4763 4764
{
	outl(value, port);
}

J
Jiri Pirko 已提交
4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775
static bool e1000_vlan_used(struct e1000_adapter *adapter)
{
	u16 vid;

	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
		return true;
	return false;
}

static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
				     bool filter_on)
L
Linus Torvalds 已提交
4776
{
J
Joe Perches 已提交
4777
	struct e1000_hw *hw = &adapter->hw;
J
Jiri Pirko 已提交
4778
	u32 rctl;
L
Linus Torvalds 已提交
4779

J
Jesse Brandeburg 已提交
4780 4781
	if (!test_bit(__E1000_DOWN, &adapter->flags))
		e1000_irq_disable(adapter);
L
Linus Torvalds 已提交
4782

J
Jiri Pirko 已提交
4783
	if (filter_on) {
4784 4785 4786
		/* enable VLAN receive filtering */
		rctl = er32(RCTL);
		rctl &= ~E1000_RCTL_CFIEN;
J
Jiri Pirko 已提交
4787
		if (!(adapter->netdev->flags & IFF_PROMISC))
4788 4789 4790
			rctl |= E1000_RCTL_VFE;
		ew32(RCTL, rctl);
		e1000_update_mng_vlan(adapter);
L
Linus Torvalds 已提交
4791
	} else {
4792 4793 4794 4795
		/* disable VLAN receive filtering */
		rctl = er32(RCTL);
		rctl &= ~E1000_RCTL_VFE;
		ew32(RCTL, rctl);
J
Jiri Pirko 已提交
4796
	}
4797

J
Jiri Pirko 已提交
4798 4799 4800 4801
	if (!test_bit(__E1000_DOWN, &adapter->flags))
		e1000_irq_enable(adapter);
}

4802 4803
static void e1000_vlan_mode(struct net_device *netdev,
	netdev_features_t features)
J
Jiri Pirko 已提交
4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818
{
	struct e1000_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
	u32 ctrl;

	if (!test_bit(__E1000_DOWN, &adapter->flags))
		e1000_irq_disable(adapter);

	ctrl = er32(CTRL);
	if (features & NETIF_F_HW_VLAN_RX) {
		/* enable VLAN tag insert/strip */
		ctrl |= E1000_CTRL_VME;
	} else {
		/* disable VLAN tag insert/strip */
		ctrl &= ~E1000_CTRL_VME;
L
Linus Torvalds 已提交
4819
	}
J
Jiri Pirko 已提交
4820
	ew32(CTRL, ctrl);
L
Linus Torvalds 已提交
4821

J
Jesse Brandeburg 已提交
4822 4823
	if (!test_bit(__E1000_DOWN, &adapter->flags))
		e1000_irq_enable(adapter);
L
Linus Torvalds 已提交
4824 4825
}

4826
static int e1000_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
L
Linus Torvalds 已提交
4827
{
4828
	struct e1000_adapter *adapter = netdev_priv(netdev);
J
Joe Perches 已提交
4829
	struct e1000_hw *hw = &adapter->hw;
4830
	u32 vfta, index;
J
Jesse Brandeburg 已提交
4831

J
Joe Perches 已提交
4832
	if ((hw->mng_cookie.status &
J
Jesse Brandeburg 已提交
4833 4834
	     E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
	    (vid == adapter->mng_vlan_id))
4835
		return 0;
J
Jiri Pirko 已提交
4836 4837 4838 4839

	if (!e1000_vlan_used(adapter))
		e1000_vlan_filter_on_off(adapter, true);

L
Linus Torvalds 已提交
4840 4841
	/* add VID to filter table */
	index = (vid >> 5) & 0x7F;
J
Joe Perches 已提交
4842
	vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
L
Linus Torvalds 已提交
4843
	vfta |= (1 << (vid & 0x1F));
J
Joe Perches 已提交
4844
	e1000_write_vfta(hw, index, vfta);
J
Jiri Pirko 已提交
4845 4846

	set_bit(vid, adapter->active_vlans);
4847 4848

	return 0;
L
Linus Torvalds 已提交
4849 4850
}

4851
static int e1000_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
L
Linus Torvalds 已提交
4852
{
4853
	struct e1000_adapter *adapter = netdev_priv(netdev);
J
Joe Perches 已提交
4854
	struct e1000_hw *hw = &adapter->hw;
4855
	u32 vfta, index;
L
Linus Torvalds 已提交
4856

J
Jesse Brandeburg 已提交
4857 4858 4859 4860
	if (!test_bit(__E1000_DOWN, &adapter->flags))
		e1000_irq_disable(adapter);
	if (!test_bit(__E1000_DOWN, &adapter->flags))
		e1000_irq_enable(adapter);
L
Linus Torvalds 已提交
4861 4862 4863

	/* remove VID from filter table */
	index = (vid >> 5) & 0x7F;
J
Joe Perches 已提交
4864
	vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
L
Linus Torvalds 已提交
4865
	vfta &= ~(1 << (vid & 0x1F));
J
Joe Perches 已提交
4866
	e1000_write_vfta(hw, index, vfta);
J
Jiri Pirko 已提交
4867 4868 4869 4870 4871

	clear_bit(vid, adapter->active_vlans);

	if (!e1000_vlan_used(adapter))
		e1000_vlan_filter_on_off(adapter, false);
4872 4873

	return 0;
L
Linus Torvalds 已提交
4874 4875
}

4876
static void e1000_restore_vlan(struct e1000_adapter *adapter)
L
Linus Torvalds 已提交
4877
{
J
Jiri Pirko 已提交
4878
	u16 vid;
L
Linus Torvalds 已提交
4879

J
Jiri Pirko 已提交
4880 4881 4882 4883 4884 4885
	if (!e1000_vlan_used(adapter))
		return;

	e1000_vlan_filter_on_off(adapter, true);
	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
		e1000_vlan_rx_add_vid(adapter->netdev, vid);
L
Linus Torvalds 已提交
4886 4887
}

4888
int e1000_set_spd_dplx(struct e1000_adapter *adapter, u32 spd, u8 dplx)
L
Linus Torvalds 已提交
4889
{
J
Joe Perches 已提交
4890 4891 4892
	struct e1000_hw *hw = &adapter->hw;

	hw->autoneg = 0;
L
Linus Torvalds 已提交
4893

4894 4895 4896 4897 4898
	/* Make sure dplx is at most 1 bit and lsb of speed is not set
	 * for the switch() below to work */
	if ((spd & 1) || (dplx & ~1))
		goto err_inval;

4899
	/* Fiber NICs only allow 1000 gbps Full duplex */
J
Joe Perches 已提交
4900
	if ((hw->media_type == e1000_media_type_fiber) &&
4901 4902 4903
	    spd != SPEED_1000 &&
	    dplx != DUPLEX_FULL)
		goto err_inval;
4904

4905
	switch (spd + dplx) {
L
Linus Torvalds 已提交
4906
	case SPEED_10 + DUPLEX_HALF:
J
Joe Perches 已提交
4907
		hw->forced_speed_duplex = e1000_10_half;
L
Linus Torvalds 已提交
4908 4909
		break;
	case SPEED_10 + DUPLEX_FULL:
J
Joe Perches 已提交
4910
		hw->forced_speed_duplex = e1000_10_full;
L
Linus Torvalds 已提交
4911 4912
		break;
	case SPEED_100 + DUPLEX_HALF:
J
Joe Perches 已提交
4913
		hw->forced_speed_duplex = e1000_100_half;
L
Linus Torvalds 已提交
4914 4915
		break;
	case SPEED_100 + DUPLEX_FULL:
J
Joe Perches 已提交
4916
		hw->forced_speed_duplex = e1000_100_full;
L
Linus Torvalds 已提交
4917 4918
		break;
	case SPEED_1000 + DUPLEX_FULL:
J
Joe Perches 已提交
4919 4920
		hw->autoneg = 1;
		hw->autoneg_advertised = ADVERTISE_1000_FULL;
L
Linus Torvalds 已提交
4921 4922 4923
		break;
	case SPEED_1000 + DUPLEX_HALF: /* not supported */
	default:
4924
		goto err_inval;
L
Linus Torvalds 已提交
4925 4926
	}
	return 0;
4927 4928 4929 4930

err_inval:
	e_err(probe, "Unsupported Speed/Duplex configuration\n");
	return -EINVAL;
L
Linus Torvalds 已提交
4931 4932
}

4933
static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake)
L
Linus Torvalds 已提交
4934 4935
{
	struct net_device *netdev = pci_get_drvdata(pdev);
4936
	struct e1000_adapter *adapter = netdev_priv(netdev);
J
Joe Perches 已提交
4937
	struct e1000_hw *hw = &adapter->hw;
4938 4939
	u32 ctrl, ctrl_ext, rctl, status;
	u32 wufc = adapter->wol;
A
Auke Kok 已提交
4940
#ifdef CONFIG_PM
4941
	int retval = 0;
A
Auke Kok 已提交
4942
#endif
L
Linus Torvalds 已提交
4943 4944 4945

	netif_device_detach(netdev);

4946 4947
	if (netif_running(netdev)) {
		WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
L
Linus Torvalds 已提交
4948
		e1000_down(adapter);
4949
	}
L
Linus Torvalds 已提交
4950

4951
#ifdef CONFIG_PM
4952
	retval = pci_save_state(pdev);
4953
	if (retval)
4954 4955 4956
		return retval;
#endif

J
Joe Perches 已提交
4957
	status = er32(STATUS);
J
Jesse Brandeburg 已提交
4958
	if (status & E1000_STATUS_LU)
L
Linus Torvalds 已提交
4959 4960
		wufc &= ~E1000_WUFC_LNKC;

J
Jesse Brandeburg 已提交
4961
	if (wufc) {
L
Linus Torvalds 已提交
4962
		e1000_setup_rctl(adapter);
4963
		e1000_set_rx_mode(netdev);
L
Linus Torvalds 已提交
4964

4965 4966
		rctl = er32(RCTL);

L
Linus Torvalds 已提交
4967
		/* turn on all-multi mode if wake on multicast is enabled */
4968
		if (wufc & E1000_WUFC_MC)
L
Linus Torvalds 已提交
4969
			rctl |= E1000_RCTL_MPE;
4970 4971 4972

		/* enable receives in the hardware */
		ew32(RCTL, rctl | E1000_RCTL_EN);
L
Linus Torvalds 已提交
4973

J
Joe Perches 已提交
4974 4975
		if (hw->mac_type >= e1000_82540) {
			ctrl = er32(CTRL);
L
Linus Torvalds 已提交
4976 4977 4978 4979 4980 4981
			/* advertise wake from D3Cold */
			#define E1000_CTRL_ADVD3WUC 0x00100000
			/* phy power management enable */
			#define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
			ctrl |= E1000_CTRL_ADVD3WUC |
				E1000_CTRL_EN_PHY_PWR_MGMT;
J
Joe Perches 已提交
4982
			ew32(CTRL, ctrl);
L
Linus Torvalds 已提交
4983 4984
		}

J
Joe Perches 已提交
4985
		if (hw->media_type == e1000_media_type_fiber ||
4986
		    hw->media_type == e1000_media_type_internal_serdes) {
L
Linus Torvalds 已提交
4987
			/* keep the laser running in D3 */
J
Joe Perches 已提交
4988
			ctrl_ext = er32(CTRL_EXT);
L
Linus Torvalds 已提交
4989
			ctrl_ext |= E1000_CTRL_EXT_SDP7_DATA;
J
Joe Perches 已提交
4990
			ew32(CTRL_EXT, ctrl_ext);
L
Linus Torvalds 已提交
4991 4992
		}

J
Joe Perches 已提交
4993 4994
		ew32(WUC, E1000_WUC_PME_EN);
		ew32(WUFC, wufc);
L
Linus Torvalds 已提交
4995
	} else {
J
Joe Perches 已提交
4996 4997
		ew32(WUC, 0);
		ew32(WUFC, 0);
L
Linus Torvalds 已提交
4998 4999
	}

5000 5001
	e1000_release_manageability(adapter);

5002 5003
	*enable_wake = !!wufc;

5004
	/* make sure adapter isn't asleep if manageability is enabled */
5005 5006
	if (adapter->en_mng_pt)
		*enable_wake = true;
L
Linus Torvalds 已提交
5007

5008 5009 5010
	if (netif_running(netdev))
		e1000_free_irq(adapter);

L
Linus Torvalds 已提交
5011
	pci_disable_device(pdev);
5012

L
Linus Torvalds 已提交
5013 5014 5015
	return 0;
}

5016
#ifdef CONFIG_PM
5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035
static int e1000_suspend(struct pci_dev *pdev, pm_message_t state)
{
	int retval;
	bool wake;

	retval = __e1000_shutdown(pdev, &wake);
	if (retval)
		return retval;

	if (wake) {
		pci_prepare_to_sleep(pdev);
	} else {
		pci_wake_from_d3(pdev, false);
		pci_set_power_state(pdev, PCI_D3hot);
	}

	return 0;
}

5036
static int e1000_resume(struct pci_dev *pdev)
L
Linus Torvalds 已提交
5037 5038
{
	struct net_device *netdev = pci_get_drvdata(pdev);
5039
	struct e1000_adapter *adapter = netdev_priv(netdev);
J
Joe Perches 已提交
5040
	struct e1000_hw *hw = &adapter->hw;
5041
	u32 err;
L
Linus Torvalds 已提交
5042

5043
	pci_set_power_state(pdev, PCI_D0);
5044
	pci_restore_state(pdev);
5045
	pci_save_state(pdev);
T
Taku Izumi 已提交
5046 5047 5048 5049 5050

	if (adapter->need_ioport)
		err = pci_enable_device(pdev);
	else
		err = pci_enable_device_mem(pdev);
5051
	if (err) {
5052
		pr_err("Cannot enable PCI device from suspend\n");
5053 5054
		return err;
	}
5055
	pci_set_master(pdev);
L
Linus Torvalds 已提交
5056

5057 5058
	pci_enable_wake(pdev, PCI_D3hot, 0);
	pci_enable_wake(pdev, PCI_D3cold, 0);
L
Linus Torvalds 已提交
5059

5060 5061 5062 5063 5064
	if (netif_running(netdev)) {
		err = e1000_request_irq(adapter);
		if (err)
			return err;
	}
5065 5066

	e1000_power_up_phy(adapter);
L
Linus Torvalds 已提交
5067
	e1000_reset(adapter);
J
Joe Perches 已提交
5068
	ew32(WUS, ~0);
L
Linus Torvalds 已提交
5069

5070 5071
	e1000_init_manageability(adapter);

J
Jesse Brandeburg 已提交
5072
	if (netif_running(netdev))
L
Linus Torvalds 已提交
5073 5074 5075 5076 5077 5078 5079
		e1000_up(adapter);

	netif_device_attach(netdev);

	return 0;
}
#endif
5080 5081 5082

static void e1000_shutdown(struct pci_dev *pdev)
{
5083 5084 5085 5086 5087 5088 5089 5090
	bool wake;

	__e1000_shutdown(pdev, &wake);

	if (system_state == SYSTEM_POWER_OFF) {
		pci_wake_from_d3(pdev, wake);
		pci_set_power_state(pdev, PCI_D3hot);
	}
5091 5092
}

L
Linus Torvalds 已提交
5093 5094 5095 5096 5097 5098
#ifdef CONFIG_NET_POLL_CONTROLLER
/*
 * Polling 'interrupt' - used by things like netconsole to send skbs
 * without having to re-enable interrupts. It's not called while
 * the interrupt routine is executing.
 */
5099
static void e1000_netpoll(struct net_device *netdev)
L
Linus Torvalds 已提交
5100
{
5101
	struct e1000_adapter *adapter = netdev_priv(netdev);
5102

L
Linus Torvalds 已提交
5103
	disable_irq(adapter->pdev->irq);
5104
	e1000_intr(adapter->pdev->irq, netdev);
L
Linus Torvalds 已提交
5105 5106 5107 5108
	enable_irq(adapter->pdev->irq);
}
#endif

A
Auke Kok 已提交
5109 5110 5111
/**
 * e1000_io_error_detected - called when PCI error is detected
 * @pdev: Pointer to PCI device
5112
 * @state: The current pci connection state
A
Auke Kok 已提交
5113 5114 5115 5116
 *
 * This function is called after a PCI bus error affecting
 * this device has been detected.
 */
5117 5118
static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
						pci_channel_state_t state)
A
Auke Kok 已提交
5119 5120
{
	struct net_device *netdev = pci_get_drvdata(pdev);
5121
	struct e1000_adapter *adapter = netdev_priv(netdev);
A
Auke Kok 已提交
5122 5123 5124

	netif_device_detach(netdev);

5125 5126 5127
	if (state == pci_channel_io_perm_failure)
		return PCI_ERS_RESULT_DISCONNECT;

A
Auke Kok 已提交
5128 5129
	if (netif_running(netdev))
		e1000_down(adapter);
5130
	pci_disable_device(pdev);
A
Auke Kok 已提交
5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145

	/* Request a slot slot reset. */
	return PCI_ERS_RESULT_NEED_RESET;
}

/**
 * e1000_io_slot_reset - called after the pci bus has been reset.
 * @pdev: Pointer to PCI device
 *
 * Restart the card from scratch, as if from a cold-boot. Implementation
 * resembles the first-half of the e1000_resume routine.
 */
static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
5146
	struct e1000_adapter *adapter = netdev_priv(netdev);
J
Joe Perches 已提交
5147
	struct e1000_hw *hw = &adapter->hw;
T
Taku Izumi 已提交
5148
	int err;
A
Auke Kok 已提交
5149

T
Taku Izumi 已提交
5150 5151 5152 5153 5154
	if (adapter->need_ioport)
		err = pci_enable_device(pdev);
	else
		err = pci_enable_device_mem(pdev);
	if (err) {
5155
		pr_err("Cannot re-enable PCI device after reset.\n");
A
Auke Kok 已提交
5156 5157 5158 5159
		return PCI_ERS_RESULT_DISCONNECT;
	}
	pci_set_master(pdev);

5160 5161
	pci_enable_wake(pdev, PCI_D3hot, 0);
	pci_enable_wake(pdev, PCI_D3cold, 0);
A
Auke Kok 已提交
5162 5163

	e1000_reset(adapter);
J
Joe Perches 已提交
5164
	ew32(WUS, ~0);
A
Auke Kok 已提交
5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179

	return PCI_ERS_RESULT_RECOVERED;
}

/**
 * e1000_io_resume - called when traffic can start flowing again.
 * @pdev: Pointer to PCI device
 *
 * This callback is called when the error recovery driver tells us that
 * its OK to resume normal operation. Implementation resembles the
 * second-half of the e1000_resume routine.
 */
static void e1000_io_resume(struct pci_dev *pdev)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
5180
	struct e1000_adapter *adapter = netdev_priv(netdev);
5181 5182

	e1000_init_manageability(adapter);
A
Auke Kok 已提交
5183 5184 5185

	if (netif_running(netdev)) {
		if (e1000_up(adapter)) {
5186
			pr_info("can't bring device back up after reset\n");
A
Auke Kok 已提交
5187 5188 5189 5190 5191 5192 5193
			return;
		}
	}

	netif_device_attach(netdev);
}

L
Linus Torvalds 已提交
5194
/* e1000_main.c */