efx.c 75.3 KB
Newer Older
1 2 3
/****************************************************************************
 * Driver for Solarflare Solarstorm network controllers and boards
 * Copyright 2005-2006 Fen Systems Ltd.
B
Ben Hutchings 已提交
4
 * Copyright 2005-2011 Solarflare Communications Inc.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation, incorporated herein by reference.
 */

#include <linux/module.h>
#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/delay.h>
#include <linux/notifier.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/in.h>
#include <linux/crc32.h>
#include <linux/ethtool.h>
22
#include <linux/topology.h>
23
#include <linux/gfp.h>
24
#include <linux/cpu_rmap.h>
25 26
#include "net_driver.h"
#include "efx.h"
B
Ben Hutchings 已提交
27
#include "nic.h"
28
#include "selftest.h"
29

30
#include "mcdi.h"
31
#include "workarounds.h"
32

33 34 35 36 37 38 39 40 41
/**************************************************************************
 *
 * Type name strings
 *
 **************************************************************************
 */

/* Loopback mode names (see LOOPBACK_MODE()) */
const unsigned int efx_loopback_mode_max = LOOPBACK_MAX;
42
const char *const efx_loopback_mode_names[] = {
43
	[LOOPBACK_NONE]		= "NONE",
44
	[LOOPBACK_DATA]		= "DATAPATH",
45 46 47
	[LOOPBACK_GMAC]		= "GMAC",
	[LOOPBACK_XGMII]	= "XGMII",
	[LOOPBACK_XGXS]		= "XGXS",
48 49 50
	[LOOPBACK_XAUI]		= "XAUI",
	[LOOPBACK_GMII]		= "GMII",
	[LOOPBACK_SGMII]	= "SGMII",
51 52 53 54 55 56
	[LOOPBACK_XGBR]		= "XGBR",
	[LOOPBACK_XFI]		= "XFI",
	[LOOPBACK_XAUI_FAR]	= "XAUI_FAR",
	[LOOPBACK_GMII_FAR]	= "GMII_FAR",
	[LOOPBACK_SGMII_FAR]	= "SGMII_FAR",
	[LOOPBACK_XFI_FAR]	= "XFI_FAR",
57 58
	[LOOPBACK_GPHY]		= "GPHY",
	[LOOPBACK_PHYXS]	= "PHYXS",
59 60
	[LOOPBACK_PCS]		= "PCS",
	[LOOPBACK_PMAPMD]	= "PMA/PMD",
61 62
	[LOOPBACK_XPORT]	= "XPORT",
	[LOOPBACK_XGMII_WS]	= "XGMII_WS",
63
	[LOOPBACK_XAUI_WS]	= "XAUI_WS",
64 65
	[LOOPBACK_XAUI_WS_FAR]  = "XAUI_WS_FAR",
	[LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR",
66
	[LOOPBACK_GMII_WS]	= "GMII_WS",
67 68
	[LOOPBACK_XFI_WS]	= "XFI_WS",
	[LOOPBACK_XFI_WS_FAR]	= "XFI_WS_FAR",
69
	[LOOPBACK_PHYXS_WS]	= "PHYXS_WS",
70 71 72
};

const unsigned int efx_reset_type_max = RESET_TYPE_MAX;
73
const char *const efx_reset_type_names[] = {
74 75 76 77 78 79 80 81 82 83
	[RESET_TYPE_INVISIBLE]     = "INVISIBLE",
	[RESET_TYPE_ALL]           = "ALL",
	[RESET_TYPE_WORLD]         = "WORLD",
	[RESET_TYPE_DISABLE]       = "DISABLE",
	[RESET_TYPE_TX_WATCHDOG]   = "TX_WATCHDOG",
	[RESET_TYPE_INT_ERROR]     = "INT_ERROR",
	[RESET_TYPE_RX_RECOVERY]   = "RX_RECOVERY",
	[RESET_TYPE_RX_DESC_FETCH] = "RX_DESC_FETCH",
	[RESET_TYPE_TX_DESC_FETCH] = "TX_DESC_FETCH",
	[RESET_TYPE_TX_SKIP]       = "TX_SKIP",
84
	[RESET_TYPE_MC_FAILURE]    = "MC_FAILURE",
85 86
};

87 88
#define EFX_MAX_MTU (9 * 1024)

89 90 91 92 93 94
/* Reset workqueue. If any NIC has a hardware failure then a reset will be
 * queued onto this work queue. This is not a per-nic work queue, because
 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised.
 */
static struct workqueue_struct *reset_workqueue;

95 96 97 98 99 100 101 102 103
/**************************************************************************
 *
 * Configurable values
 *
 *************************************************************************/

/*
 * Use separate channels for TX and RX events
 *
104 105
 * Set this to 1 to use separate channels for TX and RX. It allows us
 * to control interrupt affinity separately for TX and RX.
106
 *
107
 * This is only used in MSI-X interrupt mode
108
 */
109
static unsigned int separate_tx_channels;
110
module_param(separate_tx_channels, uint, 0444);
111 112
MODULE_PARM_DESC(separate_tx_channels,
		 "Use separate channels for TX and RX");
113 114 115 116 117 118 119

/* This is the weight assigned to each of the (per-channel) virtual
 * NAPI devices.
 */
static int napi_weight = 64;

/* This is the time (in jiffies) between invocations of the hardware
120 121 122
 * monitor.  On Falcon-based NICs, this will:
 * - Check the on-board hardware monitor;
 * - Poll the link state and reconfigure the hardware as necessary.
123
 */
S
stephen hemminger 已提交
124
static unsigned int efx_monitor_interval = 1 * HZ;
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156

/* Initial interrupt moderation settings.  They can be modified after
 * module load with ethtool.
 *
 * The default for RX should strike a balance between increasing the
 * round-trip latency and reducing overhead.
 */
static unsigned int rx_irq_mod_usec = 60;

/* Initial interrupt moderation settings.  They can be modified after
 * module load with ethtool.
 *
 * This default is chosen to ensure that a 10G link does not go idle
 * while a TX queue is stopped after it has become full.  A queue is
 * restarted when it drops below half full.  The time this takes (assuming
 * worst case 3 descriptors per packet and 1024 descriptors) is
 *   512 / 3 * 1.2 = 205 usec.
 */
static unsigned int tx_irq_mod_usec = 150;

/* This is the first interrupt mode to try out of:
 * 0 => MSI-X
 * 1 => MSI
 * 2 => legacy
 */
static unsigned int interrupt_mode;

/* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
 * i.e. the number of CPUs among which we may distribute simultaneous
 * interrupt handling.
 *
 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
157
 * The default (0) means to assign an interrupt to each core.
158 159 160 161 162
 */
static unsigned int rss_cpus;
module_param(rss_cpus, uint, 0444);
MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");

163 164 165 166
static int phy_flash_cfg;
module_param(phy_flash_cfg, int, 0644);
MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");

167
static unsigned irq_adapt_low_thresh = 8000;
168 169 170 171
module_param(irq_adapt_low_thresh, uint, 0644);
MODULE_PARM_DESC(irq_adapt_low_thresh,
		 "Threshold score for reducing IRQ moderation");

172
static unsigned irq_adapt_high_thresh = 16000;
173 174 175 176
module_param(irq_adapt_high_thresh, uint, 0644);
MODULE_PARM_DESC(irq_adapt_high_thresh,
		 "Threshold score for increasing IRQ moderation");

177 178 179 180 181 182 183
static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
			 NETIF_MSG_LINK | NETIF_MSG_IFDOWN |
			 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR |
			 NETIF_MSG_TX_ERR | NETIF_MSG_HW);
module_param(debug, uint, 0);
MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value");

184 185 186 187 188
/**************************************************************************
 *
 * Utility functions and prototypes
 *
 *************************************************************************/
189

190 191 192
static void efx_start_interrupts(struct efx_nic *efx, bool may_keep_eventq);
static void efx_stop_interrupts(struct efx_nic *efx, bool may_keep_eventq);
static void efx_remove_channel(struct efx_channel *channel);
193
static void efx_remove_channels(struct efx_nic *efx);
194
static const struct efx_channel_type efx_default_channel_type;
195
static void efx_remove_port(struct efx_nic *efx);
196
static void efx_init_napi_channel(struct efx_channel *channel);
197
static void efx_fini_napi(struct efx_nic *efx);
198
static void efx_fini_napi_channel(struct efx_channel *channel);
199 200 201
static void efx_fini_struct(struct efx_nic *efx);
static void efx_start_all(struct efx_nic *efx);
static void efx_stop_all(struct efx_nic *efx);
202 203 204

#define EFX_ASSERT_RESET_SERIALISED(efx)		\
	do {						\
205
		if ((efx->state == STATE_READY) ||	\
206
		    (efx->state == STATE_DISABLED))	\
207 208 209
			ASSERT_RTNL();			\
	} while (0)

210 211 212 213 214 215 216 217 218 219
static int efx_check_disabled(struct efx_nic *efx)
{
	if (efx->state == STATE_DISABLED) {
		netif_err(efx, drv, efx->net_dev,
			  "device is disabled due to earlier errors\n");
		return -EIO;
	}
	return 0;
}

220 221 222 223 224 225 226 227 228 229 230 231 232
/**************************************************************************
 *
 * Event queue processing
 *
 *************************************************************************/

/* Process channel's event queue
 *
 * This function is responsible for processing the event queue of a
 * single channel.  The caller must guarantee that this function will
 * never be concurrently called more than once on the same channel,
 * though different channels may be being processed concurrently.
 */
233
static int efx_process_channel(struct efx_channel *channel, int budget)
234
{
235
	int spent;
236

237
	if (unlikely(!channel->enabled))
B
Ben Hutchings 已提交
238
		return 0;
239

240
	spent = efx_nic_process_eventq(channel, budget);
241 242 243 244 245 246 247 248 249
	if (spent && efx_channel_has_rx_queue(channel)) {
		struct efx_rx_queue *rx_queue =
			efx_channel_get_rx_queue(channel);

		/* Deliver last RX packet. */
		if (channel->rx_pkt) {
			__efx_rx_packet(channel, channel->rx_pkt);
			channel->rx_pkt = NULL;
		}
250 251 252 253
		if (rx_queue->enabled) {
			efx_rx_strategy(channel);
			efx_fast_push_rx_descriptors(rx_queue);
		}
254 255
	}

256
	return spent;
257 258 259 260 261 262 263 264 265 266
}

/* Mark channel as finished processing
 *
 * Note that since we will not receive further interrupts for this
 * channel before we finish processing and call the eventq_read_ack()
 * method, there is no need to use the interrupt hold-off timers.
 */
static inline void efx_channel_processed(struct efx_channel *channel)
{
267 268 269
	/* The interrupt handler for this channel may set work_pending
	 * as soon as we acknowledge the events we've seen.  Make sure
	 * it's cleared before then. */
270
	channel->work_pending = false;
271 272
	smp_wmb();

273
	efx_nic_eventq_read_ack(channel);
274 275 276 277 278 279 280 281 282 283 284
}

/* NAPI poll handler
 *
 * NAPI guarantees serialisation of polls of the same device, which
 * provides the guarantee required by efx_process_channel().
 */
static int efx_poll(struct napi_struct *napi, int budget)
{
	struct efx_channel *channel =
		container_of(napi, struct efx_channel, napi_str);
285
	struct efx_nic *efx = channel->efx;
286
	int spent;
287

288 289 290
	netif_vdbg(efx, intr, efx->net_dev,
		   "channel %d NAPI poll executing on CPU %d\n",
		   channel->channel, raw_smp_processor_id());
291

292
	spent = efx_process_channel(channel, budget);
293

294
	if (spent < budget) {
295
		if (efx_channel_has_rx_queue(channel) &&
296 297 298 299
		    efx->irq_rx_adaptive &&
		    unlikely(++channel->irq_count == 1000)) {
			if (unlikely(channel->irq_mod_score <
				     irq_adapt_low_thresh)) {
300 301
				if (channel->irq_moderation > 1) {
					channel->irq_moderation -= 1;
302
					efx->type->push_irq_moderation(channel);
303
				}
304 305
			} else if (unlikely(channel->irq_mod_score >
					    irq_adapt_high_thresh)) {
306 307 308
				if (channel->irq_moderation <
				    efx->irq_rx_moderation) {
					channel->irq_moderation += 1;
309
					efx->type->push_irq_moderation(channel);
310
				}
311 312 313 314 315
			}
			channel->irq_count = 0;
			channel->irq_mod_score = 0;
		}

316 317
		efx_filter_rfs_expire(channel);

318
		/* There is no race here; although napi_disable() will
319
		 * only wait for napi_complete(), this isn't a problem
320 321 322
		 * since efx_channel_processed() will have no effect if
		 * interrupts have already been disabled.
		 */
323
		napi_complete(napi);
324 325 326
		efx_channel_processed(channel);
	}

327
	return spent;
328 329 330 331 332 333 334 335
}

/* Process the eventq of the specified channel immediately on this CPU
 *
 * Disable hardware generated interrupts, wait for any existing
 * processing to finish, then directly poll (and ack ) the eventq.
 * Finally reenable NAPI and interrupts.
 *
336 337
 * This is for use only during a loopback self-test.  It must not
 * deliver any packets up the stack as this can result in deadlock.
338 339 340 341 342
 */
void efx_process_channel_now(struct efx_channel *channel)
{
	struct efx_nic *efx = channel->efx;

343
	BUG_ON(channel->channel >= efx->n_channels);
344
	BUG_ON(!channel->enabled);
345
	BUG_ON(!efx->loopback_selftest);
346 347

	/* Disable interrupts and wait for ISRs to complete */
348
	efx_nic_disable_interrupts(efx);
349
	if (efx->legacy_irq) {
350
		synchronize_irq(efx->legacy_irq);
351 352
		efx->legacy_irq_enabled = false;
	}
353
	if (channel->irq)
354 355 356 357 358 359
		synchronize_irq(channel->irq);

	/* Wait for any NAPI processing to complete */
	napi_disable(&channel->napi_str);

	/* Poll the channel */
360
	efx_process_channel(channel, channel->eventq_mask + 1);
361 362 363 364 365 366

	/* Ack the eventq. This may cause an interrupt to be generated
	 * when they are reenabled */
	efx_channel_processed(channel);

	napi_enable(&channel->napi_str);
367 368
	if (efx->legacy_irq)
		efx->legacy_irq_enabled = true;
369
	efx_nic_enable_interrupts(efx);
370 371 372 373 374 375 376 377 378
}

/* Create event queue
 * Event queue memory allocations are done only once.  If the channel
 * is reset, the memory buffer will be reused; this guards against
 * errors during channel reset and also simplifies interrupt handling.
 */
static int efx_probe_eventq(struct efx_channel *channel)
{
379 380 381
	struct efx_nic *efx = channel->efx;
	unsigned long entries;

382
	netif_dbg(efx, probe, efx->net_dev,
383
		  "chan %d create event queue\n", channel->channel);
384

385 386 387 388 389 390
	/* Build an event queue with room for one event per tx and rx buffer,
	 * plus some extra for link state events and MCDI completions. */
	entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
	EFX_BUG_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
	channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;

391
	return efx_nic_probe_eventq(channel);
392 393 394
}

/* Prepare channel's event queue */
395
static void efx_init_eventq(struct efx_channel *channel)
396
{
397 398
	netif_dbg(channel->efx, drv, channel->efx->net_dev,
		  "chan %d init event queue\n", channel->channel);
399 400 401

	channel->eventq_read_ptr = 0;

402
	efx_nic_init_eventq(channel);
403 404
}

405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
/* Enable event queue processing and NAPI */
static void efx_start_eventq(struct efx_channel *channel)
{
	netif_dbg(channel->efx, ifup, channel->efx->net_dev,
		  "chan %d start event queue\n", channel->channel);

	/* The interrupt handler for this channel may set work_pending
	 * as soon as we enable it.  Make sure it's cleared before
	 * then.  Similarly, make sure it sees the enabled flag set.
	 */
	channel->work_pending = false;
	channel->enabled = true;
	smp_wmb();

	napi_enable(&channel->napi_str);
	efx_nic_eventq_read_ack(channel);
}

/* Disable event queue processing and NAPI */
static void efx_stop_eventq(struct efx_channel *channel)
{
	if (!channel->enabled)
		return;

	napi_disable(&channel->napi_str);
	channel->enabled = false;
}

433 434
static void efx_fini_eventq(struct efx_channel *channel)
{
435 436
	netif_dbg(channel->efx, drv, channel->efx->net_dev,
		  "chan %d fini event queue\n", channel->channel);
437

438
	efx_nic_fini_eventq(channel);
439 440 441 442
}

static void efx_remove_eventq(struct efx_channel *channel)
{
443 444
	netif_dbg(channel->efx, drv, channel->efx->net_dev,
		  "chan %d remove event queue\n", channel->channel);
445

446
	efx_nic_remove_eventq(channel);
447 448 449 450 451 452 453 454
}

/**************************************************************************
 *
 * Channel handling
 *
 *************************************************************************/

455
/* Allocate and initialise a channel structure. */
456 457 458 459 460 461 462 463
static struct efx_channel *
efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel)
{
	struct efx_channel *channel;
	struct efx_rx_queue *rx_queue;
	struct efx_tx_queue *tx_queue;
	int j;

464 465 466
	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
	if (!channel)
		return NULL;
467

468 469 470
	channel->efx = efx;
	channel->channel = i;
	channel->type = &efx_default_channel_type;
471

472 473 474 475 476 477
	for (j = 0; j < EFX_TXQ_TYPES; j++) {
		tx_queue = &channel->tx_queue[j];
		tx_queue->efx = efx;
		tx_queue->queue = i * EFX_TXQ_TYPES + j;
		tx_queue->channel = channel;
	}
478

479 480 481 482
	rx_queue = &channel->rx_queue;
	rx_queue->efx = efx;
	setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill,
		    (unsigned long)rx_queue);
483

484 485 486 487 488 489 490 491 492 493 494 495 496
	return channel;
}

/* Allocate and initialise a channel structure, copying parameters
 * (but not resources) from an old channel structure.
 */
static struct efx_channel *
efx_copy_channel(const struct efx_channel *old_channel)
{
	struct efx_channel *channel;
	struct efx_rx_queue *rx_queue;
	struct efx_tx_queue *tx_queue;
	int j;
497

498 499 500 501 502 503 504 505
	channel = kmalloc(sizeof(*channel), GFP_KERNEL);
	if (!channel)
		return NULL;

	*channel = *old_channel;

	channel->napi_dev = NULL;
	memset(&channel->eventq, 0, sizeof(channel->eventq));
506

507 508 509
	for (j = 0; j < EFX_TXQ_TYPES; j++) {
		tx_queue = &channel->tx_queue[j];
		if (tx_queue->channel)
510
			tx_queue->channel = channel;
511 512
		tx_queue->buffer = NULL;
		memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
513 514 515
	}

	rx_queue = &channel->rx_queue;
516 517
	rx_queue->buffer = NULL;
	memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
518 519 520 521 522 523
	setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill,
		    (unsigned long)rx_queue);

	return channel;
}

524 525 526 527 528 529
static int efx_probe_channel(struct efx_channel *channel)
{
	struct efx_tx_queue *tx_queue;
	struct efx_rx_queue *rx_queue;
	int rc;

530 531
	netif_dbg(channel->efx, probe, channel->efx->net_dev,
		  "creating channel %d\n", channel->channel);
532

533 534 535 536
	rc = channel->type->pre_probe(channel);
	if (rc)
		goto fail;

537 538
	rc = efx_probe_eventq(channel);
	if (rc)
539
		goto fail;
540 541 542 543

	efx_for_each_channel_tx_queue(tx_queue, channel) {
		rc = efx_probe_tx_queue(tx_queue);
		if (rc)
544
			goto fail;
545 546 547 548 549
	}

	efx_for_each_channel_rx_queue(rx_queue, channel) {
		rc = efx_probe_rx_queue(rx_queue);
		if (rc)
550
			goto fail;
551 552 553 554 555 556
	}

	channel->n_rx_frm_trunc = 0;

	return 0;

557 558
fail:
	efx_remove_channel(channel);
559 560 561
	return rc;
}

562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579
static void
efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len)
{
	struct efx_nic *efx = channel->efx;
	const char *type;
	int number;

	number = channel->channel;
	if (efx->tx_channel_offset == 0) {
		type = "";
	} else if (channel->channel < efx->tx_channel_offset) {
		type = "-rx";
	} else {
		type = "-tx";
		number -= efx->tx_channel_offset;
	}
	snprintf(buf, len, "%s%s-%d", efx->name, type, number);
}
580

581 582 583 584
static void efx_set_channel_names(struct efx_nic *efx)
{
	struct efx_channel *channel;

585 586 587 588
	efx_for_each_channel(channel, efx)
		channel->type->get_name(channel,
					efx->channel_name[channel->channel],
					sizeof(efx->channel_name[0]));
589 590
}

591 592 593 594 595 596 597 598
static int efx_probe_channels(struct efx_nic *efx)
{
	struct efx_channel *channel;
	int rc;

	/* Restart special buffer allocation */
	efx->next_buffer_table = 0;

599 600 601 602 603 604
	/* Probe channels in reverse, so that any 'extra' channels
	 * use the start of the buffer table. This allows the traffic
	 * channels to be resized without moving them or wasting the
	 * entries before them.
	 */
	efx_for_each_channel_rev(channel, efx) {
605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
		rc = efx_probe_channel(channel);
		if (rc) {
			netif_err(efx, probe, efx->net_dev,
				  "failed to create channel %d\n",
				  channel->channel);
			goto fail;
		}
	}
	efx_set_channel_names(efx);

	return 0;

fail:
	efx_remove_channels(efx);
	return rc;
}

622 623 624 625
/* Channels are shutdown and reinitialised whilst the NIC is running
 * to propagate configuration changes (mtu, checksum offload), or
 * to clear hardware error conditions
 */
626
static void efx_start_datapath(struct efx_nic *efx)
627 628 629 630 631
{
	struct efx_tx_queue *tx_queue;
	struct efx_rx_queue *rx_queue;
	struct efx_channel *channel;

632 633 634 635 636 637
	/* Calculate the rx buffer allocation parameters required to
	 * support the current MTU, including padding for header
	 * alignment and overruns.
	 */
	efx->rx_buffer_len = (max(EFX_PAGE_IP_ALIGN, NET_IP_ALIGN) +
			      EFX_MAX_FRAME_LEN(efx->net_dev->mtu) +
638
			      efx->type->rx_buffer_hash_size +
639
			      efx->type->rx_buffer_padding);
640 641
	efx->rx_buffer_order = get_order(efx->rx_buffer_len +
					 sizeof(struct efx_rx_page_state));
642

643 644 645 646 647 648 649 650 651 652
	/* We must keep at least one descriptor in a TX ring empty.
	 * We could avoid this when the queue size does not exactly
	 * match the hardware ring size, but it's not that important.
	 * Therefore we stop the queue when one more skb might fill
	 * the ring completely.  We wake it when half way back to
	 * empty.
	 */
	efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx);
	efx->txq_wake_thresh = efx->txq_stop_thresh / 2;

653 654
	/* Initialise the channels */
	efx_for_each_channel(channel, efx) {
655 656
		efx_for_each_channel_tx_queue(tx_queue, channel)
			efx_init_tx_queue(tx_queue);
657 658 659 660

		/* The rx buffer allocation strategy is MTU dependent */
		efx_rx_strategy(channel);

661
		efx_for_each_channel_rx_queue(rx_queue, channel) {
662
			efx_init_rx_queue(rx_queue);
663 664
			efx_nic_generate_fill_event(rx_queue);
		}
665 666 667 668 669

		WARN_ON(channel->rx_pkt != NULL);
		efx_rx_strategy(channel);
	}

670 671
	if (netif_device_present(efx->net_dev))
		netif_tx_wake_all_queues(efx->net_dev);
672 673
}

674
static void efx_stop_datapath(struct efx_nic *efx)
675 676 677 678
{
	struct efx_channel *channel;
	struct efx_tx_queue *tx_queue;
	struct efx_rx_queue *rx_queue;
679
	struct pci_dev *dev = efx->pci_dev;
680
	int rc;
681 682 683 684

	EFX_ASSERT_RESET_SERIALISED(efx);
	BUG_ON(efx->port_enabled);

685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702
	/* Only perform flush if dma is enabled */
	if (dev->is_busmaster) {
		rc = efx_nic_flush_queues(efx);

		if (rc && EFX_WORKAROUND_7803(efx)) {
			/* Schedule a reset to recover from the flush failure. The
			 * descriptor caches reference memory we're about to free,
			 * but falcon_reconfigure_mac_wrapper() won't reconnect
			 * the MACs because of the pending reset. */
			netif_err(efx, drv, efx->net_dev,
				  "Resetting to recover from flush failure\n");
			efx_schedule_reset(efx, RESET_TYPE_ALL);
		} else if (rc) {
			netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
		} else {
			netif_dbg(efx, drv, efx->net_dev,
				  "successfully flushed all queues\n");
		}
703
	}
704

705
	efx_for_each_channel(channel, efx) {
706 707 708 709 710 711 712 713 714 715
		/* RX packet processing is pipelined, so wait for the
		 * NAPI handler to complete.  At least event queue 0
		 * might be kept active by non-data events, so don't
		 * use napi_synchronize() but actually disable NAPI
		 * temporarily.
		 */
		if (efx_channel_has_rx_queue(channel)) {
			efx_stop_eventq(channel);
			efx_start_eventq(channel);
		}
716 717 718

		efx_for_each_channel_rx_queue(rx_queue, channel)
			efx_fini_rx_queue(rx_queue);
719
		efx_for_each_possible_channel_tx_queue(tx_queue, channel)
720 721 722 723 724 725 726 727 728
			efx_fini_tx_queue(tx_queue);
	}
}

static void efx_remove_channel(struct efx_channel *channel)
{
	struct efx_tx_queue *tx_queue;
	struct efx_rx_queue *rx_queue;

729 730
	netif_dbg(channel->efx, drv, channel->efx->net_dev,
		  "destroy chan %d\n", channel->channel);
731 732 733

	efx_for_each_channel_rx_queue(rx_queue, channel)
		efx_remove_rx_queue(rx_queue);
734
	efx_for_each_possible_channel_tx_queue(tx_queue, channel)
735 736 737 738
		efx_remove_tx_queue(tx_queue);
	efx_remove_eventq(channel);
}

739 740 741 742 743 744 745 746 747 748 749 750 751
static void efx_remove_channels(struct efx_nic *efx)
{
	struct efx_channel *channel;

	efx_for_each_channel(channel, efx)
		efx_remove_channel(channel);
}

int
efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
{
	struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;
	u32 old_rxq_entries, old_txq_entries;
752
	unsigned i, next_buffer_table = 0;
753 754 755 756 757
	int rc;

	rc = efx_check_disabled(efx);
	if (rc)
		return rc;
758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779

	/* Not all channels should be reallocated. We must avoid
	 * reallocating their buffer table entries.
	 */
	efx_for_each_channel(channel, efx) {
		struct efx_rx_queue *rx_queue;
		struct efx_tx_queue *tx_queue;

		if (channel->type->copy)
			continue;
		next_buffer_table = max(next_buffer_table,
					channel->eventq.index +
					channel->eventq.entries);
		efx_for_each_channel_rx_queue(rx_queue, channel)
			next_buffer_table = max(next_buffer_table,
						rx_queue->rxd.index +
						rx_queue->rxd.entries);
		efx_for_each_channel_tx_queue(tx_queue, channel)
			next_buffer_table = max(next_buffer_table,
						tx_queue->txd.index +
						tx_queue->txd.entries);
	}
780 781

	efx_stop_all(efx);
782
	efx_stop_interrupts(efx, true);
783

784
	/* Clone channels (where possible) */
785 786
	memset(other_channel, 0, sizeof(other_channel));
	for (i = 0; i < efx->n_channels; i++) {
787 788 789
		channel = efx->channel[i];
		if (channel->type->copy)
			channel = channel->type->copy(channel);
790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807
		if (!channel) {
			rc = -ENOMEM;
			goto out;
		}
		other_channel[i] = channel;
	}

	/* Swap entry counts and channel pointers */
	old_rxq_entries = efx->rxq_entries;
	old_txq_entries = efx->txq_entries;
	efx->rxq_entries = rxq_entries;
	efx->txq_entries = txq_entries;
	for (i = 0; i < efx->n_channels; i++) {
		channel = efx->channel[i];
		efx->channel[i] = other_channel[i];
		other_channel[i] = channel;
	}

808 809
	/* Restart buffer table allocation */
	efx->next_buffer_table = next_buffer_table;
810 811

	for (i = 0; i < efx->n_channels; i++) {
812 813 814 815 816 817 818
		channel = efx->channel[i];
		if (!channel->type->copy)
			continue;
		rc = efx_probe_channel(channel);
		if (rc)
			goto rollback;
		efx_init_napi_channel(efx->channel[i]);
819
	}
820

821
out:
822 823 824 825 826 827 828 829 830
	/* Destroy unused channel structures */
	for (i = 0; i < efx->n_channels; i++) {
		channel = other_channel[i];
		if (channel && channel->type->copy) {
			efx_fini_napi_channel(channel);
			efx_remove_channel(channel);
			kfree(channel);
		}
	}
831

832
	efx_start_interrupts(efx, true);
833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
	efx_start_all(efx);
	return rc;

rollback:
	/* Swap back */
	efx->rxq_entries = old_rxq_entries;
	efx->txq_entries = old_txq_entries;
	for (i = 0; i < efx->n_channels; i++) {
		channel = efx->channel[i];
		efx->channel[i] = other_channel[i];
		other_channel[i] = channel;
	}
	goto out;
}

848
void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)
849
{
850
	mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100));
851 852
}

853 854 855 856 857 858 859 860 861 862 863 864
static const struct efx_channel_type efx_default_channel_type = {
	.pre_probe		= efx_channel_dummy_op_int,
	.get_name		= efx_get_channel_name,
	.copy			= efx_copy_channel,
	.keep_eventq		= false,
};

int efx_channel_dummy_op_int(struct efx_channel *channel)
{
	return 0;
}

865 866 867 868 869 870 871 872 873 874
/**************************************************************************
 *
 * Port handling
 *
 **************************************************************************/

/* This ensures that the kernel is kept informed (via
 * netif_carrier_on/off) of the link status, and also maintains the
 * link status's stop on the port's TX queue.
 */
S
Steve Hodgson 已提交
875
void efx_link_status_changed(struct efx_nic *efx)
876
{
877 878
	struct efx_link_state *link_state = &efx->link_state;

879 880 881 882 883 884 885
	/* SFC Bug 5356: A net_dev notifier is registered, so we must ensure
	 * that no events are triggered between unregister_netdev() and the
	 * driver unloading. A more general condition is that NETDEV_CHANGE
	 * can only be generated between NETDEV_UP and NETDEV_DOWN */
	if (!netif_running(efx->net_dev))
		return;

886
	if (link_state->up != netif_carrier_ok(efx->net_dev)) {
887 888
		efx->n_link_state_changes++;

889
		if (link_state->up)
890 891 892 893 894 895
			netif_carrier_on(efx->net_dev);
		else
			netif_carrier_off(efx->net_dev);
	}

	/* Status message for kernel log */
B
Ben Hutchings 已提交
896
	if (link_state->up)
897 898 899 900 901
		netif_info(efx, link, efx->net_dev,
			   "link up at %uMbps %s-duplex (MTU %d)%s\n",
			   link_state->speed, link_state->fd ? "full" : "half",
			   efx->net_dev->mtu,
			   (efx->promiscuous ? " [PROMISC]" : ""));
B
Ben Hutchings 已提交
902
	else
903
		netif_info(efx, link, efx->net_dev, "link down\n");
904 905
}

B
Ben Hutchings 已提交
906 907 908 909 910 911 912 913 914 915 916 917 918
void efx_link_set_advertising(struct efx_nic *efx, u32 advertising)
{
	efx->link_advertising = advertising;
	if (advertising) {
		if (advertising & ADVERTISED_Pause)
			efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX);
		else
			efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
		if (advertising & ADVERTISED_Asym_Pause)
			efx->wanted_fc ^= EFX_FC_TX;
	}
}

919
void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc)
B
Ben Hutchings 已提交
920 921 922 923 924 925 926 927 928 929 930 931 932 933
{
	efx->wanted_fc = wanted_fc;
	if (efx->link_advertising) {
		if (wanted_fc & EFX_FC_RX)
			efx->link_advertising |= (ADVERTISED_Pause |
						  ADVERTISED_Asym_Pause);
		else
			efx->link_advertising &= ~(ADVERTISED_Pause |
						   ADVERTISED_Asym_Pause);
		if (wanted_fc & EFX_FC_TX)
			efx->link_advertising ^= ADVERTISED_Asym_Pause;
	}
}

934 935
static void efx_fini_port(struct efx_nic *efx);

B
Ben Hutchings 已提交
936 937 938 939 940 941 942 943
/* Push loopback/power/transmit disable settings to the PHY, and reconfigure
 * the MAC appropriately. All other PHY configuration changes are pushed
 * through phy_op->set_settings(), and pushed asynchronously to the MAC
 * through efx_monitor().
 *
 * Callers must hold the mac_lock
 */
int __efx_reconfigure_port(struct efx_nic *efx)
944
{
B
Ben Hutchings 已提交
945 946
	enum efx_phy_mode phy_mode;
	int rc;
947

B
Ben Hutchings 已提交
948
	WARN_ON(!mutex_is_locked(&efx->mac_lock));
949

950
	/* Serialise the promiscuous flag with efx_set_rx_mode. */
951 952
	netif_addr_lock_bh(efx->net_dev);
	netif_addr_unlock_bh(efx->net_dev);
953

B
Ben Hutchings 已提交
954 955
	/* Disable PHY transmit in mac level loopbacks */
	phy_mode = efx->phy_mode;
956 957 958 959 960
	if (LOOPBACK_INTERNAL(efx))
		efx->phy_mode |= PHY_MODE_TX_DISABLED;
	else
		efx->phy_mode &= ~PHY_MODE_TX_DISABLED;

B
Ben Hutchings 已提交
961
	rc = efx->type->reconfigure_port(efx);
962

B
Ben Hutchings 已提交
963 964
	if (rc)
		efx->phy_mode = phy_mode;
965

B
Ben Hutchings 已提交
966
	return rc;
967 968 969 970
}

/* Reinitialise the MAC to pick up new PHY settings, even if the port is
 * disabled. */
B
Ben Hutchings 已提交
971
int efx_reconfigure_port(struct efx_nic *efx)
972
{
B
Ben Hutchings 已提交
973 974
	int rc;

975 976 977
	EFX_ASSERT_RESET_SERIALISED(efx);

	mutex_lock(&efx->mac_lock);
B
Ben Hutchings 已提交
978
	rc = __efx_reconfigure_port(efx);
979
	mutex_unlock(&efx->mac_lock);
B
Ben Hutchings 已提交
980 981

	return rc;
982 983
}

984 985 986
/* Asynchronous work item for changing MAC promiscuity and multicast
 * hash.  Avoid a drain/rx_ingress enable by reconfiguring the current
 * MAC directly. */
987 988 989 990 991
static void efx_mac_work(struct work_struct *data)
{
	struct efx_nic *efx = container_of(data, struct efx_nic, mac_work);

	mutex_lock(&efx->mac_lock);
992
	if (efx->port_enabled)
993
		efx->type->reconfigure_mac(efx);
994 995 996
	mutex_unlock(&efx->mac_lock);
}

997 998 999 1000
static int efx_probe_port(struct efx_nic *efx)
{
	int rc;

1001
	netif_dbg(efx, probe, efx->net_dev, "create port\n");
1002

1003 1004 1005
	if (phy_flash_cfg)
		efx->phy_mode = PHY_MODE_SPECIAL;

1006 1007
	/* Connect up MAC/PHY operations table */
	rc = efx->type->probe_port(efx);
1008
	if (rc)
1009
		return rc;
1010

1011 1012
	/* Initialise MAC address to permanent address */
	memcpy(efx->net_dev->dev_addr, efx->net_dev->perm_addr, ETH_ALEN);
1013 1014 1015 1016 1017 1018 1019 1020

	return 0;
}

static int efx_init_port(struct efx_nic *efx)
{
	int rc;

1021
	netif_dbg(efx, drv, efx->net_dev, "init port\n");
1022

1023 1024
	mutex_lock(&efx->mac_lock);

1025
	rc = efx->phy_op->init(efx);
1026
	if (rc)
1027
		goto fail1;
1028

1029
	efx->port_initialized = true;
1030

B
Ben Hutchings 已提交
1031 1032
	/* Reconfigure the MAC before creating dma queues (required for
	 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */
1033
	efx->type->reconfigure_mac(efx);
B
Ben Hutchings 已提交
1034 1035 1036 1037 1038 1039

	/* Ensure the PHY advertises the correct flow control settings */
	rc = efx->phy_op->reconfigure(efx);
	if (rc)
		goto fail2;

1040
	mutex_unlock(&efx->mac_lock);
1041
	return 0;
1042

1043
fail2:
1044
	efx->phy_op->fini(efx);
1045 1046
fail1:
	mutex_unlock(&efx->mac_lock);
1047
	return rc;
1048 1049 1050 1051
}

static void efx_start_port(struct efx_nic *efx)
{
1052
	netif_dbg(efx, ifup, efx->net_dev, "start port\n");
1053 1054 1055
	BUG_ON(efx->port_enabled);

	mutex_lock(&efx->mac_lock);
1056
	efx->port_enabled = true;
1057 1058 1059

	/* efx_mac_work() might have been scheduled after efx_stop_port(),
	 * and then cancelled by efx_flush_all() */
1060
	efx->type->reconfigure_mac(efx);
1061

1062 1063 1064
	mutex_unlock(&efx->mac_lock);
}

S
Steve Hodgson 已提交
1065
/* Prevent efx_mac_work() and efx_monitor() from working */
1066 1067
static void efx_stop_port(struct efx_nic *efx)
{
1068
	netif_dbg(efx, ifdown, efx->net_dev, "stop port\n");
1069 1070

	mutex_lock(&efx->mac_lock);
1071
	efx->port_enabled = false;
1072 1073 1074
	mutex_unlock(&efx->mac_lock);

	/* Serialise against efx_set_multicast_list() */
1075 1076
	netif_addr_lock_bh(efx->net_dev);
	netif_addr_unlock_bh(efx->net_dev);
1077 1078 1079 1080
}

static void efx_fini_port(struct efx_nic *efx)
{
1081
	netif_dbg(efx, drv, efx->net_dev, "shut down port\n");
1082 1083 1084 1085

	if (!efx->port_initialized)
		return;

1086
	efx->phy_op->fini(efx);
1087
	efx->port_initialized = false;
1088

1089
	efx->link_state.up = false;
1090 1091 1092 1093 1094
	efx_link_status_changed(efx);
}

static void efx_remove_port(struct efx_nic *efx)
{
1095
	netif_dbg(efx, drv, efx->net_dev, "destroying port\n");
1096

1097
	efx->type->remove_port(efx);
1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
}

/**************************************************************************
 *
 * NIC handling
 *
 **************************************************************************/

/* This configures the PCI device to enable I/O and DMA. */
static int efx_init_io(struct efx_nic *efx)
{
	struct pci_dev *pci_dev = efx->pci_dev;
	dma_addr_t dma_mask = efx->type->max_dma_mask;
	int rc;

1113
	netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n");
1114 1115 1116

	rc = pci_enable_device(pci_dev);
	if (rc) {
1117 1118
		netif_err(efx, probe, efx->net_dev,
			  "failed to enable PCI device\n");
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
		goto fail1;
	}

	pci_set_master(pci_dev);

	/* Set the PCI DMA mask.  Try all possibilities from our
	 * genuine mask down to 32 bits, because some architectures
	 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit
	 * masks event though they reject 46 bit masks.
	 */
	while (dma_mask > 0x7fffffffUL) {
1130 1131
		if (dma_supported(&pci_dev->dev, dma_mask)) {
			rc = dma_set_mask(&pci_dev->dev, dma_mask);
1132 1133 1134
			if (rc == 0)
				break;
		}
1135 1136 1137
		dma_mask >>= 1;
	}
	if (rc) {
1138 1139
		netif_err(efx, probe, efx->net_dev,
			  "could not find a suitable DMA mask\n");
1140 1141
		goto fail2;
	}
1142 1143
	netif_dbg(efx, probe, efx->net_dev,
		  "using DMA mask %llx\n", (unsigned long long) dma_mask);
1144
	rc = dma_set_coherent_mask(&pci_dev->dev, dma_mask);
1145
	if (rc) {
1146 1147
		/* dma_set_coherent_mask() is not *allowed* to
		 * fail with a mask that dma_set_mask() accepted,
1148 1149
		 * but just in case...
		 */
1150 1151
		netif_err(efx, probe, efx->net_dev,
			  "failed to set consistent DMA mask\n");
1152 1153 1154
		goto fail2;
	}

1155 1156
	efx->membase_phys = pci_resource_start(efx->pci_dev, EFX_MEM_BAR);
	rc = pci_request_region(pci_dev, EFX_MEM_BAR, "sfc");
1157
	if (rc) {
1158 1159
		netif_err(efx, probe, efx->net_dev,
			  "request for memory BAR failed\n");
1160 1161 1162
		rc = -EIO;
		goto fail3;
	}
1163 1164
	efx->membase = ioremap_nocache(efx->membase_phys,
				       efx->type->mem_map_size);
1165
	if (!efx->membase) {
1166 1167 1168 1169
		netif_err(efx, probe, efx->net_dev,
			  "could not map memory BAR at %llx+%x\n",
			  (unsigned long long)efx->membase_phys,
			  efx->type->mem_map_size);
1170 1171 1172
		rc = -ENOMEM;
		goto fail4;
	}
1173 1174 1175 1176
	netif_dbg(efx, probe, efx->net_dev,
		  "memory BAR at %llx+%x (virtual %p)\n",
		  (unsigned long long)efx->membase_phys,
		  efx->type->mem_map_size, efx->membase);
1177 1178 1179 1180

	return 0;

 fail4:
1181
	pci_release_region(efx->pci_dev, EFX_MEM_BAR);
1182
 fail3:
1183
	efx->membase_phys = 0;
1184 1185 1186 1187 1188 1189 1190 1191
 fail2:
	pci_disable_device(efx->pci_dev);
 fail1:
	return rc;
}

static void efx_fini_io(struct efx_nic *efx)
{
1192
	netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n");
1193 1194 1195 1196 1197 1198 1199

	if (efx->membase) {
		iounmap(efx->membase);
		efx->membase = NULL;
	}

	if (efx->membase_phys) {
1200
		pci_release_region(efx->pci_dev, EFX_MEM_BAR);
1201
		efx->membase_phys = 0;
1202 1203 1204 1205 1206
	}

	pci_disable_device(efx->pci_dev);
}

1207
static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
1208
{
1209
	cpumask_var_t thread_mask;
1210
	unsigned int count;
1211
	int cpu;
1212

1213 1214 1215 1216 1217 1218 1219 1220
	if (rss_cpus) {
		count = rss_cpus;
	} else {
		if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) {
			netif_warn(efx, probe, efx->net_dev,
				   "RSS disabled due to allocation failure\n");
			return 1;
		}
1221

1222 1223 1224 1225 1226 1227 1228 1229 1230 1231
		count = 0;
		for_each_online_cpu(cpu) {
			if (!cpumask_test_cpu(cpu, thread_mask)) {
				++count;
				cpumask_or(thread_mask, thread_mask,
					   topology_thread_cpumask(cpu));
			}
		}

		free_cpumask_var(thread_mask);
R
Rusty Russell 已提交
1232 1233
	}

1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
	/* If RSS is requested for the PF *and* VFs then we can't write RSS
	 * table entries that are inaccessible to VFs
	 */
	if (efx_sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
	    count > efx_vf_size(efx)) {
		netif_warn(efx, probe, efx->net_dev,
			   "Reducing number of RSS channels from %u to %u for "
			   "VF support. Increase vf-msix-limit to use more "
			   "channels on the PF.\n",
			   count, efx_vf_size(efx));
		count = efx_vf_size(efx);
1245 1246 1247 1248 1249
	}

	return count;
}

1250 1251 1252 1253
static int
efx_init_rx_cpu_rmap(struct efx_nic *efx, struct msix_entry *xentries)
{
#ifdef CONFIG_RFS_ACCEL
1254 1255
	unsigned int i;
	int rc;
1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272

	efx->net_dev->rx_cpu_rmap = alloc_irq_cpu_rmap(efx->n_rx_channels);
	if (!efx->net_dev->rx_cpu_rmap)
		return -ENOMEM;
	for (i = 0; i < efx->n_rx_channels; i++) {
		rc = irq_cpu_rmap_add(efx->net_dev->rx_cpu_rmap,
				      xentries[i].vector);
		if (rc) {
			free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
			efx->net_dev->rx_cpu_rmap = NULL;
			return rc;
		}
	}
#endif
	return 0;
}

1273 1274 1275
/* Probe the number and type of interrupts we are able to obtain, and
 * the resulting numbers of channels and RX queues.
 */
1276
static int efx_probe_interrupts(struct efx_nic *efx)
1277
{
1278 1279
	unsigned int max_channels =
		min(efx->type->phys_addr_channels, EFX_MAX_CHANNELS);
1280 1281
	unsigned int extra_channels = 0;
	unsigned int i, j;
1282
	int rc;
1283

1284 1285 1286 1287
	for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
		if (efx->extra_channel_type[i])
			++extra_channels;

1288
	if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
1289
		struct msix_entry xentries[EFX_MAX_CHANNELS];
1290
		unsigned int n_channels;
1291

1292
		n_channels = efx_wanted_parallelism(efx);
B
Ben Hutchings 已提交
1293 1294
		if (separate_tx_channels)
			n_channels *= 2;
1295
		n_channels += extra_channels;
B
Ben Hutchings 已提交
1296
		n_channels = min(n_channels, max_channels);
1297

B
Ben Hutchings 已提交
1298
		for (i = 0; i < n_channels; i++)
1299
			xentries[i].entry = i;
B
Ben Hutchings 已提交
1300
		rc = pci_enable_msix(efx->pci_dev, xentries, n_channels);
1301
		if (rc > 0) {
1302 1303
			netif_err(efx, drv, efx->net_dev,
				  "WARNING: Insufficient MSI-X vectors"
1304
				  " available (%d < %u).\n", rc, n_channels);
1305 1306
			netif_err(efx, drv, efx->net_dev,
				  "WARNING: Performance may be reduced.\n");
B
Ben Hutchings 已提交
1307 1308
			EFX_BUG_ON_PARANOID(rc >= n_channels);
			n_channels = rc;
1309
			rc = pci_enable_msix(efx->pci_dev, xentries,
B
Ben Hutchings 已提交
1310
					     n_channels);
1311 1312 1313
		}

		if (rc == 0) {
B
Ben Hutchings 已提交
1314
			efx->n_channels = n_channels;
1315 1316
			if (n_channels > extra_channels)
				n_channels -= extra_channels;
B
Ben Hutchings 已提交
1317
			if (separate_tx_channels) {
1318 1319 1320 1321
				efx->n_tx_channels = max(n_channels / 2, 1U);
				efx->n_rx_channels = max(n_channels -
							 efx->n_tx_channels,
							 1U);
B
Ben Hutchings 已提交
1322
			} else {
1323 1324
				efx->n_tx_channels = n_channels;
				efx->n_rx_channels = n_channels;
B
Ben Hutchings 已提交
1325
			}
1326 1327 1328 1329 1330
			rc = efx_init_rx_cpu_rmap(efx, xentries);
			if (rc) {
				pci_disable_msix(efx->pci_dev);
				return rc;
			}
1331
			for (i = 0; i < efx->n_channels; i++)
1332 1333
				efx_get_channel(efx, i)->irq =
					xentries[i].vector;
1334 1335 1336
		} else {
			/* Fall back to single channel MSI */
			efx->interrupt_mode = EFX_INT_MODE_MSI;
1337 1338
			netif_err(efx, drv, efx->net_dev,
				  "could not enable MSI-X\n");
1339 1340 1341 1342 1343
		}
	}

	/* Try single interrupt MSI */
	if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
1344
		efx->n_channels = 1;
B
Ben Hutchings 已提交
1345 1346
		efx->n_rx_channels = 1;
		efx->n_tx_channels = 1;
1347 1348
		rc = pci_enable_msi(efx->pci_dev);
		if (rc == 0) {
1349
			efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
1350
		} else {
1351 1352
			netif_err(efx, drv, efx->net_dev,
				  "could not enable MSI\n");
1353 1354 1355 1356 1357 1358
			efx->interrupt_mode = EFX_INT_MODE_LEGACY;
		}
	}

	/* Assume legacy interrupts */
	if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
1359
		efx->n_channels = 1 + (separate_tx_channels ? 1 : 0);
B
Ben Hutchings 已提交
1360 1361
		efx->n_rx_channels = 1;
		efx->n_tx_channels = 1;
1362 1363
		efx->legacy_irq = efx->pci_dev->irq;
	}
1364

1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379
	/* Assign extra channels if possible */
	j = efx->n_channels;
	for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
		if (!efx->extra_channel_type[i])
			continue;
		if (efx->interrupt_mode != EFX_INT_MODE_MSIX ||
		    efx->n_channels <= extra_channels) {
			efx->extra_channel_type[i]->handle_no_channel(efx);
		} else {
			--j;
			efx_get_channel(efx, j)->type =
				efx->extra_channel_type[i];
		}
	}

1380
	/* RSS might be usable on VFs even if it is disabled on the PF */
1381
	efx->rss_spread = ((efx->n_rx_channels > 1 || !efx_sriov_wanted(efx)) ?
1382 1383
			   efx->n_rx_channels : efx_vf_size(efx));

1384
	return 0;
1385 1386
}

1387
/* Enable interrupts, then probe and start the event queues */
1388
static void efx_start_interrupts(struct efx_nic *efx, bool may_keep_eventq)
1389 1390 1391
{
	struct efx_channel *channel;

1392 1393
	BUG_ON(efx->state == STATE_DISABLED);

1394 1395 1396 1397 1398
	if (efx->legacy_irq)
		efx->legacy_irq_enabled = true;
	efx_nic_enable_interrupts(efx);

	efx_for_each_channel(channel, efx) {
1399 1400
		if (!channel->type->keep_eventq || !may_keep_eventq)
			efx_init_eventq(channel);
1401 1402 1403 1404 1405 1406
		efx_start_eventq(channel);
	}

	efx_mcdi_mode_event(efx);
}

1407
static void efx_stop_interrupts(struct efx_nic *efx, bool may_keep_eventq)
1408 1409 1410
{
	struct efx_channel *channel;

1411 1412 1413
	if (efx->state == STATE_DISABLED)
		return;

1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426
	efx_mcdi_mode_poll(efx);

	efx_nic_disable_interrupts(efx);
	if (efx->legacy_irq) {
		synchronize_irq(efx->legacy_irq);
		efx->legacy_irq_enabled = false;
	}

	efx_for_each_channel(channel, efx) {
		if (channel->irq)
			synchronize_irq(channel->irq);

		efx_stop_eventq(channel);
1427 1428
		if (!channel->type->keep_eventq || !may_keep_eventq)
			efx_fini_eventq(channel);
1429 1430 1431
	}
}

1432 1433 1434 1435 1436
static void efx_remove_interrupts(struct efx_nic *efx)
{
	struct efx_channel *channel;

	/* Remove MSI/MSI-X interrupts */
1437
	efx_for_each_channel(channel, efx)
1438 1439 1440 1441 1442 1443 1444 1445
		channel->irq = 0;
	pci_disable_msi(efx->pci_dev);
	pci_disable_msix(efx->pci_dev);

	/* Remove legacy interrupt */
	efx->legacy_irq = 0;
}

1446
static void efx_set_channels(struct efx_nic *efx)
1447
{
1448 1449 1450
	struct efx_channel *channel;
	struct efx_tx_queue *tx_queue;

1451
	efx->tx_channel_offset =
B
Ben Hutchings 已提交
1452
		separate_tx_channels ? efx->n_channels - efx->n_tx_channels : 0;
1453 1454 1455 1456 1457 1458 1459 1460 1461

	/* We need to adjust the TX queue numbers if we have separate
	 * RX-only and TX-only channels.
	 */
	efx_for_each_channel(channel, efx) {
		efx_for_each_channel_tx_queue(tx_queue, channel)
			tx_queue->queue -= (efx->tx_channel_offset *
					    EFX_TXQ_TYPES);
	}
1462 1463 1464 1465
}

static int efx_probe_nic(struct efx_nic *efx)
{
1466
	size_t i;
1467 1468
	int rc;

1469
	netif_dbg(efx, probe, efx->net_dev, "creating NIC\n");
1470 1471

	/* Carry out hardware-type specific initialisation */
1472
	rc = efx->type->probe(efx);
1473 1474 1475
	if (rc)
		return rc;

B
Ben Hutchings 已提交
1476
	/* Determine the number of channels and queues by trying to hook
1477
	 * in MSI-X interrupts. */
1478 1479 1480
	rc = efx_probe_interrupts(efx);
	if (rc)
		goto fail;
1481

1482 1483
	efx->type->dimension_resources(efx);

1484 1485
	if (efx->n_channels > 1)
		get_random_bytes(&efx->rx_hash_key, sizeof(efx->rx_hash_key));
1486
	for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++)
1487
		efx->rx_indir_table[i] =
1488
			ethtool_rxfh_indir_default(i, efx->rss_spread);
1489

1490
	efx_set_channels(efx);
1491 1492
	netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
	netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
1493 1494

	/* Initialise the interrupt moderation settings */
1495 1496
	efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true,
				true);
1497 1498

	return 0;
1499 1500 1501 1502

fail:
	efx->type->remove(efx);
	return rc;
1503 1504 1505 1506
}

static void efx_remove_nic(struct efx_nic *efx)
{
1507
	netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n");
1508 1509

	efx_remove_interrupts(efx);
1510
	efx->type->remove(efx);
1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524
}

/**************************************************************************
 *
 * NIC startup/shutdown
 *
 *************************************************************************/

static int efx_probe_all(struct efx_nic *efx)
{
	int rc;

	rc = efx_probe_nic(efx);
	if (rc) {
1525
		netif_err(efx, probe, efx->net_dev, "failed to create NIC\n");
1526 1527 1528 1529 1530
		goto fail1;
	}

	rc = efx_probe_port(efx);
	if (rc) {
1531
		netif_err(efx, probe, efx->net_dev, "failed to create port\n");
1532 1533 1534
		goto fail2;
	}

1535 1536 1537 1538 1539
	BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT);
	if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) {
		rc = -EINVAL;
		goto fail3;
	}
1540
	efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE;
1541

B
Ben Hutchings 已提交
1542 1543 1544 1545
	rc = efx_probe_filters(efx);
	if (rc) {
		netif_err(efx, probe, efx->net_dev,
			  "failed to create filter tables\n");
1546
		goto fail3;
B
Ben Hutchings 已提交
1547 1548
	}

1549 1550 1551 1552
	rc = efx_probe_channels(efx);
	if (rc)
		goto fail4;

1553 1554
	return 0;

B
Ben Hutchings 已提交
1555
 fail4:
1556
	efx_remove_filters(efx);
1557 1558 1559 1560 1561 1562 1563 1564
 fail3:
	efx_remove_port(efx);
 fail2:
	efx_remove_nic(efx);
 fail1:
	return rc;
}

1565 1566 1567 1568 1569 1570
/* If the interface is supposed to be running but is not, start
 * the hardware and software data path, regular activity for the port
 * (MAC statistics, link polling, etc.) and schedule the port to be
 * reconfigured.  Interrupts must already be enabled.  This function
 * is safe to call multiple times, so long as the NIC is not disabled.
 * Requires the RTNL lock.
1571
 */
1572 1573 1574
static void efx_start_all(struct efx_nic *efx)
{
	EFX_ASSERT_RESET_SERIALISED(efx);
1575
	BUG_ON(efx->state == STATE_DISABLED);
1576 1577 1578

	/* Check that it is appropriate to restart the interface. All
	 * of these flags are safe to read under just the rtnl lock */
1579
	if (efx->port_enabled || !netif_running(efx->net_dev))
1580 1581 1582
		return;

	efx_start_port(efx);
1583
	efx_start_datapath(efx);
1584

1585 1586 1587 1588
	/* Start the hardware monitor if there is one. Otherwise (we're link
	 * event driven), we have to poll the PHY because after an event queue
	 * flush, we could have a missed a link state change */
	if (efx->type->monitor != NULL) {
1589 1590
		queue_delayed_work(efx->workqueue, &efx->monitor_work,
				   efx_monitor_interval);
1591 1592 1593 1594 1595 1596
	} else {
		mutex_lock(&efx->mac_lock);
		if (efx->phy_op->poll(efx))
			efx_link_status_changed(efx);
		mutex_unlock(&efx->mac_lock);
	}
1597

1598
	efx->type->start_stats(efx);
1599 1600 1601 1602 1603 1604 1605
}

/* Flush all delayed work. Should only be called when no more delayed work
 * will be scheduled. This doesn't flush pending online resets (efx_reset),
 * since we're holding the rtnl_lock at this point. */
static void efx_flush_all(struct efx_nic *efx)
{
1606
	/* Make sure the hardware monitor and event self-test are stopped */
1607
	cancel_delayed_work_sync(&efx->monitor_work);
1608
	efx_selftest_async_cancel(efx);
1609
	/* Stop scheduled port reconfigurations */
1610
	cancel_work_sync(&efx->mac_work);
1611 1612
}

1613 1614 1615 1616 1617
/* Quiesce the hardware and software data path, and regular activity
 * for the port without bringing the link down.  Safe to call multiple
 * times with the NIC in almost any state, but interrupts should be
 * enabled.  Requires the RTNL lock.
 */
1618 1619 1620 1621 1622 1623 1624 1625
static void efx_stop_all(struct efx_nic *efx)
{
	EFX_ASSERT_RESET_SERIALISED(efx);

	/* port_enabled can be read safely under the rtnl lock */
	if (!efx->port_enabled)
		return;

1626
	efx->type->stop_stats(efx);
1627 1628
	efx_stop_port(efx);

S
Steve Hodgson 已提交
1629
	/* Flush efx_mac_work(), refill_workqueue, monitor_work */
1630 1631 1632 1633
	efx_flush_all(efx);

	/* Stop the kernel transmit interface late, so the watchdog
	 * timer isn't ticking over the flush */
1634 1635 1636
	netif_tx_disable(efx->net_dev);

	efx_stop_datapath(efx);
1637 1638 1639 1640
}

static void efx_remove_all(struct efx_nic *efx)
{
1641
	efx_remove_channels(efx);
1642
	efx_remove_filters(efx);
1643 1644 1645 1646 1647 1648 1649 1650 1651 1652
	efx_remove_port(efx);
	efx_remove_nic(efx);
}

/**************************************************************************
 *
 * Interrupt moderation
 *
 **************************************************************************/

1653
static unsigned int irq_mod_ticks(unsigned int usecs, unsigned int quantum_ns)
1654
{
1655 1656
	if (usecs == 0)
		return 0;
1657
	if (usecs * 1000 < quantum_ns)
1658
		return 1; /* never round down to 0 */
1659
	return usecs * 1000 / quantum_ns;
1660 1661
}

1662
/* Set interrupt moderation parameters */
1663 1664 1665
int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
			    unsigned int rx_usecs, bool rx_adaptive,
			    bool rx_may_override_tx)
1666
{
1667
	struct efx_channel *channel;
1668 1669 1670 1671 1672
	unsigned int irq_mod_max = DIV_ROUND_UP(efx->type->timer_period_max *
						efx->timer_quantum_ns,
						1000);
	unsigned int tx_ticks;
	unsigned int rx_ticks;
1673 1674 1675

	EFX_ASSERT_RESET_SERIALISED(efx);

1676
	if (tx_usecs > irq_mod_max || rx_usecs > irq_mod_max)
1677 1678
		return -EINVAL;

1679 1680 1681
	tx_ticks = irq_mod_ticks(tx_usecs, efx->timer_quantum_ns);
	rx_ticks = irq_mod_ticks(rx_usecs, efx->timer_quantum_ns);

1682 1683 1684 1685 1686 1687 1688
	if (tx_ticks != rx_ticks && efx->tx_channel_offset == 0 &&
	    !rx_may_override_tx) {
		netif_err(efx, drv, efx->net_dev, "Channels are shared. "
			  "RX and TX IRQ moderation must be equal\n");
		return -EINVAL;
	}

1689
	efx->irq_rx_adaptive = rx_adaptive;
1690
	efx->irq_rx_moderation = rx_ticks;
1691
	efx_for_each_channel(channel, efx) {
1692
		if (efx_channel_has_rx_queue(channel))
1693
			channel->irq_moderation = rx_ticks;
1694
		else if (efx_channel_has_tx_queues(channel))
1695 1696
			channel->irq_moderation = tx_ticks;
	}
1697 1698

	return 0;
1699 1700
}

1701 1702 1703
void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs,
			    unsigned int *rx_usecs, bool *rx_adaptive)
{
1704 1705 1706 1707
	/* We must round up when converting ticks to microseconds
	 * because we round down when converting the other way.
	 */

1708
	*rx_adaptive = efx->irq_rx_adaptive;
1709 1710 1711
	*rx_usecs = DIV_ROUND_UP(efx->irq_rx_moderation *
				 efx->timer_quantum_ns,
				 1000);
1712 1713 1714 1715 1716 1717 1718 1719

	/* If channels are shared between RX and TX, so is IRQ
	 * moderation.  Otherwise, IRQ moderation is the same for all
	 * TX channels and is not adaptive.
	 */
	if (efx->tx_channel_offset == 0)
		*tx_usecs = *rx_usecs;
	else
1720
		*tx_usecs = DIV_ROUND_UP(
1721
			efx->channel[efx->tx_channel_offset]->irq_moderation *
1722 1723
			efx->timer_quantum_ns,
			1000);
1724 1725
}

1726 1727 1728 1729 1730 1731
/**************************************************************************
 *
 * Hardware monitor
 *
 **************************************************************************/

1732
/* Run periodically off the general workqueue */
1733 1734 1735 1736 1737
static void efx_monitor(struct work_struct *data)
{
	struct efx_nic *efx = container_of(data, struct efx_nic,
					   monitor_work.work);

1738 1739 1740
	netif_vdbg(efx, timer, efx->net_dev,
		   "hardware monitor executing on CPU %d\n",
		   raw_smp_processor_id());
1741
	BUG_ON(efx->type->monitor == NULL);
1742 1743 1744

	/* If the mac_lock is already held then it is likely a port
	 * reconfiguration is already in place, which will likely do
1745 1746 1747 1748 1749 1750
	 * most of the work of monitor() anyway. */
	if (mutex_trylock(&efx->mac_lock)) {
		if (efx->port_enabled)
			efx->type->monitor(efx);
		mutex_unlock(&efx->mac_lock);
	}
1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766

	queue_delayed_work(efx->workqueue, &efx->monitor_work,
			   efx_monitor_interval);
}

/**************************************************************************
 *
 * ioctls
 *
 *************************************************************************/

/* Net device ioctl
 * Context: process, rtnl_lock() held.
 */
static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)
{
1767
	struct efx_nic *efx = netdev_priv(net_dev);
1768
	struct mii_ioctl_data *data = if_mii(ifr);
1769

1770 1771 1772 1773 1774 1775
	/* Convert phy_id from older PRTAD/DEVAD format */
	if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) &&
	    (data->phy_id & 0xfc00) == 0x0400)
		data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400;

	return mdio_mii_ioctl(&efx->mdio, data, cmd);
1776 1777 1778 1779 1780 1781 1782 1783
}

/**************************************************************************
 *
 * NAPI interface
 *
 **************************************************************************/

1784 1785 1786 1787 1788 1789 1790 1791 1792
static void efx_init_napi_channel(struct efx_channel *channel)
{
	struct efx_nic *efx = channel->efx;

	channel->napi_dev = efx->net_dev;
	netif_napi_add(channel->napi_dev, &channel->napi_str,
		       efx_poll, napi_weight);
}

1793
static void efx_init_napi(struct efx_nic *efx)
1794 1795 1796
{
	struct efx_channel *channel;

1797 1798
	efx_for_each_channel(channel, efx)
		efx_init_napi_channel(channel);
1799 1800 1801 1802 1803 1804 1805
}

static void efx_fini_napi_channel(struct efx_channel *channel)
{
	if (channel->napi_dev)
		netif_napi_del(&channel->napi_str);
	channel->napi_dev = NULL;
1806 1807 1808 1809 1810 1811
}

static void efx_fini_napi(struct efx_nic *efx)
{
	struct efx_channel *channel;

1812 1813
	efx_for_each_channel(channel, efx)
		efx_fini_napi_channel(channel);
1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829
}

/**************************************************************************
 *
 * Kernel netpoll interface
 *
 *************************************************************************/

#ifdef CONFIG_NET_POLL_CONTROLLER

/* Although in the common case interrupts will be disabled, this is not
 * guaranteed. However, all our work happens inside the NAPI callback,
 * so no locking is required.
 */
static void efx_netpoll(struct net_device *net_dev)
{
1830
	struct efx_nic *efx = netdev_priv(net_dev);
1831 1832
	struct efx_channel *channel;

1833
	efx_for_each_channel(channel, efx)
1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847
		efx_schedule_channel(channel);
}

#endif

/**************************************************************************
 *
 * Kernel net device interface
 *
 *************************************************************************/

/* Context: process, rtnl_lock() held. */
static int efx_net_open(struct net_device *net_dev)
{
1848
	struct efx_nic *efx = netdev_priv(net_dev);
1849 1850
	int rc;

1851 1852
	netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n",
		  raw_smp_processor_id());
1853

1854 1855 1856
	rc = efx_check_disabled(efx);
	if (rc)
		return rc;
1857 1858
	if (efx->phy_mode & PHY_MODE_SPECIAL)
		return -EBUSY;
1859 1860
	if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL))
		return -EIO;
1861

1862 1863 1864 1865
	/* Notify the kernel of the link state polled during driver load,
	 * before the monitor starts running */
	efx_link_status_changed(efx);

1866
	efx_start_all(efx);
1867
	efx_selftest_async_start(efx);
1868 1869 1870 1871 1872 1873 1874 1875 1876
	return 0;
}

/* Context: process, rtnl_lock() held.
 * Note that the kernel will ignore our return code; this method
 * should really be a void.
 */
static int efx_net_stop(struct net_device *net_dev)
{
1877
	struct efx_nic *efx = netdev_priv(net_dev);
1878

1879 1880
	netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n",
		  raw_smp_processor_id());
1881

1882 1883
	/* Stop the device and flush all the channels */
	efx_stop_all(efx);
1884 1885 1886 1887

	return 0;
}

1888
/* Context: process, dev_base_lock or RTNL held, non-blocking. */
B
Ben Hutchings 已提交
1889 1890
static struct rtnl_link_stats64 *efx_net_stats(struct net_device *net_dev,
					       struct rtnl_link_stats64 *stats)
1891
{
1892
	struct efx_nic *efx = netdev_priv(net_dev);
1893 1894
	struct efx_mac_stats *mac_stats = &efx->mac_stats;

1895
	spin_lock_bh(&efx->stats_lock);
1896

1897
	efx->type->update_stats(efx);
1898 1899 1900 1901 1902

	stats->rx_packets = mac_stats->rx_packets;
	stats->tx_packets = mac_stats->tx_packets;
	stats->rx_bytes = mac_stats->rx_bytes;
	stats->tx_bytes = mac_stats->tx_bytes;
1903
	stats->rx_dropped = efx->n_rx_nodesc_drop_cnt;
1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920
	stats->multicast = mac_stats->rx_multicast;
	stats->collisions = mac_stats->tx_collision;
	stats->rx_length_errors = (mac_stats->rx_gtjumbo +
				   mac_stats->rx_length_error);
	stats->rx_crc_errors = mac_stats->rx_bad;
	stats->rx_frame_errors = mac_stats->rx_align_error;
	stats->rx_fifo_errors = mac_stats->rx_overflow;
	stats->rx_missed_errors = mac_stats->rx_missed;
	stats->tx_window_errors = mac_stats->tx_late_collision;

	stats->rx_errors = (stats->rx_length_errors +
			    stats->rx_crc_errors +
			    stats->rx_frame_errors +
			    mac_stats->rx_symbol_error);
	stats->tx_errors = (stats->tx_window_errors +
			    mac_stats->tx_bad);

1921 1922
	spin_unlock_bh(&efx->stats_lock);

1923 1924 1925 1926 1927 1928
	return stats;
}

/* Context: netif_tx_lock held, BHs disabled. */
static void efx_watchdog(struct net_device *net_dev)
{
1929
	struct efx_nic *efx = netdev_priv(net_dev);
1930

1931 1932 1933
	netif_err(efx, tx_err, efx->net_dev,
		  "TX stuck with port_enabled=%d: resetting channels\n",
		  efx->port_enabled);
1934

1935
	efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG);
1936 1937 1938 1939 1940 1941
}


/* Context: process, rtnl_lock() held. */
static int efx_change_mtu(struct net_device *net_dev, int new_mtu)
{
1942
	struct efx_nic *efx = netdev_priv(net_dev);
1943
	int rc;
1944

1945 1946 1947
	rc = efx_check_disabled(efx);
	if (rc)
		return rc;
1948 1949 1950 1951 1952
	if (new_mtu > EFX_MAX_MTU)
		return -EINVAL;

	efx_stop_all(efx);

1953
	netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu);
1954

B
Ben Hutchings 已提交
1955
	mutex_lock(&efx->mac_lock);
1956
	net_dev->mtu = new_mtu;
1957
	efx->type->reconfigure_mac(efx);
B
Ben Hutchings 已提交
1958 1959
	mutex_unlock(&efx->mac_lock);

1960
	efx_start_all(efx);
1961
	return 0;
1962 1963 1964 1965
}

static int efx_set_mac_address(struct net_device *net_dev, void *data)
{
1966
	struct efx_nic *efx = netdev_priv(net_dev);
1967 1968 1969 1970
	struct sockaddr *addr = data;
	char *new_addr = addr->sa_data;

	if (!is_valid_ether_addr(new_addr)) {
1971 1972 1973
		netif_err(efx, drv, efx->net_dev,
			  "invalid ethernet MAC address requested: %pM\n",
			  new_addr);
1974
		return -EADDRNOTAVAIL;
1975 1976 1977
	}

	memcpy(net_dev->dev_addr, new_addr, net_dev->addr_len);
1978
	efx_sriov_mac_address_changed(efx);
1979 1980

	/* Reconfigure the MAC */
B
Ben Hutchings 已提交
1981
	mutex_lock(&efx->mac_lock);
1982
	efx->type->reconfigure_mac(efx);
B
Ben Hutchings 已提交
1983
	mutex_unlock(&efx->mac_lock);
1984 1985 1986 1987

	return 0;
}

1988
/* Context: netif_addr_lock held, BHs disabled. */
1989
static void efx_set_rx_mode(struct net_device *net_dev)
1990
{
1991
	struct efx_nic *efx = netdev_priv(net_dev);
1992
	struct netdev_hw_addr *ha;
1993 1994 1995 1996
	union efx_multicast_hash *mc_hash = &efx->multicast_hash;
	u32 crc;
	int bit;

1997
	efx->promiscuous = !!(net_dev->flags & IFF_PROMISC);
1998 1999

	/* Build multicast hash table */
2000
	if (efx->promiscuous || (net_dev->flags & IFF_ALLMULTI)) {
2001 2002 2003
		memset(mc_hash, 0xff, sizeof(*mc_hash));
	} else {
		memset(mc_hash, 0x00, sizeof(*mc_hash));
2004 2005
		netdev_for_each_mc_addr(ha, net_dev) {
			crc = ether_crc_le(ETH_ALEN, ha->addr);
2006 2007 2008 2009
			bit = crc & (EFX_MCAST_HASH_ENTRIES - 1);
			set_bit_le(bit, mc_hash->byte);
		}

2010 2011 2012 2013 2014 2015
		/* Broadcast packets go through the multicast hash filter.
		 * ether_crc_le() of the broadcast address is 0xbe2612ff
		 * so we always add bit 0xff to the mask.
		 */
		set_bit_le(0xff, mc_hash->byte);
	}
2016

2017 2018 2019
	if (efx->port_enabled)
		queue_work(efx->workqueue, &efx->mac_work);
	/* Otherwise efx_start_port() will do this */
2020 2021
}

2022
static int efx_set_features(struct net_device *net_dev, netdev_features_t data)
2023 2024 2025 2026 2027 2028 2029 2030 2031 2032
{
	struct efx_nic *efx = netdev_priv(net_dev);

	/* If disabling RX n-tuple filtering, clear existing filters */
	if (net_dev->features & ~data & NETIF_F_NTUPLE)
		efx_filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL);

	return 0;
}

S
Stephen Hemminger 已提交
2033 2034 2035
static const struct net_device_ops efx_netdev_ops = {
	.ndo_open		= efx_net_open,
	.ndo_stop		= efx_net_stop,
2036
	.ndo_get_stats64	= efx_net_stats,
S
Stephen Hemminger 已提交
2037 2038 2039 2040 2041 2042
	.ndo_tx_timeout		= efx_watchdog,
	.ndo_start_xmit		= efx_hard_start_xmit,
	.ndo_validate_addr	= eth_validate_addr,
	.ndo_do_ioctl		= efx_ioctl,
	.ndo_change_mtu		= efx_change_mtu,
	.ndo_set_mac_address	= efx_set_mac_address,
2043
	.ndo_set_rx_mode	= efx_set_rx_mode,
2044
	.ndo_set_features	= efx_set_features,
2045 2046 2047 2048 2049 2050
#ifdef CONFIG_SFC_SRIOV
	.ndo_set_vf_mac		= efx_sriov_set_vf_mac,
	.ndo_set_vf_vlan	= efx_sriov_set_vf_vlan,
	.ndo_set_vf_spoofchk	= efx_sriov_set_vf_spoofchk,
	.ndo_get_vf_config	= efx_sriov_get_vf_config,
#endif
S
Stephen Hemminger 已提交
2051 2052 2053
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller = efx_netpoll,
#endif
2054
	.ndo_setup_tc		= efx_setup_tc,
2055 2056 2057
#ifdef CONFIG_RFS_ACCEL
	.ndo_rx_flow_steer	= efx_filter_rfs,
#endif
S
Stephen Hemminger 已提交
2058 2059
};

2060 2061 2062 2063 2064 2065 2066
static void efx_update_name(struct efx_nic *efx)
{
	strcpy(efx->name, efx->net_dev->name);
	efx_mtd_rename(efx);
	efx_set_channel_names(efx);
}

2067 2068 2069
static int efx_netdev_event(struct notifier_block *this,
			    unsigned long event, void *ptr)
{
2070
	struct net_device *net_dev = ptr;
2071

2072 2073 2074
	if (net_dev->netdev_ops == &efx_netdev_ops &&
	    event == NETDEV_CHANGENAME)
		efx_update_name(netdev_priv(net_dev));
2075 2076 2077 2078 2079 2080 2081 2082

	return NOTIFY_DONE;
}

static struct notifier_block efx_netdev_notifier = {
	.notifier_call = efx_netdev_event,
};

B
Ben Hutchings 已提交
2083 2084 2085 2086 2087 2088 2089 2090
static ssize_t
show_phy_type(struct device *dev, struct device_attribute *attr, char *buf)
{
	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
	return sprintf(buf, "%d\n", efx->phy_type);
}
static DEVICE_ATTR(phy_type, 0644, show_phy_type, NULL);

2091 2092 2093
static int efx_register_netdev(struct efx_nic *efx)
{
	struct net_device *net_dev = efx->net_dev;
2094
	struct efx_channel *channel;
2095 2096 2097 2098
	int rc;

	net_dev->watchdog_timeo = 5 * HZ;
	net_dev->irq = efx->pci_dev->irq;
S
Stephen Hemminger 已提交
2099
	net_dev->netdev_ops = &efx_netdev_ops;
2100
	SET_ETHTOOL_OPS(net_dev, &efx_ethtool_ops);
2101
	net_dev->gso_max_segs = EFX_TSO_MAX_SEGS;
2102

2103
	rtnl_lock();
2104

2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117
	/* Enable resets to be scheduled and check whether any were
	 * already requested.  If so, the NIC is probably hosed so we
	 * abort.
	 */
	efx->state = STATE_READY;
	smp_mb(); /* ensure we change state before checking reset_pending */
	if (efx->reset_pending) {
		netif_err(efx, probe, efx->net_dev,
			  "aborting probe due to scheduled reset\n");
		rc = -EIO;
		goto fail_locked;
	}

2118 2119 2120
	rc = dev_alloc_name(net_dev, net_dev->name);
	if (rc < 0)
		goto fail_locked;
2121
	efx_update_name(efx);
2122 2123 2124 2125 2126

	rc = register_netdevice(net_dev);
	if (rc)
		goto fail_locked;

2127 2128
	efx_for_each_channel(channel, efx) {
		struct efx_tx_queue *tx_queue;
2129 2130
		efx_for_each_channel_tx_queue(tx_queue, channel)
			efx_init_tx_queue_core_txq(tx_queue);
2131 2132
	}

2133
	/* Always start with carrier off; PHY events will detect the link */
2134
	netif_carrier_off(net_dev);
2135

2136
	rtnl_unlock();
2137

B
Ben Hutchings 已提交
2138 2139
	rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);
	if (rc) {
2140 2141
		netif_err(efx, drv, efx->net_dev,
			  "failed to init net dev attributes\n");
B
Ben Hutchings 已提交
2142 2143 2144
		goto fail_registered;
	}

2145
	return 0;
B
Ben Hutchings 已提交
2146

2147 2148 2149
fail_registered:
	rtnl_lock();
	unregister_netdevice(net_dev);
2150
fail_locked:
2151
	efx->state = STATE_UNINIT;
2152
	rtnl_unlock();
2153
	netif_err(efx, drv, efx->net_dev, "could not register net dev\n");
2154
	return rc;
2155 2156 2157 2158
}

static void efx_unregister_netdev(struct efx_nic *efx)
{
2159
	struct efx_channel *channel;
2160 2161 2162 2163 2164
	struct efx_tx_queue *tx_queue;

	if (!efx->net_dev)
		return;

2165
	BUG_ON(netdev_priv(efx->net_dev) != efx);
2166 2167 2168 2169

	/* Free up any skbs still remaining. This has to happen before
	 * we try to unregister the netdev as running their destructors
	 * may be needed to get the device ref. count to 0. */
2170 2171 2172 2173
	efx_for_each_channel(channel, efx) {
		efx_for_each_channel_tx_queue(tx_queue, channel)
			efx_release_tx_buffers(tx_queue);
	}
2174

2175 2176
	strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
	device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2177 2178 2179 2180 2181

	rtnl_lock();
	unregister_netdevice(efx->net_dev);
	efx->state = STATE_UNINIT;
	rtnl_unlock();
2182 2183 2184 2185 2186 2187 2188 2189
}

/**************************************************************************
 *
 * Device reset and suspend
 *
 **************************************************************************/

B
Ben Hutchings 已提交
2190 2191
/* Tears down the entire software state and most of the hardware state
 * before reset.  */
B
Ben Hutchings 已提交
2192
void efx_reset_down(struct efx_nic *efx, enum reset_type method)
2193 2194 2195
{
	EFX_ASSERT_RESET_SERIALISED(efx);

B
Ben Hutchings 已提交
2196
	efx_stop_all(efx);
2197
	efx_stop_interrupts(efx, false);
2198 2199

	mutex_lock(&efx->mac_lock);
2200 2201
	if (efx->port_initialized && method != RESET_TYPE_INVISIBLE)
		efx->phy_op->fini(efx);
2202
	efx->type->fini(efx);
2203 2204
}

B
Ben Hutchings 已提交
2205 2206 2207 2208 2209
/* This function will always ensure that the locks acquired in
 * efx_reset_down() are released. A failure return code indicates
 * that we were unable to reinitialise the hardware, and the
 * driver should be disabled. If ok is false, then the rx and tx
 * engines are not restarted, pending a RESET_DISABLE. */
B
Ben Hutchings 已提交
2210
int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok)
2211 2212 2213
{
	int rc;

B
Ben Hutchings 已提交
2214
	EFX_ASSERT_RESET_SERIALISED(efx);
2215

2216
	rc = efx->type->init(efx);
2217
	if (rc) {
2218
		netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n");
2219
		goto fail;
2220 2221
	}

2222 2223 2224
	if (!ok)
		goto fail;

2225
	if (efx->port_initialized && method != RESET_TYPE_INVISIBLE) {
2226 2227 2228 2229
		rc = efx->phy_op->init(efx);
		if (rc)
			goto fail;
		if (efx->phy_op->reconfigure(efx))
2230 2231
			netif_err(efx, drv, efx->net_dev,
				  "could not restore PHY settings\n");
2232 2233
	}

2234
	efx->type->reconfigure_mac(efx);
2235

2236
	efx_start_interrupts(efx, false);
B
Ben Hutchings 已提交
2237
	efx_restore_filters(efx);
2238
	efx_sriov_reset(efx);
2239 2240 2241 2242 2243 2244 2245 2246 2247

	mutex_unlock(&efx->mac_lock);

	efx_start_all(efx);

	return 0;

fail:
	efx->port_initialized = false;
B
Ben Hutchings 已提交
2248 2249 2250

	mutex_unlock(&efx->mac_lock);

2251 2252 2253
	return rc;
}

2254 2255
/* Reset the NIC using the specified method.  Note that the reset may
 * fail, in which case the card will be left in an unusable state.
2256
 *
2257
 * Caller must hold the rtnl_lock.
2258
 */
2259
int efx_reset(struct efx_nic *efx, enum reset_type method)
2260
{
2261 2262
	int rc, rc2;
	bool disabled;
2263

2264 2265
	netif_info(efx, drv, efx->net_dev, "resetting (%s)\n",
		   RESET_TYPE(method));
2266

2267
	netif_device_detach(efx->net_dev);
B
Ben Hutchings 已提交
2268
	efx_reset_down(efx, method);
2269

2270
	rc = efx->type->reset(efx, method);
2271
	if (rc) {
2272
		netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n");
2273
		goto out;
2274 2275
	}

2276 2277 2278 2279
	/* Clear flags for the scopes we covered.  We assume the NIC and
	 * driver are now quiescent so that there is no race here.
	 */
	efx->reset_pending &= -(1 << (method + 1));
2280 2281 2282 2283 2284 2285 2286

	/* Reinitialise bus-mastering, which may have been turned off before
	 * the reset was scheduled. This is still appropriate, even in the
	 * RESET_TYPE_DISABLE since this driver generally assumes the hardware
	 * can respond to requests. */
	pci_set_master(efx->pci_dev);

2287
out:
2288
	/* Leave device stopped if necessary */
2289 2290 2291 2292 2293 2294
	disabled = rc || method == RESET_TYPE_DISABLE;
	rc2 = efx_reset_up(efx, method, !disabled);
	if (rc2) {
		disabled = true;
		if (!rc)
			rc = rc2;
2295 2296
	}

2297
	if (disabled) {
2298
		dev_close(efx->net_dev);
2299
		netif_err(efx, drv, efx->net_dev, "has been disabled\n");
2300 2301
		efx->state = STATE_DISABLED;
	} else {
2302
		netif_dbg(efx, drv, efx->net_dev, "reset complete\n");
2303
		netif_device_attach(efx->net_dev);
2304
	}
2305 2306 2307 2308 2309 2310 2311 2312
	return rc;
}

/* The worker thread exists so that code that cannot sleep can
 * schedule a reset for later.
 */
static void efx_reset_work(struct work_struct *data)
{
2313
	struct efx_nic *efx = container_of(data, struct efx_nic, reset_work);
2314
	unsigned long pending = ACCESS_ONCE(efx->reset_pending);
2315

2316
	if (!pending)
2317 2318
		return;

2319
	rtnl_lock();
2320 2321 2322 2323 2324 2325 2326 2327

	/* We checked the state in efx_schedule_reset() but it may
	 * have changed by now.  Now that we have the RTNL lock,
	 * it cannot change again.
	 */
	if (efx->state == STATE_READY)
		(void)efx_reset(efx, fls(pending) - 1);

2328
	rtnl_unlock();
2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340
}

void efx_schedule_reset(struct efx_nic *efx, enum reset_type type)
{
	enum reset_type method;

	switch (type) {
	case RESET_TYPE_INVISIBLE:
	case RESET_TYPE_ALL:
	case RESET_TYPE_WORLD:
	case RESET_TYPE_DISABLE:
		method = type;
2341 2342
		netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n",
			  RESET_TYPE(method));
2343 2344
		break;
	default:
2345
		method = efx->type->map_reset_reason(type);
2346 2347 2348
		netif_dbg(efx, drv, efx->net_dev,
			  "scheduling %s reset for %s\n",
			  RESET_TYPE(method), RESET_TYPE(type));
2349 2350
		break;
	}
2351

2352
	set_bit(method, &efx->reset_pending);
2353 2354 2355 2356 2357 2358 2359
	smp_mb(); /* ensure we change reset_pending before checking state */

	/* If we're not READY then just leave the flags set as the cue
	 * to abort probing or reschedule the reset later.
	 */
	if (ACCESS_ONCE(efx->state) != STATE_READY)
		return;
2360

2361 2362 2363 2364
	/* efx_process_channel() will no longer read events once a
	 * reset is scheduled. So switch back to poll'd MCDI completions. */
	efx_mcdi_mode_poll(efx);

2365
	queue_work(reset_workqueue, &efx->reset_work);
2366 2367 2368 2369 2370 2371 2372 2373 2374
}

/**************************************************************************
 *
 * List of NICs we support
 *
 **************************************************************************/

/* PCI device ID table */
2375
static DEFINE_PCI_DEVICE_TABLE(efx_pci_table) = {
2376 2377
	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE,
		    PCI_DEVICE_ID_SOLARFLARE_SFC4000A_0),
2378
	 .driver_data = (unsigned long) &falcon_a1_nic_type},
2379 2380
	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE,
		    PCI_DEVICE_ID_SOLARFLARE_SFC4000B),
2381
	 .driver_data = (unsigned long) &falcon_b0_nic_type},
2382
	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803),	/* SFC9020 */
2383
	 .driver_data = (unsigned long) &siena_a0_nic_type},
2384
	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813),	/* SFL9021 */
2385
	 .driver_data = (unsigned long) &siena_a0_nic_type},
2386 2387 2388 2389 2390
	{0}			/* end of list */
};

/**************************************************************************
 *
2391
 * Dummy PHY/MAC operations
2392
 *
2393
 * Can be used for some unimplemented operations
2394 2395 2396 2397 2398 2399 2400 2401 2402
 * Needed so all function pointers are valid and do not have to be tested
 * before use
 *
 **************************************************************************/
int efx_port_dummy_op_int(struct efx_nic *efx)
{
	return 0;
}
void efx_port_dummy_op_void(struct efx_nic *efx) {}
S
stephen hemminger 已提交
2403 2404

static bool efx_port_dummy_op_poll(struct efx_nic *efx)
S
Steve Hodgson 已提交
2405 2406 2407
{
	return false;
}
2408

2409
static const struct efx_phy_operations efx_dummy_phy_operations = {
2410
	.init		 = efx_port_dummy_op_int,
B
Ben Hutchings 已提交
2411
	.reconfigure	 = efx_port_dummy_op_int,
S
Steve Hodgson 已提交
2412
	.poll		 = efx_port_dummy_op_poll,
2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424
	.fini		 = efx_port_dummy_op_void,
};

/**************************************************************************
 *
 * Data housekeeping
 *
 **************************************************************************/

/* This zeroes out and then fills in the invariants in a struct
 * efx_nic (including all sub-structures).
 */
2425
static int efx_init_struct(struct efx_nic *efx,
2426 2427
			   struct pci_dev *pci_dev, struct net_device *net_dev)
{
2428
	int i;
2429 2430 2431

	/* Initialise common structures */
	spin_lock_init(&efx->biu_lock);
2432 2433 2434
#ifdef CONFIG_SFC_MTD
	INIT_LIST_HEAD(&efx->mtd_list);
#endif
2435 2436
	INIT_WORK(&efx->reset_work, efx_reset_work);
	INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor);
2437
	INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work);
2438
	efx->pci_dev = pci_dev;
2439
	efx->msg_enable = debug;
2440
	efx->state = STATE_UNINIT;
2441 2442 2443 2444 2445 2446
	strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));

	efx->net_dev = net_dev;
	spin_lock_init(&efx->stats_lock);
	mutex_init(&efx->mac_lock);
	efx->phy_op = &efx_dummy_phy_operations;
2447
	efx->mdio.dev = net_dev;
2448
	INIT_WORK(&efx->mac_work, efx_mac_work);
2449
	init_waitqueue_head(&efx->flush_wq);
2450 2451

	for (i = 0; i < EFX_MAX_CHANNELS; i++) {
2452 2453 2454
		efx->channel[i] = efx_alloc_channel(efx, i, NULL);
		if (!efx->channel[i])
			goto fail;
2455 2456 2457 2458 2459 2460 2461 2462
	}

	EFX_BUG_ON_PARANOID(efx->type->phys_addr_channels > EFX_MAX_CHANNELS);

	/* Higher numbered interrupt modes are less capable! */
	efx->interrupt_mode = max(efx->type->max_interrupt_mode,
				  interrupt_mode);

2463 2464 2465 2466
	/* Would be good to use the net_dev name, but we're too early */
	snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s",
		 pci_name(pci_dev));
	efx->workqueue = create_singlethread_workqueue(efx->workqueue_name);
2467
	if (!efx->workqueue)
2468
		goto fail;
2469

2470
	return 0;
2471 2472 2473 2474

fail:
	efx_fini_struct(efx);
	return -ENOMEM;
2475 2476 2477 2478
}

static void efx_fini_struct(struct efx_nic *efx)
{
2479 2480 2481 2482 2483
	int i;

	for (i = 0; i < EFX_MAX_CHANNELS; i++)
		kfree(efx->channel[i]);

2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500
	if (efx->workqueue) {
		destroy_workqueue(efx->workqueue);
		efx->workqueue = NULL;
	}
}

/**************************************************************************
 *
 * PCI interface
 *
 **************************************************************************/

/* Main body of final NIC shutdown code
 * This is called only at module unload (or hotplug removal).
 */
static void efx_pci_remove_main(struct efx_nic *efx)
{
2501 2502 2503 2504 2505 2506
	/* Flush reset_work. It can no longer be scheduled since we
	 * are not READY.
	 */
	BUG_ON(efx->state == STATE_READY);
	cancel_work_sync(&efx->reset_work);

2507 2508 2509 2510
#ifdef CONFIG_RFS_ACCEL
	free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
	efx->net_dev->rx_cpu_rmap = NULL;
#endif
2511
	efx_stop_interrupts(efx, false);
2512
	efx_nic_fini_interrupt(efx);
2513
	efx_fini_port(efx);
2514
	efx->type->fini(efx);
2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532
	efx_fini_napi(efx);
	efx_remove_all(efx);
}

/* Final NIC shutdown
 * This is called only at module unload (or hotplug removal).
 */
static void efx_pci_remove(struct pci_dev *pci_dev)
{
	struct efx_nic *efx;

	efx = pci_get_drvdata(pci_dev);
	if (!efx)
		return;

	/* Mark the NIC as fini, then stop the interface */
	rtnl_lock();
	dev_close(efx->net_dev);
2533
	efx_stop_interrupts(efx, false);
2534 2535
	rtnl_unlock();

2536
	efx_sriov_fini(efx);
2537 2538
	efx_unregister_netdev(efx);

2539 2540
	efx_mtd_remove(efx);

2541 2542 2543
	efx_pci_remove_main(efx);

	efx_fini_io(efx);
2544
	netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n");
2545 2546

	efx_fini_struct(efx);
2547
	pci_set_drvdata(pci_dev, NULL);
2548 2549 2550
	free_netdev(efx->net_dev);
};

2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601
/* NIC VPD information
 * Called during probe to display the part number of the
 * installed NIC.  VPD is potentially very large but this should
 * always appear within the first 512 bytes.
 */
#define SFC_VPD_LEN 512
static void efx_print_product_vpd(struct efx_nic *efx)
{
	struct pci_dev *dev = efx->pci_dev;
	char vpd_data[SFC_VPD_LEN];
	ssize_t vpd_size;
	int i, j;

	/* Get the vpd data from the device */
	vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data);
	if (vpd_size <= 0) {
		netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n");
		return;
	}

	/* Get the Read only section */
	i = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA);
	if (i < 0) {
		netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n");
		return;
	}

	j = pci_vpd_lrdt_size(&vpd_data[i]);
	i += PCI_VPD_LRDT_TAG_SIZE;
	if (i + j > vpd_size)
		j = vpd_size - i;

	/* Get the Part number */
	i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN");
	if (i < 0) {
		netif_err(efx, drv, efx->net_dev, "Part number not found\n");
		return;
	}

	j = pci_vpd_info_field_size(&vpd_data[i]);
	i += PCI_VPD_INFO_FLD_HDR_SIZE;
	if (i + j > vpd_size) {
		netif_err(efx, drv, efx->net_dev, "Incomplete part number\n");
		return;
	}

	netif_info(efx, drv, efx->net_dev,
		   "Part Number : %.*s\n", j, &vpd_data[i]);
}


2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613
/* Main body of NIC initialisation
 * This is called at module load (or hotplug insertion, theoretically).
 */
static int efx_pci_probe_main(struct efx_nic *efx)
{
	int rc;

	/* Do start-of-day initialisation */
	rc = efx_probe_all(efx);
	if (rc)
		goto fail1;

2614
	efx_init_napi(efx);
2615

2616
	rc = efx->type->init(efx);
2617
	if (rc) {
2618 2619
		netif_err(efx, probe, efx->net_dev,
			  "failed to initialise NIC\n");
2620
		goto fail3;
2621 2622 2623 2624
	}

	rc = efx_init_port(efx);
	if (rc) {
2625 2626
		netif_err(efx, probe, efx->net_dev,
			  "failed to initialise port\n");
2627
		goto fail4;
2628 2629
	}

2630
	rc = efx_nic_init_interrupt(efx);
2631
	if (rc)
2632
		goto fail5;
2633
	efx_start_interrupts(efx, false);
2634 2635 2636

	return 0;

2637
 fail5:
2638 2639
	efx_fini_port(efx);
 fail4:
2640
	efx->type->fini(efx);
2641 2642 2643 2644 2645 2646 2647 2648 2649 2650
 fail3:
	efx_fini_napi(efx);
	efx_remove_all(efx);
 fail1:
	return rc;
}

/* NIC initialisation
 *
 * This is called at module load (or hotplug insertion,
2651
 * theoretically).  It sets up PCI mappings, resets the NIC,
2652 2653 2654 2655 2656 2657 2658 2659 2660 2661
 * sets up and registers the network devices with the kernel and hooks
 * the interrupt service routine.  It does not prepare the device for
 * transmission; this is left to the first time one of the network
 * interfaces is brought up (i.e. efx_net_open).
 */
static int __devinit efx_pci_probe(struct pci_dev *pci_dev,
				   const struct pci_device_id *entry)
{
	struct net_device *net_dev;
	struct efx_nic *efx;
2662
	int rc;
2663 2664

	/* Allocate and initialise a struct net_device and struct efx_nic */
2665 2666
	net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES,
				     EFX_MAX_RX_QUEUES);
2667 2668
	if (!net_dev)
		return -ENOMEM;
2669 2670 2671
	efx = netdev_priv(net_dev);
	efx->type = (const struct efx_nic_type *) entry->driver_data;
	net_dev->features |= (efx->type->offload_features | NETIF_F_SG |
B
Ben Hutchings 已提交
2672
			      NETIF_F_HIGHDMA | NETIF_F_TSO |
2673
			      NETIF_F_RXCSUM);
2674
	if (efx->type->offload_features & NETIF_F_V6_CSUM)
B
Ben Hutchings 已提交
2675
		net_dev->features |= NETIF_F_TSO6;
2676 2677
	/* Mask for features that also apply to VLAN devices */
	net_dev->vlan_features |= (NETIF_F_ALL_CSUM | NETIF_F_SG |
2678 2679 2680 2681
				   NETIF_F_HIGHDMA | NETIF_F_ALL_TSO |
				   NETIF_F_RXCSUM);
	/* All offloads can be toggled */
	net_dev->hw_features = net_dev->features & ~NETIF_F_HIGHDMA;
2682
	pci_set_drvdata(pci_dev, efx);
2683
	SET_NETDEV_DEV(net_dev, &pci_dev->dev);
2684
	rc = efx_init_struct(efx, pci_dev, net_dev);
2685 2686 2687
	if (rc)
		goto fail1;

2688
	netif_info(efx, probe, efx->net_dev,
2689
		   "Solarflare NIC detected\n");
2690

2691 2692
	efx_print_product_vpd(efx);

2693 2694 2695 2696 2697
	/* Set up basic I/O (BAR mappings etc) */
	rc = efx_init_io(efx);
	if (rc)
		goto fail2;

2698 2699 2700
	rc = efx_pci_probe_main(efx);
	if (rc)
		goto fail3;
2701 2702 2703

	rc = efx_register_netdev(efx);
	if (rc)
2704
		goto fail4;
2705

2706 2707 2708 2709 2710
	rc = efx_sriov_init(efx);
	if (rc)
		netif_err(efx, probe, efx->net_dev,
			  "SR-IOV can't be enabled rc %d\n", rc);

2711
	netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n");
2712

2713
	/* Try to create MTDs, but allow this to fail */
2714
	rtnl_lock();
2715
	rc = efx_mtd_probe(efx);
2716
	rtnl_unlock();
2717 2718 2719 2720
	if (rc)
		netif_warn(efx, probe, efx->net_dev,
			   "failed to create MTDs (%d)\n", rc);

2721 2722 2723
	return 0;

 fail4:
2724
	efx_pci_remove_main(efx);
2725 2726 2727 2728 2729
 fail3:
	efx_fini_io(efx);
 fail2:
	efx_fini_struct(efx);
 fail1:
2730
	pci_set_drvdata(pci_dev, NULL);
S
Steve Hodgson 已提交
2731
	WARN_ON(rc > 0);
2732
	netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc);
2733 2734 2735 2736
	free_netdev(net_dev);
	return rc;
}

2737 2738 2739 2740
static int efx_pm_freeze(struct device *dev)
{
	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));

2741 2742
	rtnl_lock();

2743 2744
	if (efx->state != STATE_DISABLED) {
		efx->state = STATE_UNINIT;
2745

2746
		netif_device_detach(efx->net_dev);
2747

2748 2749 2750
		efx_stop_all(efx);
		efx_stop_interrupts(efx, false);
	}
2751

2752 2753
	rtnl_unlock();

2754 2755 2756 2757 2758 2759 2760
	return 0;
}

static int efx_pm_thaw(struct device *dev)
{
	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));

2761 2762
	rtnl_lock();

2763 2764
	if (efx->state != STATE_DISABLED) {
		efx_start_interrupts(efx, false);
2765

2766 2767 2768
		mutex_lock(&efx->mac_lock);
		efx->phy_op->reconfigure(efx);
		mutex_unlock(&efx->mac_lock);
2769

2770
		efx_start_all(efx);
2771

2772
		netif_device_attach(efx->net_dev);
2773

2774
		efx->state = STATE_READY;
2775

2776 2777
		efx->type->resume_wol(efx);
	}
2778

2779 2780
	rtnl_unlock();

2781 2782 2783
	/* Reschedule any quenched resets scheduled during efx_pm_freeze() */
	queue_work(reset_workqueue, &efx->reset_work);

2784 2785 2786 2787 2788 2789 2790 2791 2792 2793
	return 0;
}

static int efx_pm_poweroff(struct device *dev)
{
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct efx_nic *efx = pci_get_drvdata(pci_dev);

	efx->type->fini(efx);

2794
	efx->reset_pending = 0;
2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835

	pci_save_state(pci_dev);
	return pci_set_power_state(pci_dev, PCI_D3hot);
}

/* Used for both resume and restore */
static int efx_pm_resume(struct device *dev)
{
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct efx_nic *efx = pci_get_drvdata(pci_dev);
	int rc;

	rc = pci_set_power_state(pci_dev, PCI_D0);
	if (rc)
		return rc;
	pci_restore_state(pci_dev);
	rc = pci_enable_device(pci_dev);
	if (rc)
		return rc;
	pci_set_master(efx->pci_dev);
	rc = efx->type->reset(efx, RESET_TYPE_ALL);
	if (rc)
		return rc;
	rc = efx->type->init(efx);
	if (rc)
		return rc;
	efx_pm_thaw(dev);
	return 0;
}

static int efx_pm_suspend(struct device *dev)
{
	int rc;

	efx_pm_freeze(dev);
	rc = efx_pm_poweroff(dev);
	if (rc)
		efx_pm_resume(dev);
	return rc;
}

2836
static const struct dev_pm_ops efx_pm_ops = {
2837 2838 2839 2840 2841 2842 2843 2844
	.suspend	= efx_pm_suspend,
	.resume		= efx_pm_resume,
	.freeze		= efx_pm_freeze,
	.thaw		= efx_pm_thaw,
	.poweroff	= efx_pm_poweroff,
	.restore	= efx_pm_resume,
};

2845
static struct pci_driver efx_pci_driver = {
2846
	.name		= KBUILD_MODNAME,
2847 2848 2849
	.id_table	= efx_pci_table,
	.probe		= efx_pci_probe,
	.remove		= efx_pci_remove,
2850
	.driver.pm	= &efx_pm_ops,
2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872
};

/**************************************************************************
 *
 * Kernel module interface
 *
 *************************************************************************/

module_param(interrupt_mode, uint, 0444);
MODULE_PARM_DESC(interrupt_mode,
		 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");

static int __init efx_init_module(void)
{
	int rc;

	printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n");

	rc = register_netdevice_notifier(&efx_netdev_notifier);
	if (rc)
		goto err_notifier;

2873 2874 2875 2876
	rc = efx_init_sriov();
	if (rc)
		goto err_sriov;

2877 2878 2879 2880 2881
	reset_workqueue = create_singlethread_workqueue("sfc_reset");
	if (!reset_workqueue) {
		rc = -ENOMEM;
		goto err_reset;
	}
2882 2883 2884 2885 2886 2887 2888 2889

	rc = pci_register_driver(&efx_pci_driver);
	if (rc < 0)
		goto err_pci;

	return 0;

 err_pci:
2890 2891
	destroy_workqueue(reset_workqueue);
 err_reset:
2892 2893
	efx_fini_sriov();
 err_sriov:
2894 2895 2896 2897 2898 2899 2900 2901 2902 2903
	unregister_netdevice_notifier(&efx_netdev_notifier);
 err_notifier:
	return rc;
}

static void __exit efx_exit_module(void)
{
	printk(KERN_INFO "Solarflare NET driver unloading\n");

	pci_unregister_driver(&efx_pci_driver);
2904
	destroy_workqueue(reset_workqueue);
2905
	efx_fini_sriov();
2906 2907 2908 2909 2910 2911 2912
	unregister_netdevice_notifier(&efx_netdev_notifier);

}

module_init(efx_init_module);
module_exit(efx_exit_module);

2913 2914
MODULE_AUTHOR("Solarflare Communications and "
	      "Michael Brown <mbrown@fensystems.co.uk>");
2915 2916 2917
MODULE_DESCRIPTION("Solarflare Communications network driver");
MODULE_LICENSE("GPL");
MODULE_DEVICE_TABLE(pci, efx_pci_table);