efx.c 60.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/****************************************************************************
 * Driver for Solarflare Solarstorm network controllers and boards
 * Copyright 2005-2006 Fen Systems Ltd.
 * Copyright 2005-2008 Solarflare Communications Inc.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation, incorporated herein by reference.
 */

#include <linux/module.h>
#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/delay.h>
#include <linux/notifier.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/in.h>
#include <linux/crc32.h>
#include <linux/ethtool.h>
22
#include <linux/topology.h>
23 24 25 26 27
#include "net_driver.h"
#include "efx.h"
#include "mdio_10g.h"
#include "falcon.h"

28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
/**************************************************************************
 *
 * Type name strings
 *
 **************************************************************************
 */

/* Loopback mode names (see LOOPBACK_MODE()) */
const unsigned int efx_loopback_mode_max = LOOPBACK_MAX;
const char *efx_loopback_mode_names[] = {
	[LOOPBACK_NONE]		= "NONE",
	[LOOPBACK_GMAC]		= "GMAC",
	[LOOPBACK_XGMII]	= "XGMII",
	[LOOPBACK_XGXS]		= "XGXS",
	[LOOPBACK_XAUI]  	= "XAUI",
	[LOOPBACK_GPHY]		= "GPHY",
	[LOOPBACK_PHYXS]	= "PHYXS",
	[LOOPBACK_PCS]	 	= "PCS",
	[LOOPBACK_PMAPMD] 	= "PMA/PMD",
	[LOOPBACK_NETWORK]	= "NETWORK",
};

/* Interrupt mode names (see INT_MODE())) */
const unsigned int efx_interrupt_mode_max = EFX_INT_MODE_MAX;
const char *efx_interrupt_mode_names[] = {
	[EFX_INT_MODE_MSIX]   = "MSI-X",
	[EFX_INT_MODE_MSI]    = "MSI",
	[EFX_INT_MODE_LEGACY] = "legacy",
};

const unsigned int efx_reset_type_max = RESET_TYPE_MAX;
const char *efx_reset_type_names[] = {
	[RESET_TYPE_INVISIBLE]     = "INVISIBLE",
	[RESET_TYPE_ALL]           = "ALL",
	[RESET_TYPE_WORLD]         = "WORLD",
	[RESET_TYPE_DISABLE]       = "DISABLE",
	[RESET_TYPE_TX_WATCHDOG]   = "TX_WATCHDOG",
	[RESET_TYPE_INT_ERROR]     = "INT_ERROR",
	[RESET_TYPE_RX_RECOVERY]   = "RX_RECOVERY",
	[RESET_TYPE_RX_DESC_FETCH] = "RX_DESC_FETCH",
	[RESET_TYPE_TX_DESC_FETCH] = "TX_DESC_FETCH",
	[RESET_TYPE_TX_SKIP]       = "TX_SKIP",
};

72 73 74 75 76 77 78 79 80
#define EFX_MAX_MTU (9 * 1024)

/* RX slow fill workqueue. If memory allocation fails in the fast path,
 * a work item is pushed onto this work queue to retry the allocation later,
 * to avoid the NIC being starved of RX buffers. Since this is a per cpu
 * workqueue, there is nothing to be gained in making it per NIC
 */
static struct workqueue_struct *refill_workqueue;

81 82 83 84 85 86
/* Reset workqueue. If any NIC has a hardware failure then a reset will be
 * queued onto this work queue. This is not a per-nic work queue, because
 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised.
 */
static struct workqueue_struct *reset_workqueue;

87 88 89 90 91 92 93 94 95
/**************************************************************************
 *
 * Configurable values
 *
 *************************************************************************/

/*
 * Use separate channels for TX and RX events
 *
96 97
 * Set this to 1 to use separate channels for TX and RX. It allows us
 * to control interrupt affinity separately for TX and RX.
98
 *
99
 * This is only used in MSI-X interrupt mode
100
 */
101 102 103 104
static unsigned int separate_tx_channels;
module_param(separate_tx_channels, uint, 0644);
MODULE_PARM_DESC(separate_tx_channels,
		 "Use separate channels for TX and RX");
105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162

/* This is the weight assigned to each of the (per-channel) virtual
 * NAPI devices.
 */
static int napi_weight = 64;

/* This is the time (in jiffies) between invocations of the hardware
 * monitor, which checks for known hardware bugs and resets the
 * hardware and driver as necessary.
 */
unsigned int efx_monitor_interval = 1 * HZ;

/* This controls whether or not the driver will initialise devices
 * with invalid MAC addresses stored in the EEPROM or flash.  If true,
 * such devices will be initialised with a random locally-generated
 * MAC address.  This allows for loading the sfc_mtd driver to
 * reprogram the flash, even if the flash contents (including the MAC
 * address) have previously been erased.
 */
static unsigned int allow_bad_hwaddr;

/* Initial interrupt moderation settings.  They can be modified after
 * module load with ethtool.
 *
 * The default for RX should strike a balance between increasing the
 * round-trip latency and reducing overhead.
 */
static unsigned int rx_irq_mod_usec = 60;

/* Initial interrupt moderation settings.  They can be modified after
 * module load with ethtool.
 *
 * This default is chosen to ensure that a 10G link does not go idle
 * while a TX queue is stopped after it has become full.  A queue is
 * restarted when it drops below half full.  The time this takes (assuming
 * worst case 3 descriptors per packet and 1024 descriptors) is
 *   512 / 3 * 1.2 = 205 usec.
 */
static unsigned int tx_irq_mod_usec = 150;

/* This is the first interrupt mode to try out of:
 * 0 => MSI-X
 * 1 => MSI
 * 2 => legacy
 */
static unsigned int interrupt_mode;

/* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
 * i.e. the number of CPUs among which we may distribute simultaneous
 * interrupt handling.
 *
 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
 * The default (0) means to assign an interrupt to each package (level II cache)
 */
static unsigned int rss_cpus;
module_param(rss_cpus, uint, 0444);
MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");

163 164 165 166
static int phy_flash_cfg;
module_param(phy_flash_cfg, int, 0644);
MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");

167 168 169 170 171 172 173 174 175 176
static unsigned irq_adapt_low_thresh = 10000;
module_param(irq_adapt_low_thresh, uint, 0644);
MODULE_PARM_DESC(irq_adapt_low_thresh,
		 "Threshold score for reducing IRQ moderation");

static unsigned irq_adapt_high_thresh = 20000;
module_param(irq_adapt_high_thresh, uint, 0644);
MODULE_PARM_DESC(irq_adapt_high_thresh,
		 "Threshold score for increasing IRQ moderation");

177 178 179 180 181 182 183 184 185 186 187 188
/**************************************************************************
 *
 * Utility functions and prototypes
 *
 *************************************************************************/
static void efx_remove_channel(struct efx_channel *channel);
static void efx_remove_port(struct efx_nic *efx);
static void efx_fini_napi(struct efx_nic *efx);
static void efx_fini_channels(struct efx_nic *efx);

#define EFX_ASSERT_RESET_SERIALISED(efx)		\
	do {						\
189 190
		if ((efx->state == STATE_RUNNING) ||	\
		    (efx->state == STATE_DISABLED))	\
191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
			ASSERT_RTNL();			\
	} while (0)

/**************************************************************************
 *
 * Event queue processing
 *
 *************************************************************************/

/* Process channel's event queue
 *
 * This function is responsible for processing the event queue of a
 * single channel.  The caller must guarantee that this function will
 * never be concurrently called more than once on the same channel,
 * though different channels may be being processed concurrently.
 */
207
static int efx_process_channel(struct efx_channel *channel, int rx_quota)
208
{
B
Ben Hutchings 已提交
209 210
	struct efx_nic *efx = channel->efx;
	int rx_packets;
211

B
Ben Hutchings 已提交
212
	if (unlikely(efx->reset_pending != RESET_TYPE_NONE ||
213
		     !channel->enabled))
B
Ben Hutchings 已提交
214
		return 0;
215

B
Ben Hutchings 已提交
216 217 218
	rx_packets = falcon_process_eventq(channel, rx_quota);
	if (rx_packets == 0)
		return 0;
219 220 221 222 223 224 225 226 227 228

	/* Deliver last RX packet. */
	if (channel->rx_pkt) {
		__efx_rx_packet(channel, channel->rx_pkt,
				channel->rx_pkt_csummed);
		channel->rx_pkt = NULL;
	}

	efx_rx_strategy(channel);

B
Ben Hutchings 已提交
229
	efx_fast_push_rx_descriptors(&efx->rx_queue[channel->channel]);
230

B
Ben Hutchings 已提交
231
	return rx_packets;
232 233 234 235 236 237 238 239 240 241
}

/* Mark channel as finished processing
 *
 * Note that since we will not receive further interrupts for this
 * channel before we finish processing and call the eventq_read_ack()
 * method, there is no need to use the interrupt hold-off timers.
 */
static inline void efx_channel_processed(struct efx_channel *channel)
{
242 243 244
	/* The interrupt handler for this channel may set work_pending
	 * as soon as we acknowledge the events we've seen.  Make sure
	 * it's cleared before then. */
245
	channel->work_pending = false;
246 247
	smp_wmb();

248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264
	falcon_eventq_read_ack(channel);
}

/* NAPI poll handler
 *
 * NAPI guarantees serialisation of polls of the same device, which
 * provides the guarantee required by efx_process_channel().
 */
static int efx_poll(struct napi_struct *napi, int budget)
{
	struct efx_channel *channel =
		container_of(napi, struct efx_channel, napi_str);
	int rx_packets;

	EFX_TRACE(channel->efx, "channel %d NAPI poll executing on CPU %d\n",
		  channel->channel, raw_smp_processor_id());

B
Ben Hutchings 已提交
265
	rx_packets = efx_process_channel(channel, budget);
266 267

	if (rx_packets < budget) {
268 269 270 271 272 273 274
		struct efx_nic *efx = channel->efx;

		if (channel->used_flags & EFX_USED_BY_RX &&
		    efx->irq_rx_adaptive &&
		    unlikely(++channel->irq_count == 1000)) {
			if (unlikely(channel->irq_mod_score <
				     irq_adapt_low_thresh)) {
275 276 277 278
				if (channel->irq_moderation > 1) {
					channel->irq_moderation -= 1;
					falcon_set_int_moderation(channel);
				}
279 280
			} else if (unlikely(channel->irq_mod_score >
					    irq_adapt_high_thresh)) {
281 282 283 284 285
				if (channel->irq_moderation <
				    efx->irq_rx_moderation) {
					channel->irq_moderation += 1;
					falcon_set_int_moderation(channel);
				}
286 287 288 289 290
			}
			channel->irq_count = 0;
			channel->irq_mod_score = 0;
		}

291
		/* There is no race here; although napi_disable() will
292
		 * only wait for napi_complete(), this isn't a problem
293 294 295
		 * since efx_channel_processed() will have no effect if
		 * interrupts have already been disabled.
		 */
296
		napi_complete(napi);
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321
		efx_channel_processed(channel);
	}

	return rx_packets;
}

/* Process the eventq of the specified channel immediately on this CPU
 *
 * Disable hardware generated interrupts, wait for any existing
 * processing to finish, then directly poll (and ack ) the eventq.
 * Finally reenable NAPI and interrupts.
 *
 * Since we are touching interrupts the caller should hold the suspend lock
 */
void efx_process_channel_now(struct efx_channel *channel)
{
	struct efx_nic *efx = channel->efx;

	BUG_ON(!channel->used_flags);
	BUG_ON(!channel->enabled);

	/* Disable interrupts and wait for ISRs to complete */
	falcon_disable_interrupts(efx);
	if (efx->legacy_irq)
		synchronize_irq(efx->legacy_irq);
322
	if (channel->irq)
323 324 325 326 327 328
		synchronize_irq(channel->irq);

	/* Wait for any NAPI processing to complete */
	napi_disable(&channel->napi_str);

	/* Poll the channel */
329
	efx_process_channel(channel, EFX_EVQ_SIZE);
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351

	/* Ack the eventq. This may cause an interrupt to be generated
	 * when they are reenabled */
	efx_channel_processed(channel);

	napi_enable(&channel->napi_str);
	falcon_enable_interrupts(efx);
}

/* Create event queue
 * Event queue memory allocations are done only once.  If the channel
 * is reset, the memory buffer will be reused; this guards against
 * errors during channel reset and also simplifies interrupt handling.
 */
static int efx_probe_eventq(struct efx_channel *channel)
{
	EFX_LOG(channel->efx, "chan %d create event queue\n", channel->channel);

	return falcon_probe_eventq(channel);
}

/* Prepare channel's event queue */
352
static void efx_init_eventq(struct efx_channel *channel)
353 354 355 356 357
{
	EFX_LOG(channel->efx, "chan %d init event queue\n", channel->channel);

	channel->eventq_read_ptr = 0;

358
	falcon_init_eventq(channel);
359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419
}

static void efx_fini_eventq(struct efx_channel *channel)
{
	EFX_LOG(channel->efx, "chan %d fini event queue\n", channel->channel);

	falcon_fini_eventq(channel);
}

static void efx_remove_eventq(struct efx_channel *channel)
{
	EFX_LOG(channel->efx, "chan %d remove event queue\n", channel->channel);

	falcon_remove_eventq(channel);
}

/**************************************************************************
 *
 * Channel handling
 *
 *************************************************************************/

static int efx_probe_channel(struct efx_channel *channel)
{
	struct efx_tx_queue *tx_queue;
	struct efx_rx_queue *rx_queue;
	int rc;

	EFX_LOG(channel->efx, "creating channel %d\n", channel->channel);

	rc = efx_probe_eventq(channel);
	if (rc)
		goto fail1;

	efx_for_each_channel_tx_queue(tx_queue, channel) {
		rc = efx_probe_tx_queue(tx_queue);
		if (rc)
			goto fail2;
	}

	efx_for_each_channel_rx_queue(rx_queue, channel) {
		rc = efx_probe_rx_queue(rx_queue);
		if (rc)
			goto fail3;
	}

	channel->n_rx_frm_trunc = 0;

	return 0;

 fail3:
	efx_for_each_channel_rx_queue(rx_queue, channel)
		efx_remove_rx_queue(rx_queue);
 fail2:
	efx_for_each_channel_tx_queue(tx_queue, channel)
		efx_remove_tx_queue(tx_queue);
 fail1:
	return rc;
}


420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440
static void efx_set_channel_names(struct efx_nic *efx)
{
	struct efx_channel *channel;
	const char *type = "";
	int number;

	efx_for_each_channel(channel, efx) {
		number = channel->channel;
		if (efx->n_channels > efx->n_rx_queues) {
			if (channel->channel < efx->n_rx_queues) {
				type = "-rx";
			} else {
				type = "-tx";
				number -= efx->n_rx_queues;
			}
		}
		snprintf(channel->name, sizeof(channel->name),
			 "%s%s-%d", efx->name, type, number);
	}
}

441 442 443 444
/* Channels are shutdown and reinitialised whilst the NIC is running
 * to propagate configuration changes (mtu, checksum offload), or
 * to clear hardware error conditions
 */
445
static void efx_init_channels(struct efx_nic *efx)
446 447 448 449 450
{
	struct efx_tx_queue *tx_queue;
	struct efx_rx_queue *rx_queue;
	struct efx_channel *channel;

451 452 453 454 455 456 457 458
	/* Calculate the rx buffer allocation parameters required to
	 * support the current MTU, including padding for header
	 * alignment and overruns.
	 */
	efx->rx_buffer_len = (max(EFX_PAGE_IP_ALIGN, NET_IP_ALIGN) +
			      EFX_MAX_FRAME_LEN(efx->net_dev->mtu) +
			      efx->type->rx_buffer_padding);
	efx->rx_buffer_order = get_order(efx->rx_buffer_len);
459 460 461 462 463

	/* Initialise the channels */
	efx_for_each_channel(channel, efx) {
		EFX_LOG(channel->efx, "init chan %d\n", channel->channel);

464
		efx_init_eventq(channel);
465

466 467
		efx_for_each_channel_tx_queue(tx_queue, channel)
			efx_init_tx_queue(tx_queue);
468 469 470 471

		/* The rx buffer allocation strategy is MTU dependent */
		efx_rx_strategy(channel);

472 473
		efx_for_each_channel_rx_queue(rx_queue, channel)
			efx_init_rx_queue(rx_queue);
474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490

		WARN_ON(channel->rx_pkt != NULL);
		efx_rx_strategy(channel);
	}
}

/* This enables event queue processing and packet transmission.
 *
 * Note that this function is not allowed to fail, since that would
 * introduce too much complexity into the suspend/resume path.
 */
static void efx_start_channel(struct efx_channel *channel)
{
	struct efx_rx_queue *rx_queue;

	EFX_LOG(channel->efx, "starting chan %d\n", channel->channel);

491 492 493
	/* The interrupt handler for this channel may set work_pending
	 * as soon as we enable it.  Make sure it's cleared before
	 * then.  Similarly, make sure it sees the enabled flag set. */
494 495
	channel->work_pending = false;
	channel->enabled = true;
496
	smp_wmb();
497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517

	napi_enable(&channel->napi_str);

	/* Load up RX descriptors */
	efx_for_each_channel_rx_queue(rx_queue, channel)
		efx_fast_push_rx_descriptors(rx_queue);
}

/* This disables event queue processing and packet transmission.
 * This function does not guarantee that all queue processing
 * (e.g. RX refill) is complete.
 */
static void efx_stop_channel(struct efx_channel *channel)
{
	struct efx_rx_queue *rx_queue;

	if (!channel->enabled)
		return;

	EFX_LOG(channel->efx, "stop chan %d\n", channel->channel);

518
	channel->enabled = false;
519 520 521 522 523 524 525 526 527 528 529 530 531 532
	napi_disable(&channel->napi_str);

	/* Ensure that any worker threads have exited or will be no-ops */
	efx_for_each_channel_rx_queue(rx_queue, channel) {
		spin_lock_bh(&rx_queue->add_lock);
		spin_unlock_bh(&rx_queue->add_lock);
	}
}

static void efx_fini_channels(struct efx_nic *efx)
{
	struct efx_channel *channel;
	struct efx_tx_queue *tx_queue;
	struct efx_rx_queue *rx_queue;
533
	int rc;
534 535 536 537

	EFX_ASSERT_RESET_SERIALISED(efx);
	BUG_ON(efx->port_enabled);

538 539 540 541 542 543
	rc = falcon_flush_queues(efx);
	if (rc)
		EFX_ERR(efx, "failed to flush queues\n");
	else
		EFX_LOG(efx, "successfully flushed all queues\n");

544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
	efx_for_each_channel(channel, efx) {
		EFX_LOG(channel->efx, "shut down chan %d\n", channel->channel);

		efx_for_each_channel_rx_queue(rx_queue, channel)
			efx_fini_rx_queue(rx_queue);
		efx_for_each_channel_tx_queue(tx_queue, channel)
			efx_fini_tx_queue(tx_queue);
		efx_fini_eventq(channel);
	}
}

static void efx_remove_channel(struct efx_channel *channel)
{
	struct efx_tx_queue *tx_queue;
	struct efx_rx_queue *rx_queue;

	EFX_LOG(channel->efx, "destroy chan %d\n", channel->channel);

	efx_for_each_channel_rx_queue(rx_queue, channel)
		efx_remove_rx_queue(rx_queue);
	efx_for_each_channel_tx_queue(tx_queue, channel)
		efx_remove_tx_queue(tx_queue);
	efx_remove_eventq(channel);

	channel->used_flags = 0;
}

void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue, int delay)
{
	queue_delayed_work(refill_workqueue, &rx_queue->work, delay);
}

/**************************************************************************
 *
 * Port handling
 *
 **************************************************************************/

/* This ensures that the kernel is kept informed (via
 * netif_carrier_on/off) of the link status, and also maintains the
 * link status's stop on the port's TX queue.
 */
static void efx_link_status_changed(struct efx_nic *efx)
{
588 589
	struct efx_link_state *link_state = &efx->link_state;

590 591 592 593 594 595 596
	/* SFC Bug 5356: A net_dev notifier is registered, so we must ensure
	 * that no events are triggered between unregister_netdev() and the
	 * driver unloading. A more general condition is that NETDEV_CHANGE
	 * can only be generated between NETDEV_UP and NETDEV_DOWN */
	if (!netif_running(efx->net_dev))
		return;

B
Ben Hutchings 已提交
597 598 599 600 601
	if (efx->port_inhibited) {
		netif_carrier_off(efx->net_dev);
		return;
	}

602
	if (link_state->up != netif_carrier_ok(efx->net_dev)) {
603 604
		efx->n_link_state_changes++;

605
		if (link_state->up)
606 607 608 609 610 611
			netif_carrier_on(efx->net_dev);
		else
			netif_carrier_off(efx->net_dev);
	}

	/* Status message for kernel log */
612
	if (link_state->up) {
B
Ben Hutchings 已提交
613
		EFX_INFO(efx, "link up at %uMbps %s-duplex (MTU %d)%s\n",
614
			 link_state->speed, link_state->fd ? "full" : "half",
615 616 617 618 619 620 621 622
			 efx->net_dev->mtu,
			 (efx->promiscuous ? " [PROMISC]" : ""));
	} else {
		EFX_INFO(efx, "link down\n");
	}

}

623 624
static void efx_fini_port(struct efx_nic *efx);

625 626
/* This call reinitialises the MAC to pick up new PHY settings. The
 * caller must hold the mac_lock */
B
Ben Hutchings 已提交
627
void __efx_reconfigure_port(struct efx_nic *efx)
628 629 630 631 632 633
{
	WARN_ON(!mutex_is_locked(&efx->mac_lock));

	EFX_LOG(efx, "reconfiguring MAC from PHY settings on CPU %d\n",
		raw_smp_processor_id());

634 635 636 637 638 639
	/* Serialise the promiscuous flag with efx_set_multicast_list. */
	if (efx_dev_registered(efx)) {
		netif_addr_lock_bh(efx->net_dev);
		netif_addr_unlock_bh(efx->net_dev);
	}

640 641 642 643 644 645 646 647 648 649 650 651 652
	falcon_deconfigure_mac_wrapper(efx);

	/* Reconfigure the PHY, disabling transmit in mac level loopback. */
	if (LOOPBACK_INTERNAL(efx))
		efx->phy_mode |= PHY_MODE_TX_DISABLED;
	else
		efx->phy_mode &= ~PHY_MODE_TX_DISABLED;
	efx->phy_op->reconfigure(efx);

	if (falcon_switch_mac(efx))
		goto fail;

	efx->mac_op->reconfigure(efx);
653 654 655

	/* Inform kernel of loss/gain of carrier */
	efx_link_status_changed(efx);
656 657 658 659
	return;

fail:
	EFX_ERR(efx, "failed to reconfigure MAC\n");
660 661
	efx->port_enabled = false;
	efx_fini_port(efx);
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677
}

/* Reinitialise the MAC to pick up new PHY settings, even if the port is
 * disabled. */
void efx_reconfigure_port(struct efx_nic *efx)
{
	EFX_ASSERT_RESET_SERIALISED(efx);

	mutex_lock(&efx->mac_lock);
	__efx_reconfigure_port(efx);
	mutex_unlock(&efx->mac_lock);
}

/* Asynchronous efx_reconfigure_port work item. To speed up efx_flush_all()
 * we don't efx_reconfigure_port() if the port is disabled. Care is taken
 * in efx_stop_all() and efx_start_port() to prevent PHY events being lost */
678
static void efx_phy_work(struct work_struct *data)
679
{
680
	struct efx_nic *efx = container_of(data, struct efx_nic, phy_work);
681 682 683 684 685 686 687

	mutex_lock(&efx->mac_lock);
	if (efx->port_enabled)
		__efx_reconfigure_port(efx);
	mutex_unlock(&efx->mac_lock);
}

688 689 690 691 692 693 694 695 696 697
static void efx_mac_work(struct work_struct *data)
{
	struct efx_nic *efx = container_of(data, struct efx_nic, mac_work);

	mutex_lock(&efx->mac_lock);
	if (efx->port_enabled)
		efx->mac_op->irq(efx);
	mutex_unlock(&efx->mac_lock);
}

698 699 700 701 702 703 704 705 706 707 708
static int efx_probe_port(struct efx_nic *efx)
{
	int rc;

	EFX_LOG(efx, "create port\n");

	/* Connect up MAC/PHY operations table and read MAC address */
	rc = falcon_probe_port(efx);
	if (rc)
		goto err;

709 710 711
	if (phy_flash_cfg)
		efx->phy_mode = PHY_MODE_SPECIAL;

712 713 714 715
	/* Sanity check MAC address */
	if (is_valid_ether_addr(efx->mac_address)) {
		memcpy(efx->net_dev->dev_addr, efx->mac_address, ETH_ALEN);
	} else {
J
Johannes Berg 已提交
716 717
		EFX_ERR(efx, "invalid MAC address %pM\n",
			efx->mac_address);
718 719 720 721 722
		if (!allow_bad_hwaddr) {
			rc = -EINVAL;
			goto err;
		}
		random_ether_addr(efx->net_dev->dev_addr);
J
Johannes Berg 已提交
723 724
		EFX_INFO(efx, "using locally-generated MAC %pM\n",
			 efx->net_dev->dev_addr);
725 726 727 728 729 730 731 732 733 734 735 736 737 738 739
	}

	return 0;

 err:
	efx_remove_port(efx);
	return rc;
}

static int efx_init_port(struct efx_nic *efx)
{
	int rc;

	EFX_LOG(efx, "init port\n");

740
	rc = efx->phy_op->init(efx);
741 742
	if (rc)
		return rc;
743
	mutex_lock(&efx->mac_lock);
744
	efx->phy_op->reconfigure(efx);
745 746 747 748 749
	rc = falcon_switch_mac(efx);
	mutex_unlock(&efx->mac_lock);
	if (rc)
		goto fail;
	efx->mac_op->reconfigure(efx);
750

751
	efx->port_initialized = true;
752
	efx_stats_enable(efx);
753
	return 0;
754 755 756 757

fail:
	efx->phy_op->fini(efx);
	return rc;
758 759 760 761
}

/* Allow efx_reconfigure_port() to be scheduled, and close the window
 * between efx_stop_port and efx_flush_all whereby a previously scheduled
762
 * efx_phy_work()/efx_mac_work() may have been cancelled */
763 764 765 766 767 768
static void efx_start_port(struct efx_nic *efx)
{
	EFX_LOG(efx, "start port\n");
	BUG_ON(efx->port_enabled);

	mutex_lock(&efx->mac_lock);
769
	efx->port_enabled = true;
770
	__efx_reconfigure_port(efx);
771
	efx->mac_op->irq(efx);
772 773 774
	mutex_unlock(&efx->mac_lock);
}

775 776 777 778
/* Prevent efx_phy_work, efx_mac_work, and efx_monitor() from executing,
 * and efx_set_multicast_list() from scheduling efx_phy_work. efx_phy_work
 * and efx_mac_work may still be scheduled via NAPI processing until
 * efx_flush_all() is called */
779 780 781 782 783
static void efx_stop_port(struct efx_nic *efx)
{
	EFX_LOG(efx, "stop port\n");

	mutex_lock(&efx->mac_lock);
784
	efx->port_enabled = false;
785 786 787
	mutex_unlock(&efx->mac_lock);

	/* Serialise against efx_set_multicast_list() */
788
	if (efx_dev_registered(efx)) {
789 790
		netif_addr_lock_bh(efx->net_dev);
		netif_addr_unlock_bh(efx->net_dev);
791 792 793 794 795 796 797 798 799 800
	}
}

static void efx_fini_port(struct efx_nic *efx)
{
	EFX_LOG(efx, "shut down port\n");

	if (!efx->port_initialized)
		return;

801
	efx_stats_disable(efx);
802
	efx->phy_op->fini(efx);
803
	efx->port_initialized = false;
804

805
	efx->link_state.up = false;
806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864
	efx_link_status_changed(efx);
}

static void efx_remove_port(struct efx_nic *efx)
{
	EFX_LOG(efx, "destroying port\n");

	falcon_remove_port(efx);
}

/**************************************************************************
 *
 * NIC handling
 *
 **************************************************************************/

/* This configures the PCI device to enable I/O and DMA. */
static int efx_init_io(struct efx_nic *efx)
{
	struct pci_dev *pci_dev = efx->pci_dev;
	dma_addr_t dma_mask = efx->type->max_dma_mask;
	int rc;

	EFX_LOG(efx, "initialising I/O\n");

	rc = pci_enable_device(pci_dev);
	if (rc) {
		EFX_ERR(efx, "failed to enable PCI device\n");
		goto fail1;
	}

	pci_set_master(pci_dev);

	/* Set the PCI DMA mask.  Try all possibilities from our
	 * genuine mask down to 32 bits, because some architectures
	 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit
	 * masks event though they reject 46 bit masks.
	 */
	while (dma_mask > 0x7fffffffUL) {
		if (pci_dma_supported(pci_dev, dma_mask) &&
		    ((rc = pci_set_dma_mask(pci_dev, dma_mask)) == 0))
			break;
		dma_mask >>= 1;
	}
	if (rc) {
		EFX_ERR(efx, "could not find a suitable DMA mask\n");
		goto fail2;
	}
	EFX_LOG(efx, "using DMA mask %llx\n", (unsigned long long) dma_mask);
	rc = pci_set_consistent_dma_mask(pci_dev, dma_mask);
	if (rc) {
		/* pci_set_consistent_dma_mask() is not *allowed* to
		 * fail with a mask that pci_set_dma_mask() accepted,
		 * but just in case...
		 */
		EFX_ERR(efx, "failed to set consistent DMA mask\n");
		goto fail2;
	}

865 866
	efx->membase_phys = pci_resource_start(efx->pci_dev, EFX_MEM_BAR);
	rc = pci_request_region(pci_dev, EFX_MEM_BAR, "sfc");
867 868 869 870 871 872 873 874
	if (rc) {
		EFX_ERR(efx, "request for memory BAR failed\n");
		rc = -EIO;
		goto fail3;
	}
	efx->membase = ioremap_nocache(efx->membase_phys,
				       efx->type->mem_map_size);
	if (!efx->membase) {
875
		EFX_ERR(efx, "could not map memory BAR at %llx+%x\n",
876
			(unsigned long long)efx->membase_phys,
877 878 879 880
			efx->type->mem_map_size);
		rc = -ENOMEM;
		goto fail4;
	}
881 882
	EFX_LOG(efx, "memory BAR at %llx+%x (virtual %p)\n",
		(unsigned long long)efx->membase_phys,
883
		efx->type->mem_map_size, efx->membase);
884 885 886 887

	return 0;

 fail4:
888
	pci_release_region(efx->pci_dev, EFX_MEM_BAR);
889
 fail3:
890
	efx->membase_phys = 0;
891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906
 fail2:
	pci_disable_device(efx->pci_dev);
 fail1:
	return rc;
}

static void efx_fini_io(struct efx_nic *efx)
{
	EFX_LOG(efx, "shutting down I/O\n");

	if (efx->membase) {
		iounmap(efx->membase);
		efx->membase = NULL;
	}

	if (efx->membase_phys) {
907
		pci_release_region(efx->pci_dev, EFX_MEM_BAR);
908
		efx->membase_phys = 0;
909 910 911 912 913
	}

	pci_disable_device(efx->pci_dev);
}

914 915 916 917 918
/* Get number of RX queues wanted.  Return number of online CPU
 * packages in the expectation that an IRQ balancer will spread
 * interrupts across them. */
static int efx_wanted_rx_queues(void)
{
R
Rusty Russell 已提交
919
	cpumask_var_t core_mask;
920 921 922
	int count;
	int cpu;

923
	if (unlikely(!zalloc_cpumask_var(&core_mask, GFP_KERNEL))) {
R
Rusty Russell 已提交
924
		printk(KERN_WARNING
925
		       "sfc: RSS disabled due to allocation failure\n");
R
Rusty Russell 已提交
926 927 928
		return 1;
	}

929 930
	count = 0;
	for_each_online_cpu(cpu) {
R
Rusty Russell 已提交
931
		if (!cpumask_test_cpu(cpu, core_mask)) {
932
			++count;
R
Rusty Russell 已提交
933
			cpumask_or(core_mask, core_mask,
934
				   topology_core_cpumask(cpu));
935 936 937
		}
	}

R
Rusty Russell 已提交
938
	free_cpumask_var(core_mask);
939 940 941 942 943 944
	return count;
}

/* Probe the number and type of interrupts we are able to obtain, and
 * the resulting numbers of channels and RX queues.
 */
945 946
static void efx_probe_interrupts(struct efx_nic *efx)
{
947 948
	int max_channels =
		min_t(int, efx->type->phys_addr_channels, EFX_MAX_CHANNELS);
949 950 951
	int rc, i;

	if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
952 953
		struct msix_entry xentries[EFX_MAX_CHANNELS];
		int wanted_ints;
954
		int rx_queues;
955

956 957 958 959
		/* We want one RX queue and interrupt per CPU package
		 * (or as specified by the rss_cpus module parameter).
		 * We will need one channel per interrupt.
		 */
960 961 962
		rx_queues = rss_cpus ? rss_cpus : efx_wanted_rx_queues();
		wanted_ints = rx_queues + (separate_tx_channels ? 1 : 0);
		wanted_ints = min(wanted_ints, max_channels);
963

964
		for (i = 0; i < wanted_ints; i++)
965
			xentries[i].entry = i;
966
		rc = pci_enable_msix(efx->pci_dev, xentries, wanted_ints);
967
		if (rc > 0) {
968 969 970 971 972
			EFX_ERR(efx, "WARNING: Insufficient MSI-X vectors"
				" available (%d < %d).\n", rc, wanted_ints);
			EFX_ERR(efx, "WARNING: Performance may be reduced.\n");
			EFX_BUG_ON_PARANOID(rc >= wanted_ints);
			wanted_ints = rc;
973
			rc = pci_enable_msix(efx->pci_dev, xentries,
974
					     wanted_ints);
975 976 977
		}

		if (rc == 0) {
978 979 980
			efx->n_rx_queues = min(rx_queues, wanted_ints);
			efx->n_channels = wanted_ints;
			for (i = 0; i < wanted_ints; i++)
981 982 983 984 985 986 987 988 989 990
				efx->channel[i].irq = xentries[i].vector;
		} else {
			/* Fall back to single channel MSI */
			efx->interrupt_mode = EFX_INT_MODE_MSI;
			EFX_ERR(efx, "could not enable MSI-X\n");
		}
	}

	/* Try single interrupt MSI */
	if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
991
		efx->n_rx_queues = 1;
992
		efx->n_channels = 1;
993 994 995 996 997 998 999 1000 1001 1002 1003
		rc = pci_enable_msi(efx->pci_dev);
		if (rc == 0) {
			efx->channel[0].irq = efx->pci_dev->irq;
		} else {
			EFX_ERR(efx, "could not enable MSI\n");
			efx->interrupt_mode = EFX_INT_MODE_LEGACY;
		}
	}

	/* Assume legacy interrupts */
	if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
1004
		efx->n_rx_queues = 1;
1005
		efx->n_channels = 1 + (separate_tx_channels ? 1 : 0);
1006 1007 1008 1009 1010 1011 1012 1013 1014
		efx->legacy_irq = efx->pci_dev->irq;
	}
}

static void efx_remove_interrupts(struct efx_nic *efx)
{
	struct efx_channel *channel;

	/* Remove MSI/MSI-X interrupts */
1015
	efx_for_each_channel(channel, efx)
1016 1017 1018 1019 1020 1021 1022 1023
		channel->irq = 0;
	pci_disable_msi(efx->pci_dev);
	pci_disable_msix(efx->pci_dev);

	/* Remove legacy interrupt */
	efx->legacy_irq = 0;
}

1024
static void efx_set_channels(struct efx_nic *efx)
1025 1026 1027 1028
{
	struct efx_tx_queue *tx_queue;
	struct efx_rx_queue *rx_queue;

1029
	efx_for_each_tx_queue(tx_queue, efx) {
1030 1031
		if (separate_tx_channels)
			tx_queue->channel = &efx->channel[efx->n_channels-1];
1032 1033 1034 1035
		else
			tx_queue->channel = &efx->channel[0];
		tx_queue->channel->used_flags |= EFX_USED_BY_TX;
	}
1036

1037 1038 1039
	efx_for_each_rx_queue(rx_queue, efx) {
		rx_queue->channel = &efx->channel[rx_queue->queue];
		rx_queue->channel->used_flags |= EFX_USED_BY_RX;
1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057
	}
}

static int efx_probe_nic(struct efx_nic *efx)
{
	int rc;

	EFX_LOG(efx, "creating NIC\n");

	/* Carry out hardware-type specific initialisation */
	rc = falcon_probe_nic(efx);
	if (rc)
		return rc;

	/* Determine the number of channels and RX queues by trying to hook
	 * in MSI-X interrupts. */
	efx_probe_interrupts(efx);

1058
	efx_set_channels(efx);
1059 1060

	/* Initialise the interrupt moderation settings */
1061
	efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true);
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107

	return 0;
}

static void efx_remove_nic(struct efx_nic *efx)
{
	EFX_LOG(efx, "destroying NIC\n");

	efx_remove_interrupts(efx);
	falcon_remove_nic(efx);
}

/**************************************************************************
 *
 * NIC startup/shutdown
 *
 *************************************************************************/

static int efx_probe_all(struct efx_nic *efx)
{
	struct efx_channel *channel;
	int rc;

	/* Create NIC */
	rc = efx_probe_nic(efx);
	if (rc) {
		EFX_ERR(efx, "failed to create NIC\n");
		goto fail1;
	}

	/* Create port */
	rc = efx_probe_port(efx);
	if (rc) {
		EFX_ERR(efx, "failed to create port\n");
		goto fail2;
	}

	/* Create channels */
	efx_for_each_channel(channel, efx) {
		rc = efx_probe_channel(channel);
		if (rc) {
			EFX_ERR(efx, "failed to create channel %d\n",
				channel->channel);
			goto fail3;
		}
	}
1108
	efx_set_channel_names(efx);
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138

	return 0;

 fail3:
	efx_for_each_channel(channel, efx)
		efx_remove_channel(channel);
	efx_remove_port(efx);
 fail2:
	efx_remove_nic(efx);
 fail1:
	return rc;
}

/* Called after previous invocation(s) of efx_stop_all, restarts the
 * port, kernel transmit queue, NAPI processing and hardware interrupts,
 * and ensures that the port is scheduled to be reconfigured.
 * This function is safe to call multiple times when the NIC is in any
 * state. */
static void efx_start_all(struct efx_nic *efx)
{
	struct efx_channel *channel;

	EFX_ASSERT_RESET_SERIALISED(efx);

	/* Check that it is appropriate to restart the interface. All
	 * of these flags are safe to read under just the rtnl lock */
	if (efx->port_enabled)
		return;
	if ((efx->state != STATE_RUNNING) && (efx->state != STATE_INIT))
		return;
1139
	if (efx_dev_registered(efx) && !netif_running(efx->net_dev))
1140 1141 1142 1143 1144
		return;

	/* Mark the port as enabled so port reconfigurations can start, then
	 * restart the transmit interface early so the watchdog timer stops */
	efx_start_port(efx);
1145 1146
	if (efx_dev_registered(efx))
		efx_wake_queue(efx);
1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169

	efx_for_each_channel(channel, efx)
		efx_start_channel(channel);

	falcon_enable_interrupts(efx);

	/* Start hardware monitor if we're in RUNNING */
	if (efx->state == STATE_RUNNING)
		queue_delayed_work(efx->workqueue, &efx->monitor_work,
				   efx_monitor_interval);
}

/* Flush all delayed work. Should only be called when no more delayed work
 * will be scheduled. This doesn't flush pending online resets (efx_reset),
 * since we're holding the rtnl_lock at this point. */
static void efx_flush_all(struct efx_nic *efx)
{
	struct efx_rx_queue *rx_queue;

	/* Make sure the hardware monitor is stopped */
	cancel_delayed_work_sync(&efx->monitor_work);

	/* Ensure that all RX slow refills are complete. */
1170
	efx_for_each_rx_queue(rx_queue, efx)
1171 1172 1173
		cancel_delayed_work_sync(&rx_queue->work);

	/* Stop scheduled port reconfigurations */
1174 1175
	cancel_work_sync(&efx->mac_work);
	cancel_work_sync(&efx->phy_work);
1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197

}

/* Quiesce hardware and software without bringing the link down.
 * Safe to call multiple times, when the nic and interface is in any
 * state. The caller is guaranteed to subsequently be in a position
 * to modify any hardware and software state they see fit without
 * taking locks. */
static void efx_stop_all(struct efx_nic *efx)
{
	struct efx_channel *channel;

	EFX_ASSERT_RESET_SERIALISED(efx);

	/* port_enabled can be read safely under the rtnl lock */
	if (!efx->port_enabled)
		return;

	/* Disable interrupts and wait for ISR to complete */
	falcon_disable_interrupts(efx);
	if (efx->legacy_irq)
		synchronize_irq(efx->legacy_irq);
1198
	efx_for_each_channel(channel, efx) {
1199 1200
		if (channel->irq)
			synchronize_irq(channel->irq);
1201
	}
1202 1203 1204 1205 1206 1207 1208 1209 1210 1211

	/* Stop all NAPI processing and synchronous rx refills */
	efx_for_each_channel(channel, efx)
		efx_stop_channel(channel);

	/* Stop all asynchronous port reconfigurations. Since all
	 * event processing has already been stopped, there is no
	 * window to loose phy events */
	efx_stop_port(efx);

1212
	/* Flush efx_phy_work, efx_mac_work, refill_workqueue, monitor_work */
1213 1214 1215 1216
	efx_flush_all(efx);

	/* Isolate the MAC from the TX and RX engines, so that queue
	 * flushes will complete in a timely fashion. */
1217 1218
	falcon_deconfigure_mac_wrapper(efx);
	msleep(10); /* Let the Rx FIFO drain */
1219 1220 1221 1222
	falcon_drain_tx_fifo(efx);

	/* Stop the kernel transmit interface late, so the watchdog
	 * timer isn't ticking over the flush */
1223
	if (efx_dev_registered(efx)) {
1224
		efx_stop_queue(efx);
1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
		netif_tx_lock_bh(efx->net_dev);
		netif_tx_unlock_bh(efx->net_dev);
	}
}

static void efx_remove_all(struct efx_nic *efx)
{
	struct efx_channel *channel;

	efx_for_each_channel(channel, efx)
		efx_remove_channel(channel);
	efx_remove_port(efx);
	efx_remove_nic(efx);
}

/**************************************************************************
 *
 * Interrupt moderation
 *
 **************************************************************************/

1246 1247 1248 1249 1250 1251 1252 1253 1254
static unsigned irq_mod_ticks(int usecs, int resolution)
{
	if (usecs <= 0)
		return 0; /* cannot receive interrupts ahead of time :-) */
	if (usecs < resolution)
		return 1; /* never round down to 0 */
	return usecs / resolution;
}

1255
/* Set interrupt moderation parameters */
1256 1257
void efx_init_irq_moderation(struct efx_nic *efx, int tx_usecs, int rx_usecs,
			     bool rx_adaptive)
1258 1259 1260
{
	struct efx_tx_queue *tx_queue;
	struct efx_rx_queue *rx_queue;
1261 1262
	unsigned tx_ticks = irq_mod_ticks(tx_usecs, FALCON_IRQ_MOD_RESOLUTION);
	unsigned rx_ticks = irq_mod_ticks(rx_usecs, FALCON_IRQ_MOD_RESOLUTION);
1263 1264 1265 1266

	EFX_ASSERT_RESET_SERIALISED(efx);

	efx_for_each_tx_queue(tx_queue, efx)
1267
		tx_queue->channel->irq_moderation = tx_ticks;
1268

1269
	efx->irq_rx_adaptive = rx_adaptive;
1270
	efx->irq_rx_moderation = rx_ticks;
1271
	efx_for_each_rx_queue(rx_queue, efx)
1272
		rx_queue->channel->irq_moderation = rx_ticks;
1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286
}

/**************************************************************************
 *
 * Hardware monitor
 *
 **************************************************************************/

/* Run periodically off the general workqueue. Serialised against
 * efx_reconfigure_port via the mac_lock */
static void efx_monitor(struct work_struct *data)
{
	struct efx_nic *efx = container_of(data, struct efx_nic,
					   monitor_work.work);
1287
	int rc;
1288 1289 1290 1291 1292 1293 1294

	EFX_TRACE(efx, "hardware monitor executing on CPU %d\n",
		  raw_smp_processor_id());

	/* If the mac_lock is already held then it is likely a port
	 * reconfiguration is already in place, which will likely do
	 * most of the work of check_hw() anyway. */
1295 1296 1297 1298
	if (!mutex_trylock(&efx->mac_lock))
		goto out_requeue;
	if (!efx->port_enabled)
		goto out_unlock;
1299
	rc = falcon_board(efx)->type->monitor(efx);
1300 1301 1302 1303 1304
	if (rc) {
		EFX_ERR(efx, "Board sensor %s; shutting down PHY\n",
			(rc == -ERANGE) ? "reported fault" : "failed");
		efx->phy_mode |= PHY_MODE_LOW_POWER;
		falcon_sim_phy_event(efx);
1305
	}
1306 1307
	efx->phy_op->poll(efx);
	efx->mac_op->poll(efx);
1308

1309
out_unlock:
1310
	mutex_unlock(&efx->mac_lock);
1311
out_requeue:
1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326
	queue_delayed_work(efx->workqueue, &efx->monitor_work,
			   efx_monitor_interval);
}

/**************************************************************************
 *
 * ioctls
 *
 *************************************************************************/

/* Net device ioctl
 * Context: process, rtnl_lock() held.
 */
static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)
{
1327
	struct efx_nic *efx = netdev_priv(net_dev);
1328
	struct mii_ioctl_data *data = if_mii(ifr);
1329 1330 1331

	EFX_ASSERT_RESET_SERIALISED(efx);

1332 1333 1334 1335 1336 1337
	/* Convert phy_id from older PRTAD/DEVAD format */
	if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) &&
	    (data->phy_id & 0xfc00) == 0x0400)
		data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400;

	return mdio_mii_ioctl(&efx->mdio, data, cmd);
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351
}

/**************************************************************************
 *
 * NAPI interface
 *
 **************************************************************************/

static int efx_init_napi(struct efx_nic *efx)
{
	struct efx_channel *channel;

	efx_for_each_channel(channel, efx) {
		channel->napi_dev = efx->net_dev;
1352 1353
		netif_napi_add(channel->napi_dev, &channel->napi_str,
			       efx_poll, napi_weight);
1354 1355 1356 1357 1358 1359 1360 1361 1362
	}
	return 0;
}

static void efx_fini_napi(struct efx_nic *efx)
{
	struct efx_channel *channel;

	efx_for_each_channel(channel, efx) {
1363 1364
		if (channel->napi_dev)
			netif_napi_del(&channel->napi_str);
1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
		channel->napi_dev = NULL;
	}
}

/**************************************************************************
 *
 * Kernel netpoll interface
 *
 *************************************************************************/

#ifdef CONFIG_NET_POLL_CONTROLLER

/* Although in the common case interrupts will be disabled, this is not
 * guaranteed. However, all our work happens inside the NAPI callback,
 * so no locking is required.
 */
static void efx_netpoll(struct net_device *net_dev)
{
1383
	struct efx_nic *efx = netdev_priv(net_dev);
1384 1385
	struct efx_channel *channel;

1386
	efx_for_each_channel(channel, efx)
1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
		efx_schedule_channel(channel);
}

#endif

/**************************************************************************
 *
 * Kernel net device interface
 *
 *************************************************************************/

/* Context: process, rtnl_lock() held. */
static int efx_net_open(struct net_device *net_dev)
{
1401
	struct efx_nic *efx = netdev_priv(net_dev);
1402 1403 1404 1405 1406
	EFX_ASSERT_RESET_SERIALISED(efx);

	EFX_LOG(efx, "opening device %s on CPU %d\n", net_dev->name,
		raw_smp_processor_id());

1407 1408
	if (efx->state == STATE_DISABLED)
		return -EIO;
1409 1410 1411
	if (efx->phy_mode & PHY_MODE_SPECIAL)
		return -EBUSY;

1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
	efx_start_all(efx);
	return 0;
}

/* Context: process, rtnl_lock() held.
 * Note that the kernel will ignore our return code; this method
 * should really be a void.
 */
static int efx_net_stop(struct net_device *net_dev)
{
1422
	struct efx_nic *efx = netdev_priv(net_dev);
1423 1424 1425 1426

	EFX_LOG(efx, "closing %s on CPU %d\n", net_dev->name,
		raw_smp_processor_id());

1427 1428 1429 1430 1431 1432
	if (efx->state != STATE_DISABLED) {
		/* Stop the device and flush all the channels */
		efx_stop_all(efx);
		efx_fini_channels(efx);
		efx_init_channels(efx);
	}
1433 1434 1435 1436

	return 0;
}

1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450
void efx_stats_disable(struct efx_nic *efx)
{
	spin_lock(&efx->stats_lock);
	++efx->stats_disable_count;
	spin_unlock(&efx->stats_lock);
}

void efx_stats_enable(struct efx_nic *efx)
{
	spin_lock(&efx->stats_lock);
	--efx->stats_disable_count;
	spin_unlock(&efx->stats_lock);
}

1451
/* Context: process, dev_base_lock or RTNL held, non-blocking. */
1452 1453
static struct net_device_stats *efx_net_stats(struct net_device *net_dev)
{
1454
	struct efx_nic *efx = netdev_priv(net_dev);
1455 1456 1457
	struct efx_mac_stats *mac_stats = &efx->mac_stats;
	struct net_device_stats *stats = &net_dev->stats;

1458
	/* Update stats if possible, but do not wait if another thread
1459 1460
	 * is updating them or if MAC stats fetches are temporarily
	 * disabled; slightly stale stats are acceptable.
1461
	 */
1462 1463
	if (!spin_trylock(&efx->stats_lock))
		return stats;
1464
	if (!efx->stats_disable_count) {
1465
		efx->mac_op->update_stats(efx);
1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500
		falcon_update_nic_stats(efx);
	}
	spin_unlock(&efx->stats_lock);

	stats->rx_packets = mac_stats->rx_packets;
	stats->tx_packets = mac_stats->tx_packets;
	stats->rx_bytes = mac_stats->rx_bytes;
	stats->tx_bytes = mac_stats->tx_bytes;
	stats->multicast = mac_stats->rx_multicast;
	stats->collisions = mac_stats->tx_collision;
	stats->rx_length_errors = (mac_stats->rx_gtjumbo +
				   mac_stats->rx_length_error);
	stats->rx_over_errors = efx->n_rx_nodesc_drop_cnt;
	stats->rx_crc_errors = mac_stats->rx_bad;
	stats->rx_frame_errors = mac_stats->rx_align_error;
	stats->rx_fifo_errors = mac_stats->rx_overflow;
	stats->rx_missed_errors = mac_stats->rx_missed;
	stats->tx_window_errors = mac_stats->tx_late_collision;

	stats->rx_errors = (stats->rx_length_errors +
			    stats->rx_over_errors +
			    stats->rx_crc_errors +
			    stats->rx_frame_errors +
			    stats->rx_fifo_errors +
			    stats->rx_missed_errors +
			    mac_stats->rx_symbol_error);
	stats->tx_errors = (stats->tx_window_errors +
			    mac_stats->tx_bad);

	return stats;
}

/* Context: netif_tx_lock held, BHs disabled. */
static void efx_watchdog(struct net_device *net_dev)
{
1501
	struct efx_nic *efx = netdev_priv(net_dev);
1502

1503 1504 1505
	EFX_ERR(efx, "TX stuck with stop_count=%d port_enabled=%d:"
		" resetting channels\n",
		atomic_read(&efx->netif_stop_count), efx->port_enabled);
1506

1507
	efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG);
1508 1509 1510 1511 1512 1513
}


/* Context: process, rtnl_lock() held. */
static int efx_change_mtu(struct net_device *net_dev, int new_mtu)
{
1514
	struct efx_nic *efx = netdev_priv(net_dev);
1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527
	int rc = 0;

	EFX_ASSERT_RESET_SERIALISED(efx);

	if (new_mtu > EFX_MAX_MTU)
		return -EINVAL;

	efx_stop_all(efx);

	EFX_LOG(efx, "changing MTU to %d\n", new_mtu);

	efx_fini_channels(efx);
	net_dev->mtu = new_mtu;
1528
	efx_init_channels(efx);
1529 1530 1531 1532 1533 1534 1535

	efx_start_all(efx);
	return rc;
}

static int efx_set_mac_address(struct net_device *net_dev, void *data)
{
1536
	struct efx_nic *efx = netdev_priv(net_dev);
1537 1538 1539 1540 1541 1542
	struct sockaddr *addr = data;
	char *new_addr = addr->sa_data;

	EFX_ASSERT_RESET_SERIALISED(efx);

	if (!is_valid_ether_addr(new_addr)) {
J
Johannes Berg 已提交
1543 1544
		EFX_ERR(efx, "invalid ethernet MAC address requested: %pM\n",
			new_addr);
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555
		return -EINVAL;
	}

	memcpy(net_dev->dev_addr, new_addr, net_dev->addr_len);

	/* Reconfigure the MAC */
	efx_reconfigure_port(efx);

	return 0;
}

1556
/* Context: netif_addr_lock held, BHs disabled. */
1557 1558
static void efx_set_multicast_list(struct net_device *net_dev)
{
1559
	struct efx_nic *efx = netdev_priv(net_dev);
1560 1561
	struct dev_mc_list *mc_list = net_dev->mc_list;
	union efx_multicast_hash *mc_hash = &efx->multicast_hash;
1562 1563
	bool promiscuous = !!(net_dev->flags & IFF_PROMISC);
	bool changed = (efx->promiscuous != promiscuous);
1564 1565 1566 1567
	u32 crc;
	int bit;
	int i;

1568
	efx->promiscuous = promiscuous;
1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582

	/* Build multicast hash table */
	if (promiscuous || (net_dev->flags & IFF_ALLMULTI)) {
		memset(mc_hash, 0xff, sizeof(*mc_hash));
	} else {
		memset(mc_hash, 0x00, sizeof(*mc_hash));
		for (i = 0; i < net_dev->mc_count; i++) {
			crc = ether_crc_le(ETH_ALEN, mc_list->dmi_addr);
			bit = crc & (EFX_MCAST_HASH_ENTRIES - 1);
			set_bit_le(bit, mc_hash->byte);
			mc_list = mc_list->next;
		}
	}

1583 1584 1585 1586 1587
	if (!efx->port_enabled)
		/* Delay pushing settings until efx_start_port() */
		return;

	if (changed)
1588
		queue_work(efx->workqueue, &efx->phy_work);
1589

1590 1591 1592 1593
	/* Create and activate new global multicast hash table */
	falcon_set_multicast_hash(efx);
}

S
Stephen Hemminger 已提交
1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609
static const struct net_device_ops efx_netdev_ops = {
	.ndo_open		= efx_net_open,
	.ndo_stop		= efx_net_stop,
	.ndo_get_stats		= efx_net_stats,
	.ndo_tx_timeout		= efx_watchdog,
	.ndo_start_xmit		= efx_hard_start_xmit,
	.ndo_validate_addr	= eth_validate_addr,
	.ndo_do_ioctl		= efx_ioctl,
	.ndo_change_mtu		= efx_change_mtu,
	.ndo_set_mac_address	= efx_set_mac_address,
	.ndo_set_multicast_list = efx_set_multicast_list,
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller = efx_netpoll,
#endif
};

1610 1611 1612 1613 1614 1615 1616
static void efx_update_name(struct efx_nic *efx)
{
	strcpy(efx->name, efx->net_dev->name);
	efx_mtd_rename(efx);
	efx_set_channel_names(efx);
}

1617 1618 1619
static int efx_netdev_event(struct notifier_block *this,
			    unsigned long event, void *ptr)
{
1620
	struct net_device *net_dev = ptr;
1621

1622 1623 1624
	if (net_dev->netdev_ops == &efx_netdev_ops &&
	    event == NETDEV_CHANGENAME)
		efx_update_name(netdev_priv(net_dev));
1625 1626 1627 1628 1629 1630 1631 1632

	return NOTIFY_DONE;
}

static struct notifier_block efx_netdev_notifier = {
	.notifier_call = efx_netdev_event,
};

B
Ben Hutchings 已提交
1633 1634 1635 1636 1637 1638 1639 1640
static ssize_t
show_phy_type(struct device *dev, struct device_attribute *attr, char *buf)
{
	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
	return sprintf(buf, "%d\n", efx->phy_type);
}
static DEVICE_ATTR(phy_type, 0644, show_phy_type, NULL);

1641 1642 1643 1644 1645 1646 1647
static int efx_register_netdev(struct efx_nic *efx)
{
	struct net_device *net_dev = efx->net_dev;
	int rc;

	net_dev->watchdog_timeo = 5 * HZ;
	net_dev->irq = efx->pci_dev->irq;
S
Stephen Hemminger 已提交
1648
	net_dev->netdev_ops = &efx_netdev_ops;
1649 1650 1651 1652
	SET_NETDEV_DEV(net_dev, &efx->pci_dev->dev);
	SET_ETHTOOL_OPS(net_dev, &efx_ethtool_ops);

	/* Clear MAC statistics */
1653
	efx->mac_op->update_stats(efx);
1654 1655
	memset(&efx->mac_stats, 0, sizeof(efx->mac_stats));

1656
	rtnl_lock();
1657 1658 1659 1660

	rc = dev_alloc_name(net_dev, net_dev->name);
	if (rc < 0)
		goto fail_locked;
1661
	efx_update_name(efx);
1662 1663 1664 1665 1666 1667 1668 1669

	rc = register_netdevice(net_dev);
	if (rc)
		goto fail_locked;

	/* Always start with carrier off; PHY events will detect the link */
	netif_carrier_off(efx->net_dev);

1670
	rtnl_unlock();
1671

B
Ben Hutchings 已提交
1672 1673 1674 1675 1676 1677
	rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);
	if (rc) {
		EFX_ERR(efx, "failed to init net dev attributes\n");
		goto fail_registered;
	}

1678
	return 0;
B
Ben Hutchings 已提交
1679

1680 1681 1682 1683 1684
fail_locked:
	rtnl_unlock();
	EFX_ERR(efx, "could not register net dev\n");
	return rc;

B
Ben Hutchings 已提交
1685 1686 1687
fail_registered:
	unregister_netdev(net_dev);
	return rc;
1688 1689 1690 1691 1692 1693 1694 1695 1696
}

static void efx_unregister_netdev(struct efx_nic *efx)
{
	struct efx_tx_queue *tx_queue;

	if (!efx->net_dev)
		return;

1697
	BUG_ON(netdev_priv(efx->net_dev) != efx);
1698 1699 1700 1701 1702 1703 1704

	/* Free up any skbs still remaining. This has to happen before
	 * we try to unregister the netdev as running their destructors
	 * may be needed to get the device ref. count to 0. */
	efx_for_each_tx_queue(tx_queue, efx)
		efx_release_tx_buffers(tx_queue);

1705
	if (efx_dev_registered(efx)) {
1706
		strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
B
Ben Hutchings 已提交
1707
		device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
1708 1709 1710 1711 1712 1713 1714 1715 1716 1717
		unregister_netdev(efx->net_dev);
	}
}

/**************************************************************************
 *
 * Device reset and suspend
 *
 **************************************************************************/

B
Ben Hutchings 已提交
1718 1719
/* Tears down the entire software state and most of the hardware state
 * before reset.  */
1720 1721
void efx_reset_down(struct efx_nic *efx, enum reset_type method,
		    struct ethtool_cmd *ecmd)
1722 1723 1724
{
	EFX_ASSERT_RESET_SERIALISED(efx);

1725
	efx_stats_disable(efx);
B
Ben Hutchings 已提交
1726 1727
	efx_stop_all(efx);
	mutex_lock(&efx->mac_lock);
1728
	mutex_lock(&efx->spi_lock);
B
Ben Hutchings 已提交
1729

1730
	efx->phy_op->get_settings(efx, ecmd);
1731 1732

	efx_fini_channels(efx);
1733 1734
	if (efx->port_initialized && method != RESET_TYPE_INVISIBLE)
		efx->phy_op->fini(efx);
1735 1736
}

B
Ben Hutchings 已提交
1737 1738 1739 1740 1741
/* This function will always ensure that the locks acquired in
 * efx_reset_down() are released. A failure return code indicates
 * that we were unable to reinitialise the hardware, and the
 * driver should be disabled. If ok is false, then the rx and tx
 * engines are not restarted, pending a RESET_DISABLE. */
1742 1743
int efx_reset_up(struct efx_nic *efx, enum reset_type method,
		 struct ethtool_cmd *ecmd, bool ok)
1744 1745 1746
{
	int rc;

B
Ben Hutchings 已提交
1747
	EFX_ASSERT_RESET_SERIALISED(efx);
1748

B
Ben Hutchings 已提交
1749
	rc = falcon_init_nic(efx);
1750
	if (rc) {
B
Ben Hutchings 已提交
1751 1752
		EFX_ERR(efx, "failed to initialise NIC\n");
		ok = false;
1753 1754
	}

1755 1756 1757 1758 1759
	if (efx->port_initialized && method != RESET_TYPE_INVISIBLE) {
		if (ok) {
			rc = efx->phy_op->init(efx);
			if (rc)
				ok = false;
1760 1761
		}
		if (!ok)
1762 1763 1764
			efx->port_initialized = false;
	}

B
Ben Hutchings 已提交
1765 1766
	if (ok) {
		efx_init_channels(efx);
1767

1768
		if (efx->phy_op->set_settings(efx, ecmd))
B
Ben Hutchings 已提交
1769 1770 1771
			EFX_ERR(efx, "could not restore PHY settings\n");
	}

1772
	mutex_unlock(&efx->spi_lock);
B
Ben Hutchings 已提交
1773 1774
	mutex_unlock(&efx->mac_lock);

B
Ben Hutchings 已提交
1775
	if (ok) {
B
Ben Hutchings 已提交
1776
		efx_start_all(efx);
1777
		efx_stats_enable(efx);
B
Ben Hutchings 已提交
1778
	}
1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794
	return rc;
}

/* Reset the NIC as transparently as possible. Do not reset the PHY
 * Note that the reset may fail, in which case the card will be left
 * in a most-probably-unusable state.
 *
 * This function will sleep.  You cannot reset from within an atomic
 * state; use efx_schedule_reset() instead.
 *
 * Grabs the rtnl_lock.
 */
static int efx_reset(struct efx_nic *efx)
{
	struct ethtool_cmd ecmd;
	enum reset_type method = efx->reset_pending;
1795
	int rc = 0;
1796 1797 1798 1799 1800 1801 1802 1803

	/* Serialise with kernel interfaces */
	rtnl_lock();

	/* If we're not RUNNING then don't reset. Leave the reset_pending
	 * flag set so that efx_pci_probe_main will be retried */
	if (efx->state != STATE_RUNNING) {
		EFX_INFO(efx, "scheduled reset quenched. NIC not RUNNING\n");
1804
		goto out_unlock;
1805 1806
	}

1807
	EFX_INFO(efx, "resetting (%s)\n", RESET_TYPE(method));
1808

1809
	efx_reset_down(efx, method, &ecmd);
1810 1811 1812 1813

	rc = falcon_reset_hw(efx, method);
	if (rc) {
		EFX_ERR(efx, "failed to reset hardware\n");
1814
		goto out_disable;
1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827
	}

	/* Allow resets to be rescheduled. */
	efx->reset_pending = RESET_TYPE_NONE;

	/* Reinitialise bus-mastering, which may have been turned off before
	 * the reset was scheduled. This is still appropriate, even in the
	 * RESET_TYPE_DISABLE since this driver generally assumes the hardware
	 * can respond to requests. */
	pci_set_master(efx->pci_dev);

	/* Leave device stopped if necessary */
	if (method == RESET_TYPE_DISABLE) {
1828
		efx_reset_up(efx, method, &ecmd, false);
1829
		rc = -EIO;
1830
	} else {
1831
		rc = efx_reset_up(efx, method, &ecmd, true);
1832 1833
	}

1834 1835 1836 1837 1838 1839 1840 1841
out_disable:
	if (rc) {
		EFX_ERR(efx, "has been disabled\n");
		efx->state = STATE_DISABLED;
		dev_close(efx->net_dev);
	} else {
		EFX_LOG(efx, "reset complete\n");
	}
1842

1843
out_unlock:
1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885
	rtnl_unlock();
	return rc;
}

/* The worker thread exists so that code that cannot sleep can
 * schedule a reset for later.
 */
static void efx_reset_work(struct work_struct *data)
{
	struct efx_nic *nic = container_of(data, struct efx_nic, reset_work);

	efx_reset(nic);
}

void efx_schedule_reset(struct efx_nic *efx, enum reset_type type)
{
	enum reset_type method;

	if (efx->reset_pending != RESET_TYPE_NONE) {
		EFX_INFO(efx, "quenching already scheduled reset\n");
		return;
	}

	switch (type) {
	case RESET_TYPE_INVISIBLE:
	case RESET_TYPE_ALL:
	case RESET_TYPE_WORLD:
	case RESET_TYPE_DISABLE:
		method = type;
		break;
	case RESET_TYPE_RX_RECOVERY:
	case RESET_TYPE_RX_DESC_FETCH:
	case RESET_TYPE_TX_DESC_FETCH:
	case RESET_TYPE_TX_SKIP:
		method = RESET_TYPE_INVISIBLE;
		break;
	default:
		method = RESET_TYPE_ALL;
		break;
	}

	if (method != type)
1886 1887
		EFX_LOG(efx, "scheduling %s reset for %s\n",
			RESET_TYPE(method), RESET_TYPE(type));
1888
	else
1889
		EFX_LOG(efx, "scheduling %s reset\n", RESET_TYPE(method));
1890 1891 1892

	efx->reset_pending = method;

1893
	queue_work(reset_workqueue, &efx->reset_work);
1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912
}

/**************************************************************************
 *
 * List of NICs we support
 *
 **************************************************************************/

/* PCI device ID table */
static struct pci_device_id efx_pci_table[] __devinitdata = {
	{PCI_DEVICE(EFX_VENDID_SFC, FALCON_A_P_DEVID),
	 .driver_data = (unsigned long) &falcon_a_nic_type},
	{PCI_DEVICE(EFX_VENDID_SFC, FALCON_B_P_DEVID),
	 .driver_data = (unsigned long) &falcon_b_nic_type},
	{0}			/* end of list */
};

/**************************************************************************
 *
1913
 * Dummy PHY/MAC operations
1914
 *
1915
 * Can be used for some unimplemented operations
1916 1917 1918 1919 1920 1921 1922 1923 1924
 * Needed so all function pointers are valid and do not have to be tested
 * before use
 *
 **************************************************************************/
int efx_port_dummy_op_int(struct efx_nic *efx)
{
	return 0;
}
void efx_port_dummy_op_void(struct efx_nic *efx) {}
1925 1926 1927
void efx_port_dummy_op_set_id_led(struct efx_nic *efx, enum efx_led_mode mode)
{
}
1928

1929 1930
static struct efx_mac_operations efx_dummy_mac_operations = {
	.reconfigure	= efx_port_dummy_op_void,
1931 1932
	.poll		= efx_port_dummy_op_void,
	.irq		= efx_port_dummy_op_void,
1933 1934
};

1935 1936 1937
static struct efx_phy_operations efx_dummy_phy_operations = {
	.init		 = efx_port_dummy_op_int,
	.reconfigure	 = efx_port_dummy_op_void,
1938
	.poll		 = efx_port_dummy_op_void,
1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957
	.fini		 = efx_port_dummy_op_void,
	.clear_interrupt = efx_port_dummy_op_void,
};

/**************************************************************************
 *
 * Data housekeeping
 *
 **************************************************************************/

/* This zeroes out and then fills in the invariants in a struct
 * efx_nic (including all sub-structures).
 */
static int efx_init_struct(struct efx_nic *efx, struct efx_nic_type *type,
			   struct pci_dev *pci_dev, struct net_device *net_dev)
{
	struct efx_channel *channel;
	struct efx_tx_queue *tx_queue;
	struct efx_rx_queue *rx_queue;
1958
	int i;
1959 1960 1961 1962 1963

	/* Initialise common structures */
	memset(efx, 0, sizeof(*efx));
	spin_lock_init(&efx->biu_lock);
	spin_lock_init(&efx->phy_lock);
1964
	mutex_init(&efx->spi_lock);
1965 1966 1967 1968 1969 1970 1971 1972
	INIT_WORK(&efx->reset_work, efx_reset_work);
	INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor);
	efx->pci_dev = pci_dev;
	efx->state = STATE_INIT;
	efx->reset_pending = RESET_TYPE_NONE;
	strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));

	efx->net_dev = net_dev;
1973
	efx->rx_checksum_enabled = true;
1974 1975
	spin_lock_init(&efx->netif_stop_lock);
	spin_lock_init(&efx->stats_lock);
1976
	efx->stats_disable_count = 1;
1977
	mutex_init(&efx->mac_lock);
1978
	efx->mac_op = &efx_dummy_mac_operations;
1979
	efx->phy_op = &efx_dummy_phy_operations;
1980
	efx->mdio.dev = net_dev;
1981 1982
	INIT_WORK(&efx->phy_work, efx_phy_work);
	INIT_WORK(&efx->mac_work, efx_mac_work);
1983 1984 1985 1986 1987 1988
	atomic_set(&efx->netif_stop_count, 1);

	for (i = 0; i < EFX_MAX_CHANNELS; i++) {
		channel = &efx->channel[i];
		channel->efx = efx;
		channel->channel = i;
1989
		channel->work_pending = false;
1990
	}
1991
	for (i = 0; i < EFX_TX_QUEUE_COUNT; i++) {
1992 1993 1994 1995 1996
		tx_queue = &efx->tx_queue[i];
		tx_queue->efx = efx;
		tx_queue->queue = i;
		tx_queue->buffer = NULL;
		tx_queue->channel = &efx->channel[0]; /* for safety */
B
Ben Hutchings 已提交
1997
		tx_queue->tso_headers_free = NULL;
1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011
	}
	for (i = 0; i < EFX_MAX_RX_QUEUES; i++) {
		rx_queue = &efx->rx_queue[i];
		rx_queue->efx = efx;
		rx_queue->queue = i;
		rx_queue->channel = &efx->channel[0]; /* for safety */
		rx_queue->buffer = NULL;
		spin_lock_init(&rx_queue->add_lock);
		INIT_DELAYED_WORK(&rx_queue->work, efx_rx_work);
	}

	efx->type = type;

	/* As close as we can get to guaranteeing that we don't overflow */
2012 2013
	BUILD_BUG_ON(EFX_EVQ_SIZE < EFX_TXQ_SIZE + EFX_RXQ_SIZE);

2014 2015 2016 2017 2018 2019
	EFX_BUG_ON_PARANOID(efx->type->phys_addr_channels > EFX_MAX_CHANNELS);

	/* Higher numbered interrupt modes are less capable! */
	efx->interrupt_mode = max(efx->type->max_interrupt_mode,
				  interrupt_mode);

2020 2021 2022 2023
	/* Would be good to use the net_dev name, but we're too early */
	snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s",
		 pci_name(pci_dev));
	efx->workqueue = create_singlethread_workqueue(efx->workqueue_name);
2024 2025
	if (!efx->workqueue)
		return -ENOMEM;
2026

2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048
	return 0;
}

static void efx_fini_struct(struct efx_nic *efx)
{
	if (efx->workqueue) {
		destroy_workqueue(efx->workqueue);
		efx->workqueue = NULL;
	}
}

/**************************************************************************
 *
 * PCI interface
 *
 **************************************************************************/

/* Main body of final NIC shutdown code
 * This is called only at module unload (or hotplug removal).
 */
static void efx_pci_remove_main(struct efx_nic *efx)
{
2049
	falcon_fini_interrupt(efx);
2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076
	efx_fini_channels(efx);
	efx_fini_port(efx);
	efx_fini_napi(efx);
	efx_remove_all(efx);
}

/* Final NIC shutdown
 * This is called only at module unload (or hotplug removal).
 */
static void efx_pci_remove(struct pci_dev *pci_dev)
{
	struct efx_nic *efx;

	efx = pci_get_drvdata(pci_dev);
	if (!efx)
		return;

	/* Mark the NIC as fini, then stop the interface */
	rtnl_lock();
	efx->state = STATE_FINI;
	dev_close(efx->net_dev);

	/* Allow any queued efx_resets() to complete */
	rtnl_unlock();

	efx_unregister_netdev(efx);

2077 2078
	efx_mtd_remove(efx);

2079 2080 2081 2082
	/* Wait for any scheduled resets to complete. No more will be
	 * scheduled from this point because efx_stop_all() has been
	 * called, we are no longer registered with driverlink, and
	 * the net_device's have been removed. */
2083
	cancel_work_sync(&efx->reset_work);
2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113

	efx_pci_remove_main(efx);

	efx_fini_io(efx);
	EFX_LOG(efx, "shutdown successful\n");

	pci_set_drvdata(pci_dev, NULL);
	efx_fini_struct(efx);
	free_netdev(efx->net_dev);
};

/* Main body of NIC initialisation
 * This is called at module load (or hotplug insertion, theoretically).
 */
static int efx_pci_probe_main(struct efx_nic *efx)
{
	int rc;

	/* Do start-of-day initialisation */
	rc = efx_probe_all(efx);
	if (rc)
		goto fail1;

	rc = efx_init_napi(efx);
	if (rc)
		goto fail2;

	rc = falcon_init_nic(efx);
	if (rc) {
		EFX_ERR(efx, "failed to initialise NIC\n");
2114
		goto fail3;
2115 2116 2117 2118 2119
	}

	rc = efx_init_port(efx);
	if (rc) {
		EFX_ERR(efx, "failed to initialise port\n");
2120
		goto fail4;
2121 2122
	}

2123
	efx_init_channels(efx);
2124 2125 2126

	rc = falcon_init_interrupt(efx);
	if (rc)
2127
		goto fail5;
2128 2129 2130

	return 0;

2131
 fail5:
2132
	efx_fini_channels(efx);
2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163
	efx_fini_port(efx);
 fail4:
 fail3:
	efx_fini_napi(efx);
 fail2:
	efx_remove_all(efx);
 fail1:
	return rc;
}

/* NIC initialisation
 *
 * This is called at module load (or hotplug insertion,
 * theoretically).  It sets up PCI mappings, tests and resets the NIC,
 * sets up and registers the network devices with the kernel and hooks
 * the interrupt service routine.  It does not prepare the device for
 * transmission; this is left to the first time one of the network
 * interfaces is brought up (i.e. efx_net_open).
 */
static int __devinit efx_pci_probe(struct pci_dev *pci_dev,
				   const struct pci_device_id *entry)
{
	struct efx_nic_type *type = (struct efx_nic_type *) entry->driver_data;
	struct net_device *net_dev;
	struct efx_nic *efx;
	int i, rc;

	/* Allocate and initialise a struct net_device and struct efx_nic */
	net_dev = alloc_etherdev(sizeof(*efx));
	if (!net_dev)
		return -ENOMEM;
B
Ben Hutchings 已提交
2164
	net_dev->features |= (NETIF_F_IP_CSUM | NETIF_F_SG |
B
Ben Hutchings 已提交
2165 2166
			      NETIF_F_HIGHDMA | NETIF_F_TSO |
			      NETIF_F_GRO);
2167 2168
	/* Mask for features that also apply to VLAN devices */
	net_dev->vlan_features |= (NETIF_F_ALL_CSUM | NETIF_F_SG |
2169
				   NETIF_F_HIGHDMA | NETIF_F_TSO);
2170
	efx = netdev_priv(net_dev);
2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191
	pci_set_drvdata(pci_dev, efx);
	rc = efx_init_struct(efx, type, pci_dev, net_dev);
	if (rc)
		goto fail1;

	EFX_INFO(efx, "Solarflare Communications NIC detected\n");

	/* Set up basic I/O (BAR mappings etc) */
	rc = efx_init_io(efx);
	if (rc)
		goto fail2;

	/* No serialisation is required with the reset path because
	 * we're in STATE_INIT. */
	for (i = 0; i < 5; i++) {
		rc = efx_pci_probe_main(efx);

		/* Serialise against efx_reset(). No more resets will be
		 * scheduled since efx_stop_all() has been called, and we
		 * have not and never have been registered with either
		 * the rtnetlink or driverlink layers. */
2192
		cancel_work_sync(&efx->reset_work);
2193

2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204
		if (rc == 0) {
			if (efx->reset_pending != RESET_TYPE_NONE) {
				/* If there was a scheduled reset during
				 * probe, the NIC is probably hosed anyway */
				efx_pci_remove_main(efx);
				rc = -EIO;
			} else {
				break;
			}
		}

2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221
		/* Retry if a recoverably reset event has been scheduled */
		if ((efx->reset_pending != RESET_TYPE_INVISIBLE) &&
		    (efx->reset_pending != RESET_TYPE_ALL))
			goto fail3;

		efx->reset_pending = RESET_TYPE_NONE;
	}

	if (rc) {
		EFX_ERR(efx, "Could not reset NIC\n");
		goto fail4;
	}

	/* Switch to the running state before we expose the device to
	 * the OS.  This is to ensure that the initial gathering of
	 * MAC stats succeeds. */
	efx->state = STATE_RUNNING;
2222

2223 2224 2225 2226 2227
	rc = efx_register_netdev(efx);
	if (rc)
		goto fail5;

	EFX_LOG(efx, "initialisation successful\n");
2228 2229 2230 2231

	rtnl_lock();
	efx_mtd_probe(efx); /* allowed to fail */
	rtnl_unlock();
2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278
	return 0;

 fail5:
	efx_pci_remove_main(efx);
 fail4:
 fail3:
	efx_fini_io(efx);
 fail2:
	efx_fini_struct(efx);
 fail1:
	EFX_LOG(efx, "initialisation failed. rc=%d\n", rc);
	free_netdev(net_dev);
	return rc;
}

static struct pci_driver efx_pci_driver = {
	.name		= EFX_DRIVER_NAME,
	.id_table	= efx_pci_table,
	.probe		= efx_pci_probe,
	.remove		= efx_pci_remove,
};

/**************************************************************************
 *
 * Kernel module interface
 *
 *************************************************************************/

module_param(interrupt_mode, uint, 0444);
MODULE_PARM_DESC(interrupt_mode,
		 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");

static int __init efx_init_module(void)
{
	int rc;

	printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n");

	rc = register_netdevice_notifier(&efx_netdev_notifier);
	if (rc)
		goto err_notifier;

	refill_workqueue = create_workqueue("sfc_refill");
	if (!refill_workqueue) {
		rc = -ENOMEM;
		goto err_refill;
	}
2279 2280 2281 2282 2283
	reset_workqueue = create_singlethread_workqueue("sfc_reset");
	if (!reset_workqueue) {
		rc = -ENOMEM;
		goto err_reset;
	}
2284 2285 2286 2287 2288 2289 2290 2291

	rc = pci_register_driver(&efx_pci_driver);
	if (rc < 0)
		goto err_pci;

	return 0;

 err_pci:
2292 2293
	destroy_workqueue(reset_workqueue);
 err_reset:
2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305
	destroy_workqueue(refill_workqueue);
 err_refill:
	unregister_netdevice_notifier(&efx_netdev_notifier);
 err_notifier:
	return rc;
}

static void __exit efx_exit_module(void)
{
	printk(KERN_INFO "Solarflare NET driver unloading\n");

	pci_unregister_driver(&efx_pci_driver);
2306
	destroy_workqueue(reset_workqueue);
2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319
	destroy_workqueue(refill_workqueue);
	unregister_netdevice_notifier(&efx_netdev_notifier);

}

module_init(efx_init_module);
module_exit(efx_exit_module);

MODULE_AUTHOR("Michael Brown <mbrown@fensystems.co.uk> and "
	      "Solarflare Communications");
MODULE_DESCRIPTION("Solarflare Communications network driver");
MODULE_LICENSE("GPL");
MODULE_DEVICE_TABLE(pci, efx_pci_table);