myri10ge.c 114.1 KB
Newer Older
1 2 3
/*************************************************************************
 * myri10ge.c: Myricom Myri-10G Ethernet driver.
 *
4
 * Copyright (C) 2005 - 2009 Myricom, Inc.
5 6 7 8 9 10 11 12 13 14 15 16 17 18
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of Myricom, Inc. nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
19 20
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 23 24 25 26 27 28 29
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
30 31 32 33 34 35 36 37 38 39 40
 *
 *
 * If the eeprom on your board is not recent enough, you will need to get a
 * newer firmware image at:
 *   http://www.myri.com/scs/download-Myri10GE.html
 *
 * Contact Information:
 *   <help@myri.com>
 *   Myricom, Inc., 325N Santa Anita Avenue, Arcadia, CA 91006
 *************************************************************************/

41 42
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

43 44 45 46 47 48
#include <linux/tcp.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/string.h>
#include <linux/module.h>
#include <linux/pci.h>
B
Brice Goglin 已提交
49
#include <linux/dma-mapping.h>
50 51 52
#include <linux/etherdevice.h>
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
A
Andrew Gallatin 已提交
53
#include <linux/inet_lro.h>
54
#include <linux/dca.h>
55 56 57 58 59 60 61 62 63 64 65
#include <linux/ip.h>
#include <linux/inet.h>
#include <linux/in.h>
#include <linux/ethtool.h>
#include <linux/firmware.h>
#include <linux/delay.h>
#include <linux/timer.h>
#include <linux/vmalloc.h>
#include <linux/crc32.h>
#include <linux/moduleparam.h>
#include <linux/io.h>
66
#include <linux/log2.h>
67
#include <linux/slab.h>
68
#include <net/checksum.h>
A
Andrew Gallatin 已提交
69 70
#include <net/ip.h>
#include <net/tcp.h>
71 72 73 74 75 76 77 78 79 80
#include <asm/byteorder.h>
#include <asm/io.h>
#include <asm/processor.h>
#ifdef CONFIG_MTRR
#include <asm/mtrr.h>
#endif

#include "myri10ge_mcp.h"
#include "myri10ge_mcp_gen_header.h"

81
#define MYRI10GE_VERSION_STR "1.5.2-1.459"
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97

MODULE_DESCRIPTION("Myricom 10G driver (10GbE)");
MODULE_AUTHOR("Maintainer: help@myri.com");
MODULE_VERSION(MYRI10GE_VERSION_STR);
MODULE_LICENSE("Dual BSD/GPL");

#define MYRI10GE_MAX_ETHER_MTU 9014

#define MYRI10GE_ETH_STOPPED 0
#define MYRI10GE_ETH_STOPPING 1
#define MYRI10GE_ETH_STARTING 2
#define MYRI10GE_ETH_RUNNING 3
#define MYRI10GE_ETH_OPEN_FAILED 4

#define MYRI10GE_EEPROM_STRINGS_SIZE 256
#define MYRI10GE_MAX_SEND_DESC_TSO ((65536 / 2048) * 2)
A
Andrew Gallatin 已提交
98 99
#define MYRI10GE_MAX_LRO_DESCRIPTORS 8
#define MYRI10GE_LRO_MAX_PKTS 64
100

A
Al Viro 已提交
101
#define MYRI10GE_NO_CONFIRM_DATA htonl(0xffffffff)
102 103
#define MYRI10GE_NO_RESPONSE_RESULT 0xffffffff

104 105 106 107
#define MYRI10GE_ALLOC_ORDER 0
#define MYRI10GE_ALLOC_SIZE ((1 << MYRI10GE_ALLOC_ORDER) * PAGE_SIZE)
#define MYRI10GE_MAX_FRAGS_PER_FRAME (MYRI10GE_MAX_ETHER_MTU/MYRI10GE_ALLOC_SIZE + 1)

B
Brice Goglin 已提交
108 109
#define MYRI10GE_MAX_SLICES 32

110
struct myri10ge_rx_buffer_state {
111 112
	struct page *page;
	int page_offset;
113 114
	DEFINE_DMA_UNMAP_ADDR(bus);
	DEFINE_DMA_UNMAP_LEN(len);
115 116 117 118 119
};

struct myri10ge_tx_buffer_state {
	struct sk_buff *skb;
	int last;
120 121
	DEFINE_DMA_UNMAP_ADDR(bus);
	DEFINE_DMA_UNMAP_LEN(len);
122 123 124 125 126 127 128 129 130 131 132 133
};

struct myri10ge_cmd {
	u32 data0;
	u32 data1;
	u32 data2;
};

struct myri10ge_rx_buf {
	struct mcp_kreq_ether_recv __iomem *lanai;	/* lanai ptr for recv ring */
	struct mcp_kreq_ether_recv *shadow;	/* host shadow of recv ring */
	struct myri10ge_rx_buffer_state *info;
134 135 136
	struct page *page;
	dma_addr_t bus;
	int page_offset;
137
	int cnt;
138
	int fill_cnt;
139 140
	int alloc_fail;
	int mask;		/* number of rx slots -1 */
141
	int watchdog_needed;
142 143 144 145
};

struct myri10ge_tx_buf {
	struct mcp_kreq_ether_send __iomem *lanai;	/* lanai ptr for sendq */
B
Brice Goglin 已提交
146 147
	__be32 __iomem *send_go;	/* "go" doorbell ptr */
	__be32 __iomem *send_stop;	/* "stop" doorbell ptr */
148 149 150 151 152 153
	struct mcp_kreq_ether_send *req_list;	/* host shadow of sendq */
	char *req_bytes;
	struct myri10ge_tx_buffer_state *info;
	int mask;		/* number of transmit slots -1  */
	int req ____cacheline_aligned;	/* transmit slots submitted     */
	int pkt_start;		/* packets started */
154 155
	int stop_queue;
	int linearized;
156 157
	int done ____cacheline_aligned;	/* transmit slots completed     */
	int pkt_done;		/* packets completed */
158
	int wake_queue;
B
Brice Goglin 已提交
159
	int queue_active;
160 161 162 163 164 165 166
};

struct myri10ge_rx_done {
	struct mcp_slot *entry;
	dma_addr_t bus;
	int cnt;
	int idx;
A
Andrew Gallatin 已提交
167 168
	struct net_lro_mgr lro_mgr;
	struct net_lro_desc lro_desc[MYRI10GE_MAX_LRO_DESCRIPTORS];
169 170
};

171 172 173 174 175 176 177 178 179 180
struct myri10ge_slice_netstats {
	unsigned long rx_packets;
	unsigned long tx_packets;
	unsigned long rx_bytes;
	unsigned long tx_bytes;
	unsigned long rx_dropped;
	unsigned long tx_dropped;
};

struct myri10ge_slice_state {
181 182 183 184
	struct myri10ge_tx_buf tx;	/* transmit ring        */
	struct myri10ge_rx_buf rx_small;
	struct myri10ge_rx_buf rx_big;
	struct myri10ge_rx_done rx_done;
185 186 187 188 189 190 191 192 193
	struct net_device *dev;
	struct napi_struct napi;
	struct myri10ge_priv *mgp;
	struct myri10ge_slice_netstats stats;
	__be32 __iomem *irq_claim;
	struct mcp_irq_data *fw_stats;
	dma_addr_t fw_stats_bus;
	int watchdog_tx_done;
	int watchdog_tx_req;
194
	int watchdog_rx_done;
195
#ifdef CONFIG_MYRI10GE_DCA
196 197 198 199
	int cached_dca_tag;
	int cpu;
	__be32 __iomem *dca_tag;
#endif
B
Brice Goglin 已提交
200
	char irq_desc[32];
201 202 203
};

struct myri10ge_priv {
B
Brice Goglin 已提交
204
	struct myri10ge_slice_state *ss;
205
	int tx_boundary;	/* boundary transmits cannot cross */
B
Brice Goglin 已提交
206
	int num_slices;
207 208
	int running;		/* running?             */
	int csum_flag;		/* rx_csums?            */
209
	int small_bytes;
210
	int big_bytes;
211
	int max_intr_slots;
212
	struct net_device *dev;
213
	spinlock_t stats_lock;
214 215 216 217
	u8 __iomem *sram;
	int sram_size;
	unsigned long board_span;
	unsigned long iomem_base;
A
Al Viro 已提交
218
	__be32 __iomem *irq_deassert;
219 220 221 222 223
	char *mac_addr_string;
	struct mcp_cmd_response *cmd;
	dma_addr_t cmd_bus;
	struct pci_dev *pdev;
	int msi_enabled;
B
Brice Goglin 已提交
224 225
	int msix_enabled;
	struct msix_entry *msix_vectors;
226
#ifdef CONFIG_MYRI10GE_DCA
227
	int dca_enabled;
228
	int relaxed_order;
229
#endif
A
Al Viro 已提交
230
	u32 link_state;
231 232
	unsigned int rdma_tags_available;
	int intr_coal_delay;
A
Al Viro 已提交
233
	__be32 __iomem *intr_coal_delay_ptr;
234
	int mtrr;
235
	int wc_enabled;
236 237 238 239 240
	int down_cnt;
	wait_queue_head_t down_wq;
	struct work_struct watchdog_work;
	struct timer_list watchdog_timer;
	int watchdog_resets;
241
	int watchdog_pause;
242
	int pause;
243
	bool fw_name_allocated;
244 245
	char *fw_name;
	char eeprom_strings[MYRI10GE_EEPROM_STRINGS_SIZE];
246
	char *product_code_string;
247
	char fw_version[128];
248 249 250 251
	int fw_ver_major;
	int fw_ver_minor;
	int fw_ver_tiny;
	int adopted_rx_filter_bug;
252 253 254
	u8 mac_addr[6];		/* eeprom mac address */
	unsigned long serial_number;
	int vendor_specific_offset;
255
	int fw_multicast_support;
256
	u32 features;
B
Brice Goglin 已提交
257
	u32 max_tso6;
258 259 260
	u32 read_dma;
	u32 write_dma;
	u32 read_write_dma;
261 262
	u32 link_changes;
	u32 msg_enable;
263
	unsigned int board_number;
264
	int rebooted;
265 266 267 268
};

static char *myri10ge_fw_unaligned = "myri10ge_ethp_z8e.dat";
static char *myri10ge_fw_aligned = "myri10ge_eth_z8e.dat";
B
Brice Goglin 已提交
269 270
static char *myri10ge_fw_rss_unaligned = "myri10ge_rss_ethp_z8e.dat";
static char *myri10ge_fw_rss_aligned = "myri10ge_rss_eth_z8e.dat";
B
Ben Hutchings 已提交
271 272 273 274
MODULE_FIRMWARE("myri10ge_ethp_z8e.dat");
MODULE_FIRMWARE("myri10ge_eth_z8e.dat");
MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat");
MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat");
275

276
/* Careful: must be accessed under kparam_block_sysfs_write */
277 278
static char *myri10ge_fw_name = NULL;
module_param(myri10ge_fw_name, charp, S_IRUGO | S_IWUSR);
279
MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name");
280

281 282
#define MYRI10GE_MAX_BOARDS 8
static char *myri10ge_fw_names[MYRI10GE_MAX_BOARDS] =
A
Andrew Gallatin 已提交
283
    {[0 ... (MYRI10GE_MAX_BOARDS - 1)] = NULL };
284 285 286 287
module_param_array_named(myri10ge_fw_names, myri10ge_fw_names, charp, NULL,
			 0444);
MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image names per board");

288 289
static int myri10ge_ecrc_enable = 1;
module_param(myri10ge_ecrc_enable, int, S_IRUGO);
290
MODULE_PARM_DESC(myri10ge_ecrc_enable, "Enable Extended CRC on PCI-E");
291 292 293

static int myri10ge_small_bytes = -1;	/* -1 == auto */
module_param(myri10ge_small_bytes, int, S_IRUGO | S_IWUSR);
294
MODULE_PARM_DESC(myri10ge_small_bytes, "Threshold of small packets");
295 296

static int myri10ge_msi = 1;	/* enable msi by default */
297
module_param(myri10ge_msi, int, S_IRUGO | S_IWUSR);
298
MODULE_PARM_DESC(myri10ge_msi, "Enable Message Signalled Interrupts");
299

300
static int myri10ge_intr_coal_delay = 75;
301
module_param(myri10ge_intr_coal_delay, int, S_IRUGO);
302
MODULE_PARM_DESC(myri10ge_intr_coal_delay, "Interrupt coalescing delay");
303 304 305

static int myri10ge_flow_control = 1;
module_param(myri10ge_flow_control, int, S_IRUGO);
306
MODULE_PARM_DESC(myri10ge_flow_control, "Pause parameter");
307 308 309 310

static int myri10ge_deassert_wait = 1;
module_param(myri10ge_deassert_wait, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(myri10ge_deassert_wait,
311
		 "Wait when deasserting legacy interrupts");
312 313 314 315

static int myri10ge_force_firmware = 0;
module_param(myri10ge_force_firmware, int, S_IRUGO);
MODULE_PARM_DESC(myri10ge_force_firmware,
316
		 "Force firmware to assume aligned completions");
317 318 319

static int myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN;
module_param(myri10ge_initial_mtu, int, S_IRUGO);
320
MODULE_PARM_DESC(myri10ge_initial_mtu, "Initial MTU");
321 322 323

static int myri10ge_napi_weight = 64;
module_param(myri10ge_napi_weight, int, S_IRUGO);
324
MODULE_PARM_DESC(myri10ge_napi_weight, "Set NAPI weight");
325 326 327

static int myri10ge_watchdog_timeout = 1;
module_param(myri10ge_watchdog_timeout, int, S_IRUGO);
328
MODULE_PARM_DESC(myri10ge_watchdog_timeout, "Set watchdog timeout");
329 330 331 332

static int myri10ge_max_irq_loops = 1048576;
module_param(myri10ge_max_irq_loops, int, S_IRUGO);
MODULE_PARM_DESC(myri10ge_max_irq_loops,
333
		 "Set stuck legacy IRQ detection threshold");
334

335 336 337 338 339 340
#define MYRI10GE_MSG_DEFAULT NETIF_MSG_LINK

static int myri10ge_debug = -1;	/* defaults above */
module_param(myri10ge_debug, int, 0);
MODULE_PARM_DESC(myri10ge_debug, "Debug level (0=none,...,16=all)");

A
Andrew Gallatin 已提交
341 342
static int myri10ge_lro_max_pkts = MYRI10GE_LRO_MAX_PKTS;
module_param(myri10ge_lro_max_pkts, int, S_IRUGO);
343 344
MODULE_PARM_DESC(myri10ge_lro_max_pkts,
		 "Number of LRO packets to be aggregated");
A
Andrew Gallatin 已提交
345

346 347
static int myri10ge_fill_thresh = 256;
module_param(myri10ge_fill_thresh, int, S_IRUGO | S_IWUSR);
348
MODULE_PARM_DESC(myri10ge_fill_thresh, "Number of empty rx slots allowed");
349

350 351
static int myri10ge_reset_recover = 1;

B
Brice Goglin 已提交
352 353 354 355
static int myri10ge_max_slices = 1;
module_param(myri10ge_max_slices, int, S_IRUGO);
MODULE_PARM_DESC(myri10ge_max_slices, "Max tx/rx queues");

356
static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT;
B
Brice Goglin 已提交
357 358 359
module_param(myri10ge_rss_hash, int, S_IRUGO);
MODULE_PARM_DESC(myri10ge_rss_hash, "Type of RSS hashing to do");

360 361 362 363
static int myri10ge_dca = 1;
module_param(myri10ge_dca, int, S_IRUGO);
MODULE_PARM_DESC(myri10ge_dca, "Enable DCA if possible");

364 365 366 367 368 369 370
#define MYRI10GE_FW_OFFSET 1024*1024
#define MYRI10GE_HIGHPART_TO_U32(X) \
(sizeof (X) == 8) ? ((u32)((u64)(X) >> 32)) : (0)
#define MYRI10GE_LOWPART_TO_U32(X) ((u32)(X))

#define myri10ge_pio_copy(to,from,size) __iowrite64_copy(to,from,size/8)

371
static void myri10ge_set_multicast_list(struct net_device *dev);
372 373
static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb,
					 struct net_device *dev);
374

375
static inline void put_be32(__be32 val, __be32 __iomem * p)
A
Al Viro 已提交
376
{
377
	__raw_writel((__force __u32) val, (__force void __iomem *)p);
A
Al Viro 已提交
378 379
}

380 381
static struct net_device_stats *myri10ge_get_stats(struct net_device *dev);

382 383 384 385 386 387 388 389
static void set_fw_name(struct myri10ge_priv *mgp, char *name, bool allocated)
{
	if (mgp->fw_name_allocated)
		kfree(mgp->fw_name);
	mgp->fw_name = name;
	mgp->fw_name_allocated = allocated;
}

390 391 392 393 394 395 396
static int
myri10ge_send_cmd(struct myri10ge_priv *mgp, u32 cmd,
		  struct myri10ge_cmd *data, int atomic)
{
	struct mcp_cmd *buf;
	char buf_bytes[sizeof(*buf) + 8];
	struct mcp_cmd_response *response = mgp->cmd;
397
	char __iomem *cmd_addr = mgp->sram + MXGEFW_ETH_CMD;
398 399 400 401 402 403 404 405 406 407 408 409 410 411 412
	u32 dma_low, dma_high, result, value;
	int sleep_total = 0;

	/* ensure buf is aligned to 8 bytes */
	buf = (struct mcp_cmd *)ALIGN((unsigned long)buf_bytes, 8);

	buf->data0 = htonl(data->data0);
	buf->data1 = htonl(data->data1);
	buf->data2 = htonl(data->data2);
	buf->cmd = htonl(cmd);
	dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus);
	dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus);

	buf->response_addr.low = htonl(dma_low);
	buf->response_addr.high = htonl(dma_high);
A
Al Viro 已提交
413
	response->result = htonl(MYRI10GE_NO_RESPONSE_RESULT);
414 415 416 417 418 419 420 421 422 423 424 425 426
	mb();
	myri10ge_pio_copy(cmd_addr, buf, sizeof(*buf));

	/* wait up to 15ms. Longest command is the DMA benchmark,
	 * which is capped at 5ms, but runs from a timeout handler
	 * that runs every 7.8ms. So a 15ms timeout leaves us with
	 * a 2.2ms margin
	 */
	if (atomic) {
		/* if atomic is set, do not sleep,
		 * and try to get the completion quickly
		 * (1ms will be enough for those commands) */
		for (sleep_total = 0;
427 428
		     sleep_total < 1000 &&
		     response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT);
429
		     sleep_total += 10) {
430
			udelay(10);
431 432
			mb();
		}
433 434 435
	} else {
		/* use msleep for most command */
		for (sleep_total = 0;
436 437
		     sleep_total < 15 &&
		     response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT);
438 439 440 441 442 443 444 445 446 447
		     sleep_total++)
			msleep(1);
	}

	result = ntohl(response->result);
	value = ntohl(response->data);
	if (result != MYRI10GE_NO_RESPONSE_RESULT) {
		if (result == 0) {
			data->data0 = value;
			return 0;
448 449
		} else if (result == MXGEFW_CMD_UNKNOWN) {
			return -ENOSYS;
450 451
		} else if (result == MXGEFW_CMD_ERROR_UNALIGNED) {
			return -E2BIG;
B
Brice Goglin 已提交
452 453 454 455 456 457
		} else if (result == MXGEFW_CMD_ERROR_RANGE &&
			   cmd == MXGEFW_CMD_ENABLE_RSS_QUEUES &&
			   (data->
			    data1 & MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES) !=
			   0) {
			return -ERANGE;
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497
		} else {
			dev_err(&mgp->pdev->dev,
				"command %d failed, result = %d\n",
				cmd, result);
			return -ENXIO;
		}
	}

	dev_err(&mgp->pdev->dev, "command %d timed out, result = %d\n",
		cmd, result);
	return -EAGAIN;
}

/*
 * The eeprom strings on the lanaiX have the format
 * SN=x\0
 * MAC=x:x:x:x:x:x\0
 * PT:ddd mmm xx xx:xx:xx xx\0
 * PV:ddd mmm xx xx:xx:xx xx\0
 */
static int myri10ge_read_mac_addr(struct myri10ge_priv *mgp)
{
	char *ptr, *limit;
	int i;

	ptr = mgp->eeprom_strings;
	limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE;

	while (*ptr != '\0' && ptr < limit) {
		if (memcmp(ptr, "MAC=", 4) == 0) {
			ptr += 4;
			mgp->mac_addr_string = ptr;
			for (i = 0; i < 6; i++) {
				if ((ptr + 2) > limit)
					goto abort;
				mgp->mac_addr[i] =
				    simple_strtoul(ptr, &ptr, 16);
				ptr += 1;
			}
		}
498 499 500 501
		if (memcmp(ptr, "PC=", 3) == 0) {
			ptr += 3;
			mgp->product_code_string = ptr;
		}
502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523
		if (memcmp((const void *)ptr, "SN=", 3) == 0) {
			ptr += 3;
			mgp->serial_number = simple_strtoul(ptr, &ptr, 10);
		}
		while (ptr < limit && *ptr++) ;
	}

	return 0;

abort:
	dev_err(&mgp->pdev->dev, "failed to parse eeprom_strings\n");
	return -ENXIO;
}

/*
 * Enable or disable periodic RDMAs from the host to make certain
 * chipsets resend dropped PCIe messages
 */

static void myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable)
{
	char __iomem *submit;
524
	__be32 buf[16] __attribute__ ((__aligned__(8)));
525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
	u32 dma_low, dma_high;
	int i;

	/* clear confirmation addr */
	mgp->cmd->data = 0;
	mb();

	/* send a rdma command to the PCIe engine, and wait for the
	 * response in the confirmation address.  The firmware should
	 * write a -1 there to indicate it is alive and well
	 */
	dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus);
	dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus);

	buf[0] = htonl(dma_high);	/* confirm addr MSW */
	buf[1] = htonl(dma_low);	/* confirm addr LSW */
A
Al Viro 已提交
541
	buf[2] = MYRI10GE_NO_CONFIRM_DATA;	/* confirm data */
542 543 544 545
	buf[3] = htonl(dma_high);	/* dummy addr MSW */
	buf[4] = htonl(dma_low);	/* dummy addr LSW */
	buf[5] = htonl(enable);	/* enable? */

546
	submit = mgp->sram + MXGEFW_BOOT_DUMMY_RDMA;
547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570

	myri10ge_pio_copy(submit, &buf, sizeof(buf));
	for (i = 0; mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 20; i++)
		msleep(1);
	if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA)
		dev_err(&mgp->pdev->dev, "dummy rdma %s failed\n",
			(enable ? "enable" : "disable"));
}

static int
myri10ge_validate_firmware(struct myri10ge_priv *mgp,
			   struct mcp_gen_header *hdr)
{
	struct device *dev = &mgp->pdev->dev;

	/* check firmware type */
	if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) {
		dev_err(dev, "Bad firmware type: 0x%x\n", ntohl(hdr->mcp_type));
		return -EINVAL;
	}

	/* save firmware version for ethtool */
	strncpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version));

571 572
	sscanf(mgp->fw_version, "%d.%d.%d", &mgp->fw_ver_major,
	       &mgp->fw_ver_minor, &mgp->fw_ver_tiny);
573

574 575
	if (!(mgp->fw_ver_major == MXGEFW_VERSION_MAJOR &&
	      mgp->fw_ver_minor == MXGEFW_VERSION_MINOR)) {
576 577 578 579 580 581 582 583 584 585 586 587 588
		dev_err(dev, "Found firmware version %s\n", mgp->fw_version);
		dev_err(dev, "Driver needs %d.%d\n", MXGEFW_VERSION_MAJOR,
			MXGEFW_VERSION_MINOR);
		return -EINVAL;
	}
	return 0;
}

static int myri10ge_load_hotplug_firmware(struct myri10ge_priv *mgp, u32 * size)
{
	unsigned crc, reread_crc;
	const struct firmware *fw;
	struct device *dev = &mgp->pdev->dev;
589
	unsigned char *fw_readback;
590 591 592
	struct mcp_gen_header *hdr;
	size_t hdr_offset;
	int status;
593
	unsigned i;
594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611

	if ((status = request_firmware(&fw, mgp->fw_name, dev)) < 0) {
		dev_err(dev, "Unable to load %s firmware image via hotplug\n",
			mgp->fw_name);
		status = -EINVAL;
		goto abort_with_nothing;
	}

	/* check size */

	if (fw->size >= mgp->sram_size - MYRI10GE_FW_OFFSET ||
	    fw->size < MCP_HEADER_PTR_OFFSET + 4) {
		dev_err(dev, "Firmware size invalid:%d\n", (int)fw->size);
		status = -EINVAL;
		goto abort_with_fw;
	}

	/* check id */
A
Al Viro 已提交
612
	hdr_offset = ntohl(*(__be32 *) (fw->data + MCP_HEADER_PTR_OFFSET));
613 614 615 616 617 618 619 620 621 622 623 624
	if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw->size) {
		dev_err(dev, "Bad firmware file\n");
		status = -EINVAL;
		goto abort_with_fw;
	}
	hdr = (void *)(fw->data + hdr_offset);

	status = myri10ge_validate_firmware(mgp, hdr);
	if (status != 0)
		goto abort_with_fw;

	crc = crc32(~0, fw->data, fw->size);
625 626 627 628 629 630
	for (i = 0; i < fw->size; i += 256) {
		myri10ge_pio_copy(mgp->sram + MYRI10GE_FW_OFFSET + i,
				  fw->data + i,
				  min(256U, (unsigned)(fw->size - i)));
		mb();
		readb(mgp->sram);
B
Brice Goglin 已提交
631
	}
632 633 634 635 636
	fw_readback = vmalloc(fw->size);
	if (!fw_readback) {
		status = -ENOMEM;
		goto abort_with_fw;
	}
637
	/* corruption checking is good for parity recovery and buggy chipset */
638 639 640
	memcpy_fromio(fw_readback, mgp->sram + MYRI10GE_FW_OFFSET, fw->size);
	reread_crc = crc32(~0, fw_readback, fw->size);
	vfree(fw_readback);
641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664
	if (crc != reread_crc) {
		dev_err(dev, "CRC failed(fw-len=%u), got 0x%x (expect 0x%x)\n",
			(unsigned)fw->size, reread_crc, crc);
		status = -EIO;
		goto abort_with_fw;
	}
	*size = (u32) fw->size;

abort_with_fw:
	release_firmware(fw);

abort_with_nothing:
	return status;
}

static int myri10ge_adopt_running_firmware(struct myri10ge_priv *mgp)
{
	struct mcp_gen_header *hdr;
	struct device *dev = &mgp->pdev->dev;
	const size_t bytes = sizeof(struct mcp_gen_header);
	size_t hdr_offset;
	int status;

	/* find running firmware header */
A
Al Viro 已提交
665
	hdr_offset = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET));
666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682

	if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > mgp->sram_size) {
		dev_err(dev, "Running firmware has bad header offset (%d)\n",
			(int)hdr_offset);
		return -EIO;
	}

	/* copy header of running firmware from SRAM to host memory to
	 * validate firmware */
	hdr = kmalloc(bytes, GFP_KERNEL);
	if (hdr == NULL) {
		dev_err(dev, "could not malloc firmware hdr\n");
		return -ENOMEM;
	}
	memcpy_fromio(hdr, mgp->sram + hdr_offset, bytes);
	status = myri10ge_validate_firmware(mgp, hdr);
	kfree(hdr);
683 684 685 686 687 688 689 690 691 692 693 694

	/* check to see if adopted firmware has bug where adopting
	 * it will cause broadcasts to be filtered unless the NIC
	 * is kept in ALLMULTI mode */
	if (mgp->fw_ver_major == 1 && mgp->fw_ver_minor == 4 &&
	    mgp->fw_ver_tiny >= 4 && mgp->fw_ver_tiny <= 11) {
		mgp->adopted_rx_filter_bug = 1;
		dev_warn(dev, "Adopting fw %d.%d.%d: "
			 "working around rx filter bug\n",
			 mgp->fw_ver_major, mgp->fw_ver_minor,
			 mgp->fw_ver_tiny);
	}
695 696 697
	return status;
}

698
static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp)
699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723
{
	struct myri10ge_cmd cmd;
	int status;

	/* probe for IPv6 TSO support */
	mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO;
	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE,
				   &cmd, 0);
	if (status == 0) {
		mgp->max_tso6 = cmd.data0;
		mgp->features |= NETIF_F_TSO6;
	}

	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0);
	if (status != 0) {
		dev_err(&mgp->pdev->dev,
			"failed MXGEFW_CMD_GET_RX_RING_SIZE\n");
		return -ENXIO;
	}

	mgp->max_intr_slots = 2 * (cmd.data0 / sizeof(struct mcp_dma_addr));

	return 0;
}

B
Brice Goglin 已提交
724
static int myri10ge_load_firmware(struct myri10ge_priv *mgp, int adopt)
725 726
{
	char __iomem *submit;
727
	__be32 buf[16] __attribute__ ((__aligned__(8)));
728 729 730
	u32 dma_low, dma_high, size;
	int status, i;

B
Brice Goglin 已提交
731
	size = 0;
732 733
	status = myri10ge_load_hotplug_firmware(mgp, &size);
	if (status) {
B
Brice Goglin 已提交
734 735
		if (!adopt)
			return status;
736 737 738 739 740 741 742 743 744 745 746 747 748 749 750
		dev_warn(&mgp->pdev->dev, "hotplug firmware loading failed\n");

		/* Do not attempt to adopt firmware if there
		 * was a bad crc */
		if (status == -EIO)
			return status;

		status = myri10ge_adopt_running_firmware(mgp);
		if (status != 0) {
			dev_err(&mgp->pdev->dev,
				"failed to adopt running firmware\n");
			return status;
		}
		dev_info(&mgp->pdev->dev,
			 "Successfully adopted running firmware\n");
751
		if (mgp->tx_boundary == 4096) {
752 753 754 755 756 757 758 759 760
			dev_warn(&mgp->pdev->dev,
				 "Using firmware currently running on NIC"
				 ".  For optimal\n");
			dev_warn(&mgp->pdev->dev,
				 "performance consider loading optimized "
				 "firmware\n");
			dev_warn(&mgp->pdev->dev, "via hotplug\n");
		}

761
		set_fw_name(mgp, "adopted", false);
762
		mgp->tx_boundary = 2048;
763 764
		myri10ge_dummy_rdma(mgp, 1);
		status = myri10ge_get_firmware_capabilities(mgp);
765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780
		return status;
	}

	/* clear confirmation addr */
	mgp->cmd->data = 0;
	mb();

	/* send a reload command to the bootstrap MCP, and wait for the
	 *  response in the confirmation address.  The firmware should
	 * write a -1 there to indicate it is alive and well
	 */
	dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus);
	dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus);

	buf[0] = htonl(dma_high);	/* confirm addr MSW */
	buf[1] = htonl(dma_low);	/* confirm addr LSW */
A
Al Viro 已提交
781
	buf[2] = MYRI10GE_NO_CONFIRM_DATA;	/* confirm data */
782 783 784 785 786 787 788 789 790 791

	/* FIX: All newest firmware should un-protect the bottom of
	 * the sram before handoff. However, the very first interfaces
	 * do not. Therefore the handoff copy must skip the first 8 bytes
	 */
	buf[3] = htonl(MYRI10GE_FW_OFFSET + 8);	/* where the code starts */
	buf[4] = htonl(size - 8);	/* length of code */
	buf[5] = htonl(8);	/* where to copy to */
	buf[6] = htonl(0);	/* where to jump to */

792
	submit = mgp->sram + MXGEFW_BOOT_HANDOFF;
793 794 795 796 797 798

	myri10ge_pio_copy(submit, &buf, sizeof(buf));
	mb();
	msleep(1);
	mb();
	i = 0;
799 800
	while (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 9) {
		msleep(1 << i);
801 802 803 804 805 806
		i++;
	}
	if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) {
		dev_err(&mgp->pdev->dev, "handoff failed\n");
		return -ENXIO;
	}
807
	myri10ge_dummy_rdma(mgp, 1);
808
	status = myri10ge_get_firmware_capabilities(mgp);
809

810
	return status;
811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835
}

static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, u8 * addr)
{
	struct myri10ge_cmd cmd;
	int status;

	cmd.data0 = ((addr[0] << 24) | (addr[1] << 16)
		     | (addr[2] << 8) | addr[3]);

	cmd.data1 = ((addr[4] << 8) | (addr[5]));

	status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0);
	return status;
}

static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause)
{
	struct myri10ge_cmd cmd;
	int status, ctl;

	ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL;
	status = myri10ge_send_cmd(mgp, ctl, &cmd, 0);

	if (status) {
836
		netdev_err(mgp->dev, "Failed to set flow control mode\n");
837 838 839 840 841 842 843 844 845 846 847 848 849 850 851
		return status;
	}
	mgp->pause = pause;
	return 0;
}

static void
myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic)
{
	struct myri10ge_cmd cmd;
	int status, ctl;

	ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC;
	status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic);
	if (status)
852
		netdev_err(mgp->dev, "Failed to set promisc mode\n");
853 854
}

855
static int myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type)
856 857 858 859
{
	struct myri10ge_cmd cmd;
	int status;
	u32 len;
860 861
	struct page *dmatest_page;
	dma_addr_t dmatest_bus;
862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878
	char *test = " ";

	dmatest_page = alloc_page(GFP_KERNEL);
	if (!dmatest_page)
		return -ENOMEM;
	dmatest_bus = pci_map_page(mgp->pdev, dmatest_page, 0, PAGE_SIZE,
				   DMA_BIDIRECTIONAL);

	/* Run a small DMA test.
	 * The magic multipliers to the length tell the firmware
	 * to do DMA read, write, or read+write tests.  The
	 * results are returned in cmd.data0.  The upper 16
	 * bits or the return is the number of transfers completed.
	 * The lower 16 bits is the time in 0.5us ticks that the
	 * transfers took to complete.
	 */

879
	len = mgp->tx_boundary;
880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924

	cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus);
	cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus);
	cmd.data2 = len * 0x10000;
	status = myri10ge_send_cmd(mgp, test_type, &cmd, 0);
	if (status != 0) {
		test = "read";
		goto abort;
	}
	mgp->read_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff);
	cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus);
	cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus);
	cmd.data2 = len * 0x1;
	status = myri10ge_send_cmd(mgp, test_type, &cmd, 0);
	if (status != 0) {
		test = "write";
		goto abort;
	}
	mgp->write_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff);

	cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus);
	cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus);
	cmd.data2 = len * 0x10001;
	status = myri10ge_send_cmd(mgp, test_type, &cmd, 0);
	if (status != 0) {
		test = "read/write";
		goto abort;
	}
	mgp->read_write_dma = ((cmd.data0 >> 16) * len * 2 * 2) /
	    (cmd.data0 & 0xffff);

abort:
	pci_unmap_page(mgp->pdev, dmatest_bus, PAGE_SIZE, DMA_BIDIRECTIONAL);
	put_page(dmatest_page);

	if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST)
		dev_warn(&mgp->pdev->dev, "DMA %s benchmark failed: %d\n",
			 test, status);

	return status;
}

static int myri10ge_reset(struct myri10ge_priv *mgp)
{
	struct myri10ge_cmd cmd;
B
Brice Goglin 已提交
925 926
	struct myri10ge_slice_state *ss;
	int i, status;
927
	size_t bytes;
928
#ifdef CONFIG_MYRI10GE_DCA
929 930
	unsigned long dca_tag_off;
#endif
931 932 933 934 935 936 937 938 939

	/* try to send a reset command to the card to see if it
	 * is alive */
	memset(&cmd, 0, sizeof(cmd));
	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0);
	if (status != 0) {
		dev_err(&mgp->pdev->dev, "failed reset\n");
		return -ENXIO;
	}
940 941

	(void)myri10ge_dma_test(mgp, MXGEFW_DMA_TEST);
B
Brice Goglin 已提交
942 943 944 945 946 947 948 949
	/*
	 * Use non-ndis mcp_slot (eg, 4 bytes total,
	 * no toeplitz hash value returned.  Older firmware will
	 * not understand this command, but will use the correct
	 * sized mcp_slot, so we ignore error returns
	 */
	cmd.data0 = MXGEFW_RSS_MCP_SLOT_TYPE_MIN;
	(void)myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_MCP_SLOT_TYPE, &cmd, 0);
950 951 952

	/* Now exchange information about interrupts  */

B
Brice Goglin 已提交
953
	bytes = mgp->max_intr_slots * sizeof(*mgp->ss[0].rx_done.entry);
954 955
	cmd.data0 = (u32) bytes;
	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0);
B
Brice Goglin 已提交
956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983

	/*
	 * Even though we already know how many slices are supported
	 * via myri10ge_probe_slices() MXGEFW_CMD_GET_MAX_RSS_QUEUES
	 * has magic side effects, and must be called after a reset.
	 * It must be called prior to calling any RSS related cmds,
	 * including assigning an interrupt queue for anything but
	 * slice 0.  It must also be called *after*
	 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by
	 * the firmware to compute offsets.
	 */

	if (mgp->num_slices > 1) {

		/* ask the maximum number of slices it supports */
		status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES,
					   &cmd, 0);
		if (status != 0) {
			dev_err(&mgp->pdev->dev,
				"failed to get number of slices\n");
		}

		/*
		 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior
		 * to setting up the interrupt queue DMA
		 */

		cmd.data0 = mgp->num_slices;
B
Brice Goglin 已提交
984 985 986
		cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE;
		if (mgp->dev->real_num_tx_queues > 1)
			cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES;
B
Brice Goglin 已提交
987 988
		status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES,
					   &cmd, 0);
B
Brice Goglin 已提交
989 990 991 992 993

		/* Firmware older than 1.4.32 only supports multiple
		 * RX queues, so if we get an error, first retry using a
		 * single TX queue before giving up */
		if (status != 0 && mgp->dev->real_num_tx_queues > 1) {
994
			netif_set_real_num_tx_queues(mgp->dev, 1);
B
Brice Goglin 已提交
995 996 997 998 999 1000 1001
			cmd.data0 = mgp->num_slices;
			cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE;
			status = myri10ge_send_cmd(mgp,
						   MXGEFW_CMD_ENABLE_RSS_QUEUES,
						   &cmd, 0);
		}

B
Brice Goglin 已提交
1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
		if (status != 0) {
			dev_err(&mgp->pdev->dev,
				"failed to set number of slices\n");

			return status;
		}
	}
	for (i = 0; i < mgp->num_slices; i++) {
		ss = &mgp->ss[i];
		cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->rx_done.bus);
		cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->rx_done.bus);
		cmd.data2 = i;
		status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA,
					    &cmd, 0);
	};
1017 1018 1019

	status |=
	    myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd, 0);
B
Brice Goglin 已提交
1020 1021 1022 1023 1024
	for (i = 0; i < mgp->num_slices; i++) {
		ss = &mgp->ss[i];
		ss->irq_claim =
		    (__iomem __be32 *) (mgp->sram + cmd.data0 + 8 * i);
	}
1025 1026 1027
	status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET,
				    &cmd, 0);
	mgp->irq_deassert = (__iomem __be32 *) (mgp->sram + cmd.data0);
1028 1029 1030

	status |= myri10ge_send_cmd
	    (mgp, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd, 0);
A
Al Viro 已提交
1031
	mgp->intr_coal_delay_ptr = (__iomem __be32 *) (mgp->sram + cmd.data0);
1032 1033 1034 1035
	if (status != 0) {
		dev_err(&mgp->pdev->dev, "failed set interrupt parameters\n");
		return status;
	}
A
Al Viro 已提交
1036
	put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr);
1037

1038
#ifdef CONFIG_MYRI10GE_DCA
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049
	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_DCA_OFFSET, &cmd, 0);
	dca_tag_off = cmd.data0;
	for (i = 0; i < mgp->num_slices; i++) {
		ss = &mgp->ss[i];
		if (status == 0) {
			ss->dca_tag = (__iomem __be32 *)
			    (mgp->sram + dca_tag_off + 4 * i);
		} else {
			ss->dca_tag = NULL;
		}
	}
B
Brice Goglin 已提交
1050
#endif				/* CONFIG_MYRI10GE_DCA */
1051

1052
	/* reset mcp/driver shared state back to 0 */
B
Brice Goglin 已提交
1053

1054
	mgp->link_changes = 0;
B
Brice Goglin 已提交
1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
	for (i = 0; i < mgp->num_slices; i++) {
		ss = &mgp->ss[i];

		memset(ss->rx_done.entry, 0, bytes);
		ss->tx.req = 0;
		ss->tx.done = 0;
		ss->tx.pkt_start = 0;
		ss->tx.pkt_done = 0;
		ss->rx_big.cnt = 0;
		ss->rx_small.cnt = 0;
		ss->rx_done.idx = 0;
		ss->rx_done.cnt = 0;
		ss->tx.wake_queue = 0;
		ss->tx.stop_queue = 0;
	}

1071 1072
	status = myri10ge_update_mac_address(mgp, mgp->dev->dev_addr);
	myri10ge_change_pause(mgp, mgp->pause);
1073
	myri10ge_set_multicast_list(mgp->dev);
1074 1075 1076
	return status;
}

1077
#ifdef CONFIG_MYRI10GE_DCA
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
static int myri10ge_toggle_relaxed(struct pci_dev *pdev, int on)
{
	int ret, cap, err;
	u16 ctl;

	cap = pci_find_capability(pdev, PCI_CAP_ID_EXP);
	if (!cap)
		return 0;

	err = pci_read_config_word(pdev, cap + PCI_EXP_DEVCTL, &ctl);
	ret = (ctl & PCI_EXP_DEVCTL_RELAX_EN) >> 4;
	if (ret != on) {
		ctl &= ~PCI_EXP_DEVCTL_RELAX_EN;
		ctl |= (on << 4);
		pci_write_config_word(pdev, cap + PCI_EXP_DEVCTL, ctl);
	}
	return ret;
}

1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
static void
myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag)
{
	ss->cached_dca_tag = tag;
	put_be32(htonl(tag), ss->dca_tag);
}

static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss)
{
	int cpu = get_cpu();
	int tag;

	if (cpu != ss->cpu) {
1110
		tag = dca3_get_tag(&ss->mgp->pdev->dev, cpu);
1111 1112
		if (ss->cached_dca_tag != tag)
			myri10ge_write_dca(ss, cpu, tag);
1113
		ss->cpu = cpu;
1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130
	}
	put_cpu();
}

static void myri10ge_setup_dca(struct myri10ge_priv *mgp)
{
	int err, i;
	struct pci_dev *pdev = mgp->pdev;

	if (mgp->ss[0].dca_tag == NULL || mgp->dca_enabled)
		return;
	if (!myri10ge_dca) {
		dev_err(&pdev->dev, "dca disabled by administrator\n");
		return;
	}
	err = dca_add_requester(&pdev->dev);
	if (err) {
1131 1132 1133
		if (err != -ENODEV)
			dev_err(&pdev->dev,
				"dca_add_requester() failed, err=%d\n", err);
1134 1135
		return;
	}
1136
	mgp->relaxed_order = myri10ge_toggle_relaxed(pdev, 0);
1137
	mgp->dca_enabled = 1;
1138 1139 1140 1141 1142
	for (i = 0; i < mgp->num_slices; i++) {
		mgp->ss[i].cpu = -1;
		mgp->ss[i].cached_dca_tag = -1;
		myri10ge_update_dca(&mgp->ss[i]);
	 }
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152
}

static void myri10ge_teardown_dca(struct myri10ge_priv *mgp)
{
	struct pci_dev *pdev = mgp->pdev;
	int err;

	if (!mgp->dca_enabled)
		return;
	mgp->dca_enabled = 0;
1153 1154
	if (mgp->relaxed_order)
		myri10ge_toggle_relaxed(pdev, 1);
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171
	err = dca_remove_requester(&pdev->dev);
}

static int myri10ge_notify_dca_device(struct device *dev, void *data)
{
	struct myri10ge_priv *mgp;
	unsigned long event;

	mgp = dev_get_drvdata(dev);
	event = *(unsigned long *)data;

	if (event == DCA_PROVIDER_ADD)
		myri10ge_setup_dca(mgp);
	else if (event == DCA_PROVIDER_REMOVE)
		myri10ge_teardown_dca(mgp);
	return 0;
}
B
Brice Goglin 已提交
1172
#endif				/* CONFIG_MYRI10GE_DCA */
1173

1174 1175 1176 1177
static inline void
myri10ge_submit_8rx(struct mcp_kreq_ether_recv __iomem * dst,
		    struct mcp_kreq_ether_recv *src)
{
A
Al Viro 已提交
1178
	__be32 low;
1179 1180

	low = src->addr_low;
1181
	src->addr_low = htonl(DMA_BIT_MASK(32));
1182 1183 1184
	myri10ge_pio_copy(dst, src, 4 * sizeof(*src));
	mb();
	myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof(*src));
1185 1186
	mb();
	src->addr_low = low;
A
Al Viro 已提交
1187
	put_be32(low, &dst->addr_low);
1188 1189 1190
	mb();
}

A
Al Viro 已提交
1191
static inline void myri10ge_vlan_ip_csum(struct sk_buff *skb, __wsum hw_csum)
1192 1193 1194
{
	struct vlan_hdr *vh = (struct vlan_hdr *)(skb->data);

A
Al Viro 已提交
1195
	if ((skb->protocol == htons(ETH_P_8021Q)) &&
1196 1197 1198
	    (vh->h_vlan_encapsulated_proto == htons(ETH_P_IP) ||
	     vh->h_vlan_encapsulated_proto == htons(ETH_P_IPV6))) {
		skb->csum = hw_csum;
1199
		skb->ip_summed = CHECKSUM_COMPLETE;
1200 1201 1202
	}
}

1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225
static inline void
myri10ge_rx_skb_build(struct sk_buff *skb, u8 * va,
		      struct skb_frag_struct *rx_frags, int len, int hlen)
{
	struct skb_frag_struct *skb_frags;

	skb->len = skb->data_len = len;
	skb->truesize = len + sizeof(struct sk_buff);
	/* attach the page(s) */

	skb_frags = skb_shinfo(skb)->frags;
	while (len > 0) {
		memcpy(skb_frags, rx_frags, sizeof(*skb_frags));
		len -= rx_frags->size;
		skb_frags++;
		rx_frags++;
		skb_shinfo(skb)->nr_frags++;
	}

	/* pskb_may_pull is not available in irq context, but
	 * skb_pull() (for ether_pad and eth_type_trans()) requires
	 * the beginning of the packet in skb_headlen(), move it
	 * manually */
1226
	skb_copy_to_linear_data(skb, va, hlen);
1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
	skb_shinfo(skb)->frags[0].page_offset += hlen;
	skb_shinfo(skb)->frags[0].size -= hlen;
	skb->data_len -= hlen;
	skb->tail += hlen;
	skb_pull(skb, MXGEFW_PAD);
}

static void
myri10ge_alloc_rx_pages(struct myri10ge_priv *mgp, struct myri10ge_rx_buf *rx,
			int bytes, int watchdog)
{
	struct page *page;
	int idx;
1240 1241 1242
#if MYRI10GE_ALLOC_SIZE > 4096
	int end_offset;
#endif
1243 1244 1245 1246 1247 1248 1249

	if (unlikely(rx->watchdog_needed && !watchdog))
		return;

	/* try to refill entire ring */
	while (rx->fill_cnt != (rx->cnt + rx->mask + 1)) {
		idx = rx->fill_cnt & rx->mask;
1250
		if (rx->page_offset + bytes <= MYRI10GE_ALLOC_SIZE) {
1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
			/* we can use part of previous page */
			get_page(rx->page);
		} else {
			/* we need a new page */
			page =
			    alloc_pages(GFP_ATOMIC | __GFP_COMP,
					MYRI10GE_ALLOC_ORDER);
			if (unlikely(page == NULL)) {
				if (rx->fill_cnt - rx->cnt < 16)
					rx->watchdog_needed = 1;
				return;
			}
			rx->page = page;
			rx->page_offset = 0;
			rx->bus = pci_map_page(mgp->pdev, page, 0,
					       MYRI10GE_ALLOC_SIZE,
					       PCI_DMA_FROMDEVICE);
		}
		rx->info[idx].page = rx->page;
		rx->info[idx].page_offset = rx->page_offset;
		/* note that this is the address of the start of the
		 * page */
1273
		dma_unmap_addr_set(&rx->info[idx], bus, rx->bus);
1274 1275 1276 1277 1278 1279 1280
		rx->shadow[idx].addr_low =
		    htonl(MYRI10GE_LOWPART_TO_U32(rx->bus) + rx->page_offset);
		rx->shadow[idx].addr_high =
		    htonl(MYRI10GE_HIGHPART_TO_U32(rx->bus));

		/* start next packet on a cacheline boundary */
		rx->page_offset += SKB_DATA_ALIGN(bytes);
1281 1282 1283

#if MYRI10GE_ALLOC_SIZE > 4096
		/* don't cross a 4KB boundary */
1284 1285 1286
		end_offset = rx->page_offset + bytes - 1;
		if ((unsigned)(rx->page_offset ^ end_offset) > 4095)
			rx->page_offset = end_offset & ~4095;
1287
#endif
1288 1289 1290 1291
		rx->fill_cnt++;

		/* copy 8 descriptors to the firmware at a time */
		if ((idx & 7) == 7) {
B
Brice Goglin 已提交
1292 1293
			myri10ge_submit_8rx(&rx->lanai[idx - 7],
					    &rx->shadow[idx - 7]);
1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304
		}
	}
}

static inline void
myri10ge_unmap_rx_page(struct pci_dev *pdev,
		       struct myri10ge_rx_buffer_state *info, int bytes)
{
	/* unmap the recvd page if we're the only or last user of it */
	if (bytes >= MYRI10GE_ALLOC_SIZE / 2 ||
	    (info->page_offset + 2 * bytes) > MYRI10GE_ALLOC_SIZE) {
1305
		pci_unmap_page(pdev, (dma_unmap_addr(info, bus)
1306 1307 1308 1309 1310 1311 1312 1313 1314
				      & ~(MYRI10GE_ALLOC_SIZE - 1)),
			       MYRI10GE_ALLOC_SIZE, PCI_DMA_FROMDEVICE);
	}
}

#define MYRI10GE_HLEN 64	/* The number of bytes to copy from a
				 * page into an skb */

static inline int
1315 1316
myri10ge_rx_done(struct myri10ge_slice_state *ss, int len, __wsum csum,
		 int lro_enabled)
1317
{
1318
	struct myri10ge_priv *mgp = ss->mgp;
1319 1320
	struct sk_buff *skb;
	struct skb_frag_struct rx_frags[MYRI10GE_MAX_FRAGS_PER_FRAME];
1321 1322
	struct myri10ge_rx_buf *rx;
	int i, idx, hlen, remainder, bytes;
1323 1324 1325 1326
	struct pci_dev *pdev = mgp->pdev;
	struct net_device *dev = mgp->dev;
	u8 *va;

1327 1328 1329 1330 1331 1332 1333 1334
	if (len <= mgp->small_bytes) {
		rx = &ss->rx_small;
		bytes = mgp->small_bytes;
	} else {
		rx = &ss->rx_big;
		bytes = mgp->big_bytes;
	}

1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352
	len += MXGEFW_PAD;
	idx = rx->cnt & rx->mask;
	va = page_address(rx->info[idx].page) + rx->info[idx].page_offset;
	prefetch(va);
	/* Fill skb_frag_struct(s) with data from our receive */
	for (i = 0, remainder = len; remainder > 0; i++) {
		myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes);
		rx_frags[i].page = rx->info[idx].page;
		rx_frags[i].page_offset = rx->info[idx].page_offset;
		if (remainder < MYRI10GE_ALLOC_SIZE)
			rx_frags[i].size = remainder;
		else
			rx_frags[i].size = MYRI10GE_ALLOC_SIZE;
		rx->cnt++;
		idx = rx->cnt & rx->mask;
		remainder -= MYRI10GE_ALLOC_SIZE;
	}

1353
	if (lro_enabled) {
A
Andrew Gallatin 已提交
1354 1355 1356
		rx_frags[0].page_offset += MXGEFW_PAD;
		rx_frags[0].size -= MXGEFW_PAD;
		len -= MXGEFW_PAD;
1357 1358
		lro_receive_frags(&ss->rx_done.lro_mgr, rx_frags,
				  /* opaque, will come back in get_frag_header */
B
Brice Goglin 已提交
1359
				  len, len,
1360
				  (void *)(__force unsigned long)csum, csum);
B
Brice Goglin 已提交
1361

A
Andrew Gallatin 已提交
1362 1363 1364
		return 1;
	}

1365 1366
	hlen = MYRI10GE_HLEN > len ? len : MYRI10GE_HLEN;

1367 1368
	/* allocate an skb to attach the page(s) to. This is done
	 * after trying LRO, so as to avoid skb allocation overheads */
1369 1370 1371

	skb = netdev_alloc_skb(dev, MYRI10GE_HLEN + 16);
	if (unlikely(skb == NULL)) {
1372
		ss->stats.rx_dropped++;
1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386
		do {
			i--;
			put_page(rx_frags[i].page);
		} while (i != 0);
		return 0;
	}

	/* Attach the pages to the skb, and trim off any padding */
	myri10ge_rx_skb_build(skb, va, rx_frags, len, hlen);
	if (skb_shinfo(skb)->frags[0].size <= 0) {
		put_page(skb_shinfo(skb)->frags[0].page);
		skb_shinfo(skb)->nr_frags = 0;
	}
	skb->protocol = eth_type_trans(skb, dev);
1387
	skb_record_rx_queue(skb, ss - &mgp->ss[0]);
1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400

	if (mgp->csum_flag) {
		if ((skb->protocol == htons(ETH_P_IP)) ||
		    (skb->protocol == htons(ETH_P_IPV6))) {
			skb->csum = csum;
			skb->ip_summed = CHECKSUM_COMPLETE;
		} else
			myri10ge_vlan_ip_csum(skb, csum);
	}
	netif_receive_skb(skb);
	return 1;
}

1401 1402
static inline void
myri10ge_tx_done(struct myri10ge_slice_state *ss, int mcp_index)
1403
{
1404 1405
	struct pci_dev *pdev = ss->mgp->pdev;
	struct myri10ge_tx_buf *tx = &ss->tx;
B
Brice Goglin 已提交
1406
	struct netdev_queue *dev_queue;
1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420
	struct sk_buff *skb;
	int idx, len;

	while (tx->pkt_done != mcp_index) {
		idx = tx->done & tx->mask;
		skb = tx->info[idx].skb;

		/* Mark as free */
		tx->info[idx].skb = NULL;
		if (tx->info[idx].last) {
			tx->pkt_done++;
			tx->info[idx].last = 0;
		}
		tx->done++;
1421 1422
		len = dma_unmap_len(&tx->info[idx], len);
		dma_unmap_len_set(&tx->info[idx], len, 0);
1423
		if (skb) {
1424 1425
			ss->stats.tx_bytes += skb->len;
			ss->stats.tx_packets++;
1426 1427 1428
			dev_kfree_skb_irq(skb);
			if (len)
				pci_unmap_single(pdev,
1429
						 dma_unmap_addr(&tx->info[idx],
1430 1431 1432 1433 1434
								bus), len,
						 PCI_DMA_TODEVICE);
		} else {
			if (len)
				pci_unmap_page(pdev,
1435
					       dma_unmap_addr(&tx->info[idx],
1436 1437 1438 1439
							      bus), len,
					       PCI_DMA_TODEVICE);
		}
	}
B
Brice Goglin 已提交
1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455

	dev_queue = netdev_get_tx_queue(ss->dev, ss - ss->mgp->ss);
	/*
	 * Make a minimal effort to prevent the NIC from polling an
	 * idle tx queue.  If we can't get the lock we leave the queue
	 * active. In this case, either a thread was about to start
	 * using the queue anyway, or we lost a race and the NIC will
	 * waste some of its resources polling an inactive queue for a
	 * while.
	 */

	if ((ss->mgp->dev->real_num_tx_queues > 1) &&
	    __netif_tx_trylock(dev_queue)) {
		if (tx->req == tx->done) {
			tx->queue_active = 0;
			put_be32(htonl(1), tx->send_stop);
1456
			mb();
B
Brice Goglin 已提交
1457
			mmiowb();
B
Brice Goglin 已提交
1458 1459 1460 1461
		}
		__netif_tx_unlock(dev_queue);
	}

1462
	/* start the queue if we've stopped it */
1463 1464
	if (netif_tx_queue_stopped(dev_queue) &&
	    tx->req - tx->done < (tx->mask >> 1)) {
1465
		tx->wake_queue++;
B
Brice Goglin 已提交
1466
		netif_tx_wake_queue(dev_queue);
1467 1468 1469
	}
}

1470 1471
static inline int
myri10ge_clean_rx_done(struct myri10ge_slice_state *ss, int budget)
1472
{
1473 1474
	struct myri10ge_rx_done *rx_done = &ss->rx_done;
	struct myri10ge_priv *mgp = ss->mgp;
1475

1476 1477 1478 1479 1480 1481
	unsigned long rx_bytes = 0;
	unsigned long rx_packets = 0;
	unsigned long rx_ok;

	int idx = rx_done->idx;
	int cnt = rx_done->cnt;
1482
	int work_done = 0;
1483
	u16 length;
A
Al Viro 已提交
1484
	__wsum checksum;
1485

1486 1487 1488 1489 1490 1491 1492
	/*
	 * Prevent compiler from generating more than one ->features memory
	 * access to avoid theoretical race condition with functions that
	 * change NETIF_F_LRO flag at runtime.
	 */
	bool lro_enabled = ACCESS_ONCE(mgp->dev->features) & NETIF_F_LRO;

1493
	while (rx_done->entry[idx].length != 0 && work_done < budget) {
1494 1495
		length = ntohs(rx_done->entry[idx].length);
		rx_done->entry[idx].length = 0;
A
Al Viro 已提交
1496
		checksum = csum_unfold(rx_done->entry[idx].checksum);
1497
		rx_ok = myri10ge_rx_done(ss, length, checksum, lro_enabled);
1498 1499 1500
		rx_packets += rx_ok;
		rx_bytes += rx_ok * (unsigned long)length;
		cnt++;
1501
		idx = cnt & (mgp->max_intr_slots - 1);
1502
		work_done++;
1503 1504 1505
	}
	rx_done->idx = idx;
	rx_done->cnt = cnt;
1506 1507
	ss->stats.rx_packets += rx_packets;
	ss->stats.rx_bytes += rx_bytes;
1508

1509
	if (lro_enabled)
A
Andrew Gallatin 已提交
1510 1511
		lro_flush_all(&rx_done->lro_mgr);

1512
	/* restock receive rings if needed */
1513 1514
	if (ss->rx_small.fill_cnt - ss->rx_small.cnt < myri10ge_fill_thresh)
		myri10ge_alloc_rx_pages(mgp, &ss->rx_small,
1515
					mgp->small_bytes + MXGEFW_PAD, 0);
1516 1517
	if (ss->rx_big.fill_cnt - ss->rx_big.cnt < myri10ge_fill_thresh)
		myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0);
1518

1519
	return work_done;
1520 1521 1522 1523
}

static inline void myri10ge_check_statblock(struct myri10ge_priv *mgp)
{
B
Brice Goglin 已提交
1524
	struct mcp_irq_data *stats = mgp->ss[0].fw_stats;
1525 1526

	if (unlikely(stats->stats_updated)) {
1527 1528 1529 1530 1531
		unsigned link_up = ntohl(stats->link_up);
		if (mgp->link_state != link_up) {
			mgp->link_state = link_up;

			if (mgp->link_state == MXGEFW_LINK_UP) {
1532
				if (netif_msg_link(mgp))
1533
					netdev_info(mgp->dev, "link up\n");
1534
				netif_carrier_on(mgp->dev);
1535
				mgp->link_changes++;
1536
			} else {
1537
				if (netif_msg_link(mgp))
1538 1539 1540 1541
					netdev_info(mgp->dev, "link %s\n",
					    link_up == MXGEFW_LINK_MYRINET ?
					    "mismatch (Myrinet detected)" :
					    "down");
1542
				netif_carrier_off(mgp->dev);
1543
				mgp->link_changes++;
1544 1545 1546
			}
		}
		if (mgp->rdma_tags_available !=
1547
		    ntohl(stats->rdma_tags_available)) {
1548
			mgp->rdma_tags_available =
1549
			    ntohl(stats->rdma_tags_available);
1550 1551
			netdev_warn(mgp->dev, "RDMA timed out! %d tags left\n",
				    mgp->rdma_tags_available);
1552 1553 1554 1555 1556 1557 1558
		}
		mgp->down_cnt += stats->link_down;
		if (stats->link_down)
			wake_up(&mgp->down_wq);
	}
}

1559
static int myri10ge_poll(struct napi_struct *napi, int budget)
1560
{
1561 1562
	struct myri10ge_slice_state *ss =
	    container_of(napi, struct myri10ge_slice_state, napi);
1563
	int work_done;
1564

1565
#ifdef CONFIG_MYRI10GE_DCA
1566 1567 1568 1569
	if (ss->mgp->dca_enabled)
		myri10ge_update_dca(ss);
#endif

1570
	/* process as many rx events as NAPI will allow */
1571
	work_done = myri10ge_clean_rx_done(ss, budget);
1572

1573
	if (work_done < budget) {
1574
		napi_complete(napi);
1575
		put_be32(htonl(3), ss->irq_claim);
1576
	}
1577
	return work_done;
1578 1579
}

1580
static irqreturn_t myri10ge_intr(int irq, void *arg)
1581
{
1582 1583 1584 1585
	struct myri10ge_slice_state *ss = arg;
	struct myri10ge_priv *mgp = ss->mgp;
	struct mcp_irq_data *stats = ss->fw_stats;
	struct myri10ge_tx_buf *tx = &ss->tx;
1586 1587 1588
	u32 send_done_count;
	int i;

B
Brice Goglin 已提交
1589 1590 1591
	/* an interrupt on a non-zero receive-only slice is implicitly
	 * valid  since MSI-X irqs are not shared */
	if ((mgp->dev->real_num_tx_queues == 1) && (ss != mgp->ss)) {
1592
		napi_schedule(&ss->napi);
1593
		return IRQ_HANDLED;
B
Brice Goglin 已提交
1594 1595
	}

1596 1597
	/* make sure it is our IRQ, and that the DMA has finished */
	if (unlikely(!stats->valid))
1598
		return IRQ_NONE;
1599 1600 1601 1602

	/* low bit indicates receives are present, so schedule
	 * napi poll handler */
	if (stats->valid & 1)
1603
		napi_schedule(&ss->napi);
1604

B
Brice Goglin 已提交
1605
	if (!mgp->msi_enabled && !mgp->msix_enabled) {
A
Al Viro 已提交
1606
		put_be32(0, mgp->irq_deassert);
1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619
		if (!myri10ge_deassert_wait)
			stats->valid = 0;
		mb();
	} else
		stats->valid = 0;

	/* Wait for IRQ line to go low, if using INTx */
	i = 0;
	while (1) {
		i++;
		/* check for transmit completes and receives */
		send_done_count = ntohl(stats->send_done_count);
		if (send_done_count != tx->pkt_done)
1620
			myri10ge_tx_done(ss, (int)send_done_count);
1621
		if (unlikely(i > myri10ge_max_irq_loops)) {
1622
			netdev_err(mgp->dev, "irq stuck?\n");
1623 1624 1625 1626 1627 1628 1629 1630 1631
			stats->valid = 0;
			schedule_work(&mgp->watchdog_work);
		}
		if (likely(stats->valid == 0))
			break;
		cpu_relax();
		barrier();
	}

B
Brice Goglin 已提交
1632 1633 1634
	/* Only slice 0 updates stats */
	if (ss == mgp->ss)
		myri10ge_check_statblock(mgp);
1635

1636
	put_be32(htonl(3), ss->irq_claim + 1);
1637
	return IRQ_HANDLED;
1638 1639 1640 1641 1642
}

static int
myri10ge_get_settings(struct net_device *netdev, struct ethtool_cmd *cmd)
{
1643 1644 1645 1646
	struct myri10ge_priv *mgp = netdev_priv(netdev);
	char *ptr;
	int i;

1647 1648 1649
	cmd->autoneg = AUTONEG_DISABLE;
	cmd->speed = SPEED_10000;
	cmd->duplex = DUPLEX_FULL;
1650 1651 1652 1653 1654 1655 1656 1657 1658

	/*
	 * parse the product code to deterimine the interface type
	 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character
	 * after the 3rd dash in the driver's cached copy of the
	 * EEPROM's product code string.
	 */
	ptr = mgp->product_code_string;
	if (ptr == NULL) {
1659
		netdev_err(netdev, "Missing product code\n");
1660 1661 1662 1663 1664
		return 0;
	}
	for (i = 0; i < 3; i++, ptr++) {
		ptr = strchr(ptr, '-');
		if (ptr == NULL) {
1665 1666
			netdev_err(netdev, "Invalid product code %s\n",
				   mgp->product_code_string);
1667 1668 1669
			return 0;
		}
	}
1670 1671 1672 1673
	if (*ptr == '2')
		ptr++;
	if (*ptr == 'R' || *ptr == 'Q' || *ptr == 'S') {
		/* We've found either an XFP, quad ribbon fiber, or SFP+ */
1674
		cmd->port = PORT_FIBRE;
1675 1676 1677 1678
		cmd->supported |= SUPPORTED_FIBRE;
		cmd->advertising |= ADVERTISED_FIBRE;
	} else {
		cmd->port = PORT_OTHER;
1679
	}
1680 1681 1682 1683 1684
	if (*ptr == 'R' || *ptr == 'S')
		cmd->transceiver = XCVR_EXTERNAL;
	else
		cmd->transceiver = XCVR_INTERNAL;

1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702
	return 0;
}

static void
myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info)
{
	struct myri10ge_priv *mgp = netdev_priv(netdev);

	strlcpy(info->driver, "myri10ge", sizeof(info->driver));
	strlcpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version));
	strlcpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version));
	strlcpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info));
}

static int
myri10ge_get_coalesce(struct net_device *netdev, struct ethtool_coalesce *coal)
{
	struct myri10ge_priv *mgp = netdev_priv(netdev);
B
Brice Goglin 已提交
1703

1704 1705 1706 1707 1708 1709 1710 1711 1712 1713
	coal->rx_coalesce_usecs = mgp->intr_coal_delay;
	return 0;
}

static int
myri10ge_set_coalesce(struct net_device *netdev, struct ethtool_coalesce *coal)
{
	struct myri10ge_priv *mgp = netdev_priv(netdev);

	mgp->intr_coal_delay = coal->rx_coalesce_usecs;
A
Al Viro 已提交
1714
	put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr);
1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737
	return 0;
}

static void
myri10ge_get_pauseparam(struct net_device *netdev,
			struct ethtool_pauseparam *pause)
{
	struct myri10ge_priv *mgp = netdev_priv(netdev);

	pause->autoneg = 0;
	pause->rx_pause = mgp->pause;
	pause->tx_pause = mgp->pause;
}

static int
myri10ge_set_pauseparam(struct net_device *netdev,
			struct ethtool_pauseparam *pause)
{
	struct myri10ge_priv *mgp = netdev_priv(netdev);

	if (pause->tx_pause != mgp->pause)
		return myri10ge_change_pause(mgp, pause->tx_pause);
	if (pause->rx_pause != mgp->pause)
1738
		return myri10ge_change_pause(mgp, pause->rx_pause);
1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749
	if (pause->autoneg != 0)
		return -EINVAL;
	return 0;
}

static void
myri10ge_get_ringparam(struct net_device *netdev,
		       struct ethtool_ringparam *ring)
{
	struct myri10ge_priv *mgp = netdev_priv(netdev);

B
Brice Goglin 已提交
1750 1751
	ring->rx_mini_max_pending = mgp->ss[0].rx_small.mask + 1;
	ring->rx_max_pending = mgp->ss[0].rx_big.mask + 1;
1752
	ring->rx_jumbo_max_pending = 0;
1753
	ring->tx_max_pending = mgp->ss[0].tx.mask + 1;
1754 1755 1756 1757 1758 1759 1760 1761 1762
	ring->rx_mini_pending = ring->rx_mini_max_pending;
	ring->rx_pending = ring->rx_max_pending;
	ring->rx_jumbo_pending = ring->rx_jumbo_max_pending;
	ring->tx_pending = ring->tx_max_pending;
}

static u32 myri10ge_get_rx_csum(struct net_device *netdev)
{
	struct myri10ge_priv *mgp = netdev_priv(netdev);
B
Brice Goglin 已提交
1763

1764 1765 1766 1767 1768 1769 1770 1771 1772
	if (mgp->csum_flag)
		return 1;
	else
		return 0;
}

static int myri10ge_set_rx_csum(struct net_device *netdev, u32 csum_enabled)
{
	struct myri10ge_priv *mgp = netdev_priv(netdev);
1773
	int err = 0;
B
Brice Goglin 已提交
1774

1775 1776
	if (csum_enabled)
		mgp->csum_flag = MXGEFW_FLAGS_CKSUM;
1777
	else {
1778
		netdev->features &= ~NETIF_F_LRO;
1779
		mgp->csum_flag = 0;
1780 1781 1782

	}
	return err;
1783 1784
}

B
Brice Goglin 已提交
1785 1786 1787
static int myri10ge_set_tso(struct net_device *netdev, u32 tso_enabled)
{
	struct myri10ge_priv *mgp = netdev_priv(netdev);
1788
	u32 flags = mgp->features & (NETIF_F_TSO6 | NETIF_F_TSO);
B
Brice Goglin 已提交
1789 1790 1791 1792 1793 1794 1795 1796

	if (tso_enabled)
		netdev->features |= flags;
	else
		netdev->features &= ~flags;
	return 0;
}

1797
static const char myri10ge_gstrings_main_stats[][ETH_GSTRING_LEN] = {
1798 1799 1800 1801 1802 1803 1804
	"rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors",
	"tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions",
	"rx_length_errors", "rx_over_errors", "rx_crc_errors",
	"rx_frame_errors", "rx_fifo_errors", "rx_missed_errors",
	"tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors",
	"tx_heartbeat_errors", "tx_window_errors",
	/* device-specific stats */
B
Brice Goglin 已提交
1805
	"tx_boundary", "WC", "irq", "MSI", "MSIX",
1806
	"read_dma_bw_MBs", "write_dma_bw_MBs", "read_write_dma_bw_MBs",
1807
	"serial_number", "watchdog_resets",
1808
#ifdef CONFIG_MYRI10GE_DCA
1809
	"dca_capable_firmware", "dca_device_present",
1810
#endif
1811
	"link_changes", "link_up", "dropped_link_overflow",
1812 1813 1814
	"dropped_link_error_or_filtered",
	"dropped_pause", "dropped_bad_phy", "dropped_bad_crc32",
	"dropped_unicast_filtered", "dropped_multicast_filtered",
1815
	"dropped_runt", "dropped_overrun", "dropped_no_small_buffer",
1816 1817 1818 1819 1820 1821 1822 1823 1824
	"dropped_no_big_buffer"
};

static const char myri10ge_gstrings_slice_stats[][ETH_GSTRING_LEN] = {
	"----------- slice ---------",
	"tx_pkt_start", "tx_pkt_done", "tx_req", "tx_done",
	"rx_small_cnt", "rx_big_cnt",
	"wake_queue", "stop_queue", "tx_linearized", "LRO aggregated",
	    "LRO flushed",
A
Andrew Gallatin 已提交
1825
	"LRO avg aggr", "LRO no_desc"
1826 1827 1828
};

#define MYRI10GE_NET_STATS_LEN      21
1829 1830
#define MYRI10GE_MAIN_STATS_LEN  ARRAY_SIZE(myri10ge_gstrings_main_stats)
#define MYRI10GE_SLICE_STATS_LEN  ARRAY_SIZE(myri10ge_gstrings_slice_stats)
1831 1832 1833 1834

static void
myri10ge_get_strings(struct net_device *netdev, u32 stringset, u8 * data)
{
B
Brice Goglin 已提交
1835 1836 1837
	struct myri10ge_priv *mgp = netdev_priv(netdev);
	int i;

1838 1839
	switch (stringset) {
	case ETH_SS_STATS:
1840 1841 1842
		memcpy(data, *myri10ge_gstrings_main_stats,
		       sizeof(myri10ge_gstrings_main_stats));
		data += sizeof(myri10ge_gstrings_main_stats);
B
Brice Goglin 已提交
1843 1844 1845 1846 1847
		for (i = 0; i < mgp->num_slices; i++) {
			memcpy(data, *myri10ge_gstrings_slice_stats,
			       sizeof(myri10ge_gstrings_slice_stats));
			data += sizeof(myri10ge_gstrings_slice_stats);
		}
1848 1849 1850 1851
		break;
	}
}

1852
static int myri10ge_get_sset_count(struct net_device *netdev, int sset)
1853
{
B
Brice Goglin 已提交
1854 1855
	struct myri10ge_priv *mgp = netdev_priv(netdev);

1856 1857
	switch (sset) {
	case ETH_SS_STATS:
B
Brice Goglin 已提交
1858 1859
		return MYRI10GE_MAIN_STATS_LEN +
		    mgp->num_slices * MYRI10GE_SLICE_STATS_LEN;
1860 1861 1862
	default:
		return -EOPNOTSUPP;
	}
1863 1864 1865 1866 1867 1868 1869
}

static void
myri10ge_get_ethtool_stats(struct net_device *netdev,
			   struct ethtool_stats *stats, u64 * data)
{
	struct myri10ge_priv *mgp = netdev_priv(netdev);
1870
	struct myri10ge_slice_state *ss;
B
Brice Goglin 已提交
1871
	int slice;
1872 1873
	int i;

1874 1875
	/* force stats update */
	(void)myri10ge_get_stats(netdev);
1876
	for (i = 0; i < MYRI10GE_NET_STATS_LEN; i++)
1877
		data[i] = ((unsigned long *)&netdev->stats)[i];
1878

1879
	data[i++] = (unsigned int)mgp->tx_boundary;
1880
	data[i++] = (unsigned int)mgp->wc_enabled;
1881 1882
	data[i++] = (unsigned int)mgp->pdev->irq;
	data[i++] = (unsigned int)mgp->msi_enabled;
B
Brice Goglin 已提交
1883
	data[i++] = (unsigned int)mgp->msix_enabled;
1884 1885 1886 1887 1888
	data[i++] = (unsigned int)mgp->read_dma;
	data[i++] = (unsigned int)mgp->write_dma;
	data[i++] = (unsigned int)mgp->read_write_dma;
	data[i++] = (unsigned int)mgp->serial_number;
	data[i++] = (unsigned int)mgp->watchdog_resets;
1889
#ifdef CONFIG_MYRI10GE_DCA
1890 1891 1892
	data[i++] = (unsigned int)(mgp->ss[0].dca_tag != NULL);
	data[i++] = (unsigned int)(mgp->dca_enabled);
#endif
1893
	data[i++] = (unsigned int)mgp->link_changes;
1894 1895

	/* firmware stats are useful only in the first slice */
B
Brice Goglin 已提交
1896
	ss = &mgp->ss[0];
1897 1898
	data[i++] = (unsigned int)ntohl(ss->fw_stats->link_up);
	data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_link_overflow);
1899
	data[i++] =
1900 1901 1902 1903 1904
	    (unsigned int)ntohl(ss->fw_stats->dropped_link_error_or_filtered);
	data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_pause);
	data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_phy);
	data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_crc32);
	data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_unicast_filtered);
1905
	data[i++] =
1906 1907 1908 1909 1910 1911
	    (unsigned int)ntohl(ss->fw_stats->dropped_multicast_filtered);
	data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_runt);
	data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_overrun);
	data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_small_buffer);
	data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_big_buffer);

B
Brice Goglin 已提交
1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932
	for (slice = 0; slice < mgp->num_slices; slice++) {
		ss = &mgp->ss[slice];
		data[i++] = slice;
		data[i++] = (unsigned int)ss->tx.pkt_start;
		data[i++] = (unsigned int)ss->tx.pkt_done;
		data[i++] = (unsigned int)ss->tx.req;
		data[i++] = (unsigned int)ss->tx.done;
		data[i++] = (unsigned int)ss->rx_small.cnt;
		data[i++] = (unsigned int)ss->rx_big.cnt;
		data[i++] = (unsigned int)ss->tx.wake_queue;
		data[i++] = (unsigned int)ss->tx.stop_queue;
		data[i++] = (unsigned int)ss->tx.linearized;
		data[i++] = ss->rx_done.lro_mgr.stats.aggregated;
		data[i++] = ss->rx_done.lro_mgr.stats.flushed;
		if (ss->rx_done.lro_mgr.stats.flushed)
			data[i++] = ss->rx_done.lro_mgr.stats.aggregated /
			    ss->rx_done.lro_mgr.stats.flushed;
		else
			data[i++] = 0;
		data[i++] = ss->rx_done.lro_mgr.stats.no_desc;
	}
1933 1934
}

1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946
static void myri10ge_set_msglevel(struct net_device *netdev, u32 value)
{
	struct myri10ge_priv *mgp = netdev_priv(netdev);
	mgp->msg_enable = value;
}

static u32 myri10ge_get_msglevel(struct net_device *netdev)
{
	struct myri10ge_priv *mgp = netdev_priv(netdev);
	return mgp->msg_enable;
}

1947 1948 1949 1950 1951
static int myri10ge_set_flags(struct net_device *netdev, u32 value)
{
	return ethtool_op_set_flags(netdev, value, ETH_FLAG_LRO);
}

1952
static const struct ethtool_ops myri10ge_ethtool_ops = {
1953 1954 1955 1956 1957 1958 1959 1960 1961
	.get_settings = myri10ge_get_settings,
	.get_drvinfo = myri10ge_get_drvinfo,
	.get_coalesce = myri10ge_get_coalesce,
	.set_coalesce = myri10ge_set_coalesce,
	.get_pauseparam = myri10ge_get_pauseparam,
	.set_pauseparam = myri10ge_set_pauseparam,
	.get_ringparam = myri10ge_get_ringparam,
	.get_rx_csum = myri10ge_get_rx_csum,
	.set_rx_csum = myri10ge_set_rx_csum,
B
Brice Goglin 已提交
1962
	.set_tx_csum = ethtool_op_set_tx_hw_csum,
1963
	.set_sg = ethtool_op_set_sg,
B
Brice Goglin 已提交
1964
	.set_tso = myri10ge_set_tso,
1965
	.get_link = ethtool_op_get_link,
1966
	.get_strings = myri10ge_get_strings,
1967
	.get_sset_count = myri10ge_get_sset_count,
1968 1969
	.get_ethtool_stats = myri10ge_get_ethtool_stats,
	.set_msglevel = myri10ge_set_msglevel,
1970 1971
	.get_msglevel = myri10ge_get_msglevel,
	.get_flags = ethtool_op_get_flags,
1972
	.set_flags = myri10ge_set_flags
1973 1974
};

1975
static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss)
1976
{
1977
	struct myri10ge_priv *mgp = ss->mgp;
1978
	struct myri10ge_cmd cmd;
1979
	struct net_device *dev = mgp->dev;
1980 1981
	int tx_ring_size, rx_ring_size;
	int tx_ring_entries, rx_ring_entries;
B
Brice Goglin 已提交
1982
	int i, slice, status;
1983 1984 1985
	size_t bytes;

	/* get ring sizes */
B
Brice Goglin 已提交
1986 1987
	slice = ss - mgp->ss;
	cmd.data0 = slice;
1988 1989
	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0);
	tx_ring_size = cmd.data0;
B
Brice Goglin 已提交
1990
	cmd.data0 = slice;
1991
	status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0);
1992 1993
	if (status != 0)
		return status;
1994 1995 1996 1997
	rx_ring_size = cmd.data0;

	tx_ring_entries = tx_ring_size / sizeof(struct mcp_kreq_ether_send);
	rx_ring_entries = rx_ring_size / sizeof(struct mcp_dma_addr);
1998 1999
	ss->tx.mask = tx_ring_entries - 1;
	ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1;
2000

2001 2002
	status = -ENOMEM;

2003 2004 2005
	/* allocate the host shadow rings */

	bytes = 8 + (MYRI10GE_MAX_SEND_DESC_TSO + 4)
2006 2007 2008
	    * sizeof(*ss->tx.req_list);
	ss->tx.req_bytes = kzalloc(bytes, GFP_KERNEL);
	if (ss->tx.req_bytes == NULL)
2009 2010 2011
		goto abort_with_nothing;

	/* ensure req_list entries are aligned to 8 bytes */
2012 2013
	ss->tx.req_list = (struct mcp_kreq_ether_send *)
	    ALIGN((unsigned long)ss->tx.req_bytes, 8);
B
Brice Goglin 已提交
2014
	ss->tx.queue_active = 0;
2015

2016 2017 2018
	bytes = rx_ring_entries * sizeof(*ss->rx_small.shadow);
	ss->rx_small.shadow = kzalloc(bytes, GFP_KERNEL);
	if (ss->rx_small.shadow == NULL)
2019 2020
		goto abort_with_tx_req_bytes;

2021 2022 2023
	bytes = rx_ring_entries * sizeof(*ss->rx_big.shadow);
	ss->rx_big.shadow = kzalloc(bytes, GFP_KERNEL);
	if (ss->rx_big.shadow == NULL)
2024 2025 2026 2027
		goto abort_with_rx_small_shadow;

	/* allocate the host info rings */

2028 2029 2030
	bytes = tx_ring_entries * sizeof(*ss->tx.info);
	ss->tx.info = kzalloc(bytes, GFP_KERNEL);
	if (ss->tx.info == NULL)
2031 2032
		goto abort_with_rx_big_shadow;

2033 2034 2035
	bytes = rx_ring_entries * sizeof(*ss->rx_small.info);
	ss->rx_small.info = kzalloc(bytes, GFP_KERNEL);
	if (ss->rx_small.info == NULL)
2036 2037
		goto abort_with_tx_info;

2038 2039 2040
	bytes = rx_ring_entries * sizeof(*ss->rx_big.info);
	ss->rx_big.info = kzalloc(bytes, GFP_KERNEL);
	if (ss->rx_big.info == NULL)
2041 2042 2043
		goto abort_with_rx_small_info;

	/* Fill the receive rings */
2044 2045 2046 2047 2048 2049 2050 2051 2052
	ss->rx_big.cnt = 0;
	ss->rx_small.cnt = 0;
	ss->rx_big.fill_cnt = 0;
	ss->rx_small.fill_cnt = 0;
	ss->rx_small.page_offset = MYRI10GE_ALLOC_SIZE;
	ss->rx_big.page_offset = MYRI10GE_ALLOC_SIZE;
	ss->rx_small.watchdog_needed = 0;
	ss->rx_big.watchdog_needed = 0;
	myri10ge_alloc_rx_pages(mgp, &ss->rx_small,
2053
				mgp->small_bytes + MXGEFW_PAD, 0);
2054

2055
	if (ss->rx_small.fill_cnt < ss->rx_small.mask + 1) {
2056 2057
		netdev_err(dev, "slice-%d: alloced only %d small bufs\n",
			   slice, ss->rx_small.fill_cnt);
2058
		goto abort_with_rx_small_ring;
2059 2060
	}

2061 2062
	myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0);
	if (ss->rx_big.fill_cnt < ss->rx_big.mask + 1) {
2063 2064
		netdev_err(dev, "slice-%d: alloced only %d big bufs\n",
			   slice, ss->rx_big.fill_cnt);
2065
		goto abort_with_rx_big_ring;
2066 2067 2068 2069 2070
	}

	return 0;

abort_with_rx_big_ring:
2071 2072 2073
	for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) {
		int idx = i & ss->rx_big.mask;
		myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx],
2074
				       mgp->big_bytes);
2075
		put_page(ss->rx_big.info[idx].page);
2076 2077 2078
	}

abort_with_rx_small_ring:
2079 2080 2081
	for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) {
		int idx = i & ss->rx_small.mask;
		myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx],
2082
				       mgp->small_bytes + MXGEFW_PAD);
2083
		put_page(ss->rx_small.info[idx].page);
2084
	}
2085

2086
	kfree(ss->rx_big.info);
2087 2088

abort_with_rx_small_info:
2089
	kfree(ss->rx_small.info);
2090 2091

abort_with_tx_info:
2092
	kfree(ss->tx.info);
2093 2094

abort_with_rx_big_shadow:
2095
	kfree(ss->rx_big.shadow);
2096 2097

abort_with_rx_small_shadow:
2098
	kfree(ss->rx_small.shadow);
2099 2100

abort_with_tx_req_bytes:
2101 2102 2103
	kfree(ss->tx.req_bytes);
	ss->tx.req_bytes = NULL;
	ss->tx.req_list = NULL;
2104 2105 2106 2107 2108

abort_with_nothing:
	return status;
}

2109
static void myri10ge_free_rings(struct myri10ge_slice_state *ss)
2110
{
2111
	struct myri10ge_priv *mgp = ss->mgp;
2112 2113 2114 2115
	struct sk_buff *skb;
	struct myri10ge_tx_buf *tx;
	int i, len, idx;

B
Brice Goglin 已提交
2116 2117 2118 2119
	/* If not allocated, skip it */
	if (ss->tx.req_list == NULL)
		return;

2120 2121 2122 2123 2124
	for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) {
		idx = i & ss->rx_big.mask;
		if (i == ss->rx_big.fill_cnt - 1)
			ss->rx_big.info[idx].page_offset = MYRI10GE_ALLOC_SIZE;
		myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx],
2125
				       mgp->big_bytes);
2126
		put_page(ss->rx_big.info[idx].page);
2127 2128
	}

2129 2130 2131 2132
	for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) {
		idx = i & ss->rx_small.mask;
		if (i == ss->rx_small.fill_cnt - 1)
			ss->rx_small.info[idx].page_offset =
2133
			    MYRI10GE_ALLOC_SIZE;
2134
		myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx],
2135
				       mgp->small_bytes + MXGEFW_PAD);
2136
		put_page(ss->rx_small.info[idx].page);
2137
	}
2138
	tx = &ss->tx;
2139 2140 2141 2142 2143 2144 2145
	while (tx->done != tx->req) {
		idx = tx->done & tx->mask;
		skb = tx->info[idx].skb;

		/* Mark as free */
		tx->info[idx].skb = NULL;
		tx->done++;
2146 2147
		len = dma_unmap_len(&tx->info[idx], len);
		dma_unmap_len_set(&tx->info[idx], len, 0);
2148
		if (skb) {
2149
			ss->stats.tx_dropped++;
2150 2151 2152
			dev_kfree_skb_any(skb);
			if (len)
				pci_unmap_single(mgp->pdev,
2153
						 dma_unmap_addr(&tx->info[idx],
2154 2155 2156 2157 2158
								bus), len,
						 PCI_DMA_TODEVICE);
		} else {
			if (len)
				pci_unmap_page(mgp->pdev,
2159
					       dma_unmap_addr(&tx->info[idx],
2160 2161 2162 2163
							      bus), len,
					       PCI_DMA_TODEVICE);
		}
	}
2164
	kfree(ss->rx_big.info);
2165

2166
	kfree(ss->rx_small.info);
2167

2168
	kfree(ss->tx.info);
2169

2170
	kfree(ss->rx_big.shadow);
2171

2172
	kfree(ss->rx_small.shadow);
2173

2174 2175 2176
	kfree(ss->tx.req_bytes);
	ss->tx.req_bytes = NULL;
	ss->tx.req_list = NULL;
2177 2178
}

2179 2180 2181
static int myri10ge_request_irq(struct myri10ge_priv *mgp)
{
	struct pci_dev *pdev = mgp->pdev;
B
Brice Goglin 已提交
2182 2183 2184
	struct myri10ge_slice_state *ss;
	struct net_device *netdev = mgp->dev;
	int i;
2185 2186
	int status;

B
Brice Goglin 已提交
2187 2188 2189
	mgp->msi_enabled = 0;
	mgp->msix_enabled = 0;
	status = 0;
2190
	if (myri10ge_msi) {
B
Brice Goglin 已提交
2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212
		if (mgp->num_slices > 1) {
			status =
			    pci_enable_msix(pdev, mgp->msix_vectors,
					    mgp->num_slices);
			if (status == 0) {
				mgp->msix_enabled = 1;
			} else {
				dev_err(&pdev->dev,
					"Error %d setting up MSI-X\n", status);
				return status;
			}
		}
		if (mgp->msix_enabled == 0) {
			status = pci_enable_msi(pdev);
			if (status != 0) {
				dev_err(&pdev->dev,
					"Error %d setting up MSI; falling back to xPIC\n",
					status);
			} else {
				mgp->msi_enabled = 1;
			}
		}
2213
	}
B
Brice Goglin 已提交
2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242
	if (mgp->msix_enabled) {
		for (i = 0; i < mgp->num_slices; i++) {
			ss = &mgp->ss[i];
			snprintf(ss->irq_desc, sizeof(ss->irq_desc),
				 "%s:slice-%d", netdev->name, i);
			status = request_irq(mgp->msix_vectors[i].vector,
					     myri10ge_intr, 0, ss->irq_desc,
					     ss);
			if (status != 0) {
				dev_err(&pdev->dev,
					"slice %d failed to allocate IRQ\n", i);
				i--;
				while (i >= 0) {
					free_irq(mgp->msix_vectors[i].vector,
						 &mgp->ss[i]);
					i--;
				}
				pci_disable_msix(pdev);
				return status;
			}
		}
	} else {
		status = request_irq(pdev->irq, myri10ge_intr, IRQF_SHARED,
				     mgp->dev->name, &mgp->ss[0]);
		if (status != 0) {
			dev_err(&pdev->dev, "failed to allocate IRQ\n");
			if (mgp->msi_enabled)
				pci_disable_msi(pdev);
		}
2243 2244 2245 2246 2247 2248 2249
	}
	return status;
}

static void myri10ge_free_irq(struct myri10ge_priv *mgp)
{
	struct pci_dev *pdev = mgp->pdev;
B
Brice Goglin 已提交
2250
	int i;
2251

B
Brice Goglin 已提交
2252 2253 2254 2255 2256 2257
	if (mgp->msix_enabled) {
		for (i = 0; i < mgp->num_slices; i++)
			free_irq(mgp->msix_vectors[i].vector, &mgp->ss[i]);
	} else {
		free_irq(pdev->irq, &mgp->ss[0]);
	}
2258 2259
	if (mgp->msi_enabled)
		pci_disable_msi(pdev);
B
Brice Goglin 已提交
2260 2261
	if (mgp->msix_enabled)
		pci_disable_msix(pdev);
2262 2263
}

A
Andrew Gallatin 已提交
2264 2265 2266 2267 2268 2269 2270 2271 2272 2273
static int
myri10ge_get_frag_header(struct skb_frag_struct *frag, void **mac_hdr,
			 void **ip_hdr, void **tcpudp_hdr,
			 u64 * hdr_flags, void *priv)
{
	struct ethhdr *eh;
	struct vlan_ethhdr *veh;
	struct iphdr *iph;
	u8 *va = page_address(frag->page) + frag->page_offset;
	unsigned long ll_hlen;
A
Al Viro 已提交
2274 2275
	/* passed opaque through lro_receive_frags() */
	__wsum csum = (__force __wsum) (unsigned long)priv;
A
Andrew Gallatin 已提交
2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306

	/* find the mac header, aborting if not IPv4 */

	eh = (struct ethhdr *)va;
	*mac_hdr = eh;
	ll_hlen = ETH_HLEN;
	if (eh->h_proto != htons(ETH_P_IP)) {
		if (eh->h_proto == htons(ETH_P_8021Q)) {
			veh = (struct vlan_ethhdr *)va;
			if (veh->h_vlan_encapsulated_proto != htons(ETH_P_IP))
				return -1;

			ll_hlen += VLAN_HLEN;

			/*
			 *  HW checksum starts ETH_HLEN bytes into
			 *  frame, so we must subtract off the VLAN
			 *  header's checksum before csum can be used
			 */
			csum = csum_sub(csum, csum_partial(va + ETH_HLEN,
							   VLAN_HLEN, 0));
		} else {
			return -1;
		}
	}
	*hdr_flags = LRO_IPV4;

	iph = (struct iphdr *)(va + ll_hlen);
	*ip_hdr = iph;
	if (iph->protocol != IPPROTO_TCP)
		return -1;
2307 2308
	if (iph->frag_off & htons(IP_MF | IP_OFFSET))
		return -1;
A
Andrew Gallatin 已提交
2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324
	*hdr_flags |= LRO_TCP;
	*tcpudp_hdr = (u8 *) (*ip_hdr) + (iph->ihl << 2);

	/* verify the IP checksum */
	if (unlikely(ip_fast_csum((u8 *) iph, iph->ihl)))
		return -1;

	/* verify the  checksum */
	if (unlikely(csum_tcpudp_magic(iph->saddr, iph->daddr,
				       ntohs(iph->tot_len) - (iph->ihl << 2),
				       IPPROTO_TCP, csum)))
		return -1;

	return 0;
}

2325 2326 2327 2328 2329 2330 2331
static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice)
{
	struct myri10ge_cmd cmd;
	struct myri10ge_slice_state *ss;
	int status;

	ss = &mgp->ss[slice];
B
Brice Goglin 已提交
2332 2333 2334 2335 2336 2337 2338 2339
	status = 0;
	if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) {
		cmd.data0 = slice;
		status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET,
					   &cmd, 0);
		ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *)
		    (mgp->sram + cmd.data0);
	}
2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350
	cmd.data0 = slice;
	status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET,
				    &cmd, 0);
	ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *)
	    (mgp->sram + cmd.data0);

	cmd.data0 = slice;
	status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd, 0);
	ss->rx_big.lanai = (struct mcp_kreq_ether_recv __iomem *)
	    (mgp->sram + cmd.data0);

B
Brice Goglin 已提交
2351 2352 2353 2354
	ss->tx.send_go = (__iomem __be32 *)
	    (mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice);
	ss->tx.send_stop = (__iomem __be32 *)
	    (mgp->sram + MXGEFW_ETH_SEND_STOP + 64 * slice);
2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367
	return status;

}

static int myri10ge_set_stats(struct myri10ge_priv *mgp, int slice)
{
	struct myri10ge_cmd cmd;
	struct myri10ge_slice_state *ss;
	int status;

	ss = &mgp->ss[slice];
	cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->fw_stats_bus);
	cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->fw_stats_bus);
B
Brice Goglin 已提交
2368
	cmd.data2 = sizeof(struct mcp_irq_data) | (slice << 16);
2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387
	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd, 0);
	if (status == -ENOSYS) {
		dma_addr_t bus = ss->fw_stats_bus;
		if (slice != 0)
			return -EINVAL;
		bus += offsetof(struct mcp_irq_data, send_done_count);
		cmd.data0 = MYRI10GE_LOWPART_TO_U32(bus);
		cmd.data1 = MYRI10GE_HIGHPART_TO_U32(bus);
		status = myri10ge_send_cmd(mgp,
					   MXGEFW_CMD_SET_STATS_DMA_OBSOLETE,
					   &cmd, 0);
		/* Firmware cannot support multicast without STATS_DMA_V2 */
		mgp->fw_multicast_support = 0;
	} else {
		mgp->fw_multicast_support = 1;
	}
	return 0;
}

2388 2389
static int myri10ge_open(struct net_device *dev)
{
B
Brice Goglin 已提交
2390
	struct myri10ge_slice_state *ss;
2391
	struct myri10ge_priv *mgp = netdev_priv(dev);
2392
	struct myri10ge_cmd cmd;
B
Brice Goglin 已提交
2393 2394
	int i, status, big_pow2, slice;
	u8 *itable;
A
Andrew Gallatin 已提交
2395
	struct net_lro_mgr *lro_mgr;
2396 2397 2398 2399 2400 2401 2402

	if (mgp->running != MYRI10GE_ETH_STOPPED)
		return -EBUSY;

	mgp->running = MYRI10GE_ETH_STARTING;
	status = myri10ge_reset(mgp);
	if (status != 0) {
2403
		netdev_err(dev, "failed reset\n");
2404
		goto abort_with_nothing;
2405 2406
	}

B
Brice Goglin 已提交
2407 2408
	if (mgp->num_slices > 1) {
		cmd.data0 = mgp->num_slices;
B
Brice Goglin 已提交
2409 2410 2411
		cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE;
		if (mgp->dev->real_num_tx_queues > 1)
			cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES;
B
Brice Goglin 已提交
2412 2413 2414
		status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES,
					   &cmd, 0);
		if (status != 0) {
2415
			netdev_err(dev, "failed to set number of slices\n");
B
Brice Goglin 已提交
2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426
			goto abort_with_nothing;
		}
		/* setup the indirection table */
		cmd.data0 = mgp->num_slices;
		status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE,
					   &cmd, 0);

		status |= myri10ge_send_cmd(mgp,
					    MXGEFW_CMD_GET_RSS_TABLE_OFFSET,
					    &cmd, 0);
		if (status != 0) {
2427
			netdev_err(dev, "failed to setup rss tables\n");
B
Brice Goglin 已提交
2428
			goto abort_with_nothing;
B
Brice Goglin 已提交
2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440
		}

		/* just enable an identity mapping */
		itable = mgp->sram + cmd.data0;
		for (i = 0; i < mgp->num_slices; i++)
			__raw_writeb(i, &itable[i]);

		cmd.data0 = 1;
		cmd.data1 = myri10ge_rss_hash;
		status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE,
					   &cmd, 0);
		if (status != 0) {
2441
			netdev_err(dev, "failed to enable slices\n");
B
Brice Goglin 已提交
2442 2443 2444 2445
			goto abort_with_nothing;
		}
	}

2446 2447 2448 2449
	status = myri10ge_request_irq(mgp);
	if (status != 0)
		goto abort_with_nothing;

2450 2451 2452 2453 2454 2455 2456
	/* decide what small buffer size to use.  For good TCP rx
	 * performance, it is important to not receive 1514 byte
	 * frames into jumbo buffers, as it confuses the socket buffer
	 * accounting code, leading to drops and erratic performance.
	 */

	if (dev->mtu <= ETH_DATA_LEN)
2457 2458 2459 2460
		/* enough for a TCP header */
		mgp->small_bytes = (128 > SMP_CACHE_BYTES)
		    ? (128 - MXGEFW_PAD)
		    : (SMP_CACHE_BYTES - MXGEFW_PAD);
2461
	else
2462 2463
		/* enough for a vlan encapsulated ETH_DATA_LEN frame */
		mgp->small_bytes = VLAN_ETH_FRAME_LEN;
2464 2465 2466 2467 2468 2469 2470 2471 2472

	/* Override the small buffer size? */
	if (myri10ge_small_bytes > 0)
		mgp->small_bytes = myri10ge_small_bytes;

	/* Firmware needs the big buff size as a power of 2.  Lie and
	 * tell him the buffer is larger, because we only use 1
	 * buffer/pkt, and the mtu will prevent overruns.
	 */
2473
	big_pow2 = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD;
2474
	if (big_pow2 < MYRI10GE_ALLOC_SIZE / 2) {
2475
		while (!is_power_of_2(big_pow2))
2476
			big_pow2++;
2477
		mgp->big_bytes = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD;
2478 2479 2480 2481 2482
	} else {
		big_pow2 = MYRI10GE_ALLOC_SIZE;
		mgp->big_bytes = big_pow2;
	}

B
Brice Goglin 已提交
2483 2484 2485 2486 2487 2488
	/* setup the per-slice data structures */
	for (slice = 0; slice < mgp->num_slices; slice++) {
		ss = &mgp->ss[slice];

		status = myri10ge_get_txrx(mgp, slice);
		if (status != 0) {
2489
			netdev_err(dev, "failed to get ring sizes or locations\n");
B
Brice Goglin 已提交
2490 2491 2492 2493 2494
			goto abort_with_rings;
		}
		status = myri10ge_allocate_rings(ss);
		if (status != 0)
			goto abort_with_rings;
B
Brice Goglin 已提交
2495 2496 2497 2498 2499

		/* only firmware which supports multiple TX queues
		 * supports setting up the tx stats on non-zero
		 * slices */
		if (slice == 0 || mgp->dev->real_num_tx_queues > 1)
B
Brice Goglin 已提交
2500 2501
			status = myri10ge_set_stats(mgp, slice);
		if (status) {
2502
			netdev_err(dev, "Couldn't set stats DMA\n");
B
Brice Goglin 已提交
2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514
			goto abort_with_rings;
		}

		lro_mgr = &ss->rx_done.lro_mgr;
		lro_mgr->dev = dev;
		lro_mgr->features = LRO_F_NAPI;
		lro_mgr->ip_summed = CHECKSUM_COMPLETE;
		lro_mgr->ip_summed_aggr = CHECKSUM_UNNECESSARY;
		lro_mgr->max_desc = MYRI10GE_MAX_LRO_DESCRIPTORS;
		lro_mgr->lro_arr = ss->rx_done.lro_desc;
		lro_mgr->get_frag_header = myri10ge_get_frag_header;
		lro_mgr->max_aggr = myri10ge_lro_max_pkts;
2515
		lro_mgr->frag_align_pad = 2;
B
Brice Goglin 已提交
2516 2517 2518 2519 2520 2521
		if (lro_mgr->max_aggr > MAX_SKB_FRAGS)
			lro_mgr->max_aggr = MAX_SKB_FRAGS;

		/* must happen prior to any irq */
		napi_enable(&(ss)->napi);
	}
2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532

	/* now give firmware buffers sizes, and MTU */
	cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN;
	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0);
	cmd.data0 = mgp->small_bytes;
	status |=
	    myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd, 0);
	cmd.data0 = big_pow2;
	status |=
	    myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd, 0);
	if (status) {
2533
		netdev_err(dev, "Couldn't set buffer sizes\n");
2534 2535 2536
		goto abort_with_rings;
	}

B
Brice Goglin 已提交
2537 2538 2539 2540 2541 2542 2543 2544
	/*
	 * Set Linux style TSO mode; this is needed only on newer
	 *  firmware versions.  Older versions default to Linux
	 *  style TSO
	 */
	cmd.data0 = 0;
	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd, 0);
	if (status && status != -ENOSYS) {
2545
		netdev_err(dev, "Couldn't set TSO mode\n");
2546 2547 2548
		goto abort_with_rings;
	}

A
Al Viro 已提交
2549
	mgp->link_state = ~0U;
2550 2551 2552 2553
	mgp->rdma_tags_available = 15;

	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd, 0);
	if (status) {
2554
		netdev_err(dev, "Couldn't bring up link\n");
2555 2556 2557 2558 2559 2560
		goto abort_with_rings;
	}

	mgp->running = MYRI10GE_ETH_RUNNING;
	mgp->watchdog_timer.expires = jiffies + myri10ge_watchdog_timeout * HZ;
	add_timer(&mgp->watchdog_timer);
B
Brice Goglin 已提交
2561 2562
	netif_tx_wake_all_queues(dev);

2563 2564 2565
	return 0;

abort_with_rings:
2566 2567 2568 2569
	while (slice) {
		slice--;
		napi_disable(&mgp->ss[slice].napi);
	}
B
Brice Goglin 已提交
2570 2571
	for (i = 0; i < mgp->num_slices; i++)
		myri10ge_free_rings(&mgp->ss[i]);
2572

2573 2574
	myri10ge_free_irq(mgp);

2575 2576 2577 2578 2579 2580 2581
abort_with_nothing:
	mgp->running = MYRI10GE_ETH_STOPPED;
	return -ENOMEM;
}

static int myri10ge_close(struct net_device *dev)
{
2582
	struct myri10ge_priv *mgp = netdev_priv(dev);
2583 2584
	struct myri10ge_cmd cmd;
	int status, old_down_cnt;
B
Brice Goglin 已提交
2585
	int i;
2586 2587 2588 2589

	if (mgp->running != MYRI10GE_ETH_RUNNING)
		return 0;

B
Brice Goglin 已提交
2590
	if (mgp->ss[0].tx.req_bytes == NULL)
2591 2592 2593 2594
		return 0;

	del_timer_sync(&mgp->watchdog_timer);
	mgp->running = MYRI10GE_ETH_STOPPING;
B
Brice Goglin 已提交
2595 2596 2597
	for (i = 0; i < mgp->num_slices; i++) {
		napi_disable(&mgp->ss[i].napi);
	}
2598
	netif_carrier_off(dev);
B
Brice Goglin 已提交
2599 2600

	netif_tx_stop_all_queues(dev);
2601 2602 2603 2604 2605 2606
	if (mgp->rebooted == 0) {
		old_down_cnt = mgp->down_cnt;
		mb();
		status =
		    myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0);
		if (status)
2607
			netdev_err(dev, "Couldn't bring down link\n");
2608

2609 2610 2611
		wait_event_timeout(mgp->down_wq, old_down_cnt != mgp->down_cnt,
				   HZ);
		if (old_down_cnt == mgp->down_cnt)
2612
			netdev_err(dev, "never got down irq\n");
2613
	}
2614
	netif_tx_disable(dev);
2615
	myri10ge_free_irq(mgp);
B
Brice Goglin 已提交
2616 2617
	for (i = 0; i < mgp->num_slices; i++)
		myri10ge_free_rings(&mgp->ss[i]);
2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684

	mgp->running = MYRI10GE_ETH_STOPPED;
	return 0;
}

/* copy an array of struct mcp_kreq_ether_send's to the mcp.  Copy
 * backwards one at a time and handle ring wraps */

static inline void
myri10ge_submit_req_backwards(struct myri10ge_tx_buf *tx,
			      struct mcp_kreq_ether_send *src, int cnt)
{
	int idx, starting_slot;
	starting_slot = tx->req;
	while (cnt > 1) {
		cnt--;
		idx = (starting_slot + cnt) & tx->mask;
		myri10ge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src));
		mb();
	}
}

/*
 * copy an array of struct mcp_kreq_ether_send's to the mcp.  Copy
 * at most 32 bytes at a time, so as to avoid involving the software
 * pio handler in the nic.   We re-write the first segment's flags
 * to mark them valid only after writing the entire chain.
 */

static inline void
myri10ge_submit_req(struct myri10ge_tx_buf *tx, struct mcp_kreq_ether_send *src,
		    int cnt)
{
	int idx, i;
	struct mcp_kreq_ether_send __iomem *dstp, *dst;
	struct mcp_kreq_ether_send *srcp;
	u8 last_flags;

	idx = tx->req & tx->mask;

	last_flags = src->flags;
	src->flags = 0;
	mb();
	dst = dstp = &tx->lanai[idx];
	srcp = src;

	if ((idx + cnt) < tx->mask) {
		for (i = 0; i < (cnt - 1); i += 2) {
			myri10ge_pio_copy(dstp, srcp, 2 * sizeof(*src));
			mb();	/* force write every 32 bytes */
			srcp += 2;
			dstp += 2;
		}
	} else {
		/* submit all but the first request, and ensure
		 * that it is submitted below */
		myri10ge_submit_req_backwards(tx, src, cnt);
		i = 0;
	}
	if (i < cnt) {
		/* submit the first request */
		myri10ge_pio_copy(dstp, srcp, sizeof(*src));
		mb();		/* barrier before setting valid flag */
	}

	/* re-write the last 32-bits with the valid flags */
	src->flags = last_flags;
A
Al Viro 已提交
2685
	put_be32(*((__be32 *) src + 3), (__be32 __iomem *) dst + 3);
2686 2687 2688 2689 2690 2691
	tx->req += cnt;
	mb();
}

/*
 * Transmit a packet.  We need to split the packet so that a single
2692
 * segment does not cross myri10ge->tx_boundary, so this makes segment
2693 2694 2695 2696 2697 2698 2699
 * counting tricky.  So rather than try to count segments up front, we
 * just give up if there are too few segments to hold a reasonably
 * fragmented packet currently available.  If we run
 * out of segments while preparing a packet for DMA, we just linearize
 * it and try again.
 */

2700 2701
static netdev_tx_t myri10ge_xmit(struct sk_buff *skb,
				       struct net_device *dev)
2702 2703
{
	struct myri10ge_priv *mgp = netdev_priv(dev);
2704
	struct myri10ge_slice_state *ss;
2705
	struct mcp_kreq_ether_send *req;
2706
	struct myri10ge_tx_buf *tx;
2707
	struct skb_frag_struct *frag;
B
Brice Goglin 已提交
2708
	struct netdev_queue *netdev_queue;
2709
	dma_addr_t bus;
A
Al Viro 已提交
2710 2711
	u32 low;
	__be32 high_swapped;
2712 2713
	unsigned int len;
	int idx, last_idx, avail, frag_cnt, frag_idx, count, mss, max_segments;
B
Brice Goglin 已提交
2714
	u16 pseudo_hdr_offset, cksum_offset, queue;
2715 2716 2717
	int cum_len, seglen, boundary, rdma_count;
	u8 flags, odd_flag;

B
Brice Goglin 已提交
2718 2719 2720
	queue = skb_get_queue_mapping(skb);
	ss = &mgp->ss[queue];
	netdev_queue = netdev_get_tx_queue(mgp->dev, queue);
2721
	tx = &ss->tx;
B
Brice Goglin 已提交
2722

2723 2724 2725 2726 2727 2728 2729
again:
	req = tx->req_list;
	avail = tx->mask - 1 - (tx->req - tx->done);

	mss = 0;
	max_segments = MXGEFW_MAX_SEND_DESC;

2730
	if (skb_is_gso(skb)) {
2731
		mss = skb_shinfo(skb)->gso_size;
2732
		max_segments = MYRI10GE_MAX_SEND_DESC_TSO;
2733 2734 2735 2736
	}

	if ((unlikely(avail < max_segments))) {
		/* we are out of transmit resources */
2737
		tx->stop_queue++;
B
Brice Goglin 已提交
2738
		netif_tx_stop_queue(netdev_queue);
2739
		return NETDEV_TX_BUSY;
2740 2741 2742 2743 2744 2745 2746
	}

	/* Setup checksum offloading, if needed */
	cksum_offset = 0;
	pseudo_hdr_offset = 0;
	odd_flag = 0;
	flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST);
2747
	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
2748
		cksum_offset = skb_checksum_start_offset(skb);
A
Al Viro 已提交
2749
		pseudo_hdr_offset = cksum_offset + skb->csum_offset;
2750 2751
		/* If the headers are excessively large, then we must
		 * fall back to a software checksum */
B
Brice Goglin 已提交
2752 2753
		if (unlikely(!mss && (cksum_offset > 255 ||
				      pseudo_hdr_offset > 127))) {
2754
			if (skb_checksum_help(skb))
2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772
				goto drop;
			cksum_offset = 0;
			pseudo_hdr_offset = 0;
		} else {
			odd_flag = MXGEFW_FLAGS_ALIGN_ODD;
			flags |= MXGEFW_FLAGS_CKSUM;
		}
	}

	cum_len = 0;

	if (mss) {		/* TSO */
		/* this removes any CKSUM flag from before */
		flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST);

		/* negative cum_len signifies to the
		 * send loop that we are still in the
		 * header portion of the TSO packet.
B
Brice Goglin 已提交
2773
		 * TSO header can be at most 1KB long */
2774
		cum_len = -(skb_transport_offset(skb) + tcp_hdrlen(skb));
2775

B
Brice Goglin 已提交
2776 2777 2778 2779 2780 2781 2782 2783 2784
		/* for IPv6 TSO, the checksum offset stores the
		 * TCP header length, to save the firmware from
		 * the need to parse the headers */
		if (skb_is_gso_v6(skb)) {
			cksum_offset = tcp_hdrlen(skb);
			/* Can only handle headers <= max_tso6 long */
			if (unlikely(-cum_len > mgp->max_tso6))
				return myri10ge_sw_tso(skb, dev);
		}
2785 2786 2787
		/* for TSO, pseudo_hdr_offset holds mss.
		 * The firmware figures out where to put
		 * the checksum by parsing the header. */
A
Al Viro 已提交
2788
		pseudo_hdr_offset = mss;
2789 2790 2791 2792 2793 2794 2795
	} else
		/* Mark small packets, and pad out tiny packets */
	if (skb->len <= MXGEFW_SEND_SMALL_SIZE) {
		flags |= MXGEFW_FLAGS_SMALL;

		/* pad frames to at least ETH_ZLEN bytes */
		if (unlikely(skb->len < ETH_ZLEN)) {
2796
			if (skb_padto(skb, ETH_ZLEN)) {
2797 2798
				/* The packet is gone, so we must
				 * return 0 */
2799
				ss->stats.tx_dropped += 1;
2800
				return NETDEV_TX_OK;
2801 2802 2803 2804 2805 2806 2807 2808
			}
			/* adjust the len to account for the zero pad
			 * so that the nic can know how long it is */
			skb->len = ETH_ZLEN;
		}
	}

	/* map the skb for DMA */
E
Eric Dumazet 已提交
2809
	len = skb_headlen(skb);
2810 2811 2812
	idx = tx->req & tx->mask;
	tx->info[idx].skb = skb;
	bus = pci_map_single(mgp->pdev, skb->data, len, PCI_DMA_TODEVICE);
2813 2814
	dma_unmap_addr_set(&tx->info[idx], bus, bus);
	dma_unmap_len_set(&tx->info[idx], len, len);
2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840

	frag_cnt = skb_shinfo(skb)->nr_frags;
	frag_idx = 0;
	count = 0;
	rdma_count = 0;

	/* "rdma_count" is the number of RDMAs belonging to the
	 * current packet BEFORE the current send request. For
	 * non-TSO packets, this is equal to "count".
	 * For TSO packets, rdma_count needs to be reset
	 * to 0 after a segment cut.
	 *
	 * The rdma_count field of the send request is
	 * the number of RDMAs of the packet starting at
	 * that request. For TSO send requests with one ore more cuts
	 * in the middle, this is the number of RDMAs starting
	 * after the last cut in the request. All previous
	 * segments before the last cut implicitly have 1 RDMA.
	 *
	 * Since the number of RDMAs is not known beforehand,
	 * it must be filled-in retroactively - after each
	 * segmentation cut or at the end of the entire packet.
	 */

	while (1) {
		/* Break the SKB or Fragment up into pieces which
2841
		 * do not cross mgp->tx_boundary */
2842 2843 2844 2845 2846 2847 2848 2849 2850
		low = MYRI10GE_LOWPART_TO_U32(bus);
		high_swapped = htonl(MYRI10GE_HIGHPART_TO_U32(bus));
		while (len) {
			u8 flags_next;
			int cum_len_next;

			if (unlikely(count == max_segments))
				goto abort_linearize;

2851 2852
			boundary =
			    (low + mgp->tx_boundary) & ~(mgp->tx_boundary - 1);
2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885
			seglen = boundary - low;
			if (seglen > len)
				seglen = len;
			flags_next = flags & ~MXGEFW_FLAGS_FIRST;
			cum_len_next = cum_len + seglen;
			if (mss) {	/* TSO */
				(req - rdma_count)->rdma_count = rdma_count + 1;

				if (likely(cum_len >= 0)) {	/* payload */
					int next_is_first, chop;

					chop = (cum_len_next > mss);
					cum_len_next = cum_len_next % mss;
					next_is_first = (cum_len_next == 0);
					flags |= chop * MXGEFW_FLAGS_TSO_CHOP;
					flags_next |= next_is_first *
					    MXGEFW_FLAGS_FIRST;
					rdma_count |= -(chop | next_is_first);
					rdma_count += chop & !next_is_first;
				} else if (likely(cum_len_next >= 0)) {	/* header ends */
					int small;

					rdma_count = -1;
					cum_len_next = 0;
					seglen = -cum_len;
					small = (mss <= MXGEFW_SEND_SMALL_SIZE);
					flags_next = MXGEFW_FLAGS_TSO_PLD |
					    MXGEFW_FLAGS_FIRST |
					    (small * MXGEFW_FLAGS_SMALL);
				}
			}
			req->addr_high = high_swapped;
			req->addr_low = htonl(low);
A
Al Viro 已提交
2886
			req->pseudo_hdr_offset = htons(pseudo_hdr_offset);
2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899
			req->pad = 0;	/* complete solid 16-byte block; does this matter? */
			req->rdma_count = 1;
			req->length = htons(seglen);
			req->cksum_offset = cksum_offset;
			req->flags = flags | ((cum_len & 1) * odd_flag);

			low += seglen;
			len -= seglen;
			cum_len = cum_len_next;
			flags = flags_next;
			req++;
			count++;
			rdma_count++;
B
Brice Goglin 已提交
2900 2901 2902 2903 2904 2905
			if (cksum_offset != 0 && !(mss && skb_is_gso_v6(skb))) {
				if (unlikely(cksum_offset > seglen))
					cksum_offset -= seglen;
				else
					cksum_offset = 0;
			}
2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916
		}
		if (frag_idx == frag_cnt)
			break;

		/* map next fragment for DMA */
		idx = (count + tx->req) & tx->mask;
		frag = &skb_shinfo(skb)->frags[frag_idx];
		frag_idx++;
		len = frag->size;
		bus = pci_map_page(mgp->pdev, frag->page, frag->page_offset,
				   len, PCI_DMA_TODEVICE);
2917 2918
		dma_unmap_addr_set(&tx->info[idx], bus, bus);
		dma_unmap_len_set(&tx->info[idx], len, len);
2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929
	}

	(req - rdma_count)->rdma_count = rdma_count;
	if (mss)
		do {
			req--;
			req->flags |= MXGEFW_FLAGS_TSO_LAST;
		} while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP |
					 MXGEFW_FLAGS_FIRST)));
	idx = ((count - 1) + tx->req) & tx->mask;
	tx->info[idx].last = 1;
B
Brice Goglin 已提交
2930
	myri10ge_submit_req(tx, tx->req_list, count);
B
Brice Goglin 已提交
2931 2932 2933 2934 2935
	/* if using multiple tx queues, make sure NIC polls the
	 * current slice */
	if ((mgp->dev->real_num_tx_queues > 1) && tx->queue_active == 0) {
		tx->queue_active = 1;
		put_be32(htonl(1), tx->send_go);
2936
		mb();
B
Brice Goglin 已提交
2937
		mmiowb();
B
Brice Goglin 已提交
2938
	}
2939 2940
	tx->pkt_start++;
	if ((avail - count) < MXGEFW_MAX_SEND_DESC) {
2941
		tx->stop_queue++;
B
Brice Goglin 已提交
2942
		netif_tx_stop_queue(netdev_queue);
2943
	}
2944
	return NETDEV_TX_OK;
2945 2946 2947 2948 2949 2950 2951 2952 2953 2954

abort_linearize:
	/* Free any DMA resources we've alloced and clear out the skb
	 * slot so as to not trip up assertions, and to avoid a
	 * double-free if linearizing fails */

	last_idx = (idx + 1) & tx->mask;
	idx = tx->req & tx->mask;
	tx->info[idx].skb = NULL;
	do {
2955
		len = dma_unmap_len(&tx->info[idx], len);
2956 2957 2958
		if (len) {
			if (tx->info[idx].skb != NULL)
				pci_unmap_single(mgp->pdev,
2959
						 dma_unmap_addr(&tx->info[idx],
2960 2961 2962 2963
								bus), len,
						 PCI_DMA_TODEVICE);
			else
				pci_unmap_page(mgp->pdev,
2964
					       dma_unmap_addr(&tx->info[idx],
2965 2966
							      bus), len,
					       PCI_DMA_TODEVICE);
2967
			dma_unmap_len_set(&tx->info[idx], len, 0);
2968 2969 2970 2971
			tx->info[idx].skb = NULL;
		}
		idx = (idx + 1) & tx->mask;
	} while (idx != last_idx);
H
Herbert Xu 已提交
2972
	if (skb_is_gso(skb)) {
2973
		netdev_err(mgp->dev, "TSO but wanted to linearize?!?!?\n");
2974 2975 2976
		goto drop;
	}

A
Andrew Morton 已提交
2977
	if (skb_linearize(skb))
2978 2979
		goto drop;

2980
	tx->linearized++;
2981 2982 2983 2984
	goto again;

drop:
	dev_kfree_skb_any(skb);
2985
	ss->stats.tx_dropped += 1;
2986
	return NETDEV_TX_OK;
2987 2988 2989

}

2990 2991
static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb,
					 struct net_device *dev)
B
Brice Goglin 已提交
2992 2993
{
	struct sk_buff *segs, *curr;
2994
	struct myri10ge_priv *mgp = netdev_priv(dev);
2995
	struct myri10ge_slice_state *ss;
2996
	netdev_tx_t status;
B
Brice Goglin 已提交
2997 2998

	segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO6);
2999
	if (IS_ERR(segs))
B
Brice Goglin 已提交
3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018
		goto drop;

	while (segs) {
		curr = segs;
		segs = segs->next;
		curr->next = NULL;
		status = myri10ge_xmit(curr, dev);
		if (status != 0) {
			dev_kfree_skb_any(curr);
			if (segs != NULL) {
				curr = segs;
				segs = segs->next;
				curr->next = NULL;
				dev_kfree_skb_any(segs);
			}
			goto drop;
		}
	}
	dev_kfree_skb_any(skb);
3019
	return NETDEV_TX_OK;
B
Brice Goglin 已提交
3020 3021

drop:
3022
	ss = &mgp->ss[skb_get_queue_mapping(skb)];
B
Brice Goglin 已提交
3023
	dev_kfree_skb_any(skb);
3024
	ss->stats.tx_dropped += 1;
3025
	return NETDEV_TX_OK;
B
Brice Goglin 已提交
3026 3027
}

3028 3029 3030
static struct net_device_stats *myri10ge_get_stats(struct net_device *dev)
{
	struct myri10ge_priv *mgp = netdev_priv(dev);
B
Brice Goglin 已提交
3031
	struct myri10ge_slice_netstats *slice_stats;
3032
	struct net_device_stats *stats = &dev->stats;
B
Brice Goglin 已提交
3033 3034
	int i;

3035
	spin_lock(&mgp->stats_lock);
B
Brice Goglin 已提交
3036 3037 3038 3039 3040 3041 3042 3043 3044 3045
	memset(stats, 0, sizeof(*stats));
	for (i = 0; i < mgp->num_slices; i++) {
		slice_stats = &mgp->ss[i].stats;
		stats->rx_packets += slice_stats->rx_packets;
		stats->tx_packets += slice_stats->tx_packets;
		stats->rx_bytes += slice_stats->rx_bytes;
		stats->tx_bytes += slice_stats->tx_bytes;
		stats->rx_dropped += slice_stats->rx_dropped;
		stats->tx_dropped += slice_stats->tx_dropped;
	}
3046
	spin_unlock(&mgp->stats_lock);
B
Brice Goglin 已提交
3047
	return stats;
3048 3049 3050 3051
}

static void myri10ge_set_multicast_list(struct net_device *dev)
{
3052
	struct myri10ge_priv *mgp = netdev_priv(dev);
3053
	struct myri10ge_cmd cmd;
3054
	struct netdev_hw_addr *ha;
3055
	__be32 data[2] = { 0, 0 };
3056 3057
	int err;

3058 3059
	/* can be called from atomic contexts,
	 * pass 1 to force atomicity in myri10ge_send_cmd() */
3060 3061 3062
	myri10ge_change_promisc(mgp, dev->flags & IFF_PROMISC, 1);

	/* This firmware is known to not support multicast */
3063
	if (!mgp->fw_multicast_support)
3064 3065 3066 3067 3068 3069
		return;

	/* Disable multicast filtering */

	err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1);
	if (err != 0) {
3070 3071
		netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n",
			   err);
3072 3073 3074
		goto abort;
	}

3075
	if ((dev->flags & IFF_ALLMULTI) || mgp->adopted_rx_filter_bug) {
3076 3077 3078 3079 3080 3081 3082 3083 3084
		/* request to disable multicast filtering, so quit here */
		return;
	}

	/* Flush the filters */

	err = myri10ge_send_cmd(mgp, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS,
				&cmd, 1);
	if (err != 0) {
3085 3086
		netdev_err(dev, "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, error status: %d\n",
			   err);
3087 3088 3089 3090
		goto abort;
	}

	/* Walk the multicast list, and add each address */
3091 3092
	netdev_for_each_mc_addr(ha, dev) {
		memcpy(data, &ha->addr, 6);
A
Al Viro 已提交
3093 3094
		cmd.data0 = ntohl(data[0]);
		cmd.data1 = ntohl(data[1]);
3095 3096 3097 3098
		err = myri10ge_send_cmd(mgp, MXGEFW_JOIN_MULTICAST_GROUP,
					&cmd, 1);

		if (err != 0) {
3099
			netdev_err(dev, "Failed MXGEFW_JOIN_MULTICAST_GROUP, error status:%d %pM\n",
3100
				   err, ha->addr);
3101 3102 3103 3104 3105 3106
			goto abort;
		}
	}
	/* Enable multicast filtering */
	err = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_ALLMULTI, &cmd, 1);
	if (err != 0) {
3107 3108
		netdev_err(dev, "Failed MXGEFW_DISABLE_ALLMULTI, error status: %d\n",
			   err);
3109 3110 3111 3112 3113 3114 3115
		goto abort;
	}

	return;

abort:
	return;
3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128
}

static int myri10ge_set_mac_address(struct net_device *dev, void *addr)
{
	struct sockaddr *sa = addr;
	struct myri10ge_priv *mgp = netdev_priv(dev);
	int status;

	if (!is_valid_ether_addr(sa->sa_data))
		return -EADDRNOTAVAIL;

	status = myri10ge_update_mac_address(mgp, sa->sa_data);
	if (status != 0) {
3129 3130
		netdev_err(dev, "changing mac address failed with %d\n",
			   status);
3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144
		return status;
	}

	/* change the dev structure */
	memcpy(dev->dev_addr, sa->sa_data, 6);
	return 0;
}

static int myri10ge_change_mtu(struct net_device *dev, int new_mtu)
{
	struct myri10ge_priv *mgp = netdev_priv(dev);
	int error = 0;

	if ((new_mtu < 68) || (ETH_HLEN + new_mtu > MYRI10GE_MAX_ETHER_MTU)) {
3145
		netdev_err(dev, "new mtu (%d) is not valid\n", new_mtu);
3146 3147
		return -EINVAL;
	}
3148
	netdev_info(dev, "changing mtu from %d to %d\n", dev->mtu, new_mtu);
3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185
	if (mgp->running) {
		/* if we change the mtu on an active device, we must
		 * reset the device so the firmware sees the change */
		myri10ge_close(dev);
		dev->mtu = new_mtu;
		myri10ge_open(dev);
	} else
		dev->mtu = new_mtu;

	return error;
}

/*
 * Enable ECRC to align PCI-E Completion packets on an 8-byte boundary.
 * Only do it if the bridge is a root port since we don't want to disturb
 * any other device, except if forced with myri10ge_ecrc_enable > 1.
 */

static void myri10ge_enable_ecrc(struct myri10ge_priv *mgp)
{
	struct pci_dev *bridge = mgp->pdev->bus->self;
	struct device *dev = &mgp->pdev->dev;
	unsigned cap;
	unsigned err_cap;
	u16 val;
	u8 ext_type;
	int ret;

	if (!myri10ge_ecrc_enable || !bridge)
		return;

	/* check that the bridge is a root port */
	cap = pci_find_capability(bridge, PCI_CAP_ID_EXP);
	pci_read_config_word(bridge, cap + PCI_CAP_FLAGS, &val);
	ext_type = (val & PCI_EXP_FLAGS_TYPE) >> 4;
	if (ext_type != PCI_EXP_TYPE_ROOT_PORT) {
		if (myri10ge_ecrc_enable > 1) {
3186
			struct pci_dev *prev_bridge, *old_bridge = bridge;
3187 3188 3189 3190

			/* Walk the hierarchy up to the root port
			 * where ECRC has to be enabled */
			do {
3191
				prev_bridge = bridge;
3192
				bridge = bridge->bus->self;
3193
				if (!bridge || prev_bridge == bridge) {
3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249
					dev_err(dev,
						"Failed to find root port"
						" to force ECRC\n");
					return;
				}
				cap =
				    pci_find_capability(bridge, PCI_CAP_ID_EXP);
				pci_read_config_word(bridge,
						     cap + PCI_CAP_FLAGS, &val);
				ext_type = (val & PCI_EXP_FLAGS_TYPE) >> 4;
			} while (ext_type != PCI_EXP_TYPE_ROOT_PORT);

			dev_info(dev,
				 "Forcing ECRC on non-root port %s"
				 " (enabling on root port %s)\n",
				 pci_name(old_bridge), pci_name(bridge));
		} else {
			dev_err(dev,
				"Not enabling ECRC on non-root port %s\n",
				pci_name(bridge));
			return;
		}
	}

	cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR);
	if (!cap)
		return;

	ret = pci_read_config_dword(bridge, cap + PCI_ERR_CAP, &err_cap);
	if (ret) {
		dev_err(dev, "failed reading ext-conf-space of %s\n",
			pci_name(bridge));
		dev_err(dev, "\t pci=nommconf in use? "
			"or buggy/incomplete/absent ACPI MCFG attr?\n");
		return;
	}
	if (!(err_cap & PCI_ERR_CAP_ECRC_GENC))
		return;

	err_cap |= PCI_ERR_CAP_ECRC_GENE;
	pci_write_config_dword(bridge, cap + PCI_ERR_CAP, err_cap);
	dev_info(dev, "Enabled ECRC on upstream bridge %s\n", pci_name(bridge));
}

/*
 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput
 * when the PCI-E Completion packets are aligned on an 8-byte
 * boundary.  Some PCI-E chip sets always align Completion packets; on
 * the ones that do not, the alignment can be enforced by enabling
 * ECRC generation (if supported).
 *
 * When PCI-E Completion packets are not aligned, it is actually more
 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB.
 *
 * If the driver can neither enable ECRC nor verify that it has
 * already been enabled, then it must use a firmware image which works
B
Brice Goglin 已提交
3250
 * around unaligned completion packets (myri10ge_rss_ethp_z8e.dat), and it
3251
 * should also ensure that it never gives the device a Read-DMA which is
3252
 * larger than 2KB by setting the tx_boundary to 2KB.  If ECRC is
B
Brice Goglin 已提交
3253
 * enabled, then the driver should use the aligned (myri10ge_rss_eth_z8e.dat)
3254
 * firmware image, and set tx_boundary to 4KB.
3255 3256
 */

3257
static void myri10ge_firmware_probe(struct myri10ge_priv *mgp)
3258
{
3259 3260
	struct pci_dev *pdev = mgp->pdev;
	struct device *dev = &pdev->dev;
B
Brice Goglin 已提交
3261
	int status;
3262

3263
	mgp->tx_boundary = 4096;
3264 3265 3266 3267
	/*
	 * Verify the max read request size was set to 4KB
	 * before trying the test with 4KB.
	 */
B
Brice Goglin 已提交
3268 3269
	status = pcie_get_readrq(pdev);
	if (status < 0) {
3270 3271 3272
		dev_err(dev, "Couldn't read max read req size: %d\n", status);
		goto abort;
	}
B
Brice Goglin 已提交
3273 3274
	if (status != 4096) {
		dev_warn(dev, "Max Read Request size != 4096 (%d)\n", status);
3275
		mgp->tx_boundary = 2048;
3276 3277 3278 3279 3280
	}
	/*
	 * load the optimized firmware (which assumes aligned PCIe
	 * completions) in order to see if it works on this host.
	 */
3281
	set_fw_name(mgp, myri10ge_fw_aligned, false);
B
Brice Goglin 已提交
3282
	status = myri10ge_load_firmware(mgp, 1);
3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307
	if (status != 0) {
		goto abort;
	}

	/*
	 * Enable ECRC if possible
	 */
	myri10ge_enable_ecrc(mgp);

	/*
	 * Run a DMA test which watches for unaligned completions and
	 * aborts on the first one seen.
	 */

	status = myri10ge_dma_test(mgp, MXGEFW_CMD_UNALIGNED_TEST);
	if (status == 0)
		return;		/* keep the aligned firmware */

	if (status != -E2BIG)
		dev_warn(dev, "DMA test failed: %d\n", status);
	if (status == -ENOSYS)
		dev_warn(dev, "Falling back to ethp! "
			 "Please install up to date fw\n");
abort:
	/* fall back to using the unaligned firmware */
3308
	mgp->tx_boundary = 2048;
3309
	set_fw_name(mgp, myri10ge_fw_unaligned, false);
3310

3311 3312 3313 3314
}

static void myri10ge_select_firmware(struct myri10ge_priv *mgp)
{
3315 3316
	int overridden = 0;

3317
	if (myri10ge_force_firmware == 0) {
3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330
		int link_width, exp_cap;
		u16 lnk;

		exp_cap = pci_find_capability(mgp->pdev, PCI_CAP_ID_EXP);
		pci_read_config_word(mgp->pdev, exp_cap + PCI_EXP_LNKSTA, &lnk);
		link_width = (lnk >> 4) & 0x3f;

		/* Check to see if Link is less than 8 or if the
		 * upstream bridge is known to provide aligned
		 * completions */
		if (link_width < 8) {
			dev_info(&mgp->pdev->dev, "PCIE x%d Link\n",
				 link_width);
3331
			mgp->tx_boundary = 4096;
3332
			set_fw_name(mgp, myri10ge_fw_aligned, false);
3333 3334
		} else {
			myri10ge_firmware_probe(mgp);
3335 3336 3337 3338 3339
		}
	} else {
		if (myri10ge_force_firmware == 1) {
			dev_info(&mgp->pdev->dev,
				 "Assuming aligned completions (forced)\n");
3340
			mgp->tx_boundary = 4096;
3341
			set_fw_name(mgp, myri10ge_fw_aligned, false);
3342 3343 3344
		} else {
			dev_info(&mgp->pdev->dev,
				 "Assuming unaligned completions (forced)\n");
3345
			mgp->tx_boundary = 2048;
3346
			set_fw_name(mgp, myri10ge_fw_unaligned, false);
3347 3348
		}
	}
3349 3350

	kparam_block_sysfs_write(myri10ge_fw_name);
3351
	if (myri10ge_fw_name != NULL) {
3352 3353 3354 3355 3356
		char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL);
		if (fw_name) {
			overridden = 1;
			set_fw_name(mgp, fw_name, true);
		}
3357
	}
3358 3359
	kparam_unblock_sysfs_write(myri10ge_fw_name);

3360 3361 3362
	if (mgp->board_number < MYRI10GE_MAX_BOARDS &&
	    myri10ge_fw_names[mgp->board_number] != NULL &&
	    strlen(myri10ge_fw_names[mgp->board_number])) {
3363
		set_fw_name(mgp, myri10ge_fw_names[mgp->board_number], false);
3364 3365 3366 3367 3368
		overridden = 1;
	}
	if (overridden)
		dev_info(&mgp->pdev->dev, "overriding firmware to %s\n",
			 mgp->fw_name);
3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383
}

#ifdef CONFIG_PM
static int myri10ge_suspend(struct pci_dev *pdev, pm_message_t state)
{
	struct myri10ge_priv *mgp;
	struct net_device *netdev;

	mgp = pci_get_drvdata(pdev);
	if (mgp == NULL)
		return -EINVAL;
	netdev = mgp->dev;

	netif_device_detach(netdev);
	if (netif_running(netdev)) {
3384
		netdev_info(netdev, "closing\n");
3385 3386 3387 3388 3389
		rtnl_lock();
		myri10ge_close(netdev);
		rtnl_unlock();
	}
	myri10ge_dummy_rdma(mgp, 0);
3390
	pci_save_state(pdev);
3391
	pci_disable_device(pdev);
3392 3393

	return pci_set_power_state(pdev, pci_choose_state(pdev, state));
3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410
}

static int myri10ge_resume(struct pci_dev *pdev)
{
	struct myri10ge_priv *mgp;
	struct net_device *netdev;
	int status;
	u16 vendor;

	mgp = pci_get_drvdata(pdev);
	if (mgp == NULL)
		return -EINVAL;
	netdev = mgp->dev;
	pci_set_power_state(pdev, 0);	/* zeros conf space as a side effect */
	msleep(5);		/* give card time to respond */
	pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor);
	if (vendor == 0xffff) {
3411
		netdev_err(mgp->dev, "device disappeared!\n");
3412 3413
		return -EIO;
	}
3414

3415
	pci_restore_state(pdev);
B
Brice Goglin 已提交
3416 3417

	status = pci_enable_device(pdev);
3418
	if (status) {
B
Brice Goglin 已提交
3419
		dev_err(&pdev->dev, "failed to enable device\n");
3420
		return status;
B
Brice Goglin 已提交
3421 3422
	}

3423 3424 3425
	pci_set_master(pdev);

	myri10ge_reset(mgp);
3426
	myri10ge_dummy_rdma(mgp, 1);
3427 3428 3429

	/* Save configuration space to be restored if the
	 * nic resets due to a parity error */
3430
	pci_save_state(pdev);
3431 3432 3433

	if (netif_running(netdev)) {
		rtnl_lock();
3434
		status = myri10ge_open(netdev);
3435
		rtnl_unlock();
3436 3437 3438
		if (status != 0)
			goto abort_with_enabled;

3439 3440 3441 3442 3443
	}
	netif_device_attach(netdev);

	return 0;

B
Brice Goglin 已提交
3444 3445
abort_with_enabled:
	pci_disable_device(pdev);
3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469
	return -EIO;

}
#endif				/* CONFIG_PM */

static u32 myri10ge_read_reboot(struct myri10ge_priv *mgp)
{
	struct pci_dev *pdev = mgp->pdev;
	int vs = mgp->vendor_specific_offset;
	u32 reboot;

	/*enter read32 mode */
	pci_write_config_byte(pdev, vs + 0x10, 0x3);

	/*read REBOOT_STATUS (0xfffffff0) */
	pci_write_config_dword(pdev, vs + 0x18, 0xfffffff0);
	pci_read_config_dword(pdev, vs + 0x14, &reboot);
	return reboot;
}

/*
 * This watchdog is used to check whether the board has suffered
 * from a parity error and needs to be recovered.
 */
D
David Howells 已提交
3470
static void myri10ge_watchdog(struct work_struct *work)
3471
{
D
David Howells 已提交
3472
	struct myri10ge_priv *mgp =
3473
	    container_of(work, struct myri10ge_priv, watchdog_work);
3474
	struct myri10ge_tx_buf *tx;
3475
	u32 reboot;
3476
	int status, rebooted;
B
Brice Goglin 已提交
3477
	int i;
3478 3479 3480 3481
	u16 cmd, vendor;

	mgp->watchdog_resets++;
	pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd);
3482
	rebooted = 0;
3483 3484 3485 3486 3487
	if ((cmd & PCI_COMMAND_MASTER) == 0) {
		/* Bus master DMA disabled?  Check to see
		 * if the card rebooted due to a parity error
		 * For now, just report it */
		reboot = myri10ge_read_reboot(mgp);
3488 3489 3490
		netdev_err(mgp->dev, "NIC rebooted (0x%x),%s resetting\n",
			   reboot,
			   myri10ge_reset_recover ? "" : " not");
3491 3492
		if (myri10ge_reset_recover == 0)
			return;
3493 3494 3495 3496
		rtnl_lock();
		mgp->rebooted = 1;
		rebooted = 1;
		myri10ge_close(mgp->dev);
3497
		myri10ge_reset_recover--;
3498
		mgp->rebooted = 0;
3499 3500 3501 3502 3503 3504 3505
		/*
		 * A rebooted nic will come back with config space as
		 * it was after power was applied to PCIe bus.
		 * Attempt to restore config space which was saved
		 * when the driver was loaded, or the last time the
		 * nic was resumed from power saving mode.
		 */
3506
		pci_restore_state(mgp->pdev);
3507 3508

		/* save state again for accounting reasons */
3509
		pci_save_state(mgp->pdev);
3510

3511 3512 3513 3514 3515 3516 3517
	} else {
		/* if we get back -1's from our slot, perhaps somebody
		 * powered off our card.  Don't try to reset it in
		 * this case */
		if (cmd == 0xffff) {
			pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor);
			if (vendor == 0xffff) {
3518
				netdev_err(mgp->dev, "device disappeared!\n");
3519 3520 3521 3522 3523
				return;
			}
		}
		/* Perhaps it is a software error.  Try to reset */

3524
		netdev_err(mgp->dev, "device timeout, resetting\n");
B
Brice Goglin 已提交
3525 3526
		for (i = 0; i < mgp->num_slices; i++) {
			tx = &mgp->ss[i].tx;
3527 3528 3529 3530 3531
			netdev_err(mgp->dev, "(%d): %d %d %d %d %d %d\n",
				   i, tx->queue_active, tx->req,
				   tx->done, tx->pkt_start, tx->pkt_done,
				   (int)ntohl(mgp->ss[i].fw_stats->
					      send_done_count));
B
Brice Goglin 已提交
3532
			msleep(2000);
3533 3534 3535 3536 3537
			netdev_info(mgp->dev, "(%d): %d %d %d %d %d %d\n",
				    i, tx->queue_active, tx->req,
				    tx->done, tx->pkt_start, tx->pkt_done,
				    (int)ntohl(mgp->ss[i].fw_stats->
					       send_done_count));
B
Brice Goglin 已提交
3538
		}
3539
	}
B
Brice Goglin 已提交
3540

3541 3542 3543 3544
	if (!rebooted) {
		rtnl_lock();
		myri10ge_close(mgp->dev);
	}
B
Brice Goglin 已提交
3545
	status = myri10ge_load_firmware(mgp, 1);
3546
	if (status != 0)
3547
		netdev_err(mgp->dev, "failed to load firmware\n");
3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562
	else
		myri10ge_open(mgp->dev);
	rtnl_unlock();
}

/*
 * We use our own timer routine rather than relying upon
 * netdev->tx_timeout because we have a very large hardware transmit
 * queue.  Due to the large queue, the netdev->tx_timeout function
 * cannot detect a NIC with a parity error in a timely fashion if the
 * NIC is lightly loaded.
 */
static void myri10ge_watchdog_timer(unsigned long arg)
{
	struct myri10ge_priv *mgp;
3563
	struct myri10ge_slice_state *ss;
3564
	int i, reset_needed, busy_slice_cnt;
3565
	u32 rx_pause_cnt;
3566
	u16 cmd;
3567 3568

	mgp = (struct myri10ge_priv *)arg;
3569

B
Brice Goglin 已提交
3570
	rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause);
3571
	busy_slice_cnt = 0;
B
Brice Goglin 已提交
3572 3573
	for (i = 0, reset_needed = 0;
	     i < mgp->num_slices && reset_needed == 0; ++i) {
3574

B
Brice Goglin 已提交
3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597
		ss = &mgp->ss[i];
		if (ss->rx_small.watchdog_needed) {
			myri10ge_alloc_rx_pages(mgp, &ss->rx_small,
						mgp->small_bytes + MXGEFW_PAD,
						1);
			if (ss->rx_small.fill_cnt - ss->rx_small.cnt >=
			    myri10ge_fill_thresh)
				ss->rx_small.watchdog_needed = 0;
		}
		if (ss->rx_big.watchdog_needed) {
			myri10ge_alloc_rx_pages(mgp, &ss->rx_big,
						mgp->big_bytes, 1);
			if (ss->rx_big.fill_cnt - ss->rx_big.cnt >=
			    myri10ge_fill_thresh)
				ss->rx_big.watchdog_needed = 0;
		}

		if (ss->tx.req != ss->tx.done &&
		    ss->tx.done == ss->watchdog_tx_done &&
		    ss->watchdog_tx_req != ss->watchdog_tx_done) {
			/* nic seems like it might be stuck.. */
			if (rx_pause_cnt != mgp->watchdog_pause) {
				if (net_ratelimit())
3598 3599
					netdev_err(mgp->dev, "slice %d: TX paused, check link partner\n",
						   i);
B
Brice Goglin 已提交
3600
			} else {
3601
				netdev_warn(mgp->dev, "slice %d stuck:", i);
B
Brice Goglin 已提交
3602 3603
				reset_needed = 1;
			}
3604
		}
3605 3606 3607 3608
		if (ss->watchdog_tx_done != ss->tx.done ||
		    ss->watchdog_rx_done != ss->rx_done.cnt) {
			busy_slice_cnt++;
		}
B
Brice Goglin 已提交
3609 3610
		ss->watchdog_tx_done = ss->tx.done;
		ss->watchdog_tx_req = ss->tx.req;
3611 3612 3613 3614 3615 3616 3617 3618 3619 3620
		ss->watchdog_rx_done = ss->rx_done.cnt;
	}
	/* if we've sent or received no traffic, poll the NIC to
	 * ensure it is still there.  Otherwise, we risk not noticing
	 * an error in a timely fashion */
	if (busy_slice_cnt == 0) {
		pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd);
		if ((cmd & PCI_COMMAND_MASTER) == 0) {
			reset_needed = 1;
		}
3621 3622
	}
	mgp->watchdog_pause = rx_pause_cnt;
B
Brice Goglin 已提交
3623 3624 3625 3626 3627 3628 3629 3630

	if (reset_needed) {
		schedule_work(&mgp->watchdog_work);
	} else {
		/* rearm timer */
		mod_timer(&mgp->watchdog_timer,
			  jiffies + myri10ge_watchdog_timeout * HZ);
	}
3631 3632
}

3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656
static void myri10ge_free_slices(struct myri10ge_priv *mgp)
{
	struct myri10ge_slice_state *ss;
	struct pci_dev *pdev = mgp->pdev;
	size_t bytes;
	int i;

	if (mgp->ss == NULL)
		return;

	for (i = 0; i < mgp->num_slices; i++) {
		ss = &mgp->ss[i];
		if (ss->rx_done.entry != NULL) {
			bytes = mgp->max_intr_slots *
			    sizeof(*ss->rx_done.entry);
			dma_free_coherent(&pdev->dev, bytes,
					  ss->rx_done.entry, ss->rx_done.bus);
			ss->rx_done.entry = NULL;
		}
		if (ss->fw_stats != NULL) {
			bytes = sizeof(*ss->fw_stats);
			dma_free_coherent(&pdev->dev, bytes,
					  ss->fw_stats, ss->fw_stats_bus);
			ss->fw_stats = NULL;
S
Stanislaw Gruszka 已提交
3657
			netif_napi_del(&ss->napi);
3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713
		}
	}
	kfree(mgp->ss);
	mgp->ss = NULL;
}

static int myri10ge_alloc_slices(struct myri10ge_priv *mgp)
{
	struct myri10ge_slice_state *ss;
	struct pci_dev *pdev = mgp->pdev;
	size_t bytes;
	int i;

	bytes = sizeof(*mgp->ss) * mgp->num_slices;
	mgp->ss = kzalloc(bytes, GFP_KERNEL);
	if (mgp->ss == NULL) {
		return -ENOMEM;
	}

	for (i = 0; i < mgp->num_slices; i++) {
		ss = &mgp->ss[i];
		bytes = mgp->max_intr_slots * sizeof(*ss->rx_done.entry);
		ss->rx_done.entry = dma_alloc_coherent(&pdev->dev, bytes,
						       &ss->rx_done.bus,
						       GFP_KERNEL);
		if (ss->rx_done.entry == NULL)
			goto abort;
		memset(ss->rx_done.entry, 0, bytes);
		bytes = sizeof(*ss->fw_stats);
		ss->fw_stats = dma_alloc_coherent(&pdev->dev, bytes,
						  &ss->fw_stats_bus,
						  GFP_KERNEL);
		if (ss->fw_stats == NULL)
			goto abort;
		ss->mgp = mgp;
		ss->dev = mgp->dev;
		netif_napi_add(ss->dev, &ss->napi, myri10ge_poll,
			       myri10ge_napi_weight);
	}
	return 0;
abort:
	myri10ge_free_slices(mgp);
	return -ENOMEM;
}

/*
 * This function determines the number of slices supported.
 * The number slices is the minumum of the number of CPUS,
 * the number of MSI-X irqs supported, the number of slices
 * supported by the firmware
 */
static void myri10ge_probe_slices(struct myri10ge_priv *mgp)
{
	struct myri10ge_cmd cmd;
	struct pci_dev *pdev = mgp->pdev;
	char *old_fw;
3714
	bool old_allocated;
3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726
	int i, status, ncpus, msix_cap;

	mgp->num_slices = 1;
	msix_cap = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
	ncpus = num_online_cpus();

	if (myri10ge_max_slices == 1 || msix_cap == 0 ||
	    (myri10ge_max_slices == -1 && ncpus < 2))
		return;

	/* try to load the slice aware rss firmware */
	old_fw = mgp->fw_name;
3727 3728 3729 3730
	old_allocated = mgp->fw_name_allocated;
	/* don't free old_fw if we override it. */
	mgp->fw_name_allocated = false;

3731 3732 3733
	if (myri10ge_fw_name != NULL) {
		dev_info(&mgp->pdev->dev, "overriding rss firmware to %s\n",
			 myri10ge_fw_name);
3734
		set_fw_name(mgp, myri10ge_fw_name, false);
3735
	} else if (old_fw == myri10ge_fw_aligned)
3736
		set_fw_name(mgp, myri10ge_fw_rss_aligned, false);
3737
	else
3738
		set_fw_name(mgp, myri10ge_fw_rss_unaligned, false);
3739 3740 3741
	status = myri10ge_load_firmware(mgp, 0);
	if (status != 0) {
		dev_info(&pdev->dev, "Rss firmware not found\n");
3742 3743
		if (old_allocated)
			kfree(old_fw);
3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789
		return;
	}

	/* hit the board with a reset to ensure it is alive */
	memset(&cmd, 0, sizeof(cmd));
	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0);
	if (status != 0) {
		dev_err(&mgp->pdev->dev, "failed reset\n");
		goto abort_with_fw;
	}

	mgp->max_intr_slots = cmd.data0 / sizeof(struct mcp_slot);

	/* tell it the size of the interrupt queues */
	cmd.data0 = mgp->max_intr_slots * sizeof(struct mcp_slot);
	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0);
	if (status != 0) {
		dev_err(&mgp->pdev->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n");
		goto abort_with_fw;
	}

	/* ask the maximum number of slices it supports */
	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd, 0);
	if (status != 0)
		goto abort_with_fw;
	else
		mgp->num_slices = cmd.data0;

	/* Only allow multiple slices if MSI-X is usable */
	if (!myri10ge_msi) {
		goto abort_with_fw;
	}

	/* if the admin did not specify a limit to how many
	 * slices we should use, cap it automatically to the
	 * number of CPUs currently online */
	if (myri10ge_max_slices == -1)
		myri10ge_max_slices = ncpus;

	if (mgp->num_slices > myri10ge_max_slices)
		mgp->num_slices = myri10ge_max_slices;

	/* Now try to allocate as many MSI-X vectors as we have
	 * slices. We give up on MSI-X if we can only get a single
	 * vector. */

3790 3791
	mgp->msix_vectors = kcalloc(mgp->num_slices, sizeof(*mgp->msix_vectors),
				    GFP_KERNEL);
3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807
	if (mgp->msix_vectors == NULL)
		goto disable_msix;
	for (i = 0; i < mgp->num_slices; i++) {
		mgp->msix_vectors[i].entry = i;
	}

	while (mgp->num_slices > 1) {
		/* make sure it is a power of two */
		while (!is_power_of_2(mgp->num_slices))
			mgp->num_slices--;
		if (mgp->num_slices == 1)
			goto disable_msix;
		status = pci_enable_msix(pdev, mgp->msix_vectors,
					 mgp->num_slices);
		if (status == 0) {
			pci_disable_msix(pdev);
3808 3809
			if (old_allocated)
				kfree(old_fw);
3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825
			return;
		}
		if (status > 0)
			mgp->num_slices = status;
		else
			goto disable_msix;
	}

disable_msix:
	if (mgp->msix_vectors != NULL) {
		kfree(mgp->msix_vectors);
		mgp->msix_vectors = NULL;
	}

abort_with_fw:
	mgp->num_slices = 1;
3826
	set_fw_name(mgp, old_fw, old_allocated);
3827 3828 3829
	myri10ge_load_firmware(mgp, 0);
}

3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840
static const struct net_device_ops myri10ge_netdev_ops = {
	.ndo_open		= myri10ge_open,
	.ndo_stop		= myri10ge_close,
	.ndo_start_xmit		= myri10ge_xmit,
	.ndo_get_stats		= myri10ge_get_stats,
	.ndo_validate_addr	= eth_validate_addr,
	.ndo_change_mtu		= myri10ge_change_mtu,
	.ndo_set_multicast_list = myri10ge_set_multicast_list,
	.ndo_set_mac_address	= myri10ge_set_mac_address,
};

3841 3842 3843 3844 3845 3846 3847 3848
static int myri10ge_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
	struct net_device *netdev;
	struct myri10ge_priv *mgp;
	struct device *dev = &pdev->dev;
	int i;
	int status = -ENXIO;
	int dac_enabled;
3849
	unsigned hdr_offset, ss_offset;
3850
	static int board_number;
3851

B
Brice Goglin 已提交
3852
	netdev = alloc_etherdev_mq(sizeof(*mgp), MYRI10GE_MAX_SLICES);
3853 3854 3855 3856 3857
	if (netdev == NULL) {
		dev_err(dev, "Could not allocate ethernet device\n");
		return -ENOMEM;
	}

M
Maik Hampel 已提交
3858 3859
	SET_NETDEV_DEV(netdev, &pdev->dev);

3860 3861 3862 3863 3864 3865
	mgp = netdev_priv(netdev);
	mgp->dev = netdev;
	mgp->pdev = pdev;
	mgp->csum_flag = MXGEFW_FLAGS_CKSUM;
	mgp->pause = myri10ge_flow_control;
	mgp->intr_coal_delay = myri10ge_intr_coal_delay;
3866
	mgp->msg_enable = netif_msg_init(myri10ge_debug, MYRI10GE_MSG_DEFAULT);
3867
	mgp->board_number = board_number;
3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881
	init_waitqueue_head(&mgp->down_wq);

	if (pci_enable_device(pdev)) {
		dev_err(&pdev->dev, "pci_enable_device call failed\n");
		status = -ENODEV;
		goto abort_with_netdev;
	}

	/* Find the vendor-specific cap so we can check
	 * the reboot register later on */
	mgp->vendor_specific_offset
	    = pci_find_capability(pdev, PCI_CAP_ID_VNDR);

	/* Set our max read request to 4KB */
B
Brice Goglin 已提交
3882
	status = pcie_set_readrq(pdev, 4096);
3883 3884 3885
	if (status != 0) {
		dev_err(&pdev->dev, "Error %d writing PCI_EXP_DEVCTL\n",
			status);
3886
		goto abort_with_enabled;
3887 3888 3889 3890
	}

	pci_set_master(pdev);
	dac_enabled = 1;
3891
	status = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
3892 3893 3894
	if (status != 0) {
		dac_enabled = 0;
		dev_err(&pdev->dev,
3895 3896
			"64-bit pci address mask was refused, "
			"trying 32-bit\n");
3897
		status = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3898 3899 3900
	}
	if (status != 0) {
		dev_err(&pdev->dev, "Error %d setting DMA mask\n", status);
3901
		goto abort_with_enabled;
3902
	}
3903
	(void)pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
B
Brice Goglin 已提交
3904 3905
	mgp->cmd = dma_alloc_coherent(&pdev->dev, sizeof(*mgp->cmd),
				      &mgp->cmd_bus, GFP_KERNEL);
3906
	if (mgp->cmd == NULL)
3907
		goto abort_with_enabled;
3908 3909 3910 3911

	mgp->board_span = pci_resource_len(pdev, 0);
	mgp->iomem_base = pci_resource_start(pdev, 0);
	mgp->mtrr = -1;
3912
	mgp->wc_enabled = 0;
3913 3914 3915
#ifdef CONFIG_MTRR
	mgp->mtrr = mtrr_add(mgp->iomem_base, mgp->board_span,
			     MTRR_TYPE_WRCOMB, 1);
3916 3917
	if (mgp->mtrr >= 0)
		mgp->wc_enabled = 1;
3918
#endif
B
Brice Goglin 已提交
3919
	mgp->sram = ioremap_wc(mgp->iomem_base, mgp->board_span);
3920 3921 3922 3923
	if (mgp->sram == NULL) {
		dev_err(&pdev->dev, "ioremap failed for %ld bytes at 0x%lx\n",
			mgp->board_span, mgp->iomem_base);
		status = -ENXIO;
B
Brice Goglin 已提交
3924
		goto abort_with_mtrr;
3925
	}
3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936
	hdr_offset =
	    ntohl(__raw_readl(mgp->sram + MCP_HEADER_PTR_OFFSET)) & 0xffffc;
	ss_offset = hdr_offset + offsetof(struct mcp_gen_header, string_specs);
	mgp->sram_size = ntohl(__raw_readl(mgp->sram + ss_offset));
	if (mgp->sram_size > mgp->board_span ||
	    mgp->sram_size <= MYRI10GE_FW_OFFSET) {
		dev_err(&pdev->dev,
			"invalid sram_size %dB or board span %ldB\n",
			mgp->sram_size, mgp->board_span);
		goto abort_with_ioremap;
	}
3937
	memcpy_fromio(mgp->eeprom_strings,
3938
		      mgp->sram + mgp->sram_size, MYRI10GE_EEPROM_STRINGS_SIZE);
3939 3940 3941 3942 3943 3944 3945 3946
	memset(mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE - 2, 0, 2);
	status = myri10ge_read_mac_addr(mgp);
	if (status)
		goto abort_with_ioremap;

	for (i = 0; i < ETH_ALEN; i++)
		netdev->dev_addr[i] = mgp->mac_addr[i];

3947 3948
	myri10ge_select_firmware(mgp);

B
Brice Goglin 已提交
3949
	status = myri10ge_load_firmware(mgp, 1);
3950 3951
	if (status != 0) {
		dev_err(&pdev->dev, "failed to load firmware\n");
B
Brice Goglin 已提交
3952 3953 3954 3955 3956 3957 3958
		goto abort_with_ioremap;
	}
	myri10ge_probe_slices(mgp);
	status = myri10ge_alloc_slices(mgp);
	if (status != 0) {
		dev_err(&pdev->dev, "failed to alloc slice state\n");
		goto abort_with_firmware;
3959
	}
3960 3961
	netif_set_real_num_tx_queues(netdev, mgp->num_slices);
	netif_set_real_num_rx_queues(netdev, mgp->num_slices);
3962 3963 3964
	status = myri10ge_reset(mgp);
	if (status != 0) {
		dev_err(&pdev->dev, "failed reset\n");
B
Brice Goglin 已提交
3965
		goto abort_with_slices;
3966
	}
3967
#ifdef CONFIG_MYRI10GE_DCA
3968 3969
	myri10ge_setup_dca(mgp);
#endif
3970 3971 3972 3973 3974
	pci_set_drvdata(pdev, mgp);
	if ((myri10ge_initial_mtu + ETH_HLEN) > MYRI10GE_MAX_ETHER_MTU)
		myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN;
	if ((myri10ge_initial_mtu + ETH_HLEN) < 68)
		myri10ge_initial_mtu = 68;
3975 3976

	netdev->netdev_ops = &myri10ge_netdev_ops;
3977 3978
	netdev->mtu = myri10ge_initial_mtu;
	netdev->base_addr = mgp->iomem_base;
B
Brice Goglin 已提交
3979
	netdev->features = mgp->features;
B
Brice Goglin 已提交
3980

3981 3982
	if (dac_enabled)
		netdev->features |= NETIF_F_HIGHDMA;
3983
	netdev->features |= NETIF_F_LRO;
3984

3985 3986 3987 3988 3989 3990
	netdev->vlan_features |= mgp->features;
	if (mgp->fw_ver_tiny < 37)
		netdev->vlan_features &= ~NETIF_F_TSO6;
	if (mgp->fw_ver_tiny < 32)
		netdev->vlan_features &= ~NETIF_F_TSO;

3991 3992 3993 3994 3995 3996 3997 3998 3999
	/* make sure we can get an irq, and that MSI can be
	 * setup (if available).  Also ensure netdev->irq
	 * is set to correct value if MSI is enabled */
	status = myri10ge_request_irq(mgp);
	if (status != 0)
		goto abort_with_firmware;
	netdev->irq = pdev->irq;
	myri10ge_free_irq(mgp);

4000 4001
	/* Save configuration space to be restored if the
	 * nic resets due to a parity error */
4002
	pci_save_state(pdev);
4003 4004 4005 4006 4007

	/* Setup the watchdog timer */
	setup_timer(&mgp->watchdog_timer, myri10ge_watchdog_timer,
		    (unsigned long)mgp);

4008
	spin_lock_init(&mgp->stats_lock);
4009
	SET_ETHTOOL_OPS(netdev, &myri10ge_ethtool_ops);
D
David Howells 已提交
4010
	INIT_WORK(&mgp->watchdog_work, myri10ge_watchdog);
4011 4012 4013
	status = register_netdev(netdev);
	if (status != 0) {
		dev_err(&pdev->dev, "register_netdev failed: %d\n", status);
4014
		goto abort_with_state;
4015
	}
B
Brice Goglin 已提交
4016 4017 4018 4019 4020 4021 4022 4023 4024
	if (mgp->msix_enabled)
		dev_info(dev, "%d MSI-X IRQs, tx bndry %d, fw %s, WC %s\n",
			 mgp->num_slices, mgp->tx_boundary, mgp->fw_name,
			 (mgp->wc_enabled ? "Enabled" : "Disabled"));
	else
		dev_info(dev, "%s IRQ %d, tx bndry %d, fw %s, WC %s\n",
			 mgp->msi_enabled ? "MSI" : "xPIC",
			 netdev->irq, mgp->tx_boundary, mgp->fw_name,
			 (mgp->wc_enabled ? "Enabled" : "Disabled"));
4025

4026
	board_number++;
4027 4028
	return 0;

4029
abort_with_state:
4030
	pci_restore_state(pdev);
4031

B
Brice Goglin 已提交
4032 4033 4034
abort_with_slices:
	myri10ge_free_slices(mgp);

4035 4036 4037 4038
abort_with_firmware:
	myri10ge_dummy_rdma(mgp, 0);

abort_with_ioremap:
4039 4040 4041 4042
	if (mgp->mac_addr_string != NULL)
		dev_err(&pdev->dev,
			"myri10ge_probe() failed: MAC=%s, SN=%ld\n",
			mgp->mac_addr_string, mgp->serial_number);
4043 4044
	iounmap(mgp->sram);

B
Brice Goglin 已提交
4045
abort_with_mtrr:
4046 4047 4048 4049
#ifdef CONFIG_MTRR
	if (mgp->mtrr >= 0)
		mtrr_del(mgp->mtrr, mgp->iomem_base, mgp->board_span);
#endif
B
Brice Goglin 已提交
4050 4051
	dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd),
			  mgp->cmd, mgp->cmd_bus);
4052

4053 4054
abort_with_enabled:
	pci_disable_device(pdev);
4055

4056
abort_with_netdev:
4057
	set_fw_name(mgp, NULL, false);
4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077
	free_netdev(netdev);
	return status;
}

/*
 * myri10ge_remove
 *
 * Does what is necessary to shutdown one Myrinet device. Called
 *   once for each Myrinet card by the kernel when a module is
 *   unloaded.
 */
static void myri10ge_remove(struct pci_dev *pdev)
{
	struct myri10ge_priv *mgp;
	struct net_device *netdev;

	mgp = pci_get_drvdata(pdev);
	if (mgp == NULL)
		return;

4078
	cancel_work_sync(&mgp->watchdog_work);
4079 4080 4081
	netdev = mgp->dev;
	unregister_netdev(netdev);

4082
#ifdef CONFIG_MYRI10GE_DCA
4083 4084
	myri10ge_teardown_dca(mgp);
#endif
4085 4086
	myri10ge_dummy_rdma(mgp, 0);

4087
	/* avoid a memory leak */
4088
	pci_restore_state(pdev);
4089

4090 4091 4092 4093 4094 4095
	iounmap(mgp->sram);

#ifdef CONFIG_MTRR
	if (mgp->mtrr >= 0)
		mtrr_del(mgp->mtrr, mgp->iomem_base, mgp->board_span);
#endif
B
Brice Goglin 已提交
4096 4097 4098
	myri10ge_free_slices(mgp);
	if (mgp->msix_vectors != NULL)
		kfree(mgp->msix_vectors);
B
Brice Goglin 已提交
4099 4100
	dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd),
			  mgp->cmd, mgp->cmd_bus);
4101

4102
	set_fw_name(mgp, NULL, false);
4103
	free_netdev(netdev);
4104
	pci_disable_device(pdev);
4105 4106 4107
	pci_set_drvdata(pdev, NULL);
}

B
Brice Goglin 已提交
4108
#define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E 	0x0008
4109
#define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9	0x0009
4110

4111
static DEFINE_PCI_DEVICE_TABLE(myri10ge_pci_tbl) = {
B
Brice Goglin 已提交
4112
	{PCI_DEVICE(PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E)},
4113 4114
	{PCI_DEVICE
	 (PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9)},
4115 4116 4117
	{0},
};

B
Brice Goglin 已提交
4118 4119
MODULE_DEVICE_TABLE(pci, myri10ge_pci_tbl);

4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130
static struct pci_driver myri10ge_driver = {
	.name = "myri10ge",
	.probe = myri10ge_probe,
	.remove = myri10ge_remove,
	.id_table = myri10ge_pci_tbl,
#ifdef CONFIG_PM
	.suspend = myri10ge_suspend,
	.resume = myri10ge_resume,
#endif
};

4131
#ifdef CONFIG_MYRI10GE_DCA
4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148
static int
myri10ge_notify_dca(struct notifier_block *nb, unsigned long event, void *p)
{
	int err = driver_for_each_device(&myri10ge_driver.driver,
					 NULL, &event,
					 myri10ge_notify_dca_device);

	if (err)
		return NOTIFY_BAD;
	return NOTIFY_DONE;
}

static struct notifier_block myri10ge_dca_notifier = {
	.notifier_call = myri10ge_notify_dca,
	.next = NULL,
	.priority = 0,
};
B
Brice Goglin 已提交
4149
#endif				/* CONFIG_MYRI10GE_DCA */
4150

4151 4152
static __init int myri10ge_init_module(void)
{
4153
	pr_info("Version %s\n", MYRI10GE_VERSION_STR);
B
Brice Goglin 已提交
4154

B
Brice Goglin 已提交
4155
	if (myri10ge_rss_hash > MXGEFW_RSS_HASH_TYPE_MAX) {
4156 4157
		pr_err("Illegal rssh hash type %d, defaulting to source port\n",
		       myri10ge_rss_hash);
B
Brice Goglin 已提交
4158 4159
		myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_PORT;
	}
4160
#ifdef CONFIG_MYRI10GE_DCA
4161 4162
	dca_register_notify(&myri10ge_dca_notifier);
#endif
B
Brice Goglin 已提交
4163 4164
	if (myri10ge_max_slices > MYRI10GE_MAX_SLICES)
		myri10ge_max_slices = MYRI10GE_MAX_SLICES;
B
Brice Goglin 已提交
4165

4166 4167 4168 4169 4170 4171 4172
	return pci_register_driver(&myri10ge_driver);
}

module_init(myri10ge_init_module);

static __exit void myri10ge_cleanup_module(void)
{
4173
#ifdef CONFIG_MYRI10GE_DCA
4174 4175
	dca_unregister_notify(&myri10ge_dca_notifier);
#endif
4176 4177 4178 4179
	pci_unregister_driver(&myri10ge_driver);
}

module_exit(myri10ge_cleanup_module);