mlx4_en.h 24.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
/*
 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#ifndef _MLX4_EN_H_
#define _MLX4_EN_H_

J
Jiri Pirko 已提交
37
#include <linux/bitops.h>
38 39 40 41
#include <linux/compiler.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/netdevice.h>
J
Jiri Pirko 已提交
42
#include <linux/if_vlan.h>
43
#include <linux/net_tstamp.h>
A
Amir Vadai 已提交
44 45 46
#ifdef CONFIG_MLX4_EN_DCB
#include <linux/dcbnl.h>
#endif
47
#include <linux/cpu_rmap.h>
S
Shawn Bohrer 已提交
48
#include <linux/ptp_clock_kernel.h>
49 50 51 52 53 54

#include <linux/mlx4/device.h>
#include <linux/mlx4/qp.h>
#include <linux/mlx4/cq.h>
#include <linux/mlx4/srq.h>
#include <linux/mlx4/doorbell.h>
55
#include <linux/mlx4/cmd.h>
56 57

#include "en_port.h"
58
#include "mlx4_stats.h"
59 60

#define DRV_NAME	"mlx4_en"
61 62
#define DRV_VERSION	"2.2-1"
#define DRV_RELDATE	"Feb 2014"
63 64 65 66 67 68 69 70 71 72

#define MLX4_EN_MSG_LEVEL	(NETIF_MSG_LINK | NETIF_MSG_IFDOWN)

/*
 * Device constants
 */


#define MLX4_EN_PAGE_SHIFT	12
#define MLX4_EN_PAGE_SIZE	(1 << MLX4_EN_PAGE_SHIFT)
73 74
#define DEF_RX_RINGS		16
#define MAX_RX_RINGS		128
75
#define MIN_RX_RINGS		4
76 77 78 79 80 81 82
#define TXBB_SIZE		64
#define HEADROOM		(2048 / TXBB_SIZE + 1)
#define STAMP_STRIDE		64
#define STAMP_DWORDS		(STAMP_STRIDE / 4)
#define STAMP_SHIFT		31
#define STAMP_VAL		0x7fffffff
#define STATS_DELAY		(HZ / 4)
A
Amir Vadai 已提交
83
#define SERVICE_TASK_DELAY	(HZ / 4)
84
#define MAX_NUM_OF_FS_RULES	256
85

86 87 88
#define MLX4_EN_FILTER_HASH_SHIFT 4
#define MLX4_EN_FILTER_EXPIRY_QUOTA 60

89 90 91 92 93 94 95 96
/* Typical TSO descriptor with 16 gather entries is 352 bytes... */
#define MAX_DESC_SIZE		512
#define MAX_DESC_TXBBS		(MAX_DESC_SIZE / TXBB_SIZE)

/*
 * OS related constants and tunables
 */

97 98
#define MLX4_EN_PRIV_FLAGS_BLUEFLAME 1

99 100
#define MLX4_EN_WATCHDOG_TIMEOUT	(15 * HZ)

101 102
/* Use the maximum between 16384 and a single page */
#define MLX4_EN_ALLOC_SIZE	PAGE_ALIGN(16384)
103 104

#define MLX4_EN_ALLOC_PREFER_ORDER	PAGE_ALLOC_COSTLY_ORDER
105

106
/* Receive fragment sizes; we use at most 3 fragments (for 9600 byte MTU
107 108
 * and 4K allocations) */
enum {
109 110
	FRAG_SZ0 = 1536 - NET_IP_ALIGN,
	FRAG_SZ1 = 4096,
111 112 113 114 115
	FRAG_SZ2 = 4096,
	FRAG_SZ3 = MLX4_EN_ALLOC_SIZE
};
#define MLX4_EN_MAX_RX_FRAGS	4

116 117 118 119
/* Maximum ring sizes */
#define MLX4_EN_MAX_TX_SIZE	8192
#define MLX4_EN_MAX_RX_SIZE	8192

120
/* Minimum ring size for our page-allocation scheme to work */
121 122 123
#define MLX4_EN_MIN_RX_SIZE	(MLX4_EN_ALLOC_SIZE / SMP_CACHE_BYTES)
#define MLX4_EN_MIN_TX_SIZE	(4096 / TXBB_SIZE)

Y
Yevgeny Petrilin 已提交
124
#define MLX4_EN_SMALL_PKT_SIZE		64
125
#define MLX4_EN_MIN_TX_RING_P_UP	1
126
#define MLX4_EN_MAX_TX_RING_P_UP	32
A
Amir Vadai 已提交
127
#define MLX4_EN_NUM_UP			8
Y
Yevgeny Petrilin 已提交
128
#define MLX4_EN_DEF_TX_RING_SIZE	512
129
#define MLX4_EN_DEF_RX_RING_SIZE  	1024
130 131
#define MAX_TX_RINGS			(MLX4_EN_MAX_TX_RING_P_UP * \
					 MLX4_EN_NUM_UP)
132

133 134
#define MLX4_EN_DEFAULT_TX_WORK		256

135 136
/* Target number of packets to coalesce with interrupt moderation */
#define MLX4_EN_RX_COAL_TARGET	44
137 138
#define MLX4_EN_RX_COAL_TIME	0x10

139
#define MLX4_EN_TX_COAL_PKTS	16
E
Eric Dumazet 已提交
140
#define MLX4_EN_TX_COAL_TIME	0x10
141 142 143 144 145 146 147 148

#define MLX4_EN_RX_RATE_LOW		400000
#define MLX4_EN_RX_COAL_TIME_LOW	0
#define MLX4_EN_RX_RATE_HIGH		450000
#define MLX4_EN_RX_COAL_TIME_HIGH	128
#define MLX4_EN_RX_SIZE_THRESH		1024
#define MLX4_EN_RX_RATE_THRESH		(1000000 / MLX4_EN_RX_COAL_TIME_HIGH)
#define MLX4_EN_SAMPLE_INTERVAL		0
149
#define MLX4_EN_AVG_PKT_SMALL		256
150 151 152 153 154 155

#define MLX4_EN_AUTO_CONF	0xffff

#define MLX4_EN_DEF_RX_PAUSE	1
#define MLX4_EN_DEF_TX_PAUSE	1

156
/* Interval between successive polls in the Tx routine when polling is used
157 158 159 160 161 162
   instead of interrupts (in per-core Tx rings) - should be power of 2 */
#define MLX4_EN_TX_POLL_MODER	16
#define MLX4_EN_TX_POLL_TIMEOUT	(HZ / 4)

#define SMALL_PACKET_SIZE      (256 - NET_IP_ALIGN)
#define HEADER_COPY_SIZE       (128 - NET_IP_ALIGN)
163
#define MLX4_LOOPBACK_TEST_PAYLOAD (HEADER_COPY_SIZE - ETH_HLEN)
164 165 166 167

#define MLX4_EN_MIN_MTU		46
#define ETH_BCAST		0xffffffffffffULL

168 169 170
#define MLX4_EN_LOOPBACK_RETRIES	5
#define MLX4_EN_LOOPBACK_TIMEOUT	100

171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
#ifdef MLX4_EN_PERF_STAT
/* Number of samples to 'average' */
#define AVG_SIZE			128
#define AVG_FACTOR			1024

#define INC_PERF_COUNTER(cnt)		(++(cnt))
#define ADD_PERF_COUNTER(cnt, add)	((cnt) += (add))
#define AVG_PERF_COUNTER(cnt, sample) \
	((cnt) = ((cnt) * (AVG_SIZE - 1) + (sample) * AVG_FACTOR) / AVG_SIZE)
#define GET_PERF_COUNTER(cnt)		(cnt)
#define GET_AVG_PERF_COUNTER(cnt)	((cnt) / AVG_FACTOR)

#else

#define INC_PERF_COUNTER(cnt)		do {} while (0)
#define ADD_PERF_COUNTER(cnt, add)	do {} while (0)
#define AVG_PERF_COUNTER(cnt, sample)	do {} while (0)
#define GET_PERF_COUNTER(cnt)		(0)
#define GET_AVG_PERF_COUNTER(cnt)	(0)
#endif /* MLX4_EN_PERF_STAT */

192 193 194 195 196 197 198
/* Constants for TX flow */
enum {
	MAX_INLINE = 104, /* 128 - 16 - 4 - 4 */
	MAX_BF = 256,
	MIN_PKT_LEN = 17,
};

199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
/*
 * Configurables
 */

enum cq_type {
	RX = 0,
	TX = 1,
};


/*
 * Useful macros
 */
#define ROUNDUP_LOG2(x)		ilog2(roundup_pow_of_two(x))
#define XNOR(x, y)		(!(x) == !(y))


struct mlx4_en_tx_info {
	struct sk_buff *skb;
218 219
	dma_addr_t	map0_dma;
	u32		map0_byte_count;
220 221 222 223 224 225
	u32		nr_txbb;
	u32		nr_bytes;
	u8		linear;
	u8		data_offset;
	u8		inl;
	u8		ts_requested;
226
	u8		nr_maps;
227
} ____cacheline_aligned_in_smp;
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246


#define MLX4_EN_BIT_DESC_OWN	0x80000000
#define CTRL_SIZE	sizeof(struct mlx4_wqe_ctrl_seg)
#define MLX4_EN_MEMTYPE_PAD	0x100
#define DS_SIZE		sizeof(struct mlx4_wqe_data_seg)


struct mlx4_en_tx_desc {
	struct mlx4_wqe_ctrl_seg ctrl;
	union {
		struct mlx4_wqe_data_seg data; /* at least one data segment */
		struct mlx4_wqe_lso_seg lso;
		struct mlx4_wqe_inline_seg inl;
	};
};

#define MLX4_EN_USE_SRQ		0x01000000

247 248 249
#define MLX4_EN_CX3_LOW_ID	0x1000
#define MLX4_EN_CX3_HIGH_ID	0x1005

250
struct mlx4_en_rx_alloc {
251 252
	struct page	*page;
	dma_addr_t	dma;
253 254
	u32		page_offset;
	u32		page_size;
255 256 257
};

struct mlx4_en_tx_ring {
258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
	/* cache line used and dirtied in tx completion
	 * (mlx4_en_free_tx_buf())
	 */
	u32			last_nr_txbb;
	u32			cons;
	unsigned long		wake_queue;

	/* cache line used and dirtied in mlx4_en_xmit() */
	u32			prod ____cacheline_aligned_in_smp;
	unsigned long		bytes;
	unsigned long		packets;
	unsigned long		tx_csum;
	unsigned long		tso_packets;
	unsigned long		xmit_more;
	struct mlx4_bf		bf;
	unsigned long		queue_stopped;

	/* Following part should be mostly read */
	cpumask_t		affinity_mask;
	struct mlx4_qp		qp;
278
	struct mlx4_hwq_resources wqres;
279 280 281 282 283
	u32			size; /* number of TXBBs */
	u32			size_mask;
	u16			stride;
	u16			cqn;	/* index of port CQ associated with this ring */
	u32			buf_size;
284 285
	__be32			doorbell_qpn;
	__be32			mr_key;
286 287 288 289 290 291 292 293 294 295 296 297
	void			*buf;
	struct mlx4_en_tx_info	*tx_info;
	u8			*bounce_buf;
	struct mlx4_qp_context	context;
	int			qpn;
	enum mlx4_qp_state	qp_state;
	u8			queue_index;
	bool			bf_enabled;
	bool			bf_alloced;
	struct netdev_queue	*tx_queue;
	int			hwtstamp_tx_type;
} ____cacheline_aligned_in_smp;
298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315

struct mlx4_en_rx_desc {
	/* actual number of entries depends on rx ring stride */
	struct mlx4_wqe_data_seg data[0];
};

struct mlx4_en_rx_ring {
	struct mlx4_hwq_resources wqres;
	struct mlx4_en_rx_alloc page_alloc[MLX4_EN_MAX_RX_FRAGS];
	u32 size ;	/* number of Rx descs*/
	u32 actual_size;
	u32 size_mask;
	u16 stride;
	u16 log_stride;
	u16 cqn;	/* index of port CQ associated with this ring */
	u32 prod;
	u32 cons;
	u32 buf_size;
316
	u8  fcs_del;
317 318 319 320
	void *buf;
	void *rx_info;
	unsigned long bytes;
	unsigned long packets;
321
#ifdef CONFIG_NET_RX_BUSY_POLL
322 323 324 325
	unsigned long yields;
	unsigned long misses;
	unsigned long cleaned;
#endif
326 327
	unsigned long csum_ok;
	unsigned long csum_none;
328
	unsigned long csum_complete;
329
	int hwtstamp_rx_filter;
Y
Yuval Atias 已提交
330
	cpumask_var_t affinity_mask;
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
};

struct mlx4_en_cq {
	struct mlx4_cq          mcq;
	struct mlx4_hwq_resources wqres;
	int                     ring;
	struct net_device      *dev;
	struct napi_struct	napi;
	int size;
	int buf_size;
	unsigned vector;
	enum cq_type is_tx;
	u16 moder_time;
	u16 moder_cnt;
	struct mlx4_cqe *buf;
#define MLX4_EN_OPCODE_ERROR	0x1e
347

348
#ifdef CONFIG_NET_RX_BUSY_POLL
349 350 351 352 353 354 355 356 357 358
	unsigned int state;
#define MLX4_EN_CQ_STATE_IDLE        0
#define MLX4_EN_CQ_STATE_NAPI     1    /* NAPI owns this CQ */
#define MLX4_EN_CQ_STATE_POLL     2    /* poll owns this CQ */
#define MLX4_CQ_LOCKED (MLX4_EN_CQ_STATE_NAPI | MLX4_EN_CQ_STATE_POLL)
#define MLX4_EN_CQ_STATE_NAPI_YIELD  4    /* NAPI yielded this CQ */
#define MLX4_EN_CQ_STATE_POLL_YIELD  8    /* poll yielded this CQ */
#define CQ_YIELD (MLX4_EN_CQ_STATE_NAPI_YIELD | MLX4_EN_CQ_STATE_POLL_YIELD)
#define CQ_USER_PEND (MLX4_EN_CQ_STATE_POLL | MLX4_EN_CQ_STATE_POLL_YIELD)
	spinlock_t poll_lock; /* protects from LLS/napi conflicts */
359
#endif  /* CONFIG_NET_RX_BUSY_POLL */
360
	struct irq_desc *irq_desc;
361 362 363 364 365 366 367 368
};

struct mlx4_en_port_profile {
	u32 flags;
	u32 tx_ring_num;
	u32 rx_ring_num;
	u32 tx_ring_size;
	u32 rx_ring_size;
369 370 371 372
	u8 rx_pause;
	u8 rx_ppp;
	u8 tx_pause;
	u8 tx_ppp;
373
	int rss_rings;
374
	int inline_thold;
375 376 377
};

struct mlx4_en_profile {
Y
Yevgeny Petrilin 已提交
378
	int udp_rss;
379 380 381 382
	u8 rss_mask;
	u32 active_ports;
	u32 small_pkt_int;
	u8 no_reset;
383
	u8 num_tx_rings_p_up;
384 385 386 387 388 389 390 391
	struct mlx4_en_port_profile prof[MLX4_MAX_PORTS + 1];
};

struct mlx4_en_dev {
	struct mlx4_dev         *dev;
	struct pci_dev		*pdev;
	struct mutex		state_lock;
	struct net_device       *pndev[MLX4_MAX_PORTS + 1];
392
	struct net_device       *upper[MLX4_MAX_PORTS + 1];
393 394 395 396 397 398 399 400 401 402 403
	u32                     port_cnt;
	bool			device_up;
	struct mlx4_en_profile  profile;
	u32			LSO_support;
	struct workqueue_struct *workqueue;
	struct device           *dma_device;
	void __iomem            *uar_map;
	struct mlx4_uar         priv_uar;
	struct mlx4_mr		mr;
	u32                     priv_pdn;
	spinlock_t              uar_lock;
404
	u8			mac_removed[MLX4_MAX_PORTS + 1];
S
Shawn Bohrer 已提交
405 406
	rwlock_t		clock_lock;
	u32			nominal_c_mult;
407 408 409
	struct cyclecounter	cycles;
	struct timecounter	clock;
	unsigned long		last_overflow_check;
A
Amir Vadai 已提交
410
	unsigned long		overflow_period;
S
Shawn Bohrer 已提交
411 412
	struct ptp_clock	*ptp_clock;
	struct ptp_clock_info	ptp_clock_info;
413
	struct notifier_block	nb;
414 415 416 417 418
};


struct mlx4_en_rss_map {
	int base_qpn;
419 420
	struct mlx4_qp qps[MAX_RX_RINGS];
	enum mlx4_qp_state state[MAX_RX_RINGS];
421 422 423 424
	struct mlx4_qp indir_qp;
	enum mlx4_qp_state indir_state;
};

425 426 427 428 429
enum mlx4_en_port_flag {
	MLX4_EN_PORT_ANC = 1<<0, /* Auto-negotiation complete */
	MLX4_EN_PORT_ANE = 1<<1, /* Auto-negotiation enabled */
};

430 431 432
struct mlx4_en_port_state {
	int link_state;
	int link_speed;
433 434
	int transceiver;
	u32 flags;
435 436
};

437 438 439 440 441 442 443 444 445 446
enum mlx4_en_mclist_act {
	MCLIST_NONE,
	MCLIST_REM,
	MCLIST_ADD,
};

struct mlx4_en_mc_list {
	struct list_head	list;
	enum mlx4_en_mclist_act	action;
	u8			addr[ETH_ALEN];
447
	u64			reg_id;
448
	u64			tunnel_reg_id;
449 450
};

451 452 453 454 455 456
struct mlx4_en_frag_info {
	u16 frag_size;
	u16 frag_prefix_size;
	u16 frag_stride;
};

A
Amir Vadai 已提交
457 458 459 460 461 462 463 464 465
#ifdef CONFIG_MLX4_EN_DCB
/* Minimal TC BW - setting to 0 will block traffic */
#define MLX4_EN_BW_MIN 1
#define MLX4_EN_BW_MAX 100 /* Utilize 100% of the line */

#define MLX4_EN_TC_ETS 7

#endif

466
struct ethtool_flow_id {
467
	struct list_head list;
468 469 470 471
	struct ethtool_rx_flow_spec flow_spec;
	u64 id;
};

472 473 474 475 476 477 478 479
enum {
	MLX4_EN_FLAG_PROMISC		= (1 << 0),
	MLX4_EN_FLAG_MC_PROMISC		= (1 << 1),
	/* whether we need to enable hardware loopback by putting dmac
	 * in Tx WQE
	 */
	MLX4_EN_FLAG_ENABLE_HW_LOOPBACK	= (1 << 2),
	/* whether we need to drop packets that hardware loopback-ed */
480
	MLX4_EN_FLAG_RX_FILTER_NEEDED	= (1 << 3),
481 482
	MLX4_EN_FLAG_FORCE_PROMISC	= (1 << 4),
	MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP	= (1 << 5),
483 484
};

485 486 487
#define MLX4_EN_MAC_HASH_SIZE (1 << BITS_PER_BYTE)
#define MLX4_EN_MAC_HASH_IDX 5

488 489 490 491 492
struct mlx4_en_stats_bitmap {
	DECLARE_BITMAP(bitmap, NUM_ALL_STATS);
	struct mutex mutex; /* for mutual access to stats bitmap */
};

493 494 495 496
struct mlx4_en_priv {
	struct mlx4_en_dev *mdev;
	struct mlx4_en_port_profile *prof;
	struct net_device *dev;
J
Jiri Pirko 已提交
497
	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
498 499
	struct net_device_stats stats;
	struct net_device_stats ret_stats;
500
	struct mlx4_en_port_state port_state;
501
	spinlock_t stats_lock;
502
	struct ethtool_flow_id ethtool_rules[MAX_NUM_OF_FS_RULES];
503 504
	/* To allow rules removal while port is going down */
	struct list_head ethtool_list;
505

506
	unsigned long last_moder_packets[MAX_RX_RINGS];
507
	unsigned long last_moder_tx_packets;
508
	unsigned long last_moder_bytes[MAX_RX_RINGS];
509
	unsigned long last_moder_jiffies;
510
	int last_moder_time[MAX_RX_RINGS];
511 512 513 514 515 516 517 518 519 520 521
	u16 rx_usecs;
	u16 rx_frames;
	u16 tx_usecs;
	u16 tx_frames;
	u32 pkt_rate_low;
	u16 rx_usecs_low;
	u32 pkt_rate_high;
	u16 rx_usecs_high;
	u16 sample_interval;
	u16 adaptive_rx_coal;
	u32 msg_enable;
522 523
	u32 loopback_ok;
	u32 validate_loopback;
524 525 526 527 528 529 530 531 532

	struct mlx4_hwq_resources res;
	int link_state;
	int last_link_state;
	bool port_up;
	int port;
	int registered;
	int allocated;
	int stride;
533
	unsigned char current_mac[ETH_ALEN + 2];
534 535 536
	int mac_index;
	unsigned max_mtu;
	int base_qpn;
O
Or Gerlitz 已提交
537
	int cqe_factor;
538
	int cqe_size;
539 540

	struct mlx4_en_rss_map rss_map;
541
	__be32 ctrl_flags;
542
	u32 flags;
543
	u8 num_tx_rings_p_up;
544
	u32 tx_work_limit;
545 546 547 548 549 550 551
	u32 tx_ring_num;
	u32 rx_ring_num;
	u32 rx_skb_size;
	struct mlx4_en_frag_info frag_info[MLX4_EN_MAX_RX_FRAGS];
	u16 num_frags;
	u16 log_rx_info;

552 553 554 555
	struct mlx4_en_tx_ring **tx_ring;
	struct mlx4_en_rx_ring *rx_ring[MAX_RX_RINGS];
	struct mlx4_en_cq **tx_cq;
	struct mlx4_en_cq *rx_cq[MAX_RX_RINGS];
556
	struct mlx4_qp drop_qp;
557
	struct work_struct rx_mode_task;
558 559 560
	struct work_struct watchdog_task;
	struct work_struct linkstate_task;
	struct delayed_work stats_task;
A
Amir Vadai 已提交
561
	struct delayed_work service_task;
562
#ifdef CONFIG_MLX4_EN_VXLAN
O
Or Gerlitz 已提交
563 564
	struct work_struct vxlan_add_task;
	struct work_struct vxlan_del_task;
565
#endif
566 567
	struct mlx4_en_perf_stats pstats;
	struct mlx4_en_pkt_stats pkstats;
568 569 570 571
	struct mlx4_en_flow_stats_rx rx_priority_flowstats[MLX4_NUM_PRIORITIES];
	struct mlx4_en_flow_stats_tx tx_priority_flowstats[MLX4_NUM_PRIORITIES];
	struct mlx4_en_flow_stats_rx rx_flowstats;
	struct mlx4_en_flow_stats_tx tx_flowstats;
572
	struct mlx4_en_port_stats port_stats;
573
	struct mlx4_en_stats_bitmap stats_bitmap;
574 575
	struct list_head mc_list;
	struct list_head curr_list;
576
	u64 broadcast_id;
577
	struct mlx4_en_stat_out_mbox hw_stats;
E
Eli Cohen 已提交
578
	int vids[128];
Y
Yevgeny Petrilin 已提交
579
	bool wol;
580
	struct device *ddev;
581
	int base_tx_qpn;
582
	struct hlist_head mac_hash[MLX4_EN_MAC_HASH_SIZE];
583
	struct hwtstamp_config hwtstamp_config;
A
Amir Vadai 已提交
584 585 586

#ifdef CONFIG_MLX4_EN_DCB
	struct ieee_ets ets;
587
	u16 maxrate[IEEE_8021QAZ_MAX_TCS];
588
	enum dcbnl_cndd_states cndd_state[IEEE_8021QAZ_MAX_TCS];
A
Amir Vadai 已提交
589
#endif
590 591 592 593 594 595
#ifdef CONFIG_RFS_ACCEL
	spinlock_t filters_lock;
	int last_filter_id;
	struct list_head filters;
	struct hlist_head filter_hash[1 << MLX4_EN_FILTER_HASH_SHIFT];
#endif
596
	u64 tunnel_reg_id;
O
Or Gerlitz 已提交
597
	__be16 vxlan_port;
598 599

	u32 pflags;
E
Eric Dumazet 已提交
600
	u8 rss_key[MLX4_EN_RSS_KEY_SIZE];
601
	u8 rss_hash_fn;
Y
Yevgeny Petrilin 已提交
602 603 604 605 606
};

enum mlx4_en_wol {
	MLX4_EN_WOL_MAGIC = (1ULL << 61),
	MLX4_EN_WOL_ENABLED = (1ULL << 62),
607 608
};

609
struct mlx4_mac_entry {
610
	struct hlist_node hlist;
611 612
	unsigned char mac[ETH_ALEN + 2];
	u64 reg_id;
613
	struct rcu_head rcu;
614 615
};

616 617 618 619 620
static inline struct mlx4_cqe *mlx4_en_get_cqe(void *buf, int idx, int cqe_sz)
{
	return buf + idx * cqe_sz;
}

621
#ifdef CONFIG_NET_RX_BUSY_POLL
622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666
static inline void mlx4_en_cq_init_lock(struct mlx4_en_cq *cq)
{
	spin_lock_init(&cq->poll_lock);
	cq->state = MLX4_EN_CQ_STATE_IDLE;
}

/* called from the device poll rutine to get ownership of a cq */
static inline bool mlx4_en_cq_lock_napi(struct mlx4_en_cq *cq)
{
	int rc = true;
	spin_lock(&cq->poll_lock);
	if (cq->state & MLX4_CQ_LOCKED) {
		WARN_ON(cq->state & MLX4_EN_CQ_STATE_NAPI);
		cq->state |= MLX4_EN_CQ_STATE_NAPI_YIELD;
		rc = false;
	} else
		/* we don't care if someone yielded */
		cq->state = MLX4_EN_CQ_STATE_NAPI;
	spin_unlock(&cq->poll_lock);
	return rc;
}

/* returns true is someone tried to get the cq while napi had it */
static inline bool mlx4_en_cq_unlock_napi(struct mlx4_en_cq *cq)
{
	int rc = false;
	spin_lock(&cq->poll_lock);
	WARN_ON(cq->state & (MLX4_EN_CQ_STATE_POLL |
			       MLX4_EN_CQ_STATE_NAPI_YIELD));

	if (cq->state & MLX4_EN_CQ_STATE_POLL_YIELD)
		rc = true;
	cq->state = MLX4_EN_CQ_STATE_IDLE;
	spin_unlock(&cq->poll_lock);
	return rc;
}

/* called from mlx4_en_low_latency_poll() */
static inline bool mlx4_en_cq_lock_poll(struct mlx4_en_cq *cq)
{
	int rc = true;
	spin_lock_bh(&cq->poll_lock);
	if ((cq->state & MLX4_CQ_LOCKED)) {
		struct net_device *dev = cq->dev;
		struct mlx4_en_priv *priv = netdev_priv(dev);
667
		struct mlx4_en_rx_ring *rx_ring = priv->rx_ring[cq->ring];
668 669 670

		cq->state |= MLX4_EN_CQ_STATE_POLL_YIELD;
		rc = false;
671
		rx_ring->yields++;
672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693
	} else
		/* preserve yield marks */
		cq->state |= MLX4_EN_CQ_STATE_POLL;
	spin_unlock_bh(&cq->poll_lock);
	return rc;
}

/* returns true if someone tried to get the cq while it was locked */
static inline bool mlx4_en_cq_unlock_poll(struct mlx4_en_cq *cq)
{
	int rc = false;
	spin_lock_bh(&cq->poll_lock);
	WARN_ON(cq->state & (MLX4_EN_CQ_STATE_NAPI));

	if (cq->state & MLX4_EN_CQ_STATE_POLL_YIELD)
		rc = true;
	cq->state = MLX4_EN_CQ_STATE_IDLE;
	spin_unlock_bh(&cq->poll_lock);
	return rc;
}

/* true if a socket is polling, even if it did not get the lock */
694
static inline bool mlx4_en_cq_busy_polling(struct mlx4_en_cq *cq)
695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723
{
	WARN_ON(!(cq->state & MLX4_CQ_LOCKED));
	return cq->state & CQ_USER_PEND;
}
#else
static inline void mlx4_en_cq_init_lock(struct mlx4_en_cq *cq)
{
}

static inline bool mlx4_en_cq_lock_napi(struct mlx4_en_cq *cq)
{
	return true;
}

static inline bool mlx4_en_cq_unlock_napi(struct mlx4_en_cq *cq)
{
	return false;
}

static inline bool mlx4_en_cq_lock_poll(struct mlx4_en_cq *cq)
{
	return false;
}

static inline bool mlx4_en_cq_unlock_poll(struct mlx4_en_cq *cq)
{
	return false;
}

724
static inline bool mlx4_en_cq_busy_polling(struct mlx4_en_cq *cq)
725 726 727
{
	return false;
}
728
#endif /* CONFIG_NET_RX_BUSY_POLL */
729

730
#define MLX4_EN_WOL_DO_MODIFY (1ULL << 63)
731

732 733 734
void mlx4_en_update_loopback_state(struct net_device *dev,
				   netdev_features_t features);

735 736 737 738
void mlx4_en_destroy_netdev(struct net_device *dev);
int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
			struct mlx4_en_port_profile *prof);

739
int mlx4_en_start_port(struct net_device *dev);
740
void mlx4_en_stop_port(struct net_device *dev, int detach);
741

742
void mlx4_en_set_stats_bitmap(struct mlx4_dev *dev,
743 744 745
			      struct mlx4_en_stats_bitmap *stats_bitmap,
			      u8 rx_ppp, u8 rx_pause,
			      u8 tx_ppp, u8 tx_pause);
746

747
void mlx4_en_free_resources(struct mlx4_en_priv *priv);
748 749
int mlx4_en_alloc_resources(struct mlx4_en_priv *priv);

750
int mlx4_en_create_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq,
751
		      int entries, int ring, enum cq_type mode, int node);
752
void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq);
753 754
int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
			int cq_idx);
755 756 757 758 759
void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
int mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);

void mlx4_en_tx_irq(struct mlx4_cq *mcq);
760
u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb,
761
			 void *accel_priv, select_queue_fallback_t fallback);
762
netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev);
763

764 765
int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
			   struct mlx4_en_tx_ring **pring,
766
			   u32 size, u16 stride,
767
			   int node, int queue_index);
768 769
void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
			     struct mlx4_en_tx_ring **pring);
770 771
int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv,
			     struct mlx4_en_tx_ring *ring,
772
			     int cq, int user_prio);
773 774
void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv,
				struct mlx4_en_tx_ring *ring);
775
void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev);
776
int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
777
			   struct mlx4_en_rx_ring **pring,
778
			   u32 size, u16 stride, int node);
779
void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
780
			     struct mlx4_en_rx_ring **pring,
781
			     u32 size, u16 stride);
782 783 784 785 786 787 788
int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv);
void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
				struct mlx4_en_rx_ring *ring);
int mlx4_en_process_rx_cq(struct net_device *dev,
			  struct mlx4_en_cq *cq,
			  int budget);
int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget);
789
int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget);
790
void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride,
791 792
		int is_tx, int rss, int qpn, int cqn, int user_prio,
		struct mlx4_qp_context *context);
793
void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event);
794 795 796 797 798 799
int mlx4_en_map_buffer(struct mlx4_buf *buf);
void mlx4_en_unmap_buffer(struct mlx4_buf *buf);

void mlx4_en_calc_rx_buf(struct net_device *dev);
int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv);
void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv);
800 801
int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv);
void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv);
802 803 804 805
int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring);
void mlx4_en_rx_irq(struct mlx4_cq *mcq);

int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode);
J
Jiri Pirko 已提交
806
int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, struct mlx4_en_priv *priv);
807 808

int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset);
809 810
int mlx4_en_QUERY_PORT(struct mlx4_en_dev *mdev, u8 port);

A
Amir Vadai 已提交
811 812
#ifdef CONFIG_MLX4_EN_DCB
extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_ops;
813
extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_pfc_ops;
A
Amir Vadai 已提交
814 815
#endif

816 817
int mlx4_en_setup_tc(struct net_device *dev, u8 up);

818
#ifdef CONFIG_RFS_ACCEL
819
void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv);
820 821
#endif

822 823
#define MLX4_EN_NUM_SELF_TEST	5
void mlx4_en_ex_selftest(struct net_device *dev, u32 *flags, u64 *buf);
A
Amir Vadai 已提交
824
void mlx4_en_ptp_overflow_check(struct mlx4_en_dev *mdev);
825

826 827 828 829 830 831
#define DEV_FEATURE_CHANGED(dev, new_features, feature) \
	((dev->features & feature) ^ (new_features & feature))

int mlx4_en_reset_config(struct net_device *dev,
			 struct hwtstamp_config ts_config,
			 netdev_features_t new_features);
832 833 834 835
void mlx4_en_update_pfc_stats_bitmap(struct mlx4_dev *dev,
				     struct mlx4_en_stats_bitmap *stats_bitmap,
				     u8 rx_ppp, u8 rx_pause,
				     u8 tx_ppp, u8 tx_pause);
836 837 838
int mlx4_en_netdev_event(struct notifier_block *this,
			 unsigned long event, void *ptr);

839
/*
840 841 842 843 844 845 846
 * Functions for time stamping
 */
u64 mlx4_en_get_cqe_ts(struct mlx4_cqe *cqe);
void mlx4_en_fill_hwtstamps(struct mlx4_en_dev *mdev,
			    struct skb_shared_hwtstamps *hwts,
			    u64 timestamp);
void mlx4_en_init_timestamp(struct mlx4_en_dev *mdev);
S
Shawn Bohrer 已提交
847
void mlx4_en_remove_timestamp(struct mlx4_en_dev *mdev);
848 849

/* Globals
850 851
 */
extern const struct ethtool_ops mlx4_en_ethtool_ops;
852 853 854 855 856 857 858



/*
 * printk / logging functions
 */

859
__printf(3, 4)
860 861
void en_print(const char *level, const struct mlx4_en_priv *priv,
	      const char *format, ...);
862

J
Joe Perches 已提交
863 864 865 866
#define en_dbg(mlevel, priv, format, ...)				\
do {									\
	if (NETIF_MSG_##mlevel & (priv)->msg_enable)			\
		en_print(KERN_DEBUG, priv, format, ##__VA_ARGS__);	\
867
} while (0)
J
Joe Perches 已提交
868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883
#define en_warn(priv, format, ...)					\
	en_print(KERN_WARNING, priv, format, ##__VA_ARGS__)
#define en_err(priv, format, ...)					\
	en_print(KERN_ERR, priv, format, ##__VA_ARGS__)
#define en_info(priv, format, ...)					\
	en_print(KERN_INFO, priv, format, ##__VA_ARGS__)

#define mlx4_err(mdev, format, ...)					\
	pr_err(DRV_NAME " %s: " format,					\
	       dev_name(&(mdev)->pdev->dev), ##__VA_ARGS__)
#define mlx4_info(mdev, format, ...)					\
	pr_info(DRV_NAME " %s: " format,				\
		dev_name(&(mdev)->pdev->dev), ##__VA_ARGS__)
#define mlx4_warn(mdev, format, ...)					\
	pr_warn(DRV_NAME " %s: " format,				\
		dev_name(&(mdev)->pdev->dev), ##__VA_ARGS__)
884

885
#endif